fix(WRK-22,WRK-24,WRK-25,WRK-27,IPC-26): worker write-seam hardening
WRK-22/IPC-26: tombstone a WriteAsync/WriteBatchAsync cancelled while waiting for the write lock (PendingFrame.Claimed under _gate; DequeueNext skips cancelled, claims the frame it returns) so a cancelled write never reaches the wire unless already claimed mid-write (documented residual). WRK-25: add WriteBatchAsync; RunEventDrainLoopAsync submits the drained event batch through it, so a burst of N events costs one flush not N. IPC-30 oversized-event structured fault preserved via FindOversizedEvent. WRK-24: reject a below-1024 negotiated frame maximum at the handshake (MinNegotiableFrameBytes, matching GatewayOptionsValidator floor). WRK-27: alarm poll advertises StaCallInProgress on the heartbeat snapshot so the watchdog suppresses to the ceiling, not the grace. Docs (WorkerFrameProtocol.md, MxAccessWorkerInstanceDesign.md) and the 2026-07-12 remediation registers/change-log updated in the same commit.
This commit is contained in:
@@ -481,6 +481,192 @@ public sealed class WorkerFrameProtocolTests
|
||||
Assert.Equal(WorkerFrameProtocolErrorCode.InvalidConfiguration, exception.ErrorCode);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies a negotiated frame maximum below the worker floor is rejected as
|
||||
/// <c>InvalidConfiguration</c> (WRK-24), and that exactly the floor is adopted. The floor matches
|
||||
/// the gateway's own <c>GatewayOptionsValidator.MinimumMaxMessageBytes</c>, so the worker never
|
||||
/// rejects a value the gateway's validator accepts, yet a nonsensical tiny value faults at the
|
||||
/// handshake instead of leaving a session that fails every later frame.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void AdoptNegotiatedMaxMessageBytes_BelowFloor_ThrowsInvalidConfiguration()
|
||||
{
|
||||
WorkerFrameProtocolOptions belowFloor = CreateOptions();
|
||||
WorkerFrameProtocolException exception = Assert.Throws<WorkerFrameProtocolException>(
|
||||
() => belowFloor.AdoptNegotiatedMaxMessageBytes(512));
|
||||
Assert.Equal(WorkerFrameProtocolErrorCode.InvalidConfiguration, exception.ErrorCode);
|
||||
|
||||
// Boundary: exactly the floor is accepted.
|
||||
WorkerFrameProtocolOptions atFloor = CreateOptions();
|
||||
atFloor.AdoptNegotiatedMaxMessageBytes((uint)WorkerFrameProtocolOptions.MinNegotiableFrameBytes);
|
||||
Assert.Equal(WorkerFrameProtocolOptions.MinNegotiableFrameBytes, atFloor.MaxMessageBytes);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// WRK-22 / IPC-26. A <c>WriteAsync</c> cancelled while it waits for the write lock must never
|
||||
/// have its frame written by the next lock-holder. Writer A holds the lock mid-write (blocked in
|
||||
/// the stream); an event write is queued and then cancelled; when A is released and a later
|
||||
/// control frame drains, the wire carries A's frame and the control frame only — the cancelled
|
||||
/// event envelope is tombstoned and skipped.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task WriteAsync_CancelledWhileWaitingForLock_FrameIsNeverWritten()
|
||||
{
|
||||
WorkerFrameProtocolOptions options = CreateOptions();
|
||||
using GatedWriteStream stream = new();
|
||||
WorkerFrameWriter writer = new(stream, options);
|
||||
|
||||
// Writer A occupies the writer and blocks inside the stream, holding the write lock.
|
||||
Task firstWrite = writer.WriteAsync(CreateGatewayHelloEnvelope(), WorkerFrameWritePriority.Control);
|
||||
await AwaitWithTimeoutAsync(stream.FirstWriteStarted);
|
||||
|
||||
// Queue an event write with its own CTS while the lock is held, then cancel it.
|
||||
using CancellationTokenSource cts = new();
|
||||
Task cancelledWrite = writer.WriteAsync(CreateEventEnvelope(), WorkerFrameWritePriority.Event, cts.Token);
|
||||
await Task.Delay(50);
|
||||
cts.Cancel();
|
||||
await Assert.ThrowsAnyAsync<OperationCanceledException>(async () => await cancelledWrite);
|
||||
|
||||
// Release A, then drive a fresh control write.
|
||||
stream.ReleaseFirstWrite();
|
||||
await AwaitWithTimeoutAsync(firstWrite);
|
||||
await writer.WriteAsync(CreateGatewayHelloEnvelope(), WorkerFrameWritePriority.Control);
|
||||
|
||||
stream.Position = 0;
|
||||
WorkerFrameReader reader = new(stream, options);
|
||||
WorkerEnvelope frame1 = await reader.ReadAsync();
|
||||
WorkerEnvelope frame2 = await reader.ReadAsync();
|
||||
Assert.Equal(WorkerEnvelope.BodyOneofCase.GatewayHello, frame1.BodyCase);
|
||||
Assert.Equal(WorkerEnvelope.BodyOneofCase.GatewayHello, frame2.BodyCase);
|
||||
// The cancelled event never reached the wire — no third frame, and sequences stay contiguous.
|
||||
Assert.Equal(stream.Length, stream.Position);
|
||||
Assert.Equal(1UL, frame1.Sequence);
|
||||
Assert.Equal(2UL, frame2.Sequence);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// WRK-22 / IPC-26, the review's shutdown scenario. A cancelled event frame queued before a
|
||||
/// shutdown-ack control frame must not trail the ack on the wire: the tombstone rule plus the
|
||||
/// control-before-event scheduler keeps the ack the last frame written.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task WriteAsync_CancelledEventFrame_DoesNotTrailShutdownAck()
|
||||
{
|
||||
WorkerFrameProtocolOptions options = CreateOptions();
|
||||
using GatedWriteStream stream = new();
|
||||
WorkerFrameWriter writer = new(stream, options);
|
||||
|
||||
Task firstWrite = writer.WriteAsync(CreateGatewayHelloEnvelope(), WorkerFrameWritePriority.Control);
|
||||
await AwaitWithTimeoutAsync(stream.FirstWriteStarted);
|
||||
|
||||
using CancellationTokenSource cts = new();
|
||||
Task cancelledEvent = writer.WriteAsync(CreateEventEnvelope(), WorkerFrameWritePriority.Event, cts.Token);
|
||||
await Task.Delay(50);
|
||||
cts.Cancel();
|
||||
await Assert.ThrowsAnyAsync<OperationCanceledException>(async () => await cancelledEvent);
|
||||
|
||||
// The shutdown ack (a control frame) is queued behind the still-blocked first write.
|
||||
Task ackWrite = writer.WriteAsync(CreateShutdownAckEnvelope(), WorkerFrameWritePriority.Control);
|
||||
await Task.Delay(50);
|
||||
|
||||
stream.ReleaseFirstWrite();
|
||||
await AwaitWithTimeoutAsync(Task.WhenAll(firstWrite, ackWrite));
|
||||
|
||||
stream.Position = 0;
|
||||
WorkerFrameReader reader = new(stream, options);
|
||||
WorkerEnvelope frame1 = await reader.ReadAsync();
|
||||
WorkerEnvelope frame2 = await reader.ReadAsync();
|
||||
Assert.Equal(WorkerEnvelope.BodyOneofCase.GatewayHello, frame1.BodyCase);
|
||||
// The ack is the last frame — the cancelled event did not trail it.
|
||||
Assert.Equal(WorkerEnvelope.BodyOneofCase.WorkerShutdownAck, frame2.BodyCase);
|
||||
Assert.Equal(stream.Length, stream.Position);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// WRK-25. The batch entry point enqueues a whole event burst under one lock acquisition and
|
||||
/// drains it together, so N events cost exactly one flush and reach the wire in batch order with
|
||||
/// monotonic sequences — the coalescing WRK-12 shipped, now on the event hot path.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task WriteBatchAsync_FlushesOnceAndPreservesOrder()
|
||||
{
|
||||
const int count = 8;
|
||||
WorkerFrameProtocolOptions options = CreateOptions();
|
||||
using FlushCountingStream stream = new();
|
||||
WorkerFrameWriter writer = new(stream, options);
|
||||
|
||||
WorkerEnvelope[] batch = new WorkerEnvelope[count];
|
||||
for (int index = 0; index < count; index++)
|
||||
{
|
||||
batch[index] = CreateEventEnvelope(workerSequence: (ulong)(100 + index));
|
||||
}
|
||||
|
||||
await writer.WriteBatchAsync(batch, WorkerFrameWritePriority.Event);
|
||||
|
||||
// The whole batch was queued before the single lock wait, so it drained in one pass => one flush.
|
||||
Assert.Equal(1, stream.FlushCount);
|
||||
|
||||
stream.Position = 0;
|
||||
WorkerFrameReader reader = new(stream, options);
|
||||
for (int index = 0; index < count; index++)
|
||||
{
|
||||
WorkerEnvelope frame = await reader.ReadAsync();
|
||||
Assert.Equal(WorkerEnvelope.BodyOneofCase.WorkerEvent, frame.BodyCase);
|
||||
// Wire order matches batch order.
|
||||
Assert.Equal((ulong)(100 + index), frame.WorkerEvent.Event.WorkerSequence);
|
||||
// Write-time stamped sequence is monotonic 1..count.
|
||||
Assert.Equal((ulong)(index + 1), frame.Sequence);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// WRK-25. Control-before-event still holds mid-batch: a control frame queued while a batch is
|
||||
/// draining jumps ahead of the batch's remaining events.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task WriteBatchAsync_ControlFrameQueuedDuringBatch_JumpsRemainingEvents()
|
||||
{
|
||||
WorkerFrameProtocolOptions options = CreateOptions();
|
||||
using GatedWriteStream stream = new();
|
||||
WorkerFrameWriter writer = new(stream, options);
|
||||
|
||||
WorkerEnvelope[] batch = new[]
|
||||
{
|
||||
CreateEventEnvelope(),
|
||||
CreateEventEnvelope(),
|
||||
CreateEventEnvelope(),
|
||||
};
|
||||
|
||||
// The batch takes the lock and blocks writing its first event frame inside the stream.
|
||||
Task batchWrite = writer.WriteBatchAsync(batch, WorkerFrameWritePriority.Event);
|
||||
await AwaitWithTimeoutAsync(stream.FirstWriteStarted);
|
||||
|
||||
// A control frame queued mid-drain must jump the batch's remaining events.
|
||||
Task controlWrite = writer.WriteAsync(CreateGatewayHelloEnvelope(), WorkerFrameWritePriority.Control);
|
||||
await Task.Delay(50);
|
||||
|
||||
stream.ReleaseFirstWrite();
|
||||
await AwaitWithTimeoutAsync(Task.WhenAll(batchWrite, controlWrite));
|
||||
|
||||
stream.Position = 0;
|
||||
WorkerFrameReader reader = new(stream, options);
|
||||
WorkerEnvelope f1 = await reader.ReadAsync();
|
||||
WorkerEnvelope f2 = await reader.ReadAsync();
|
||||
WorkerEnvelope f3 = await reader.ReadAsync();
|
||||
WorkerEnvelope f4 = await reader.ReadAsync();
|
||||
|
||||
// First event was already writing when the control frame queued; the control frame then jumps
|
||||
// ahead of the two remaining events.
|
||||
Assert.Equal(WorkerEnvelope.BodyOneofCase.WorkerEvent, f1.BodyCase);
|
||||
Assert.Equal(WorkerEnvelope.BodyOneofCase.GatewayHello, f2.BodyCase);
|
||||
Assert.Equal(WorkerEnvelope.BodyOneofCase.WorkerEvent, f3.BodyCase);
|
||||
Assert.Equal(WorkerEnvelope.BodyOneofCase.WorkerEvent, f4.BodyCase);
|
||||
}
|
||||
|
||||
private static WorkerEnvelope CreateGatewayHelloEnvelope(ulong sequence = 1)
|
||||
{
|
||||
return new WorkerEnvelope
|
||||
@@ -522,6 +708,47 @@ public sealed class WorkerFrameProtocolTests
|
||||
};
|
||||
}
|
||||
|
||||
private static WorkerEnvelope CreateEventEnvelope(ulong workerSequence)
|
||||
{
|
||||
WorkerEnvelope envelope = CreateEventEnvelope();
|
||||
envelope.WorkerEvent.Event.WorkerSequence = workerSequence;
|
||||
return envelope;
|
||||
}
|
||||
|
||||
private static WorkerEnvelope CreateShutdownAckEnvelope()
|
||||
{
|
||||
return new WorkerEnvelope
|
||||
{
|
||||
ProtocolVersion = GatewayContractInfo.WorkerProtocolVersion,
|
||||
SessionId = SessionId,
|
||||
WorkerShutdownAck = new WorkerShutdownAck
|
||||
{
|
||||
Status = new ProtocolStatus
|
||||
{
|
||||
Code = ProtocolStatusCode.Ok,
|
||||
Message = "OK",
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// A MemoryStream that counts FlushAsync calls without gating any write, so a batch write can be
|
||||
// asserted to flush exactly once.
|
||||
private sealed class FlushCountingStream : MemoryStream
|
||||
{
|
||||
private int _flushCount;
|
||||
|
||||
/// <summary>Gets the number of <see cref="FlushAsync"/> calls observed so far.</summary>
|
||||
public int FlushCount => Volatile.Read(ref _flushCount);
|
||||
|
||||
/// <inheritdoc />
|
||||
public override Task FlushAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
Interlocked.Increment(ref _flushCount);
|
||||
return base.FlushAsync(cancellationToken);
|
||||
}
|
||||
}
|
||||
|
||||
// A MemoryStream whose first WriteAsync blocks until released, so a test can queue additional frames
|
||||
// behind an in-progress write and observe the writer's priority ordering.
|
||||
private sealed class GatedWriteStream : MemoryStream
|
||||
|
||||
@@ -1112,6 +1112,184 @@ public sealed class WorkerPipeSessionTests
|
||||
await SendShutdownAndWaitAsync(pipePair, runTask, cancellation.Token);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// WRK-27. An STA call outside the command dispatcher (the alarm poll) advertises itself on
|
||||
/// the heartbeat snapshot's <c>StaCallInProgress</c> flag, and the watchdog grants it the same
|
||||
/// grace-to-ceiling suppression as a dispatched command: stale STA activity within the ceiling
|
||||
/// does not fault while the flag is set, but stale activity beyond the ceiling faults anyway.
|
||||
/// This closes the 15 s-vs-75 s asymmetry between polls and commands.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task Watchdog_StaCallInProgress_SuppressedUntilCeiling()
|
||||
{
|
||||
using CancellationTokenSource cancellation = new(TimeSpan.FromSeconds(10));
|
||||
|
||||
// Phase 1 — within the ceiling: stale beyond grace, empty correlation id, StaCallInProgress
|
||||
// set. The default 75 s ceiling is far beyond the 5 s staleness, so the watchdog must suppress.
|
||||
using (PipePair pipePair = await PipePair.CreateAsync(cancellation.Token))
|
||||
{
|
||||
FakeRuntimeSession runtime = new();
|
||||
runtime.SetSnapshot(new WorkerRuntimeHeartbeatSnapshot(
|
||||
DateTimeOffset.UtcNow - TimeSpan.FromSeconds(5),
|
||||
pendingCommandCount: 0,
|
||||
outboundEventQueueDepth: 0,
|
||||
lastEventSequence: 0,
|
||||
currentCommandCorrelationId: string.Empty,
|
||||
staCallInProgress: true));
|
||||
WorkerPipeSession session = CreatePipeSession(
|
||||
pipePair.WorkerStream,
|
||||
runtime,
|
||||
new WorkerPipeSessionOptions
|
||||
{
|
||||
HeartbeatInterval = TimeSpan.FromMilliseconds(20),
|
||||
HeartbeatGrace = TimeSpan.FromMilliseconds(50),
|
||||
});
|
||||
Task runTask = session.RunAsync(cancellation.Token);
|
||||
await CompleteGatewayHandshakeAsync(pipePair, cancellation.Token);
|
||||
|
||||
const int framesToInspect = 6;
|
||||
int heartbeatsObserved = 0;
|
||||
for (int index = 0; index < framesToInspect; index++)
|
||||
{
|
||||
WorkerEnvelope envelope = await pipePair.GatewayReader.ReadAsync(cancellation.Token);
|
||||
Assert.NotEqual(WorkerEnvelope.BodyOneofCase.WorkerFault, envelope.BodyCase);
|
||||
if (envelope.BodyCase == WorkerEnvelope.BodyOneofCase.WorkerHeartbeat)
|
||||
{
|
||||
heartbeatsObserved++;
|
||||
}
|
||||
}
|
||||
|
||||
Assert.True(
|
||||
heartbeatsObserved >= 2,
|
||||
$"Expected multiple heartbeats during the in-progress STA-call window; observed {heartbeatsObserved}.");
|
||||
|
||||
await SendShutdownAndWaitAsync(pipePair, runTask, cancellation.Token);
|
||||
}
|
||||
|
||||
// Phase 2 — beyond the ceiling: same StaCallInProgress flag, but staleness (5 s) exceeds the
|
||||
// 200 ms ceiling, so the watchdog must fire even with the poll in progress.
|
||||
using (PipePair pipePair = await PipePair.CreateAsync(cancellation.Token))
|
||||
{
|
||||
FakeRuntimeSession runtime = new();
|
||||
runtime.SetSnapshot(new WorkerRuntimeHeartbeatSnapshot(
|
||||
DateTimeOffset.UtcNow - TimeSpan.FromSeconds(5),
|
||||
pendingCommandCount: 0,
|
||||
outboundEventQueueDepth: 0,
|
||||
lastEventSequence: 0,
|
||||
currentCommandCorrelationId: string.Empty,
|
||||
staCallInProgress: true));
|
||||
WorkerPipeSession session = CreatePipeSession(
|
||||
pipePair.WorkerStream,
|
||||
runtime,
|
||||
new WorkerPipeSessionOptions
|
||||
{
|
||||
HeartbeatInterval = TimeSpan.FromMilliseconds(20),
|
||||
HeartbeatGrace = TimeSpan.FromMilliseconds(50),
|
||||
HeartbeatStuckCeiling = TimeSpan.FromMilliseconds(200),
|
||||
});
|
||||
Task runTask = session.RunAsync(cancellation.Token);
|
||||
await CompleteGatewayHandshakeAsync(pipePair, cancellation.Token);
|
||||
|
||||
WorkerEnvelope fault = await ReadUntilAsync(
|
||||
pipePair.GatewayReader,
|
||||
WorkerEnvelope.BodyOneofCase.WorkerFault,
|
||||
cancellation.Token);
|
||||
|
||||
Assert.Equal(WorkerFaultCategory.StaHung, fault.WorkerFault.Category);
|
||||
|
||||
await SendShutdownAndWaitAsync(pipePair, runTask, cancellation.Token);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// WRK-25. The event drain loop submits a whole drained batch through the writer's batch entry
|
||||
/// point, so a burst of 128 events costs one flush, not 128 — the assertion the WRK-12
|
||||
/// tracking claim needed to actually hold on the event hot path.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task EventBurst_DrainLoopCoalescesFlushes()
|
||||
{
|
||||
using CancellationTokenSource cancellation = new(TimeSpan.FromSeconds(15));
|
||||
using PipePair pipePair = await PipePair.CreateAsync(cancellation.Token);
|
||||
FakeRuntimeSession runtime = new();
|
||||
// A far-off heartbeat interval keeps heartbeat flushes out of the measurement window.
|
||||
FlushCountingPassthroughStream countingStream = new(pipePair.WorkerStream);
|
||||
WorkerPipeSession session = CreatePipeSession(
|
||||
countingStream,
|
||||
runtime,
|
||||
new WorkerPipeSessionOptions
|
||||
{
|
||||
HeartbeatInterval = TimeSpan.FromMinutes(5),
|
||||
HeartbeatGrace = TimeSpan.FromSeconds(30),
|
||||
});
|
||||
Task runTask = session.RunAsync(cancellation.Token);
|
||||
await CompleteGatewayHandshakeAsync(pipePair, cancellation.Token);
|
||||
|
||||
// Let the idle drain loop settle (no events yet → no flushes) and record the baseline.
|
||||
await Task.Delay(100, cancellation.Token);
|
||||
int baselineFlushes = countingStream.FlushCount;
|
||||
|
||||
// Enqueue a full 128-event batch atomically so the drain loop sees it as one batch.
|
||||
const int burst = 128;
|
||||
List<WorkerEvent> batch = new(burst);
|
||||
for (int index = 0; index < burst; index++)
|
||||
{
|
||||
batch.Add(CreateWorkerEvent(sequence: (ulong)(index + 1)));
|
||||
}
|
||||
|
||||
runtime.EnqueueEvents(batch);
|
||||
|
||||
// Drain all 128 events off the gateway side.
|
||||
for (int index = 0; index < burst; index++)
|
||||
{
|
||||
await ReadUntilAsync(
|
||||
pipePair.GatewayReader,
|
||||
WorkerEnvelope.BodyOneofCase.WorkerEvent,
|
||||
cancellation.Token);
|
||||
}
|
||||
|
||||
// The whole burst cost exactly one additional flush.
|
||||
Assert.Equal(1, countingStream.FlushCount - baselineFlushes);
|
||||
|
||||
await SendShutdownAndWaitAsync(pipePair, runTask, cancellation.Token);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// WRK-24. A GatewayHello negotiating a frame maximum below the worker floor faults at the
|
||||
/// handshake with a fault frame rather than being adopted — mirroring the above-ceiling
|
||||
/// handshake behavior — so a nonsensical tiny value never leaves a session that fails every
|
||||
/// later frame. No message loop is entered.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task Handshake_GatewayHelloWithTinyMaxFrameBytes_FaultsAtHandshake()
|
||||
{
|
||||
WorkerFrameProtocolOptions options = CreateOptions();
|
||||
using MemoryStream inbound = new();
|
||||
await new WorkerFrameWriter(inbound, options)
|
||||
.WriteAsync(CreateGatewayHelloEnvelope(maxFrameBytes: 512));
|
||||
inbound.Position = 0;
|
||||
using MemoryStream outbound = new();
|
||||
WorkerPipeSession session = CreateSession(inbound, outbound, options);
|
||||
bool initialized = false;
|
||||
|
||||
WorkerFrameProtocolException exception =
|
||||
await Assert.ThrowsAsync<WorkerFrameProtocolException>(
|
||||
async () => await session.CompleteStartupHandshakeAsync(
|
||||
_ =>
|
||||
{
|
||||
initialized = true;
|
||||
return Task.CompletedTask;
|
||||
}));
|
||||
|
||||
Assert.False(initialized);
|
||||
Assert.Equal(WorkerFrameProtocolErrorCode.InvalidConfiguration, exception.ErrorCode);
|
||||
WorkerEnvelope fault = Assert.Single(ReadWrittenFrames(outbound, options));
|
||||
Assert.Equal(WorkerEnvelope.BodyOneofCase.WorkerFault, fault.BodyCase);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Regression test: a long in-flight STA command that keeps pumping
|
||||
/// must NOT self-fault as <c>StaHung</c>, and its reply must still be
|
||||
@@ -1976,6 +2154,78 @@ public sealed class WorkerPipeSessionTests
|
||||
}
|
||||
}
|
||||
|
||||
// Wraps the worker side of the pipe and counts FlushAsync calls so a test can assert the event
|
||||
// drain loop coalesces a burst into a single flush. Delegates every other operation to the inner
|
||||
// stream; does not own the inner stream's lifetime (PipePair disposes it).
|
||||
private sealed class FlushCountingPassthroughStream : Stream
|
||||
{
|
||||
private readonly Stream inner;
|
||||
private int flushCount;
|
||||
|
||||
/// <summary>Initializes the passthrough over the given inner stream.</summary>
|
||||
/// <param name="inner">The stream to delegate to.</param>
|
||||
public FlushCountingPassthroughStream(Stream inner)
|
||||
{
|
||||
this.inner = inner;
|
||||
}
|
||||
|
||||
/// <summary>Gets the number of <see cref="FlushAsync"/> calls observed so far.</summary>
|
||||
public int FlushCount => Volatile.Read(ref flushCount);
|
||||
|
||||
/// <inheritdoc />
|
||||
public override bool CanRead => inner.CanRead;
|
||||
|
||||
/// <inheritdoc />
|
||||
public override bool CanSeek => inner.CanSeek;
|
||||
|
||||
/// <inheritdoc />
|
||||
public override bool CanWrite => inner.CanWrite;
|
||||
|
||||
/// <inheritdoc />
|
||||
public override long Length => inner.Length;
|
||||
|
||||
/// <inheritdoc />
|
||||
public override long Position
|
||||
{
|
||||
get => inner.Position;
|
||||
set => inner.Position = value;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public override void Flush()
|
||||
{
|
||||
Interlocked.Increment(ref flushCount);
|
||||
inner.Flush();
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public override Task FlushAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
Interlocked.Increment(ref flushCount);
|
||||
return inner.FlushAsync(cancellationToken);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public override int Read(byte[] buffer, int offset, int count) => inner.Read(buffer, offset, count);
|
||||
|
||||
/// <inheritdoc />
|
||||
public override Task<int> ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken)
|
||||
=> inner.ReadAsync(buffer, offset, count, cancellationToken);
|
||||
|
||||
/// <inheritdoc />
|
||||
public override long Seek(long offset, SeekOrigin origin) => inner.Seek(offset, origin);
|
||||
|
||||
/// <inheritdoc />
|
||||
public override void SetLength(long value) => inner.SetLength(value);
|
||||
|
||||
/// <inheritdoc />
|
||||
public override void Write(byte[] buffer, int offset, int count) => inner.Write(buffer, offset, count);
|
||||
|
||||
/// <inheritdoc />
|
||||
public override Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken)
|
||||
=> inner.WriteAsync(buffer, offset, count, cancellationToken);
|
||||
}
|
||||
|
||||
private sealed class PipePair : IDisposable
|
||||
{
|
||||
private readonly NamedPipeServerStream gatewayStream;
|
||||
|
||||
@@ -438,6 +438,53 @@ public sealed class MxAccessStaSessionTests
|
||||
Assert.Contains("alarm poll failed", fault.DiagnosticMessage, StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// WRK-27. While the alarm poll's PollOnce is executing on the STA, a heartbeat captured mid-poll
|
||||
/// must report <see cref="WorkerRuntimeHeartbeatSnapshot.StaCallInProgress"/> so the watchdog
|
||||
/// grants the poll the same grace-to-ceiling suppression as a dispatched command; once the poll
|
||||
/// returns the flag clears. PollOnce is blocked on a gate so the heartbeat can be captured while
|
||||
/// the STA call is genuinely in flight.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task CaptureHeartbeat_DuringAlarmPoll_ReportsStaCallInProgress()
|
||||
{
|
||||
FakeAlarmCommandHandler handler = new() { BlockPoll = true };
|
||||
FakeMxAccessComObjectFactory factory = new();
|
||||
FakeMxAccessEventSink eventSink = new();
|
||||
using StaRuntime runtime = CreateRuntime();
|
||||
using MxAccessStaSession session = new(
|
||||
runtime,
|
||||
factory,
|
||||
eventSink,
|
||||
new MxAccessEventQueue(),
|
||||
(_eq, _affinity, _comFactory) => handler);
|
||||
|
||||
await session.StartAsync("session-1", workerProcessId: 1);
|
||||
|
||||
// Wait until PollOnce is blocked mid-call on the STA thread.
|
||||
Assert.True(
|
||||
handler.WaitForPollEntered(TimeSpan.FromSeconds(5)),
|
||||
"Expected the alarm poll to start within 5 seconds.");
|
||||
|
||||
// Captured mid-poll, the heartbeat advertises the in-progress STA call.
|
||||
Assert.True(session.CaptureHeartbeat().StaCallInProgress);
|
||||
|
||||
// Release the poll and stop blocking; the flag clears once the poll returns.
|
||||
handler.BlockPoll = false;
|
||||
handler.ReleasePoll();
|
||||
|
||||
using CancellationTokenSource timeout = new(TimeSpan.FromSeconds(5));
|
||||
while (session.CaptureHeartbeat().StaCallInProgress && !timeout.IsCancellationRequested)
|
||||
{
|
||||
await Task.Delay(25, CancellationToken.None);
|
||||
}
|
||||
|
||||
Assert.False(
|
||||
session.CaptureHeartbeat().StaCallInProgress,
|
||||
"Expected StaCallInProgress to clear once the alarm poll returned.");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The STA-affinity guard throws when an
|
||||
/// IMxAccessAlarmConsumer call is attempted off the thread that created
|
||||
@@ -472,6 +519,8 @@ public sealed class MxAccessStaSessionTests
|
||||
private sealed class FakeAlarmCommandHandler : IAlarmCommandHandler
|
||||
{
|
||||
private readonly object gate = new object();
|
||||
private readonly ManualResetEventSlim pollEntered = new(false);
|
||||
private readonly ManualResetEventSlim releasePoll = new(false);
|
||||
private int pollCount;
|
||||
private int? lastPollThreadId;
|
||||
|
||||
@@ -484,6 +533,17 @@ public sealed class MxAccessStaSessionTests
|
||||
/// <summary>Exception thrown by PollOnce; null to succeed.</summary>
|
||||
public Exception? PollException { get; set; }
|
||||
|
||||
/// <summary>When set, <see cref="PollOnce"/> blocks until <see cref="ReleasePoll"/> is called.</summary>
|
||||
public bool BlockPoll { get; set; }
|
||||
|
||||
/// <summary>Waits until a blocking <see cref="PollOnce"/> has entered and is blocked.</summary>
|
||||
/// <param name="timeout">Maximum time to wait.</param>
|
||||
/// <returns>True if a poll entered within the timeout.</returns>
|
||||
public bool WaitForPollEntered(TimeSpan timeout) => pollEntered.Wait(timeout);
|
||||
|
||||
/// <summary>Releases a <see cref="PollOnce"/> blocked on the gate.</summary>
|
||||
public void ReleasePoll() => releasePoll.Set();
|
||||
|
||||
/// <summary>Gets the count of PollOnce calls.</summary>
|
||||
public int PollCount
|
||||
{
|
||||
@@ -533,6 +593,12 @@ public sealed class MxAccessStaSessionTests
|
||||
lastPollThreadId = Thread.CurrentThread.ManagedThreadId;
|
||||
}
|
||||
|
||||
if (BlockPoll)
|
||||
{
|
||||
pollEntered.Set();
|
||||
releasePoll.Wait(TimeSpan.FromSeconds(10));
|
||||
}
|
||||
|
||||
if (PollException is not null)
|
||||
{
|
||||
throw PollException;
|
||||
@@ -540,6 +606,10 @@ public sealed class MxAccessStaSessionTests
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public void Dispose() { }
|
||||
public void Dispose()
|
||||
{
|
||||
pollEntered.Dispose();
|
||||
releasePoll.Dispose();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -372,6 +372,23 @@ internal sealed class FakeRuntimeSession : IWorkerRuntimeSession
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Enqueues a batch of worker events atomically under one lock so the drain loop cannot
|
||||
/// observe a partial batch. Lets a test assert the drain loop coalesces a whole batch into one
|
||||
/// flush (WRK-25) without racing a mid-enqueue drain that would split the batch.
|
||||
/// </summary>
|
||||
/// <param name="workerEvents">The events to enqueue in order.</param>
|
||||
public void EnqueueEvents(IEnumerable<WorkerEvent> workerEvents)
|
||||
{
|
||||
lock (gate)
|
||||
{
|
||||
foreach (WorkerEvent workerEvent in workerEvents)
|
||||
{
|
||||
events.Enqueue(workerEvent);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public void Dispose()
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user