feat(worker): adopt negotiated frame max, bound drain, priority write scheduler (IPC-02/04 + WRK-04/07 worker half)
Worker half of the Wave 3 size/backpressure + write-ordering pass: - IPC-02: the worker adopts GatewayHello.max_frame_bytes during the handshake (WorkerFrameProtocolOptions.AdoptNegotiatedMaxMessageBytes) instead of a hard-coded default; 0 keeps the default, a value above a 256 MiB ceiling is rejected. Reader and writer share the options instance, applied before the message loop. - IPC-04: DrainEvents caps each reply at MaxDrainEventsPerReply (10_000) and treats max_events = 0 as that cap rather than 'drain the entire queue', so one diagnostic drain cannot pack a session-killing reply frame. - WRK-04: WorkerFrameWriter stamps the envelope Sequence at the actual point of writing (under the write lock) instead of at envelope creation, so the on-wire order and the stamped sequence always agree under concurrent producers. - WRK-07: the writer is now a cooperative priority scheduler — callers enqueue at Control or Event priority and the draining lock-holder writes all control frames before any event frame, so replies/faults/heartbeats jump ahead of an event backlog. Per-frame validation/size rejections fail only that frame; a stream write failure fails all queued frames. Tests: monotonic gap-free sequence under concurrency, control-before-event priority (gated stream), negotiated-max adoption, DrainEvents zero-bound. Worker builds x86 only — verified on windev.
This commit is contained in:
Joseph Doherty
@@ -1,5 +1,7 @@
|
||||
using System;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using ZB.MOM.WW.MxGateway.Contracts;
|
||||
using ZB.MOM.WW.MxGateway.Contracts.Proto;
|
||||
@@ -305,6 +307,101 @@ public sealed class WorkerFrameProtocolTests
|
||||
Nonce);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies that under concurrent writers every frame receives a distinct, gap-free sequence in
|
||||
/// strictly increasing on-wire order — the sequence is stamped by the writer at write time, so the
|
||||
/// wire order and the stamped sequence always agree (WRK-04).
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task WriteAsync_UnderConcurrentCalls_StampsGapFreeMonotonicSequence()
|
||||
{
|
||||
const int frameCount = 50;
|
||||
WorkerFrameProtocolOptions options = CreateOptions();
|
||||
using MemoryStream stream = new();
|
||||
WorkerFrameWriter writer = new(stream, options);
|
||||
|
||||
await Task.WhenAll(
|
||||
Enumerable.Range(0, frameCount).Select(_ => writer.WriteAsync(CreateEventEnvelope())));
|
||||
|
||||
stream.Position = 0;
|
||||
WorkerFrameReader reader = new(stream, options);
|
||||
ulong[] sequences = new ulong[frameCount];
|
||||
for (int index = 0; index < frameCount; index++)
|
||||
{
|
||||
sequences[index] = (await reader.ReadAsync()).Sequence;
|
||||
}
|
||||
|
||||
// On-wire order is strictly increasing 1..frameCount with no gaps or duplicates.
|
||||
Assert.Equal(Enumerable.Range(1, frameCount).Select(value => (ulong)value), sequences);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies that when a control frame and an event frame are both queued behind an in-progress
|
||||
/// write, the draining lock-holder writes the control frame first even though the event was queued
|
||||
/// earlier (WRK-07).
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task WriteAsync_WhenControlAndEventQueuedTogether_WritesControlFirst()
|
||||
{
|
||||
WorkerFrameProtocolOptions options = CreateOptions();
|
||||
using GatedWriteStream stream = new();
|
||||
WorkerFrameWriter writer = new(stream, options);
|
||||
|
||||
// First write occupies the writer and blocks inside the stream, holding the write lock.
|
||||
Task firstWrite = writer.WriteAsync(CreateGatewayHelloEnvelope(), WorkerFrameWritePriority.Control);
|
||||
await AwaitWithTimeoutAsync(stream.FirstWriteStarted);
|
||||
|
||||
// Queue an event first, then a control frame, while the writer is blocked. Both wait for the lock.
|
||||
Task eventWrite = writer.WriteAsync(CreateEventEnvelope(), WorkerFrameWritePriority.Event);
|
||||
Task controlWrite = writer.WriteAsync(CreateGatewayHelloEnvelope(), WorkerFrameWritePriority.Control);
|
||||
await Task.Delay(50);
|
||||
|
||||
stream.ReleaseFirstWrite();
|
||||
await AwaitWithTimeoutAsync(Task.WhenAll(firstWrite, eventWrite, controlWrite));
|
||||
|
||||
stream.Position = 0;
|
||||
WorkerFrameReader reader = new(stream, options);
|
||||
WorkerEnvelope frame1 = await reader.ReadAsync();
|
||||
WorkerEnvelope frame2 = await reader.ReadAsync();
|
||||
WorkerEnvelope frame3 = await reader.ReadAsync();
|
||||
|
||||
Assert.Equal(WorkerEnvelope.BodyOneofCase.GatewayHello, frame1.BodyCase);
|
||||
// The control frame jumped ahead of the earlier-queued event.
|
||||
Assert.Equal(WorkerEnvelope.BodyOneofCase.GatewayHello, frame2.BodyCase);
|
||||
Assert.Equal(WorkerEnvelope.BodyOneofCase.WorkerEvent, frame3.BodyCase);
|
||||
}
|
||||
|
||||
/// <summary>Verifies a zero negotiated frame maximum keeps the constructor default (IPC-02).</summary>
|
||||
[Fact]
|
||||
public void AdoptNegotiatedMaxMessageBytes_WithZero_KeepsDefault()
|
||||
{
|
||||
WorkerFrameProtocolOptions options = CreateOptions();
|
||||
int original = options.MaxMessageBytes;
|
||||
options.AdoptNegotiatedMaxMessageBytes(0);
|
||||
Assert.Equal(original, options.MaxMessageBytes);
|
||||
}
|
||||
|
||||
/// <summary>Verifies an in-range negotiated frame maximum is adopted (IPC-02).</summary>
|
||||
[Fact]
|
||||
public void AdoptNegotiatedMaxMessageBytes_WithInRangeValue_Adopts()
|
||||
{
|
||||
WorkerFrameProtocolOptions options = CreateOptions();
|
||||
options.AdoptNegotiatedMaxMessageBytes(4 * 1024 * 1024);
|
||||
Assert.Equal(4 * 1024 * 1024, options.MaxMessageBytes);
|
||||
}
|
||||
|
||||
/// <summary>Verifies a negotiated frame maximum above the worker ceiling is rejected (IPC-02).</summary>
|
||||
[Fact]
|
||||
public void AdoptNegotiatedMaxMessageBytes_AboveCeiling_Throws()
|
||||
{
|
||||
WorkerFrameProtocolOptions options = CreateOptions();
|
||||
WorkerFrameProtocolException exception = Assert.Throws<WorkerFrameProtocolException>(
|
||||
() => options.AdoptNegotiatedMaxMessageBytes((uint)WorkerFrameProtocolOptions.MaxNegotiableFrameBytes + 1));
|
||||
Assert.Equal(WorkerFrameProtocolErrorCode.InvalidConfiguration, exception.ErrorCode);
|
||||
}
|
||||
|
||||
private static WorkerEnvelope CreateGatewayHelloEnvelope(ulong sequence = 1)
|
||||
{
|
||||
return new WorkerEnvelope
|
||||
@@ -321,4 +418,63 @@ public sealed class WorkerFrameProtocolTests
|
||||
};
|
||||
}
|
||||
|
||||
// net48 has no Task.WaitAsync(TimeSpan); fail the test rather than hang if the writer misbehaves.
|
||||
private static async Task AwaitWithTimeoutAsync(Task task)
|
||||
{
|
||||
Task completed = await Task.WhenAny(task, Task.Delay(TimeSpan.FromSeconds(5)));
|
||||
if (completed != task)
|
||||
{
|
||||
throw new TimeoutException("Timed out waiting for the frame writer.");
|
||||
}
|
||||
|
||||
await task;
|
||||
}
|
||||
|
||||
private static WorkerEnvelope CreateEventEnvelope()
|
||||
{
|
||||
return new WorkerEnvelope
|
||||
{
|
||||
ProtocolVersion = GatewayContractInfo.WorkerProtocolVersion,
|
||||
SessionId = SessionId,
|
||||
WorkerEvent = new WorkerEvent
|
||||
{
|
||||
Event = new MxEvent { SessionId = SessionId },
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// A MemoryStream whose first WriteAsync blocks until released, so a test can queue additional frames
|
||||
// behind an in-progress write and observe the writer's priority ordering.
|
||||
private sealed class GatedWriteStream : MemoryStream
|
||||
{
|
||||
private readonly SemaphoreSlim _release = new SemaphoreSlim(0);
|
||||
private readonly TaskCompletionSource<bool> _firstWriteStarted =
|
||||
new TaskCompletionSource<bool>(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
private int _writeCount;
|
||||
|
||||
public Task FirstWriteStarted => _firstWriteStarted.Task;
|
||||
|
||||
public void ReleaseFirstWrite() => _release.Release();
|
||||
|
||||
public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken)
|
||||
{
|
||||
if (Interlocked.Increment(ref _writeCount) == 1)
|
||||
{
|
||||
_firstWriteStarted.TrySetResult(true);
|
||||
await _release.WaitAsync(cancellationToken);
|
||||
}
|
||||
|
||||
await base.WriteAsync(buffer, offset, count, cancellationToken);
|
||||
}
|
||||
|
||||
protected override void Dispose(bool disposing)
|
||||
{
|
||||
if (disposing)
|
||||
{
|
||||
_release.Dispose();
|
||||
}
|
||||
|
||||
base.Dispose(disposing);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user