Joseph Doherty
f40409a688
Tracking IDs are review bookkeeping that rots in shipped source; the rationale they tagged is kept. Comment-only change (CommentChecker 458 -> 16 false positives).
1482 lines
69 KiB
C#
1482 lines
69 KiB
C#
using System.IO.Pipes;
|
||
using Google.Protobuf.WellKnownTypes;
|
||
using ZB.MOM.WW.MxGateway.Contracts;
|
||
using ZB.MOM.WW.MxGateway.Contracts.Proto;
|
||
using ZB.MOM.WW.MxGateway.Server.Metrics;
|
||
using ZB.MOM.WW.MxGateway.Server.Workers;
|
||
using ZB.MOM.WW.MxGateway.Tests.Gateway.Workers.Fakes;
|
||
using ZB.MOM.WW.MxGateway.Tests.TestSupport;
|
||
|
||
namespace ZB.MOM.WW.MxGateway.Tests.Gateway.Workers;
|
||
|
||
public sealed class WorkerClientTests
|
||
{
|
||
private const string SessionId = "session-worker-client";
|
||
private const string Nonce = "nonce-worker-client";
|
||
private const int WorkerProcessId = 4321;
|
||
|
||
/// <summary>Upper bound on the events <see cref="EmitEventsUntilStoppedAsync"/> writes.</summary>
|
||
private const ulong EmittedEventCeiling = 2000;
|
||
|
||
private static readonly TimeSpan TestTimeout = TimeSpan.FromSeconds(5);
|
||
|
||
/// <summary>Verifies that StartAsync enters ready state after receiving worker hello and ready messages.</summary>
|
||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||
[Fact]
|
||
public async Task StartAsync_WithWorkerHelloAndReady_EntersReadyState()
|
||
{
|
||
await using PipePair pipePair = await PipePair.CreateAsync();
|
||
await using WorkerClient client = CreateClient(pipePair);
|
||
|
||
await CompleteHandshakeAsync(client, pipePair);
|
||
|
||
Assert.Equal(WorkerClientState.Ready, client.State);
|
||
Assert.Equal(WorkerProcessId, client.ProcessId);
|
||
}
|
||
|
||
/// <summary>
|
||
/// The <c>GatewayHello</c> carries the negotiated worker-frame maximum so the worker adopts the
|
||
/// configured limit instead of its own default; a regression to sending <c>0</c> means
|
||
/// "older gateway, use default" to the worker and would silently downgrade the negotiated limit.
|
||
/// The adoption half is asserted only in the Windows-only worker suite, so the gateway half is
|
||
/// pinned here, in the portable suite. Both the default and an override are covered so
|
||
/// the assertion tracks configuration rather than a constant.
|
||
/// </summary>
|
||
/// <param name="maxMessageBytes">Configured worker-frame maximum to negotiate.</param>
|
||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||
[Theory]
|
||
[InlineData(WorkerFrameProtocolOptions.DefaultMaxMessageBytes)]
|
||
[InlineData(2 * 1024 * 1024)]
|
||
public async Task StartAsync_SendsGatewayHelloWithConfiguredMaxFrameBytes(int maxMessageBytes)
|
||
{
|
||
await using FakeWorkerHarness harness =
|
||
await FakeWorkerHarness.CreateConnectedPairAsync(maxMessageBytes: maxMessageBytes);
|
||
await using WorkerClient client = harness.CreateClient();
|
||
|
||
Task startTask = client.StartAsync(CancellationToken.None);
|
||
WorkerEnvelope gatewayHello = await harness.CompleteStartupAsync().WaitAsync(TestTimeout);
|
||
await startTask.WaitAsync(TestTimeout);
|
||
|
||
Assert.NotEqual(0u, gatewayHello.GatewayHello.MaxFrameBytes);
|
||
Assert.Equal((uint)maxMessageBytes, gatewayHello.GatewayHello.MaxFrameBytes);
|
||
}
|
||
|
||
/// <summary>Verifies that InvokeAsync completes a pending command when a matching reply arrives.</summary>
|
||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||
[Fact]
|
||
public async Task InvokeAsync_WithMatchingReply_CompletesPendingCommand()
|
||
{
|
||
await using PipePair pipePair = await PipePair.CreateAsync();
|
||
await using WorkerClient client = CreateClient(pipePair);
|
||
await CompleteHandshakeAsync(client, pipePair);
|
||
|
||
Task<WorkerCommandReply> invokeTask = client.InvokeAsync(
|
||
CreateCommand(MxCommandKind.Ping),
|
||
TestTimeout,
|
||
CancellationToken.None);
|
||
|
||
WorkerEnvelope commandEnvelope = await pipePair.WorkerReader.ReadAsync().AsTask().WaitAsync(TestTimeout);
|
||
Assert.Equal(WorkerEnvelope.BodyOneofCase.WorkerCommand, commandEnvelope.BodyCase);
|
||
Assert.False(string.IsNullOrWhiteSpace(commandEnvelope.CorrelationId));
|
||
|
||
await pipePair.WriteAsync(
|
||
CreateCommandReplyEnvelope(commandEnvelope.CorrelationId, MxCommandKind.Ping));
|
||
|
||
WorkerCommandReply reply = await invokeTask.WaitAsync(TestTimeout);
|
||
|
||
Assert.Equal(commandEnvelope.CorrelationId, reply.Reply.CorrelationId);
|
||
Assert.Equal(MxCommandKind.Ping, reply.Reply.Kind);
|
||
}
|
||
|
||
/// <summary>
|
||
/// Verifies that a command whose serialized envelope exceeds the negotiated worker-frame maximum
|
||
/// fails only that command with <see cref="WorkerClientErrorCode.CommandTooLarge"/> at the enqueue
|
||
/// boundary, leaving the client ready for subsequent commands. Without the pre-check the
|
||
/// oversized frame would reach the write loop and fault the whole session.
|
||
/// </summary>
|
||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||
[Fact]
|
||
public async Task InvokeAsync_WhenCommandExceedsFrameMax_FailsOnlyThatCommandAndStaysReady()
|
||
{
|
||
await using PipePair pipePair = await PipePair.CreateAsync();
|
||
await using WorkerClient client = CreateClient(pipePair, maxMessageBytes: 4096);
|
||
await CompleteHandshakeAsync(client, pipePair);
|
||
|
||
WorkerCommand oversized = new()
|
||
{
|
||
Command = new MxCommand
|
||
{
|
||
Kind = MxCommandKind.Write,
|
||
Write = new WriteCommand
|
||
{
|
||
ServerHandle = 1,
|
||
ItemHandle = 2,
|
||
Value = new MxValue { StringValue = new string('x', 8192) },
|
||
},
|
||
},
|
||
};
|
||
|
||
WorkerClientException exception = await Assert.ThrowsAsync<WorkerClientException>(
|
||
async () => await client.InvokeAsync(oversized, TestTimeout, CancellationToken.None));
|
||
Assert.Equal(WorkerClientErrorCode.CommandTooLarge, exception.ErrorCode);
|
||
Assert.Equal(WorkerClientState.Ready, client.State);
|
||
|
||
// A subsequent normally-sized command still round-trips: the session was not faulted.
|
||
Task<WorkerCommandReply> nextInvoke = client.InvokeAsync(
|
||
CreateCommand(MxCommandKind.Ping),
|
||
TestTimeout,
|
||
CancellationToken.None);
|
||
WorkerEnvelope commandEnvelope = await pipePair.WorkerReader.ReadAsync().AsTask().WaitAsync(TestTimeout);
|
||
await pipePair.WriteAsync(
|
||
CreateCommandReplyEnvelope(commandEnvelope.CorrelationId, MxCommandKind.Ping));
|
||
WorkerCommandReply reply = await nextInvoke.WaitAsync(TestTimeout);
|
||
Assert.Equal(MxCommandKind.Ping, reply.Reply.Kind);
|
||
Assert.Equal(WorkerClientState.Ready, client.State);
|
||
}
|
||
|
||
/// <summary>Verifies that InvokeAsync ignores late replies and keeps the client ready.</summary>
|
||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||
[Fact]
|
||
public async Task InvokeAsync_WithLateReply_IgnoresLateReplyAndKeepsClientReady()
|
||
{
|
||
await using PipePair pipePair = await PipePair.CreateAsync();
|
||
await using WorkerClient client = CreateClient(pipePair);
|
||
await CompleteHandshakeAsync(client, pipePair);
|
||
|
||
Task<WorkerCommandReply> timedOutInvokeTask = client.InvokeAsync(
|
||
CreateCommand(MxCommandKind.Ping),
|
||
TimeSpan.FromMilliseconds(50),
|
||
CancellationToken.None);
|
||
WorkerEnvelope timedOutCommand = await pipePair.WorkerReader.ReadAsync().AsTask().WaitAsync(TestTimeout);
|
||
|
||
WorkerClientException exception = await Assert.ThrowsAsync<WorkerClientException>(
|
||
async () => await timedOutInvokeTask);
|
||
Assert.Equal(WorkerClientErrorCode.CommandTimeout, exception.ErrorCode);
|
||
|
||
// Send the stale reply for the already-timed-out command, then the second
|
||
// command's reply. The pipe is FIFO, so the read loop processes (and discards)
|
||
// the stale reply before the second reply — no fixed Task.Delay needed.
|
||
await pipePair.WriteAsync(
|
||
CreateCommandReplyEnvelope(timedOutCommand.CorrelationId, MxCommandKind.Ping));
|
||
|
||
Task<WorkerCommandReply> secondInvokeTask = client.InvokeAsync(
|
||
CreateCommand(MxCommandKind.GetWorkerInfo),
|
||
TestTimeout,
|
||
CancellationToken.None);
|
||
|
||
// The timeout also emits a WorkerCancel for the abandoned correlation, which sits
|
||
// ahead of the second command on the FIFO pipe; skip it rather than mistaking it for the
|
||
// command this assertion is about.
|
||
WorkerEnvelope secondCommand = await ReadNextCommandAsync(pipePair, timedOutCommand.CorrelationId);
|
||
await pipePair.WriteAsync(
|
||
CreateCommandReplyEnvelope(secondCommand.CorrelationId, MxCommandKind.GetWorkerInfo));
|
||
|
||
WorkerCommandReply reply = await secondInvokeTask.WaitAsync(TestTimeout);
|
||
|
||
Assert.Equal(WorkerClientState.Ready, client.State);
|
||
Assert.Equal(MxCommandKind.GetWorkerInfo, reply.Reply.Kind);
|
||
}
|
||
|
||
/// <summary>
|
||
/// A command timeout abandons the gateway-side wait, but the worker keeps the correlation on
|
||
/// its single STA queue and would still run it — so the gateway forwards a <c>WorkerCancel</c>
|
||
/// for the abandoned correlation id. Without it, a client that retries after a
|
||
/// timeout stacks work the worker still intends to execute. Asserted on the wire because the
|
||
/// cancel is protocol behavior the worker depends on, not an internal detail.
|
||
/// </summary>
|
||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||
[Fact]
|
||
public async Task InvokeAsync_WhenCommandTimesOut_SendsWorkerCancelForThatCorrelation()
|
||
{
|
||
await using PipePair pipePair = await PipePair.CreateAsync();
|
||
await using WorkerClient client = CreateClient(pipePair);
|
||
await CompleteHandshakeAsync(client, pipePair);
|
||
|
||
// Advise rather than a control command: control commands bypass the worker's STA queue, so a
|
||
// data command is the case the cancel actually exists for.
|
||
Task<WorkerCommandReply> invokeTask = client.InvokeAsync(
|
||
CreateCommand(MxCommandKind.Advise),
|
||
TimeSpan.FromMilliseconds(50),
|
||
CancellationToken.None);
|
||
|
||
WorkerEnvelope commandEnvelope = await pipePair.WorkerReader.ReadAsync().AsTask().WaitAsync(TestTimeout);
|
||
Assert.Equal(WorkerEnvelope.BodyOneofCase.WorkerCommand, commandEnvelope.BodyCase);
|
||
|
||
WorkerClientException exception = await Assert.ThrowsAsync<WorkerClientException>(
|
||
async () => await invokeTask.WaitAsync(TestTimeout));
|
||
Assert.Equal(WorkerClientErrorCode.CommandTimeout, exception.ErrorCode);
|
||
|
||
WorkerEnvelope cancelEnvelope = await pipePair.WorkerReader.ReadAsync().AsTask().WaitAsync(TestTimeout);
|
||
|
||
Assert.Equal(WorkerEnvelope.BodyOneofCase.WorkerCancel, cancelEnvelope.BodyCase);
|
||
Assert.Equal(commandEnvelope.CorrelationId, cancelEnvelope.CorrelationId);
|
||
Assert.False(string.IsNullOrWhiteSpace(cancelEnvelope.WorkerCancel.Reason));
|
||
Assert.True(
|
||
cancelEnvelope.Sequence > commandEnvelope.Sequence,
|
||
$"The cancel arrived with sequence {cancelEnvelope.Sequence} after {commandEnvelope.Sequence}; "
|
||
+ "envelope sequences must be strictly increasing in wire order.");
|
||
|
||
// The timeout fails one command; it is not a session fault.
|
||
Assert.Equal(WorkerClientState.Ready, client.State);
|
||
}
|
||
|
||
/// <summary>
|
||
/// Caller cancellation abandons the correlation exactly as a gateway timeout does — a client
|
||
/// deadline shorter than the gateway's command timeout, or a disconnect, is the common
|
||
/// abandonment path — so it forwards the same <c>WorkerCancel</c>. Without it a
|
||
/// retry storm of deadline-cancelled calls leaves the worker's single STA working through
|
||
/// commands nobody is waiting for while live callers queue behind them. Twin of
|
||
/// <see cref="InvokeAsync_WhenCommandTimesOut_SendsWorkerCancelForThatCorrelation"/>, driven
|
||
/// by the caller's token instead of the timer.
|
||
/// </summary>
|
||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||
[Fact]
|
||
public async Task InvokeAsync_WhenCallerCancels_SendsWorkerCancelForThatCorrelation()
|
||
{
|
||
await using PipePair pipePair = await PipePair.CreateAsync();
|
||
await using WorkerClient client = CreateClient(pipePair);
|
||
await CompleteHandshakeAsync(client, pipePair);
|
||
|
||
using CancellationTokenSource callerCts = new();
|
||
|
||
// Advise rather than a control command: control commands bypass the worker's STA queue, so a
|
||
// data command is the case the cancel actually exists for. The timeout is the full test
|
||
// timeout so the cancellation, not the timer, is unambiguously what abandons the command.
|
||
Task<WorkerCommandReply> invokeTask = client.InvokeAsync(
|
||
CreateCommand(MxCommandKind.Advise),
|
||
TestTimeout,
|
||
callerCts.Token);
|
||
|
||
WorkerEnvelope commandEnvelope = await pipePair.WorkerReader.ReadAsync().AsTask().WaitAsync(TestTimeout);
|
||
Assert.Equal(WorkerEnvelope.BodyOneofCase.WorkerCommand, commandEnvelope.BodyCase);
|
||
|
||
await callerCts.CancelAsync();
|
||
|
||
await Assert.ThrowsAnyAsync<OperationCanceledException>(
|
||
async () => await invokeTask.WaitAsync(TestTimeout));
|
||
|
||
WorkerEnvelope cancelEnvelope = await pipePair.WorkerReader.ReadAsync().AsTask().WaitAsync(TestTimeout);
|
||
|
||
Assert.Equal(WorkerEnvelope.BodyOneofCase.WorkerCancel, cancelEnvelope.BodyCase);
|
||
Assert.Equal(commandEnvelope.CorrelationId, cancelEnvelope.CorrelationId);
|
||
Assert.False(string.IsNullOrWhiteSpace(cancelEnvelope.WorkerCancel.Reason));
|
||
Assert.True(
|
||
cancelEnvelope.Sequence > commandEnvelope.Sequence,
|
||
$"The cancel arrived with sequence {cancelEnvelope.Sequence} after {commandEnvelope.Sequence}; "
|
||
+ "envelope sequences must be strictly increasing in wire order.");
|
||
|
||
// One caller gave up; the session did not fault.
|
||
Assert.Equal(WorkerClientState.Ready, client.State);
|
||
}
|
||
|
||
/// <summary>
|
||
/// The cancel is forwarded for a caller that abandoned its command, never for the client's own
|
||
/// teardown: once the stop token has fired the outbound channel is being completed and
|
||
/// every pending command is failed anyway, so a cancel would at best be pointless and at worst
|
||
/// a write racing teardown. Asserted on the wire — the guard in the cancellation arm is the
|
||
/// code-level expression of it, and the completed outbound channel is the second line of
|
||
/// defense, so this pins the observable contract rather than either mechanism.
|
||
/// </summary>
|
||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||
[Fact]
|
||
public async Task InvokeAsync_WhenClientStopsUnderInFlightCommand_DoesNotSendWorkerCancel()
|
||
{
|
||
await using PipePair pipePair = await PipePair.CreateAsync();
|
||
await using WorkerClient client = CreateClient(pipePair);
|
||
await CompleteHandshakeAsync(client, pipePair);
|
||
|
||
using CancellationTokenSource callerCts = new();
|
||
Task<WorkerCommandReply> invokeTask = client.InvokeAsync(
|
||
CreateCommand(MxCommandKind.Advise),
|
||
TestTimeout,
|
||
callerCts.Token);
|
||
|
||
WorkerEnvelope commandEnvelope = await pipePair.WorkerReader.ReadAsync().AsTask().WaitAsync(TestTimeout);
|
||
Assert.Equal(WorkerEnvelope.BodyOneofCase.WorkerCommand, commandEnvelope.BodyCase);
|
||
|
||
// Read ahead before tearing down so anything the teardown path puts on the wire is caught.
|
||
// Exceptions are swallowed: the read is expected to be still pending at the assertion, and
|
||
// the stream is disposed underneath it when the test ends.
|
||
Task<WorkerEnvelope?> pendingRead = Task.Run(async () =>
|
||
{
|
||
try
|
||
{
|
||
return await pipePair.WorkerReader.ReadAsync();
|
||
}
|
||
catch (Exception)
|
||
{
|
||
return null;
|
||
}
|
||
});
|
||
|
||
// Kill fires the client's stop token and then fails the pending command; a caller token that
|
||
// is wired to gateway shutdown (as the session's is) cancels in the same window.
|
||
client.Kill("test teardown");
|
||
await callerCts.CancelAsync();
|
||
|
||
await Assert.ThrowsAnyAsync<Exception>(async () => await invokeTask.WaitAsync(TestTimeout));
|
||
|
||
await Task.Delay(TimeSpan.FromMilliseconds(250));
|
||
if (pendingRead.IsCompleted)
|
||
{
|
||
WorkerEnvelope? unexpected = await pendingRead;
|
||
Assert.True(
|
||
unexpected is null || unexpected.BodyCase != WorkerEnvelope.BodyOneofCase.WorkerCancel,
|
||
"A stopping worker client must not put a WorkerCancel on the wire.");
|
||
}
|
||
}
|
||
|
||
/// <summary>
|
||
/// A <c>WorkerFault</c> arriving in place of a handshake frame is the worker reporting a
|
||
/// structured startup failure — most often MXAccess COM creation — not a protocol problem, so
|
||
/// the gateway surfaces <see cref="WorkerClientErrorCode.WorkerFaulted"/> with the fault's
|
||
/// category and diagnostic rather than a generic protocol violation naming only the body type. This is the window where the information matters most: first session open on a
|
||
/// freshly deployed host, where the message reaches the operator log and the client's
|
||
/// <c>OpenFailed</c> detail.
|
||
/// </summary>
|
||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||
[Fact]
|
||
public async Task StartAsync_WhenWorkerFaultsDuringHandshake_SurfacesFaultCategoryAndDiagnostic()
|
||
{
|
||
const string diagnostic = "CoCreateInstance for LMXProxy.LMXProxyServer.1 failed (0x80040154).";
|
||
|
||
await using FakeWorkerHarness harness = await FakeWorkerHarness.CreateConnectedPairAsync();
|
||
await using WorkerClient client = harness.CreateClient();
|
||
|
||
Task startTask = client.StartAsync(CancellationToken.None);
|
||
|
||
WorkerEnvelope gatewayHello = await harness.ReadGatewayEnvelopeAsync().WaitAsync(TestTimeout);
|
||
Assert.Equal(WorkerEnvelope.BodyOneofCase.GatewayHello, gatewayHello.BodyCase);
|
||
|
||
// The hello succeeds; the fault replaces the WorkerReady, which is where a COM creation
|
||
// failure actually lands (the worker creates MXAccess after answering the hello).
|
||
await harness.SendWorkerHelloAsync();
|
||
await harness.EmitFaultAsync(WorkerFaultCategory.MxaccessCreationFailed, diagnostic);
|
||
|
||
WorkerClientException exception = await Assert.ThrowsAsync<WorkerClientException>(
|
||
async () => await startTask.WaitAsync(TestTimeout));
|
||
|
||
Assert.Equal(WorkerClientErrorCode.WorkerFaulted, exception.ErrorCode);
|
||
Assert.Contains(
|
||
WorkerFaultCategory.MxaccessCreationFailed.ToString(),
|
||
exception.Message,
|
||
StringComparison.Ordinal);
|
||
Assert.Contains(diagnostic, exception.Message, StringComparison.Ordinal);
|
||
}
|
||
|
||
/// <summary>
|
||
/// The envelope <c>sequence</c> is a monotonic per-sender counter (gateway.md), so the values
|
||
/// observed on the pipe must be strictly increasing in wire order. Stamping the sequence when
|
||
/// the envelope is constructed lets two concurrent invokes stamp 1 and 2 but enqueue 2 then 1; stamping on the single-consumer write loop makes wire order and sequence order the
|
||
/// same thing by construction.
|
||
/// </summary>
|
||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||
[Fact]
|
||
public async Task ConcurrentInvokesEmitStrictlyIncreasingSequencesOnTheWire()
|
||
{
|
||
const int commandCount = 32;
|
||
await using PipePair pipePair = await PipePair.CreateAsync();
|
||
await using WorkerClient client = CreateClient(pipePair);
|
||
await CompleteHandshakeAsync(client, pipePair);
|
||
|
||
Task<WorkerCommandReply>[] invokeTasks = Enumerable.Range(0, commandCount)
|
||
.Select(_ => Task.Run(async () => await client.InvokeAsync(
|
||
CreateCommand(MxCommandKind.Ping),
|
||
TestTimeout,
|
||
CancellationToken.None)))
|
||
.ToArray();
|
||
|
||
ulong previousSequence = 0;
|
||
for (int index = 0; index < commandCount; index++)
|
||
{
|
||
WorkerEnvelope commandEnvelope = await pipePair.WorkerReader.ReadAsync().AsTask().WaitAsync(TestTimeout);
|
||
Assert.Equal(WorkerEnvelope.BodyOneofCase.WorkerCommand, commandEnvelope.BodyCase);
|
||
Assert.True(
|
||
commandEnvelope.Sequence > previousSequence,
|
||
$"Command {index} arrived with sequence {commandEnvelope.Sequence} after {previousSequence}; "
|
||
+ "envelope sequences must be strictly increasing in wire order.");
|
||
previousSequence = commandEnvelope.Sequence;
|
||
|
||
await pipePair.WriteAsync(
|
||
CreateCommandReplyEnvelope(commandEnvelope.CorrelationId, MxCommandKind.Ping));
|
||
}
|
||
|
||
await Task.WhenAll(invokeTasks).WaitAsync(TestTimeout);
|
||
Assert.Equal(WorkerClientState.Ready, client.State);
|
||
}
|
||
|
||
/// <summary>Verifies that ReadEventsAsync yields events in pipe order from the worker.</summary>
|
||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||
[Fact]
|
||
public async Task ReadEventsAsync_WithWorkerEvents_YieldsEventsInPipeOrder()
|
||
{
|
||
await using PipePair pipePair = await PipePair.CreateAsync();
|
||
await using WorkerClient client = CreateClient(pipePair);
|
||
await CompleteHandshakeAsync(client, pipePair);
|
||
using CancellationTokenSource cancellationTokenSource = new(TestTimeout);
|
||
|
||
await using IAsyncEnumerator<WorkerEvent> events =
|
||
client.ReadEventsAsync(cancellationTokenSource.Token).GetAsyncEnumerator(cancellationTokenSource.Token);
|
||
|
||
await pipePair.WriteAsync(
|
||
CreateEventEnvelope(sequence: 11, MxEventFamily.OnDataChange));
|
||
await pipePair.WriteAsync(
|
||
CreateEventEnvelope(sequence: 12, MxEventFamily.OperationComplete));
|
||
|
||
Assert.True(await events.MoveNextAsync());
|
||
Assert.Equal((ulong)11, events.Current.Event.WorkerSequence);
|
||
Assert.Equal(MxEventFamily.OnDataChange, events.Current.Event.Family);
|
||
|
||
Assert.True(await events.MoveNextAsync());
|
||
Assert.Equal((ulong)12, events.Current.Event.WorkerSequence);
|
||
Assert.Equal(MxEventFamily.OperationComplete, events.Current.Event.Family);
|
||
}
|
||
|
||
/// <summary>
|
||
/// The worker event channel is single-reader: a second <see cref="WorkerClient.ReadEventsAsync"/>
|
||
/// enumerator must throw rather than silently split events between two consumers.
|
||
/// </summary>
|
||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||
[Fact]
|
||
public async Task ReadEventsAsync_SecondEnumerator_Throws()
|
||
{
|
||
await using PipePair pipePair = await PipePair.CreateAsync();
|
||
await using WorkerClient client = CreateClient(pipePair);
|
||
await CompleteHandshakeAsync(client, pipePair);
|
||
using CancellationTokenSource cancellationTokenSource = new(TestTimeout);
|
||
|
||
// First call claims the single reader.
|
||
IAsyncEnumerable<WorkerEvent> first = client.ReadEventsAsync(cancellationTokenSource.Token);
|
||
await using IAsyncEnumerator<WorkerEvent> firstEnumerator = first.GetAsyncEnumerator(cancellationTokenSource.Token);
|
||
|
||
// A second call must fail loudly at call time rather than racing the first for events.
|
||
Assert.Throws<InvalidOperationException>(() => client.ReadEventsAsync(cancellationTokenSource.Token));
|
||
}
|
||
|
||
/// <summary>Verifies that the read loop faults the client when the event queue overflows.</summary>
|
||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||
[Fact]
|
||
public async Task ReadLoop_WhenEventQueueOverflows_FaultsClient()
|
||
{
|
||
await using PipePair pipePair = await PipePair.CreateAsync();
|
||
await using WorkerClient client = CreateClient(
|
||
pipePair,
|
||
new WorkerClientOptions
|
||
{
|
||
EventChannelCapacity = 1,
|
||
EventChannelFullModeTimeout = TimeSpan.FromMilliseconds(50),
|
||
HeartbeatGrace = TimeSpan.FromSeconds(30),
|
||
HeartbeatCheckInterval = TimeSpan.FromSeconds(30),
|
||
});
|
||
await CompleteHandshakeAsync(client, pipePair);
|
||
|
||
await pipePair.WriteAsync(
|
||
CreateEventEnvelope(sequence: 11, MxEventFamily.OnDataChange));
|
||
await pipePair.WriteAsync(
|
||
CreateEventEnvelope(sequence: 12, MxEventFamily.OnDataChange));
|
||
|
||
await WaitUntilAsync(
|
||
() => client.State == WorkerClientState.Faulted,
|
||
TestTimeout);
|
||
|
||
Assert.Equal(WorkerClientState.Faulted, client.State);
|
||
}
|
||
|
||
/// <summary>
|
||
/// Verifies that a command reply arriving on the pipe after events is dispatched promptly even
|
||
/// when the event channel is full and has no consumer — event enqueue is decoupled from the read
|
||
/// loop, so a blocked event writer cannot delay a reply. The event full-mode timeout is
|
||
/// set far above the command timeout: without the decoupling the read loop would block behind the
|
||
/// full event channel and the in-flight InvokeAsync would hit CommandTimeout.
|
||
/// </summary>
|
||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||
[Fact]
|
||
public async Task ReadLoop_WhenEventChannelFull_DispatchesCommandReplyWithoutBlocking()
|
||
{
|
||
await using PipePair pipePair = await PipePair.CreateAsync();
|
||
await using WorkerClient client = CreateClient(
|
||
pipePair,
|
||
new WorkerClientOptions
|
||
{
|
||
EventChannelCapacity = 1,
|
||
// Much larger than TestTimeout: the read loop must not block for this long behind the
|
||
// full event channel while a reply waits.
|
||
EventChannelFullModeTimeout = TimeSpan.FromSeconds(30),
|
||
HeartbeatGrace = TimeSpan.FromSeconds(60),
|
||
HeartbeatCheckInterval = TimeSpan.FromSeconds(60),
|
||
});
|
||
await CompleteHandshakeAsync(client, pipePair);
|
||
|
||
// No StreamEvents consumer is attached, so the capacity-1 event channel fills and the event
|
||
// writer blocks on the second event's timed WriteAsync.
|
||
await pipePair.WriteAsync(
|
||
CreateEventEnvelope(sequence: 11, MxEventFamily.OnDataChange));
|
||
await pipePair.WriteAsync(
|
||
CreateEventEnvelope(sequence: 12, MxEventFamily.OnDataChange));
|
||
|
||
Task<WorkerCommandReply> invokeTask = client.InvokeAsync(
|
||
CreateCommand(MxCommandKind.Ping),
|
||
TestTimeout,
|
||
CancellationToken.None);
|
||
WorkerEnvelope commandEnvelope = await pipePair.WorkerReader.ReadAsync().AsTask().WaitAsync(TestTimeout);
|
||
await pipePair.WriteAsync(
|
||
CreateCommandReplyEnvelope(commandEnvelope.CorrelationId, MxCommandKind.Ping));
|
||
|
||
WorkerCommandReply reply = await invokeTask.WaitAsync(TestTimeout);
|
||
Assert.Equal(MxCommandKind.Ping, reply.Reply.Kind);
|
||
Assert.Equal(WorkerClientState.Ready, client.State);
|
||
}
|
||
|
||
/// <summary>
|
||
/// Verifies that when the client faults it kills the owned worker process.
|
||
/// The assertion waits on <see cref="FakeWorkerProcess.WaitForExitAsync"/>, which
|
||
/// completes exactly when <c>Kill</c> runs, instead of polling <c>client.State</c>.
|
||
/// Polling state is racy: <see cref="WorkerClient.SetFaulted"/> publishes the
|
||
/// <c>Faulted</c> state before it calls <c>KillOwnedProcess</c>, so a state-based
|
||
/// wait can observe <c>Faulted</c> while <c>KillCount</c> is still 0.
|
||
/// </summary>
|
||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||
[Fact]
|
||
public async Task ReadLoop_WhenClientFaults_KillsOwnedWorkerProcess()
|
||
{
|
||
await using PipePair pipePair = await PipePair.CreateAsync();
|
||
FakeWorkerProcess process = new(WorkerProcessId);
|
||
await using WorkerClient client = CreateClient(
|
||
pipePair,
|
||
new WorkerClientOptions
|
||
{
|
||
EventChannelCapacity = 1,
|
||
EventChannelFullModeTimeout = TimeSpan.FromMilliseconds(50),
|
||
HeartbeatGrace = TimeSpan.FromSeconds(30),
|
||
HeartbeatCheckInterval = TimeSpan.FromSeconds(30),
|
||
},
|
||
processHandle: CreateProcessHandle(process));
|
||
await CompleteHandshakeAsync(client, pipePair);
|
||
|
||
await pipePair.WriteAsync(
|
||
CreateEventEnvelope(sequence: 11, MxEventFamily.OnDataChange));
|
||
await pipePair.WriteAsync(
|
||
CreateEventEnvelope(sequence: 12, MxEventFamily.OnDataChange));
|
||
|
||
// Deterministic: this completes the instant Kill() runs, with no timing window.
|
||
using CancellationTokenSource exitTimeout = new(TestTimeout);
|
||
await process.WaitForExitAsync(exitTimeout.Token);
|
||
|
||
Assert.Equal(WorkerClientState.Faulted, client.State);
|
||
Assert.Equal(1, process.KillCount);
|
||
Assert.True(process.KillEntireProcessTree);
|
||
Assert.True(process.HasExited);
|
||
}
|
||
|
||
/// <summary>
|
||
/// Verifies that a worker faulting mid-command — the pipe dropping while an
|
||
/// <see cref="WorkerClient.InvokeAsync"/> is still pending — completes the pending
|
||
/// invoke task with a <see cref="WorkerClientException"/> carrying the
|
||
/// pipe-disconnected error code rather than hanging until the command timeout.
|
||
/// </summary>
|
||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||
[Fact]
|
||
public async Task InvokeAsync_WhenPipeDisconnectsMidCommand_FailsPendingInvokeWithPipeDisconnected()
|
||
{
|
||
await using PipePair pipePair = await PipePair.CreateAsync();
|
||
await using WorkerClient client = CreateClient(pipePair);
|
||
await CompleteHandshakeAsync(client, pipePair);
|
||
|
||
Task<WorkerCommandReply> invokeTask = client.InvokeAsync(
|
||
CreateCommand(MxCommandKind.Ping),
|
||
TestTimeout,
|
||
CancellationToken.None);
|
||
|
||
// The worker received the command but disconnects before replying.
|
||
WorkerEnvelope commandEnvelope = await pipePair.WorkerReader.ReadAsync().AsTask().WaitAsync(TestTimeout);
|
||
Assert.Equal(WorkerEnvelope.BodyOneofCase.WorkerCommand, commandEnvelope.BodyCase);
|
||
await pipePair.DisposeWorkerSideAsync();
|
||
|
||
WorkerClientException exception = await Assert.ThrowsAsync<WorkerClientException>(
|
||
async () => await invokeTask.WaitAsync(TestTimeout));
|
||
|
||
Assert.Equal(WorkerClientErrorCode.PipeDisconnected, exception.ErrorCode);
|
||
await WaitUntilAsync(() => client.State == WorkerClientState.Faulted, TestTimeout);
|
||
Assert.Equal(WorkerClientState.Faulted, client.State);
|
||
}
|
||
|
||
/// <summary>
|
||
/// Verifies that a worker emitting a <c>WorkerFault</c> envelope while an
|
||
/// <see cref="WorkerClient.InvokeAsync"/> is pending completes the pending invoke
|
||
/// task with a <see cref="WorkerClientException"/> carrying the worker-faulted
|
||
/// error code.
|
||
/// </summary>
|
||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||
[Fact]
|
||
public async Task InvokeAsync_WhenWorkerFaultsMidCommand_FailsPendingInvokeWithWorkerFaulted()
|
||
{
|
||
await using PipePair pipePair = await PipePair.CreateAsync();
|
||
await using WorkerClient client = CreateClient(pipePair);
|
||
await CompleteHandshakeAsync(client, pipePair);
|
||
|
||
Task<WorkerCommandReply> invokeTask = client.InvokeAsync(
|
||
CreateCommand(MxCommandKind.Ping),
|
||
TestTimeout,
|
||
CancellationToken.None);
|
||
|
||
WorkerEnvelope commandEnvelope = await pipePair.WorkerReader.ReadAsync().AsTask().WaitAsync(TestTimeout);
|
||
Assert.Equal(WorkerEnvelope.BodyOneofCase.WorkerCommand, commandEnvelope.BodyCase);
|
||
await pipePair.WriteAsync(CreateWorkerFaultEnvelope("scripted mid-command fault"));
|
||
|
||
WorkerClientException exception = await Assert.ThrowsAsync<WorkerClientException>(
|
||
async () => await invokeTask.WaitAsync(TestTimeout));
|
||
|
||
Assert.Equal(WorkerClientErrorCode.WorkerFaulted, exception.ErrorCode);
|
||
await WaitUntilAsync(() => client.State == WorkerClientState.Faulted, TestTimeout);
|
||
Assert.Equal(WorkerClientState.Faulted, client.State);
|
||
}
|
||
|
||
/// <summary>Verifies that pipe disconnect faults the client.</summary>
|
||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||
[Fact]
|
||
public async Task ReadLoop_WhenPipeDisconnects_FaultsClient()
|
||
{
|
||
await using PipePair pipePair = await PipePair.CreateAsync();
|
||
await using WorkerClient client = CreateClient(pipePair);
|
||
await CompleteHandshakeAsync(client, pipePair);
|
||
|
||
await pipePair.DisposeWorkerSideAsync();
|
||
|
||
await WaitUntilAsync(
|
||
() => client.State == WorkerClientState.Faulted,
|
||
TestTimeout);
|
||
|
||
Assert.Equal(WorkerClientState.Faulted, client.State);
|
||
}
|
||
|
||
/// <summary>Verifies that the read loop stops the running worker metric when the pipe disconnects.</summary>
|
||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||
[Fact]
|
||
public async Task ReadLoop_WhenPipeDisconnects_StopsRunningWorkerMetric()
|
||
{
|
||
await using PipePair pipePair = await PipePair.CreateAsync();
|
||
using GatewayMetrics metrics = new();
|
||
await using WorkerClient client = CreateClient(pipePair, metrics: metrics);
|
||
await CompleteHandshakeAsync(client, pipePair);
|
||
|
||
Assert.Equal(1, metrics.GetSnapshot().WorkersRunning);
|
||
|
||
await pipePair.DisposeWorkerSideAsync();
|
||
|
||
await WaitUntilAsync(
|
||
() => client.State == WorkerClientState.Faulted
|
||
&& metrics.GetSnapshot().WorkersRunning == 0,
|
||
TestTimeout);
|
||
|
||
GatewayMetricsSnapshot snapshot = metrics.GetSnapshot();
|
||
Assert.Equal(0, snapshot.WorkersRunning);
|
||
Assert.Equal(1, snapshot.WorkerExits);
|
||
}
|
||
|
||
/// <summary>Verifies that DisposeAsync returns within a bounded timeout when the pipe read is blocked.</summary>
|
||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||
[Fact]
|
||
public async Task DisposeAsync_WhenPipeReadIsBlocked_ReturnsWithinBoundedTimeout()
|
||
{
|
||
await using PipePair pipePair = await PipePair.CreateAsync();
|
||
WorkerClient client = CreateClient(pipePair);
|
||
await CompleteHandshakeAsync(client, pipePair);
|
||
|
||
DateTimeOffset startedAt = DateTimeOffset.UtcNow;
|
||
await client.DisposeAsync().AsTask().WaitAsync(TestTimeout);
|
||
TimeSpan elapsed = DateTimeOffset.UtcNow - startedAt;
|
||
|
||
Assert.True(
|
||
elapsed < TimeSpan.FromSeconds(4),
|
||
$"DisposeAsync took {elapsed.TotalMilliseconds:N0}ms.");
|
||
}
|
||
|
||
/// <summary>Verifies that DisposeAsync kills the still-running owned worker process before disposing.</summary>
|
||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||
[Fact]
|
||
public async Task DisposeAsync_WhenOwnedWorkerStillRuns_KillsProcessBeforeDisposing()
|
||
{
|
||
await using PipePair pipePair = await PipePair.CreateAsync();
|
||
FakeWorkerProcess process = new(WorkerProcessId);
|
||
WorkerClient client = CreateClient(pipePair, processHandle: CreateProcessHandle(process));
|
||
|
||
await client.DisposeAsync().AsTask().WaitAsync(TestTimeout);
|
||
|
||
Assert.Equal(1, process.KillCount);
|
||
Assert.True(process.KillEntireProcessTree);
|
||
Assert.True(process.Disposed);
|
||
}
|
||
|
||
/// <summary>
|
||
/// Disposing a still-<c>Ready</c> client while the worker is emitting events must be
|
||
/// an ordinary teardown, not a fault. <c>DisposeAsync</c> completes the same three channels
|
||
/// <c>MarkClosed</c> and <c>SetFaulted</c> do, so it must enter a terminal state first;
|
||
/// otherwise the read loop — already inside <c>DispatchEnvelope</c> when the staging channel
|
||
/// is completed, or reading a stream that is torn down underneath it — misdiagnoses the
|
||
/// completed channel as the sustained-backpressure overflow and raises a spurious
|
||
/// <c>ProtocolViolation</c> fault, a <c>worker-event-staging</c> overflow metric and a
|
||
/// redundant kill during an intentional dispose. The event channel is sized far above the
|
||
/// burst so a genuine overflow is impossible, and <c>DisposeAsync</c> awaits its background
|
||
/// tasks, so the metrics read afterwards is not racing a late fault.
|
||
/// </summary>
|
||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||
[Fact]
|
||
public async Task DisposeAsync_WhileWorkerIsEmittingEvents_ClosesWithoutFaultingOrOverflowing()
|
||
{
|
||
// The burst is capped at EmittedEventCeiling and the staging channel holds 2 x capacity, so
|
||
// even if every emitted event were staged and none consumed the staging bound is unreachable:
|
||
// an overflow metric in this test can only be the dispose-race misdiagnosis.
|
||
const int capacity = 2048;
|
||
using GatewayMetrics metrics = new();
|
||
await using PipePair pipePair = await PipePair.CreateAsync();
|
||
WorkerClient client = CreateClient(
|
||
pipePair,
|
||
new WorkerClientOptions
|
||
{
|
||
EventChannelCapacity = capacity,
|
||
EventChannelFullModeTimeout = TimeSpan.FromMinutes(5),
|
||
HeartbeatGrace = TimeSpan.FromSeconds(30),
|
||
HeartbeatCheckInterval = TimeSpan.FromSeconds(30),
|
||
},
|
||
metrics: metrics);
|
||
await CompleteHandshakeAsync(client, pipePair);
|
||
|
||
using CancellationTokenSource emitCts = new(TestTimeout);
|
||
Task emitTask = EmitEventsUntilStoppedAsync(pipePair, emitCts.Token);
|
||
|
||
// Proves the read loop is mid-stream — not parked before its first event — when dispose runs.
|
||
await WaitUntilAsync(() => metrics.GetSnapshot().WorkerEventQueueDepth > 0, TestTimeout);
|
||
|
||
await client.DisposeAsync().AsTask().WaitAsync(TestTimeout);
|
||
await emitCts.CancelAsync();
|
||
await emitTask.WaitAsync(TestTimeout);
|
||
|
||
Assert.Equal(WorkerClientState.Closed, client.State);
|
||
|
||
GatewayMetricsSnapshot snapshot = metrics.GetSnapshot();
|
||
Assert.Equal(0L, snapshot.Faults);
|
||
Assert.Equal(0L, snapshot.QueueOverflows);
|
||
Assert.Equal(0L, snapshot.WorkerKills);
|
||
|
||
// Disposal is now a terminal transition, so the worker that reached Ready is recorded as
|
||
// stopped exactly once — the running-worker gauge no longer leaks on this path.
|
||
Assert.Equal(0, snapshot.WorkersRunning);
|
||
Assert.Equal(1L, snapshot.WorkerExits);
|
||
}
|
||
|
||
/// <summary>
|
||
/// Writes worker events from the fake worker side until cancelled or until
|
||
/// <see cref="EmittedEventCeiling"/> events have been written, so the gateway read loop has a
|
||
/// continuous stream to dispatch while a test tears the client down. The ceiling keeps the
|
||
/// burst structurally below the staging bound the caller configures. Every failure is
|
||
/// swallowed: disposal closes the gateway end of the pipe underneath this writer, which is
|
||
/// the situation under test rather than a test failure.
|
||
/// </summary>
|
||
/// <param name="pipePair">The connected pipe pair whose worker side emits the events.</param>
|
||
/// <param name="cancellationToken">Stops the emission.</param>
|
||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||
private static async Task EmitEventsUntilStoppedAsync(
|
||
PipePair pipePair,
|
||
CancellationToken cancellationToken)
|
||
{
|
||
await Task.Yield();
|
||
try
|
||
{
|
||
for (ulong sequence = 1;
|
||
sequence <= EmittedEventCeiling && !cancellationToken.IsCancellationRequested;
|
||
sequence++)
|
||
{
|
||
await pipePair.WorkerWriter.WriteAsync(
|
||
CreateEventEnvelope(sequence, MxEventFamily.OnDataChange),
|
||
cancellationToken);
|
||
}
|
||
}
|
||
catch (Exception)
|
||
{
|
||
// Broken pipe, disposed stream or cancellation — all expected once dispose starts.
|
||
}
|
||
}
|
||
|
||
/// <summary>
|
||
/// Verifies that a heartbeat envelope updates the last-heartbeat timestamp and worker
|
||
/// process id. Uses a <see cref="ManualTimeProvider"/> so the timestamp advance is
|
||
/// deterministic instead of relying on a wall-clock <c>Task.Delay</c> exceeding
|
||
/// <see cref="DateTimeOffset.UtcNow"/> resolution.
|
||
/// </summary>
|
||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||
[Fact]
|
||
public async Task ReadLoop_WhenHeartbeatArrives_UpdatesLastHeartbeatAndWorkerProcess()
|
||
{
|
||
ManualTimeProvider clock = new(DateTimeOffset.Parse("2026-05-18T12:00:00Z", System.Globalization.CultureInfo.InvariantCulture));
|
||
await using PipePair pipePair = await PipePair.CreateAsync();
|
||
await using WorkerClient client = CreateClient(pipePair, timeProvider: clock);
|
||
await CompleteHandshakeAsync(client, pipePair);
|
||
DateTimeOffset previousHeartbeat = client.LastHeartbeatAt;
|
||
|
||
clock.Advance(TimeSpan.FromSeconds(1));
|
||
await pipePair.WriteAsync(CreateHeartbeatEnvelope(workerProcessId: 9876));
|
||
|
||
await WaitUntilAsync(
|
||
() => client.ProcessId == 9876 && client.LastHeartbeatAt > previousHeartbeat,
|
||
TestTimeout);
|
||
|
||
Assert.Equal(WorkerClientState.Ready, client.State);
|
||
Assert.Equal(previousHeartbeat + TimeSpan.FromSeconds(1), client.LastHeartbeatAt);
|
||
}
|
||
|
||
/// <summary>
|
||
/// Verifies that the heartbeat monitor faults the client when the heartbeat expires.
|
||
/// Uses an injected <see cref="ManualTimeProvider"/> so the grace comparison is deterministic
|
||
/// instead of depending on real wall-clock advance; the monitor's
|
||
/// <see cref="WorkerClientOptions.HeartbeatCheckInterval"/> timer stays on the real clock and
|
||
/// observes the manually-advanced grace on its next tick.
|
||
/// </summary>
|
||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||
[Fact]
|
||
public async Task HeartbeatMonitor_WhenHeartbeatExpires_FaultsClient()
|
||
{
|
||
ManualTimeProvider clock = new(DateTimeOffset.Parse("2026-05-20T12:00:00Z", System.Globalization.CultureInfo.InvariantCulture));
|
||
await using PipePair pipePair = await PipePair.CreateAsync();
|
||
await using WorkerClient client = CreateClient(
|
||
pipePair,
|
||
new WorkerClientOptions
|
||
{
|
||
HeartbeatGrace = TimeSpan.FromMilliseconds(80),
|
||
HeartbeatCheckInterval = TimeSpan.FromMilliseconds(20),
|
||
EventChannelCapacity = 8,
|
||
},
|
||
timeProvider: clock);
|
||
await CompleteHandshakeAsync(client, pipePair);
|
||
|
||
clock.Advance(TimeSpan.FromSeconds(2));
|
||
|
||
await WaitUntilAsync(
|
||
() => client.State == WorkerClientState.Faulted,
|
||
TestTimeout);
|
||
|
||
Assert.Equal(WorkerClientState.Faulted, client.State);
|
||
}
|
||
|
||
/// <summary>
|
||
/// While a command is in flight on the
|
||
/// gateway↔worker pipe and the oldest pending command is younger
|
||
/// than <see cref="WorkerClientOptions.HeartbeatStuckCeiling"/>, the
|
||
/// heartbeat watchdog must NOT fault on heartbeat-expired alone — the
|
||
/// gap is more likely caused by pipe-write contention than by a hung
|
||
/// worker.
|
||
/// </summary>
|
||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||
[Fact]
|
||
public async Task HeartbeatMonitor_WhenCommandInFlightWithinCeiling_DoesNotFaultOnExpiredHeartbeat()
|
||
{
|
||
ManualTimeProvider clock = new(DateTimeOffset.Parse("2026-05-20T13:00:00Z", System.Globalization.CultureInfo.InvariantCulture));
|
||
await using PipePair pipePair = await PipePair.CreateAsync();
|
||
await using WorkerClient client = CreateClient(
|
||
pipePair,
|
||
new WorkerClientOptions
|
||
{
|
||
HeartbeatGrace = TimeSpan.FromMilliseconds(80),
|
||
HeartbeatCheckInterval = TimeSpan.FromMilliseconds(20),
|
||
EventChannelCapacity = 8,
|
||
HeartbeatStuckCeiling = TimeSpan.FromSeconds(30),
|
||
},
|
||
timeProvider: clock);
|
||
await CompleteHandshakeAsync(client, pipePair);
|
||
|
||
// Begin a command that the test never replies to — keeps the
|
||
// PendingCommand alive in `_pendingCommands` for the duration.
|
||
Task<WorkerCommandReply> pendingInvoke = client.InvokeAsync(
|
||
CreateCommand(MxCommandKind.Ping),
|
||
TestTimeout,
|
||
CancellationToken.None);
|
||
WorkerEnvelope commandEnvelope = await pipePair.WorkerReader.ReadAsync().AsTask().WaitAsync(TestTimeout);
|
||
Assert.Equal(WorkerEnvelope.BodyOneofCase.WorkerCommand, commandEnvelope.BodyCase);
|
||
|
||
// Advance well past HeartbeatGrace but well within HeartbeatStuckCeiling.
|
||
clock.Advance(TimeSpan.FromSeconds(2));
|
||
|
||
// Give the heartbeat monitor a few real check-intervals to observe the gap.
|
||
await Task.Delay(TimeSpan.FromMilliseconds(150));
|
||
|
||
Assert.Equal(WorkerClientState.Ready, client.State);
|
||
Assert.False(pendingInvoke.IsCompleted);
|
||
}
|
||
|
||
/// <summary>
|
||
/// Once the oldest pending command exceeds
|
||
/// <see cref="WorkerClientOptions.HeartbeatStuckCeiling"/>, the
|
||
/// heartbeat watchdog fires anyway — a truly stuck COM call shouldn't
|
||
/// keep the watchdog suppressed indefinitely.
|
||
/// </summary>
|
||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||
[Fact]
|
||
public async Task HeartbeatMonitor_WhenPendingCommandExceedsStuckCeiling_FaultsClient()
|
||
{
|
||
ManualTimeProvider clock = new(DateTimeOffset.Parse("2026-05-20T13:00:00Z", System.Globalization.CultureInfo.InvariantCulture));
|
||
await using PipePair pipePair = await PipePair.CreateAsync();
|
||
await using WorkerClient client = CreateClient(
|
||
pipePair,
|
||
new WorkerClientOptions
|
||
{
|
||
HeartbeatGrace = TimeSpan.FromMilliseconds(80),
|
||
HeartbeatCheckInterval = TimeSpan.FromMilliseconds(20),
|
||
EventChannelCapacity = 8,
|
||
HeartbeatStuckCeiling = TimeSpan.FromMilliseconds(200),
|
||
},
|
||
timeProvider: clock);
|
||
await CompleteHandshakeAsync(client, pipePair);
|
||
|
||
Task<WorkerCommandReply> pendingInvoke = client.InvokeAsync(
|
||
CreateCommand(MxCommandKind.Ping),
|
||
TestTimeout,
|
||
CancellationToken.None);
|
||
await pipePair.WorkerReader.ReadAsync().AsTask().WaitAsync(TestTimeout);
|
||
|
||
// Advance the clock past HeartbeatStuckCeiling. The worker pipe's
|
||
// PendingCommand.StartTimestamp uses TimeProvider.GetTimestamp(), so the
|
||
// ManualTimeProvider's GetElapsedTime sees the advanced gap.
|
||
clock.Advance(TimeSpan.FromSeconds(2));
|
||
|
||
await WaitUntilAsync(
|
||
() => client.State == WorkerClientState.Faulted,
|
||
TestTimeout);
|
||
|
||
Assert.Equal(WorkerClientState.Faulted, client.State);
|
||
}
|
||
|
||
/// <summary>
|
||
/// A transient burst that exceeds
|
||
/// <see cref="WorkerClientOptions.EventChannelCapacity"/> must be
|
||
/// absorbed for up to <see cref="WorkerClientOptions.EventChannelFullModeTimeout"/>
|
||
/// (the channel is configured for <c>BoundedChannelFullMode.Wait</c>);
|
||
/// only when the wait elapses without progress is the worker faulted,
|
||
/// and the diagnostic must name the channel capacity, depth, and
|
||
/// actionable remediation.
|
||
/// </summary>
|
||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||
[Fact]
|
||
public async Task EnqueueWorkerEvent_WhenChannelFullPastTimeout_FaultsWithRichDiagnostic()
|
||
{
|
||
await using PipePair pipePair = await PipePair.CreateAsync();
|
||
await using WorkerClient client = CreateClient(
|
||
pipePair,
|
||
new WorkerClientOptions
|
||
{
|
||
EventChannelCapacity = 4,
|
||
EventChannelFullModeTimeout = TimeSpan.FromMilliseconds(100),
|
||
HeartbeatGrace = TimeSpan.FromSeconds(30),
|
||
HeartbeatCheckInterval = TimeSpan.FromSeconds(1),
|
||
});
|
||
await CompleteHandshakeAsync(client, pipePair);
|
||
|
||
// Fill the 4-slot channel and write exactly one more to force the
|
||
// overflow path. The gateway never opens a StreamEvents consumer, so
|
||
// the events stay buffered. Exactly five events are written: the
|
||
// worker client faults while reading the fifth, after which its read
|
||
// loop stops — a sixth event would never be drained and its pipe
|
||
// write would block forever on a full OS pipe buffer.
|
||
for (ulong sequence = 1; sequence <= 5; sequence++)
|
||
{
|
||
await pipePair.WriteAsync(
|
||
CreateEventEnvelope(sequence: sequence, MxEventFamily.OnDataChange));
|
||
}
|
||
|
||
await WaitUntilAsync(
|
||
() => client.State == WorkerClientState.Faulted,
|
||
TestTimeout);
|
||
|
||
Assert.Equal(WorkerClientState.Faulted, client.State);
|
||
|
||
// Reading the events channel after fault throws the propagated
|
||
// WorkerClientException carrying the rich diagnostic message. The
|
||
// drain is bounded by TestTimeout so a regression that leaves the
|
||
// channel uncompleted fails the test instead of hanging it.
|
||
using CancellationTokenSource drainTimeout = new(TestTimeout);
|
||
WorkerClientException fault = await Assert.ThrowsAsync<WorkerClientException>(async () =>
|
||
{
|
||
await foreach (WorkerEvent _ in client.ReadEventsAsync(drainTimeout.Token))
|
||
{
|
||
}
|
||
});
|
||
Assert.Contains("Worker event channel rejected", fault.Message);
|
||
Assert.Contains("of 4 capacity", fault.Message);
|
||
Assert.Contains("StreamEvents", fault.Message);
|
||
Assert.Contains("MxGateway:Events:QueueCapacity", fault.Message);
|
||
}
|
||
|
||
/// <summary>
|
||
/// The staging channel between the read loop and the event writer is bounded at
|
||
/// 2 × <see cref="WorkerClientOptions.EventChannelCapacity"/>, so a consumer that drains
|
||
/// slower than the worker produces faults the client at the bound instead of growing
|
||
/// gateway memory silently. The full-mode timeout here is far longer than the test could
|
||
/// ever wait, which proves the fault came from the staging bound and not from the timed
|
||
/// write in <c>EnqueueWorkerEventAsync</c>. A command reply interleaved before the fault
|
||
/// must still complete: the read loop never blocks behind events (the read-loop guarantee).
|
||
/// </summary>
|
||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||
[Fact]
|
||
public async Task StagingChannelOverflowFaultsWorkerWithoutWaitingForFullModeTimeout()
|
||
{
|
||
const int capacity = 4;
|
||
const int stagingBound = 2 * capacity;
|
||
using GatewayMetrics metrics = new();
|
||
await using PipePair pipePair = await PipePair.CreateAsync();
|
||
await using WorkerClient client = CreateClient(
|
||
pipePair,
|
||
new WorkerClientOptions
|
||
{
|
||
EventChannelCapacity = capacity,
|
||
// Five minutes: the timed-write fault physically cannot be the trigger.
|
||
EventChannelFullModeTimeout = TimeSpan.FromMinutes(5),
|
||
HeartbeatGrace = TimeSpan.FromSeconds(30),
|
||
HeartbeatCheckInterval = TimeSpan.FromSeconds(30),
|
||
},
|
||
metrics: metrics);
|
||
await CompleteHandshakeAsync(client, pipePair);
|
||
|
||
// The staging channel alone holds 2 × capacity, and the event writer only ever removes
|
||
// from it, so this batch cannot fault however the writer happens to be scheduled. Waiting
|
||
// on the gauge (rather than a delay) proves the read loop consumed every one of them.
|
||
ulong sequence = 1;
|
||
for (; sequence <= (ulong)stagingBound; sequence++)
|
||
{
|
||
await pipePair.WriteAsync(
|
||
CreateEventEnvelope(sequence, MxEventFamily.OnDataChange));
|
||
}
|
||
|
||
await WaitUntilAsync(
|
||
() => metrics.GetSnapshot().WorkerEventQueueDepth == stagingBound,
|
||
TestTimeout);
|
||
Assert.Equal(WorkerClientState.Ready, client.State);
|
||
|
||
// The event path is backed up with no consumer attached, yet a command still round-trips.
|
||
Task<WorkerCommandReply> invokeTask = client.InvokeAsync(
|
||
CreateCommand(MxCommandKind.Ping),
|
||
TestTimeout,
|
||
CancellationToken.None);
|
||
WorkerEnvelope commandEnvelope = await pipePair.WorkerReader.ReadAsync().AsTask().WaitAsync(TestTimeout);
|
||
await pipePair.WriteAsync(
|
||
CreateCommandReplyEnvelope(commandEnvelope.CorrelationId, MxCommandKind.Ping));
|
||
WorkerCommandReply reply = await invokeTask.WaitAsync(TestTimeout);
|
||
Assert.Equal(MxCommandKind.Ping, reply.Reply.Kind);
|
||
Assert.Equal(WorkerClientState.Ready, client.State);
|
||
|
||
// The absolute ceiling is capacity (consumer channel) + 1 (in flight in the blocked event
|
||
// writer) + 2 × capacity (staging). Push comfortably past it. The events are tiny, so the
|
||
// ones the stopped read loop never drains stay in the OS pipe buffer instead of blocking.
|
||
for (int extra = 0; extra < 3 * capacity; extra++, sequence++)
|
||
{
|
||
await pipePair.WriteAsync(
|
||
CreateEventEnvelope(sequence, MxEventFamily.OnDataChange));
|
||
}
|
||
|
||
await WaitUntilAsync(() => client.State == WorkerClientState.Faulted, TestTimeout);
|
||
Assert.Equal(WorkerClientState.Faulted, client.State);
|
||
|
||
using CancellationTokenSource drainTimeout = new(TestTimeout);
|
||
WorkerClientException fault = await Assert.ThrowsAsync<WorkerClientException>(async () =>
|
||
{
|
||
await foreach (WorkerEvent _ in client.ReadEventsAsync(drainTimeout.Token))
|
||
{
|
||
}
|
||
});
|
||
|
||
Assert.Equal(WorkerClientErrorCode.ProtocolViolation, fault.ErrorCode);
|
||
Assert.Contains("staging", fault.Message, StringComparison.OrdinalIgnoreCase);
|
||
Assert.Contains($"{2 * capacity}", fault.Message, StringComparison.Ordinal);
|
||
Assert.Contains("StreamEvents", fault.Message, StringComparison.Ordinal);
|
||
Assert.Contains("MxGateway:Events:QueueCapacity", fault.Message, StringComparison.Ordinal);
|
||
|
||
// The timed-write diagnostic must not be what fired.
|
||
Assert.DoesNotContain("Worker event channel rejected", fault.Message, StringComparison.Ordinal);
|
||
}
|
||
|
||
/// <summary>
|
||
/// The worker event queue-depth gauge counts staged *and* queued events, so a
|
||
/// backlog held in the staging channel is visible rather than invisible. Depth is
|
||
/// incremented at staging and decremented when the consumer reads, so the single counter
|
||
/// reports total undelivered events and returns to zero once drained.
|
||
/// </summary>
|
||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||
[Fact]
|
||
public async Task WorkerEventQueueDepthGaugeCountsStagedEvents()
|
||
{
|
||
const int capacity = 4;
|
||
const int eventCount = 8;
|
||
using GatewayMetrics metrics = new();
|
||
await using PipePair pipePair = await PipePair.CreateAsync();
|
||
await using WorkerClient client = CreateClient(
|
||
pipePair,
|
||
new WorkerClientOptions
|
||
{
|
||
EventChannelCapacity = capacity,
|
||
EventChannelFullModeTimeout = TimeSpan.FromMinutes(5),
|
||
HeartbeatGrace = TimeSpan.FromSeconds(30),
|
||
HeartbeatCheckInterval = TimeSpan.FromSeconds(30),
|
||
},
|
||
metrics: metrics);
|
||
await CompleteHandshakeAsync(client, pipePair);
|
||
|
||
// Above EventChannelCapacity but below the 2× staging bound: no consumer, no fault.
|
||
for (ulong sequence = 1; sequence <= eventCount; sequence++)
|
||
{
|
||
await pipePair.WriteAsync(
|
||
CreateEventEnvelope(sequence, MxEventFamily.OnDataChange));
|
||
}
|
||
|
||
await WaitUntilAsync(
|
||
() => metrics.GetSnapshot().WorkerEventQueueDepth == eventCount,
|
||
TestTimeout);
|
||
Assert.Equal(eventCount, metrics.GetSnapshot().WorkerEventQueueDepth);
|
||
Assert.Equal(WorkerClientState.Ready, client.State);
|
||
|
||
using CancellationTokenSource drainTimeout = new(TestTimeout);
|
||
await using IAsyncEnumerator<WorkerEvent> events = client
|
||
.ReadEventsAsync(drainTimeout.Token)
|
||
.GetAsyncEnumerator(drainTimeout.Token);
|
||
for (int read = 0; read < eventCount; read++)
|
||
{
|
||
Assert.True(await events.MoveNextAsync());
|
||
}
|
||
|
||
await WaitUntilAsync(
|
||
() => metrics.GetSnapshot().WorkerEventQueueDepth == 0,
|
||
TestTimeout);
|
||
Assert.Equal(0, metrics.GetSnapshot().WorkerEventQueueDepth);
|
||
}
|
||
|
||
/// <summary>
|
||
/// A faulted client can remain retained for diagnostics, but its abandoned event
|
||
/// backlog must stop contributing to the gateway-wide queue-depth gauge as soon as the
|
||
/// client enters a terminal state rather than waiting for disposal.
|
||
/// </summary>
|
||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||
[Fact]
|
||
public async Task WorkerEventQueueDepthGauge_WhenClientFaults_StopsCountingRetainedClient()
|
||
{
|
||
using GatewayMetrics metrics = new();
|
||
await using PipePair pipePair = await PipePair.CreateAsync();
|
||
await using WorkerClient client = CreateClient(
|
||
pipePair,
|
||
new WorkerClientOptions
|
||
{
|
||
EventChannelCapacity = 4,
|
||
EventChannelFullModeTimeout = TimeSpan.FromMinutes(5),
|
||
HeartbeatGrace = TimeSpan.FromSeconds(30),
|
||
HeartbeatCheckInterval = TimeSpan.FromSeconds(30),
|
||
},
|
||
metrics: metrics);
|
||
await CompleteHandshakeAsync(client, pipePair);
|
||
|
||
await pipePair.WriteAsync(CreateEventEnvelope(sequence: 1, MxEventFamily.OnDataChange));
|
||
await WaitUntilAsync(
|
||
() => metrics.GetSnapshot().WorkerEventQueueDepth == 1,
|
||
TestTimeout);
|
||
|
||
await pipePair.WriteAsync(CreateWorkerFaultEnvelope("scripted retained-client fault"));
|
||
await WaitUntilAsync(() => client.State == WorkerClientState.Faulted, TestTimeout);
|
||
|
||
Assert.Equal(WorkerClientState.Faulted, client.State);
|
||
Assert.Equal(0, metrics.GetSnapshot().WorkerEventQueueDepth);
|
||
}
|
||
|
||
private static WorkerClient CreateClient(
|
||
PipePair pipePair,
|
||
WorkerClientOptions? options = null,
|
||
GatewayMetrics? metrics = null,
|
||
WorkerProcessHandle? processHandle = null,
|
||
TimeProvider? timeProvider = null,
|
||
int maxMessageBytes = WorkerFrameProtocolOptions.DefaultMaxMessageBytes)
|
||
{
|
||
WorkerFrameProtocolOptions frameOptions = new(
|
||
SessionId,
|
||
GatewayContractInfo.WorkerProtocolVersion,
|
||
maxMessageBytes);
|
||
WorkerClientConnection connection = new(
|
||
SessionId,
|
||
Nonce,
|
||
pipePair.GatewayStream,
|
||
frameOptions,
|
||
processHandle);
|
||
|
||
return new WorkerClient(connection, options, metrics, timeProvider);
|
||
}
|
||
|
||
private static WorkerProcessHandle CreateProcessHandle(FakeWorkerProcess process)
|
||
{
|
||
return new WorkerProcessHandle(
|
||
process,
|
||
new WorkerProcessCommandLine("ZB.MOM.WW.MxGateway.Worker.exe", []),
|
||
DateTimeOffset.UtcNow);
|
||
}
|
||
|
||
private static async Task CompleteHandshakeAsync(
|
||
WorkerClient client,
|
||
PipePair pipePair)
|
||
{
|
||
Task startTask = client.StartAsync(CancellationToken.None);
|
||
|
||
WorkerEnvelope gatewayHello = await pipePair.WorkerReader.ReadAsync().AsTask().WaitAsync(TestTimeout);
|
||
Assert.Equal(WorkerEnvelope.BodyOneofCase.GatewayHello, gatewayHello.BodyCase);
|
||
Assert.Equal(Nonce, gatewayHello.GatewayHello.Nonce);
|
||
Assert.Equal(GatewayContractInfo.WorkerProtocolVersion, gatewayHello.GatewayHello.SupportedProtocolVersion);
|
||
|
||
await pipePair.WriteAsync(CreateWorkerHelloEnvelope());
|
||
await pipePair.WriteAsync(CreateWorkerReadyEnvelope());
|
||
await startTask.WaitAsync(TestTimeout);
|
||
}
|
||
|
||
private static WorkerCommand CreateCommand(MxCommandKind kind)
|
||
{
|
||
return new WorkerCommand
|
||
{
|
||
Command = new MxCommand
|
||
{
|
||
Kind = kind,
|
||
},
|
||
};
|
||
}
|
||
|
||
private static WorkerEnvelope CreateWorkerHelloEnvelope()
|
||
{
|
||
return CreateWorkerEnvelope(
|
||
correlationId: string.Empty,
|
||
sequence: 1,
|
||
envelope => envelope.WorkerHello = new WorkerHello
|
||
{
|
||
ProtocolVersion = GatewayContractInfo.WorkerProtocolVersion,
|
||
Nonce = Nonce,
|
||
WorkerProcessId = WorkerProcessId,
|
||
WorkerVersion = "fake-worker",
|
||
});
|
||
}
|
||
|
||
private static WorkerEnvelope CreateWorkerReadyEnvelope()
|
||
{
|
||
return CreateWorkerEnvelope(
|
||
correlationId: string.Empty,
|
||
sequence: 2,
|
||
envelope => envelope.WorkerReady = new WorkerReady
|
||
{
|
||
WorkerProcessId = WorkerProcessId,
|
||
MxaccessProgid = "LMXProxy.LMXProxyServer.1",
|
||
MxaccessClsid = "{C30B52F5-2CB5-4760-AF0A-3A344A7EB5DC}",
|
||
});
|
||
}
|
||
|
||
private static WorkerEnvelope CreateCommandReplyEnvelope(
|
||
string correlationId,
|
||
MxCommandKind kind)
|
||
{
|
||
return CreateWorkerEnvelope(
|
||
correlationId,
|
||
sequence: 10,
|
||
envelope => envelope.WorkerCommandReply = new WorkerCommandReply
|
||
{
|
||
Reply = new MxCommandReply
|
||
{
|
||
SessionId = SessionId,
|
||
CorrelationId = correlationId,
|
||
Kind = kind,
|
||
},
|
||
});
|
||
}
|
||
|
||
private static WorkerEnvelope CreateEventEnvelope(
|
||
ulong sequence,
|
||
MxEventFamily family)
|
||
{
|
||
return CreateWorkerEnvelope(
|
||
correlationId: string.Empty,
|
||
sequence,
|
||
envelope => envelope.WorkerEvent = new WorkerEvent
|
||
{
|
||
Event = new MxEvent
|
||
{
|
||
SessionId = SessionId,
|
||
Family = family,
|
||
WorkerSequence = sequence,
|
||
},
|
||
});
|
||
}
|
||
|
||
private static WorkerEnvelope CreateWorkerFaultEnvelope(string diagnosticMessage)
|
||
{
|
||
return CreateWorkerEnvelope(
|
||
correlationId: string.Empty,
|
||
sequence: 30,
|
||
envelope => envelope.WorkerFault = new WorkerFault
|
||
{
|
||
Category = WorkerFaultCategory.MxaccessCommandFailed,
|
||
DiagnosticMessage = diagnosticMessage,
|
||
ProtocolStatus = new ProtocolStatus
|
||
{
|
||
Code = ProtocolStatusCode.WorkerUnavailable,
|
||
Message = diagnosticMessage,
|
||
},
|
||
});
|
||
}
|
||
|
||
private static WorkerEnvelope CreateHeartbeatEnvelope(int workerProcessId)
|
||
{
|
||
return CreateWorkerEnvelope(
|
||
correlationId: string.Empty,
|
||
sequence: 20,
|
||
envelope => envelope.WorkerHeartbeat = new WorkerHeartbeat
|
||
{
|
||
WorkerProcessId = workerProcessId,
|
||
State = WorkerState.Ready,
|
||
LastStaActivityTimestamp = Timestamp.FromDateTimeOffset(DateTimeOffset.UtcNow),
|
||
PendingCommandCount = 0,
|
||
OutboundEventQueueDepth = 0,
|
||
});
|
||
}
|
||
|
||
private static WorkerEnvelope CreateWorkerEnvelope(
|
||
string correlationId,
|
||
ulong sequence,
|
||
Action<WorkerEnvelope> setBody)
|
||
{
|
||
WorkerEnvelope envelope = new()
|
||
{
|
||
ProtocolVersion = GatewayContractInfo.WorkerProtocolVersion,
|
||
SessionId = SessionId,
|
||
Sequence = sequence,
|
||
CorrelationId = correlationId,
|
||
};
|
||
setBody(envelope);
|
||
|
||
return envelope;
|
||
}
|
||
|
||
/// <summary>
|
||
/// Reads gateway envelopes until a <c>WorkerCommand</c> arrives, skipping the <c>WorkerCancel</c>
|
||
/// a command timeout emits for <paramref name="canceledCorrelationId"/>. Anything else on the pipe
|
||
/// fails the test rather than being skipped: the point of the skip is to tolerate exactly the one
|
||
/// known interleaving, not to make the assertion blind to unexpected gateway traffic.
|
||
/// </summary>
|
||
/// <param name="pipePair">The connected pipe pair whose worker side is read.</param>
|
||
/// <param name="canceledCorrelationId">Correlation id of the timed-out command whose cancel is expected.</param>
|
||
/// <returns>The next command envelope written by the gateway.</returns>
|
||
private static async Task<WorkerEnvelope> ReadNextCommandAsync(
|
||
PipePair pipePair,
|
||
string canceledCorrelationId)
|
||
{
|
||
while (true)
|
||
{
|
||
WorkerEnvelope envelope = await pipePair.WorkerReader.ReadAsync().AsTask().WaitAsync(TestTimeout);
|
||
if (envelope.BodyCase == WorkerEnvelope.BodyOneofCase.WorkerCommand)
|
||
{
|
||
return envelope;
|
||
}
|
||
|
||
Assert.Equal(WorkerEnvelope.BodyOneofCase.WorkerCancel, envelope.BodyCase);
|
||
Assert.Equal(canceledCorrelationId, envelope.CorrelationId);
|
||
}
|
||
}
|
||
|
||
private static async Task WaitUntilAsync(
|
||
Func<bool> predicate,
|
||
TimeSpan timeout)
|
||
{
|
||
using CancellationTokenSource cancellationTokenSource = new(timeout);
|
||
while (!predicate())
|
||
{
|
||
await Task.Delay(TimeSpan.FromMilliseconds(10), cancellationTokenSource.Token);
|
||
}
|
||
}
|
||
|
||
private sealed class PipePair : IAsyncDisposable
|
||
{
|
||
private readonly NamedPipeClientStream _workerStream;
|
||
private bool _workerSideDisposed;
|
||
|
||
private PipePair(
|
||
NamedPipeServerStream gatewayStream,
|
||
NamedPipeClientStream workerStream)
|
||
{
|
||
GatewayStream = gatewayStream;
|
||
_workerStream = workerStream;
|
||
WorkerReader = new WorkerFrameReader(_workerStream, new WorkerFrameProtocolOptions(SessionId));
|
||
WorkerWriter = new WorkerFrameWriter(_workerStream, new WorkerFrameProtocolOptions(SessionId));
|
||
}
|
||
|
||
/// <summary>The gateway side of the named pipe connection.</summary>
|
||
public NamedPipeServerStream GatewayStream { get; }
|
||
|
||
/// <summary>Frame reader for worker messages.</summary>
|
||
public WorkerFrameReader WorkerReader { get; }
|
||
|
||
/// <summary>Frame writer for worker messages.</summary>
|
||
public WorkerFrameWriter WorkerWriter { get; }
|
||
|
||
/// <summary>
|
||
/// Writes one envelope from the fake worker side, bounded by <see cref="TestTimeout"/>.
|
||
/// </summary>
|
||
/// <remarks>
|
||
/// Every test-side write goes through here rather than <see cref="WorkerWriter"/> directly so
|
||
/// that a write which cannot complete fails this test instead of hanging it. An unbounded
|
||
/// write is not a local problem: a test method that never returns leaves xUnit's assembly
|
||
/// runner awaiting it forever, so <c>ITestAssemblyFinished</c> is never raised and the whole
|
||
/// <c>testhost</c> process never exits — the run reports no failure at all, just a wedge.
|
||
/// </remarks>
|
||
/// <param name="envelope">The envelope to write.</param>
|
||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||
public async Task WriteAsync(WorkerEnvelope envelope)
|
||
{
|
||
using CancellationTokenSource writeTimeout = new(TestTimeout);
|
||
try
|
||
{
|
||
await WorkerWriter.WriteAsync(envelope, writeTimeout.Token);
|
||
}
|
||
catch (OperationCanceledException) when (writeTimeout.IsCancellationRequested)
|
||
{
|
||
Assert.Fail(
|
||
$"The fake worker's pipe write did not complete within {TestTimeout}. The gateway " +
|
||
"side has stopped reading and the pipe buffer is full.");
|
||
}
|
||
}
|
||
|
||
/// <summary>Creates a connected pipe pair for testing.</summary>
|
||
/// <returns>The connected <see cref="PipePair"/>.</returns>
|
||
public static async Task<PipePair> CreateAsync()
|
||
{
|
||
string pipeName = $"mxgw-wc-{Guid.NewGuid():N}";
|
||
NamedPipeServerStream gatewayStream = TestNamedPipe.CreateServer(pipeName);
|
||
NamedPipeClientStream workerStream = new(
|
||
".",
|
||
pipeName,
|
||
PipeDirection.InOut,
|
||
PipeOptions.Asynchronous);
|
||
|
||
Task waitForConnectionTask = gatewayStream.WaitForConnectionAsync();
|
||
await workerStream.ConnectAsync();
|
||
await waitForConnectionTask;
|
||
|
||
return new PipePair(gatewayStream, workerStream);
|
||
}
|
||
|
||
/// <summary>Disposes the worker side of the pipe.</summary>
|
||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||
public async ValueTask DisposeWorkerSideAsync()
|
||
{
|
||
if (_workerSideDisposed)
|
||
{
|
||
return;
|
||
}
|
||
|
||
await _workerStream.DisposeAsync();
|
||
_workerSideDisposed = true;
|
||
}
|
||
|
||
/// <summary>Disposes the duplex stream.</summary>
|
||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||
public async ValueTask DisposeAsync()
|
||
{
|
||
await DisposeWorkerSideAsync();
|
||
await GatewayStream.DisposeAsync();
|
||
}
|
||
}
|
||
|
||
}
|