Merge fix/c3-ipc-boundaries: IPC-33 drain budget bounded under the gRPC send cap + validator upper coupling, IPC-39 pre-check sequence allowance, IPC-34 WorkerCancel on caller cancellation + CallerCancelled code, IPC-35 handshake WorkerFault surfaces category

2026-08-20 09:33:55 -04:00
parent 9f0c3fd9fa ab41362a5c
commit cfbc99581b
15 changed files with 430 additions and 62 deletions
@@ -216,6 +216,151 @@ public sealed class WorkerClientTests
        Assert.Equal(WorkerClientState.Ready, client.State);
    }

+    /// <summary>
+    ///     Caller cancellation abandons the correlation exactly as a gateway timeout does — a client
+    ///     deadline shorter than the gateway's command timeout, or a disconnect, is the common
+    ///     abandonment path — so it forwards the same <c>WorkerCancel</c> (IPC-34). Without it a
+    ///     retry storm of deadline-cancelled calls leaves the worker's single STA working through
+    ///     commands nobody is waiting for while live callers queue behind them. Twin of
+    ///     <see cref="InvokeAsync_WhenCommandTimesOut_SendsWorkerCancelForThatCorrelation"/>, driven
+    ///     by the caller's token instead of the timer.
+    /// </summary>
+    /// <returns>A task that represents the asynchronous operation.</returns>
+    [Fact]
+    public async Task InvokeAsync_WhenCallerCancels_SendsWorkerCancelForThatCorrelation()
+    {
+        await using PipePair pipePair = await PipePair.CreateAsync();
+        await using WorkerClient client = CreateClient(pipePair);
+        await CompleteHandshakeAsync(client, pipePair);
+
+        using CancellationTokenSource callerCts = new();
+
+        // Advise rather than a control command: control commands bypass the worker's STA queue, so a
+        // data command is the case the cancel actually exists for. The timeout is the full test
+        // timeout so the cancellation, not the timer, is unambiguously what abandons the command.
+        Task<WorkerCommandReply> invokeTask = client.InvokeAsync(
+            CreateCommand(MxCommandKind.Advise),
+            TestTimeout,
+            callerCts.Token);
+
+        WorkerEnvelope commandEnvelope = await pipePair.WorkerReader.ReadAsync().AsTask().WaitAsync(TestTimeout);
+        Assert.Equal(WorkerEnvelope.BodyOneofCase.WorkerCommand, commandEnvelope.BodyCase);
+
+        await callerCts.CancelAsync();
+
+        await Assert.ThrowsAnyAsync<OperationCanceledException>(
+            async () => await invokeTask.WaitAsync(TestTimeout));
+
+        WorkerEnvelope cancelEnvelope = await pipePair.WorkerReader.ReadAsync().AsTask().WaitAsync(TestTimeout);
+
+        Assert.Equal(WorkerEnvelope.BodyOneofCase.WorkerCancel, cancelEnvelope.BodyCase);
+        Assert.Equal(commandEnvelope.CorrelationId, cancelEnvelope.CorrelationId);
+        Assert.False(string.IsNullOrWhiteSpace(cancelEnvelope.WorkerCancel.Reason));
+        Assert.True(
+            cancelEnvelope.Sequence > commandEnvelope.Sequence,
+            $"The cancel arrived with sequence {cancelEnvelope.Sequence} after {commandEnvelope.Sequence}; "
+            + "envelope sequences must be strictly increasing in wire order.");
+
+        // One caller gave up; the session did not fault.
+        Assert.Equal(WorkerClientState.Ready, client.State);
+    }
+
+    /// <summary>
+    ///     The cancel is forwarded for a caller that abandoned its command, never for the client's own
+    ///     teardown (IPC-34): once the stop token has fired the outbound channel is being completed and
+    ///     every pending command is failed anyway, so a cancel would at best be pointless and at worst
+    ///     a write racing teardown. Asserted on the wire — the guard in the cancellation arm is the
+    ///     code-level expression of it, and the completed outbound channel is the second line of
+    ///     defense, so this pins the observable contract rather than either mechanism.
+    /// </summary>
+    /// <returns>A task that represents the asynchronous operation.</returns>
+    [Fact]
+    public async Task InvokeAsync_WhenClientStopsUnderInFlightCommand_DoesNotSendWorkerCancel()
+    {
+        await using PipePair pipePair = await PipePair.CreateAsync();
+        await using WorkerClient client = CreateClient(pipePair);
+        await CompleteHandshakeAsync(client, pipePair);
+
+        using CancellationTokenSource callerCts = new();
+        Task<WorkerCommandReply> invokeTask = client.InvokeAsync(
+            CreateCommand(MxCommandKind.Advise),
+            TestTimeout,
+            callerCts.Token);
+
+        WorkerEnvelope commandEnvelope = await pipePair.WorkerReader.ReadAsync().AsTask().WaitAsync(TestTimeout);
+        Assert.Equal(WorkerEnvelope.BodyOneofCase.WorkerCommand, commandEnvelope.BodyCase);
+
+        // Read ahead before tearing down so anything the teardown path puts on the wire is caught.
+        // Exceptions are swallowed: the read is expected to be still pending at the assertion, and
+        // the stream is disposed underneath it when the test ends.
+        Task<WorkerEnvelope?> pendingRead = Task.Run(async () =>
+        {
+            try
+            {
+                return await pipePair.WorkerReader.ReadAsync();
+            }
+            catch (Exception)
+            {
+                return null;
+            }
+        });
+
+        // Kill fires the client's stop token and then fails the pending command; a caller token that
+        // is wired to gateway shutdown (as the session's is) cancels in the same window.
+        client.Kill("test teardown");
+        await callerCts.CancelAsync();
+
+        await Assert.ThrowsAnyAsync<Exception>(async () => await invokeTask.WaitAsync(TestTimeout));
+
+        await Task.Delay(TimeSpan.FromMilliseconds(250));
+        if (pendingRead.IsCompleted)
+        {
+            WorkerEnvelope? unexpected = await pendingRead;
+            Assert.True(
+                unexpected is null || unexpected.BodyCase != WorkerEnvelope.BodyOneofCase.WorkerCancel,
+                "A stopping worker client must not put a WorkerCancel on the wire.");
+        }
+    }
+
+    /// <summary>
+    ///     A <c>WorkerFault</c> arriving in place of a handshake frame is the worker reporting a
+    ///     structured startup failure — most often MXAccess COM creation — not a protocol problem, so
+    ///     the gateway surfaces <see cref="WorkerClientErrorCode.WorkerFaulted"/> with the fault's
+    ///     category and diagnostic rather than a generic protocol violation naming only the body type
+    ///     (IPC-35). This is the window where the information matters most: first session open on a
+    ///     freshly deployed host, where the message reaches the operator log and the client's
+    ///     <c>OpenFailed</c> detail.
+    /// </summary>
+    /// <returns>A task that represents the asynchronous operation.</returns>
+    [Fact]
+    public async Task StartAsync_WhenWorkerFaultsDuringHandshake_SurfacesFaultCategoryAndDiagnostic()
+    {
+        const string diagnostic = "CoCreateInstance for LMXProxy.LMXProxyServer.1 failed (0x80040154).";
+
+        await using FakeWorkerHarness harness = await FakeWorkerHarness.CreateConnectedPairAsync();
+        await using WorkerClient client = harness.CreateClient();
+
+        Task startTask = client.StartAsync(CancellationToken.None);
+
+        WorkerEnvelope gatewayHello = await harness.ReadGatewayEnvelopeAsync().WaitAsync(TestTimeout);
+        Assert.Equal(WorkerEnvelope.BodyOneofCase.GatewayHello, gatewayHello.BodyCase);
+
+        // The hello succeeds; the fault replaces the WorkerReady, which is where a COM creation
+        // failure actually lands (the worker creates MXAccess after answering the hello).
+        await harness.SendWorkerHelloAsync();
+        await harness.EmitFaultAsync(WorkerFaultCategory.MxaccessCreationFailed, diagnostic);
+
+        WorkerClientException exception = await Assert.ThrowsAsync<WorkerClientException>(
+            async () => await startTask.WaitAsync(TestTimeout));
+
+        Assert.Equal(WorkerClientErrorCode.WorkerFaulted, exception.ErrorCode);
+        Assert.Contains(
+            WorkerFaultCategory.MxaccessCreationFailed.ToString(),
+            exception.Message,
+            StringComparison.Ordinal);
+        Assert.Contains(diagnostic, exception.Message, StringComparison.Ordinal);
+    }
+
    /// <summary>
    ///     The envelope <c>sequence</c> is a monotonic per-sender counter (gateway.md), so the values
    ///     observed on the pipe must be strictly increasing in wire order. Stamping the sequence when