Merge fix/c3-ipc-boundaries: IPC-33 drain budget bounded under the gRPC send cap + validator upper coupling, IPC-39 pre-check sequence allowance, IPC-34 WorkerCancel on caller cancellation + CallerCancelled code, IPC-35 handshake WorkerFault surfaces category

src/ZB.MOM.WW.MxGateway.Tests/Configuration/GatewayOptionsTests.cs

@@ -86,6 +86,11 @@ public sealed class GatewayOptionsTests
                 ["MxGateway:Events:QueueCapacity"] = "256",
                 ["MxGateway:Dashboard:Enabled"] = "false",
                 ["MxGateway:Protocol:MaxGrpcMessageBytes"] = "8388608",
+                // The frame-headroom rule is two-sided (IPC-33): the worker-frame maximum must sit
+                // between one and two 64 KiB envelope reserves above the gRPC cap, so moving the
+                // gRPC cap without moving this one no longer validates. Kept at one reserve above,
+                // the same shape as the shipped default.
+                ["MxGateway:Worker:MaxMessageBytes"] = "8454144",
                 // Blank-by-default bind password must be supplied so enabled LDAP validates on bind.
                 ["MxGateway:Ldap:ServiceAccountPassword"] = "test-bind-password"
             });

src/ZB.MOM.WW.MxGateway.Tests/Configuration/GatewayOptionsValidatorTests.cs

@@ -1,5 +1,6 @@
 using Microsoft.Extensions.Options;
 using ZB.MOM.WW.MxGateway.Server.Configuration;
+using ZB.MOM.WW.MxGateway.Server.Workers;
 using LdapTransport = ZB.MOM.WW.Auth.Abstractions.Ldap.LdapTransport;
 
 namespace ZB.MOM.WW.MxGateway.Tests.Configuration;
@@ -1256,6 +1257,57 @@ public sealed class GatewayOptionsValidatorTests
         Assert.Contains(result.Failures!, f => f.Contains("MaxMessageBytes") && f.Contains("reserve"));
     }
 
+    /// <summary>
+    /// Verifies the headroom rule's upper half (IPC-33): a worker-frame maximum more than two
+    /// envelope reserves above the gRPC cap is rejected. The worker budgets a DrainEvents reply at
+    /// the frame maximum less two reserves and the gateway returns that same reply through
+    /// <c>Invoke</c>, so a wider gap lets the worker pack a reply the pipe carries and the gRPC send
+    /// cap refuses — with the drained events already removed from the worker queue.
+    /// </summary>
+    [Fact]
+    public void Validate_Fails_WhenWorkerFrameMaxExceedsGrpcMaxByMoreThanTwoReserves()
+    {
+        const int grpcMax = 16 * 1024 * 1024;
+        int tooLarge = grpcMax + (2 * WorkerFrameProtocolOptions.EnvelopeOverheadReserveBytes) + 1;
+
+        ValidateOptionsResult result = new GatewayOptionsValidator().Validate(
+            null,
+            WithWorkerAndProtocol(
+                new WorkerOptions { MaxMessageBytes = tooLarge },
+                new ProtocolOptions { MaxGrpcMessageBytes = grpcMax }));
+
+        Assert.True(result.Failed);
+        Assert.Contains(
+            result.Failures!,
+            f => f.Contains("MaxMessageBytes", StringComparison.Ordinal)
+                && f.Contains("at most", StringComparison.Ordinal)
+                && f.Contains("DrainEvents", StringComparison.Ordinal));
+    }
+
+    /// <summary>
+    /// Verifies both edges of the two-sided headroom rule are themselves accepted: exactly one
+    /// reserve above the gRPC cap (the shipped default's shape) and exactly two reserves above it
+    /// (the widest legal gap). Pinning the boundaries keeps a future retune of the reserve from
+    /// silently making the shipped default illegal.
+    /// </summary>
+    /// <param name="reserveMultiple">Number of envelope reserves between the gRPC cap and the frame maximum.</param>
+    [Theory]
+    [InlineData(1)]
+    [InlineData(2)]
+    public void Validate_Succeeds_AtBothEdgesOfTheFrameHeadroomBand(int reserveMultiple)
+    {
+        const int grpcMax = 16 * 1024 * 1024;
+        int workerMax = grpcMax + (reserveMultiple * WorkerFrameProtocolOptions.EnvelopeOverheadReserveBytes);
+
+        ValidateOptionsResult result = new GatewayOptionsValidator().Validate(
+            null,
+            WithWorkerAndProtocol(
+                new WorkerOptions { MaxMessageBytes = workerMax },
+                new ProtocolOptions { MaxGrpcMessageBytes = grpcMax }));
+
+        Assert.True(result.Succeeded);
+    }
+
     /// <summary>
     /// Verifies the shipped worker event-queue capacity default passes validation and still matches
     /// the worker-side <c>MxAccessEventQueue.DefaultCapacity</c> the environment variable falls back

src/ZB.MOM.WW.MxGateway.Tests/Gateway/Workers/WorkerClientTests.cs

@@ -216,6 +216,151 @@ public sealed class WorkerClientTests
         Assert.Equal(WorkerClientState.Ready, client.State);
     }
 
+    /// <summary>
+    ///     Caller cancellation abandons the correlation exactly as a gateway timeout does — a client
+    ///     deadline shorter than the gateway's command timeout, or a disconnect, is the common
+    ///     abandonment path — so it forwards the same <c>WorkerCancel</c> (IPC-34). Without it a
+    ///     retry storm of deadline-cancelled calls leaves the worker's single STA working through
+    ///     commands nobody is waiting for while live callers queue behind them. Twin of
+    ///     <see cref="InvokeAsync_WhenCommandTimesOut_SendsWorkerCancelForThatCorrelation"/>, driven
+    ///     by the caller's token instead of the timer.
+    /// </summary>
+    /// <returns>A task that represents the asynchronous operation.</returns>
+    [Fact]
+    public async Task InvokeAsync_WhenCallerCancels_SendsWorkerCancelForThatCorrelation()
+    {
+        await using PipePair pipePair = await PipePair.CreateAsync();
+        await using WorkerClient client = CreateClient(pipePair);
+        await CompleteHandshakeAsync(client, pipePair);
+
+        using CancellationTokenSource callerCts = new();
+
+        // Advise rather than a control command: control commands bypass the worker's STA queue, so a
+        // data command is the case the cancel actually exists for. The timeout is the full test
+        // timeout so the cancellation, not the timer, is unambiguously what abandons the command.
+        Task<WorkerCommandReply> invokeTask = client.InvokeAsync(
+            CreateCommand(MxCommandKind.Advise),
+            TestTimeout,
+            callerCts.Token);
+
+        WorkerEnvelope commandEnvelope = await pipePair.WorkerReader.ReadAsync().AsTask().WaitAsync(TestTimeout);
+        Assert.Equal(WorkerEnvelope.BodyOneofCase.WorkerCommand, commandEnvelope.BodyCase);
+
+        await callerCts.CancelAsync();
+
+        await Assert.ThrowsAnyAsync<OperationCanceledException>(
+            async () => await invokeTask.WaitAsync(TestTimeout));
+
+        WorkerEnvelope cancelEnvelope = await pipePair.WorkerReader.ReadAsync().AsTask().WaitAsync(TestTimeout);
+
+        Assert.Equal(WorkerEnvelope.BodyOneofCase.WorkerCancel, cancelEnvelope.BodyCase);
+        Assert.Equal(commandEnvelope.CorrelationId, cancelEnvelope.CorrelationId);
+        Assert.False(string.IsNullOrWhiteSpace(cancelEnvelope.WorkerCancel.Reason));
+        Assert.True(
+            cancelEnvelope.Sequence > commandEnvelope.Sequence,
+            $"The cancel arrived with sequence {cancelEnvelope.Sequence} after {commandEnvelope.Sequence}; "
+            + "envelope sequences must be strictly increasing in wire order.");
+
+        // One caller gave up; the session did not fault.
+        Assert.Equal(WorkerClientState.Ready, client.State);
+    }
+
+    /// <summary>
+    ///     The cancel is forwarded for a caller that abandoned its command, never for the client's own
+    ///     teardown (IPC-34): once the stop token has fired the outbound channel is being completed and
+    ///     every pending command is failed anyway, so a cancel would at best be pointless and at worst
+    ///     a write racing teardown. Asserted on the wire — the guard in the cancellation arm is the
+    ///     code-level expression of it, and the completed outbound channel is the second line of
+    ///     defense, so this pins the observable contract rather than either mechanism.
+    /// </summary>
+    /// <returns>A task that represents the asynchronous operation.</returns>
+    [Fact]
+    public async Task InvokeAsync_WhenClientStopsUnderInFlightCommand_DoesNotSendWorkerCancel()
+    {
+        await using PipePair pipePair = await PipePair.CreateAsync();
+        await using WorkerClient client = CreateClient(pipePair);
+        await CompleteHandshakeAsync(client, pipePair);
+
+        using CancellationTokenSource callerCts = new();
+        Task<WorkerCommandReply> invokeTask = client.InvokeAsync(
+            CreateCommand(MxCommandKind.Advise),
+            TestTimeout,
+            callerCts.Token);
+
+        WorkerEnvelope commandEnvelope = await pipePair.WorkerReader.ReadAsync().AsTask().WaitAsync(TestTimeout);
+        Assert.Equal(WorkerEnvelope.BodyOneofCase.WorkerCommand, commandEnvelope.BodyCase);
+
+        // Read ahead before tearing down so anything the teardown path puts on the wire is caught.
+        // Exceptions are swallowed: the read is expected to be still pending at the assertion, and
+        // the stream is disposed underneath it when the test ends.
+        Task<WorkerEnvelope?> pendingRead = Task.Run(async () =>
+        {
+            try
+            {
+                return await pipePair.WorkerReader.ReadAsync();
+            }
+            catch (Exception)
+            {
+                return null;
+            }
+        });
+
+        // Kill fires the client's stop token and then fails the pending command; a caller token that
+        // is wired to gateway shutdown (as the session's is) cancels in the same window.
+        client.Kill("test teardown");
+        await callerCts.CancelAsync();
+
+        await Assert.ThrowsAnyAsync<Exception>(async () => await invokeTask.WaitAsync(TestTimeout));
+
+        await Task.Delay(TimeSpan.FromMilliseconds(250));
+        if (pendingRead.IsCompleted)
+        {
+            WorkerEnvelope? unexpected = await pendingRead;
+            Assert.True(
+                unexpected is null || unexpected.BodyCase != WorkerEnvelope.BodyOneofCase.WorkerCancel,
+                "A stopping worker client must not put a WorkerCancel on the wire.");
+        }
+    }
+
+    /// <summary>
+    ///     A <c>WorkerFault</c> arriving in place of a handshake frame is the worker reporting a
+    ///     structured startup failure — most often MXAccess COM creation — not a protocol problem, so
+    ///     the gateway surfaces <see cref="WorkerClientErrorCode.WorkerFaulted"/> with the fault's
+    ///     category and diagnostic rather than a generic protocol violation naming only the body type
+    ///     (IPC-35). This is the window where the information matters most: first session open on a
+    ///     freshly deployed host, where the message reaches the operator log and the client's
+    ///     <c>OpenFailed</c> detail.
+    /// </summary>
+    /// <returns>A task that represents the asynchronous operation.</returns>
+    [Fact]
+    public async Task StartAsync_WhenWorkerFaultsDuringHandshake_SurfacesFaultCategoryAndDiagnostic()
+    {
+        const string diagnostic = "CoCreateInstance for LMXProxy.LMXProxyServer.1 failed (0x80040154).";
+
+        await using FakeWorkerHarness harness = await FakeWorkerHarness.CreateConnectedPairAsync();
+        await using WorkerClient client = harness.CreateClient();
+
+        Task startTask = client.StartAsync(CancellationToken.None);
+
+        WorkerEnvelope gatewayHello = await harness.ReadGatewayEnvelopeAsync().WaitAsync(TestTimeout);
+        Assert.Equal(WorkerEnvelope.BodyOneofCase.GatewayHello, gatewayHello.BodyCase);
+
+        // The hello succeeds; the fault replaces the WorkerReady, which is where a COM creation
+        // failure actually lands (the worker creates MXAccess after answering the hello).
+        await harness.SendWorkerHelloAsync();
+        await harness.EmitFaultAsync(WorkerFaultCategory.MxaccessCreationFailed, diagnostic);
+
+        WorkerClientException exception = await Assert.ThrowsAsync<WorkerClientException>(
+            async () => await startTask.WaitAsync(TestTimeout));
+
+        Assert.Equal(WorkerClientErrorCode.WorkerFaulted, exception.ErrorCode);
+        Assert.Contains(
+            WorkerFaultCategory.MxaccessCreationFailed.ToString(),
+            exception.Message,
+            StringComparison.Ordinal);
+        Assert.Contains(diagnostic, exception.Message, StringComparison.Ordinal);
+    }
+
     /// <summary>
     ///     The envelope <c>sequence</c> is a monotonic per-sender counter (gateway.md), so the values
     ///     observed on the pipe must be strictly increasing in wire order. Stamping the sequence when