feat(ipc): negotiate worker frame max, add gRPC headroom, bound DrainEvents (IPC-02/03/04 gateway half)

Proto foundation + gateway-side of the size/backpressure pass:

- IPC-02: add GatewayHello.max_frame_bytes (regen Generated/); gateway sends
  its negotiated worker-frame max in the handshake so the worker can adopt it
  instead of a hard-coded default. Worker read-half lands separately.
- IPC-03: give the pipe frame max envelope-overhead headroom above the public
  gRPC cap (WorkerFrameProtocolOptions.EnvelopeOverheadReserveBytes = 64 KiB;
  default Worker.MaxMessageBytes bumped to 16 MiB + reserve), cross-validate the
  headroom at startup, and pre-check command envelope size in WorkerClient so an
  oversized command fails only that correlation (ResourceExhausted) instead of
  faulting the whole session.
- IPC-04: reject DrainEvents max_events above a public ceiling in the request
  validator (worker per-reply cap lands with the worker half).

Docs: GatewayConfiguration, WorkerFrameProtocol, gateway.md.
Tests: headroom validation, DrainEvents bound, oversized-command per-command
failure (pipe-harness, verified on windev).
This commit is contained in:
Joseph Doherty
2026-07-09 08:49:59 -04:00
parent e1a505d662
commit c8b3a2281a
16 changed files with 362 additions and 65 deletions
@@ -771,4 +771,50 @@ public sealed class GatewayOptionsValidatorTests
Assert.True(result.Failed);
Assert.Contains(result.Failures!, f => f.Contains("ApiKeyFailureTrackedPeers"));
}
private static GatewayOptions WithWorkerAndProtocol(WorkerOptions worker, ProtocolOptions protocol)
{
GatewayOptions source = ValidOptions();
return new GatewayOptions
{
Authentication = source.Authentication,
Ldap = source.Ldap,
Worker = worker,
Sessions = source.Sessions,
Events = source.Events,
Dashboard = source.Dashboard,
Protocol = protocol,
Alarms = source.Alarms,
Tls = source.Tls,
};
}
/// <summary>
/// Verifies the default worker-frame maximum keeps the required envelope-overhead reserve above
/// the default public gRPC cap, so a stock configuration passes the IPC-03 headroom check.
/// </summary>
[Fact]
public void Validate_Succeeds_WhenWorkerFrameMaxHasEnvelopeHeadroom()
{
ValidateOptionsResult result = new GatewayOptionsValidator().Validate(null, ValidOptions());
Assert.True(result.Succeeded);
}
/// <summary>
/// Verifies that a worker-frame maximum equal to the gRPC cap (zero headroom) fails validation:
/// a maximally-sized accepted gRPC payload would not fit one worker frame once wrapped in a
/// WorkerEnvelope, faulting the whole session on the outbound write (IPC-03).
/// </summary>
[Fact]
public void Validate_Fails_WhenWorkerFrameMaxEqualsGrpcMaxWithoutHeadroom()
{
const int grpcMax = 16 * 1024 * 1024;
ValidateOptionsResult result = new GatewayOptionsValidator().Validate(
null,
WithWorkerAndProtocol(
new WorkerOptions { MaxMessageBytes = grpcMax },
new ProtocolOptions { MaxGrpcMessageBytes = grpcMax }));
Assert.True(result.Failed);
Assert.Contains(result.Failures!, f => f.Contains("MaxMessageBytes") && f.Contains("reserve"));
}
}
@@ -0,0 +1,45 @@
using Grpc.Core;
using ZB.MOM.WW.MxGateway.Contracts.Proto;
using ZB.MOM.WW.MxGateway.Server.Grpc;
namespace ZB.MOM.WW.MxGateway.Tests.Gateway.Grpc;
public sealed class MxAccessGrpcRequestValidatorTests
{
private static MxCommandRequest DrainRequest(uint maxEvents) => new()
{
SessionId = "session-1",
Command = new MxCommand
{
Kind = MxCommandKind.DrainEvents,
DrainEvents = new DrainEventsCommand { MaxEvents = maxEvents },
},
};
/// <summary>
/// Verifies a DrainEvents request within the per-request ceiling passes validation, including the
/// <c>max_events = 0</c> "worker default cap" sentinel (IPC-04).
/// </summary>
[Theory]
[InlineData(0u)]
[InlineData(1u)]
[InlineData(10_000u)]
public void ValidateInvoke_AllowsDrainEvents_WithinCeiling(uint maxEvents)
{
MxAccessGrpcRequestValidator validator = new();
validator.ValidateInvoke(DrainRequest(maxEvents));
}
/// <summary>
/// Verifies a DrainEvents request above the per-request ceiling is rejected with InvalidArgument
/// so one accepted request cannot pack an unbounded reply frame (IPC-04).
/// </summary>
[Fact]
public void ValidateInvoke_RejectsDrainEvents_AboveCeiling()
{
MxAccessGrpcRequestValidator validator = new();
RpcException exception = Assert.Throws<RpcException>(() => validator.ValidateInvoke(DrainRequest(10_001)));
Assert.Equal(StatusCode.InvalidArgument, exception.StatusCode);
Assert.Contains("max_events", exception.Status.Detail, StringComparison.Ordinal);
}
}
@@ -56,6 +56,52 @@ public sealed class WorkerClientTests
Assert.Equal(MxCommandKind.Ping, reply.Reply.Kind);
}
/// <summary>
/// Verifies that a command whose serialized envelope exceeds the negotiated worker-frame maximum
/// fails only that command with <see cref="WorkerClientErrorCode.CommandTooLarge"/> at the enqueue
/// boundary, leaving the client ready for subsequent commands (IPC-03). Without the pre-check the
/// oversized frame would reach the write loop and fault the whole session.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task InvokeAsync_WhenCommandExceedsFrameMax_FailsOnlyThatCommandAndStaysReady()
{
await using PipePair pipePair = await PipePair.CreateAsync();
await using WorkerClient client = CreateClient(pipePair, maxMessageBytes: 4096);
await CompleteHandshakeAsync(client, pipePair);
WorkerCommand oversized = new()
{
Command = new MxCommand
{
Kind = MxCommandKind.Write,
Write = new WriteCommand
{
ServerHandle = 1,
ItemHandle = 2,
Value = new MxValue { StringValue = new string('x', 8192) },
},
},
};
WorkerClientException exception = await Assert.ThrowsAsync<WorkerClientException>(
async () => await client.InvokeAsync(oversized, TestTimeout, CancellationToken.None));
Assert.Equal(WorkerClientErrorCode.CommandTooLarge, exception.ErrorCode);
Assert.Equal(WorkerClientState.Ready, client.State);
// A subsequent normally-sized command still round-trips: the session was not faulted.
Task<WorkerCommandReply> nextInvoke = client.InvokeAsync(
CreateCommand(MxCommandKind.Ping),
TestTimeout,
CancellationToken.None);
WorkerEnvelope commandEnvelope = await pipePair.WorkerReader.ReadAsync().AsTask().WaitAsync(TestTimeout);
await pipePair.WorkerWriter.WriteAsync(
CreateCommandReplyEnvelope(commandEnvelope.CorrelationId, MxCommandKind.Ping));
WorkerCommandReply reply = await nextInvoke.WaitAsync(TestTimeout);
Assert.Equal(MxCommandKind.Ping, reply.Reply.Kind);
Assert.Equal(WorkerClientState.Ready, client.State);
}
/// <summary>Verifies that InvokeAsync ignores late replies and keeps the client ready.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
@@ -564,9 +610,13 @@ public sealed class WorkerClientTests
WorkerClientOptions? options = null,
GatewayMetrics? metrics = null,
WorkerProcessHandle? processHandle = null,
TimeProvider? timeProvider = null)
TimeProvider? timeProvider = null,
int maxMessageBytes = WorkerFrameProtocolOptions.DefaultMaxMessageBytes)
{
WorkerFrameProtocolOptions frameOptions = new(SessionId);
WorkerFrameProtocolOptions frameOptions = new(
SessionId,
GatewayContractInfo.WorkerProtocolVersion,
maxMessageBytes);
WorkerClientConnection connection = new(
SessionId,
Nonce,