feat(ipc): negotiate worker frame max, add gRPC headroom, bound DrainEvents (IPC-02/03/04 gateway half)

Proto foundation + gateway-side of the size/backpressure pass:

- IPC-02: add GatewayHello.max_frame_bytes (regen Generated/); gateway sends
  its negotiated worker-frame max in the handshake so the worker can adopt it
  instead of a hard-coded default. Worker read-half lands separately.
- IPC-03: give the pipe frame max envelope-overhead headroom above the public
  gRPC cap (WorkerFrameProtocolOptions.EnvelopeOverheadReserveBytes = 64 KiB;
  default Worker.MaxMessageBytes bumped to 16 MiB + reserve), cross-validate the
  headroom at startup, and pre-check command envelope size in WorkerClient so an
  oversized command fails only that correlation (ResourceExhausted) instead of
  faulting the whole session.
- IPC-04: reject DrainEvents max_events above a public ceiling in the request
  validator (worker per-reply cap lands with the worker half).

Docs: GatewayConfiguration, WorkerFrameProtocol, gateway.md.
Tests: headroom validation, DrainEvents bound, oversized-command per-command
failure (pipe-harness, verified on windev).
This commit is contained in:
Joseph Doherty
2026-07-09 08:49:59 -04:00
parent e1a505d662
commit c8b3a2281a
16 changed files with 362 additions and 65 deletions
@@ -1,6 +1,7 @@
using ZB.MOM.WW.Auth.Abstractions.Ldap;
using ZB.MOM.WW.Configuration;
using ZB.MOM.WW.MxGateway.Contracts;
using ZB.MOM.WW.MxGateway.Server.Workers;
namespace ZB.MOM.WW.MxGateway.Server.Configuration;
@@ -46,6 +47,7 @@ public sealed class GatewayOptionsValidator : OptionsValidatorBase<GatewayOption
ValidateEvents(options.Events, builder);
ValidateDashboard(options.Dashboard, builder, _isProduction);
ValidateProtocol(options.Protocol, builder);
ValidateFrameSizeHeadroom(options.Worker, options.Protocol, builder);
ValidateAlarms(options.Alarms, builder);
ValidateTls(options.Tls, builder);
ValidateSecurity(options.Security, builder);
@@ -466,6 +468,35 @@ public sealed class GatewayOptionsValidator : OptionsValidatorBase<GatewayOption
}
}
// The worker-frame (pipe) maximum must stay above the public gRPC cap by the envelope-overhead
// reserve, otherwise a maximally-sized accepted gRPC payload does not fit one worker frame once
// wrapped in a WorkerEnvelope and the outbound write faults the whole session (IPC-03). Fail fast
// at startup rather than mid-traffic. Only checked when both knobs are themselves in range so the
// message is not doubled up with the individual range errors.
private static void ValidateFrameSizeHeadroom(
WorkerOptions worker,
ProtocolOptions protocol,
ValidationBuilder builder)
{
bool workerInRange = worker.MaxMessageBytes is >= MinimumMaxMessageBytes and <= MaximumMaxMessageBytes;
bool grpcInRange = protocol.MaxGrpcMessageBytes is >= MinimumMaxMessageBytes and <= MaximumMaxMessageBytes;
if (!workerInRange || !grpcInRange)
{
return;
}
long requiredWorkerMax =
(long)protocol.MaxGrpcMessageBytes + WorkerFrameProtocolOptions.EnvelopeOverheadReserveBytes;
if (worker.MaxMessageBytes < requiredWorkerMax)
{
builder.Add(
$"MxGateway:Worker:MaxMessageBytes ({worker.MaxMessageBytes}) must be at least "
+ $"MxGateway:Protocol:MaxGrpcMessageBytes ({protocol.MaxGrpcMessageBytes}) plus the "
+ $"{WorkerFrameProtocolOptions.EnvelopeOverheadReserveBytes}-byte worker-frame envelope reserve "
+ $"(>= {requiredWorkerMax}).");
}
}
private static void AddIfBlank(string? value, string message, ValidationBuilder builder)
{
builder.RequireThat(!string.IsNullOrWhiteSpace(value), message);
@@ -33,6 +33,14 @@ public sealed class WorkerOptions
/// <summary>The grace period in seconds after a heartbeat before considering the worker unresponsive.</summary>
public int HeartbeatGraceSeconds { get; init; } = 15;
/// <summary>The maximum message size in bytes for IPC communication.</summary>
public int MaxMessageBytes { get; init; } = 16 * 1024 * 1024;
/// <summary>
/// The maximum worker-frame (pipe) message size in bytes. Must stay at least
/// <see cref="Workers.WorkerFrameProtocolOptions.EnvelopeOverheadReserveBytes"/> above
/// <see cref="ProtocolOptions.MaxGrpcMessageBytes"/> so a maximally-sized accepted gRPC payload
/// still fits one worker frame (IPC-03); the gateway conveys this value to the worker in the
/// handshake (<c>GatewayHello.max_frame_bytes</c>, IPC-02). Default is the 16 MB public gRPC cap
/// plus that reserve.
/// </summary>
public int MaxMessageBytes { get; init; } =
(16 * 1024 * 1024) + Workers.WorkerFrameProtocolOptions.EnvelopeOverheadReserveBytes;
}