feat(events): distinguish ResourceExhausted causes with an mxgw-reason trailer (mxaccessgw#216)

Event-queue overflow, alarm-feed overflow, subscriber cap, session limit and
oversized commands all surfaced as ResourceExhausted. The status code is
unchanged; the gateway now attaches a machine-readable `mxgw-reason` trailer
(constants in Contracts ResourceExhaustedReasons). The .NET client maps
ResourceExhausted to MxGatewayResourceExhaustedException (an
MxGatewayException subclass; additive) exposing Reason/IsOverflow/
IsSubscriberLimit. Documented in docs/ClientBehaviorFixtures.md; the other
clients get a doc note only.

Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
Joseph Doherty
2026-11-09 15:02:24 -05:00
parent 716c845fc7
commit fccc21ba8e
9 changed files with 253 additions and 5 deletions
@@ -149,6 +149,57 @@ public sealed class MxGatewayClientWireTests
() => session.RegisterAsync("wire-test-client")); () => session.RegisterAsync("wire-test-client"));
} }
/// <summary>
/// A <c>RESOURCE_EXHAUSTED</c> status carrying the <c>mxgw-reason</c> trailer surfaces the
/// reason as a typed property, so overflow and limit rejections can be told apart.
/// </summary>
/// <param name="reason">The reason code the fake gateway attaches.</param>
/// <param name="overflow">Whether the client should classify it as an overflow.</param>
/// <param name="limit">Whether the client should classify it as a subscriber-limit rejection.</param>
/// <returns>A task that represents the asynchronous operation.</returns>
[Theory]
[InlineData("event-queue-overflow", true, false)]
[InlineData("subscriber-limit", false, true)]
public async Task ResourceExhaustedStatus_ExposesReasonFromTrailer(string reason, bool overflow, bool limit)
{
await using WireFakeGatewayServer server = await WireFakeGatewayServer.StartAsync(
service =>
{
service.ExhaustInvoke = true;
service.ExhaustedReason = reason;
});
await using MxGatewayClient client = server.CreateClient(ApiKey);
MxGatewaySession session = await client.OpenSessionAsync(
new OpenSessionRequest { ClientSessionName = "wire-test" });
MxGatewayResourceExhaustedException exception =
await Assert.ThrowsAsync<MxGatewayResourceExhaustedException>(
() => session.RegisterAsync("wire-test-client"));
Assert.Equal(reason, exception.Reason);
Assert.Equal(overflow, exception.IsOverflow);
Assert.Equal(limit, exception.IsSubscriberLimit);
Assert.IsAssignableFrom<MxGatewayException>(exception);
}
/// <summary>A <c>RESOURCE_EXHAUSTED</c> from an older gateway (no trailer) has a null reason.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task ResourceExhaustedStatus_WithoutTrailer_HasNullReason()
{
await using WireFakeGatewayServer server = await WireFakeGatewayServer.StartAsync(
service => service.ExhaustInvoke = true);
await using MxGatewayClient client = server.CreateClient(ApiKey);
MxGatewaySession session = await client.OpenSessionAsync(
new OpenSessionRequest { ClientSessionName = "wire-test" });
MxGatewayResourceExhaustedException exception =
await Assert.ThrowsAsync<MxGatewayResourceExhaustedException>(
() => session.RegisterAsync("wire-test-client"));
Assert.Null(exception.Reason);
}
private static async Task<List<MxEvent>> CollectAsync(IAsyncEnumerable<MxEvent> stream) private static async Task<List<MxEvent>> CollectAsync(IAsyncEnumerable<MxEvent> stream)
{ {
List<MxEvent> events = []; List<MxEvent> events = [];
@@ -140,6 +140,16 @@ internal sealed class WireFakeGatewayServer : IAsyncDisposable
/// </summary> /// </summary>
public bool DenyInvoke { get; set; } public bool DenyInvoke { get; set; }
/// <summary>
/// Gets or sets a value indicating whether <c>Invoke</c> fails with
/// <see cref="StatusCode.ResourceExhausted"/>; <see cref="ExhaustedReason"/> (when set) rides the
/// <c>mxgw-reason</c> trailer the way the real gateway attaches it.
/// </summary>
public bool ExhaustInvoke { get; set; }
/// <summary>Gets or sets the reason-code trailer value sent with the exhausted failure.</summary>
public string? ExhaustedReason { get; set; }
/// <summary> /// <summary>
/// Gets or sets the replay-gap sentinel emitted at the head of the event stream. /// Gets or sets the replay-gap sentinel emitted at the head of the event stream.
/// </summary> /// </summary>
@@ -189,6 +199,17 @@ internal sealed class WireFakeGatewayServer : IAsyncDisposable
throw new RpcException(new Status(StatusCode.PermissionDenied, "invoke scope required")); throw new RpcException(new Status(StatusCode.PermissionDenied, "invoke scope required"));
} }
if (ExhaustInvoke)
{
Metadata trailers = [];
if (ExhaustedReason is not null)
{
trailers.Add(ZB.MOM.WW.MxGateway.Contracts.ResourceExhaustedReasons.TrailerKey, ExhaustedReason);
}
throw new RpcException(new Status(StatusCode.ResourceExhausted, "exhausted"), trailers);
}
return Task.FromResult(new MxCommandReply return Task.FromResult(new MxCommandReply
{ {
SessionId = request.SessionId, SessionId = request.SessionId,
@@ -255,6 +255,10 @@ internal sealed class GrpcMxGatewayClientTransport(
StatusCode.PermissionDenied => new MxGatewayAuthorizationException( StatusCode.PermissionDenied => new MxGatewayAuthorizationException(
exception.Status.Detail, exception.Status.Detail,
innerException: exception), innerException: exception),
StatusCode.ResourceExhausted => new MxGatewayResourceExhaustedException(
exception.Status.Detail,
MxGatewayResourceExhaustedException.ReadReason(exception),
exception),
_ => new MxGatewayException(exception.Status.Detail, exception), _ => new MxGatewayException(exception.Status.Detail, exception),
}; };
} }
@@ -0,0 +1,51 @@
using Grpc.Core;
using ZB.MOM.WW.MxGateway.Contracts;
namespace ZB.MOM.WW.MxGateway.Client;
/// <summary>
/// Exception thrown when the gateway answers gRPC <c>ResourceExhausted</c>. The
/// <see cref="Reason"/> distinguishes "you fell behind and events were lost" (re-subscribe and
/// resync) from "a limit was reached" (back off, do not retry immediately). It derives from
/// <see cref="MxGatewayException"/>, so existing catch blocks keep working.
/// </summary>
public sealed class MxGatewayResourceExhaustedException : MxGatewayException
{
/// <summary>Initializes a new instance.</summary>
/// <param name="message">Diagnostic message describing the failure.</param>
/// <param name="reason">The gateway's machine-readable reason code, or null when absent (older gateway).</param>
/// <param name="innerException">The underlying <see cref="RpcException"/>.</param>
public MxGatewayResourceExhaustedException(string message, string? reason, Exception? innerException = null)
: base(message, innerException)
{
Reason = reason;
}
/// <summary>
/// Gets the reason code from the <c>mxgw-reason</c> trailer (see <see cref="ResourceExhaustedReasons"/>),
/// or null when the gateway sent none.
/// </summary>
public string? Reason { get; }
/// <summary>Gets a value indicating whether events were lost because the subscriber fell behind (event-stream or alarm-feed overflow); the caller should re-subscribe and resync.</summary>
public bool IsOverflow =>
Reason is ResourceExhaustedReasons.EventQueueOverflow or ResourceExhaustedReasons.AlarmFeedOverflow;
/// <summary>Gets a value indicating whether the failure was a subscriber-count limit rejection.</summary>
public bool IsSubscriberLimit =>
Reason is ResourceExhaustedReasons.SubscriberLimit or ResourceExhaustedReasons.SubscriberAlreadyActive;
internal static string? ReadReason(RpcException exception)
{
foreach (Metadata.Entry entry in exception.Trailers)
{
if (!entry.IsBinary
&& string.Equals(entry.Key, ResourceExhaustedReasons.TrailerKey, StringComparison.OrdinalIgnoreCase))
{
return entry.Value;
}
}
return null;
}
}
+28
View File
@@ -137,6 +137,34 @@ covers only what each client does on its own side of the wire.
| Python | none; `Session.stream_events` is an async iterator over the gRPC call | n/a | gRPC flow control applies back-pressure to the server | the same stream | | Python | none; `Session.stream_events` is an async iterator over the gRPC call | n/a | gRPC flow control applies back-pressure to the server | the same stream |
| Rust | none; `EventStream` is a boxed `Stream` over the tonic response (the design requires any helper channel to be bounded) | n/a | gRPC flow control applies back-pressure to the server | the same stream | | Rust | none; `EventStream` is a boxed `Stream` over the tonic response (the design requires any helper channel to be bounded) | n/a | gRPC flow control applies back-pressure to the server | the same stream |
### ResourceExhausted Reason Trailer
The gateway uses gRPC `ResourceExhausted` for several different conditions. The status
code is unchanged, but every such failure now also carries a machine-readable reason in
the `mxgw-reason` response trailer (ASCII value), so a client can tell "you fell behind and
events were lost" (re-subscribe and resync) from "a limit was reached" (back off, do not
retry immediately):
| `mxgw-reason` | Meaning | Client action |
|---|---|---|
| `event-queue-overflow` | The `StreamEvents` subscriber fell behind its bounded queue and was disconnected; events were lost. | Reopen with the last seen `afterWorkerSequence` and resync. |
| `alarm-feed-overflow` | The `StreamAlarms` subscriber fell behind and was dropped. | Reconnect and re-snapshot. |
| `subscriber-limit` | The per-session event-subscriber cap was reached. | Back off. |
| `subscriber-already-active` | Only one subscriber is allowed and one is attached. | Back off. |
| `session-limit` | The gateway session limit was reached. | Back off. |
| `command-too-large` | A command exceeded the maximum frame size. | Do not retry unchanged. |
A missing trailer (an older gateway, or the API-key failure throttle's `ResourceExhausted`)
means "reason unknown"; treat it as the generic status. Constants live in
`ZB.MOM.WW.MxGateway.Contracts.ResourceExhaustedReasons`.
- **.NET** (additive): `ResourceExhausted` now surfaces as `MxGatewayResourceExhaustedException`
(a subclass of `MxGatewayException`, so existing catches keep working) with `Reason`,
`IsOverflow` and `IsSubscriberLimit`.
- **Go, Java, Python, Rust**: no typed surface yet (doc note only). Read the `mxgw-reason`
trailer from the failing call's trailing metadata (`status.Trailer()` / `Metadata` /
`trailing_metadata()` / tonic `Status::metadata()`).
After a slow-consumer cancel or an `EventQueueOverflow` disconnect, reopen with After a slow-consumer cancel or an `EventQueueOverflow` disconnect, reopen with
the last seen `afterWorkerSequence`; if it predates the replay ring the stream the last seen `afterWorkerSequence`; if it predates the replay ring the stream
opens with the replay-gap sentinel described in `Event Streams` above. opens with the replay-gap sentinel described in `Event Streams` above.
@@ -0,0 +1,30 @@
namespace ZB.MOM.WW.MxGateway.Contracts;
/// <summary>
/// Machine-readable reason codes the gateway attaches to gRPC <c>ResourceExhausted</c> failures
/// via the <see cref="TrailerKey"/> response trailer, so a client can tell "you fell behind and
/// events were lost" from "a limit was reached". The gRPC status code itself is unchanged.
/// </summary>
public static class ResourceExhaustedReasons
{
/// <summary>Response-trailer key carrying the reason code (gRPC trailer keys are lower-case ASCII).</summary>
public const string TrailerKey = "mxgw-reason";
/// <summary>The subscriber fell behind its bounded event queue and was disconnected; events were lost. Re-subscribe and resync.</summary>
public const string EventQueueOverflow = "event-queue-overflow";
/// <summary>The alarm-feed subscriber fell behind its bounded queue and was dropped. Reconnect and re-snapshot.</summary>
public const string AlarmFeedOverflow = "alarm-feed-overflow";
/// <summary>The per-session event-subscriber cap was reached. Back off; do not retry immediately.</summary>
public const string SubscriberLimit = "subscriber-limit";
/// <summary>The session already has an active event subscriber and only one is allowed.</summary>
public const string SubscriberAlreadyActive = "subscriber-already-active";
/// <summary>The gateway session limit was reached.</summary>
public const string SessionLimit = "session-limit";
/// <summary>A command exceeded the maximum frame size.</summary>
public const string CommandTooLarge = "command-too-large";
}
@@ -261,7 +261,9 @@ public sealed class MxAccessGatewayService(
// client-side slowness condition with a client-side remedy (reconnect and // client-side slowness condition with a client-side remedy (reconnect and
// re-snapshot), so it maps to ResourceExhausted like a StreamEvents overflow rather // re-snapshot), so it maps to ResourceExhausted like a StreamEvents overflow rather
// than reaching MapException's warning-with-stack + Unavailable fallback. // than reaching MapException's warning-with-stack + Unavailable fallback.
throw new RpcException(new Status(StatusCode.ResourceExhausted, overflow.Message)); throw new RpcException(
new Status(StatusCode.ResourceExhausted, overflow.Message),
ReasonTrailers(ResourceExhaustedReasons.AlarmFeedOverflow));
} }
catch (Exception exception) when (exception is not RpcException) catch (Exception exception) when (exception is not RpcException)
{ {
@@ -1113,9 +1115,24 @@ public sealed class MxAccessGatewayService(
_ => StatusCode.Unavailable, _ => StatusCode.Unavailable,
}; };
return new RpcException(new Status(statusCode, exception.Message)); string? reason = exception.ErrorCode switch
{
SessionManagerErrorCode.EventSubscriberAlreadyActive => ResourceExhaustedReasons.SubscriberAlreadyActive,
SessionManagerErrorCode.EventSubscriberLimitReached => ResourceExhaustedReasons.SubscriberLimit,
SessionManagerErrorCode.EventQueueOverflow => ResourceExhaustedReasons.EventQueueOverflow,
SessionManagerErrorCode.SessionLimitExceeded => ResourceExhaustedReasons.SessionLimit,
_ => null,
};
return reason is null
? new RpcException(new Status(statusCode, exception.Message))
: new RpcException(new Status(statusCode, exception.Message), ReasonTrailers(reason));
} }
/// <summary>Builds the response trailers carrying a <see cref="ResourceExhaustedReasons"/> code.</summary>
private static Metadata ReasonTrailers(string reason) =>
[new Metadata.Entry(ResourceExhaustedReasons.TrailerKey, reason)];
private static RpcException MapWorkerClientException(WorkerClientException exception) private static RpcException MapWorkerClientException(WorkerClientException exception)
{ {
StatusCode statusCode = exception.ErrorCode switch StatusCode statusCode = exception.ErrorCode switch
@@ -1129,6 +1146,8 @@ public sealed class MxAccessGatewayService(
_ => StatusCode.Unavailable, _ => StatusCode.Unavailable,
}; };
return new RpcException(new Status(statusCode, exception.Message)); return exception.ErrorCode == WorkerClientErrorCode.CommandTooLarge
? new RpcException(new Status(statusCode, exception.Message), ReasonTrailers(ResourceExhaustedReasons.CommandTooLarge))
: new RpcException(new Status(statusCode, exception.Message));
} }
} }
@@ -324,6 +324,11 @@ public sealed class GatewayEndToEndMultiSubscriberTests(ITestOutputHelper output
// EventSubscriberLimitReached contains "maximum". // EventSubscriberLimitReached contains "maximum".
Assert.Contains("maximum", capException.Status.Detail, StringComparison.OrdinalIgnoreCase); Assert.Contains("maximum", capException.Status.Detail, StringComparison.OrdinalIgnoreCase);
// #216: the machine-readable reason distinguishes the cap from an event-queue overflow.
Assert.Equal(
ResourceExhaustedReasons.SubscriberLimit,
capException.Trailers.GetValue(ResourceExhaustedReasons.TrailerKey));
// The first two streams must still be live (not completed). // The first two streams must still be live (not completed).
Assert.False(stream1Task.IsCompleted, "Sub1 must still be streaming after sub3 was rejected."); Assert.False(stream1Task.IsCompleted, "Sub1 must still be streaming after sub3 was rejected.");
Assert.False(stream2Task.IsCompleted, "Sub2 must still be streaming after sub3 was rejected."); Assert.False(stream2Task.IsCompleted, "Sub2 must still be streaming after sub3 was rejected.");
@@ -668,16 +668,46 @@ public sealed class MxAccessGatewayServiceTests
new TestServerCallContext())); new TestServerCallContext()));
Assert.Equal(StatusCode.ResourceExhausted, exception.StatusCode); Assert.Equal(StatusCode.ResourceExhausted, exception.StatusCode);
Assert.Equal(ResourceExhaustedReasons.AlarmFeedOverflow, exception.Trailers.GetValue(ResourceExhaustedReasons.TrailerKey));
Assert.DoesNotContain(logger.Entries, entry => entry.Level >= LogLevel.Warning); Assert.DoesNotContain(logger.Entries, entry => entry.Level >= LogLevel.Warning);
} }
/// <summary>
/// Event-queue overflow and the subscriber cap both stay <c>ResourceExhausted</c> but carry
/// distinct machine-readable reason trailers (#216).
/// </summary>
/// <param name="errorCode">The session-manager fault.</param>
/// <param name="expectedReason">The reason trailer expected.</param>
/// <returns>A task that represents the asynchronous operation.</returns>
[Theory]
[InlineData(SessionManagerErrorCode.EventQueueOverflow, ResourceExhaustedReasons.EventQueueOverflow)]
[InlineData(SessionManagerErrorCode.EventSubscriberLimitReached, ResourceExhaustedReasons.SubscriberLimit)]
public async Task StreamEvents_ResourceExhaustedFaults_CarryDistinguishingReasonTrailer(
SessionManagerErrorCode errorCode,
string expectedReason)
{
MxAccessGatewayService service = CreateService(
new FakeSessionManager(),
eventStreamFault: new SessionManagerException(errorCode, "fault"));
RpcException exception = await Assert.ThrowsAsync<RpcException>(
() => service.StreamEvents(
new StreamEventsRequest { SessionId = "session-1" },
new RecordingServerStreamWriter<MxEvent>(),
new TestServerCallContext()));
Assert.Equal(StatusCode.ResourceExhausted, exception.StatusCode);
Assert.Equal(expectedReason, exception.Trailers.GetValue(ResourceExhaustedReasons.TrailerKey));
}
private static MxAccessGatewayService CreateService( private static MxAccessGatewayService CreateService(
FakeSessionManager sessionManager, FakeSessionManager sessionManager,
IGatewayRequestIdentityAccessor? identityAccessor = null, IGatewayRequestIdentityAccessor? identityAccessor = null,
GatewayMetrics? metrics = null, GatewayMetrics? metrics = null,
IConstraintEnforcer? constraintEnforcer = null, IConstraintEnforcer? constraintEnforcer = null,
IGatewayAlarmService? alarmService = null, IGatewayAlarmService? alarmService = null,
ILogger<MxAccessGatewayService>? logger = null) ILogger<MxAccessGatewayService>? logger = null,
Exception? eventStreamFault = null)
{ {
return new MxAccessGatewayService( return new MxAccessGatewayService(
sessionManager, sessionManager,
@@ -685,7 +715,7 @@ public sealed class MxAccessGatewayServiceTests
constraintEnforcer ?? new AllowAllConstraintEnforcer(), constraintEnforcer ?? new AllowAllConstraintEnforcer(),
new MxAccessGrpcRequestValidator(), new MxAccessGrpcRequestValidator(),
new MxAccessGrpcMapper(), new MxAccessGrpcMapper(),
new FakeEventStreamService(sessionManager), new FakeEventStreamService(sessionManager) { StreamFault = eventStreamFault },
metrics ?? new GatewayMetrics(), metrics ?? new GatewayMetrics(),
logger ?? NullLogger<MxAccessGatewayService>.Instance, logger ?? NullLogger<MxAccessGatewayService>.Instance,
alarmService ?? new FakeGatewayAlarmService()); alarmService ?? new FakeGatewayAlarmService());
@@ -917,6 +947,9 @@ public sealed class MxAccessGatewayServiceTests
private sealed class FakeEventStreamService(FakeSessionManager sessionManager) : IEventStreamService private sealed class FakeEventStreamService(FakeSessionManager sessionManager) : IEventStreamService
{ {
/// <summary>Gets or sets a fault thrown when the stream starts.</summary>
public Exception? StreamFault { get; set; }
/// <inheritdoc /> /// <inheritdoc />
public async IAsyncEnumerable<MxEvent> StreamEventsAsync( public async IAsyncEnumerable<MxEvent> StreamEventsAsync(
StreamEventsRequest request, StreamEventsRequest request,
@@ -924,6 +957,12 @@ public sealed class MxAccessGatewayServiceTests
[EnumeratorCancellation] CancellationToken cancellationToken) [EnumeratorCancellation] CancellationToken cancellationToken)
{ {
sessionManager.RecordReadEventsSessionId(request.SessionId); sessionManager.RecordReadEventsSessionId(request.SessionId);
if (StreamFault is not null)
{
await Task.Yield();
throw StreamFault;
}
foreach (WorkerEvent workerEvent in sessionManager.Events) foreach (WorkerEvent workerEvent in sessionManager.Events)
{ {
cancellationToken.ThrowIfCancellationRequested(); cancellationToken.ThrowIfCancellationRequested();