diff --git a/clients/dotnet/ZB.MOM.WW.MxGateway.Client.Tests/MxGatewayClientWireTests.cs b/clients/dotnet/ZB.MOM.WW.MxGateway.Client.Tests/MxGatewayClientWireTests.cs
index e093a892..102764da 100644
--- a/clients/dotnet/ZB.MOM.WW.MxGateway.Client.Tests/MxGatewayClientWireTests.cs
+++ b/clients/dotnet/ZB.MOM.WW.MxGateway.Client.Tests/MxGatewayClientWireTests.cs
@@ -149,6 +149,57 @@ public sealed class MxGatewayClientWireTests
() => session.RegisterAsync("wire-test-client"));
}
+ ///
+ /// A RESOURCE_EXHAUSTED status carrying the mxgw-reason trailer surfaces the
+ /// reason as a typed property, so overflow and limit rejections can be told apart.
+ ///
+ /// The reason code the fake gateway attaches.
+ /// Whether the client should classify it as an overflow.
+ /// Whether the client should classify it as a subscriber-limit rejection.
+ /// A task that represents the asynchronous operation.
+ [Theory]
+ [InlineData("event-queue-overflow", true, false)]
+ [InlineData("subscriber-limit", false, true)]
+ public async Task ResourceExhaustedStatus_ExposesReasonFromTrailer(string reason, bool overflow, bool limit)
+ {
+ await using WireFakeGatewayServer server = await WireFakeGatewayServer.StartAsync(
+ service =>
+ {
+ service.ExhaustInvoke = true;
+ service.ExhaustedReason = reason;
+ });
+ await using MxGatewayClient client = server.CreateClient(ApiKey);
+ MxGatewaySession session = await client.OpenSessionAsync(
+ new OpenSessionRequest { ClientSessionName = "wire-test" });
+
+ MxGatewayResourceExhaustedException exception =
+ await Assert.ThrowsAsync(
+ () => session.RegisterAsync("wire-test-client"));
+
+ Assert.Equal(reason, exception.Reason);
+ Assert.Equal(overflow, exception.IsOverflow);
+ Assert.Equal(limit, exception.IsSubscriberLimit);
+ Assert.IsAssignableFrom(exception);
+ }
+
+ /// A RESOURCE_EXHAUSTED from an older gateway (no trailer) has a null reason.
+ /// A task that represents the asynchronous operation.
+ [Fact]
+ public async Task ResourceExhaustedStatus_WithoutTrailer_HasNullReason()
+ {
+ await using WireFakeGatewayServer server = await WireFakeGatewayServer.StartAsync(
+ service => service.ExhaustInvoke = true);
+ await using MxGatewayClient client = server.CreateClient(ApiKey);
+ MxGatewaySession session = await client.OpenSessionAsync(
+ new OpenSessionRequest { ClientSessionName = "wire-test" });
+
+ MxGatewayResourceExhaustedException exception =
+ await Assert.ThrowsAsync(
+ () => session.RegisterAsync("wire-test-client"));
+
+ Assert.Null(exception.Reason);
+ }
+
private static async Task> CollectAsync(IAsyncEnumerable stream)
{
List events = [];
diff --git a/clients/dotnet/ZB.MOM.WW.MxGateway.Client.Tests/WireFakeGatewayServer.cs b/clients/dotnet/ZB.MOM.WW.MxGateway.Client.Tests/WireFakeGatewayServer.cs
index 3cfc9bd9..328af2e4 100644
--- a/clients/dotnet/ZB.MOM.WW.MxGateway.Client.Tests/WireFakeGatewayServer.cs
+++ b/clients/dotnet/ZB.MOM.WW.MxGateway.Client.Tests/WireFakeGatewayServer.cs
@@ -140,6 +140,16 @@ internal sealed class WireFakeGatewayServer : IAsyncDisposable
///
public bool DenyInvoke { get; set; }
+ ///
+ /// Gets or sets a value indicating whether Invoke fails with
+ /// ; (when set) rides the
+ /// mxgw-reason trailer the way the real gateway attaches it.
+ ///
+ public bool ExhaustInvoke { get; set; }
+
+ /// Gets or sets the reason-code trailer value sent with the exhausted failure.
+ public string? ExhaustedReason { get; set; }
+
///
/// Gets or sets the replay-gap sentinel emitted at the head of the event stream.
///
@@ -189,6 +199,17 @@ internal sealed class WireFakeGatewayServer : IAsyncDisposable
throw new RpcException(new Status(StatusCode.PermissionDenied, "invoke scope required"));
}
+ if (ExhaustInvoke)
+ {
+ Metadata trailers = [];
+ if (ExhaustedReason is not null)
+ {
+ trailers.Add(ZB.MOM.WW.MxGateway.Contracts.ResourceExhaustedReasons.TrailerKey, ExhaustedReason);
+ }
+
+ throw new RpcException(new Status(StatusCode.ResourceExhausted, "exhausted"), trailers);
+ }
+
return Task.FromResult(new MxCommandReply
{
SessionId = request.SessionId,
diff --git a/clients/dotnet/ZB.MOM.WW.MxGateway.Client/GrpcMxGatewayClientTransport.cs b/clients/dotnet/ZB.MOM.WW.MxGateway.Client/GrpcMxGatewayClientTransport.cs
index 2198041a..0a9f6b52 100644
--- a/clients/dotnet/ZB.MOM.WW.MxGateway.Client/GrpcMxGatewayClientTransport.cs
+++ b/clients/dotnet/ZB.MOM.WW.MxGateway.Client/GrpcMxGatewayClientTransport.cs
@@ -255,6 +255,10 @@ internal sealed class GrpcMxGatewayClientTransport(
StatusCode.PermissionDenied => new MxGatewayAuthorizationException(
exception.Status.Detail,
innerException: exception),
+ StatusCode.ResourceExhausted => new MxGatewayResourceExhaustedException(
+ exception.Status.Detail,
+ MxGatewayResourceExhaustedException.ReadReason(exception),
+ exception),
_ => new MxGatewayException(exception.Status.Detail, exception),
};
}
diff --git a/clients/dotnet/ZB.MOM.WW.MxGateway.Client/MxGatewayResourceExhaustedException.cs b/clients/dotnet/ZB.MOM.WW.MxGateway.Client/MxGatewayResourceExhaustedException.cs
new file mode 100644
index 00000000..785f100c
--- /dev/null
+++ b/clients/dotnet/ZB.MOM.WW.MxGateway.Client/MxGatewayResourceExhaustedException.cs
@@ -0,0 +1,51 @@
+using Grpc.Core;
+using ZB.MOM.WW.MxGateway.Contracts;
+
+namespace ZB.MOM.WW.MxGateway.Client;
+
+///
+/// Exception thrown when the gateway answers gRPC ResourceExhausted. The
+/// distinguishes "you fell behind and events were lost" (re-subscribe and
+/// resync) from "a limit was reached" (back off, do not retry immediately). It derives from
+/// , so existing catch blocks keep working.
+///
+public sealed class MxGatewayResourceExhaustedException : MxGatewayException
+{
+ /// Initializes a new instance.
+ /// Diagnostic message describing the failure.
+ /// The gateway's machine-readable reason code, or null when absent (older gateway).
+ /// The underlying .
+ public MxGatewayResourceExhaustedException(string message, string? reason, Exception? innerException = null)
+ : base(message, innerException)
+ {
+ Reason = reason;
+ }
+
+ ///
+ /// Gets the reason code from the mxgw-reason trailer (see ),
+ /// or null when the gateway sent none.
+ ///
+ public string? Reason { get; }
+
+ /// Gets a value indicating whether events were lost because the subscriber fell behind (event-stream or alarm-feed overflow); the caller should re-subscribe and resync.
+ public bool IsOverflow =>
+ Reason is ResourceExhaustedReasons.EventQueueOverflow or ResourceExhaustedReasons.AlarmFeedOverflow;
+
+ /// Gets a value indicating whether the failure was a subscriber-count limit rejection.
+ public bool IsSubscriberLimit =>
+ Reason is ResourceExhaustedReasons.SubscriberLimit or ResourceExhaustedReasons.SubscriberAlreadyActive;
+
+ internal static string? ReadReason(RpcException exception)
+ {
+ foreach (Metadata.Entry entry in exception.Trailers)
+ {
+ if (!entry.IsBinary
+ && string.Equals(entry.Key, ResourceExhaustedReasons.TrailerKey, StringComparison.OrdinalIgnoreCase))
+ {
+ return entry.Value;
+ }
+ }
+
+ return null;
+ }
+}
diff --git a/docs/ClientBehaviorFixtures.md b/docs/ClientBehaviorFixtures.md
index d18817c5..5504827b 100644
--- a/docs/ClientBehaviorFixtures.md
+++ b/docs/ClientBehaviorFixtures.md
@@ -137,6 +137,34 @@ covers only what each client does on its own side of the wire.
| Python | none; `Session.stream_events` is an async iterator over the gRPC call | n/a | gRPC flow control applies back-pressure to the server | the same stream |
| Rust | none; `EventStream` is a boxed `Stream` over the tonic response (the design requires any helper channel to be bounded) | n/a | gRPC flow control applies back-pressure to the server | the same stream |
+### ResourceExhausted Reason Trailer
+
+The gateway uses gRPC `ResourceExhausted` for several different conditions. The status
+code is unchanged, but every such failure now also carries a machine-readable reason in
+the `mxgw-reason` response trailer (ASCII value), so a client can tell "you fell behind and
+events were lost" (re-subscribe and resync) from "a limit was reached" (back off, do not
+retry immediately):
+
+| `mxgw-reason` | Meaning | Client action |
+|---|---|---|
+| `event-queue-overflow` | The `StreamEvents` subscriber fell behind its bounded queue and was disconnected; events were lost. | Reopen with the last seen `afterWorkerSequence` and resync. |
+| `alarm-feed-overflow` | The `StreamAlarms` subscriber fell behind and was dropped. | Reconnect and re-snapshot. |
+| `subscriber-limit` | The per-session event-subscriber cap was reached. | Back off. |
+| `subscriber-already-active` | Only one subscriber is allowed and one is attached. | Back off. |
+| `session-limit` | The gateway session limit was reached. | Back off. |
+| `command-too-large` | A command exceeded the maximum frame size. | Do not retry unchanged. |
+
+A missing trailer (an older gateway, or the API-key failure throttle's `ResourceExhausted`)
+means "reason unknown"; treat it as the generic status. Constants live in
+`ZB.MOM.WW.MxGateway.Contracts.ResourceExhaustedReasons`.
+
+- **.NET** (additive): `ResourceExhausted` now surfaces as `MxGatewayResourceExhaustedException`
+ (a subclass of `MxGatewayException`, so existing catches keep working) with `Reason`,
+ `IsOverflow` and `IsSubscriberLimit`.
+- **Go, Java, Python, Rust**: no typed surface yet (doc note only). Read the `mxgw-reason`
+ trailer from the failing call's trailing metadata (`status.Trailer()` / `Metadata` /
+ `trailing_metadata()` / tonic `Status::metadata()`).
+
After a slow-consumer cancel or an `EventQueueOverflow` disconnect, reopen with
the last seen `afterWorkerSequence`; if it predates the replay ring the stream
opens with the replay-gap sentinel described in `Event Streams` above.
diff --git a/src/ZB.MOM.WW.MxGateway.Contracts/ResourceExhaustedReasons.cs b/src/ZB.MOM.WW.MxGateway.Contracts/ResourceExhaustedReasons.cs
new file mode 100644
index 00000000..7cbfbd52
--- /dev/null
+++ b/src/ZB.MOM.WW.MxGateway.Contracts/ResourceExhaustedReasons.cs
@@ -0,0 +1,30 @@
+namespace ZB.MOM.WW.MxGateway.Contracts;
+
+///
+/// Machine-readable reason codes the gateway attaches to gRPC ResourceExhausted failures
+/// via the response trailer, so a client can tell "you fell behind and
+/// events were lost" from "a limit was reached". The gRPC status code itself is unchanged.
+///
+public static class ResourceExhaustedReasons
+{
+ /// Response-trailer key carrying the reason code (gRPC trailer keys are lower-case ASCII).
+ public const string TrailerKey = "mxgw-reason";
+
+ /// The subscriber fell behind its bounded event queue and was disconnected; events were lost. Re-subscribe and resync.
+ public const string EventQueueOverflow = "event-queue-overflow";
+
+ /// The alarm-feed subscriber fell behind its bounded queue and was dropped. Reconnect and re-snapshot.
+ public const string AlarmFeedOverflow = "alarm-feed-overflow";
+
+ /// The per-session event-subscriber cap was reached. Back off; do not retry immediately.
+ public const string SubscriberLimit = "subscriber-limit";
+
+ /// The session already has an active event subscriber and only one is allowed.
+ public const string SubscriberAlreadyActive = "subscriber-already-active";
+
+ /// The gateway session limit was reached.
+ public const string SessionLimit = "session-limit";
+
+ /// A command exceeded the maximum frame size.
+ public const string CommandTooLarge = "command-too-large";
+}
diff --git a/src/ZB.MOM.WW.MxGateway.Server/Grpc/MxAccessGatewayService.cs b/src/ZB.MOM.WW.MxGateway.Server/Grpc/MxAccessGatewayService.cs
index 2028932c..ef6eb92f 100644
--- a/src/ZB.MOM.WW.MxGateway.Server/Grpc/MxAccessGatewayService.cs
+++ b/src/ZB.MOM.WW.MxGateway.Server/Grpc/MxAccessGatewayService.cs
@@ -261,7 +261,9 @@ public sealed class MxAccessGatewayService(
// client-side slowness condition with a client-side remedy (reconnect and
// re-snapshot), so it maps to ResourceExhausted like a StreamEvents overflow rather
// than reaching MapException's warning-with-stack + Unavailable fallback.
- throw new RpcException(new Status(StatusCode.ResourceExhausted, overflow.Message));
+ throw new RpcException(
+ new Status(StatusCode.ResourceExhausted, overflow.Message),
+ ReasonTrailers(ResourceExhaustedReasons.AlarmFeedOverflow));
}
catch (Exception exception) when (exception is not RpcException)
{
@@ -1113,9 +1115,24 @@ public sealed class MxAccessGatewayService(
_ => StatusCode.Unavailable,
};
- return new RpcException(new Status(statusCode, exception.Message));
+ string? reason = exception.ErrorCode switch
+ {
+ SessionManagerErrorCode.EventSubscriberAlreadyActive => ResourceExhaustedReasons.SubscriberAlreadyActive,
+ SessionManagerErrorCode.EventSubscriberLimitReached => ResourceExhaustedReasons.SubscriberLimit,
+ SessionManagerErrorCode.EventQueueOverflow => ResourceExhaustedReasons.EventQueueOverflow,
+ SessionManagerErrorCode.SessionLimitExceeded => ResourceExhaustedReasons.SessionLimit,
+ _ => null,
+ };
+
+ return reason is null
+ ? new RpcException(new Status(statusCode, exception.Message))
+ : new RpcException(new Status(statusCode, exception.Message), ReasonTrailers(reason));
}
+ /// Builds the response trailers carrying a code.
+ private static Metadata ReasonTrailers(string reason) =>
+ [new Metadata.Entry(ResourceExhaustedReasons.TrailerKey, reason)];
+
private static RpcException MapWorkerClientException(WorkerClientException exception)
{
StatusCode statusCode = exception.ErrorCode switch
@@ -1129,6 +1146,8 @@ public sealed class MxAccessGatewayService(
_ => StatusCode.Unavailable,
};
- return new RpcException(new Status(statusCode, exception.Message));
+ return exception.ErrorCode == WorkerClientErrorCode.CommandTooLarge
+ ? new RpcException(new Status(statusCode, exception.Message), ReasonTrailers(ResourceExhaustedReasons.CommandTooLarge))
+ : new RpcException(new Status(statusCode, exception.Message));
}
}
diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/GatewayEndToEndMultiSubscriberTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/GatewayEndToEndMultiSubscriberTests.cs
index 1b5791b8..a616e15f 100644
--- a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/GatewayEndToEndMultiSubscriberTests.cs
+++ b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/GatewayEndToEndMultiSubscriberTests.cs
@@ -324,6 +324,11 @@ public sealed class GatewayEndToEndMultiSubscriberTests(ITestOutputHelper output
// EventSubscriberLimitReached contains "maximum".
Assert.Contains("maximum", capException.Status.Detail, StringComparison.OrdinalIgnoreCase);
+ // #216: the machine-readable reason distinguishes the cap from an event-queue overflow.
+ Assert.Equal(
+ ResourceExhaustedReasons.SubscriberLimit,
+ capException.Trailers.GetValue(ResourceExhaustedReasons.TrailerKey));
+
// The first two streams must still be live (not completed).
Assert.False(stream1Task.IsCompleted, "Sub1 must still be streaming after sub3 was rejected.");
Assert.False(stream2Task.IsCompleted, "Sub2 must still be streaming after sub3 was rejected.");
diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Grpc/MxAccessGatewayServiceTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Grpc/MxAccessGatewayServiceTests.cs
index 5cebc7a2..24b708c7 100644
--- a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Grpc/MxAccessGatewayServiceTests.cs
+++ b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Grpc/MxAccessGatewayServiceTests.cs
@@ -668,16 +668,46 @@ public sealed class MxAccessGatewayServiceTests
new TestServerCallContext()));
Assert.Equal(StatusCode.ResourceExhausted, exception.StatusCode);
+ Assert.Equal(ResourceExhaustedReasons.AlarmFeedOverflow, exception.Trailers.GetValue(ResourceExhaustedReasons.TrailerKey));
Assert.DoesNotContain(logger.Entries, entry => entry.Level >= LogLevel.Warning);
}
+ ///
+ /// Event-queue overflow and the subscriber cap both stay ResourceExhausted but carry
+ /// distinct machine-readable reason trailers (#216).
+ ///
+ /// The session-manager fault.
+ /// The reason trailer expected.
+ /// A task that represents the asynchronous operation.
+ [Theory]
+ [InlineData(SessionManagerErrorCode.EventQueueOverflow, ResourceExhaustedReasons.EventQueueOverflow)]
+ [InlineData(SessionManagerErrorCode.EventSubscriberLimitReached, ResourceExhaustedReasons.SubscriberLimit)]
+ public async Task StreamEvents_ResourceExhaustedFaults_CarryDistinguishingReasonTrailer(
+ SessionManagerErrorCode errorCode,
+ string expectedReason)
+ {
+ MxAccessGatewayService service = CreateService(
+ new FakeSessionManager(),
+ eventStreamFault: new SessionManagerException(errorCode, "fault"));
+
+ RpcException exception = await Assert.ThrowsAsync(
+ () => service.StreamEvents(
+ new StreamEventsRequest { SessionId = "session-1" },
+ new RecordingServerStreamWriter(),
+ new TestServerCallContext()));
+
+ Assert.Equal(StatusCode.ResourceExhausted, exception.StatusCode);
+ Assert.Equal(expectedReason, exception.Trailers.GetValue(ResourceExhaustedReasons.TrailerKey));
+ }
+
private static MxAccessGatewayService CreateService(
FakeSessionManager sessionManager,
IGatewayRequestIdentityAccessor? identityAccessor = null,
GatewayMetrics? metrics = null,
IConstraintEnforcer? constraintEnforcer = null,
IGatewayAlarmService? alarmService = null,
- ILogger? logger = null)
+ ILogger? logger = null,
+ Exception? eventStreamFault = null)
{
return new MxAccessGatewayService(
sessionManager,
@@ -685,7 +715,7 @@ public sealed class MxAccessGatewayServiceTests
constraintEnforcer ?? new AllowAllConstraintEnforcer(),
new MxAccessGrpcRequestValidator(),
new MxAccessGrpcMapper(),
- new FakeEventStreamService(sessionManager),
+ new FakeEventStreamService(sessionManager) { StreamFault = eventStreamFault },
metrics ?? new GatewayMetrics(),
logger ?? NullLogger.Instance,
alarmService ?? new FakeGatewayAlarmService());
@@ -917,6 +947,9 @@ public sealed class MxAccessGatewayServiceTests
private sealed class FakeEventStreamService(FakeSessionManager sessionManager) : IEventStreamService
{
+ /// Gets or sets a fault thrown when the stream starts.
+ public Exception? StreamFault { get; set; }
+
///
public async IAsyncEnumerable StreamEventsAsync(
StreamEventsRequest request,
@@ -924,6 +957,12 @@ public sealed class MxAccessGatewayServiceTests
[EnumeratorCancellation] CancellationToken cancellationToken)
{
sessionManager.RecordReadEventsSessionId(request.SessionId);
+ if (StreamFault is not null)
+ {
+ await Task.Yield();
+ throw StreamFault;
+ }
+
foreach (WorkerEvent workerEvent in sessionManager.Events)
{
cancellationToken.ThrowIfCancellationRequested();