diff --git a/clients/dotnet/ZB.MOM.WW.MxGateway.Client.Tests/MxGatewayClientWireTests.cs b/clients/dotnet/ZB.MOM.WW.MxGateway.Client.Tests/MxGatewayClientWireTests.cs index e093a892..102764da 100644 --- a/clients/dotnet/ZB.MOM.WW.MxGateway.Client.Tests/MxGatewayClientWireTests.cs +++ b/clients/dotnet/ZB.MOM.WW.MxGateway.Client.Tests/MxGatewayClientWireTests.cs @@ -149,6 +149,57 @@ public sealed class MxGatewayClientWireTests () => session.RegisterAsync("wire-test-client")); } + /// + /// A RESOURCE_EXHAUSTED status carrying the mxgw-reason trailer surfaces the + /// reason as a typed property, so overflow and limit rejections can be told apart. + /// + /// The reason code the fake gateway attaches. + /// Whether the client should classify it as an overflow. + /// Whether the client should classify it as a subscriber-limit rejection. + /// A task that represents the asynchronous operation. + [Theory] + [InlineData("event-queue-overflow", true, false)] + [InlineData("subscriber-limit", false, true)] + public async Task ResourceExhaustedStatus_ExposesReasonFromTrailer(string reason, bool overflow, bool limit) + { + await using WireFakeGatewayServer server = await WireFakeGatewayServer.StartAsync( + service => + { + service.ExhaustInvoke = true; + service.ExhaustedReason = reason; + }); + await using MxGatewayClient client = server.CreateClient(ApiKey); + MxGatewaySession session = await client.OpenSessionAsync( + new OpenSessionRequest { ClientSessionName = "wire-test" }); + + MxGatewayResourceExhaustedException exception = + await Assert.ThrowsAsync( + () => session.RegisterAsync("wire-test-client")); + + Assert.Equal(reason, exception.Reason); + Assert.Equal(overflow, exception.IsOverflow); + Assert.Equal(limit, exception.IsSubscriberLimit); + Assert.IsAssignableFrom(exception); + } + + /// A RESOURCE_EXHAUSTED from an older gateway (no trailer) has a null reason. + /// A task that represents the asynchronous operation. + [Fact] + public async Task ResourceExhaustedStatus_WithoutTrailer_HasNullReason() + { + await using WireFakeGatewayServer server = await WireFakeGatewayServer.StartAsync( + service => service.ExhaustInvoke = true); + await using MxGatewayClient client = server.CreateClient(ApiKey); + MxGatewaySession session = await client.OpenSessionAsync( + new OpenSessionRequest { ClientSessionName = "wire-test" }); + + MxGatewayResourceExhaustedException exception = + await Assert.ThrowsAsync( + () => session.RegisterAsync("wire-test-client")); + + Assert.Null(exception.Reason); + } + private static async Task> CollectAsync(IAsyncEnumerable stream) { List events = []; diff --git a/clients/dotnet/ZB.MOM.WW.MxGateway.Client.Tests/WireFakeGatewayServer.cs b/clients/dotnet/ZB.MOM.WW.MxGateway.Client.Tests/WireFakeGatewayServer.cs index 3cfc9bd9..328af2e4 100644 --- a/clients/dotnet/ZB.MOM.WW.MxGateway.Client.Tests/WireFakeGatewayServer.cs +++ b/clients/dotnet/ZB.MOM.WW.MxGateway.Client.Tests/WireFakeGatewayServer.cs @@ -140,6 +140,16 @@ internal sealed class WireFakeGatewayServer : IAsyncDisposable /// public bool DenyInvoke { get; set; } + /// + /// Gets or sets a value indicating whether Invoke fails with + /// ; (when set) rides the + /// mxgw-reason trailer the way the real gateway attaches it. + /// + public bool ExhaustInvoke { get; set; } + + /// Gets or sets the reason-code trailer value sent with the exhausted failure. + public string? ExhaustedReason { get; set; } + /// /// Gets or sets the replay-gap sentinel emitted at the head of the event stream. /// @@ -189,6 +199,17 @@ internal sealed class WireFakeGatewayServer : IAsyncDisposable throw new RpcException(new Status(StatusCode.PermissionDenied, "invoke scope required")); } + if (ExhaustInvoke) + { + Metadata trailers = []; + if (ExhaustedReason is not null) + { + trailers.Add(ZB.MOM.WW.MxGateway.Contracts.ResourceExhaustedReasons.TrailerKey, ExhaustedReason); + } + + throw new RpcException(new Status(StatusCode.ResourceExhausted, "exhausted"), trailers); + } + return Task.FromResult(new MxCommandReply { SessionId = request.SessionId, diff --git a/clients/dotnet/ZB.MOM.WW.MxGateway.Client/GrpcMxGatewayClientTransport.cs b/clients/dotnet/ZB.MOM.WW.MxGateway.Client/GrpcMxGatewayClientTransport.cs index 2198041a..0a9f6b52 100644 --- a/clients/dotnet/ZB.MOM.WW.MxGateway.Client/GrpcMxGatewayClientTransport.cs +++ b/clients/dotnet/ZB.MOM.WW.MxGateway.Client/GrpcMxGatewayClientTransport.cs @@ -255,6 +255,10 @@ internal sealed class GrpcMxGatewayClientTransport( StatusCode.PermissionDenied => new MxGatewayAuthorizationException( exception.Status.Detail, innerException: exception), + StatusCode.ResourceExhausted => new MxGatewayResourceExhaustedException( + exception.Status.Detail, + MxGatewayResourceExhaustedException.ReadReason(exception), + exception), _ => new MxGatewayException(exception.Status.Detail, exception), }; } diff --git a/clients/dotnet/ZB.MOM.WW.MxGateway.Client/MxGatewayResourceExhaustedException.cs b/clients/dotnet/ZB.MOM.WW.MxGateway.Client/MxGatewayResourceExhaustedException.cs new file mode 100644 index 00000000..785f100c --- /dev/null +++ b/clients/dotnet/ZB.MOM.WW.MxGateway.Client/MxGatewayResourceExhaustedException.cs @@ -0,0 +1,51 @@ +using Grpc.Core; +using ZB.MOM.WW.MxGateway.Contracts; + +namespace ZB.MOM.WW.MxGateway.Client; + +/// +/// Exception thrown when the gateway answers gRPC ResourceExhausted. The +/// distinguishes "you fell behind and events were lost" (re-subscribe and +/// resync) from "a limit was reached" (back off, do not retry immediately). It derives from +/// , so existing catch blocks keep working. +/// +public sealed class MxGatewayResourceExhaustedException : MxGatewayException +{ + /// Initializes a new instance. + /// Diagnostic message describing the failure. + /// The gateway's machine-readable reason code, or null when absent (older gateway). + /// The underlying . + public MxGatewayResourceExhaustedException(string message, string? reason, Exception? innerException = null) + : base(message, innerException) + { + Reason = reason; + } + + /// + /// Gets the reason code from the mxgw-reason trailer (see ), + /// or null when the gateway sent none. + /// + public string? Reason { get; } + + /// Gets a value indicating whether events were lost because the subscriber fell behind (event-stream or alarm-feed overflow); the caller should re-subscribe and resync. + public bool IsOverflow => + Reason is ResourceExhaustedReasons.EventQueueOverflow or ResourceExhaustedReasons.AlarmFeedOverflow; + + /// Gets a value indicating whether the failure was a subscriber-count limit rejection. + public bool IsSubscriberLimit => + Reason is ResourceExhaustedReasons.SubscriberLimit or ResourceExhaustedReasons.SubscriberAlreadyActive; + + internal static string? ReadReason(RpcException exception) + { + foreach (Metadata.Entry entry in exception.Trailers) + { + if (!entry.IsBinary + && string.Equals(entry.Key, ResourceExhaustedReasons.TrailerKey, StringComparison.OrdinalIgnoreCase)) + { + return entry.Value; + } + } + + return null; + } +} diff --git a/docs/ClientBehaviorFixtures.md b/docs/ClientBehaviorFixtures.md index d18817c5..5504827b 100644 --- a/docs/ClientBehaviorFixtures.md +++ b/docs/ClientBehaviorFixtures.md @@ -137,6 +137,34 @@ covers only what each client does on its own side of the wire. | Python | none; `Session.stream_events` is an async iterator over the gRPC call | n/a | gRPC flow control applies back-pressure to the server | the same stream | | Rust | none; `EventStream` is a boxed `Stream` over the tonic response (the design requires any helper channel to be bounded) | n/a | gRPC flow control applies back-pressure to the server | the same stream | +### ResourceExhausted Reason Trailer + +The gateway uses gRPC `ResourceExhausted` for several different conditions. The status +code is unchanged, but every such failure now also carries a machine-readable reason in +the `mxgw-reason` response trailer (ASCII value), so a client can tell "you fell behind and +events were lost" (re-subscribe and resync) from "a limit was reached" (back off, do not +retry immediately): + +| `mxgw-reason` | Meaning | Client action | +|---|---|---| +| `event-queue-overflow` | The `StreamEvents` subscriber fell behind its bounded queue and was disconnected; events were lost. | Reopen with the last seen `afterWorkerSequence` and resync. | +| `alarm-feed-overflow` | The `StreamAlarms` subscriber fell behind and was dropped. | Reconnect and re-snapshot. | +| `subscriber-limit` | The per-session event-subscriber cap was reached. | Back off. | +| `subscriber-already-active` | Only one subscriber is allowed and one is attached. | Back off. | +| `session-limit` | The gateway session limit was reached. | Back off. | +| `command-too-large` | A command exceeded the maximum frame size. | Do not retry unchanged. | + +A missing trailer (an older gateway, or the API-key failure throttle's `ResourceExhausted`) +means "reason unknown"; treat it as the generic status. Constants live in +`ZB.MOM.WW.MxGateway.Contracts.ResourceExhaustedReasons`. + +- **.NET** (additive): `ResourceExhausted` now surfaces as `MxGatewayResourceExhaustedException` + (a subclass of `MxGatewayException`, so existing catches keep working) with `Reason`, + `IsOverflow` and `IsSubscriberLimit`. +- **Go, Java, Python, Rust**: no typed surface yet (doc note only). Read the `mxgw-reason` + trailer from the failing call's trailing metadata (`status.Trailer()` / `Metadata` / + `trailing_metadata()` / tonic `Status::metadata()`). + After a slow-consumer cancel or an `EventQueueOverflow` disconnect, reopen with the last seen `afterWorkerSequence`; if it predates the replay ring the stream opens with the replay-gap sentinel described in `Event Streams` above. diff --git a/src/ZB.MOM.WW.MxGateway.Contracts/ResourceExhaustedReasons.cs b/src/ZB.MOM.WW.MxGateway.Contracts/ResourceExhaustedReasons.cs new file mode 100644 index 00000000..7cbfbd52 --- /dev/null +++ b/src/ZB.MOM.WW.MxGateway.Contracts/ResourceExhaustedReasons.cs @@ -0,0 +1,30 @@ +namespace ZB.MOM.WW.MxGateway.Contracts; + +/// +/// Machine-readable reason codes the gateway attaches to gRPC ResourceExhausted failures +/// via the response trailer, so a client can tell "you fell behind and +/// events were lost" from "a limit was reached". The gRPC status code itself is unchanged. +/// +public static class ResourceExhaustedReasons +{ + /// Response-trailer key carrying the reason code (gRPC trailer keys are lower-case ASCII). + public const string TrailerKey = "mxgw-reason"; + + /// The subscriber fell behind its bounded event queue and was disconnected; events were lost. Re-subscribe and resync. + public const string EventQueueOverflow = "event-queue-overflow"; + + /// The alarm-feed subscriber fell behind its bounded queue and was dropped. Reconnect and re-snapshot. + public const string AlarmFeedOverflow = "alarm-feed-overflow"; + + /// The per-session event-subscriber cap was reached. Back off; do not retry immediately. + public const string SubscriberLimit = "subscriber-limit"; + + /// The session already has an active event subscriber and only one is allowed. + public const string SubscriberAlreadyActive = "subscriber-already-active"; + + /// The gateway session limit was reached. + public const string SessionLimit = "session-limit"; + + /// A command exceeded the maximum frame size. + public const string CommandTooLarge = "command-too-large"; +} diff --git a/src/ZB.MOM.WW.MxGateway.Server/Grpc/MxAccessGatewayService.cs b/src/ZB.MOM.WW.MxGateway.Server/Grpc/MxAccessGatewayService.cs index 2028932c..ef6eb92f 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Grpc/MxAccessGatewayService.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Grpc/MxAccessGatewayService.cs @@ -261,7 +261,9 @@ public sealed class MxAccessGatewayService( // client-side slowness condition with a client-side remedy (reconnect and // re-snapshot), so it maps to ResourceExhausted like a StreamEvents overflow rather // than reaching MapException's warning-with-stack + Unavailable fallback. - throw new RpcException(new Status(StatusCode.ResourceExhausted, overflow.Message)); + throw new RpcException( + new Status(StatusCode.ResourceExhausted, overflow.Message), + ReasonTrailers(ResourceExhaustedReasons.AlarmFeedOverflow)); } catch (Exception exception) when (exception is not RpcException) { @@ -1113,9 +1115,24 @@ public sealed class MxAccessGatewayService( _ => StatusCode.Unavailable, }; - return new RpcException(new Status(statusCode, exception.Message)); + string? reason = exception.ErrorCode switch + { + SessionManagerErrorCode.EventSubscriberAlreadyActive => ResourceExhaustedReasons.SubscriberAlreadyActive, + SessionManagerErrorCode.EventSubscriberLimitReached => ResourceExhaustedReasons.SubscriberLimit, + SessionManagerErrorCode.EventQueueOverflow => ResourceExhaustedReasons.EventQueueOverflow, + SessionManagerErrorCode.SessionLimitExceeded => ResourceExhaustedReasons.SessionLimit, + _ => null, + }; + + return reason is null + ? new RpcException(new Status(statusCode, exception.Message)) + : new RpcException(new Status(statusCode, exception.Message), ReasonTrailers(reason)); } + /// Builds the response trailers carrying a code. + private static Metadata ReasonTrailers(string reason) => + [new Metadata.Entry(ResourceExhaustedReasons.TrailerKey, reason)]; + private static RpcException MapWorkerClientException(WorkerClientException exception) { StatusCode statusCode = exception.ErrorCode switch @@ -1129,6 +1146,8 @@ public sealed class MxAccessGatewayService( _ => StatusCode.Unavailable, }; - return new RpcException(new Status(statusCode, exception.Message)); + return exception.ErrorCode == WorkerClientErrorCode.CommandTooLarge + ? new RpcException(new Status(statusCode, exception.Message), ReasonTrailers(ResourceExhaustedReasons.CommandTooLarge)) + : new RpcException(new Status(statusCode, exception.Message)); } } diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/GatewayEndToEndMultiSubscriberTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/GatewayEndToEndMultiSubscriberTests.cs index 1b5791b8..a616e15f 100644 --- a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/GatewayEndToEndMultiSubscriberTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/GatewayEndToEndMultiSubscriberTests.cs @@ -324,6 +324,11 @@ public sealed class GatewayEndToEndMultiSubscriberTests(ITestOutputHelper output // EventSubscriberLimitReached contains "maximum". Assert.Contains("maximum", capException.Status.Detail, StringComparison.OrdinalIgnoreCase); + // #216: the machine-readable reason distinguishes the cap from an event-queue overflow. + Assert.Equal( + ResourceExhaustedReasons.SubscriberLimit, + capException.Trailers.GetValue(ResourceExhaustedReasons.TrailerKey)); + // The first two streams must still be live (not completed). Assert.False(stream1Task.IsCompleted, "Sub1 must still be streaming after sub3 was rejected."); Assert.False(stream2Task.IsCompleted, "Sub2 must still be streaming after sub3 was rejected."); diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Grpc/MxAccessGatewayServiceTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Grpc/MxAccessGatewayServiceTests.cs index 5cebc7a2..24b708c7 100644 --- a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Grpc/MxAccessGatewayServiceTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Grpc/MxAccessGatewayServiceTests.cs @@ -668,16 +668,46 @@ public sealed class MxAccessGatewayServiceTests new TestServerCallContext())); Assert.Equal(StatusCode.ResourceExhausted, exception.StatusCode); + Assert.Equal(ResourceExhaustedReasons.AlarmFeedOverflow, exception.Trailers.GetValue(ResourceExhaustedReasons.TrailerKey)); Assert.DoesNotContain(logger.Entries, entry => entry.Level >= LogLevel.Warning); } + /// + /// Event-queue overflow and the subscriber cap both stay ResourceExhausted but carry + /// distinct machine-readable reason trailers (#216). + /// + /// The session-manager fault. + /// The reason trailer expected. + /// A task that represents the asynchronous operation. + [Theory] + [InlineData(SessionManagerErrorCode.EventQueueOverflow, ResourceExhaustedReasons.EventQueueOverflow)] + [InlineData(SessionManagerErrorCode.EventSubscriberLimitReached, ResourceExhaustedReasons.SubscriberLimit)] + public async Task StreamEvents_ResourceExhaustedFaults_CarryDistinguishingReasonTrailer( + SessionManagerErrorCode errorCode, + string expectedReason) + { + MxAccessGatewayService service = CreateService( + new FakeSessionManager(), + eventStreamFault: new SessionManagerException(errorCode, "fault")); + + RpcException exception = await Assert.ThrowsAsync( + () => service.StreamEvents( + new StreamEventsRequest { SessionId = "session-1" }, + new RecordingServerStreamWriter(), + new TestServerCallContext())); + + Assert.Equal(StatusCode.ResourceExhausted, exception.StatusCode); + Assert.Equal(expectedReason, exception.Trailers.GetValue(ResourceExhaustedReasons.TrailerKey)); + } + private static MxAccessGatewayService CreateService( FakeSessionManager sessionManager, IGatewayRequestIdentityAccessor? identityAccessor = null, GatewayMetrics? metrics = null, IConstraintEnforcer? constraintEnforcer = null, IGatewayAlarmService? alarmService = null, - ILogger? logger = null) + ILogger? logger = null, + Exception? eventStreamFault = null) { return new MxAccessGatewayService( sessionManager, @@ -685,7 +715,7 @@ public sealed class MxAccessGatewayServiceTests constraintEnforcer ?? new AllowAllConstraintEnforcer(), new MxAccessGrpcRequestValidator(), new MxAccessGrpcMapper(), - new FakeEventStreamService(sessionManager), + new FakeEventStreamService(sessionManager) { StreamFault = eventStreamFault }, metrics ?? new GatewayMetrics(), logger ?? NullLogger.Instance, alarmService ?? new FakeGatewayAlarmService()); @@ -917,6 +947,9 @@ public sealed class MxAccessGatewayServiceTests private sealed class FakeEventStreamService(FakeSessionManager sessionManager) : IEventStreamService { + /// Gets or sets a fault thrown when the stream starts. + public Exception? StreamFault { get; set; } + /// public async IAsyncEnumerable StreamEventsAsync( StreamEventsRequest request, @@ -924,6 +957,12 @@ public sealed class MxAccessGatewayServiceTests [EnumeratorCancellation] CancellationToken cancellationToken) { sessionManager.RecordReadEventsSessionId(request.SessionId); + if (StreamFault is not null) + { + await Task.Yield(); + throw StreamFault; + } + foreach (WorkerEvent workerEvent in sessionManager.Events) { cancellationToken.ThrowIfCancellationRequested();