feat(events): distinguish ResourceExhausted causes with an mxgw-reason trailer (mxaccessgw#216)

Event-queue overflow, alarm-feed overflow, subscriber cap, session limit and
oversized commands all surfaced as ResourceExhausted. The status code is
unchanged; the gateway now attaches a machine-readable `mxgw-reason` trailer
(constants in Contracts ResourceExhaustedReasons). The .NET client maps
ResourceExhausted to MxGatewayResourceExhaustedException (an
MxGatewayException subclass; additive) exposing Reason/IsOverflow/
IsSubscriberLimit. Documented in docs/ClientBehaviorFixtures.md; the other
clients get a doc note only.

Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
Joseph Doherty
2026-11-09 15:02:24 -05:00
parent 716c845fc7
commit fccc21ba8e
9 changed files with 253 additions and 5 deletions
@@ -149,6 +149,57 @@ public sealed class MxGatewayClientWireTests
() => session.RegisterAsync("wire-test-client"));
}
/// <summary>
/// A <c>RESOURCE_EXHAUSTED</c> status carrying the <c>mxgw-reason</c> trailer surfaces the
/// reason as a typed property, so overflow and limit rejections can be told apart.
/// </summary>
/// <param name="reason">The reason code the fake gateway attaches.</param>
/// <param name="overflow">Whether the client should classify it as an overflow.</param>
/// <param name="limit">Whether the client should classify it as a subscriber-limit rejection.</param>
/// <returns>A task that represents the asynchronous operation.</returns>
[Theory]
[InlineData("event-queue-overflow", true, false)]
[InlineData("subscriber-limit", false, true)]
public async Task ResourceExhaustedStatus_ExposesReasonFromTrailer(string reason, bool overflow, bool limit)
{
await using WireFakeGatewayServer server = await WireFakeGatewayServer.StartAsync(
service =>
{
service.ExhaustInvoke = true;
service.ExhaustedReason = reason;
});
await using MxGatewayClient client = server.CreateClient(ApiKey);
MxGatewaySession session = await client.OpenSessionAsync(
new OpenSessionRequest { ClientSessionName = "wire-test" });
MxGatewayResourceExhaustedException exception =
await Assert.ThrowsAsync<MxGatewayResourceExhaustedException>(
() => session.RegisterAsync("wire-test-client"));
Assert.Equal(reason, exception.Reason);
Assert.Equal(overflow, exception.IsOverflow);
Assert.Equal(limit, exception.IsSubscriberLimit);
Assert.IsAssignableFrom<MxGatewayException>(exception);
}
/// <summary>A <c>RESOURCE_EXHAUSTED</c> from an older gateway (no trailer) has a null reason.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task ResourceExhaustedStatus_WithoutTrailer_HasNullReason()
{
await using WireFakeGatewayServer server = await WireFakeGatewayServer.StartAsync(
service => service.ExhaustInvoke = true);
await using MxGatewayClient client = server.CreateClient(ApiKey);
MxGatewaySession session = await client.OpenSessionAsync(
new OpenSessionRequest { ClientSessionName = "wire-test" });
MxGatewayResourceExhaustedException exception =
await Assert.ThrowsAsync<MxGatewayResourceExhaustedException>(
() => session.RegisterAsync("wire-test-client"));
Assert.Null(exception.Reason);
}
private static async Task<List<MxEvent>> CollectAsync(IAsyncEnumerable<MxEvent> stream)
{
List<MxEvent> events = [];
@@ -140,6 +140,16 @@ internal sealed class WireFakeGatewayServer : IAsyncDisposable
/// </summary>
public bool DenyInvoke { get; set; }
/// <summary>
/// Gets or sets a value indicating whether <c>Invoke</c> fails with
/// <see cref="StatusCode.ResourceExhausted"/>; <see cref="ExhaustedReason"/> (when set) rides the
/// <c>mxgw-reason</c> trailer the way the real gateway attaches it.
/// </summary>
public bool ExhaustInvoke { get; set; }
/// <summary>Gets or sets the reason-code trailer value sent with the exhausted failure.</summary>
public string? ExhaustedReason { get; set; }
/// <summary>
/// Gets or sets the replay-gap sentinel emitted at the head of the event stream.
/// </summary>
@@ -189,6 +199,17 @@ internal sealed class WireFakeGatewayServer : IAsyncDisposable
throw new RpcException(new Status(StatusCode.PermissionDenied, "invoke scope required"));
}
if (ExhaustInvoke)
{
Metadata trailers = [];
if (ExhaustedReason is not null)
{
trailers.Add(ZB.MOM.WW.MxGateway.Contracts.ResourceExhaustedReasons.TrailerKey, ExhaustedReason);
}
throw new RpcException(new Status(StatusCode.ResourceExhausted, "exhausted"), trailers);
}
return Task.FromResult(new MxCommandReply
{
SessionId = request.SessionId,
@@ -255,6 +255,10 @@ internal sealed class GrpcMxGatewayClientTransport(
StatusCode.PermissionDenied => new MxGatewayAuthorizationException(
exception.Status.Detail,
innerException: exception),
StatusCode.ResourceExhausted => new MxGatewayResourceExhaustedException(
exception.Status.Detail,
MxGatewayResourceExhaustedException.ReadReason(exception),
exception),
_ => new MxGatewayException(exception.Status.Detail, exception),
};
}
@@ -0,0 +1,51 @@
using Grpc.Core;
using ZB.MOM.WW.MxGateway.Contracts;
namespace ZB.MOM.WW.MxGateway.Client;
/// <summary>
/// Exception thrown when the gateway answers gRPC <c>ResourceExhausted</c>. The
/// <see cref="Reason"/> distinguishes "you fell behind and events were lost" (re-subscribe and
/// resync) from "a limit was reached" (back off, do not retry immediately). It derives from
/// <see cref="MxGatewayException"/>, so existing catch blocks keep working.
/// </summary>
public sealed class MxGatewayResourceExhaustedException : MxGatewayException
{
/// <summary>Initializes a new instance.</summary>
/// <param name="message">Diagnostic message describing the failure.</param>
/// <param name="reason">The gateway's machine-readable reason code, or null when absent (older gateway).</param>
/// <param name="innerException">The underlying <see cref="RpcException"/>.</param>
public MxGatewayResourceExhaustedException(string message, string? reason, Exception? innerException = null)
: base(message, innerException)
{
Reason = reason;
}
/// <summary>
/// Gets the reason code from the <c>mxgw-reason</c> trailer (see <see cref="ResourceExhaustedReasons"/>),
/// or null when the gateway sent none.
/// </summary>
public string? Reason { get; }
/// <summary>Gets a value indicating whether events were lost because the subscriber fell behind (event-stream or alarm-feed overflow); the caller should re-subscribe and resync.</summary>
public bool IsOverflow =>
Reason is ResourceExhaustedReasons.EventQueueOverflow or ResourceExhaustedReasons.AlarmFeedOverflow;
/// <summary>Gets a value indicating whether the failure was a subscriber-count limit rejection.</summary>
public bool IsSubscriberLimit =>
Reason is ResourceExhaustedReasons.SubscriberLimit or ResourceExhaustedReasons.SubscriberAlreadyActive;
internal static string? ReadReason(RpcException exception)
{
foreach (Metadata.Entry entry in exception.Trailers)
{
if (!entry.IsBinary
&& string.Equals(entry.Key, ResourceExhaustedReasons.TrailerKey, StringComparison.OrdinalIgnoreCase))
{
return entry.Value;
}
}
return null;
}
}