feat(events): distinguish ResourceExhausted causes with an mxgw-reason trailer (mxaccessgw#216)
Event-queue overflow, alarm-feed overflow, subscriber cap, session limit and oversized commands all surfaced as ResourceExhausted. The status code is unchanged; the gateway now attaches a machine-readable `mxgw-reason` trailer (constants in Contracts ResourceExhaustedReasons). The .NET client maps ResourceExhausted to MxGatewayResourceExhaustedException (an MxGatewayException subclass; additive) exposing Reason/IsOverflow/ IsSubscriberLimit. Documented in docs/ClientBehaviorFixtures.md; the other clients get a doc note only. Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
@@ -0,0 +1,30 @@
|
||||
namespace ZB.MOM.WW.MxGateway.Contracts;
|
||||
|
||||
/// <summary>
|
||||
/// Machine-readable reason codes the gateway attaches to gRPC <c>ResourceExhausted</c> failures
|
||||
/// via the <see cref="TrailerKey"/> response trailer, so a client can tell "you fell behind and
|
||||
/// events were lost" from "a limit was reached". The gRPC status code itself is unchanged.
|
||||
/// </summary>
|
||||
public static class ResourceExhaustedReasons
|
||||
{
|
||||
/// <summary>Response-trailer key carrying the reason code (gRPC trailer keys are lower-case ASCII).</summary>
|
||||
public const string TrailerKey = "mxgw-reason";
|
||||
|
||||
/// <summary>The subscriber fell behind its bounded event queue and was disconnected; events were lost. Re-subscribe and resync.</summary>
|
||||
public const string EventQueueOverflow = "event-queue-overflow";
|
||||
|
||||
/// <summary>The alarm-feed subscriber fell behind its bounded queue and was dropped. Reconnect and re-snapshot.</summary>
|
||||
public const string AlarmFeedOverflow = "alarm-feed-overflow";
|
||||
|
||||
/// <summary>The per-session event-subscriber cap was reached. Back off; do not retry immediately.</summary>
|
||||
public const string SubscriberLimit = "subscriber-limit";
|
||||
|
||||
/// <summary>The session already has an active event subscriber and only one is allowed.</summary>
|
||||
public const string SubscriberAlreadyActive = "subscriber-already-active";
|
||||
|
||||
/// <summary>The gateway session limit was reached.</summary>
|
||||
public const string SessionLimit = "session-limit";
|
||||
|
||||
/// <summary>A command exceeded the maximum frame size.</summary>
|
||||
public const string CommandTooLarge = "command-too-large";
|
||||
}
|
||||
@@ -261,7 +261,9 @@ public sealed class MxAccessGatewayService(
|
||||
// client-side slowness condition with a client-side remedy (reconnect and
|
||||
// re-snapshot), so it maps to ResourceExhausted like a StreamEvents overflow rather
|
||||
// than reaching MapException's warning-with-stack + Unavailable fallback.
|
||||
throw new RpcException(new Status(StatusCode.ResourceExhausted, overflow.Message));
|
||||
throw new RpcException(
|
||||
new Status(StatusCode.ResourceExhausted, overflow.Message),
|
||||
ReasonTrailers(ResourceExhaustedReasons.AlarmFeedOverflow));
|
||||
}
|
||||
catch (Exception exception) when (exception is not RpcException)
|
||||
{
|
||||
@@ -1113,9 +1115,24 @@ public sealed class MxAccessGatewayService(
|
||||
_ => StatusCode.Unavailable,
|
||||
};
|
||||
|
||||
return new RpcException(new Status(statusCode, exception.Message));
|
||||
string? reason = exception.ErrorCode switch
|
||||
{
|
||||
SessionManagerErrorCode.EventSubscriberAlreadyActive => ResourceExhaustedReasons.SubscriberAlreadyActive,
|
||||
SessionManagerErrorCode.EventSubscriberLimitReached => ResourceExhaustedReasons.SubscriberLimit,
|
||||
SessionManagerErrorCode.EventQueueOverflow => ResourceExhaustedReasons.EventQueueOverflow,
|
||||
SessionManagerErrorCode.SessionLimitExceeded => ResourceExhaustedReasons.SessionLimit,
|
||||
_ => null,
|
||||
};
|
||||
|
||||
return reason is null
|
||||
? new RpcException(new Status(statusCode, exception.Message))
|
||||
: new RpcException(new Status(statusCode, exception.Message), ReasonTrailers(reason));
|
||||
}
|
||||
|
||||
/// <summary>Builds the response trailers carrying a <see cref="ResourceExhaustedReasons"/> code.</summary>
|
||||
private static Metadata ReasonTrailers(string reason) =>
|
||||
[new Metadata.Entry(ResourceExhaustedReasons.TrailerKey, reason)];
|
||||
|
||||
private static RpcException MapWorkerClientException(WorkerClientException exception)
|
||||
{
|
||||
StatusCode statusCode = exception.ErrorCode switch
|
||||
@@ -1129,6 +1146,8 @@ public sealed class MxAccessGatewayService(
|
||||
_ => StatusCode.Unavailable,
|
||||
};
|
||||
|
||||
return new RpcException(new Status(statusCode, exception.Message));
|
||||
return exception.ErrorCode == WorkerClientErrorCode.CommandTooLarge
|
||||
? new RpcException(new Status(statusCode, exception.Message), ReasonTrailers(ResourceExhaustedReasons.CommandTooLarge))
|
||||
: new RpcException(new Status(statusCode, exception.Message));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -324,6 +324,11 @@ public sealed class GatewayEndToEndMultiSubscriberTests(ITestOutputHelper output
|
||||
// EventSubscriberLimitReached contains "maximum".
|
||||
Assert.Contains("maximum", capException.Status.Detail, StringComparison.OrdinalIgnoreCase);
|
||||
|
||||
// #216: the machine-readable reason distinguishes the cap from an event-queue overflow.
|
||||
Assert.Equal(
|
||||
ResourceExhaustedReasons.SubscriberLimit,
|
||||
capException.Trailers.GetValue(ResourceExhaustedReasons.TrailerKey));
|
||||
|
||||
// The first two streams must still be live (not completed).
|
||||
Assert.False(stream1Task.IsCompleted, "Sub1 must still be streaming after sub3 was rejected.");
|
||||
Assert.False(stream2Task.IsCompleted, "Sub2 must still be streaming after sub3 was rejected.");
|
||||
|
||||
@@ -668,16 +668,46 @@ public sealed class MxAccessGatewayServiceTests
|
||||
new TestServerCallContext()));
|
||||
|
||||
Assert.Equal(StatusCode.ResourceExhausted, exception.StatusCode);
|
||||
Assert.Equal(ResourceExhaustedReasons.AlarmFeedOverflow, exception.Trailers.GetValue(ResourceExhaustedReasons.TrailerKey));
|
||||
Assert.DoesNotContain(logger.Entries, entry => entry.Level >= LogLevel.Warning);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Event-queue overflow and the subscriber cap both stay <c>ResourceExhausted</c> but carry
|
||||
/// distinct machine-readable reason trailers (#216).
|
||||
/// </summary>
|
||||
/// <param name="errorCode">The session-manager fault.</param>
|
||||
/// <param name="expectedReason">The reason trailer expected.</param>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Theory]
|
||||
[InlineData(SessionManagerErrorCode.EventQueueOverflow, ResourceExhaustedReasons.EventQueueOverflow)]
|
||||
[InlineData(SessionManagerErrorCode.EventSubscriberLimitReached, ResourceExhaustedReasons.SubscriberLimit)]
|
||||
public async Task StreamEvents_ResourceExhaustedFaults_CarryDistinguishingReasonTrailer(
|
||||
SessionManagerErrorCode errorCode,
|
||||
string expectedReason)
|
||||
{
|
||||
MxAccessGatewayService service = CreateService(
|
||||
new FakeSessionManager(),
|
||||
eventStreamFault: new SessionManagerException(errorCode, "fault"));
|
||||
|
||||
RpcException exception = await Assert.ThrowsAsync<RpcException>(
|
||||
() => service.StreamEvents(
|
||||
new StreamEventsRequest { SessionId = "session-1" },
|
||||
new RecordingServerStreamWriter<MxEvent>(),
|
||||
new TestServerCallContext()));
|
||||
|
||||
Assert.Equal(StatusCode.ResourceExhausted, exception.StatusCode);
|
||||
Assert.Equal(expectedReason, exception.Trailers.GetValue(ResourceExhaustedReasons.TrailerKey));
|
||||
}
|
||||
|
||||
private static MxAccessGatewayService CreateService(
|
||||
FakeSessionManager sessionManager,
|
||||
IGatewayRequestIdentityAccessor? identityAccessor = null,
|
||||
GatewayMetrics? metrics = null,
|
||||
IConstraintEnforcer? constraintEnforcer = null,
|
||||
IGatewayAlarmService? alarmService = null,
|
||||
ILogger<MxAccessGatewayService>? logger = null)
|
||||
ILogger<MxAccessGatewayService>? logger = null,
|
||||
Exception? eventStreamFault = null)
|
||||
{
|
||||
return new MxAccessGatewayService(
|
||||
sessionManager,
|
||||
@@ -685,7 +715,7 @@ public sealed class MxAccessGatewayServiceTests
|
||||
constraintEnforcer ?? new AllowAllConstraintEnforcer(),
|
||||
new MxAccessGrpcRequestValidator(),
|
||||
new MxAccessGrpcMapper(),
|
||||
new FakeEventStreamService(sessionManager),
|
||||
new FakeEventStreamService(sessionManager) { StreamFault = eventStreamFault },
|
||||
metrics ?? new GatewayMetrics(),
|
||||
logger ?? NullLogger<MxAccessGatewayService>.Instance,
|
||||
alarmService ?? new FakeGatewayAlarmService());
|
||||
@@ -917,6 +947,9 @@ public sealed class MxAccessGatewayServiceTests
|
||||
|
||||
private sealed class FakeEventStreamService(FakeSessionManager sessionManager) : IEventStreamService
|
||||
{
|
||||
/// <summary>Gets or sets a fault thrown when the stream starts.</summary>
|
||||
public Exception? StreamFault { get; set; }
|
||||
|
||||
/// <inheritdoc />
|
||||
public async IAsyncEnumerable<MxEvent> StreamEventsAsync(
|
||||
StreamEventsRequest request,
|
||||
@@ -924,6 +957,12 @@ public sealed class MxAccessGatewayServiceTests
|
||||
[EnumeratorCancellation] CancellationToken cancellationToken)
|
||||
{
|
||||
sessionManager.RecordReadEventsSessionId(request.SessionId);
|
||||
if (StreamFault is not null)
|
||||
{
|
||||
await Task.Yield();
|
||||
throw StreamFault;
|
||||
}
|
||||
|
||||
foreach (WorkerEvent workerEvent in sessionManager.Events)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
Reference in New Issue
Block a user