fix(grpc): typed alarm-feed overflow, memoized alarm snapshot, cancellable stream writes, uniform reply correlation id

- StreamAlarms: slow subscribers are completed with
  AlarmFeedSubscriberOverflowException and surface as ResourceExhausted
  (like a StreamEvents overflow) instead of warning-with-stack + Unavailable.
- GatewayAlarmMonitor.StreamAsync reuses the memoized alarm-set projection
  instead of cloning the whole set under the broadcast lock per attach; the
  prefix filter runs outside the lock.
- StreamEvents/StreamAlarms pass the call token to response writes; a write
  failure after client cancel maps to Cancelled without a warning.
- Every MxCommandReply.correlation_id now echoes the request's
  client_correlation_id (previously only constraint-denied replies did;
  normal replies leaked the internal envelope id).
This commit is contained in:
Joseph Doherty
2026-10-22 06:27:36 -04:00
parent 19cc74f21a
commit 0b4a1682c8
11 changed files with 478 additions and 14 deletions
@@ -144,6 +144,12 @@ public sealed class MxAccessGatewayService(
publicReply = bulkConstraintPlan.MergeDeniedInto(publicReply);
}
// MxCommandReply.correlation_id always echoes the caller's client_correlation_id —
// the same value the constraint-denied replies carry — so a client can match every
// reply to its request. The worker reply carries the gateway's internal envelope id,
// which is a gateway↔worker routing detail and is overwritten here, empty included.
publicReply.CorrelationId = request.ClientCorrelationId;
session.TrackCommandReply(commandToInvoke, publicReply);
return publicReply;
}
@@ -172,7 +178,10 @@ public sealed class MxAccessGatewayService(
// volume gateway path. Stopwatch.GetTimestamp/GetElapsedTime measure the exact
// same wall-clock span the former Stopwatch.StartNew()/.Elapsed did.
long sendStartTimestamp = Stopwatch.GetTimestamp();
await responseStream.WriteAsync(publicEvent).ConfigureAwait(false);
// The call token releases a write parked on a slow or vanished client when the
// call is cancelled, instead of leaving it to the transport's own timeout.
await responseStream.WriteAsync(publicEvent, context.CancellationToken).ConfigureAwait(false);
metrics.RecordEventStreamSend(
publicEvent.Family.ToString(),
Stopwatch.GetElapsedTime(sendStartTimestamp));
@@ -180,6 +189,11 @@ public sealed class MxAccessGatewayService(
}
catch (Exception exception) when (exception is not RpcException)
{
if (IsClientAbort(exception, context))
{
throw ClientCancelled();
}
throw MapException(exception);
}
}
@@ -225,11 +239,24 @@ public sealed class MxAccessGatewayService(
.WithCancellation(context.CancellationToken)
.ConfigureAwait(false))
{
await responseStream.WriteAsync(message).ConfigureAwait(false);
await responseStream.WriteAsync(message, context.CancellationToken).ConfigureAwait(false);
}
}
catch (AlarmFeedSubscriberOverflowException overflow)
{
// The monitor dropped this subscriber for falling behind its bounded queue. That is a
// client-side slowness condition with a client-side remedy (reconnect and
// re-snapshot), so it maps to ResourceExhausted like a StreamEvents overflow rather
// than reaching MapException's warning-with-stack + Unavailable fallback.
throw new RpcException(new Status(StatusCode.ResourceExhausted, overflow.Message));
}
catch (Exception exception) when (exception is not RpcException)
{
if (IsClientAbort(exception, context))
{
throw ClientCancelled();
}
throw MapException(exception);
}
}
@@ -264,6 +291,17 @@ public sealed class MxAccessGatewayService(
}
}
// A response write fails with an IOException (or an operation-cancelled exception) when the
// client disconnects mid-stream. With the call token already cancelled that is a routine
// client abort, not a gateway fault: it maps to Cancelled with no warning. The same exception
// on a live call token is a genuine transport failure and keeps the MapException path.
private static bool IsClientAbort(Exception exception, ServerCallContext context) =>
context.CancellationToken.IsCancellationRequested
&& exception is IOException or OperationCanceledException;
private static RpcException ClientCancelled() =>
new(new Status(StatusCode.Cancelled, "gRPC request was canceled."));
private string? ResolveClientIdentity()
{
return identityAccessor.Current?.DisplayName ?? identityAccessor.Current?.KeyId;