fix(grpc): typed alarm-feed overflow, memoized alarm snapshot, cancellable stream writes, uniform reply correlation id

- StreamAlarms: slow subscribers are completed with
  AlarmFeedSubscriberOverflowException and surface as ResourceExhausted
  (like a StreamEvents overflow) instead of warning-with-stack + Unavailable.
- GatewayAlarmMonitor.StreamAsync reuses the memoized alarm-set projection
  instead of cloning the whole set under the broadcast lock per attach; the
  prefix filter runs outside the lock.
- StreamEvents/StreamAlarms pass the call token to response writes; a write
  failure after client cancel maps to Cancelled without a warning.
- Every MxCommandReply.correlation_id now echoes the request's
  client_correlation_id (previously only constraint-denied replies did;
  normal replies leaked the internal envelope id).
This commit is contained in:
Joseph Doherty
2026-10-22 06:27:36 -04:00
parent 19cc74f21a
commit 0b4a1682c8
11 changed files with 478 additions and 14 deletions
@@ -967,7 +967,9 @@ public sealed class GatewayAlarmMonitor : BackgroundService, IGatewayAlarmServic
}
// Caller holds _sync. Pushes a feed message to every matching subscriber;
// a subscriber that has fallen behind is completed with an error and dropped.
// a subscriber that has fallen behind is completed with an AlarmFeedSubscriberOverflowException
// and dropped. The typed fault is what lets StreamAlarms answer ResourceExhausted instead of
// treating a slow client as a gateway failure.
private void Broadcast(AlarmFeedMessage message, string reference)
{
for (int index = _subscribers.Count - 1; index >= 0; index--)
@@ -980,8 +982,7 @@ public sealed class GatewayAlarmMonitor : BackgroundService, IGatewayAlarmServic
if (!subscriber.Channel.Writer.TryWrite(message))
{
subscriber.Channel.Writer.TryComplete(new InvalidOperationException(
"Alarm feed subscriber fell behind and was dropped; reconnect to re-snapshot."));
subscriber.Channel.Writer.TryComplete(new AlarmFeedSubscriberOverflowException());
_subscribers.RemoveAt(index);
}
}
@@ -997,8 +998,7 @@ public sealed class GatewayAlarmMonitor : BackgroundService, IGatewayAlarmServic
Subscriber subscriber = _subscribers[index];
if (!subscriber.Channel.Writer.TryWrite(message))
{
subscriber.Channel.Writer.TryComplete(new InvalidOperationException(
"Alarm feed subscriber fell behind and was dropped; reconnect to re-snapshot."));
subscriber.Channel.Writer.TryComplete(new AlarmFeedSubscriberOverflowException());
_subscribers.RemoveAt(index);
}
}
@@ -1046,9 +1046,14 @@ public sealed class GatewayAlarmMonitor : BackgroundService, IGatewayAlarmServic
_subscribers.Add(subscriber);
providerStatus = BuildProviderStatus();
snapshotTruncated = _snapshotTruncated;
snapshot = _alarms.Values
.Where(alarm => prefix.Length == 0
|| alarm.AlarmFullReference.StartsWith(prefix, StringComparison.Ordinal))
// Take the memoized projection rather than cloning the whole set per attach: the
// clone runs at most once per alarm-set generation (shared with CurrentAlarms), so a
// burst of reconnecting subscribers no longer holds the broadcast lock for one full
// copy each. The array is never mutated once built — a write to _alarms replaces it
// with null — so capturing the reference here pins exactly the generation this
// subscriber registered against, which is what keeps snapshot-then-live ordering.
snapshot = _currentAlarmsProjection ??= _alarms.Values
.Select(alarm => alarm.Clone())
.ToArray();
}
@@ -1068,8 +1073,15 @@ public sealed class GatewayAlarmMonitor : BackgroundService, IGatewayAlarmServic
SnapshotStatus = new AlarmSnapshotStatus { Truncated = snapshotTruncated },
};
// The prefix filter runs here, outside _sync, over the immutable projection.
foreach (ActiveAlarmSnapshot alarm in snapshot)
{
if (prefix.Length != 0
&& !alarm.AlarmFullReference.StartsWith(prefix, StringComparison.Ordinal))
{
continue;
}
yield return new AlarmFeedMessage { ActiveAlarm = alarm };
}