perf(alarms): memoize CurrentAlarms projection, invalidate on mutation

This commit is contained in:
Joseph Doherty
2026-08-15 12:20:51 -04:00
parent ca34a2d65d
commit f1e26fed4f
2 changed files with 71 additions and 1 deletions
@@ -34,6 +34,13 @@ public sealed class GatewayAlarmMonitor : BackgroundService, IGatewayAlarmServic
private readonly Dictionary<string, ActiveAlarmSnapshot> _alarms = new(StringComparer.Ordinal); private readonly Dictionary<string, ActiveAlarmSnapshot> _alarms = new(StringComparer.Ordinal);
private readonly List<Subscriber> _subscribers = []; private readonly List<Subscriber> _subscribers = [];
// Memoized CurrentAlarms projection, guarded by _sync: the cloned, read-only view of _alarms
// handed to the dashboard and the QueryActiveAlarms RPC. Cloning the whole set per read held
// _sync — the broadcast lock — for the length of the copy, so a polled dashboard stalled every
// ApplyTransition/Broadcast behind it. Null means "not built for the current generation":
// every path that writes _alarms must null this under _sync, or readers keep a stale set.
private ActiveAlarmSnapshot[]? _currentAlarmsProjection;
// NEXT-03 dedup tombstones, guarded by _sync: alarm instances whose Clear was synthesized by // NEXT-03 dedup tombstones, guarded by _sync: alarm instances whose Clear was synthesized by
// the most recent reconcile pass, keyed by reference with the instance's original raise // the most recent reconcile pass, keyed by reference with the instance's original raise
// timestamp as the identity marker. A buffered live Clear for the same instance is a duplicate // timestamp as the identity marker. A buffered live Clear for the same instance is a duplicate
@@ -93,7 +100,12 @@ public sealed class GatewayAlarmMonitor : BackgroundService, IGatewayAlarmServic
{ {
lock (_sync) lock (_sync)
{ {
return _alarms.Values.Select(alarm => alarm.Clone()).ToArray(); // Same clone semantics as an uncached read — callers still get instances no
// mutation can leak back into the cache — but built once per alarm-set
// generation instead of once per caller.
return _currentAlarmsProjection ??= _alarms.Values
.Select(alarm => alarm.Clone())
.ToArray();
} }
} }
} }
@@ -422,6 +434,11 @@ public sealed class GatewayAlarmMonitor : BackgroundService, IGatewayAlarmServic
if (transition.TransitionKind == AlarmTransitionKind.Clear) if (transition.TransitionKind == AlarmTransitionKind.Clear)
{ {
bool wasKnown = _alarms.Remove(reference); bool wasKnown = _alarms.Remove(reference);
if (wasKnown)
{
_currentAlarmsProjection = null;
}
if (!wasKnown && IsDuplicateOfReconcileClear(reference, transition)) if (!wasKnown && IsDuplicateOfReconcileClear(reference, transition))
{ {
return; return;
@@ -433,6 +450,7 @@ public sealed class GatewayAlarmMonitor : BackgroundService, IGatewayAlarmServic
bool duplicate = _alarms.TryGetValue(reference, out ActiveAlarmSnapshot? existing) bool duplicate = _alarms.TryGetValue(reference, out ActiveAlarmSnapshot? existing)
&& IsDuplicateOfCachedState(existing, snapshot); && IsDuplicateOfCachedState(existing, snapshot);
_alarms[reference] = snapshot; _alarms[reference] = snapshot;
_currentAlarmsProjection = null;
if (duplicate) if (duplicate)
{ {
return; return;
@@ -650,6 +668,8 @@ public sealed class GatewayAlarmMonitor : BackgroundService, IGatewayAlarmServic
{ {
_alarms[incoming.Key] = incoming.Value; _alarms[incoming.Key] = incoming.Value;
} }
_currentAlarmsProjection = null;
} }
} }
@@ -696,6 +716,7 @@ public sealed class GatewayAlarmMonitor : BackgroundService, IGatewayAlarmServic
lock (_sync) lock (_sync)
{ {
_alarms.Clear(); _alarms.Clear();
_currentAlarmsProjection = null;
} }
} }
@@ -282,6 +282,55 @@ public sealed class GatewayAlarmMonitorAttachOrderTests
await monitor.StopAsync(CancellationToken.None); await monitor.StopAsync(CancellationToken.None);
} }
/// <summary>
/// <see cref="GatewayAlarmMonitor.CurrentAlarms"/> clones the whole active-alarm set under
/// the broadcast lock, so rebuilding it per read stalls every transition and broadcast
/// behind the copy once the dashboard polls a large alarm set. The projection is memoized
/// for as long as the set is unchanged, and every mutation must invalidate it — a stale
/// projection would hide live transitions from the dashboard and the QueryActiveAlarms RPC.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task CurrentAlarmsProjectionIsMemoizedUntilTheAlarmSetChanges()
{
using GatewayMetrics metrics = new();
await using FakeSessionManager sessions = new();
using GatewayAlarmMonitor monitor = CreateMonitor(sessions, metrics);
using CancellationTokenSource cts = new();
await monitor.StartAsync(cts.Token);
await sessions.WaitForSubscribeStartAsync(WaitTimeout);
// Seed through a reconcile (forced by a provider-mode probe) so the cache holds one
// unacked alarm and no further mutation is in flight.
sessions.SetReconcileSnapshot(Snapshot(AlarmConditionState.Active));
sessions.EmitEvent(ProviderModeProbe(1));
await WaitUntilAsync(
() => monitor.CurrentAlarms.Any(alarm => alarm.AlarmFullReference == AlarmReference
&& alarm.CurrentState == AlarmConditionState.Active),
WaitTimeout);
IReadOnlyList<ActiveAlarmSnapshot> first = monitor.CurrentAlarms;
Assert.Same(first, monitor.CurrentAlarms);
// A live Acknowledge replaces the cached snapshot, so the next read must rebuild.
sessions.EmitEvent(Transition(2, AlarmTransitionKind.Acknowledge));
await WaitUntilAsync(
() => monitor.CurrentAlarms.Any(alarm => alarm.AlarmFullReference == AlarmReference
&& alarm.CurrentState == AlarmConditionState.ActiveAcked),
WaitTimeout);
IReadOnlyList<ActiveAlarmSnapshot> second = monitor.CurrentAlarms;
Assert.NotSame(first, second);
Assert.Same(second, monitor.CurrentAlarms);
// The pre-transition projection is a snapshot of the old generation, not a live view.
Assert.Equal(AlarmConditionState.Active, Assert.Single(first).CurrentState);
await cts.CancelAsync();
await monitor.StopAsync(CancellationToken.None);
}
private static GatewayAlarmMonitor CreateMonitor(FakeSessionManager sessions, GatewayMetrics metrics) private static GatewayAlarmMonitor CreateMonitor(FakeSessionManager sessions, GatewayMetrics metrics)
{ {
AlarmsOptions options = new() AlarmsOptions options = new()