perf(dashboard): idle-gate the snapshot tick; cache static config; bound key-list refresh

The snapshot publisher broadcast to Clients.All on every ~1s tick forever, with
zero viewers. Each tick cost a session-registry snapshot and sort, a metrics
snapshot that copies dictionaries under the global metrics lock, a full rebuild
of EffectiveGatewayConfiguration, and a SQLite read of the API key table.

DashboardSnapshotHub now counts live connections into the singleton
DashboardSnapshotHubConnectionCounter (clamped at zero, since SignalR can call
OnDisconnectedAsync for a connection whose OnConnectedAsync faulted). The
publisher drives the snapshot enumerator by hand instead of await foreach: with
no connections it does not call MoveNextAsync at all, so the producing iterator
stays suspended at its yield and no snapshot is built — the gate removes the
build, not just the broadcast. It re-checks once a second, so the first viewer
resumes the tick within about one interval; that viewer is seeded immediately by
DashboardPageBase's synchronous GetSnapshot() and by the hub's OnConnectedAsync.

Two per-tick costs are bounded independently of the gate: the effective
configuration is startup-static (options are bound once at boot and never
reloaded), so it is built once and cached; and the API key summaries refresh at
most every 15s, since the list only changes when an operator creates, rotates,
or revokes a key. Only a successful refresh restarts the interval, so a failed
or timed-out read is still retried on the next tick with the previous summaries
left on screen.
This commit is contained in:
Joseph Doherty
2026-08-15 12:21:49 -04:00
parent e04b1c9199
commit 77c5731b7b
7 changed files with 338 additions and 30 deletions
@@ -51,6 +51,7 @@ public static class DashboardServiceCollectionExtensions
// subscriber bookkeeping they share with the broadcaster must outlive them.
services.AddSingleton<Hubs.EventsHubViewerRegistry>();
services.AddSingleton<Hubs.IDashboardEventBroadcaster, Hubs.DashboardEventBroadcaster>();
services.AddSingleton<Hubs.DashboardSnapshotHubConnectionCounter>();
services.AddHostedService<Hubs.DashboardSnapshotPublisher>();
services.AddHostedService<Hubs.AlarmsHubPublisher>();
services.AddHttpContextAccessor();
@@ -16,6 +16,17 @@ public sealed class DashboardSnapshotService : IDashboardSnapshotService
{
private const string HealthyStatus = "Healthy";
/// <summary>
/// Minimum spacing between API key list reads. The list is a SQLite query whose
/// content only changes when an operator creates, rotates, or revokes a key, so
/// refreshing it on every ~1s snapshot tick buys nothing; the dashboard still sees
/// a key change within this interval.
/// </summary>
private static readonly TimeSpan ApiKeySummaryRefreshInterval = TimeSpan.FromSeconds(15);
/// <summary>Sentinel for "the API key summaries have never been refreshed".</summary>
private const long NeverRefreshedTicks = long.MinValue;
private readonly ISessionRegistry _sessionRegistry;
private readonly GatewayMetrics _metrics;
private readonly IGatewayConfigurationProvider _configurationProvider;
@@ -30,6 +41,13 @@ public sealed class DashboardSnapshotService : IDashboardSnapshotService
private readonly ILogger<DashboardSnapshotService> _logger;
private readonly SemaphoreSlim _apiKeySummaryRefreshGate = new(1, 1);
private IReadOnlyList<DashboardApiKeySummary> _apiKeySummaries = Array.Empty<DashboardApiKeySummary>();
private long _apiKeySummariesRefreshedAtTicks = NeverRefreshedTicks;
// The effective configuration is built from IOptions<GatewayOptions> and is startup-static:
// the gateway binds options once at boot and never reloads them, so this projection cannot
// change for the process lifetime. Build it once instead of re-projecting the whole option
// tree on every snapshot tick. A racing first build is harmless — the projection is pure,
// so either winner stores equivalent content.
private EffectiveGatewayConfiguration? _effectiveConfiguration;
// Memoizes ONLY the O(N) template/category breakdown against the cache sequence. The shared
// library bumps Sequence only on a heavy refresh that replaces the object set, so an unchanged
// sequence means the breakdown is unchanged and can be reused — keeping the ~1s snapshot tick
@@ -100,10 +118,23 @@ public sealed class DashboardSnapshotService : IDashboardSnapshotService
Metrics: CreateMetricSummaries(metricsSnapshot),
Faults: CreateFaultSummaries(sessions, generatedAt),
ApiKeys: Volatile.Read(ref _apiKeySummaries),
Configuration: _configurationProvider.GetEffectiveConfiguration(),
Configuration: ResolveEffectiveConfiguration(),
Galaxy: ResolveGalaxySummary());
}
private EffectiveGatewayConfiguration ResolveEffectiveConfiguration()
{
EffectiveGatewayConfiguration? cached = Volatile.Read(ref _effectiveConfiguration);
if (cached is not null)
{
return cached;
}
EffectiveGatewayConfiguration configuration = _configurationProvider.GetEffectiveConfiguration();
Volatile.Write(ref _effectiveConfiguration, configuration);
return configuration;
}
private DashboardGalaxySummary ResolveGalaxySummary()
{
GalaxyHierarchyCacheEntry entry = _galaxyHierarchyCache.Current;
@@ -255,6 +286,17 @@ public sealed class DashboardSnapshotService : IDashboardSnapshotService
private async Task RefreshApiKeySummariesAsync(CancellationToken cancellationToken)
{
DateTimeOffset now = _timeProvider.GetUtcNow();
long lastRefreshedAtTicks = Interlocked.Read(ref _apiKeySummariesRefreshedAtTicks);
if (lastRefreshedAtTicks != NeverRefreshedTicks
&& now.UtcTicks - lastRefreshedAtTicks < ApiKeySummaryRefreshInterval.Ticks)
{
// Inside the refresh window: reuse the cached summaries rather than
// re-reading the API key table on this tick. Only a *successful* refresh
// moves the timestamp, so a failed read is retried on the next tick.
return;
}
if (!await _apiKeySummaryRefreshGate.WaitAsync(0, cancellationToken).ConfigureAwait(false))
{
return;
@@ -278,6 +320,7 @@ public sealed class DashboardSnapshotService : IDashboardSnapshotService
.ToArray();
Volatile.Write(ref _apiKeySummaries, summaries);
Interlocked.Exchange(ref _apiKeySummariesRefreshedAtTicks, now.UtcTicks);
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
@@ -9,8 +9,15 @@ namespace ZB.MOM.WW.MxGateway.Server.Dashboard.Hubs;
/// immediately via <see cref="OnConnectedAsync"/>; subsequent refreshes are
/// broadcast by <see cref="DashboardSnapshotPublisher"/>.
/// </summary>
/// <remarks>
/// Connections are counted into <see cref="DashboardSnapshotHubConnectionCounter"/>
/// so <see cref="DashboardSnapshotPublisher"/> can stop building and broadcasting
/// snapshots while nobody is watching.
/// </remarks>
[Authorize(Policy = DashboardAuthenticationDefaults.HubClientsPolicy)]
public sealed class DashboardSnapshotHub(IDashboardSnapshotService snapshotService) : Hub
public sealed class DashboardSnapshotHub(
IDashboardSnapshotService snapshotService,
DashboardSnapshotHubConnectionCounter connectionCounter) : Hub
{
/// <summary>Method name used to push snapshot updates to clients.</summary>
public const string SnapshotMessage = "SnapshotUpdated";
@@ -18,7 +25,57 @@ public sealed class DashboardSnapshotHub(IDashboardSnapshotService snapshotServi
/// <inheritdoc />
public override async Task OnConnectedAsync()
{
// Count the viewer before seeding it, so the publisher resumes its tick
// no later than the first snapshot this connection renders.
connectionCounter.Increment();
await Clients.Caller.SendAsync(SnapshotMessage, snapshotService.GetSnapshot()).ConfigureAwait(false);
await base.OnConnectedAsync().ConfigureAwait(false);
}
/// <inheritdoc />
public override async Task OnDisconnectedAsync(Exception? exception)
{
connectionCounter.Decrement();
await base.OnDisconnectedAsync(exception).ConfigureAwait(false);
}
}
/// <summary>
/// Process-wide count of live <see cref="DashboardSnapshotHub"/> connections.
/// Registered as a singleton and read by <see cref="DashboardSnapshotPublisher"/>
/// to idle-gate the snapshot tick: with no dashboard connected there is nothing
/// to broadcast to, so no snapshot is built.
/// </summary>
public sealed class DashboardSnapshotHubConnectionCounter
{
private int _count;
/// <summary>Gets the number of live snapshot hub connections.</summary>
public int Count => Volatile.Read(ref _count);
/// <summary>Records a new snapshot hub connection.</summary>
/// <returns>The connection count after the increment.</returns>
public int Increment()
{
return Interlocked.Increment(ref _count);
}
/// <summary>
/// Records a snapshot hub disconnection. The count is clamped at zero: SignalR
/// can invoke <c>OnDisconnectedAsync</c> for a connection whose
/// <c>OnConnectedAsync</c> faulted, and a negative count would idle-gate the
/// publisher while viewers are still attached.
/// </summary>
/// <returns>The connection count after the decrement.</returns>
public int Decrement()
{
int updated = Interlocked.Decrement(ref _count);
if (updated >= 0)
{
return updated;
}
Interlocked.CompareExchange(ref _count, 0, updated);
return 0;
}
}
@@ -9,6 +9,7 @@ namespace ZB.MOM.WW.MxGateway.Server.Dashboard.Hubs;
/// gateway process; clients listen via the hub.
/// </summary>
/// <remarks>
/// <para>
/// <see cref="ExecuteAsync"/> wraps the snapshot subscription in
/// a reconnect loop with a configurable retry delay (5s by default,
/// mirroring <see cref="AlarmsHubPublisher"/>). A transient failure inside
@@ -16,44 +17,67 @@ namespace ZB.MOM.WW.MxGateway.Server.Dashboard.Hubs;
/// one-time logger-init failure or a transient SQL error from the Galaxy
/// summary projection — would otherwise end the BackgroundService with no
/// reconnect, taking the dashboard offline until process restart.
/// </para>
/// <para>
/// The loop is idle-gated on <see cref="DashboardSnapshotHubConnectionCounter"/>.
/// Each snapshot costs a session-registry snapshot and sort, a metrics snapshot
/// that copies dictionaries under the global metrics lock, and (periodically) a
/// SQLite read of the API key table — work with no consumer when no dashboard is
/// connected. While the count is zero the publisher does not advance the snapshot
/// enumerator at all, so the producing iterator stays suspended and builds nothing.
/// </para>
/// </remarks>
public sealed class DashboardSnapshotPublisher : BackgroundService
{
private static readonly TimeSpan DefaultReconnectDelay = TimeSpan.FromSeconds(5);
private static readonly TimeSpan DefaultIdlePollInterval = TimeSpan.FromSeconds(1);
private readonly IDashboardSnapshotService _snapshotService;
private readonly IHubContext<DashboardSnapshotHub> _hubContext;
private readonly DashboardSnapshotHubConnectionCounter _connectionCounter;
private readonly ILogger<DashboardSnapshotPublisher> _logger;
private readonly TimeSpan _reconnectDelay;
private readonly TimeSpan _idlePollInterval;
/// <summary>Initializes a new instance of the DashboardSnapshotPublisher class.</summary>
/// <param name="snapshotService">The snapshot service to subscribe to.</param>
/// <param name="hubContext">The SignalR hub context for broadcasting.</param>
/// <param name="connectionCounter">Live snapshot hub connection count used to idle-gate the tick.</param>
/// <param name="logger">The logger instance.</param>
public DashboardSnapshotPublisher(
IDashboardSnapshotService snapshotService,
IHubContext<DashboardSnapshotHub> hubContext,
DashboardSnapshotHubConnectionCounter connectionCounter,
ILogger<DashboardSnapshotPublisher> logger)
: this(snapshotService, hubContext, logger, DefaultReconnectDelay)
: this(snapshotService, hubContext, connectionCounter, logger, DefaultReconnectDelay, DefaultIdlePollInterval)
{
}
/// <summary>Initializes a new instance of the DashboardSnapshotPublisher class with custom reconnect delay.</summary>
/// <remarks>Internal hook for testing: tests inject a very short reconnect delay so assertions don't wait full 5s.</remarks>
/// <summary>Initializes a new instance of the DashboardSnapshotPublisher class with custom cadences.</summary>
/// <remarks>
/// Internal hook for testing: tests inject a very short reconnect delay so assertions
/// don't wait the full 5s, and a short idle poll so the resume-from-idle path is fast.
/// </remarks>
/// <param name="snapshotService">The snapshot service to subscribe to.</param>
/// <param name="hubContext">The SignalR hub context for broadcasting.</param>
/// <param name="connectionCounter">Live snapshot hub connection count used to idle-gate the tick.</param>
/// <param name="logger">The logger instance.</param>
/// <param name="reconnectDelay">The delay before reconnecting after a subscription failure.</param>
/// <param name="idlePollInterval">How often the idle publisher re-checks for a connected viewer.</param>
internal DashboardSnapshotPublisher(
IDashboardSnapshotService snapshotService,
IHubContext<DashboardSnapshotHub> hubContext,
DashboardSnapshotHubConnectionCounter connectionCounter,
ILogger<DashboardSnapshotPublisher> logger,
TimeSpan reconnectDelay)
TimeSpan reconnectDelay,
TimeSpan idlePollInterval)
{
_snapshotService = snapshotService;
_hubContext = hubContext;
_connectionCounter = connectionCounter;
_logger = logger;
_reconnectDelay = reconnectDelay;
_idlePollInterval = idlePollInterval;
}
/// <inheritdoc />
@@ -66,15 +90,31 @@ public sealed class DashboardSnapshotPublisher : BackgroundService
{
try
{
await foreach (DashboardSnapshot snapshot in _snapshotService
// Enumerated by hand rather than with await foreach: the snapshot is
// built inside the producer's MoveNextAsync, so not calling MoveNextAsync
// is what makes the idle gate skip the build and not just the broadcast.
await using IAsyncEnumerator<DashboardSnapshot> snapshots = _snapshotService
.WatchSnapshotsAsync(stoppingToken)
.ConfigureAwait(false))
.GetAsyncEnumerator(stoppingToken);
while (!stoppingToken.IsCancellationRequested)
{
if (stoppingToken.IsCancellationRequested)
if (_connectionCounter.Count == 0)
{
// Nobody is watching: leave the producer suspended and re-check
// shortly. The first viewer to connect resumes the tick, and is
// seeded directly by the hub's OnConnectedAsync meanwhile.
await Task.Delay(_idlePollInterval, stoppingToken).ConfigureAwait(false);
continue;
}
if (!await snapshots.MoveNextAsync().ConfigureAwait(false))
{
break;
}
DashboardSnapshot snapshot = snapshots.Current;
try
{
await _hubContext.Clients