perf(dashboard): idle-gate the snapshot tick; cache static config; bound key-list refresh
The snapshot publisher broadcast to Clients.All on every ~1s tick forever, with zero viewers. Each tick cost a session-registry snapshot and sort, a metrics snapshot that copies dictionaries under the global metrics lock, a full rebuild of EffectiveGatewayConfiguration, and a SQLite read of the API key table. DashboardSnapshotHub now counts live connections into the singleton DashboardSnapshotHubConnectionCounter (clamped at zero, since SignalR can call OnDisconnectedAsync for a connection whose OnConnectedAsync faulted). The publisher drives the snapshot enumerator by hand instead of await foreach: with no connections it does not call MoveNextAsync at all, so the producing iterator stays suspended at its yield and no snapshot is built — the gate removes the build, not just the broadcast. It re-checks once a second, so the first viewer resumes the tick within about one interval; that viewer is seeded immediately by DashboardPageBase's synchronous GetSnapshot() and by the hub's OnConnectedAsync. Two per-tick costs are bounded independently of the gate: the effective configuration is startup-static (options are bound once at boot and never reloaded), so it is built once and cached; and the API key summaries refresh at most every 15s, since the list only changes when an operator creates, rotates, or revokes a key. Only a successful refresh restarts the interval, so a failed or timed-out read is still retried on the next tick with the previous summaries left on screen.
This commit is contained in:
@@ -51,6 +51,7 @@ public static class DashboardServiceCollectionExtensions
|
||||
// subscriber bookkeeping they share with the broadcaster must outlive them.
|
||||
services.AddSingleton<Hubs.EventsHubViewerRegistry>();
|
||||
services.AddSingleton<Hubs.IDashboardEventBroadcaster, Hubs.DashboardEventBroadcaster>();
|
||||
services.AddSingleton<Hubs.DashboardSnapshotHubConnectionCounter>();
|
||||
services.AddHostedService<Hubs.DashboardSnapshotPublisher>();
|
||||
services.AddHostedService<Hubs.AlarmsHubPublisher>();
|
||||
services.AddHttpContextAccessor();
|
||||
|
||||
@@ -16,6 +16,17 @@ public sealed class DashboardSnapshotService : IDashboardSnapshotService
|
||||
{
|
||||
private const string HealthyStatus = "Healthy";
|
||||
|
||||
/// <summary>
|
||||
/// Minimum spacing between API key list reads. The list is a SQLite query whose
|
||||
/// content only changes when an operator creates, rotates, or revokes a key, so
|
||||
/// refreshing it on every ~1s snapshot tick buys nothing; the dashboard still sees
|
||||
/// a key change within this interval.
|
||||
/// </summary>
|
||||
private static readonly TimeSpan ApiKeySummaryRefreshInterval = TimeSpan.FromSeconds(15);
|
||||
|
||||
/// <summary>Sentinel for "the API key summaries have never been refreshed".</summary>
|
||||
private const long NeverRefreshedTicks = long.MinValue;
|
||||
|
||||
private readonly ISessionRegistry _sessionRegistry;
|
||||
private readonly GatewayMetrics _metrics;
|
||||
private readonly IGatewayConfigurationProvider _configurationProvider;
|
||||
@@ -30,6 +41,13 @@ public sealed class DashboardSnapshotService : IDashboardSnapshotService
|
||||
private readonly ILogger<DashboardSnapshotService> _logger;
|
||||
private readonly SemaphoreSlim _apiKeySummaryRefreshGate = new(1, 1);
|
||||
private IReadOnlyList<DashboardApiKeySummary> _apiKeySummaries = Array.Empty<DashboardApiKeySummary>();
|
||||
private long _apiKeySummariesRefreshedAtTicks = NeverRefreshedTicks;
|
||||
// The effective configuration is built from IOptions<GatewayOptions> and is startup-static:
|
||||
// the gateway binds options once at boot and never reloads them, so this projection cannot
|
||||
// change for the process lifetime. Build it once instead of re-projecting the whole option
|
||||
// tree on every snapshot tick. A racing first build is harmless — the projection is pure,
|
||||
// so either winner stores equivalent content.
|
||||
private EffectiveGatewayConfiguration? _effectiveConfiguration;
|
||||
// Memoizes ONLY the O(N) template/category breakdown against the cache sequence. The shared
|
||||
// library bumps Sequence only on a heavy refresh that replaces the object set, so an unchanged
|
||||
// sequence means the breakdown is unchanged and can be reused — keeping the ~1s snapshot tick
|
||||
@@ -100,10 +118,23 @@ public sealed class DashboardSnapshotService : IDashboardSnapshotService
|
||||
Metrics: CreateMetricSummaries(metricsSnapshot),
|
||||
Faults: CreateFaultSummaries(sessions, generatedAt),
|
||||
ApiKeys: Volatile.Read(ref _apiKeySummaries),
|
||||
Configuration: _configurationProvider.GetEffectiveConfiguration(),
|
||||
Configuration: ResolveEffectiveConfiguration(),
|
||||
Galaxy: ResolveGalaxySummary());
|
||||
}
|
||||
|
||||
private EffectiveGatewayConfiguration ResolveEffectiveConfiguration()
|
||||
{
|
||||
EffectiveGatewayConfiguration? cached = Volatile.Read(ref _effectiveConfiguration);
|
||||
if (cached is not null)
|
||||
{
|
||||
return cached;
|
||||
}
|
||||
|
||||
EffectiveGatewayConfiguration configuration = _configurationProvider.GetEffectiveConfiguration();
|
||||
Volatile.Write(ref _effectiveConfiguration, configuration);
|
||||
return configuration;
|
||||
}
|
||||
|
||||
private DashboardGalaxySummary ResolveGalaxySummary()
|
||||
{
|
||||
GalaxyHierarchyCacheEntry entry = _galaxyHierarchyCache.Current;
|
||||
@@ -255,6 +286,17 @@ public sealed class DashboardSnapshotService : IDashboardSnapshotService
|
||||
|
||||
private async Task RefreshApiKeySummariesAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
DateTimeOffset now = _timeProvider.GetUtcNow();
|
||||
long lastRefreshedAtTicks = Interlocked.Read(ref _apiKeySummariesRefreshedAtTicks);
|
||||
if (lastRefreshedAtTicks != NeverRefreshedTicks
|
||||
&& now.UtcTicks - lastRefreshedAtTicks < ApiKeySummaryRefreshInterval.Ticks)
|
||||
{
|
||||
// Inside the refresh window: reuse the cached summaries rather than
|
||||
// re-reading the API key table on this tick. Only a *successful* refresh
|
||||
// moves the timestamp, so a failed read is retried on the next tick.
|
||||
return;
|
||||
}
|
||||
|
||||
if (!await _apiKeySummaryRefreshGate.WaitAsync(0, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
return;
|
||||
@@ -278,6 +320,7 @@ public sealed class DashboardSnapshotService : IDashboardSnapshotService
|
||||
.ToArray();
|
||||
|
||||
Volatile.Write(ref _apiKeySummaries, summaries);
|
||||
Interlocked.Exchange(ref _apiKeySummariesRefreshedAtTicks, now.UtcTicks);
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
|
||||
@@ -9,8 +9,15 @@ namespace ZB.MOM.WW.MxGateway.Server.Dashboard.Hubs;
|
||||
/// immediately via <see cref="OnConnectedAsync"/>; subsequent refreshes are
|
||||
/// broadcast by <see cref="DashboardSnapshotPublisher"/>.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Connections are counted into <see cref="DashboardSnapshotHubConnectionCounter"/>
|
||||
/// so <see cref="DashboardSnapshotPublisher"/> can stop building and broadcasting
|
||||
/// snapshots while nobody is watching.
|
||||
/// </remarks>
|
||||
[Authorize(Policy = DashboardAuthenticationDefaults.HubClientsPolicy)]
|
||||
public sealed class DashboardSnapshotHub(IDashboardSnapshotService snapshotService) : Hub
|
||||
public sealed class DashboardSnapshotHub(
|
||||
IDashboardSnapshotService snapshotService,
|
||||
DashboardSnapshotHubConnectionCounter connectionCounter) : Hub
|
||||
{
|
||||
/// <summary>Method name used to push snapshot updates to clients.</summary>
|
||||
public const string SnapshotMessage = "SnapshotUpdated";
|
||||
@@ -18,7 +25,57 @@ public sealed class DashboardSnapshotHub(IDashboardSnapshotService snapshotServi
|
||||
/// <inheritdoc />
|
||||
public override async Task OnConnectedAsync()
|
||||
{
|
||||
// Count the viewer before seeding it, so the publisher resumes its tick
|
||||
// no later than the first snapshot this connection renders.
|
||||
connectionCounter.Increment();
|
||||
await Clients.Caller.SendAsync(SnapshotMessage, snapshotService.GetSnapshot()).ConfigureAwait(false);
|
||||
await base.OnConnectedAsync().ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public override async Task OnDisconnectedAsync(Exception? exception)
|
||||
{
|
||||
connectionCounter.Decrement();
|
||||
await base.OnDisconnectedAsync(exception).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Process-wide count of live <see cref="DashboardSnapshotHub"/> connections.
|
||||
/// Registered as a singleton and read by <see cref="DashboardSnapshotPublisher"/>
|
||||
/// to idle-gate the snapshot tick: with no dashboard connected there is nothing
|
||||
/// to broadcast to, so no snapshot is built.
|
||||
/// </summary>
|
||||
public sealed class DashboardSnapshotHubConnectionCounter
|
||||
{
|
||||
private int _count;
|
||||
|
||||
/// <summary>Gets the number of live snapshot hub connections.</summary>
|
||||
public int Count => Volatile.Read(ref _count);
|
||||
|
||||
/// <summary>Records a new snapshot hub connection.</summary>
|
||||
/// <returns>The connection count after the increment.</returns>
|
||||
public int Increment()
|
||||
{
|
||||
return Interlocked.Increment(ref _count);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Records a snapshot hub disconnection. The count is clamped at zero: SignalR
|
||||
/// can invoke <c>OnDisconnectedAsync</c> for a connection whose
|
||||
/// <c>OnConnectedAsync</c> faulted, and a negative count would idle-gate the
|
||||
/// publisher while viewers are still attached.
|
||||
/// </summary>
|
||||
/// <returns>The connection count after the decrement.</returns>
|
||||
public int Decrement()
|
||||
{
|
||||
int updated = Interlocked.Decrement(ref _count);
|
||||
if (updated >= 0)
|
||||
{
|
||||
return updated;
|
||||
}
|
||||
|
||||
Interlocked.CompareExchange(ref _count, 0, updated);
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -9,6 +9,7 @@ namespace ZB.MOM.WW.MxGateway.Server.Dashboard.Hubs;
|
||||
/// gateway process; clients listen via the hub.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <see cref="ExecuteAsync"/> wraps the snapshot subscription in
|
||||
/// a reconnect loop with a configurable retry delay (5s by default,
|
||||
/// mirroring <see cref="AlarmsHubPublisher"/>). A transient failure inside
|
||||
@@ -16,44 +17,67 @@ namespace ZB.MOM.WW.MxGateway.Server.Dashboard.Hubs;
|
||||
/// one-time logger-init failure or a transient SQL error from the Galaxy
|
||||
/// summary projection — would otherwise end the BackgroundService with no
|
||||
/// reconnect, taking the dashboard offline until process restart.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// The loop is idle-gated on <see cref="DashboardSnapshotHubConnectionCounter"/>.
|
||||
/// Each snapshot costs a session-registry snapshot and sort, a metrics snapshot
|
||||
/// that copies dictionaries under the global metrics lock, and (periodically) a
|
||||
/// SQLite read of the API key table — work with no consumer when no dashboard is
|
||||
/// connected. While the count is zero the publisher does not advance the snapshot
|
||||
/// enumerator at all, so the producing iterator stays suspended and builds nothing.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public sealed class DashboardSnapshotPublisher : BackgroundService
|
||||
{
|
||||
private static readonly TimeSpan DefaultReconnectDelay = TimeSpan.FromSeconds(5);
|
||||
private static readonly TimeSpan DefaultIdlePollInterval = TimeSpan.FromSeconds(1);
|
||||
|
||||
private readonly IDashboardSnapshotService _snapshotService;
|
||||
private readonly IHubContext<DashboardSnapshotHub> _hubContext;
|
||||
private readonly DashboardSnapshotHubConnectionCounter _connectionCounter;
|
||||
private readonly ILogger<DashboardSnapshotPublisher> _logger;
|
||||
private readonly TimeSpan _reconnectDelay;
|
||||
private readonly TimeSpan _idlePollInterval;
|
||||
|
||||
/// <summary>Initializes a new instance of the DashboardSnapshotPublisher class.</summary>
|
||||
/// <param name="snapshotService">The snapshot service to subscribe to.</param>
|
||||
/// <param name="hubContext">The SignalR hub context for broadcasting.</param>
|
||||
/// <param name="connectionCounter">Live snapshot hub connection count used to idle-gate the tick.</param>
|
||||
/// <param name="logger">The logger instance.</param>
|
||||
public DashboardSnapshotPublisher(
|
||||
IDashboardSnapshotService snapshotService,
|
||||
IHubContext<DashboardSnapshotHub> hubContext,
|
||||
DashboardSnapshotHubConnectionCounter connectionCounter,
|
||||
ILogger<DashboardSnapshotPublisher> logger)
|
||||
: this(snapshotService, hubContext, logger, DefaultReconnectDelay)
|
||||
: this(snapshotService, hubContext, connectionCounter, logger, DefaultReconnectDelay, DefaultIdlePollInterval)
|
||||
{
|
||||
}
|
||||
|
||||
/// <summary>Initializes a new instance of the DashboardSnapshotPublisher class with custom reconnect delay.</summary>
|
||||
/// <remarks>Internal hook for testing: tests inject a very short reconnect delay so assertions don't wait full 5s.</remarks>
|
||||
/// <summary>Initializes a new instance of the DashboardSnapshotPublisher class with custom cadences.</summary>
|
||||
/// <remarks>
|
||||
/// Internal hook for testing: tests inject a very short reconnect delay so assertions
|
||||
/// don't wait the full 5s, and a short idle poll so the resume-from-idle path is fast.
|
||||
/// </remarks>
|
||||
/// <param name="snapshotService">The snapshot service to subscribe to.</param>
|
||||
/// <param name="hubContext">The SignalR hub context for broadcasting.</param>
|
||||
/// <param name="connectionCounter">Live snapshot hub connection count used to idle-gate the tick.</param>
|
||||
/// <param name="logger">The logger instance.</param>
|
||||
/// <param name="reconnectDelay">The delay before reconnecting after a subscription failure.</param>
|
||||
/// <param name="idlePollInterval">How often the idle publisher re-checks for a connected viewer.</param>
|
||||
internal DashboardSnapshotPublisher(
|
||||
IDashboardSnapshotService snapshotService,
|
||||
IHubContext<DashboardSnapshotHub> hubContext,
|
||||
DashboardSnapshotHubConnectionCounter connectionCounter,
|
||||
ILogger<DashboardSnapshotPublisher> logger,
|
||||
TimeSpan reconnectDelay)
|
||||
TimeSpan reconnectDelay,
|
||||
TimeSpan idlePollInterval)
|
||||
{
|
||||
_snapshotService = snapshotService;
|
||||
_hubContext = hubContext;
|
||||
_connectionCounter = connectionCounter;
|
||||
_logger = logger;
|
||||
_reconnectDelay = reconnectDelay;
|
||||
_idlePollInterval = idlePollInterval;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
@@ -66,15 +90,31 @@ public sealed class DashboardSnapshotPublisher : BackgroundService
|
||||
{
|
||||
try
|
||||
{
|
||||
await foreach (DashboardSnapshot snapshot in _snapshotService
|
||||
// Enumerated by hand rather than with await foreach: the snapshot is
|
||||
// built inside the producer's MoveNextAsync, so not calling MoveNextAsync
|
||||
// is what makes the idle gate skip the build and not just the broadcast.
|
||||
await using IAsyncEnumerator<DashboardSnapshot> snapshots = _snapshotService
|
||||
.WatchSnapshotsAsync(stoppingToken)
|
||||
.ConfigureAwait(false))
|
||||
.GetAsyncEnumerator(stoppingToken);
|
||||
|
||||
while (!stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
if (stoppingToken.IsCancellationRequested)
|
||||
if (_connectionCounter.Count == 0)
|
||||
{
|
||||
// Nobody is watching: leave the producer suspended and re-check
|
||||
// shortly. The first viewer to connect resumes the tick, and is
|
||||
// seeded directly by the hub's OnConnectedAsync meanwhile.
|
||||
await Task.Delay(_idlePollInterval, stoppingToken).ConfigureAwait(false);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!await snapshots.MoveNextAsync().ConfigureAwait(false))
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
DashboardSnapshot snapshot = snapshots.Current;
|
||||
|
||||
try
|
||||
{
|
||||
await _hubContext.Clients
|
||||
|
||||
Reference in New Issue
Block a user