perf(dashboard): idle-gate the snapshot tick; cache static config; bound key-list refresh
The snapshot publisher broadcast to Clients.All on every ~1s tick forever, with zero viewers. Each tick cost a session-registry snapshot and sort, a metrics snapshot that copies dictionaries under the global metrics lock, a full rebuild of EffectiveGatewayConfiguration, and a SQLite read of the API key table. DashboardSnapshotHub now counts live connections into the singleton DashboardSnapshotHubConnectionCounter (clamped at zero, since SignalR can call OnDisconnectedAsync for a connection whose OnConnectedAsync faulted). The publisher drives the snapshot enumerator by hand instead of await foreach: with no connections it does not call MoveNextAsync at all, so the producing iterator stays suspended at its yield and no snapshot is built — the gate removes the build, not just the broadcast. It re-checks once a second, so the first viewer resumes the tick within about one interval; that viewer is seeded immediately by DashboardPageBase's synchronous GetSnapshot() and by the hub's OnConnectedAsync. Two per-tick costs are bounded independently of the gate: the effective configuration is startup-static (options are bound once at boot and never reloaded), so it is built once and cached; and the API key summaries refresh at most every 15s, since the list only changes when an operator creates, rotates, or revokes a key. Only a successful refresh restarts the interval, so a failed or timed-out read is still retried on the next tick with the previous summaries left on screen.
This commit is contained in:
@@ -394,18 +394,6 @@ its lease expires. One session means one worker process backs every dashboard
|
|||||||
circuit; all access is serialised so the worker sees one in-flight command at a
|
circuit; all access is serialised so the worker sees one in-flight command at a
|
||||||
time. Tag reads go through `GatewaySession.SubscribeBulkAsync` / `ReadBulkAsync`.
|
time. Tag reads go through `GatewaySession.SubscribeBulkAsync` / `ReadBulkAsync`.
|
||||||
|
|
||||||
The advise set that backs those reads is capped at 256 tags (one browse page plus
|
|
||||||
headroom) and evicted least-recently-read-first. Without the cap every tag any
|
|
||||||
viewer ever inspected stayed advised on the single dashboard worker until the
|
|
||||||
session faulted, so browsing a large galaxy accreted unbounded live MXAccess
|
|
||||||
subscriptions — and the event churn they feed — on one x86 process. Reading a tag
|
|
||||||
already in the set marks it most-recently-read; subscribing past the cap unadvises
|
|
||||||
the oldest entries with `GatewaySession.UnsubscribeBulkAsync` in one batch before
|
|
||||||
the new ones are advised. Tags read in the same call are never evicted to make
|
|
||||||
room for each other. A failed unadvise does not fail the read: the tags are
|
|
||||||
dropped from tracking anyway (they re-subscribe if read again), because the
|
|
||||||
session-invalidation path already handles gateway/worker drift.
|
|
||||||
|
|
||||||
The Alarms page does **not** use the dashboard session: alarm data comes from
|
The Alarms page does **not** use the dashboard session: alarm data comes from
|
||||||
the gateway's always-on central monitor. `QueryAlarmsAsync` reads
|
the gateway's always-on central monitor. `QueryAlarmsAsync` reads
|
||||||
`IGatewayAlarmService.CurrentAlarms` — the monitor's in-process cache — so the
|
`IGatewayAlarmService.CurrentAlarms` — the monitor's in-process cache — so the
|
||||||
|
|||||||
@@ -51,6 +51,7 @@ public static class DashboardServiceCollectionExtensions
|
|||||||
// subscriber bookkeeping they share with the broadcaster must outlive them.
|
// subscriber bookkeeping they share with the broadcaster must outlive them.
|
||||||
services.AddSingleton<Hubs.EventsHubViewerRegistry>();
|
services.AddSingleton<Hubs.EventsHubViewerRegistry>();
|
||||||
services.AddSingleton<Hubs.IDashboardEventBroadcaster, Hubs.DashboardEventBroadcaster>();
|
services.AddSingleton<Hubs.IDashboardEventBroadcaster, Hubs.DashboardEventBroadcaster>();
|
||||||
|
services.AddSingleton<Hubs.DashboardSnapshotHubConnectionCounter>();
|
||||||
services.AddHostedService<Hubs.DashboardSnapshotPublisher>();
|
services.AddHostedService<Hubs.DashboardSnapshotPublisher>();
|
||||||
services.AddHostedService<Hubs.AlarmsHubPublisher>();
|
services.AddHostedService<Hubs.AlarmsHubPublisher>();
|
||||||
services.AddHttpContextAccessor();
|
services.AddHttpContextAccessor();
|
||||||
|
|||||||
@@ -16,6 +16,17 @@ public sealed class DashboardSnapshotService : IDashboardSnapshotService
|
|||||||
{
|
{
|
||||||
private const string HealthyStatus = "Healthy";
|
private const string HealthyStatus = "Healthy";
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Minimum spacing between API key list reads. The list is a SQLite query whose
|
||||||
|
/// content only changes when an operator creates, rotates, or revokes a key, so
|
||||||
|
/// refreshing it on every ~1s snapshot tick buys nothing; the dashboard still sees
|
||||||
|
/// a key change within this interval.
|
||||||
|
/// </summary>
|
||||||
|
private static readonly TimeSpan ApiKeySummaryRefreshInterval = TimeSpan.FromSeconds(15);
|
||||||
|
|
||||||
|
/// <summary>Sentinel for "the API key summaries have never been refreshed".</summary>
|
||||||
|
private const long NeverRefreshedTicks = long.MinValue;
|
||||||
|
|
||||||
private readonly ISessionRegistry _sessionRegistry;
|
private readonly ISessionRegistry _sessionRegistry;
|
||||||
private readonly GatewayMetrics _metrics;
|
private readonly GatewayMetrics _metrics;
|
||||||
private readonly IGatewayConfigurationProvider _configurationProvider;
|
private readonly IGatewayConfigurationProvider _configurationProvider;
|
||||||
@@ -30,6 +41,13 @@ public sealed class DashboardSnapshotService : IDashboardSnapshotService
|
|||||||
private readonly ILogger<DashboardSnapshotService> _logger;
|
private readonly ILogger<DashboardSnapshotService> _logger;
|
||||||
private readonly SemaphoreSlim _apiKeySummaryRefreshGate = new(1, 1);
|
private readonly SemaphoreSlim _apiKeySummaryRefreshGate = new(1, 1);
|
||||||
private IReadOnlyList<DashboardApiKeySummary> _apiKeySummaries = Array.Empty<DashboardApiKeySummary>();
|
private IReadOnlyList<DashboardApiKeySummary> _apiKeySummaries = Array.Empty<DashboardApiKeySummary>();
|
||||||
|
private long _apiKeySummariesRefreshedAtTicks = NeverRefreshedTicks;
|
||||||
|
// The effective configuration is built from IOptions<GatewayOptions> and is startup-static:
|
||||||
|
// the gateway binds options once at boot and never reloads them, so this projection cannot
|
||||||
|
// change for the process lifetime. Build it once instead of re-projecting the whole option
|
||||||
|
// tree on every snapshot tick. A racing first build is harmless — the projection is pure,
|
||||||
|
// so either winner stores equivalent content.
|
||||||
|
private EffectiveGatewayConfiguration? _effectiveConfiguration;
|
||||||
// Memoizes ONLY the O(N) template/category breakdown against the cache sequence. The shared
|
// Memoizes ONLY the O(N) template/category breakdown against the cache sequence. The shared
|
||||||
// library bumps Sequence only on a heavy refresh that replaces the object set, so an unchanged
|
// library bumps Sequence only on a heavy refresh that replaces the object set, so an unchanged
|
||||||
// sequence means the breakdown is unchanged and can be reused — keeping the ~1s snapshot tick
|
// sequence means the breakdown is unchanged and can be reused — keeping the ~1s snapshot tick
|
||||||
@@ -100,10 +118,23 @@ public sealed class DashboardSnapshotService : IDashboardSnapshotService
|
|||||||
Metrics: CreateMetricSummaries(metricsSnapshot),
|
Metrics: CreateMetricSummaries(metricsSnapshot),
|
||||||
Faults: CreateFaultSummaries(sessions, generatedAt),
|
Faults: CreateFaultSummaries(sessions, generatedAt),
|
||||||
ApiKeys: Volatile.Read(ref _apiKeySummaries),
|
ApiKeys: Volatile.Read(ref _apiKeySummaries),
|
||||||
Configuration: _configurationProvider.GetEffectiveConfiguration(),
|
Configuration: ResolveEffectiveConfiguration(),
|
||||||
Galaxy: ResolveGalaxySummary());
|
Galaxy: ResolveGalaxySummary());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private EffectiveGatewayConfiguration ResolveEffectiveConfiguration()
|
||||||
|
{
|
||||||
|
EffectiveGatewayConfiguration? cached = Volatile.Read(ref _effectiveConfiguration);
|
||||||
|
if (cached is not null)
|
||||||
|
{
|
||||||
|
return cached;
|
||||||
|
}
|
||||||
|
|
||||||
|
EffectiveGatewayConfiguration configuration = _configurationProvider.GetEffectiveConfiguration();
|
||||||
|
Volatile.Write(ref _effectiveConfiguration, configuration);
|
||||||
|
return configuration;
|
||||||
|
}
|
||||||
|
|
||||||
private DashboardGalaxySummary ResolveGalaxySummary()
|
private DashboardGalaxySummary ResolveGalaxySummary()
|
||||||
{
|
{
|
||||||
GalaxyHierarchyCacheEntry entry = _galaxyHierarchyCache.Current;
|
GalaxyHierarchyCacheEntry entry = _galaxyHierarchyCache.Current;
|
||||||
@@ -255,6 +286,17 @@ public sealed class DashboardSnapshotService : IDashboardSnapshotService
|
|||||||
|
|
||||||
private async Task RefreshApiKeySummariesAsync(CancellationToken cancellationToken)
|
private async Task RefreshApiKeySummariesAsync(CancellationToken cancellationToken)
|
||||||
{
|
{
|
||||||
|
DateTimeOffset now = _timeProvider.GetUtcNow();
|
||||||
|
long lastRefreshedAtTicks = Interlocked.Read(ref _apiKeySummariesRefreshedAtTicks);
|
||||||
|
if (lastRefreshedAtTicks != NeverRefreshedTicks
|
||||||
|
&& now.UtcTicks - lastRefreshedAtTicks < ApiKeySummaryRefreshInterval.Ticks)
|
||||||
|
{
|
||||||
|
// Inside the refresh window: reuse the cached summaries rather than
|
||||||
|
// re-reading the API key table on this tick. Only a *successful* refresh
|
||||||
|
// moves the timestamp, so a failed read is retried on the next tick.
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
if (!await _apiKeySummaryRefreshGate.WaitAsync(0, cancellationToken).ConfigureAwait(false))
|
if (!await _apiKeySummaryRefreshGate.WaitAsync(0, cancellationToken).ConfigureAwait(false))
|
||||||
{
|
{
|
||||||
return;
|
return;
|
||||||
@@ -278,6 +320,7 @@ public sealed class DashboardSnapshotService : IDashboardSnapshotService
|
|||||||
.ToArray();
|
.ToArray();
|
||||||
|
|
||||||
Volatile.Write(ref _apiKeySummaries, summaries);
|
Volatile.Write(ref _apiKeySummaries, summaries);
|
||||||
|
Interlocked.Exchange(ref _apiKeySummariesRefreshedAtTicks, now.UtcTicks);
|
||||||
}
|
}
|
||||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -9,8 +9,15 @@ namespace ZB.MOM.WW.MxGateway.Server.Dashboard.Hubs;
|
|||||||
/// immediately via <see cref="OnConnectedAsync"/>; subsequent refreshes are
|
/// immediately via <see cref="OnConnectedAsync"/>; subsequent refreshes are
|
||||||
/// broadcast by <see cref="DashboardSnapshotPublisher"/>.
|
/// broadcast by <see cref="DashboardSnapshotPublisher"/>.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
|
/// <remarks>
|
||||||
|
/// Connections are counted into <see cref="DashboardSnapshotHubConnectionCounter"/>
|
||||||
|
/// so <see cref="DashboardSnapshotPublisher"/> can stop building and broadcasting
|
||||||
|
/// snapshots while nobody is watching.
|
||||||
|
/// </remarks>
|
||||||
[Authorize(Policy = DashboardAuthenticationDefaults.HubClientsPolicy)]
|
[Authorize(Policy = DashboardAuthenticationDefaults.HubClientsPolicy)]
|
||||||
public sealed class DashboardSnapshotHub(IDashboardSnapshotService snapshotService) : Hub
|
public sealed class DashboardSnapshotHub(
|
||||||
|
IDashboardSnapshotService snapshotService,
|
||||||
|
DashboardSnapshotHubConnectionCounter connectionCounter) : Hub
|
||||||
{
|
{
|
||||||
/// <summary>Method name used to push snapshot updates to clients.</summary>
|
/// <summary>Method name used to push snapshot updates to clients.</summary>
|
||||||
public const string SnapshotMessage = "SnapshotUpdated";
|
public const string SnapshotMessage = "SnapshotUpdated";
|
||||||
@@ -18,7 +25,57 @@ public sealed class DashboardSnapshotHub(IDashboardSnapshotService snapshotServi
|
|||||||
/// <inheritdoc />
|
/// <inheritdoc />
|
||||||
public override async Task OnConnectedAsync()
|
public override async Task OnConnectedAsync()
|
||||||
{
|
{
|
||||||
|
// Count the viewer before seeding it, so the publisher resumes its tick
|
||||||
|
// no later than the first snapshot this connection renders.
|
||||||
|
connectionCounter.Increment();
|
||||||
await Clients.Caller.SendAsync(SnapshotMessage, snapshotService.GetSnapshot()).ConfigureAwait(false);
|
await Clients.Caller.SendAsync(SnapshotMessage, snapshotService.GetSnapshot()).ConfigureAwait(false);
|
||||||
await base.OnConnectedAsync().ConfigureAwait(false);
|
await base.OnConnectedAsync().ConfigureAwait(false);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <inheritdoc />
|
||||||
|
public override async Task OnDisconnectedAsync(Exception? exception)
|
||||||
|
{
|
||||||
|
connectionCounter.Decrement();
|
||||||
|
await base.OnDisconnectedAsync(exception).ConfigureAwait(false);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Process-wide count of live <see cref="DashboardSnapshotHub"/> connections.
|
||||||
|
/// Registered as a singleton and read by <see cref="DashboardSnapshotPublisher"/>
|
||||||
|
/// to idle-gate the snapshot tick: with no dashboard connected there is nothing
|
||||||
|
/// to broadcast to, so no snapshot is built.
|
||||||
|
/// </summary>
|
||||||
|
public sealed class DashboardSnapshotHubConnectionCounter
|
||||||
|
{
|
||||||
|
private int _count;
|
||||||
|
|
||||||
|
/// <summary>Gets the number of live snapshot hub connections.</summary>
|
||||||
|
public int Count => Volatile.Read(ref _count);
|
||||||
|
|
||||||
|
/// <summary>Records a new snapshot hub connection.</summary>
|
||||||
|
/// <returns>The connection count after the increment.</returns>
|
||||||
|
public int Increment()
|
||||||
|
{
|
||||||
|
return Interlocked.Increment(ref _count);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Records a snapshot hub disconnection. The count is clamped at zero: SignalR
|
||||||
|
/// can invoke <c>OnDisconnectedAsync</c> for a connection whose
|
||||||
|
/// <c>OnConnectedAsync</c> faulted, and a negative count would idle-gate the
|
||||||
|
/// publisher while viewers are still attached.
|
||||||
|
/// </summary>
|
||||||
|
/// <returns>The connection count after the decrement.</returns>
|
||||||
|
public int Decrement()
|
||||||
|
{
|
||||||
|
int updated = Interlocked.Decrement(ref _count);
|
||||||
|
if (updated >= 0)
|
||||||
|
{
|
||||||
|
return updated;
|
||||||
|
}
|
||||||
|
|
||||||
|
Interlocked.CompareExchange(ref _count, 0, updated);
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ namespace ZB.MOM.WW.MxGateway.Server.Dashboard.Hubs;
|
|||||||
/// gateway process; clients listen via the hub.
|
/// gateway process; clients listen via the hub.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <remarks>
|
/// <remarks>
|
||||||
|
/// <para>
|
||||||
/// <see cref="ExecuteAsync"/> wraps the snapshot subscription in
|
/// <see cref="ExecuteAsync"/> wraps the snapshot subscription in
|
||||||
/// a reconnect loop with a configurable retry delay (5s by default,
|
/// a reconnect loop with a configurable retry delay (5s by default,
|
||||||
/// mirroring <see cref="AlarmsHubPublisher"/>). A transient failure inside
|
/// mirroring <see cref="AlarmsHubPublisher"/>). A transient failure inside
|
||||||
@@ -16,44 +17,67 @@ namespace ZB.MOM.WW.MxGateway.Server.Dashboard.Hubs;
|
|||||||
/// one-time logger-init failure or a transient SQL error from the Galaxy
|
/// one-time logger-init failure or a transient SQL error from the Galaxy
|
||||||
/// summary projection — would otherwise end the BackgroundService with no
|
/// summary projection — would otherwise end the BackgroundService with no
|
||||||
/// reconnect, taking the dashboard offline until process restart.
|
/// reconnect, taking the dashboard offline until process restart.
|
||||||
|
/// </para>
|
||||||
|
/// <para>
|
||||||
|
/// The loop is idle-gated on <see cref="DashboardSnapshotHubConnectionCounter"/>.
|
||||||
|
/// Each snapshot costs a session-registry snapshot and sort, a metrics snapshot
|
||||||
|
/// that copies dictionaries under the global metrics lock, and (periodically) a
|
||||||
|
/// SQLite read of the API key table — work with no consumer when no dashboard is
|
||||||
|
/// connected. While the count is zero the publisher does not advance the snapshot
|
||||||
|
/// enumerator at all, so the producing iterator stays suspended and builds nothing.
|
||||||
|
/// </para>
|
||||||
/// </remarks>
|
/// </remarks>
|
||||||
public sealed class DashboardSnapshotPublisher : BackgroundService
|
public sealed class DashboardSnapshotPublisher : BackgroundService
|
||||||
{
|
{
|
||||||
private static readonly TimeSpan DefaultReconnectDelay = TimeSpan.FromSeconds(5);
|
private static readonly TimeSpan DefaultReconnectDelay = TimeSpan.FromSeconds(5);
|
||||||
|
private static readonly TimeSpan DefaultIdlePollInterval = TimeSpan.FromSeconds(1);
|
||||||
|
|
||||||
private readonly IDashboardSnapshotService _snapshotService;
|
private readonly IDashboardSnapshotService _snapshotService;
|
||||||
private readonly IHubContext<DashboardSnapshotHub> _hubContext;
|
private readonly IHubContext<DashboardSnapshotHub> _hubContext;
|
||||||
|
private readonly DashboardSnapshotHubConnectionCounter _connectionCounter;
|
||||||
private readonly ILogger<DashboardSnapshotPublisher> _logger;
|
private readonly ILogger<DashboardSnapshotPublisher> _logger;
|
||||||
private readonly TimeSpan _reconnectDelay;
|
private readonly TimeSpan _reconnectDelay;
|
||||||
|
private readonly TimeSpan _idlePollInterval;
|
||||||
|
|
||||||
/// <summary>Initializes a new instance of the DashboardSnapshotPublisher class.</summary>
|
/// <summary>Initializes a new instance of the DashboardSnapshotPublisher class.</summary>
|
||||||
/// <param name="snapshotService">The snapshot service to subscribe to.</param>
|
/// <param name="snapshotService">The snapshot service to subscribe to.</param>
|
||||||
/// <param name="hubContext">The SignalR hub context for broadcasting.</param>
|
/// <param name="hubContext">The SignalR hub context for broadcasting.</param>
|
||||||
|
/// <param name="connectionCounter">Live snapshot hub connection count used to idle-gate the tick.</param>
|
||||||
/// <param name="logger">The logger instance.</param>
|
/// <param name="logger">The logger instance.</param>
|
||||||
public DashboardSnapshotPublisher(
|
public DashboardSnapshotPublisher(
|
||||||
IDashboardSnapshotService snapshotService,
|
IDashboardSnapshotService snapshotService,
|
||||||
IHubContext<DashboardSnapshotHub> hubContext,
|
IHubContext<DashboardSnapshotHub> hubContext,
|
||||||
|
DashboardSnapshotHubConnectionCounter connectionCounter,
|
||||||
ILogger<DashboardSnapshotPublisher> logger)
|
ILogger<DashboardSnapshotPublisher> logger)
|
||||||
: this(snapshotService, hubContext, logger, DefaultReconnectDelay)
|
: this(snapshotService, hubContext, connectionCounter, logger, DefaultReconnectDelay, DefaultIdlePollInterval)
|
||||||
{
|
{
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>Initializes a new instance of the DashboardSnapshotPublisher class with custom reconnect delay.</summary>
|
/// <summary>Initializes a new instance of the DashboardSnapshotPublisher class with custom cadences.</summary>
|
||||||
/// <remarks>Internal hook for testing: tests inject a very short reconnect delay so assertions don't wait full 5s.</remarks>
|
/// <remarks>
|
||||||
|
/// Internal hook for testing: tests inject a very short reconnect delay so assertions
|
||||||
|
/// don't wait the full 5s, and a short idle poll so the resume-from-idle path is fast.
|
||||||
|
/// </remarks>
|
||||||
/// <param name="snapshotService">The snapshot service to subscribe to.</param>
|
/// <param name="snapshotService">The snapshot service to subscribe to.</param>
|
||||||
/// <param name="hubContext">The SignalR hub context for broadcasting.</param>
|
/// <param name="hubContext">The SignalR hub context for broadcasting.</param>
|
||||||
|
/// <param name="connectionCounter">Live snapshot hub connection count used to idle-gate the tick.</param>
|
||||||
/// <param name="logger">The logger instance.</param>
|
/// <param name="logger">The logger instance.</param>
|
||||||
/// <param name="reconnectDelay">The delay before reconnecting after a subscription failure.</param>
|
/// <param name="reconnectDelay">The delay before reconnecting after a subscription failure.</param>
|
||||||
|
/// <param name="idlePollInterval">How often the idle publisher re-checks for a connected viewer.</param>
|
||||||
internal DashboardSnapshotPublisher(
|
internal DashboardSnapshotPublisher(
|
||||||
IDashboardSnapshotService snapshotService,
|
IDashboardSnapshotService snapshotService,
|
||||||
IHubContext<DashboardSnapshotHub> hubContext,
|
IHubContext<DashboardSnapshotHub> hubContext,
|
||||||
|
DashboardSnapshotHubConnectionCounter connectionCounter,
|
||||||
ILogger<DashboardSnapshotPublisher> logger,
|
ILogger<DashboardSnapshotPublisher> logger,
|
||||||
TimeSpan reconnectDelay)
|
TimeSpan reconnectDelay,
|
||||||
|
TimeSpan idlePollInterval)
|
||||||
{
|
{
|
||||||
_snapshotService = snapshotService;
|
_snapshotService = snapshotService;
|
||||||
_hubContext = hubContext;
|
_hubContext = hubContext;
|
||||||
|
_connectionCounter = connectionCounter;
|
||||||
_logger = logger;
|
_logger = logger;
|
||||||
_reconnectDelay = reconnectDelay;
|
_reconnectDelay = reconnectDelay;
|
||||||
|
_idlePollInterval = idlePollInterval;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <inheritdoc />
|
/// <inheritdoc />
|
||||||
@@ -66,15 +90,31 @@ public sealed class DashboardSnapshotPublisher : BackgroundService
|
|||||||
{
|
{
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
await foreach (DashboardSnapshot snapshot in _snapshotService
|
// Enumerated by hand rather than with await foreach: the snapshot is
|
||||||
|
// built inside the producer's MoveNextAsync, so not calling MoveNextAsync
|
||||||
|
// is what makes the idle gate skip the build and not just the broadcast.
|
||||||
|
await using IAsyncEnumerator<DashboardSnapshot> snapshots = _snapshotService
|
||||||
.WatchSnapshotsAsync(stoppingToken)
|
.WatchSnapshotsAsync(stoppingToken)
|
||||||
.ConfigureAwait(false))
|
.GetAsyncEnumerator(stoppingToken);
|
||||||
|
|
||||||
|
while (!stoppingToken.IsCancellationRequested)
|
||||||
{
|
{
|
||||||
if (stoppingToken.IsCancellationRequested)
|
if (_connectionCounter.Count == 0)
|
||||||
|
{
|
||||||
|
// Nobody is watching: leave the producer suspended and re-check
|
||||||
|
// shortly. The first viewer to connect resumes the tick, and is
|
||||||
|
// seeded directly by the hub's OnConnectedAsync meanwhile.
|
||||||
|
await Task.Delay(_idlePollInterval, stoppingToken).ConfigureAwait(false);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!await snapshots.MoveNextAsync().ConfigureAwait(false))
|
||||||
{
|
{
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
DashboardSnapshot snapshot = snapshots.Current;
|
||||||
|
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
await _hubContext.Clients
|
await _hubContext.Clients
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ namespace ZB.MOM.WW.MxGateway.Tests.Gateway.Dashboard;
|
|||||||
public sealed class DashboardSnapshotPublisherTests
|
public sealed class DashboardSnapshotPublisherTests
|
||||||
{
|
{
|
||||||
private static readonly TimeSpan TestTimeout = TimeSpan.FromSeconds(5);
|
private static readonly TimeSpan TestTimeout = TimeSpan.FromSeconds(5);
|
||||||
|
private static readonly TimeSpan IdlePollInterval = TimeSpan.FromMilliseconds(10);
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// A transient failure inside
|
/// A transient failure inside
|
||||||
@@ -28,8 +29,10 @@ public sealed class DashboardSnapshotPublisherTests
|
|||||||
DashboardSnapshotPublisher publisher = new(
|
DashboardSnapshotPublisher publisher = new(
|
||||||
snapshotService,
|
snapshotService,
|
||||||
hubContext,
|
hubContext,
|
||||||
|
ConnectedCounter(),
|
||||||
NullLogger<DashboardSnapshotPublisher>.Instance,
|
NullLogger<DashboardSnapshotPublisher>.Instance,
|
||||||
reconnectDelay);
|
reconnectDelay,
|
||||||
|
IdlePollInterval);
|
||||||
|
|
||||||
using CancellationTokenSource cts = new();
|
using CancellationTokenSource cts = new();
|
||||||
Task execute = publisher.StartAsync(cts.Token);
|
Task execute = publisher.StartAsync(cts.Token);
|
||||||
@@ -73,8 +76,10 @@ public sealed class DashboardSnapshotPublisherTests
|
|||||||
DashboardSnapshotPublisher publisher = new(
|
DashboardSnapshotPublisher publisher = new(
|
||||||
snapshotService,
|
snapshotService,
|
||||||
hubContext,
|
hubContext,
|
||||||
|
ConnectedCounter(),
|
||||||
NullLogger<DashboardSnapshotPublisher>.Instance,
|
NullLogger<DashboardSnapshotPublisher>.Instance,
|
||||||
reconnectDelay);
|
reconnectDelay,
|
||||||
|
IdlePollInterval);
|
||||||
|
|
||||||
using CancellationTokenSource cts = new();
|
using CancellationTokenSource cts = new();
|
||||||
Task execute = publisher.StartAsync(cts.Token);
|
Task execute = publisher.StartAsync(cts.Token);
|
||||||
@@ -88,6 +93,54 @@ public sealed class DashboardSnapshotPublisherTests
|
|||||||
Assert.True(snapshotService.SubscribeCount >= 2);
|
Assert.True(snapshotService.SubscribeCount >= 2);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// With no dashboard connected there is nobody to broadcast to, so the publisher must
|
||||||
|
/// not advance the snapshot enumerator at all — every pull costs a registry snapshot and
|
||||||
|
/// sort, a locked metrics dictionary copy, and periodically a SQLite key-table read.
|
||||||
|
/// The first viewer to connect resumes the tick.
|
||||||
|
/// </summary>
|
||||||
|
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||||
|
[Fact]
|
||||||
|
public async Task ExecuteAsync_WhenNoHubConnections_DoesNotPullSnapshots()
|
||||||
|
{
|
||||||
|
CountingSnapshotService snapshotService = new();
|
||||||
|
RecordingHubContext hubContext = new();
|
||||||
|
DashboardSnapshotHubConnectionCounter connectionCounter = new();
|
||||||
|
DashboardSnapshotPublisher publisher = new(
|
||||||
|
snapshotService,
|
||||||
|
hubContext,
|
||||||
|
connectionCounter,
|
||||||
|
NullLogger<DashboardSnapshotPublisher>.Instance,
|
||||||
|
TimeSpan.FromMilliseconds(50),
|
||||||
|
IdlePollInterval);
|
||||||
|
|
||||||
|
using CancellationTokenSource cts = new();
|
||||||
|
await publisher.StartAsync(cts.Token).WaitAsync(TestTimeout);
|
||||||
|
|
||||||
|
// Long enough for many idle polls at IdlePollInterval.
|
||||||
|
await Task.Delay(TimeSpan.FromMilliseconds(250));
|
||||||
|
|
||||||
|
Assert.Equal(0, snapshotService.PullCount);
|
||||||
|
Assert.Equal(0, hubContext.SendCount);
|
||||||
|
|
||||||
|
connectionCounter.Increment();
|
||||||
|
await WaitUntilAsync(() => hubContext.SendCount >= 1);
|
||||||
|
|
||||||
|
await cts.CancelAsync();
|
||||||
|
await publisher.StopAsync(CancellationToken.None);
|
||||||
|
|
||||||
|
Assert.True(snapshotService.PullCount >= 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>Creates a connection counter that already has one live viewer.</summary>
|
||||||
|
/// <returns>A counter reporting a single connection.</returns>
|
||||||
|
private static DashboardSnapshotHubConnectionCounter ConnectedCounter()
|
||||||
|
{
|
||||||
|
DashboardSnapshotHubConnectionCounter counter = new();
|
||||||
|
counter.Increment();
|
||||||
|
return counter;
|
||||||
|
}
|
||||||
|
|
||||||
private static async Task WaitUntilAsync(Func<bool> predicate)
|
private static async Task WaitUntilAsync(Func<bool> predicate)
|
||||||
{
|
{
|
||||||
using CancellationTokenSource cancellationTokenSource = new(TestTimeout);
|
using CancellationTokenSource cancellationTokenSource = new(TestTimeout);
|
||||||
@@ -174,6 +227,34 @@ public sealed class DashboardSnapshotPublisherTests
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private sealed class CountingSnapshotService : IDashboardSnapshotService
|
||||||
|
{
|
||||||
|
private int _pullCount;
|
||||||
|
|
||||||
|
/// <summary>Gets the number of snapshots the publisher pulled from this source.</summary>
|
||||||
|
public int PullCount => Volatile.Read(ref _pullCount);
|
||||||
|
|
||||||
|
/// <inheritdoc />
|
||||||
|
public DashboardSnapshot GetSnapshot()
|
||||||
|
{
|
||||||
|
return null!;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <inheritdoc />
|
||||||
|
public async IAsyncEnumerable<DashboardSnapshot> WatchSnapshotsAsync(
|
||||||
|
[EnumeratorCancellation] CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
while (!cancellationToken.IsCancellationRequested)
|
||||||
|
{
|
||||||
|
// Short cadence so a listening publisher pulls quickly; the counter only
|
||||||
|
// moves when the publisher actually advances the enumerator.
|
||||||
|
await Task.Delay(TimeSpan.FromMilliseconds(5), cancellationToken).ConfigureAwait(false);
|
||||||
|
Interlocked.Increment(ref _pullCount);
|
||||||
|
yield return GetSnapshot();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private sealed class RecordingHubContext : IHubContext<DashboardSnapshotHub>
|
private sealed class RecordingHubContext : IHubContext<DashboardSnapshotHub>
|
||||||
{
|
{
|
||||||
private readonly RecordingHubClients _clients = new();
|
private readonly RecordingHubClients _clients = new();
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
using System.Globalization;
|
using System.Globalization;
|
||||||
using Microsoft.Extensions.Options;
|
using Microsoft.Extensions.Options;
|
||||||
|
using Microsoft.Extensions.Time.Testing;
|
||||||
using ZB.MOM.WW.Auth.Abstractions.ApiKeys;
|
using ZB.MOM.WW.Auth.Abstractions.ApiKeys;
|
||||||
using ZB.MOM.WW.GalaxyRepository;
|
using ZB.MOM.WW.GalaxyRepository;
|
||||||
using ZB.MOM.WW.GalaxyRepository.Grpc;
|
using ZB.MOM.WW.GalaxyRepository.Grpc;
|
||||||
@@ -457,6 +458,7 @@ public sealed class DashboardSnapshotServiceTests
|
|||||||
CreatedUtc: DateTimeOffset.Parse("2026-04-28T12:00:00Z", CultureInfo.InvariantCulture),
|
CreatedUtc: DateTimeOffset.Parse("2026-04-28T12:00:00Z", CultureInfo.InvariantCulture),
|
||||||
LastUsedUtc: null,
|
LastUsedUtc: null,
|
||||||
RevokedUtc: null));
|
RevokedUtc: null));
|
||||||
|
FakeTimeProvider timeProvider = new(DateTimeOffset.Parse("2026-08-15T00:00:00Z", CultureInfo.InvariantCulture));
|
||||||
DashboardSnapshotService service = CreateService(
|
DashboardSnapshotService service = CreateService(
|
||||||
new SessionRegistry(),
|
new SessionRegistry(),
|
||||||
metrics,
|
metrics,
|
||||||
@@ -464,11 +466,12 @@ public sealed class DashboardSnapshotServiceTests
|
|||||||
{
|
{
|
||||||
Dashboard = new DashboardOptions
|
Dashboard = new DashboardOptions
|
||||||
{
|
{
|
||||||
SnapshotIntervalMilliseconds = 1,
|
SnapshotIntervalMilliseconds = 1000,
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
apiKeyAdminStore: apiKeyAdminStore);
|
apiKeyAdminStore: apiKeyAdminStore,
|
||||||
using CancellationTokenSource cancellation = new(TimeSpan.FromSeconds(2));
|
timeProvider: timeProvider);
|
||||||
|
using CancellationTokenSource cancellation = new(TimeSpan.FromSeconds(30));
|
||||||
await using IAsyncEnumerator<DashboardSnapshot> enumerator = service
|
await using IAsyncEnumerator<DashboardSnapshot> enumerator = service
|
||||||
.WatchSnapshotsAsync(cancellation.Token)
|
.WatchSnapshotsAsync(cancellation.Token)
|
||||||
.GetAsyncEnumerator(cancellation.Token);
|
.GetAsyncEnumerator(cancellation.Token);
|
||||||
@@ -477,14 +480,86 @@ public sealed class DashboardSnapshotServiceTests
|
|||||||
DashboardSnapshot first = enumerator.Current;
|
DashboardSnapshot first = enumerator.Current;
|
||||||
apiKeyAdminStore.FailNext = true;
|
apiKeyAdminStore.FailNext = true;
|
||||||
|
|
||||||
Assert.True(await enumerator.MoveNextAsync());
|
// Advance past the key-summary refresh interval so the second tick really
|
||||||
DashboardSnapshot second = enumerator.Current;
|
// does attempt a refresh — that attempt is the one that fails.
|
||||||
|
DashboardSnapshot second = await NextSnapshotAsync(enumerator, timeProvider, TimeSpan.FromSeconds(20));
|
||||||
|
|
||||||
Assert.Equal("operator01", Assert.Single(first.ApiKeys).KeyId);
|
Assert.Equal("operator01", Assert.Single(first.ApiKeys).KeyId);
|
||||||
Assert.Equal("operator01", Assert.Single(second.ApiKeys).KeyId);
|
Assert.Equal("operator01", Assert.Single(second.ApiKeys).KeyId);
|
||||||
Assert.Equal(2, apiKeyAdminStore.ListCount);
|
Assert.Equal(2, apiKeyAdminStore.ListCount);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// The API key list is a SQLite read; at the default 1s snapshot cadence it would run
|
||||||
|
/// ~86k times a day against a table that changes by hand. Ticks inside the refresh
|
||||||
|
/// interval must reuse the cached summaries and not touch the store.
|
||||||
|
/// </summary>
|
||||||
|
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||||
|
[Fact]
|
||||||
|
public async Task WatchSnapshotsAsync_WhenTicksFallInsideRefreshInterval_ListsApiKeysOnce()
|
||||||
|
{
|
||||||
|
using GatewayMetrics metrics = new();
|
||||||
|
CountingApiKeyAdminStore apiKeyAdminStore = new(
|
||||||
|
new ApiKeyListItem(
|
||||||
|
KeyId: "operator01",
|
||||||
|
KeyPrefix: "mxgw",
|
||||||
|
DisplayName: "Operator",
|
||||||
|
Scopes: new HashSet<string>([GatewayScopes.MetadataRead], StringComparer.Ordinal),
|
||||||
|
ConstraintsJson: null,
|
||||||
|
CreatedUtc: DateTimeOffset.Parse("2026-04-28T12:00:00Z", CultureInfo.InvariantCulture),
|
||||||
|
LastUsedUtc: null,
|
||||||
|
RevokedUtc: null));
|
||||||
|
FakeTimeProvider timeProvider = new(DateTimeOffset.Parse("2026-08-15T00:00:00Z", CultureInfo.InvariantCulture));
|
||||||
|
DashboardSnapshotService service = CreateService(
|
||||||
|
new SessionRegistry(),
|
||||||
|
metrics,
|
||||||
|
new GatewayOptions
|
||||||
|
{
|
||||||
|
Dashboard = new DashboardOptions
|
||||||
|
{
|
||||||
|
SnapshotIntervalMilliseconds = 1000,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
apiKeyAdminStore: apiKeyAdminStore,
|
||||||
|
timeProvider: timeProvider);
|
||||||
|
using CancellationTokenSource cancellation = new(TimeSpan.FromSeconds(30));
|
||||||
|
await using IAsyncEnumerator<DashboardSnapshot> enumerator = service
|
||||||
|
.WatchSnapshotsAsync(cancellation.Token)
|
||||||
|
.GetAsyncEnumerator(cancellation.Token);
|
||||||
|
|
||||||
|
Assert.True(await enumerator.MoveNextAsync());
|
||||||
|
Assert.Equal(1, apiKeyAdminStore.ListCount);
|
||||||
|
|
||||||
|
// Two more 1s ticks, both well inside the 15s refresh interval.
|
||||||
|
DashboardSnapshot second = await NextSnapshotAsync(enumerator, timeProvider, TimeSpan.FromSeconds(1));
|
||||||
|
await NextSnapshotAsync(enumerator, timeProvider, TimeSpan.FromSeconds(1));
|
||||||
|
|
||||||
|
Assert.Equal(1, apiKeyAdminStore.ListCount);
|
||||||
|
Assert.Equal("operator01", Assert.Single(second.ApiKeys).KeyId);
|
||||||
|
|
||||||
|
// A tick past the interval refreshes again, so an added or revoked key still
|
||||||
|
// reaches the dashboard within the interval.
|
||||||
|
await NextSnapshotAsync(enumerator, timeProvider, TimeSpan.FromSeconds(20));
|
||||||
|
|
||||||
|
Assert.Equal(2, apiKeyAdminStore.ListCount);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// The effective configuration is startup-static, so the snapshot must hand out the
|
||||||
|
/// same instance instead of rebuilding the whole option tree on every tick.
|
||||||
|
/// </summary>
|
||||||
|
[Fact]
|
||||||
|
public void GetSnapshot_ReusesTheSameEffectiveConfigurationInstance()
|
||||||
|
{
|
||||||
|
using GatewayMetrics metrics = new();
|
||||||
|
DashboardSnapshotService service = CreateService(new SessionRegistry(), metrics);
|
||||||
|
|
||||||
|
DashboardSnapshot first = service.GetSnapshot();
|
||||||
|
DashboardSnapshot second = service.GetSnapshot();
|
||||||
|
|
||||||
|
Assert.Same(first.Configuration, second.Configuration);
|
||||||
|
}
|
||||||
|
|
||||||
/// <summary>Verifies that snapshot service disposes cleanly when subscriber cancels.</summary>
|
/// <summary>Verifies that snapshot service disposes cleanly when subscriber cancels.</summary>
|
||||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||||
[Fact]
|
[Fact]
|
||||||
@@ -513,12 +588,34 @@ public sealed class DashboardSnapshotServiceTests
|
|||||||
Assert.False(hasNext);
|
Assert.False(hasNext);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Advances the fake clock past the next snapshot tick and returns the snapshot it
|
||||||
|
/// produces. <c>MoveNextAsync</c> is started before the advance because the iterator
|
||||||
|
/// creates its <see cref="PeriodicTimer"/> synchronously on that call — the timer must
|
||||||
|
/// exist before the clock moves or the tick is missed.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="enumerator">The snapshot enumerator being driven.</param>
|
||||||
|
/// <param name="timeProvider">The fake clock backing the snapshot timer.</param>
|
||||||
|
/// <param name="advance">How far to advance the clock.</param>
|
||||||
|
/// <returns>The snapshot produced by the tick.</returns>
|
||||||
|
private static async Task<DashboardSnapshot> NextSnapshotAsync(
|
||||||
|
IAsyncEnumerator<DashboardSnapshot> enumerator,
|
||||||
|
FakeTimeProvider timeProvider,
|
||||||
|
TimeSpan advance)
|
||||||
|
{
|
||||||
|
ValueTask<bool> pending = enumerator.MoveNextAsync();
|
||||||
|
timeProvider.Advance(advance);
|
||||||
|
Assert.True(await pending.AsTask().WaitAsync(TimeSpan.FromSeconds(10)));
|
||||||
|
return enumerator.Current;
|
||||||
|
}
|
||||||
|
|
||||||
private static DashboardSnapshotService CreateService(
|
private static DashboardSnapshotService CreateService(
|
||||||
SessionRegistry registry,
|
SessionRegistry registry,
|
||||||
GatewayMetrics metrics,
|
GatewayMetrics metrics,
|
||||||
GatewayOptions? options = null,
|
GatewayOptions? options = null,
|
||||||
IGalaxyHierarchyCache? galaxyHierarchyCache = null,
|
IGalaxyHierarchyCache? galaxyHierarchyCache = null,
|
||||||
IApiKeyAdminStore? apiKeyAdminStore = null)
|
IApiKeyAdminStore? apiKeyAdminStore = null,
|
||||||
|
TimeProvider? timeProvider = null)
|
||||||
{
|
{
|
||||||
GatewayOptions resolvedOptions = options ?? new GatewayOptions
|
GatewayOptions resolvedOptions = options ?? new GatewayOptions
|
||||||
{
|
{
|
||||||
@@ -535,7 +632,8 @@ public sealed class DashboardSnapshotServiceTests
|
|||||||
configurationProvider,
|
configurationProvider,
|
||||||
galaxyHierarchyCache ?? new StubGalaxyHierarchyCache(GalaxyHierarchyCacheEntry.Empty),
|
galaxyHierarchyCache ?? new StubGalaxyHierarchyCache(GalaxyHierarchyCacheEntry.Empty),
|
||||||
apiKeyAdminStore ?? new FakeApiKeyAdminStore(),
|
apiKeyAdminStore ?? new FakeApiKeyAdminStore(),
|
||||||
Options.Create(resolvedOptions));
|
Options.Create(resolvedOptions),
|
||||||
|
timeProvider);
|
||||||
}
|
}
|
||||||
|
|
||||||
private sealed class StubGalaxyHierarchyCache(GalaxyHierarchyCacheEntry current) : IGalaxyHierarchyCache
|
private sealed class StubGalaxyHierarchyCache(GalaxyHierarchyCacheEntry current) : IGalaxyHierarchyCache
|
||||||
|
|||||||
Reference in New Issue
Block a user