perf(dashboard): idle-gate the snapshot tick; cache static config; bound key-list refresh

The snapshot publisher broadcast to Clients.All on every ~1s tick forever, with zero viewers. Each tick cost a session-registry snapshot and sort, a metrics snapshot that copies dictionaries under the global metrics lock, a full rebuild of EffectiveGatewayConfiguration, and a SQLite read of the API key table. DashboardSnapshotHub now counts live connections into the singleton DashboardSnapshotHubConnectionCounter (clamped at zero, since SignalR can call OnDisconnectedAsync for a connection whose OnConnectedAsync faulted). The publisher drives the snapshot enumerator by hand instead of await foreach: with no connections it does not call MoveNextAsync at all, so the producing iterator stays suspended at its yield and no snapshot is built — the gate removes the build, not just the broadcast. It re-checks once a second, so the first viewer resumes the tick within about one interval; that viewer is seeded immediately by DashboardPageBase's synchronous GetSnapshot() and by the hub's OnConnectedAsync. Two per-tick costs are bounded independently of the gate: the effective configuration is startup-static (options are bound once at boot and never reloaded), so it is built once and cached; and the API key summaries refresh at most every 15s, since the list only changes when an operator creates, rotates, or revokes a key. Only a successful refresh restarts the interval, so a failed or timed-out read is still retried on the next tick with the previous summaries left on screen.
2026-08-15 12:21:49 -04:00
parent e04b1c9199
commit 77c5731b7b
7 changed files with 338 additions and 30 deletions
@@ -9,6 +9,7 @@ namespace ZB.MOM.WW.MxGateway.Tests.Gateway.Dashboard;
 public sealed class DashboardSnapshotPublisherTests
 {
    private static readonly TimeSpan TestTimeout = TimeSpan.FromSeconds(5);
+    private static readonly TimeSpan IdlePollInterval = TimeSpan.FromMilliseconds(10);

    /// <summary>
    /// A transient failure inside
@@ -28,8 +29,10 @@ public sealed class DashboardSnapshotPublisherTests
        DashboardSnapshotPublisher publisher = new(
            snapshotService,
            hubContext,
+            ConnectedCounter(),
            NullLogger<DashboardSnapshotPublisher>.Instance,
-            reconnectDelay);
+            reconnectDelay,
+            IdlePollInterval);

        using CancellationTokenSource cts = new();
        Task execute = publisher.StartAsync(cts.Token);
@@ -73,8 +76,10 @@ public sealed class DashboardSnapshotPublisherTests
        DashboardSnapshotPublisher publisher = new(
            snapshotService,
            hubContext,
+            ConnectedCounter(),
            NullLogger<DashboardSnapshotPublisher>.Instance,
-            reconnectDelay);
+            reconnectDelay,
+            IdlePollInterval);

        using CancellationTokenSource cts = new();
        Task execute = publisher.StartAsync(cts.Token);
@@ -88,6 +93,54 @@ public sealed class DashboardSnapshotPublisherTests
        Assert.True(snapshotService.SubscribeCount >= 2);
    }

+    /// <summary>
+    /// With no dashboard connected there is nobody to broadcast to, so the publisher must
+    /// not advance the snapshot enumerator at all — every pull costs a registry snapshot and
+    /// sort, a locked metrics dictionary copy, and periodically a SQLite key-table read.
+    /// The first viewer to connect resumes the tick.
+    /// </summary>
+    /// <returns>A task that represents the asynchronous operation.</returns>
+    [Fact]
+    public async Task ExecuteAsync_WhenNoHubConnections_DoesNotPullSnapshots()
+    {
+        CountingSnapshotService snapshotService = new();
+        RecordingHubContext hubContext = new();
+        DashboardSnapshotHubConnectionCounter connectionCounter = new();
+        DashboardSnapshotPublisher publisher = new(
+            snapshotService,
+            hubContext,
+            connectionCounter,
+            NullLogger<DashboardSnapshotPublisher>.Instance,
+            TimeSpan.FromMilliseconds(50),
+            IdlePollInterval);
+
+        using CancellationTokenSource cts = new();
+        await publisher.StartAsync(cts.Token).WaitAsync(TestTimeout);
+
+        // Long enough for many idle polls at IdlePollInterval.
+        await Task.Delay(TimeSpan.FromMilliseconds(250));
+
+        Assert.Equal(0, snapshotService.PullCount);
+        Assert.Equal(0, hubContext.SendCount);
+
+        connectionCounter.Increment();
+        await WaitUntilAsync(() => hubContext.SendCount >= 1);
+
+        await cts.CancelAsync();
+        await publisher.StopAsync(CancellationToken.None);
+
+        Assert.True(snapshotService.PullCount >= 1);
+    }
+
+    /// <summary>Creates a connection counter that already has one live viewer.</summary>
+    /// <returns>A counter reporting a single connection.</returns>
+    private static DashboardSnapshotHubConnectionCounter ConnectedCounter()
+    {
+        DashboardSnapshotHubConnectionCounter counter = new();
+        counter.Increment();
+        return counter;
+    }
+
    private static async Task WaitUntilAsync(Func<bool> predicate)
    {
        using CancellationTokenSource cancellationTokenSource = new(TestTimeout);
@@ -174,6 +227,34 @@ public sealed class DashboardSnapshotPublisherTests
        }
    }

+    private sealed class CountingSnapshotService : IDashboardSnapshotService
+    {
+        private int _pullCount;
+
+        /// <summary>Gets the number of snapshots the publisher pulled from this source.</summary>
+        public int PullCount => Volatile.Read(ref _pullCount);
+
+        /// <inheritdoc />
+        public DashboardSnapshot GetSnapshot()
+        {
+            return null!;
+        }
+
+        /// <inheritdoc />
+        public async IAsyncEnumerable<DashboardSnapshot> WatchSnapshotsAsync(
+            [EnumeratorCancellation] CancellationToken cancellationToken)
+        {
+            while (!cancellationToken.IsCancellationRequested)
+            {
+                // Short cadence so a listening publisher pulls quickly; the counter only
+                // moves when the publisher actually advances the enumerator.
+                await Task.Delay(TimeSpan.FromMilliseconds(5), cancellationToken).ConfigureAwait(false);
+                Interlocked.Increment(ref _pullCount);
+                yield return GetSnapshot();
+            }
+        }
+    }
+
    private sealed class RecordingHubContext : IHubContext<DashboardSnapshotHub>
    {
        private readonly RecordingHubClients _clients = new();
@@ -1,5 +1,6 @@
 using System.Globalization;
 using Microsoft.Extensions.Options;
+using Microsoft.Extensions.Time.Testing;
 using ZB.MOM.WW.Auth.Abstractions.ApiKeys;
 using ZB.MOM.WW.GalaxyRepository;
 using ZB.MOM.WW.GalaxyRepository.Grpc;
@@ -457,6 +458,7 @@ public sealed class DashboardSnapshotServiceTests
                CreatedUtc: DateTimeOffset.Parse("2026-04-28T12:00:00Z", CultureInfo.InvariantCulture),
                LastUsedUtc: null,
                RevokedUtc: null));
+        FakeTimeProvider timeProvider = new(DateTimeOffset.Parse("2026-08-15T00:00:00Z", CultureInfo.InvariantCulture));
        DashboardSnapshotService service = CreateService(
            new SessionRegistry(),
            metrics,
@@ -464,11 +466,12 @@ public sealed class DashboardSnapshotServiceTests
            {
                Dashboard = new DashboardOptions
                {
-                    SnapshotIntervalMilliseconds = 1,
+                    SnapshotIntervalMilliseconds = 1000,
                },
            },
-            apiKeyAdminStore: apiKeyAdminStore);
-        using CancellationTokenSource cancellation = new(TimeSpan.FromSeconds(2));
+            apiKeyAdminStore: apiKeyAdminStore,
+            timeProvider: timeProvider);
+        using CancellationTokenSource cancellation = new(TimeSpan.FromSeconds(30));
        await using IAsyncEnumerator<DashboardSnapshot> enumerator = service
            .WatchSnapshotsAsync(cancellation.Token)
            .GetAsyncEnumerator(cancellation.Token);
@@ -477,14 +480,86 @@ public sealed class DashboardSnapshotServiceTests
        DashboardSnapshot first = enumerator.Current;
        apiKeyAdminStore.FailNext = true;

-        Assert.True(await enumerator.MoveNextAsync());
-        DashboardSnapshot second = enumerator.Current;
+        // Advance past the key-summary refresh interval so the second tick really
+        // does attempt a refresh — that attempt is the one that fails.
+        DashboardSnapshot second = await NextSnapshotAsync(enumerator, timeProvider, TimeSpan.FromSeconds(20));

        Assert.Equal("operator01", Assert.Single(first.ApiKeys).KeyId);
        Assert.Equal("operator01", Assert.Single(second.ApiKeys).KeyId);
        Assert.Equal(2, apiKeyAdminStore.ListCount);
    }

+    /// <summary>
+    /// The API key list is a SQLite read; at the default 1s snapshot cadence it would run
+    /// ~86k times a day against a table that changes by hand. Ticks inside the refresh
+    /// interval must reuse the cached summaries and not touch the store.
+    /// </summary>
+    /// <returns>A task that represents the asynchronous operation.</returns>
+    [Fact]
+    public async Task WatchSnapshotsAsync_WhenTicksFallInsideRefreshInterval_ListsApiKeysOnce()
+    {
+        using GatewayMetrics metrics = new();
+        CountingApiKeyAdminStore apiKeyAdminStore = new(
+            new ApiKeyListItem(
+                KeyId: "operator01",
+                KeyPrefix: "mxgw",
+                DisplayName: "Operator",
+                Scopes: new HashSet<string>([GatewayScopes.MetadataRead], StringComparer.Ordinal),
+                ConstraintsJson: null,
+                CreatedUtc: DateTimeOffset.Parse("2026-04-28T12:00:00Z", CultureInfo.InvariantCulture),
+                LastUsedUtc: null,
+                RevokedUtc: null));
+        FakeTimeProvider timeProvider = new(DateTimeOffset.Parse("2026-08-15T00:00:00Z", CultureInfo.InvariantCulture));
+        DashboardSnapshotService service = CreateService(
+            new SessionRegistry(),
+            metrics,
+            new GatewayOptions
+            {
+                Dashboard = new DashboardOptions
+                {
+                    SnapshotIntervalMilliseconds = 1000,
+                },
+            },
+            apiKeyAdminStore: apiKeyAdminStore,
+            timeProvider: timeProvider);
+        using CancellationTokenSource cancellation = new(TimeSpan.FromSeconds(30));
+        await using IAsyncEnumerator<DashboardSnapshot> enumerator = service
+            .WatchSnapshotsAsync(cancellation.Token)
+            .GetAsyncEnumerator(cancellation.Token);
+
+        Assert.True(await enumerator.MoveNextAsync());
+        Assert.Equal(1, apiKeyAdminStore.ListCount);
+
+        // Two more 1s ticks, both well inside the 15s refresh interval.
+        DashboardSnapshot second = await NextSnapshotAsync(enumerator, timeProvider, TimeSpan.FromSeconds(1));
+        await NextSnapshotAsync(enumerator, timeProvider, TimeSpan.FromSeconds(1));
+
+        Assert.Equal(1, apiKeyAdminStore.ListCount);
+        Assert.Equal("operator01", Assert.Single(second.ApiKeys).KeyId);
+
+        // A tick past the interval refreshes again, so an added or revoked key still
+        // reaches the dashboard within the interval.
+        await NextSnapshotAsync(enumerator, timeProvider, TimeSpan.FromSeconds(20));
+
+        Assert.Equal(2, apiKeyAdminStore.ListCount);
+    }
+
+    /// <summary>
+    /// The effective configuration is startup-static, so the snapshot must hand out the
+    /// same instance instead of rebuilding the whole option tree on every tick.
+    /// </summary>
+    [Fact]
+    public void GetSnapshot_ReusesTheSameEffectiveConfigurationInstance()
+    {
+        using GatewayMetrics metrics = new();
+        DashboardSnapshotService service = CreateService(new SessionRegistry(), metrics);
+
+        DashboardSnapshot first = service.GetSnapshot();
+        DashboardSnapshot second = service.GetSnapshot();
+
+        Assert.Same(first.Configuration, second.Configuration);
+    }
+
    /// <summary>Verifies that snapshot service disposes cleanly when subscriber cancels.</summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [Fact]
@@ -513,12 +588,34 @@ public sealed class DashboardSnapshotServiceTests
        Assert.False(hasNext);
    }

+    /// <summary>
+    /// Advances the fake clock past the next snapshot tick and returns the snapshot it
+    /// produces. <c>MoveNextAsync</c> is started before the advance because the iterator
+    /// creates its <see cref="PeriodicTimer"/> synchronously on that call — the timer must
+    /// exist before the clock moves or the tick is missed.
+    /// </summary>
+    /// <param name="enumerator">The snapshot enumerator being driven.</param>
+    /// <param name="timeProvider">The fake clock backing the snapshot timer.</param>
+    /// <param name="advance">How far to advance the clock.</param>
+    /// <returns>The snapshot produced by the tick.</returns>
+    private static async Task<DashboardSnapshot> NextSnapshotAsync(
+        IAsyncEnumerator<DashboardSnapshot> enumerator,
+        FakeTimeProvider timeProvider,
+        TimeSpan advance)
+    {
+        ValueTask<bool> pending = enumerator.MoveNextAsync();
+        timeProvider.Advance(advance);
+        Assert.True(await pending.AsTask().WaitAsync(TimeSpan.FromSeconds(10)));
+        return enumerator.Current;
+    }
+
    private static DashboardSnapshotService CreateService(
        SessionRegistry registry,
        GatewayMetrics metrics,
        GatewayOptions? options = null,
        IGalaxyHierarchyCache? galaxyHierarchyCache = null,
-        IApiKeyAdminStore? apiKeyAdminStore = null)
+        IApiKeyAdminStore? apiKeyAdminStore = null,
+        TimeProvider? timeProvider = null)
    {
        GatewayOptions resolvedOptions = options ?? new GatewayOptions
        {
@@ -535,7 +632,8 @@ public sealed class DashboardSnapshotServiceTests
            configurationProvider,
            galaxyHierarchyCache ?? new StubGalaxyHierarchyCache(GalaxyHierarchyCacheEntry.Empty),
            apiKeyAdminStore ?? new FakeApiKeyAdminStore(),
-            Options.Create(resolvedOptions));
+            Options.Create(resolvedOptions),
+            timeProvider);
    }

    private sealed class StubGalaxyHierarchyCache(GalaxyHierarchyCacheEntry current) : IGalaxyHierarchyCache