From e69cd0321e1651939cbb5e02c12785f81ee0a463 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Mon, 9 Nov 2026 12:30:04 -0500 Subject: [PATCH] fix(dashboard): close three low hub/page races (mxaccessgw#174) D11: BrowsePage search CTS is cancelled+disposed exactly once under a lock. D12: the hub refreshes the cached health status before seeding a new connection. D13: DashboardEventBroadcaster gains DisposeAsync that awaits the pumps (bounded). Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV --- .../Components/Pages/BrowsePage.razor | 46 +++++++++++-------- .../Dashboard/DashboardSnapshotService.cs | 2 +- .../Hubs/DashboardEventBroadcaster.cs | 41 +++++++++++++++-- .../Dashboard/Hubs/DashboardSnapshotHub.cs | 13 ++++++ .../Dashboard/IDashboardSnapshotService.cs | 10 ++++ .../DashboardEventBroadcasterTests.cs | 23 ++++++++++ .../DashboardSnapshotServiceTests.cs | 17 +++++++ 7 files changed, 128 insertions(+), 24 deletions(-) diff --git a/src/ZB.MOM.WW.MxGateway.Server/Dashboard/Components/Pages/BrowsePage.razor b/src/ZB.MOM.WW.MxGateway.Server/Dashboard/Components/Pages/BrowsePage.razor index 33abb90f..69c4e0c8 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Dashboard/Components/Pages/BrowsePage.razor +++ b/src/ZB.MOM.WW.MxGateway.Server/Dashboard/Components/Pages/BrowsePage.razor @@ -239,6 +239,7 @@ private bool _searchPending; private string? _searchError; private CancellationTokenSource? _searchCts; + private readonly object _searchLock = new(); private Task? _searchTask; private readonly List _subscribed = []; private readonly Dictionary _values = new(StringComparer.Ordinal); @@ -367,8 +368,7 @@ private void ScheduleSearch(string query) { - CancelQuietly(_searchCts); - _searchCts = null; + ReleaseSearchSource(null); _searchError = null; if (query.Trim().Length == 0) @@ -378,10 +378,15 @@ return; } - // Linked to the page token, so disposal cancels an in-flight search too. The - // search task owns the source and disposes it when it finishes. + // Linked to the page token, so disposal cancels an in-flight search too. Whoever + // clears _searchCts (a newer keystroke or the finishing task) cancels and disposes + // the source, under _searchLock, so it is disposed exactly once. CancellationTokenSource searchCts = CancellationTokenSource.CreateLinkedTokenSource(_cts.Token); - _searchCts = searchCts; + lock (_searchLock) + { + _searchCts = searchCts; + } + _searchPending = true; _searchTask = RunSearchAsync(query, searchCts); } @@ -427,25 +432,30 @@ } finally { - searchCts.Dispose(); + // Only releases when this search is still the current one; a newer keystroke + // has already cancelled and disposed it otherwise. + ReleaseSearchSource(searchCts); } } - // The search task disposes its own source, so a cancel can race that disposal. - private static void CancelQuietly(CancellationTokenSource? source) + // Swap-and-dispose under a lock (R21-D11): the field is cleared and the source is + // cancelled and disposed in one critical section, so a keystroke cancel and the + // finishing task can never both touch a disposed source. A task already running + // keeps its captured token; a disposed source's token still reports cancellation. + // When 'only' is non-null the release applies only if that source is still current. + private void ReleaseSearchSource(CancellationTokenSource? only) { - if (source is null) + lock (_searchLock) { - return; - } + CancellationTokenSource? current = _searchCts; + if (current is null || (only is not null && !ReferenceEquals(current, only))) + { + return; + } - try - { - source.Cancel(); - } - catch (ObjectDisposedException) - { - // The search already finished; there is nothing left to cancel. + _searchCts = null; + current.Cancel(); + current.Dispose(); } } diff --git a/src/ZB.MOM.WW.MxGateway.Server/Dashboard/DashboardSnapshotService.cs b/src/ZB.MOM.WW.MxGateway.Server/Dashboard/DashboardSnapshotService.cs index f6b97502..5ffffcca 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Dashboard/DashboardSnapshotService.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Dashboard/DashboardSnapshotService.cs @@ -227,7 +227,7 @@ public sealed class DashboardSnapshotService : IDashboardSnapshotService /// /// Token to cancel the evaluation. /// A task that represents the asynchronous operation. - private async Task RefreshGatewayStatusAsync(CancellationToken cancellationToken) + public async Task RefreshGatewayStatusAsync(CancellationToken cancellationToken) { if (_healthCheckService is null) { diff --git a/src/ZB.MOM.WW.MxGateway.Server/Dashboard/Hubs/DashboardEventBroadcaster.cs b/src/ZB.MOM.WW.MxGateway.Server/Dashboard/Hubs/DashboardEventBroadcaster.cs index 1287aaf2..16d20ecd 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Dashboard/Hubs/DashboardEventBroadcaster.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Dashboard/Hubs/DashboardEventBroadcaster.cs @@ -38,7 +38,7 @@ namespace ZB.MOM.WW.MxGateway.Server.Dashboard.Hubs; /// reports the last one gone. /// /// -public sealed class DashboardEventBroadcaster : IDashboardEventBroadcaster, IDashboardSessionEventSubscriber, IDisposable +public sealed class DashboardEventBroadcaster : IDashboardEventBroadcaster, IDashboardSessionEventSubscriber, IDisposable, IAsyncDisposable { /// /// Queue depth per in-process subscriber. The consumer is a Blazor page @@ -173,17 +173,46 @@ public sealed class DashboardEventBroadcaster : IDashboardEventBroadcaster, IDas } /// - /// Stops every hub pump and detaches from the viewer registry. The singleton is - /// disposed with the host; a pump's in-flight send is bounded by its timeout. + /// Stops every hub pump and detaches from the viewer registry without waiting for + /// the pumps to drain. The host prefers . /// - public void Dispose() + public void Dispose() => _ = Shutdown(); + + /// + /// Stops every hub pump, detaches from the viewer registry, and awaits the pump + /// drains for at most the hub send timeout plus a small margin, so a pump task is + /// not left running past host shutdown (R21-D13). A wedged send is abandoned after + /// the bound; it is already time-bounded per send. + /// + /// A task that completes when the pumps drained or the bound elapsed. + public async ValueTask DisposeAsync() + { + HubPump[] pumps = Shutdown(); + if (pumps.Length == 0) + { + return; + } + + try + { + await Task.WhenAll(pumps.Select(pump => pump.Completion)) + .WaitAsync(_hubSendTimeout + TimeSpan.FromSeconds(1)) + .ConfigureAwait(false); + } + catch (TimeoutException) + { + _logger.LogDebug("Dashboard event hub pumps did not drain within the shutdown bound."); + } + } + + private HubPump[] Shutdown() { HubPump[] pumps; lock (_syncRoot) { if (_disposed) { - return; + return []; } _disposed = true; @@ -196,6 +225,8 @@ public sealed class DashboardEventBroadcaster : IDashboardEventBroadcaster, IDas { pump.Complete(); } + + return pumps; } /// diff --git a/src/ZB.MOM.WW.MxGateway.Server/Dashboard/Hubs/DashboardSnapshotHub.cs b/src/ZB.MOM.WW.MxGateway.Server/Dashboard/Hubs/DashboardSnapshotHub.cs index aa70d7ec..0824eabd 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Dashboard/Hubs/DashboardSnapshotHub.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Dashboard/Hubs/DashboardSnapshotHub.cs @@ -43,6 +43,19 @@ public sealed class DashboardSnapshotHub( // Count the viewer before seeding it, so the publisher resumes its tick // no later than the first snapshot this connection renders. connectionCounter.Increment(); + + // The cached health status only advances on a publisher tick, and the publisher is + // suspended while nobody watches; refresh it so the seed is not from before the idle + // period (R21-D12). A failed refresh falls back to the cached value. + try + { + await snapshotService.RefreshGatewayStatusAsync(Context.ConnectionAborted).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + // Connection already gone; the seed below fails or is moot. + } + DashboardSnapshot snapshot = snapshotService.GetSnapshot(); await Clients.Caller.SendAsync(ConfigurationMessage, snapshot.Configuration).ConfigureAwait(false); await Clients.Caller.SendAsync(SnapshotMessage, snapshot).ConfigureAwait(false); diff --git a/src/ZB.MOM.WW.MxGateway.Server/Dashboard/IDashboardSnapshotService.cs b/src/ZB.MOM.WW.MxGateway.Server/Dashboard/IDashboardSnapshotService.cs index d72adfae..02ca7d23 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Dashboard/IDashboardSnapshotService.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Dashboard/IDashboardSnapshotService.cs @@ -17,4 +17,14 @@ public interface IDashboardSnapshotService /// Token to cancel the asynchronous operation. /// An asynchronous stream of dashboard snapshots. IAsyncEnumerable WatchSnapshotsAsync(CancellationToken cancellationToken); + + /// + /// Re-evaluates the cached gateway health status now. The status otherwise refreshes + /// only on a watch tick, which is suspended while no dashboard viewer is connected, + /// so a first viewer would be seeded with a status from before the idle period. + /// The default does nothing for snapshot sources that have no cached status. + /// + /// Token to cancel the refresh. + /// A task that represents the asynchronous operation. + Task RefreshGatewayStatusAsync(CancellationToken cancellationToken) => Task.CompletedTask; } diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Dashboard/DashboardEventBroadcasterTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Dashboard/DashboardEventBroadcasterTests.cs index d4b36884..6b1195f1 100644 --- a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Dashboard/DashboardEventBroadcasterTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Dashboard/DashboardEventBroadcasterTests.cs @@ -71,6 +71,29 @@ public sealed class DashboardEventBroadcasterTests Assert.Equal(1, hubContext.SendCount); } + /// DisposeAsync awaits the hub pumps so queued sends finish before it returns (R21-D13). + /// A task that represents the asynchronous operation. + [Fact] + public async Task DisposeAsync_AwaitsPumpDrain() + { + CapturingHubContext hubContext = new(); + EventsHubViewerRegistry viewers = new(); + viewers.AddViewer("conn-1", "session-1"); + DashboardEventBroadcaster broadcaster = Create(hubContext, showTagValues: false, viewers); + + for (int i = 0; i < 20; i++) + { + broadcaster.Publish("session-1", BuildEventWithValue()); + } + + await broadcaster.DisposeAsync(); + + // Every queued event was drained before DisposeAsync returned: nothing is left + // in flight on a pump task past shutdown. + Assert.Equal(20, hubContext.SendCount); + Assert.Equal(0, broadcaster.ActiveHubPumpCount); + } + /// /// A hub viewer that cannot keep up costs a bounded queue, not an unbounded pile of /// pending sends: while one send is stuck, further events drop oldest-first at the diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Dashboard/DashboardSnapshotServiceTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Dashboard/DashboardSnapshotServiceTests.cs index 81b33570..acaf569d 100644 --- a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Dashboard/DashboardSnapshotServiceTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Dashboard/DashboardSnapshotServiceTests.cs @@ -68,6 +68,23 @@ public sealed class DashboardSnapshotServiceTests Assert.Equal(1, health.CheckCount); } + /// RefreshGatewayStatusAsync updates the cached status without a watch tick (R21-D12). + /// A task that represents the asynchronous operation. + [Fact] + public async Task RefreshGatewayStatusAsync_UpdatesCachedStatusWithoutWatching() + { + using GatewayMetrics metrics = new(); + StubHealthCheckService health = new(HealthStatus.Degraded); + DashboardSnapshotService service = CreateService(new SessionRegistry(), metrics, healthCheckService: health); + + Assert.NotEqual(HealthStatus.Degraded.ToString(), service.GetSnapshot().GatewayStatus); + + await service.RefreshGatewayStatusAsync(CancellationToken.None); + + Assert.Equal(HealthStatus.Degraded.ToString(), service.GetSnapshot().GatewayStatus); + Assert.Equal(1, health.CheckCount); + } + /// A health evaluation that throws is reported as Unknown, never as Healthy. /// A task that represents the asynchronous operation. [Fact]