fix(dashboard): close three low hub/page races (mxaccessgw#174)

D11: BrowsePage search CTS is cancelled+disposed exactly once under a lock.
D12: the hub refreshes the cached health status before seeding a new connection.
D13: DashboardEventBroadcaster gains DisposeAsync that awaits the pumps (bounded).

Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
Joseph Doherty
2026-11-09 12:30:04 -05:00
parent f3a452810e
commit e69cd0321e
7 changed files with 128 additions and 24 deletions
@@ -239,6 +239,7 @@
private bool _searchPending; private bool _searchPending;
private string? _searchError; private string? _searchError;
private CancellationTokenSource? _searchCts; private CancellationTokenSource? _searchCts;
private readonly object _searchLock = new();
private Task? _searchTask; private Task? _searchTask;
private readonly List<string> _subscribed = []; private readonly List<string> _subscribed = [];
private readonly Dictionary<string, DashboardTagValue> _values = new(StringComparer.Ordinal); private readonly Dictionary<string, DashboardTagValue> _values = new(StringComparer.Ordinal);
@@ -367,8 +368,7 @@
private void ScheduleSearch(string query) private void ScheduleSearch(string query)
{ {
CancelQuietly(_searchCts); ReleaseSearchSource(null);
_searchCts = null;
_searchError = null; _searchError = null;
if (query.Trim().Length == 0) if (query.Trim().Length == 0)
@@ -378,10 +378,15 @@
return; return;
} }
// Linked to the page token, so disposal cancels an in-flight search too. The // Linked to the page token, so disposal cancels an in-flight search too. Whoever
// search task owns the source and disposes it when it finishes. // clears _searchCts (a newer keystroke or the finishing task) cancels and disposes
// the source, under _searchLock, so it is disposed exactly once.
CancellationTokenSource searchCts = CancellationTokenSource.CreateLinkedTokenSource(_cts.Token); CancellationTokenSource searchCts = CancellationTokenSource.CreateLinkedTokenSource(_cts.Token);
_searchCts = searchCts; lock (_searchLock)
{
_searchCts = searchCts;
}
_searchPending = true; _searchPending = true;
_searchTask = RunSearchAsync(query, searchCts); _searchTask = RunSearchAsync(query, searchCts);
} }
@@ -427,25 +432,30 @@
} }
finally finally
{ {
searchCts.Dispose(); // Only releases when this search is still the current one; a newer keystroke
// has already cancelled and disposed it otherwise.
ReleaseSearchSource(searchCts);
} }
} }
// The search task disposes its own source, so a cancel can race that disposal. // Swap-and-dispose under a lock (R21-D11): the field is cleared and the source is
private static void CancelQuietly(CancellationTokenSource? source) // cancelled and disposed in one critical section, so a keystroke cancel and the
// finishing task can never both touch a disposed source. A task already running
// keeps its captured token; a disposed source's token still reports cancellation.
// When 'only' is non-null the release applies only if that source is still current.
private void ReleaseSearchSource(CancellationTokenSource? only)
{ {
if (source is null) lock (_searchLock)
{ {
return; CancellationTokenSource? current = _searchCts;
} if (current is null || (only is not null && !ReferenceEquals(current, only)))
{
return;
}
try _searchCts = null;
{ current.Cancel();
source.Cancel(); current.Dispose();
}
catch (ObjectDisposedException)
{
// The search already finished; there is nothing left to cancel.
} }
} }
@@ -227,7 +227,7 @@ public sealed class DashboardSnapshotService : IDashboardSnapshotService
/// </summary> /// </summary>
/// <param name="cancellationToken">Token to cancel the evaluation.</param> /// <param name="cancellationToken">Token to cancel the evaluation.</param>
/// <returns>A task that represents the asynchronous operation.</returns> /// <returns>A task that represents the asynchronous operation.</returns>
private async Task RefreshGatewayStatusAsync(CancellationToken cancellationToken) public async Task RefreshGatewayStatusAsync(CancellationToken cancellationToken)
{ {
if (_healthCheckService is null) if (_healthCheckService is null)
{ {
@@ -38,7 +38,7 @@ namespace ZB.MOM.WW.MxGateway.Server.Dashboard.Hubs;
/// <see cref="EventsHubViewerRegistry.HubViewersReleased"/> reports the last one gone. /// <see cref="EventsHubViewerRegistry.HubViewersReleased"/> reports the last one gone.
/// </para> /// </para>
/// </remarks> /// </remarks>
public sealed class DashboardEventBroadcaster : IDashboardEventBroadcaster, IDashboardSessionEventSubscriber, IDisposable public sealed class DashboardEventBroadcaster : IDashboardEventBroadcaster, IDashboardSessionEventSubscriber, IDisposable, IAsyncDisposable
{ {
/// <summary> /// <summary>
/// Queue depth per in-process subscriber. The consumer is a Blazor page /// Queue depth per in-process subscriber. The consumer is a Blazor page
@@ -173,17 +173,46 @@ public sealed class DashboardEventBroadcaster : IDashboardEventBroadcaster, IDas
} }
/// <summary> /// <summary>
/// Stops every hub pump and detaches from the viewer registry. The singleton is /// Stops every hub pump and detaches from the viewer registry without waiting for
/// disposed with the host; a pump's in-flight send is bounded by its timeout. /// the pumps to drain. The host prefers <see cref="DisposeAsync"/>.
/// </summary> /// </summary>
public void Dispose() public void Dispose() => _ = Shutdown();
/// <summary>
/// Stops every hub pump, detaches from the viewer registry, and awaits the pump
/// drains for at most the hub send timeout plus a small margin, so a pump task is
/// not left running past host shutdown (R21-D13). A wedged send is abandoned after
/// the bound; it is already time-bounded per send.
/// </summary>
/// <returns>A task that completes when the pumps drained or the bound elapsed.</returns>
public async ValueTask DisposeAsync()
{
HubPump[] pumps = Shutdown();
if (pumps.Length == 0)
{
return;
}
try
{
await Task.WhenAll(pumps.Select(pump => pump.Completion))
.WaitAsync(_hubSendTimeout + TimeSpan.FromSeconds(1))
.ConfigureAwait(false);
}
catch (TimeoutException)
{
_logger.LogDebug("Dashboard event hub pumps did not drain within the shutdown bound.");
}
}
private HubPump[] Shutdown()
{ {
HubPump[] pumps; HubPump[] pumps;
lock (_syncRoot) lock (_syncRoot)
{ {
if (_disposed) if (_disposed)
{ {
return; return [];
} }
_disposed = true; _disposed = true;
@@ -196,6 +225,8 @@ public sealed class DashboardEventBroadcaster : IDashboardEventBroadcaster, IDas
{ {
pump.Complete(); pump.Complete();
} }
return pumps;
} }
/// <summary> /// <summary>
@@ -43,6 +43,19 @@ public sealed class DashboardSnapshotHub(
// Count the viewer before seeding it, so the publisher resumes its tick // Count the viewer before seeding it, so the publisher resumes its tick
// no later than the first snapshot this connection renders. // no later than the first snapshot this connection renders.
connectionCounter.Increment(); connectionCounter.Increment();
// The cached health status only advances on a publisher tick, and the publisher is
// suspended while nobody watches; refresh it so the seed is not from before the idle
// period (R21-D12). A failed refresh falls back to the cached value.
try
{
await snapshotService.RefreshGatewayStatusAsync(Context.ConnectionAborted).ConfigureAwait(false);
}
catch (OperationCanceledException)
{
// Connection already gone; the seed below fails or is moot.
}
DashboardSnapshot snapshot = snapshotService.GetSnapshot(); DashboardSnapshot snapshot = snapshotService.GetSnapshot();
await Clients.Caller.SendAsync(ConfigurationMessage, snapshot.Configuration).ConfigureAwait(false); await Clients.Caller.SendAsync(ConfigurationMessage, snapshot.Configuration).ConfigureAwait(false);
await Clients.Caller.SendAsync(SnapshotMessage, snapshot).ConfigureAwait(false); await Clients.Caller.SendAsync(SnapshotMessage, snapshot).ConfigureAwait(false);
@@ -17,4 +17,14 @@ public interface IDashboardSnapshotService
/// <param name="cancellationToken">Token to cancel the asynchronous operation.</param> /// <param name="cancellationToken">Token to cancel the asynchronous operation.</param>
/// <returns>An asynchronous stream of dashboard snapshots.</returns> /// <returns>An asynchronous stream of dashboard snapshots.</returns>
IAsyncEnumerable<DashboardSnapshot> WatchSnapshotsAsync(CancellationToken cancellationToken); IAsyncEnumerable<DashboardSnapshot> WatchSnapshotsAsync(CancellationToken cancellationToken);
/// <summary>
/// Re-evaluates the cached gateway health status now. The status otherwise refreshes
/// only on a watch tick, which is suspended while no dashboard viewer is connected,
/// so a first viewer would be seeded with a status from before the idle period.
/// The default does nothing for snapshot sources that have no cached status.
/// </summary>
/// <param name="cancellationToken">Token to cancel the refresh.</param>
/// <returns>A task that represents the asynchronous operation.</returns>
Task RefreshGatewayStatusAsync(CancellationToken cancellationToken) => Task.CompletedTask;
} }
@@ -71,6 +71,29 @@ public sealed class DashboardEventBroadcasterTests
Assert.Equal(1, hubContext.SendCount); Assert.Equal(1, hubContext.SendCount);
} }
/// <summary>DisposeAsync awaits the hub pumps so queued sends finish before it returns (R21-D13).</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task DisposeAsync_AwaitsPumpDrain()
{
CapturingHubContext hubContext = new();
EventsHubViewerRegistry viewers = new();
viewers.AddViewer("conn-1", "session-1");
DashboardEventBroadcaster broadcaster = Create(hubContext, showTagValues: false, viewers);
for (int i = 0; i < 20; i++)
{
broadcaster.Publish("session-1", BuildEventWithValue());
}
await broadcaster.DisposeAsync();
// Every queued event was drained before DisposeAsync returned: nothing is left
// in flight on a pump task past shutdown.
Assert.Equal(20, hubContext.SendCount);
Assert.Equal(0, broadcaster.ActiveHubPumpCount);
}
/// <summary> /// <summary>
/// A hub viewer that cannot keep up costs a bounded queue, not an unbounded pile of /// A hub viewer that cannot keep up costs a bounded queue, not an unbounded pile of
/// pending sends: while one send is stuck, further events drop oldest-first at the /// pending sends: while one send is stuck, further events drop oldest-first at the
@@ -68,6 +68,23 @@ public sealed class DashboardSnapshotServiceTests
Assert.Equal(1, health.CheckCount); Assert.Equal(1, health.CheckCount);
} }
/// <summary>RefreshGatewayStatusAsync updates the cached status without a watch tick (R21-D12).</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task RefreshGatewayStatusAsync_UpdatesCachedStatusWithoutWatching()
{
using GatewayMetrics metrics = new();
StubHealthCheckService health = new(HealthStatus.Degraded);
DashboardSnapshotService service = CreateService(new SessionRegistry(), metrics, healthCheckService: health);
Assert.NotEqual(HealthStatus.Degraded.ToString(), service.GetSnapshot().GatewayStatus);
await service.RefreshGatewayStatusAsync(CancellationToken.None);
Assert.Equal(HealthStatus.Degraded.ToString(), service.GetSnapshot().GatewayStatus);
Assert.Equal(1, health.CheckCount);
}
/// <summary>A health evaluation that throws is reported as Unknown, never as Healthy.</summary> /// <summary>A health evaluation that throws is reported as Unknown, never as Healthy.</summary>
/// <returns>A task that represents the asynchronous operation.</returns> /// <returns>A task that represents the asynchronous operation.</returns>
[Fact] [Fact]