fix(dashboard): close three low hub/page races (mxaccessgw#174)
D11: BrowsePage search CTS is cancelled+disposed exactly once under a lock. D12: the hub refreshes the cached health status before seeding a new connection. D13: DashboardEventBroadcaster gains DisposeAsync that awaits the pumps (bounded). Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
@@ -239,6 +239,7 @@
|
||||
private bool _searchPending;
|
||||
private string? _searchError;
|
||||
private CancellationTokenSource? _searchCts;
|
||||
private readonly object _searchLock = new();
|
||||
private Task? _searchTask;
|
||||
private readonly List<string> _subscribed = [];
|
||||
private readonly Dictionary<string, DashboardTagValue> _values = new(StringComparer.Ordinal);
|
||||
@@ -367,8 +368,7 @@
|
||||
|
||||
private void ScheduleSearch(string query)
|
||||
{
|
||||
CancelQuietly(_searchCts);
|
||||
_searchCts = null;
|
||||
ReleaseSearchSource(null);
|
||||
_searchError = null;
|
||||
|
||||
if (query.Trim().Length == 0)
|
||||
@@ -378,10 +378,15 @@
|
||||
return;
|
||||
}
|
||||
|
||||
// Linked to the page token, so disposal cancels an in-flight search too. The
|
||||
// search task owns the source and disposes it when it finishes.
|
||||
// Linked to the page token, so disposal cancels an in-flight search too. Whoever
|
||||
// clears _searchCts (a newer keystroke or the finishing task) cancels and disposes
|
||||
// the source, under _searchLock, so it is disposed exactly once.
|
||||
CancellationTokenSource searchCts = CancellationTokenSource.CreateLinkedTokenSource(_cts.Token);
|
||||
_searchCts = searchCts;
|
||||
lock (_searchLock)
|
||||
{
|
||||
_searchCts = searchCts;
|
||||
}
|
||||
|
||||
_searchPending = true;
|
||||
_searchTask = RunSearchAsync(query, searchCts);
|
||||
}
|
||||
@@ -427,25 +432,30 @@
|
||||
}
|
||||
finally
|
||||
{
|
||||
searchCts.Dispose();
|
||||
// Only releases when this search is still the current one; a newer keystroke
|
||||
// has already cancelled and disposed it otherwise.
|
||||
ReleaseSearchSource(searchCts);
|
||||
}
|
||||
}
|
||||
|
||||
// The search task disposes its own source, so a cancel can race that disposal.
|
||||
private static void CancelQuietly(CancellationTokenSource? source)
|
||||
// Swap-and-dispose under a lock (R21-D11): the field is cleared and the source is
|
||||
// cancelled and disposed in one critical section, so a keystroke cancel and the
|
||||
// finishing task can never both touch a disposed source. A task already running
|
||||
// keeps its captured token; a disposed source's token still reports cancellation.
|
||||
// When 'only' is non-null the release applies only if that source is still current.
|
||||
private void ReleaseSearchSource(CancellationTokenSource? only)
|
||||
{
|
||||
if (source is null)
|
||||
lock (_searchLock)
|
||||
{
|
||||
return;
|
||||
}
|
||||
CancellationTokenSource? current = _searchCts;
|
||||
if (current is null || (only is not null && !ReferenceEquals(current, only)))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
source.Cancel();
|
||||
}
|
||||
catch (ObjectDisposedException)
|
||||
{
|
||||
// The search already finished; there is nothing left to cancel.
|
||||
_searchCts = null;
|
||||
current.Cancel();
|
||||
current.Dispose();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -227,7 +227,7 @@ public sealed class DashboardSnapshotService : IDashboardSnapshotService
|
||||
/// </summary>
|
||||
/// <param name="cancellationToken">Token to cancel the evaluation.</param>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
private async Task RefreshGatewayStatusAsync(CancellationToken cancellationToken)
|
||||
public async Task RefreshGatewayStatusAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
if (_healthCheckService is null)
|
||||
{
|
||||
|
||||
@@ -38,7 +38,7 @@ namespace ZB.MOM.WW.MxGateway.Server.Dashboard.Hubs;
|
||||
/// <see cref="EventsHubViewerRegistry.HubViewersReleased"/> reports the last one gone.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public sealed class DashboardEventBroadcaster : IDashboardEventBroadcaster, IDashboardSessionEventSubscriber, IDisposable
|
||||
public sealed class DashboardEventBroadcaster : IDashboardEventBroadcaster, IDashboardSessionEventSubscriber, IDisposable, IAsyncDisposable
|
||||
{
|
||||
/// <summary>
|
||||
/// Queue depth per in-process subscriber. The consumer is a Blazor page
|
||||
@@ -173,17 +173,46 @@ public sealed class DashboardEventBroadcaster : IDashboardEventBroadcaster, IDas
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Stops every hub pump and detaches from the viewer registry. The singleton is
|
||||
/// disposed with the host; a pump's in-flight send is bounded by its timeout.
|
||||
/// Stops every hub pump and detaches from the viewer registry without waiting for
|
||||
/// the pumps to drain. The host prefers <see cref="DisposeAsync"/>.
|
||||
/// </summary>
|
||||
public void Dispose()
|
||||
public void Dispose() => _ = Shutdown();
|
||||
|
||||
/// <summary>
|
||||
/// Stops every hub pump, detaches from the viewer registry, and awaits the pump
|
||||
/// drains for at most the hub send timeout plus a small margin, so a pump task is
|
||||
/// not left running past host shutdown (R21-D13). A wedged send is abandoned after
|
||||
/// the bound; it is already time-bounded per send.
|
||||
/// </summary>
|
||||
/// <returns>A task that completes when the pumps drained or the bound elapsed.</returns>
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
HubPump[] pumps = Shutdown();
|
||||
if (pumps.Length == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
await Task.WhenAll(pumps.Select(pump => pump.Completion))
|
||||
.WaitAsync(_hubSendTimeout + TimeSpan.FromSeconds(1))
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
catch (TimeoutException)
|
||||
{
|
||||
_logger.LogDebug("Dashboard event hub pumps did not drain within the shutdown bound.");
|
||||
}
|
||||
}
|
||||
|
||||
private HubPump[] Shutdown()
|
||||
{
|
||||
HubPump[] pumps;
|
||||
lock (_syncRoot)
|
||||
{
|
||||
if (_disposed)
|
||||
{
|
||||
return;
|
||||
return [];
|
||||
}
|
||||
|
||||
_disposed = true;
|
||||
@@ -196,6 +225,8 @@ public sealed class DashboardEventBroadcaster : IDashboardEventBroadcaster, IDas
|
||||
{
|
||||
pump.Complete();
|
||||
}
|
||||
|
||||
return pumps;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
||||
@@ -43,6 +43,19 @@ public sealed class DashboardSnapshotHub(
|
||||
// Count the viewer before seeding it, so the publisher resumes its tick
|
||||
// no later than the first snapshot this connection renders.
|
||||
connectionCounter.Increment();
|
||||
|
||||
// The cached health status only advances on a publisher tick, and the publisher is
|
||||
// suspended while nobody watches; refresh it so the seed is not from before the idle
|
||||
// period (R21-D12). A failed refresh falls back to the cached value.
|
||||
try
|
||||
{
|
||||
await snapshotService.RefreshGatewayStatusAsync(Context.ConnectionAborted).ConfigureAwait(false);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
// Connection already gone; the seed below fails or is moot.
|
||||
}
|
||||
|
||||
DashboardSnapshot snapshot = snapshotService.GetSnapshot();
|
||||
await Clients.Caller.SendAsync(ConfigurationMessage, snapshot.Configuration).ConfigureAwait(false);
|
||||
await Clients.Caller.SendAsync(SnapshotMessage, snapshot).ConfigureAwait(false);
|
||||
|
||||
@@ -17,4 +17,14 @@ public interface IDashboardSnapshotService
|
||||
/// <param name="cancellationToken">Token to cancel the asynchronous operation.</param>
|
||||
/// <returns>An asynchronous stream of dashboard snapshots.</returns>
|
||||
IAsyncEnumerable<DashboardSnapshot> WatchSnapshotsAsync(CancellationToken cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// Re-evaluates the cached gateway health status now. The status otherwise refreshes
|
||||
/// only on a watch tick, which is suspended while no dashboard viewer is connected,
|
||||
/// so a first viewer would be seeded with a status from before the idle period.
|
||||
/// The default does nothing for snapshot sources that have no cached status.
|
||||
/// </summary>
|
||||
/// <param name="cancellationToken">Token to cancel the refresh.</param>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
Task RefreshGatewayStatusAsync(CancellationToken cancellationToken) => Task.CompletedTask;
|
||||
}
|
||||
|
||||
@@ -71,6 +71,29 @@ public sealed class DashboardEventBroadcasterTests
|
||||
Assert.Equal(1, hubContext.SendCount);
|
||||
}
|
||||
|
||||
/// <summary>DisposeAsync awaits the hub pumps so queued sends finish before it returns (R21-D13).</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task DisposeAsync_AwaitsPumpDrain()
|
||||
{
|
||||
CapturingHubContext hubContext = new();
|
||||
EventsHubViewerRegistry viewers = new();
|
||||
viewers.AddViewer("conn-1", "session-1");
|
||||
DashboardEventBroadcaster broadcaster = Create(hubContext, showTagValues: false, viewers);
|
||||
|
||||
for (int i = 0; i < 20; i++)
|
||||
{
|
||||
broadcaster.Publish("session-1", BuildEventWithValue());
|
||||
}
|
||||
|
||||
await broadcaster.DisposeAsync();
|
||||
|
||||
// Every queued event was drained before DisposeAsync returned: nothing is left
|
||||
// in flight on a pump task past shutdown.
|
||||
Assert.Equal(20, hubContext.SendCount);
|
||||
Assert.Equal(0, broadcaster.ActiveHubPumpCount);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A hub viewer that cannot keep up costs a bounded queue, not an unbounded pile of
|
||||
/// pending sends: while one send is stuck, further events drop oldest-first at the
|
||||
|
||||
@@ -68,6 +68,23 @@ public sealed class DashboardSnapshotServiceTests
|
||||
Assert.Equal(1, health.CheckCount);
|
||||
}
|
||||
|
||||
/// <summary>RefreshGatewayStatusAsync updates the cached status without a watch tick (R21-D12).</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task RefreshGatewayStatusAsync_UpdatesCachedStatusWithoutWatching()
|
||||
{
|
||||
using GatewayMetrics metrics = new();
|
||||
StubHealthCheckService health = new(HealthStatus.Degraded);
|
||||
DashboardSnapshotService service = CreateService(new SessionRegistry(), metrics, healthCheckService: health);
|
||||
|
||||
Assert.NotEqual(HealthStatus.Degraded.ToString(), service.GetSnapshot().GatewayStatus);
|
||||
|
||||
await service.RefreshGatewayStatusAsync(CancellationToken.None);
|
||||
|
||||
Assert.Equal(HealthStatus.Degraded.ToString(), service.GetSnapshot().GatewayStatus);
|
||||
Assert.Equal(1, health.CheckCount);
|
||||
}
|
||||
|
||||
/// <summary>A health evaluation that throws is reported as Unknown, never as Healthy.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
|
||||
Reference in New Issue
Block a user