fix(dashboard): close three low hub/page races (mxaccessgw#174)
D11: BrowsePage search CTS is cancelled+disposed exactly once under a lock. D12: the hub refreshes the cached health status before seeding a new connection. D13: DashboardEventBroadcaster gains DisposeAsync that awaits the pumps (bounded). Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
@@ -239,6 +239,7 @@
|
|||||||
private bool _searchPending;
|
private bool _searchPending;
|
||||||
private string? _searchError;
|
private string? _searchError;
|
||||||
private CancellationTokenSource? _searchCts;
|
private CancellationTokenSource? _searchCts;
|
||||||
|
private readonly object _searchLock = new();
|
||||||
private Task? _searchTask;
|
private Task? _searchTask;
|
||||||
private readonly List<string> _subscribed = [];
|
private readonly List<string> _subscribed = [];
|
||||||
private readonly Dictionary<string, DashboardTagValue> _values = new(StringComparer.Ordinal);
|
private readonly Dictionary<string, DashboardTagValue> _values = new(StringComparer.Ordinal);
|
||||||
@@ -367,8 +368,7 @@
|
|||||||
|
|
||||||
private void ScheduleSearch(string query)
|
private void ScheduleSearch(string query)
|
||||||
{
|
{
|
||||||
CancelQuietly(_searchCts);
|
ReleaseSearchSource(null);
|
||||||
_searchCts = null;
|
|
||||||
_searchError = null;
|
_searchError = null;
|
||||||
|
|
||||||
if (query.Trim().Length == 0)
|
if (query.Trim().Length == 0)
|
||||||
@@ -378,10 +378,15 @@
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Linked to the page token, so disposal cancels an in-flight search too. The
|
// Linked to the page token, so disposal cancels an in-flight search too. Whoever
|
||||||
// search task owns the source and disposes it when it finishes.
|
// clears _searchCts (a newer keystroke or the finishing task) cancels and disposes
|
||||||
|
// the source, under _searchLock, so it is disposed exactly once.
|
||||||
CancellationTokenSource searchCts = CancellationTokenSource.CreateLinkedTokenSource(_cts.Token);
|
CancellationTokenSource searchCts = CancellationTokenSource.CreateLinkedTokenSource(_cts.Token);
|
||||||
_searchCts = searchCts;
|
lock (_searchLock)
|
||||||
|
{
|
||||||
|
_searchCts = searchCts;
|
||||||
|
}
|
||||||
|
|
||||||
_searchPending = true;
|
_searchPending = true;
|
||||||
_searchTask = RunSearchAsync(query, searchCts);
|
_searchTask = RunSearchAsync(query, searchCts);
|
||||||
}
|
}
|
||||||
@@ -427,25 +432,30 @@
|
|||||||
}
|
}
|
||||||
finally
|
finally
|
||||||
{
|
{
|
||||||
searchCts.Dispose();
|
// Only releases when this search is still the current one; a newer keystroke
|
||||||
|
// has already cancelled and disposed it otherwise.
|
||||||
|
ReleaseSearchSource(searchCts);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// The search task disposes its own source, so a cancel can race that disposal.
|
// Swap-and-dispose under a lock (R21-D11): the field is cleared and the source is
|
||||||
private static void CancelQuietly(CancellationTokenSource? source)
|
// cancelled and disposed in one critical section, so a keystroke cancel and the
|
||||||
|
// finishing task can never both touch a disposed source. A task already running
|
||||||
|
// keeps its captured token; a disposed source's token still reports cancellation.
|
||||||
|
// When 'only' is non-null the release applies only if that source is still current.
|
||||||
|
private void ReleaseSearchSource(CancellationTokenSource? only)
|
||||||
{
|
{
|
||||||
if (source is null)
|
lock (_searchLock)
|
||||||
{
|
{
|
||||||
return;
|
CancellationTokenSource? current = _searchCts;
|
||||||
}
|
if (current is null || (only is not null && !ReferenceEquals(current, only)))
|
||||||
|
{
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
try
|
_searchCts = null;
|
||||||
{
|
current.Cancel();
|
||||||
source.Cancel();
|
current.Dispose();
|
||||||
}
|
|
||||||
catch (ObjectDisposedException)
|
|
||||||
{
|
|
||||||
// The search already finished; there is nothing left to cancel.
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -227,7 +227,7 @@ public sealed class DashboardSnapshotService : IDashboardSnapshotService
|
|||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="cancellationToken">Token to cancel the evaluation.</param>
|
/// <param name="cancellationToken">Token to cancel the evaluation.</param>
|
||||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||||
private async Task RefreshGatewayStatusAsync(CancellationToken cancellationToken)
|
public async Task RefreshGatewayStatusAsync(CancellationToken cancellationToken)
|
||||||
{
|
{
|
||||||
if (_healthCheckService is null)
|
if (_healthCheckService is null)
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -38,7 +38,7 @@ namespace ZB.MOM.WW.MxGateway.Server.Dashboard.Hubs;
|
|||||||
/// <see cref="EventsHubViewerRegistry.HubViewersReleased"/> reports the last one gone.
|
/// <see cref="EventsHubViewerRegistry.HubViewersReleased"/> reports the last one gone.
|
||||||
/// </para>
|
/// </para>
|
||||||
/// </remarks>
|
/// </remarks>
|
||||||
public sealed class DashboardEventBroadcaster : IDashboardEventBroadcaster, IDashboardSessionEventSubscriber, IDisposable
|
public sealed class DashboardEventBroadcaster : IDashboardEventBroadcaster, IDashboardSessionEventSubscriber, IDisposable, IAsyncDisposable
|
||||||
{
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Queue depth per in-process subscriber. The consumer is a Blazor page
|
/// Queue depth per in-process subscriber. The consumer is a Blazor page
|
||||||
@@ -173,17 +173,46 @@ public sealed class DashboardEventBroadcaster : IDashboardEventBroadcaster, IDas
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Stops every hub pump and detaches from the viewer registry. The singleton is
|
/// Stops every hub pump and detaches from the viewer registry without waiting for
|
||||||
/// disposed with the host; a pump's in-flight send is bounded by its timeout.
|
/// the pumps to drain. The host prefers <see cref="DisposeAsync"/>.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public void Dispose()
|
public void Dispose() => _ = Shutdown();
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Stops every hub pump, detaches from the viewer registry, and awaits the pump
|
||||||
|
/// drains for at most the hub send timeout plus a small margin, so a pump task is
|
||||||
|
/// not left running past host shutdown (R21-D13). A wedged send is abandoned after
|
||||||
|
/// the bound; it is already time-bounded per send.
|
||||||
|
/// </summary>
|
||||||
|
/// <returns>A task that completes when the pumps drained or the bound elapsed.</returns>
|
||||||
|
public async ValueTask DisposeAsync()
|
||||||
|
{
|
||||||
|
HubPump[] pumps = Shutdown();
|
||||||
|
if (pumps.Length == 0)
|
||||||
|
{
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
await Task.WhenAll(pumps.Select(pump => pump.Completion))
|
||||||
|
.WaitAsync(_hubSendTimeout + TimeSpan.FromSeconds(1))
|
||||||
|
.ConfigureAwait(false);
|
||||||
|
}
|
||||||
|
catch (TimeoutException)
|
||||||
|
{
|
||||||
|
_logger.LogDebug("Dashboard event hub pumps did not drain within the shutdown bound.");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private HubPump[] Shutdown()
|
||||||
{
|
{
|
||||||
HubPump[] pumps;
|
HubPump[] pumps;
|
||||||
lock (_syncRoot)
|
lock (_syncRoot)
|
||||||
{
|
{
|
||||||
if (_disposed)
|
if (_disposed)
|
||||||
{
|
{
|
||||||
return;
|
return [];
|
||||||
}
|
}
|
||||||
|
|
||||||
_disposed = true;
|
_disposed = true;
|
||||||
@@ -196,6 +225,8 @@ public sealed class DashboardEventBroadcaster : IDashboardEventBroadcaster, IDas
|
|||||||
{
|
{
|
||||||
pump.Complete();
|
pump.Complete();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
return pumps;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
|
|||||||
@@ -43,6 +43,19 @@ public sealed class DashboardSnapshotHub(
|
|||||||
// Count the viewer before seeding it, so the publisher resumes its tick
|
// Count the viewer before seeding it, so the publisher resumes its tick
|
||||||
// no later than the first snapshot this connection renders.
|
// no later than the first snapshot this connection renders.
|
||||||
connectionCounter.Increment();
|
connectionCounter.Increment();
|
||||||
|
|
||||||
|
// The cached health status only advances on a publisher tick, and the publisher is
|
||||||
|
// suspended while nobody watches; refresh it so the seed is not from before the idle
|
||||||
|
// period (R21-D12). A failed refresh falls back to the cached value.
|
||||||
|
try
|
||||||
|
{
|
||||||
|
await snapshotService.RefreshGatewayStatusAsync(Context.ConnectionAborted).ConfigureAwait(false);
|
||||||
|
}
|
||||||
|
catch (OperationCanceledException)
|
||||||
|
{
|
||||||
|
// Connection already gone; the seed below fails or is moot.
|
||||||
|
}
|
||||||
|
|
||||||
DashboardSnapshot snapshot = snapshotService.GetSnapshot();
|
DashboardSnapshot snapshot = snapshotService.GetSnapshot();
|
||||||
await Clients.Caller.SendAsync(ConfigurationMessage, snapshot.Configuration).ConfigureAwait(false);
|
await Clients.Caller.SendAsync(ConfigurationMessage, snapshot.Configuration).ConfigureAwait(false);
|
||||||
await Clients.Caller.SendAsync(SnapshotMessage, snapshot).ConfigureAwait(false);
|
await Clients.Caller.SendAsync(SnapshotMessage, snapshot).ConfigureAwait(false);
|
||||||
|
|||||||
@@ -17,4 +17,14 @@ public interface IDashboardSnapshotService
|
|||||||
/// <param name="cancellationToken">Token to cancel the asynchronous operation.</param>
|
/// <param name="cancellationToken">Token to cancel the asynchronous operation.</param>
|
||||||
/// <returns>An asynchronous stream of dashboard snapshots.</returns>
|
/// <returns>An asynchronous stream of dashboard snapshots.</returns>
|
||||||
IAsyncEnumerable<DashboardSnapshot> WatchSnapshotsAsync(CancellationToken cancellationToken);
|
IAsyncEnumerable<DashboardSnapshot> WatchSnapshotsAsync(CancellationToken cancellationToken);
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Re-evaluates the cached gateway health status now. The status otherwise refreshes
|
||||||
|
/// only on a watch tick, which is suspended while no dashboard viewer is connected,
|
||||||
|
/// so a first viewer would be seeded with a status from before the idle period.
|
||||||
|
/// The default does nothing for snapshot sources that have no cached status.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="cancellationToken">Token to cancel the refresh.</param>
|
||||||
|
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||||
|
Task RefreshGatewayStatusAsync(CancellationToken cancellationToken) => Task.CompletedTask;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -71,6 +71,29 @@ public sealed class DashboardEventBroadcasterTests
|
|||||||
Assert.Equal(1, hubContext.SendCount);
|
Assert.Equal(1, hubContext.SendCount);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <summary>DisposeAsync awaits the hub pumps so queued sends finish before it returns (R21-D13).</summary>
|
||||||
|
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||||
|
[Fact]
|
||||||
|
public async Task DisposeAsync_AwaitsPumpDrain()
|
||||||
|
{
|
||||||
|
CapturingHubContext hubContext = new();
|
||||||
|
EventsHubViewerRegistry viewers = new();
|
||||||
|
viewers.AddViewer("conn-1", "session-1");
|
||||||
|
DashboardEventBroadcaster broadcaster = Create(hubContext, showTagValues: false, viewers);
|
||||||
|
|
||||||
|
for (int i = 0; i < 20; i++)
|
||||||
|
{
|
||||||
|
broadcaster.Publish("session-1", BuildEventWithValue());
|
||||||
|
}
|
||||||
|
|
||||||
|
await broadcaster.DisposeAsync();
|
||||||
|
|
||||||
|
// Every queued event was drained before DisposeAsync returned: nothing is left
|
||||||
|
// in flight on a pump task past shutdown.
|
||||||
|
Assert.Equal(20, hubContext.SendCount);
|
||||||
|
Assert.Equal(0, broadcaster.ActiveHubPumpCount);
|
||||||
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// A hub viewer that cannot keep up costs a bounded queue, not an unbounded pile of
|
/// A hub viewer that cannot keep up costs a bounded queue, not an unbounded pile of
|
||||||
/// pending sends: while one send is stuck, further events drop oldest-first at the
|
/// pending sends: while one send is stuck, further events drop oldest-first at the
|
||||||
|
|||||||
@@ -68,6 +68,23 @@ public sealed class DashboardSnapshotServiceTests
|
|||||||
Assert.Equal(1, health.CheckCount);
|
Assert.Equal(1, health.CheckCount);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <summary>RefreshGatewayStatusAsync updates the cached status without a watch tick (R21-D12).</summary>
|
||||||
|
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||||
|
[Fact]
|
||||||
|
public async Task RefreshGatewayStatusAsync_UpdatesCachedStatusWithoutWatching()
|
||||||
|
{
|
||||||
|
using GatewayMetrics metrics = new();
|
||||||
|
StubHealthCheckService health = new(HealthStatus.Degraded);
|
||||||
|
DashboardSnapshotService service = CreateService(new SessionRegistry(), metrics, healthCheckService: health);
|
||||||
|
|
||||||
|
Assert.NotEqual(HealthStatus.Degraded.ToString(), service.GetSnapshot().GatewayStatus);
|
||||||
|
|
||||||
|
await service.RefreshGatewayStatusAsync(CancellationToken.None);
|
||||||
|
|
||||||
|
Assert.Equal(HealthStatus.Degraded.ToString(), service.GetSnapshot().GatewayStatus);
|
||||||
|
Assert.Equal(1, health.CheckCount);
|
||||||
|
}
|
||||||
|
|
||||||
/// <summary>A health evaluation that throws is reported as Unknown, never as Healthy.</summary>
|
/// <summary>A health evaluation that throws is reported as Unknown, never as Healthy.</summary>
|
||||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||||
[Fact]
|
[Fact]
|
||||||
|
|||||||
Reference in New Issue
Block a user