fix(adminui): fleet/hosts poll ticks no longer overlap; cancel on dispose

This commit is contained in:
Joseph Doherty
2026-08-16 04:03:31 -04:00
parent d6a4fbd648
commit 4b6a93bb78
2 changed files with 81 additions and 26 deletions
@@ -106,6 +106,14 @@ else
private DateTime? _lastRefreshUtc; private DateTime? _lastRefreshUtc;
private Timer? _timer; private Timer? _timer;
// Overlap guard: a tick whose DB query outruns the refresh interval must not stack behind the
// next one. A manual Refresh landing on top of an in-flight tick is a no-op by the same guard.
private int _tickRunning;
// Page-lifetime cancellation: cancelled in Dispose() so a tick in flight when the circuit goes
// away aborts its query instead of running it to completion against a component that's gone.
private readonly CancellationTokenSource _cts = new();
protected override async Task OnInitializedAsync() protected override async Task OnInitializedAsync()
{ {
await LoadAsync(); await LoadAsync();
@@ -118,17 +126,18 @@ else
private async Task LoadAsync() private async Task LoadAsync()
{ {
_refreshing = true; if (Interlocked.Exchange(ref _tickRunning, 1) == 1) return;
StateHasChanged();
try try
{ {
await using var db = await DbFactory.CreateDbContextAsync(); _refreshing = true;
StateHasChanged();
await using var db = await DbFactory.CreateDbContextAsync(_cts.Token);
// Project the most-recent NodeDeploymentState per node — that's the row the // Project the most-recent NodeDeploymentState per node — that's the row the
// DriverHostActor most recently touched, regardless of which deployment it was for. // DriverHostActor most recently touched, regardless of which deployment it was for.
var states = await db.NodeDeploymentStates.AsNoTracking() var states = await db.NodeDeploymentStates.AsNoTracking()
.GroupBy(s => s.NodeId) .GroupBy(s => s.NodeId)
.Select(g => g.OrderByDescending(s => s.StartedAtUtc).First()) .Select(g => g.OrderByDescending(s => s.StartedAtUtc).First())
.ToListAsync(); .ToListAsync(_cts.Token);
var byNode = states.ToDictionary(s => s.NodeId); var byNode = states.ToDictionary(s => s.NodeId);
// Union with current Akka driver members so a freshly-joined node that has no // Union with current Akka driver members so a freshly-joined node that has no
@@ -151,8 +160,15 @@ else
}).ToList(); }).ToList();
_lastRefreshUtc = DateTime.UtcNow; _lastRefreshUtc = DateTime.UtcNow;
} }
catch (Exception ex) when (ex is OperationCanceledException or ObjectDisposedException)
{
// Circuit disposed while a tick was in flight — a cancelled tick is a clean exit, not a
// fault (the discarded timer task would otherwise swallow it silently).
}
finally finally
{ {
// Release first: a StateHasChanged on a torn-down circuit must never leave the guard set.
Volatile.Write(ref _tickRunning, 0);
_refreshing = false; _refreshing = false;
StateHasChanged(); StateHasChanged();
} }
@@ -168,7 +184,14 @@ else
private static string StatusLabel(NodeDeploymentStatus? status) => status?.ToString() ?? "waiting"; private static string StatusLabel(NodeDeploymentStatus? status) => status?.ToString() ?? "waiting";
public void Dispose() => _timer?.Dispose(); public void Dispose()
{
// Cancel BEFORE disposing the timer so a tick already in flight aborts its query rather than
// running to completion; the CTS is disposed last, once nothing new can observe the token.
_cts.Cancel();
_timer?.Dispose();
_cts.Dispose();
}
private sealed record NodeRow( private sealed record NodeRow(
string NodeId, string NodeId,
@@ -329,44 +329,66 @@ else
// HostsDriverRow instance and would otherwise close a dialog an operator just opened. // HostsDriverRow instance and would otherwise close a dialog an operator just opened.
private string? _controlsDriverId; private string? _controlsDriverId;
// Overlap guard: a tick whose config load outruns the 5 s interval must not stack behind the
// next one. A manual Refresh landing on top of an in-flight tick is a no-op by the same guard.
private int _tickRunning;
// Page-lifetime cancellation: cancelled in DisposeAsync() so a tick in flight when the circuit
// goes away aborts its queries instead of running them against a component that's gone.
private readonly CancellationTokenSource _cts = new();
protected override async Task OnInitializedAsync() protected override async Task OnInitializedAsync()
{ {
Refresh(); Refresh();
await LoadConfigAsync(); await LoadConfigAsync();
RebuildDriverGroups(); RebuildDriverGroups();
DriverStore.SnapshotChanged += OnSnapshotChanged; DriverStore.SnapshotChanged += OnSnapshotChanged;
_timer = new Timer(_ => _ = InvokeAsync(async () => _timer = new Timer(_ => _ = InvokeAsync(TickAsync), null,
{
try
{
Refresh();
await LoadConfigAsync();
RebuildDriverGroups();
StateHasChanged();
}
catch (Exception ex) when (ex is ObjectDisposedException or OperationCanceledException)
{
// Circuit disposed while a tick was in flight — ignore (the discarded task would
// otherwise swallow this silently). Mirrors DriverStatusPanel's drain-on-dispose.
}
}), null,
TimeSpan.FromSeconds(RefreshIntervalSeconds), TimeSpan.FromSeconds(RefreshIntervalSeconds),
TimeSpan.FromSeconds(RefreshIntervalSeconds)); TimeSpan.FromSeconds(RefreshIntervalSeconds));
} }
private async Task RefreshAsync() private async Task TickAsync()
{ {
_refreshing = true; if (Interlocked.Exchange(ref _tickRunning, 1) == 1) return;
StateHasChanged();
try try
{ {
Refresh();
await LoadConfigAsync();
RebuildDriverGroups();
StateHasChanged();
}
catch (Exception ex) when (ex is ObjectDisposedException or OperationCanceledException)
{
// Circuit disposed while a tick was in flight — ignore (the discarded task would
// otherwise swallow this silently). Mirrors DriverStatusPanel's drain-on-dispose.
}
finally
{
Volatile.Write(ref _tickRunning, 0);
}
}
private async Task RefreshAsync()
{
if (Interlocked.Exchange(ref _tickRunning, 1) == 1) return;
try
{
_refreshing = true;
StateHasChanged();
await Task.Yield(); await Task.Yield();
Refresh(); Refresh();
await LoadConfigAsync(); await LoadConfigAsync();
RebuildDriverGroups(); RebuildDriverGroups();
} }
catch (Exception ex) when (ex is ObjectDisposedException or OperationCanceledException)
{
// Same clean-exit tolerance as the timer path.
}
finally finally
{ {
// Release first: a StateHasChanged on a torn-down circuit must never leave the guard set.
Volatile.Write(ref _tickRunning, 0);
_refreshing = false; _refreshing = false;
StateHasChanged(); StateHasChanged();
} }
@@ -379,13 +401,19 @@ else
{ {
try try
{ {
await using var db = await DbFactory.CreateDbContextAsync(); await using var db = await DbFactory.CreateDbContextAsync(_cts.Token);
_nodes = await db.ClusterNodes.AsNoTracking() _nodes = await db.ClusterNodes.AsNoTracking()
.Select(n => new HostsNodeInfo(n.ClusterId, n.NodeId, n.Host, n.OpcUaPort)) .Select(n => new HostsNodeInfo(n.ClusterId, n.NodeId, n.Host, n.OpcUaPort))
.ToListAsync(); .ToListAsync(_cts.Token);
_instances = await db.DriverInstances.AsNoTracking() _instances = await db.DriverInstances.AsNoTracking()
.Select(d => new HostsDriverInstanceInfo(d.DriverInstanceId, d.ClusterId, d.Name, d.DriverType, d.Enabled)) .Select(d => new HostsDriverInstanceInfo(d.DriverInstanceId, d.ClusterId, d.Name, d.DriverType, d.Enabled))
.ToListAsync(); .ToListAsync(_cts.Token);
}
catch (OperationCanceledException)
{
// The circuit went away mid-load. Rethrow so the caller treats it as a clean exit — it
// must NOT fall into the swallow-and-clear below, which would blank the cached config.
throw;
} }
catch (Exception ex) catch (Exception ex)
{ {
@@ -487,9 +515,13 @@ else
{ {
// Unsubscribe first so the singleton store can't invoke a handler on a disposed component. // Unsubscribe first so the singleton store can't invoke a handler on a disposed component.
DriverStore.SnapshotChanged -= OnSnapshotChanged; DriverStore.SnapshotChanged -= OnSnapshotChanged;
// Cancel before draining so a tick already in flight aborts its queries rather than running
// them to completion against a component that's gone.
_cts.Cancel();
// Drain the timer so an in-flight callback can't touch a component that's already gone // Drain the timer so an in-flight callback can't touch a component that's already gone
// (System.Threading.Timer's async dispose awaits any in-flight callback — .NET 6+). // (System.Threading.Timer's async dispose awaits any in-flight callback — .NET 6+).
if (_timer is not null) await _timer.DisposeAsync(); if (_timer is not null) await _timer.DisposeAsync();
_cts.Dispose();
} }
private sealed record MemberRow( private sealed record MemberRow(