fix(adminui): fleet/hosts poll ticks no longer overlap; cancel on dispose

2026-08-16 04:03:31 -04:00
parent d6a4fbd648
commit 4b6a93bb78
2 changed files with 81 additions and 26 deletions
@@ -106,6 +106,14 @@ else
    private DateTime? _lastRefreshUtc;
    private Timer? _timer;

+    // Overlap guard: a tick whose DB query outruns the refresh interval must not stack behind the
+    // next one. A manual Refresh landing on top of an in-flight tick is a no-op by the same guard.
+    private int _tickRunning;
+
+    // Page-lifetime cancellation: cancelled in Dispose() so a tick in flight when the circuit goes
+    // away aborts its query instead of running it to completion against a component that's gone.
+    private readonly CancellationTokenSource _cts = new();
+
    protected override async Task OnInitializedAsync()
    {
        await LoadAsync();
@@ -118,17 +126,18 @@ else

    private async Task LoadAsync()
    {
-        _refreshing = true;
-        StateHasChanged();
+        if (Interlocked.Exchange(ref _tickRunning, 1) == 1) return;
        try
        {
-            await using var db = await DbFactory.CreateDbContextAsync();
+            _refreshing = true;
+            StateHasChanged();
+            await using var db = await DbFactory.CreateDbContextAsync(_cts.Token);
            // Project the most-recent NodeDeploymentState per node — that's the row the
            // DriverHostActor most recently touched, regardless of which deployment it was for.
            var states = await db.NodeDeploymentStates.AsNoTracking()
                .GroupBy(s => s.NodeId)
                .Select(g => g.OrderByDescending(s => s.StartedAtUtc).First())
-                .ToListAsync();
+                .ToListAsync(_cts.Token);
            var byNode = states.ToDictionary(s => s.NodeId);

            // Union with current Akka driver members so a freshly-joined node that has no
@@ -151,8 +160,15 @@ else
            }).ToList();
            _lastRefreshUtc = DateTime.UtcNow;
        }
+        catch (Exception ex) when (ex is OperationCanceledException or ObjectDisposedException)
+        {
+            // Circuit disposed while a tick was in flight — a cancelled tick is a clean exit, not a
+            // fault (the discarded timer task would otherwise swallow it silently).
+        }
        finally
        {
+            // Release first: a StateHasChanged on a torn-down circuit must never leave the guard set.
+            Volatile.Write(ref _tickRunning, 0);
            _refreshing = false;
            StateHasChanged();
        }
@@ -168,7 +184,14 @@ else

    private static string StatusLabel(NodeDeploymentStatus? status) => status?.ToString() ?? "waiting";

-    public void Dispose() => _timer?.Dispose();
+    public void Dispose()
+    {
+        // Cancel BEFORE disposing the timer so a tick already in flight aborts its query rather than
+        // running to completion; the CTS is disposed last, once nothing new can observe the token.
+        _cts.Cancel();
+        _timer?.Dispose();
+        _cts.Dispose();
+    }

    private sealed record NodeRow(
        string NodeId,
@@ -329,44 +329,66 @@ else
    // HostsDriverRow instance and would otherwise close a dialog an operator just opened.
    private string? _controlsDriverId;

+    // Overlap guard: a tick whose config load outruns the 5 s interval must not stack behind the
+    // next one. A manual Refresh landing on top of an in-flight tick is a no-op by the same guard.
+    private int _tickRunning;
+
+    // Page-lifetime cancellation: cancelled in DisposeAsync() so a tick in flight when the circuit
+    // goes away aborts its queries instead of running them against a component that's gone.
+    private readonly CancellationTokenSource _cts = new();
+
    protected override async Task OnInitializedAsync()
    {
        Refresh();
        await LoadConfigAsync();
        RebuildDriverGroups();
        DriverStore.SnapshotChanged += OnSnapshotChanged;
-        _timer = new Timer(_ => _ = InvokeAsync(async () =>
-            {
-                try
-                {
-                    Refresh();
-                    await LoadConfigAsync();
-                    RebuildDriverGroups();
-                    StateHasChanged();
-                }
-                catch (Exception ex) when (ex is ObjectDisposedException or OperationCanceledException)
-                {
-                    // Circuit disposed while a tick was in flight — ignore (the discarded task would
-                    // otherwise swallow this silently). Mirrors DriverStatusPanel's drain-on-dispose.
-                }
-            }), null,
+        _timer = new Timer(_ => _ = InvokeAsync(TickAsync), null,
            TimeSpan.FromSeconds(RefreshIntervalSeconds),
            TimeSpan.FromSeconds(RefreshIntervalSeconds));
    }

-    private async Task RefreshAsync()
+    private async Task TickAsync()
    {
-        _refreshing = true;
-        StateHasChanged();
+        if (Interlocked.Exchange(ref _tickRunning, 1) == 1) return;
        try
        {
+            Refresh();
+            await LoadConfigAsync();
+            RebuildDriverGroups();
+            StateHasChanged();
+        }
+        catch (Exception ex) when (ex is ObjectDisposedException or OperationCanceledException)
+        {
+            // Circuit disposed while a tick was in flight — ignore (the discarded task would
+            // otherwise swallow this silently). Mirrors DriverStatusPanel's drain-on-dispose.
+        }
+        finally
+        {
+            Volatile.Write(ref _tickRunning, 0);
+        }
+    }
+
+    private async Task RefreshAsync()
+    {
+        if (Interlocked.Exchange(ref _tickRunning, 1) == 1) return;
+        try
+        {
+            _refreshing = true;
+            StateHasChanged();
            await Task.Yield();
            Refresh();
            await LoadConfigAsync();
            RebuildDriverGroups();
        }
+        catch (Exception ex) when (ex is ObjectDisposedException or OperationCanceledException)
+        {
+            // Same clean-exit tolerance as the timer path.
+        }
        finally
        {
+            // Release first: a StateHasChanged on a torn-down circuit must never leave the guard set.
+            Volatile.Write(ref _tickRunning, 0);
            _refreshing = false;
            StateHasChanged();
        }
@@ -379,13 +401,19 @@ else
    {
        try
        {
-            await using var db = await DbFactory.CreateDbContextAsync();
+            await using var db = await DbFactory.CreateDbContextAsync(_cts.Token);
            _nodes = await db.ClusterNodes.AsNoTracking()
                .Select(n => new HostsNodeInfo(n.ClusterId, n.NodeId, n.Host, n.OpcUaPort))
-                .ToListAsync();
+                .ToListAsync(_cts.Token);
            _instances = await db.DriverInstances.AsNoTracking()
                .Select(d => new HostsDriverInstanceInfo(d.DriverInstanceId, d.ClusterId, d.Name, d.DriverType, d.Enabled))
-                .ToListAsync();
+                .ToListAsync(_cts.Token);
+        }
+        catch (OperationCanceledException)
+        {
+            // The circuit went away mid-load. Rethrow so the caller treats it as a clean exit — it
+            // must NOT fall into the swallow-and-clear below, which would blank the cached config.
+            throw;
        }
        catch (Exception ex)
        {
@@ -487,9 +515,13 @@ else
    {
        // Unsubscribe first so the singleton store can't invoke a handler on a disposed component.
        DriverStore.SnapshotChanged -= OnSnapshotChanged;
+        // Cancel before draining so a tick already in flight aborts its queries rather than running
+        // them to completion against a component that's gone.
+        _cts.Cancel();
        // Drain the timer so an in-flight callback can't touch a component that's already gone
        // (System.Threading.Timer's async dispose awaits any in-flight callback — .NET 6+).
        if (_timer is not null) await _timer.DisposeAsync();
+        _cts.Dispose();
    }

    private sealed record MemberRow(