Preserve Fleet and Hosts readings through refresh failures

This commit is contained in:
Joseph Doherty
2026-11-13 23:30:42 -05:00
parent b4f7672d2a
commit 7c888d1dca
12 changed files with 848 additions and 54 deletions
@@ -12,6 +12,8 @@
@using ZB.MOM.WW.OtOpcUa.Configuration
@using ZB.MOM.WW.OtOpcUa.Configuration.Entities
@using ZB.MOM.WW.OtOpcUa.Configuration.Enums
@using Microsoft.Extensions.Logging
@inject ILogger<Fleet> Logger
@inject IClusterRoleInfo Cluster
@inject IDbContextFactory<OtOpcUaConfigDbContext> DbFactory
@implements IAsyncDisposable
@@ -23,18 +25,25 @@
</div>
<div class="d-flex flex-wrap align-items-center mb-3 gap-2">
<button class="btn btn-sm btn-outline-primary" @onclick="RefreshAsync" disabled="@_refreshing">
<button class="btn btn-sm btn-outline-primary" @onclick="RefreshAsync" aria-disabled="@_refreshing.ToString().ToLowerInvariant()" aria-busy="@_refreshing.ToString().ToLowerInvariant()">
@if (_refreshing) { <span class="spinner-border spinner-border-sm me-1" /> }
Refresh
</button>
<span class="text-muted small">
Auto-refresh every @RefreshIntervalSeconds s. Last updated: @(_lastRefreshUtc?.ToString("HH:mm:ss 'UTC'") ?? "—")
Auto-refresh every @RefreshIntervalSeconds s. Last successful reading:
@if (_lastRefreshUtc is { } fetched) { <time datetime="@fetched.ToString("O")">@fetched.ToString("yyyy-MM-dd HH:mm:ss 'UTC'")</time> } else { <span>Not yet available</span> }
</span>
<span class="small" aria-live="polite" aria-atomic="true">@_manualFeedback</span>
</div>
@if (_refreshFailedAtUtc is { } failed)
{
<p class="panel notice" role="status">Fleet status is unavailable since @failed.ToString("yyyy-MM-dd HH:mm:ss 'UTC'"). @(_lastRefreshUtc is null ? "No successful reading is available yet." : "Showing the last successful reading; it may be stale.") Use Refresh to retry.</p>
}
@if (_rows is null)
{
<p>Loading…</p>
@if (_refreshFailedAtUtc is null) { <p>Loading…</p> }
}
else if (_rows.Count == 0)
{
@@ -116,7 +125,9 @@ else
private List<NodeRow>? _rows;
private bool _refreshing;
private string? _manualFeedback;
private DateTime? _lastRefreshUtc;
private DateTime? _refreshFailedAtUtc;
private Timer? _timer;
// Overlap guard: a tick whose DB query outruns the refresh interval must not stack behind the
@@ -134,16 +145,25 @@ else
protected override async Task OnInitializedAsync()
{
await LoadAsync();
if (_disposed) return;
_timer = new Timer(_ => _ = InvokeAsync(LoadAsync), null,
TimeSpan.FromSeconds(RefreshIntervalSeconds),
TimeSpan.FromSeconds(RefreshIntervalSeconds));
}
private async Task RefreshAsync() => await LoadAsync();
private async Task RefreshAsync()
{
if (_disposed || Volatile.Read(ref _tickRunning) == 1) return;
_manualFeedback = "Refreshing fleet status…";
await LoadAsync();
if (_disposed) return;
_manualFeedback = _refreshFailedAtUtc is null ? "Fleet status refreshed." : null;
StateHasChanged();
}
private async Task LoadAsync()
{
if (Interlocked.Exchange(ref _tickRunning, 1) == 1) return;
if (_disposed || Interlocked.Exchange(ref _tickRunning, 1) == 1) return;
try
{
_refreshing = true;
@@ -155,6 +175,7 @@ else
.GroupBy(s => s.NodeId)
.Select(g => g.OrderByDescending(s => s.StartedAtUtc).First())
.ToListAsync(_cts.Token);
if (_disposed) return;
var byNode = states.ToDictionary(s => s.NodeId);
// Union with current Akka driver members so a freshly-joined node that has no
@@ -176,12 +197,21 @@ else
FailureReason: state?.FailureReason);
}).ToList();
_lastRefreshUtc = DateTime.UtcNow;
_refreshFailedAtUtc = null;
}
catch (Exception ex) when (ex is OperationCanceledException or ObjectDisposedException)
{
// Circuit disposed while a tick was in flight — a cancelled tick is a clean exit, not a
// fault (the discarded timer task would otherwise swallow it silently).
}
catch (Exception ex)
{
if (!_disposed)
{
Logger.LogWarning(ex, "/fleet: failed to refresh deployment status; retaining last successful reading.");
_refreshFailedAtUtc ??= DateTime.UtcNow;
}
}
finally
{
// Release first: a StateHasChanged on a torn-down circuit must never leave the guard set.
@@ -30,19 +30,31 @@
<span class="spacer"></span>
</div>
<div class="d-flex align-items-center mb-3 gap-2">
<button class="btn btn-sm btn-outline-primary" @onclick="RefreshAsync" disabled="@_refreshing">
<div class="d-flex flex-wrap align-items-center mb-3 gap-2">
<button class="btn btn-sm btn-outline-primary" @onclick="RefreshAsync" aria-disabled="@_refreshing.ToString().ToLowerInvariant()" aria-busy="@_refreshing.ToString().ToLowerInvariant()">
@if (_refreshing) { <span class="spinner-border spinner-border-sm me-1" /> }
Refresh
</button>
<span class="text-muted small">
Auto-refresh every @RefreshIntervalSeconds s. Last updated: @(_lastRefreshUtc?.ToString("HH:mm:ss 'UTC'") ?? "—")
Auto-refresh every @RefreshIntervalSeconds s. Membership read:
@if (_lastRefreshUtc is { } membership) { <time datetime="@membership.ToString("O")">@membership.ToString("yyyy-MM-dd HH:mm:ss 'UTC'")</time> } else { <span>Not yet available</span> }
<span class="d-block">Configuration read: @if (_lastConfigUtc is { } config) { <time data-testid="config-fetched" datetime="@config.ToString("O")">@config.ToString("yyyy-MM-dd HH:mm:ss 'UTC'")</time> } else { <span>Not yet available</span> }</span>
</span>
<span class="small" aria-live="polite" aria-atomic="true">@_manualFeedback</span>
</div>
@if (_configFailedAtUtc is { } configFailed)
{
<p class="panel notice" role="status">Host configuration is unavailable since @configFailed.ToString("yyyy-MM-dd HH:mm:ss 'UTC'"). @(_lastConfigUtc is null ? "No successful configuration reading is available yet; names and configured endpoints are unknown." : "Showing the last successful configuration; it may be stale.") Live health snapshots are independent. Use Refresh to retry.</p>
}
@if (_membershipFailedAtUtc is { } membershipFailed)
{
<p class="panel notice" role="status">Cluster membership refresh has failed since @membershipFailed.ToString("yyyy-MM-dd HH:mm:ss 'UTC'"). @(_lastRefreshUtc is null ? "Membership is unavailable." : "Showing the last successful membership reading; it may be stale.") Use Refresh to retry.</p>
}
@if (_rows is null)
{
<p>Loading…</p>
@if (_membershipFailedAtUtc is null) { <p>Loading…</p> }
}
else if (_rows.Count == 0)
{
@@ -351,7 +363,11 @@ else
private List<MemberRow>? _rows;
private bool _refreshing;
private string? _manualFeedback;
private DateTime? _lastRefreshUtc;
private DateTime? _lastConfigUtc;
private DateTime? _configFailedAtUtc;
private DateTime? _membershipFailedAtUtc;
private Timer? _timer;
private IReadOnlyList<HostsClusterGroup>? _driverGroups;
@@ -380,9 +396,9 @@ else
protected override async Task OnInitializedAsync()
{
Refresh();
await LoadConfigAsync();
RebuildDriverGroups();
_driversDirty.Mark();
await TickAsync();
if (_disposed) return;
DriverStore.SnapshotChanged += OnSnapshotChanged;
_timer = new Timer(_ => _ = InvokeAsync(TickAsync), null,
TimeSpan.FromSeconds(RefreshIntervalSeconds),
@@ -391,11 +407,12 @@ else
private async Task TickAsync()
{
if (Interlocked.Exchange(ref _tickRunning, 1) == 1) return;
if (_disposed || Interlocked.Exchange(ref _tickRunning, 1) == 1) return;
try
{
Refresh();
var configChanged = await LoadConfigAsync();
if (_disposed) return;
// Re-project the driver rows only when a health snapshot or the config actually changed
// since the last tick (R04 P4). The snapshot handler just marks the flag, so N drivers
// publishing every 5 s cost one rebuild per tick per tab, not N rebuilds + N full renders.
@@ -415,14 +432,16 @@ else
private async Task RefreshAsync()
{
if (Interlocked.Exchange(ref _tickRunning, 1) == 1) return;
if (_disposed || Interlocked.Exchange(ref _tickRunning, 1) == 1) return;
try
{
_refreshing = true;
_manualFeedback = "Refreshing host readings…";
StateHasChanged();
await Task.Yield();
Refresh();
await LoadConfigAsync();
if (_disposed) return;
// A manual refresh always rebuilds; clear the flag so the next tick does not rebuild again.
_driversDirty.TryConsume();
RebuildDriverGroups();
@@ -436,6 +455,7 @@ else
// Release first: a StateHasChanged on a torn-down circuit must never leave the guard set.
Volatile.Write(ref _tickRunning, 0);
_refreshing = false;
_manualFeedback = _configFailedAtUtc is null && _membershipFailedAtUtc is null ? "Host readings refreshed." : null;
// This one sits OUTSIDE the try, so it has no catch to fall into — and a render
// request against a disposed renderer throws. Skip it once the page is gone; there
// is nothing left to render to.
@@ -444,8 +464,8 @@ else
}
// Load the per-cluster node + driver-instance config from the ConfigDB. Kept cheap and
// swallow-on-failure so a transient DB hiccup dims the enrichment (rows fall back to id/—)
// without crashing the page. The live health feed (the snapshot store) is independent of this.
// retain-on-failure so a transient DB hiccup preserves the last successful enrichment
// and reports its age without crashing the page. The live health feed (the snapshot store) is independent of this.
// Returns whether the loaded config differs from the previous load (records compare by value).
private async Task<bool> LoadConfigAsync()
{
@@ -455,28 +475,30 @@ else
{
await using var db = await DbFactory.CreateDbContextAsync(_cts.Token);
// Ordered: the change test below compares with SequenceEqual (review r0 M9).
_nodes = await db.ClusterNodes.AsNoTracking()
var nodes = await db.ClusterNodes.AsNoTracking()
.OrderBy(n => n.ClusterId).ThenBy(n => n.NodeId)
.Select(n => new HostsNodeInfo(n.ClusterId, n.NodeId, n.Host, n.OpcUaPort))
.ToListAsync(_cts.Token);
_instances = await db.DriverInstances.AsNoTracking()
var instances = await db.DriverInstances.AsNoTracking()
.OrderBy(d => d.DriverInstanceId)
.Select(d => new HostsDriverInstanceInfo(d.DriverInstanceId, d.ClusterId, d.Name, d.DriverType, d.Enabled))
.ToListAsync(_cts.Token);
if (_disposed) return false;
// Evict snapshots of driver instances that are neither AUTHORED nor DEPLOYED, and stale. The
// stores have no other removal path, so without this a removed driver stays on this page as a
// frozen row until the admin node restarts. DriverInstances is the authored (draft) table: a
// driver deleted in /raw keeps running until the next deploy, and a quiet one (Faulted, idle,
// tagless) may not publish at all meanwhile, so the deployed set — not staleness — is what keeps
// it (review D-b r1, I1). Unknown deployed set ⇒ no eviction this sweep.
// Only a SUCCESSFUL load reaches here; the failure path below blanks _instances and must
// never sweep the stores.
// Only a successful configuration read reaches here; a failed read must never sweep stores.
var evicted = 0;
var deployed = await Deployed.LoadAsync(_cts.Token);
if (_disposed) return false;
if (deployed is not null)
{
var configured = _instances
var configured = instances
.Select(i => i.DriverInstanceId)
.ToHashSet(StringComparer.OrdinalIgnoreCase);
configured.UnionWith(deployed);
@@ -485,6 +507,11 @@ else
ResilienceStore.EvictUnconfiguredStale(configured, staleBefore);
}
if (_disposed) return false;
_nodes = nodes;
_instances = instances;
_lastConfigUtc = DateTime.UtcNow;
_configFailedAtUtc = null;
return evicted > 0
|| !_nodes.SequenceEqual(previousNodes)
|| !_instances.SequenceEqual(previousInstances);
@@ -493,19 +520,17 @@ else
{
// The circuit went away mid-load. Rethrow so the caller treats it as a clean exit — it
// must NOT fall into the swallow-and-clear below, which would blank the cached config.
// ObjectDisposedException belongs here for the same reason: teardown disposes _cts (and
// with it the DbContext's token registration) while a query may still be running, and
// that shape reaches this method as ODE rather than OCE. Sending it to the generic catch
// blanked _nodes/_instances on the way out — the exact swallow-and-clear this rethrow
// exists to prevent, and permanent, because a torn-down page never ticks again.
// ObjectDisposedException likewise belongs to the clean exit path during teardown.
throw;
}
catch (Exception ex)
{
Logger.LogWarning(ex, "/hosts: failed to load driver-instance config; rows show ids only.");
_nodes = new();
_instances = new();
return true;
if (!_disposed)
{
Logger.LogWarning(ex, "/hosts: failed to load driver-instance config; retaining last successful configuration.");
_configFailedAtUtc ??= DateTime.UtcNow;
}
return false;
}
}
@@ -542,31 +567,43 @@ else
private void Refresh()
{
var cluster = Akka.Cluster.Cluster.Get(ActorSystem);
var state = cluster.State;
var unreachable = state.Unreachable
.Select(m => m.Address.ToString()).ToHashSet();
var selfAddress = cluster.SelfAddress.ToString();
_rows = state.Members.Select(m =>
try
{
var address = m.Address.ToString();
var hostPort = $"{m.Address.Host ?? "?"}:{m.Address.Port ?? 0}";
var leaderRoles = m.Roles
.Where(role => cluster.State.RoleLeader(role)?.ToString() == address)
.OrderBy(s => s, StringComparer.OrdinalIgnoreCase)
.ToList();
return new MemberRow(
Address: hostPort,
Status: m.Status.ToString(),
Roles: m.Roles.OrderBy(s => s, StringComparer.OrdinalIgnoreCase).ToList(),
LeaderRoles: leaderRoles,
Unreachable: unreachable.Contains(address),
IsSelf: address == selfAddress);
})
.OrderBy(r => r.Address, StringComparer.OrdinalIgnoreCase)
.ToList();
_lastRefreshUtc = DateTime.UtcNow;
var cluster = Akka.Cluster.Cluster.Get(ActorSystem);
var state = cluster.State;
var unreachable = state.Unreachable
.Select(m => m.Address.ToString()).ToHashSet();
var selfAddress = cluster.SelfAddress.ToString();
_rows = state.Members.Select(m =>
{
var address = m.Address.ToString();
var hostPort = $"{m.Address.Host ?? "?"}:{m.Address.Port ?? 0}";
var leaderRoles = m.Roles
.Where(role => cluster.State.RoleLeader(role)?.ToString() == address)
.OrderBy(s => s, StringComparer.OrdinalIgnoreCase)
.ToList();
return new MemberRow(
Address: hostPort,
Status: m.Status.ToString(),
Roles: m.Roles.OrderBy(s => s, StringComparer.OrdinalIgnoreCase).ToList(),
LeaderRoles: leaderRoles,
Unreachable: unreachable.Contains(address),
IsSelf: address == selfAddress);
})
.OrderBy(r => r.Address, StringComparer.OrdinalIgnoreCase)
.ToList();
_lastRefreshUtc = DateTime.UtcNow;
_membershipFailedAtUtc = null;
}
catch (Exception ex) when (ex is not OperationCanceledException && ex is not ObjectDisposedException)
{
if (!_disposed)
{
Logger.LogWarning(ex, "/hosts: failed to refresh cluster membership; retaining last successful reading.");
_membershipFailedAtUtc ??= DateTime.UtcNow;
}
}
}
private static string StatusChipClass(string status, bool unreachable) => (status, unreachable) switch