Preserve Fleet and Hosts readings through refresh failures
This commit is contained in:
@@ -12,6 +12,8 @@
|
||||
@using ZB.MOM.WW.OtOpcUa.Configuration
|
||||
@using ZB.MOM.WW.OtOpcUa.Configuration.Entities
|
||||
@using ZB.MOM.WW.OtOpcUa.Configuration.Enums
|
||||
@using Microsoft.Extensions.Logging
|
||||
@inject ILogger<Fleet> Logger
|
||||
@inject IClusterRoleInfo Cluster
|
||||
@inject IDbContextFactory<OtOpcUaConfigDbContext> DbFactory
|
||||
@implements IAsyncDisposable
|
||||
@@ -23,18 +25,25 @@
|
||||
</div>
|
||||
|
||||
<div class="d-flex flex-wrap align-items-center mb-3 gap-2">
|
||||
<button class="btn btn-sm btn-outline-primary" @onclick="RefreshAsync" disabled="@_refreshing">
|
||||
<button class="btn btn-sm btn-outline-primary" @onclick="RefreshAsync" aria-disabled="@_refreshing.ToString().ToLowerInvariant()" aria-busy="@_refreshing.ToString().ToLowerInvariant()">
|
||||
@if (_refreshing) { <span class="spinner-border spinner-border-sm me-1" /> }
|
||||
Refresh
|
||||
</button>
|
||||
<span class="text-muted small">
|
||||
Auto-refresh every @RefreshIntervalSeconds s. Last updated: @(_lastRefreshUtc?.ToString("HH:mm:ss 'UTC'") ?? "—")
|
||||
Auto-refresh every @RefreshIntervalSeconds s. Last successful reading:
|
||||
@if (_lastRefreshUtc is { } fetched) { <time datetime="@fetched.ToString("O")">@fetched.ToString("yyyy-MM-dd HH:mm:ss 'UTC'")</time> } else { <span>Not yet available</span> }
|
||||
</span>
|
||||
<span class="small" aria-live="polite" aria-atomic="true">@_manualFeedback</span>
|
||||
</div>
|
||||
|
||||
@if (_refreshFailedAtUtc is { } failed)
|
||||
{
|
||||
<p class="panel notice" role="status">Fleet status is unavailable since @failed.ToString("yyyy-MM-dd HH:mm:ss 'UTC'"). @(_lastRefreshUtc is null ? "No successful reading is available yet." : "Showing the last successful reading; it may be stale.") Use Refresh to retry.</p>
|
||||
}
|
||||
|
||||
@if (_rows is null)
|
||||
{
|
||||
<p>Loading…</p>
|
||||
@if (_refreshFailedAtUtc is null) { <p>Loading…</p> }
|
||||
}
|
||||
else if (_rows.Count == 0)
|
||||
{
|
||||
@@ -116,7 +125,9 @@ else
|
||||
|
||||
private List<NodeRow>? _rows;
|
||||
private bool _refreshing;
|
||||
private string? _manualFeedback;
|
||||
private DateTime? _lastRefreshUtc;
|
||||
private DateTime? _refreshFailedAtUtc;
|
||||
private Timer? _timer;
|
||||
|
||||
// Overlap guard: a tick whose DB query outruns the refresh interval must not stack behind the
|
||||
@@ -134,16 +145,25 @@ else
|
||||
protected override async Task OnInitializedAsync()
|
||||
{
|
||||
await LoadAsync();
|
||||
if (_disposed) return;
|
||||
_timer = new Timer(_ => _ = InvokeAsync(LoadAsync), null,
|
||||
TimeSpan.FromSeconds(RefreshIntervalSeconds),
|
||||
TimeSpan.FromSeconds(RefreshIntervalSeconds));
|
||||
}
|
||||
|
||||
private async Task RefreshAsync() => await LoadAsync();
|
||||
private async Task RefreshAsync()
|
||||
{
|
||||
if (_disposed || Volatile.Read(ref _tickRunning) == 1) return;
|
||||
_manualFeedback = "Refreshing fleet status…";
|
||||
await LoadAsync();
|
||||
if (_disposed) return;
|
||||
_manualFeedback = _refreshFailedAtUtc is null ? "Fleet status refreshed." : null;
|
||||
StateHasChanged();
|
||||
}
|
||||
|
||||
private async Task LoadAsync()
|
||||
{
|
||||
if (Interlocked.Exchange(ref _tickRunning, 1) == 1) return;
|
||||
if (_disposed || Interlocked.Exchange(ref _tickRunning, 1) == 1) return;
|
||||
try
|
||||
{
|
||||
_refreshing = true;
|
||||
@@ -155,6 +175,7 @@ else
|
||||
.GroupBy(s => s.NodeId)
|
||||
.Select(g => g.OrderByDescending(s => s.StartedAtUtc).First())
|
||||
.ToListAsync(_cts.Token);
|
||||
if (_disposed) return;
|
||||
var byNode = states.ToDictionary(s => s.NodeId);
|
||||
|
||||
// Union with current Akka driver members so a freshly-joined node that has no
|
||||
@@ -176,12 +197,21 @@ else
|
||||
FailureReason: state?.FailureReason);
|
||||
}).ToList();
|
||||
_lastRefreshUtc = DateTime.UtcNow;
|
||||
_refreshFailedAtUtc = null;
|
||||
}
|
||||
catch (Exception ex) when (ex is OperationCanceledException or ObjectDisposedException)
|
||||
{
|
||||
// Circuit disposed while a tick was in flight — a cancelled tick is a clean exit, not a
|
||||
// fault (the discarded timer task would otherwise swallow it silently).
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
if (!_disposed)
|
||||
{
|
||||
Logger.LogWarning(ex, "/fleet: failed to refresh deployment status; retaining last successful reading.");
|
||||
_refreshFailedAtUtc ??= DateTime.UtcNow;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
// Release first: a StateHasChanged on a torn-down circuit must never leave the guard set.
|
||||
|
||||
@@ -30,19 +30,31 @@
|
||||
<span class="spacer"></span>
|
||||
</div>
|
||||
|
||||
<div class="d-flex align-items-center mb-3 gap-2">
|
||||
<button class="btn btn-sm btn-outline-primary" @onclick="RefreshAsync" disabled="@_refreshing">
|
||||
<div class="d-flex flex-wrap align-items-center mb-3 gap-2">
|
||||
<button class="btn btn-sm btn-outline-primary" @onclick="RefreshAsync" aria-disabled="@_refreshing.ToString().ToLowerInvariant()" aria-busy="@_refreshing.ToString().ToLowerInvariant()">
|
||||
@if (_refreshing) { <span class="spinner-border spinner-border-sm me-1" /> }
|
||||
Refresh
|
||||
</button>
|
||||
<span class="text-muted small">
|
||||
Auto-refresh every @RefreshIntervalSeconds s. Last updated: @(_lastRefreshUtc?.ToString("HH:mm:ss 'UTC'") ?? "—")
|
||||
Auto-refresh every @RefreshIntervalSeconds s. Membership read:
|
||||
@if (_lastRefreshUtc is { } membership) { <time datetime="@membership.ToString("O")">@membership.ToString("yyyy-MM-dd HH:mm:ss 'UTC'")</time> } else { <span>Not yet available</span> }
|
||||
<span class="d-block">Configuration read: @if (_lastConfigUtc is { } config) { <time data-testid="config-fetched" datetime="@config.ToString("O")">@config.ToString("yyyy-MM-dd HH:mm:ss 'UTC'")</time> } else { <span>Not yet available</span> }</span>
|
||||
</span>
|
||||
<span class="small" aria-live="polite" aria-atomic="true">@_manualFeedback</span>
|
||||
</div>
|
||||
|
||||
@if (_configFailedAtUtc is { } configFailed)
|
||||
{
|
||||
<p class="panel notice" role="status">Host configuration is unavailable since @configFailed.ToString("yyyy-MM-dd HH:mm:ss 'UTC'"). @(_lastConfigUtc is null ? "No successful configuration reading is available yet; names and configured endpoints are unknown." : "Showing the last successful configuration; it may be stale.") Live health snapshots are independent. Use Refresh to retry.</p>
|
||||
}
|
||||
@if (_membershipFailedAtUtc is { } membershipFailed)
|
||||
{
|
||||
<p class="panel notice" role="status">Cluster membership refresh has failed since @membershipFailed.ToString("yyyy-MM-dd HH:mm:ss 'UTC'"). @(_lastRefreshUtc is null ? "Membership is unavailable." : "Showing the last successful membership reading; it may be stale.") Use Refresh to retry.</p>
|
||||
}
|
||||
|
||||
@if (_rows is null)
|
||||
{
|
||||
<p>Loading…</p>
|
||||
@if (_membershipFailedAtUtc is null) { <p>Loading…</p> }
|
||||
}
|
||||
else if (_rows.Count == 0)
|
||||
{
|
||||
@@ -351,7 +363,11 @@ else
|
||||
|
||||
private List<MemberRow>? _rows;
|
||||
private bool _refreshing;
|
||||
private string? _manualFeedback;
|
||||
private DateTime? _lastRefreshUtc;
|
||||
private DateTime? _lastConfigUtc;
|
||||
private DateTime? _configFailedAtUtc;
|
||||
private DateTime? _membershipFailedAtUtc;
|
||||
private Timer? _timer;
|
||||
|
||||
private IReadOnlyList<HostsClusterGroup>? _driverGroups;
|
||||
@@ -380,9 +396,9 @@ else
|
||||
|
||||
protected override async Task OnInitializedAsync()
|
||||
{
|
||||
Refresh();
|
||||
await LoadConfigAsync();
|
||||
RebuildDriverGroups();
|
||||
_driversDirty.Mark();
|
||||
await TickAsync();
|
||||
if (_disposed) return;
|
||||
DriverStore.SnapshotChanged += OnSnapshotChanged;
|
||||
_timer = new Timer(_ => _ = InvokeAsync(TickAsync), null,
|
||||
TimeSpan.FromSeconds(RefreshIntervalSeconds),
|
||||
@@ -391,11 +407,12 @@ else
|
||||
|
||||
private async Task TickAsync()
|
||||
{
|
||||
if (Interlocked.Exchange(ref _tickRunning, 1) == 1) return;
|
||||
if (_disposed || Interlocked.Exchange(ref _tickRunning, 1) == 1) return;
|
||||
try
|
||||
{
|
||||
Refresh();
|
||||
var configChanged = await LoadConfigAsync();
|
||||
if (_disposed) return;
|
||||
// Re-project the driver rows only when a health snapshot or the config actually changed
|
||||
// since the last tick (R04 P4). The snapshot handler just marks the flag, so N drivers
|
||||
// publishing every 5 s cost one rebuild per tick per tab, not N rebuilds + N full renders.
|
||||
@@ -415,14 +432,16 @@ else
|
||||
|
||||
private async Task RefreshAsync()
|
||||
{
|
||||
if (Interlocked.Exchange(ref _tickRunning, 1) == 1) return;
|
||||
if (_disposed || Interlocked.Exchange(ref _tickRunning, 1) == 1) return;
|
||||
try
|
||||
{
|
||||
_refreshing = true;
|
||||
_manualFeedback = "Refreshing host readings…";
|
||||
StateHasChanged();
|
||||
await Task.Yield();
|
||||
Refresh();
|
||||
await LoadConfigAsync();
|
||||
if (_disposed) return;
|
||||
// A manual refresh always rebuilds; clear the flag so the next tick does not rebuild again.
|
||||
_driversDirty.TryConsume();
|
||||
RebuildDriverGroups();
|
||||
@@ -436,6 +455,7 @@ else
|
||||
// Release first: a StateHasChanged on a torn-down circuit must never leave the guard set.
|
||||
Volatile.Write(ref _tickRunning, 0);
|
||||
_refreshing = false;
|
||||
_manualFeedback = _configFailedAtUtc is null && _membershipFailedAtUtc is null ? "Host readings refreshed." : null;
|
||||
// This one sits OUTSIDE the try, so it has no catch to fall into — and a render
|
||||
// request against a disposed renderer throws. Skip it once the page is gone; there
|
||||
// is nothing left to render to.
|
||||
@@ -444,8 +464,8 @@ else
|
||||
}
|
||||
|
||||
// Load the per-cluster node + driver-instance config from the ConfigDB. Kept cheap and
|
||||
// swallow-on-failure so a transient DB hiccup dims the enrichment (rows fall back to id/—)
|
||||
// without crashing the page. The live health feed (the snapshot store) is independent of this.
|
||||
// retain-on-failure so a transient DB hiccup preserves the last successful enrichment
|
||||
// and reports its age without crashing the page. The live health feed (the snapshot store) is independent of this.
|
||||
// Returns whether the loaded config differs from the previous load (records compare by value).
|
||||
private async Task<bool> LoadConfigAsync()
|
||||
{
|
||||
@@ -455,28 +475,30 @@ else
|
||||
{
|
||||
await using var db = await DbFactory.CreateDbContextAsync(_cts.Token);
|
||||
// Ordered: the change test below compares with SequenceEqual (review r0 M9).
|
||||
_nodes = await db.ClusterNodes.AsNoTracking()
|
||||
var nodes = await db.ClusterNodes.AsNoTracking()
|
||||
.OrderBy(n => n.ClusterId).ThenBy(n => n.NodeId)
|
||||
.Select(n => new HostsNodeInfo(n.ClusterId, n.NodeId, n.Host, n.OpcUaPort))
|
||||
.ToListAsync(_cts.Token);
|
||||
_instances = await db.DriverInstances.AsNoTracking()
|
||||
var instances = await db.DriverInstances.AsNoTracking()
|
||||
.OrderBy(d => d.DriverInstanceId)
|
||||
.Select(d => new HostsDriverInstanceInfo(d.DriverInstanceId, d.ClusterId, d.Name, d.DriverType, d.Enabled))
|
||||
.ToListAsync(_cts.Token);
|
||||
|
||||
if (_disposed) return false;
|
||||
|
||||
// Evict snapshots of driver instances that are neither AUTHORED nor DEPLOYED, and stale. The
|
||||
// stores have no other removal path, so without this a removed driver stays on this page as a
|
||||
// frozen row until the admin node restarts. DriverInstances is the authored (draft) table: a
|
||||
// driver deleted in /raw keeps running until the next deploy, and a quiet one (Faulted, idle,
|
||||
// tagless) may not publish at all meanwhile, so the deployed set — not staleness — is what keeps
|
||||
// it (review D-b r1, I1). Unknown deployed set ⇒ no eviction this sweep.
|
||||
// Only a SUCCESSFUL load reaches here; the failure path below blanks _instances and must
|
||||
// never sweep the stores.
|
||||
// Only a successful configuration read reaches here; a failed read must never sweep stores.
|
||||
var evicted = 0;
|
||||
var deployed = await Deployed.LoadAsync(_cts.Token);
|
||||
if (_disposed) return false;
|
||||
if (deployed is not null)
|
||||
{
|
||||
var configured = _instances
|
||||
var configured = instances
|
||||
.Select(i => i.DriverInstanceId)
|
||||
.ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
configured.UnionWith(deployed);
|
||||
@@ -485,6 +507,11 @@ else
|
||||
ResilienceStore.EvictUnconfiguredStale(configured, staleBefore);
|
||||
}
|
||||
|
||||
if (_disposed) return false;
|
||||
_nodes = nodes;
|
||||
_instances = instances;
|
||||
_lastConfigUtc = DateTime.UtcNow;
|
||||
_configFailedAtUtc = null;
|
||||
return evicted > 0
|
||||
|| !_nodes.SequenceEqual(previousNodes)
|
||||
|| !_instances.SequenceEqual(previousInstances);
|
||||
@@ -493,19 +520,17 @@ else
|
||||
{
|
||||
// The circuit went away mid-load. Rethrow so the caller treats it as a clean exit — it
|
||||
// must NOT fall into the swallow-and-clear below, which would blank the cached config.
|
||||
// ObjectDisposedException belongs here for the same reason: teardown disposes _cts (and
|
||||
// with it the DbContext's token registration) while a query may still be running, and
|
||||
// that shape reaches this method as ODE rather than OCE. Sending it to the generic catch
|
||||
// blanked _nodes/_instances on the way out — the exact swallow-and-clear this rethrow
|
||||
// exists to prevent, and permanent, because a torn-down page never ticks again.
|
||||
// ObjectDisposedException likewise belongs to the clean exit path during teardown.
|
||||
throw;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Logger.LogWarning(ex, "/hosts: failed to load driver-instance config; rows show ids only.");
|
||||
_nodes = new();
|
||||
_instances = new();
|
||||
return true;
|
||||
if (!_disposed)
|
||||
{
|
||||
Logger.LogWarning(ex, "/hosts: failed to load driver-instance config; retaining last successful configuration.");
|
||||
_configFailedAtUtc ??= DateTime.UtcNow;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -542,31 +567,43 @@ else
|
||||
|
||||
private void Refresh()
|
||||
{
|
||||
var cluster = Akka.Cluster.Cluster.Get(ActorSystem);
|
||||
var state = cluster.State;
|
||||
var unreachable = state.Unreachable
|
||||
.Select(m => m.Address.ToString()).ToHashSet();
|
||||
var selfAddress = cluster.SelfAddress.ToString();
|
||||
|
||||
_rows = state.Members.Select(m =>
|
||||
try
|
||||
{
|
||||
var address = m.Address.ToString();
|
||||
var hostPort = $"{m.Address.Host ?? "?"}:{m.Address.Port ?? 0}";
|
||||
var leaderRoles = m.Roles
|
||||
.Where(role => cluster.State.RoleLeader(role)?.ToString() == address)
|
||||
.OrderBy(s => s, StringComparer.OrdinalIgnoreCase)
|
||||
.ToList();
|
||||
return new MemberRow(
|
||||
Address: hostPort,
|
||||
Status: m.Status.ToString(),
|
||||
Roles: m.Roles.OrderBy(s => s, StringComparer.OrdinalIgnoreCase).ToList(),
|
||||
LeaderRoles: leaderRoles,
|
||||
Unreachable: unreachable.Contains(address),
|
||||
IsSelf: address == selfAddress);
|
||||
})
|
||||
.OrderBy(r => r.Address, StringComparer.OrdinalIgnoreCase)
|
||||
.ToList();
|
||||
_lastRefreshUtc = DateTime.UtcNow;
|
||||
var cluster = Akka.Cluster.Cluster.Get(ActorSystem);
|
||||
var state = cluster.State;
|
||||
var unreachable = state.Unreachable
|
||||
.Select(m => m.Address.ToString()).ToHashSet();
|
||||
var selfAddress = cluster.SelfAddress.ToString();
|
||||
|
||||
_rows = state.Members.Select(m =>
|
||||
{
|
||||
var address = m.Address.ToString();
|
||||
var hostPort = $"{m.Address.Host ?? "?"}:{m.Address.Port ?? 0}";
|
||||
var leaderRoles = m.Roles
|
||||
.Where(role => cluster.State.RoleLeader(role)?.ToString() == address)
|
||||
.OrderBy(s => s, StringComparer.OrdinalIgnoreCase)
|
||||
.ToList();
|
||||
return new MemberRow(
|
||||
Address: hostPort,
|
||||
Status: m.Status.ToString(),
|
||||
Roles: m.Roles.OrderBy(s => s, StringComparer.OrdinalIgnoreCase).ToList(),
|
||||
LeaderRoles: leaderRoles,
|
||||
Unreachable: unreachable.Contains(address),
|
||||
IsSelf: address == selfAddress);
|
||||
})
|
||||
.OrderBy(r => r.Address, StringComparer.OrdinalIgnoreCase)
|
||||
.ToList();
|
||||
_lastRefreshUtc = DateTime.UtcNow;
|
||||
_membershipFailedAtUtc = null;
|
||||
}
|
||||
catch (Exception ex) when (ex is not OperationCanceledException && ex is not ObjectDisposedException)
|
||||
{
|
||||
if (!_disposed)
|
||||
{
|
||||
Logger.LogWarning(ex, "/hosts: failed to refresh cluster membership; retaining last successful reading.");
|
||||
_membershipFailedAtUtc ??= DateTime.UtcNow;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static string StatusChipClass(string status, bool unreachable) => (status, unreachable) switch
|
||||
|
||||
Reference in New Issue
Block a user