Merge #786 fleet diagnostics: Faulted devices counts distinct devices across the pair, Faulted only; Stopped column

Claude-Session: https://claude.ai/code/session_01GT67AD99zQQqpLCVm3p7xk
This commit is contained in:
Joseph Doherty
2026-11-11 11:58:51 -05:00
6 changed files with 437 additions and 10 deletions
@@ -86,9 +86,17 @@ else
<div class="agg-label">Drivers</div>
<div class="agg-value numeric">@_snapshot.Drivers.Count</div>
</div>
<div class="agg-card alert">
<div class="agg-label">Faulted devices</div>
<div class="agg-value numeric">@_snapshot.Drivers.Sum(d => d.FaultedDevices)</div>
@* #786: distinct devices in Faulted across every node of the selected node's cluster (a device faulted on
both nodes of a redundant pair counts once) — Stopped is not a fault and is a per-node column below.
See FleetDiagnosticsAggregation for the identity key. *@
<div class="agg-card alert"
title="Distinct devices in Faulted state across the nodes of @(_clusterId ?? "this node's cluster") that answered (@_clusterAnswered of @_clusterNodeCount). Stopped devices are not counted; see the per-driver Stopped column.">
<div class="agg-label">Faulted devices@(_clusterId is { } cid ? $" (cluster {cid})" : "")</div>
<div class="agg-value numeric">@_clusterFaulted</div>
@if (_clusterNodeCount > 1)
{
<div class="small text-muted">@_clusterAnswered of @_clusterNodeCount nodes answered</div>
}
</div>
</section>
@@ -124,6 +132,7 @@ else
<th>State</th>
<th class="numeric">Connected devices</th>
<th class="numeric">Faulted devices</th>
<th class="numeric" title="Hosts the driver reports Stopped (e.g. taken off scan) — not counted as faulted">Stopped devices</th>
<th title="When the driver child last published a changed health snapshot (its spawn time until the first)">Last health publish</th>
</tr>
</thead>
@@ -144,6 +153,7 @@ else
<td><span class="chip @StateChipClass(d.State)">@d.State</span></td>
<td class="numeric">@d.ConnectedDevices</td>
<td class="numeric">@d.FaultedDevices</td>
<td class="numeric">@d.StoppedDevices</td>
<td>@d.LastChangeUtc.ToString("u")</td>
</tr>
}
@@ -165,6 +175,12 @@ else
private NodeDiagnosticsSnapshot? _snapshot;
private IReadOnlyDictionary<string, string> _displayNames = new Dictionary<string, string>();
private bool _loading;
// #786 cluster-wide faulted-device tile: the selected node's cluster, how many of its nodes answered, and the
// distinct faulted-device count across them.
private string? _clusterId;
private int _clusterNodeCount;
private int _clusterAnswered;
private int _clusterFaulted;
private DateTime? _fetchedUtc;
// Page-lifetime cancellation: a fetch in flight when the circuit goes away is abandoned.
@@ -222,6 +238,7 @@ else
// The client is fail-safe by contract: a node that does not answer yields an empty snapshot.
_snapshot = await Diagnostics.GetDiagnosticsAsync(NodeId.Parse(_selectedNode), _cts.Token);
_fetchedUtc = DateTime.UtcNow;
await LoadClusterFaultedAsync(_snapshot);
_displayNames = await LoadDisplayNamesAsync(_snapshot);
}
catch (Exception ex) when (ex is OperationCanceledException or ObjectDisposedException)
@@ -235,6 +252,48 @@ else
}
}
// #786: fetch the selected node's cluster peers too and count distinct faulted devices across all of them.
// The ConfigDb lookup is enrichment: on failure the tile falls back to the selected node alone.
private async Task LoadClusterFaultedAsync(NodeDiagnosticsSnapshot selected)
{
string? clusterId = null;
IReadOnlyList<string> nodeIds = new[] { selected.NodeId.Value };
try
{
await using var db = await DbFactory.CreateDbContextAsync(_cts.Token);
(clusterId, nodeIds) = await FleetDiagnosticsAggregation.LoadClusterNodeIdsAsync(db, selected.NodeId.Value, _cts.Token);
}
catch (Exception ex) when (ex is not OperationCanceledException and not ObjectDisposedException)
{
// ConfigDb hiccup — count this node only.
}
var snapshots = new List<NodeDiagnosticsSnapshot> { selected };
var peers = nodeIds.Where(n => !string.Equals(n, selected.NodeId.Value, StringComparison.OrdinalIgnoreCase)).ToList();
var peerSnapshots = await Task.WhenAll(peers.Select(FetchPeerAsync));
snapshots.AddRange(peerSnapshots.OfType<NodeDiagnosticsSnapshot>());
_clusterId = clusterId;
_clusterNodeCount = nodeIds.Count;
// The client degrades an unreachable node to an empty snapshot; one with drivers clearly answered. An empty
// answer is indistinguishable from "no answer", so it is counted as not answered — it contributes nothing either way.
_clusterAnswered = 1 + peerSnapshots.Count(p => p is { Drivers.Count: > 0 });
_clusterFaulted = FleetDiagnosticsAggregation.DistinctFaultedDevices(snapshots);
}
private async Task<NodeDiagnosticsSnapshot?> FetchPeerAsync(string nodeId)
{
try
{
return await Diagnostics.GetDiagnosticsAsync(NodeId.Parse(nodeId), _cts.Token);
}
catch (Exception ex) when (ex is not OperationCanceledException and not ObjectDisposedException)
{
// A malformed peer row (NodeId.Parse) — the peer just does not contribute.
return null;
}
}
// The authored name for a driver row, else its id (no ConfigDb row, or the name lookup failed).
private string DisplayNameOf(DriverInstanceDiagnostics d) =>
_displayNames.TryGetValue(d.Name, out var name) ? name : d.Name;