fix(adminui): Faulted-devices tile counts distinct Faulted devices across the cluster pair (#786)

Owner decision 2026-11-11: /fleet/diagnostics "Faulted devices" counts DISTINCT
devices across the nodes of the selected node's cluster, Faulted state only.

- DriverHostActor: FaultedDevices = HostState.Faulted only (Stopped no longer
  counted); new StoppedDevices + FaultedHostNames. Counting extracted to the
  testable DiagnosticsFromHealth.
- DriverInstanceDiagnostics: StoppedDevices (default 0) and FaultedHostNames
  (default null) are additive with defaults, so a pre-#786 sender's snapshot
  still deserializes over Akka's default JSON serializer (pinned by a test).
- FleetDiagnosticsAggregation (pure): identity key = (driver-instance string
  id, HostName), case-insensitive; a single-host driver / nameless pre-#786
  count is combined by MAX across nodes, never summed. Cluster peers come from
  enabled ClusterNode rows of the selected node's ClusterId.
- FleetDiagnostics.razor: tile fetches the cluster peers and shows the
  distinct count + "n of m nodes answered"; per-driver table gains a Stopped
  devices column.

Claude-Session: https://claude.ai/code/session_01GT67AD99zQQqpLCVm3p7xk
This commit is contained in:
Joseph Doherty
2026-11-11 11:58:35 -05:00
parent e857135e8b
commit 43bdeff2b6
6 changed files with 437 additions and 10 deletions
@@ -86,9 +86,17 @@ else
<div class="agg-label">Drivers</div>
<div class="agg-value numeric">@_snapshot.Drivers.Count</div>
</div>
<div class="agg-card alert">
<div class="agg-label">Faulted devices</div>
<div class="agg-value numeric">@_snapshot.Drivers.Sum(d => d.FaultedDevices)</div>
@* #786: distinct devices in Faulted across every node of the selected node's cluster (a device faulted on
both nodes of a redundant pair counts once) — Stopped is not a fault and is a per-node column below.
See FleetDiagnosticsAggregation for the identity key. *@
<div class="agg-card alert"
title="Distinct devices in Faulted state across the nodes of @(_clusterId ?? "this node's cluster") that answered (@_clusterAnswered of @_clusterNodeCount). Stopped devices are not counted; see the per-driver Stopped column.">
<div class="agg-label">Faulted devices@(_clusterId is { } cid ? $" (cluster {cid})" : "")</div>
<div class="agg-value numeric">@_clusterFaulted</div>
@if (_clusterNodeCount > 1)
{
<div class="small text-muted">@_clusterAnswered of @_clusterNodeCount nodes answered</div>
}
</div>
</section>
@@ -124,6 +132,7 @@ else
<th>State</th>
<th class="numeric">Connected devices</th>
<th class="numeric">Faulted devices</th>
<th class="numeric" title="Hosts the driver reports Stopped (e.g. taken off scan) — not counted as faulted">Stopped devices</th>
<th title="When the driver child last published a changed health snapshot (its spawn time until the first)">Last health publish</th>
</tr>
</thead>
@@ -144,6 +153,7 @@ else
<td><span class="chip @StateChipClass(d.State)">@d.State</span></td>
<td class="numeric">@d.ConnectedDevices</td>
<td class="numeric">@d.FaultedDevices</td>
<td class="numeric">@d.StoppedDevices</td>
<td>@d.LastChangeUtc.ToString("u")</td>
</tr>
}
@@ -165,6 +175,12 @@ else
private NodeDiagnosticsSnapshot? _snapshot;
private IReadOnlyDictionary<string, string> _displayNames = new Dictionary<string, string>();
private bool _loading;
// #786 cluster-wide faulted-device tile: the selected node's cluster, how many of its nodes answered, and the
// distinct faulted-device count across them.
private string? _clusterId;
private int _clusterNodeCount;
private int _clusterAnswered;
private int _clusterFaulted;
private DateTime? _fetchedUtc;
// Page-lifetime cancellation: a fetch in flight when the circuit goes away is abandoned.
@@ -222,6 +238,7 @@ else
// The client is fail-safe by contract: a node that does not answer yields an empty snapshot.
_snapshot = await Diagnostics.GetDiagnosticsAsync(NodeId.Parse(_selectedNode), _cts.Token);
_fetchedUtc = DateTime.UtcNow;
await LoadClusterFaultedAsync(_snapshot);
_displayNames = await LoadDisplayNamesAsync(_snapshot);
}
catch (Exception ex) when (ex is OperationCanceledException or ObjectDisposedException)
@@ -235,6 +252,48 @@ else
}
}
// #786: fetch the selected node's cluster peers too and count distinct faulted devices across all of them.
// The ConfigDb lookup is enrichment: on failure the tile falls back to the selected node alone.
private async Task LoadClusterFaultedAsync(NodeDiagnosticsSnapshot selected)
{
string? clusterId = null;
IReadOnlyList<string> nodeIds = new[] { selected.NodeId.Value };
try
{
await using var db = await DbFactory.CreateDbContextAsync(_cts.Token);
(clusterId, nodeIds) = await FleetDiagnosticsAggregation.LoadClusterNodeIdsAsync(db, selected.NodeId.Value, _cts.Token);
}
catch (Exception ex) when (ex is not OperationCanceledException and not ObjectDisposedException)
{
// ConfigDb hiccup — count this node only.
}
var snapshots = new List<NodeDiagnosticsSnapshot> { selected };
var peers = nodeIds.Where(n => !string.Equals(n, selected.NodeId.Value, StringComparison.OrdinalIgnoreCase)).ToList();
var peerSnapshots = await Task.WhenAll(peers.Select(FetchPeerAsync));
snapshots.AddRange(peerSnapshots.OfType<NodeDiagnosticsSnapshot>());
_clusterId = clusterId;
_clusterNodeCount = nodeIds.Count;
// The client degrades an unreachable node to an empty snapshot; one with drivers clearly answered. An empty
// answer is indistinguishable from "no answer", so it is counted as not answered — it contributes nothing either way.
_clusterAnswered = 1 + peerSnapshots.Count(p => p is { Drivers.Count: > 0 });
_clusterFaulted = FleetDiagnosticsAggregation.DistinctFaultedDevices(snapshots);
}
private async Task<NodeDiagnosticsSnapshot?> FetchPeerAsync(string nodeId)
{
try
{
return await Diagnostics.GetDiagnosticsAsync(NodeId.Parse(nodeId), _cts.Token);
}
catch (Exception ex) when (ex is not OperationCanceledException and not ObjectDisposedException)
{
// A malformed peer row (NodeId.Parse) — the peer just does not contribute.
return null;
}
}
// The authored name for a driver row, else its id (no ConfigDb row, or the name lookup failed).
private string DisplayNameOf(DriverInstanceDiagnostics d) =>
_displayNames.TryGetValue(d.Name, out var name) ? name : d.Name;