fix(adminui): Faulted-devices tile counts distinct Faulted devices across the cluster pair (#786)
Owner decision 2026-11-11: /fleet/diagnostics "Faulted devices" counts DISTINCT devices across the nodes of the selected node's cluster, Faulted state only. - DriverHostActor: FaultedDevices = HostState.Faulted only (Stopped no longer counted); new StoppedDevices + FaultedHostNames. Counting extracted to the testable DiagnosticsFromHealth. - DriverInstanceDiagnostics: StoppedDevices (default 0) and FaultedHostNames (default null) are additive with defaults, so a pre-#786 sender's snapshot still deserializes over Akka's default JSON serializer (pinned by a test). - FleetDiagnosticsAggregation (pure): identity key = (driver-instance string id, HostName), case-insensitive; a single-host driver / nameless pre-#786 count is combined by MAX across nodes, never summed. Cluster peers come from enabled ClusterNode rows of the selected node's ClusterId. - FleetDiagnostics.razor: tile fetches the cluster peers and shows the distinct count + "n of m nodes answered"; per-driver table gains a Stopped devices column. Claude-Session: https://claude.ai/code/session_01GT67AD99zQQqpLCVm3p7xk
This commit is contained in:
@@ -86,9 +86,17 @@ else
|
||||
<div class="agg-label">Drivers</div>
|
||||
<div class="agg-value numeric">@_snapshot.Drivers.Count</div>
|
||||
</div>
|
||||
<div class="agg-card alert">
|
||||
<div class="agg-label">Faulted devices</div>
|
||||
<div class="agg-value numeric">@_snapshot.Drivers.Sum(d => d.FaultedDevices)</div>
|
||||
@* #786: distinct devices in Faulted across every node of the selected node's cluster (a device faulted on
|
||||
both nodes of a redundant pair counts once) — Stopped is not a fault and is a per-node column below.
|
||||
See FleetDiagnosticsAggregation for the identity key. *@
|
||||
<div class="agg-card alert"
|
||||
title="Distinct devices in Faulted state across the nodes of @(_clusterId ?? "this node's cluster") that answered (@_clusterAnswered of @_clusterNodeCount). Stopped devices are not counted; see the per-driver Stopped column.">
|
||||
<div class="agg-label">Faulted devices@(_clusterId is { } cid ? $" (cluster {cid})" : "")</div>
|
||||
<div class="agg-value numeric">@_clusterFaulted</div>
|
||||
@if (_clusterNodeCount > 1)
|
||||
{
|
||||
<div class="small text-muted">@_clusterAnswered of @_clusterNodeCount nodes answered</div>
|
||||
}
|
||||
</div>
|
||||
</section>
|
||||
|
||||
@@ -124,6 +132,7 @@ else
|
||||
<th>State</th>
|
||||
<th class="numeric">Connected devices</th>
|
||||
<th class="numeric">Faulted devices</th>
|
||||
<th class="numeric" title="Hosts the driver reports Stopped (e.g. taken off scan) — not counted as faulted">Stopped devices</th>
|
||||
<th title="When the driver child last published a changed health snapshot (its spawn time until the first)">Last health publish</th>
|
||||
</tr>
|
||||
</thead>
|
||||
@@ -144,6 +153,7 @@ else
|
||||
<td><span class="chip @StateChipClass(d.State)">@d.State</span></td>
|
||||
<td class="numeric">@d.ConnectedDevices</td>
|
||||
<td class="numeric">@d.FaultedDevices</td>
|
||||
<td class="numeric">@d.StoppedDevices</td>
|
||||
<td>@d.LastChangeUtc.ToString("u")</td>
|
||||
</tr>
|
||||
}
|
||||
@@ -165,6 +175,12 @@ else
|
||||
private NodeDiagnosticsSnapshot? _snapshot;
|
||||
private IReadOnlyDictionary<string, string> _displayNames = new Dictionary<string, string>();
|
||||
private bool _loading;
|
||||
// #786 cluster-wide faulted-device tile: the selected node's cluster, how many of its nodes answered, and the
|
||||
// distinct faulted-device count across them.
|
||||
private string? _clusterId;
|
||||
private int _clusterNodeCount;
|
||||
private int _clusterAnswered;
|
||||
private int _clusterFaulted;
|
||||
private DateTime? _fetchedUtc;
|
||||
|
||||
// Page-lifetime cancellation: a fetch in flight when the circuit goes away is abandoned.
|
||||
@@ -222,6 +238,7 @@ else
|
||||
// The client is fail-safe by contract: a node that does not answer yields an empty snapshot.
|
||||
_snapshot = await Diagnostics.GetDiagnosticsAsync(NodeId.Parse(_selectedNode), _cts.Token);
|
||||
_fetchedUtc = DateTime.UtcNow;
|
||||
await LoadClusterFaultedAsync(_snapshot);
|
||||
_displayNames = await LoadDisplayNamesAsync(_snapshot);
|
||||
}
|
||||
catch (Exception ex) when (ex is OperationCanceledException or ObjectDisposedException)
|
||||
@@ -235,6 +252,48 @@ else
|
||||
}
|
||||
}
|
||||
|
||||
// #786: fetch the selected node's cluster peers too and count distinct faulted devices across all of them.
|
||||
// The ConfigDb lookup is enrichment: on failure the tile falls back to the selected node alone.
|
||||
private async Task LoadClusterFaultedAsync(NodeDiagnosticsSnapshot selected)
|
||||
{
|
||||
string? clusterId = null;
|
||||
IReadOnlyList<string> nodeIds = new[] { selected.NodeId.Value };
|
||||
try
|
||||
{
|
||||
await using var db = await DbFactory.CreateDbContextAsync(_cts.Token);
|
||||
(clusterId, nodeIds) = await FleetDiagnosticsAggregation.LoadClusterNodeIdsAsync(db, selected.NodeId.Value, _cts.Token);
|
||||
}
|
||||
catch (Exception ex) when (ex is not OperationCanceledException and not ObjectDisposedException)
|
||||
{
|
||||
// ConfigDb hiccup — count this node only.
|
||||
}
|
||||
|
||||
var snapshots = new List<NodeDiagnosticsSnapshot> { selected };
|
||||
var peers = nodeIds.Where(n => !string.Equals(n, selected.NodeId.Value, StringComparison.OrdinalIgnoreCase)).ToList();
|
||||
var peerSnapshots = await Task.WhenAll(peers.Select(FetchPeerAsync));
|
||||
snapshots.AddRange(peerSnapshots.OfType<NodeDiagnosticsSnapshot>());
|
||||
|
||||
_clusterId = clusterId;
|
||||
_clusterNodeCount = nodeIds.Count;
|
||||
// The client degrades an unreachable node to an empty snapshot; one with drivers clearly answered. An empty
|
||||
// answer is indistinguishable from "no answer", so it is counted as not answered — it contributes nothing either way.
|
||||
_clusterAnswered = 1 + peerSnapshots.Count(p => p is { Drivers.Count: > 0 });
|
||||
_clusterFaulted = FleetDiagnosticsAggregation.DistinctFaultedDevices(snapshots);
|
||||
}
|
||||
|
||||
private async Task<NodeDiagnosticsSnapshot?> FetchPeerAsync(string nodeId)
|
||||
{
|
||||
try
|
||||
{
|
||||
return await Diagnostics.GetDiagnosticsAsync(NodeId.Parse(nodeId), _cts.Token);
|
||||
}
|
||||
catch (Exception ex) when (ex is not OperationCanceledException and not ObjectDisposedException)
|
||||
{
|
||||
// A malformed peer row (NodeId.Parse) — the peer just does not contribute.
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// The authored name for a driver row, else its id (no ConfigDb row, or the name lookup failed).
|
||||
private string DisplayNameOf(DriverInstanceDiagnostics d) =>
|
||||
_displayNames.TryGetValue(d.Name, out var name) ? name : d.Name;
|
||||
|
||||
Reference in New Issue
Block a user