333 lines
15 KiB
Plaintext
333 lines
15 KiB
Plaintext
@page "/fleet/diagnostics"
|
|
@* Per-node driver diagnostics, fetched on demand from the selected driver node's DriverHostActor over
|
|
Akka (IFleetDiagnosticsClient → GetDiagnostics → NodeDiagnosticsSnapshot). The node answers from
|
|
its live child set + each child's last-published health; the "running from cache" flag is the one
|
|
fact /hosts cannot show (a node booted from its artifact cache looks healthy but is frozen).
|
|
Reach it as /fleet/diagnostics?node=<host:port> (the NodeId shown on /fleet and /hosts), or pick
|
|
a driver-role member from the selector. *@
|
|
@attribute [Authorize(Policy = AdminUiPolicies.AuthenticatedRead)]
|
|
@rendermode RenderMode.InteractiveServer
|
|
@using Microsoft.EntityFrameworkCore
|
|
@using ZB.MOM.WW.OtOpcUa.AdminUI.Clients
|
|
@using ZB.MOM.WW.OtOpcUa.Cluster
|
|
@using ZB.MOM.WW.OtOpcUa.Commons.Interfaces
|
|
@using ZB.MOM.WW.OtOpcUa.Commons.Types
|
|
@using ZB.MOM.WW.OtOpcUa.Configuration
|
|
@inject IClusterRoleInfo Cluster
|
|
@inject IFleetDiagnosticsClient Diagnostics
|
|
@inject IDbContextFactory<OtOpcUaConfigDbContext> DbFactory
|
|
@inject NavigationManager Nav
|
|
@implements IDisposable
|
|
|
|
<PageErrorBoundary>
|
|
<PageTitle>Node diagnostics · OtOpcUa</PageTitle>
|
|
<div class="page-head">
|
|
<h1>Node diagnostics</h1>
|
|
<span class="spacer"></span>
|
|
<a class="btn btn-sm btn-outline-secondary" href="/fleet">Back to fleet status</a>
|
|
</div>
|
|
|
|
<div class="d-flex align-items-center mb-3 gap-2 flex-wrap">
|
|
<label class="small text-muted mb-0" for="diag-node">Node</label>
|
|
<select id="diag-node" class="form-select form-select-sm" style="width:auto"
|
|
value="@_selectedNode" @onchange="OnNodeSelected">
|
|
<option value="">— select a driver node —</option>
|
|
@foreach (var member in _members)
|
|
{
|
|
<option value="@member">@member</option>
|
|
}
|
|
@if (!string.IsNullOrEmpty(_selectedNode) && !_members.Contains(_selectedNode))
|
|
{
|
|
<option value="@_selectedNode">@_selectedNode (not an Up member)</option>
|
|
}
|
|
</select>
|
|
<input type="text" class="form-control form-control-sm" style="width:220px"
|
|
placeholder="or type host:port…" @bind="_typedNode" @bind:event="oninput"
|
|
@onkeydown="OnTypedKeyDown" />
|
|
<button class="btn btn-sm btn-outline-primary" @onclick="UseTypedNodeAsync"
|
|
disabled="@(string.IsNullOrWhiteSpace(_typedNode))">Go</button>
|
|
<button class="btn btn-sm btn-outline-primary" @onclick="RefreshAsync"
|
|
disabled="@(_loading || string.IsNullOrEmpty(_selectedNode))">
|
|
@if (_loading) { <span class="spinner-border spinner-border-sm me-1" /> }
|
|
Refresh
|
|
</button>
|
|
<span class="text-muted small">
|
|
Last fetched: @(_fetchedUtc?.ToString("HH:mm:ss 'UTC'") ?? "—")
|
|
</span>
|
|
</div>
|
|
|
|
@if (string.IsNullOrEmpty(_selectedNode))
|
|
{
|
|
<section class="panel notice rise" style="animation-delay:.02s">
|
|
Pick a driver-role node to fetch its driver diagnostics. The node answers from its own
|
|
DriverHostActor, so this works for any node reachable over Akka remoting from this admin node.
|
|
</section>
|
|
}
|
|
else if (_snapshot is null)
|
|
{
|
|
<p>@(_loading ? "Fetching…" : "No snapshot yet.")</p>
|
|
}
|
|
else
|
|
{
|
|
<section class="agg-grid rise" style="animation-delay:.02s">
|
|
<div class="agg-card">
|
|
<div class="agg-label">Node</div>
|
|
<div class="agg-value mono">@_snapshot.NodeId.Value</div>
|
|
</div>
|
|
<div class="agg-card">
|
|
<div class="agg-label">Revision</div>
|
|
<div class="agg-value mono" title="@(_snapshot.CurrentRevision?.Value ?? "")">
|
|
@(_snapshot.CurrentRevision is { } rev ? DisplayText.Abbreviate(rev.Value, 12) : "—")
|
|
</div>
|
|
</div>
|
|
<div class="agg-card @(_snapshot.RunningFromCache ? "caution" : "")">
|
|
<div class="agg-label">Config source</div>
|
|
<div class="agg-value">@(_snapshot.RunningFromCache ? "cache (frozen)" : "live")</div>
|
|
</div>
|
|
<div class="agg-card">
|
|
<div class="agg-label">Drivers</div>
|
|
<div class="agg-value numeric">@_snapshot.Drivers.Count</div>
|
|
</div>
|
|
@* #786: distinct devices in Faulted across every node of the selected node's cluster (a device faulted on
|
|
both nodes of a redundant pair counts once) — Stopped is not a fault and is a per-node column below.
|
|
See FleetDiagnosticsAggregation for the identity key. *@
|
|
<div class="agg-card alert"
|
|
title="Distinct devices in Faulted state across the nodes of @(_clusterId ?? "this node's cluster") that answered (@_clusterAnswered of @_clusterNodeCount). Stopped devices are not counted; see the per-driver Stopped column.">
|
|
<div class="agg-label">Faulted devices@(_clusterId is { } cid ? $" (cluster {cid})" : "")</div>
|
|
<div class="agg-value numeric">@_clusterFaulted</div>
|
|
@if (_clusterNodeCount > 1)
|
|
{
|
|
<div class="small text-muted">@_clusterAnswered of @_clusterNodeCount nodes answered</div>
|
|
}
|
|
</div>
|
|
</section>
|
|
|
|
@if (_snapshot.RunningFromCache)
|
|
{
|
|
<section class="panel notice rise" style="animation-delay:.05s">
|
|
This node booted its configuration from its local artifact cache because central was unreachable.
|
|
It serves its last-known-good address space, but no deployment can reach it until central is back.
|
|
</section>
|
|
}
|
|
|
|
<section class="panel rise" style="animation-delay:.08s">
|
|
<h2 class="panel-head">
|
|
Drivers
|
|
<span class="text-muted small ms-2">as of @_snapshot.AsOfUtc.ToString("u")</span>
|
|
</h2>
|
|
@if (_snapshot.Drivers.Count == 0)
|
|
{
|
|
<div class="p-3 text-muted">
|
|
Empty snapshot: the node has no driver children — or it did not answer within the client's
|
|
3 s budget (an unreachable node degrades to an empty snapshot; see the AdminUI log for the
|
|
FleetDiagnosticsClient warning).
|
|
</div>
|
|
}
|
|
else
|
|
{
|
|
<div class="table-wrap" role="region" aria-label="Driver diagnostics" tabindex="0">
|
|
<table class="data-table">
|
|
<thead>
|
|
<tr>
|
|
<th scope="col">Driver instance</th>
|
|
<th scope="col">Row id</th>
|
|
<th scope="col">State</th>
|
|
<th scope="col" class="numeric">Connected devices</th>
|
|
<th scope="col" class="numeric">Faulted devices</th>
|
|
<th scope="col" class="numeric" title="Hosts the driver reports Stopped (e.g. taken off scan) — not counted as faulted">Stopped devices</th>
|
|
<th scope="col" title="When the driver child last published a changed health snapshot (its spawn time until the first)">Last health publish</th>
|
|
</tr>
|
|
</thead>
|
|
<tbody>
|
|
@foreach (var d in _snapshot.Drivers.OrderBy(d => DisplayNameOf(d), StringComparer.OrdinalIgnoreCase))
|
|
{
|
|
<tr>
|
|
<td>
|
|
@* A-F5: the wire carries the string instance id; show the authored name
|
|
(joined from the ConfigDb) with the id beneath it. *@
|
|
@DisplayNameOf(d)
|
|
@if (!string.Equals(DisplayNameOf(d), d.Name, StringComparison.Ordinal))
|
|
{
|
|
<div class="mono small text-muted">@d.Name</div>
|
|
}
|
|
</td>
|
|
<td><span class="mono" title="@d.DriverInstanceId">@(d.DriverInstanceId == Guid.Empty ? "—" : DisplayText.Abbreviate(d.DriverInstanceId.ToString(), 13))</span></td>
|
|
<td><span class="chip @StateChipClass(d.State)">@d.State</span></td>
|
|
<td class="numeric">@d.ConnectedDevices</td>
|
|
<td class="numeric">@d.FaultedDevices</td>
|
|
<td class="numeric">@d.StoppedDevices</td>
|
|
<td>@d.LastChangeUtc.ToString("u")</td>
|
|
</tr>
|
|
}
|
|
</tbody>
|
|
</table>
|
|
</div>
|
|
}
|
|
</section>
|
|
}
|
|
</PageErrorBoundary>
|
|
|
|
@code {
|
|
[SupplyParameterFromQuery(Name = "node")]
|
|
public string? Node { get; set; }
|
|
|
|
private IReadOnlyList<string> _members = Array.Empty<string>();
|
|
private string _selectedNode = "";
|
|
private string _typedNode = "";
|
|
private NodeDiagnosticsSnapshot? _snapshot;
|
|
private IReadOnlyDictionary<string, string> _displayNames = new Dictionary<string, string>();
|
|
private bool _loading;
|
|
// #786 cluster-wide faulted-device tile: the selected node's cluster, how many of its nodes answered, and the
|
|
// distinct faulted-device count across them.
|
|
private string? _clusterId;
|
|
private int _clusterNodeCount;
|
|
private int _clusterAnswered;
|
|
private int _clusterFaulted;
|
|
private DateTime? _fetchedUtc;
|
|
|
|
// Page-lifetime cancellation: a fetch in flight when the circuit goes away is abandoned.
|
|
private readonly CancellationTokenSource _cts = new();
|
|
private volatile bool _disposed;
|
|
|
|
protected override async Task OnParametersSetAsync()
|
|
{
|
|
_members = Cluster.MembersWithRole(RoleParser.Driver)
|
|
.Select(n => n.Value)
|
|
.OrderBy(s => s, StringComparer.OrdinalIgnoreCase)
|
|
.ToList();
|
|
var requested = Node?.Trim() ?? "";
|
|
if (!string.Equals(requested, _selectedNode, StringComparison.Ordinal))
|
|
{
|
|
_selectedNode = requested;
|
|
_snapshot = null;
|
|
if (!string.IsNullOrEmpty(_selectedNode)) await LoadAsync();
|
|
}
|
|
}
|
|
|
|
private Task OnNodeSelected(ChangeEventArgs e)
|
|
{
|
|
var value = e.Value?.ToString() ?? "";
|
|
Navigate(value);
|
|
return Task.CompletedTask;
|
|
}
|
|
|
|
private Task UseTypedNodeAsync()
|
|
{
|
|
Navigate(_typedNode.Trim());
|
|
return Task.CompletedTask;
|
|
}
|
|
|
|
private Task OnTypedKeyDown(KeyboardEventArgs e)
|
|
{
|
|
if (e.Key == "Enter") Navigate(_typedNode.Trim());
|
|
return Task.CompletedTask;
|
|
}
|
|
|
|
// The node id lives in the URL so a diagnostics view is linkable from /fleet and survives a reload.
|
|
private void Navigate(string node) =>
|
|
Nav.NavigateTo(string.IsNullOrEmpty(node)
|
|
? "/fleet/diagnostics"
|
|
: FleetDiagnosticsLinks.For(node));
|
|
|
|
private async Task RefreshAsync() => await LoadAsync();
|
|
|
|
private async Task LoadAsync()
|
|
{
|
|
if (_loading || string.IsNullOrEmpty(_selectedNode)) return;
|
|
_loading = true;
|
|
try
|
|
{
|
|
// The client is fail-safe by contract: a node that does not answer yields an empty snapshot.
|
|
_snapshot = await Diagnostics.GetDiagnosticsAsync(NodeId.Parse(_selectedNode), _cts.Token);
|
|
_fetchedUtc = DateTime.UtcNow;
|
|
await LoadClusterFaultedAsync(_snapshot);
|
|
_displayNames = await LoadDisplayNamesAsync(_snapshot);
|
|
}
|
|
catch (Exception ex) when (ex is OperationCanceledException or ObjectDisposedException)
|
|
{
|
|
// Circuit gone mid-fetch — a clean exit, not a fault.
|
|
}
|
|
finally
|
|
{
|
|
_loading = false;
|
|
if (!_disposed) StateHasChanged();
|
|
}
|
|
}
|
|
|
|
// #786: fetch the selected node's cluster peers too and count distinct faulted devices across all of them.
|
|
// The ConfigDb lookup is enrichment: on failure the tile falls back to the selected node alone.
|
|
private async Task LoadClusterFaultedAsync(NodeDiagnosticsSnapshot selected)
|
|
{
|
|
string? clusterId = null;
|
|
IReadOnlyList<string> nodeIds = new[] { selected.NodeId.Value };
|
|
try
|
|
{
|
|
await using var db = await DbFactory.CreateDbContextAsync(_cts.Token);
|
|
(clusterId, nodeIds) = await FleetDiagnosticsAggregation.LoadClusterNodeIdsAsync(db, selected.NodeId.Value, _cts.Token);
|
|
}
|
|
catch (Exception ex) when (ex is not OperationCanceledException and not ObjectDisposedException)
|
|
{
|
|
// ConfigDb hiccup — count this node only.
|
|
}
|
|
|
|
var snapshots = new List<NodeDiagnosticsSnapshot> { selected };
|
|
var peers = nodeIds.Where(n => !string.Equals(n, selected.NodeId.Value, StringComparison.OrdinalIgnoreCase)).ToList();
|
|
var peerSnapshots = await Task.WhenAll(peers.Select(FetchPeerAsync));
|
|
snapshots.AddRange(peerSnapshots.OfType<NodeDiagnosticsSnapshot>());
|
|
|
|
_clusterId = clusterId;
|
|
_clusterNodeCount = nodeIds.Count;
|
|
// The client degrades an unreachable node to an empty snapshot; one with drivers clearly answered. An empty
|
|
// answer is indistinguishable from "no answer", so it is counted as not answered — it contributes nothing either way.
|
|
_clusterAnswered = 1 + peerSnapshots.Count(p => p is { Drivers.Count: > 0 });
|
|
_clusterFaulted = FleetDiagnosticsAggregation.DistinctFaultedDevices(snapshots);
|
|
}
|
|
|
|
private async Task<NodeDiagnosticsSnapshot?> FetchPeerAsync(string nodeId)
|
|
{
|
|
try
|
|
{
|
|
return await Diagnostics.GetDiagnosticsAsync(NodeId.Parse(nodeId), _cts.Token);
|
|
}
|
|
catch (Exception ex) when (ex is not OperationCanceledException and not ObjectDisposedException)
|
|
{
|
|
// A malformed peer row (NodeId.Parse) — the peer just does not contribute.
|
|
return null;
|
|
}
|
|
}
|
|
|
|
// The authored name for a driver row, else its id (no ConfigDb row, or the name lookup failed).
|
|
private string DisplayNameOf(DriverInstanceDiagnostics d) =>
|
|
_displayNames.TryGetValue(d.Name, out var name) ? name : d.Name;
|
|
|
|
private async Task<IReadOnlyDictionary<string, string>> LoadDisplayNamesAsync(NodeDiagnosticsSnapshot snapshot)
|
|
{
|
|
try
|
|
{
|
|
await using var db = await DbFactory.CreateDbContextAsync(_cts.Token);
|
|
return await FleetDiagnosticsLinks.LoadDisplayNamesAsync(
|
|
db, snapshot.Drivers.Select(d => d.Name).ToList(), _cts.Token);
|
|
}
|
|
catch (Exception ex) when (ex is not OperationCanceledException and not ObjectDisposedException)
|
|
{
|
|
// Names are enrichment only: a ConfigDb hiccup falls back to the ids, never fails the page.
|
|
return new Dictionary<string, string>();
|
|
}
|
|
}
|
|
|
|
private static string StateChipClass(string state) => state switch
|
|
{
|
|
"Healthy" => "chip-ok",
|
|
"Degraded" or "Connecting" or "Reconnecting" => "chip-caution",
|
|
"Faulted" => "chip-alert",
|
|
_ => "chip-idle",
|
|
};
|
|
|
|
public void Dispose()
|
|
{
|
|
_disposed = true;
|
|
_cts.Cancel();
|
|
_cts.Dispose();
|
|
}
|
|
}
|