Files
lmxopcua/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Components/Pages/Hosts.razor
T

627 lines
33 KiB
Plaintext

@page "/hosts"
@* Two views, top to bottom. (1) Akka cluster topology: each member's NodeId (host:port), roles,
leader status — the cluster-membership view. (2) Driver instances: live driver-health grouped by
cluster. Each health snapshot carries its reporting node (DriverHealthChanged.ReportingNodeId, #847), so the
rows are one per (driver instance, node) — a driver Faulted on one node and Healthy on its peer shows both. The section
reads the in-process driver-health snapshot store directly + reloads its config from the ConfigDB. *@
@attribute [Authorize(Policy = AdminUiPolicies.AuthenticatedRead)]
@rendermode RenderMode.InteractiveServer
@using Akka.Actor
@using Akka.Cluster
@using Microsoft.EntityFrameworkCore
@using Microsoft.Extensions.Logging
@using ZB.MOM.WW.OtOpcUa.Configuration
@using ZB.MOM.WW.OtOpcUa.AdminUI.Components.Shared.Drivers
@using ZB.MOM.WW.OtOpcUa.AdminUI.Hosts
@using ZB.MOM.WW.OtOpcUa.AdminUI.Hubs
@using ZB.MOM.WW.OtOpcUa.Commons.Messages.Drivers
@inject ActorSystem ActorSystem
@inject IDriverStatusSnapshotStore DriverStore
@inject IDriverResilienceStatusStore ResilienceStore
@inject IDbContextFactory<OtOpcUaConfigDbContext> DbFactory
@inject DeployedDriverSet Deployed
@inject Microsoft.Extensions.Logging.ILogger<Hosts> Logger
@implements IAsyncDisposable
<PageErrorBoundary>
<div class="d-flex justify-content-between align-items-center mb-3">
<h4 class="mb-0">Cluster hosts</h4>
</div>
<div class="d-flex align-items-center mb-3 gap-2">
<button class="btn btn-sm btn-outline-primary" @onclick="RefreshAsync" disabled="@_refreshing">
@if (_refreshing) { <span class="spinner-border spinner-border-sm me-1" /> }
Refresh
</button>
<span class="text-muted small">
Auto-refresh every @RefreshIntervalSeconds s. Last updated: @(_lastRefreshUtc?.ToString("HH:mm:ss 'UTC'") ?? "—")
</span>
</div>
@if (_rows is null)
{
<p>Loading…</p>
}
else if (_rows.Count == 0)
{
<section class="panel notice rise" style="animation-delay:.08s">
No cluster members visible. The local node may still be joining.
</section>
}
else
{
<section class="agg-grid rise" style="animation-delay:.08s">
<div class="agg-card">
<div class="agg-label">Members</div>
<div class="agg-value numeric">@_rows.Count</div>
</div>
<div class="agg-card">
<div class="agg-label">Up</div>
<div class="agg-value numeric">@_rows.Count(r => r.Status == "Up")</div>
</div>
<div class="agg-card caution">
<div class="agg-label">Joining/Leaving</div>
<div class="agg-value numeric">@_rows.Count(r => r.Status is "Joining" or "Leaving" or "Exiting")</div>
</div>
<div class="agg-card alert">
<div class="agg-label">Unreachable</div>
<div class="agg-value numeric">@_rows.Count(r => r.Unreachable)</div>
</div>
</section>
<section class="panel rise" style="animation-delay:.14s">
<div class="panel-head">Members</div>
<div class="table-wrap">
<table class="data-table">
<thead>
<tr>
<th>Address</th>
<th>Status</th>
<th>Roles</th>
<th>Leader for</th>
</tr>
</thead>
<tbody>
@foreach (var r in _rows)
{
<tr>
<td>
<span class="mono">@r.Address</span>
@if (r.IsSelf) { <span class="chip chip-idle ms-2">self</span> }
</td>
<td>
<span class="chip @StatusChipClass(r.Status, r.Unreachable)">
@(r.Unreachable ? $"{r.Status} (unreachable)" : r.Status)
</span>
</td>
<td>
@foreach (var role in r.Roles)
{
<span class="chip chip-idle me-1">@role</span>
}
</td>
<td>
@if (r.LeaderRoles.Count == 0)
{
<span class="text-muted">—</span>
}
else
{
@foreach (var role in r.LeaderRoles)
{
<span class="chip chip-ok me-1">@role</span>
}
}
</td>
</tr>
}
</tbody>
</table>
</div>
</section>
}
@if (_driverGroups is null)
{
<p>Loading…</p>
}
else if (_driverGroups.Count == 0)
{
<section class="panel notice rise" style="animation-delay:.2s">
No driver instances reporting yet.
</section>
}
else
{
@* One panel per cluster group (was: one panel nesting every group) so the sections read as
separate cards instead of a continuous run of headings. *@
@foreach (var g in _driverGroups)
{
<section class="panel rise mt-3" style="animation-delay:.2s" @key="@($"group-{g.ClusterId}")">
<div class="panel-head">
Driver instances · Cluster <span class="mono">@g.ClusterId</span>
<span class="ms-2">
@foreach (var n in g.Nodes)
{
<span class="chip chip-idle me-1">@n.NodeId (@n.Host:@n.OpcUaPort)</span>
}
</span>
</div>
<div class="table-wrap">
<table class="data-table">
<thead>
<tr>
<th>Driver</th>
<th title="The node reporting this row — each node of a redundant pair runs the driver, so a driver has one row per node">Node</th>
<th>Type</th>
<th>Status</th>
<th>Hosts</th>
<th>Tags</th>
<th>Last read</th>
<th>Errors/5 min</th>
<th>Last error</th>
<th></th>
</tr>
</thead>
<tbody>
@if (g.Drivers.Count == 0)
{
<tr>
<td colspan="10"><span class="text-muted">No drivers.</span></td>
</tr>
}
else
{
@foreach (var d in g.Drivers)
{
<tr @key="@($"row-{d.RowKey}")">
<td>
@(d.Name ?? d.DriverInstanceId)
@if (d.RediscoveryNeededUtc is not null)
{
@* The driver reported its remote's tag set may have changed. Advisory
only — v3 authors raw tags via /raw browse-commit, so nothing is
grafted at runtime and an operator must re-browse to pick it up. *@
<span class="chip chip-warn ms-1"
title="@($"Reported {d.RediscoveryNeededUtc:u}: {d.RediscoveryReason ?? "tag set may have changed"}. The served address space is unchanged — re-browse this device under /raw and commit anything new.")">
re-browse
</span>
}
@if (d.Name is not null)
{
@* Own line — inline after the name the two identifiers ran together. *@
<div class="mono small text-muted">@d.DriverInstanceId</div>
}
@if (d.DiagnosticItems.Count > 0)
{
@* Driver-reported diagnostics (Gitea #640), e.g. S7 negotiated PDU size.
Renders nothing for a driver with none. *@
<div class="small text-muted" data-testid="driver-diagnostics">
@foreach (var kv in d.DiagnosticItems)
{
<div class="mono">@kv.Key: @kv.Value</div>
}
</div>
}
</td>
<td>
@if (d.NodeId is null)
{
<span class="text-muted" title="Reported by a node that predates per-node health (#847)">—</span>
}
else
{
<span class="mono">@d.NodeId</span>
}
</td>
<td>@(d.DriverType ?? "—")</td>
<td><span class="chip @DriverChipClass(d.State)">@d.State</span></td>
<td>
@* Per-host connectivity (Gitea #521). The point of this column is the case
the driver-level Status chip cannot express: a multi-device driver stays
Healthy in aggregate while ONE of its PLCs is unreachable. A driver with
no probe shows "—" — deliberately distinct from a probe reporting zero
hosts, which shows "0 hosts". *@
@if (d.HostStatuses is null)
{
<span class="text-muted">—</span>
}
else if (d.DegradedHosts.Count == 0)
{
<span class="text-muted small">@d.HostStatuses.Count @(d.HostStatuses.Count == 1 ? "host" : "hosts")</span>
}
else
{
<span class="chip chip-warn"
title="@DegradedHostTitle(d)">
@d.DegradedHosts.Count / @d.HostStatuses.Count down
</span>
}
</td>
<td>
@* Authored tags the driver is NOT polling (Gitea #541). This column
exists because a skip is not an error: the Status chip stays
Healthy, Errors/5 min stays 0, Last error stays empty, and Last
read keeps advancing off the tags that DID map. On the Makino in
#540 that read as a perfectly healthy driver ignoring 20 of its 22
tags. "—" means NO ANSWER — either the driver cannot report skips,
or it faulted before building a tag table — which is distinct from
the positive claim "this driver skipped nothing". A pre-fix build
rendered "0 tags" for two faulted drivers on the docker-dev rig
that had two authored tags apiece. *@
@if (d.SkippedTags is null)
{
<span class="text-muted">—</span>
}
else if (!d.HasSkippedTags)
{
<span class="text-muted small">@d.SkippedTags.AuthoredCount @(d.SkippedTags.AuthoredCount == 1 ? "tag" : "tags")</span>
}
else
{
<span class="chip @(d.SkippedTags.SkippedEverything ? "chip-bad" : "chip-warn")"
title="@d.SkippedTagsTitle()">
@d.SkippedTags.SkippedCount / @d.SkippedTags.AuthoredCount skipped
</span>
}
</td>
<td>@(d.LastSuccessfulReadUtc?.ToString("HH:mm:ss 'UTC'") ?? "—")</td>
<td class="numeric">@d.ErrorCount5Min</td>
<td>
@* Abbreviated so one long driver exception can't blow the table out
sideways (never slice a DB/driver string in Razor — #504); the full
text rides the tooltip and the Controls dialog. *@
<span class="text-muted small" title="@d.LastError">@(DisplayText.Abbreviate(d.LastError, 60) is { Length: > 0 } err ? err : "—")</span>
</td>
<td>
@* Opens the driver's status panel in a dialog (was an inline full-width
expander row — #546). Shown to every AuthenticatedRead viewer — the
panel carries read-only detail (resilience/breaker state, last error)
as well as the operator actions, and self-gates only the
Reconnect/Restart buttons on DriverOperator. *@
<button type="button"
class="btn btn-sm btn-outline-secondary"
@onclick="() => OpenControls(d.DriverInstanceId)"
title="Show live status, resilience and driver controls">
Controls
</button>
</td>
</tr>
}
}
</tbody>
</table>
</div>
</section>
}
@* Controls dialog (#546, reworked from an inline expander row — the full-width panel inside the
table made an already-dense page unreadable). Only one dialog at a time; the id survives the
5 s RebuildDriverGroups because it is held on the component, and the row is re-resolved from
the CURRENT groups each render so the header/Enabled always track live state. A driver that
vanishes from the snapshot (undeployed) closes the dialog by rendering nothing.
Enabled participates in the panel's key on purpose. The panel decides whether to subscribe
ONCE, in OnInitializedAsync (`if (!Enabled) return;`), and has no OnParametersSet hook — so a
live config edit that flips Enabled (surfaced by the 5 s LoadConfigAsync refresh) would
re-render the panel's "Disabled — not deployed" markup while its two store subscriptions and
its staleness timer stayed wired underneath. Keying on the flag recycles the component:
the old instance is disposed (unsubscribing both stores, draining both timers) and a fresh
one mounts against the new value. Closing the dialog disposes the panel the same way. *@
@if (FindControlsDriver() is var (cg, cd) && cd is not null && cg is not null)
{
<div class="modal-backdrop fade show" style="display:block"></div>
<div class="modal fade show" tabindex="-1" role="dialog" style="display:block">
<div class="modal-dialog modal-lg" role="document">
<div class="modal-content">
<div class="modal-header">
<h5 class="modal-title">
@(cd.Name ?? cd.DriverInstanceId)
<span class="mono small text-muted ms-2">@cd.DriverInstanceId</span>
<span class="chip chip-idle ms-2">@cg.ClusterId</span>
</h5>
<button type="button" class="btn-close" aria-label="Close" @onclick="CloseControls"></button>
</div>
<div class="modal-body">
<DriverStatusPanel @key="@($"detail-{cd.DriverInstanceId}-{cd.Enabled}")"
DriverInstanceId="@cd.DriverInstanceId"
ClusterId="@cg.ClusterId"
Enabled="@cd.Enabled" />
</div>
<div class="modal-footer">
<button type="button" class="btn btn-outline-secondary" @onclick="CloseControls">Close</button>
</div>
</div>
</div>
</div>
}
}
</PageErrorBoundary>
@code {
private const int RefreshIntervalSeconds = 5;
// Second guard only: a snapshot of a driver neither authored nor deployed must also be this old before it
// is evicted. NOT a liveness proof — DriverInstanceActor publishes on a 30 s tick and dedups on a health
// fingerprint, so a running Faulted/idle/tagless driver can go hours without publishing; the deployed
// set (DeployedDriverSet) is what keeps such a driver's row.
private static readonly TimeSpan EvictUnconfiguredAfter = TimeSpan.FromSeconds(60);
private List<MemberRow>? _rows;
private bool _refreshing;
private DateTime? _lastRefreshUtc;
private Timer? _timer;
private IReadOnlyList<HostsClusterGroup>? _driverGroups;
private List<HostsNodeInfo> _nodes = new();
private List<HostsDriverInstanceInfo> _instances = new();
// The driver instance whose Controls dialog is open (Gitea #546). Held on the component, NOT on
// the row records, so it survives the 5 s timer's RebuildDriverGroups — which replaces every
// HostsDriverRow instance and would otherwise close a dialog an operator just opened.
private string? _controlsDriverId;
// Overlap guard: a tick whose config load outruns the 5 s interval must not stack behind the
// next one. A manual Refresh landing on top of an in-flight tick is a no-op by the same guard.
private int _tickRunning;
// Set by OnSnapshotChanged (bridge actor thread), consumed by TickAsync.
private readonly CoalescingDirtyFlag _driversDirty = new();
// Page-lifetime cancellation: cancelled in DisposeAsync() so a tick in flight when the circuit
// goes away aborts its queries instead of running them against a component that's gone.
private readonly CancellationTokenSource _cts = new();
// Set first thing in DisposeAsync. A tick that resumes after teardown must not call
// StateHasChanged — the renderer is gone and it throws, inside a task nobody awaits.
private volatile bool _disposed;
protected override async Task OnInitializedAsync()
{
Refresh();
await LoadConfigAsync();
RebuildDriverGroups();
DriverStore.SnapshotChanged += OnSnapshotChanged;
_timer = new Timer(_ => _ = InvokeAsync(TickAsync), null,
TimeSpan.FromSeconds(RefreshIntervalSeconds),
TimeSpan.FromSeconds(RefreshIntervalSeconds));
}
private async Task TickAsync()
{
if (Interlocked.Exchange(ref _tickRunning, 1) == 1) return;
try
{
Refresh();
var configChanged = await LoadConfigAsync();
// Re-project the driver rows only when a health snapshot or the config actually changed
// since the last tick (R04 P4). The snapshot handler just marks the flag, so N drivers
// publishing every 5 s cost one rebuild per tick per tab, not N rebuilds + N full renders.
if (_driversDirty.TryConsume() | configChanged) RebuildDriverGroups();
StateHasChanged();
}
catch (Exception ex) when (ex is ObjectDisposedException or OperationCanceledException)
{
// Circuit disposed while a tick was in flight — ignore (the discarded task would
// otherwise swallow this silently). Mirrors DriverStatusPanel's drain-on-dispose.
}
finally
{
Volatile.Write(ref _tickRunning, 0);
}
}
private async Task RefreshAsync()
{
if (Interlocked.Exchange(ref _tickRunning, 1) == 1) return;
try
{
_refreshing = true;
StateHasChanged();
await Task.Yield();
Refresh();
await LoadConfigAsync();
// A manual refresh always rebuilds; clear the flag so the next tick does not rebuild again.
_driversDirty.TryConsume();
RebuildDriverGroups();
}
catch (Exception ex) when (ex is ObjectDisposedException or OperationCanceledException)
{
// Same clean-exit tolerance as the timer path.
}
finally
{
// Release first: a StateHasChanged on a torn-down circuit must never leave the guard set.
Volatile.Write(ref _tickRunning, 0);
_refreshing = false;
// This one sits OUTSIDE the try, so it has no catch to fall into — and a render
// request against a disposed renderer throws. Skip it once the page is gone; there
// is nothing left to render to.
if (!_disposed) StateHasChanged();
}
}
// Load the per-cluster node + driver-instance config from the ConfigDB. Kept cheap and
// swallow-on-failure so a transient DB hiccup dims the enrichment (rows fall back to id/—)
// without crashing the page. The live health feed (the snapshot store) is independent of this.
// Returns whether the loaded config differs from the previous load (records compare by value).
private async Task<bool> LoadConfigAsync()
{
var previousNodes = _nodes;
var previousInstances = _instances;
try
{
await using var db = await DbFactory.CreateDbContextAsync(_cts.Token);
// Ordered: the change test below compares with SequenceEqual (review r0 M9).
_nodes = await db.ClusterNodes.AsNoTracking()
.OrderBy(n => n.ClusterId).ThenBy(n => n.NodeId)
.Select(n => new HostsNodeInfo(n.ClusterId, n.NodeId, n.Host, n.OpcUaPort))
.ToListAsync(_cts.Token);
_instances = await db.DriverInstances.AsNoTracking()
.OrderBy(d => d.DriverInstanceId)
.Select(d => new HostsDriverInstanceInfo(d.DriverInstanceId, d.ClusterId, d.Name, d.DriverType, d.Enabled))
.ToListAsync(_cts.Token);
// Evict snapshots of driver instances that are neither AUTHORED nor DEPLOYED, and stale. The
// stores have no other removal path, so without this a removed driver stays on this page as a
// frozen row until the admin node restarts. DriverInstances is the authored (draft) table: a
// driver deleted in /raw keeps running until the next deploy, and a quiet one (Faulted, idle,
// tagless) may not publish at all meanwhile, so the deployed set — not staleness — is what keeps
// it (review D-b r1, I1). Unknown deployed set ⇒ no eviction this sweep.
// Only a SUCCESSFUL load reaches here; the failure path below blanks _instances and must
// never sweep the stores.
var evicted = 0;
var deployed = await Deployed.LoadAsync(_cts.Token);
if (deployed is not null)
{
var configured = _instances
.Select(i => i.DriverInstanceId)
.ToHashSet(StringComparer.OrdinalIgnoreCase);
configured.UnionWith(deployed);
var staleBefore = DateTime.UtcNow - EvictUnconfiguredAfter;
evicted = DriverStore.EvictUnconfiguredStale(configured, staleBefore);
ResilienceStore.EvictUnconfiguredStale(configured, staleBefore);
}
return evicted > 0
|| !_nodes.SequenceEqual(previousNodes)
|| !_instances.SequenceEqual(previousInstances);
}
catch (Exception ex) when (ex is OperationCanceledException or ObjectDisposedException)
{
// The circuit went away mid-load. Rethrow so the caller treats it as a clean exit — it
// must NOT fall into the swallow-and-clear below, which would blank the cached config.
// ObjectDisposedException belongs here for the same reason: teardown disposes _cts (and
// with it the DbContext's token registration) while a query may still be running, and
// that shape reaches this method as ODE rather than OCE. Sending it to the generic catch
// blanked _nodes/_instances on the way out — the exact swallow-and-clear this rethrow
// exists to prevent, and permanent, because a torn-down page never ticks again.
throw;
}
catch (Exception ex)
{
Logger.LogWarning(ex, "/hosts: failed to load driver-instance config; rows show ids only.");
_nodes = new();
_instances = new();
return true;
}
}
// Re-project the cluster-grouped driver rows from the latest live snapshots + cached config.
private void RebuildDriverGroups()
=> _driverGroups = HostsDriverView.Build(DriverStore.GetAll(), _nodes, _instances);
// Open/close the Controls dialog. Ids are globally unique, so no cluster qualifier is needed.
private void OpenControls(string driverInstanceId) => _controlsDriverId = driverInstanceId;
private void CloseControls() => _controlsDriverId = null;
// Resolve the open dialog's driver from the CURRENT groups (the row records are replaced on
// every rebuild, so the dialog must never cache one). Null row ⇒ the driver vanished from the
// snapshot ⇒ the dialog renders nothing, i.e. self-closes.
private (HostsClusterGroup? Group, HostsDriverRow? Driver) FindControlsDriver()
{
if (_controlsDriverId is null || _driverGroups is null) return (null, null);
foreach (var g in _driverGroups)
{
var d = g.Drivers.FirstOrDefault(x =>
string.Equals(x.DriverInstanceId, _controlsDriverId, StringComparison.OrdinalIgnoreCase));
if (d is not null) return (g, d);
}
return (null, null);
}
// Raised by the snapshot store on the bridge actor's thread for every driver instance. It only
// marks the rows dirty: the 5 s tick does the rebuild + render (R04 P4). Rendering here cost one
// full re-projection and full-page diff per driver publish per open tab (200 drivers ≈ 40/s).
// A health change therefore shows within one tick; the open Controls dialog's DriverStatusPanel
// subscribes to the store itself and stays live.
private void OnSnapshotChanged(DriverHealthChanged _) => _driversDirty.Mark();
private void Refresh()
{
var cluster = Akka.Cluster.Cluster.Get(ActorSystem);
var state = cluster.State;
var unreachable = state.Unreachable
.Select(m => m.Address.ToString()).ToHashSet();
var selfAddress = cluster.SelfAddress.ToString();
_rows = state.Members.Select(m =>
{
var address = m.Address.ToString();
var hostPort = $"{m.Address.Host ?? "?"}:{m.Address.Port ?? 0}";
var leaderRoles = m.Roles
.Where(role => cluster.State.RoleLeader(role)?.ToString() == address)
.OrderBy(s => s, StringComparer.OrdinalIgnoreCase)
.ToList();
return new MemberRow(
Address: hostPort,
Status: m.Status.ToString(),
Roles: m.Roles.OrderBy(s => s, StringComparer.OrdinalIgnoreCase).ToList(),
LeaderRoles: leaderRoles,
Unreachable: unreachable.Contains(address),
IsSelf: address == selfAddress);
})
.OrderBy(r => r.Address, StringComparer.OrdinalIgnoreCase)
.ToList();
_lastRefreshUtc = DateTime.UtcNow;
}
private static string StatusChipClass(string status, bool unreachable) => (status, unreachable) switch
{
(_, true) => "chip-alert",
("Up", _) => "chip-ok",
("Joining", _) or ("Leaving", _) or ("Exiting", _) or ("WeaklyUp", _) => "chip-caution",
("Down", _) or ("Removed", _) => "chip-alert",
_ => "chip-idle",
};
// Map DriverState string → chip CSS class (mirrors DriverStatusPanel.ChipClass). Since #546 that
// panel renders inside this page's expanded rows, so the two mappings are visible side by side —
// they must stay in step or one driver shows two different colours for one state.
private static string DriverChipClass(string? state) => state switch
{
"Healthy" => "chip-ok",
"Degraded" => "chip-warn",
"Connecting" => "chip-warn",
"Reconnecting" => "chip-warn",
"Faulted" => "chip-bad",
_ => "chip-idle",
};
// Tooltip listing each host that is not Running, with its state and when it last changed. Built in
// C# rather than inline in the markup so the string is composed once per render, and so the row can
// never be the thing that 500s the page — string.Join over an already-materialised list does no
// indexing (cf. #504, where slicing a short DB string in Razor took down the whole page).
private static string DegradedHostTitle(HostsDriverRow d) =>
string.Join(" · ", d.DegradedHosts.Select(h => $"{h.HostName}: {h.State} since {h.LastChangedUtc:u}"));
public async ValueTask DisposeAsync()
{
_disposed = true;
// Unsubscribe first so the singleton store can't invoke a handler on a disposed component.
DriverStore.SnapshotChanged -= OnSnapshotChanged;
// Cancel before draining so a tick already in flight aborts its queries rather than running
// them to completion against a component that's gone.
_cts.Cancel();
// Drain the timer so an in-flight callback can't touch a component that's already gone
// (System.Threading.Timer's async dispose awaits any in-flight callback — .NET 6+).
if (_timer is not null) await _timer.DisposeAsync();
// ...but the callback body is `_ = InvokeAsync(TickAsync)`, so that await covered only the
// SCHEDULING of the tick, never the tick itself: LoadConfigAsync can still be running
// against _cts.Token right now. Disposing the source out from under it turns the clean
// cancellation the line above arranged into an ObjectDisposedException mid-query, so skip
// the dispose while a tick holds it. An undisposed CancellationTokenSource with no timer
// and no materialised WaitHandle costs nothing and is collected with the component.
if (Volatile.Read(ref _tickRunning) == 0) _cts.Dispose();
}
private sealed record MemberRow(
string Address,
string Status,
IReadOnlyCollection<string> Roles,
IReadOnlyCollection<string> LeaderRoles,
bool Unreachable,
bool IsSelf);
}