Files
lmxopcua/src/ZB.MOM.WW.OtOpcUa.Admin/Components/Pages/Hosts.razor
Joseph Doherty d06cc01a48 Admin /hosts red-badge + resilience columns + Polly telemetry observer. Closes task #164 (the remaining slice of Phase 6.1 Stream E.3 after the earlier publisher + hub PR). Three cooperating pieces wired together so the operator-facing /hosts table actually reflects the live Polly counters that the pipeline builder is producing. DriverResiliencePipelineBuilder gains an optional DriverResilienceStatusTracker ctor param — when non-null, every built pipeline wires Polly's OnRetry/OnOpened/OnClosed strategy-options callbacks into the tracker. OnRetry → tracker.RecordFailure (so ConsecutiveFailures climbs per retry), OnOpened → tracker.RecordBreakerOpen (stamps LastCircuitBreakerOpenUtc), OnClosed → tracker.RecordSuccess (resets the failure counter once the target recovers). Absent tracker = silent, preserving the unit-test constructor path + any deployment that doesn't care about resilience observability. Cancellation stays excluded from the failure count via the existing ShouldHandle predicate. HostStatusService.HostStatusRow extends with four new fields — ConsecutiveFailures, LastCircuitBreakerOpenUtc, CurrentBulkheadDepth, LastRecycleUtc — populated via a second LEFT JOIN onto DriverInstanceResilienceStatuses keyed on (DriverInstanceId, HostName). LEFT JOIN because brand-new hosts haven't been sampled yet; a missing row means zero failures + never-opened breaker, which is the correct default. New FailureFlagThreshold constant (=3, matches plan decision #143's conservative half-of-breaker convention) + IsFlagged predicate so the UI can pre-warn before the breaker actually trips. Hosts.razor paints three new columns between State and Last-transition — Fail# (bold red when flagged), In-flight (bulkhead-depth proxy), Breaker-opened (relative age). Per-row "Flagged" red badge alongside State when IsFlagged is true. Above the first cluster table, a red alert banner summarises the flagged-host count when ≥1 host is flagged, so operators see the problem before scanning rows. Three new tests in DriverResiliencePipelineBuilderTests — Tracker_RecordsFailure_OnEveryRetry verifies ConsecutiveFailures reaches RetryCount after a transient-forever operation, Tracker_StampsBreakerOpen_WhenBreakerTrips verifies LastBreakerOpenUtc is set after threshold failures on a Write pipeline, Tracker_IsolatesCounters_PerHost verifies one dead host does not leak failure counts into a healthy sibling. Full suite — Core.Tests 14/14 resilience-builder tests passing (11 existing + 3 new), Admin.Tests 72/72 passing, Admin project builds 0 errors. SignalR live push of status changes + browser visual review are deliberately left to a follow-up — this PR keeps the structural change minimal (polling refresh already exists in the page's 10s timer; SignalR would be a structural add that touches hub registration + client subscription).
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-04-19 21:35:54 -04:00

185 lines
7.5 KiB
Plaintext

@page "/hosts"
@using Microsoft.EntityFrameworkCore
@using ZB.MOM.WW.OtOpcUa.Admin.Services
@using ZB.MOM.WW.OtOpcUa.Configuration.Enums
@inject IServiceScopeFactory ScopeFactory
@implements IDisposable
<h1 class="mb-4">Driver host status</h1>
<div class="d-flex align-items-center mb-3 gap-2">
<button class="btn btn-sm btn-outline-primary" @onclick="RefreshAsync" disabled="@_refreshing">
@if (_refreshing) { <span class="spinner-border spinner-border-sm me-1" /> }
Refresh
</button>
<span class="text-muted small">
Auto-refresh every @RefreshIntervalSeconds s. Last updated: @(_lastRefreshUtc?.ToString("HH:mm:ss 'UTC'") ?? "—")
</span>
</div>
<div class="alert alert-info small mb-4">
Each row is one host reported by a driver instance on a server node. Galaxy drivers report
per-Platform / per-AppEngine entries; Modbus drivers report the PLC endpoint. Rows age out
of the Server's publisher on every 10-second heartbeat — rows whose LastSeen is older than
30s are flagged Stale, which usually means the owning Server process has crashed or lost
its DB connection.
</div>
@if (_rows is null)
{
<p>Loading…</p>
}
else if (_rows.Count == 0)
{
<div class="alert alert-secondary">
No host-status rows yet. The Server publishes its first tick 2s after startup; if this list stays empty, check that the Server is running and the driver implements <code>IHostConnectivityProbe</code>.
</div>
}
else
{
<div class="row g-3 mb-4">
<div class="col-md-3"><div class="card"><div class="card-body">
<h6 class="text-muted mb-1">Hosts</h6>
<div class="fs-3">@_rows.Count</div>
</div></div></div>
<div class="col-md-3"><div class="card border-success"><div class="card-body">
<h6 class="text-muted mb-1">Running</h6>
<div class="fs-3 text-success">@_rows.Count(r => r.State == DriverHostState.Running && !HostStatusService.IsStale(r))</div>
</div></div></div>
<div class="col-md-3"><div class="card border-warning"><div class="card-body">
<h6 class="text-muted mb-1">Stale</h6>
<div class="fs-3 text-warning">@_rows.Count(HostStatusService.IsStale)</div>
</div></div></div>
<div class="col-md-3"><div class="card border-danger"><div class="card-body">
<h6 class="text-muted mb-1">Faulted</h6>
<div class="fs-3 text-danger">@_rows.Count(r => r.State == DriverHostState.Faulted)</div>
</div></div></div>
</div>
@if (_rows.Any(HostStatusService.IsFlagged))
{
var flaggedCount = _rows.Count(HostStatusService.IsFlagged);
<div class="alert alert-danger small mb-3">
<strong>@flaggedCount host@(flaggedCount == 1 ? "" : "s")</strong>
reporting ≥ @HostStatusService.FailureFlagThreshold consecutive failures — circuit breaker
may trip soon. Inspect the resilience columns below to locate.
</div>
}
@foreach (var cluster in _rows.GroupBy(r => r.ClusterId ?? "(unassigned)").OrderBy(g => g.Key))
{
<h2 class="h5 mt-4">Cluster: <code>@cluster.Key</code></h2>
<table class="table table-sm table-hover align-middle">
<thead>
<tr>
<th>Node</th>
<th>Driver</th>
<th>Host</th>
<th>State</th>
<th class="text-end" title="Consecutive failures — resets when a call succeeds or the breaker closes">Fail#</th>
<th class="text-end" title="In-flight capability calls (bulkhead-depth proxy)">In-flight</th>
<th>Breaker opened</th>
<th>Last transition</th>
<th>Last seen</th>
<th>Detail</th>
</tr>
</thead>
<tbody>
@foreach (var r in cluster)
{
<tr class="@RowClass(r)">
<td><code>@r.NodeId</code></td>
<td><code>@r.DriverInstanceId</code></td>
<td>@r.HostName</td>
<td>
<span class="badge @StateBadge(r.State)">@r.State</span>
@if (HostStatusService.IsStale(r))
{
<span class="badge bg-warning text-dark ms-1">Stale</span>
}
@if (HostStatusService.IsFlagged(r))
{
<span class="badge bg-danger ms-1" title="≥ @HostStatusService.FailureFlagThreshold consecutive failures">Flagged</span>
}
</td>
<td class="text-end small @(HostStatusService.IsFlagged(r) ? "text-danger fw-bold" : "")">
@r.ConsecutiveFailures
</td>
<td class="text-end small">@r.CurrentBulkheadDepth</td>
<td class="small">
@(r.LastCircuitBreakerOpenUtc is null ? "—" : FormatAge(r.LastCircuitBreakerOpenUtc.Value))
</td>
<td class="small">@FormatAge(r.StateChangedUtc)</td>
<td class="small @(HostStatusService.IsStale(r) ? "text-warning" : "")">@FormatAge(r.LastSeenUtc)</td>
<td class="text-truncate small" style="max-width: 240px;" title="@r.Detail">@r.Detail</td>
</tr>
}
</tbody>
</table>
}
}
@code {
// Mirrors HostStatusPublisher.HeartbeatInterval — polling ahead of the broadcaster
// produces stale-looking rows mid-cycle.
private const int RefreshIntervalSeconds = 10;
private List<HostStatusRow>? _rows;
private bool _refreshing;
private DateTime? _lastRefreshUtc;
private Timer? _timer;
protected override async Task OnInitializedAsync()
{
await RefreshAsync();
_timer = new Timer(async _ => await InvokeAsync(RefreshAsync),
state: null,
dueTime: TimeSpan.FromSeconds(RefreshIntervalSeconds),
period: TimeSpan.FromSeconds(RefreshIntervalSeconds));
}
private async Task RefreshAsync()
{
if (_refreshing) return;
_refreshing = true;
try
{
using var scope = ScopeFactory.CreateScope();
var svc = scope.ServiceProvider.GetRequiredService<HostStatusService>();
_rows = (await svc.ListAsync()).ToList();
_lastRefreshUtc = DateTime.UtcNow;
}
finally
{
_refreshing = false;
StateHasChanged();
}
}
private static string RowClass(HostStatusRow r) => r.State switch
{
DriverHostState.Faulted => "table-danger",
_ when HostStatusService.IsStale(r) => "table-warning",
_ => "",
};
private static string StateBadge(DriverHostState s) => s switch
{
DriverHostState.Running => "bg-success",
DriverHostState.Stopped => "bg-secondary",
DriverHostState.Faulted => "bg-danger",
_ => "bg-secondary",
};
private static string FormatAge(DateTime t)
{
var age = DateTime.UtcNow - t;
if (age.TotalSeconds < 60) return $"{(int)age.TotalSeconds}s ago";
if (age.TotalMinutes < 60) return $"{(int)age.TotalMinutes}m ago";
if (age.TotalHours < 24) return $"{(int)age.TotalHours}h ago";
return t.ToString("yyyy-MM-dd HH:mm 'UTC'");
}
public void Dispose() => _timer?.Dispose();
}