Joseph Doherty
935 lines
49 KiB
Plaintext
935 lines
49 KiB
Plaintext
@page "/monitoring/health"
|
|
@attribute [Authorize]
|
|
@using ZB.MOM.WW.ScadaBridge.CentralUI.Components.Health
|
|
@using ZB.MOM.WW.ScadaBridge.CentralUI.Services
|
|
@using ZB.MOM.WW.ScadaBridge.Commons.Types
|
|
@using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums
|
|
@using ZB.MOM.WW.ScadaBridge.Commons.Entities.Sites
|
|
@using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories
|
|
@using ZB.MOM.WW.ScadaBridge.HealthMonitoring
|
|
@using ZB.MOM.WW.ScadaBridge.Commons.Messages.Notification
|
|
@using ZB.MOM.WW.ScadaBridge.Commons.Messages.Audit
|
|
@using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit
|
|
@using ZB.MOM.WW.ScadaBridge.Commons.Types.Kpi
|
|
@using ZB.MOM.WW.ScadaBridge.Communication
|
|
@implements IDisposable
|
|
@inject ICentralHealthAggregator HealthAggregator
|
|
@inject ISiteRepository SiteRepository
|
|
@inject IKpiSnapshotCache KpiSnapshots
|
|
@inject IKpiHistoryQueryService KpiHistory
|
|
@inject Microsoft.Extensions.Options.IOptions<HealthMonitoringOptions> HealthOptions
|
|
|
|
<div class="container-fluid mt-3">
|
|
<div class="d-flex justify-content-between align-items-center mb-3">
|
|
<h4 class="mb-0">Health Dashboard</h4>
|
|
<div>
|
|
<span class="text-muted small me-2">Auto-refresh: @(_autoRefreshSeconds)s</span>
|
|
<button class="btn btn-outline-secondary btn-sm" @onclick="RefreshNow">Refresh Now</button>
|
|
</div>
|
|
</div>
|
|
|
|
@* Headline KPI groups. Each is a self-contained panel, so each gets its own card
|
|
boundary — the same treatment the Site Health Trends group below already uses,
|
|
including tiles-that-are-cards nested in a card body (Trends does exactly that
|
|
with KpiTrendChart). The two component-rendered groups own their heading and
|
|
"View details" link internally, so the Notification Outbox group keeps its
|
|
heading inside the card body too rather than promoting it to a card-header:
|
|
uniformity across the panels beats matching the Trends card, whose card-header
|
|
exists to carry the site selector and window toggle. card-body pb-0 lets each
|
|
tile row's own mb-3 supply the bottom padding instead of doubling it. *@
|
|
|
|
@* Notification Outbox headline KPIs — a central concern, shown regardless of site reports *@
|
|
<div class="card mb-3">
|
|
<div class="card-body pb-0">
|
|
<div class="d-flex justify-content-between align-items-center mb-2">
|
|
<h6 class="text-muted mb-0">Notification Outbox</h6>
|
|
<a class="small" href="/notifications/kpis">View details →</a>
|
|
</div>
|
|
<div class="row g-3 mb-3">
|
|
<div class="col-lg-4 col-md-6 col-12">
|
|
<div class="card h-100">
|
|
<div class="card-body text-center">
|
|
<h3 class="mb-0">@OutboxTileValue(_outboxKpi.QueueDepth)</h3>
|
|
<small class="text-muted">Queue Depth</small>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<div class="col-lg-4 col-md-6 col-12">
|
|
<div class="card h-100 @(_outboxKpiAvailable && _outboxKpi.StuckCount > 0 ? "border-warning" : "")">
|
|
<div class="card-body text-center">
|
|
<h3 class="mb-0 @(_outboxKpiAvailable && _outboxKpi.StuckCount > 0 ? "text-warning" : "")">@OutboxTileValue(_outboxKpi.StuckCount)</h3>
|
|
<small class="text-muted">Stuck</small>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<div class="col-lg-4 col-md-6 col-12">
|
|
<div class="card h-100 @(_outboxKpiAvailable && _outboxKpi.ParkedCount > 0 ? "border-danger" : "")">
|
|
<div class="card-body text-center">
|
|
<h3 class="mb-0 @(_outboxKpiAvailable && _outboxKpi.ParkedCount > 0 ? "text-danger" : "")">@OutboxTileValue(_outboxKpi.ParkedCount)</h3>
|
|
<small class="text-muted">Parked</small>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
@if (!_outboxKpiAvailable && _outboxKpiError != null)
|
|
{
|
|
<div class="text-muted small mb-3">Notification Outbox KPIs unavailable: @_outboxKpiError</div>
|
|
}
|
|
</div>
|
|
</div>
|
|
|
|
@* Site Call Audit (#22) Task 7 — three KPI tiles for the Site Call channel
|
|
(buffered / stuck / parked). Refreshed alongside the site states. *@
|
|
<div class="card mb-3">
|
|
<div class="card-body pb-0">
|
|
<SiteCallKpiTiles Snapshot="@_siteCallKpi"
|
|
IsAvailable="@_siteCallKpiAvailable"
|
|
ErrorMessage="@_siteCallKpiError"
|
|
PerNodeSnapshots="@_siteCallNodeKpis"
|
|
PerNodeAvailable="@_siteCallNodeKpiAvailable" />
|
|
</div>
|
|
</div>
|
|
|
|
@* Three KPI tiles for the Audit channel
|
|
(volume / error rate / backlog). Refreshed alongside the site states. *@
|
|
<div class="card mb-3">
|
|
<div class="card-body pb-0">
|
|
<AuditKpiTiles Snapshot="@_auditKpi"
|
|
IsAvailable="@_auditKpiAvailable"
|
|
ErrorMessage="@_auditKpiError" />
|
|
</div>
|
|
</div>
|
|
|
|
@* Site Health Trends (M6) — per-site Site Health KPI history. Loads on a
|
|
separate path from the 10s tile-refresh timer so a trend-query fault can
|
|
never disturb the live dashboard or its polling loop. The site selector
|
|
reuses the site keys already loaded into _siteStates; the window toggle
|
|
drives the time range. Both re-query independently. *@
|
|
<div class="card mb-3" data-test="site-health-trends">
|
|
<div class="card-header d-flex justify-content-between align-items-center py-2">
|
|
<div class="d-flex align-items-center">
|
|
<h6 class="text-muted mb-0 me-3">Site Health Trends</h6>
|
|
@if (_trendSiteKeys.Count > 0)
|
|
{
|
|
@* OnTrendSiteChangedAsync is async — Blazor wraps it in an EventCallback so the returned Task IS awaited; keep the Async signature, do not change it to a void/fire-and-forget handler. This comment MUST stay OUTSIDE the <select> start tag: a Razor comment placed between attributes renders as an invalid attribute name and crashes the SignalR circuit (InvalidCharacterError on setAttribute). *@
|
|
<select class="form-select form-select-sm" style="width:auto"
|
|
data-test="site-health-trends-site"
|
|
value="@_trendSiteId"
|
|
@onchange="OnTrendSiteChangedAsync">
|
|
@foreach (var key in _trendSiteKeys)
|
|
{
|
|
<option value="@key">@TrendSiteLabel(key)</option>
|
|
}
|
|
</select>
|
|
}
|
|
</div>
|
|
<div class="btn-group btn-group-sm" role="group" aria-label="Trend window">
|
|
<button type="button"
|
|
class="btn @(_trendWindowHours == 24 ? "btn-primary" : "btn-outline-secondary")"
|
|
@onclick="() => SetTrendWindowAsync(24)" disabled="@_trendsLoading">24h</button>
|
|
<button type="button"
|
|
class="btn @(_trendWindowHours == 168 ? "btn-primary" : "btn-outline-secondary")"
|
|
@onclick="() => SetTrendWindowAsync(168)" disabled="@_trendsLoading">7d</button>
|
|
<button type="button"
|
|
class="btn @(_trendWindowHours == 720 ? "btn-primary" : "btn-outline-secondary")"
|
|
@onclick="() => SetTrendWindowAsync(720)" disabled="@_trendsLoading">30d</button>
|
|
<button type="button"
|
|
class="btn @(_trendWindowHours == 2160 ? "btn-primary" : "btn-outline-secondary")"
|
|
@onclick="() => SetTrendWindowAsync(2160)" disabled="@_trendsLoading">90d</button>
|
|
</div>
|
|
</div>
|
|
<div class="card-body p-3">
|
|
@if (_trendSiteKeys.Count == 0)
|
|
{
|
|
<span class="text-muted small">No sites available for trends yet.</span>
|
|
}
|
|
else
|
|
{
|
|
<div class="row g-3">
|
|
<div class="col-lg-3 col-md-6">
|
|
<KpiTrendChart Title="Connections Down"
|
|
Points="@_connectionsDownSeries"
|
|
IsAvailable="@_connectionsDownAvailable"
|
|
ErrorMessage="@_connectionsDownError" />
|
|
</div>
|
|
<div class="col-lg-3 col-md-6">
|
|
<KpiTrendChart Title="Dead Letters"
|
|
Points="@_deadLettersSeries"
|
|
IsAvailable="@_deadLettersAvailable"
|
|
ErrorMessage="@_deadLettersError" />
|
|
</div>
|
|
<div class="col-lg-3 col-md-6">
|
|
<KpiTrendChart Title="Script Errors"
|
|
Points="@_scriptErrorsSeries"
|
|
IsAvailable="@_scriptErrorsAvailable"
|
|
ErrorMessage="@_scriptErrorsError" />
|
|
</div>
|
|
<div class="col-lg-3 col-md-6">
|
|
<KpiTrendChart Title="S&F Buffer Depth"
|
|
Points="@_sfBufferDepthSeries"
|
|
IsAvailable="@_sfBufferDepthAvailable"
|
|
ErrorMessage="@_sfBufferDepthError" />
|
|
</div>
|
|
</div>
|
|
}
|
|
</div>
|
|
</div>
|
|
|
|
@if (_siteStates.Count == 0)
|
|
{
|
|
<div class="alert alert-info">No site health reports received yet.</div>
|
|
}
|
|
else
|
|
{
|
|
@* Overview cards — the fourth headline tile row, so it takes the same card
|
|
boundary as the three above it. It was also the only tile row on the page
|
|
with no heading at all; it now carries one for the same reason. *@
|
|
<div class="card mb-3">
|
|
<div class="card-body pb-0">
|
|
<div class="d-flex justify-content-between align-items-center mb-2">
|
|
<h6 class="text-muted mb-0">Sites</h6>
|
|
</div>
|
|
<div class="row g-3 mb-3">
|
|
<div class="col-lg-4 col-md-6 col-12">
|
|
<div class="card border-success h-100">
|
|
<div class="card-body text-center">
|
|
<h3 class="mb-0 text-success">@_siteStates.Values.Count(s => s.IsOnline)</h3>
|
|
<small class="text-muted">Sites Online</small>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<div class="col-lg-4 col-md-6 col-12">
|
|
<div class="card border-danger h-100">
|
|
<div class="card-body text-center">
|
|
<h3 class="mb-0 text-danger">@_siteStates.Values.Count(s => !s.IsOnline)</h3>
|
|
<small class="text-muted">Sites Offline</small>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<div class="col-lg-4 col-md-6 col-12">
|
|
<div class="card border-warning h-100">
|
|
<div class="card-body text-center">
|
|
<h3 class="mb-0 text-warning">@_siteStates.Values.Count(SiteHasActiveErrors)</h3>
|
|
<small class="text-muted">Sites with active errors</small>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
@* Per-site detail cards — central cluster pinned to the top, then sites alphabetically *@
|
|
@foreach (var (siteId, state) in _siteStates.OrderBy(s => s.Key == CentralHealthReportLoop.CentralSiteId ? 0 : 1).ThenBy(s => s.Key))
|
|
{
|
|
var isCentral = siteId == CentralHealthReportLoop.CentralSiteId;
|
|
var siteName = isCentral ? "Central Cluster" : GetSiteName(siteId);
|
|
var detailsCollapseId = $"site-details-{siteId}";
|
|
<div class="card mb-3">
|
|
<div class="card-header d-flex justify-content-between align-items-center py-2">
|
|
@* Display name and machine id are two different things and are read
|
|
for two different reasons, so they get two lines: chips + name on
|
|
the first, the site identifier on its own beneath it. Reference
|
|
shape: Pages/Admin/Sites.razor (card title + monospace identifier). *@
|
|
<div>
|
|
<div class="d-flex align-items-center flex-wrap gap-2">
|
|
@if (state.IsOnline)
|
|
{
|
|
<span class="badge bg-success" aria-label="State: Online">@OnlineGlyph Online</span>
|
|
}
|
|
else
|
|
{
|
|
<span class="badge bg-danger" aria-label="State: Offline">@OfflineGlyph Offline</span>
|
|
}
|
|
<strong class="fs-5">@siteName</strong>
|
|
@if (state.IsOnline && state.IsMetricsStale)
|
|
{
|
|
<span class="badge bg-warning text-dark"
|
|
title="Site is heartbeating but has sent no health report for over @StaleTimeoutDisplay — the metrics pipeline may be dead.">
|
|
Metrics stale
|
|
</span>
|
|
}
|
|
@if (!state.IsOnline && state.LastStatusChangeAt is { } changedAt)
|
|
{
|
|
<small class="text-muted">offline since @changedAt.ToString("u")</small>
|
|
}
|
|
</div>
|
|
@if (!isCentral)
|
|
{
|
|
<div class="small text-muted font-monospace">@siteId</div>
|
|
}
|
|
</div>
|
|
<div class="d-flex align-items-center gap-3">
|
|
@* Admin-only; the control gates itself, and disables when the pair has
|
|
no online standby to take over. Central acts on the local cluster;
|
|
a site is asked over the command/control relay. *@
|
|
<CentralFailoverControl OnlineCentralNodeCount="@OnlineNodeCount(state)"
|
|
SiteId="@(isCentral ? null : siteId)" />
|
|
<small class="text-muted">
|
|
Last report: <TimestampDisplay Value="@state.LastReportReceivedAt" Format="HH:mm:ss" NullText="awaiting first report" />
|
|
| Last heartbeat: <TimestampDisplay Value="@state.LastHeartbeatAt" Format="HH:mm:ss" />
|
|
| Seq: @state.LastSequenceNumber
|
|
</small>
|
|
</div>
|
|
</div>
|
|
<div class="card-body p-3">
|
|
@if (state.LatestReport != null)
|
|
{
|
|
var report = state.LatestReport;
|
|
<div class="row g-3">
|
|
@* Each logical group below sits in its own card, with the group's
|
|
head promoted to a real card-header, instead of h6 rules stacked
|
|
inside one undifferentiated body. Columns 3 and 4 keep their
|
|
existing single collapse region (and its id) spanning both of
|
|
their sub-sections: the disclosure toggle IS the card-header and
|
|
the two sub-heads live inside the one card body. *@
|
|
|
|
@* Column 1: Nodes *@
|
|
<div class="col-md-6">
|
|
<div class="card h-100">
|
|
<div class="card-header py-2">
|
|
<h6 class="text-muted mb-0">Nodes (@NodeRowCount(state))</h6>
|
|
</div>
|
|
<div class="card-body p-2">
|
|
<table class="table table-sm table-borderless mb-0">
|
|
<tbody>
|
|
@if (report.ClusterNodes is { Count: > 0 })
|
|
{
|
|
@foreach (var node in report.ClusterNodes)
|
|
{
|
|
<tr>
|
|
<td class="small">@node.Hostname</td>
|
|
<td>
|
|
<span class="badge @(node.IsOnline ? "bg-success" : "bg-danger")"
|
|
aria-label="State: @(node.IsOnline ? "Online" : "Offline")">
|
|
@(node.IsOnline ? OnlineGlyph : OfflineGlyph) @(node.IsOnline ? "Online" : "Offline")
|
|
</span>
|
|
</td>
|
|
<td>
|
|
<span class="badge @(node.Role == "Primary" ? "bg-primary" : "bg-secondary")"
|
|
aria-label="State: @node.Role">
|
|
@(node.Role == "Primary" ? PrimaryGlyph : StandbyGlyph) @node.Role
|
|
</span>
|
|
</td>
|
|
</tr>
|
|
}
|
|
}
|
|
else
|
|
{
|
|
<tr>
|
|
<td class="small">@(report.NodeHostname != "" ? report.NodeHostname : "Node")</td>
|
|
<td>
|
|
<span class="badge @(state.IsOnline ? "bg-success" : "bg-danger")"
|
|
aria-label="State: @(state.IsOnline ? "Online" : "Offline")">
|
|
@(state.IsOnline ? OnlineGlyph : OfflineGlyph) @(state.IsOnline ? "Online" : "Offline")
|
|
</span>
|
|
</td>
|
|
<td>
|
|
@{
|
|
var roleLabel = report.NodeRole == "Active" ? "Primary" : "Standby";
|
|
}
|
|
<span class="badge @(report.NodeRole == "Active" ? "bg-primary" : "bg-secondary")"
|
|
aria-label="State: @roleLabel">
|
|
@(roleLabel == "Primary" ? PrimaryGlyph : StandbyGlyph) @roleLabel
|
|
</span>
|
|
</td>
|
|
</tr>
|
|
}
|
|
</tbody>
|
|
</table>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
@* Column 2: Data Connections (collapsible) *@
|
|
<div class="col-md-6">
|
|
<div class="card h-100">
|
|
<div class="card-header py-2">
|
|
<button class="btn btn-link btn-sm p-0 text-decoration-none"
|
|
data-bs-toggle="collapse"
|
|
data-bs-target="@($"#{detailsCollapseId}-conns")"
|
|
aria-expanded="false">
|
|
Data Connections (@report.DataConnectionStatuses.Count)
|
|
</button>
|
|
</div>
|
|
<div class="collapse" id="@($"{detailsCollapseId}-conns")">
|
|
<div class="card-body p-2">
|
|
@if (report.DataConnectionStatuses.Count == 0)
|
|
{
|
|
<span class="text-muted small">None</span>
|
|
}
|
|
else
|
|
{
|
|
@foreach (var (connName, health) in report.DataConnectionStatuses)
|
|
{
|
|
var endpoint = report.DataConnectionEndpoints?.GetValueOrDefault(connName);
|
|
var quality = report.DataConnectionTagQuality?.GetValueOrDefault(connName);
|
|
var endpointText = endpoint ?? health.ToString();
|
|
<div class="mb-2">
|
|
@* Both halves are site-supplied strings — a long connection
|
|
name or endpoint URL would otherwise wrap this flex row
|
|
into several lines. Clip both, full value on the title. *@
|
|
<div class="d-flex justify-content-between gap-2">
|
|
<strong class="small cell-clip cell-clip-sm" title="@connName">@connName</strong>
|
|
<span class="small cell-clip cell-clip-sm text-end" title="@endpointText">@endpointText</span>
|
|
</div>
|
|
@if (quality != null)
|
|
{
|
|
<table class="table table-sm table-borderless mb-0">
|
|
<tbody>
|
|
<tr>
|
|
<td class="small text-muted py-0">Tags good</td>
|
|
<td class="small text-end py-0">@quality.Good.ToString("N0")</td>
|
|
</tr>
|
|
<tr>
|
|
<td class="small text-muted py-0">Tags bad</td>
|
|
<td class="small text-end py-0">@quality.Bad.ToString("N0")</td>
|
|
</tr>
|
|
<tr>
|
|
<td class="small text-muted py-0">Tags uncertain</td>
|
|
<td class="small text-end py-0">@quality.Uncertain.ToString("N0")</td>
|
|
</tr>
|
|
</tbody>
|
|
</table>
|
|
}
|
|
</div>
|
|
}
|
|
}
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
@* Column 3: Instances + Store-and-Forward (collapsible) *@
|
|
<div class="col-md-6">
|
|
<div class="card h-100">
|
|
<div class="card-header py-2">
|
|
<button class="btn btn-link btn-sm p-0 text-decoration-none"
|
|
data-bs-toggle="collapse"
|
|
data-bs-target="@($"#{detailsCollapseId}-queues")"
|
|
aria-expanded="false">
|
|
Instances & Queues
|
|
</button>
|
|
</div>
|
|
<div class="collapse" id="@($"{detailsCollapseId}-queues")">
|
|
<div class="card-body p-2">
|
|
<h6 class="text-muted mb-2 border-bottom pb-1">Instances</h6>
|
|
<table class="table table-sm table-borderless mb-0">
|
|
<tbody>
|
|
<tr>
|
|
<td class="small">Deployed</td>
|
|
<td class="text-end">@report.DeployedInstanceCount</td>
|
|
</tr>
|
|
<tr>
|
|
<td class="small">Enabled</td>
|
|
<td class="text-end text-success">@report.EnabledInstanceCount</td>
|
|
</tr>
|
|
<tr>
|
|
<td class="small">Disabled</td>
|
|
<td class="text-end">@report.DisabledInstanceCount</td>
|
|
</tr>
|
|
</tbody>
|
|
</table>
|
|
|
|
<h6 class="text-muted mb-2 mt-3 border-bottom pb-1">
|
|
Store-and-Forward Buffers (@report.StoreAndForwardBufferDepths.Count)
|
|
</h6>
|
|
@if (report.StoreAndForwardBufferDepths.Count == 0)
|
|
{
|
|
<span class="text-muted small">Empty</span>
|
|
}
|
|
else
|
|
{
|
|
@foreach (var (category, depth) in report.StoreAndForwardBufferDepths)
|
|
{
|
|
<div class="d-flex justify-content-between mb-1">
|
|
<span class="small">@category</span>
|
|
<span class="badge @(depth > 0 ? "bg-warning text-dark" : "bg-light text-dark")">@depth</span>
|
|
</div>
|
|
}
|
|
}
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
@* Column 4: Error Counts + Parked Messages (collapsible) *@
|
|
<div class="col-md-6">
|
|
<div class="card h-100">
|
|
<div class="card-header py-2">
|
|
<button class="btn btn-link btn-sm p-0 text-decoration-none"
|
|
data-bs-toggle="collapse"
|
|
data-bs-target="@($"#{detailsCollapseId}-errors")"
|
|
aria-expanded="false">
|
|
Errors & Parked Messages
|
|
</button>
|
|
</div>
|
|
<div class="collapse" id="@($"{detailsCollapseId}-errors")">
|
|
<div class="card-body p-2">
|
|
<h6 class="text-muted mb-2 border-bottom pb-1">Error Counts</h6>
|
|
<table class="table table-sm table-borderless mb-0">
|
|
<tbody>
|
|
<tr>
|
|
<td class="small">Script Errors</td>
|
|
<td class="text-end">
|
|
<span class="@(report.ScriptErrorCount > 0 ? "text-danger fw-bold" : "")">@report.ScriptErrorCount</span>
|
|
</td>
|
|
</tr>
|
|
<tr>
|
|
<td class="small">Alarm Eval Errors</td>
|
|
<td class="text-end">
|
|
<span class="@(report.AlarmEvaluationErrorCount > 0 ? "text-warning fw-bold" : "")">@report.AlarmEvaluationErrorCount</span>
|
|
</td>
|
|
</tr>
|
|
<tr>
|
|
<td class="small">Dead Letters</td>
|
|
<td class="text-end">
|
|
<span class="@(report.DeadLetterCount > 0 ? "text-danger fw-bold" : "")">@report.DeadLetterCount</span>
|
|
</td>
|
|
</tr>
|
|
</tbody>
|
|
</table>
|
|
|
|
<h6 class="text-muted mb-2 mt-3 border-bottom pb-1">Parked Messages</h6>
|
|
@if (report.ParkedMessageCount == 0)
|
|
{
|
|
<span class="text-muted small">Empty</span>
|
|
}
|
|
else
|
|
{
|
|
<span class="badge bg-warning text-dark">@report.ParkedMessageCount</span>
|
|
}
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
}
|
|
else
|
|
{
|
|
<span class="text-muted">No report data available.</span>
|
|
}
|
|
</div>
|
|
</div>
|
|
}
|
|
}
|
|
</div>
|
|
|
|
@code {
|
|
// Shape-coded status glyphs to pair with badge colour.
|
|
private const string OnlineGlyph = "●"; // ●
|
|
private const string OfflineGlyph = "○"; // ○
|
|
private const string PrimaryGlyph = "▲"; // ▲
|
|
private const string StandbyGlyph = "△"; // △
|
|
|
|
// Human-friendly rendering of the configured metrics-stale window for the
|
|
// "Metrics stale" badge tooltip (e.g. "2 minutes").
|
|
private string StaleTimeoutDisplay =>
|
|
FormatDuration(HealthOptions.Value.MetricsStaleTimeout);
|
|
|
|
// Online central nodes, from the same ClusterNodes list the Nodes column renders.
|
|
// Fewer than two means no standby, which disables the manual-failover control. This is
|
|
// a display-side guard only — AkkaManualFailoverService re-checks against live cluster
|
|
// membership, which is the authoritative answer.
|
|
private static int OnlineNodeCount(SiteHealthState state) =>
|
|
state.LatestReport?.ClusterNodes is { Count: > 0 } nodes
|
|
? nodes.Count(n => n.IsOnline)
|
|
: (state.IsOnline ? 1 : 0);
|
|
|
|
// Rows the Nodes group actually renders: the reported cluster-node list when the
|
|
// site sends one, otherwise the single synthesized fallback row. Shown in that
|
|
// group's card-header so the count is legible without reading the table — the same
|
|
// metadata-in-the-header treatment the Data Connections group already had inline.
|
|
private static int NodeRowCount(SiteHealthState state) =>
|
|
state.LatestReport?.ClusterNodes is { Count: > 0 } nodes ? nodes.Count : 1;
|
|
|
|
private static string FormatDuration(TimeSpan span) =>
|
|
span.TotalMinutes >= 1 && span == TimeSpan.FromMinutes(Math.Round(span.TotalMinutes))
|
|
? $"{span.TotalMinutes:0} minute{(span.TotalMinutes == 1 ? "" : "s")}"
|
|
: $"{span.TotalSeconds:0} seconds";
|
|
|
|
private IReadOnlyDictionary<string, SiteHealthState> _siteStates = new Dictionary<string, SiteHealthState>();
|
|
private Dictionary<string, string> _siteNames = new();
|
|
private Timer? _refreshTimer;
|
|
private readonly ZB.MOM.WW.ScadaBridge.CentralUI.Services.PollGate _pollGate = new();
|
|
private int _autoRefreshSeconds = 10;
|
|
|
|
// Notification Outbox headline KPIs, refreshed alongside the site states.
|
|
private NotificationKpiResponse _outboxKpi =
|
|
new(
|
|
CorrelationId: string.Empty,
|
|
Success: false,
|
|
ErrorMessage: null,
|
|
QueueDepth: 0,
|
|
StuckCount: 0,
|
|
ParkedCount: 0,
|
|
DeliveredLastInterval: 0,
|
|
OldestPendingAge: null);
|
|
private bool _outboxKpiAvailable;
|
|
private string? _outboxKpiError;
|
|
|
|
// Audit KPI tiles. Volume + error rate come
|
|
// from a 1h aggregate over the central AuditLog table; backlog sums the
|
|
// per-site SiteAuditBacklog.PendingCount via the health aggregator.
|
|
private AuditLogKpiSnapshot? _auditKpi;
|
|
private bool _auditKpiAvailable;
|
|
private string? _auditKpiError;
|
|
|
|
// Site Call Audit (#22) Task 7 — Site Call KPI tiles. Point-in-time counts
|
|
// from the central SiteCalls table, fetched alongside the site states. The
|
|
// SiteCallKpiResponse message doubles as the snapshot the tile takes.
|
|
private SiteCallKpiResponse? _siteCallKpi;
|
|
private bool _siteCallKpiAvailable;
|
|
private string? _siteCallKpiError;
|
|
|
|
// Per-node Site Call KPI breakdown (M5.2 per-node stuck-count KPIs).
|
|
// Passed to SiteCallKpiTiles as an optional sub-table.
|
|
private IReadOnlyList<SiteCallNodeKpiSnapshot> _siteCallNodeKpis =
|
|
Array.Empty<SiteCallNodeKpiSnapshot>();
|
|
private bool _siteCallNodeKpiAvailable;
|
|
|
|
// ── Site Health Trends (M6) ───────────────────────────────────────────────
|
|
// Per-site Site Health KPI history, loaded on a path entirely separate from
|
|
// the 10s tile-refresh timer (LoadSiteHealthTrendsAsync, never called from
|
|
// the timer tick). The site keys are a snapshot of the dashboard's site set,
|
|
// captured each time trends load so the selector mirrors the live cards.
|
|
// Window in hours: 24h (default) or 168h (7d). Changing the selected site OR
|
|
// the window re-queries. Each metric chart carries its own availability +
|
|
// error so one failed GetSeriesAsync degrades a single chart, never the
|
|
// dashboard.
|
|
private IReadOnlyList<string> _trendSiteKeys = Array.Empty<string>();
|
|
private string? _trendSiteId;
|
|
private int _trendWindowHours = 24;
|
|
private bool _trendsLoading;
|
|
|
|
private IReadOnlyList<KpiSeriesPoint>? _connectionsDownSeries;
|
|
private bool _connectionsDownAvailable = true;
|
|
private string? _connectionsDownError;
|
|
|
|
private IReadOnlyList<KpiSeriesPoint>? _deadLettersSeries;
|
|
private bool _deadLettersAvailable = true;
|
|
private string? _deadLettersError;
|
|
|
|
private IReadOnlyList<KpiSeriesPoint>? _scriptErrorsSeries;
|
|
private bool _scriptErrorsAvailable = true;
|
|
private string? _scriptErrorsError;
|
|
|
|
private IReadOnlyList<KpiSeriesPoint>? _sfBufferDepthSeries;
|
|
private bool _sfBufferDepthAvailable = true;
|
|
private string? _sfBufferDepthError;
|
|
|
|
private static bool SiteHasActiveErrors(SiteHealthState state)
|
|
{
|
|
var report = state.LatestReport;
|
|
if (report == null) return false;
|
|
return report.ScriptErrorCount > 0
|
|
|| report.AlarmEvaluationErrorCount > 0
|
|
|| report.DeadLetterCount > 0;
|
|
}
|
|
|
|
protected override async Task OnInitializedAsync()
|
|
{
|
|
// Load site names for display
|
|
try
|
|
{
|
|
var sites = await SiteRepository.GetAllSitesAsync();
|
|
_siteNames = sites.ToDictionary(s => s.SiteIdentifier, s => s.Name);
|
|
}
|
|
catch
|
|
{
|
|
// Non-fatal — fall back to showing siteId only
|
|
}
|
|
|
|
// First load rides whatever the shared KPI cache already has fresh — a second
|
|
// operator opening the dashboard inside the window costs no extra query.
|
|
await RefreshAllAsync(forceRefresh: false);
|
|
|
|
// Site Health Trends (M6) load on their own path — never from the
|
|
// timer tick below — so a trend-query fault can't disturb the live tile
|
|
// refresh. Seed the selector from the sites just loaded into _siteStates
|
|
// and query the default site.
|
|
await LoadSiteHealthTrendsAsync();
|
|
|
|
_refreshTimer = new Timer(_ =>
|
|
{
|
|
if (!_pollGate.TryEnter()) return;
|
|
InvokeAsync(async () =>
|
|
{
|
|
try
|
|
{
|
|
await RefreshAllAsync(forceRefresh: false);
|
|
StateHasChanged();
|
|
}
|
|
finally
|
|
{
|
|
_pollGate.Exit();
|
|
}
|
|
});
|
|
}, null, TimeSpan.FromSeconds(_autoRefreshSeconds), TimeSpan.FromSeconds(_autoRefreshSeconds));
|
|
}
|
|
|
|
// Re-query when the operator picks a different site. Best-effort: the load
|
|
// itself swallows faults per-chart.
|
|
private async Task OnTrendSiteChangedAsync(ChangeEventArgs e)
|
|
{
|
|
var selected = e.Value?.ToString();
|
|
if (string.IsNullOrEmpty(selected) || selected == _trendSiteId)
|
|
{
|
|
return;
|
|
}
|
|
|
|
_trendSiteId = selected;
|
|
await LoadSiteHealthTrendsAsync(refreshSiteKeys: false);
|
|
}
|
|
|
|
// Re-query when the window toggle changes (24h ↔ 7d).
|
|
private async Task SetTrendWindowAsync(int windowHours)
|
|
{
|
|
if (_trendWindowHours == windowHours)
|
|
{
|
|
return;
|
|
}
|
|
|
|
_trendWindowHours = windowHours;
|
|
await LoadSiteHealthTrendsAsync(refreshSiteKeys: false);
|
|
}
|
|
|
|
// Loads the four Site Health trend series for the selected site over the
|
|
// selected window. Deliberately decoupled from RefreshNow / the 10s timer:
|
|
// a fault here degrades the affected chart(s) only and never propagates to
|
|
// the tile-refresh loop.
|
|
//
|
|
// refreshSiteKeys re-snapshots the dashboard's site set into the selector
|
|
// (true on init); the site-change / window-toggle paths pass false so a
|
|
// mid-interaction site addition/removal can't yank the operator's choice.
|
|
private async Task LoadSiteHealthTrendsAsync(bool refreshSiteKeys = true)
|
|
{
|
|
if (refreshSiteKeys)
|
|
{
|
|
// Mirror the dashboard ordering: central cluster pinned first, then
|
|
// sites alphabetically — the same comparer the detail cards use.
|
|
_trendSiteKeys = _siteStates.Keys
|
|
.OrderBy(k => k == CentralHealthReportLoop.CentralSiteId ? 0 : 1)
|
|
.ThenBy(k => k)
|
|
.ToList();
|
|
|
|
// Default to the first site (or keep a still-valid prior selection).
|
|
if (_trendSiteId == null || !_trendSiteKeys.Contains(_trendSiteId))
|
|
{
|
|
_trendSiteId = _trendSiteKeys.FirstOrDefault();
|
|
}
|
|
}
|
|
|
|
if (string.IsNullOrEmpty(_trendSiteId))
|
|
{
|
|
return;
|
|
}
|
|
|
|
_trendsLoading = true;
|
|
try
|
|
{
|
|
var toUtc = DateTime.UtcNow;
|
|
var fromUtc = toUtc - TimeSpan.FromHours(_trendWindowHours);
|
|
var siteId = _trendSiteId;
|
|
|
|
(_connectionsDownSeries, _connectionsDownAvailable, _connectionsDownError) =
|
|
await LoadTrendSeriesAsync(KpiMetrics.SiteHealth.ConnectionsDown, siteId, fromUtc, toUtc);
|
|
(_deadLettersSeries, _deadLettersAvailable, _deadLettersError) =
|
|
await LoadTrendSeriesAsync(KpiMetrics.SiteHealth.DeadLetters, siteId, fromUtc, toUtc);
|
|
(_scriptErrorsSeries, _scriptErrorsAvailable, _scriptErrorsError) =
|
|
await LoadTrendSeriesAsync(KpiMetrics.SiteHealth.ScriptErrors, siteId, fromUtc, toUtc);
|
|
(_sfBufferDepthSeries, _sfBufferDepthAvailable, _sfBufferDepthError) =
|
|
await LoadTrendSeriesAsync(KpiMetrics.SiteHealth.SfBufferDepth, siteId, fromUtc, toUtc);
|
|
}
|
|
finally
|
|
{
|
|
_trendsLoading = false;
|
|
}
|
|
}
|
|
|
|
// Single best-effort series fetch. Site Health metrics are Site-scoped, so
|
|
// scope = KpiScopes.Site and scopeKey = the selected site id. On any fault
|
|
// the chart falls back to the unavailable placeholder — a failure here must
|
|
// NEVER break the dashboard.
|
|
private async Task<(IReadOnlyList<KpiSeriesPoint>?, bool, string?)> LoadTrendSeriesAsync(
|
|
string metric, string siteId, DateTime fromUtc, DateTime toUtc)
|
|
{
|
|
try
|
|
{
|
|
var series = await KpiHistory.GetSeriesAsync(
|
|
KpiSources.SiteHealth, metric, KpiScopes.Site, siteId, fromUtc, toUtc);
|
|
return (series, true, null);
|
|
}
|
|
catch
|
|
{
|
|
return (null, false, "Trend data unavailable.");
|
|
}
|
|
}
|
|
|
|
private string TrendSiteLabel(string siteKey) =>
|
|
siteKey == CentralHealthReportLoop.CentralSiteId
|
|
? "Central Cluster"
|
|
: $"{GetSiteName(siteKey)} ({siteKey})";
|
|
|
|
// Per-tile budget for one refresh tick. The underlying KPI Asks carry the
|
|
// 30s CommunicationOptions.QueryTimeout, so without this a single hung
|
|
// singleton stalled the WHOLE tick (and, with a 10s timer, stacked ticks)
|
|
// rather than degrading its own tile. Cancelling here abandons only this
|
|
// circuit's wait — the shared KPI flight continues and its result lands in
|
|
// the process-level cache for the next tick.
|
|
private static readonly TimeSpan TileTimeout = TimeSpan.FromSeconds(5);
|
|
|
|
// The operator's explicit "Refresh Now" bypasses the shared cache's freshness
|
|
// window; the timer tick and the first load ride whatever is already fresh
|
|
// (which is the whole point — N dashboards share one query per window).
|
|
private Task RefreshNow() => RefreshAllAsync(forceRefresh: true);
|
|
|
|
// Every tile loads concurrently, each with its own short budget, so one slow
|
|
// KPI degrades one tile instead of the tick. Every loader swallows its own
|
|
// faults, so Task.WhenAll here can never observe an exception.
|
|
private async Task RefreshAllAsync(bool forceRefresh)
|
|
{
|
|
_siteStates = HealthAggregator.GetAllSiteStates();
|
|
await Task.WhenAll(
|
|
LoadOutboxKpis(forceRefresh),
|
|
LoadSiteCallKpis(forceRefresh),
|
|
LoadSiteCallNodeKpis(forceRefresh),
|
|
LoadAuditKpis(forceRefresh));
|
|
}
|
|
|
|
private async Task LoadOutboxKpis(bool forceRefresh)
|
|
{
|
|
try
|
|
{
|
|
using var timeout = new CancellationTokenSource(TileTimeout);
|
|
var response = await KpiSnapshots.GetNotificationKpisAsync(forceRefresh, timeout.Token);
|
|
if (response.Success)
|
|
{
|
|
_outboxKpi = response;
|
|
_outboxKpiAvailable = true;
|
|
_outboxKpiError = null;
|
|
}
|
|
else
|
|
{
|
|
_outboxKpiAvailable = false;
|
|
_outboxKpiError = response.ErrorMessage ?? "KPI query failed.";
|
|
}
|
|
}
|
|
catch (OperationCanceledException)
|
|
{
|
|
_outboxKpiAvailable = false;
|
|
_outboxKpiError = "KPI query timed out.";
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
_outboxKpiAvailable = false;
|
|
_outboxKpiError = $"KPI query failed: {ex.Message}";
|
|
}
|
|
}
|
|
|
|
// Site Call KPI loader: wraps the service call so a transient fault degrades
|
|
// the three Site Call tiles to em dashes with an inline error rather than
|
|
// killing the dashboard. Mirrors LoadOutboxKpis's error handling shape — a
|
|
// response with Success == false (repository fault) and an Ask that threw
|
|
// (transport fault) both collapse to "unavailable".
|
|
private async Task LoadSiteCallKpis(bool forceRefresh)
|
|
{
|
|
try
|
|
{
|
|
using var timeout = new CancellationTokenSource(TileTimeout);
|
|
var response = await KpiSnapshots.GetSiteCallKpisAsync(forceRefresh, timeout.Token);
|
|
if (response.Success)
|
|
{
|
|
_siteCallKpi = response;
|
|
_siteCallKpiAvailable = true;
|
|
_siteCallKpiError = null;
|
|
}
|
|
else
|
|
{
|
|
_siteCallKpiAvailable = false;
|
|
_siteCallKpiError = response.ErrorMessage ?? "KPI query failed.";
|
|
}
|
|
}
|
|
catch (OperationCanceledException)
|
|
{
|
|
_siteCallKpiAvailable = false;
|
|
_siteCallKpiError = "KPI query timed out.";
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
_siteCallKpiAvailable = false;
|
|
_siteCallKpiError = $"KPI query failed: {ex.Message}";
|
|
}
|
|
}
|
|
|
|
// Per-node site-call KPI loader (M5.2). Best-effort; a fault silently
|
|
// suppresses the per-node sub-table rather than degrading the dashboard.
|
|
private async Task LoadSiteCallNodeKpis(bool forceRefresh)
|
|
{
|
|
try
|
|
{
|
|
using var timeout = new CancellationTokenSource(TileTimeout);
|
|
var response = await KpiSnapshots.GetPerNodeSiteCallKpisAsync(forceRefresh, timeout.Token);
|
|
if (response.Success)
|
|
{
|
|
_siteCallNodeKpis = response.Nodes;
|
|
_siteCallNodeKpiAvailable = true;
|
|
}
|
|
else
|
|
{
|
|
_siteCallNodeKpiAvailable = false;
|
|
}
|
|
}
|
|
catch
|
|
{
|
|
_siteCallNodeKpiAvailable = false;
|
|
}
|
|
}
|
|
|
|
// Tiles show the numeric KPI when available, or an em dash when the outbox
|
|
// KPI query failed — matching how the page renders other unavailable data.
|
|
private string OutboxTileValue(int value) =>
|
|
_outboxKpiAvailable ? value.ToString() : "—";
|
|
|
|
// Audit KPI loader: wraps the service call so a transient DB outage degrades
|
|
// the three tiles to em dashes with an inline error rather than killing the
|
|
// dashboard. Mirrors LoadOutboxKpis's error handling shape.
|
|
private async Task LoadAuditKpis(bool forceRefresh)
|
|
{
|
|
try
|
|
{
|
|
using var timeout = new CancellationTokenSource(TileTimeout);
|
|
_auditKpi = await KpiSnapshots.GetAuditKpiSnapshotAsync(forceRefresh, timeout.Token);
|
|
_auditKpiAvailable = true;
|
|
_auditKpiError = null;
|
|
}
|
|
catch (OperationCanceledException)
|
|
{
|
|
_auditKpiAvailable = false;
|
|
_auditKpiError = "KPI query timed out.";
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
_auditKpiAvailable = false;
|
|
_auditKpiError = $"KPI query failed: {ex.Message}";
|
|
}
|
|
}
|
|
|
|
private string GetSiteName(string siteId)
|
|
{
|
|
return _siteNames.GetValueOrDefault(siteId, siteId);
|
|
}
|
|
|
|
private static string GetConnectionHealthBadge(ConnectionHealth health) => health switch
|
|
{
|
|
ConnectionHealth.Connected => "bg-success",
|
|
ConnectionHealth.Connecting => "bg-warning text-dark",
|
|
ConnectionHealth.Disconnected => "bg-danger",
|
|
_ => "bg-secondary"
|
|
};
|
|
|
|
public void Dispose()
|
|
{
|
|
_refreshTimer?.Dispose();
|
|
}
|
|
}
|