Joseph Doherty
c8e2f4da02
Central and each site are SEPARATE Akka clusters, so central cannot act on a
site's membership -- it asks. New TriggerSiteFailover/SiteFailoverAck contract
travels the existing ClusterClient command/control channel (mirroring the
RetryParkedOperation relay); the site's own SiteCommunicationActor performs the
graceful Leave and acks the outcome.
- ClusterFailoverCoordinator moved out of Host into Communication/ClusterState,
beside ActiveNodeEvaluator. Both paths now share ONE oldest-Up implementation;
SiteCommunicationActor cannot reference Host, and the two definitions must not
drift or the node asked to leave stops being the singleton host.
- Site scope is the SITE-SPECIFIC role (site-{SiteId}), not the base Site role --
site singletons are placed on the former, so the base role would move the wrong
node. Pinned by a unit test asserting the role string and by a real-cluster test.
- Site-side guards: refuses a command addressed to another site (a misroute must
never fail over a site the operator did not select), refuses when there is no
peer, and reports a fault as an ack rather than throwing into supervision --
a restart there would drop central's Ask into a bare timeout and lose the reason.
- Ack is sent before the Leave takes effect so it still reaches central.
- UI: the same control now serves both scopes via a SiteId parameter. The site
confirmation deliberately does NOT claim the admin's page will disconnect --
it won't, and crying wolf there devalues the central warning that is real. A
site refusal and an unreachable site surface distinctly.
- Rolling upgrade: a site on an older binary has no handler, so the message
dead-letters and the Ask times out, reported as "site did not respond". That
is the honest outcome; documented on the contract.
Fallout fixed: HealthPageTests now renders the page inside
CascadingAuthenticationState with the real policy set and IAuthorizationService,
because the cards embed an AuthorizeView. That mirrors production, where the
layout supplies the cascading value.
815 lines
40 KiB
Plaintext
815 lines
40 KiB
Plaintext
@page "/monitoring/health"
|
|
@attribute [Authorize]
|
|
@using ZB.MOM.WW.ScadaBridge.CentralUI.Components.Health
|
|
@using ZB.MOM.WW.ScadaBridge.CentralUI.Services
|
|
@using ZB.MOM.WW.ScadaBridge.Commons.Types
|
|
@using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums
|
|
@using ZB.MOM.WW.ScadaBridge.Commons.Entities.Sites
|
|
@using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories
|
|
@using ZB.MOM.WW.ScadaBridge.HealthMonitoring
|
|
@using ZB.MOM.WW.ScadaBridge.Commons.Messages.Notification
|
|
@using ZB.MOM.WW.ScadaBridge.Commons.Messages.Audit
|
|
@using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit
|
|
@using ZB.MOM.WW.ScadaBridge.Commons.Types.Kpi
|
|
@using ZB.MOM.WW.ScadaBridge.Communication
|
|
@implements IDisposable
|
|
@inject ICentralHealthAggregator HealthAggregator
|
|
@inject ISiteRepository SiteRepository
|
|
@inject CommunicationService CommunicationService
|
|
@inject IAuditLogQueryService AuditLogQueryService
|
|
@inject IKpiHistoryQueryService KpiHistory
|
|
@inject Microsoft.Extensions.Options.IOptions<HealthMonitoringOptions> HealthOptions
|
|
|
|
<div class="container-fluid mt-3">
|
|
<div class="d-flex justify-content-between align-items-center mb-3">
|
|
<h4 class="mb-0">Health Dashboard</h4>
|
|
<div>
|
|
<span class="text-muted small me-2">Auto-refresh: @(_autoRefreshSeconds)s</span>
|
|
<button class="btn btn-outline-secondary btn-sm" @onclick="RefreshNow">Refresh Now</button>
|
|
</div>
|
|
</div>
|
|
|
|
@* Notification Outbox headline KPIs — a central concern, shown regardless of site reports *@
|
|
<div class="d-flex justify-content-between align-items-center mb-2">
|
|
<h6 class="text-muted mb-0">Notification Outbox</h6>
|
|
<a class="small" href="/notifications/kpis">View details →</a>
|
|
</div>
|
|
<div class="row g-3 mb-3">
|
|
<div class="col-lg-4 col-md-6 col-12">
|
|
<div class="card h-100">
|
|
<div class="card-body text-center">
|
|
<h3 class="mb-0">@OutboxTileValue(_outboxKpi.QueueDepth)</h3>
|
|
<small class="text-muted">Queue Depth</small>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<div class="col-lg-4 col-md-6 col-12">
|
|
<div class="card h-100 @(_outboxKpiAvailable && _outboxKpi.StuckCount > 0 ? "border-warning" : "")">
|
|
<div class="card-body text-center">
|
|
<h3 class="mb-0 @(_outboxKpiAvailable && _outboxKpi.StuckCount > 0 ? "text-warning" : "")">@OutboxTileValue(_outboxKpi.StuckCount)</h3>
|
|
<small class="text-muted">Stuck</small>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<div class="col-lg-4 col-md-6 col-12">
|
|
<div class="card h-100 @(_outboxKpiAvailable && _outboxKpi.ParkedCount > 0 ? "border-danger" : "")">
|
|
<div class="card-body text-center">
|
|
<h3 class="mb-0 @(_outboxKpiAvailable && _outboxKpi.ParkedCount > 0 ? "text-danger" : "")">@OutboxTileValue(_outboxKpi.ParkedCount)</h3>
|
|
<small class="text-muted">Parked</small>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
@if (!_outboxKpiAvailable && _outboxKpiError != null)
|
|
{
|
|
<div class="text-muted small mb-3">Notification Outbox KPIs unavailable: @_outboxKpiError</div>
|
|
}
|
|
|
|
@* Site Call Audit (#22) Task 7 — three KPI tiles for the Site Call channel
|
|
(buffered / stuck / parked). Refreshed alongside the site states. *@
|
|
<SiteCallKpiTiles Snapshot="@_siteCallKpi"
|
|
IsAvailable="@_siteCallKpiAvailable"
|
|
ErrorMessage="@_siteCallKpiError"
|
|
PerNodeSnapshots="@_siteCallNodeKpis"
|
|
PerNodeAvailable="@_siteCallNodeKpiAvailable" />
|
|
|
|
@* Three KPI tiles for the Audit channel
|
|
(volume / error rate / backlog). Refreshed alongside the site states. *@
|
|
<AuditKpiTiles Snapshot="@_auditKpi"
|
|
IsAvailable="@_auditKpiAvailable"
|
|
ErrorMessage="@_auditKpiError" />
|
|
|
|
@* Site Health Trends (M6) — per-site Site Health KPI history. Loads on a
|
|
separate path from the 10s tile-refresh timer so a trend-query fault can
|
|
never disturb the live dashboard or its polling loop. The site selector
|
|
reuses the site keys already loaded into _siteStates; the window toggle
|
|
drives the time range. Both re-query independently. *@
|
|
<div class="card mb-3" data-test="site-health-trends">
|
|
<div class="card-header d-flex justify-content-between align-items-center py-2">
|
|
<div class="d-flex align-items-center">
|
|
<h6 class="text-muted mb-0 me-3">Site Health Trends</h6>
|
|
@if (_trendSiteKeys.Count > 0)
|
|
{
|
|
@* OnTrendSiteChangedAsync is async — Blazor wraps it in an EventCallback so the returned Task IS awaited; keep the Async signature, do not change it to a void/fire-and-forget handler. This comment MUST stay OUTSIDE the <select> start tag: a Razor comment placed between attributes renders as an invalid attribute name and crashes the SignalR circuit (InvalidCharacterError on setAttribute). *@
|
|
<select class="form-select form-select-sm" style="width:auto"
|
|
data-test="site-health-trends-site"
|
|
value="@_trendSiteId"
|
|
@onchange="OnTrendSiteChangedAsync">
|
|
@foreach (var key in _trendSiteKeys)
|
|
{
|
|
<option value="@key">@TrendSiteLabel(key)</option>
|
|
}
|
|
</select>
|
|
}
|
|
</div>
|
|
<div class="btn-group btn-group-sm" role="group" aria-label="Trend window">
|
|
<button type="button"
|
|
class="btn @(_trendWindowHours == 24 ? "btn-primary" : "btn-outline-secondary")"
|
|
@onclick="() => SetTrendWindowAsync(24)" disabled="@_trendsLoading">24h</button>
|
|
<button type="button"
|
|
class="btn @(_trendWindowHours == 168 ? "btn-primary" : "btn-outline-secondary")"
|
|
@onclick="() => SetTrendWindowAsync(168)" disabled="@_trendsLoading">7d</button>
|
|
<button type="button"
|
|
class="btn @(_trendWindowHours == 720 ? "btn-primary" : "btn-outline-secondary")"
|
|
@onclick="() => SetTrendWindowAsync(720)" disabled="@_trendsLoading">30d</button>
|
|
<button type="button"
|
|
class="btn @(_trendWindowHours == 2160 ? "btn-primary" : "btn-outline-secondary")"
|
|
@onclick="() => SetTrendWindowAsync(2160)" disabled="@_trendsLoading">90d</button>
|
|
</div>
|
|
</div>
|
|
<div class="card-body p-3">
|
|
@if (_trendSiteKeys.Count == 0)
|
|
{
|
|
<span class="text-muted small">No sites available for trends yet.</span>
|
|
}
|
|
else
|
|
{
|
|
<div class="row g-3">
|
|
<div class="col-lg-3 col-md-6">
|
|
<KpiTrendChart Title="Connections Down"
|
|
Points="@_connectionsDownSeries"
|
|
IsAvailable="@_connectionsDownAvailable"
|
|
ErrorMessage="@_connectionsDownError" />
|
|
</div>
|
|
<div class="col-lg-3 col-md-6">
|
|
<KpiTrendChart Title="Dead Letters"
|
|
Points="@_deadLettersSeries"
|
|
IsAvailable="@_deadLettersAvailable"
|
|
ErrorMessage="@_deadLettersError" />
|
|
</div>
|
|
<div class="col-lg-3 col-md-6">
|
|
<KpiTrendChart Title="Script Errors"
|
|
Points="@_scriptErrorsSeries"
|
|
IsAvailable="@_scriptErrorsAvailable"
|
|
ErrorMessage="@_scriptErrorsError" />
|
|
</div>
|
|
<div class="col-lg-3 col-md-6">
|
|
<KpiTrendChart Title="S&F Buffer Depth"
|
|
Points="@_sfBufferDepthSeries"
|
|
IsAvailable="@_sfBufferDepthAvailable"
|
|
ErrorMessage="@_sfBufferDepthError" />
|
|
</div>
|
|
</div>
|
|
}
|
|
</div>
|
|
</div>
|
|
|
|
@if (_siteStates.Count == 0)
|
|
{
|
|
<div class="alert alert-info">No site health reports received yet.</div>
|
|
}
|
|
else
|
|
{
|
|
@* Overview cards *@
|
|
<div class="row g-3 mb-3">
|
|
<div class="col-lg-4 col-md-6 col-12">
|
|
<div class="card border-success h-100">
|
|
<div class="card-body text-center">
|
|
<h3 class="mb-0 text-success">@_siteStates.Values.Count(s => s.IsOnline)</h3>
|
|
<small class="text-muted">Sites Online</small>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<div class="col-lg-4 col-md-6 col-12">
|
|
<div class="card border-danger h-100">
|
|
<div class="card-body text-center">
|
|
<h3 class="mb-0 text-danger">@_siteStates.Values.Count(s => !s.IsOnline)</h3>
|
|
<small class="text-muted">Sites Offline</small>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<div class="col-lg-4 col-md-6 col-12">
|
|
<div class="card border-warning h-100">
|
|
<div class="card-body text-center">
|
|
<h3 class="mb-0 text-warning">@_siteStates.Values.Count(SiteHasActiveErrors)</h3>
|
|
<small class="text-muted">Sites with active errors</small>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
@* Per-site detail cards — central cluster pinned to the top, then sites alphabetically *@
|
|
@foreach (var (siteId, state) in _siteStates.OrderBy(s => s.Key == CentralHealthReportLoop.CentralSiteId ? 0 : 1).ThenBy(s => s.Key))
|
|
{
|
|
var isCentral = siteId == CentralHealthReportLoop.CentralSiteId;
|
|
var siteName = isCentral ? "Central Cluster" : GetSiteName(siteId);
|
|
var detailsCollapseId = $"site-details-{siteId}";
|
|
<div class="card mb-3">
|
|
<div class="card-header d-flex justify-content-between align-items-center py-2">
|
|
<div>
|
|
@if (state.IsOnline)
|
|
{
|
|
<span class="badge bg-success me-2" aria-label="State: Online">@OnlineGlyph Online</span>
|
|
}
|
|
else
|
|
{
|
|
<span class="badge bg-danger me-2" aria-label="State: Offline">@OfflineGlyph Offline</span>
|
|
}
|
|
<strong class="fs-5">@siteName@(isCentral ? "" : $" ({siteId})")</strong>
|
|
@if (state.IsOnline && state.IsMetricsStale)
|
|
{
|
|
<span class="badge bg-warning text-dark ms-2"
|
|
title="Site is heartbeating but has sent no health report for over @StaleTimeoutDisplay — the metrics pipeline may be dead.">
|
|
Metrics stale
|
|
</span>
|
|
}
|
|
@if (!state.IsOnline && state.LastStatusChangeAt is { } changedAt)
|
|
{
|
|
<small class="text-muted ms-2">offline since @changedAt.ToString("u")</small>
|
|
}
|
|
</div>
|
|
<div class="d-flex align-items-center gap-3">
|
|
@* Admin-only; the control gates itself, and disables when the pair has
|
|
no online standby to take over. Central acts on the local cluster;
|
|
a site is asked over the command/control relay. *@
|
|
<CentralFailoverControl OnlineCentralNodeCount="@OnlineNodeCount(state)"
|
|
SiteId="@(isCentral ? null : siteId)" />
|
|
<small class="text-muted">
|
|
Last report: <TimestampDisplay Value="@state.LastReportReceivedAt" Format="HH:mm:ss" NullText="awaiting first report" />
|
|
| Last heartbeat: <TimestampDisplay Value="@state.LastHeartbeatAt" Format="HH:mm:ss" />
|
|
| Seq: @state.LastSequenceNumber
|
|
</small>
|
|
</div>
|
|
</div>
|
|
<div class="card-body p-3">
|
|
@if (state.LatestReport != null)
|
|
{
|
|
var report = state.LatestReport;
|
|
<div class="row g-3">
|
|
@* Column 1: Nodes *@
|
|
<div class="col-md-6">
|
|
<h6 class="text-muted mb-2 border-bottom pb-1">Nodes</h6>
|
|
<table class="table table-sm table-borderless mb-0">
|
|
<tbody>
|
|
@if (report.ClusterNodes is { Count: > 0 })
|
|
{
|
|
@foreach (var node in report.ClusterNodes)
|
|
{
|
|
<tr>
|
|
<td class="small">@node.Hostname</td>
|
|
<td>
|
|
<span class="badge @(node.IsOnline ? "bg-success" : "bg-danger")"
|
|
aria-label="State: @(node.IsOnline ? "Online" : "Offline")">
|
|
@(node.IsOnline ? OnlineGlyph : OfflineGlyph) @(node.IsOnline ? "Online" : "Offline")
|
|
</span>
|
|
</td>
|
|
<td>
|
|
<span class="badge @(node.Role == "Primary" ? "bg-primary" : "bg-secondary")"
|
|
aria-label="State: @node.Role">
|
|
@(node.Role == "Primary" ? PrimaryGlyph : StandbyGlyph) @node.Role
|
|
</span>
|
|
</td>
|
|
</tr>
|
|
}
|
|
}
|
|
else
|
|
{
|
|
<tr>
|
|
<td class="small">@(report.NodeHostname != "" ? report.NodeHostname : "Node")</td>
|
|
<td>
|
|
<span class="badge @(state.IsOnline ? "bg-success" : "bg-danger")"
|
|
aria-label="State: @(state.IsOnline ? "Online" : "Offline")">
|
|
@(state.IsOnline ? OnlineGlyph : OfflineGlyph) @(state.IsOnline ? "Online" : "Offline")
|
|
</span>
|
|
</td>
|
|
<td>
|
|
@{
|
|
var roleLabel = report.NodeRole == "Active" ? "Primary" : "Standby";
|
|
}
|
|
<span class="badge @(report.NodeRole == "Active" ? "bg-primary" : "bg-secondary")"
|
|
aria-label="State: @roleLabel">
|
|
@(roleLabel == "Primary" ? PrimaryGlyph : StandbyGlyph) @roleLabel
|
|
</span>
|
|
</td>
|
|
</tr>
|
|
}
|
|
</tbody>
|
|
</table>
|
|
</div>
|
|
|
|
@* Column 2: Data Connections (collapsible) *@
|
|
<div class="col-md-6">
|
|
<button class="btn btn-link btn-sm p-0 text-decoration-none mb-2"
|
|
data-bs-toggle="collapse"
|
|
data-bs-target="@($"#{detailsCollapseId}-conns")"
|
|
aria-expanded="false">
|
|
Data Connections (@report.DataConnectionStatuses.Count)
|
|
</button>
|
|
<div class="collapse" id="@($"{detailsCollapseId}-conns")">
|
|
@if (report.DataConnectionStatuses.Count == 0)
|
|
{
|
|
<span class="text-muted small">None</span>
|
|
}
|
|
else
|
|
{
|
|
@foreach (var (connName, health) in report.DataConnectionStatuses)
|
|
{
|
|
var endpoint = report.DataConnectionEndpoints?.GetValueOrDefault(connName);
|
|
var quality = report.DataConnectionTagQuality?.GetValueOrDefault(connName);
|
|
<div class="mb-2">
|
|
<div class="d-flex justify-content-between">
|
|
<strong class="small">@connName</strong>
|
|
<span class="small">@(endpoint ?? health.ToString())</span>
|
|
</div>
|
|
@if (quality != null)
|
|
{
|
|
<table class="table table-sm table-borderless mb-0">
|
|
<tbody>
|
|
<tr>
|
|
<td class="small text-muted py-0">Tags good</td>
|
|
<td class="small text-end py-0">@quality.Good.ToString("N0")</td>
|
|
</tr>
|
|
<tr>
|
|
<td class="small text-muted py-0">Tags bad</td>
|
|
<td class="small text-end py-0">@quality.Bad.ToString("N0")</td>
|
|
</tr>
|
|
<tr>
|
|
<td class="small text-muted py-0">Tags uncertain</td>
|
|
<td class="small text-end py-0">@quality.Uncertain.ToString("N0")</td>
|
|
</tr>
|
|
</tbody>
|
|
</table>
|
|
}
|
|
</div>
|
|
}
|
|
}
|
|
</div>
|
|
</div>
|
|
|
|
@* Column 3: Instances + Store-and-Forward (collapsible) *@
|
|
<div class="col-md-6">
|
|
<button class="btn btn-link btn-sm p-0 text-decoration-none mb-2"
|
|
data-bs-toggle="collapse"
|
|
data-bs-target="@($"#{detailsCollapseId}-queues")"
|
|
aria-expanded="false">
|
|
Instances & Queues
|
|
</button>
|
|
<div class="collapse" id="@($"{detailsCollapseId}-queues")">
|
|
<h6 class="text-muted mb-2 border-bottom pb-1">Instances</h6>
|
|
<table class="table table-sm table-borderless mb-0">
|
|
<tbody>
|
|
<tr>
|
|
<td class="small">Deployed</td>
|
|
<td class="text-end">@report.DeployedInstanceCount</td>
|
|
</tr>
|
|
<tr>
|
|
<td class="small">Enabled</td>
|
|
<td class="text-end text-success">@report.EnabledInstanceCount</td>
|
|
</tr>
|
|
<tr>
|
|
<td class="small">Disabled</td>
|
|
<td class="text-end">@report.DisabledInstanceCount</td>
|
|
</tr>
|
|
</tbody>
|
|
</table>
|
|
|
|
<h6 class="text-muted mb-2 mt-3 border-bottom pb-1">Store-and-Forward Buffers</h6>
|
|
@if (report.StoreAndForwardBufferDepths.Count == 0)
|
|
{
|
|
<span class="text-muted small">Empty</span>
|
|
}
|
|
else
|
|
{
|
|
@foreach (var (category, depth) in report.StoreAndForwardBufferDepths)
|
|
{
|
|
<div class="d-flex justify-content-between mb-1">
|
|
<span class="small">@category</span>
|
|
<span class="badge @(depth > 0 ? "bg-warning text-dark" : "bg-light text-dark")">@depth</span>
|
|
</div>
|
|
}
|
|
}
|
|
</div>
|
|
</div>
|
|
|
|
@* Column 4: Error Counts + Parked Messages (collapsible) *@
|
|
<div class="col-md-6">
|
|
<button class="btn btn-link btn-sm p-0 text-decoration-none mb-2"
|
|
data-bs-toggle="collapse"
|
|
data-bs-target="@($"#{detailsCollapseId}-errors")"
|
|
aria-expanded="false">
|
|
Errors & Parked Messages
|
|
</button>
|
|
<div class="collapse" id="@($"{detailsCollapseId}-errors")">
|
|
<h6 class="text-muted mb-2 border-bottom pb-1">Error Counts</h6>
|
|
<table class="table table-sm table-borderless mb-0">
|
|
<tbody>
|
|
<tr>
|
|
<td class="small">Script Errors</td>
|
|
<td class="text-end">
|
|
<span class="@(report.ScriptErrorCount > 0 ? "text-danger fw-bold" : "")">@report.ScriptErrorCount</span>
|
|
</td>
|
|
</tr>
|
|
<tr>
|
|
<td class="small">Alarm Eval Errors</td>
|
|
<td class="text-end">
|
|
<span class="@(report.AlarmEvaluationErrorCount > 0 ? "text-warning fw-bold" : "")">@report.AlarmEvaluationErrorCount</span>
|
|
</td>
|
|
</tr>
|
|
<tr>
|
|
<td class="small">Dead Letters</td>
|
|
<td class="text-end">
|
|
<span class="@(report.DeadLetterCount > 0 ? "text-danger fw-bold" : "")">@report.DeadLetterCount</span>
|
|
</td>
|
|
</tr>
|
|
</tbody>
|
|
</table>
|
|
|
|
<h6 class="text-muted mb-2 mt-3 border-bottom pb-1">Parked Messages</h6>
|
|
@if (report.ParkedMessageCount == 0)
|
|
{
|
|
<span class="text-muted small">Empty</span>
|
|
}
|
|
else
|
|
{
|
|
<span class="badge bg-warning text-dark">@report.ParkedMessageCount</span>
|
|
}
|
|
</div>
|
|
</div>
|
|
</div>
|
|
}
|
|
else
|
|
{
|
|
<span class="text-muted">No report data available.</span>
|
|
}
|
|
</div>
|
|
</div>
|
|
}
|
|
}
|
|
</div>
|
|
|
|
@code {
|
|
// Shape-coded status glyphs to pair with badge colour.
|
|
private const string OnlineGlyph = "●"; // ●
|
|
private const string OfflineGlyph = "○"; // ○
|
|
private const string PrimaryGlyph = "▲"; // ▲
|
|
private const string StandbyGlyph = "△"; // △
|
|
|
|
// Human-friendly rendering of the configured metrics-stale window for the
|
|
// "Metrics stale" badge tooltip (e.g. "2 minutes").
|
|
private string StaleTimeoutDisplay =>
|
|
FormatDuration(HealthOptions.Value.MetricsStaleTimeout);
|
|
|
|
// Online central nodes, from the same ClusterNodes list the Nodes column renders.
|
|
// Fewer than two means no standby, which disables the manual-failover control. This is
|
|
// a display-side guard only — AkkaManualFailoverService re-checks against live cluster
|
|
// membership, which is the authoritative answer.
|
|
private static int OnlineNodeCount(SiteHealthState state) =>
|
|
state.LatestReport?.ClusterNodes is { Count: > 0 } nodes
|
|
? nodes.Count(n => n.IsOnline)
|
|
: (state.IsOnline ? 1 : 0);
|
|
|
|
private static string FormatDuration(TimeSpan span) =>
|
|
span.TotalMinutes >= 1 && span == TimeSpan.FromMinutes(Math.Round(span.TotalMinutes))
|
|
? $"{span.TotalMinutes:0} minute{(span.TotalMinutes == 1 ? "" : "s")}"
|
|
: $"{span.TotalSeconds:0} seconds";
|
|
|
|
private IReadOnlyDictionary<string, SiteHealthState> _siteStates = new Dictionary<string, SiteHealthState>();
|
|
private Dictionary<string, string> _siteNames = new();
|
|
private Timer? _refreshTimer;
|
|
private readonly ZB.MOM.WW.ScadaBridge.CentralUI.Services.PollGate _pollGate = new();
|
|
private int _autoRefreshSeconds = 10;
|
|
|
|
// Notification Outbox headline KPIs, refreshed alongside the site states.
|
|
private NotificationKpiResponse _outboxKpi =
|
|
new(
|
|
CorrelationId: string.Empty,
|
|
Success: false,
|
|
ErrorMessage: null,
|
|
QueueDepth: 0,
|
|
StuckCount: 0,
|
|
ParkedCount: 0,
|
|
DeliveredLastInterval: 0,
|
|
OldestPendingAge: null);
|
|
private bool _outboxKpiAvailable;
|
|
private string? _outboxKpiError;
|
|
|
|
// Audit KPI tiles. Volume + error rate come
|
|
// from a 1h aggregate over the central AuditLog table; backlog sums the
|
|
// per-site SiteAuditBacklog.PendingCount via the health aggregator.
|
|
private AuditLogKpiSnapshot? _auditKpi;
|
|
private bool _auditKpiAvailable;
|
|
private string? _auditKpiError;
|
|
|
|
// Site Call Audit (#22) Task 7 — Site Call KPI tiles. Point-in-time counts
|
|
// from the central SiteCalls table, fetched alongside the site states. The
|
|
// SiteCallKpiResponse message doubles as the snapshot the tile takes.
|
|
private SiteCallKpiResponse? _siteCallKpi;
|
|
private bool _siteCallKpiAvailable;
|
|
private string? _siteCallKpiError;
|
|
|
|
// Per-node Site Call KPI breakdown (M5.2 per-node stuck-count KPIs).
|
|
// Passed to SiteCallKpiTiles as an optional sub-table.
|
|
private IReadOnlyList<SiteCallNodeKpiSnapshot> _siteCallNodeKpis =
|
|
Array.Empty<SiteCallNodeKpiSnapshot>();
|
|
private bool _siteCallNodeKpiAvailable;
|
|
|
|
// ── Site Health Trends (M6) ───────────────────────────────────────────────
|
|
// Per-site Site Health KPI history, loaded on a path entirely separate from
|
|
// the 10s tile-refresh timer (LoadSiteHealthTrendsAsync, never called from
|
|
// the timer tick). The site keys are a snapshot of the dashboard's site set,
|
|
// captured each time trends load so the selector mirrors the live cards.
|
|
// Window in hours: 24h (default) or 168h (7d). Changing the selected site OR
|
|
// the window re-queries. Each metric chart carries its own availability +
|
|
// error so one failed GetSeriesAsync degrades a single chart, never the
|
|
// dashboard.
|
|
private IReadOnlyList<string> _trendSiteKeys = Array.Empty<string>();
|
|
private string? _trendSiteId;
|
|
private int _trendWindowHours = 24;
|
|
private bool _trendsLoading;
|
|
|
|
private IReadOnlyList<KpiSeriesPoint>? _connectionsDownSeries;
|
|
private bool _connectionsDownAvailable = true;
|
|
private string? _connectionsDownError;
|
|
|
|
private IReadOnlyList<KpiSeriesPoint>? _deadLettersSeries;
|
|
private bool _deadLettersAvailable = true;
|
|
private string? _deadLettersError;
|
|
|
|
private IReadOnlyList<KpiSeriesPoint>? _scriptErrorsSeries;
|
|
private bool _scriptErrorsAvailable = true;
|
|
private string? _scriptErrorsError;
|
|
|
|
private IReadOnlyList<KpiSeriesPoint>? _sfBufferDepthSeries;
|
|
private bool _sfBufferDepthAvailable = true;
|
|
private string? _sfBufferDepthError;
|
|
|
|
private static bool SiteHasActiveErrors(SiteHealthState state)
|
|
{
|
|
var report = state.LatestReport;
|
|
if (report == null) return false;
|
|
return report.ScriptErrorCount > 0
|
|
|| report.AlarmEvaluationErrorCount > 0
|
|
|| report.DeadLetterCount > 0;
|
|
}
|
|
|
|
protected override async Task OnInitializedAsync()
|
|
{
|
|
// Load site names for display
|
|
try
|
|
{
|
|
var sites = await SiteRepository.GetAllSitesAsync();
|
|
_siteNames = sites.ToDictionary(s => s.SiteIdentifier, s => s.Name);
|
|
}
|
|
catch
|
|
{
|
|
// Non-fatal — fall back to showing siteId only
|
|
}
|
|
|
|
await RefreshNow();
|
|
|
|
// Site Health Trends (M6) load on their own path — never from the
|
|
// timer tick below — so a trend-query fault can't disturb the live tile
|
|
// refresh. Seed the selector from the sites just loaded into _siteStates
|
|
// and query the default site.
|
|
await LoadSiteHealthTrendsAsync();
|
|
|
|
_refreshTimer = new Timer(_ =>
|
|
{
|
|
if (!_pollGate.TryEnter()) return;
|
|
InvokeAsync(async () =>
|
|
{
|
|
try
|
|
{
|
|
await RefreshNow();
|
|
StateHasChanged();
|
|
}
|
|
finally
|
|
{
|
|
_pollGate.Exit();
|
|
}
|
|
});
|
|
}, null, TimeSpan.FromSeconds(_autoRefreshSeconds), TimeSpan.FromSeconds(_autoRefreshSeconds));
|
|
}
|
|
|
|
// Re-query when the operator picks a different site. Best-effort: the load
|
|
// itself swallows faults per-chart.
|
|
private async Task OnTrendSiteChangedAsync(ChangeEventArgs e)
|
|
{
|
|
var selected = e.Value?.ToString();
|
|
if (string.IsNullOrEmpty(selected) || selected == _trendSiteId)
|
|
{
|
|
return;
|
|
}
|
|
|
|
_trendSiteId = selected;
|
|
await LoadSiteHealthTrendsAsync(refreshSiteKeys: false);
|
|
}
|
|
|
|
// Re-query when the window toggle changes (24h ↔ 7d).
|
|
private async Task SetTrendWindowAsync(int windowHours)
|
|
{
|
|
if (_trendWindowHours == windowHours)
|
|
{
|
|
return;
|
|
}
|
|
|
|
_trendWindowHours = windowHours;
|
|
await LoadSiteHealthTrendsAsync(refreshSiteKeys: false);
|
|
}
|
|
|
|
// Loads the four Site Health trend series for the selected site over the
|
|
// selected window. Deliberately decoupled from RefreshNow / the 10s timer:
|
|
// a fault here degrades the affected chart(s) only and never propagates to
|
|
// the tile-refresh loop.
|
|
//
|
|
// refreshSiteKeys re-snapshots the dashboard's site set into the selector
|
|
// (true on init); the site-change / window-toggle paths pass false so a
|
|
// mid-interaction site addition/removal can't yank the operator's choice.
|
|
private async Task LoadSiteHealthTrendsAsync(bool refreshSiteKeys = true)
|
|
{
|
|
if (refreshSiteKeys)
|
|
{
|
|
// Mirror the dashboard ordering: central cluster pinned first, then
|
|
// sites alphabetically — the same comparer the detail cards use.
|
|
_trendSiteKeys = _siteStates.Keys
|
|
.OrderBy(k => k == CentralHealthReportLoop.CentralSiteId ? 0 : 1)
|
|
.ThenBy(k => k)
|
|
.ToList();
|
|
|
|
// Default to the first site (or keep a still-valid prior selection).
|
|
if (_trendSiteId == null || !_trendSiteKeys.Contains(_trendSiteId))
|
|
{
|
|
_trendSiteId = _trendSiteKeys.FirstOrDefault();
|
|
}
|
|
}
|
|
|
|
if (string.IsNullOrEmpty(_trendSiteId))
|
|
{
|
|
return;
|
|
}
|
|
|
|
_trendsLoading = true;
|
|
try
|
|
{
|
|
var toUtc = DateTime.UtcNow;
|
|
var fromUtc = toUtc - TimeSpan.FromHours(_trendWindowHours);
|
|
var siteId = _trendSiteId;
|
|
|
|
(_connectionsDownSeries, _connectionsDownAvailable, _connectionsDownError) =
|
|
await LoadTrendSeriesAsync(KpiMetrics.SiteHealth.ConnectionsDown, siteId, fromUtc, toUtc);
|
|
(_deadLettersSeries, _deadLettersAvailable, _deadLettersError) =
|
|
await LoadTrendSeriesAsync(KpiMetrics.SiteHealth.DeadLetters, siteId, fromUtc, toUtc);
|
|
(_scriptErrorsSeries, _scriptErrorsAvailable, _scriptErrorsError) =
|
|
await LoadTrendSeriesAsync(KpiMetrics.SiteHealth.ScriptErrors, siteId, fromUtc, toUtc);
|
|
(_sfBufferDepthSeries, _sfBufferDepthAvailable, _sfBufferDepthError) =
|
|
await LoadTrendSeriesAsync(KpiMetrics.SiteHealth.SfBufferDepth, siteId, fromUtc, toUtc);
|
|
}
|
|
finally
|
|
{
|
|
_trendsLoading = false;
|
|
}
|
|
}
|
|
|
|
// Single best-effort series fetch. Site Health metrics are Site-scoped, so
|
|
// scope = KpiScopes.Site and scopeKey = the selected site id. On any fault
|
|
// the chart falls back to the unavailable placeholder — a failure here must
|
|
// NEVER break the dashboard.
|
|
private async Task<(IReadOnlyList<KpiSeriesPoint>?, bool, string?)> LoadTrendSeriesAsync(
|
|
string metric, string siteId, DateTime fromUtc, DateTime toUtc)
|
|
{
|
|
try
|
|
{
|
|
var series = await KpiHistory.GetSeriesAsync(
|
|
KpiSources.SiteHealth, metric, KpiScopes.Site, siteId, fromUtc, toUtc);
|
|
return (series, true, null);
|
|
}
|
|
catch
|
|
{
|
|
return (null, false, "Trend data unavailable.");
|
|
}
|
|
}
|
|
|
|
private string TrendSiteLabel(string siteKey) =>
|
|
siteKey == CentralHealthReportLoop.CentralSiteId
|
|
? "Central Cluster"
|
|
: $"{GetSiteName(siteKey)} ({siteKey})";
|
|
|
|
private async Task RefreshNow()
|
|
{
|
|
_siteStates = HealthAggregator.GetAllSiteStates();
|
|
await LoadOutboxKpis();
|
|
await Task.WhenAll(LoadSiteCallKpis(), LoadSiteCallNodeKpis());
|
|
await LoadAuditKpis();
|
|
}
|
|
|
|
private async Task LoadOutboxKpis()
|
|
{
|
|
try
|
|
{
|
|
var response = await CommunicationService.GetNotificationKpisAsync(
|
|
new NotificationKpiRequest(Guid.NewGuid().ToString("N")));
|
|
if (response.Success)
|
|
{
|
|
_outboxKpi = response;
|
|
_outboxKpiAvailable = true;
|
|
_outboxKpiError = null;
|
|
}
|
|
else
|
|
{
|
|
_outboxKpiAvailable = false;
|
|
_outboxKpiError = response.ErrorMessage ?? "KPI query failed.";
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
_outboxKpiAvailable = false;
|
|
_outboxKpiError = $"KPI query failed: {ex.Message}";
|
|
}
|
|
}
|
|
|
|
// Site Call KPI loader: wraps the service call so a transient fault degrades
|
|
// the three Site Call tiles to em dashes with an inline error rather than
|
|
// killing the dashboard. Mirrors LoadOutboxKpis's error handling shape — a
|
|
// response with Success == false (repository fault) and an Ask that threw
|
|
// (transport fault) both collapse to "unavailable".
|
|
private async Task LoadSiteCallKpis()
|
|
{
|
|
try
|
|
{
|
|
var response = await CommunicationService.GetSiteCallKpisAsync(
|
|
new SiteCallKpiRequest(Guid.NewGuid().ToString("N")));
|
|
if (response.Success)
|
|
{
|
|
_siteCallKpi = response;
|
|
_siteCallKpiAvailable = true;
|
|
_siteCallKpiError = null;
|
|
}
|
|
else
|
|
{
|
|
_siteCallKpiAvailable = false;
|
|
_siteCallKpiError = response.ErrorMessage ?? "KPI query failed.";
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
_siteCallKpiAvailable = false;
|
|
_siteCallKpiError = $"KPI query failed: {ex.Message}";
|
|
}
|
|
}
|
|
|
|
// Per-node site-call KPI loader (M5.2). Best-effort; a fault silently
|
|
// suppresses the per-node sub-table rather than degrading the dashboard.
|
|
private async Task LoadSiteCallNodeKpis()
|
|
{
|
|
try
|
|
{
|
|
var response = await CommunicationService.GetPerNodeSiteCallKpisAsync(
|
|
new PerNodeSiteCallKpiRequest(Guid.NewGuid().ToString("N")));
|
|
if (response.Success)
|
|
{
|
|
_siteCallNodeKpis = response.Nodes;
|
|
_siteCallNodeKpiAvailable = true;
|
|
}
|
|
else
|
|
{
|
|
_siteCallNodeKpiAvailable = false;
|
|
}
|
|
}
|
|
catch
|
|
{
|
|
_siteCallNodeKpiAvailable = false;
|
|
}
|
|
}
|
|
|
|
// Tiles show the numeric KPI when available, or an em dash when the outbox
|
|
// KPI query failed — matching how the page renders other unavailable data.
|
|
private string OutboxTileValue(int value) =>
|
|
_outboxKpiAvailable ? value.ToString() : "—";
|
|
|
|
// Audit KPI loader: wraps the service call so a transient DB outage degrades
|
|
// the three tiles to em dashes with an inline error rather than killing the
|
|
// dashboard. Mirrors LoadOutboxKpis's error handling shape.
|
|
private async Task LoadAuditKpis()
|
|
{
|
|
try
|
|
{
|
|
_auditKpi = await AuditLogQueryService.GetKpiSnapshotAsync();
|
|
_auditKpiAvailable = true;
|
|
_auditKpiError = null;
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
_auditKpiAvailable = false;
|
|
_auditKpiError = $"KPI query failed: {ex.Message}";
|
|
}
|
|
}
|
|
|
|
private string GetSiteName(string siteId)
|
|
{
|
|
return _siteNames.GetValueOrDefault(siteId, siteId);
|
|
}
|
|
|
|
private static string GetConnectionHealthBadge(ConnectionHealth health) => health switch
|
|
{
|
|
ConnectionHealth.Connected => "bg-success",
|
|
ConnectionHealth.Connecting => "bg-warning text-dark",
|
|
ConnectionHealth.Disconnected => "bg-danger",
|
|
_ => "bg-secondary"
|
|
};
|
|
|
|
public void Dispose()
|
|
{
|
|
_refreshTimer?.Dispose();
|
|
}
|
|
}
|