@page "/monitoring/health" @attribute [Authorize] @using ZB.MOM.WW.ScadaBridge.CentralUI.Components.Health @using ZB.MOM.WW.ScadaBridge.CentralUI.Services @using ZB.MOM.WW.ScadaBridge.Commons.Types @using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums @using ZB.MOM.WW.ScadaBridge.Commons.Entities.Sites @using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories @using ZB.MOM.WW.ScadaBridge.HealthMonitoring @using ZB.MOM.WW.ScadaBridge.Commons.Messages.Notification @using ZB.MOM.WW.ScadaBridge.Commons.Messages.Audit @using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit @using ZB.MOM.WW.ScadaBridge.Commons.Types.Kpi @using ZB.MOM.WW.ScadaBridge.Communication @implements IDisposable @inject ICentralHealthAggregator HealthAggregator @inject ISiteRepository SiteRepository @inject CommunicationService CommunicationService @inject IAuditLogQueryService AuditLogQueryService @inject IKpiHistoryQueryService KpiHistory @inject Microsoft.Extensions.Options.IOptions HealthOptions

Health Dashboard

Auto-refresh: @(_autoRefreshSeconds)s
@* Notification Outbox headline KPIs — a central concern, shown regardless of site reports *@
Notification Outbox
View details →

@OutboxTileValue(_outboxKpi.QueueDepth)

Queue Depth

@OutboxTileValue(_outboxKpi.StuckCount)

Stuck

@OutboxTileValue(_outboxKpi.ParkedCount)

Parked
@if (!_outboxKpiAvailable && _outboxKpiError != null) {
Notification Outbox KPIs unavailable: @_outboxKpiError
} @* Site Call Audit (#22) Task 7 — three KPI tiles for the Site Call channel (buffered / stuck / parked). Refreshed alongside the site states. *@ @* Three KPI tiles for the Audit channel (volume / error rate / backlog). Refreshed alongside the site states. *@ @* Site Health Trends (M6) — per-site Site Health KPI history. Loads on a separate path from the 10s tile-refresh timer so a trend-query fault can never disturb the live dashboard or its polling loop. The site selector reuses the site keys already loaded into _siteStates; the window toggle drives the time range. Both re-query independently. *@
Site Health Trends
@if (_trendSiteKeys.Count > 0) { @* OnTrendSiteChangedAsync is async — Blazor wraps it in an EventCallback so the returned Task IS awaited; keep the Async signature, do not change it to a void/fire-and-forget handler. This comment MUST stay OUTSIDE the @foreach (var key in _trendSiteKeys) { } }
@if (_trendSiteKeys.Count == 0) { No sites available for trends yet. } else {
}
@if (_siteStates.Count == 0) {
No site health reports received yet.
} else { @* Overview cards *@

@_siteStates.Values.Count(s => s.IsOnline)

Sites Online

@_siteStates.Values.Count(s => !s.IsOnline)

Sites Offline

@_siteStates.Values.Count(SiteHasActiveErrors)

Sites with active errors
@* Per-site detail cards — central cluster pinned to the top, then sites alphabetically *@ @foreach (var (siteId, state) in _siteStates.OrderBy(s => s.Key == CentralHealthReportLoop.CentralSiteId ? 0 : 1).ThenBy(s => s.Key)) { var isCentral = siteId == CentralHealthReportLoop.CentralSiteId; var siteName = isCentral ? "Central Cluster" : GetSiteName(siteId); var detailsCollapseId = $"site-details-{siteId}";
@if (state.IsOnline) { @OnlineGlyph Online } else { @OfflineGlyph Offline } @siteName@(isCentral ? "" : $" ({siteId})") @if (state.IsOnline && state.IsMetricsStale) { Metrics stale } @if (!state.IsOnline && state.LastStatusChangeAt is { } changedAt) { offline since @changedAt.ToString("u") }
Last report: | Last heartbeat: | Seq: @state.LastSequenceNumber
@if (state.LatestReport != null) { var report = state.LatestReport;
@* Column 1: Nodes *@
Nodes
@if (report.ClusterNodes is { Count: > 0 }) { @foreach (var node in report.ClusterNodes) { } } else { }
@node.Hostname @(node.IsOnline ? OnlineGlyph : OfflineGlyph) @(node.IsOnline ? "Online" : "Offline") @(node.Role == "Primary" ? PrimaryGlyph : StandbyGlyph) @node.Role
@(report.NodeHostname != "" ? report.NodeHostname : "Node") @(state.IsOnline ? OnlineGlyph : OfflineGlyph) @(state.IsOnline ? "Online" : "Offline") @{ var roleLabel = report.NodeRole == "Active" ? "Primary" : "Standby"; } @(roleLabel == "Primary" ? PrimaryGlyph : StandbyGlyph) @roleLabel
@* Column 2: Data Connections (collapsible) *@
@if (report.DataConnectionStatuses.Count == 0) { None } else { @foreach (var (connName, health) in report.DataConnectionStatuses) { var endpoint = report.DataConnectionEndpoints?.GetValueOrDefault(connName); var quality = report.DataConnectionTagQuality?.GetValueOrDefault(connName);
@connName @(endpoint ?? health.ToString())
@if (quality != null) {
Tags good @quality.Good.ToString("N0")
Tags bad @quality.Bad.ToString("N0")
Tags uncertain @quality.Uncertain.ToString("N0")
}
} }
@* Column 3: Instances + Store-and-Forward (collapsible) *@
Instances
Deployed @report.DeployedInstanceCount
Enabled @report.EnabledInstanceCount
Disabled @report.DisabledInstanceCount
Store-and-Forward Buffers
@if (report.StoreAndForwardBufferDepths.Count == 0) { Empty } else { @foreach (var (category, depth) in report.StoreAndForwardBufferDepths) {
@category @depth
} }
@* Column 4: Error Counts + Parked Messages (collapsible) *@
Error Counts
Script Errors @report.ScriptErrorCount
Alarm Eval Errors @report.AlarmEvaluationErrorCount
Dead Letters @report.DeadLetterCount
Parked Messages
@if (report.ParkedMessageCount == 0) { Empty } else { @report.ParkedMessageCount }
} else { No report data available. }
} }
@code { // Shape-coded status glyphs to pair with badge colour. private const string OnlineGlyph = "●"; // ● private const string OfflineGlyph = "○"; // ○ private const string PrimaryGlyph = "▲"; // ▲ private const string StandbyGlyph = "△"; // △ // Human-friendly rendering of the configured metrics-stale window for the // "Metrics stale" badge tooltip (e.g. "2 minutes"). private string StaleTimeoutDisplay => FormatDuration(HealthOptions.Value.MetricsStaleTimeout); private static string FormatDuration(TimeSpan span) => span.TotalMinutes >= 1 && span == TimeSpan.FromMinutes(Math.Round(span.TotalMinutes)) ? $"{span.TotalMinutes:0} minute{(span.TotalMinutes == 1 ? "" : "s")}" : $"{span.TotalSeconds:0} seconds"; private IReadOnlyDictionary _siteStates = new Dictionary(); private Dictionary _siteNames = new(); private Timer? _refreshTimer; private readonly ZB.MOM.WW.ScadaBridge.CentralUI.Services.PollGate _pollGate = new(); private int _autoRefreshSeconds = 10; // Notification Outbox headline KPIs, refreshed alongside the site states. private NotificationKpiResponse _outboxKpi = new( CorrelationId: string.Empty, Success: false, ErrorMessage: null, QueueDepth: 0, StuckCount: 0, ParkedCount: 0, DeliveredLastInterval: 0, OldestPendingAge: null); private bool _outboxKpiAvailable; private string? _outboxKpiError; // Audit KPI tiles. Volume + error rate come // from a 1h aggregate over the central AuditLog table; backlog sums the // per-site SiteAuditBacklog.PendingCount via the health aggregator. private AuditLogKpiSnapshot? _auditKpi; private bool _auditKpiAvailable; private string? _auditKpiError; // Site Call Audit (#22) Task 7 — Site Call KPI tiles. Point-in-time counts // from the central SiteCalls table, fetched alongside the site states. The // SiteCallKpiResponse message doubles as the snapshot the tile takes. private SiteCallKpiResponse? _siteCallKpi; private bool _siteCallKpiAvailable; private string? _siteCallKpiError; // Per-node Site Call KPI breakdown (M5.2 per-node stuck-count KPIs). // Passed to SiteCallKpiTiles as an optional sub-table. private IReadOnlyList _siteCallNodeKpis = Array.Empty(); private bool _siteCallNodeKpiAvailable; // ── Site Health Trends (M6) ─────────────────────────────────────────────── // Per-site Site Health KPI history, loaded on a path entirely separate from // the 10s tile-refresh timer (LoadSiteHealthTrendsAsync, never called from // the timer tick). The site keys are a snapshot of the dashboard's site set, // captured each time trends load so the selector mirrors the live cards. // Window in hours: 24h (default) or 168h (7d). Changing the selected site OR // the window re-queries. Each metric chart carries its own availability + // error so one failed GetSeriesAsync degrades a single chart, never the // dashboard. private IReadOnlyList _trendSiteKeys = Array.Empty(); private string? _trendSiteId; private int _trendWindowHours = 24; private bool _trendsLoading; private IReadOnlyList? _connectionsDownSeries; private bool _connectionsDownAvailable = true; private string? _connectionsDownError; private IReadOnlyList? _deadLettersSeries; private bool _deadLettersAvailable = true; private string? _deadLettersError; private IReadOnlyList? _scriptErrorsSeries; private bool _scriptErrorsAvailable = true; private string? _scriptErrorsError; private IReadOnlyList? _sfBufferDepthSeries; private bool _sfBufferDepthAvailable = true; private string? _sfBufferDepthError; private static bool SiteHasActiveErrors(SiteHealthState state) { var report = state.LatestReport; if (report == null) return false; return report.ScriptErrorCount > 0 || report.AlarmEvaluationErrorCount > 0 || report.DeadLetterCount > 0; } protected override async Task OnInitializedAsync() { // Load site names for display try { var sites = await SiteRepository.GetAllSitesAsync(); _siteNames = sites.ToDictionary(s => s.SiteIdentifier, s => s.Name); } catch { // Non-fatal — fall back to showing siteId only } await RefreshNow(); // Site Health Trends (M6) load on their own path — never from the // timer tick below — so a trend-query fault can't disturb the live tile // refresh. Seed the selector from the sites just loaded into _siteStates // and query the default site. await LoadSiteHealthTrendsAsync(); _refreshTimer = new Timer(_ => { if (!_pollGate.TryEnter()) return; InvokeAsync(async () => { try { await RefreshNow(); StateHasChanged(); } finally { _pollGate.Exit(); } }); }, null, TimeSpan.FromSeconds(_autoRefreshSeconds), TimeSpan.FromSeconds(_autoRefreshSeconds)); } // Re-query when the operator picks a different site. Best-effort: the load // itself swallows faults per-chart. private async Task OnTrendSiteChangedAsync(ChangeEventArgs e) { var selected = e.Value?.ToString(); if (string.IsNullOrEmpty(selected) || selected == _trendSiteId) { return; } _trendSiteId = selected; await LoadSiteHealthTrendsAsync(refreshSiteKeys: false); } // Re-query when the window toggle changes (24h ↔ 7d). private async Task SetTrendWindowAsync(int windowHours) { if (_trendWindowHours == windowHours) { return; } _trendWindowHours = windowHours; await LoadSiteHealthTrendsAsync(refreshSiteKeys: false); } // Loads the four Site Health trend series for the selected site over the // selected window. Deliberately decoupled from RefreshNow / the 10s timer: // a fault here degrades the affected chart(s) only and never propagates to // the tile-refresh loop. // // refreshSiteKeys re-snapshots the dashboard's site set into the selector // (true on init); the site-change / window-toggle paths pass false so a // mid-interaction site addition/removal can't yank the operator's choice. private async Task LoadSiteHealthTrendsAsync(bool refreshSiteKeys = true) { if (refreshSiteKeys) { // Mirror the dashboard ordering: central cluster pinned first, then // sites alphabetically — the same comparer the detail cards use. _trendSiteKeys = _siteStates.Keys .OrderBy(k => k == CentralHealthReportLoop.CentralSiteId ? 0 : 1) .ThenBy(k => k) .ToList(); // Default to the first site (or keep a still-valid prior selection). if (_trendSiteId == null || !_trendSiteKeys.Contains(_trendSiteId)) { _trendSiteId = _trendSiteKeys.FirstOrDefault(); } } if (string.IsNullOrEmpty(_trendSiteId)) { return; } _trendsLoading = true; try { var toUtc = DateTime.UtcNow; var fromUtc = toUtc - TimeSpan.FromHours(_trendWindowHours); var siteId = _trendSiteId; (_connectionsDownSeries, _connectionsDownAvailable, _connectionsDownError) = await LoadTrendSeriesAsync(KpiMetrics.SiteHealth.ConnectionsDown, siteId, fromUtc, toUtc); (_deadLettersSeries, _deadLettersAvailable, _deadLettersError) = await LoadTrendSeriesAsync(KpiMetrics.SiteHealth.DeadLetters, siteId, fromUtc, toUtc); (_scriptErrorsSeries, _scriptErrorsAvailable, _scriptErrorsError) = await LoadTrendSeriesAsync(KpiMetrics.SiteHealth.ScriptErrors, siteId, fromUtc, toUtc); (_sfBufferDepthSeries, _sfBufferDepthAvailable, _sfBufferDepthError) = await LoadTrendSeriesAsync(KpiMetrics.SiteHealth.SfBufferDepth, siteId, fromUtc, toUtc); } finally { _trendsLoading = false; } } // Single best-effort series fetch. Site Health metrics are Site-scoped, so // scope = KpiScopes.Site and scopeKey = the selected site id. On any fault // the chart falls back to the unavailable placeholder — a failure here must // NEVER break the dashboard. private async Task<(IReadOnlyList?, bool, string?)> LoadTrendSeriesAsync( string metric, string siteId, DateTime fromUtc, DateTime toUtc) { try { var series = await KpiHistory.GetSeriesAsync( KpiSources.SiteHealth, metric, KpiScopes.Site, siteId, fromUtc, toUtc); return (series, true, null); } catch { return (null, false, "Trend data unavailable."); } } private string TrendSiteLabel(string siteKey) => siteKey == CentralHealthReportLoop.CentralSiteId ? "Central Cluster" : $"{GetSiteName(siteKey)} ({siteKey})"; private async Task RefreshNow() { _siteStates = HealthAggregator.GetAllSiteStates(); await LoadOutboxKpis(); await Task.WhenAll(LoadSiteCallKpis(), LoadSiteCallNodeKpis()); await LoadAuditKpis(); } private async Task LoadOutboxKpis() { try { var response = await CommunicationService.GetNotificationKpisAsync( new NotificationKpiRequest(Guid.NewGuid().ToString("N"))); if (response.Success) { _outboxKpi = response; _outboxKpiAvailable = true; _outboxKpiError = null; } else { _outboxKpiAvailable = false; _outboxKpiError = response.ErrorMessage ?? "KPI query failed."; } } catch (Exception ex) { _outboxKpiAvailable = false; _outboxKpiError = $"KPI query failed: {ex.Message}"; } } // Site Call KPI loader: wraps the service call so a transient fault degrades // the three Site Call tiles to em dashes with an inline error rather than // killing the dashboard. Mirrors LoadOutboxKpis's error handling shape — a // response with Success == false (repository fault) and an Ask that threw // (transport fault) both collapse to "unavailable". private async Task LoadSiteCallKpis() { try { var response = await CommunicationService.GetSiteCallKpisAsync( new SiteCallKpiRequest(Guid.NewGuid().ToString("N"))); if (response.Success) { _siteCallKpi = response; _siteCallKpiAvailable = true; _siteCallKpiError = null; } else { _siteCallKpiAvailable = false; _siteCallKpiError = response.ErrorMessage ?? "KPI query failed."; } } catch (Exception ex) { _siteCallKpiAvailable = false; _siteCallKpiError = $"KPI query failed: {ex.Message}"; } } // Per-node site-call KPI loader (M5.2). Best-effort; a fault silently // suppresses the per-node sub-table rather than degrading the dashboard. private async Task LoadSiteCallNodeKpis() { try { var response = await CommunicationService.GetPerNodeSiteCallKpisAsync( new PerNodeSiteCallKpiRequest(Guid.NewGuid().ToString("N"))); if (response.Success) { _siteCallNodeKpis = response.Nodes; _siteCallNodeKpiAvailable = true; } else { _siteCallNodeKpiAvailable = false; } } catch { _siteCallNodeKpiAvailable = false; } } // Tiles show the numeric KPI when available, or an em dash when the outbox // KPI query failed — matching how the page renders other unavailable data. private string OutboxTileValue(int value) => _outboxKpiAvailable ? value.ToString() : "—"; // Audit KPI loader: wraps the service call so a transient DB outage degrades // the three tiles to em dashes with an inline error rather than killing the // dashboard. Mirrors LoadOutboxKpis's error handling shape. private async Task LoadAuditKpis() { try { _auditKpi = await AuditLogQueryService.GetKpiSnapshotAsync(); _auditKpiAvailable = true; _auditKpiError = null; } catch (Exception ex) { _auditKpiAvailable = false; _auditKpiError = $"KPI query failed: {ex.Message}"; } } private string GetSiteName(string siteId) { return _siteNames.GetValueOrDefault(siteId, siteId); } private static string GetConnectionHealthBadge(ConnectionHealth health) => health switch { ConnectionHealth.Connected => "bg-success", ConnectionHealth.Connecting => "bg-warning text-dark", ConnectionHealth.Disconnected => "bg-danger", _ => "bg-secondary" }; public void Dispose() { _refreshTimer?.Dispose(); } }