perf(ui): shared KPI cache, live-cache-backed alarm summary, coalesced debug renders

This commit is contained in:
Joseph Doherty
2026-08-14 20:53:38 -04:00
parent ee193cd2bb
commit 8c0b36b2fa
19 changed files with 1554 additions and 64 deletions
@@ -291,13 +291,56 @@
.OrderBy(a => a.AttributeName)
.ToList();
/// <summary>Attribute composition forest, rebuilt from the live latest-per-name dictionary.</summary>
private IReadOnlyList<DebugTreeNode> AttributeForest =>
DebugTreeBuilder.BuildAttributeTree(_attributeValues.Values, _attrFilter);
// ── Tree memoization (arch-review WP2.4) ──────────────────────────────────
// Both forests used to be rebuilt from scratch on EVERY render — and a render
// fired per streamed event. A busy instance therefore rebuilt two whole
// composition trees hundreds of times a second, on the circuit's single
// dispatcher thread. The rebuild now sits behind a version stamp bumped
// whenever the underlying dictionaries change (a coalesced stream flush,
// Connect, Disconnect), so an unrelated re-render (tab switch, toast, status
// strip tick) reuses the forest instead of rebuilding it. Reusing the same
// instance also lets Blazor skip re-rendering the TreeView subtree entirely.
//
// The cached key includes the dictionary Count as well as the version: the
// trees must never go stale if the dictionaries are populated by some path
// that forgets to bump the stamp.
private int _dataVersion;
/// <summary>Alarm composition forest, rebuilt from the live latest-per-name dictionary.</summary>
private IReadOnlyList<DebugTreeNode> AlarmForest =>
DebugTreeBuilder.BuildAlarmTree(_alarmStates.Values, _alarmFilter);
private IReadOnlyList<DebugTreeNode>? _attrForestCache;
private (int Version, int Count, string Filter) _attrForestKey = (-1, -1, string.Empty);
private IReadOnlyList<DebugTreeNode>? _alarmForestCache;
private (int Version, int Count, string Filter) _alarmForestKey = (-1, -1, string.Empty);
/// <summary>Attribute composition forest, rebuilt only when the underlying data or filter changed.</summary>
private IReadOnlyList<DebugTreeNode> AttributeForest
{
get
{
var key = (_dataVersion, _attributeValues.Count, _attrFilter);
if (_attrForestCache is null || _attrForestKey != key)
{
_attrForestCache = DebugTreeBuilder.BuildAttributeTree(_attributeValues.Values, _attrFilter);
_attrForestKey = key;
}
return _attrForestCache;
}
}
/// <summary>Alarm composition forest, rebuilt only when the underlying data or filter changed.</summary>
private IReadOnlyList<DebugTreeNode> AlarmForest
{
get
{
var key = (_dataVersion, _alarmStates.Count, _alarmFilter);
if (_alarmForestCache is null || _alarmForestKey != key)
{
_alarmForestCache = DebugTreeBuilder.BuildAlarmTree(_alarmStates.Values, _alarmFilter);
_alarmForestKey = key;
}
return _alarmForestCache;
}
}
private DebugStreamSession? _session;
private ToastNotification _toast = default!;
@@ -458,6 +501,9 @@
foreach (var al in session.InitialSnapshot.AlarmStates)
_alarmStates[al.AlarmName] = al;
// Snapshot seeding is a data change like any other — bump the stamp so
// the memoized forests rebuild from it.
_dataVersion++;
_snapshot = session.InitialSnapshot;
_connected = true;
@@ -492,6 +538,11 @@
_snapshot = null;
_attributeValues.Clear();
_alarmStates.Clear();
// Drop anything the stream buffered for the session we just tore down, so a
// late flush can't repopulate a disconnected view.
_pendingAttributes.Clear();
_pendingAlarms.Clear();
_dataVersion++;
}
/// <summary>
@@ -508,37 +559,108 @@
_toast.ShowInfo("Cleared previous session — select a site and instance to begin.", autoDismissMs: 5000);
}
// ── Render coalescing (arch-review WP2.4) ─────────────────────────────────
// Every streamed event used to marshal onto the dispatcher and call
// StateHasChanged individually, so a chatty instance drove one full render
// (and two full tree rebuilds) per value change. Events now land in a
// thread-safe pending map keyed by name — repeated updates to the same tag
// inside one window collapse to the latest — and exactly ONE dispatcher
// marshal per window drains them, bumps the version stamp and renders once.
private const int RenderCoalesceMs = 250;
private readonly System.Collections.Concurrent.ConcurrentDictionary<string, AttributeValueChanged> _pendingAttributes = new();
private readonly System.Collections.Concurrent.ConcurrentDictionary<string, AlarmStateChanged> _pendingAlarms = new();
/// <summary>0/1 flag: whether a coalesce window is already armed. Interlocked-guarded.</summary>
private int _flushArmed;
/// <summary>
/// Handles one debug-stream event. The callback is invoked on an Akka/gRPC
/// thread, but <see cref="_attributeValues"/>/<see cref="_alarmStates"/> are
/// <see cref="Dictionary{TKey,TValue}"/> instances also enumerated by the
/// render thread (the tree forests + <see cref="FilteredAttributeValues"/> are
/// built from them). <c>Dictionary</c> is not thread-safe (CentralUI-021): a
/// write racing an enumeration can throw or corrupt the buckets. The mutation
/// is therefore marshalled onto the renderer's dispatcher via
/// <see cref="SafeInvokeAsync"/> so every access to the dictionaries — read
/// and write — happens on the render thread.
/// write racing an enumeration can throw or corrupt the buckets. The event is
/// therefore parked in a concurrent pending map here and applied on the
/// renderer's dispatcher by <see cref="FlushPendingAsync"/>, so every access to
/// the render dictionaries — read and write — still happens on one thread.
/// </summary>
private void HandleStreamEvent(object evt)
{
// CentralUI-009: the component may have been disposed while this event
// was in flight on the Akka/gRPC thread.
if (_disposed) return;
_ = SafeInvokeAsync(() =>
switch (evt)
{
case AttributeValueChanged av:
_pendingAttributes[av.AttributeName] = av;
break;
case AlarmStateChanged al:
_pendingAlarms[al.AlarmName] = al;
break;
default:
// Unknown event type — nothing to apply, no render needed.
return;
}
// Arm the window if it isn't already. Losers of the CAS have nothing to do:
// their event is in the pending map and the armed flush will pick it up.
if (Interlocked.CompareExchange(ref _flushArmed, 1, 0) == 0)
{
_ = FlushPendingAsync();
}
}
/// <summary>
/// Waits out the coalesce window, then applies everything buffered in one
/// dispatcher pass and renders once.
/// </summary>
private async Task FlushPendingAsync()
{
try
{
await Task.Delay(RenderCoalesceMs);
}
catch
{
// Delay never faults in practice; if it ever did, fall through and drain.
}
if (_disposed)
{
Volatile.Write(ref _flushArmed, 0);
return;
}
await SafeInvokeAsync(() =>
{
// Disarm BEFORE draining: an event arriving mid-drain then arms a fresh
// window rather than being stranded until the next unrelated event.
Volatile.Write(ref _flushArmed, 0);
if (_disposed) return;
switch (evt)
var applied = false;
foreach (var key in _pendingAttributes.Keys)
{
case AttributeValueChanged av:
if (_pendingAttributes.TryRemove(key, out var av))
{
_attributeValues[av.AttributeName] = av;
break;
case AlarmStateChanged al:
_alarmStates[al.AlarmName] = al;
break;
default:
// Unknown event type — no re-render needed.
return;
applied = true;
}
}
foreach (var key in _pendingAlarms.Keys)
{
if (_pendingAlarms.TryRemove(key, out var al))
{
_alarmStates[al.AlarmName] = al;
applied = true;
}
}
if (!applied) return;
_dataVersion++;
StateHasChanged();
});
}
@@ -236,13 +236,66 @@
DeploymentStatusNotifier.StatusChanged += OnDeploymentStatusChanged;
}
// ── Push coalescing (arch-review WP2.4) ───────────────────────────────────
// One reload = every deployment record + every instance, re-read and re-filtered.
// The notifier fires per status WRITE, so a site deploy of N instances produced
// 2N+ of those full reloads back to back, per circuit. The reload is now
// leading-edge debounced: the first push after an idle gap still reloads
// immediately (a single deployment stays as responsive as before), and every
// push inside the window is absorbed into ONE trailing reload.
private const int ReloadDebounceMs = 500;
private DateTimeOffset _lastReloadAt = DateTimeOffset.MinValue;
/// <summary>Coalescing timer for pushes arriving inside the debounce window. Disposed with the component.</summary>
private Timer? _coalesceTimer;
private void OnDeploymentStatusChanged(ZB.MOM.WW.ScadaBridge.DeploymentManager.DeploymentStatusChange change)
{
// CentralUI-022: a callback racing disposal must not touch the component.
if (_disposed || !_autoRefresh) return;
lock (_coalesceLock)
{
if (_disposed) return;
var sinceLast = DateTimeOffset.UtcNow - _lastReloadAt;
if (sinceLast >= TimeSpan.FromMilliseconds(ReloadDebounceMs))
{
// Idle — reload straight away (leading edge).
_lastReloadAt = DateTimeOffset.UtcNow;
}
else
{
// Inside the window: arm one trailing reload for the remainder, and
// let any further push in this window ride it.
if (_coalesceTimer is not null) return;
var delay = TimeSpan.FromMilliseconds(ReloadDebounceMs) - sinceLast;
_coalesceTimer = new Timer(_ => OnCoalesceElapsed(), null, delay, Timeout.InfiniteTimeSpan);
return;
}
}
_ = DispatchReloadAsync();
}
/// <summary>Trailing edge of the debounce: disarm, stamp, and run the one coalesced reload.</summary>
private void OnCoalesceElapsed()
{
lock (_coalesceLock)
{
_coalesceTimer?.Dispose();
_coalesceTimer = null;
if (_disposed) return;
_lastReloadAt = DateTimeOffset.UtcNow;
}
_ = DispatchReloadAsync();
}
private readonly object _coalesceLock = new();
/// <summary>
/// Reloads the deployment table on the renderer's dispatcher, guarded
/// against the component being disposed mid-flight (CentralUI-022):
@@ -361,5 +414,10 @@
// status change reaches this disposed component.
_disposed = true;
DeploymentStatusNotifier.StatusChanged -= OnDeploymentStatusChanged;
lock (_coalesceLock)
{
_coalesceTimer?.Dispose();
_coalesceTimer = null;
}
}
}
@@ -1,5 +1,6 @@
@page "/monitoring/alarms"
@attribute [Authorize(Policy = ZB.MOM.WW.ScadaBridge.Security.AuthorizationPolicies.RequireDeployment)]
@using Microsoft.AspNetCore.Components.Web.Virtualization
@using ZB.MOM.WW.ScadaBridge.CentralUI.Components.Shared
@using ZB.MOM.WW.ScadaBridge.CentralUI.Services
@using ZB.MOM.WW.ScadaBridge.Commons.Entities.Sites
@@ -196,15 +197,25 @@
@onkeydown='e => OnHeaderKeyDown(e, "severity")' @onkeydown:preventDefault="_preventHeaderDefault">Severity @SortGlyph("severity")</th>
</tr>
</thead>
@* Row markup is identical on both paths — the same <tr>, the same cells,
the same data-test hook. Only WHO enumerates it differs: a plain
foreach for the ordinary case, and Virtualize once the flat table
gets long enough that rendering every row per delta dominates the
circuit (arch-review WP2.4). Virtualize needs <tr> spacers or the
browser hoists its <div>s out of the table and the layout collapses. *@
<tbody>
@foreach (var row in _visibleRows)
@if (_visibleRows.Count <= VirtualizeThreshold)
{
<tr data-test="alarm-summary-row">
<td class="font-monospace small">@row.InstanceUniqueName</td>
<td>@row.Alarm.AlarmName</td>
<td><AlarmStateBadges Alarm="row.Alarm" /></td>
<td class="text-end font-monospace">@row.Alarm.Condition.Severity</td>
</tr>
@foreach (var row in _visibleRows)
{
@AlarmRow(row)
}
}
else
{
<Virtualize Items="_visibleRows" Context="row" ItemSize="37" SpacerElement="tr">
@AlarmRow(row)
</Virtualize>
}
</tbody>
</table>
@@ -222,7 +233,23 @@
// P4 (arch-review): the filtered+sorted view is memoized rather than recomputed
// on every render. RecomputeVisibleRows() refreshes it whenever the inputs change
// (a fresh snapshot in RefreshAsync, a filter change via @bind:after, or a sort).
private IReadOnlyList<AlarmSummaryRow> _visibleRows = Array.Empty<AlarmSummaryRow>();
// Concrete List because Virtualize binds ICollection<T>, not IReadOnlyList<T>.
private List<AlarmSummaryRow> _visibleRows = new();
// Above this many visible rows the flat table switches to Virtualize. Below it the
// plain foreach is cheaper than the virtualization machinery (and needs no JS), and
// an operator filtered down to a handful of alarms should never pay for either.
private const int VirtualizeThreshold = 150;
// One row template, shared by the plain and virtualized paths so the two can never
// drift visually.
private RenderFragment<AlarmSummaryRow> AlarmRow => row =>
@<tr data-test="alarm-summary-row">
<td class="font-monospace small">@row.InstanceUniqueName</td>
<td>@row.Alarm.AlarmName</td>
<td><AlarmStateBadges Alarm="row.Alarm" /></td>
<td class="text-end font-monospace">@row.Alarm.Condition.Severity</td>
</tr>;
private IReadOnlyList<string> _notReporting = Array.Empty<string>();
private AlarmRollup _rollup = new(0, 0, 0, new Dictionary<AlarmKind, int>());
@@ -15,8 +15,7 @@
@implements IDisposable
@inject ICentralHealthAggregator HealthAggregator
@inject ISiteRepository SiteRepository
@inject CommunicationService CommunicationService
@inject IAuditLogQueryService AuditLogQueryService
@inject IKpiSnapshotCache KpiSnapshots
@inject IKpiHistoryQueryService KpiHistory
@inject Microsoft.Extensions.Options.IOptions<HealthMonitoringOptions> HealthOptions
@@ -640,7 +639,9 @@
// Non-fatal — fall back to showing siteId only
}
await RefreshNow();
// First load rides whatever the shared KPI cache already has fresh — a second
// operator opening the dashboard inside the window costs no extra query.
await RefreshAllAsync(forceRefresh: false);
// Site Health Trends (M6) load on their own path — never from the
// timer tick below — so a trend-query fault can't disturb the live tile
@@ -655,7 +656,7 @@
{
try
{
await RefreshNow();
await RefreshAllAsync(forceRefresh: false);
StateHasChanged();
}
finally
@@ -769,20 +770,38 @@
? "Central Cluster"
: $"{GetSiteName(siteKey)} ({siteKey})";
private async Task RefreshNow()
// Per-tile budget for one refresh tick. The underlying KPI Asks carry the
// 30s CommunicationOptions.QueryTimeout, so without this a single hung
// singleton stalled the WHOLE tick (and, with a 10s timer, stacked ticks)
// rather than degrading its own tile. Cancelling here abandons only this
// circuit's wait — the shared KPI flight continues and its result lands in
// the process-level cache for the next tick.
private static readonly TimeSpan TileTimeout = TimeSpan.FromSeconds(5);
// The operator's explicit "Refresh Now" bypasses the shared cache's freshness
// window; the timer tick and the first load ride whatever is already fresh
// (which is the whole point — N dashboards share one query per window).
private Task RefreshNow() => RefreshAllAsync(forceRefresh: true);
// Every tile loads concurrently, each with its own short budget, so one slow
// KPI degrades one tile instead of the tick. Every loader swallows its own
// faults, so Task.WhenAll here can never observe an exception.
private async Task RefreshAllAsync(bool forceRefresh)
{
_siteStates = HealthAggregator.GetAllSiteStates();
await LoadOutboxKpis();
await Task.WhenAll(LoadSiteCallKpis(), LoadSiteCallNodeKpis());
await LoadAuditKpis();
await Task.WhenAll(
LoadOutboxKpis(forceRefresh),
LoadSiteCallKpis(forceRefresh),
LoadSiteCallNodeKpis(forceRefresh),
LoadAuditKpis(forceRefresh));
}
private async Task LoadOutboxKpis()
private async Task LoadOutboxKpis(bool forceRefresh)
{
try
{
var response = await CommunicationService.GetNotificationKpisAsync(
new NotificationKpiRequest(Guid.NewGuid().ToString("N")));
using var timeout = new CancellationTokenSource(TileTimeout);
var response = await KpiSnapshots.GetNotificationKpisAsync(forceRefresh, timeout.Token);
if (response.Success)
{
_outboxKpi = response;
@@ -795,6 +814,11 @@
_outboxKpiError = response.ErrorMessage ?? "KPI query failed.";
}
}
catch (OperationCanceledException)
{
_outboxKpiAvailable = false;
_outboxKpiError = "KPI query timed out.";
}
catch (Exception ex)
{
_outboxKpiAvailable = false;
@@ -807,12 +831,12 @@
// killing the dashboard. Mirrors LoadOutboxKpis's error handling shape — a
// response with Success == false (repository fault) and an Ask that threw
// (transport fault) both collapse to "unavailable".
private async Task LoadSiteCallKpis()
private async Task LoadSiteCallKpis(bool forceRefresh)
{
try
{
var response = await CommunicationService.GetSiteCallKpisAsync(
new SiteCallKpiRequest(Guid.NewGuid().ToString("N")));
using var timeout = new CancellationTokenSource(TileTimeout);
var response = await KpiSnapshots.GetSiteCallKpisAsync(forceRefresh, timeout.Token);
if (response.Success)
{
_siteCallKpi = response;
@@ -825,6 +849,11 @@
_siteCallKpiError = response.ErrorMessage ?? "KPI query failed.";
}
}
catch (OperationCanceledException)
{
_siteCallKpiAvailable = false;
_siteCallKpiError = "KPI query timed out.";
}
catch (Exception ex)
{
_siteCallKpiAvailable = false;
@@ -834,12 +863,12 @@
// Per-node site-call KPI loader (M5.2). Best-effort; a fault silently
// suppresses the per-node sub-table rather than degrading the dashboard.
private async Task LoadSiteCallNodeKpis()
private async Task LoadSiteCallNodeKpis(bool forceRefresh)
{
try
{
var response = await CommunicationService.GetPerNodeSiteCallKpisAsync(
new PerNodeSiteCallKpiRequest(Guid.NewGuid().ToString("N")));
using var timeout = new CancellationTokenSource(TileTimeout);
var response = await KpiSnapshots.GetPerNodeSiteCallKpisAsync(forceRefresh, timeout.Token);
if (response.Success)
{
_siteCallNodeKpis = response.Nodes;
@@ -864,14 +893,20 @@
// Audit KPI loader: wraps the service call so a transient DB outage degrades
// the three tiles to em dashes with an inline error rather than killing the
// dashboard. Mirrors LoadOutboxKpis's error handling shape.
private async Task LoadAuditKpis()
private async Task LoadAuditKpis(bool forceRefresh)
{
try
{
_auditKpi = await AuditLogQueryService.GetKpiSnapshotAsync();
using var timeout = new CancellationTokenSource(TileTimeout);
_auditKpi = await KpiSnapshots.GetAuditKpiSnapshotAsync(forceRefresh, timeout.Token);
_auditKpiAvailable = true;
_auditKpiError = null;
}
catch (OperationCanceledException)
{
_auditKpiAvailable = false;
_auditKpiError = "KPI query timed out.";
}
catch (Exception ex)
{
_auditKpiAvailable = false;
@@ -7,7 +7,7 @@
@using ZB.MOM.WW.ScadaBridge.Commons.Types.Kpi
@using ZB.MOM.WW.ScadaBridge.Communication
@using ZB.MOM.WW.ScadaBridge.CentralUI.Services
@inject CommunicationService CommunicationService
@inject IKpiSnapshotCache KpiSnapshots
@inject ISiteRepository SiteRepository
@inject IKpiHistoryQueryService KpiHistory
@inject ILogger<NotificationKpis> Logger
@@ -250,15 +250,24 @@
Logger.LogWarning(ex, "Failed to load sites for the KPI per-site breakdown.");
}
await RefreshAll();
// First load rides whatever the shared KPI cache already holds fresh; the
// Refresh button forces a new query (see RefreshAll).
await LoadAllAsync(forceRefresh: false);
}
private async Task RefreshAll()
// The operator's explicit Refresh bypasses the shared cache's freshness window.
private Task RefreshAll() => LoadAllAsync(forceRefresh: true);
private async Task LoadAllAsync(bool forceRefresh)
{
_loading = true;
// Race-free despite all tasks mutating component fields: Blazor Server runs
// every continuation on the circuit's single-threaded synchronization context.
await Task.WhenAll(LoadGlobalKpis(), LoadPerSiteKpis(), LoadPerNodeKpis(), LoadTrends());
await Task.WhenAll(
LoadGlobalKpis(forceRefresh),
LoadPerSiteKpis(forceRefresh),
LoadPerNodeKpis(forceRefresh),
LoadTrends());
_loading = false;
}
@@ -320,12 +329,11 @@
}
}
private async Task LoadGlobalKpis()
private async Task LoadGlobalKpis(bool forceRefresh)
{
try
{
var response = await CommunicationService.GetNotificationKpisAsync(
new NotificationKpiRequest(Guid.NewGuid().ToString("N")));
var response = await KpiSnapshots.GetNotificationKpisAsync(forceRefresh);
if (response.Success)
{
_kpi = response;
@@ -342,12 +350,11 @@
}
}
private async Task LoadPerSiteKpis()
private async Task LoadPerSiteKpis(bool forceRefresh)
{
try
{
var response = await CommunicationService.GetPerSiteNotificationKpisAsync(
new PerSiteNotificationKpiRequest(Guid.NewGuid().ToString("N")));
var response = await KpiSnapshots.GetPerSiteNotificationKpisAsync(forceRefresh);
if (response.Success)
{
_perSite = response.Sites;
@@ -364,12 +371,11 @@
}
}
private async Task LoadPerNodeKpis()
private async Task LoadPerNodeKpis(bool forceRefresh)
{
try
{
var response = await CommunicationService.GetPerNodeNotificationKpisAsync(
new PerNodeNotificationKpiRequest(Guid.NewGuid().ToString("N")));
var response = await KpiSnapshots.GetPerNodeNotificationKpisAsync(forceRefresh);
if (response.Success)
{
_perNode = response.Nodes;