perf(ui): shared KPI cache, live-cache-backed alarm summary, coalesced debug renders

This commit is contained in:
Joseph Doherty
2026-08-14 20:53:38 -04:00
parent ee193cd2bb
commit 8c0b36b2fa
19 changed files with 1554 additions and 64 deletions
@@ -291,13 +291,56 @@
.OrderBy(a => a.AttributeName)
.ToList();
/// <summary>Attribute composition forest, rebuilt from the live latest-per-name dictionary.</summary>
private IReadOnlyList<DebugTreeNode> AttributeForest =>
DebugTreeBuilder.BuildAttributeTree(_attributeValues.Values, _attrFilter);
// ── Tree memoization (arch-review WP2.4) ──────────────────────────────────
// Both forests used to be rebuilt from scratch on EVERY render — and a render
// fired per streamed event. A busy instance therefore rebuilt two whole
// composition trees hundreds of times a second, on the circuit's single
// dispatcher thread. The rebuild now sits behind a version stamp bumped
// whenever the underlying dictionaries change (a coalesced stream flush,
// Connect, Disconnect), so an unrelated re-render (tab switch, toast, status
// strip tick) reuses the forest instead of rebuilding it. Reusing the same
// instance also lets Blazor skip re-rendering the TreeView subtree entirely.
//
// The cached key includes the dictionary Count as well as the version: the
// trees must never go stale if the dictionaries are populated by some path
// that forgets to bump the stamp.
private int _dataVersion;
/// <summary>Alarm composition forest, rebuilt from the live latest-per-name dictionary.</summary>
private IReadOnlyList<DebugTreeNode> AlarmForest =>
DebugTreeBuilder.BuildAlarmTree(_alarmStates.Values, _alarmFilter);
private IReadOnlyList<DebugTreeNode>? _attrForestCache;
private (int Version, int Count, string Filter) _attrForestKey = (-1, -1, string.Empty);
private IReadOnlyList<DebugTreeNode>? _alarmForestCache;
private (int Version, int Count, string Filter) _alarmForestKey = (-1, -1, string.Empty);
/// <summary>Attribute composition forest, rebuilt only when the underlying data or filter changed.</summary>
private IReadOnlyList<DebugTreeNode> AttributeForest
{
get
{
var key = (_dataVersion, _attributeValues.Count, _attrFilter);
if (_attrForestCache is null || _attrForestKey != key)
{
_attrForestCache = DebugTreeBuilder.BuildAttributeTree(_attributeValues.Values, _attrFilter);
_attrForestKey = key;
}
return _attrForestCache;
}
}
/// <summary>Alarm composition forest, rebuilt only when the underlying data or filter changed.</summary>
private IReadOnlyList<DebugTreeNode> AlarmForest
{
get
{
var key = (_dataVersion, _alarmStates.Count, _alarmFilter);
if (_alarmForestCache is null || _alarmForestKey != key)
{
_alarmForestCache = DebugTreeBuilder.BuildAlarmTree(_alarmStates.Values, _alarmFilter);
_alarmForestKey = key;
}
return _alarmForestCache;
}
}
private DebugStreamSession? _session;
private ToastNotification _toast = default!;
@@ -458,6 +501,9 @@
foreach (var al in session.InitialSnapshot.AlarmStates)
_alarmStates[al.AlarmName] = al;
// Snapshot seeding is a data change like any other — bump the stamp so
// the memoized forests rebuild from it.
_dataVersion++;
_snapshot = session.InitialSnapshot;
_connected = true;
@@ -492,6 +538,11 @@
_snapshot = null;
_attributeValues.Clear();
_alarmStates.Clear();
// Drop anything the stream buffered for the session we just tore down, so a
// late flush can't repopulate a disconnected view.
_pendingAttributes.Clear();
_pendingAlarms.Clear();
_dataVersion++;
}
/// <summary>
@@ -508,37 +559,108 @@
_toast.ShowInfo("Cleared previous session — select a site and instance to begin.", autoDismissMs: 5000);
}
// ── Render coalescing (arch-review WP2.4) ─────────────────────────────────
// Every streamed event used to marshal onto the dispatcher and call
// StateHasChanged individually, so a chatty instance drove one full render
// (and two full tree rebuilds) per value change. Events now land in a
// thread-safe pending map keyed by name — repeated updates to the same tag
// inside one window collapse to the latest — and exactly ONE dispatcher
// marshal per window drains them, bumps the version stamp and renders once.
private const int RenderCoalesceMs = 250;
private readonly System.Collections.Concurrent.ConcurrentDictionary<string, AttributeValueChanged> _pendingAttributes = new();
private readonly System.Collections.Concurrent.ConcurrentDictionary<string, AlarmStateChanged> _pendingAlarms = new();
/// <summary>0/1 flag: whether a coalesce window is already armed. Interlocked-guarded.</summary>
private int _flushArmed;
/// <summary>
/// Handles one debug-stream event. The callback is invoked on an Akka/gRPC
/// thread, but <see cref="_attributeValues"/>/<see cref="_alarmStates"/> are
/// <see cref="Dictionary{TKey,TValue}"/> instances also enumerated by the
/// render thread (the tree forests + <see cref="FilteredAttributeValues"/> are
/// built from them). <c>Dictionary</c> is not thread-safe (CentralUI-021): a
/// write racing an enumeration can throw or corrupt the buckets. The mutation
/// is therefore marshalled onto the renderer's dispatcher via
/// <see cref="SafeInvokeAsync"/> so every access to the dictionaries — read
/// and write — happens on the render thread.
/// write racing an enumeration can throw or corrupt the buckets. The event is
/// therefore parked in a concurrent pending map here and applied on the
/// renderer's dispatcher by <see cref="FlushPendingAsync"/>, so every access to
/// the render dictionaries — read and write — still happens on one thread.
/// </summary>
private void HandleStreamEvent(object evt)
{
// CentralUI-009: the component may have been disposed while this event
// was in flight on the Akka/gRPC thread.
if (_disposed) return;
_ = SafeInvokeAsync(() =>
switch (evt)
{
case AttributeValueChanged av:
_pendingAttributes[av.AttributeName] = av;
break;
case AlarmStateChanged al:
_pendingAlarms[al.AlarmName] = al;
break;
default:
// Unknown event type — nothing to apply, no render needed.
return;
}
// Arm the window if it isn't already. Losers of the CAS have nothing to do:
// their event is in the pending map and the armed flush will pick it up.
if (Interlocked.CompareExchange(ref _flushArmed, 1, 0) == 0)
{
_ = FlushPendingAsync();
}
}
/// <summary>
/// Waits out the coalesce window, then applies everything buffered in one
/// dispatcher pass and renders once.
/// </summary>
private async Task FlushPendingAsync()
{
try
{
await Task.Delay(RenderCoalesceMs);
}
catch
{
// Delay never faults in practice; if it ever did, fall through and drain.
}
if (_disposed)
{
Volatile.Write(ref _flushArmed, 0);
return;
}
await SafeInvokeAsync(() =>
{
// Disarm BEFORE draining: an event arriving mid-drain then arms a fresh
// window rather than being stranded until the next unrelated event.
Volatile.Write(ref _flushArmed, 0);
if (_disposed) return;
switch (evt)
var applied = false;
foreach (var key in _pendingAttributes.Keys)
{
case AttributeValueChanged av:
if (_pendingAttributes.TryRemove(key, out var av))
{
_attributeValues[av.AttributeName] = av;
break;
case AlarmStateChanged al:
_alarmStates[al.AlarmName] = al;
break;
default:
// Unknown event type — no re-render needed.
return;
applied = true;
}
}
foreach (var key in _pendingAlarms.Keys)
{
if (_pendingAlarms.TryRemove(key, out var al))
{
_alarmStates[al.AlarmName] = al;
applied = true;
}
}
if (!applied) return;
_dataVersion++;
StateHasChanged();
});
}
@@ -236,13 +236,66 @@
DeploymentStatusNotifier.StatusChanged += OnDeploymentStatusChanged;
}
// ── Push coalescing (arch-review WP2.4) ───────────────────────────────────
// One reload = every deployment record + every instance, re-read and re-filtered.
// The notifier fires per status WRITE, so a site deploy of N instances produced
// 2N+ of those full reloads back to back, per circuit. The reload is now
// leading-edge debounced: the first push after an idle gap still reloads
// immediately (a single deployment stays as responsive as before), and every
// push inside the window is absorbed into ONE trailing reload.
private const int ReloadDebounceMs = 500;
private DateTimeOffset _lastReloadAt = DateTimeOffset.MinValue;
/// <summary>Coalescing timer for pushes arriving inside the debounce window. Disposed with the component.</summary>
private Timer? _coalesceTimer;
private void OnDeploymentStatusChanged(ZB.MOM.WW.ScadaBridge.DeploymentManager.DeploymentStatusChange change)
{
// CentralUI-022: a callback racing disposal must not touch the component.
if (_disposed || !_autoRefresh) return;
lock (_coalesceLock)
{
if (_disposed) return;
var sinceLast = DateTimeOffset.UtcNow - _lastReloadAt;
if (sinceLast >= TimeSpan.FromMilliseconds(ReloadDebounceMs))
{
// Idle — reload straight away (leading edge).
_lastReloadAt = DateTimeOffset.UtcNow;
}
else
{
// Inside the window: arm one trailing reload for the remainder, and
// let any further push in this window ride it.
if (_coalesceTimer is not null) return;
var delay = TimeSpan.FromMilliseconds(ReloadDebounceMs) - sinceLast;
_coalesceTimer = new Timer(_ => OnCoalesceElapsed(), null, delay, Timeout.InfiniteTimeSpan);
return;
}
}
_ = DispatchReloadAsync();
}
/// <summary>Trailing edge of the debounce: disarm, stamp, and run the one coalesced reload.</summary>
private void OnCoalesceElapsed()
{
lock (_coalesceLock)
{
_coalesceTimer?.Dispose();
_coalesceTimer = null;
if (_disposed) return;
_lastReloadAt = DateTimeOffset.UtcNow;
}
_ = DispatchReloadAsync();
}
private readonly object _coalesceLock = new();
/// <summary>
/// Reloads the deployment table on the renderer's dispatcher, guarded
/// against the component being disposed mid-flight (CentralUI-022):
@@ -361,5 +414,10 @@
// status change reaches this disposed component.
_disposed = true;
DeploymentStatusNotifier.StatusChanged -= OnDeploymentStatusChanged;
lock (_coalesceLock)
{
_coalesceTimer?.Dispose();
_coalesceTimer = null;
}
}
}
@@ -1,5 +1,6 @@
@page "/monitoring/alarms"
@attribute [Authorize(Policy = ZB.MOM.WW.ScadaBridge.Security.AuthorizationPolicies.RequireDeployment)]
@using Microsoft.AspNetCore.Components.Web.Virtualization
@using ZB.MOM.WW.ScadaBridge.CentralUI.Components.Shared
@using ZB.MOM.WW.ScadaBridge.CentralUI.Services
@using ZB.MOM.WW.ScadaBridge.Commons.Entities.Sites
@@ -196,15 +197,25 @@
@onkeydown='e => OnHeaderKeyDown(e, "severity")' @onkeydown:preventDefault="_preventHeaderDefault">Severity @SortGlyph("severity")</th>
</tr>
</thead>
@* Row markup is identical on both paths — the same <tr>, the same cells,
the same data-test hook. Only WHO enumerates it differs: a plain
foreach for the ordinary case, and Virtualize once the flat table
gets long enough that rendering every row per delta dominates the
circuit (arch-review WP2.4). Virtualize needs <tr> spacers or the
browser hoists its <div>s out of the table and the layout collapses. *@
<tbody>
@foreach (var row in _visibleRows)
@if (_visibleRows.Count <= VirtualizeThreshold)
{
<tr data-test="alarm-summary-row">
<td class="font-monospace small">@row.InstanceUniqueName</td>
<td>@row.Alarm.AlarmName</td>
<td><AlarmStateBadges Alarm="row.Alarm" /></td>
<td class="text-end font-monospace">@row.Alarm.Condition.Severity</td>
</tr>
@foreach (var row in _visibleRows)
{
@AlarmRow(row)
}
}
else
{
<Virtualize Items="_visibleRows" Context="row" ItemSize="37" SpacerElement="tr">
@AlarmRow(row)
</Virtualize>
}
</tbody>
</table>
@@ -222,7 +233,23 @@
// P4 (arch-review): the filtered+sorted view is memoized rather than recomputed
// on every render. RecomputeVisibleRows() refreshes it whenever the inputs change
// (a fresh snapshot in RefreshAsync, a filter change via @bind:after, or a sort).
private IReadOnlyList<AlarmSummaryRow> _visibleRows = Array.Empty<AlarmSummaryRow>();
// Concrete List because Virtualize binds ICollection<T>, not IReadOnlyList<T>.
private List<AlarmSummaryRow> _visibleRows = new();
// Above this many visible rows the flat table switches to Virtualize. Below it the
// plain foreach is cheaper than the virtualization machinery (and needs no JS), and
// an operator filtered down to a handful of alarms should never pay for either.
private const int VirtualizeThreshold = 150;
// One row template, shared by the plain and virtualized paths so the two can never
// drift visually.
private RenderFragment<AlarmSummaryRow> AlarmRow => row =>
@<tr data-test="alarm-summary-row">
<td class="font-monospace small">@row.InstanceUniqueName</td>
<td>@row.Alarm.AlarmName</td>
<td><AlarmStateBadges Alarm="row.Alarm" /></td>
<td class="text-end font-monospace">@row.Alarm.Condition.Severity</td>
</tr>;
private IReadOnlyList<string> _notReporting = Array.Empty<string>();
private AlarmRollup _rollup = new(0, 0, 0, new Dictionary<AlarmKind, int>());
@@ -15,8 +15,7 @@
@implements IDisposable
@inject ICentralHealthAggregator HealthAggregator
@inject ISiteRepository SiteRepository
@inject CommunicationService CommunicationService
@inject IAuditLogQueryService AuditLogQueryService
@inject IKpiSnapshotCache KpiSnapshots
@inject IKpiHistoryQueryService KpiHistory
@inject Microsoft.Extensions.Options.IOptions<HealthMonitoringOptions> HealthOptions
@@ -640,7 +639,9 @@
// Non-fatal — fall back to showing siteId only
}
await RefreshNow();
// First load rides whatever the shared KPI cache already has fresh — a second
// operator opening the dashboard inside the window costs no extra query.
await RefreshAllAsync(forceRefresh: false);
// Site Health Trends (M6) load on their own path — never from the
// timer tick below — so a trend-query fault can't disturb the live tile
@@ -655,7 +656,7 @@
{
try
{
await RefreshNow();
await RefreshAllAsync(forceRefresh: false);
StateHasChanged();
}
finally
@@ -769,20 +770,38 @@
? "Central Cluster"
: $"{GetSiteName(siteKey)} ({siteKey})";
private async Task RefreshNow()
// Per-tile budget for one refresh tick. The underlying KPI Asks carry the
// 30s CommunicationOptions.QueryTimeout, so without this a single hung
// singleton stalled the WHOLE tick (and, with a 10s timer, stacked ticks)
// rather than degrading its own tile. Cancelling here abandons only this
// circuit's wait — the shared KPI flight continues and its result lands in
// the process-level cache for the next tick.
private static readonly TimeSpan TileTimeout = TimeSpan.FromSeconds(5);
// The operator's explicit "Refresh Now" bypasses the shared cache's freshness
// window; the timer tick and the first load ride whatever is already fresh
// (which is the whole point — N dashboards share one query per window).
private Task RefreshNow() => RefreshAllAsync(forceRefresh: true);
// Every tile loads concurrently, each with its own short budget, so one slow
// KPI degrades one tile instead of the tick. Every loader swallows its own
// faults, so Task.WhenAll here can never observe an exception.
private async Task RefreshAllAsync(bool forceRefresh)
{
_siteStates = HealthAggregator.GetAllSiteStates();
await LoadOutboxKpis();
await Task.WhenAll(LoadSiteCallKpis(), LoadSiteCallNodeKpis());
await LoadAuditKpis();
await Task.WhenAll(
LoadOutboxKpis(forceRefresh),
LoadSiteCallKpis(forceRefresh),
LoadSiteCallNodeKpis(forceRefresh),
LoadAuditKpis(forceRefresh));
}
private async Task LoadOutboxKpis()
private async Task LoadOutboxKpis(bool forceRefresh)
{
try
{
var response = await CommunicationService.GetNotificationKpisAsync(
new NotificationKpiRequest(Guid.NewGuid().ToString("N")));
using var timeout = new CancellationTokenSource(TileTimeout);
var response = await KpiSnapshots.GetNotificationKpisAsync(forceRefresh, timeout.Token);
if (response.Success)
{
_outboxKpi = response;
@@ -795,6 +814,11 @@
_outboxKpiError = response.ErrorMessage ?? "KPI query failed.";
}
}
catch (OperationCanceledException)
{
_outboxKpiAvailable = false;
_outboxKpiError = "KPI query timed out.";
}
catch (Exception ex)
{
_outboxKpiAvailable = false;
@@ -807,12 +831,12 @@
// killing the dashboard. Mirrors LoadOutboxKpis's error handling shape — a
// response with Success == false (repository fault) and an Ask that threw
// (transport fault) both collapse to "unavailable".
private async Task LoadSiteCallKpis()
private async Task LoadSiteCallKpis(bool forceRefresh)
{
try
{
var response = await CommunicationService.GetSiteCallKpisAsync(
new SiteCallKpiRequest(Guid.NewGuid().ToString("N")));
using var timeout = new CancellationTokenSource(TileTimeout);
var response = await KpiSnapshots.GetSiteCallKpisAsync(forceRefresh, timeout.Token);
if (response.Success)
{
_siteCallKpi = response;
@@ -825,6 +849,11 @@
_siteCallKpiError = response.ErrorMessage ?? "KPI query failed.";
}
}
catch (OperationCanceledException)
{
_siteCallKpiAvailable = false;
_siteCallKpiError = "KPI query timed out.";
}
catch (Exception ex)
{
_siteCallKpiAvailable = false;
@@ -834,12 +863,12 @@
// Per-node site-call KPI loader (M5.2). Best-effort; a fault silently
// suppresses the per-node sub-table rather than degrading the dashboard.
private async Task LoadSiteCallNodeKpis()
private async Task LoadSiteCallNodeKpis(bool forceRefresh)
{
try
{
var response = await CommunicationService.GetPerNodeSiteCallKpisAsync(
new PerNodeSiteCallKpiRequest(Guid.NewGuid().ToString("N")));
using var timeout = new CancellationTokenSource(TileTimeout);
var response = await KpiSnapshots.GetPerNodeSiteCallKpisAsync(forceRefresh, timeout.Token);
if (response.Success)
{
_siteCallNodeKpis = response.Nodes;
@@ -864,14 +893,20 @@
// Audit KPI loader: wraps the service call so a transient DB outage degrades
// the three tiles to em dashes with an inline error rather than killing the
// dashboard. Mirrors LoadOutboxKpis's error handling shape.
private async Task LoadAuditKpis()
private async Task LoadAuditKpis(bool forceRefresh)
{
try
{
_auditKpi = await AuditLogQueryService.GetKpiSnapshotAsync();
using var timeout = new CancellationTokenSource(TileTimeout);
_auditKpi = await KpiSnapshots.GetAuditKpiSnapshotAsync(forceRefresh, timeout.Token);
_auditKpiAvailable = true;
_auditKpiError = null;
}
catch (OperationCanceledException)
{
_auditKpiAvailable = false;
_auditKpiError = "KPI query timed out.";
}
catch (Exception ex)
{
_auditKpiAvailable = false;
@@ -7,7 +7,7 @@
@using ZB.MOM.WW.ScadaBridge.Commons.Types.Kpi
@using ZB.MOM.WW.ScadaBridge.Communication
@using ZB.MOM.WW.ScadaBridge.CentralUI.Services
@inject CommunicationService CommunicationService
@inject IKpiSnapshotCache KpiSnapshots
@inject ISiteRepository SiteRepository
@inject IKpiHistoryQueryService KpiHistory
@inject ILogger<NotificationKpis> Logger
@@ -250,15 +250,24 @@
Logger.LogWarning(ex, "Failed to load sites for the KPI per-site breakdown.");
}
await RefreshAll();
// First load rides whatever the shared KPI cache already holds fresh; the
// Refresh button forces a new query (see RefreshAll).
await LoadAllAsync(forceRefresh: false);
}
private async Task RefreshAll()
// The operator's explicit Refresh bypasses the shared cache's freshness window.
private Task RefreshAll() => LoadAllAsync(forceRefresh: true);
private async Task LoadAllAsync(bool forceRefresh)
{
_loading = true;
// Race-free despite all tasks mutating component fields: Blazor Server runs
// every continuation on the circuit's single-threaded synchronization context.
await Task.WhenAll(LoadGlobalKpis(), LoadPerSiteKpis(), LoadPerNodeKpis(), LoadTrends());
await Task.WhenAll(
LoadGlobalKpis(forceRefresh),
LoadPerSiteKpis(forceRefresh),
LoadPerNodeKpis(forceRefresh),
LoadTrends());
_loading = false;
}
@@ -320,12 +329,11 @@
}
}
private async Task LoadGlobalKpis()
private async Task LoadGlobalKpis(bool forceRefresh)
{
try
{
var response = await CommunicationService.GetNotificationKpisAsync(
new NotificationKpiRequest(Guid.NewGuid().ToString("N")));
var response = await KpiSnapshots.GetNotificationKpisAsync(forceRefresh);
if (response.Success)
{
_kpi = response;
@@ -342,12 +350,11 @@
}
}
private async Task LoadPerSiteKpis()
private async Task LoadPerSiteKpis(bool forceRefresh)
{
try
{
var response = await CommunicationService.GetPerSiteNotificationKpisAsync(
new PerSiteNotificationKpiRequest(Guid.NewGuid().ToString("N")));
var response = await KpiSnapshots.GetPerSiteNotificationKpisAsync(forceRefresh);
if (response.Success)
{
_perSite = response.Sites;
@@ -364,12 +371,11 @@
}
}
private async Task LoadPerNodeKpis()
private async Task LoadPerNodeKpis(bool forceRefresh)
{
try
{
var response = await CommunicationService.GetPerNodeNotificationKpisAsync(
new PerNodeNotificationKpiRequest(Guid.NewGuid().ToString("N")));
var response = await KpiSnapshots.GetPerNodeNotificationKpisAsync(forceRefresh);
if (response.Success)
{
_perNode = response.Nodes;
@@ -179,6 +179,31 @@ public class ScriptAnalysisService
private const int SandboxMaxCallSharedDepth = 16;
/// <summary>
/// Process-wide cap on concurrently executing Test Runs (arch-review WP2.4).
/// <para>
/// A sandbox run occupies a thread-pool thread for its whole duration — the
/// <c>SandboxScriptHost</c> attribute accessors are synchronous by contract and block on
/// cross-site I/O (<c>GetAwaiter().GetResult()</c>), so the thread is <em>parked</em>, not
/// merely busy, for up to the 10s sandbox timeout. Unbounded, a handful of Designers
/// each hitting Run on a slow-bound script could park enough pool threads to starve the
/// Blazor circuits sharing this process. The gate bounds that: excess runs wait for a
/// slot instead of taking a thread, and the wait is charged to the caller's own timeout.
/// </para>
/// <para>
/// Static because <c>ScriptAnalysisService</c> is registered scoped — a per-instance gate
/// would be per-request and bound nothing.
/// </para>
/// </summary>
private static readonly SemaphoreSlim SandboxRunGate =
new(SandboxMaxConcurrentRuns, SandboxMaxConcurrentRuns);
/// <summary>
/// Slot count for <see cref="SandboxRunGate"/>. Deliberately small: Test Run is an
/// authoring convenience on a node whose real job is serving circuits.
/// </summary>
private const int SandboxMaxConcurrentRuns = 4;
/// <summary>
/// Compiles and runs a script in the central process. The globals surface
/// depends on <see cref="SandboxRunRequest.Kind"/>: template and shared
@@ -428,8 +453,15 @@ public class ScriptAnalysisService
using var errorScope = captureError.BeginCapture(captured);
var stopwatch = Stopwatch.StartNew();
var gateAcquired = false;
try
{
// Bound concurrent Test Runs (see SandboxRunGate). The wait is charged to the
// caller's own linked token, so queue time counts against the sandbox timeout
// and a queued run reports Timeout rather than hanging.
await SandboxRunGate.WaitAsync(linkedCts.Token).ConfigureAwait(false);
gateAcquired = true;
// Run on a thread-pool thread with no SynchronizationContext: a
// bound script's Instance.SetAttribute / Attributes[...] block
// synchronously on cross-site I/O (the API surface is sync by
@@ -480,6 +512,10 @@ public class ScriptAnalysisService
$"{inner.GetType().Name}: {inner.Message}",
SandboxErrorKind.RuntimeError, stopwatch.ElapsedMilliseconds, null);
}
finally
{
if (gateAcquired) SandboxRunGate.Release();
}
// outScope / errorScope are disposed by their `using` declarations when the
// method returns, restoring the previous capture scope on this call-tree
// without touching process-global Console state.
@@ -99,7 +99,17 @@ public static class ServiceCollectionExtensions
// facade over CommunicationService.RequestDebugSnapshotAsync (the same
// single-shot Ask the Debug View uses) — and flattens the alarm states.
services.AddScoped<IInstanceSnapshotClient, CommunicationInstanceSnapshotClient>();
services.AddScoped<IAlarmSummaryService, AlarmSummaryService>();
// The fan-out itself stays scoped (it holds scoped repositories + the snapshot
// client), but the page resolves it through SharedAlarmSummaryService — a process
// singleton that memoizes the fan-out per site with single-flight, so N circuits
// watching one site cost ONE fan-out per window instead of N (arch-review WP2.4).
services.AddScoped<AlarmSummaryService>();
services.AddSingleton<IAlarmSummaryService, SharedAlarmSummaryService>();
// Process-level memoized point-in-time KPI snapshots (arch-review WP2.4, finding
// #8's per-circuit half). Singleton on purpose: the memo has to outlive any one
// circuit for N dashboards to share a single SQL round per refresh window.
services.AddSingleton<IKpiSnapshotCache, KpiSnapshotCache>();
// Secured Writes: dispatches the two-person secured-write commands
// (submit / approve / reject / list) to the central ManagementActor through the
@@ -128,7 +128,20 @@ public sealed class AlarmSummaryService : IAlarmSummaryService
}
/// <inheritdoc/>
public AlarmSummaryResult BuildFromLiveAlarms(IReadOnlyList<AlarmStateChanged> alarms)
public AlarmSummaryResult BuildFromLiveAlarms(IReadOnlyList<AlarmStateChanged> alarms) =>
BuildFromLiveAlarmsCore(alarms);
/// <inheritdoc/>
public AlarmRollup ComputeRollup(IReadOnlyList<AlarmSummaryRow> rows) => ComputeRollupCore(rows);
/// <summary>
/// Pure flattening of a live-cache snapshot into summary rows. Shared with
/// <see cref="SharedAlarmSummaryService"/>, which memoizes the fan-out but must produce
/// byte-identical rows on the live path — one implementation, no drift.
/// </summary>
/// <param name="alarms">The live-cache alarm snapshot.</param>
/// <returns>The flattened, deterministically ordered rows (never any not-reporting names).</returns>
internal static AlarmSummaryResult BuildFromLiveAlarmsCore(IReadOnlyList<AlarmStateChanged> alarms)
{
ArgumentNullException.ThrowIfNull(alarms);
@@ -147,8 +160,14 @@ public sealed class AlarmSummaryService : IAlarmSummaryService
return new AlarmSummaryResult(orderedRows, Array.Empty<string>());
}
/// <inheritdoc/>
public AlarmRollup ComputeRollup(IReadOnlyList<AlarmSummaryRow> rows)
/// <summary>
/// Pure roll-up over already-flattened rows. Shared with
/// <see cref="SharedAlarmSummaryService"/> for the same one-implementation reason as
/// <see cref="BuildFromLiveAlarmsCore"/>.
/// </summary>
/// <param name="rows">The rows to roll up.</param>
/// <returns>The active/worst-severity/unacked/by-kind roll-up.</returns>
internal static AlarmRollup ComputeRollupCore(IReadOnlyList<AlarmSummaryRow> rows)
{
ArgumentNullException.ThrowIfNull(rows);
@@ -0,0 +1,70 @@
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Audit;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Notification;
using ZB.MOM.WW.ScadaBridge.Commons.Types;
namespace ZB.MOM.WW.ScadaBridge.CentralUI.Services;
/// <summary>
/// Process-level memoized point-in-time KPI snapshots shared by every Blazor circuit on
/// this node (arch-review WP2.4, finding #8 — the per-circuit polling half).
/// <para>
/// Every KPI here is a <em>global, point-in-time aggregate</em>: it does not vary by user,
/// site scope, or role, so N circuits asking for it inside one refresh window are asking
/// the identical question. Without memoization, ten Health dashboards meant ten Asks →
/// ten aggregate SQL round trips every ten seconds. Each accessor is independently
/// memoized with a short TTL and single-flight production, so the fleet costs ONE round
/// trip per KPI per window regardless of viewer count.
/// </para>
/// <para>
/// Faults are never memoized: a failed query is re-attempted by the next caller, and the
/// caller still sees the exception exactly as it would from a direct query — the pages'
/// existing per-tile "unavailable" degradation is unchanged.
/// </para>
/// </summary>
public interface IKpiSnapshotCache
{
/// <summary>Global Notification Outbox KPIs (queue depth, stuck, parked, delivered-last-interval).</summary>
/// <param name="forceRefresh">Bypass a still-fresh value (operator "Refresh"); an in-flight round is still joined.</param>
/// <param name="cancellationToken">Cancels this caller's wait, never the shared flight.</param>
/// <returns>The memoized notification KPI response.</returns>
Task<NotificationKpiResponse> GetNotificationKpisAsync(
bool forceRefresh = false, CancellationToken cancellationToken = default);
/// <summary>Per-source-site Notification Outbox KPI breakdown.</summary>
/// <param name="forceRefresh">Bypass a still-fresh value; an in-flight round is still joined.</param>
/// <param name="cancellationToken">Cancels this caller's wait, never the shared flight.</param>
/// <returns>The memoized per-site notification KPI response.</returns>
Task<PerSiteNotificationKpiResponse> GetPerSiteNotificationKpisAsync(
bool forceRefresh = false, CancellationToken cancellationToken = default);
/// <summary>Per-node Notification Outbox KPI breakdown (grouped by <c>SourceNode</c>).</summary>
/// <param name="forceRefresh">Bypass a still-fresh value; an in-flight round is still joined.</param>
/// <param name="cancellationToken">Cancels this caller's wait, never the shared flight.</param>
/// <returns>The memoized per-node notification KPI response.</returns>
Task<PerNodeNotificationKpiResponse> GetPerNodeNotificationKpisAsync(
bool forceRefresh = false, CancellationToken cancellationToken = default);
/// <summary>Global Site Call Audit KPIs (buffered, stuck, parked).</summary>
/// <param name="forceRefresh">Bypass a still-fresh value; an in-flight round is still joined.</param>
/// <param name="cancellationToken">Cancels this caller's wait, never the shared flight.</param>
/// <returns>The memoized site call KPI response.</returns>
Task<SiteCallKpiResponse> GetSiteCallKpisAsync(
bool forceRefresh = false, CancellationToken cancellationToken = default);
/// <summary>Per-node Site Call Audit KPI breakdown (grouped by <c>SourceNode</c>).</summary>
/// <param name="forceRefresh">Bypass a still-fresh value; an in-flight round is still joined.</param>
/// <param name="cancellationToken">Cancels this caller's wait, never the shared flight.</param>
/// <returns>The memoized per-node site call KPI response.</returns>
Task<PerNodeSiteCallKpiResponse> GetPerNodeSiteCallKpisAsync(
bool forceRefresh = false, CancellationToken cancellationToken = default);
/// <summary>
/// Audit Log KPI snapshot (1h volume + error rate over the central <c>AuditLog</c> table,
/// plus the summed per-site backlog).
/// </summary>
/// <param name="forceRefresh">Bypass a still-fresh value; an in-flight round is still joined.</param>
/// <param name="cancellationToken">Cancels this caller's wait, never the shared flight.</param>
/// <returns>The memoized audit KPI snapshot.</returns>
Task<AuditLogKpiSnapshot> GetAuditKpiSnapshotAsync(
bool forceRefresh = false, CancellationToken cancellationToken = default);
}
@@ -0,0 +1,122 @@
using Microsoft.Extensions.DependencyInjection;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Audit;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Notification;
using ZB.MOM.WW.ScadaBridge.Commons.Types;
using ZB.MOM.WW.ScadaBridge.Communication;
namespace ZB.MOM.WW.ScadaBridge.CentralUI.Services;
/// <summary>
/// Default <see cref="IKpiSnapshotCache"/> — a DI <b>singleton</b> holding one
/// <see cref="SingleFlightMemo{T}"/> per KPI query.
/// <para>
/// Registered as a singleton on purpose: the whole point is that the memo outlives any one
/// circuit. It depends only on <see cref="CommunicationService"/> (itself a singleton) and
/// <see cref="IServiceScopeFactory"/> — the audit KPI resolves its scoped query service from
/// a fresh scope per flight, the same shape <c>AuditLogQueryService</c> uses internally so a
/// KPI round never shares a circuit-scoped <c>DbContext</c>.
/// </para>
/// </summary>
public sealed class KpiSnapshotCache : IKpiSnapshotCache
{
/// <summary>
/// Freshness window. Sized just under the Health dashboard's 10s tick so each tick still
/// yields ONE fresh query for the whole node (rather than one per circuit), while a
/// circuit whose tick lands mid-window is served the memoized value instead of duplicating it.
/// </summary>
internal static readonly TimeSpan Ttl = TimeSpan.FromSeconds(8);
private readonly CommunicationService _communication;
private readonly IServiceScopeFactory _scopeFactory;
private readonly SingleFlightMemo<NotificationKpiResponse> _notificationKpis;
private readonly SingleFlightMemo<PerSiteNotificationKpiResponse> _perSiteNotificationKpis;
private readonly SingleFlightMemo<PerNodeNotificationKpiResponse> _perNodeNotificationKpis;
private readonly SingleFlightMemo<SiteCallKpiResponse> _siteCallKpis;
private readonly SingleFlightMemo<PerNodeSiteCallKpiResponse> _perNodeSiteCallKpis;
private readonly SingleFlightMemo<AuditLogKpiSnapshot> _auditKpis;
/// <summary>
/// Initializes the shared KPI snapshot cache.
/// </summary>
/// <param name="communication">Central-side comms used to Ask the outbox / site-call singletons.</param>
/// <param name="scopeFactory">Opens a fresh DI scope per audit-KPI flight.</param>
public KpiSnapshotCache(CommunicationService communication, IServiceScopeFactory scopeFactory)
: this(communication, scopeFactory, Ttl, clock: null)
{
}
/// <summary>
/// Test seam: same cache with an explicit TTL and clock so freshness can be asserted
/// without real delays.
/// </summary>
/// <param name="communication">Central-side comms used to Ask the outbox / site-call singletons.</param>
/// <param name="scopeFactory">Opens a fresh DI scope per audit-KPI flight.</param>
/// <param name="ttl">Freshness window.</param>
/// <param name="clock">Clock used for freshness.</param>
internal KpiSnapshotCache(
CommunicationService communication,
IServiceScopeFactory scopeFactory,
TimeSpan ttl,
Func<DateTimeOffset>? clock)
{
_communication = communication ?? throw new ArgumentNullException(nameof(communication));
_scopeFactory = scopeFactory ?? throw new ArgumentNullException(nameof(scopeFactory));
_notificationKpis = new SingleFlightMemo<NotificationKpiResponse>(ttl, clock);
_perSiteNotificationKpis = new SingleFlightMemo<PerSiteNotificationKpiResponse>(ttl, clock);
_perNodeNotificationKpis = new SingleFlightMemo<PerNodeNotificationKpiResponse>(ttl, clock);
_siteCallKpis = new SingleFlightMemo<SiteCallKpiResponse>(ttl, clock);
_perNodeSiteCallKpis = new SingleFlightMemo<PerNodeSiteCallKpiResponse>(ttl, clock);
_auditKpis = new SingleFlightMemo<AuditLogKpiSnapshot>(ttl, clock);
}
/// <inheritdoc/>
public Task<NotificationKpiResponse> GetNotificationKpisAsync(
bool forceRefresh = false, CancellationToken cancellationToken = default) =>
_notificationKpis.GetAsync(
() => _communication.GetNotificationKpisAsync(new NotificationKpiRequest(NewCorrelationId())),
forceRefresh, cancellationToken);
/// <inheritdoc/>
public Task<PerSiteNotificationKpiResponse> GetPerSiteNotificationKpisAsync(
bool forceRefresh = false, CancellationToken cancellationToken = default) =>
_perSiteNotificationKpis.GetAsync(
() => _communication.GetPerSiteNotificationKpisAsync(new PerSiteNotificationKpiRequest(NewCorrelationId())),
forceRefresh, cancellationToken);
/// <inheritdoc/>
public Task<PerNodeNotificationKpiResponse> GetPerNodeNotificationKpisAsync(
bool forceRefresh = false, CancellationToken cancellationToken = default) =>
_perNodeNotificationKpis.GetAsync(
() => _communication.GetPerNodeNotificationKpisAsync(new PerNodeNotificationKpiRequest(NewCorrelationId())),
forceRefresh, cancellationToken);
/// <inheritdoc/>
public Task<SiteCallKpiResponse> GetSiteCallKpisAsync(
bool forceRefresh = false, CancellationToken cancellationToken = default) =>
_siteCallKpis.GetAsync(
() => _communication.GetSiteCallKpisAsync(new SiteCallKpiRequest(NewCorrelationId())),
forceRefresh, cancellationToken);
/// <inheritdoc/>
public Task<PerNodeSiteCallKpiResponse> GetPerNodeSiteCallKpisAsync(
bool forceRefresh = false, CancellationToken cancellationToken = default) =>
_perNodeSiteCallKpis.GetAsync(
() => _communication.GetPerNodeSiteCallKpisAsync(new PerNodeSiteCallKpiRequest(NewCorrelationId())),
forceRefresh, cancellationToken);
/// <inheritdoc/>
public Task<AuditLogKpiSnapshot> GetAuditKpiSnapshotAsync(
bool forceRefresh = false, CancellationToken cancellationToken = default) =>
_auditKpis.GetAsync(LoadAuditKpisAsync, forceRefresh, cancellationToken);
private async Task<AuditLogKpiSnapshot> LoadAuditKpisAsync()
{
await using var scope = _scopeFactory.CreateAsyncScope();
var queryService = scope.ServiceProvider.GetRequiredService<IAuditLogQueryService>();
return await queryService.GetKpiSnapshotAsync();
}
private static string NewCorrelationId() => Guid.NewGuid().ToString("N");
}
@@ -0,0 +1,115 @@
using System.Collections.Concurrent;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Options;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Streaming;
using ZB.MOM.WW.ScadaBridge.Communication;
namespace ZB.MOM.WW.ScadaBridge.CentralUI.Services;
/// <summary>
/// Process-level memoizing façade over <see cref="AlarmSummaryService"/> (arch-review WP2.4).
/// <para>
/// The Alarm Summary page polls every 15s <em>per circuit</em>, and each poll fans one debug
/// snapshot Ask out to every Enabled instance on the site. Ten operators watching one site
/// meant ten independent fan-outs per 15s across the same instances — on top of the shared
/// live-alarm aggregator already seeding and reconciling from the identical fan-out. The
/// question is not user-specific, so this service collapses it: one memo slot per site,
/// single-flight, so the node costs ONE fan-out per site per window no matter how many
/// viewers are watching.
/// </para>
/// <para>
/// <b>The freshness window follows the live cache.</b> While
/// <see cref="ISiteAlarmLiveCache.IsLive"/> is true the page deliberately ignores the poll's
/// alarm rows (the live deltas own them — arch-review R2 N5), so the only thing the poll
/// still supplies is the not-reporting list, and the window widens to the aggregator's own
/// reconcile interval. While the cache is cold the poll is the page's full-rebuild safety
/// net, so the window stays just under the page's 15s tick and every tick gets fresh data.
/// </para>
/// </summary>
public sealed class SharedAlarmSummaryService : IAlarmSummaryService
{
/// <summary>
/// Freshness window while the live alarm cache is NOT serving this site: the poll is the
/// page's authoritative rebuild path, so it must stay under the 15s page tick.
/// </summary>
internal static readonly TimeSpan ColdCacheTtl = TimeSpan.FromSeconds(12);
private readonly IServiceScopeFactory _scopeFactory;
private readonly ISiteAlarmLiveCache _liveCache;
private readonly TimeSpan _liveCacheTtl;
private readonly Func<DateTimeOffset>? _clock;
private readonly ConcurrentDictionary<int, SingleFlightMemo<AlarmSummaryResult>> _bySite = new();
/// <summary>
/// Initializes the shared alarm summary façade.
/// </summary>
/// <param name="scopeFactory">Opens a fresh DI scope per fan-out (fresh repositories, off any circuit scope).</param>
/// <param name="liveCache">The shared live alarm cache, consulted only for its per-site liveness.</param>
/// <param name="options">Communication options; supplies the aggregator reconcile interval.</param>
public SharedAlarmSummaryService(
IServiceScopeFactory scopeFactory,
ISiteAlarmLiveCache liveCache,
IOptions<CommunicationOptions> options)
: this(scopeFactory, liveCache, (options ?? throw new ArgumentNullException(nameof(options)))
.Value.LiveAlarmCacheReconcileInterval, clock: null)
{
}
/// <summary>
/// Test seam: same façade with an explicit live-cache window and clock.
/// </summary>
/// <param name="scopeFactory">Opens a fresh DI scope per fan-out.</param>
/// <param name="liveCache">The shared live alarm cache.</param>
/// <param name="liveCacheTtl">Freshness window used while the live cache is serving the site.</param>
/// <param name="clock">Clock used for freshness.</param>
internal SharedAlarmSummaryService(
IServiceScopeFactory scopeFactory,
ISiteAlarmLiveCache liveCache,
TimeSpan liveCacheTtl,
Func<DateTimeOffset>? clock)
{
_scopeFactory = scopeFactory ?? throw new ArgumentNullException(nameof(scopeFactory));
_liveCache = liveCache ?? throw new ArgumentNullException(nameof(liveCache));
// Never shorter than the cold window — a misconfigured reconcile interval must not
// silently make the memo useless.
_liveCacheTtl = liveCacheTtl > ColdCacheTtl ? liveCacheTtl : ColdCacheTtl;
_clock = clock;
}
/// <inheritdoc/>
public Task<AlarmSummaryResult> GetSiteAlarmsAsync(
int siteId, CancellationToken cancellationToken = default)
{
var memo = _bySite.GetOrAdd(
siteId,
_ => new SingleFlightMemo<AlarmSummaryResult>(ColdCacheTtl, _clock));
var ttl = _liveCache.IsLive(siteId) ? _liveCacheTtl : ColdCacheTtl;
return memo.GetAsync(
() => FanOutAsync(siteId),
forceRefresh: false,
cancellationToken,
ttlOverride: ttl);
}
/// <inheritdoc/>
public AlarmSummaryResult BuildFromLiveAlarms(IReadOnlyList<AlarmStateChanged> alarms) =>
AlarmSummaryService.BuildFromLiveAlarmsCore(alarms);
/// <inheritdoc/>
public AlarmRollup ComputeRollup(IReadOnlyList<AlarmSummaryRow> rows) =>
AlarmSummaryService.ComputeRollupCore(rows);
/// <summary>
/// One shared fan-out. Deliberately runs with no caller cancellation token: the flight is
/// shared, so one circuit navigating away must not cancel the round the others are awaiting.
/// </summary>
private async Task<AlarmSummaryResult> FanOutAsync(int siteId)
{
await using var scope = _scopeFactory.CreateAsyncScope();
var inner = scope.ServiceProvider.GetRequiredService<AlarmSummaryService>();
return await inner.GetSiteAlarmsAsync(siteId, CancellationToken.None);
}
}
@@ -0,0 +1,136 @@
namespace ZB.MOM.WW.ScadaBridge.CentralUI.Services;
/// <summary>
/// A process-level memo slot: one value, a time-to-live, and single-flight production.
/// <para>
/// Blazor Server runs one circuit per connected browser, so a page that polls on a timer
/// runs its query <em>per circuit</em>. Ten operators on the Health dashboard meant ten
/// identical KPI round trips every ten seconds. This slot collapses them: the first caller
/// in a TTL window starts the flight, every concurrent caller awaits the <em>same</em>
/// <see cref="Task{TResult}"/>, and callers arriving after it completes are served the
/// memoized value until the TTL expires.
/// </para>
/// <para>
/// Failures are never cached — a faulted flight is discarded so the next caller retries.
/// The caller's own exception handling is therefore unchanged: it still sees the fault
/// (all callers of that one flight see it), and the next window starts clean.
/// </para>
/// </summary>
/// <typeparam name="T">The memoized value type.</typeparam>
internal sealed class SingleFlightMemo<T>
{
private readonly object _lock = new();
private readonly TimeSpan _ttl;
private readonly Func<DateTimeOffset> _clock;
/// <summary>The current flight, completed or not. Null until the first call.</summary>
private Task<T>? _current;
/// <summary>Wall-clock instant after which <see cref="_current"/> is stale. Only meaningful when it completed successfully.</summary>
private DateTimeOffset _freshUntil;
/// <summary>
/// Initializes a memo slot.
/// </summary>
/// <param name="ttl">How long a successfully produced value stays fresh.</param>
/// <param name="clock">Clock used for freshness, injectable so tests need no real delay.</param>
public SingleFlightMemo(TimeSpan ttl, Func<DateTimeOffset>? clock = null)
{
_ttl = ttl;
_clock = clock ?? (() => DateTimeOffset.UtcNow);
}
/// <summary>
/// Number of times the factory has actually been invoked. Diagnostics and tests only.
/// </summary>
public int FlightCount { get; private set; }
/// <summary>
/// Returns the memoized value, starting a flight if none is fresh.
/// </summary>
/// <param name="factory">Produces a fresh value. Invoked outside the slot's lock.</param>
/// <param name="forceRefresh">
/// When <see langword="true"/>, a completed-but-still-fresh value is discarded and a new
/// flight starts. An <em>in-flight</em> round is still joined rather than duplicated, so a
/// burst of operator "Refresh" clicks remains one round trip.
/// </param>
/// <param name="cancellationToken">
/// Cancels this caller's wait only — never the shared flight, which other callers may still
/// be awaiting.
/// </param>
/// <param name="ttlOverride">
/// Freshness window for <em>this</em> flight, overriding the slot's default. Used where the
/// acceptable staleness depends on state the caller knows (the Alarm Summary poll needs a
/// far shorter window while the live alarm cache is cold than while it is serving deltas).
/// </param>
/// <returns>The memoized (or freshly produced) value.</returns>
public Task<T> GetAsync(
Func<Task<T>> factory,
bool forceRefresh = false,
CancellationToken cancellationToken = default,
TimeSpan? ttlOverride = null)
{
ArgumentNullException.ThrowIfNull(factory);
TaskCompletionSource<T>? started = null;
Task<T> result;
lock (_lock)
{
var current = _current;
if (current is { IsCompleted: false })
{
// A round is already in progress — join it, force or not.
result = current;
}
else if (!forceRefresh
&& current is { IsCompletedSuccessfully: true }
&& _clock() < _freshUntil)
{
result = current;
}
else
{
// RunContinuationsAsynchronously: a caller's continuation must never run
// inline on the thread that completes the flight (that thread is another
// circuit's dispatcher).
started = new TaskCompletionSource<T>(TaskCreationOptions.RunContinuationsAsynchronously);
_current = started.Task;
FlightCount++;
result = started.Task;
}
}
// The factory runs OUTSIDE the lock: it does I/O (an Ask, a SQL round trip) and must
// never be able to re-enter this slot on the calling thread while the lock is held.
if (started is not null)
{
_ = RunAsync(factory, started, ttlOverride ?? _ttl);
}
return cancellationToken.CanBeCanceled ? result.WaitAsync(cancellationToken) : result;
}
private async Task RunAsync(Func<Task<T>> factory, TaskCompletionSource<T> completion, TimeSpan ttl)
{
try
{
var value = await factory().ConfigureAwait(false);
lock (_lock)
{
_freshUntil = _clock() + ttl;
}
completion.SetResult(value);
}
catch (Exception ex)
{
// A failure is not memoized: leave _freshUntil in the past so the next caller
// starts a new flight instead of re-serving the fault for the whole TTL.
lock (_lock)
{
_freshUntil = default;
}
completion.SetException(ex);
}
}
}