fix(dashboard): bound EventsHub delivery with one drop-oldest pump per session

Publish fired a group SendAsync per event and returned, so a slow remote
viewer accumulated unbounded pending sends plus cloned MxEvents. Hub delivery
now goes through a 256-deep DropOldest channel per hub-watched session with a
single drain task and time-bounded sends. The registry counts hub viewers
separately and raises HubViewersReleased so the pump is torn down with the
last remote viewer; in-process-only sessions start no pump.
This commit is contained in:
Joseph Doherty
2026-10-22 06:13:05 -04:00
parent 488649ce99
commit f1db002b87
5 changed files with 638 additions and 114 deletions
@@ -0,0 +1,52 @@
using Microsoft.AspNetCore.SignalR;
namespace ZB.MOM.WW.MxGateway.Server.Dashboard.Hubs;
/// <summary>
/// Time-bounded SignalR send shared by the dashboard hub publishers. A group or
/// broadcast send completes only once every target connection has accepted the
/// frame, so one remote viewer with a full transport buffer would otherwise hold
/// the publisher's single drain loop for as long as that viewer stays slow.
/// </summary>
internal static class BoundedHubSend
{
/// <summary>
/// Sends <paramref name="argument"/> as <paramref name="method"/> and waits at most
/// <paramref name="timeout"/> for it. The timeout cancels the send's own token, so
/// the lifetime manager abandons the pending write to the slow connection, and the
/// wait itself is bounded too, so a proxy that ignores the token cannot stall the
/// caller either.
/// </summary>
/// <param name="proxy">Target client proxy (a group or all connections).</param>
/// <param name="method">Client method name.</param>
/// <param name="argument">The single message argument.</param>
/// <param name="timeout">Upper bound on the send.</param>
/// <param name="stoppingToken">Host shutdown token; its cancellation propagates.</param>
/// <returns>
/// <see langword="true"/> when the send completed in time; <see langword="false"/>
/// when it was abandoned at the timeout.
/// </returns>
/// <exception cref="OperationCanceledException"><paramref name="stoppingToken"/> was cancelled.</exception>
public static async Task<bool> SendAsync(
IClientProxy proxy,
string method,
object argument,
TimeSpan timeout,
CancellationToken stoppingToken)
{
using CancellationTokenSource bounded = CancellationTokenSource.CreateLinkedTokenSource(stoppingToken);
bounded.CancelAfter(timeout);
try
{
await proxy.SendAsync(method, argument, bounded.Token)
.WaitAsync(bounded.Token)
.ConfigureAwait(false);
return true;
}
catch (OperationCanceledException) when (!stoppingToken.IsCancellationRequested)
{
return false;
}
}
}
@@ -10,10 +10,11 @@ namespace ZB.MOM.WW.MxGateway.Server.Dashboard.Hubs;
/// Broadcasts MxEvents to the two dashboard audiences for a session: remote
/// <see cref="EventsHub"/> clients subscribed to the session's group, and
/// in-process subscribers opened through
/// <see cref="IDashboardSessionEventSubscriber.Subscribe"/>. Fire-and-forget: we
/// hand the send to the hub context and return immediately so the source gRPC
/// stream is never blocked. Errors are logged once and dropped — keeping the
/// SignalR mirror best-effort preserves the gRPC contract that exists today.
/// <see cref="IDashboardSessionEventSubscriber.Subscribe"/>. <see cref="Publish"/>
/// never blocks: both audiences are fed through bounded drop-oldest queues, so the
/// source gRPC stream is never held up by a slow dashboard viewer. Errors are
/// logged and dropped — keeping the SignalR mirror best-effort preserves the gRPC
/// contract.
/// </summary>
/// <remarks>
/// When <c>MxGateway:Dashboard:ShowTagValues</c> is false (the default), tag
@@ -26,31 +27,45 @@ namespace ZB.MOM.WW.MxGateway.Server.Dashboard.Hubs;
/// <see cref="EventsHub"/>), while the redaction decides what a permitted
/// subscriber sees — so the value-leak seam stays closed whatever the ACL
/// admits.
/// <para>
/// Hub delivery goes through one <see cref="HubPump"/> per watched session: a
/// <see cref="HubQueueCapacity"/>-deep drop-oldest channel drained by a single task
/// that sends one event at a time, each send time-bounded. An earlier version fired
/// a group send per event and returned, so a slow remote viewer accumulated an
/// unbounded number of pending sends — each holding a cloned event — inside the
/// SignalR lifetime manager. The pump is started by the first event published while
/// a hub connection watches the session and torn down when
/// <see cref="EventsHubViewerRegistry.HubViewersReleased"/> reports the last one gone.
/// </para>
/// </remarks>
/// <param name="hubContext">Hub context used to send to the session's group.</param>
/// <param name="viewerRegistry">
/// Live-subscriber registry consulted before any per-event work is done. Both
/// audiences register here — hub connections by their SignalR connection id,
/// in-process subscriptions by a synthetic one — so the gate stays a single
/// source of truth.
/// </param>
/// <param name="options">Gateway options supplying <c>Dashboard:ShowTagValues</c>.</param>
/// <param name="logger">Logger for best-effort mirror failures.</param>
public sealed class DashboardEventBroadcaster(
IHubContext<EventsHub> hubContext,
EventsHubViewerRegistry viewerRegistry,
IOptions<GatewayOptions> options,
ILogger<DashboardEventBroadcaster> logger) : IDashboardEventBroadcaster, IDashboardSessionEventSubscriber
public sealed class DashboardEventBroadcaster : IDashboardEventBroadcaster, IDashboardSessionEventSubscriber, IDisposable
{
/// <summary>
/// Queue depth per in-process subscriber. The consumer is a Blazor page
/// rendering the newest handful of events, so a burst it cannot keep up with
/// is dropped oldest-first rather than allowed to grow — same best-effort
/// contract the SignalR mirror already has.
/// contract the SignalR mirror has.
/// </summary>
private const int InProcessQueueCapacity = 256;
private readonly bool _showTagValues = options.Value.Dashboard.ShowTagValues;
/// <summary>
/// Queue depth of a session's hub-delivery pump. Same depth and drop policy as
/// the in-process feed: the remote viewer is a live view, so a burst it cannot
/// keep up with loses its oldest events rather than buffering without bound.
/// </summary>
internal const int HubQueueCapacity = 256;
private static readonly TimeSpan DefaultHubSendTimeout = TimeSpan.FromSeconds(5);
private readonly IHubContext<EventsHub> _hubContext;
private readonly EventsHubViewerRegistry _viewerRegistry;
private readonly ILogger<DashboardEventBroadcaster> _logger;
private readonly TimeSpan _hubSendTimeout;
private readonly bool _showTagValues;
private readonly object _syncRoot = new();
@@ -63,6 +78,9 @@ public sealed class DashboardEventBroadcaster(
private readonly Dictionary<string, InProcessSubscription[]> _inProcessSubscribers =
new(StringComparer.Ordinal);
/// <summary>Live hub-delivery pumps per session. Guarded by <see cref="_syncRoot"/>.</summary>
private readonly Dictionary<string, HubPump> _hubPumps = new(StringComparer.Ordinal);
/// <summary>
/// Total live in-process subscribers, read without the lock so the common
/// case — nobody has a session-details page open — never contends on it.
@@ -70,6 +88,61 @@ public sealed class DashboardEventBroadcaster(
/// </summary>
private int _inProcessSubscriberCount;
private bool _disposed;
/// <summary>Initializes a new instance of the <see cref="DashboardEventBroadcaster"/> class.</summary>
/// <param name="hubContext">Hub context used to send to the session's group.</param>
/// <param name="viewerRegistry">
/// Live-subscriber registry consulted before any per-event work is done. Both
/// audiences register here — hub connections by their SignalR connection id,
/// in-process subscriptions by a synthetic one — so the gate stays a single
/// source of truth.
/// </param>
/// <param name="options">Gateway options supplying <c>Dashboard:ShowTagValues</c>.</param>
/// <param name="logger">Logger for best-effort mirror failures.</param>
public DashboardEventBroadcaster(
IHubContext<EventsHub> hubContext,
EventsHubViewerRegistry viewerRegistry,
IOptions<GatewayOptions> options,
ILogger<DashboardEventBroadcaster> logger)
: this(hubContext, viewerRegistry, options, logger, DefaultHubSendTimeout)
{
}
/// <summary>Initializes a new instance with a custom hub send timeout (test hook).</summary>
/// <param name="hubContext">Hub context used to send to the session's group.</param>
/// <param name="viewerRegistry">Live-subscriber registry.</param>
/// <param name="options">Gateway options supplying <c>Dashboard:ShowTagValues</c>.</param>
/// <param name="logger">Logger for best-effort mirror failures.</param>
/// <param name="hubSendTimeout">Upper bound on one group send before it is abandoned.</param>
internal DashboardEventBroadcaster(
IHubContext<EventsHub> hubContext,
EventsHubViewerRegistry viewerRegistry,
IOptions<GatewayOptions> options,
ILogger<DashboardEventBroadcaster> logger,
TimeSpan hubSendTimeout)
{
ArgumentNullException.ThrowIfNull(options);
_hubContext = hubContext ?? throw new ArgumentNullException(nameof(hubContext));
_viewerRegistry = viewerRegistry ?? throw new ArgumentNullException(nameof(viewerRegistry));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_hubSendTimeout = hubSendTimeout;
_showTagValues = options.Value.Dashboard.ShowTagValues;
_viewerRegistry.HubViewersReleased += OnHubViewersReleased;
}
/// <summary>Gets the number of live hub-delivery pumps (one per hub-watched session).</summary>
internal int ActiveHubPumpCount
{
get
{
lock (_syncRoot)
{
return _hubPumps.Count;
}
}
}
/// <inheritdoc />
public void Publish(string sessionId, MxEvent mxEvent)
{
@@ -81,46 +154,126 @@ public sealed class DashboardEventBroadcaster(
// Every session's dashboard-mirror subscriber calls Publish for every event,
// whether or not a browser is on that session's page. Without this gate the
// steady state — no dashboard viewer at all — still paid a deep protobuf
// clone (redaction is on by default) plus a send to an empty SignalR group
// per event. Bail before both.
if (!viewerRegistry.HasViewers(sessionId))
// clone (redaction is on by default) per event. Bail before it.
if (!_viewerRegistry.HasViewers(sessionId))
{
return;
}
MxEvent outbound = _showTagValues ? mxEvent : RedactValues(mxEvent);
// In-process delivery first: it is synchronous, cannot throw, and must not be
// skipped by the early return the hub send's guard clause takes.
DeliverInProcess(sessionId, outbound);
// Wrap the Task acquisition in a try/catch so a hypothetical synchronous throw
// from SendAsync (e.g. an implementation that throws before returning the Task)
// cannot escape Publish. The interface contract is never-throw; fire-and-forget.
Task send;
// In-process-only viewers need no hub send at all; the pump exists only
// while a remote connection is in the session's group.
if (_viewerRegistry.HasHubViewers(sessionId))
{
EnqueueForHub(sessionId, outbound);
}
}
/// <summary>
/// Stops every hub pump and detaches from the viewer registry. The singleton is
/// disposed with the host; a pump's in-flight send is bounded by its timeout.
/// </summary>
public void Dispose()
{
HubPump[] pumps;
lock (_syncRoot)
{
if (_disposed)
{
return;
}
_disposed = true;
pumps = [.. _hubPumps.Values];
_hubPumps.Clear();
}
_viewerRegistry.HubViewersReleased -= OnHubViewersReleased;
foreach (HubPump pump in pumps)
{
pump.Complete();
}
}
/// <summary>
/// Queues an event on the session's hub pump, starting the pump on first use.
/// The write happens under the lock so it cannot land on a pump that a
/// concurrent teardown has already completed.
/// </summary>
/// <param name="sessionId">Session the event belongs to.</param>
/// <param name="outbound">The event as the dashboard should see it.</param>
private void EnqueueForHub(string sessionId, MxEvent outbound)
{
lock (_syncRoot)
{
if (_disposed)
{
return;
}
if (!_hubPumps.TryGetValue(sessionId, out HubPump? pump))
{
pump = new HubPump(this, sessionId, HubQueueCapacity);
_hubPumps[sessionId] = pump;
}
pump.TryWrite(outbound);
}
}
/// <summary>
/// Tears down a session's hub pump once its last hub viewer has gone. Re-checks
/// the registry under the lock: a viewer that re-subscribed in between keeps the
/// pump, and one that arrives after the teardown gets a fresh pump on the next
/// event.
/// </summary>
/// <param name="sessionId">Session whose last hub viewer left.</param>
private void OnHubViewersReleased(string sessionId)
{
HubPump? pump;
lock (_syncRoot)
{
if (_viewerRegistry.HasHubViewers(sessionId) || !_hubPumps.Remove(sessionId, out pump))
{
return;
}
}
pump.Complete();
}
/// <summary>
/// Sends one event to the session's group, bounded by the hub send timeout.
/// Never throws: a failed or abandoned send is logged and the pump moves on.
/// </summary>
/// <param name="sessionId">Session whose group receives the event.</param>
/// <param name="outbound">The event to send.</param>
/// <returns>A task that completes when the send finished or was abandoned.</returns>
private async Task SendToHubAsync(string sessionId, MxEvent outbound)
{
try
{
send = hubContext.Clients
.Group(EventsHub.GroupName(sessionId))
.SendAsync(EventsHub.EventMessage, outbound);
bool sent = await BoundedHubSend.SendAsync(
_hubContext.Clients.Group(EventsHub.GroupName(sessionId)),
EventsHub.EventMessage,
outbound,
_hubSendTimeout,
CancellationToken.None)
.ConfigureAwait(false);
if (!sent)
{
_logger.LogDebug(
"Dashboard event mirror to session {SessionId} timed out after {Timeout}; continuing.",
sessionId,
_hubSendTimeout);
}
}
catch (Exception ex)
{
logger.LogDebug(ex, "Dashboard event mirror to session {SessionId} threw synchronously.", sessionId);
return;
}
if (!send.IsCompletedSuccessfully)
{
_ = send.ContinueWith(
t =>
{
if (t.Exception is { } ex)
{
logger.LogDebug(ex, "Dashboard event mirror to session {SessionId} failed.", sessionId);
}
},
TaskScheduler.Default);
_logger.LogDebug(ex, "Dashboard event mirror to session {SessionId} failed.", sessionId);
}
}
@@ -133,7 +286,7 @@ public sealed class DashboardEventBroadcaster(
// usable for a subscriber that has no SignalR connection behind it. The
// "inproc-" prefix cannot collide with a SignalR connection id and makes the
// origin obvious in a debugger.
string connectionId = "inproc-" + Guid.NewGuid().ToString("N");
string connectionId = EventsHubViewerRegistry.InProcessConnectionIdPrefix + Guid.NewGuid().ToString("N");
InProcessSubscription subscription = new(this, sessionId, connectionId, InProcessQueueCapacity);
// Register before the subscriber becomes a delivery target, exactly as
@@ -142,7 +295,7 @@ public sealed class DashboardEventBroadcaster(
// Publish's gate still reports the session unwatched, silently dropping events
// it should receive. The cost of this order is at worst a redaction clone that
// reaches nobody for the width of the window.
viewerRegistry.AddViewer(connectionId, sessionId);
_viewerRegistry.AddViewer(connectionId, sessionId);
lock (_syncRoot)
{
@@ -227,12 +380,12 @@ public sealed class DashboardEventBroadcaster(
}
}
viewerRegistry.RemoveViewer(subscription.ConnectionId, subscription.SessionId);
_viewerRegistry.RemoveViewer(subscription.ConnectionId, subscription.SessionId);
// The synthetic connection id is used once and never reconnects, so nothing
// else will ever call ReleaseConnection for it; without this the registry
// would retain an empty per-connection entry per subscription ever opened.
viewerRegistry.ReleaseConnection(subscription.ConnectionId);
_viewerRegistry.ReleaseConnection(subscription.ConnectionId);
}
/// <summary>
@@ -328,4 +481,56 @@ public sealed class DashboardEventBroadcaster(
_channel.Writer.TryComplete();
}
}
/// <summary>
/// One session's hub-delivery pump: a bounded drop-oldest channel and the single
/// task that drains it into the session's SignalR group, one bounded send at a
/// time. The drain is the only reader, so at most one send per session is ever
/// pending in the lifetime manager.
/// </summary>
private sealed class HubPump
{
private readonly Channel<MxEvent> _channel;
/// <summary>Initializes a new instance of the <see cref="HubPump"/> class and starts its drain.</summary>
/// <param name="owner">Broadcaster whose send path the drain uses.</param>
/// <param name="sessionId">Session whose group the pump feeds.</param>
/// <param name="capacity">Queue depth before the oldest queued event is dropped.</param>
internal HubPump(DashboardEventBroadcaster owner, string sessionId, int capacity)
{
_channel = Channel.CreateBounded<MxEvent>(new BoundedChannelOptions(capacity)
{
// DropOldest, not Wait: Publish runs on the gRPC event pipeline and
// must never block, and the newest events are the ones a live view wants.
FullMode = BoundedChannelFullMode.DropOldest,
SingleReader = true,
SingleWriter = false,
});
// Task.Run so the drain never starts inline on the publishing thread.
Completion = Task.Run(() => DrainAsync(owner, sessionId));
}
/// <summary>Gets the drain task; it completes once the channel is completed and emptied.</summary>
internal Task Completion { get; }
/// <summary>Queues an event, dropping the oldest queued one when full. Never blocks.</summary>
/// <param name="mxEvent">The event to queue.</param>
internal void TryWrite(MxEvent mxEvent) => _channel.Writer.TryWrite(mxEvent);
/// <summary>Completes the channel so the drain exits once it is empty.</summary>
internal void Complete() => _channel.Writer.TryComplete();
private async Task DrainAsync(DashboardEventBroadcaster owner, string sessionId)
{
ChannelReader<MxEvent> reader = _channel.Reader;
while (await reader.WaitToReadAsync().ConfigureAwait(false))
{
while (reader.TryRead(out MxEvent? next))
{
await owner.SendToHubAsync(sessionId, next).ConfigureAwait(false);
}
}
}
}
}
@@ -19,11 +19,27 @@ namespace ZB.MOM.WW.MxGateway.Server.Dashboard.Hubs;
/// simply goes away never calls <c>UnsubscribeSession</c>; the hub's
/// <c>OnDisconnectedAsync</c> releases everything the connection held.
/// </para>
/// <para>
/// Hub viewers are also counted on their own, separately from the in-process
/// subscribers that register under an <see cref="InProcessConnectionIdPrefix"/>
/// id: <see cref="DashboardEventBroadcaster"/> runs one bounded hub-delivery pump
/// per session only while a remote connection is watching it, and tears that pump
/// down on <see cref="HubViewersReleased"/>.
/// </para>
/// </remarks>
public sealed class EventsHubViewerRegistry
{
/// <summary>
/// Prefix of the synthetic connection ids in-process subscribers register under.
/// A SignalR connection id never carries it, so it is what tells the two
/// audiences apart for the hub-only count.
/// </summary>
public const string InProcessConnectionIdPrefix = "inproc-";
private readonly ConcurrentDictionary<string, int> _viewersBySession = new(StringComparer.Ordinal);
private readonly ConcurrentDictionary<string, int> _hubViewersBySession = new(StringComparer.Ordinal);
private readonly ConcurrentDictionary<string, ConcurrentDictionary<string, byte>> _sessionsByConnection =
new(StringComparer.Ordinal);
@@ -53,8 +69,19 @@ public sealed class EventsHubViewerRegistry
}
_viewersBySession.AddOrUpdate(sessionId, 1, static (_, count) => count + 1);
if (IsHubConnection(connectionId))
{
_hubViewersBySession.AddOrUpdate(sessionId, 1, static (_, count) => count + 1);
}
}
/// <summary>
/// Raised with a session id when its last remote hub viewer leaves (unsubscribe
/// or disconnect). In-process subscribers do not raise it. Handlers run on the
/// releasing thread and must not throw or block.
/// </summary>
public event Action<string>? HubViewersReleased;
/// <summary>
/// Records that <paramref name="connectionId"/> stopped watching
/// <paramref name="sessionId"/>. A removal with no matching
@@ -75,7 +102,7 @@ public sealed class EventsHubViewerRegistry
return;
}
ReleaseSession(sessionId);
Release(connectionId, sessionId);
}
/// <summary>
@@ -107,7 +134,7 @@ public sealed class EventsHubViewerRegistry
// same detached set must not let the session be decremented twice.
if (sessions.TryRemove(sessionId, out _))
{
ReleaseSession(sessionId);
Release(connectionId, sessionId);
}
}
}
@@ -120,32 +147,63 @@ public sealed class EventsHubViewerRegistry
&& _viewersBySession.TryGetValue(sessionId, out int count)
&& count > 0;
/// <summary>
/// Gets a value indicating whether any remote <see cref="EventsHub"/> connection —
/// as opposed to an in-process subscriber — is watching the session.
/// </summary>
/// <param name="sessionId">Session id to test.</param>
/// <returns><see langword="true"/> when at least one hub connection is subscribed.</returns>
public bool HasHubViewers(string sessionId) =>
!string.IsNullOrEmpty(sessionId)
&& _hubViewersBySession.TryGetValue(sessionId, out int count)
&& count > 0;
private static bool IsHubConnection(string connectionId) =>
!connectionId.StartsWith(InProcessConnectionIdPrefix, StringComparison.Ordinal);
/// <summary>
/// Releases one subscription from both counts it was recorded in, and raises
/// <see cref="HubViewersReleased"/> when it was the session's last hub viewer.
/// </summary>
/// <param name="connectionId">Connection that held the subscription.</param>
/// <param name="sessionId">Session id whose counts are released.</param>
private void Release(string connectionId, string sessionId)
{
ReleaseSession(_viewersBySession, sessionId);
if (IsHubConnection(connectionId) && ReleaseSession(_hubViewersBySession, sessionId))
{
HubViewersReleased?.Invoke(sessionId);
}
}
/// <summary>
/// Decrements the session's viewer count, dropping the entry entirely at
/// zero so the dictionary does not grow one key per session ever viewed.
/// The compare-and-swap loop keeps the decrement correct against a
/// concurrent <see cref="AddViewer"/> on the same session.
/// </summary>
/// <param name="counts">The count map to decrement.</param>
/// <param name="sessionId">Session id whose count is released.</param>
private void ReleaseSession(string sessionId)
/// <returns><see langword="true"/> when this release dropped the count to zero.</returns>
private static bool ReleaseSession(ConcurrentDictionary<string, int> counts, string sessionId)
{
while (true)
{
if (!_viewersBySession.TryGetValue(sessionId, out int count))
if (!counts.TryGetValue(sessionId, out int count))
{
return;
return false;
}
if (count <= 1)
{
if (_viewersBySession.TryRemove(new KeyValuePair<string, int>(sessionId, count)))
if (counts.TryRemove(new KeyValuePair<string, int>(sessionId, count)))
{
return;
return true;
}
}
else if (_viewersBySession.TryUpdate(sessionId, count - 1, count))
else if (counts.TryUpdate(sessionId, count - 1, count))
{
return;
return false;
}
}
}