fix(dashboard): bound EventsHub delivery with one drop-oldest pump per session

Publish fired a group SendAsync per event and returned, so a slow remote
viewer accumulated unbounded pending sends plus cloned MxEvents. Hub delivery
now goes through a 256-deep DropOldest channel per hub-watched session with a
single drain task and time-bounded sends. The registry counts hub viewers
separately and raises HubViewersReleased so the pump is torn down with the
last remote viewer; in-process-only sessions start no pump.
This commit is contained in:
Joseph Doherty
2026-10-22 06:13:05 -04:00
parent 488649ce99
commit f1db002b87
5 changed files with 638 additions and 114 deletions
@@ -10,10 +10,11 @@ namespace ZB.MOM.WW.MxGateway.Server.Dashboard.Hubs;
/// Broadcasts MxEvents to the two dashboard audiences for a session: remote
/// <see cref="EventsHub"/> clients subscribed to the session's group, and
/// in-process subscribers opened through
/// <see cref="IDashboardSessionEventSubscriber.Subscribe"/>. Fire-and-forget: we
/// hand the send to the hub context and return immediately so the source gRPC
/// stream is never blocked. Errors are logged once and dropped — keeping the
/// SignalR mirror best-effort preserves the gRPC contract that exists today.
/// <see cref="IDashboardSessionEventSubscriber.Subscribe"/>. <see cref="Publish"/>
/// never blocks: both audiences are fed through bounded drop-oldest queues, so the
/// source gRPC stream is never held up by a slow dashboard viewer. Errors are
/// logged and dropped — keeping the SignalR mirror best-effort preserves the gRPC
/// contract.
/// </summary>
/// <remarks>
/// When <c>MxGateway:Dashboard:ShowTagValues</c> is false (the default), tag
@@ -26,31 +27,45 @@ namespace ZB.MOM.WW.MxGateway.Server.Dashboard.Hubs;
/// <see cref="EventsHub"/>), while the redaction decides what a permitted
/// subscriber sees — so the value-leak seam stays closed whatever the ACL
/// admits.
/// <para>
/// Hub delivery goes through one <see cref="HubPump"/> per watched session: a
/// <see cref="HubQueueCapacity"/>-deep drop-oldest channel drained by a single task
/// that sends one event at a time, each send time-bounded. An earlier version fired
/// a group send per event and returned, so a slow remote viewer accumulated an
/// unbounded number of pending sends — each holding a cloned event — inside the
/// SignalR lifetime manager. The pump is started by the first event published while
/// a hub connection watches the session and torn down when
/// <see cref="EventsHubViewerRegistry.HubViewersReleased"/> reports the last one gone.
/// </para>
/// </remarks>
/// <param name="hubContext">Hub context used to send to the session's group.</param>
/// <param name="viewerRegistry">
/// Live-subscriber registry consulted before any per-event work is done. Both
/// audiences register here — hub connections by their SignalR connection id,
/// in-process subscriptions by a synthetic one — so the gate stays a single
/// source of truth.
/// </param>
/// <param name="options">Gateway options supplying <c>Dashboard:ShowTagValues</c>.</param>
/// <param name="logger">Logger for best-effort mirror failures.</param>
public sealed class DashboardEventBroadcaster(
IHubContext<EventsHub> hubContext,
EventsHubViewerRegistry viewerRegistry,
IOptions<GatewayOptions> options,
ILogger<DashboardEventBroadcaster> logger) : IDashboardEventBroadcaster, IDashboardSessionEventSubscriber
public sealed class DashboardEventBroadcaster : IDashboardEventBroadcaster, IDashboardSessionEventSubscriber, IDisposable
{
/// <summary>
/// Queue depth per in-process subscriber. The consumer is a Blazor page
/// rendering the newest handful of events, so a burst it cannot keep up with
/// is dropped oldest-first rather than allowed to grow — same best-effort
/// contract the SignalR mirror already has.
/// contract the SignalR mirror has.
/// </summary>
private const int InProcessQueueCapacity = 256;
private readonly bool _showTagValues = options.Value.Dashboard.ShowTagValues;
/// <summary>
/// Queue depth of a session's hub-delivery pump. Same depth and drop policy as
/// the in-process feed: the remote viewer is a live view, so a burst it cannot
/// keep up with loses its oldest events rather than buffering without bound.
/// </summary>
internal const int HubQueueCapacity = 256;
private static readonly TimeSpan DefaultHubSendTimeout = TimeSpan.FromSeconds(5);
private readonly IHubContext<EventsHub> _hubContext;
private readonly EventsHubViewerRegistry _viewerRegistry;
private readonly ILogger<DashboardEventBroadcaster> _logger;
private readonly TimeSpan _hubSendTimeout;
private readonly bool _showTagValues;
private readonly object _syncRoot = new();
@@ -63,6 +78,9 @@ public sealed class DashboardEventBroadcaster(
private readonly Dictionary<string, InProcessSubscription[]> _inProcessSubscribers =
new(StringComparer.Ordinal);
/// <summary>Live hub-delivery pumps per session. Guarded by <see cref="_syncRoot"/>.</summary>
private readonly Dictionary<string, HubPump> _hubPumps = new(StringComparer.Ordinal);
/// <summary>
/// Total live in-process subscribers, read without the lock so the common
/// case — nobody has a session-details page open — never contends on it.
@@ -70,6 +88,61 @@ public sealed class DashboardEventBroadcaster(
/// </summary>
private int _inProcessSubscriberCount;
private bool _disposed;
/// <summary>Initializes a new instance of the <see cref="DashboardEventBroadcaster"/> class.</summary>
/// <param name="hubContext">Hub context used to send to the session's group.</param>
/// <param name="viewerRegistry">
/// Live-subscriber registry consulted before any per-event work is done. Both
/// audiences register here — hub connections by their SignalR connection id,
/// in-process subscriptions by a synthetic one — so the gate stays a single
/// source of truth.
/// </param>
/// <param name="options">Gateway options supplying <c>Dashboard:ShowTagValues</c>.</param>
/// <param name="logger">Logger for best-effort mirror failures.</param>
public DashboardEventBroadcaster(
IHubContext<EventsHub> hubContext,
EventsHubViewerRegistry viewerRegistry,
IOptions<GatewayOptions> options,
ILogger<DashboardEventBroadcaster> logger)
: this(hubContext, viewerRegistry, options, logger, DefaultHubSendTimeout)
{
}
/// <summary>Initializes a new instance with a custom hub send timeout (test hook).</summary>
/// <param name="hubContext">Hub context used to send to the session's group.</param>
/// <param name="viewerRegistry">Live-subscriber registry.</param>
/// <param name="options">Gateway options supplying <c>Dashboard:ShowTagValues</c>.</param>
/// <param name="logger">Logger for best-effort mirror failures.</param>
/// <param name="hubSendTimeout">Upper bound on one group send before it is abandoned.</param>
internal DashboardEventBroadcaster(
IHubContext<EventsHub> hubContext,
EventsHubViewerRegistry viewerRegistry,
IOptions<GatewayOptions> options,
ILogger<DashboardEventBroadcaster> logger,
TimeSpan hubSendTimeout)
{
ArgumentNullException.ThrowIfNull(options);
_hubContext = hubContext ?? throw new ArgumentNullException(nameof(hubContext));
_viewerRegistry = viewerRegistry ?? throw new ArgumentNullException(nameof(viewerRegistry));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_hubSendTimeout = hubSendTimeout;
_showTagValues = options.Value.Dashboard.ShowTagValues;
_viewerRegistry.HubViewersReleased += OnHubViewersReleased;
}
/// <summary>Gets the number of live hub-delivery pumps (one per hub-watched session).</summary>
internal int ActiveHubPumpCount
{
get
{
lock (_syncRoot)
{
return _hubPumps.Count;
}
}
}
/// <inheritdoc />
public void Publish(string sessionId, MxEvent mxEvent)
{
@@ -81,46 +154,126 @@ public sealed class DashboardEventBroadcaster(
// Every session's dashboard-mirror subscriber calls Publish for every event,
// whether or not a browser is on that session's page. Without this gate the
// steady state — no dashboard viewer at all — still paid a deep protobuf
// clone (redaction is on by default) plus a send to an empty SignalR group
// per event. Bail before both.
if (!viewerRegistry.HasViewers(sessionId))
// clone (redaction is on by default) per event. Bail before it.
if (!_viewerRegistry.HasViewers(sessionId))
{
return;
}
MxEvent outbound = _showTagValues ? mxEvent : RedactValues(mxEvent);
// In-process delivery first: it is synchronous, cannot throw, and must not be
// skipped by the early return the hub send's guard clause takes.
DeliverInProcess(sessionId, outbound);
// Wrap the Task acquisition in a try/catch so a hypothetical synchronous throw
// from SendAsync (e.g. an implementation that throws before returning the Task)
// cannot escape Publish. The interface contract is never-throw; fire-and-forget.
Task send;
// In-process-only viewers need no hub send at all; the pump exists only
// while a remote connection is in the session's group.
if (_viewerRegistry.HasHubViewers(sessionId))
{
EnqueueForHub(sessionId, outbound);
}
}
/// <summary>
/// Stops every hub pump and detaches from the viewer registry. The singleton is
/// disposed with the host; a pump's in-flight send is bounded by its timeout.
/// </summary>
public void Dispose()
{
HubPump[] pumps;
lock (_syncRoot)
{
if (_disposed)
{
return;
}
_disposed = true;
pumps = [.. _hubPumps.Values];
_hubPumps.Clear();
}
_viewerRegistry.HubViewersReleased -= OnHubViewersReleased;
foreach (HubPump pump in pumps)
{
pump.Complete();
}
}
/// <summary>
/// Queues an event on the session's hub pump, starting the pump on first use.
/// The write happens under the lock so it cannot land on a pump that a
/// concurrent teardown has already completed.
/// </summary>
/// <param name="sessionId">Session the event belongs to.</param>
/// <param name="outbound">The event as the dashboard should see it.</param>
private void EnqueueForHub(string sessionId, MxEvent outbound)
{
lock (_syncRoot)
{
if (_disposed)
{
return;
}
if (!_hubPumps.TryGetValue(sessionId, out HubPump? pump))
{
pump = new HubPump(this, sessionId, HubQueueCapacity);
_hubPumps[sessionId] = pump;
}
pump.TryWrite(outbound);
}
}
/// <summary>
/// Tears down a session's hub pump once its last hub viewer has gone. Re-checks
/// the registry under the lock: a viewer that re-subscribed in between keeps the
/// pump, and one that arrives after the teardown gets a fresh pump on the next
/// event.
/// </summary>
/// <param name="sessionId">Session whose last hub viewer left.</param>
private void OnHubViewersReleased(string sessionId)
{
HubPump? pump;
lock (_syncRoot)
{
if (_viewerRegistry.HasHubViewers(sessionId) || !_hubPumps.Remove(sessionId, out pump))
{
return;
}
}
pump.Complete();
}
/// <summary>
/// Sends one event to the session's group, bounded by the hub send timeout.
/// Never throws: a failed or abandoned send is logged and the pump moves on.
/// </summary>
/// <param name="sessionId">Session whose group receives the event.</param>
/// <param name="outbound">The event to send.</param>
/// <returns>A task that completes when the send finished or was abandoned.</returns>
private async Task SendToHubAsync(string sessionId, MxEvent outbound)
{
try
{
send = hubContext.Clients
.Group(EventsHub.GroupName(sessionId))
.SendAsync(EventsHub.EventMessage, outbound);
bool sent = await BoundedHubSend.SendAsync(
_hubContext.Clients.Group(EventsHub.GroupName(sessionId)),
EventsHub.EventMessage,
outbound,
_hubSendTimeout,
CancellationToken.None)
.ConfigureAwait(false);
if (!sent)
{
_logger.LogDebug(
"Dashboard event mirror to session {SessionId} timed out after {Timeout}; continuing.",
sessionId,
_hubSendTimeout);
}
}
catch (Exception ex)
{
logger.LogDebug(ex, "Dashboard event mirror to session {SessionId} threw synchronously.", sessionId);
return;
}
if (!send.IsCompletedSuccessfully)
{
_ = send.ContinueWith(
t =>
{
if (t.Exception is { } ex)
{
logger.LogDebug(ex, "Dashboard event mirror to session {SessionId} failed.", sessionId);
}
},
TaskScheduler.Default);
_logger.LogDebug(ex, "Dashboard event mirror to session {SessionId} failed.", sessionId);
}
}
@@ -133,7 +286,7 @@ public sealed class DashboardEventBroadcaster(
// usable for a subscriber that has no SignalR connection behind it. The
// "inproc-" prefix cannot collide with a SignalR connection id and makes the
// origin obvious in a debugger.
string connectionId = "inproc-" + Guid.NewGuid().ToString("N");
string connectionId = EventsHubViewerRegistry.InProcessConnectionIdPrefix + Guid.NewGuid().ToString("N");
InProcessSubscription subscription = new(this, sessionId, connectionId, InProcessQueueCapacity);
// Register before the subscriber becomes a delivery target, exactly as
@@ -142,7 +295,7 @@ public sealed class DashboardEventBroadcaster(
// Publish's gate still reports the session unwatched, silently dropping events
// it should receive. The cost of this order is at worst a redaction clone that
// reaches nobody for the width of the window.
viewerRegistry.AddViewer(connectionId, sessionId);
_viewerRegistry.AddViewer(connectionId, sessionId);
lock (_syncRoot)
{
@@ -227,12 +380,12 @@ public sealed class DashboardEventBroadcaster(
}
}
viewerRegistry.RemoveViewer(subscription.ConnectionId, subscription.SessionId);
_viewerRegistry.RemoveViewer(subscription.ConnectionId, subscription.SessionId);
// The synthetic connection id is used once and never reconnects, so nothing
// else will ever call ReleaseConnection for it; without this the registry
// would retain an empty per-connection entry per subscription ever opened.
viewerRegistry.ReleaseConnection(subscription.ConnectionId);
_viewerRegistry.ReleaseConnection(subscription.ConnectionId);
}
/// <summary>
@@ -328,4 +481,56 @@ public sealed class DashboardEventBroadcaster(
_channel.Writer.TryComplete();
}
}
/// <summary>
/// One session's hub-delivery pump: a bounded drop-oldest channel and the single
/// task that drains it into the session's SignalR group, one bounded send at a
/// time. The drain is the only reader, so at most one send per session is ever
/// pending in the lifetime manager.
/// </summary>
private sealed class HubPump
{
private readonly Channel<MxEvent> _channel;
/// <summary>Initializes a new instance of the <see cref="HubPump"/> class and starts its drain.</summary>
/// <param name="owner">Broadcaster whose send path the drain uses.</param>
/// <param name="sessionId">Session whose group the pump feeds.</param>
/// <param name="capacity">Queue depth before the oldest queued event is dropped.</param>
internal HubPump(DashboardEventBroadcaster owner, string sessionId, int capacity)
{
_channel = Channel.CreateBounded<MxEvent>(new BoundedChannelOptions(capacity)
{
// DropOldest, not Wait: Publish runs on the gRPC event pipeline and
// must never block, and the newest events are the ones a live view wants.
FullMode = BoundedChannelFullMode.DropOldest,
SingleReader = true,
SingleWriter = false,
});
// Task.Run so the drain never starts inline on the publishing thread.
Completion = Task.Run(() => DrainAsync(owner, sessionId));
}
/// <summary>Gets the drain task; it completes once the channel is completed and emptied.</summary>
internal Task Completion { get; }
/// <summary>Queues an event, dropping the oldest queued one when full. Never blocks.</summary>
/// <param name="mxEvent">The event to queue.</param>
internal void TryWrite(MxEvent mxEvent) => _channel.Writer.TryWrite(mxEvent);
/// <summary>Completes the channel so the drain exits once it is empty.</summary>
internal void Complete() => _channel.Writer.TryComplete();
private async Task DrainAsync(DashboardEventBroadcaster owner, string sessionId)
{
ChannelReader<MxEvent> reader = _channel.Reader;
while (await reader.WaitToReadAsync().ConfigureAwait(false))
{
while (reader.TryRead(out MxEvent? next))
{
await owner.SendToHubAsync(sessionId, next).ConfigureAwait(false);
}
}
}
}
}