fix(dashboard): bound EventsHub delivery with one drop-oldest pump per session
Publish fired a group SendAsync per event and returned, so a slow remote viewer accumulated unbounded pending sends plus cloned MxEvents. Hub delivery now goes through a 256-deep DropOldest channel per hub-watched session with a single drain task and time-bounded sends. The registry counts hub viewers separately and raises HubViewersReleased so the pump is torn down with the last remote viewer; in-process-only sessions start no pump.
This commit is contained in:
@@ -10,10 +10,11 @@ namespace ZB.MOM.WW.MxGateway.Server.Dashboard.Hubs;
|
||||
/// Broadcasts MxEvents to the two dashboard audiences for a session: remote
|
||||
/// <see cref="EventsHub"/> clients subscribed to the session's group, and
|
||||
/// in-process subscribers opened through
|
||||
/// <see cref="IDashboardSessionEventSubscriber.Subscribe"/>. Fire-and-forget: we
|
||||
/// hand the send to the hub context and return immediately so the source gRPC
|
||||
/// stream is never blocked. Errors are logged once and dropped — keeping the
|
||||
/// SignalR mirror best-effort preserves the gRPC contract that exists today.
|
||||
/// <see cref="IDashboardSessionEventSubscriber.Subscribe"/>. <see cref="Publish"/>
|
||||
/// never blocks: both audiences are fed through bounded drop-oldest queues, so the
|
||||
/// source gRPC stream is never held up by a slow dashboard viewer. Errors are
|
||||
/// logged and dropped — keeping the SignalR mirror best-effort preserves the gRPC
|
||||
/// contract.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// When <c>MxGateway:Dashboard:ShowTagValues</c> is false (the default), tag
|
||||
@@ -26,31 +27,45 @@ namespace ZB.MOM.WW.MxGateway.Server.Dashboard.Hubs;
|
||||
/// <see cref="EventsHub"/>), while the redaction decides what a permitted
|
||||
/// subscriber sees — so the value-leak seam stays closed whatever the ACL
|
||||
/// admits.
|
||||
/// <para>
|
||||
/// Hub delivery goes through one <see cref="HubPump"/> per watched session: a
|
||||
/// <see cref="HubQueueCapacity"/>-deep drop-oldest channel drained by a single task
|
||||
/// that sends one event at a time, each send time-bounded. An earlier version fired
|
||||
/// a group send per event and returned, so a slow remote viewer accumulated an
|
||||
/// unbounded number of pending sends — each holding a cloned event — inside the
|
||||
/// SignalR lifetime manager. The pump is started by the first event published while
|
||||
/// a hub connection watches the session and torn down when
|
||||
/// <see cref="EventsHubViewerRegistry.HubViewersReleased"/> reports the last one gone.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
/// <param name="hubContext">Hub context used to send to the session's group.</param>
|
||||
/// <param name="viewerRegistry">
|
||||
/// Live-subscriber registry consulted before any per-event work is done. Both
|
||||
/// audiences register here — hub connections by their SignalR connection id,
|
||||
/// in-process subscriptions by a synthetic one — so the gate stays a single
|
||||
/// source of truth.
|
||||
/// </param>
|
||||
/// <param name="options">Gateway options supplying <c>Dashboard:ShowTagValues</c>.</param>
|
||||
/// <param name="logger">Logger for best-effort mirror failures.</param>
|
||||
public sealed class DashboardEventBroadcaster(
|
||||
IHubContext<EventsHub> hubContext,
|
||||
EventsHubViewerRegistry viewerRegistry,
|
||||
IOptions<GatewayOptions> options,
|
||||
ILogger<DashboardEventBroadcaster> logger) : IDashboardEventBroadcaster, IDashboardSessionEventSubscriber
|
||||
public sealed class DashboardEventBroadcaster : IDashboardEventBroadcaster, IDashboardSessionEventSubscriber, IDisposable
|
||||
{
|
||||
/// <summary>
|
||||
/// Queue depth per in-process subscriber. The consumer is a Blazor page
|
||||
/// rendering the newest handful of events, so a burst it cannot keep up with
|
||||
/// is dropped oldest-first rather than allowed to grow — same best-effort
|
||||
/// contract the SignalR mirror already has.
|
||||
/// contract the SignalR mirror has.
|
||||
/// </summary>
|
||||
private const int InProcessQueueCapacity = 256;
|
||||
|
||||
private readonly bool _showTagValues = options.Value.Dashboard.ShowTagValues;
|
||||
/// <summary>
|
||||
/// Queue depth of a session's hub-delivery pump. Same depth and drop policy as
|
||||
/// the in-process feed: the remote viewer is a live view, so a burst it cannot
|
||||
/// keep up with loses its oldest events rather than buffering without bound.
|
||||
/// </summary>
|
||||
internal const int HubQueueCapacity = 256;
|
||||
|
||||
private static readonly TimeSpan DefaultHubSendTimeout = TimeSpan.FromSeconds(5);
|
||||
|
||||
private readonly IHubContext<EventsHub> _hubContext;
|
||||
|
||||
private readonly EventsHubViewerRegistry _viewerRegistry;
|
||||
|
||||
private readonly ILogger<DashboardEventBroadcaster> _logger;
|
||||
|
||||
private readonly TimeSpan _hubSendTimeout;
|
||||
|
||||
private readonly bool _showTagValues;
|
||||
|
||||
private readonly object _syncRoot = new();
|
||||
|
||||
@@ -63,6 +78,9 @@ public sealed class DashboardEventBroadcaster(
|
||||
private readonly Dictionary<string, InProcessSubscription[]> _inProcessSubscribers =
|
||||
new(StringComparer.Ordinal);
|
||||
|
||||
/// <summary>Live hub-delivery pumps per session. Guarded by <see cref="_syncRoot"/>.</summary>
|
||||
private readonly Dictionary<string, HubPump> _hubPumps = new(StringComparer.Ordinal);
|
||||
|
||||
/// <summary>
|
||||
/// Total live in-process subscribers, read without the lock so the common
|
||||
/// case — nobody has a session-details page open — never contends on it.
|
||||
@@ -70,6 +88,61 @@ public sealed class DashboardEventBroadcaster(
|
||||
/// </summary>
|
||||
private int _inProcessSubscriberCount;
|
||||
|
||||
private bool _disposed;
|
||||
|
||||
/// <summary>Initializes a new instance of the <see cref="DashboardEventBroadcaster"/> class.</summary>
|
||||
/// <param name="hubContext">Hub context used to send to the session's group.</param>
|
||||
/// <param name="viewerRegistry">
|
||||
/// Live-subscriber registry consulted before any per-event work is done. Both
|
||||
/// audiences register here — hub connections by their SignalR connection id,
|
||||
/// in-process subscriptions by a synthetic one — so the gate stays a single
|
||||
/// source of truth.
|
||||
/// </param>
|
||||
/// <param name="options">Gateway options supplying <c>Dashboard:ShowTagValues</c>.</param>
|
||||
/// <param name="logger">Logger for best-effort mirror failures.</param>
|
||||
public DashboardEventBroadcaster(
|
||||
IHubContext<EventsHub> hubContext,
|
||||
EventsHubViewerRegistry viewerRegistry,
|
||||
IOptions<GatewayOptions> options,
|
||||
ILogger<DashboardEventBroadcaster> logger)
|
||||
: this(hubContext, viewerRegistry, options, logger, DefaultHubSendTimeout)
|
||||
{
|
||||
}
|
||||
|
||||
/// <summary>Initializes a new instance with a custom hub send timeout (test hook).</summary>
|
||||
/// <param name="hubContext">Hub context used to send to the session's group.</param>
|
||||
/// <param name="viewerRegistry">Live-subscriber registry.</param>
|
||||
/// <param name="options">Gateway options supplying <c>Dashboard:ShowTagValues</c>.</param>
|
||||
/// <param name="logger">Logger for best-effort mirror failures.</param>
|
||||
/// <param name="hubSendTimeout">Upper bound on one group send before it is abandoned.</param>
|
||||
internal DashboardEventBroadcaster(
|
||||
IHubContext<EventsHub> hubContext,
|
||||
EventsHubViewerRegistry viewerRegistry,
|
||||
IOptions<GatewayOptions> options,
|
||||
ILogger<DashboardEventBroadcaster> logger,
|
||||
TimeSpan hubSendTimeout)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(options);
|
||||
_hubContext = hubContext ?? throw new ArgumentNullException(nameof(hubContext));
|
||||
_viewerRegistry = viewerRegistry ?? throw new ArgumentNullException(nameof(viewerRegistry));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
_hubSendTimeout = hubSendTimeout;
|
||||
_showTagValues = options.Value.Dashboard.ShowTagValues;
|
||||
_viewerRegistry.HubViewersReleased += OnHubViewersReleased;
|
||||
}
|
||||
|
||||
/// <summary>Gets the number of live hub-delivery pumps (one per hub-watched session).</summary>
|
||||
internal int ActiveHubPumpCount
|
||||
{
|
||||
get
|
||||
{
|
||||
lock (_syncRoot)
|
||||
{
|
||||
return _hubPumps.Count;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public void Publish(string sessionId, MxEvent mxEvent)
|
||||
{
|
||||
@@ -81,46 +154,126 @@ public sealed class DashboardEventBroadcaster(
|
||||
// Every session's dashboard-mirror subscriber calls Publish for every event,
|
||||
// whether or not a browser is on that session's page. Without this gate the
|
||||
// steady state — no dashboard viewer at all — still paid a deep protobuf
|
||||
// clone (redaction is on by default) plus a send to an empty SignalR group
|
||||
// per event. Bail before both.
|
||||
if (!viewerRegistry.HasViewers(sessionId))
|
||||
// clone (redaction is on by default) per event. Bail before it.
|
||||
if (!_viewerRegistry.HasViewers(sessionId))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
MxEvent outbound = _showTagValues ? mxEvent : RedactValues(mxEvent);
|
||||
|
||||
// In-process delivery first: it is synchronous, cannot throw, and must not be
|
||||
// skipped by the early return the hub send's guard clause takes.
|
||||
DeliverInProcess(sessionId, outbound);
|
||||
|
||||
// Wrap the Task acquisition in a try/catch so a hypothetical synchronous throw
|
||||
// from SendAsync (e.g. an implementation that throws before returning the Task)
|
||||
// cannot escape Publish. The interface contract is never-throw; fire-and-forget.
|
||||
Task send;
|
||||
// In-process-only viewers need no hub send at all; the pump exists only
|
||||
// while a remote connection is in the session's group.
|
||||
if (_viewerRegistry.HasHubViewers(sessionId))
|
||||
{
|
||||
EnqueueForHub(sessionId, outbound);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Stops every hub pump and detaches from the viewer registry. The singleton is
|
||||
/// disposed with the host; a pump's in-flight send is bounded by its timeout.
|
||||
/// </summary>
|
||||
public void Dispose()
|
||||
{
|
||||
HubPump[] pumps;
|
||||
lock (_syncRoot)
|
||||
{
|
||||
if (_disposed)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
_disposed = true;
|
||||
pumps = [.. _hubPumps.Values];
|
||||
_hubPumps.Clear();
|
||||
}
|
||||
|
||||
_viewerRegistry.HubViewersReleased -= OnHubViewersReleased;
|
||||
foreach (HubPump pump in pumps)
|
||||
{
|
||||
pump.Complete();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Queues an event on the session's hub pump, starting the pump on first use.
|
||||
/// The write happens under the lock so it cannot land on a pump that a
|
||||
/// concurrent teardown has already completed.
|
||||
/// </summary>
|
||||
/// <param name="sessionId">Session the event belongs to.</param>
|
||||
/// <param name="outbound">The event as the dashboard should see it.</param>
|
||||
private void EnqueueForHub(string sessionId, MxEvent outbound)
|
||||
{
|
||||
lock (_syncRoot)
|
||||
{
|
||||
if (_disposed)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if (!_hubPumps.TryGetValue(sessionId, out HubPump? pump))
|
||||
{
|
||||
pump = new HubPump(this, sessionId, HubQueueCapacity);
|
||||
_hubPumps[sessionId] = pump;
|
||||
}
|
||||
|
||||
pump.TryWrite(outbound);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Tears down a session's hub pump once its last hub viewer has gone. Re-checks
|
||||
/// the registry under the lock: a viewer that re-subscribed in between keeps the
|
||||
/// pump, and one that arrives after the teardown gets a fresh pump on the next
|
||||
/// event.
|
||||
/// </summary>
|
||||
/// <param name="sessionId">Session whose last hub viewer left.</param>
|
||||
private void OnHubViewersReleased(string sessionId)
|
||||
{
|
||||
HubPump? pump;
|
||||
lock (_syncRoot)
|
||||
{
|
||||
if (_viewerRegistry.HasHubViewers(sessionId) || !_hubPumps.Remove(sessionId, out pump))
|
||||
{
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
pump.Complete();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Sends one event to the session's group, bounded by the hub send timeout.
|
||||
/// Never throws: a failed or abandoned send is logged and the pump moves on.
|
||||
/// </summary>
|
||||
/// <param name="sessionId">Session whose group receives the event.</param>
|
||||
/// <param name="outbound">The event to send.</param>
|
||||
/// <returns>A task that completes when the send finished or was abandoned.</returns>
|
||||
private async Task SendToHubAsync(string sessionId, MxEvent outbound)
|
||||
{
|
||||
try
|
||||
{
|
||||
send = hubContext.Clients
|
||||
.Group(EventsHub.GroupName(sessionId))
|
||||
.SendAsync(EventsHub.EventMessage, outbound);
|
||||
bool sent = await BoundedHubSend.SendAsync(
|
||||
_hubContext.Clients.Group(EventsHub.GroupName(sessionId)),
|
||||
EventsHub.EventMessage,
|
||||
outbound,
|
||||
_hubSendTimeout,
|
||||
CancellationToken.None)
|
||||
.ConfigureAwait(false);
|
||||
if (!sent)
|
||||
{
|
||||
_logger.LogDebug(
|
||||
"Dashboard event mirror to session {SessionId} timed out after {Timeout}; continuing.",
|
||||
sessionId,
|
||||
_hubSendTimeout);
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
logger.LogDebug(ex, "Dashboard event mirror to session {SessionId} threw synchronously.", sessionId);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!send.IsCompletedSuccessfully)
|
||||
{
|
||||
_ = send.ContinueWith(
|
||||
t =>
|
||||
{
|
||||
if (t.Exception is { } ex)
|
||||
{
|
||||
logger.LogDebug(ex, "Dashboard event mirror to session {SessionId} failed.", sessionId);
|
||||
}
|
||||
},
|
||||
TaskScheduler.Default);
|
||||
_logger.LogDebug(ex, "Dashboard event mirror to session {SessionId} failed.", sessionId);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -133,7 +286,7 @@ public sealed class DashboardEventBroadcaster(
|
||||
// usable for a subscriber that has no SignalR connection behind it. The
|
||||
// "inproc-" prefix cannot collide with a SignalR connection id and makes the
|
||||
// origin obvious in a debugger.
|
||||
string connectionId = "inproc-" + Guid.NewGuid().ToString("N");
|
||||
string connectionId = EventsHubViewerRegistry.InProcessConnectionIdPrefix + Guid.NewGuid().ToString("N");
|
||||
InProcessSubscription subscription = new(this, sessionId, connectionId, InProcessQueueCapacity);
|
||||
|
||||
// Register before the subscriber becomes a delivery target, exactly as
|
||||
@@ -142,7 +295,7 @@ public sealed class DashboardEventBroadcaster(
|
||||
// Publish's gate still reports the session unwatched, silently dropping events
|
||||
// it should receive. The cost of this order is at worst a redaction clone that
|
||||
// reaches nobody for the width of the window.
|
||||
viewerRegistry.AddViewer(connectionId, sessionId);
|
||||
_viewerRegistry.AddViewer(connectionId, sessionId);
|
||||
|
||||
lock (_syncRoot)
|
||||
{
|
||||
@@ -227,12 +380,12 @@ public sealed class DashboardEventBroadcaster(
|
||||
}
|
||||
}
|
||||
|
||||
viewerRegistry.RemoveViewer(subscription.ConnectionId, subscription.SessionId);
|
||||
_viewerRegistry.RemoveViewer(subscription.ConnectionId, subscription.SessionId);
|
||||
|
||||
// The synthetic connection id is used once and never reconnects, so nothing
|
||||
// else will ever call ReleaseConnection for it; without this the registry
|
||||
// would retain an empty per-connection entry per subscription ever opened.
|
||||
viewerRegistry.ReleaseConnection(subscription.ConnectionId);
|
||||
_viewerRegistry.ReleaseConnection(subscription.ConnectionId);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -328,4 +481,56 @@ public sealed class DashboardEventBroadcaster(
|
||||
_channel.Writer.TryComplete();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// One session's hub-delivery pump: a bounded drop-oldest channel and the single
|
||||
/// task that drains it into the session's SignalR group, one bounded send at a
|
||||
/// time. The drain is the only reader, so at most one send per session is ever
|
||||
/// pending in the lifetime manager.
|
||||
/// </summary>
|
||||
private sealed class HubPump
|
||||
{
|
||||
private readonly Channel<MxEvent> _channel;
|
||||
|
||||
/// <summary>Initializes a new instance of the <see cref="HubPump"/> class and starts its drain.</summary>
|
||||
/// <param name="owner">Broadcaster whose send path the drain uses.</param>
|
||||
/// <param name="sessionId">Session whose group the pump feeds.</param>
|
||||
/// <param name="capacity">Queue depth before the oldest queued event is dropped.</param>
|
||||
internal HubPump(DashboardEventBroadcaster owner, string sessionId, int capacity)
|
||||
{
|
||||
_channel = Channel.CreateBounded<MxEvent>(new BoundedChannelOptions(capacity)
|
||||
{
|
||||
// DropOldest, not Wait: Publish runs on the gRPC event pipeline and
|
||||
// must never block, and the newest events are the ones a live view wants.
|
||||
FullMode = BoundedChannelFullMode.DropOldest,
|
||||
SingleReader = true,
|
||||
SingleWriter = false,
|
||||
});
|
||||
|
||||
// Task.Run so the drain never starts inline on the publishing thread.
|
||||
Completion = Task.Run(() => DrainAsync(owner, sessionId));
|
||||
}
|
||||
|
||||
/// <summary>Gets the drain task; it completes once the channel is completed and emptied.</summary>
|
||||
internal Task Completion { get; }
|
||||
|
||||
/// <summary>Queues an event, dropping the oldest queued one when full. Never blocks.</summary>
|
||||
/// <param name="mxEvent">The event to queue.</param>
|
||||
internal void TryWrite(MxEvent mxEvent) => _channel.Writer.TryWrite(mxEvent);
|
||||
|
||||
/// <summary>Completes the channel so the drain exits once it is empty.</summary>
|
||||
internal void Complete() => _channel.Writer.TryComplete();
|
||||
|
||||
private async Task DrainAsync(DashboardEventBroadcaster owner, string sessionId)
|
||||
{
|
||||
ChannelReader<MxEvent> reader = _channel.Reader;
|
||||
while (await reader.WaitToReadAsync().ConfigureAwait(false))
|
||||
{
|
||||
while (reader.TryRead(out MxEvent? next))
|
||||
{
|
||||
await owner.SendToHubAsync(sessionId, next).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user