Files
mxaccessgw/src/ZB.MOM.WW.MxGateway.Server/Dashboard/Hubs/AlarmsHubPublisher.cs
T
Joseph Doherty 3688fa19e6 fix(dashboard): time-bound alarm hub sends and mark each resubscribe
AlarmsHubPublisher awaited every group send unbounded inside the StreamAsync
loop, so one slow client stalled the drain until the monitor dropped the
subscriber and the resubscribe replayed the full snapshot to everyone. Each
broadcast is now bounded (2s) and abandoned on timeout, and every
(re)subscription is preceded by an AlarmFeedReset frame so clients discard
state before the replayed snapshot.
2026-10-22 06:14:53 -04:00

221 lines
9.9 KiB
C#

using Microsoft.AspNetCore.SignalR;
using Microsoft.Extensions.Options;
using ZB.MOM.WW.MxGateway.Contracts.Proto;
using ZB.MOM.WW.MxGateway.Server.Alarms;
using ZB.MOM.WW.MxGateway.Server.Configuration;
namespace ZB.MOM.WW.MxGateway.Server.Dashboard.Hubs;
/// <summary>
/// Background service that subscribes to
/// <see cref="IGatewayAlarmService.StreamAsync"/> (no filter) and re-broadcasts
/// every <see cref="AlarmFeedMessage"/> to every <see cref="AlarmsHub"/>
/// client. The hub itself is session-less; clients filter / route messages
/// in the browser.
/// </summary>
/// <remarks>
/// When <c>MxGateway:Dashboard:ShowTagValues</c> is false (the default), the
/// alarm value fields are stripped from a redacted copy before the message
/// reaches any browser client — the same rule
/// <see cref="DashboardEventBroadcaster"/> applies to the events-hub mirror, so
/// the two SignalR seams cannot disagree about whether values leave the gateway.
/// The source message is never mutated: it fans out from one feed to the gRPC
/// <c>StreamAlarms</c> subscribers and the alarms page as well, and none of
/// those audiences is subject to this dashboard-display flag.
/// <para>
/// The broadcast stays deliberately unfiltered. Audience is decided once, at hub
/// admission — <see cref="AlarmsHub"/> carries
/// <see cref="DashboardAuthenticationDefaults.AlarmsPolicy"/>, so only an authenticated caller
/// with a real dashboard role is ever in <see cref="AlarmsHub.AllAlarmsGroup"/> — and the alarm
/// monitor is gateway-central with no per-session provenance, so there is no per-connection
/// narrowing this loop could apply even in principle. Adding a filter here would imply a
/// per-caller scope the feed does not carry.
/// </para>
/// <para>
/// Each broadcast is time-bounded (<see cref="DefaultSendTimeout"/>). A group send
/// completes only once every member accepted the frame, so awaiting it unbounded
/// let one slow browser stall this loop; the stalled loop stopped draining its
/// monitor subscription, the monitor dropped it as a slow subscriber, and the
/// resubscribe replayed the whole active-alarm snapshot to every client. A send that
/// misses the bound is abandoned and the loop moves on.
/// </para>
/// <para>
/// Every subscription — the first and each resubscribe — is preceded by an
/// <see cref="AlarmsHub.ResetMessage"/> frame. The monitor primes every new
/// subscription with a full snapshot, so a client that keeps accumulating
/// <c>active_alarm</c> rows across a resubscribe would double-count or keep alarms
/// that cleared while the feed was down; the marker tells it to discard its state
/// and rebuild from the snapshot that follows.
/// </para>
/// </remarks>
public sealed class AlarmsHubPublisher : BackgroundService
{
/// <summary>Upper bound on one alarm broadcast before it is abandoned.</summary>
internal static readonly TimeSpan DefaultSendTimeout = TimeSpan.FromSeconds(2);
private readonly IGatewayAlarmService _alarmService;
private readonly IHubContext<AlarmsHub> _hubContext;
private readonly ILogger<AlarmsHubPublisher> _logger;
private readonly TimeSpan _sendTimeout;
private readonly bool _showTagValues;
private long _subscriptionGeneration;
/// <summary>Initializes a new instance of the <see cref="AlarmsHubPublisher"/> class.</summary>
/// <param name="alarmService">The gateway's central alarm feed.</param>
/// <param name="hubContext">Hub context used to broadcast to the alarms group.</param>
/// <param name="options">Gateway options supplying <c>Dashboard:ShowTagValues</c>.</param>
/// <param name="logger">Logger for best-effort broadcast failures.</param>
public AlarmsHubPublisher(
IGatewayAlarmService alarmService,
IHubContext<AlarmsHub> hubContext,
IOptions<GatewayOptions> options,
ILogger<AlarmsHubPublisher> logger)
: this(alarmService, hubContext, options, logger, DefaultSendTimeout)
{
}
/// <summary>Initializes a new instance with a custom send timeout (test hook).</summary>
/// <param name="alarmService">The gateway's central alarm feed.</param>
/// <param name="hubContext">Hub context used to broadcast to the alarms group.</param>
/// <param name="options">Gateway options supplying <c>Dashboard:ShowTagValues</c>.</param>
/// <param name="logger">Logger for best-effort broadcast failures.</param>
/// <param name="sendTimeout">Upper bound on one broadcast.</param>
internal AlarmsHubPublisher(
IGatewayAlarmService alarmService,
IHubContext<AlarmsHub> hubContext,
IOptions<GatewayOptions> options,
ILogger<AlarmsHubPublisher> logger,
TimeSpan sendTimeout)
{
ArgumentNullException.ThrowIfNull(options);
_alarmService = alarmService ?? throw new ArgumentNullException(nameof(alarmService));
_hubContext = hubContext ?? throw new ArgumentNullException(nameof(hubContext));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_sendTimeout = sendTimeout;
_showTagValues = options.Value.Dashboard.ShowTagValues;
}
/// <inheritdoc />
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
// Loop forever — when StreamAsync completes (monitor restart, etc.)
// reopen the subscription. The hosted-service lifetime ends only
// when the host stops.
while (!stoppingToken.IsCancellationRequested)
{
try
{
// Sent before the subscription opens, so it reaches clients ahead of
// the snapshot the monitor primes the new subscription with.
long generation = Interlocked.Increment(ref _subscriptionGeneration);
await BroadcastAsync(
AlarmsHub.ResetMessage,
new AlarmFeedReset(generation),
stoppingToken)
.ConfigureAwait(false);
await foreach (AlarmFeedMessage message in _alarmService
.StreamAsync(alarmFilterPrefix: null, stoppingToken)
.ConfigureAwait(false))
{
if (stoppingToken.IsCancellationRequested)
{
break;
}
AlarmFeedMessage outbound = _showTagValues ? message : RedactValues(message);
await BroadcastAsync(AlarmsHub.AlarmMessage, outbound, stoppingToken).ConfigureAwait(false);
}
}
catch (OperationCanceledException)
{
return;
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Alarm subscription faulted; reconnecting in 5s.");
try
{
await Task.Delay(TimeSpan.FromSeconds(5), stoppingToken).ConfigureAwait(false);
}
catch (OperationCanceledException)
{
return;
}
}
}
}
/// <summary>
/// Sends one frame to the alarms group within the send timeout. A slow or failed
/// send is logged and dropped so the feed keeps draining; only host shutdown
/// propagates.
/// </summary>
/// <param name="method">Client method name.</param>
/// <param name="argument">The frame payload.</param>
/// <param name="stoppingToken">Host shutdown token.</param>
/// <returns>A task that completes when the send finished or was abandoned.</returns>
private async Task BroadcastAsync(string method, object argument, CancellationToken stoppingToken)
{
try
{
bool sent = await BoundedHubSend.SendAsync(
_hubContext.Clients.Group(AlarmsHub.AllAlarmsGroup),
method,
argument,
_sendTimeout,
stoppingToken)
.ConfigureAwait(false);
if (!sent)
{
_logger.LogDebug(
"Alarm broadcast {Method} exceeded {Timeout} and was abandoned; continuing.",
method,
_sendTimeout);
}
}
catch (Exception ex) when (ex is not OperationCanceledException)
{
_logger.LogWarning(ex, "Alarm broadcast failed; continuing.");
}
}
/// <summary>
/// Produces a copy of <paramref name="source"/> with the alarm value fields
/// cleared, leaving every other field — reference, severity, state, category,
/// operator, timestamps — intact so the alarms page still renders the row.
/// </summary>
/// <param name="source">The message as the alarm feed produced it.</param>
/// <returns>
/// A redacted deep clone for the two value-bearing payload arms; the source
/// instance itself for every other arm, which carries no value to strip.
/// New payload arms therefore pass through unchanged by default — the switch
/// names only the arms that have something to redact, so adding a valueless
/// arm to the contract needs no change here.
/// </returns>
private static AlarmFeedMessage RedactValues(AlarmFeedMessage source)
{
switch (source.PayloadCase)
{
case AlarmFeedMessage.PayloadOneofCase.Transition:
{
AlarmFeedMessage redacted = source.Clone();
redacted.Transition.CurrentValue = null;
redacted.Transition.LimitValue = null;
return redacted;
}
case AlarmFeedMessage.PayloadOneofCase.ActiveAlarm:
{
AlarmFeedMessage redacted = source.Clone();
redacted.ActiveAlarm.CurrentValue = null;
redacted.ActiveAlarm.LimitValue = null;
return redacted;
}
default:
return source;
}
}
}