fix(dashboard): time-bound alarm hub sends and mark each resubscribe
AlarmsHubPublisher awaited every group send unbounded inside the StreamAsync loop, so one slow client stalled the drain until the monitor dropped the subscriber and the resubscribe replayed the full snapshot to everyone. Each broadcast is now bounded (2s) and abandoned on timeout, and every (re)subscription is preceded by an AlarmFeedReset frame so clients discard state before the replayed snapshot.
This commit is contained in:
@@ -15,7 +15,9 @@ namespace ZB.MOM.WW.MxGateway.Tests.Gateway.Dashboard;
|
||||
/// both value-bearing payload arms are stripped from the copy broadcast to
|
||||
/// browser clients when the flag is off, present when it is on, and the source
|
||||
/// <see cref="AlarmFeedMessage"/> — shared with the gRPC <c>StreamAlarms</c>
|
||||
/// subscribers and the alarms page — is never mutated.
|
||||
/// subscribers and the alarms page — is never mutated. Also verifies that a
|
||||
/// slow client cannot stall the feed and that every (re)subscription is announced
|
||||
/// with a reset marker ahead of its snapshot.
|
||||
/// </summary>
|
||||
public sealed class AlarmsHubPublisherTests
|
||||
{
|
||||
@@ -136,6 +138,81 @@ public sealed class AlarmsHubPublisherTests
|
||||
Assert.True(hubContext.Sent[2].SnapshotStatus.Truncated);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Every subscription — the first and each resubscribe after the monitor ends a
|
||||
/// stream — is preceded by a reset marker, so a client knows to discard its
|
||||
/// state before the replayed snapshot arrives instead of double-counting it.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task ExecuteAsync_OnResubscribe_SendsResetMarkerBeforeTheReplayedSnapshot()
|
||||
{
|
||||
AlarmFeedMessage first = BuildActiveAlarm();
|
||||
AlarmFeedMessage replayed = BuildActiveAlarm();
|
||||
ScriptedAlarmService alarmService = new([first], [replayed]);
|
||||
CapturingHubContext hubContext = new();
|
||||
AlarmsHubPublisher publisher = CreatePublisher(alarmService, hubContext, AlarmsHubPublisher.DefaultSendTimeout);
|
||||
|
||||
using CancellationTokenSource cts = new();
|
||||
await publisher.StartAsync(cts.Token).WaitAsync(TestTimeout);
|
||||
await WaitUntilAsync(() => hubContext.Sent.Count >= 2);
|
||||
await cts.CancelAsync();
|
||||
await publisher.StopAsync(CancellationToken.None);
|
||||
|
||||
IReadOnlyList<(string Method, object? Argument)> frames = hubContext.Frames;
|
||||
Assert.Equal(
|
||||
[AlarmsHub.ResetMessage, AlarmsHub.AlarmMessage, AlarmsHub.ResetMessage, AlarmsHub.AlarmMessage],
|
||||
frames.Select(frame => frame.Method).Take(4));
|
||||
Assert.Equal(1, Assert.IsType<AlarmFeedReset>(frames[0].Argument).Generation);
|
||||
Assert.Equal(2, Assert.IsType<AlarmFeedReset>(frames[2].Argument).Generation);
|
||||
Assert.Same(first, frames[1].Argument);
|
||||
Assert.Same(replayed, frames[3].Argument);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A broadcast that never completes — one client whose transport is wedged — is
|
||||
/// abandoned at the send timeout, so the loop keeps draining the monitor
|
||||
/// subscription instead of stalling until the monitor drops it.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task ExecuteAsync_WhenABroadcastNeverCompletes_AbandonsItAndKeepsDraining()
|
||||
{
|
||||
AlarmFeedMessage stuck = BuildTransition();
|
||||
AlarmFeedMessage next = BuildActiveAlarm();
|
||||
ScriptedAlarmService alarmService = new([stuck, next]);
|
||||
CapturingHubContext hubContext = new();
|
||||
hubContext.Proxy.StallFirstAlarm = true;
|
||||
AlarmsHubPublisher publisher = CreatePublisher(alarmService, hubContext, TimeSpan.FromMilliseconds(50));
|
||||
|
||||
using CancellationTokenSource cts = new();
|
||||
await publisher.StartAsync(cts.Token).WaitAsync(TestTimeout);
|
||||
await WaitUntilAsync(() => hubContext.Sent.Count >= 2);
|
||||
await cts.CancelAsync();
|
||||
await publisher.StopAsync(CancellationToken.None);
|
||||
|
||||
Assert.Equal("Tank02.Level.Lo", hubContext.Sent[1].ActiveAlarm.AlarmFullReference);
|
||||
}
|
||||
|
||||
private static AlarmsHubPublisher CreatePublisher(
|
||||
IGatewayAlarmService alarmService,
|
||||
CapturingHubContext hubContext,
|
||||
TimeSpan sendTimeout,
|
||||
bool showTagValues = true)
|
||||
{
|
||||
GatewayOptions gatewayOptions = new()
|
||||
{
|
||||
Dashboard = new DashboardOptions { ShowTagValues = showTagValues },
|
||||
};
|
||||
|
||||
return new AlarmsHubPublisher(
|
||||
alarmService,
|
||||
hubContext,
|
||||
Options.Create(gatewayOptions),
|
||||
NullLogger<AlarmsHubPublisher>.Instance,
|
||||
sendTimeout);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Starts the publisher over a scripted feed, waits until every scripted
|
||||
/// message has been broadcast, and stops it.
|
||||
@@ -149,16 +226,11 @@ public sealed class AlarmsHubPublisherTests
|
||||
{
|
||||
ScriptedAlarmService alarmService = new(messages);
|
||||
CapturingHubContext hubContext = new();
|
||||
GatewayOptions gatewayOptions = new()
|
||||
{
|
||||
Dashboard = new DashboardOptions { ShowTagValues = showTagValues },
|
||||
};
|
||||
|
||||
AlarmsHubPublisher publisher = new(
|
||||
AlarmsHubPublisher publisher = CreatePublisher(
|
||||
alarmService,
|
||||
hubContext,
|
||||
Options.Create(gatewayOptions),
|
||||
NullLogger<AlarmsHubPublisher>.Instance);
|
||||
AlarmsHubPublisher.DefaultSendTimeout,
|
||||
showTagValues);
|
||||
|
||||
using CancellationTokenSource cts = new();
|
||||
await publisher.StartAsync(cts.Token).WaitAsync(TestTimeout);
|
||||
@@ -221,12 +293,15 @@ public sealed class AlarmsHubPublisherTests
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Yields a scripted message list once and then stays open until cancelled,
|
||||
/// so the publisher's reconnect loop never re-subscribes mid-test.
|
||||
/// Yields one scripted batch per subscription. Every batch but the last ends its
|
||||
/// stream — the monitor completing a subscription — so the publisher resubscribes;
|
||||
/// the last batch stays open until cancelled, so the reconnect loop then settles.
|
||||
/// </summary>
|
||||
/// <param name="messages">The messages to yield, in order.</param>
|
||||
private sealed class ScriptedAlarmService(IReadOnlyList<AlarmFeedMessage> messages) : IGatewayAlarmService
|
||||
/// <param name="batches">The messages each successive subscription yields, in order.</param>
|
||||
private sealed class ScriptedAlarmService(params IReadOnlyList<AlarmFeedMessage>[] batches) : IGatewayAlarmService
|
||||
{
|
||||
private int _subscriptions;
|
||||
|
||||
/// <inheritdoc />
|
||||
public GatewayAlarmMonitorState State => GatewayAlarmMonitorState.Monitoring;
|
||||
|
||||
@@ -247,11 +322,24 @@ public sealed class AlarmsHubPublisherTests
|
||||
string? alarmFilterPrefix,
|
||||
[EnumeratorCancellation] CancellationToken cancellationToken)
|
||||
{
|
||||
int index = Interlocked.Increment(ref _subscriptions) - 1;
|
||||
IReadOnlyList<AlarmFeedMessage> messages = batches[Math.Min(index, batches.Length - 1)];
|
||||
if (index >= batches.Length)
|
||||
{
|
||||
// Past the script: stay open and quiet rather than replaying the last batch.
|
||||
messages = [];
|
||||
}
|
||||
|
||||
foreach (AlarmFeedMessage message in messages)
|
||||
{
|
||||
yield return message;
|
||||
}
|
||||
|
||||
if (index < batches.Length - 1)
|
||||
{
|
||||
yield break;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken).ConfigureAwait(false);
|
||||
@@ -277,8 +365,14 @@ public sealed class AlarmsHubPublisherTests
|
||||
/// <summary>Gets the group manager.</summary>
|
||||
public IGroupManager Groups { get; } = new NoopGroupManager();
|
||||
|
||||
/// <summary>Gets every message the publisher broadcast, in order.</summary>
|
||||
/// <summary>Gets the capturing proxy every client lookup resolves to.</summary>
|
||||
public CapturingClientProxy Proxy => _clients.GroupProxy;
|
||||
|
||||
/// <summary>Gets every alarm message the publisher broadcast, in order.</summary>
|
||||
public IReadOnlyList<AlarmFeedMessage> Sent => _clients.GroupProxy.Sent;
|
||||
|
||||
/// <summary>Gets every frame the publisher broadcast — alarm messages and reset markers — in order.</summary>
|
||||
public IReadOnlyList<(string Method, object? Argument)> Frames => _clients.GroupProxy.Frames;
|
||||
}
|
||||
|
||||
private sealed class CapturingHubClients : IHubClients
|
||||
@@ -334,6 +428,26 @@ public sealed class AlarmsHubPublisherTests
|
||||
private sealed class CapturingClientProxy : IClientProxy
|
||||
{
|
||||
private readonly List<AlarmFeedMessage> _sent = [];
|
||||
private readonly List<(string Method, object? Argument)> _frames = [];
|
||||
private bool _stalled;
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets a value indicating whether the first alarm send never completes
|
||||
/// and ignores its cancellation token — a wedged client transport.
|
||||
/// </summary>
|
||||
public bool StallFirstAlarm { get; set; }
|
||||
|
||||
/// <summary>Gets every frame sent through this proxy, in order.</summary>
|
||||
public IReadOnlyList<(string Method, object? Argument)> Frames
|
||||
{
|
||||
get
|
||||
{
|
||||
lock (_sent)
|
||||
{
|
||||
return [.. _frames];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Gets every alarm message sent through this proxy, in order.</summary>
|
||||
public IReadOnlyList<AlarmFeedMessage> Sent
|
||||
@@ -347,22 +461,29 @@ public sealed class AlarmsHubPublisherTests
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Records the broadcast message and completes synchronously.</summary>
|
||||
/// <summary>Records the broadcast frame and completes synchronously unless stalled.</summary>
|
||||
/// <param name="method">The SignalR method name.</param>
|
||||
/// <param name="args">The method arguments.</param>
|
||||
/// <param name="cancellationToken">Token to observe for cancellation.</param>
|
||||
/// <returns>A completed task.</returns>
|
||||
/// <returns>A completed task, or a never-completing one for the stalled send.</returns>
|
||||
public Task SendCoreAsync(string method, object?[] args, CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (args.Length > 0 && args[0] is AlarmFeedMessage message)
|
||||
bool stall = false;
|
||||
lock (_sent)
|
||||
{
|
||||
lock (_sent)
|
||||
_frames.Add((method, args.Length > 0 ? args[0] : null));
|
||||
if (args.Length > 0 && args[0] is AlarmFeedMessage message)
|
||||
{
|
||||
_sent.Add(message);
|
||||
if (StallFirstAlarm && !_stalled)
|
||||
{
|
||||
_stalled = true;
|
||||
stall = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return Task.CompletedTask;
|
||||
return stall ? new TaskCompletionSource().Task : Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user