test: fake time for dashboard snapshot publisher; signal-based waits (mxaccessgw#205)

DashboardSnapshotPublisher takes an optional TimeProvider (default
TimeProvider.System) driving the idle poll, the reconnect delay and, via
BoundedHubSend, the send timeout. Its tests now run on a fake clock with
exact assertions (reconnect gap == delay, 25 idle polls pull nothing, a
wedged send is abandoned at the fake send timeout).

DashboardSnapshotFeed has no time dependence, so no seam was added; its
tests lose the 100 ms sleep and poll loops in favour of CountSignal
signals. The Task.Delay(25) poll copies in the alarm-monitor tests and
several 5-10 ms poll loops now share TestSupport/Eventually.

Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
Joseph Doherty
2026-11-09 13:05:43 -05:00
parent 0c0c7b86f2
commit 82d29beafd
16 changed files with 262 additions and 275 deletions
@@ -22,6 +22,7 @@ internal static class BoundedHubSend
/// <param name="argument">The single message argument.</param> /// <param name="argument">The single message argument.</param>
/// <param name="timeout">Upper bound on the send.</param> /// <param name="timeout">Upper bound on the send.</param>
/// <param name="stoppingToken">Host shutdown token; its cancellation propagates.</param> /// <param name="stoppingToken">Host shutdown token; its cancellation propagates.</param>
/// <param name="timeProvider">Clock the timeout runs on; defaults to <see cref="TimeProvider.System"/>.</param>
/// <returns> /// <returns>
/// <see langword="true"/> when the send completed in time; <see langword="false"/> /// <see langword="true"/> when the send completed in time; <see langword="false"/>
/// when it was abandoned at the timeout. /// when it was abandoned at the timeout.
@@ -32,10 +33,11 @@ internal static class BoundedHubSend
string method, string method,
object argument, object argument,
TimeSpan timeout, TimeSpan timeout,
CancellationToken stoppingToken) CancellationToken stoppingToken,
TimeProvider? timeProvider = null)
{ {
using CancellationTokenSource bounded = CancellationTokenSource.CreateLinkedTokenSource(stoppingToken); using CancellationTokenSource timeoutSource = new(timeout, timeProvider ?? TimeProvider.System);
bounded.CancelAfter(timeout); using CancellationTokenSource bounded = CancellationTokenSource.CreateLinkedTokenSource(stoppingToken, timeoutSource.Token);
try try
{ {
@@ -48,18 +48,21 @@ public sealed class DashboardSnapshotPublisher : BackgroundService
private readonly TimeSpan _reconnectDelay; private readonly TimeSpan _reconnectDelay;
private readonly TimeSpan _idlePollInterval; private readonly TimeSpan _idlePollInterval;
private readonly TimeSpan _sendTimeout; private readonly TimeSpan _sendTimeout;
private readonly TimeProvider _timeProvider;
/// <summary>Initializes a new instance of the DashboardSnapshotPublisher class.</summary> /// <summary>Initializes a new instance of the DashboardSnapshotPublisher class.</summary>
/// <param name="snapshotService">The snapshot service to subscribe to.</param> /// <param name="snapshotService">The snapshot service to subscribe to.</param>
/// <param name="hubContext">The SignalR hub context for broadcasting.</param> /// <param name="hubContext">The SignalR hub context for broadcasting.</param>
/// <param name="connectionCounter">Live snapshot hub connection count used to idle-gate the tick.</param> /// <param name="connectionCounter">Live snapshot hub connection count used to idle-gate the tick.</param>
/// <param name="logger">The logger instance.</param> /// <param name="logger">The logger instance.</param>
/// <param name="timeProvider">Clock for the idle poll, reconnect delay and send timeout; defaults to <see cref="TimeProvider.System"/>.</param>
public DashboardSnapshotPublisher( public DashboardSnapshotPublisher(
IDashboardSnapshotService snapshotService, IDashboardSnapshotService snapshotService,
IHubContext<DashboardSnapshotHub> hubContext, IHubContext<DashboardSnapshotHub> hubContext,
DashboardSnapshotHubConnectionCounter connectionCounter, DashboardSnapshotHubConnectionCounter connectionCounter,
ILogger<DashboardSnapshotPublisher> logger) ILogger<DashboardSnapshotPublisher> logger,
: this(snapshotService, hubContext, connectionCounter, logger, DefaultReconnectDelay, DefaultIdlePollInterval) TimeProvider? timeProvider = null)
: this(snapshotService, hubContext, connectionCounter, logger, DefaultReconnectDelay, DefaultIdlePollInterval, null, timeProvider)
{ {
} }
@@ -75,6 +78,7 @@ public sealed class DashboardSnapshotPublisher : BackgroundService
/// <param name="reconnectDelay">The delay before reconnecting after a subscription failure.</param> /// <param name="reconnectDelay">The delay before reconnecting after a subscription failure.</param>
/// <param name="idlePollInterval">How often the idle publisher re-checks for a connected viewer.</param> /// <param name="idlePollInterval">How often the idle publisher re-checks for a connected viewer.</param>
/// <param name="sendTimeout">Upper bound on one broadcast; defaults to one second.</param> /// <param name="sendTimeout">Upper bound on one broadcast; defaults to one second.</param>
/// <param name="timeProvider">Clock for the idle poll, reconnect delay and send timeout; defaults to <see cref="TimeProvider.System"/>.</param>
internal DashboardSnapshotPublisher( internal DashboardSnapshotPublisher(
IDashboardSnapshotService snapshotService, IDashboardSnapshotService snapshotService,
IHubContext<DashboardSnapshotHub> hubContext, IHubContext<DashboardSnapshotHub> hubContext,
@@ -82,8 +86,10 @@ public sealed class DashboardSnapshotPublisher : BackgroundService
ILogger<DashboardSnapshotPublisher> logger, ILogger<DashboardSnapshotPublisher> logger,
TimeSpan reconnectDelay, TimeSpan reconnectDelay,
TimeSpan idlePollInterval, TimeSpan idlePollInterval,
TimeSpan? sendTimeout = null) TimeSpan? sendTimeout = null,
TimeProvider? timeProvider = null)
{ {
_timeProvider = timeProvider ?? TimeProvider.System;
_snapshotService = snapshotService; _snapshotService = snapshotService;
_hubContext = hubContext; _hubContext = hubContext;
_connectionCounter = connectionCounter; _connectionCounter = connectionCounter;
@@ -117,7 +123,7 @@ public sealed class DashboardSnapshotPublisher : BackgroundService
// Nobody is watching: leave the producer suspended and re-check // Nobody is watching: leave the producer suspended and re-check
// shortly. The first viewer to connect resumes the tick, and is // shortly. The first viewer to connect resumes the tick, and is
// seeded directly by the hub's OnConnectedAsync meanwhile. // seeded directly by the hub's OnConnectedAsync meanwhile.
await Task.Delay(_idlePollInterval, stoppingToken).ConfigureAwait(false); await Task.Delay(_idlePollInterval, _timeProvider, stoppingToken).ConfigureAwait(false);
continue; continue;
} }
@@ -135,7 +141,8 @@ public sealed class DashboardSnapshotPublisher : BackgroundService
DashboardSnapshotHub.SnapshotMessage, DashboardSnapshotHub.SnapshotMessage,
snapshot, snapshot,
_sendTimeout, _sendTimeout,
stoppingToken) stoppingToken,
_timeProvider)
.ConfigureAwait(false); .ConfigureAwait(false);
if (!sent) if (!sent)
{ {
@@ -159,7 +166,7 @@ public sealed class DashboardSnapshotPublisher : BackgroundService
_logger.LogWarning(ex, "Snapshot subscription faulted; reconnecting in {DelaySeconds:F1}s.", _reconnectDelay.TotalSeconds); _logger.LogWarning(ex, "Snapshot subscription faulted; reconnecting in {DelaySeconds:F1}s.", _reconnectDelay.TotalSeconds);
try try
{ {
await Task.Delay(_reconnectDelay, stoppingToken).ConfigureAwait(false); await Task.Delay(_reconnectDelay, _timeProvider, stoppingToken).ConfigureAwait(false);
} }
catch (OperationCanceledException) catch (OperationCanceledException)
{ {
@@ -357,28 +357,19 @@ public sealed class AlarmFailoverEndToEndTests
await monitor.StopAsync(CancellationToken.None); await monitor.StopAsync(CancellationToken.None);
} }
private static async Task<AlarmFeedMessage> WaitForAsync( private static Task<AlarmFeedMessage> WaitForAsync(
List<AlarmFeedMessage> received, List<AlarmFeedMessage> received,
Func<AlarmFeedMessage, bool> predicate, Func<AlarmFeedMessage, bool> predicate,
TimeSpan timeout) TimeSpan timeout)
{ => Eventually.WaitForAsync(
DateTime deadline = DateTime.UtcNow + timeout; () =>
while (DateTime.UtcNow < deadline)
{ {
lock (received) lock (received)
{ {
AlarmFeedMessage? match = received.FirstOrDefault(predicate); return received.FirstOrDefault(predicate);
if (match is not null)
{
return match;
}
}
await Task.Delay(25);
}
throw new TimeoutException("No matching AlarmFeedMessage was received in time.");
} }
},
timeout);
/// <summary><see cref="IAlarmWatchListResolver"/> that returns a fixed watch-list.</summary> /// <summary><see cref="IAlarmWatchListResolver"/> that returns a fixed watch-list.</summary>
private sealed class StubWatchListResolver(IReadOnlyList<AlarmSubtagTarget> targets) : IAlarmWatchListResolver private sealed class StubWatchListResolver(IReadOnlyList<AlarmSubtagTarget> targets) : IAlarmWatchListResolver
@@ -603,21 +603,8 @@ public sealed class AlarmTruncationSignalTests
alarms); alarms);
} }
private static async Task WaitUntilAsync(Func<bool> condition, TimeSpan timeout) private static Task WaitUntilAsync(Func<bool> condition, TimeSpan timeout)
{ => Eventually.WaitUntilAsync(condition, timeout);
DateTime deadline = DateTime.UtcNow + timeout;
while (DateTime.UtcNow < deadline)
{
if (condition())
{
return;
}
await Task.Delay(25);
}
throw new TimeoutException("Condition was not met in time.");
}
/// <summary><see cref="IAlarmWatchListResolver"/> that resolves an empty watch-list.</summary> /// <summary><see cref="IAlarmWatchListResolver"/> that resolves an empty watch-list.</summary>
private sealed class StubWatchListResolver : IAlarmWatchListResolver private sealed class StubWatchListResolver : IAlarmWatchListResolver
@@ -515,44 +515,22 @@ public sealed class GatewayAlarmMonitorAttachOrderTests
return mxEvent; return mxEvent;
} }
private static async Task<AlarmFeedMessage> WaitForAsync( private static Task<AlarmFeedMessage> WaitForAsync(
List<AlarmFeedMessage> received, List<AlarmFeedMessage> received,
Func<AlarmFeedMessage, bool> predicate, Func<AlarmFeedMessage, bool> predicate,
TimeSpan timeout) TimeSpan timeout)
{ => Eventually.WaitForAsync(
DateTime deadline = DateTime.UtcNow + timeout; () =>
while (DateTime.UtcNow < deadline)
{ {
lock (received) lock (received)
{ {
AlarmFeedMessage? match = received.FirstOrDefault(predicate); return received.FirstOrDefault(predicate);
if (match is not null)
{
return match;
}
} }
},
timeout);
await Task.Delay(25); private static Task WaitUntilAsync(Func<bool> condition, TimeSpan timeout)
} => Eventually.WaitUntilAsync(condition, timeout);
throw new TimeoutException("No matching AlarmFeedMessage was received in time.");
}
private static async Task WaitUntilAsync(Func<bool> condition, TimeSpan timeout)
{
DateTime deadline = DateTime.UtcNow + timeout;
while (DateTime.UtcNow < deadline)
{
if (condition())
{
return;
}
await Task.Delay(25);
}
throw new TimeoutException("Condition was not met in time.");
}
/// <summary><see cref="IAlarmWatchListResolver"/> that resolves an empty watch-list.</summary> /// <summary><see cref="IAlarmWatchListResolver"/> that resolves an empty watch-list.</summary>
private sealed class StubWatchListResolver : IAlarmWatchListResolver private sealed class StubWatchListResolver : IAlarmWatchListResolver
@@ -896,44 +896,22 @@ public sealed class GatewayAlarmMonitorProviderModeTests
NullLogger<GatewayAlarmMonitor>.Instance); NullLogger<GatewayAlarmMonitor>.Instance);
} }
private static async Task<AlarmFeedMessage> WaitForAsync( private static Task<AlarmFeedMessage> WaitForAsync(
List<AlarmFeedMessage> received, List<AlarmFeedMessage> received,
Func<AlarmFeedMessage, bool> predicate, Func<AlarmFeedMessage, bool> predicate,
TimeSpan timeout) TimeSpan timeout)
{ => Eventually.WaitForAsync(
DateTime deadline = DateTime.UtcNow + timeout; () =>
while (DateTime.UtcNow < deadline)
{ {
lock (received) lock (received)
{ {
AlarmFeedMessage? match = received.FirstOrDefault(predicate); return received.FirstOrDefault(predicate);
if (match is not null)
{
return match;
}
} }
},
timeout);
await Task.Delay(25); private static Task WaitUntilAsync(Func<bool> condition, TimeSpan timeout)
} => Eventually.WaitUntilAsync(condition, timeout);
throw new TimeoutException("No matching AlarmFeedMessage was received in time.");
}
private static async Task WaitUntilAsync(Func<bool> condition, TimeSpan timeout)
{
DateTime deadline = DateTime.UtcNow + timeout;
while (DateTime.UtcNow < deadline)
{
if (condition())
{
return;
}
await Task.Delay(25);
}
throw new TimeoutException("Condition was not met in time.");
}
/// <summary> /// <summary>
/// <see cref="IAlarmWatchListResolver"/> that hands out a scripted sequence of /// <see cref="IAlarmWatchListResolver"/> that hands out a scripted sequence of
@@ -6,6 +6,7 @@ using ZB.MOM.WW.MxGateway.Contracts.Proto;
using ZB.MOM.WW.MxGateway.Server.Alarms; using ZB.MOM.WW.MxGateway.Server.Alarms;
using ZB.MOM.WW.MxGateway.Server.Configuration; using ZB.MOM.WW.MxGateway.Server.Configuration;
using ZB.MOM.WW.MxGateway.Server.Dashboard.Hubs; using ZB.MOM.WW.MxGateway.Server.Dashboard.Hubs;
using ZB.MOM.WW.MxGateway.Tests.TestSupport;
namespace ZB.MOM.WW.MxGateway.Tests.Gateway.Dashboard; namespace ZB.MOM.WW.MxGateway.Tests.Gateway.Dashboard;
@@ -241,14 +242,8 @@ public sealed class AlarmsHubPublisherTests
return hubContext; return hubContext;
} }
private static async Task WaitUntilAsync(Func<bool> predicate) private static Task WaitUntilAsync(Func<bool> predicate)
{ => Eventually.WaitUntilAsync(predicate, TestTimeout);
using CancellationTokenSource cancellationTokenSource = new(TestTimeout);
while (!predicate())
{
await Task.Delay(TimeSpan.FromMilliseconds(5), cancellationTokenSource.Token);
}
}
/// <summary>Builds a value-bearing <c>transition</c> feed message.</summary> /// <summary>Builds a value-bearing <c>transition</c> feed message.</summary>
/// <returns>The message.</returns> /// <returns>The message.</returns>
@@ -1,6 +1,7 @@
using System.Runtime.CompilerServices; using System.Runtime.CompilerServices;
using System.Threading.Channels; using System.Threading.Channels;
using ZB.MOM.WW.MxGateway.Server.Dashboard; using ZB.MOM.WW.MxGateway.Server.Dashboard;
using ZB.MOM.WW.MxGateway.Tests.TestSupport;
namespace ZB.MOM.WW.MxGateway.Tests.Gateway.Dashboard; namespace ZB.MOM.WW.MxGateway.Tests.Gateway.Dashboard;
@@ -23,7 +24,7 @@ public sealed class DashboardSnapshotFeedTests
/// </summary> /// </summary>
/// <returns>A task that represents the asynchronous operation.</returns> /// <returns>A task that represents the asynchronous operation.</returns>
[Fact] [Fact]
public async Task WatchAsync_WithNoSubscribers_NeverEnumeratesUnderlyingWatch() public Task WatchAsync_WithNoSubscribers_NeverEnumeratesUnderlyingWatch()
{ {
FakeSnapshotService service = new(); FakeSnapshotService service = new();
DashboardSnapshotFeed feed = new(service); DashboardSnapshotFeed feed = new(service);
@@ -31,9 +32,9 @@ public sealed class DashboardSnapshotFeedTests
// Obtaining the enumerable without enumerating it must not subscribe // Obtaining the enumerable without enumerating it must not subscribe
// either: the pump starts on the first MoveNextAsync, not before. // either: the pump starts on the first MoveNextAsync, not before.
_ = feed.WatchAsync(CancellationToken.None); _ = feed.WatchAsync(CancellationToken.None);
await Task.Delay(TimeSpan.FromMilliseconds(100));
Assert.Equal(0, service.EnumerationCount); Assert.Equal(0, service.EnumerationCount);
return Task.CompletedTask;
} }
/// <summary> /// <summary>
@@ -53,7 +54,7 @@ public sealed class DashboardSnapshotFeedTests
IAsyncEnumerator<DashboardSnapshot> first = IAsyncEnumerator<DashboardSnapshot> first =
feed.WatchAsync(firstCancellation.Token).GetAsyncEnumerator(firstCancellation.Token); feed.WatchAsync(firstCancellation.Token).GetAsyncEnumerator(firstCancellation.Token);
Task<bool> firstMove = first.MoveNextAsync().AsTask(); Task<bool> firstMove = first.MoveNextAsync().AsTask();
await WaitUntilAsync(() => service.EnumerationCount >= 1); await service.Enumerations.WaitForAsync(1);
IAsyncEnumerator<DashboardSnapshot> second = IAsyncEnumerator<DashboardSnapshot> second =
feed.WatchAsync(secondCancellation.Token).GetAsyncEnumerator(secondCancellation.Token); feed.WatchAsync(secondCancellation.Token).GetAsyncEnumerator(secondCancellation.Token);
@@ -92,19 +93,19 @@ public sealed class DashboardSnapshotFeedTests
IAsyncEnumerator<DashboardSnapshot> first = IAsyncEnumerator<DashboardSnapshot> first =
feed.WatchAsync(firstCancellation.Token).GetAsyncEnumerator(firstCancellation.Token); feed.WatchAsync(firstCancellation.Token).GetAsyncEnumerator(firstCancellation.Token);
Task<bool> firstMove = first.MoveNextAsync().AsTask(); Task<bool> firstMove = first.MoveNextAsync().AsTask();
await WaitUntilAsync(() => service.EnumerationCount >= 1); await service.Enumerations.WaitForAsync(1);
await firstCancellation.CancelAsync(); await firstCancellation.CancelAsync();
await DrainAsync(first, firstMove); await DrainAsync(first, firstMove);
await WaitUntilAsync(() => service.CompletedEnumerationCount >= 1); await service.Completions.WaitForAsync(1);
Assert.True(service.LastEnumerationWasCancelled); Assert.True(service.LastEnumerationWasCancelled);
using CancellationTokenSource secondCancellation = new(); using CancellationTokenSource secondCancellation = new();
IAsyncEnumerator<DashboardSnapshot> second = IAsyncEnumerator<DashboardSnapshot> second =
feed.WatchAsync(secondCancellation.Token).GetAsyncEnumerator(secondCancellation.Token); feed.WatchAsync(secondCancellation.Token).GetAsyncEnumerator(secondCancellation.Token);
Task<bool> secondMove = second.MoveNextAsync().AsTask(); Task<bool> secondMove = second.MoveNextAsync().AsTask();
await WaitUntilAsync(() => service.EnumerationCount >= 2); await service.Enumerations.WaitForAsync(2);
Assert.Equal(2, service.EnumerationCount); Assert.Equal(2, service.EnumerationCount);
@@ -131,7 +132,7 @@ public sealed class DashboardSnapshotFeedTests
IAsyncEnumerator<DashboardSnapshot> fast = IAsyncEnumerator<DashboardSnapshot> fast =
feed.WatchAsync(fastCancellation.Token).GetAsyncEnumerator(fastCancellation.Token); feed.WatchAsync(fastCancellation.Token).GetAsyncEnumerator(fastCancellation.Token);
Task<bool> fastMove = fast.MoveNextAsync().AsTask(); Task<bool> fastMove = fast.MoveNextAsync().AsTask();
await WaitUntilAsync(() => service.EnumerationCount >= 1); await service.Enumerations.WaitForAsync(1);
// The slow subscriber registers but never advances until the very end. // The slow subscriber registers but never advances until the very end.
IAsyncEnumerator<DashboardSnapshot> slow = IAsyncEnumerator<DashboardSnapshot> slow =
@@ -179,7 +180,7 @@ public sealed class DashboardSnapshotFeedTests
IAsyncEnumerator<DashboardSnapshot> first = IAsyncEnumerator<DashboardSnapshot> first =
feed.WatchAsync(firstCancellation.Token).GetAsyncEnumerator(firstCancellation.Token); feed.WatchAsync(firstCancellation.Token).GetAsyncEnumerator(firstCancellation.Token);
Task<bool> firstMove = first.MoveNextAsync().AsTask(); Task<bool> firstMove = first.MoveNextAsync().AsTask();
await WaitUntilAsync(() => service.EnumerationCount >= 1); await service.Enumerations.WaitForAsync(1);
service.Fault(new InvalidOperationException("simulated snapshot source failure")); service.Fault(new InvalidOperationException("simulated snapshot source failure"));
@@ -192,7 +193,7 @@ public sealed class DashboardSnapshotFeedTests
IAsyncEnumerator<DashboardSnapshot> second = IAsyncEnumerator<DashboardSnapshot> second =
feed.WatchAsync(secondCancellation.Token).GetAsyncEnumerator(secondCancellation.Token); feed.WatchAsync(secondCancellation.Token).GetAsyncEnumerator(secondCancellation.Token);
Task<bool> secondMove = second.MoveNextAsync().AsTask(); Task<bool> secondMove = second.MoveNextAsync().AsTask();
await WaitUntilAsync(() => service.EnumerationCount >= 2); await service.Enumerations.WaitForAsync(2);
await PushUntilAsync(service, secondMove); await PushUntilAsync(service, secondMove);
Assert.True(await secondMove.WaitAsync(TestTimeout)); Assert.True(await secondMove.WaitAsync(TestTimeout));
@@ -219,7 +220,7 @@ public sealed class DashboardSnapshotFeedTests
IAsyncEnumerator<DashboardSnapshot> first = IAsyncEnumerator<DashboardSnapshot> first =
feed.WatchAsync(firstCancellation.Token).GetAsyncEnumerator(firstCancellation.Token); feed.WatchAsync(firstCancellation.Token).GetAsyncEnumerator(firstCancellation.Token);
Task<bool> firstMove = first.MoveNextAsync().AsTask(); Task<bool> firstMove = first.MoveNextAsync().AsTask();
await WaitUntilAsync(() => service.EnumerationCount >= 1); await service.Enumerations.WaitForAsync(1);
service.Fault(new InvalidOperationException("simulated snapshot source failure")); service.Fault(new InvalidOperationException("simulated snapshot source failure"));
@@ -245,7 +246,7 @@ public sealed class DashboardSnapshotFeedTests
// ...and the one that joined mid-unwind is served by a restarted enumeration // ...and the one that joined mid-unwind is served by a restarted enumeration
// instead of inheriting the failure. // instead of inheriting the failure.
await WaitUntilAsync(() => service.EnumerationCount >= 2); await service.Enumerations.WaitForAsync(2);
await PushUntilAsync(service, secondMove); await PushUntilAsync(service, secondMove);
Assert.True(await secondMove.WaitAsync(TestTimeout)); Assert.True(await secondMove.WaitAsync(TestTimeout));
Assert.StartsWith("push-", second.Current.GatewayVersion, StringComparison.Ordinal); Assert.StartsWith("push-", second.Current.GatewayVersion, StringComparison.Ordinal);
@@ -273,7 +274,7 @@ public sealed class DashboardSnapshotFeedTests
IAsyncEnumerator<DashboardSnapshot> oldSubscriber = IAsyncEnumerator<DashboardSnapshot> oldSubscriber =
feed.WatchAsync(oldCancellation.Token).GetAsyncEnumerator(oldCancellation.Token); feed.WatchAsync(oldCancellation.Token).GetAsyncEnumerator(oldCancellation.Token);
Task<bool> oldMove = oldSubscriber.MoveNextAsync().AsTask(); Task<bool> oldMove = oldSubscriber.MoveNextAsync().AsTask();
await WaitUntilAsync(() => service.EnumerationCount >= 1); await service.Enumerations.WaitForAsync(1);
service.Fault(new InvalidOperationException("simulated snapshot source failure")); service.Fault(new InvalidOperationException("simulated snapshot source failure"));
await service.DisposalReached.WaitAsync(TestTimeout); await service.DisposalReached.WaitAsync(TestTimeout);
@@ -287,8 +288,9 @@ public sealed class DashboardSnapshotFeedTests
await joinerCancellation.CancelAsync(); await joinerCancellation.CancelAsync();
// The joiner's unwind is a continuation of its cancelled channel read; give it time to // The joiner's unwind is a continuation of its cancelled channel read; give it time to
// run its unsubscribe before the dying pump is released. Only the interleaving depends // run its unsubscribe before the dying pump is released. The unsubscribe cannot be
// on this delay — the assertions below hold either way. // awaited (it waits on the parked pump) and exposes no signal, so this stays a real
// delay; only the interleaving depends on it — the assertions below hold either way.
await Task.Delay(TimeSpan.FromMilliseconds(150)); await Task.Delay(TimeSpan.FromMilliseconds(150));
service.ReleaseDisposal(); service.ReleaseDisposal();
@@ -312,7 +314,7 @@ public sealed class DashboardSnapshotFeedTests
feed.WatchAsync(nextCancellation.Token).GetAsyncEnumerator(nextCancellation.Token); feed.WatchAsync(nextCancellation.Token).GetAsyncEnumerator(nextCancellation.Token);
Task<bool> nextMove = next.MoveNextAsync().AsTask(); Task<bool> nextMove = next.MoveNextAsync().AsTask();
await WaitUntilAsync(() => service.EnumerationCount > enumerationsBefore); await service.Enumerations.WaitForAsync(enumerationsBefore + 1);
Assert.True(await nextMove.WaitAsync(TestTimeout)); Assert.True(await nextMove.WaitAsync(TestTimeout));
Assert.Equal("orphan-check", next.Current.GatewayVersion); Assert.Equal("orphan-check", next.Current.GatewayVersion);
@@ -384,15 +386,6 @@ public sealed class DashboardSnapshotFeedTests
} }
} }
private static async Task WaitUntilAsync(Func<bool> predicate)
{
using CancellationTokenSource cancellation = new(TestTimeout);
while (!predicate())
{
await Task.Delay(TimeSpan.FromMilliseconds(5), cancellation.Token);
}
}
/// <summary> /// <summary>
/// Snapshot source under the feed's control: counts enumerations, records how /// Snapshot source under the feed's control: counts enumerations, records how
/// each one ended, and lets the test drive snapshots (or a fault) into the /// each one ended, and lets the test drive snapshots (or a fault) into the
@@ -403,13 +396,17 @@ public sealed class DashboardSnapshotFeedTests
private readonly Channel<object> _pushes = Channel.CreateUnbounded<object>(); private readonly Channel<object> _pushes = Channel.CreateUnbounded<object>();
private readonly TaskCompletionSource _disposalReached = new(TaskCreationOptions.RunContinuationsAsynchronously); private readonly TaskCompletionSource _disposalReached = new(TaskCreationOptions.RunContinuationsAsynchronously);
private readonly TaskCompletionSource _disposalRelease = new(TaskCreationOptions.RunContinuationsAsynchronously); private readonly TaskCompletionSource _disposalRelease = new(TaskCreationOptions.RunContinuationsAsynchronously);
private int _enumerationCount;
private int _completedEnumerationCount;
private volatile bool _lastEnumerationWasCancelled; private volatile bool _lastEnumerationWasCancelled;
private volatile bool _holdDisposal; private volatile bool _holdDisposal;
/// <summary>Gets the number of times the feed started enumerating this source.</summary> /// <summary>Gets the number of times the feed started enumerating this source.</summary>
public int EnumerationCount => Volatile.Read(ref _enumerationCount); public int EnumerationCount => Enumerations.Count;
/// <summary>Gets the signal raised each time the feed starts enumerating this source.</summary>
public CountSignal Enumerations { get; } = new();
/// <summary>Gets the signal raised each time an enumeration finishes.</summary>
public CountSignal Completions { get; } = new();
/// <summary>Gets a task that completes when a held enumerator disposal is reached.</summary> /// <summary>Gets a task that completes when a held enumerator disposal is reached.</summary>
public Task DisposalReached => _disposalReached.Task; public Task DisposalReached => _disposalReached.Task;
@@ -421,7 +418,7 @@ public sealed class DashboardSnapshotFeedTests
public int PendingPushCount => _pushes.Reader.Count; public int PendingPushCount => _pushes.Reader.Count;
/// <summary>Gets the number of enumerations that have finished (cancelled, faulted, or completed).</summary> /// <summary>Gets the number of enumerations that have finished (cancelled, faulted, or completed).</summary>
public int CompletedEnumerationCount => Volatile.Read(ref _completedEnumerationCount); public int CompletedEnumerationCount => Completions.Count;
/// <summary>Gets a value indicating whether the most recently finished enumeration ended cancelled.</summary> /// <summary>Gets a value indicating whether the most recently finished enumeration ended cancelled.</summary>
public bool LastEnumerationWasCancelled => _lastEnumerationWasCancelled; public bool LastEnumerationWasCancelled => _lastEnumerationWasCancelled;
@@ -464,7 +461,7 @@ public sealed class DashboardSnapshotFeedTests
private async IAsyncEnumerable<DashboardSnapshot> EnumerateAsync( private async IAsyncEnumerable<DashboardSnapshot> EnumerateAsync(
[EnumeratorCancellation] CancellationToken cancellationToken) [EnumeratorCancellation] CancellationToken cancellationToken)
{ {
Interlocked.Increment(ref _enumerationCount); Enumerations.Signal();
try try
{ {
await foreach (object item in _pushes.Reader.ReadAllAsync(cancellationToken).ConfigureAwait(false)) await foreach (object item in _pushes.Reader.ReadAllAsync(cancellationToken).ConfigureAwait(false))
@@ -480,7 +477,7 @@ public sealed class DashboardSnapshotFeedTests
finally finally
{ {
_lastEnumerationWasCancelled = cancellationToken.IsCancellationRequested; _lastEnumerationWasCancelled = cancellationToken.IsCancellationRequested;
Interlocked.Increment(ref _completedEnumerationCount); Completions.Signal();
} }
} }
@@ -1,8 +1,10 @@
using System.Runtime.CompilerServices; using System.Runtime.CompilerServices;
using Microsoft.AspNetCore.SignalR; using Microsoft.AspNetCore.SignalR;
using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Time.Testing;
using ZB.MOM.WW.MxGateway.Server.Dashboard; using ZB.MOM.WW.MxGateway.Server.Dashboard;
using ZB.MOM.WW.MxGateway.Server.Dashboard.Hubs; using ZB.MOM.WW.MxGateway.Server.Dashboard.Hubs;
using ZB.MOM.WW.MxGateway.Tests.TestSupport;
namespace ZB.MOM.WW.MxGateway.Tests.Gateway.Dashboard; namespace ZB.MOM.WW.MxGateway.Tests.Gateway.Dashboard;
@@ -18,97 +20,94 @@ public sealed class DashboardSnapshotPublisherTests
/// reconnect delay and then re-open the subscription. Before the fix, /// reconnect delay and then re-open the subscription. Before the fix,
/// the publisher exited on the first non-cancellation exception and /// the publisher exited on the first non-cancellation exception and
/// the dashboard's snapshot stream went silent until process restart. /// the dashboard's snapshot stream went silent until process restart.
/// Driven by a fake clock, so the delay is exact and nothing sleeps.
/// </summary> /// </summary>
/// <returns>A task that represents the asynchronous operation.</returns> /// <returns>A task that represents the asynchronous operation.</returns>
[Fact] [Fact]
public async Task ExecuteAsync_WhenSnapshotServiceThrowsOnce_ReconnectsAfterDelay() public async Task ExecuteAsync_WhenSnapshotServiceThrowsOnce_ReconnectsAfterDelay()
{ {
ThrowOnceThenYieldSnapshotService snapshotService = new();
RecordingHubContext hubContext = new();
TimeSpan reconnectDelay = TimeSpan.FromMilliseconds(50); TimeSpan reconnectDelay = TimeSpan.FromMilliseconds(50);
TickObservingTimeProvider clock = new() { WatchedDueTime = reconnectDelay };
ThrowOnceThenYieldSnapshotService snapshotService = new(clock);
RecordingHubContext hubContext = new();
DashboardSnapshotPublisher publisher = new( DashboardSnapshotPublisher publisher = new(
snapshotService, snapshotService,
hubContext, hubContext,
ConnectedCounter(), ConnectedCounter(),
NullLogger<DashboardSnapshotPublisher>.Instance, NullLogger<DashboardSnapshotPublisher>.Instance,
reconnectDelay, reconnectDelay,
IdlePollInterval); IdlePollInterval,
timeProvider: clock);
using CancellationTokenSource cts = new(); using CancellationTokenSource cts = new();
Task execute = publisher.StartAsync(cts.Token); await publisher.StartAsync(cts.Token).WaitAsync(TestTimeout);
await execute.WaitAsync(TestTimeout);
// The publisher's first WatchSnapshotsAsync call throws; the second // The first subscribe throws; the publisher then arms the reconnect delay on the
// call yields one snapshot. We block here until the publisher has // fake clock. Wait for that timer so the advances below cannot be lost.
// made the second subscribe attempt AND broadcast its first await snapshotService.Subscribes.WaitForAsync(1);
// snapshot — proving the publisher did NOT exit on the throw. await clock.WaitForWatchedTimersAsync(1, TestTimeout);
await WaitUntilAsync(() => snapshotService.SubscribeCount >= 2);
await WaitUntilAsync(() => hubContext.SendCount >= 1); // The delay is honoured: just short of it, no second subscribe may have happened.
clock.Advance(reconnectDelay - TimeSpan.FromMilliseconds(1));
Assert.Equal(1, snapshotService.Subscribes.Count);
clock.Advance(TimeSpan.FromMilliseconds(1));
await snapshotService.Subscribes.WaitForAsync(2);
await hubContext.Proxy.Sends.WaitForAsync(1);
await cts.CancelAsync();
await publisher.StopAsync(CancellationToken.None);
DateTimeOffset firstThrowAt = snapshotService.FirstThrowAt DateTimeOffset firstThrowAt = snapshotService.FirstThrowAt
?? throw new InvalidOperationException("First subscribe did not record a throw timestamp."); ?? throw new InvalidOperationException("First subscribe did not record a throw timestamp.");
DateTimeOffset secondSubscribeAt = snapshotService.SecondSubscribeAt DateTimeOffset secondSubscribeAt = snapshotService.SecondSubscribeAt
?? throw new InvalidOperationException("Second subscribe did not record a timestamp."); ?? throw new InvalidOperationException("Second subscribe did not record a timestamp.");
await cts.CancelAsync(); // Fake time is exact: the gap is precisely the reconnect delay.
await publisher.StopAsync(CancellationToken.None); Assert.Equal(reconnectDelay, secondSubscribeAt - firstThrowAt);
Assert.True(snapshotService.SubscribeCount >= 2,
$"Expected at least 2 subscribe calls, got {snapshotService.SubscribeCount}.");
Assert.True(hubContext.SendCount >= 1);
// The gap is measured from the moment the first subscribe actually
// threw (inside the fake) to the moment the second subscribe began
// (also inside the fake). This isolates the publisher's
// Task.Delay(reconnectDelay) — no StartAsync / scheduling overhead in
// the baseline. The 10ms slack absorbs Task.Delay's coarse Windows
// timer quantum (~15ms) when the underlying scheduler wakes early.
TimeSpan gap = secondSubscribeAt - firstThrowAt;
Assert.True(gap >= reconnectDelay - TimeSpan.FromMilliseconds(10),
$"Expected reconnect gap >= {reconnectDelay.TotalMilliseconds}ms; got {gap.TotalMilliseconds}ms.");
} }
/// <summary> /// <summary>
/// Sanity: a normal completion of WatchSnapshotsAsync (no exception) /// Sanity: a normal completion of WatchSnapshotsAsync (no exception)
/// also reconnects after the delay — exits only on host shutdown. /// is re-subscribed; the publisher exits only on host shutdown.
/// </summary> /// </summary>
/// <returns>A task that represents the asynchronous operation.</returns> /// <returns>A task that represents the asynchronous operation.</returns>
[Fact] [Fact]
public async Task ExecuteAsync_WhenSnapshotServiceCompletes_ReconnectsAfterDelay() public async Task ExecuteAsync_WhenSnapshotServiceCompletes_Resubscribes()
{ {
CompleteImmediatelySnapshotService snapshotService = new(); CompleteImmediatelySnapshotService snapshotService = new();
RecordingHubContext hubContext = new(); RecordingHubContext hubContext = new();
TimeSpan reconnectDelay = TimeSpan.FromMilliseconds(50);
DashboardSnapshotPublisher publisher = new( DashboardSnapshotPublisher publisher = new(
snapshotService, snapshotService,
hubContext, hubContext,
ConnectedCounter(), ConnectedCounter(),
NullLogger<DashboardSnapshotPublisher>.Instance, NullLogger<DashboardSnapshotPublisher>.Instance,
reconnectDelay, TimeSpan.FromMilliseconds(50),
IdlePollInterval); IdlePollInterval,
timeProvider: new FakeTimeProvider());
using CancellationTokenSource cts = new(); using CancellationTokenSource cts = new();
Task execute = publisher.StartAsync(cts.Token); await publisher.StartAsync(cts.Token).WaitAsync(TestTimeout);
await execute.WaitAsync(TestTimeout);
await WaitUntilAsync(() => snapshotService.SubscribeCount >= 2); await snapshotService.Subscribes.WaitForAsync(2);
await cts.CancelAsync(); await cts.CancelAsync();
await publisher.StopAsync(CancellationToken.None); await publisher.StopAsync(CancellationToken.None);
Assert.True(snapshotService.SubscribeCount >= 2); Assert.True(snapshotService.Subscribes.Count >= 2);
} }
/// <summary> /// <summary>
/// With no dashboard connected there is nobody to broadcast to, so the publisher must /// With no dashboard connected there is nobody to broadcast to, so the publisher must
/// not advance the snapshot enumerator at all — every pull costs a registry snapshot and /// not advance the snapshot enumerator at all — every pull costs a registry snapshot and
/// sort, a locked metrics dictionary copy, and periodically a SQLite key-table read. /// sort, a locked metrics dictionary copy, and periodically a SQLite key-table read.
/// The first viewer to connect resumes the tick. /// The first viewer to connect resumes the tick. Many idle polls are driven on a fake clock.
/// </summary> /// </summary>
/// <returns>A task that represents the asynchronous operation.</returns> /// <returns>A task that represents the asynchronous operation.</returns>
[Fact] [Fact]
public async Task ExecuteAsync_WhenNoHubConnections_DoesNotPullSnapshots() public async Task ExecuteAsync_WhenNoHubConnections_DoesNotPullSnapshots()
{ {
TickObservingTimeProvider clock = new() { WatchedDueTime = IdlePollInterval };
CountingSnapshotService snapshotService = new(); CountingSnapshotService snapshotService = new();
RecordingHubContext hubContext = new(); RecordingHubContext hubContext = new();
DashboardSnapshotHubConnectionCounter connectionCounter = new(); DashboardSnapshotHubConnectionCounter connectionCounter = new();
@@ -118,19 +117,30 @@ public sealed class DashboardSnapshotPublisherTests
connectionCounter, connectionCounter,
NullLogger<DashboardSnapshotPublisher>.Instance, NullLogger<DashboardSnapshotPublisher>.Instance,
TimeSpan.FromMilliseconds(50), TimeSpan.FromMilliseconds(50),
IdlePollInterval); IdlePollInterval,
timeProvider: clock);
using CancellationTokenSource cts = new(); using CancellationTokenSource cts = new();
await publisher.StartAsync(cts.Token).WaitAsync(TestTimeout); await publisher.StartAsync(cts.Token).WaitAsync(TestTimeout);
// Long enough for many idle polls at IdlePollInterval. // A source that is ready to yield every time it is pulled, so the only thing
await Task.Delay(TimeSpan.FromMilliseconds(250)); // holding the publisher back is the idle gate.
snapshotService.Tick(100);
// Drive 25 idle polls; each wake-up re-arms exactly one new idle timer.
await clock.WaitForWatchedTimersAsync(1, TestTimeout);
for (int poll = 1; poll <= 25; poll++)
{
clock.Advance(IdlePollInterval);
await clock.WaitForWatchedTimersAsync(poll + 1, TestTimeout);
}
Assert.Equal(0, snapshotService.PullCount); Assert.Equal(0, snapshotService.PullCount);
Assert.Equal(0, hubContext.SendCount); Assert.Equal(0, hubContext.Proxy.Sends.Count);
connectionCounter.Increment(); connectionCounter.Increment();
await WaitUntilAsync(() => hubContext.SendCount >= 1); clock.Advance(IdlePollInterval);
await hubContext.Proxy.Sends.WaitForAsync(1);
await cts.CancelAsync(); await cts.CancelAsync();
await publisher.StopAsync(CancellationToken.None); await publisher.StopAsync(CancellationToken.None);
@@ -141,12 +151,14 @@ public sealed class DashboardSnapshotPublisherTests
/// <summary> /// <summary>
/// A broadcast that never completes — one remote client whose transport is wedged — /// A broadcast that never completes — one remote client whose transport is wedged —
/// is abandoned at the send timeout, so later ticks still go out instead of every /// is abandoned at the send timeout, so later ticks still go out instead of every
/// viewer waiting on the slowest one. /// viewer waiting on the slowest one. The send timeout runs on the fake clock.
/// </summary> /// </summary>
/// <returns>A task that represents the asynchronous operation.</returns> /// <returns>A task that represents the asynchronous operation.</returns>
[Fact] [Fact]
public async Task ExecuteAsync_WhenABroadcastNeverCompletes_LaterTicksStillGoOut() public async Task ExecuteAsync_WhenABroadcastNeverCompletes_LaterTicksStillGoOut()
{ {
TimeSpan sendTimeout = TimeSpan.FromMilliseconds(50);
TickObservingTimeProvider clock = new() { WatchedDueTime = sendTimeout };
CountingSnapshotService snapshotService = new(); CountingSnapshotService snapshotService = new();
RecordingHubContext hubContext = new(); RecordingHubContext hubContext = new();
hubContext.Proxy.StallFirstSend = true; hubContext.Proxy.StallFirstSend = true;
@@ -155,13 +167,23 @@ public sealed class DashboardSnapshotPublisherTests
hubContext, hubContext,
ConnectedCounter(), ConnectedCounter(),
NullLogger<DashboardSnapshotPublisher>.Instance, NullLogger<DashboardSnapshotPublisher>.Instance,
TimeSpan.FromMilliseconds(50), TimeSpan.FromSeconds(5),
IdlePollInterval, IdlePollInterval,
sendTimeout: TimeSpan.FromMilliseconds(50)); sendTimeout: sendTimeout,
timeProvider: clock);
snapshotService.Tick(3);
using CancellationTokenSource cts = new(); using CancellationTokenSource cts = new();
await publisher.StartAsync(cts.Token).WaitAsync(TestTimeout); await publisher.StartAsync(cts.Token).WaitAsync(TestTimeout);
await WaitUntilAsync(() => hubContext.SendCount >= 3);
// The first send is wedged and its timeout timer is armed; nothing else goes out
// until the fake clock reaches the timeout.
await hubContext.Proxy.Sends.WaitForAsync(1);
await clock.WaitForWatchedTimersAsync(1, TestTimeout);
Assert.Equal(1, hubContext.Proxy.Sends.Count);
clock.Advance(sendTimeout);
await hubContext.Proxy.Sends.WaitForAsync(3);
await cts.CancelAsync(); await cts.CancelAsync();
await publisher.StopAsync(CancellationToken.None); await publisher.StopAsync(CancellationToken.None);
@@ -176,29 +198,15 @@ public sealed class DashboardSnapshotPublisherTests
return counter; return counter;
} }
private static async Task WaitUntilAsync(Func<bool> predicate) private sealed class ThrowOnceThenYieldSnapshotService(TimeProvider clock) : IDashboardSnapshotService
{ {
using CancellationTokenSource cancellationTokenSource = new(TestTimeout); /// <summary>Gets the subscription attempts.</summary>
while (!predicate()) public CountSignal Subscribes { get; } = new();
{
await Task.Delay(TimeSpan.FromMilliseconds(5), cancellationTokenSource.Token);
}
}
private sealed class ThrowOnceThenYieldSnapshotService : IDashboardSnapshotService /// <summary>Gets the clock instant the first <c>WatchSnapshotsAsync</c> throws.</summary>
{
/// <summary>Gets the number of subscription attempts.</summary>
public int SubscribeCount { get; private set; }
/// <summary>
/// The wall-clock instant the first <c>WatchSnapshotsAsync</c> throws.
/// The reconnect-gap assertion is measured against this timestamp (NOT the
/// pre-<c>StartAsync</c> wall clock) so scheduling overhead is not baselined
/// into the lower bound.
/// </summary>
public DateTimeOffset? FirstThrowAt { get; private set; } public DateTimeOffset? FirstThrowAt { get; private set; }
/// <summary>Gets the wall-clock instant of the second subscription attempt.</summary> /// <summary>Gets the clock instant of the second subscription attempt.</summary>
public DateTimeOffset? SecondSubscribeAt { get; private set; } public DateTimeOffset? SecondSubscribeAt { get; private set; }
/// <inheritdoc /> /// <inheritdoc />
@@ -211,19 +219,18 @@ public sealed class DashboardSnapshotPublisherTests
public async IAsyncEnumerable<DashboardSnapshot> WatchSnapshotsAsync( public async IAsyncEnumerable<DashboardSnapshot> WatchSnapshotsAsync(
[EnumeratorCancellation] CancellationToken cancellationToken) [EnumeratorCancellation] CancellationToken cancellationToken)
{ {
SubscribeCount++; if (Subscribes.Count == 0)
int call = SubscribeCount;
if (call == 1)
{ {
// First call: throw after a brief yield so the publisher // First call: throw after a brief yield so the publisher
// observes us as a live producer that failed. // observes us as a live producer that failed.
await Task.Yield(); await Task.Yield();
FirstThrowAt = DateTimeOffset.UtcNow; FirstThrowAt = clock.GetUtcNow();
Subscribes.Signal();
throw new InvalidOperationException("simulated transient snapshot failure"); throw new InvalidOperationException("simulated transient snapshot failure");
} }
SecondSubscribeAt = DateTimeOffset.UtcNow; SecondSubscribeAt = clock.GetUtcNow();
Subscribes.Signal();
yield return GetSnapshot(); yield return GetSnapshot();
// Stay open until cancelled so the publisher's inner await // Stay open until cancelled so the publisher's inner await
@@ -240,8 +247,8 @@ public sealed class DashboardSnapshotPublisherTests
private sealed class CompleteImmediatelySnapshotService : IDashboardSnapshotService private sealed class CompleteImmediatelySnapshotService : IDashboardSnapshotService
{ {
/// <summary>Gets the number of subscription attempts.</summary> /// <summary>Gets the subscription attempts.</summary>
public int SubscribeCount { get; private set; } public CountSignal Subscribes { get; } = new();
/// <inheritdoc /> /// <inheritdoc />
public DashboardSnapshot GetSnapshot() public DashboardSnapshot GetSnapshot()
@@ -255,7 +262,7 @@ public sealed class DashboardSnapshotPublisherTests
[EnumeratorCancellation] CancellationToken cancellationToken) [EnumeratorCancellation] CancellationToken cancellationToken)
#pragma warning restore CS1998 #pragma warning restore CS1998
{ {
SubscribeCount++; Subscribes.Signal();
// Yield nothing and complete immediately — simulates a transient // Yield nothing and complete immediately — simulates a transient
// upstream disconnect that completes cleanly. // upstream disconnect that completes cleanly.
yield break; yield break;
@@ -264,11 +271,16 @@ public sealed class DashboardSnapshotPublisherTests
private sealed class CountingSnapshotService : IDashboardSnapshotService private sealed class CountingSnapshotService : IDashboardSnapshotService
{ {
private readonly SemaphoreSlim _ticks = new(0);
private int _pullCount; private int _pullCount;
/// <summary>Gets the number of snapshots the publisher pulled from this source.</summary> /// <summary>Gets the number of snapshots the publisher pulled from this source.</summary>
public int PullCount => Volatile.Read(ref _pullCount); public int PullCount => Volatile.Read(ref _pullCount);
/// <summary>Lets the source yield <paramref name="count"/> more snapshots.</summary>
/// <param name="count">Number of snapshots to allow.</param>
public void Tick(int count) => _ticks.Release(count);
/// <inheritdoc /> /// <inheritdoc />
public DashboardSnapshot GetSnapshot() public DashboardSnapshot GetSnapshot()
{ {
@@ -281,9 +293,9 @@ public sealed class DashboardSnapshotPublisherTests
{ {
while (!cancellationToken.IsCancellationRequested) while (!cancellationToken.IsCancellationRequested)
{ {
// Short cadence so a listening publisher pulls quickly; the counter only // The counter only moves when the publisher actually advances the enumerator
// moves when the publisher actually advances the enumerator. // and a tick is available.
await Task.Delay(TimeSpan.FromMilliseconds(5), cancellationToken).ConfigureAwait(false); await _ticks.WaitAsync(cancellationToken).ConfigureAwait(false);
Interlocked.Increment(ref _pullCount); Interlocked.Increment(ref _pullCount);
yield return GetSnapshot(); yield return GetSnapshot();
} }
@@ -303,8 +315,6 @@ public sealed class DashboardSnapshotPublisherTests
/// <summary>Gets the recording proxy every client lookup resolves to.</summary> /// <summary>Gets the recording proxy every client lookup resolves to.</summary>
public RecordingClientProxy Proxy => _clients.AllProxy; public RecordingClientProxy Proxy => _clients.AllProxy;
/// <summary>Gets the number of send calls recorded.</summary>
public int SendCount => _clients.AllProxy.SendCount;
} }
private sealed class RecordingHubClients : IHubClients private sealed class RecordingHubClients : IHubClients
@@ -359,16 +369,14 @@ public sealed class DashboardSnapshotPublisherTests
private sealed class RecordingClientProxy : IClientProxy private sealed class RecordingClientProxy : IClientProxy
{ {
private int _sendCount;
/// <summary> /// <summary>
/// Gets or sets a value indicating whether the first send never completes and /// Gets or sets a value indicating whether the first send never completes and
/// ignores its cancellation token — a wedged client transport. /// ignores its cancellation token — a wedged client transport.
/// </summary> /// </summary>
public bool StallFirstSend { get; set; } public bool StallFirstSend { get; set; }
/// <summary>Gets the number of send calls recorded.</summary> /// <summary>Gets the recorded send calls.</summary>
public int SendCount => Volatile.Read(ref _sendCount); public CountSignal Sends { get; } = new();
/// <summary>Records a send call and completes asynchronously.</summary> /// <summary>Records a send call and completes asynchronously.</summary>
/// <param name="method">The SignalR method name.</param> /// <param name="method">The SignalR method name.</param>
@@ -377,8 +385,8 @@ public sealed class DashboardSnapshotPublisherTests
/// <returns>A task that represents the asynchronous operation.</returns> /// <returns>A task that represents the asynchronous operation.</returns>
public Task SendCoreAsync(string method, object?[] args, CancellationToken cancellationToken = default) public Task SendCoreAsync(string method, object?[] args, CancellationToken cancellationToken = default)
{ {
int count = Interlocked.Increment(ref _sendCount); Sends.Signal();
return StallFirstSend && count == 1 ? new TaskCompletionSource().Task : Task.CompletedTask; return StallFirstSend && Sends.Count == 1 ? new TaskCompletionSource().Task : Task.CompletedTask;
} }
} }
@@ -5,6 +5,7 @@ using ZB.MOM.WW.MxGateway.Server.Metrics;
using ZB.MOM.WW.MxGateway.Server.Sessions; using ZB.MOM.WW.MxGateway.Server.Sessions;
using ZB.MOM.WW.MxGateway.Server.Workers; using ZB.MOM.WW.MxGateway.Server.Workers;
using ZB.MOM.WW.MxGateway.Tests.Gateway.Workers.Fakes; using ZB.MOM.WW.MxGateway.Tests.Gateway.Workers.Fakes;
using ZB.MOM.WW.MxGateway.Tests.TestSupport;
namespace ZB.MOM.WW.MxGateway.Tests.Gateway.Sessions; namespace ZB.MOM.WW.MxGateway.Tests.Gateway.Sessions;
@@ -198,19 +199,8 @@ public sealed class GatewaySessionWorkerExitTests
}; };
} }
private static async Task WaitUntilAsync(Func<bool> condition) private static Task WaitUntilAsync(Func<bool> condition)
{ => Eventually.WaitUntilAsync(condition, TestTimeout);
DateTime deadline = DateTime.UtcNow + TestTimeout;
while (!condition())
{
if (DateTime.UtcNow > deadline)
{
Assert.Fail("Condition was not met within the test timeout.");
}
await Task.Delay(10);
}
}
private sealed class SingleClientFactory(IWorkerClient client) : ISessionWorkerClientFactory private sealed class SingleClientFactory(IWorkerClient client) : ISessionWorkerClientFactory
{ {
@@ -4,6 +4,7 @@ using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Time.Testing; using Microsoft.Extensions.Time.Testing;
using ZB.MOM.WW.MxGateway.Contracts.Proto; using ZB.MOM.WW.MxGateway.Contracts.Proto;
using ZB.MOM.WW.MxGateway.Server.Sessions; using ZB.MOM.WW.MxGateway.Server.Sessions;
using ZB.MOM.WW.MxGateway.Tests.TestSupport;
namespace ZB.MOM.WW.MxGateway.Tests.Gateway.Sessions; namespace ZB.MOM.WW.MxGateway.Tests.Gateway.Sessions;
@@ -743,14 +744,7 @@ public sealed class SessionEventDistributorTests
Assert.Equal(SessionManagerErrorCode.EventQueueOverflow, fault.ErrorCode); Assert.Equal(SessionManagerErrorCode.EventQueueOverflow, fault.ErrorCode);
// Wait for the handler to fire (it runs on the pump thread). // Wait for the handler to fire (it runs on the pump thread).
await Task.Run(async () => await Eventually.WaitUntilAsync(() => Volatile.Read(ref observedIsOnlySubscriberSet) != 0, ReadTimeout);
{
using CancellationTokenSource cts = new(ReadTimeout);
while (Volatile.Read(ref observedIsOnlySubscriberSet) == 0)
{
await Task.Delay(10, cts.Token);
}
});
// isOnlySubscriber must be FALSE even though the internal subscriber was the ONLY // isOnlySubscriber must be FALSE even though the internal subscriber was the ONLY
// subscriber — CountExternalSubscribers excludes it, so a FailFast policy on the // subscriber — CountExternalSubscribers excludes it, so a FailFast policy on the
@@ -802,14 +796,7 @@ public sealed class SessionEventDistributorTests
async () => await DrainUntilFaultAsync(external.Reader)); async () => await DrainUntilFaultAsync(external.Reader));
Assert.Equal(SessionManagerErrorCode.EventQueueOverflow, fault.ErrorCode); Assert.Equal(SessionManagerErrorCode.EventQueueOverflow, fault.ErrorCode);
await Task.Run(async () => await Eventually.WaitUntilAsync(() => Volatile.Read(ref observedSet) != 0, ReadTimeout);
{
using CancellationTokenSource cts = new(ReadTimeout);
while (Volatile.Read(ref observedSet) == 0)
{
await Task.Delay(10, cts.Token);
}
});
// Guard: ensure the handler actually fired before asserting its observed value. // Guard: ensure the handler actually fired before asserting its observed value.
// Without this the test could pass vacuously if the overflow never triggered. // Without this the test could pass vacuously if the overflow never triggered.
@@ -358,15 +358,9 @@ public sealed class FakeWorkerHarnessTests
return new WorkerCommand { Command = command }; return new WorkerCommand { Command = command };
} }
private static async Task WaitUntilAsync( private static Task WaitUntilAsync(
Func<bool> predicate, Func<bool> predicate,
TimeSpan timeout) TimeSpan timeout)
{ => Eventually.WaitUntilAsync(predicate, timeout);
using CancellationTokenSource cancellationTokenSource = new(timeout);
while (!predicate())
{
await Task.Delay(TimeSpan.FromMilliseconds(10), cancellationTokenSource.Token);
}
}
} }
@@ -1522,16 +1522,10 @@ public sealed class WorkerClientTests
} }
} }
private static async Task WaitUntilAsync( private static Task WaitUntilAsync(
Func<bool> predicate, Func<bool> predicate,
TimeSpan timeout) TimeSpan timeout)
{ => Eventually.WaitUntilAsync(predicate, timeout);
using CancellationTokenSource cancellationTokenSource = new(timeout);
while (!predicate())
{
await Task.Delay(TimeSpan.FromMilliseconds(10), cancellationTokenSource.Token);
}
}
private sealed class PipePair : IAsyncDisposable private sealed class PipePair : IAsyncDisposable
{ {
@@ -4,6 +4,7 @@ using ZB.MOM.WW.Auth.Abstractions.ApiKeys;
using ZB.MOM.WW.MxGateway.Server.Security.Authentication; using ZB.MOM.WW.MxGateway.Server.Security.Authentication;
using LibApiKeyIdentity = ZB.MOM.WW.Auth.Abstractions.ApiKeys.ApiKeyIdentity; using LibApiKeyIdentity = ZB.MOM.WW.Auth.Abstractions.ApiKeys.ApiKeyIdentity;
using ZB.MOM.WW.MxGateway.Tests.TestSupport;
namespace ZB.MOM.WW.MxGateway.Tests.Security.Authentication; namespace ZB.MOM.WW.MxGateway.Tests.Security.Authentication;
@@ -267,11 +268,7 @@ public sealed class CachingApiKeyVerifierTests
cache.Compact(1.0); cache.Compact(1.0);
DateTime deadline = DateTime.UtcNow + TimeSpan.FromSeconds(5); await Eventually.WaitUntilAsync(() => verifier.IndexedCacheKeyCount == 0, TimeSpan.FromSeconds(5));
while (verifier.IndexedCacheKeyCount != 0 && DateTime.UtcNow < deadline)
{
await Task.Delay(10);
}
Assert.Equal(0, verifier.IndexedCacheKeyCount); Assert.Equal(0, verifier.IndexedCacheKeyCount);
Assert.Equal(0, verifier.IndexedKeyIdCount); Assert.Equal(0, verifier.IndexedKeyIdCount);
@@ -0,0 +1,36 @@
namespace ZB.MOM.WW.MxGateway.Tests.TestSupport;
/// <summary>
/// Counts events and lets a test wait for the nth one without polling. The timed wait result
/// is asserted, so a missed event fails the test instead of silently continuing (mxaccessgw#221).
/// </summary>
public sealed class CountSignal
{
private static readonly TimeSpan WaitTimeout = TimeSpan.FromSeconds(5);
private readonly SemaphoreSlim _signal = new(0);
private int _count;
private int _consumed;
/// <summary>Gets the number of events signalled so far.</summary>
public int Count => Volatile.Read(ref _count);
/// <summary>Records one event.</summary>
public void Signal()
{
Interlocked.Increment(ref _count);
_signal.Release();
}
/// <summary>Waits until at least <paramref name="total"/> events have been signalled; fails on timeout.</summary>
/// <param name="total">Cumulative event count to wait for.</param>
/// <returns>A task that represents the asynchronous operation.</returns>
public async Task WaitForAsync(int total)
{
while (_consumed < total)
{
Assert.True(await _signal.WaitAsync(WaitTimeout), $"Timed out waiting for event {total} (saw {Count}).");
_consumed++;
}
}
}
@@ -0,0 +1,46 @@
namespace ZB.MOM.WW.MxGateway.Tests.TestSupport;
/// <summary>
/// Bounded positive-wait helpers for conditions that no production type signals (for example
/// a monitor's current-alarm count). The wait fails with <see cref="TimeoutException"/> when the
/// condition never holds; use a signal (<see cref="CountSignal"/>, a TCS) where one exists.
/// </summary>
public static class Eventually
{
private static readonly TimeSpan PollInterval = TimeSpan.FromMilliseconds(2);
/// <summary>Waits until <paramref name="condition"/> holds.</summary>
/// <param name="condition">Condition to poll.</param>
/// <param name="timeout">Real-time bound on the wait.</param>
/// <returns>A task that represents the asynchronous operation.</returns>
/// <exception cref="TimeoutException">The condition did not hold within the bound.</exception>
public static async Task WaitUntilAsync(Func<bool> condition, TimeSpan timeout)
{
using CancellationTokenSource cts = new(timeout);
while (!condition())
{
try
{
await Task.Delay(PollInterval, cts.Token).ConfigureAwait(false);
}
catch (OperationCanceledException)
{
throw new TimeoutException("Condition was not met in time.");
}
}
}
/// <summary>Waits until <paramref name="probe"/> returns a non-null value and returns it.</summary>
/// <typeparam name="T">Reference type produced by the probe.</typeparam>
/// <param name="probe">Probe to poll.</param>
/// <param name="timeout">Real-time bound on the wait.</param>
/// <returns>The first non-null probe result.</returns>
/// <exception cref="TimeoutException">The probe never returned a value within the bound.</exception>
public static async Task<T> WaitForAsync<T>(Func<T?> probe, TimeSpan timeout)
where T : class
{
T? result = null;
await WaitUntilAsync(() => (result = probe()) is not null, timeout).ConfigureAwait(false);
return result!;
}
}