diff --git a/src/ZB.MOM.WW.MxGateway.Server/Dashboard/Hubs/BoundedHubSend.cs b/src/ZB.MOM.WW.MxGateway.Server/Dashboard/Hubs/BoundedHubSend.cs
index 4d14f370..943e9240 100644
--- a/src/ZB.MOM.WW.MxGateway.Server/Dashboard/Hubs/BoundedHubSend.cs
+++ b/src/ZB.MOM.WW.MxGateway.Server/Dashboard/Hubs/BoundedHubSend.cs
@@ -22,6 +22,7 @@ internal static class BoundedHubSend
/// The single message argument.
/// Upper bound on the send.
/// Host shutdown token; its cancellation propagates.
+ /// Clock the timeout runs on; defaults to .
///
/// when the send completed in time;
/// when it was abandoned at the timeout.
@@ -32,10 +33,11 @@ internal static class BoundedHubSend
string method,
object argument,
TimeSpan timeout,
- CancellationToken stoppingToken)
+ CancellationToken stoppingToken,
+ TimeProvider? timeProvider = null)
{
- using CancellationTokenSource bounded = CancellationTokenSource.CreateLinkedTokenSource(stoppingToken);
- bounded.CancelAfter(timeout);
+ using CancellationTokenSource timeoutSource = new(timeout, timeProvider ?? TimeProvider.System);
+ using CancellationTokenSource bounded = CancellationTokenSource.CreateLinkedTokenSource(stoppingToken, timeoutSource.Token);
try
{
diff --git a/src/ZB.MOM.WW.MxGateway.Server/Dashboard/Hubs/DashboardSnapshotPublisher.cs b/src/ZB.MOM.WW.MxGateway.Server/Dashboard/Hubs/DashboardSnapshotPublisher.cs
index 3ad8c5fd..bfbfdf85 100644
--- a/src/ZB.MOM.WW.MxGateway.Server/Dashboard/Hubs/DashboardSnapshotPublisher.cs
+++ b/src/ZB.MOM.WW.MxGateway.Server/Dashboard/Hubs/DashboardSnapshotPublisher.cs
@@ -48,18 +48,21 @@ public sealed class DashboardSnapshotPublisher : BackgroundService
private readonly TimeSpan _reconnectDelay;
private readonly TimeSpan _idlePollInterval;
private readonly TimeSpan _sendTimeout;
+ private readonly TimeProvider _timeProvider;
/// Initializes a new instance of the DashboardSnapshotPublisher class.
/// The snapshot service to subscribe to.
/// The SignalR hub context for broadcasting.
/// Live snapshot hub connection count used to idle-gate the tick.
/// The logger instance.
+ /// Clock for the idle poll, reconnect delay and send timeout; defaults to .
public DashboardSnapshotPublisher(
IDashboardSnapshotService snapshotService,
IHubContext hubContext,
DashboardSnapshotHubConnectionCounter connectionCounter,
- ILogger logger)
- : this(snapshotService, hubContext, connectionCounter, logger, DefaultReconnectDelay, DefaultIdlePollInterval)
+ ILogger logger,
+ TimeProvider? timeProvider = null)
+ : this(snapshotService, hubContext, connectionCounter, logger, DefaultReconnectDelay, DefaultIdlePollInterval, null, timeProvider)
{
}
@@ -75,6 +78,7 @@ public sealed class DashboardSnapshotPublisher : BackgroundService
/// The delay before reconnecting after a subscription failure.
/// How often the idle publisher re-checks for a connected viewer.
/// Upper bound on one broadcast; defaults to one second.
+ /// Clock for the idle poll, reconnect delay and send timeout; defaults to .
internal DashboardSnapshotPublisher(
IDashboardSnapshotService snapshotService,
IHubContext hubContext,
@@ -82,8 +86,10 @@ public sealed class DashboardSnapshotPublisher : BackgroundService
ILogger logger,
TimeSpan reconnectDelay,
TimeSpan idlePollInterval,
- TimeSpan? sendTimeout = null)
+ TimeSpan? sendTimeout = null,
+ TimeProvider? timeProvider = null)
{
+ _timeProvider = timeProvider ?? TimeProvider.System;
_snapshotService = snapshotService;
_hubContext = hubContext;
_connectionCounter = connectionCounter;
@@ -117,7 +123,7 @@ public sealed class DashboardSnapshotPublisher : BackgroundService
// Nobody is watching: leave the producer suspended and re-check
// shortly. The first viewer to connect resumes the tick, and is
// seeded directly by the hub's OnConnectedAsync meanwhile.
- await Task.Delay(_idlePollInterval, stoppingToken).ConfigureAwait(false);
+ await Task.Delay(_idlePollInterval, _timeProvider, stoppingToken).ConfigureAwait(false);
continue;
}
@@ -135,7 +141,8 @@ public sealed class DashboardSnapshotPublisher : BackgroundService
DashboardSnapshotHub.SnapshotMessage,
snapshot,
_sendTimeout,
- stoppingToken)
+ stoppingToken,
+ _timeProvider)
.ConfigureAwait(false);
if (!sent)
{
@@ -159,7 +166,7 @@ public sealed class DashboardSnapshotPublisher : BackgroundService
_logger.LogWarning(ex, "Snapshot subscription faulted; reconnecting in {DelaySeconds:F1}s.", _reconnectDelay.TotalSeconds);
try
{
- await Task.Delay(_reconnectDelay, stoppingToken).ConfigureAwait(false);
+ await Task.Delay(_reconnectDelay, _timeProvider, stoppingToken).ConfigureAwait(false);
}
catch (OperationCanceledException)
{
diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Alarms/AlarmFailoverEndToEndTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Alarms/AlarmFailoverEndToEndTests.cs
index 3a11ff92..b6313ab8 100644
--- a/src/ZB.MOM.WW.MxGateway.Tests/Alarms/AlarmFailoverEndToEndTests.cs
+++ b/src/ZB.MOM.WW.MxGateway.Tests/Alarms/AlarmFailoverEndToEndTests.cs
@@ -357,28 +357,19 @@ public sealed class AlarmFailoverEndToEndTests
await monitor.StopAsync(CancellationToken.None);
}
- private static async Task WaitForAsync(
+ private static Task WaitForAsync(
List received,
Func predicate,
TimeSpan timeout)
- {
- DateTime deadline = DateTime.UtcNow + timeout;
- while (DateTime.UtcNow < deadline)
- {
- lock (received)
+ => Eventually.WaitForAsync(
+ () =>
{
- AlarmFeedMessage? match = received.FirstOrDefault(predicate);
- if (match is not null)
+ lock (received)
{
- return match;
+ return received.FirstOrDefault(predicate);
}
- }
-
- await Task.Delay(25);
- }
-
- throw new TimeoutException("No matching AlarmFeedMessage was received in time.");
- }
+ },
+ timeout);
/// that returns a fixed watch-list.
private sealed class StubWatchListResolver(IReadOnlyList targets) : IAlarmWatchListResolver
diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Alarms/AlarmTruncationSignalTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Alarms/AlarmTruncationSignalTests.cs
index 2852da78..753e90fa 100644
--- a/src/ZB.MOM.WW.MxGateway.Tests/Alarms/AlarmTruncationSignalTests.cs
+++ b/src/ZB.MOM.WW.MxGateway.Tests/Alarms/AlarmTruncationSignalTests.cs
@@ -603,21 +603,8 @@ public sealed class AlarmTruncationSignalTests
alarms);
}
- private static async Task WaitUntilAsync(Func condition, TimeSpan timeout)
- {
- DateTime deadline = DateTime.UtcNow + timeout;
- while (DateTime.UtcNow < deadline)
- {
- if (condition())
- {
- return;
- }
-
- await Task.Delay(25);
- }
-
- throw new TimeoutException("Condition was not met in time.");
- }
+ private static Task WaitUntilAsync(Func condition, TimeSpan timeout)
+ => Eventually.WaitUntilAsync(condition, timeout);
/// that resolves an empty watch-list.
private sealed class StubWatchListResolver : IAlarmWatchListResolver
diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Alarms/GatewayAlarmMonitorAttachOrderTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Alarms/GatewayAlarmMonitorAttachOrderTests.cs
index fee7c4aa..9b63dcd3 100644
--- a/src/ZB.MOM.WW.MxGateway.Tests/Alarms/GatewayAlarmMonitorAttachOrderTests.cs
+++ b/src/ZB.MOM.WW.MxGateway.Tests/Alarms/GatewayAlarmMonitorAttachOrderTests.cs
@@ -515,44 +515,22 @@ public sealed class GatewayAlarmMonitorAttachOrderTests
return mxEvent;
}
- private static async Task WaitForAsync(
+ private static Task WaitForAsync(
List received,
Func predicate,
TimeSpan timeout)
- {
- DateTime deadline = DateTime.UtcNow + timeout;
- while (DateTime.UtcNow < deadline)
- {
- lock (received)
+ => Eventually.WaitForAsync(
+ () =>
{
- AlarmFeedMessage? match = received.FirstOrDefault(predicate);
- if (match is not null)
+ lock (received)
{
- return match;
+ return received.FirstOrDefault(predicate);
}
- }
+ },
+ timeout);
- await Task.Delay(25);
- }
-
- throw new TimeoutException("No matching AlarmFeedMessage was received in time.");
- }
-
- private static async Task WaitUntilAsync(Func condition, TimeSpan timeout)
- {
- DateTime deadline = DateTime.UtcNow + timeout;
- while (DateTime.UtcNow < deadline)
- {
- if (condition())
- {
- return;
- }
-
- await Task.Delay(25);
- }
-
- throw new TimeoutException("Condition was not met in time.");
- }
+ private static Task WaitUntilAsync(Func condition, TimeSpan timeout)
+ => Eventually.WaitUntilAsync(condition, timeout);
/// that resolves an empty watch-list.
private sealed class StubWatchListResolver : IAlarmWatchListResolver
diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Alarms/GatewayAlarmMonitorProviderModeTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Alarms/GatewayAlarmMonitorProviderModeTests.cs
index 643d2b69..fa87ceef 100644
--- a/src/ZB.MOM.WW.MxGateway.Tests/Alarms/GatewayAlarmMonitorProviderModeTests.cs
+++ b/src/ZB.MOM.WW.MxGateway.Tests/Alarms/GatewayAlarmMonitorProviderModeTests.cs
@@ -896,44 +896,22 @@ public sealed class GatewayAlarmMonitorProviderModeTests
NullLogger.Instance);
}
- private static async Task WaitForAsync(
+ private static Task WaitForAsync(
List received,
Func predicate,
TimeSpan timeout)
- {
- DateTime deadline = DateTime.UtcNow + timeout;
- while (DateTime.UtcNow < deadline)
- {
- lock (received)
+ => Eventually.WaitForAsync(
+ () =>
{
- AlarmFeedMessage? match = received.FirstOrDefault(predicate);
- if (match is not null)
+ lock (received)
{
- return match;
+ return received.FirstOrDefault(predicate);
}
- }
+ },
+ timeout);
- await Task.Delay(25);
- }
-
- throw new TimeoutException("No matching AlarmFeedMessage was received in time.");
- }
-
- private static async Task WaitUntilAsync(Func condition, TimeSpan timeout)
- {
- DateTime deadline = DateTime.UtcNow + timeout;
- while (DateTime.UtcNow < deadline)
- {
- if (condition())
- {
- return;
- }
-
- await Task.Delay(25);
- }
-
- throw new TimeoutException("Condition was not met in time.");
- }
+ private static Task WaitUntilAsync(Func condition, TimeSpan timeout)
+ => Eventually.WaitUntilAsync(condition, timeout);
///
/// that hands out a scripted sequence of
diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Dashboard/AlarmsHubPublisherTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Dashboard/AlarmsHubPublisherTests.cs
index 21711f8b..2cda0437 100644
--- a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Dashboard/AlarmsHubPublisherTests.cs
+++ b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Dashboard/AlarmsHubPublisherTests.cs
@@ -6,6 +6,7 @@ using ZB.MOM.WW.MxGateway.Contracts.Proto;
using ZB.MOM.WW.MxGateway.Server.Alarms;
using ZB.MOM.WW.MxGateway.Server.Configuration;
using ZB.MOM.WW.MxGateway.Server.Dashboard.Hubs;
+using ZB.MOM.WW.MxGateway.Tests.TestSupport;
namespace ZB.MOM.WW.MxGateway.Tests.Gateway.Dashboard;
@@ -241,14 +242,8 @@ public sealed class AlarmsHubPublisherTests
return hubContext;
}
- private static async Task WaitUntilAsync(Func predicate)
- {
- using CancellationTokenSource cancellationTokenSource = new(TestTimeout);
- while (!predicate())
- {
- await Task.Delay(TimeSpan.FromMilliseconds(5), cancellationTokenSource.Token);
- }
- }
+ private static Task WaitUntilAsync(Func predicate)
+ => Eventually.WaitUntilAsync(predicate, TestTimeout);
/// Builds a value-bearing transition feed message.
/// The message.
diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Dashboard/DashboardSnapshotFeedTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Dashboard/DashboardSnapshotFeedTests.cs
index 8c0091ed..1c590e12 100644
--- a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Dashboard/DashboardSnapshotFeedTests.cs
+++ b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Dashboard/DashboardSnapshotFeedTests.cs
@@ -1,6 +1,7 @@
using System.Runtime.CompilerServices;
using System.Threading.Channels;
using ZB.MOM.WW.MxGateway.Server.Dashboard;
+using ZB.MOM.WW.MxGateway.Tests.TestSupport;
namespace ZB.MOM.WW.MxGateway.Tests.Gateway.Dashboard;
@@ -23,7 +24,7 @@ public sealed class DashboardSnapshotFeedTests
///
/// A task that represents the asynchronous operation.
[Fact]
- public async Task WatchAsync_WithNoSubscribers_NeverEnumeratesUnderlyingWatch()
+ public Task WatchAsync_WithNoSubscribers_NeverEnumeratesUnderlyingWatch()
{
FakeSnapshotService service = new();
DashboardSnapshotFeed feed = new(service);
@@ -31,9 +32,9 @@ public sealed class DashboardSnapshotFeedTests
// Obtaining the enumerable without enumerating it must not subscribe
// either: the pump starts on the first MoveNextAsync, not before.
_ = feed.WatchAsync(CancellationToken.None);
- await Task.Delay(TimeSpan.FromMilliseconds(100));
Assert.Equal(0, service.EnumerationCount);
+ return Task.CompletedTask;
}
///
@@ -53,7 +54,7 @@ public sealed class DashboardSnapshotFeedTests
IAsyncEnumerator first =
feed.WatchAsync(firstCancellation.Token).GetAsyncEnumerator(firstCancellation.Token);
Task firstMove = first.MoveNextAsync().AsTask();
- await WaitUntilAsync(() => service.EnumerationCount >= 1);
+ await service.Enumerations.WaitForAsync(1);
IAsyncEnumerator second =
feed.WatchAsync(secondCancellation.Token).GetAsyncEnumerator(secondCancellation.Token);
@@ -92,19 +93,19 @@ public sealed class DashboardSnapshotFeedTests
IAsyncEnumerator first =
feed.WatchAsync(firstCancellation.Token).GetAsyncEnumerator(firstCancellation.Token);
Task firstMove = first.MoveNextAsync().AsTask();
- await WaitUntilAsync(() => service.EnumerationCount >= 1);
+ await service.Enumerations.WaitForAsync(1);
await firstCancellation.CancelAsync();
await DrainAsync(first, firstMove);
- await WaitUntilAsync(() => service.CompletedEnumerationCount >= 1);
+ await service.Completions.WaitForAsync(1);
Assert.True(service.LastEnumerationWasCancelled);
using CancellationTokenSource secondCancellation = new();
IAsyncEnumerator second =
feed.WatchAsync(secondCancellation.Token).GetAsyncEnumerator(secondCancellation.Token);
Task secondMove = second.MoveNextAsync().AsTask();
- await WaitUntilAsync(() => service.EnumerationCount >= 2);
+ await service.Enumerations.WaitForAsync(2);
Assert.Equal(2, service.EnumerationCount);
@@ -131,7 +132,7 @@ public sealed class DashboardSnapshotFeedTests
IAsyncEnumerator fast =
feed.WatchAsync(fastCancellation.Token).GetAsyncEnumerator(fastCancellation.Token);
Task fastMove = fast.MoveNextAsync().AsTask();
- await WaitUntilAsync(() => service.EnumerationCount >= 1);
+ await service.Enumerations.WaitForAsync(1);
// The slow subscriber registers but never advances until the very end.
IAsyncEnumerator slow =
@@ -179,7 +180,7 @@ public sealed class DashboardSnapshotFeedTests
IAsyncEnumerator first =
feed.WatchAsync(firstCancellation.Token).GetAsyncEnumerator(firstCancellation.Token);
Task firstMove = first.MoveNextAsync().AsTask();
- await WaitUntilAsync(() => service.EnumerationCount >= 1);
+ await service.Enumerations.WaitForAsync(1);
service.Fault(new InvalidOperationException("simulated snapshot source failure"));
@@ -192,7 +193,7 @@ public sealed class DashboardSnapshotFeedTests
IAsyncEnumerator second =
feed.WatchAsync(secondCancellation.Token).GetAsyncEnumerator(secondCancellation.Token);
Task secondMove = second.MoveNextAsync().AsTask();
- await WaitUntilAsync(() => service.EnumerationCount >= 2);
+ await service.Enumerations.WaitForAsync(2);
await PushUntilAsync(service, secondMove);
Assert.True(await secondMove.WaitAsync(TestTimeout));
@@ -219,7 +220,7 @@ public sealed class DashboardSnapshotFeedTests
IAsyncEnumerator first =
feed.WatchAsync(firstCancellation.Token).GetAsyncEnumerator(firstCancellation.Token);
Task firstMove = first.MoveNextAsync().AsTask();
- await WaitUntilAsync(() => service.EnumerationCount >= 1);
+ await service.Enumerations.WaitForAsync(1);
service.Fault(new InvalidOperationException("simulated snapshot source failure"));
@@ -245,7 +246,7 @@ public sealed class DashboardSnapshotFeedTests
// ...and the one that joined mid-unwind is served by a restarted enumeration
// instead of inheriting the failure.
- await WaitUntilAsync(() => service.EnumerationCount >= 2);
+ await service.Enumerations.WaitForAsync(2);
await PushUntilAsync(service, secondMove);
Assert.True(await secondMove.WaitAsync(TestTimeout));
Assert.StartsWith("push-", second.Current.GatewayVersion, StringComparison.Ordinal);
@@ -273,7 +274,7 @@ public sealed class DashboardSnapshotFeedTests
IAsyncEnumerator oldSubscriber =
feed.WatchAsync(oldCancellation.Token).GetAsyncEnumerator(oldCancellation.Token);
Task oldMove = oldSubscriber.MoveNextAsync().AsTask();
- await WaitUntilAsync(() => service.EnumerationCount >= 1);
+ await service.Enumerations.WaitForAsync(1);
service.Fault(new InvalidOperationException("simulated snapshot source failure"));
await service.DisposalReached.WaitAsync(TestTimeout);
@@ -287,8 +288,9 @@ public sealed class DashboardSnapshotFeedTests
await joinerCancellation.CancelAsync();
// The joiner's unwind is a continuation of its cancelled channel read; give it time to
- // run its unsubscribe before the dying pump is released. Only the interleaving depends
- // on this delay — the assertions below hold either way.
+ // run its unsubscribe before the dying pump is released. The unsubscribe cannot be
+ // awaited (it waits on the parked pump) and exposes no signal, so this stays a real
+ // delay; only the interleaving depends on it — the assertions below hold either way.
await Task.Delay(TimeSpan.FromMilliseconds(150));
service.ReleaseDisposal();
@@ -312,7 +314,7 @@ public sealed class DashboardSnapshotFeedTests
feed.WatchAsync(nextCancellation.Token).GetAsyncEnumerator(nextCancellation.Token);
Task nextMove = next.MoveNextAsync().AsTask();
- await WaitUntilAsync(() => service.EnumerationCount > enumerationsBefore);
+ await service.Enumerations.WaitForAsync(enumerationsBefore + 1);
Assert.True(await nextMove.WaitAsync(TestTimeout));
Assert.Equal("orphan-check", next.Current.GatewayVersion);
@@ -384,15 +386,6 @@ public sealed class DashboardSnapshotFeedTests
}
}
- private static async Task WaitUntilAsync(Func predicate)
- {
- using CancellationTokenSource cancellation = new(TestTimeout);
- while (!predicate())
- {
- await Task.Delay(TimeSpan.FromMilliseconds(5), cancellation.Token);
- }
- }
-
///
/// Snapshot source under the feed's control: counts enumerations, records how
/// each one ended, and lets the test drive snapshots (or a fault) into the
@@ -403,13 +396,17 @@ public sealed class DashboardSnapshotFeedTests
private readonly Channel
/// A task that represents the asynchronous operation.
[Fact]
public async Task ExecuteAsync_WhenSnapshotServiceThrowsOnce_ReconnectsAfterDelay()
{
- ThrowOnceThenYieldSnapshotService snapshotService = new();
- RecordingHubContext hubContext = new();
TimeSpan reconnectDelay = TimeSpan.FromMilliseconds(50);
+ TickObservingTimeProvider clock = new() { WatchedDueTime = reconnectDelay };
+ ThrowOnceThenYieldSnapshotService snapshotService = new(clock);
+ RecordingHubContext hubContext = new();
DashboardSnapshotPublisher publisher = new(
snapshotService,
hubContext,
ConnectedCounter(),
NullLogger.Instance,
reconnectDelay,
- IdlePollInterval);
+ IdlePollInterval,
+ timeProvider: clock);
using CancellationTokenSource cts = new();
- Task execute = publisher.StartAsync(cts.Token);
- await execute.WaitAsync(TestTimeout);
+ await publisher.StartAsync(cts.Token).WaitAsync(TestTimeout);
- // The publisher's first WatchSnapshotsAsync call throws; the second
- // call yields one snapshot. We block here until the publisher has
- // made the second subscribe attempt AND broadcast its first
- // snapshot — proving the publisher did NOT exit on the throw.
- await WaitUntilAsync(() => snapshotService.SubscribeCount >= 2);
- await WaitUntilAsync(() => hubContext.SendCount >= 1);
+ // The first subscribe throws; the publisher then arms the reconnect delay on the
+ // fake clock. Wait for that timer so the advances below cannot be lost.
+ await snapshotService.Subscribes.WaitForAsync(1);
+ await clock.WaitForWatchedTimersAsync(1, TestTimeout);
+
+ // The delay is honoured: just short of it, no second subscribe may have happened.
+ clock.Advance(reconnectDelay - TimeSpan.FromMilliseconds(1));
+ Assert.Equal(1, snapshotService.Subscribes.Count);
+
+ clock.Advance(TimeSpan.FromMilliseconds(1));
+ await snapshotService.Subscribes.WaitForAsync(2);
+ await hubContext.Proxy.Sends.WaitForAsync(1);
+
+ await cts.CancelAsync();
+ await publisher.StopAsync(CancellationToken.None);
DateTimeOffset firstThrowAt = snapshotService.FirstThrowAt
?? throw new InvalidOperationException("First subscribe did not record a throw timestamp.");
DateTimeOffset secondSubscribeAt = snapshotService.SecondSubscribeAt
?? throw new InvalidOperationException("Second subscribe did not record a timestamp.");
- await cts.CancelAsync();
- await publisher.StopAsync(CancellationToken.None);
-
- Assert.True(snapshotService.SubscribeCount >= 2,
- $"Expected at least 2 subscribe calls, got {snapshotService.SubscribeCount}.");
- Assert.True(hubContext.SendCount >= 1);
-
- // The gap is measured from the moment the first subscribe actually
- // threw (inside the fake) to the moment the second subscribe began
- // (also inside the fake). This isolates the publisher's
- // Task.Delay(reconnectDelay) — no StartAsync / scheduling overhead in
- // the baseline. The 10ms slack absorbs Task.Delay's coarse Windows
- // timer quantum (~15ms) when the underlying scheduler wakes early.
- TimeSpan gap = secondSubscribeAt - firstThrowAt;
- Assert.True(gap >= reconnectDelay - TimeSpan.FromMilliseconds(10),
- $"Expected reconnect gap >= {reconnectDelay.TotalMilliseconds}ms; got {gap.TotalMilliseconds}ms.");
+ // Fake time is exact: the gap is precisely the reconnect delay.
+ Assert.Equal(reconnectDelay, secondSubscribeAt - firstThrowAt);
}
///
/// Sanity: a normal completion of WatchSnapshotsAsync (no exception)
- /// also reconnects after the delay — exits only on host shutdown.
+ /// is re-subscribed; the publisher exits only on host shutdown.
///
/// A task that represents the asynchronous operation.
[Fact]
- public async Task ExecuteAsync_WhenSnapshotServiceCompletes_ReconnectsAfterDelay()
+ public async Task ExecuteAsync_WhenSnapshotServiceCompletes_Resubscribes()
{
CompleteImmediatelySnapshotService snapshotService = new();
RecordingHubContext hubContext = new();
- TimeSpan reconnectDelay = TimeSpan.FromMilliseconds(50);
DashboardSnapshotPublisher publisher = new(
snapshotService,
hubContext,
ConnectedCounter(),
NullLogger.Instance,
- reconnectDelay,
- IdlePollInterval);
+ TimeSpan.FromMilliseconds(50),
+ IdlePollInterval,
+ timeProvider: new FakeTimeProvider());
using CancellationTokenSource cts = new();
- Task execute = publisher.StartAsync(cts.Token);
- await execute.WaitAsync(TestTimeout);
+ await publisher.StartAsync(cts.Token).WaitAsync(TestTimeout);
- await WaitUntilAsync(() => snapshotService.SubscribeCount >= 2);
+ await snapshotService.Subscribes.WaitForAsync(2);
await cts.CancelAsync();
await publisher.StopAsync(CancellationToken.None);
- Assert.True(snapshotService.SubscribeCount >= 2);
+ Assert.True(snapshotService.Subscribes.Count >= 2);
}
///
/// With no dashboard connected there is nobody to broadcast to, so the publisher must
/// not advance the snapshot enumerator at all — every pull costs a registry snapshot and
/// sort, a locked metrics dictionary copy, and periodically a SQLite key-table read.
- /// The first viewer to connect resumes the tick.
+ /// The first viewer to connect resumes the tick. Many idle polls are driven on a fake clock.
///
/// A task that represents the asynchronous operation.
[Fact]
public async Task ExecuteAsync_WhenNoHubConnections_DoesNotPullSnapshots()
{
+ TickObservingTimeProvider clock = new() { WatchedDueTime = IdlePollInterval };
CountingSnapshotService snapshotService = new();
RecordingHubContext hubContext = new();
DashboardSnapshotHubConnectionCounter connectionCounter = new();
@@ -118,19 +117,30 @@ public sealed class DashboardSnapshotPublisherTests
connectionCounter,
NullLogger.Instance,
TimeSpan.FromMilliseconds(50),
- IdlePollInterval);
+ IdlePollInterval,
+ timeProvider: clock);
using CancellationTokenSource cts = new();
await publisher.StartAsync(cts.Token).WaitAsync(TestTimeout);
- // Long enough for many idle polls at IdlePollInterval.
- await Task.Delay(TimeSpan.FromMilliseconds(250));
+ // A source that is ready to yield every time it is pulled, so the only thing
+ // holding the publisher back is the idle gate.
+ snapshotService.Tick(100);
+
+ // Drive 25 idle polls; each wake-up re-arms exactly one new idle timer.
+ await clock.WaitForWatchedTimersAsync(1, TestTimeout);
+ for (int poll = 1; poll <= 25; poll++)
+ {
+ clock.Advance(IdlePollInterval);
+ await clock.WaitForWatchedTimersAsync(poll + 1, TestTimeout);
+ }
Assert.Equal(0, snapshotService.PullCount);
- Assert.Equal(0, hubContext.SendCount);
+ Assert.Equal(0, hubContext.Proxy.Sends.Count);
connectionCounter.Increment();
- await WaitUntilAsync(() => hubContext.SendCount >= 1);
+ clock.Advance(IdlePollInterval);
+ await hubContext.Proxy.Sends.WaitForAsync(1);
await cts.CancelAsync();
await publisher.StopAsync(CancellationToken.None);
@@ -141,12 +151,14 @@ public sealed class DashboardSnapshotPublisherTests
///
/// A broadcast that never completes — one remote client whose transport is wedged —
/// is abandoned at the send timeout, so later ticks still go out instead of every
- /// viewer waiting on the slowest one.
+ /// viewer waiting on the slowest one. The send timeout runs on the fake clock.
///
/// A task that represents the asynchronous operation.
[Fact]
public async Task ExecuteAsync_WhenABroadcastNeverCompletes_LaterTicksStillGoOut()
{
+ TimeSpan sendTimeout = TimeSpan.FromMilliseconds(50);
+ TickObservingTimeProvider clock = new() { WatchedDueTime = sendTimeout };
CountingSnapshotService snapshotService = new();
RecordingHubContext hubContext = new();
hubContext.Proxy.StallFirstSend = true;
@@ -155,13 +167,23 @@ public sealed class DashboardSnapshotPublisherTests
hubContext,
ConnectedCounter(),
NullLogger.Instance,
- TimeSpan.FromMilliseconds(50),
+ TimeSpan.FromSeconds(5),
IdlePollInterval,
- sendTimeout: TimeSpan.FromMilliseconds(50));
+ sendTimeout: sendTimeout,
+ timeProvider: clock);
+ snapshotService.Tick(3);
using CancellationTokenSource cts = new();
await publisher.StartAsync(cts.Token).WaitAsync(TestTimeout);
- await WaitUntilAsync(() => hubContext.SendCount >= 3);
+
+ // The first send is wedged and its timeout timer is armed; nothing else goes out
+ // until the fake clock reaches the timeout.
+ await hubContext.Proxy.Sends.WaitForAsync(1);
+ await clock.WaitForWatchedTimersAsync(1, TestTimeout);
+ Assert.Equal(1, hubContext.Proxy.Sends.Count);
+
+ clock.Advance(sendTimeout);
+ await hubContext.Proxy.Sends.WaitForAsync(3);
await cts.CancelAsync();
await publisher.StopAsync(CancellationToken.None);
@@ -176,29 +198,15 @@ public sealed class DashboardSnapshotPublisherTests
return counter;
}
- private static async Task WaitUntilAsync(Func predicate)
+ private sealed class ThrowOnceThenYieldSnapshotService(TimeProvider clock) : IDashboardSnapshotService
{
- using CancellationTokenSource cancellationTokenSource = new(TestTimeout);
- while (!predicate())
- {
- await Task.Delay(TimeSpan.FromMilliseconds(5), cancellationTokenSource.Token);
- }
- }
+ /// Gets the subscription attempts.
+ public CountSignal Subscribes { get; } = new();
- private sealed class ThrowOnceThenYieldSnapshotService : IDashboardSnapshotService
- {
- /// Gets the number of subscription attempts.
- public int SubscribeCount { get; private set; }
-
- ///
- /// The wall-clock instant the first WatchSnapshotsAsync throws.
- /// The reconnect-gap assertion is measured against this timestamp (NOT the
- /// pre-StartAsync wall clock) so scheduling overhead is not baselined
- /// into the lower bound.
- ///
+ /// Gets the clock instant the first WatchSnapshotsAsync throws.
public DateTimeOffset? FirstThrowAt { get; private set; }
- /// Gets the wall-clock instant of the second subscription attempt.
+ /// Gets the clock instant of the second subscription attempt.
public DateTimeOffset? SecondSubscribeAt { get; private set; }
///
@@ -211,19 +219,18 @@ public sealed class DashboardSnapshotPublisherTests
public async IAsyncEnumerable WatchSnapshotsAsync(
[EnumeratorCancellation] CancellationToken cancellationToken)
{
- SubscribeCount++;
- int call = SubscribeCount;
-
- if (call == 1)
+ if (Subscribes.Count == 0)
{
// First call: throw after a brief yield so the publisher
// observes us as a live producer that failed.
await Task.Yield();
- FirstThrowAt = DateTimeOffset.UtcNow;
+ FirstThrowAt = clock.GetUtcNow();
+ Subscribes.Signal();
throw new InvalidOperationException("simulated transient snapshot failure");
}
- SecondSubscribeAt = DateTimeOffset.UtcNow;
+ SecondSubscribeAt = clock.GetUtcNow();
+ Subscribes.Signal();
yield return GetSnapshot();
// Stay open until cancelled so the publisher's inner await
@@ -240,8 +247,8 @@ public sealed class DashboardSnapshotPublisherTests
private sealed class CompleteImmediatelySnapshotService : IDashboardSnapshotService
{
- /// Gets the number of subscription attempts.
- public int SubscribeCount { get; private set; }
+ /// Gets the subscription attempts.
+ public CountSignal Subscribes { get; } = new();
///
public DashboardSnapshot GetSnapshot()
@@ -255,7 +262,7 @@ public sealed class DashboardSnapshotPublisherTests
[EnumeratorCancellation] CancellationToken cancellationToken)
#pragma warning restore CS1998
{
- SubscribeCount++;
+ Subscribes.Signal();
// Yield nothing and complete immediately — simulates a transient
// upstream disconnect that completes cleanly.
yield break;
@@ -264,11 +271,16 @@ public sealed class DashboardSnapshotPublisherTests
private sealed class CountingSnapshotService : IDashboardSnapshotService
{
+ private readonly SemaphoreSlim _ticks = new(0);
private int _pullCount;
/// Gets the number of snapshots the publisher pulled from this source.
public int PullCount => Volatile.Read(ref _pullCount);
+ /// Lets the source yield more snapshots.
+ /// Number of snapshots to allow.
+ public void Tick(int count) => _ticks.Release(count);
+
///
public DashboardSnapshot GetSnapshot()
{
@@ -281,9 +293,9 @@ public sealed class DashboardSnapshotPublisherTests
{
while (!cancellationToken.IsCancellationRequested)
{
- // Short cadence so a listening publisher pulls quickly; the counter only
- // moves when the publisher actually advances the enumerator.
- await Task.Delay(TimeSpan.FromMilliseconds(5), cancellationToken).ConfigureAwait(false);
+ // The counter only moves when the publisher actually advances the enumerator
+ // and a tick is available.
+ await _ticks.WaitAsync(cancellationToken).ConfigureAwait(false);
Interlocked.Increment(ref _pullCount);
yield return GetSnapshot();
}
@@ -303,8 +315,6 @@ public sealed class DashboardSnapshotPublisherTests
/// Gets the recording proxy every client lookup resolves to.
public RecordingClientProxy Proxy => _clients.AllProxy;
- /// Gets the number of send calls recorded.
- public int SendCount => _clients.AllProxy.SendCount;
}
private sealed class RecordingHubClients : IHubClients
@@ -359,16 +369,14 @@ public sealed class DashboardSnapshotPublisherTests
private sealed class RecordingClientProxy : IClientProxy
{
- private int _sendCount;
-
///
/// Gets or sets a value indicating whether the first send never completes and
/// ignores its cancellation token — a wedged client transport.
///
public bool StallFirstSend { get; set; }
- /// Gets the number of send calls recorded.
- public int SendCount => Volatile.Read(ref _sendCount);
+ /// Gets the recorded send calls.
+ public CountSignal Sends { get; } = new();
/// Records a send call and completes asynchronously.
/// The SignalR method name.
@@ -377,8 +385,8 @@ public sealed class DashboardSnapshotPublisherTests
/// A task that represents the asynchronous operation.
public Task SendCoreAsync(string method, object?[] args, CancellationToken cancellationToken = default)
{
- int count = Interlocked.Increment(ref _sendCount);
- return StallFirstSend && count == 1 ? new TaskCompletionSource().Task : Task.CompletedTask;
+ Sends.Signal();
+ return StallFirstSend && Sends.Count == 1 ? new TaskCompletionSource().Task : Task.CompletedTask;
}
}
diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Sessions/GatewaySessionWorkerExitTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Sessions/GatewaySessionWorkerExitTests.cs
index cb1764ef..9339e1ed 100644
--- a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Sessions/GatewaySessionWorkerExitTests.cs
+++ b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Sessions/GatewaySessionWorkerExitTests.cs
@@ -5,6 +5,7 @@ using ZB.MOM.WW.MxGateway.Server.Metrics;
using ZB.MOM.WW.MxGateway.Server.Sessions;
using ZB.MOM.WW.MxGateway.Server.Workers;
using ZB.MOM.WW.MxGateway.Tests.Gateway.Workers.Fakes;
+using ZB.MOM.WW.MxGateway.Tests.TestSupport;
namespace ZB.MOM.WW.MxGateway.Tests.Gateway.Sessions;
@@ -198,19 +199,8 @@ public sealed class GatewaySessionWorkerExitTests
};
}
- private static async Task WaitUntilAsync(Func condition)
- {
- DateTime deadline = DateTime.UtcNow + TestTimeout;
- while (!condition())
- {
- if (DateTime.UtcNow > deadline)
- {
- Assert.Fail("Condition was not met within the test timeout.");
- }
-
- await Task.Delay(10);
- }
- }
+ private static Task WaitUntilAsync(Func condition)
+ => Eventually.WaitUntilAsync(condition, TestTimeout);
private sealed class SingleClientFactory(IWorkerClient client) : ISessionWorkerClientFactory
{
diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Sessions/SessionEventDistributorTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Sessions/SessionEventDistributorTests.cs
index a42b8682..4f828c2a 100644
--- a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Sessions/SessionEventDistributorTests.cs
+++ b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Sessions/SessionEventDistributorTests.cs
@@ -4,6 +4,7 @@ using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Time.Testing;
using ZB.MOM.WW.MxGateway.Contracts.Proto;
using ZB.MOM.WW.MxGateway.Server.Sessions;
+using ZB.MOM.WW.MxGateway.Tests.TestSupport;
namespace ZB.MOM.WW.MxGateway.Tests.Gateway.Sessions;
@@ -743,14 +744,7 @@ public sealed class SessionEventDistributorTests
Assert.Equal(SessionManagerErrorCode.EventQueueOverflow, fault.ErrorCode);
// Wait for the handler to fire (it runs on the pump thread).
- await Task.Run(async () =>
- {
- using CancellationTokenSource cts = new(ReadTimeout);
- while (Volatile.Read(ref observedIsOnlySubscriberSet) == 0)
- {
- await Task.Delay(10, cts.Token);
- }
- });
+ await Eventually.WaitUntilAsync(() => Volatile.Read(ref observedIsOnlySubscriberSet) != 0, ReadTimeout);
// isOnlySubscriber must be FALSE even though the internal subscriber was the ONLY
// subscriber — CountExternalSubscribers excludes it, so a FailFast policy on the
@@ -802,14 +796,7 @@ public sealed class SessionEventDistributorTests
async () => await DrainUntilFaultAsync(external.Reader));
Assert.Equal(SessionManagerErrorCode.EventQueueOverflow, fault.ErrorCode);
- await Task.Run(async () =>
- {
- using CancellationTokenSource cts = new(ReadTimeout);
- while (Volatile.Read(ref observedSet) == 0)
- {
- await Task.Delay(10, cts.Token);
- }
- });
+ await Eventually.WaitUntilAsync(() => Volatile.Read(ref observedSet) != 0, ReadTimeout);
// Guard: ensure the handler actually fired before asserting its observed value.
// Without this the test could pass vacuously if the overflow never triggered.
diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Workers/FakeWorkerHarnessTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Workers/FakeWorkerHarnessTests.cs
index 44f3644f..5fac3dc3 100644
--- a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Workers/FakeWorkerHarnessTests.cs
+++ b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Workers/FakeWorkerHarnessTests.cs
@@ -358,15 +358,9 @@ public sealed class FakeWorkerHarnessTests
return new WorkerCommand { Command = command };
}
- private static async Task WaitUntilAsync(
+ private static Task WaitUntilAsync(
Func predicate,
TimeSpan timeout)
- {
- using CancellationTokenSource cancellationTokenSource = new(timeout);
- while (!predicate())
- {
- await Task.Delay(TimeSpan.FromMilliseconds(10), cancellationTokenSource.Token);
- }
- }
+ => Eventually.WaitUntilAsync(predicate, timeout);
}
diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Workers/WorkerClientTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Workers/WorkerClientTests.cs
index 2262b00f..40682d17 100644
--- a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Workers/WorkerClientTests.cs
+++ b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Workers/WorkerClientTests.cs
@@ -1522,16 +1522,10 @@ public sealed class WorkerClientTests
}
}
- private static async Task WaitUntilAsync(
+ private static Task WaitUntilAsync(
Func predicate,
TimeSpan timeout)
- {
- using CancellationTokenSource cancellationTokenSource = new(timeout);
- while (!predicate())
- {
- await Task.Delay(TimeSpan.FromMilliseconds(10), cancellationTokenSource.Token);
- }
- }
+ => Eventually.WaitUntilAsync(predicate, timeout);
private sealed class PipePair : IAsyncDisposable
{
diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Security/Authentication/CachingApiKeyVerifierTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Security/Authentication/CachingApiKeyVerifierTests.cs
index 7278d642..d0d9d513 100644
--- a/src/ZB.MOM.WW.MxGateway.Tests/Security/Authentication/CachingApiKeyVerifierTests.cs
+++ b/src/ZB.MOM.WW.MxGateway.Tests/Security/Authentication/CachingApiKeyVerifierTests.cs
@@ -4,6 +4,7 @@ using ZB.MOM.WW.Auth.Abstractions.ApiKeys;
using ZB.MOM.WW.MxGateway.Server.Security.Authentication;
using LibApiKeyIdentity = ZB.MOM.WW.Auth.Abstractions.ApiKeys.ApiKeyIdentity;
+using ZB.MOM.WW.MxGateway.Tests.TestSupport;
namespace ZB.MOM.WW.MxGateway.Tests.Security.Authentication;
@@ -267,11 +268,7 @@ public sealed class CachingApiKeyVerifierTests
cache.Compact(1.0);
- DateTime deadline = DateTime.UtcNow + TimeSpan.FromSeconds(5);
- while (verifier.IndexedCacheKeyCount != 0 && DateTime.UtcNow < deadline)
- {
- await Task.Delay(10);
- }
+ await Eventually.WaitUntilAsync(() => verifier.IndexedCacheKeyCount == 0, TimeSpan.FromSeconds(5));
Assert.Equal(0, verifier.IndexedCacheKeyCount);
Assert.Equal(0, verifier.IndexedKeyIdCount);
diff --git a/src/ZB.MOM.WW.MxGateway.Tests/TestSupport/CountSignal.cs b/src/ZB.MOM.WW.MxGateway.Tests/TestSupport/CountSignal.cs
new file mode 100644
index 00000000..ff7fa6ef
--- /dev/null
+++ b/src/ZB.MOM.WW.MxGateway.Tests/TestSupport/CountSignal.cs
@@ -0,0 +1,36 @@
+namespace ZB.MOM.WW.MxGateway.Tests.TestSupport;
+
+///
+/// Counts events and lets a test wait for the nth one without polling. The timed wait result
+/// is asserted, so a missed event fails the test instead of silently continuing (mxaccessgw#221).
+///
+public sealed class CountSignal
+{
+ private static readonly TimeSpan WaitTimeout = TimeSpan.FromSeconds(5);
+
+ private readonly SemaphoreSlim _signal = new(0);
+ private int _count;
+ private int _consumed;
+
+ /// Gets the number of events signalled so far.
+ public int Count => Volatile.Read(ref _count);
+
+ /// Records one event.
+ public void Signal()
+ {
+ Interlocked.Increment(ref _count);
+ _signal.Release();
+ }
+
+ /// Waits until at least events have been signalled; fails on timeout.
+ /// Cumulative event count to wait for.
+ /// A task that represents the asynchronous operation.
+ public async Task WaitForAsync(int total)
+ {
+ while (_consumed < total)
+ {
+ Assert.True(await _signal.WaitAsync(WaitTimeout), $"Timed out waiting for event {total} (saw {Count}).");
+ _consumed++;
+ }
+ }
+}
diff --git a/src/ZB.MOM.WW.MxGateway.Tests/TestSupport/Eventually.cs b/src/ZB.MOM.WW.MxGateway.Tests/TestSupport/Eventually.cs
new file mode 100644
index 00000000..788d0f3b
--- /dev/null
+++ b/src/ZB.MOM.WW.MxGateway.Tests/TestSupport/Eventually.cs
@@ -0,0 +1,46 @@
+namespace ZB.MOM.WW.MxGateway.Tests.TestSupport;
+
+///
+/// Bounded positive-wait helpers for conditions that no production type signals (for example
+/// a monitor's current-alarm count). The wait fails with when the
+/// condition never holds; use a signal (, a TCS) where one exists.
+///
+public static class Eventually
+{
+ private static readonly TimeSpan PollInterval = TimeSpan.FromMilliseconds(2);
+
+ /// Waits until holds.
+ /// Condition to poll.
+ /// Real-time bound on the wait.
+ /// A task that represents the asynchronous operation.
+ /// The condition did not hold within the bound.
+ public static async Task WaitUntilAsync(Func condition, TimeSpan timeout)
+ {
+ using CancellationTokenSource cts = new(timeout);
+ while (!condition())
+ {
+ try
+ {
+ await Task.Delay(PollInterval, cts.Token).ConfigureAwait(false);
+ }
+ catch (OperationCanceledException)
+ {
+ throw new TimeoutException("Condition was not met in time.");
+ }
+ }
+ }
+
+ /// Waits until returns a non-null value and returns it.
+ /// Reference type produced by the probe.
+ /// Probe to poll.
+ /// Real-time bound on the wait.
+ /// The first non-null probe result.
+ /// The probe never returned a value within the bound.
+ public static async Task WaitForAsync(Func probe, TimeSpan timeout)
+ where T : class
+ {
+ T? result = null;
+ await WaitUntilAsync(() => (result = probe()) is not null, timeout).ConfigureAwait(false);
+ return result!;
+ }
+}