diff --git a/src/ZB.MOM.WW.MxGateway.Server/Dashboard/Hubs/BoundedHubSend.cs b/src/ZB.MOM.WW.MxGateway.Server/Dashboard/Hubs/BoundedHubSend.cs index 4d14f370..943e9240 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Dashboard/Hubs/BoundedHubSend.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Dashboard/Hubs/BoundedHubSend.cs @@ -22,6 +22,7 @@ internal static class BoundedHubSend /// The single message argument. /// Upper bound on the send. /// Host shutdown token; its cancellation propagates. + /// Clock the timeout runs on; defaults to . /// /// when the send completed in time; /// when it was abandoned at the timeout. @@ -32,10 +33,11 @@ internal static class BoundedHubSend string method, object argument, TimeSpan timeout, - CancellationToken stoppingToken) + CancellationToken stoppingToken, + TimeProvider? timeProvider = null) { - using CancellationTokenSource bounded = CancellationTokenSource.CreateLinkedTokenSource(stoppingToken); - bounded.CancelAfter(timeout); + using CancellationTokenSource timeoutSource = new(timeout, timeProvider ?? TimeProvider.System); + using CancellationTokenSource bounded = CancellationTokenSource.CreateLinkedTokenSource(stoppingToken, timeoutSource.Token); try { diff --git a/src/ZB.MOM.WW.MxGateway.Server/Dashboard/Hubs/DashboardSnapshotPublisher.cs b/src/ZB.MOM.WW.MxGateway.Server/Dashboard/Hubs/DashboardSnapshotPublisher.cs index 3ad8c5fd..bfbfdf85 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Dashboard/Hubs/DashboardSnapshotPublisher.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Dashboard/Hubs/DashboardSnapshotPublisher.cs @@ -48,18 +48,21 @@ public sealed class DashboardSnapshotPublisher : BackgroundService private readonly TimeSpan _reconnectDelay; private readonly TimeSpan _idlePollInterval; private readonly TimeSpan _sendTimeout; + private readonly TimeProvider _timeProvider; /// Initializes a new instance of the DashboardSnapshotPublisher class. /// The snapshot service to subscribe to. /// The SignalR hub context for broadcasting. /// Live snapshot hub connection count used to idle-gate the tick. /// The logger instance. + /// Clock for the idle poll, reconnect delay and send timeout; defaults to . public DashboardSnapshotPublisher( IDashboardSnapshotService snapshotService, IHubContext hubContext, DashboardSnapshotHubConnectionCounter connectionCounter, - ILogger logger) - : this(snapshotService, hubContext, connectionCounter, logger, DefaultReconnectDelay, DefaultIdlePollInterval) + ILogger logger, + TimeProvider? timeProvider = null) + : this(snapshotService, hubContext, connectionCounter, logger, DefaultReconnectDelay, DefaultIdlePollInterval, null, timeProvider) { } @@ -75,6 +78,7 @@ public sealed class DashboardSnapshotPublisher : BackgroundService /// The delay before reconnecting after a subscription failure. /// How often the idle publisher re-checks for a connected viewer. /// Upper bound on one broadcast; defaults to one second. + /// Clock for the idle poll, reconnect delay and send timeout; defaults to . internal DashboardSnapshotPublisher( IDashboardSnapshotService snapshotService, IHubContext hubContext, @@ -82,8 +86,10 @@ public sealed class DashboardSnapshotPublisher : BackgroundService ILogger logger, TimeSpan reconnectDelay, TimeSpan idlePollInterval, - TimeSpan? sendTimeout = null) + TimeSpan? sendTimeout = null, + TimeProvider? timeProvider = null) { + _timeProvider = timeProvider ?? TimeProvider.System; _snapshotService = snapshotService; _hubContext = hubContext; _connectionCounter = connectionCounter; @@ -117,7 +123,7 @@ public sealed class DashboardSnapshotPublisher : BackgroundService // Nobody is watching: leave the producer suspended and re-check // shortly. The first viewer to connect resumes the tick, and is // seeded directly by the hub's OnConnectedAsync meanwhile. - await Task.Delay(_idlePollInterval, stoppingToken).ConfigureAwait(false); + await Task.Delay(_idlePollInterval, _timeProvider, stoppingToken).ConfigureAwait(false); continue; } @@ -135,7 +141,8 @@ public sealed class DashboardSnapshotPublisher : BackgroundService DashboardSnapshotHub.SnapshotMessage, snapshot, _sendTimeout, - stoppingToken) + stoppingToken, + _timeProvider) .ConfigureAwait(false); if (!sent) { @@ -159,7 +166,7 @@ public sealed class DashboardSnapshotPublisher : BackgroundService _logger.LogWarning(ex, "Snapshot subscription faulted; reconnecting in {DelaySeconds:F1}s.", _reconnectDelay.TotalSeconds); try { - await Task.Delay(_reconnectDelay, stoppingToken).ConfigureAwait(false); + await Task.Delay(_reconnectDelay, _timeProvider, stoppingToken).ConfigureAwait(false); } catch (OperationCanceledException) { diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Alarms/AlarmFailoverEndToEndTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Alarms/AlarmFailoverEndToEndTests.cs index 3a11ff92..b6313ab8 100644 --- a/src/ZB.MOM.WW.MxGateway.Tests/Alarms/AlarmFailoverEndToEndTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Tests/Alarms/AlarmFailoverEndToEndTests.cs @@ -357,28 +357,19 @@ public sealed class AlarmFailoverEndToEndTests await monitor.StopAsync(CancellationToken.None); } - private static async Task WaitForAsync( + private static Task WaitForAsync( List received, Func predicate, TimeSpan timeout) - { - DateTime deadline = DateTime.UtcNow + timeout; - while (DateTime.UtcNow < deadline) - { - lock (received) + => Eventually.WaitForAsync( + () => { - AlarmFeedMessage? match = received.FirstOrDefault(predicate); - if (match is not null) + lock (received) { - return match; + return received.FirstOrDefault(predicate); } - } - - await Task.Delay(25); - } - - throw new TimeoutException("No matching AlarmFeedMessage was received in time."); - } + }, + timeout); /// that returns a fixed watch-list. private sealed class StubWatchListResolver(IReadOnlyList targets) : IAlarmWatchListResolver diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Alarms/AlarmTruncationSignalTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Alarms/AlarmTruncationSignalTests.cs index 2852da78..753e90fa 100644 --- a/src/ZB.MOM.WW.MxGateway.Tests/Alarms/AlarmTruncationSignalTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Tests/Alarms/AlarmTruncationSignalTests.cs @@ -603,21 +603,8 @@ public sealed class AlarmTruncationSignalTests alarms); } - private static async Task WaitUntilAsync(Func condition, TimeSpan timeout) - { - DateTime deadline = DateTime.UtcNow + timeout; - while (DateTime.UtcNow < deadline) - { - if (condition()) - { - return; - } - - await Task.Delay(25); - } - - throw new TimeoutException("Condition was not met in time."); - } + private static Task WaitUntilAsync(Func condition, TimeSpan timeout) + => Eventually.WaitUntilAsync(condition, timeout); /// that resolves an empty watch-list. private sealed class StubWatchListResolver : IAlarmWatchListResolver diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Alarms/GatewayAlarmMonitorAttachOrderTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Alarms/GatewayAlarmMonitorAttachOrderTests.cs index fee7c4aa..9b63dcd3 100644 --- a/src/ZB.MOM.WW.MxGateway.Tests/Alarms/GatewayAlarmMonitorAttachOrderTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Tests/Alarms/GatewayAlarmMonitorAttachOrderTests.cs @@ -515,44 +515,22 @@ public sealed class GatewayAlarmMonitorAttachOrderTests return mxEvent; } - private static async Task WaitForAsync( + private static Task WaitForAsync( List received, Func predicate, TimeSpan timeout) - { - DateTime deadline = DateTime.UtcNow + timeout; - while (DateTime.UtcNow < deadline) - { - lock (received) + => Eventually.WaitForAsync( + () => { - AlarmFeedMessage? match = received.FirstOrDefault(predicate); - if (match is not null) + lock (received) { - return match; + return received.FirstOrDefault(predicate); } - } + }, + timeout); - await Task.Delay(25); - } - - throw new TimeoutException("No matching AlarmFeedMessage was received in time."); - } - - private static async Task WaitUntilAsync(Func condition, TimeSpan timeout) - { - DateTime deadline = DateTime.UtcNow + timeout; - while (DateTime.UtcNow < deadline) - { - if (condition()) - { - return; - } - - await Task.Delay(25); - } - - throw new TimeoutException("Condition was not met in time."); - } + private static Task WaitUntilAsync(Func condition, TimeSpan timeout) + => Eventually.WaitUntilAsync(condition, timeout); /// that resolves an empty watch-list. private sealed class StubWatchListResolver : IAlarmWatchListResolver diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Alarms/GatewayAlarmMonitorProviderModeTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Alarms/GatewayAlarmMonitorProviderModeTests.cs index 643d2b69..fa87ceef 100644 --- a/src/ZB.MOM.WW.MxGateway.Tests/Alarms/GatewayAlarmMonitorProviderModeTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Tests/Alarms/GatewayAlarmMonitorProviderModeTests.cs @@ -896,44 +896,22 @@ public sealed class GatewayAlarmMonitorProviderModeTests NullLogger.Instance); } - private static async Task WaitForAsync( + private static Task WaitForAsync( List received, Func predicate, TimeSpan timeout) - { - DateTime deadline = DateTime.UtcNow + timeout; - while (DateTime.UtcNow < deadline) - { - lock (received) + => Eventually.WaitForAsync( + () => { - AlarmFeedMessage? match = received.FirstOrDefault(predicate); - if (match is not null) + lock (received) { - return match; + return received.FirstOrDefault(predicate); } - } + }, + timeout); - await Task.Delay(25); - } - - throw new TimeoutException("No matching AlarmFeedMessage was received in time."); - } - - private static async Task WaitUntilAsync(Func condition, TimeSpan timeout) - { - DateTime deadline = DateTime.UtcNow + timeout; - while (DateTime.UtcNow < deadline) - { - if (condition()) - { - return; - } - - await Task.Delay(25); - } - - throw new TimeoutException("Condition was not met in time."); - } + private static Task WaitUntilAsync(Func condition, TimeSpan timeout) + => Eventually.WaitUntilAsync(condition, timeout); /// /// that hands out a scripted sequence of diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Dashboard/AlarmsHubPublisherTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Dashboard/AlarmsHubPublisherTests.cs index 21711f8b..2cda0437 100644 --- a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Dashboard/AlarmsHubPublisherTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Dashboard/AlarmsHubPublisherTests.cs @@ -6,6 +6,7 @@ using ZB.MOM.WW.MxGateway.Contracts.Proto; using ZB.MOM.WW.MxGateway.Server.Alarms; using ZB.MOM.WW.MxGateway.Server.Configuration; using ZB.MOM.WW.MxGateway.Server.Dashboard.Hubs; +using ZB.MOM.WW.MxGateway.Tests.TestSupport; namespace ZB.MOM.WW.MxGateway.Tests.Gateway.Dashboard; @@ -241,14 +242,8 @@ public sealed class AlarmsHubPublisherTests return hubContext; } - private static async Task WaitUntilAsync(Func predicate) - { - using CancellationTokenSource cancellationTokenSource = new(TestTimeout); - while (!predicate()) - { - await Task.Delay(TimeSpan.FromMilliseconds(5), cancellationTokenSource.Token); - } - } + private static Task WaitUntilAsync(Func predicate) + => Eventually.WaitUntilAsync(predicate, TestTimeout); /// Builds a value-bearing transition feed message. /// The message. diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Dashboard/DashboardSnapshotFeedTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Dashboard/DashboardSnapshotFeedTests.cs index 8c0091ed..1c590e12 100644 --- a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Dashboard/DashboardSnapshotFeedTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Dashboard/DashboardSnapshotFeedTests.cs @@ -1,6 +1,7 @@ using System.Runtime.CompilerServices; using System.Threading.Channels; using ZB.MOM.WW.MxGateway.Server.Dashboard; +using ZB.MOM.WW.MxGateway.Tests.TestSupport; namespace ZB.MOM.WW.MxGateway.Tests.Gateway.Dashboard; @@ -23,7 +24,7 @@ public sealed class DashboardSnapshotFeedTests /// /// A task that represents the asynchronous operation. [Fact] - public async Task WatchAsync_WithNoSubscribers_NeverEnumeratesUnderlyingWatch() + public Task WatchAsync_WithNoSubscribers_NeverEnumeratesUnderlyingWatch() { FakeSnapshotService service = new(); DashboardSnapshotFeed feed = new(service); @@ -31,9 +32,9 @@ public sealed class DashboardSnapshotFeedTests // Obtaining the enumerable without enumerating it must not subscribe // either: the pump starts on the first MoveNextAsync, not before. _ = feed.WatchAsync(CancellationToken.None); - await Task.Delay(TimeSpan.FromMilliseconds(100)); Assert.Equal(0, service.EnumerationCount); + return Task.CompletedTask; } /// @@ -53,7 +54,7 @@ public sealed class DashboardSnapshotFeedTests IAsyncEnumerator first = feed.WatchAsync(firstCancellation.Token).GetAsyncEnumerator(firstCancellation.Token); Task firstMove = first.MoveNextAsync().AsTask(); - await WaitUntilAsync(() => service.EnumerationCount >= 1); + await service.Enumerations.WaitForAsync(1); IAsyncEnumerator second = feed.WatchAsync(secondCancellation.Token).GetAsyncEnumerator(secondCancellation.Token); @@ -92,19 +93,19 @@ public sealed class DashboardSnapshotFeedTests IAsyncEnumerator first = feed.WatchAsync(firstCancellation.Token).GetAsyncEnumerator(firstCancellation.Token); Task firstMove = first.MoveNextAsync().AsTask(); - await WaitUntilAsync(() => service.EnumerationCount >= 1); + await service.Enumerations.WaitForAsync(1); await firstCancellation.CancelAsync(); await DrainAsync(first, firstMove); - await WaitUntilAsync(() => service.CompletedEnumerationCount >= 1); + await service.Completions.WaitForAsync(1); Assert.True(service.LastEnumerationWasCancelled); using CancellationTokenSource secondCancellation = new(); IAsyncEnumerator second = feed.WatchAsync(secondCancellation.Token).GetAsyncEnumerator(secondCancellation.Token); Task secondMove = second.MoveNextAsync().AsTask(); - await WaitUntilAsync(() => service.EnumerationCount >= 2); + await service.Enumerations.WaitForAsync(2); Assert.Equal(2, service.EnumerationCount); @@ -131,7 +132,7 @@ public sealed class DashboardSnapshotFeedTests IAsyncEnumerator fast = feed.WatchAsync(fastCancellation.Token).GetAsyncEnumerator(fastCancellation.Token); Task fastMove = fast.MoveNextAsync().AsTask(); - await WaitUntilAsync(() => service.EnumerationCount >= 1); + await service.Enumerations.WaitForAsync(1); // The slow subscriber registers but never advances until the very end. IAsyncEnumerator slow = @@ -179,7 +180,7 @@ public sealed class DashboardSnapshotFeedTests IAsyncEnumerator first = feed.WatchAsync(firstCancellation.Token).GetAsyncEnumerator(firstCancellation.Token); Task firstMove = first.MoveNextAsync().AsTask(); - await WaitUntilAsync(() => service.EnumerationCount >= 1); + await service.Enumerations.WaitForAsync(1); service.Fault(new InvalidOperationException("simulated snapshot source failure")); @@ -192,7 +193,7 @@ public sealed class DashboardSnapshotFeedTests IAsyncEnumerator second = feed.WatchAsync(secondCancellation.Token).GetAsyncEnumerator(secondCancellation.Token); Task secondMove = second.MoveNextAsync().AsTask(); - await WaitUntilAsync(() => service.EnumerationCount >= 2); + await service.Enumerations.WaitForAsync(2); await PushUntilAsync(service, secondMove); Assert.True(await secondMove.WaitAsync(TestTimeout)); @@ -219,7 +220,7 @@ public sealed class DashboardSnapshotFeedTests IAsyncEnumerator first = feed.WatchAsync(firstCancellation.Token).GetAsyncEnumerator(firstCancellation.Token); Task firstMove = first.MoveNextAsync().AsTask(); - await WaitUntilAsync(() => service.EnumerationCount >= 1); + await service.Enumerations.WaitForAsync(1); service.Fault(new InvalidOperationException("simulated snapshot source failure")); @@ -245,7 +246,7 @@ public sealed class DashboardSnapshotFeedTests // ...and the one that joined mid-unwind is served by a restarted enumeration // instead of inheriting the failure. - await WaitUntilAsync(() => service.EnumerationCount >= 2); + await service.Enumerations.WaitForAsync(2); await PushUntilAsync(service, secondMove); Assert.True(await secondMove.WaitAsync(TestTimeout)); Assert.StartsWith("push-", second.Current.GatewayVersion, StringComparison.Ordinal); @@ -273,7 +274,7 @@ public sealed class DashboardSnapshotFeedTests IAsyncEnumerator oldSubscriber = feed.WatchAsync(oldCancellation.Token).GetAsyncEnumerator(oldCancellation.Token); Task oldMove = oldSubscriber.MoveNextAsync().AsTask(); - await WaitUntilAsync(() => service.EnumerationCount >= 1); + await service.Enumerations.WaitForAsync(1); service.Fault(new InvalidOperationException("simulated snapshot source failure")); await service.DisposalReached.WaitAsync(TestTimeout); @@ -287,8 +288,9 @@ public sealed class DashboardSnapshotFeedTests await joinerCancellation.CancelAsync(); // The joiner's unwind is a continuation of its cancelled channel read; give it time to - // run its unsubscribe before the dying pump is released. Only the interleaving depends - // on this delay — the assertions below hold either way. + // run its unsubscribe before the dying pump is released. The unsubscribe cannot be + // awaited (it waits on the parked pump) and exposes no signal, so this stays a real + // delay; only the interleaving depends on it — the assertions below hold either way. await Task.Delay(TimeSpan.FromMilliseconds(150)); service.ReleaseDisposal(); @@ -312,7 +314,7 @@ public sealed class DashboardSnapshotFeedTests feed.WatchAsync(nextCancellation.Token).GetAsyncEnumerator(nextCancellation.Token); Task nextMove = next.MoveNextAsync().AsTask(); - await WaitUntilAsync(() => service.EnumerationCount > enumerationsBefore); + await service.Enumerations.WaitForAsync(enumerationsBefore + 1); Assert.True(await nextMove.WaitAsync(TestTimeout)); Assert.Equal("orphan-check", next.Current.GatewayVersion); @@ -384,15 +386,6 @@ public sealed class DashboardSnapshotFeedTests } } - private static async Task WaitUntilAsync(Func predicate) - { - using CancellationTokenSource cancellation = new(TestTimeout); - while (!predicate()) - { - await Task.Delay(TimeSpan.FromMilliseconds(5), cancellation.Token); - } - } - /// /// Snapshot source under the feed's control: counts enumerations, records how /// each one ended, and lets the test drive snapshots (or a fault) into the @@ -403,13 +396,17 @@ public sealed class DashboardSnapshotFeedTests private readonly Channel _pushes = Channel.CreateUnbounded(); private readonly TaskCompletionSource _disposalReached = new(TaskCreationOptions.RunContinuationsAsynchronously); private readonly TaskCompletionSource _disposalRelease = new(TaskCreationOptions.RunContinuationsAsynchronously); - private int _enumerationCount; - private int _completedEnumerationCount; private volatile bool _lastEnumerationWasCancelled; private volatile bool _holdDisposal; /// Gets the number of times the feed started enumerating this source. - public int EnumerationCount => Volatile.Read(ref _enumerationCount); + public int EnumerationCount => Enumerations.Count; + + /// Gets the signal raised each time the feed starts enumerating this source. + public CountSignal Enumerations { get; } = new(); + + /// Gets the signal raised each time an enumeration finishes. + public CountSignal Completions { get; } = new(); /// Gets a task that completes when a held enumerator disposal is reached. public Task DisposalReached => _disposalReached.Task; @@ -421,7 +418,7 @@ public sealed class DashboardSnapshotFeedTests public int PendingPushCount => _pushes.Reader.Count; /// Gets the number of enumerations that have finished (cancelled, faulted, or completed). - public int CompletedEnumerationCount => Volatile.Read(ref _completedEnumerationCount); + public int CompletedEnumerationCount => Completions.Count; /// Gets a value indicating whether the most recently finished enumeration ended cancelled. public bool LastEnumerationWasCancelled => _lastEnumerationWasCancelled; @@ -464,7 +461,7 @@ public sealed class DashboardSnapshotFeedTests private async IAsyncEnumerable EnumerateAsync( [EnumeratorCancellation] CancellationToken cancellationToken) { - Interlocked.Increment(ref _enumerationCount); + Enumerations.Signal(); try { await foreach (object item in _pushes.Reader.ReadAllAsync(cancellationToken).ConfigureAwait(false)) @@ -480,7 +477,7 @@ public sealed class DashboardSnapshotFeedTests finally { _lastEnumerationWasCancelled = cancellationToken.IsCancellationRequested; - Interlocked.Increment(ref _completedEnumerationCount); + Completions.Signal(); } } diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Dashboard/DashboardSnapshotPublisherTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Dashboard/DashboardSnapshotPublisherTests.cs index b9833779..27210cd2 100644 --- a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Dashboard/DashboardSnapshotPublisherTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Dashboard/DashboardSnapshotPublisherTests.cs @@ -1,8 +1,10 @@ using System.Runtime.CompilerServices; using Microsoft.AspNetCore.SignalR; using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; using ZB.MOM.WW.MxGateway.Server.Dashboard; using ZB.MOM.WW.MxGateway.Server.Dashboard.Hubs; +using ZB.MOM.WW.MxGateway.Tests.TestSupport; namespace ZB.MOM.WW.MxGateway.Tests.Gateway.Dashboard; @@ -18,97 +20,94 @@ public sealed class DashboardSnapshotPublisherTests /// reconnect delay and then re-open the subscription. Before the fix, /// the publisher exited on the first non-cancellation exception and /// the dashboard's snapshot stream went silent until process restart. + /// Driven by a fake clock, so the delay is exact and nothing sleeps. /// /// A task that represents the asynchronous operation. [Fact] public async Task ExecuteAsync_WhenSnapshotServiceThrowsOnce_ReconnectsAfterDelay() { - ThrowOnceThenYieldSnapshotService snapshotService = new(); - RecordingHubContext hubContext = new(); TimeSpan reconnectDelay = TimeSpan.FromMilliseconds(50); + TickObservingTimeProvider clock = new() { WatchedDueTime = reconnectDelay }; + ThrowOnceThenYieldSnapshotService snapshotService = new(clock); + RecordingHubContext hubContext = new(); DashboardSnapshotPublisher publisher = new( snapshotService, hubContext, ConnectedCounter(), NullLogger.Instance, reconnectDelay, - IdlePollInterval); + IdlePollInterval, + timeProvider: clock); using CancellationTokenSource cts = new(); - Task execute = publisher.StartAsync(cts.Token); - await execute.WaitAsync(TestTimeout); + await publisher.StartAsync(cts.Token).WaitAsync(TestTimeout); - // The publisher's first WatchSnapshotsAsync call throws; the second - // call yields one snapshot. We block here until the publisher has - // made the second subscribe attempt AND broadcast its first - // snapshot — proving the publisher did NOT exit on the throw. - await WaitUntilAsync(() => snapshotService.SubscribeCount >= 2); - await WaitUntilAsync(() => hubContext.SendCount >= 1); + // The first subscribe throws; the publisher then arms the reconnect delay on the + // fake clock. Wait for that timer so the advances below cannot be lost. + await snapshotService.Subscribes.WaitForAsync(1); + await clock.WaitForWatchedTimersAsync(1, TestTimeout); + + // The delay is honoured: just short of it, no second subscribe may have happened. + clock.Advance(reconnectDelay - TimeSpan.FromMilliseconds(1)); + Assert.Equal(1, snapshotService.Subscribes.Count); + + clock.Advance(TimeSpan.FromMilliseconds(1)); + await snapshotService.Subscribes.WaitForAsync(2); + await hubContext.Proxy.Sends.WaitForAsync(1); + + await cts.CancelAsync(); + await publisher.StopAsync(CancellationToken.None); DateTimeOffset firstThrowAt = snapshotService.FirstThrowAt ?? throw new InvalidOperationException("First subscribe did not record a throw timestamp."); DateTimeOffset secondSubscribeAt = snapshotService.SecondSubscribeAt ?? throw new InvalidOperationException("Second subscribe did not record a timestamp."); - await cts.CancelAsync(); - await publisher.StopAsync(CancellationToken.None); - - Assert.True(snapshotService.SubscribeCount >= 2, - $"Expected at least 2 subscribe calls, got {snapshotService.SubscribeCount}."); - Assert.True(hubContext.SendCount >= 1); - - // The gap is measured from the moment the first subscribe actually - // threw (inside the fake) to the moment the second subscribe began - // (also inside the fake). This isolates the publisher's - // Task.Delay(reconnectDelay) — no StartAsync / scheduling overhead in - // the baseline. The 10ms slack absorbs Task.Delay's coarse Windows - // timer quantum (~15ms) when the underlying scheduler wakes early. - TimeSpan gap = secondSubscribeAt - firstThrowAt; - Assert.True(gap >= reconnectDelay - TimeSpan.FromMilliseconds(10), - $"Expected reconnect gap >= {reconnectDelay.TotalMilliseconds}ms; got {gap.TotalMilliseconds}ms."); + // Fake time is exact: the gap is precisely the reconnect delay. + Assert.Equal(reconnectDelay, secondSubscribeAt - firstThrowAt); } /// /// Sanity: a normal completion of WatchSnapshotsAsync (no exception) - /// also reconnects after the delay — exits only on host shutdown. + /// is re-subscribed; the publisher exits only on host shutdown. /// /// A task that represents the asynchronous operation. [Fact] - public async Task ExecuteAsync_WhenSnapshotServiceCompletes_ReconnectsAfterDelay() + public async Task ExecuteAsync_WhenSnapshotServiceCompletes_Resubscribes() { CompleteImmediatelySnapshotService snapshotService = new(); RecordingHubContext hubContext = new(); - TimeSpan reconnectDelay = TimeSpan.FromMilliseconds(50); DashboardSnapshotPublisher publisher = new( snapshotService, hubContext, ConnectedCounter(), NullLogger.Instance, - reconnectDelay, - IdlePollInterval); + TimeSpan.FromMilliseconds(50), + IdlePollInterval, + timeProvider: new FakeTimeProvider()); using CancellationTokenSource cts = new(); - Task execute = publisher.StartAsync(cts.Token); - await execute.WaitAsync(TestTimeout); + await publisher.StartAsync(cts.Token).WaitAsync(TestTimeout); - await WaitUntilAsync(() => snapshotService.SubscribeCount >= 2); + await snapshotService.Subscribes.WaitForAsync(2); await cts.CancelAsync(); await publisher.StopAsync(CancellationToken.None); - Assert.True(snapshotService.SubscribeCount >= 2); + Assert.True(snapshotService.Subscribes.Count >= 2); } /// /// With no dashboard connected there is nobody to broadcast to, so the publisher must /// not advance the snapshot enumerator at all — every pull costs a registry snapshot and /// sort, a locked metrics dictionary copy, and periodically a SQLite key-table read. - /// The first viewer to connect resumes the tick. + /// The first viewer to connect resumes the tick. Many idle polls are driven on a fake clock. /// /// A task that represents the asynchronous operation. [Fact] public async Task ExecuteAsync_WhenNoHubConnections_DoesNotPullSnapshots() { + TickObservingTimeProvider clock = new() { WatchedDueTime = IdlePollInterval }; CountingSnapshotService snapshotService = new(); RecordingHubContext hubContext = new(); DashboardSnapshotHubConnectionCounter connectionCounter = new(); @@ -118,19 +117,30 @@ public sealed class DashboardSnapshotPublisherTests connectionCounter, NullLogger.Instance, TimeSpan.FromMilliseconds(50), - IdlePollInterval); + IdlePollInterval, + timeProvider: clock); using CancellationTokenSource cts = new(); await publisher.StartAsync(cts.Token).WaitAsync(TestTimeout); - // Long enough for many idle polls at IdlePollInterval. - await Task.Delay(TimeSpan.FromMilliseconds(250)); + // A source that is ready to yield every time it is pulled, so the only thing + // holding the publisher back is the idle gate. + snapshotService.Tick(100); + + // Drive 25 idle polls; each wake-up re-arms exactly one new idle timer. + await clock.WaitForWatchedTimersAsync(1, TestTimeout); + for (int poll = 1; poll <= 25; poll++) + { + clock.Advance(IdlePollInterval); + await clock.WaitForWatchedTimersAsync(poll + 1, TestTimeout); + } Assert.Equal(0, snapshotService.PullCount); - Assert.Equal(0, hubContext.SendCount); + Assert.Equal(0, hubContext.Proxy.Sends.Count); connectionCounter.Increment(); - await WaitUntilAsync(() => hubContext.SendCount >= 1); + clock.Advance(IdlePollInterval); + await hubContext.Proxy.Sends.WaitForAsync(1); await cts.CancelAsync(); await publisher.StopAsync(CancellationToken.None); @@ -141,12 +151,14 @@ public sealed class DashboardSnapshotPublisherTests /// /// A broadcast that never completes — one remote client whose transport is wedged — /// is abandoned at the send timeout, so later ticks still go out instead of every - /// viewer waiting on the slowest one. + /// viewer waiting on the slowest one. The send timeout runs on the fake clock. /// /// A task that represents the asynchronous operation. [Fact] public async Task ExecuteAsync_WhenABroadcastNeverCompletes_LaterTicksStillGoOut() { + TimeSpan sendTimeout = TimeSpan.FromMilliseconds(50); + TickObservingTimeProvider clock = new() { WatchedDueTime = sendTimeout }; CountingSnapshotService snapshotService = new(); RecordingHubContext hubContext = new(); hubContext.Proxy.StallFirstSend = true; @@ -155,13 +167,23 @@ public sealed class DashboardSnapshotPublisherTests hubContext, ConnectedCounter(), NullLogger.Instance, - TimeSpan.FromMilliseconds(50), + TimeSpan.FromSeconds(5), IdlePollInterval, - sendTimeout: TimeSpan.FromMilliseconds(50)); + sendTimeout: sendTimeout, + timeProvider: clock); + snapshotService.Tick(3); using CancellationTokenSource cts = new(); await publisher.StartAsync(cts.Token).WaitAsync(TestTimeout); - await WaitUntilAsync(() => hubContext.SendCount >= 3); + + // The first send is wedged and its timeout timer is armed; nothing else goes out + // until the fake clock reaches the timeout. + await hubContext.Proxy.Sends.WaitForAsync(1); + await clock.WaitForWatchedTimersAsync(1, TestTimeout); + Assert.Equal(1, hubContext.Proxy.Sends.Count); + + clock.Advance(sendTimeout); + await hubContext.Proxy.Sends.WaitForAsync(3); await cts.CancelAsync(); await publisher.StopAsync(CancellationToken.None); @@ -176,29 +198,15 @@ public sealed class DashboardSnapshotPublisherTests return counter; } - private static async Task WaitUntilAsync(Func predicate) + private sealed class ThrowOnceThenYieldSnapshotService(TimeProvider clock) : IDashboardSnapshotService { - using CancellationTokenSource cancellationTokenSource = new(TestTimeout); - while (!predicate()) - { - await Task.Delay(TimeSpan.FromMilliseconds(5), cancellationTokenSource.Token); - } - } + /// Gets the subscription attempts. + public CountSignal Subscribes { get; } = new(); - private sealed class ThrowOnceThenYieldSnapshotService : IDashboardSnapshotService - { - /// Gets the number of subscription attempts. - public int SubscribeCount { get; private set; } - - /// - /// The wall-clock instant the first WatchSnapshotsAsync throws. - /// The reconnect-gap assertion is measured against this timestamp (NOT the - /// pre-StartAsync wall clock) so scheduling overhead is not baselined - /// into the lower bound. - /// + /// Gets the clock instant the first WatchSnapshotsAsync throws. public DateTimeOffset? FirstThrowAt { get; private set; } - /// Gets the wall-clock instant of the second subscription attempt. + /// Gets the clock instant of the second subscription attempt. public DateTimeOffset? SecondSubscribeAt { get; private set; } /// @@ -211,19 +219,18 @@ public sealed class DashboardSnapshotPublisherTests public async IAsyncEnumerable WatchSnapshotsAsync( [EnumeratorCancellation] CancellationToken cancellationToken) { - SubscribeCount++; - int call = SubscribeCount; - - if (call == 1) + if (Subscribes.Count == 0) { // First call: throw after a brief yield so the publisher // observes us as a live producer that failed. await Task.Yield(); - FirstThrowAt = DateTimeOffset.UtcNow; + FirstThrowAt = clock.GetUtcNow(); + Subscribes.Signal(); throw new InvalidOperationException("simulated transient snapshot failure"); } - SecondSubscribeAt = DateTimeOffset.UtcNow; + SecondSubscribeAt = clock.GetUtcNow(); + Subscribes.Signal(); yield return GetSnapshot(); // Stay open until cancelled so the publisher's inner await @@ -240,8 +247,8 @@ public sealed class DashboardSnapshotPublisherTests private sealed class CompleteImmediatelySnapshotService : IDashboardSnapshotService { - /// Gets the number of subscription attempts. - public int SubscribeCount { get; private set; } + /// Gets the subscription attempts. + public CountSignal Subscribes { get; } = new(); /// public DashboardSnapshot GetSnapshot() @@ -255,7 +262,7 @@ public sealed class DashboardSnapshotPublisherTests [EnumeratorCancellation] CancellationToken cancellationToken) #pragma warning restore CS1998 { - SubscribeCount++; + Subscribes.Signal(); // Yield nothing and complete immediately — simulates a transient // upstream disconnect that completes cleanly. yield break; @@ -264,11 +271,16 @@ public sealed class DashboardSnapshotPublisherTests private sealed class CountingSnapshotService : IDashboardSnapshotService { + private readonly SemaphoreSlim _ticks = new(0); private int _pullCount; /// Gets the number of snapshots the publisher pulled from this source. public int PullCount => Volatile.Read(ref _pullCount); + /// Lets the source yield more snapshots. + /// Number of snapshots to allow. + public void Tick(int count) => _ticks.Release(count); + /// public DashboardSnapshot GetSnapshot() { @@ -281,9 +293,9 @@ public sealed class DashboardSnapshotPublisherTests { while (!cancellationToken.IsCancellationRequested) { - // Short cadence so a listening publisher pulls quickly; the counter only - // moves when the publisher actually advances the enumerator. - await Task.Delay(TimeSpan.FromMilliseconds(5), cancellationToken).ConfigureAwait(false); + // The counter only moves when the publisher actually advances the enumerator + // and a tick is available. + await _ticks.WaitAsync(cancellationToken).ConfigureAwait(false); Interlocked.Increment(ref _pullCount); yield return GetSnapshot(); } @@ -303,8 +315,6 @@ public sealed class DashboardSnapshotPublisherTests /// Gets the recording proxy every client lookup resolves to. public RecordingClientProxy Proxy => _clients.AllProxy; - /// Gets the number of send calls recorded. - public int SendCount => _clients.AllProxy.SendCount; } private sealed class RecordingHubClients : IHubClients @@ -359,16 +369,14 @@ public sealed class DashboardSnapshotPublisherTests private sealed class RecordingClientProxy : IClientProxy { - private int _sendCount; - /// /// Gets or sets a value indicating whether the first send never completes and /// ignores its cancellation token — a wedged client transport. /// public bool StallFirstSend { get; set; } - /// Gets the number of send calls recorded. - public int SendCount => Volatile.Read(ref _sendCount); + /// Gets the recorded send calls. + public CountSignal Sends { get; } = new(); /// Records a send call and completes asynchronously. /// The SignalR method name. @@ -377,8 +385,8 @@ public sealed class DashboardSnapshotPublisherTests /// A task that represents the asynchronous operation. public Task SendCoreAsync(string method, object?[] args, CancellationToken cancellationToken = default) { - int count = Interlocked.Increment(ref _sendCount); - return StallFirstSend && count == 1 ? new TaskCompletionSource().Task : Task.CompletedTask; + Sends.Signal(); + return StallFirstSend && Sends.Count == 1 ? new TaskCompletionSource().Task : Task.CompletedTask; } } diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Sessions/GatewaySessionWorkerExitTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Sessions/GatewaySessionWorkerExitTests.cs index cb1764ef..9339e1ed 100644 --- a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Sessions/GatewaySessionWorkerExitTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Sessions/GatewaySessionWorkerExitTests.cs @@ -5,6 +5,7 @@ using ZB.MOM.WW.MxGateway.Server.Metrics; using ZB.MOM.WW.MxGateway.Server.Sessions; using ZB.MOM.WW.MxGateway.Server.Workers; using ZB.MOM.WW.MxGateway.Tests.Gateway.Workers.Fakes; +using ZB.MOM.WW.MxGateway.Tests.TestSupport; namespace ZB.MOM.WW.MxGateway.Tests.Gateway.Sessions; @@ -198,19 +199,8 @@ public sealed class GatewaySessionWorkerExitTests }; } - private static async Task WaitUntilAsync(Func condition) - { - DateTime deadline = DateTime.UtcNow + TestTimeout; - while (!condition()) - { - if (DateTime.UtcNow > deadline) - { - Assert.Fail("Condition was not met within the test timeout."); - } - - await Task.Delay(10); - } - } + private static Task WaitUntilAsync(Func condition) + => Eventually.WaitUntilAsync(condition, TestTimeout); private sealed class SingleClientFactory(IWorkerClient client) : ISessionWorkerClientFactory { diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Sessions/SessionEventDistributorTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Sessions/SessionEventDistributorTests.cs index a42b8682..4f828c2a 100644 --- a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Sessions/SessionEventDistributorTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Sessions/SessionEventDistributorTests.cs @@ -4,6 +4,7 @@ using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Time.Testing; using ZB.MOM.WW.MxGateway.Contracts.Proto; using ZB.MOM.WW.MxGateway.Server.Sessions; +using ZB.MOM.WW.MxGateway.Tests.TestSupport; namespace ZB.MOM.WW.MxGateway.Tests.Gateway.Sessions; @@ -743,14 +744,7 @@ public sealed class SessionEventDistributorTests Assert.Equal(SessionManagerErrorCode.EventQueueOverflow, fault.ErrorCode); // Wait for the handler to fire (it runs on the pump thread). - await Task.Run(async () => - { - using CancellationTokenSource cts = new(ReadTimeout); - while (Volatile.Read(ref observedIsOnlySubscriberSet) == 0) - { - await Task.Delay(10, cts.Token); - } - }); + await Eventually.WaitUntilAsync(() => Volatile.Read(ref observedIsOnlySubscriberSet) != 0, ReadTimeout); // isOnlySubscriber must be FALSE even though the internal subscriber was the ONLY // subscriber — CountExternalSubscribers excludes it, so a FailFast policy on the @@ -802,14 +796,7 @@ public sealed class SessionEventDistributorTests async () => await DrainUntilFaultAsync(external.Reader)); Assert.Equal(SessionManagerErrorCode.EventQueueOverflow, fault.ErrorCode); - await Task.Run(async () => - { - using CancellationTokenSource cts = new(ReadTimeout); - while (Volatile.Read(ref observedSet) == 0) - { - await Task.Delay(10, cts.Token); - } - }); + await Eventually.WaitUntilAsync(() => Volatile.Read(ref observedSet) != 0, ReadTimeout); // Guard: ensure the handler actually fired before asserting its observed value. // Without this the test could pass vacuously if the overflow never triggered. diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Workers/FakeWorkerHarnessTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Workers/FakeWorkerHarnessTests.cs index 44f3644f..5fac3dc3 100644 --- a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Workers/FakeWorkerHarnessTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Workers/FakeWorkerHarnessTests.cs @@ -358,15 +358,9 @@ public sealed class FakeWorkerHarnessTests return new WorkerCommand { Command = command }; } - private static async Task WaitUntilAsync( + private static Task WaitUntilAsync( Func predicate, TimeSpan timeout) - { - using CancellationTokenSource cancellationTokenSource = new(timeout); - while (!predicate()) - { - await Task.Delay(TimeSpan.FromMilliseconds(10), cancellationTokenSource.Token); - } - } + => Eventually.WaitUntilAsync(predicate, timeout); } diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Workers/WorkerClientTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Workers/WorkerClientTests.cs index 2262b00f..40682d17 100644 --- a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Workers/WorkerClientTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Workers/WorkerClientTests.cs @@ -1522,16 +1522,10 @@ public sealed class WorkerClientTests } } - private static async Task WaitUntilAsync( + private static Task WaitUntilAsync( Func predicate, TimeSpan timeout) - { - using CancellationTokenSource cancellationTokenSource = new(timeout); - while (!predicate()) - { - await Task.Delay(TimeSpan.FromMilliseconds(10), cancellationTokenSource.Token); - } - } + => Eventually.WaitUntilAsync(predicate, timeout); private sealed class PipePair : IAsyncDisposable { diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Security/Authentication/CachingApiKeyVerifierTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Security/Authentication/CachingApiKeyVerifierTests.cs index 7278d642..d0d9d513 100644 --- a/src/ZB.MOM.WW.MxGateway.Tests/Security/Authentication/CachingApiKeyVerifierTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Tests/Security/Authentication/CachingApiKeyVerifierTests.cs @@ -4,6 +4,7 @@ using ZB.MOM.WW.Auth.Abstractions.ApiKeys; using ZB.MOM.WW.MxGateway.Server.Security.Authentication; using LibApiKeyIdentity = ZB.MOM.WW.Auth.Abstractions.ApiKeys.ApiKeyIdentity; +using ZB.MOM.WW.MxGateway.Tests.TestSupport; namespace ZB.MOM.WW.MxGateway.Tests.Security.Authentication; @@ -267,11 +268,7 @@ public sealed class CachingApiKeyVerifierTests cache.Compact(1.0); - DateTime deadline = DateTime.UtcNow + TimeSpan.FromSeconds(5); - while (verifier.IndexedCacheKeyCount != 0 && DateTime.UtcNow < deadline) - { - await Task.Delay(10); - } + await Eventually.WaitUntilAsync(() => verifier.IndexedCacheKeyCount == 0, TimeSpan.FromSeconds(5)); Assert.Equal(0, verifier.IndexedCacheKeyCount); Assert.Equal(0, verifier.IndexedKeyIdCount); diff --git a/src/ZB.MOM.WW.MxGateway.Tests/TestSupport/CountSignal.cs b/src/ZB.MOM.WW.MxGateway.Tests/TestSupport/CountSignal.cs new file mode 100644 index 00000000..ff7fa6ef --- /dev/null +++ b/src/ZB.MOM.WW.MxGateway.Tests/TestSupport/CountSignal.cs @@ -0,0 +1,36 @@ +namespace ZB.MOM.WW.MxGateway.Tests.TestSupport; + +/// +/// Counts events and lets a test wait for the nth one without polling. The timed wait result +/// is asserted, so a missed event fails the test instead of silently continuing (mxaccessgw#221). +/// +public sealed class CountSignal +{ + private static readonly TimeSpan WaitTimeout = TimeSpan.FromSeconds(5); + + private readonly SemaphoreSlim _signal = new(0); + private int _count; + private int _consumed; + + /// Gets the number of events signalled so far. + public int Count => Volatile.Read(ref _count); + + /// Records one event. + public void Signal() + { + Interlocked.Increment(ref _count); + _signal.Release(); + } + + /// Waits until at least events have been signalled; fails on timeout. + /// Cumulative event count to wait for. + /// A task that represents the asynchronous operation. + public async Task WaitForAsync(int total) + { + while (_consumed < total) + { + Assert.True(await _signal.WaitAsync(WaitTimeout), $"Timed out waiting for event {total} (saw {Count})."); + _consumed++; + } + } +} diff --git a/src/ZB.MOM.WW.MxGateway.Tests/TestSupport/Eventually.cs b/src/ZB.MOM.WW.MxGateway.Tests/TestSupport/Eventually.cs new file mode 100644 index 00000000..788d0f3b --- /dev/null +++ b/src/ZB.MOM.WW.MxGateway.Tests/TestSupport/Eventually.cs @@ -0,0 +1,46 @@ +namespace ZB.MOM.WW.MxGateway.Tests.TestSupport; + +/// +/// Bounded positive-wait helpers for conditions that no production type signals (for example +/// a monitor's current-alarm count). The wait fails with when the +/// condition never holds; use a signal (, a TCS) where one exists. +/// +public static class Eventually +{ + private static readonly TimeSpan PollInterval = TimeSpan.FromMilliseconds(2); + + /// Waits until holds. + /// Condition to poll. + /// Real-time bound on the wait. + /// A task that represents the asynchronous operation. + /// The condition did not hold within the bound. + public static async Task WaitUntilAsync(Func condition, TimeSpan timeout) + { + using CancellationTokenSource cts = new(timeout); + while (!condition()) + { + try + { + await Task.Delay(PollInterval, cts.Token).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + throw new TimeoutException("Condition was not met in time."); + } + } + } + + /// Waits until returns a non-null value and returns it. + /// Reference type produced by the probe. + /// Probe to poll. + /// Real-time bound on the wait. + /// The first non-null probe result. + /// The probe never returned a value within the bound. + public static async Task WaitForAsync(Func probe, TimeSpan timeout) + where T : class + { + T? result = null; + await WaitUntilAsync(() => (result = probe()) is not null, timeout).ConfigureAwait(false); + return result!; + } +}