diff --git a/src/ZB.MOM.WW.MxGateway.Server/Dashboard/DashboardSnapshotFeed.cs b/src/ZB.MOM.WW.MxGateway.Server/Dashboard/DashboardSnapshotFeed.cs index 301104fb..38d5398d 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Dashboard/DashboardSnapshotFeed.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Dashboard/DashboardSnapshotFeed.cs @@ -53,6 +53,12 @@ public sealed class DashboardSnapshotFeed : IDashboardSnapshotFeed /// Last generation handed out; only ever incremented under _gate. private long _lastGeneration = NoGeneration; + /// + /// Completed (and cleared) the next time the live generation stops accepting subscribers; + /// created lazily by , so production never allocates it. + /// + private TaskCompletionSource? _idleGateClosed; + /// Initializes a new instance of the class. /// Snapshot source to multicast. /// Optional logger for pump faults. @@ -64,6 +70,44 @@ public sealed class DashboardSnapshotFeed : IDashboardSnapshotFeed _logger = logger ?? NullLogger.Instance; } + /// + /// Gets a value indicating whether the feed is fully idle: no generation is accepting + /// subscribers and the most recent pump has exited, so nothing is enumerating the + /// snapshot source. Test seam for the idle gate. + /// + internal bool IsPumpIdle + { + get + { + lock (_gate) + { + return _pumpCancellation is null && _pumpTask.IsCompleted; + } + } + } + + /// + /// Completes once no generation is accepting subscribers — the idle gate has closed, either + /// because the last subscriber of the live generation left or because its source ended. The + /// pump that served the generation may still be unwinding; see . + /// Test seam: it lets a test wait for an unsubscribe it cannot await directly. + /// + /// Bounds the wait. + /// A task that completes when no generation is live. + internal Task WhenIdleGateClosedAsync(CancellationToken cancellationToken) + { + lock (_gate) + { + if (_pumpCancellation is null) + { + return Task.CompletedTask; + } + + _idleGateClosed ??= new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + return _idleGateClosed.Task.WaitAsync(cancellationToken); + } + } + /// public async IAsyncEnumerable WatchAsync( [EnumeratorCancellation] CancellationToken cancellationToken) @@ -140,8 +184,7 @@ public sealed class DashboardSnapshotFeed : IDashboardSnapshotFeed } cancellation = _pumpCancellation; - _pumpCancellation = null; - _generation = NoGeneration; + CloseIdleGateLocked(); pump = _pumpTask; } @@ -311,8 +354,19 @@ public sealed class DashboardSnapshotFeed : IDashboardSnapshotFeed return; } + CloseIdleGateLocked(); + } + + /// + /// Stops the live generation from accepting subscribers and wakes any + /// waiter. Must be called while holding _gate. + /// + private void CloseIdleGateLocked() + { _generation = NoGeneration; _pumpCancellation = null; + _idleGateClosed?.TrySetResult(); + _idleGateClosed = null; } /// diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Dashboard/AlarmsPageTruncationBannerTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Dashboard/AlarmsPageTruncationBannerTests.cs index d14339b7..92877454 100644 --- a/src/ZB.MOM.WW.MxGateway.Tests/Dashboard/AlarmsPageTruncationBannerTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Tests/Dashboard/AlarmsPageTruncationBannerTests.cs @@ -1,4 +1,3 @@ -using System.Diagnostics; using System.Runtime.CompilerServices; using System.Threading.Channels; using Microsoft.AspNetCore.Components.Web.HtmlRendering; @@ -199,25 +198,15 @@ public sealed class AlarmsPageTruncationBannerTests return renderer.Dispatcher.InvokeAsync(page.ToHtmlString); } - private static async Task WaitForHtmlAsync( + private static Task WaitForHtmlAsync( HtmlRenderer renderer, HtmlRootComponent page, Func predicate, string expectation) - { - Stopwatch elapsed = Stopwatch.StartNew(); - while (elapsed.Elapsed < RenderWaitTimeout) - { - if (predicate(await HtmlAsync(renderer, page))) - { - return; - } - - await Task.Delay(TimeSpan.FromMilliseconds(20)); - } - - Assert.Fail($"Timed out after {RenderWaitTimeout.TotalSeconds:N0}s waiting for the {expectation}."); - } + => Eventually.WaitUntilAsync( + async () => predicate(await HtmlAsync(renderer, page)), + RenderWaitTimeout, + $"the {expectation}"); private static async Task RenderAsync(bool snapshotTruncated) { diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Dashboard/DashboardEventBroadcasterTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Dashboard/DashboardEventBroadcasterTests.cs index 6b1195f1..3acf5853 100644 --- a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Dashboard/DashboardEventBroadcasterTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Dashboard/DashboardEventBroadcasterTests.cs @@ -4,6 +4,7 @@ using Microsoft.Extensions.Options; using ZB.MOM.WW.MxGateway.Contracts.Proto; using ZB.MOM.WW.MxGateway.Server.Configuration; using ZB.MOM.WW.MxGateway.Server.Dashboard.Hubs; +using ZB.MOM.WW.MxGateway.Tests.TestSupport; namespace ZB.MOM.WW.MxGateway.Tests.Gateway.Dashboard; @@ -448,19 +449,8 @@ public sealed class DashboardEventBroadcasterTests () => LastArgument is MxEvent sent && sent.ItemHandle == itemHandle, $"a send of item handle {itemHandle}"); - private static async Task WaitUntilAsync(Func condition, string description) - { - DateTime deadline = DateTime.UtcNow.AddSeconds(10); - while (!condition()) - { - if (DateTime.UtcNow > deadline) - { - Assert.Fail($"Timed out waiting for {description}."); - } - - await Task.Delay(5); - } - } + private static Task WaitUntilAsync(Func condition, string description) + => Eventually.WaitUntilAsync(condition, TimeSpan.FromSeconds(10), description); } private sealed class CapturingHubClients : IHubClients diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Dashboard/DashboardSnapshotFeedTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Dashboard/DashboardSnapshotFeedTests.cs index 1c590e12..72904ce2 100644 --- a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Dashboard/DashboardSnapshotFeedTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Dashboard/DashboardSnapshotFeedTests.cs @@ -287,11 +287,12 @@ public sealed class DashboardSnapshotFeedTests Task joinerMove = joiner.MoveNextAsync().AsTask(); await joinerCancellation.CancelAsync(); - // The joiner's unwind is a continuation of its cancelled channel read; give it time to - // run its unsubscribe before the dying pump is released. The unsubscribe cannot be - // awaited (it waits on the parked pump) and exposes no signal, so this stays a real - // delay; only the interleaving depends on it — the assertions below hold either way. - await Task.Delay(TimeSpan.FromMilliseconds(150)); + // The joiner's unwind is a continuation of its cancelled channel read, and its unsubscribe + // cannot be awaited directly (it then waits on the parked pump). Wait instead for the idle + // gate it closes: the joiner started the live generation, so the gate closes only once its + // unsubscribe has cancelled that generation. A global "is the list empty" check — the + // defect under test — never closes it, because the old subscriber still lingers. + await feed.WhenIdleGateClosedAsync(CancellationToken.None).WaitAsync(TestTimeout); service.ReleaseDisposal(); @@ -301,10 +302,12 @@ public sealed class DashboardSnapshotFeedTests // Completes only once the joiner's unsubscribe has awaited its generation's pump. await DrainAsync(joiner, joinerMove); - // Nobody is watching, so nothing may consume a snapshot: a surviving pump would drain - // this push within its first read. + // Nobody is watching, so no pump may be left enumerating the source... + Assert.True(feed.IsPumpIdle); + + // ...and so nothing consumes a snapshot pushed now. With no pump running the push stays + // queued, so the count is checked without a settling delay. service.Push(CreateSnapshot("orphan-check")); - await Task.Delay(TimeSpan.FromMilliseconds(150)); Assert.Equal(1, service.PendingPushCount); // ...and the next viewer still starts cleanly, picking up the queued snapshot. diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Grpc/EventStreamServiceTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Grpc/EventStreamServiceTests.cs index 85b305cb..bda97b3e 100644 --- a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Grpc/EventStreamServiceTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Grpc/EventStreamServiceTests.cs @@ -9,6 +9,7 @@ using ZB.MOM.WW.MxGateway.Server.Grpc; using ZB.MOM.WW.MxGateway.Server.Metrics; using ZB.MOM.WW.MxGateway.Server.Sessions; using ZB.MOM.WW.MxGateway.Server.Workers; +using ZB.MOM.WW.MxGateway.Tests.TestSupport; namespace ZB.MOM.WW.MxGateway.Tests.Gateway.Grpc; @@ -892,27 +893,13 @@ public sealed class EventStreamServiceTests } // The real-clock deadline here is load-sensitive on a wide host (windev runs this suite - // 36-way parallel). Surfacing the unmet condition instead of letting the bare - // TaskCanceledException escape is what makes such a failure diagnosable rather than a + // 36-way parallel). Eventually names the unmet condition instead of letting a bare + // TaskCanceledException escape, which is what makes such a failure diagnosable rather than a // mystery cancellation attributed to "the environment". - private static async Task WaitUntilAsync( + private static Task WaitUntilAsync( Func predicate, [CallerArgumentExpression(nameof(predicate))] string? predicateExpression = null) - { - using CancellationTokenSource cancellationTokenSource = new(TestTimeout); - while (!predicate()) - { - try - { - await Task.Delay(TimeSpan.FromMilliseconds(10), cancellationTokenSource.Token); - } - catch (OperationCanceledException) - { - Assert.Fail( - $"Timed out after {TestTimeout} waiting for condition: {predicateExpression}"); - } - } - } + => Eventually.WaitUntilAsync(predicate, TestTimeout, predicateExpression); /// Fake session manager for testing event streams. private sealed class FakeSessionManager : ISessionManager diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Sessions/GatewaySessionDashboardMirrorTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Sessions/GatewaySessionDashboardMirrorTests.cs index 1d549330..526db388 100644 --- a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Sessions/GatewaySessionDashboardMirrorTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Sessions/GatewaySessionDashboardMirrorTests.cs @@ -294,21 +294,8 @@ public sealed class GatewaySessionDashboardMirrorTests return new WorkerEvent { Event = mxEvent }; } - private static async Task WaitUntilAsync(Func predicate, [CallerArgumentExpression(nameof(predicate))] string? condition = null) - { - using CancellationTokenSource cancellationTokenSource = new(TestTimeout); - try - { - while (!predicate()) - { - await Task.Delay(TimeSpan.FromMilliseconds(10), cancellationTokenSource.Token); - } - } - catch (OperationCanceledException) - { - Assert.Fail($"Timed out after {TestTimeout.TotalSeconds}s waiting for: {condition}"); - } - } + private static Task WaitUntilAsync(Func predicate, [CallerArgumentExpression(nameof(predicate))] string? condition = null) + => Eventually.WaitUntilAsync(predicate, TestTimeout, condition); private sealed class ThrowingDashboardEventBroadcaster : IDashboardEventBroadcaster { diff --git a/src/ZB.MOM.WW.MxGateway.Tests/TestSupport/Eventually.cs b/src/ZB.MOM.WW.MxGateway.Tests/TestSupport/Eventually.cs index 788d0f3b..f04f3d57 100644 --- a/src/ZB.MOM.WW.MxGateway.Tests/TestSupport/Eventually.cs +++ b/src/ZB.MOM.WW.MxGateway.Tests/TestSupport/Eventually.cs @@ -1,3 +1,5 @@ +using System.Runtime.CompilerServices; + namespace ZB.MOM.WW.MxGateway.Tests.TestSupport; /// @@ -12,12 +14,38 @@ public static class Eventually /// Waits until holds. /// Condition to poll. /// Real-time bound on the wait. + /// + /// What is being waited for, named in the timeout message. Defaults to the source text of + /// , so an unmet wait says which condition it was. + /// /// A task that represents the asynchronous operation. /// The condition did not hold within the bound. - public static async Task WaitUntilAsync(Func condition, TimeSpan timeout) + public static Task WaitUntilAsync( + Func condition, + TimeSpan timeout, + [CallerArgumentExpression(nameof(condition))] string? description = null) { + ArgumentNullException.ThrowIfNull(condition); + return WaitUntilAsync(() => Task.FromResult(condition()), timeout, description); + } + + /// Waits until the asynchronous holds. + /// Asynchronous condition to poll; each probe is awaited before the next. + /// Real-time bound on the wait. + /// + /// What is being waited for, named in the timeout message. Defaults to the source text of + /// . + /// + /// A task that represents the asynchronous operation. + /// The condition did not hold within the bound. + public static async Task WaitUntilAsync( + Func> condition, + TimeSpan timeout, + [CallerArgumentExpression(nameof(condition))] string? description = null) + { + ArgumentNullException.ThrowIfNull(condition); using CancellationTokenSource cts = new(timeout); - while (!condition()) + while (!await condition().ConfigureAwait(false)) { try { @@ -25,7 +53,8 @@ public static class Eventually } catch (OperationCanceledException) { - throw new TimeoutException("Condition was not met in time."); + throw new TimeoutException( + $"Condition was not met within {timeout.TotalSeconds:0.###}s: {description}"); } } } @@ -34,13 +63,20 @@ public static class Eventually /// Reference type produced by the probe. /// Probe to poll. /// Real-time bound on the wait. + /// + /// What is being waited for, named in the timeout message. Defaults to the source text of + /// . + /// /// The first non-null probe result. /// The probe never returned a value within the bound. - public static async Task WaitForAsync(Func probe, TimeSpan timeout) + public static async Task WaitForAsync( + Func probe, + TimeSpan timeout, + [CallerArgumentExpression(nameof(probe))] string? description = null) where T : class { T? result = null; - await WaitUntilAsync(() => (result = probe()) is not null, timeout).ConfigureAwait(false); + await WaitUntilAsync(() => (result = probe()) is not null, timeout, description).ConfigureAwait(false); return result!; } }