test: pump-idle signal for DashboardSnapshotFeed; fold private wait helpers into Eventually (mxaccessgw#223)

The MxGateway.Tests part of #223 (the Worker.Tests items are handled separately).

- DashboardSnapshotFeed gains two internal test seams: WhenIdleGateClosedAsync, a
  TCS-backed signal that completes once no pump generation is accepting subscribers,
  and IsPumpIdle (no live generation and the latest pump has exited). The TCS is created
  lazily by a waiter, so production never allocates it.
- WatchAsync_WhenAJoinerLeavesWhileAnOldGenerationLingers_LeavesNoPumpRunning drops both
  150 ms real sleeps: it waits on the idle-gate signal for the joiner's unsubscribe, and
  asserts IsPumpIdle before the orphan-push check, which is then immediate. Reverting the
  per-generation idle check to the old global "list is empty" check now fails the test
  (5 s TimeoutException on the gate), where before the sleep simply elapsed.
- Eventually.WaitUntilAsync/WaitForAsync take an optional description (defaulting to the
  condition's source text via CallerArgumentExpression) for the timeout message, and gain an
  async-condition overload. The hand-rolled polling loops in EventStreamServiceTests,
  GatewaySessionDashboardMirrorTests, DashboardEventBroadcasterTests and
  AlarmsPageTruncationBannerTests now delegate to it, keeping their timeouts and messages.

Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
Joseph Doherty
2026-11-09 13:26:38 -05:00
parent 22964e192b
commit 09b3ecddf2
7 changed files with 123 additions and 77 deletions
@@ -53,6 +53,12 @@ public sealed class DashboardSnapshotFeed : IDashboardSnapshotFeed
/// <summary>Last generation handed out; only ever incremented under <c>_gate</c>.</summary>
private long _lastGeneration = NoGeneration;
/// <summary>
/// Completed (and cleared) the next time the live generation stops accepting subscribers;
/// created lazily by <see cref="WhenIdleGateClosedAsync"/>, so production never allocates it.
/// </summary>
private TaskCompletionSource? _idleGateClosed;
/// <summary>Initializes a new instance of the <see cref="DashboardSnapshotFeed"/> class.</summary>
/// <param name="snapshotService">Snapshot source to multicast.</param>
/// <param name="logger">Optional logger for pump faults.</param>
@@ -64,6 +70,44 @@ public sealed class DashboardSnapshotFeed : IDashboardSnapshotFeed
_logger = logger ?? NullLogger<DashboardSnapshotFeed>.Instance;
}
/// <summary>
/// Gets a value indicating whether the feed is fully idle: no generation is accepting
/// subscribers and the most recent pump has exited, so nothing is enumerating the
/// snapshot source. Test seam for the idle gate.
/// </summary>
internal bool IsPumpIdle
{
get
{
lock (_gate)
{
return _pumpCancellation is null && _pumpTask.IsCompleted;
}
}
}
/// <summary>
/// Completes once no generation is accepting subscribers — the idle gate has closed, either
/// because the last subscriber of the live generation left or because its source ended. The
/// pump that served the generation may still be unwinding; see <see cref="IsPumpIdle"/>.
/// Test seam: it lets a test wait for an unsubscribe it cannot await directly.
/// </summary>
/// <param name="cancellationToken">Bounds the wait.</param>
/// <returns>A task that completes when no generation is live.</returns>
internal Task WhenIdleGateClosedAsync(CancellationToken cancellationToken)
{
lock (_gate)
{
if (_pumpCancellation is null)
{
return Task.CompletedTask;
}
_idleGateClosed ??= new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
return _idleGateClosed.Task.WaitAsync(cancellationToken);
}
}
/// <inheritdoc />
public async IAsyncEnumerable<DashboardSnapshot> WatchAsync(
[EnumeratorCancellation] CancellationToken cancellationToken)
@@ -140,8 +184,7 @@ public sealed class DashboardSnapshotFeed : IDashboardSnapshotFeed
}
cancellation = _pumpCancellation;
_pumpCancellation = null;
_generation = NoGeneration;
CloseIdleGateLocked();
pump = _pumpTask;
}
@@ -311,8 +354,19 @@ public sealed class DashboardSnapshotFeed : IDashboardSnapshotFeed
return;
}
CloseIdleGateLocked();
}
/// <summary>
/// Stops the live generation from accepting subscribers and wakes any
/// <see cref="WhenIdleGateClosedAsync"/> waiter. Must be called while holding <c>_gate</c>.
/// </summary>
private void CloseIdleGateLocked()
{
_generation = NoGeneration;
_pumpCancellation = null;
_idleGateClosed?.TrySetResult();
_idleGateClosed = null;
}
/// <summary>
@@ -1,4 +1,3 @@
using System.Diagnostics;
using System.Runtime.CompilerServices;
using System.Threading.Channels;
using Microsoft.AspNetCore.Components.Web.HtmlRendering;
@@ -199,25 +198,15 @@ public sealed class AlarmsPageTruncationBannerTests
return renderer.Dispatcher.InvokeAsync(page.ToHtmlString);
}
private static async Task WaitForHtmlAsync(
private static Task WaitForHtmlAsync(
HtmlRenderer renderer,
HtmlRootComponent page,
Func<string, bool> predicate,
string expectation)
{
Stopwatch elapsed = Stopwatch.StartNew();
while (elapsed.Elapsed < RenderWaitTimeout)
{
if (predicate(await HtmlAsync(renderer, page)))
{
return;
}
await Task.Delay(TimeSpan.FromMilliseconds(20));
}
Assert.Fail($"Timed out after {RenderWaitTimeout.TotalSeconds:N0}s waiting for the {expectation}.");
}
=> Eventually.WaitUntilAsync(
async () => predicate(await HtmlAsync(renderer, page)),
RenderWaitTimeout,
$"the {expectation}");
private static async Task<string> RenderAsync(bool snapshotTruncated)
{
@@ -4,6 +4,7 @@ using Microsoft.Extensions.Options;
using ZB.MOM.WW.MxGateway.Contracts.Proto;
using ZB.MOM.WW.MxGateway.Server.Configuration;
using ZB.MOM.WW.MxGateway.Server.Dashboard.Hubs;
using ZB.MOM.WW.MxGateway.Tests.TestSupport;
namespace ZB.MOM.WW.MxGateway.Tests.Gateway.Dashboard;
@@ -448,19 +449,8 @@ public sealed class DashboardEventBroadcasterTests
() => LastArgument is MxEvent sent && sent.ItemHandle == itemHandle,
$"a send of item handle {itemHandle}");
private static async Task WaitUntilAsync(Func<bool> condition, string description)
{
DateTime deadline = DateTime.UtcNow.AddSeconds(10);
while (!condition())
{
if (DateTime.UtcNow > deadline)
{
Assert.Fail($"Timed out waiting for {description}.");
}
await Task.Delay(5);
}
}
private static Task WaitUntilAsync(Func<bool> condition, string description)
=> Eventually.WaitUntilAsync(condition, TimeSpan.FromSeconds(10), description);
}
private sealed class CapturingHubClients : IHubClients
@@ -287,11 +287,12 @@ public sealed class DashboardSnapshotFeedTests
Task<bool> joinerMove = joiner.MoveNextAsync().AsTask();
await joinerCancellation.CancelAsync();
// The joiner's unwind is a continuation of its cancelled channel read; give it time to
// run its unsubscribe before the dying pump is released. The unsubscribe cannot be
// awaited (it waits on the parked pump) and exposes no signal, so this stays a real
// delay; only the interleaving depends on it — the assertions below hold either way.
await Task.Delay(TimeSpan.FromMilliseconds(150));
// The joiner's unwind is a continuation of its cancelled channel read, and its unsubscribe
// cannot be awaited directly (it then waits on the parked pump). Wait instead for the idle
// gate it closes: the joiner started the live generation, so the gate closes only once its
// unsubscribe has cancelled that generation. A global "is the list empty" check — the
// defect under test — never closes it, because the old subscriber still lingers.
await feed.WhenIdleGateClosedAsync(CancellationToken.None).WaitAsync(TestTimeout);
service.ReleaseDisposal();
@@ -301,10 +302,12 @@ public sealed class DashboardSnapshotFeedTests
// Completes only once the joiner's unsubscribe has awaited its generation's pump.
await DrainAsync(joiner, joinerMove);
// Nobody is watching, so nothing may consume a snapshot: a surviving pump would drain
// this push within its first read.
// Nobody is watching, so no pump may be left enumerating the source...
Assert.True(feed.IsPumpIdle);
// ...and so nothing consumes a snapshot pushed now. With no pump running the push stays
// queued, so the count is checked without a settling delay.
service.Push(CreateSnapshot("orphan-check"));
await Task.Delay(TimeSpan.FromMilliseconds(150));
Assert.Equal(1, service.PendingPushCount);
// ...and the next viewer still starts cleanly, picking up the queued snapshot.
@@ -9,6 +9,7 @@ using ZB.MOM.WW.MxGateway.Server.Grpc;
using ZB.MOM.WW.MxGateway.Server.Metrics;
using ZB.MOM.WW.MxGateway.Server.Sessions;
using ZB.MOM.WW.MxGateway.Server.Workers;
using ZB.MOM.WW.MxGateway.Tests.TestSupport;
namespace ZB.MOM.WW.MxGateway.Tests.Gateway.Grpc;
@@ -892,27 +893,13 @@ public sealed class EventStreamServiceTests
}
// The real-clock deadline here is load-sensitive on a wide host (windev runs this suite
// 36-way parallel). Surfacing the unmet condition instead of letting the bare
// TaskCanceledException escape is what makes such a failure diagnosable rather than a
// 36-way parallel). Eventually names the unmet condition instead of letting a bare
// TaskCanceledException escape, which is what makes such a failure diagnosable rather than a
// mystery cancellation attributed to "the environment".
private static async Task WaitUntilAsync(
private static Task WaitUntilAsync(
Func<bool> predicate,
[CallerArgumentExpression(nameof(predicate))] string? predicateExpression = null)
{
using CancellationTokenSource cancellationTokenSource = new(TestTimeout);
while (!predicate())
{
try
{
await Task.Delay(TimeSpan.FromMilliseconds(10), cancellationTokenSource.Token);
}
catch (OperationCanceledException)
{
Assert.Fail(
$"Timed out after {TestTimeout} waiting for condition: {predicateExpression}");
}
}
}
=> Eventually.WaitUntilAsync(predicate, TestTimeout, predicateExpression);
/// <summary>Fake session manager for testing event streams.</summary>
private sealed class FakeSessionManager : ISessionManager
@@ -294,21 +294,8 @@ public sealed class GatewaySessionDashboardMirrorTests
return new WorkerEvent { Event = mxEvent };
}
private static async Task WaitUntilAsync(Func<bool> predicate, [CallerArgumentExpression(nameof(predicate))] string? condition = null)
{
using CancellationTokenSource cancellationTokenSource = new(TestTimeout);
try
{
while (!predicate())
{
await Task.Delay(TimeSpan.FromMilliseconds(10), cancellationTokenSource.Token);
}
}
catch (OperationCanceledException)
{
Assert.Fail($"Timed out after {TestTimeout.TotalSeconds}s waiting for: {condition}");
}
}
private static Task WaitUntilAsync(Func<bool> predicate, [CallerArgumentExpression(nameof(predicate))] string? condition = null)
=> Eventually.WaitUntilAsync(predicate, TestTimeout, condition);
private sealed class ThrowingDashboardEventBroadcaster : IDashboardEventBroadcaster
{
@@ -1,3 +1,5 @@
using System.Runtime.CompilerServices;
namespace ZB.MOM.WW.MxGateway.Tests.TestSupport;
/// <summary>
@@ -12,12 +14,38 @@ public static class Eventually
/// <summary>Waits until <paramref name="condition"/> holds.</summary>
/// <param name="condition">Condition to poll.</param>
/// <param name="timeout">Real-time bound on the wait.</param>
/// <param name="description">
/// What is being waited for, named in the timeout message. Defaults to the source text of
/// <paramref name="condition"/>, so an unmet wait says which condition it was.
/// </param>
/// <returns>A task that represents the asynchronous operation.</returns>
/// <exception cref="TimeoutException">The condition did not hold within the bound.</exception>
public static async Task WaitUntilAsync(Func<bool> condition, TimeSpan timeout)
public static Task WaitUntilAsync(
Func<bool> condition,
TimeSpan timeout,
[CallerArgumentExpression(nameof(condition))] string? description = null)
{
ArgumentNullException.ThrowIfNull(condition);
return WaitUntilAsync(() => Task.FromResult(condition()), timeout, description);
}
/// <summary>Waits until the asynchronous <paramref name="condition"/> holds.</summary>
/// <param name="condition">Asynchronous condition to poll; each probe is awaited before the next.</param>
/// <param name="timeout">Real-time bound on the wait.</param>
/// <param name="description">
/// What is being waited for, named in the timeout message. Defaults to the source text of
/// <paramref name="condition"/>.
/// </param>
/// <returns>A task that represents the asynchronous operation.</returns>
/// <exception cref="TimeoutException">The condition did not hold within the bound.</exception>
public static async Task WaitUntilAsync(
Func<Task<bool>> condition,
TimeSpan timeout,
[CallerArgumentExpression(nameof(condition))] string? description = null)
{
ArgumentNullException.ThrowIfNull(condition);
using CancellationTokenSource cts = new(timeout);
while (!condition())
while (!await condition().ConfigureAwait(false))
{
try
{
@@ -25,7 +53,8 @@ public static class Eventually
}
catch (OperationCanceledException)
{
throw new TimeoutException("Condition was not met in time.");
throw new TimeoutException(
$"Condition was not met within {timeout.TotalSeconds:0.###}s: {description}");
}
}
}
@@ -34,13 +63,20 @@ public static class Eventually
/// <typeparam name="T">Reference type produced by the probe.</typeparam>
/// <param name="probe">Probe to poll.</param>
/// <param name="timeout">Real-time bound on the wait.</param>
/// <param name="description">
/// What is being waited for, named in the timeout message. Defaults to the source text of
/// <paramref name="probe"/>.
/// </param>
/// <returns>The first non-null probe result.</returns>
/// <exception cref="TimeoutException">The probe never returned a value within the bound.</exception>
public static async Task<T> WaitForAsync<T>(Func<T?> probe, TimeSpan timeout)
public static async Task<T> WaitForAsync<T>(
Func<T?> probe,
TimeSpan timeout,
[CallerArgumentExpression(nameof(probe))] string? description = null)
where T : class
{
T? result = null;
await WaitUntilAsync(() => (result = probe()) is not null, timeout).ConfigureAwait(false);
await WaitUntilAsync(() => (result = probe()) is not null, timeout, description).ConfigureAwait(false);
return result!;
}
}