test: pump-idle signal for DashboardSnapshotFeed; fold private wait helpers into Eventually (mxaccessgw#223)
The MxGateway.Tests part of #223 (the Worker.Tests items are handled separately). - DashboardSnapshotFeed gains two internal test seams: WhenIdleGateClosedAsync, a TCS-backed signal that completes once no pump generation is accepting subscribers, and IsPumpIdle (no live generation and the latest pump has exited). The TCS is created lazily by a waiter, so production never allocates it. - WatchAsync_WhenAJoinerLeavesWhileAnOldGenerationLingers_LeavesNoPumpRunning drops both 150 ms real sleeps: it waits on the idle-gate signal for the joiner's unsubscribe, and asserts IsPumpIdle before the orphan-push check, which is then immediate. Reverting the per-generation idle check to the old global "list is empty" check now fails the test (5 s TimeoutException on the gate), where before the sleep simply elapsed. - Eventually.WaitUntilAsync/WaitForAsync take an optional description (defaulting to the condition's source text via CallerArgumentExpression) for the timeout message, and gain an async-condition overload. The hand-rolled polling loops in EventStreamServiceTests, GatewaySessionDashboardMirrorTests, DashboardEventBroadcasterTests and AlarmsPageTruncationBannerTests now delegate to it, keeping their timeouts and messages. Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
@@ -53,6 +53,12 @@ public sealed class DashboardSnapshotFeed : IDashboardSnapshotFeed
|
|||||||
/// <summary>Last generation handed out; only ever incremented under <c>_gate</c>.</summary>
|
/// <summary>Last generation handed out; only ever incremented under <c>_gate</c>.</summary>
|
||||||
private long _lastGeneration = NoGeneration;
|
private long _lastGeneration = NoGeneration;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Completed (and cleared) the next time the live generation stops accepting subscribers;
|
||||||
|
/// created lazily by <see cref="WhenIdleGateClosedAsync"/>, so production never allocates it.
|
||||||
|
/// </summary>
|
||||||
|
private TaskCompletionSource? _idleGateClosed;
|
||||||
|
|
||||||
/// <summary>Initializes a new instance of the <see cref="DashboardSnapshotFeed"/> class.</summary>
|
/// <summary>Initializes a new instance of the <see cref="DashboardSnapshotFeed"/> class.</summary>
|
||||||
/// <param name="snapshotService">Snapshot source to multicast.</param>
|
/// <param name="snapshotService">Snapshot source to multicast.</param>
|
||||||
/// <param name="logger">Optional logger for pump faults.</param>
|
/// <param name="logger">Optional logger for pump faults.</param>
|
||||||
@@ -64,6 +70,44 @@ public sealed class DashboardSnapshotFeed : IDashboardSnapshotFeed
|
|||||||
_logger = logger ?? NullLogger<DashboardSnapshotFeed>.Instance;
|
_logger = logger ?? NullLogger<DashboardSnapshotFeed>.Instance;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets a value indicating whether the feed is fully idle: no generation is accepting
|
||||||
|
/// subscribers and the most recent pump has exited, so nothing is enumerating the
|
||||||
|
/// snapshot source. Test seam for the idle gate.
|
||||||
|
/// </summary>
|
||||||
|
internal bool IsPumpIdle
|
||||||
|
{
|
||||||
|
get
|
||||||
|
{
|
||||||
|
lock (_gate)
|
||||||
|
{
|
||||||
|
return _pumpCancellation is null && _pumpTask.IsCompleted;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Completes once no generation is accepting subscribers — the idle gate has closed, either
|
||||||
|
/// because the last subscriber of the live generation left or because its source ended. The
|
||||||
|
/// pump that served the generation may still be unwinding; see <see cref="IsPumpIdle"/>.
|
||||||
|
/// Test seam: it lets a test wait for an unsubscribe it cannot await directly.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="cancellationToken">Bounds the wait.</param>
|
||||||
|
/// <returns>A task that completes when no generation is live.</returns>
|
||||||
|
internal Task WhenIdleGateClosedAsync(CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
lock (_gate)
|
||||||
|
{
|
||||||
|
if (_pumpCancellation is null)
|
||||||
|
{
|
||||||
|
return Task.CompletedTask;
|
||||||
|
}
|
||||||
|
|
||||||
|
_idleGateClosed ??= new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||||
|
return _idleGateClosed.Task.WaitAsync(cancellationToken);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// <inheritdoc />
|
/// <inheritdoc />
|
||||||
public async IAsyncEnumerable<DashboardSnapshot> WatchAsync(
|
public async IAsyncEnumerable<DashboardSnapshot> WatchAsync(
|
||||||
[EnumeratorCancellation] CancellationToken cancellationToken)
|
[EnumeratorCancellation] CancellationToken cancellationToken)
|
||||||
@@ -140,8 +184,7 @@ public sealed class DashboardSnapshotFeed : IDashboardSnapshotFeed
|
|||||||
}
|
}
|
||||||
|
|
||||||
cancellation = _pumpCancellation;
|
cancellation = _pumpCancellation;
|
||||||
_pumpCancellation = null;
|
CloseIdleGateLocked();
|
||||||
_generation = NoGeneration;
|
|
||||||
pump = _pumpTask;
|
pump = _pumpTask;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -311,8 +354,19 @@ public sealed class DashboardSnapshotFeed : IDashboardSnapshotFeed
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
CloseIdleGateLocked();
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Stops the live generation from accepting subscribers and wakes any
|
||||||
|
/// <see cref="WhenIdleGateClosedAsync"/> waiter. Must be called while holding <c>_gate</c>.
|
||||||
|
/// </summary>
|
||||||
|
private void CloseIdleGateLocked()
|
||||||
|
{
|
||||||
_generation = NoGeneration;
|
_generation = NoGeneration;
|
||||||
_pumpCancellation = null;
|
_pumpCancellation = null;
|
||||||
|
_idleGateClosed?.TrySetResult();
|
||||||
|
_idleGateClosed = null;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
|
|||||||
@@ -1,4 +1,3 @@
|
|||||||
using System.Diagnostics;
|
|
||||||
using System.Runtime.CompilerServices;
|
using System.Runtime.CompilerServices;
|
||||||
using System.Threading.Channels;
|
using System.Threading.Channels;
|
||||||
using Microsoft.AspNetCore.Components.Web.HtmlRendering;
|
using Microsoft.AspNetCore.Components.Web.HtmlRendering;
|
||||||
@@ -199,25 +198,15 @@ public sealed class AlarmsPageTruncationBannerTests
|
|||||||
return renderer.Dispatcher.InvokeAsync(page.ToHtmlString);
|
return renderer.Dispatcher.InvokeAsync(page.ToHtmlString);
|
||||||
}
|
}
|
||||||
|
|
||||||
private static async Task WaitForHtmlAsync(
|
private static Task WaitForHtmlAsync(
|
||||||
HtmlRenderer renderer,
|
HtmlRenderer renderer,
|
||||||
HtmlRootComponent page,
|
HtmlRootComponent page,
|
||||||
Func<string, bool> predicate,
|
Func<string, bool> predicate,
|
||||||
string expectation)
|
string expectation)
|
||||||
{
|
=> Eventually.WaitUntilAsync(
|
||||||
Stopwatch elapsed = Stopwatch.StartNew();
|
async () => predicate(await HtmlAsync(renderer, page)),
|
||||||
while (elapsed.Elapsed < RenderWaitTimeout)
|
RenderWaitTimeout,
|
||||||
{
|
$"the {expectation}");
|
||||||
if (predicate(await HtmlAsync(renderer, page)))
|
|
||||||
{
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
await Task.Delay(TimeSpan.FromMilliseconds(20));
|
|
||||||
}
|
|
||||||
|
|
||||||
Assert.Fail($"Timed out after {RenderWaitTimeout.TotalSeconds:N0}s waiting for the {expectation}.");
|
|
||||||
}
|
|
||||||
|
|
||||||
private static async Task<string> RenderAsync(bool snapshotTruncated)
|
private static async Task<string> RenderAsync(bool snapshotTruncated)
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ using Microsoft.Extensions.Options;
|
|||||||
using ZB.MOM.WW.MxGateway.Contracts.Proto;
|
using ZB.MOM.WW.MxGateway.Contracts.Proto;
|
||||||
using ZB.MOM.WW.MxGateway.Server.Configuration;
|
using ZB.MOM.WW.MxGateway.Server.Configuration;
|
||||||
using ZB.MOM.WW.MxGateway.Server.Dashboard.Hubs;
|
using ZB.MOM.WW.MxGateway.Server.Dashboard.Hubs;
|
||||||
|
using ZB.MOM.WW.MxGateway.Tests.TestSupport;
|
||||||
|
|
||||||
namespace ZB.MOM.WW.MxGateway.Tests.Gateway.Dashboard;
|
namespace ZB.MOM.WW.MxGateway.Tests.Gateway.Dashboard;
|
||||||
|
|
||||||
@@ -448,19 +449,8 @@ public sealed class DashboardEventBroadcasterTests
|
|||||||
() => LastArgument is MxEvent sent && sent.ItemHandle == itemHandle,
|
() => LastArgument is MxEvent sent && sent.ItemHandle == itemHandle,
|
||||||
$"a send of item handle {itemHandle}");
|
$"a send of item handle {itemHandle}");
|
||||||
|
|
||||||
private static async Task WaitUntilAsync(Func<bool> condition, string description)
|
private static Task WaitUntilAsync(Func<bool> condition, string description)
|
||||||
{
|
=> Eventually.WaitUntilAsync(condition, TimeSpan.FromSeconds(10), description);
|
||||||
DateTime deadline = DateTime.UtcNow.AddSeconds(10);
|
|
||||||
while (!condition())
|
|
||||||
{
|
|
||||||
if (DateTime.UtcNow > deadline)
|
|
||||||
{
|
|
||||||
Assert.Fail($"Timed out waiting for {description}.");
|
|
||||||
}
|
|
||||||
|
|
||||||
await Task.Delay(5);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private sealed class CapturingHubClients : IHubClients
|
private sealed class CapturingHubClients : IHubClients
|
||||||
|
|||||||
@@ -287,11 +287,12 @@ public sealed class DashboardSnapshotFeedTests
|
|||||||
Task<bool> joinerMove = joiner.MoveNextAsync().AsTask();
|
Task<bool> joinerMove = joiner.MoveNextAsync().AsTask();
|
||||||
await joinerCancellation.CancelAsync();
|
await joinerCancellation.CancelAsync();
|
||||||
|
|
||||||
// The joiner's unwind is a continuation of its cancelled channel read; give it time to
|
// The joiner's unwind is a continuation of its cancelled channel read, and its unsubscribe
|
||||||
// run its unsubscribe before the dying pump is released. The unsubscribe cannot be
|
// cannot be awaited directly (it then waits on the parked pump). Wait instead for the idle
|
||||||
// awaited (it waits on the parked pump) and exposes no signal, so this stays a real
|
// gate it closes: the joiner started the live generation, so the gate closes only once its
|
||||||
// delay; only the interleaving depends on it — the assertions below hold either way.
|
// unsubscribe has cancelled that generation. A global "is the list empty" check — the
|
||||||
await Task.Delay(TimeSpan.FromMilliseconds(150));
|
// defect under test — never closes it, because the old subscriber still lingers.
|
||||||
|
await feed.WhenIdleGateClosedAsync(CancellationToken.None).WaitAsync(TestTimeout);
|
||||||
|
|
||||||
service.ReleaseDisposal();
|
service.ReleaseDisposal();
|
||||||
|
|
||||||
@@ -301,10 +302,12 @@ public sealed class DashboardSnapshotFeedTests
|
|||||||
// Completes only once the joiner's unsubscribe has awaited its generation's pump.
|
// Completes only once the joiner's unsubscribe has awaited its generation's pump.
|
||||||
await DrainAsync(joiner, joinerMove);
|
await DrainAsync(joiner, joinerMove);
|
||||||
|
|
||||||
// Nobody is watching, so nothing may consume a snapshot: a surviving pump would drain
|
// Nobody is watching, so no pump may be left enumerating the source...
|
||||||
// this push within its first read.
|
Assert.True(feed.IsPumpIdle);
|
||||||
|
|
||||||
|
// ...and so nothing consumes a snapshot pushed now. With no pump running the push stays
|
||||||
|
// queued, so the count is checked without a settling delay.
|
||||||
service.Push(CreateSnapshot("orphan-check"));
|
service.Push(CreateSnapshot("orphan-check"));
|
||||||
await Task.Delay(TimeSpan.FromMilliseconds(150));
|
|
||||||
Assert.Equal(1, service.PendingPushCount);
|
Assert.Equal(1, service.PendingPushCount);
|
||||||
|
|
||||||
// ...and the next viewer still starts cleanly, picking up the queued snapshot.
|
// ...and the next viewer still starts cleanly, picking up the queued snapshot.
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ using ZB.MOM.WW.MxGateway.Server.Grpc;
|
|||||||
using ZB.MOM.WW.MxGateway.Server.Metrics;
|
using ZB.MOM.WW.MxGateway.Server.Metrics;
|
||||||
using ZB.MOM.WW.MxGateway.Server.Sessions;
|
using ZB.MOM.WW.MxGateway.Server.Sessions;
|
||||||
using ZB.MOM.WW.MxGateway.Server.Workers;
|
using ZB.MOM.WW.MxGateway.Server.Workers;
|
||||||
|
using ZB.MOM.WW.MxGateway.Tests.TestSupport;
|
||||||
|
|
||||||
namespace ZB.MOM.WW.MxGateway.Tests.Gateway.Grpc;
|
namespace ZB.MOM.WW.MxGateway.Tests.Gateway.Grpc;
|
||||||
|
|
||||||
@@ -892,27 +893,13 @@ public sealed class EventStreamServiceTests
|
|||||||
}
|
}
|
||||||
|
|
||||||
// The real-clock deadline here is load-sensitive on a wide host (windev runs this suite
|
// The real-clock deadline here is load-sensitive on a wide host (windev runs this suite
|
||||||
// 36-way parallel). Surfacing the unmet condition instead of letting the bare
|
// 36-way parallel). Eventually names the unmet condition instead of letting a bare
|
||||||
// TaskCanceledException escape is what makes such a failure diagnosable rather than a
|
// TaskCanceledException escape, which is what makes such a failure diagnosable rather than a
|
||||||
// mystery cancellation attributed to "the environment".
|
// mystery cancellation attributed to "the environment".
|
||||||
private static async Task WaitUntilAsync(
|
private static Task WaitUntilAsync(
|
||||||
Func<bool> predicate,
|
Func<bool> predicate,
|
||||||
[CallerArgumentExpression(nameof(predicate))] string? predicateExpression = null)
|
[CallerArgumentExpression(nameof(predicate))] string? predicateExpression = null)
|
||||||
{
|
=> Eventually.WaitUntilAsync(predicate, TestTimeout, predicateExpression);
|
||||||
using CancellationTokenSource cancellationTokenSource = new(TestTimeout);
|
|
||||||
while (!predicate())
|
|
||||||
{
|
|
||||||
try
|
|
||||||
{
|
|
||||||
await Task.Delay(TimeSpan.FromMilliseconds(10), cancellationTokenSource.Token);
|
|
||||||
}
|
|
||||||
catch (OperationCanceledException)
|
|
||||||
{
|
|
||||||
Assert.Fail(
|
|
||||||
$"Timed out after {TestTimeout} waiting for condition: {predicateExpression}");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>Fake session manager for testing event streams.</summary>
|
/// <summary>Fake session manager for testing event streams.</summary>
|
||||||
private sealed class FakeSessionManager : ISessionManager
|
private sealed class FakeSessionManager : ISessionManager
|
||||||
|
|||||||
+2
-15
@@ -294,21 +294,8 @@ public sealed class GatewaySessionDashboardMirrorTests
|
|||||||
return new WorkerEvent { Event = mxEvent };
|
return new WorkerEvent { Event = mxEvent };
|
||||||
}
|
}
|
||||||
|
|
||||||
private static async Task WaitUntilAsync(Func<bool> predicate, [CallerArgumentExpression(nameof(predicate))] string? condition = null)
|
private static Task WaitUntilAsync(Func<bool> predicate, [CallerArgumentExpression(nameof(predicate))] string? condition = null)
|
||||||
{
|
=> Eventually.WaitUntilAsync(predicate, TestTimeout, condition);
|
||||||
using CancellationTokenSource cancellationTokenSource = new(TestTimeout);
|
|
||||||
try
|
|
||||||
{
|
|
||||||
while (!predicate())
|
|
||||||
{
|
|
||||||
await Task.Delay(TimeSpan.FromMilliseconds(10), cancellationTokenSource.Token);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
catch (OperationCanceledException)
|
|
||||||
{
|
|
||||||
Assert.Fail($"Timed out after {TestTimeout.TotalSeconds}s waiting for: {condition}");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private sealed class ThrowingDashboardEventBroadcaster : IDashboardEventBroadcaster
|
private sealed class ThrowingDashboardEventBroadcaster : IDashboardEventBroadcaster
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -1,3 +1,5 @@
|
|||||||
|
using System.Runtime.CompilerServices;
|
||||||
|
|
||||||
namespace ZB.MOM.WW.MxGateway.Tests.TestSupport;
|
namespace ZB.MOM.WW.MxGateway.Tests.TestSupport;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
@@ -12,12 +14,38 @@ public static class Eventually
|
|||||||
/// <summary>Waits until <paramref name="condition"/> holds.</summary>
|
/// <summary>Waits until <paramref name="condition"/> holds.</summary>
|
||||||
/// <param name="condition">Condition to poll.</param>
|
/// <param name="condition">Condition to poll.</param>
|
||||||
/// <param name="timeout">Real-time bound on the wait.</param>
|
/// <param name="timeout">Real-time bound on the wait.</param>
|
||||||
|
/// <param name="description">
|
||||||
|
/// What is being waited for, named in the timeout message. Defaults to the source text of
|
||||||
|
/// <paramref name="condition"/>, so an unmet wait says which condition it was.
|
||||||
|
/// </param>
|
||||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||||
/// <exception cref="TimeoutException">The condition did not hold within the bound.</exception>
|
/// <exception cref="TimeoutException">The condition did not hold within the bound.</exception>
|
||||||
public static async Task WaitUntilAsync(Func<bool> condition, TimeSpan timeout)
|
public static Task WaitUntilAsync(
|
||||||
|
Func<bool> condition,
|
||||||
|
TimeSpan timeout,
|
||||||
|
[CallerArgumentExpression(nameof(condition))] string? description = null)
|
||||||
{
|
{
|
||||||
|
ArgumentNullException.ThrowIfNull(condition);
|
||||||
|
return WaitUntilAsync(() => Task.FromResult(condition()), timeout, description);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>Waits until the asynchronous <paramref name="condition"/> holds.</summary>
|
||||||
|
/// <param name="condition">Asynchronous condition to poll; each probe is awaited before the next.</param>
|
||||||
|
/// <param name="timeout">Real-time bound on the wait.</param>
|
||||||
|
/// <param name="description">
|
||||||
|
/// What is being waited for, named in the timeout message. Defaults to the source text of
|
||||||
|
/// <paramref name="condition"/>.
|
||||||
|
/// </param>
|
||||||
|
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||||
|
/// <exception cref="TimeoutException">The condition did not hold within the bound.</exception>
|
||||||
|
public static async Task WaitUntilAsync(
|
||||||
|
Func<Task<bool>> condition,
|
||||||
|
TimeSpan timeout,
|
||||||
|
[CallerArgumentExpression(nameof(condition))] string? description = null)
|
||||||
|
{
|
||||||
|
ArgumentNullException.ThrowIfNull(condition);
|
||||||
using CancellationTokenSource cts = new(timeout);
|
using CancellationTokenSource cts = new(timeout);
|
||||||
while (!condition())
|
while (!await condition().ConfigureAwait(false))
|
||||||
{
|
{
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
@@ -25,7 +53,8 @@ public static class Eventually
|
|||||||
}
|
}
|
||||||
catch (OperationCanceledException)
|
catch (OperationCanceledException)
|
||||||
{
|
{
|
||||||
throw new TimeoutException("Condition was not met in time.");
|
throw new TimeoutException(
|
||||||
|
$"Condition was not met within {timeout.TotalSeconds:0.###}s: {description}");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -34,13 +63,20 @@ public static class Eventually
|
|||||||
/// <typeparam name="T">Reference type produced by the probe.</typeparam>
|
/// <typeparam name="T">Reference type produced by the probe.</typeparam>
|
||||||
/// <param name="probe">Probe to poll.</param>
|
/// <param name="probe">Probe to poll.</param>
|
||||||
/// <param name="timeout">Real-time bound on the wait.</param>
|
/// <param name="timeout">Real-time bound on the wait.</param>
|
||||||
|
/// <param name="description">
|
||||||
|
/// What is being waited for, named in the timeout message. Defaults to the source text of
|
||||||
|
/// <paramref name="probe"/>.
|
||||||
|
/// </param>
|
||||||
/// <returns>The first non-null probe result.</returns>
|
/// <returns>The first non-null probe result.</returns>
|
||||||
/// <exception cref="TimeoutException">The probe never returned a value within the bound.</exception>
|
/// <exception cref="TimeoutException">The probe never returned a value within the bound.</exception>
|
||||||
public static async Task<T> WaitForAsync<T>(Func<T?> probe, TimeSpan timeout)
|
public static async Task<T> WaitForAsync<T>(
|
||||||
|
Func<T?> probe,
|
||||||
|
TimeSpan timeout,
|
||||||
|
[CallerArgumentExpression(nameof(probe))] string? description = null)
|
||||||
where T : class
|
where T : class
|
||||||
{
|
{
|
||||||
T? result = null;
|
T? result = null;
|
||||||
await WaitUntilAsync(() => (result = probe()) is not null, timeout).ConfigureAwait(false);
|
await WaitUntilAsync(() => (result = probe()) is not null, timeout, description).ConfigureAwait(false);
|
||||||
return result!;
|
return result!;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user