Files
mxaccessgw/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Dashboard/DashboardSnapshotFeedTests.cs
T
Joseph Doherty 09b3ecddf2 test: pump-idle signal for DashboardSnapshotFeed; fold private wait helpers into Eventually (mxaccessgw#223)
The MxGateway.Tests part of #223 (the Worker.Tests items are handled separately).

- DashboardSnapshotFeed gains two internal test seams: WhenIdleGateClosedAsync, a
  TCS-backed signal that completes once no pump generation is accepting subscribers,
  and IsPumpIdle (no live generation and the latest pump has exited). The TCS is created
  lazily by a waiter, so production never allocates it.
- WatchAsync_WhenAJoinerLeavesWhileAnOldGenerationLingers_LeavesNoPumpRunning drops both
  150 ms real sleeps: it waits on the idle-gate signal for the joiner's unsubscribe, and
  asserts IsPumpIdle before the orphan-push check, which is then immediate. Reverting the
  per-generation idle check to the old global "list is empty" check now fails the test
  (5 s TimeoutException on the gate), where before the sleep simply elapsed.
- Eventually.WaitUntilAsync/WaitForAsync take an optional description (defaulting to the
  condition's source text via CallerArgumentExpression) for the timeout message, and gain an
  async-condition overload. The hand-rolled polling loops in EventStreamServiceTests,
  GatewaySessionDashboardMirrorTests, DashboardEventBroadcasterTests and
  AlarmsPageTruncationBannerTests now delegate to it, keeping their timeouts and messages.

Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
2026-11-09 13:36:34 -05:00

523 lines
24 KiB
C#

using System.Runtime.CompilerServices;
using System.Threading.Channels;
using ZB.MOM.WW.MxGateway.Server.Dashboard;
using ZB.MOM.WW.MxGateway.Tests.TestSupport;
namespace ZB.MOM.WW.MxGateway.Tests.Gateway.Dashboard;
/// <summary>
/// Covers the in-process snapshot fan-out that replaced the dashboard pages'
/// loopback <c>/hubs/snapshot</c> connections. The invariants under test are the
/// ones that make the feed cheaper than the hub hop: exactly one underlying
/// <see cref="IDashboardSnapshotService.WatchSnapshotsAsync"/> enumeration for any
/// number of viewers, nothing at all while nobody is watching, and a slow viewer
/// that can neither buffer without bound nor stall the others.
/// </summary>
public sealed class DashboardSnapshotFeedTests
{
private static readonly TimeSpan TestTimeout = TimeSpan.FromSeconds(5);
/// <summary>
/// With no page subscribed, the feed must not touch the snapshot service at
/// all — no timer, no snapshot build. This is the whole point of the idle
/// gate: an unattended gateway does no dashboard work.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public Task WatchAsync_WithNoSubscribers_NeverEnumeratesUnderlyingWatch()
{
FakeSnapshotService service = new();
DashboardSnapshotFeed feed = new(service);
// Obtaining the enumerable without enumerating it must not subscribe
// either: the pump starts on the first MoveNextAsync, not before.
_ = feed.WatchAsync(CancellationToken.None);
Assert.Equal(0, service.EnumerationCount);
return Task.CompletedTask;
}
/// <summary>
/// Two viewers share one underlying enumeration and both see the same
/// pushed snapshot. Before the feed, each page opened its own SignalR
/// connection and the publisher pulled its own snapshot stream.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task WatchAsync_WithTwoSubscribers_SharesASingleUnderlyingEnumeration()
{
FakeSnapshotService service = new();
DashboardSnapshotFeed feed = new(service);
using CancellationTokenSource firstCancellation = new();
using CancellationTokenSource secondCancellation = new();
IAsyncEnumerator<DashboardSnapshot> first =
feed.WatchAsync(firstCancellation.Token).GetAsyncEnumerator(firstCancellation.Token);
Task<bool> firstMove = first.MoveNextAsync().AsTask();
await service.Enumerations.WaitForAsync(1);
IAsyncEnumerator<DashboardSnapshot> second =
feed.WatchAsync(secondCancellation.Token).GetAsyncEnumerator(secondCancellation.Token);
Task<bool> secondMove = second.MoveNextAsync().AsTask();
// Push until both have observed a snapshot: a subscriber only becomes
// visible to the pump once its MoveNextAsync has registered the channel,
// so a single push could race the second registration.
await PushUntilAsync(service, Task.WhenAll(firstMove, secondMove));
Assert.True(await firstMove.WaitAsync(TestTimeout));
Assert.True(await secondMove.WaitAsync(TestTimeout));
Assert.StartsWith("push-", first.Current.GatewayVersion, StringComparison.Ordinal);
Assert.StartsWith("push-", second.Current.GatewayVersion, StringComparison.Ordinal);
Assert.Equal(1, service.EnumerationCount);
await firstCancellation.CancelAsync();
await secondCancellation.CancelAsync();
await DrainAsync(first, firstMove);
await DrainAsync(second, secondMove);
}
/// <summary>
/// The last viewer leaving must cancel the underlying enumeration (idle
/// gate re-armed), and the next viewer must restart it — the rapid
/// unsubscribe/resubscribe path a page navigation exercises.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task WatchAsync_WhenLastSubscriberLeaves_CancelsPumpAndRestartsForTheNext()
{
FakeSnapshotService service = new();
DashboardSnapshotFeed feed = new(service);
using CancellationTokenSource firstCancellation = new();
IAsyncEnumerator<DashboardSnapshot> first =
feed.WatchAsync(firstCancellation.Token).GetAsyncEnumerator(firstCancellation.Token);
Task<bool> firstMove = first.MoveNextAsync().AsTask();
await service.Enumerations.WaitForAsync(1);
await firstCancellation.CancelAsync();
await DrainAsync(first, firstMove);
await service.Completions.WaitForAsync(1);
Assert.True(service.LastEnumerationWasCancelled);
using CancellationTokenSource secondCancellation = new();
IAsyncEnumerator<DashboardSnapshot> second =
feed.WatchAsync(secondCancellation.Token).GetAsyncEnumerator(secondCancellation.Token);
Task<bool> secondMove = second.MoveNextAsync().AsTask();
await service.Enumerations.WaitForAsync(2);
Assert.Equal(2, service.EnumerationCount);
await secondCancellation.CancelAsync();
await DrainAsync(second, secondMove);
}
/// <summary>
/// A viewer that is not reading must not stall the pump or accumulate
/// snapshots: its bounded channel drops the oldest, so its next read is the
/// newest snapshot the pump has broadcast, not a backlog head. The fast
/// reader's progress is what makes the assertion deterministic — once it has
/// seen the third snapshot the pump has provably broadcast all three.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task WatchAsync_WithASlowSubscriber_KeepsOnlyTheNewestSnapshot()
{
FakeSnapshotService service = new();
DashboardSnapshotFeed feed = new(service);
using CancellationTokenSource fastCancellation = new();
using CancellationTokenSource slowCancellation = new();
IAsyncEnumerator<DashboardSnapshot> fast =
feed.WatchAsync(fastCancellation.Token).GetAsyncEnumerator(fastCancellation.Token);
Task<bool> fastMove = fast.MoveNextAsync().AsTask();
await service.Enumerations.WaitForAsync(1);
// The slow subscriber registers but never advances until the very end.
IAsyncEnumerator<DashboardSnapshot> slow =
feed.WatchAsync(slowCancellation.Token).GetAsyncEnumerator(slowCancellation.Token);
Task<bool> slowMove = slow.MoveNextAsync().AsTask();
await PushUntilAsync(service, Task.WhenAll(fastMove, slowMove));
Assert.True(await fastMove.WaitAsync(TestTimeout));
Assert.True(await slowMove.WaitAsync(TestTimeout));
service.Push(CreateSnapshot("s1"));
service.Push(CreateSnapshot("s2"));
service.Push(CreateSnapshot("s3"));
// Drain the fast reader until it sees s3; that proves the pump broadcast
// all three to every subscriber, so the slow channel now holds exactly s3.
string fastLatest = fast.Current.GatewayVersion;
while (fastLatest != "s3")
{
Assert.True(await fast.MoveNextAsync().AsTask().WaitAsync(TestTimeout));
fastLatest = fast.Current.GatewayVersion;
}
Assert.True(await slow.MoveNextAsync().AsTask().WaitAsync(TestTimeout));
Assert.Equal("s3", slow.Current.GatewayVersion);
await fastCancellation.CancelAsync();
await slowCancellation.CancelAsync();
await DrainAsync(fast, Task.FromResult(true));
await DrainAsync(slow, Task.FromResult(true));
}
/// <summary>
/// A fault in the underlying watch is surfaced to the current viewers rather
/// than silently hanging them, and it resets the feed so the next viewer
/// starts a fresh pump instead of attaching to a dead one.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task WatchAsync_WhenUnderlyingWatchFaults_PropagatesAndRestartsForTheNextSubscriber()
{
FakeSnapshotService service = new();
DashboardSnapshotFeed feed = new(service);
using CancellationTokenSource firstCancellation = new();
IAsyncEnumerator<DashboardSnapshot> first =
feed.WatchAsync(firstCancellation.Token).GetAsyncEnumerator(firstCancellation.Token);
Task<bool> firstMove = first.MoveNextAsync().AsTask();
await service.Enumerations.WaitForAsync(1);
service.Fault(new InvalidOperationException("simulated snapshot source failure"));
InvalidOperationException failure =
await Assert.ThrowsAsync<InvalidOperationException>(() => firstMove.WaitAsync(TestTimeout));
Assert.Equal("simulated snapshot source failure", failure.Message);
await first.DisposeAsync();
using CancellationTokenSource secondCancellation = new();
IAsyncEnumerator<DashboardSnapshot> second =
feed.WatchAsync(secondCancellation.Token).GetAsyncEnumerator(secondCancellation.Token);
Task<bool> secondMove = second.MoveNextAsync().AsTask();
await service.Enumerations.WaitForAsync(2);
await PushUntilAsync(service, secondMove);
Assert.True(await secondMove.WaitAsync(TestTimeout));
await secondCancellation.CancelAsync();
await DrainAsync(second, secondMove);
}
/// <summary>
/// The race the generation tagging exists for: a page subscribes in the window between
/// the source failing and the dying pump detaching its subscribers. Without generations
/// the newcomer joined the doomed pump, was detached with its error, and no pump ever
/// restarted (only a first subscriber started one) — that page froze for good.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task WatchAsync_WhenASubscriberJoinsWhileAFaultedPumpUnwinds_IsServedByAFreshPump()
{
FakeSnapshotService service = new();
service.HoldDisposal();
DashboardSnapshotFeed feed = new(service);
using CancellationTokenSource firstCancellation = new();
IAsyncEnumerator<DashboardSnapshot> first =
feed.WatchAsync(firstCancellation.Token).GetAsyncEnumerator(firstCancellation.Token);
Task<bool> firstMove = first.MoveNextAsync().AsTask();
await service.Enumerations.WaitForAsync(1);
service.Fault(new InvalidOperationException("simulated snapshot source failure"));
// The pump has observed the failure and is parked disposing the enumerator — the
// exact window in which a page used to attach itself to a doomed pump.
await service.DisposalReached.WaitAsync(TestTimeout);
using CancellationTokenSource secondCancellation = new();
IAsyncEnumerator<DashboardSnapshot> second =
feed.WatchAsync(secondCancellation.Token).GetAsyncEnumerator(secondCancellation.Token);
// An async iterator body runs synchronously up to its first await, so the
// subscription is registered by the time MoveNextAsync hands back its task.
Task<bool> secondMove = second.MoveNextAsync().AsTask();
service.ReleaseDisposal();
// The subscriber that was there when the source broke still learns about it...
InvalidOperationException failure =
await Assert.ThrowsAsync<InvalidOperationException>(() => firstMove.WaitAsync(TestTimeout));
Assert.Equal("simulated snapshot source failure", failure.Message);
await first.DisposeAsync();
// ...and the one that joined mid-unwind is served by a restarted enumeration
// instead of inheriting the failure.
await service.Enumerations.WaitForAsync(2);
await PushUntilAsync(service, secondMove);
Assert.True(await secondMove.WaitAsync(TestTimeout));
Assert.StartsWith("push-", second.Current.GatewayVersion, StringComparison.Ordinal);
await secondCancellation.CancelAsync();
await DrainAsync(second, secondMove);
}
/// <summary>
/// The idle gate is per generation, not per subscriber count. A viewer that joins while a
/// pump unwinds starts a new generation, and the old generation's viewers linger in the
/// list until that pump's reset runs — so a global "is the list empty" check let the new
/// viewer leave without cancelling the generation it had just started, leaving a pump
/// enumerating the snapshot source with nobody watching it.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task WatchAsync_WhenAJoinerLeavesWhileAnOldGenerationLingers_LeavesNoPumpRunning()
{
FakeSnapshotService service = new();
service.HoldDisposal();
DashboardSnapshotFeed feed = new(service);
using CancellationTokenSource oldCancellation = new();
IAsyncEnumerator<DashboardSnapshot> oldSubscriber =
feed.WatchAsync(oldCancellation.Token).GetAsyncEnumerator(oldCancellation.Token);
Task<bool> oldMove = oldSubscriber.MoveNextAsync().AsTask();
await service.Enumerations.WaitForAsync(1);
service.Fault(new InvalidOperationException("simulated snapshot source failure"));
await service.DisposalReached.WaitAsync(TestTimeout);
// Joins mid-unwind (starting a fresh generation) and leaves again before the dying
// pump has detached the subscriber that is still lingering in the list.
using CancellationTokenSource joinerCancellation = new();
IAsyncEnumerator<DashboardSnapshot> joiner =
feed.WatchAsync(joinerCancellation.Token).GetAsyncEnumerator(joinerCancellation.Token);
Task<bool> joinerMove = joiner.MoveNextAsync().AsTask();
await joinerCancellation.CancelAsync();
// The joiner's unwind is a continuation of its cancelled channel read, and its unsubscribe
// cannot be awaited directly (it then waits on the parked pump). Wait instead for the idle
// gate it closes: the joiner started the live generation, so the gate closes only once its
// unsubscribe has cancelled that generation. A global "is the list empty" check — the
// defect under test — never closes it, because the old subscriber still lingers.
await feed.WhenIdleGateClosedAsync(CancellationToken.None).WaitAsync(TestTimeout);
service.ReleaseDisposal();
await Assert.ThrowsAsync<InvalidOperationException>(() => oldMove.WaitAsync(TestTimeout));
await oldSubscriber.DisposeAsync();
// Completes only once the joiner's unsubscribe has awaited its generation's pump.
await DrainAsync(joiner, joinerMove);
// Nobody is watching, so no pump may be left enumerating the source...
Assert.True(feed.IsPumpIdle);
// ...and so nothing consumes a snapshot pushed now. With no pump running the push stays
// queued, so the count is checked without a settling delay.
service.Push(CreateSnapshot("orphan-check"));
Assert.Equal(1, service.PendingPushCount);
// ...and the next viewer still starts cleanly, picking up the queued snapshot.
int enumerationsBefore = service.EnumerationCount;
using CancellationTokenSource nextCancellation = new();
IAsyncEnumerator<DashboardSnapshot> next =
feed.WatchAsync(nextCancellation.Token).GetAsyncEnumerator(nextCancellation.Token);
Task<bool> nextMove = next.MoveNextAsync().AsTask();
await service.Enumerations.WaitForAsync(enumerationsBefore + 1);
Assert.True(await nextMove.WaitAsync(TestTimeout));
Assert.Equal("orphan-check", next.Current.GatewayVersion);
await nextCancellation.CancelAsync();
await DrainAsync(next, nextMove);
}
/// <summary>Builds a snapshot whose version string identifies it in assertions.</summary>
/// <param name="version">Identity marker carried in <c>GatewayVersion</c>.</param>
/// <returns>A snapshot carrying the supplied identity marker.</returns>
private static DashboardSnapshot CreateSnapshot(string version)
{
return new DashboardSnapshot(
GeneratedAt: DateTimeOffset.UnixEpoch,
GatewayStartedAt: DateTimeOffset.UnixEpoch,
GatewayUptime: TimeSpan.Zero,
GatewayStatus: "Healthy",
GatewayVersion: version,
Sessions: Array.Empty<DashboardSessionSummary>(),
Workers: Array.Empty<DashboardWorkerSummary>(),
Metrics: Array.Empty<DashboardMetricSummary>(),
Faults: Array.Empty<DashboardFaultSummary>(),
ApiKeys: Array.Empty<DashboardApiKeySummary>(),
Configuration: null!,
Galaxy: null!);
}
/// <summary>
/// Pushes snapshots until the supplied task completes, so a test never
/// depends on a single push landing after a subscriber has registered.
/// </summary>
/// <param name="service">Fake snapshot source to push through.</param>
/// <param name="until">Task whose completion stops the pushes.</param>
/// <returns>A task that represents the asynchronous operation.</returns>
private static async Task PushUntilAsync(FakeSnapshotService service, Task until)
{
using CancellationTokenSource cancellation = new(TestTimeout);
int sequence = 0;
while (!until.IsCompleted)
{
service.Push(CreateSnapshot($"push-{sequence++}"));
await Task.Delay(TimeSpan.FromMilliseconds(5), cancellation.Token);
}
}
/// <summary>
/// Observes the cancellation of a pending enumeration and disposes the
/// enumerator, mirroring how <c>await foreach</c> unwinds a cancelled watch.
/// </summary>
/// <param name="enumerator">Enumerator to unwind.</param>
/// <param name="pending">The in-flight move, if any.</param>
/// <returns>A task that represents the asynchronous operation.</returns>
private static async Task DrainAsync(IAsyncEnumerator<DashboardSnapshot> enumerator, Task pending)
{
try
{
await pending.WaitAsync(TestTimeout);
}
catch (OperationCanceledException)
{
}
try
{
await enumerator.DisposeAsync();
}
catch (OperationCanceledException)
{
}
}
/// <summary>
/// Snapshot source under the feed's control: counts enumerations, records how
/// each one ended, and lets the test drive snapshots (or a fault) into the
/// live enumeration.
/// </summary>
private sealed class FakeSnapshotService : IDashboardSnapshotService
{
private readonly Channel<object> _pushes = Channel.CreateUnbounded<object>();
private readonly TaskCompletionSource _disposalReached = new(TaskCreationOptions.RunContinuationsAsynchronously);
private readonly TaskCompletionSource _disposalRelease = new(TaskCreationOptions.RunContinuationsAsynchronously);
private volatile bool _lastEnumerationWasCancelled;
private volatile bool _holdDisposal;
/// <summary>Gets the number of times the feed started enumerating this source.</summary>
public int EnumerationCount => Enumerations.Count;
/// <summary>Gets the signal raised each time the feed starts enumerating this source.</summary>
public CountSignal Enumerations { get; } = new();
/// <summary>Gets the signal raised each time an enumeration finishes.</summary>
public CountSignal Completions { get; } = new();
/// <summary>Gets a task that completes when a held enumerator disposal is reached.</summary>
public Task DisposalReached => _disposalReached.Task;
/// <summary>
/// Gets the number of queued snapshots no enumeration has taken yet. A live pump
/// drains this even with nobody watching, so a stable count proves the feed is idle.
/// </summary>
public int PendingPushCount => _pushes.Reader.Count;
/// <summary>Gets the number of enumerations that have finished (cancelled, faulted, or completed).</summary>
public int CompletedEnumerationCount => Completions.Count;
/// <summary>Gets a value indicating whether the most recently finished enumeration ended cancelled.</summary>
public bool LastEnumerationWasCancelled => _lastEnumerationWasCancelled;
/// <summary>Queues a snapshot for the live enumeration to yield.</summary>
/// <param name="snapshot">Snapshot to yield.</param>
public void Push(DashboardSnapshot snapshot) => _pushes.Writer.TryWrite(snapshot);
/// <summary>Queues a failure for the live enumeration to throw.</summary>
/// <param name="error">Exception to throw from the enumeration.</param>
public void Fault(Exception error) => _pushes.Writer.TryWrite(error);
/// <summary>
/// Parks enumerator disposal until <see cref="ReleaseDisposal"/>, which holds the pump
/// in the window between observing the source's failure and detaching its subscribers.
/// </summary>
public void HoldDisposal() => _holdDisposal = true;
/// <summary>Releases a held enumerator disposal.</summary>
public void ReleaseDisposal() => _disposalRelease.TrySetResult();
/// <inheritdoc />
public DashboardSnapshot GetSnapshot() => CreateSnapshot("current");
/// <inheritdoc />
public IAsyncEnumerable<DashboardSnapshot> WatchSnapshotsAsync(CancellationToken cancellationToken)
=> new GatedEnumerable(this);
private async Task OnDisposingAsync()
{
if (!_holdDisposal)
{
return;
}
_disposalReached.TrySetResult();
await _disposalRelease.Task.ConfigureAwait(false);
}
private async IAsyncEnumerable<DashboardSnapshot> EnumerateAsync(
[EnumeratorCancellation] CancellationToken cancellationToken)
{
Enumerations.Signal();
try
{
await foreach (object item in _pushes.Reader.ReadAllAsync(cancellationToken).ConfigureAwait(false))
{
if (item is Exception error)
{
throw error;
}
yield return (DashboardSnapshot)item;
}
}
finally
{
_lastEnumerationWasCancelled = cancellationToken.IsCancellationRequested;
Completions.Signal();
}
}
/// <summary>
/// Wraps the iterator so disposal is a control point of its own: the feed ends a pump
/// generation when MoveNextAsync fails, which is strictly before this disposal runs.
/// </summary>
/// <param name="owner">The fake whose enumeration is being wrapped.</param>
private sealed class GatedEnumerable(FakeSnapshotService owner) : IAsyncEnumerable<DashboardSnapshot>
{
/// <summary>Creates a gated enumerator over the fake's enumeration.</summary>
/// <param name="cancellationToken">Token to observe for cancellation.</param>
/// <returns>The gated enumerator.</returns>
public IAsyncEnumerator<DashboardSnapshot> GetAsyncEnumerator(
CancellationToken cancellationToken = default)
=> new GatedEnumerator(owner, owner.EnumerateAsync(cancellationToken).GetAsyncEnumerator(cancellationToken));
}
private sealed class GatedEnumerator(
FakeSnapshotService owner,
IAsyncEnumerator<DashboardSnapshot> inner) : IAsyncEnumerator<DashboardSnapshot>
{
/// <summary>Gets the current snapshot.</summary>
public DashboardSnapshot Current => inner.Current;
/// <summary>Advances the wrapped enumeration.</summary>
/// <returns>A task that yields whether another snapshot is available.</returns>
public ValueTask<bool> MoveNextAsync() => inner.MoveNextAsync();
/// <summary>Parks while the fake holds disposal, then disposes the wrapped enumeration.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
public async ValueTask DisposeAsync()
{
await owner.OnDisposingAsync().ConfigureAwait(false);
await inner.DisposeAsync().ConfigureAwait(false);
}
}
}
}