feat(dashboard): in-process snapshot feed replaces the pages' loopback /hubs/snapshot hop
This commit is contained in:
Joseph Doherty
@@ -0,0 +1,333 @@
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Threading.Channels;
|
||||
using ZB.MOM.WW.MxGateway.Server.Dashboard;
|
||||
|
||||
namespace ZB.MOM.WW.MxGateway.Tests.Gateway.Dashboard;
|
||||
|
||||
/// <summary>
|
||||
/// Covers the in-process snapshot fan-out that replaced the dashboard pages'
|
||||
/// loopback <c>/hubs/snapshot</c> connections. The invariants under test are the
|
||||
/// ones that make the feed cheaper than the hub hop: exactly one underlying
|
||||
/// <see cref="IDashboardSnapshotService.WatchSnapshotsAsync"/> enumeration for any
|
||||
/// number of viewers, nothing at all while nobody is watching, and a slow viewer
|
||||
/// that can neither buffer without bound nor stall the others.
|
||||
/// </summary>
|
||||
public sealed class DashboardSnapshotFeedTests
|
||||
{
|
||||
private static readonly TimeSpan TestTimeout = TimeSpan.FromSeconds(5);
|
||||
|
||||
/// <summary>
|
||||
/// With no page subscribed, the feed must not touch the snapshot service at
|
||||
/// all — no timer, no snapshot build. This is the whole point of the idle
|
||||
/// gate: an unattended gateway does no dashboard work.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task WatchAsync_WithNoSubscribers_NeverEnumeratesUnderlyingWatch()
|
||||
{
|
||||
FakeSnapshotService service = new();
|
||||
DashboardSnapshotFeed feed = new(service);
|
||||
|
||||
// Obtaining the enumerable without enumerating it must not subscribe
|
||||
// either: the pump starts on the first MoveNextAsync, not before.
|
||||
_ = feed.WatchAsync(CancellationToken.None);
|
||||
await Task.Delay(TimeSpan.FromMilliseconds(100));
|
||||
|
||||
Assert.Equal(0, service.EnumerationCount);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Two viewers share one underlying enumeration and both see the same
|
||||
/// pushed snapshot. Before the feed, each page opened its own SignalR
|
||||
/// connection and the publisher pulled its own snapshot stream.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task WatchAsync_WithTwoSubscribers_SharesASingleUnderlyingEnumeration()
|
||||
{
|
||||
FakeSnapshotService service = new();
|
||||
DashboardSnapshotFeed feed = new(service);
|
||||
|
||||
using CancellationTokenSource firstCancellation = new();
|
||||
using CancellationTokenSource secondCancellation = new();
|
||||
IAsyncEnumerator<DashboardSnapshot> first =
|
||||
feed.WatchAsync(firstCancellation.Token).GetAsyncEnumerator(firstCancellation.Token);
|
||||
Task<bool> firstMove = first.MoveNextAsync().AsTask();
|
||||
await WaitUntilAsync(() => service.EnumerationCount >= 1);
|
||||
|
||||
IAsyncEnumerator<DashboardSnapshot> second =
|
||||
feed.WatchAsync(secondCancellation.Token).GetAsyncEnumerator(secondCancellation.Token);
|
||||
Task<bool> secondMove = second.MoveNextAsync().AsTask();
|
||||
|
||||
// Push until both have observed a snapshot: a subscriber only becomes
|
||||
// visible to the pump once its MoveNextAsync has registered the channel,
|
||||
// so a single push could race the second registration.
|
||||
await PushUntilAsync(service, Task.WhenAll(firstMove, secondMove));
|
||||
|
||||
Assert.True(await firstMove.WaitAsync(TestTimeout));
|
||||
Assert.True(await secondMove.WaitAsync(TestTimeout));
|
||||
Assert.StartsWith("push-", first.Current.GatewayVersion, StringComparison.Ordinal);
|
||||
Assert.StartsWith("push-", second.Current.GatewayVersion, StringComparison.Ordinal);
|
||||
Assert.Equal(1, service.EnumerationCount);
|
||||
|
||||
await firstCancellation.CancelAsync();
|
||||
await secondCancellation.CancelAsync();
|
||||
await DrainAsync(first, firstMove);
|
||||
await DrainAsync(second, secondMove);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The last viewer leaving must cancel the underlying enumeration (idle
|
||||
/// gate re-armed), and the next viewer must restart it — the rapid
|
||||
/// unsubscribe/resubscribe path a page navigation exercises.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task WatchAsync_WhenLastSubscriberLeaves_CancelsPumpAndRestartsForTheNext()
|
||||
{
|
||||
FakeSnapshotService service = new();
|
||||
DashboardSnapshotFeed feed = new(service);
|
||||
|
||||
using CancellationTokenSource firstCancellation = new();
|
||||
IAsyncEnumerator<DashboardSnapshot> first =
|
||||
feed.WatchAsync(firstCancellation.Token).GetAsyncEnumerator(firstCancellation.Token);
|
||||
Task<bool> firstMove = first.MoveNextAsync().AsTask();
|
||||
await WaitUntilAsync(() => service.EnumerationCount >= 1);
|
||||
|
||||
await firstCancellation.CancelAsync();
|
||||
await DrainAsync(first, firstMove);
|
||||
|
||||
await WaitUntilAsync(() => service.CompletedEnumerationCount >= 1);
|
||||
Assert.True(service.LastEnumerationWasCancelled);
|
||||
|
||||
using CancellationTokenSource secondCancellation = new();
|
||||
IAsyncEnumerator<DashboardSnapshot> second =
|
||||
feed.WatchAsync(secondCancellation.Token).GetAsyncEnumerator(secondCancellation.Token);
|
||||
Task<bool> secondMove = second.MoveNextAsync().AsTask();
|
||||
await WaitUntilAsync(() => service.EnumerationCount >= 2);
|
||||
|
||||
Assert.Equal(2, service.EnumerationCount);
|
||||
|
||||
await secondCancellation.CancelAsync();
|
||||
await DrainAsync(second, secondMove);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A viewer that is not reading must not stall the pump or accumulate
|
||||
/// snapshots: its bounded channel drops the oldest, so its next read is the
|
||||
/// newest snapshot the pump has broadcast, not a backlog head. The fast
|
||||
/// reader's progress is what makes the assertion deterministic — once it has
|
||||
/// seen the third snapshot the pump has provably broadcast all three.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task WatchAsync_WithASlowSubscriber_KeepsOnlyTheNewestSnapshot()
|
||||
{
|
||||
FakeSnapshotService service = new();
|
||||
DashboardSnapshotFeed feed = new(service);
|
||||
|
||||
using CancellationTokenSource fastCancellation = new();
|
||||
using CancellationTokenSource slowCancellation = new();
|
||||
IAsyncEnumerator<DashboardSnapshot> fast =
|
||||
feed.WatchAsync(fastCancellation.Token).GetAsyncEnumerator(fastCancellation.Token);
|
||||
Task<bool> fastMove = fast.MoveNextAsync().AsTask();
|
||||
await WaitUntilAsync(() => service.EnumerationCount >= 1);
|
||||
|
||||
// The slow subscriber registers but never advances until the very end.
|
||||
IAsyncEnumerator<DashboardSnapshot> slow =
|
||||
feed.WatchAsync(slowCancellation.Token).GetAsyncEnumerator(slowCancellation.Token);
|
||||
Task<bool> slowMove = slow.MoveNextAsync().AsTask();
|
||||
await PushUntilAsync(service, Task.WhenAll(fastMove, slowMove));
|
||||
Assert.True(await fastMove.WaitAsync(TestTimeout));
|
||||
Assert.True(await slowMove.WaitAsync(TestTimeout));
|
||||
|
||||
service.Push(CreateSnapshot("s1"));
|
||||
service.Push(CreateSnapshot("s2"));
|
||||
service.Push(CreateSnapshot("s3"));
|
||||
|
||||
// Drain the fast reader until it sees s3; that proves the pump broadcast
|
||||
// all three to every subscriber, so the slow channel now holds exactly s3.
|
||||
string fastLatest = fast.Current.GatewayVersion;
|
||||
while (fastLatest != "s3")
|
||||
{
|
||||
Assert.True(await fast.MoveNextAsync().AsTask().WaitAsync(TestTimeout));
|
||||
fastLatest = fast.Current.GatewayVersion;
|
||||
}
|
||||
|
||||
Assert.True(await slow.MoveNextAsync().AsTask().WaitAsync(TestTimeout));
|
||||
Assert.Equal("s3", slow.Current.GatewayVersion);
|
||||
|
||||
await fastCancellation.CancelAsync();
|
||||
await slowCancellation.CancelAsync();
|
||||
await DrainAsync(fast, Task.FromResult(true));
|
||||
await DrainAsync(slow, Task.FromResult(true));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A fault in the underlying watch is surfaced to the current viewers rather
|
||||
/// than silently hanging them, and it resets the feed so the next viewer
|
||||
/// starts a fresh pump instead of attaching to a dead one.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task WatchAsync_WhenUnderlyingWatchFaults_PropagatesAndRestartsForTheNextSubscriber()
|
||||
{
|
||||
FakeSnapshotService service = new();
|
||||
DashboardSnapshotFeed feed = new(service);
|
||||
|
||||
using CancellationTokenSource firstCancellation = new();
|
||||
IAsyncEnumerator<DashboardSnapshot> first =
|
||||
feed.WatchAsync(firstCancellation.Token).GetAsyncEnumerator(firstCancellation.Token);
|
||||
Task<bool> firstMove = first.MoveNextAsync().AsTask();
|
||||
await WaitUntilAsync(() => service.EnumerationCount >= 1);
|
||||
|
||||
service.Fault(new InvalidOperationException("simulated snapshot source failure"));
|
||||
|
||||
InvalidOperationException failure =
|
||||
await Assert.ThrowsAsync<InvalidOperationException>(() => firstMove.WaitAsync(TestTimeout));
|
||||
Assert.Equal("simulated snapshot source failure", failure.Message);
|
||||
await first.DisposeAsync();
|
||||
|
||||
using CancellationTokenSource secondCancellation = new();
|
||||
IAsyncEnumerator<DashboardSnapshot> second =
|
||||
feed.WatchAsync(secondCancellation.Token).GetAsyncEnumerator(secondCancellation.Token);
|
||||
Task<bool> secondMove = second.MoveNextAsync().AsTask();
|
||||
await WaitUntilAsync(() => service.EnumerationCount >= 2);
|
||||
|
||||
await PushUntilAsync(service, secondMove);
|
||||
Assert.True(await secondMove.WaitAsync(TestTimeout));
|
||||
|
||||
await secondCancellation.CancelAsync();
|
||||
await DrainAsync(second, secondMove);
|
||||
}
|
||||
|
||||
/// <summary>Builds a snapshot whose version string identifies it in assertions.</summary>
|
||||
/// <param name="version">Identity marker carried in <c>GatewayVersion</c>.</param>
|
||||
/// <returns>A snapshot carrying the supplied identity marker.</returns>
|
||||
private static DashboardSnapshot CreateSnapshot(string version)
|
||||
{
|
||||
return new DashboardSnapshot(
|
||||
GeneratedAt: DateTimeOffset.UnixEpoch,
|
||||
GatewayStartedAt: DateTimeOffset.UnixEpoch,
|
||||
GatewayUptime: TimeSpan.Zero,
|
||||
GatewayStatus: "Healthy",
|
||||
GatewayVersion: version,
|
||||
Sessions: Array.Empty<DashboardSessionSummary>(),
|
||||
Workers: Array.Empty<DashboardWorkerSummary>(),
|
||||
Metrics: Array.Empty<DashboardMetricSummary>(),
|
||||
Faults: Array.Empty<DashboardFaultSummary>(),
|
||||
ApiKeys: Array.Empty<DashboardApiKeySummary>(),
|
||||
Configuration: null!,
|
||||
Galaxy: null!);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Pushes snapshots until the supplied task completes, so a test never
|
||||
/// depends on a single push landing after a subscriber has registered.
|
||||
/// </summary>
|
||||
/// <param name="service">Fake snapshot source to push through.</param>
|
||||
/// <param name="until">Task whose completion stops the pushes.</param>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
private static async Task PushUntilAsync(FakeSnapshotService service, Task until)
|
||||
{
|
||||
using CancellationTokenSource cancellation = new(TestTimeout);
|
||||
int sequence = 0;
|
||||
while (!until.IsCompleted)
|
||||
{
|
||||
service.Push(CreateSnapshot($"push-{sequence++}"));
|
||||
await Task.Delay(TimeSpan.FromMilliseconds(5), cancellation.Token);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Observes the cancellation of a pending enumeration and disposes the
|
||||
/// enumerator, mirroring how <c>await foreach</c> unwinds a cancelled watch.
|
||||
/// </summary>
|
||||
/// <param name="enumerator">Enumerator to unwind.</param>
|
||||
/// <param name="pending">The in-flight move, if any.</param>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
private static async Task DrainAsync(IAsyncEnumerator<DashboardSnapshot> enumerator, Task pending)
|
||||
{
|
||||
try
|
||||
{
|
||||
await pending.WaitAsync(TestTimeout);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
await enumerator.DisposeAsync();
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task WaitUntilAsync(Func<bool> predicate)
|
||||
{
|
||||
using CancellationTokenSource cancellation = new(TestTimeout);
|
||||
while (!predicate())
|
||||
{
|
||||
await Task.Delay(TimeSpan.FromMilliseconds(5), cancellation.Token);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Snapshot source under the feed's control: counts enumerations, records how
|
||||
/// each one ended, and lets the test drive snapshots (or a fault) into the
|
||||
/// live enumeration.
|
||||
/// </summary>
|
||||
private sealed class FakeSnapshotService : IDashboardSnapshotService
|
||||
{
|
||||
private readonly Channel<object> _pushes = Channel.CreateUnbounded<object>();
|
||||
private int _enumerationCount;
|
||||
private int _completedEnumerationCount;
|
||||
private volatile bool _lastEnumerationWasCancelled;
|
||||
|
||||
/// <summary>Gets the number of times the feed started enumerating this source.</summary>
|
||||
public int EnumerationCount => Volatile.Read(ref _enumerationCount);
|
||||
|
||||
/// <summary>Gets the number of enumerations that have finished (cancelled, faulted, or completed).</summary>
|
||||
public int CompletedEnumerationCount => Volatile.Read(ref _completedEnumerationCount);
|
||||
|
||||
/// <summary>Gets a value indicating whether the most recently finished enumeration ended cancelled.</summary>
|
||||
public bool LastEnumerationWasCancelled => _lastEnumerationWasCancelled;
|
||||
|
||||
/// <summary>Queues a snapshot for the live enumeration to yield.</summary>
|
||||
/// <param name="snapshot">Snapshot to yield.</param>
|
||||
public void Push(DashboardSnapshot snapshot) => _pushes.Writer.TryWrite(snapshot);
|
||||
|
||||
/// <summary>Queues a failure for the live enumeration to throw.</summary>
|
||||
/// <param name="error">Exception to throw from the enumeration.</param>
|
||||
public void Fault(Exception error) => _pushes.Writer.TryWrite(error);
|
||||
|
||||
/// <inheritdoc />
|
||||
public DashboardSnapshot GetSnapshot() => CreateSnapshot("current");
|
||||
|
||||
/// <inheritdoc />
|
||||
public async IAsyncEnumerable<DashboardSnapshot> WatchSnapshotsAsync(
|
||||
[EnumeratorCancellation] CancellationToken cancellationToken)
|
||||
{
|
||||
Interlocked.Increment(ref _enumerationCount);
|
||||
try
|
||||
{
|
||||
await foreach (object item in _pushes.Reader.ReadAllAsync(cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
if (item is Exception error)
|
||||
{
|
||||
throw error;
|
||||
}
|
||||
|
||||
yield return (DashboardSnapshot)item;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
_lastEnumerationWasCancelled = cancellationToken.IsCancellationRequested;
|
||||
Interlocked.Increment(ref _completedEnumerationCount);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user