test: fake time for dashboard snapshot publisher; signal-based waits (mxaccessgw#205)

DashboardSnapshotPublisher takes an optional TimeProvider (default
TimeProvider.System) driving the idle poll, the reconnect delay and, via
BoundedHubSend, the send timeout. Its tests now run on a fake clock with
exact assertions (reconnect gap == delay, 25 idle polls pull nothing, a
wedged send is abandoned at the fake send timeout).

DashboardSnapshotFeed has no time dependence, so no seam was added; its
tests lose the 100 ms sleep and poll loops in favour of CountSignal
signals. The Task.Delay(25) poll copies in the alarm-monitor tests and
several 5-10 ms poll loops now share TestSupport/Eventually.

Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
Joseph Doherty
2026-11-09 13:05:43 -05:00
parent 0c0c7b86f2
commit 82d29beafd
16 changed files with 262 additions and 275 deletions
@@ -357,28 +357,19 @@ public sealed class AlarmFailoverEndToEndTests
await monitor.StopAsync(CancellationToken.None);
}
private static async Task<AlarmFeedMessage> WaitForAsync(
private static Task<AlarmFeedMessage> WaitForAsync(
List<AlarmFeedMessage> received,
Func<AlarmFeedMessage, bool> predicate,
TimeSpan timeout)
{
DateTime deadline = DateTime.UtcNow + timeout;
while (DateTime.UtcNow < deadline)
{
lock (received)
=> Eventually.WaitForAsync(
() =>
{
AlarmFeedMessage? match = received.FirstOrDefault(predicate);
if (match is not null)
lock (received)
{
return match;
return received.FirstOrDefault(predicate);
}
}
await Task.Delay(25);
}
throw new TimeoutException("No matching AlarmFeedMessage was received in time.");
}
},
timeout);
/// <summary><see cref="IAlarmWatchListResolver"/> that returns a fixed watch-list.</summary>
private sealed class StubWatchListResolver(IReadOnlyList<AlarmSubtagTarget> targets) : IAlarmWatchListResolver
@@ -603,21 +603,8 @@ public sealed class AlarmTruncationSignalTests
alarms);
}
private static async Task WaitUntilAsync(Func<bool> condition, TimeSpan timeout)
{
DateTime deadline = DateTime.UtcNow + timeout;
while (DateTime.UtcNow < deadline)
{
if (condition())
{
return;
}
await Task.Delay(25);
}
throw new TimeoutException("Condition was not met in time.");
}
private static Task WaitUntilAsync(Func<bool> condition, TimeSpan timeout)
=> Eventually.WaitUntilAsync(condition, timeout);
/// <summary><see cref="IAlarmWatchListResolver"/> that resolves an empty watch-list.</summary>
private sealed class StubWatchListResolver : IAlarmWatchListResolver
@@ -515,44 +515,22 @@ public sealed class GatewayAlarmMonitorAttachOrderTests
return mxEvent;
}
private static async Task<AlarmFeedMessage> WaitForAsync(
private static Task<AlarmFeedMessage> WaitForAsync(
List<AlarmFeedMessage> received,
Func<AlarmFeedMessage, bool> predicate,
TimeSpan timeout)
{
DateTime deadline = DateTime.UtcNow + timeout;
while (DateTime.UtcNow < deadline)
{
lock (received)
=> Eventually.WaitForAsync(
() =>
{
AlarmFeedMessage? match = received.FirstOrDefault(predicate);
if (match is not null)
lock (received)
{
return match;
return received.FirstOrDefault(predicate);
}
}
},
timeout);
await Task.Delay(25);
}
throw new TimeoutException("No matching AlarmFeedMessage was received in time.");
}
private static async Task WaitUntilAsync(Func<bool> condition, TimeSpan timeout)
{
DateTime deadline = DateTime.UtcNow + timeout;
while (DateTime.UtcNow < deadline)
{
if (condition())
{
return;
}
await Task.Delay(25);
}
throw new TimeoutException("Condition was not met in time.");
}
private static Task WaitUntilAsync(Func<bool> condition, TimeSpan timeout)
=> Eventually.WaitUntilAsync(condition, timeout);
/// <summary><see cref="IAlarmWatchListResolver"/> that resolves an empty watch-list.</summary>
private sealed class StubWatchListResolver : IAlarmWatchListResolver
@@ -896,44 +896,22 @@ public sealed class GatewayAlarmMonitorProviderModeTests
NullLogger<GatewayAlarmMonitor>.Instance);
}
private static async Task<AlarmFeedMessage> WaitForAsync(
private static Task<AlarmFeedMessage> WaitForAsync(
List<AlarmFeedMessage> received,
Func<AlarmFeedMessage, bool> predicate,
TimeSpan timeout)
{
DateTime deadline = DateTime.UtcNow + timeout;
while (DateTime.UtcNow < deadline)
{
lock (received)
=> Eventually.WaitForAsync(
() =>
{
AlarmFeedMessage? match = received.FirstOrDefault(predicate);
if (match is not null)
lock (received)
{
return match;
return received.FirstOrDefault(predicate);
}
}
},
timeout);
await Task.Delay(25);
}
throw new TimeoutException("No matching AlarmFeedMessage was received in time.");
}
private static async Task WaitUntilAsync(Func<bool> condition, TimeSpan timeout)
{
DateTime deadline = DateTime.UtcNow + timeout;
while (DateTime.UtcNow < deadline)
{
if (condition())
{
return;
}
await Task.Delay(25);
}
throw new TimeoutException("Condition was not met in time.");
}
private static Task WaitUntilAsync(Func<bool> condition, TimeSpan timeout)
=> Eventually.WaitUntilAsync(condition, timeout);
/// <summary>
/// <see cref="IAlarmWatchListResolver"/> that hands out a scripted sequence of
@@ -6,6 +6,7 @@ using ZB.MOM.WW.MxGateway.Contracts.Proto;
using ZB.MOM.WW.MxGateway.Server.Alarms;
using ZB.MOM.WW.MxGateway.Server.Configuration;
using ZB.MOM.WW.MxGateway.Server.Dashboard.Hubs;
using ZB.MOM.WW.MxGateway.Tests.TestSupport;
namespace ZB.MOM.WW.MxGateway.Tests.Gateway.Dashboard;
@@ -241,14 +242,8 @@ public sealed class AlarmsHubPublisherTests
return hubContext;
}
private static async Task WaitUntilAsync(Func<bool> predicate)
{
using CancellationTokenSource cancellationTokenSource = new(TestTimeout);
while (!predicate())
{
await Task.Delay(TimeSpan.FromMilliseconds(5), cancellationTokenSource.Token);
}
}
private static Task WaitUntilAsync(Func<bool> predicate)
=> Eventually.WaitUntilAsync(predicate, TestTimeout);
/// <summary>Builds a value-bearing <c>transition</c> feed message.</summary>
/// <returns>The message.</returns>
@@ -1,6 +1,7 @@
using System.Runtime.CompilerServices;
using System.Threading.Channels;
using ZB.MOM.WW.MxGateway.Server.Dashboard;
using ZB.MOM.WW.MxGateway.Tests.TestSupport;
namespace ZB.MOM.WW.MxGateway.Tests.Gateway.Dashboard;
@@ -23,7 +24,7 @@ public sealed class DashboardSnapshotFeedTests
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task WatchAsync_WithNoSubscribers_NeverEnumeratesUnderlyingWatch()
public Task WatchAsync_WithNoSubscribers_NeverEnumeratesUnderlyingWatch()
{
FakeSnapshotService service = new();
DashboardSnapshotFeed feed = new(service);
@@ -31,9 +32,9 @@ public sealed class DashboardSnapshotFeedTests
// Obtaining the enumerable without enumerating it must not subscribe
// either: the pump starts on the first MoveNextAsync, not before.
_ = feed.WatchAsync(CancellationToken.None);
await Task.Delay(TimeSpan.FromMilliseconds(100));
Assert.Equal(0, service.EnumerationCount);
return Task.CompletedTask;
}
/// <summary>
@@ -53,7 +54,7 @@ public sealed class DashboardSnapshotFeedTests
IAsyncEnumerator<DashboardSnapshot> first =
feed.WatchAsync(firstCancellation.Token).GetAsyncEnumerator(firstCancellation.Token);
Task<bool> firstMove = first.MoveNextAsync().AsTask();
await WaitUntilAsync(() => service.EnumerationCount >= 1);
await service.Enumerations.WaitForAsync(1);
IAsyncEnumerator<DashboardSnapshot> second =
feed.WatchAsync(secondCancellation.Token).GetAsyncEnumerator(secondCancellation.Token);
@@ -92,19 +93,19 @@ public sealed class DashboardSnapshotFeedTests
IAsyncEnumerator<DashboardSnapshot> first =
feed.WatchAsync(firstCancellation.Token).GetAsyncEnumerator(firstCancellation.Token);
Task<bool> firstMove = first.MoveNextAsync().AsTask();
await WaitUntilAsync(() => service.EnumerationCount >= 1);
await service.Enumerations.WaitForAsync(1);
await firstCancellation.CancelAsync();
await DrainAsync(first, firstMove);
await WaitUntilAsync(() => service.CompletedEnumerationCount >= 1);
await service.Completions.WaitForAsync(1);
Assert.True(service.LastEnumerationWasCancelled);
using CancellationTokenSource secondCancellation = new();
IAsyncEnumerator<DashboardSnapshot> second =
feed.WatchAsync(secondCancellation.Token).GetAsyncEnumerator(secondCancellation.Token);
Task<bool> secondMove = second.MoveNextAsync().AsTask();
await WaitUntilAsync(() => service.EnumerationCount >= 2);
await service.Enumerations.WaitForAsync(2);
Assert.Equal(2, service.EnumerationCount);
@@ -131,7 +132,7 @@ public sealed class DashboardSnapshotFeedTests
IAsyncEnumerator<DashboardSnapshot> fast =
feed.WatchAsync(fastCancellation.Token).GetAsyncEnumerator(fastCancellation.Token);
Task<bool> fastMove = fast.MoveNextAsync().AsTask();
await WaitUntilAsync(() => service.EnumerationCount >= 1);
await service.Enumerations.WaitForAsync(1);
// The slow subscriber registers but never advances until the very end.
IAsyncEnumerator<DashboardSnapshot> slow =
@@ -179,7 +180,7 @@ public sealed class DashboardSnapshotFeedTests
IAsyncEnumerator<DashboardSnapshot> first =
feed.WatchAsync(firstCancellation.Token).GetAsyncEnumerator(firstCancellation.Token);
Task<bool> firstMove = first.MoveNextAsync().AsTask();
await WaitUntilAsync(() => service.EnumerationCount >= 1);
await service.Enumerations.WaitForAsync(1);
service.Fault(new InvalidOperationException("simulated snapshot source failure"));
@@ -192,7 +193,7 @@ public sealed class DashboardSnapshotFeedTests
IAsyncEnumerator<DashboardSnapshot> second =
feed.WatchAsync(secondCancellation.Token).GetAsyncEnumerator(secondCancellation.Token);
Task<bool> secondMove = second.MoveNextAsync().AsTask();
await WaitUntilAsync(() => service.EnumerationCount >= 2);
await service.Enumerations.WaitForAsync(2);
await PushUntilAsync(service, secondMove);
Assert.True(await secondMove.WaitAsync(TestTimeout));
@@ -219,7 +220,7 @@ public sealed class DashboardSnapshotFeedTests
IAsyncEnumerator<DashboardSnapshot> first =
feed.WatchAsync(firstCancellation.Token).GetAsyncEnumerator(firstCancellation.Token);
Task<bool> firstMove = first.MoveNextAsync().AsTask();
await WaitUntilAsync(() => service.EnumerationCount >= 1);
await service.Enumerations.WaitForAsync(1);
service.Fault(new InvalidOperationException("simulated snapshot source failure"));
@@ -245,7 +246,7 @@ public sealed class DashboardSnapshotFeedTests
// ...and the one that joined mid-unwind is served by a restarted enumeration
// instead of inheriting the failure.
await WaitUntilAsync(() => service.EnumerationCount >= 2);
await service.Enumerations.WaitForAsync(2);
await PushUntilAsync(service, secondMove);
Assert.True(await secondMove.WaitAsync(TestTimeout));
Assert.StartsWith("push-", second.Current.GatewayVersion, StringComparison.Ordinal);
@@ -273,7 +274,7 @@ public sealed class DashboardSnapshotFeedTests
IAsyncEnumerator<DashboardSnapshot> oldSubscriber =
feed.WatchAsync(oldCancellation.Token).GetAsyncEnumerator(oldCancellation.Token);
Task<bool> oldMove = oldSubscriber.MoveNextAsync().AsTask();
await WaitUntilAsync(() => service.EnumerationCount >= 1);
await service.Enumerations.WaitForAsync(1);
service.Fault(new InvalidOperationException("simulated snapshot source failure"));
await service.DisposalReached.WaitAsync(TestTimeout);
@@ -287,8 +288,9 @@ public sealed class DashboardSnapshotFeedTests
await joinerCancellation.CancelAsync();
// The joiner's unwind is a continuation of its cancelled channel read; give it time to
// run its unsubscribe before the dying pump is released. Only the interleaving depends
// on this delay — the assertions below hold either way.
// run its unsubscribe before the dying pump is released. The unsubscribe cannot be
// awaited (it waits on the parked pump) and exposes no signal, so this stays a real
// delay; only the interleaving depends on it — the assertions below hold either way.
await Task.Delay(TimeSpan.FromMilliseconds(150));
service.ReleaseDisposal();
@@ -312,7 +314,7 @@ public sealed class DashboardSnapshotFeedTests
feed.WatchAsync(nextCancellation.Token).GetAsyncEnumerator(nextCancellation.Token);
Task<bool> nextMove = next.MoveNextAsync().AsTask();
await WaitUntilAsync(() => service.EnumerationCount > enumerationsBefore);
await service.Enumerations.WaitForAsync(enumerationsBefore + 1);
Assert.True(await nextMove.WaitAsync(TestTimeout));
Assert.Equal("orphan-check", next.Current.GatewayVersion);
@@ -384,15 +386,6 @@ public sealed class DashboardSnapshotFeedTests
}
}
private static async Task WaitUntilAsync(Func<bool> predicate)
{
using CancellationTokenSource cancellation = new(TestTimeout);
while (!predicate())
{
await Task.Delay(TimeSpan.FromMilliseconds(5), cancellation.Token);
}
}
/// <summary>
/// Snapshot source under the feed's control: counts enumerations, records how
/// each one ended, and lets the test drive snapshots (or a fault) into the
@@ -403,13 +396,17 @@ public sealed class DashboardSnapshotFeedTests
private readonly Channel<object> _pushes = Channel.CreateUnbounded<object>();
private readonly TaskCompletionSource _disposalReached = new(TaskCreationOptions.RunContinuationsAsynchronously);
private readonly TaskCompletionSource _disposalRelease = new(TaskCreationOptions.RunContinuationsAsynchronously);
private int _enumerationCount;
private int _completedEnumerationCount;
private volatile bool _lastEnumerationWasCancelled;
private volatile bool _holdDisposal;
/// <summary>Gets the number of times the feed started enumerating this source.</summary>
public int EnumerationCount => Volatile.Read(ref _enumerationCount);
public int EnumerationCount => Enumerations.Count;
/// <summary>Gets the signal raised each time the feed starts enumerating this source.</summary>
public CountSignal Enumerations { get; } = new();
/// <summary>Gets the signal raised each time an enumeration finishes.</summary>
public CountSignal Completions { get; } = new();
/// <summary>Gets a task that completes when a held enumerator disposal is reached.</summary>
public Task DisposalReached => _disposalReached.Task;
@@ -421,7 +418,7 @@ public sealed class DashboardSnapshotFeedTests
public int PendingPushCount => _pushes.Reader.Count;
/// <summary>Gets the number of enumerations that have finished (cancelled, faulted, or completed).</summary>
public int CompletedEnumerationCount => Volatile.Read(ref _completedEnumerationCount);
public int CompletedEnumerationCount => Completions.Count;
/// <summary>Gets a value indicating whether the most recently finished enumeration ended cancelled.</summary>
public bool LastEnumerationWasCancelled => _lastEnumerationWasCancelled;
@@ -464,7 +461,7 @@ public sealed class DashboardSnapshotFeedTests
private async IAsyncEnumerable<DashboardSnapshot> EnumerateAsync(
[EnumeratorCancellation] CancellationToken cancellationToken)
{
Interlocked.Increment(ref _enumerationCount);
Enumerations.Signal();
try
{
await foreach (object item in _pushes.Reader.ReadAllAsync(cancellationToken).ConfigureAwait(false))
@@ -480,7 +477,7 @@ public sealed class DashboardSnapshotFeedTests
finally
{
_lastEnumerationWasCancelled = cancellationToken.IsCancellationRequested;
Interlocked.Increment(ref _completedEnumerationCount);
Completions.Signal();
}
}
@@ -1,8 +1,10 @@
using System.Runtime.CompilerServices;
using Microsoft.AspNetCore.SignalR;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Time.Testing;
using ZB.MOM.WW.MxGateway.Server.Dashboard;
using ZB.MOM.WW.MxGateway.Server.Dashboard.Hubs;
using ZB.MOM.WW.MxGateway.Tests.TestSupport;
namespace ZB.MOM.WW.MxGateway.Tests.Gateway.Dashboard;
@@ -18,97 +20,94 @@ public sealed class DashboardSnapshotPublisherTests
/// reconnect delay and then re-open the subscription. Before the fix,
/// the publisher exited on the first non-cancellation exception and
/// the dashboard's snapshot stream went silent until process restart.
/// Driven by a fake clock, so the delay is exact and nothing sleeps.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task ExecuteAsync_WhenSnapshotServiceThrowsOnce_ReconnectsAfterDelay()
{
ThrowOnceThenYieldSnapshotService snapshotService = new();
RecordingHubContext hubContext = new();
TimeSpan reconnectDelay = TimeSpan.FromMilliseconds(50);
TickObservingTimeProvider clock = new() { WatchedDueTime = reconnectDelay };
ThrowOnceThenYieldSnapshotService snapshotService = new(clock);
RecordingHubContext hubContext = new();
DashboardSnapshotPublisher publisher = new(
snapshotService,
hubContext,
ConnectedCounter(),
NullLogger<DashboardSnapshotPublisher>.Instance,
reconnectDelay,
IdlePollInterval);
IdlePollInterval,
timeProvider: clock);
using CancellationTokenSource cts = new();
Task execute = publisher.StartAsync(cts.Token);
await execute.WaitAsync(TestTimeout);
await publisher.StartAsync(cts.Token).WaitAsync(TestTimeout);
// The publisher's first WatchSnapshotsAsync call throws; the second
// call yields one snapshot. We block here until the publisher has
// made the second subscribe attempt AND broadcast its first
// snapshot — proving the publisher did NOT exit on the throw.
await WaitUntilAsync(() => snapshotService.SubscribeCount >= 2);
await WaitUntilAsync(() => hubContext.SendCount >= 1);
// The first subscribe throws; the publisher then arms the reconnect delay on the
// fake clock. Wait for that timer so the advances below cannot be lost.
await snapshotService.Subscribes.WaitForAsync(1);
await clock.WaitForWatchedTimersAsync(1, TestTimeout);
// The delay is honoured: just short of it, no second subscribe may have happened.
clock.Advance(reconnectDelay - TimeSpan.FromMilliseconds(1));
Assert.Equal(1, snapshotService.Subscribes.Count);
clock.Advance(TimeSpan.FromMilliseconds(1));
await snapshotService.Subscribes.WaitForAsync(2);
await hubContext.Proxy.Sends.WaitForAsync(1);
await cts.CancelAsync();
await publisher.StopAsync(CancellationToken.None);
DateTimeOffset firstThrowAt = snapshotService.FirstThrowAt
?? throw new InvalidOperationException("First subscribe did not record a throw timestamp.");
DateTimeOffset secondSubscribeAt = snapshotService.SecondSubscribeAt
?? throw new InvalidOperationException("Second subscribe did not record a timestamp.");
await cts.CancelAsync();
await publisher.StopAsync(CancellationToken.None);
Assert.True(snapshotService.SubscribeCount >= 2,
$"Expected at least 2 subscribe calls, got {snapshotService.SubscribeCount}.");
Assert.True(hubContext.SendCount >= 1);
// The gap is measured from the moment the first subscribe actually
// threw (inside the fake) to the moment the second subscribe began
// (also inside the fake). This isolates the publisher's
// Task.Delay(reconnectDelay) — no StartAsync / scheduling overhead in
// the baseline. The 10ms slack absorbs Task.Delay's coarse Windows
// timer quantum (~15ms) when the underlying scheduler wakes early.
TimeSpan gap = secondSubscribeAt - firstThrowAt;
Assert.True(gap >= reconnectDelay - TimeSpan.FromMilliseconds(10),
$"Expected reconnect gap >= {reconnectDelay.TotalMilliseconds}ms; got {gap.TotalMilliseconds}ms.");
// Fake time is exact: the gap is precisely the reconnect delay.
Assert.Equal(reconnectDelay, secondSubscribeAt - firstThrowAt);
}
/// <summary>
/// Sanity: a normal completion of WatchSnapshotsAsync (no exception)
/// also reconnects after the delay — exits only on host shutdown.
/// is re-subscribed; the publisher exits only on host shutdown.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task ExecuteAsync_WhenSnapshotServiceCompletes_ReconnectsAfterDelay()
public async Task ExecuteAsync_WhenSnapshotServiceCompletes_Resubscribes()
{
CompleteImmediatelySnapshotService snapshotService = new();
RecordingHubContext hubContext = new();
TimeSpan reconnectDelay = TimeSpan.FromMilliseconds(50);
DashboardSnapshotPublisher publisher = new(
snapshotService,
hubContext,
ConnectedCounter(),
NullLogger<DashboardSnapshotPublisher>.Instance,
reconnectDelay,
IdlePollInterval);
TimeSpan.FromMilliseconds(50),
IdlePollInterval,
timeProvider: new FakeTimeProvider());
using CancellationTokenSource cts = new();
Task execute = publisher.StartAsync(cts.Token);
await execute.WaitAsync(TestTimeout);
await publisher.StartAsync(cts.Token).WaitAsync(TestTimeout);
await WaitUntilAsync(() => snapshotService.SubscribeCount >= 2);
await snapshotService.Subscribes.WaitForAsync(2);
await cts.CancelAsync();
await publisher.StopAsync(CancellationToken.None);
Assert.True(snapshotService.SubscribeCount >= 2);
Assert.True(snapshotService.Subscribes.Count >= 2);
}
/// <summary>
/// With no dashboard connected there is nobody to broadcast to, so the publisher must
/// not advance the snapshot enumerator at all — every pull costs a registry snapshot and
/// sort, a locked metrics dictionary copy, and periodically a SQLite key-table read.
/// The first viewer to connect resumes the tick.
/// The first viewer to connect resumes the tick. Many idle polls are driven on a fake clock.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task ExecuteAsync_WhenNoHubConnections_DoesNotPullSnapshots()
{
TickObservingTimeProvider clock = new() { WatchedDueTime = IdlePollInterval };
CountingSnapshotService snapshotService = new();
RecordingHubContext hubContext = new();
DashboardSnapshotHubConnectionCounter connectionCounter = new();
@@ -118,19 +117,30 @@ public sealed class DashboardSnapshotPublisherTests
connectionCounter,
NullLogger<DashboardSnapshotPublisher>.Instance,
TimeSpan.FromMilliseconds(50),
IdlePollInterval);
IdlePollInterval,
timeProvider: clock);
using CancellationTokenSource cts = new();
await publisher.StartAsync(cts.Token).WaitAsync(TestTimeout);
// Long enough for many idle polls at IdlePollInterval.
await Task.Delay(TimeSpan.FromMilliseconds(250));
// A source that is ready to yield every time it is pulled, so the only thing
// holding the publisher back is the idle gate.
snapshotService.Tick(100);
// Drive 25 idle polls; each wake-up re-arms exactly one new idle timer.
await clock.WaitForWatchedTimersAsync(1, TestTimeout);
for (int poll = 1; poll <= 25; poll++)
{
clock.Advance(IdlePollInterval);
await clock.WaitForWatchedTimersAsync(poll + 1, TestTimeout);
}
Assert.Equal(0, snapshotService.PullCount);
Assert.Equal(0, hubContext.SendCount);
Assert.Equal(0, hubContext.Proxy.Sends.Count);
connectionCounter.Increment();
await WaitUntilAsync(() => hubContext.SendCount >= 1);
clock.Advance(IdlePollInterval);
await hubContext.Proxy.Sends.WaitForAsync(1);
await cts.CancelAsync();
await publisher.StopAsync(CancellationToken.None);
@@ -141,12 +151,14 @@ public sealed class DashboardSnapshotPublisherTests
/// <summary>
/// A broadcast that never completes — one remote client whose transport is wedged —
/// is abandoned at the send timeout, so later ticks still go out instead of every
/// viewer waiting on the slowest one.
/// viewer waiting on the slowest one. The send timeout runs on the fake clock.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task ExecuteAsync_WhenABroadcastNeverCompletes_LaterTicksStillGoOut()
{
TimeSpan sendTimeout = TimeSpan.FromMilliseconds(50);
TickObservingTimeProvider clock = new() { WatchedDueTime = sendTimeout };
CountingSnapshotService snapshotService = new();
RecordingHubContext hubContext = new();
hubContext.Proxy.StallFirstSend = true;
@@ -155,13 +167,23 @@ public sealed class DashboardSnapshotPublisherTests
hubContext,
ConnectedCounter(),
NullLogger<DashboardSnapshotPublisher>.Instance,
TimeSpan.FromMilliseconds(50),
TimeSpan.FromSeconds(5),
IdlePollInterval,
sendTimeout: TimeSpan.FromMilliseconds(50));
sendTimeout: sendTimeout,
timeProvider: clock);
snapshotService.Tick(3);
using CancellationTokenSource cts = new();
await publisher.StartAsync(cts.Token).WaitAsync(TestTimeout);
await WaitUntilAsync(() => hubContext.SendCount >= 3);
// The first send is wedged and its timeout timer is armed; nothing else goes out
// until the fake clock reaches the timeout.
await hubContext.Proxy.Sends.WaitForAsync(1);
await clock.WaitForWatchedTimersAsync(1, TestTimeout);
Assert.Equal(1, hubContext.Proxy.Sends.Count);
clock.Advance(sendTimeout);
await hubContext.Proxy.Sends.WaitForAsync(3);
await cts.CancelAsync();
await publisher.StopAsync(CancellationToken.None);
@@ -176,29 +198,15 @@ public sealed class DashboardSnapshotPublisherTests
return counter;
}
private static async Task WaitUntilAsync(Func<bool> predicate)
private sealed class ThrowOnceThenYieldSnapshotService(TimeProvider clock) : IDashboardSnapshotService
{
using CancellationTokenSource cancellationTokenSource = new(TestTimeout);
while (!predicate())
{
await Task.Delay(TimeSpan.FromMilliseconds(5), cancellationTokenSource.Token);
}
}
/// <summary>Gets the subscription attempts.</summary>
public CountSignal Subscribes { get; } = new();
private sealed class ThrowOnceThenYieldSnapshotService : IDashboardSnapshotService
{
/// <summary>Gets the number of subscription attempts.</summary>
public int SubscribeCount { get; private set; }
/// <summary>
/// The wall-clock instant the first <c>WatchSnapshotsAsync</c> throws.
/// The reconnect-gap assertion is measured against this timestamp (NOT the
/// pre-<c>StartAsync</c> wall clock) so scheduling overhead is not baselined
/// into the lower bound.
/// </summary>
/// <summary>Gets the clock instant the first <c>WatchSnapshotsAsync</c> throws.</summary>
public DateTimeOffset? FirstThrowAt { get; private set; }
/// <summary>Gets the wall-clock instant of the second subscription attempt.</summary>
/// <summary>Gets the clock instant of the second subscription attempt.</summary>
public DateTimeOffset? SecondSubscribeAt { get; private set; }
/// <inheritdoc />
@@ -211,19 +219,18 @@ public sealed class DashboardSnapshotPublisherTests
public async IAsyncEnumerable<DashboardSnapshot> WatchSnapshotsAsync(
[EnumeratorCancellation] CancellationToken cancellationToken)
{
SubscribeCount++;
int call = SubscribeCount;
if (call == 1)
if (Subscribes.Count == 0)
{
// First call: throw after a brief yield so the publisher
// observes us as a live producer that failed.
await Task.Yield();
FirstThrowAt = DateTimeOffset.UtcNow;
FirstThrowAt = clock.GetUtcNow();
Subscribes.Signal();
throw new InvalidOperationException("simulated transient snapshot failure");
}
SecondSubscribeAt = DateTimeOffset.UtcNow;
SecondSubscribeAt = clock.GetUtcNow();
Subscribes.Signal();
yield return GetSnapshot();
// Stay open until cancelled so the publisher's inner await
@@ -240,8 +247,8 @@ public sealed class DashboardSnapshotPublisherTests
private sealed class CompleteImmediatelySnapshotService : IDashboardSnapshotService
{
/// <summary>Gets the number of subscription attempts.</summary>
public int SubscribeCount { get; private set; }
/// <summary>Gets the subscription attempts.</summary>
public CountSignal Subscribes { get; } = new();
/// <inheritdoc />
public DashboardSnapshot GetSnapshot()
@@ -255,7 +262,7 @@ public sealed class DashboardSnapshotPublisherTests
[EnumeratorCancellation] CancellationToken cancellationToken)
#pragma warning restore CS1998
{
SubscribeCount++;
Subscribes.Signal();
// Yield nothing and complete immediately — simulates a transient
// upstream disconnect that completes cleanly.
yield break;
@@ -264,11 +271,16 @@ public sealed class DashboardSnapshotPublisherTests
private sealed class CountingSnapshotService : IDashboardSnapshotService
{
private readonly SemaphoreSlim _ticks = new(0);
private int _pullCount;
/// <summary>Gets the number of snapshots the publisher pulled from this source.</summary>
public int PullCount => Volatile.Read(ref _pullCount);
/// <summary>Lets the source yield <paramref name="count"/> more snapshots.</summary>
/// <param name="count">Number of snapshots to allow.</param>
public void Tick(int count) => _ticks.Release(count);
/// <inheritdoc />
public DashboardSnapshot GetSnapshot()
{
@@ -281,9 +293,9 @@ public sealed class DashboardSnapshotPublisherTests
{
while (!cancellationToken.IsCancellationRequested)
{
// Short cadence so a listening publisher pulls quickly; the counter only
// moves when the publisher actually advances the enumerator.
await Task.Delay(TimeSpan.FromMilliseconds(5), cancellationToken).ConfigureAwait(false);
// The counter only moves when the publisher actually advances the enumerator
// and a tick is available.
await _ticks.WaitAsync(cancellationToken).ConfigureAwait(false);
Interlocked.Increment(ref _pullCount);
yield return GetSnapshot();
}
@@ -303,8 +315,6 @@ public sealed class DashboardSnapshotPublisherTests
/// <summary>Gets the recording proxy every client lookup resolves to.</summary>
public RecordingClientProxy Proxy => _clients.AllProxy;
/// <summary>Gets the number of send calls recorded.</summary>
public int SendCount => _clients.AllProxy.SendCount;
}
private sealed class RecordingHubClients : IHubClients
@@ -359,16 +369,14 @@ public sealed class DashboardSnapshotPublisherTests
private sealed class RecordingClientProxy : IClientProxy
{
private int _sendCount;
/// <summary>
/// Gets or sets a value indicating whether the first send never completes and
/// ignores its cancellation token — a wedged client transport.
/// </summary>
public bool StallFirstSend { get; set; }
/// <summary>Gets the number of send calls recorded.</summary>
public int SendCount => Volatile.Read(ref _sendCount);
/// <summary>Gets the recorded send calls.</summary>
public CountSignal Sends { get; } = new();
/// <summary>Records a send call and completes asynchronously.</summary>
/// <param name="method">The SignalR method name.</param>
@@ -377,8 +385,8 @@ public sealed class DashboardSnapshotPublisherTests
/// <returns>A task that represents the asynchronous operation.</returns>
public Task SendCoreAsync(string method, object?[] args, CancellationToken cancellationToken = default)
{
int count = Interlocked.Increment(ref _sendCount);
return StallFirstSend && count == 1 ? new TaskCompletionSource().Task : Task.CompletedTask;
Sends.Signal();
return StallFirstSend && Sends.Count == 1 ? new TaskCompletionSource().Task : Task.CompletedTask;
}
}
@@ -5,6 +5,7 @@ using ZB.MOM.WW.MxGateway.Server.Metrics;
using ZB.MOM.WW.MxGateway.Server.Sessions;
using ZB.MOM.WW.MxGateway.Server.Workers;
using ZB.MOM.WW.MxGateway.Tests.Gateway.Workers.Fakes;
using ZB.MOM.WW.MxGateway.Tests.TestSupport;
namespace ZB.MOM.WW.MxGateway.Tests.Gateway.Sessions;
@@ -198,19 +199,8 @@ public sealed class GatewaySessionWorkerExitTests
};
}
private static async Task WaitUntilAsync(Func<bool> condition)
{
DateTime deadline = DateTime.UtcNow + TestTimeout;
while (!condition())
{
if (DateTime.UtcNow > deadline)
{
Assert.Fail("Condition was not met within the test timeout.");
}
await Task.Delay(10);
}
}
private static Task WaitUntilAsync(Func<bool> condition)
=> Eventually.WaitUntilAsync(condition, TestTimeout);
private sealed class SingleClientFactory(IWorkerClient client) : ISessionWorkerClientFactory
{
@@ -4,6 +4,7 @@ using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Time.Testing;
using ZB.MOM.WW.MxGateway.Contracts.Proto;
using ZB.MOM.WW.MxGateway.Server.Sessions;
using ZB.MOM.WW.MxGateway.Tests.TestSupport;
namespace ZB.MOM.WW.MxGateway.Tests.Gateway.Sessions;
@@ -743,14 +744,7 @@ public sealed class SessionEventDistributorTests
Assert.Equal(SessionManagerErrorCode.EventQueueOverflow, fault.ErrorCode);
// Wait for the handler to fire (it runs on the pump thread).
await Task.Run(async () =>
{
using CancellationTokenSource cts = new(ReadTimeout);
while (Volatile.Read(ref observedIsOnlySubscriberSet) == 0)
{
await Task.Delay(10, cts.Token);
}
});
await Eventually.WaitUntilAsync(() => Volatile.Read(ref observedIsOnlySubscriberSet) != 0, ReadTimeout);
// isOnlySubscriber must be FALSE even though the internal subscriber was the ONLY
// subscriber — CountExternalSubscribers excludes it, so a FailFast policy on the
@@ -802,14 +796,7 @@ public sealed class SessionEventDistributorTests
async () => await DrainUntilFaultAsync(external.Reader));
Assert.Equal(SessionManagerErrorCode.EventQueueOverflow, fault.ErrorCode);
await Task.Run(async () =>
{
using CancellationTokenSource cts = new(ReadTimeout);
while (Volatile.Read(ref observedSet) == 0)
{
await Task.Delay(10, cts.Token);
}
});
await Eventually.WaitUntilAsync(() => Volatile.Read(ref observedSet) != 0, ReadTimeout);
// Guard: ensure the handler actually fired before asserting its observed value.
// Without this the test could pass vacuously if the overflow never triggered.
@@ -358,15 +358,9 @@ public sealed class FakeWorkerHarnessTests
return new WorkerCommand { Command = command };
}
private static async Task WaitUntilAsync(
private static Task WaitUntilAsync(
Func<bool> predicate,
TimeSpan timeout)
{
using CancellationTokenSource cancellationTokenSource = new(timeout);
while (!predicate())
{
await Task.Delay(TimeSpan.FromMilliseconds(10), cancellationTokenSource.Token);
}
}
=> Eventually.WaitUntilAsync(predicate, timeout);
}
@@ -1522,16 +1522,10 @@ public sealed class WorkerClientTests
}
}
private static async Task WaitUntilAsync(
private static Task WaitUntilAsync(
Func<bool> predicate,
TimeSpan timeout)
{
using CancellationTokenSource cancellationTokenSource = new(timeout);
while (!predicate())
{
await Task.Delay(TimeSpan.FromMilliseconds(10), cancellationTokenSource.Token);
}
}
=> Eventually.WaitUntilAsync(predicate, timeout);
private sealed class PipePair : IAsyncDisposable
{
@@ -4,6 +4,7 @@ using ZB.MOM.WW.Auth.Abstractions.ApiKeys;
using ZB.MOM.WW.MxGateway.Server.Security.Authentication;
using LibApiKeyIdentity = ZB.MOM.WW.Auth.Abstractions.ApiKeys.ApiKeyIdentity;
using ZB.MOM.WW.MxGateway.Tests.TestSupport;
namespace ZB.MOM.WW.MxGateway.Tests.Security.Authentication;
@@ -267,11 +268,7 @@ public sealed class CachingApiKeyVerifierTests
cache.Compact(1.0);
DateTime deadline = DateTime.UtcNow + TimeSpan.FromSeconds(5);
while (verifier.IndexedCacheKeyCount != 0 && DateTime.UtcNow < deadline)
{
await Task.Delay(10);
}
await Eventually.WaitUntilAsync(() => verifier.IndexedCacheKeyCount == 0, TimeSpan.FromSeconds(5));
Assert.Equal(0, verifier.IndexedCacheKeyCount);
Assert.Equal(0, verifier.IndexedKeyIdCount);
@@ -0,0 +1,36 @@
namespace ZB.MOM.WW.MxGateway.Tests.TestSupport;
/// <summary>
/// Counts events and lets a test wait for the nth one without polling. The timed wait result
/// is asserted, so a missed event fails the test instead of silently continuing (mxaccessgw#221).
/// </summary>
public sealed class CountSignal
{
private static readonly TimeSpan WaitTimeout = TimeSpan.FromSeconds(5);
private readonly SemaphoreSlim _signal = new(0);
private int _count;
private int _consumed;
/// <summary>Gets the number of events signalled so far.</summary>
public int Count => Volatile.Read(ref _count);
/// <summary>Records one event.</summary>
public void Signal()
{
Interlocked.Increment(ref _count);
_signal.Release();
}
/// <summary>Waits until at least <paramref name="total"/> events have been signalled; fails on timeout.</summary>
/// <param name="total">Cumulative event count to wait for.</param>
/// <returns>A task that represents the asynchronous operation.</returns>
public async Task WaitForAsync(int total)
{
while (_consumed < total)
{
Assert.True(await _signal.WaitAsync(WaitTimeout), $"Timed out waiting for event {total} (saw {Count}).");
_consumed++;
}
}
}
@@ -0,0 +1,46 @@
namespace ZB.MOM.WW.MxGateway.Tests.TestSupport;
/// <summary>
/// Bounded positive-wait helpers for conditions that no production type signals (for example
/// a monitor's current-alarm count). The wait fails with <see cref="TimeoutException"/> when the
/// condition never holds; use a signal (<see cref="CountSignal"/>, a TCS) where one exists.
/// </summary>
public static class Eventually
{
private static readonly TimeSpan PollInterval = TimeSpan.FromMilliseconds(2);
/// <summary>Waits until <paramref name="condition"/> holds.</summary>
/// <param name="condition">Condition to poll.</param>
/// <param name="timeout">Real-time bound on the wait.</param>
/// <returns>A task that represents the asynchronous operation.</returns>
/// <exception cref="TimeoutException">The condition did not hold within the bound.</exception>
public static async Task WaitUntilAsync(Func<bool> condition, TimeSpan timeout)
{
using CancellationTokenSource cts = new(timeout);
while (!condition())
{
try
{
await Task.Delay(PollInterval, cts.Token).ConfigureAwait(false);
}
catch (OperationCanceledException)
{
throw new TimeoutException("Condition was not met in time.");
}
}
}
/// <summary>Waits until <paramref name="probe"/> returns a non-null value and returns it.</summary>
/// <typeparam name="T">Reference type produced by the probe.</typeparam>
/// <param name="probe">Probe to poll.</param>
/// <param name="timeout">Real-time bound on the wait.</param>
/// <returns>The first non-null probe result.</returns>
/// <exception cref="TimeoutException">The probe never returned a value within the bound.</exception>
public static async Task<T> WaitForAsync<T>(Func<T?> probe, TimeSpan timeout)
where T : class
{
T? result = null;
await WaitUntilAsync(() => (result = probe()) is not null, timeout).ConfigureAwait(false);
return result!;
}
}