using System.Runtime.CompilerServices; using Microsoft.AspNetCore.SignalR; using Microsoft.Extensions.Logging.Abstractions; using ZB.MOM.WW.MxGateway.Server.Dashboard; using ZB.MOM.WW.MxGateway.Server.Dashboard.Hubs; namespace ZB.MOM.WW.MxGateway.Tests.Gateway.Dashboard; public sealed class DashboardSnapshotPublisherTests { private static readonly TimeSpan TestTimeout = TimeSpan.FromSeconds(5); private static readonly TimeSpan IdlePollInterval = TimeSpan.FromMilliseconds(10); /// /// A transient failure inside /// must not /// end the BackgroundService; the publisher must wait the configured /// reconnect delay and then re-open the subscription. Before the fix, /// the publisher exited on the first non-cancellation exception and /// the dashboard's snapshot stream went silent until process restart. /// /// A task that represents the asynchronous operation. [Fact] public async Task ExecuteAsync_WhenSnapshotServiceThrowsOnce_ReconnectsAfterDelay() { ThrowOnceThenYieldSnapshotService snapshotService = new(); RecordingHubContext hubContext = new(); TimeSpan reconnectDelay = TimeSpan.FromMilliseconds(50); DashboardSnapshotPublisher publisher = new( snapshotService, hubContext, ConnectedCounter(), NullLogger.Instance, reconnectDelay, IdlePollInterval); using CancellationTokenSource cts = new(); Task execute = publisher.StartAsync(cts.Token); await execute.WaitAsync(TestTimeout); // The publisher's first WatchSnapshotsAsync call throws; the second // call yields one snapshot. We block here until the publisher has // made the second subscribe attempt AND broadcast its first // snapshot — proving the publisher did NOT exit on the throw. await WaitUntilAsync(() => snapshotService.SubscribeCount >= 2); await WaitUntilAsync(() => hubContext.SendCount >= 1); DateTimeOffset firstThrowAt = snapshotService.FirstThrowAt ?? throw new InvalidOperationException("First subscribe did not record a throw timestamp."); DateTimeOffset secondSubscribeAt = snapshotService.SecondSubscribeAt ?? throw new InvalidOperationException("Second subscribe did not record a timestamp."); await cts.CancelAsync(); await publisher.StopAsync(CancellationToken.None); Assert.True(snapshotService.SubscribeCount >= 2, $"Expected at least 2 subscribe calls, got {snapshotService.SubscribeCount}."); Assert.True(hubContext.SendCount >= 1); // The gap is measured from the moment the first subscribe actually // threw (inside the fake) to the moment the second subscribe began // (also inside the fake). This isolates the publisher's // Task.Delay(reconnectDelay) — no StartAsync / scheduling overhead in // the baseline. The 10ms slack absorbs Task.Delay's coarse Windows // timer quantum (~15ms) when the underlying scheduler wakes early. TimeSpan gap = secondSubscribeAt - firstThrowAt; Assert.True(gap >= reconnectDelay - TimeSpan.FromMilliseconds(10), $"Expected reconnect gap >= {reconnectDelay.TotalMilliseconds}ms; got {gap.TotalMilliseconds}ms."); } /// /// Sanity: a normal completion of WatchSnapshotsAsync (no exception) /// also reconnects after the delay — exits only on host shutdown. /// /// A task that represents the asynchronous operation. [Fact] public async Task ExecuteAsync_WhenSnapshotServiceCompletes_ReconnectsAfterDelay() { CompleteImmediatelySnapshotService snapshotService = new(); RecordingHubContext hubContext = new(); TimeSpan reconnectDelay = TimeSpan.FromMilliseconds(50); DashboardSnapshotPublisher publisher = new( snapshotService, hubContext, ConnectedCounter(), NullLogger.Instance, reconnectDelay, IdlePollInterval); using CancellationTokenSource cts = new(); Task execute = publisher.StartAsync(cts.Token); await execute.WaitAsync(TestTimeout); await WaitUntilAsync(() => snapshotService.SubscribeCount >= 2); await cts.CancelAsync(); await publisher.StopAsync(CancellationToken.None); Assert.True(snapshotService.SubscribeCount >= 2); } /// /// With no dashboard connected there is nobody to broadcast to, so the publisher must /// not advance the snapshot enumerator at all — every pull costs a registry snapshot and /// sort, a locked metrics dictionary copy, and periodically a SQLite key-table read. /// The first viewer to connect resumes the tick. /// /// A task that represents the asynchronous operation. [Fact] public async Task ExecuteAsync_WhenNoHubConnections_DoesNotPullSnapshots() { CountingSnapshotService snapshotService = new(); RecordingHubContext hubContext = new(); DashboardSnapshotHubConnectionCounter connectionCounter = new(); DashboardSnapshotPublisher publisher = new( snapshotService, hubContext, connectionCounter, NullLogger.Instance, TimeSpan.FromMilliseconds(50), IdlePollInterval); using CancellationTokenSource cts = new(); await publisher.StartAsync(cts.Token).WaitAsync(TestTimeout); // Long enough for many idle polls at IdlePollInterval. await Task.Delay(TimeSpan.FromMilliseconds(250)); Assert.Equal(0, snapshotService.PullCount); Assert.Equal(0, hubContext.SendCount); connectionCounter.Increment(); await WaitUntilAsync(() => hubContext.SendCount >= 1); await cts.CancelAsync(); await publisher.StopAsync(CancellationToken.None); Assert.True(snapshotService.PullCount >= 1); } /// Creates a connection counter that already has one live viewer. /// A counter reporting a single connection. private static DashboardSnapshotHubConnectionCounter ConnectedCounter() { DashboardSnapshotHubConnectionCounter counter = new(); counter.Increment(); return counter; } private static async Task WaitUntilAsync(Func predicate) { using CancellationTokenSource cancellationTokenSource = new(TestTimeout); while (!predicate()) { await Task.Delay(TimeSpan.FromMilliseconds(5), cancellationTokenSource.Token); } } private sealed class ThrowOnceThenYieldSnapshotService : IDashboardSnapshotService { /// Gets the number of subscription attempts. public int SubscribeCount { get; private set; } /// /// The wall-clock instant the first WatchSnapshotsAsync throws. /// The reconnect-gap assertion is measured against this timestamp (NOT the /// pre-StartAsync wall clock) so scheduling overhead is not baselined /// into the lower bound. /// public DateTimeOffset? FirstThrowAt { get; private set; } /// Gets the wall-clock instant of the second subscription attempt. public DateTimeOffset? SecondSubscribeAt { get; private set; } /// public DashboardSnapshot GetSnapshot() { return null!; } /// public async IAsyncEnumerable WatchSnapshotsAsync( [EnumeratorCancellation] CancellationToken cancellationToken) { SubscribeCount++; int call = SubscribeCount; if (call == 1) { // First call: throw after a brief yield so the publisher // observes us as a live producer that failed. await Task.Yield(); FirstThrowAt = DateTimeOffset.UtcNow; throw new InvalidOperationException("simulated transient snapshot failure"); } SecondSubscribeAt = DateTimeOffset.UtcNow; yield return GetSnapshot(); // Stay open until cancelled so the publisher's inner await // foreach doesn't immediately re-loop. try { await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken).ConfigureAwait(false); } catch (OperationCanceledException) { } } } private sealed class CompleteImmediatelySnapshotService : IDashboardSnapshotService { /// Gets the number of subscription attempts. public int SubscribeCount { get; private set; } /// public DashboardSnapshot GetSnapshot() { return null!; } /// #pragma warning disable CS1998 // async without await — IAsyncEnumerable contract requires async signature public async IAsyncEnumerable WatchSnapshotsAsync( [EnumeratorCancellation] CancellationToken cancellationToken) #pragma warning restore CS1998 { SubscribeCount++; // Yield nothing and complete immediately — simulates a transient // upstream disconnect that completes cleanly. yield break; } } private sealed class CountingSnapshotService : IDashboardSnapshotService { private int _pullCount; /// Gets the number of snapshots the publisher pulled from this source. public int PullCount => Volatile.Read(ref _pullCount); /// public DashboardSnapshot GetSnapshot() { return null!; } /// public async IAsyncEnumerable WatchSnapshotsAsync( [EnumeratorCancellation] CancellationToken cancellationToken) { while (!cancellationToken.IsCancellationRequested) { // Short cadence so a listening publisher pulls quickly; the counter only // moves when the publisher actually advances the enumerator. await Task.Delay(TimeSpan.FromMilliseconds(5), cancellationToken).ConfigureAwait(false); Interlocked.Increment(ref _pullCount); yield return GetSnapshot(); } } } private sealed class RecordingHubContext : IHubContext { private readonly RecordingHubClients _clients = new(); /// Gets the hub clients. public IHubClients Clients => _clients; /// Gets the group manager. public IGroupManager Groups { get; } = new NoopGroupManager(); /// Gets the number of send calls recorded. public int SendCount => _clients.AllProxy.SendCount; } private sealed class RecordingHubClients : IHubClients { /// Gets the recording client proxy for all clients. public RecordingClientProxy AllProxy { get; } = new(); /// Gets a client proxy targeting all clients. public IClientProxy All => AllProxy; /// Gets a client proxy excluding specified connections. /// Connection identifiers to exclude. /// The recording client proxy shared by this fake. public IClientProxy AllExcept(IReadOnlyList excludedConnectionIds) => AllProxy; /// Gets a client proxy for a specific connection. /// The connection identifier. /// The recording client proxy shared by this fake. public IClientProxy Client(string connectionId) => AllProxy; /// Gets a client proxy for specified connections. /// The connection identifiers. /// The recording client proxy shared by this fake. public IClientProxy Clients(IReadOnlyList connectionIds) => AllProxy; /// Gets a client proxy for a group. /// The group name. /// The recording client proxy shared by this fake. public IClientProxy Group(string groupName) => AllProxy; /// Gets a client proxy for a group excluding specified connections. /// The group name. /// Connection identifiers to exclude. /// The recording client proxy shared by this fake. public IClientProxy GroupExcept(string groupName, IReadOnlyList excludedConnectionIds) => AllProxy; /// Gets a client proxy for specified groups. /// The group names. /// The recording client proxy shared by this fake. public IClientProxy Groups(IReadOnlyList groupNames) => AllProxy; /// Gets a client proxy for a specific user. /// The user identifier. /// The recording client proxy shared by this fake. public IClientProxy User(string userId) => AllProxy; /// Gets a client proxy for specified users. /// The user identifiers. /// The recording client proxy shared by this fake. public IClientProxy Users(IReadOnlyList userIds) => AllProxy; } private sealed class RecordingClientProxy : IClientProxy { private int _sendCount; /// Gets the number of send calls recorded. public int SendCount => Volatile.Read(ref _sendCount); /// Records a send call and completes asynchronously. /// The SignalR method name. /// The method arguments. /// Token to observe for cancellation. /// A task that represents the asynchronous operation. public Task SendCoreAsync(string method, object?[] args, CancellationToken cancellationToken = default) { Interlocked.Increment(ref _sendCount); return Task.CompletedTask; } } private sealed class NoopGroupManager : IGroupManager { /// Completes immediately without performing group addition. /// The connection identifier. /// The group name. /// Token to observe for cancellation. /// A task that represents the asynchronous operation. public Task AddToGroupAsync(string connectionId, string groupName, CancellationToken cancellationToken = default) => Task.CompletedTask; /// Completes immediately without performing group removal. /// The connection identifier. /// The group name. /// Token to observe for cancellation. /// A task that represents the asynchronous operation. public Task RemoveFromGroupAsync(string connectionId, string groupName, CancellationToken cancellationToken = default) => Task.CompletedTask; } }