fix(dashboard): generation-tagged feed subscribers survive pump teardown races; drain-timeout observability

src/ZB.MOM.WW.MxGateway.Tests/Gateway/Dashboard/DashboardSnapshotFeedTests.cs

@@ -201,6 +201,59 @@ public sealed class DashboardSnapshotFeedTests
         await DrainAsync(second, secondMove);
     }
 
+    /// <summary>
+    /// The race the generation tagging exists for: a page subscribes in the window between
+    /// the source failing and the dying pump detaching its subscribers. Without generations
+    /// the newcomer joined the doomed pump, was detached with its error, and no pump ever
+    /// restarted (only a first subscriber started one) — that page froze for good.
+    /// </summary>
+    /// <returns>A task that represents the asynchronous operation.</returns>
+    [Fact]
+    public async Task WatchAsync_WhenASubscriberJoinsWhileAFaultedPumpUnwinds_IsServedByAFreshPump()
+    {
+        FakeSnapshotService service = new();
+        service.HoldDisposal();
+        DashboardSnapshotFeed feed = new(service);
+
+        using CancellationTokenSource firstCancellation = new();
+        IAsyncEnumerator<DashboardSnapshot> first =
+            feed.WatchAsync(firstCancellation.Token).GetAsyncEnumerator(firstCancellation.Token);
+        Task<bool> firstMove = first.MoveNextAsync().AsTask();
+        await WaitUntilAsync(() => service.EnumerationCount >= 1);
+
+        service.Fault(new InvalidOperationException("simulated snapshot source failure"));
+
+        // The pump has observed the failure and is parked disposing the enumerator — the
+        // exact window in which a page used to attach itself to a doomed pump.
+        await service.DisposalReached.WaitAsync(TestTimeout);
+
+        using CancellationTokenSource secondCancellation = new();
+        IAsyncEnumerator<DashboardSnapshot> second =
+            feed.WatchAsync(secondCancellation.Token).GetAsyncEnumerator(secondCancellation.Token);
+
+        // An async iterator body runs synchronously up to its first await, so the
+        // subscription is registered by the time MoveNextAsync hands back its task.
+        Task<bool> secondMove = second.MoveNextAsync().AsTask();
+
+        service.ReleaseDisposal();
+
+        // The subscriber that was there when the source broke still learns about it...
+        InvalidOperationException failure =
+            await Assert.ThrowsAsync<InvalidOperationException>(() => firstMove.WaitAsync(TestTimeout));
+        Assert.Equal("simulated snapshot source failure", failure.Message);
+        await first.DisposeAsync();
+
+        // ...and the one that joined mid-unwind is served by a restarted enumeration
+        // instead of inheriting the failure.
+        await WaitUntilAsync(() => service.EnumerationCount >= 2);
+        await PushUntilAsync(service, secondMove);
+        Assert.True(await secondMove.WaitAsync(TestTimeout));
+        Assert.StartsWith("push-", second.Current.GatewayVersion, StringComparison.Ordinal);
+
+        await secondCancellation.CancelAsync();
+        await DrainAsync(second, secondMove);
+    }
+
     /// <summary>Builds a snapshot whose version string identifies it in assertions.</summary>
     /// <param name="version">Identity marker carried in <c>GatewayVersion</c>.</param>
     /// <returns>A snapshot carrying the supplied identity marker.</returns>
@@ -282,13 +335,19 @@ public sealed class DashboardSnapshotFeedTests
     private sealed class FakeSnapshotService : IDashboardSnapshotService
     {
         private readonly Channel<object> _pushes = Channel.CreateUnbounded<object>();
+        private readonly TaskCompletionSource _disposalReached = new(TaskCreationOptions.RunContinuationsAsynchronously);
+        private readonly TaskCompletionSource _disposalRelease = new(TaskCreationOptions.RunContinuationsAsynchronously);
         private int _enumerationCount;
         private int _completedEnumerationCount;
         private volatile bool _lastEnumerationWasCancelled;
+        private volatile bool _holdDisposal;
 
         /// <summary>Gets the number of times the feed started enumerating this source.</summary>
         public int EnumerationCount => Volatile.Read(ref _enumerationCount);
 
+        /// <summary>Gets a task that completes when a held enumerator disposal is reached.</summary>
+        public Task DisposalReached => _disposalReached.Task;
+
         /// <summary>Gets the number of enumerations that have finished (cancelled, faulted, or completed).</summary>
         public int CompletedEnumerationCount => Volatile.Read(ref _completedEnumerationCount);
 
@@ -303,11 +362,34 @@ public sealed class DashboardSnapshotFeedTests
         /// <param name="error">Exception to throw from the enumeration.</param>
         public void Fault(Exception error) => _pushes.Writer.TryWrite(error);
 
+        /// <summary>
+        /// Parks enumerator disposal until <see cref="ReleaseDisposal"/>, which holds the pump
+        /// in the window between observing the source's failure and detaching its subscribers.
+        /// </summary>
+        public void HoldDisposal() => _holdDisposal = true;
+
+        /// <summary>Releases a held enumerator disposal.</summary>
+        public void ReleaseDisposal() => _disposalRelease.TrySetResult();
+
         /// <inheritdoc />
         public DashboardSnapshot GetSnapshot() => CreateSnapshot("current");
 
         /// <inheritdoc />
-        public async IAsyncEnumerable<DashboardSnapshot> WatchSnapshotsAsync(
+        public IAsyncEnumerable<DashboardSnapshot> WatchSnapshotsAsync(CancellationToken cancellationToken)
+            => new GatedEnumerable(this);
+
+        private async Task OnDisposingAsync()
+        {
+            if (!_holdDisposal)
+            {
+                return;
+            }
+
+            _disposalReached.TrySetResult();
+            await _disposalRelease.Task.ConfigureAwait(false);
+        }
+
+        private async IAsyncEnumerable<DashboardSnapshot> EnumerateAsync(
             [EnumeratorCancellation] CancellationToken cancellationToken)
         {
             Interlocked.Increment(ref _enumerationCount);
@@ -329,5 +411,40 @@ public sealed class DashboardSnapshotFeedTests
                 Interlocked.Increment(ref _completedEnumerationCount);
             }
         }
+
+        /// <summary>
+        /// Wraps the iterator so disposal is a control point of its own: the feed ends a pump
+        /// generation when MoveNextAsync fails, which is strictly before this disposal runs.
+        /// </summary>
+        /// <param name="owner">The fake whose enumeration is being wrapped.</param>
+        private sealed class GatedEnumerable(FakeSnapshotService owner) : IAsyncEnumerable<DashboardSnapshot>
+        {
+            /// <summary>Creates a gated enumerator over the fake's enumeration.</summary>
+            /// <param name="cancellationToken">Token to observe for cancellation.</param>
+            /// <returns>The gated enumerator.</returns>
+            public IAsyncEnumerator<DashboardSnapshot> GetAsyncEnumerator(
+                CancellationToken cancellationToken = default)
+                => new GatedEnumerator(owner, owner.EnumerateAsync(cancellationToken).GetAsyncEnumerator(cancellationToken));
+        }
+
+        private sealed class GatedEnumerator(
+            FakeSnapshotService owner,
+            IAsyncEnumerator<DashboardSnapshot> inner) : IAsyncEnumerator<DashboardSnapshot>
+        {
+            /// <summary>Gets the current snapshot.</summary>
+            public DashboardSnapshot Current => inner.Current;
+
+            /// <summary>Advances the wrapped enumeration.</summary>
+            /// <returns>A task that yields whether another snapshot is available.</returns>
+            public ValueTask<bool> MoveNextAsync() => inner.MoveNextAsync();
+
+            /// <summary>Parks while the fake holds disposal, then disposes the wrapped enumeration.</summary>
+            /// <returns>A task that represents the asynchronous operation.</returns>
+            public async ValueTask DisposeAsync()
+            {
+                await owner.OnDisposingAsync().ConfigureAwait(false);
+                await inner.DisposeAsync().ConfigureAwait(false);
+            }
+        }
     }
 }