Fix remaining reliability findings

2026-04-28 06:38:05 -04:00
parent b0041c5d18
commit 047d875fe6
8 changed files with 200 additions and 217 deletions
@@ -111,6 +111,46 @@ public sealed class EventStreamServiceTests
        await WaitUntilAsync(() => metrics.GetSnapshot().GrpcEventStreamQueueDepth == 0);
    }

+    [Fact]
+    public async Task StreamEventsAsync_WithConcurrentStreams_TracksAggregateQueueDepth()
+    {
+        FakeWorkerClient firstWorkerClient = new();
+        FakeWorkerClient secondWorkerClient = new();
+        GatewaySession firstSession = CreateReadySession(firstWorkerClient, "session-events-1");
+        GatewaySession secondSession = CreateReadySession(secondWorkerClient, "session-events-2");
+        using GatewayMetrics metrics = new();
+        EventStreamService service = CreateService(
+            new FakeSessionManager(firstSession, secondSession),
+            metrics,
+            queueCapacity: 8);
+        for (ulong sequence = 1; sequence <= 3; sequence++)
+        {
+            firstWorkerClient.Events.Add(CreateWorkerEvent(sequence, MxEventFamily.OnDataChange));
+            secondWorkerClient.Events.Add(CreateWorkerEvent(sequence, MxEventFamily.OnDataChange));
+        }
+
+        firstWorkerClient.CompleteAfterConfiguredEvents = true;
+        secondWorkerClient.CompleteAfterConfiguredEvents = true;
+        await using IAsyncEnumerator<MxEvent> firstSubscriber = service
+            .StreamEventsAsync(CreateRequest(firstSession.SessionId), CancellationToken.None)
+            .GetAsyncEnumerator();
+        await using IAsyncEnumerator<MxEvent> secondSubscriber = service
+            .StreamEventsAsync(CreateRequest(secondSession.SessionId), CancellationToken.None)
+            .GetAsyncEnumerator();
+
+        Assert.True(await firstSubscriber.MoveNextAsync().AsTask().WaitAsync(TestTimeout));
+        Assert.True(await secondSubscriber.MoveNextAsync().AsTask().WaitAsync(TestTimeout));
+        await WaitUntilAsync(() => metrics.GetSnapshot().GrpcEventStreamQueueDepth == 4);
+
+        await firstSubscriber.DisposeAsync();
+
+        await WaitUntilAsync(() => metrics.GetSnapshot().GrpcEventStreamQueueDepth == 2);
+
+        await secondSubscriber.DisposeAsync();
+
+        await WaitUntilAsync(() => metrics.GetSnapshot().GrpcEventStreamQueueDepth == 0);
+    }
+
    [Fact]
    public async Task StreamEventsAsync_WhenStreamQueueOverflows_FaultsSessionAndReportsOverflow()
    {
@@ -255,10 +295,12 @@ public sealed class EventStreamServiceTests
        };
    }

-    private static GatewaySession CreateReadySession(FakeWorkerClient workerClient)
+    private static GatewaySession CreateReadySession(
+        FakeWorkerClient workerClient,
+        string sessionId = "session-events")
    {
        GatewaySession session = new(
-            "session-events",
+            sessionId,
            GatewayContractInfo.DefaultBackendName,
            "pipe",
            "nonce",
@@ -317,22 +359,28 @@ public sealed class EventStreamServiceTests
        }
    }

-    private sealed class FakeSessionManager(GatewaySession session) : ISessionManager
+    private sealed class FakeSessionManager : ISessionManager
    {
+        private readonly IReadOnlyDictionary<string, GatewaySession> _sessions;
+
+        public FakeSessionManager(params GatewaySession[] sessions)
+        {
+            _sessions = sessions.ToDictionary(session => session.SessionId, StringComparer.Ordinal);
+        }
+
        public Task<GatewaySession> OpenSessionAsync(
            SessionOpenRequest request,
            string? clientIdentity,
            CancellationToken cancellationToken)
        {
-            return Task.FromResult(session);
+            return Task.FromResult(_sessions.Values.First());
        }

        public bool TryGetSession(
            string sessionId,
            out GatewaySession gatewaySession)
        {
-            gatewaySession = session;
-            return string.Equals(sessionId, session.SessionId, StringComparison.Ordinal);
+            return _sessions.TryGetValue(sessionId, out gatewaySession!);
        }

        public Task<WorkerCommandReply> InvokeAsync(
@@ -347,7 +395,7 @@ public sealed class EventStreamServiceTests
            string sessionId,
            CancellationToken cancellationToken)
        {
-            return session.ReadEventsAsync(cancellationToken);
+            return _sessions[sessionId].ReadEventsAsync(cancellationToken);
        }

        public Task<SessionCloseResult> CloseSessionAsync(