using System.Runtime.CompilerServices; using Grpc.Core; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Options; using ZB.MOM.WW.MxGateway.Contracts; using ZB.MOM.WW.MxGateway.Contracts.Proto; using ZB.MOM.WW.MxGateway.Server.Configuration; using ZB.MOM.WW.MxGateway.Server.Grpc; using ZB.MOM.WW.MxGateway.Server.Metrics; using ZB.MOM.WW.MxGateway.Server.Sessions; using ZB.MOM.WW.MxGateway.Server.Workers; namespace ZB.MOM.WW.MxGateway.Tests.Gateway.Grpc; public sealed class EventStreamServiceTests { private static readonly TimeSpan TestTimeout = TimeSpan.FromSeconds(5); /// Verifies that events from the worker stream maintain their original sequence order. /// A task that represents the asynchronous operation. [Fact] public async Task StreamEventsAsync_YieldsEventsInWorkerOrder() { FakeWorkerClient workerClient = new(); GatewaySession session = CreateReadySession(workerClient); FakeSessionManager sessionManager = new(session); using GatewayMetrics metrics = new(); EventStreamService service = CreateService(sessionManager, metrics: metrics); workerClient.Events.Add(CreateWorkerEvent(sequence: 10, MxEventFamily.OnDataChange)); workerClient.Events.Add(CreateWorkerEvent(sequence: 11, MxEventFamily.OnWriteComplete)); workerClient.CompleteAfterConfiguredEvents = true; List events = await CollectEventsAsync(service, session.SessionId); Assert.Equal([10UL, 11UL], events.Select(mxEvent => mxEvent.WorkerSequence).ToArray()); Assert.Equal(MxEventFamily.OnDataChange, events[0].Family); Assert.Equal(MxEventFamily.OnWriteComplete, events[1].Family); Assert.Equal(1, metrics.GetSnapshot().StreamDisconnects); } /// /// Owner-scoped attach: the API key that opened a session may attach its event /// stream — the caller key equals the session owner, so streaming proceeds normally. /// /// A task that represents the asynchronous operation. [Fact] public async Task StreamEventsAsync_WhenCallerKeyMatchesOwner_Streams() { FakeWorkerClient workerClient = new(); GatewaySession session = CreateReadySession(workerClient, ownerKeyId: "key-owner"); EventStreamService service = CreateService(new FakeSessionManager(session)); workerClient.Events.Add(CreateWorkerEvent(sequence: 5, MxEventFamily.OnDataChange)); workerClient.CompleteAfterConfiguredEvents = true; List events = []; await foreach (MxEvent mxEvent in service .StreamEventsAsync(CreateRequest(session.SessionId), callerKeyId: "key-owner", CancellationToken.None) .WithCancellation(CancellationToken.None)) { events.Add(mxEvent); } Assert.Equal([5UL], events.Select(mxEvent => mxEvent.WorkerSequence).ToArray()); } /// /// Owner-scoped attach, security control: a caller whose API key differs from /// the key that opened the session is rejected with a /// fault before any events are streamed — closing the reconnect/fan-out trust-boundary hole. /// /// A task that represents the asynchronous operation. [Fact] public async Task StreamEventsAsync_WhenCallerKeyDiffersFromOwner_ThrowsPermissionDenied() { FakeWorkerClient workerClient = new(); GatewaySession session = CreateReadySession(workerClient, ownerKeyId: "key-owner"); EventStreamService service = CreateService(new FakeSessionManager(session)); SessionManagerException exception = await Assert.ThrowsAsync(async () => { await foreach (MxEvent _ in service .StreamEventsAsync(CreateRequest(session.SessionId), callerKeyId: "key-intruder", CancellationToken.None) .WithCancellation(CancellationToken.None)) { // No event should be yielded — the owner check runs before the first attach. } }); Assert.Equal(SessionManagerErrorCode.PermissionDenied, exception.ErrorCode); Assert.Equal(0, session.ActiveEventSubscriberCount); } /// Verifies that a second event subscriber is rejected when one is already active. /// A task that represents the asynchronous operation. [Fact] public async Task StreamEventsAsync_WhenSecondSubscriberStarts_RejectsClearly() { FakeWorkerClient workerClient = new(); GatewaySession session = CreateReadySession(workerClient); EventStreamService service = CreateService(new FakeSessionManager(session)); using CancellationTokenSource firstSubscriberCancellation = new(); await using IAsyncEnumerator firstSubscriber = service .StreamEventsAsync(CreateRequest(session.SessionId), callerKeyId: null, firstSubscriberCancellation.Token) .GetAsyncEnumerator(firstSubscriberCancellation.Token); Task firstMoveTask = firstSubscriber.MoveNextAsync().AsTask(); await WaitUntilAsync(() => session.ActiveEventSubscriberCount == 1); await using IAsyncEnumerator secondSubscriber = service .StreamEventsAsync(CreateRequest(session.SessionId), callerKeyId: null, CancellationToken.None) .GetAsyncEnumerator(); SessionManagerException exception = await Assert.ThrowsAsync( async () => await secondSubscriber.MoveNextAsync().AsTask().WaitAsync(TestTimeout)); Assert.Equal(SessionManagerErrorCode.EventSubscriberAlreadyActive, exception.ErrorCode); await firstSubscriberCancellation.CancelAsync(); await Assert.ThrowsAnyAsync( async () => await firstMoveTask.WaitAsync(TestTimeout)); await firstSubscriber.DisposeAsync(); await WaitUntilAsync(() => session.ActiveEventSubscriberCount == 0); } /// Verifies that canceling an event stream detaches the subscriber cleanly. /// A task that represents the asynchronous operation. [Fact] public async Task StreamEventsAsync_WhenCanceled_DetachesSubscriber() { FakeWorkerClient workerClient = new(); GatewaySession session = CreateReadySession(workerClient); EventStreamService service = CreateService(new FakeSessionManager(session)); using CancellationTokenSource cancellationTokenSource = new(); await using IAsyncEnumerator subscriber = service .StreamEventsAsync(CreateRequest(session.SessionId), callerKeyId: null, cancellationTokenSource.Token) .GetAsyncEnumerator(cancellationTokenSource.Token); Task moveTask = subscriber.MoveNextAsync().AsTask(); await WaitUntilAsync(() => session.ActiveEventSubscriberCount == 1); await cancellationTokenSource.CancelAsync(); await Assert.ThrowsAnyAsync( async () => await moveTask.WaitAsync(TestTimeout)); await subscriber.DisposeAsync(); await WaitUntilAsync(() => session.ActiveEventSubscriberCount == 0); } /// Verifies that disposing an event stream with buffered events resets the queue depth metric. /// A task that represents the asynchronous operation. [Fact] public async Task StreamEventsAsync_WhenDisposedWithBufferedEvents_ResetsStreamQueueDepth() { FakeWorkerClient workerClient = new(); GatewaySession session = CreateReadySession(workerClient); using GatewayMetrics metrics = new(); EventStreamService service = CreateService( new FakeSessionManager(session), metrics, queueCapacity: 8); workerClient.Events.Add(CreateWorkerEvent(sequence: 1, MxEventFamily.OnDataChange)); workerClient.Events.Add(CreateWorkerEvent(sequence: 2, MxEventFamily.OnDataChange)); workerClient.Events.Add(CreateWorkerEvent(sequence: 3, MxEventFamily.OnDataChange)); workerClient.CompleteAfterConfiguredEvents = true; await using IAsyncEnumerator subscriber = service .StreamEventsAsync(CreateRequest(session.SessionId), callerKeyId: null, CancellationToken.None) .GetAsyncEnumerator(); Assert.True(await subscriber.MoveNextAsync().AsTask().WaitAsync(TestTimeout)); await WaitUntilAsync(() => metrics.GetSnapshot().GrpcEventStreamQueueDepth > 0); await subscriber.DisposeAsync(); await WaitUntilAsync(() => metrics.GetSnapshot().GrpcEventStreamQueueDepth == 0); } /// Verifies that queue depth metrics correctly track concurrent event streams across multiple sessions. /// A task that represents the asynchronous operation. [Fact] public async Task StreamEventsAsync_WithConcurrentStreams_TracksAggregateQueueDepth() { FakeWorkerClient firstWorkerClient = new(); FakeWorkerClient secondWorkerClient = new(); GatewaySession firstSession = CreateReadySession(firstWorkerClient, "session-events-1"); GatewaySession secondSession = CreateReadySession(secondWorkerClient, "session-events-2"); using GatewayMetrics metrics = new(); EventStreamService service = CreateService( new FakeSessionManager(firstSession, secondSession), metrics, queueCapacity: 8); for (ulong sequence = 1; sequence <= 3; sequence++) { firstWorkerClient.Events.Add(CreateWorkerEvent(sequence, MxEventFamily.OnDataChange)); secondWorkerClient.Events.Add(CreateWorkerEvent(sequence, MxEventFamily.OnDataChange)); } firstWorkerClient.CompleteAfterConfiguredEvents = true; secondWorkerClient.CompleteAfterConfiguredEvents = true; await using IAsyncEnumerator firstSubscriber = service .StreamEventsAsync(CreateRequest(firstSession.SessionId), callerKeyId: null, CancellationToken.None) .GetAsyncEnumerator(); await using IAsyncEnumerator secondSubscriber = service .StreamEventsAsync(CreateRequest(secondSession.SessionId), callerKeyId: null, CancellationToken.None) .GetAsyncEnumerator(); Assert.True(await firstSubscriber.MoveNextAsync().AsTask().WaitAsync(TestTimeout)); Assert.True(await secondSubscriber.MoveNextAsync().AsTask().WaitAsync(TestTimeout)); await WaitUntilAsync(() => metrics.GetSnapshot().GrpcEventStreamQueueDepth == 4); await firstSubscriber.DisposeAsync(); await WaitUntilAsync(() => metrics.GetSnapshot().GrpcEventStreamQueueDepth == 2); await secondSubscriber.DisposeAsync(); await WaitUntilAsync(() => metrics.GetSnapshot().GrpcEventStreamQueueDepth == 0); } /// /// A per-subscriber channel overflow in the session's /// faults the whole session under the legacy /// single-subscriber FailFast policy (the default, single-subscriber mode) and records /// the overflow + fault metrics. The distributor completes this subscriber's channel /// with the overflow fault, which surfaces here as the same /// the pre-epic per-RPC /// overflow produced. /// /// A task that represents the asynchronous operation. [Fact] public async Task StreamEventsAsync_WhenStreamQueueOverflows_FaultsSessionAndReportsOverflow() { FakeWorkerClient workerClient = new(); using GatewayMetrics metrics = new(); GatewaySession session = CreateReadySession( workerClient, queueCapacity: 1, metrics: metrics, backpressurePolicy: EventBackpressurePolicy.FailFast); EventStreamService service = CreateService( new FakeSessionManager(session), metrics, queueCapacity: 1); for (ulong sequence = 1; sequence <= 50; sequence++) { workerClient.Events.Add(CreateWorkerEvent(sequence, MxEventFamily.OnDataChange)); } workerClient.CompleteAfterConfiguredEvents = true; await using IAsyncEnumerator subscriber = service .StreamEventsAsync(CreateRequest(session.SessionId), callerKeyId: null, CancellationToken.None) .GetAsyncEnumerator(); // The pump fans 50 events into a subscriber channel with capacity 1 faster than this // single reader drains, so one of the reads observes the terminal overflow fault. SessionManagerException exception = await Assert.ThrowsAsync( async () => { while (await subscriber.MoveNextAsync().AsTask().WaitAsync(TestTimeout)) { } }); Assert.Equal(SessionManagerErrorCode.EventQueueOverflow, exception.ErrorCode); await WaitUntilAsync(() => session.State == SessionState.Faulted); Assert.Equal(SessionState.Faulted, session.State); GatewayMetricsSnapshot snapshot = metrics.GetSnapshot(); Assert.Equal(1, snapshot.QueueOverflows); Assert.Equal(1, snapshot.Faults); // The finally block in StreamEventsAsync calls StreamDisconnected("Detached") on the // overflow+fault path too; pin it here so a regression removing that call is caught. Assert.Equal(1, snapshot.StreamDisconnects); } /// /// Under the DisconnectSubscriber policy a per-subscriber channel overflow /// disconnects only that subscriber's stream (terminal /// ) and records the overflow /// metric, but leaves the session and records no /// fault. The session, pump, and any other subscribers are unaffected. /// /// A task that represents the asynchronous operation. [Fact] public async Task StreamEventsAsync_WhenStreamQueueOverflowsWithDisconnectPolicy_LeavesSessionReady() { FakeWorkerClient workerClient = new(); using GatewayMetrics metrics = new(); GatewaySession session = CreateReadySession( workerClient, queueCapacity: 1, metrics: metrics, backpressurePolicy: EventBackpressurePolicy.DisconnectSubscriber); EventStreamService service = CreateService( new FakeSessionManager(session), metrics, queueCapacity: 1, backpressurePolicy: EventBackpressurePolicy.DisconnectSubscriber); for (ulong sequence = 1; sequence <= 50; sequence++) { workerClient.Events.Add(CreateWorkerEvent(sequence, MxEventFamily.OnDataChange)); } workerClient.CompleteAfterConfiguredEvents = true; await using IAsyncEnumerator subscriber = service .StreamEventsAsync(CreateRequest(session.SessionId), callerKeyId: null, CancellationToken.None) .GetAsyncEnumerator(); SessionManagerException exception = await Assert.ThrowsAsync( async () => { while (await subscriber.MoveNextAsync().AsTask().WaitAsync(TestTimeout)) { } }); Assert.Equal(SessionManagerErrorCode.EventQueueOverflow, exception.ErrorCode); Assert.Equal(SessionState.Ready, session.State); GatewayMetricsSnapshot snapshot = metrics.GetSnapshot(); Assert.Equal(1, snapshot.QueueOverflows); Assert.Equal(0, snapshot.Faults); Assert.Equal(1, snapshot.StreamDisconnects); } /// Verifies that the event stream does not synthesize OperationComplete events from write completions. /// A task that represents the asynchronous operation. [Fact] public async Task StreamEventsAsync_DoesNotSynthesizeOperationComplete() { FakeWorkerClient workerClient = new(); GatewaySession session = CreateReadySession(workerClient); EventStreamService service = CreateService(new FakeSessionManager(session)); workerClient.Events.Add(CreateWorkerEvent(sequence: 10, MxEventFamily.OnWriteComplete)); workerClient.CompleteAfterConfiguredEvents = true; List events = await CollectEventsAsync(service, session.SessionId); MxEvent mxEvent = Assert.Single(events); Assert.Equal(MxEventFamily.OnWriteComplete, mxEvent.Family); Assert.DoesNotContain(events, candidate => candidate.Family == MxEventFamily.OperationComplete); } /// Verifies that a terminal fault from the worker event stream propagates and faults the session. /// A task that represents the asynchronous operation. [Fact] public async Task StreamEventsAsync_WhenWorkerEventStreamFaults_PropagatesTerminalFault() { FakeWorkerClient workerClient = new() { TerminalException = new WorkerClientException( WorkerClientErrorCode.WorkerFaulted, "worker terminal fault"), }; GatewaySession session = CreateReadySession(workerClient); using GatewayMetrics metrics = new(); EventStreamService service = CreateService(new FakeSessionManager(session), metrics); await using IAsyncEnumerator subscriber = service .StreamEventsAsync(CreateRequest(session.SessionId), callerKeyId: null, CancellationToken.None) .GetAsyncEnumerator(); WorkerClientException exception = await Assert.ThrowsAsync( async () => await subscriber.MoveNextAsync().AsTask().WaitAsync(TestTimeout)); Assert.Equal(WorkerClientErrorCode.WorkerFaulted, exception.ErrorCode); Assert.Equal(SessionState.Faulted, session.State); Assert.Equal(1, metrics.GetSnapshot().Faults); } /// /// Resuming with AfterWorkerSequence inside the retained window replays exactly /// the newer retained events (in order, no dup) then live, with NO ReplayGap sentinel. /// /// A task that represents the asynchronous operation. [Fact] public async Task StreamEventsAsync_ResumeWithinRetainedWindow_ReplaysNewerThenLive_NoSentinel() { System.Threading.Channels.Channel live = System.Threading.Channels.Channel.CreateUnbounded(); FakeWorkerClient workerClient = new() { LiveEvents = live }; for (ulong sequence = 1; sequence <= 5; sequence++) { workerClient.Events.Add(CreateWorkerEvent(sequence, MxEventFamily.OnDataChange)); } GatewaySession session = CreateReadySession(workerClient); EventStreamService service = CreateService(new FakeSessionManager(session)); // Prime: drain the static 1..5 through a first subscriber so the replay ring retains them. await PrimeReplayAsync(service, session.SessionId, expectedCount: 5); // Resume after sequence 2: retained window [1..5] covers it — replay 3,4,5 then live. await using IAsyncEnumerator resume = service .StreamEventsAsync(CreateRequest(session.SessionId, afterWorkerSequence: 2), callerKeyId: null, CancellationToken.None) .GetAsyncEnumerator(); MxEvent r3 = await ReadNextAsync(resume); MxEvent r4 = await ReadNextAsync(resume); MxEvent r5 = await ReadNextAsync(resume); Assert.Equal(new ulong[] { 3, 4, 5 }, new[] { r3.WorkerSequence, r4.WorkerSequence, r5.WorkerSequence }); Assert.Null(r3.ReplayGap); // No sentinel anywhere; next is a LIVE event. live.Writer.TryWrite(CreateWorkerEvent(6, MxEventFamily.OnDataChange)); MxEvent liveEvent = await ReadNextAsync(resume); Assert.Equal(6ul, liveEvent.WorkerSequence); Assert.Null(liveEvent.ReplayGap); } /// /// Resuming with AfterWorkerSequence older than the oldest retained yields the /// ReplayGap sentinel FIRST (correct requested/oldest), then the retained tail, then live. /// /// A task that represents the asynchronous operation. [Fact] public async Task StreamEventsAsync_ResumeOlderThanOldestRetained_EmitsSentinelFirst_ThenTailThenLive() { System.Threading.Channels.Channel live = System.Threading.Channels.Channel.CreateUnbounded(); FakeWorkerClient workerClient = new() { LiveEvents = live }; for (ulong sequence = 1; sequence <= 5; sequence++) { workerClient.Events.Add(CreateWorkerEvent(sequence, MxEventFamily.OnDataChange)); } // Replay capacity 3 retains only 3,4,5; 1,2 are evicted. GatewaySession session = CreateReadySession(workerClient, replayBufferCapacity: 3); EventStreamService service = CreateService(new FakeSessionManager(session)); await PrimeReplayAsync(service, session.SessionId, expectedCount: 5); // Resume after 1: events 1,2 are below the oldest retained (3) and were evicted, so // they are unrecoverable => sentinel first, then the retained tail 3,4,5, then live. await using IAsyncEnumerator realResume = service .StreamEventsAsync(CreateRequest(session.SessionId, afterWorkerSequence: 1), callerKeyId: null, CancellationToken.None) .GetAsyncEnumerator(); MxEvent sentinel = await ReadNextAsync(realResume); Assert.NotNull(sentinel.ReplayGap); Assert.Equal(1ul, sentinel.ReplayGap.RequestedAfterSequence); Assert.Equal(3ul, sentinel.ReplayGap.OldestAvailableSequence); Assert.Equal(MxEventFamily.Unspecified, sentinel.Family); Assert.Equal(session.SessionId, sentinel.SessionId); MxEvent r3 = await ReadNextAsync(realResume); MxEvent r4 = await ReadNextAsync(realResume); MxEvent r5 = await ReadNextAsync(realResume); Assert.Equal(new ulong[] { 3, 4, 5 }, new[] { r3.WorkerSequence, r4.WorkerSequence, r5.WorkerSequence }); Assert.Null(r3.ReplayGap); live.Writer.TryWrite(CreateWorkerEvent(6, MxEventFamily.OnDataChange)); MxEvent liveEvent = await ReadNextAsync(realResume); Assert.Equal(6ul, liveEvent.WorkerSequence); } /// /// The replay→live boundary is contiguous — no duplicate and no skip — even /// when events span the handoff. /// /// A task that represents the asynchronous operation. [Fact] public async Task StreamEventsAsync_ResumeHandoff_IsContiguous_NoDuplicateNoSkip() { System.Threading.Channels.Channel live = System.Threading.Channels.Channel.CreateUnbounded(); FakeWorkerClient workerClient = new() { LiveEvents = live }; for (ulong sequence = 1; sequence <= 4; sequence++) { workerClient.Events.Add(CreateWorkerEvent(sequence, MxEventFamily.OnDataChange)); } GatewaySession session = CreateReadySession(workerClient); EventStreamService service = CreateService(new FakeSessionManager(session)); await PrimeReplayAsync(service, session.SessionId, expectedCount: 4); // Resume after 2: replay 3,4 then live 5,6,7. Collect across the boundary and assert // the full sequence is contiguous with no duplicate and no skip. await using IAsyncEnumerator resume = service .StreamEventsAsync(CreateRequest(session.SessionId, afterWorkerSequence: 2), callerKeyId: null, CancellationToken.None) .GetAsyncEnumerator(); List collected = []; collected.Add((await ReadNextAsync(resume)).WorkerSequence); // 3 collected.Add((await ReadNextAsync(resume)).WorkerSequence); // 4 for (ulong sequence = 5; sequence <= 7; sequence++) { live.Writer.TryWrite(CreateWorkerEvent(sequence, MxEventFamily.OnDataChange)); collected.Add((await ReadNextAsync(resume)).WorkerSequence); } Assert.Equal(new ulong[] { 3, 4, 5, 6, 7 }, collected); } /// /// The per-item filter applies to REPLAYED events identically to live — a /// replayed event at/below the requested watermark is never delivered. /// /// A task that represents the asynchronous operation. [Fact] public async Task StreamEventsAsync_ResumeReplay_AppliesPerItemFilter_DropsAtOrBelowWatermark() { System.Threading.Channels.Channel live = System.Threading.Channels.Channel.CreateUnbounded(); FakeWorkerClient workerClient = new() { LiveEvents = live }; for (ulong sequence = 1; sequence <= 5; sequence++) { workerClient.Events.Add(CreateWorkerEvent(sequence, MxEventFamily.OnDataChange)); } GatewaySession session = CreateReadySession(workerClient); EventStreamService service = CreateService(new FakeSessionManager(session)); await PrimeReplayAsync(service, session.SessionId, expectedCount: 5); // Resume after 3: only 4,5 may be delivered. Events 1,2,3 — present in the ring but at // or below the watermark — must be filtered out of the replay, never seen. The first two // reads must be exactly 4 then 5 (no sentinel, no <=3 event); a live tag confirms the // stream resumed live strictly after 5. await using IAsyncEnumerator resume = service .StreamEventsAsync(CreateRequest(session.SessionId, afterWorkerSequence: 3), callerKeyId: null, CancellationToken.None) .GetAsyncEnumerator(); MxEvent first = await ReadNextAsync(resume); MxEvent second = await ReadNextAsync(resume); Assert.Equal(4ul, first.WorkerSequence); Assert.Equal(5ul, second.WorkerSequence); Assert.Null(first.ReplayGap); Assert.Null(second.ReplayGap); // The very next delivered event is the live 6 — proving nothing <=3 slipped in and the // handoff resumed strictly after the replay tail. live.Writer.TryWrite(CreateWorkerEvent(6, MxEventFamily.OnDataChange)); MxEvent liveEvent = await ReadNextAsync(resume); Assert.Equal(6ul, liveEvent.WorkerSequence); } /// /// AfterWorkerSequence == 0 is a fresh stream (not a resume) — no replay, no /// sentinel, just live events as before. /// /// A task that represents the asynchronous operation. [Fact] public async Task StreamEventsAsync_FreshStreamAfterSequenceZero_NoReplayNoSentinel() { FakeWorkerClient workerClient = new(); for (ulong sequence = 1; sequence <= 3; sequence++) { workerClient.Events.Add(CreateWorkerEvent(sequence, MxEventFamily.OnDataChange)); } workerClient.CompleteAfterConfiguredEvents = true; GatewaySession session = CreateReadySession(workerClient); EventStreamService service = CreateService(new FakeSessionManager(session)); List events = await CollectEventsAsync(service, session.SessionId); Assert.Equal(new ulong[] { 1, 2, 3 }, events.Select(e => e.WorkerSequence)); Assert.DoesNotContain(events, e => e.ReplayGap is not null); } // Drains the first `expectedCount` events through a throwaway subscriber so the session's // replay ring retains them, then disposes the subscriber. The pump (started on first // attach) keeps running for the session, so subsequent resume attaches see the retained // events. private static async Task PrimeReplayAsync( EventStreamService service, string sessionId, int expectedCount) { await using IAsyncEnumerator primer = service .StreamEventsAsync(CreateRequest(sessionId), callerKeyId: null, CancellationToken.None) .GetAsyncEnumerator(); for (int i = 0; i < expectedCount; i++) { await ReadNextAsync(primer); } } private static async Task ReadNextAsync(IAsyncEnumerator enumerator) { Assert.True(await enumerator.MoveNextAsync().AsTask().WaitAsync(TestTimeout)); return enumerator.Current; } private static EventStreamService CreateService( FakeSessionManager sessionManager, GatewayMetrics? metrics = null, int queueCapacity = 8, EventBackpressurePolicy backpressurePolicy = EventBackpressurePolicy.FailFast) { return new EventStreamService( sessionManager, Options.Create(new GatewayOptions { Events = new EventOptions { QueueCapacity = queueCapacity, BackpressurePolicy = backpressurePolicy, }, }), metrics ?? new GatewayMetrics()); } private static async Task> CollectEventsAsync( EventStreamService service, string sessionId) { List events = []; await foreach (MxEvent mxEvent in service .StreamEventsAsync(CreateRequest(sessionId), callerKeyId: null, CancellationToken.None) .WithCancellation(CancellationToken.None)) { events.Add(mxEvent); } return events; } private static StreamEventsRequest CreateRequest(string sessionId, ulong afterWorkerSequence = 0) { return new StreamEventsRequest { SessionId = sessionId, AfterWorkerSequence = afterWorkerSequence, }; } private static GatewaySession CreateReadySession( FakeWorkerClient workerClient, string sessionId = "session-events", int queueCapacity = 8, GatewayMetrics? metrics = null, EventBackpressurePolicy backpressurePolicy = EventBackpressurePolicy.FailFast, int replayBufferCapacity = 1024, string? ownerKeyId = null) { // The per-subscriber overflow policy now lives in the session's // SessionEventDistributor, so the session must share the same metrics sink and // backpressure policy the overflow assertions observe. queueCapacity flows into the // distributor's per-subscriber channel bound, which is what overflows. GatewaySession session = new( sessionId, GatewayContractInfo.DefaultBackendName, "pipe", "nonce", "client", ownerKeyId: ownerKeyId, "client-session", "client-correlation", TimeSpan.FromSeconds(30), TimeSpan.FromSeconds(30), TimeSpan.FromSeconds(10), TimeSpan.FromMinutes(30), DateTimeOffset.UtcNow, new SessionEventStreaming( new MxAccessGrpcMapper(), new EventOptions { QueueCapacity = queueCapacity, BackpressurePolicy = backpressurePolicy, ReplayBufferCapacity = replayBufferCapacity, ReplayRetentionSeconds = 0, }, NullLogger.Instance, TimeProvider.System, metrics ?? new GatewayMetrics())); session.AttachWorkerClient(workerClient); session.MarkReady(); return session; } private static WorkerEvent CreateWorkerEvent( ulong sequence, MxEventFamily family) { MxEvent mxEvent = new() { SessionId = "session-events", Family = family, WorkerSequence = sequence, }; switch (family) { case MxEventFamily.OnDataChange: mxEvent.OnDataChange = new OnDataChangeEvent(); break; case MxEventFamily.OnWriteComplete: mxEvent.OnWriteComplete = new OnWriteCompleteEvent(); break; case MxEventFamily.OperationComplete: mxEvent.OperationComplete = new OperationCompleteEvent(); break; case MxEventFamily.OnBufferedDataChange: mxEvent.OnBufferedDataChange = new OnBufferedDataChangeEvent(); break; } return new WorkerEvent { Event = mxEvent, }; } // The real-clock deadline here is load-sensitive on a wide host (windev runs this suite // 36-way parallel). Surfacing the unmet condition instead of letting the bare // TaskCanceledException escape is what makes such a failure diagnosable rather than a // mystery cancellation attributed to "the environment". private static async Task WaitUntilAsync( Func predicate, [CallerArgumentExpression(nameof(predicate))] string? predicateExpression = null) { using CancellationTokenSource cancellationTokenSource = new(TestTimeout); while (!predicate()) { try { await Task.Delay(TimeSpan.FromMilliseconds(10), cancellationTokenSource.Token); } catch (OperationCanceledException) { Assert.Fail( $"Timed out after {TestTimeout} waiting for condition: {predicateExpression}"); } } } /// Fake session manager for testing event streams. private sealed class FakeSessionManager : ISessionManager { private readonly IReadOnlyDictionary _sessions; /// Initializes a new instance of the FakeSessionManager. /// Sessions to manage. public FakeSessionManager(params GatewaySession[] sessions) { _sessions = sessions.ToDictionary(session => session.SessionId, StringComparer.Ordinal); } /// public Task OpenSessionAsync( SessionOpenRequest request, string? clientIdentity, string? ownerKeyId, CancellationToken cancellationToken) { return Task.FromResult(_sessions.Values.First()); } /// public bool TryGetSession( string sessionId, out GatewaySession gatewaySession) { return _sessions.TryGetValue(sessionId, out gatewaySession!); } /// public Task InvokeAsync( string sessionId, WorkerCommand command, CancellationToken cancellationToken) { return Task.FromResult(new WorkerCommandReply()); } /// public IAsyncEnumerable ReadEventsAsync( string sessionId, CancellationToken cancellationToken) { return _sessions[sessionId].ReadEventsAsync(cancellationToken); } /// public Task CloseSessionAsync( string sessionId, CancellationToken cancellationToken) { return Task.FromResult(new SessionCloseResult(sessionId, SessionState.Closed, AlreadyClosed: false)); } /// public Task KillWorkerAsync( string sessionId, string reason, CancellationToken cancellationToken) { return Task.FromResult(new SessionCloseResult(sessionId, SessionState.Closed, AlreadyClosed: false)); } /// public Task CloseExpiredLeasesAsync( DateTimeOffset now, CancellationToken cancellationToken) { return Task.FromResult(0); } /// public Task ShutdownAsync(CancellationToken cancellationToken) { return Task.CompletedTask; } } /// Fake worker client for testing event streams. private sealed class FakeWorkerClient : IWorkerClient { /// Gets the list of queued worker events. public List Events { get; } = []; /// Gets or sets whether to complete the event stream after configured events are yielded. public bool CompleteAfterConfiguredEvents { get; set; } /// /// Optional live channel source. When set, the worker drains the static /// first, then streams from this channel until it completes, /// letting a test feed events on demand (e.g. to exercise replay→live handoff). /// public System.Threading.Channels.Channel? LiveEvents { get; init; } /// Gets or sets an optional exception to throw as a terminal event stream fault. public Exception? TerminalException { get; init; } /// public string SessionId { get; } = "session-events"; /// public int? ProcessId { get; } = 4321; /// public WorkerClientState State { get; private set; } = WorkerClientState.Ready; /// public DateTimeOffset LastHeartbeatAt { get; } = DateTimeOffset.UtcNow; /// public Task StartAsync(CancellationToken cancellationToken) { return Task.CompletedTask; } /// public Task InvokeAsync( WorkerCommand command, TimeSpan timeout, CancellationToken cancellationToken) { return Task.FromResult(new WorkerCommandReply()); } /// public async IAsyncEnumerable ReadEventsAsync( [EnumeratorCancellation] CancellationToken cancellationToken) { foreach (WorkerEvent workerEvent in Events) { cancellationToken.ThrowIfCancellationRequested(); yield return workerEvent; } if (TerminalException is not null) { throw TerminalException; } if (LiveEvents is not null) { await foreach (WorkerEvent liveEvent in LiveEvents.Reader .ReadAllAsync(cancellationToken) .ConfigureAwait(false)) { yield return liveEvent; } yield break; } if (CompleteAfterConfiguredEvents) { yield break; } await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken); } /// public Task ShutdownAsync( TimeSpan timeout, CancellationToken cancellationToken) { State = WorkerClientState.Closed; return Task.CompletedTask; } /// public void Kill(string reason) { State = WorkerClientState.Faulted; } /// No-op disposal; the fake holds no unmanaged resources. /// A task that represents the asynchronous operation. public ValueTask DisposeAsync() { return ValueTask.CompletedTask; } } }