using System.Runtime.CompilerServices; using Grpc.Core; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Options; using ZB.MOM.WW.MxGateway.Contracts; using ZB.MOM.WW.MxGateway.Contracts.Proto; using ZB.MOM.WW.MxGateway.Server.Configuration; using ZB.MOM.WW.MxGateway.Server.Grpc; using ZB.MOM.WW.MxGateway.Server.Metrics; using ZB.MOM.WW.MxGateway.Server.Sessions; using ZB.MOM.WW.MxGateway.Server.Workers; namespace ZB.MOM.WW.MxGateway.Tests.Gateway.Grpc; public sealed class EventStreamServiceTests { private static readonly TimeSpan TestTimeout = TimeSpan.FromSeconds(5); /// Verifies that events from the worker stream maintain their original sequence order. [Fact] public async Task StreamEventsAsync_YieldsEventsInWorkerOrder() { FakeWorkerClient workerClient = new(); GatewaySession session = CreateReadySession(workerClient); FakeSessionManager sessionManager = new(session); using GatewayMetrics metrics = new(); EventStreamService service = CreateService(sessionManager, metrics: metrics); workerClient.Events.Add(CreateWorkerEvent(sequence: 10, MxEventFamily.OnDataChange)); workerClient.Events.Add(CreateWorkerEvent(sequence: 11, MxEventFamily.OnWriteComplete)); workerClient.CompleteAfterConfiguredEvents = true; List events = await CollectEventsAsync(service, session.SessionId); Assert.Equal([10UL, 11UL], events.Select(mxEvent => mxEvent.WorkerSequence).ToArray()); Assert.Equal(MxEventFamily.OnDataChange, events[0].Family); Assert.Equal(MxEventFamily.OnWriteComplete, events[1].Family); Assert.Equal(1, metrics.GetSnapshot().StreamDisconnects); } /// Verifies that a second event subscriber is rejected when one is already active. [Fact] public async Task StreamEventsAsync_WhenSecondSubscriberStarts_RejectsClearly() { FakeWorkerClient workerClient = new(); GatewaySession session = CreateReadySession(workerClient); EventStreamService service = CreateService(new FakeSessionManager(session)); using CancellationTokenSource firstSubscriberCancellation = new(); await using IAsyncEnumerator firstSubscriber = service .StreamEventsAsync(CreateRequest(session.SessionId), firstSubscriberCancellation.Token) .GetAsyncEnumerator(firstSubscriberCancellation.Token); Task firstMoveTask = firstSubscriber.MoveNextAsync().AsTask(); await WaitUntilAsync(() => session.ActiveEventSubscriberCount == 1); await using IAsyncEnumerator secondSubscriber = service .StreamEventsAsync(CreateRequest(session.SessionId), CancellationToken.None) .GetAsyncEnumerator(); SessionManagerException exception = await Assert.ThrowsAsync( async () => await secondSubscriber.MoveNextAsync().AsTask().WaitAsync(TestTimeout)); Assert.Equal(SessionManagerErrorCode.EventSubscriberAlreadyActive, exception.ErrorCode); await firstSubscriberCancellation.CancelAsync(); await Assert.ThrowsAnyAsync( async () => await firstMoveTask.WaitAsync(TestTimeout)); await firstSubscriber.DisposeAsync(); await WaitUntilAsync(() => session.ActiveEventSubscriberCount == 0); } /// Verifies that canceling an event stream detaches the subscriber cleanly. [Fact] public async Task StreamEventsAsync_WhenCanceled_DetachesSubscriber() { FakeWorkerClient workerClient = new(); GatewaySession session = CreateReadySession(workerClient); EventStreamService service = CreateService(new FakeSessionManager(session)); using CancellationTokenSource cancellationTokenSource = new(); await using IAsyncEnumerator subscriber = service .StreamEventsAsync(CreateRequest(session.SessionId), cancellationTokenSource.Token) .GetAsyncEnumerator(cancellationTokenSource.Token); Task moveTask = subscriber.MoveNextAsync().AsTask(); await WaitUntilAsync(() => session.ActiveEventSubscriberCount == 1); await cancellationTokenSource.CancelAsync(); await Assert.ThrowsAnyAsync( async () => await moveTask.WaitAsync(TestTimeout)); await subscriber.DisposeAsync(); await WaitUntilAsync(() => session.ActiveEventSubscriberCount == 0); } /// Verifies that disposing an event stream with buffered events resets the queue depth metric. [Fact] public async Task StreamEventsAsync_WhenDisposedWithBufferedEvents_ResetsStreamQueueDepth() { FakeWorkerClient workerClient = new(); GatewaySession session = CreateReadySession(workerClient); using GatewayMetrics metrics = new(); EventStreamService service = CreateService( new FakeSessionManager(session), metrics, queueCapacity: 8); workerClient.Events.Add(CreateWorkerEvent(sequence: 1, MxEventFamily.OnDataChange)); workerClient.Events.Add(CreateWorkerEvent(sequence: 2, MxEventFamily.OnDataChange)); workerClient.Events.Add(CreateWorkerEvent(sequence: 3, MxEventFamily.OnDataChange)); workerClient.CompleteAfterConfiguredEvents = true; await using IAsyncEnumerator subscriber = service .StreamEventsAsync(CreateRequest(session.SessionId), CancellationToken.None) .GetAsyncEnumerator(); Assert.True(await subscriber.MoveNextAsync().AsTask().WaitAsync(TestTimeout)); await WaitUntilAsync(() => metrics.GetSnapshot().GrpcEventStreamQueueDepth > 0); await subscriber.DisposeAsync(); await WaitUntilAsync(() => metrics.GetSnapshot().GrpcEventStreamQueueDepth == 0); } /// Verifies that queue depth metrics correctly track concurrent event streams across multiple sessions. [Fact] public async Task StreamEventsAsync_WithConcurrentStreams_TracksAggregateQueueDepth() { FakeWorkerClient firstWorkerClient = new(); FakeWorkerClient secondWorkerClient = new(); GatewaySession firstSession = CreateReadySession(firstWorkerClient, "session-events-1"); GatewaySession secondSession = CreateReadySession(secondWorkerClient, "session-events-2"); using GatewayMetrics metrics = new(); EventStreamService service = CreateService( new FakeSessionManager(firstSession, secondSession), metrics, queueCapacity: 8); for (ulong sequence = 1; sequence <= 3; sequence++) { firstWorkerClient.Events.Add(CreateWorkerEvent(sequence, MxEventFamily.OnDataChange)); secondWorkerClient.Events.Add(CreateWorkerEvent(sequence, MxEventFamily.OnDataChange)); } firstWorkerClient.CompleteAfterConfiguredEvents = true; secondWorkerClient.CompleteAfterConfiguredEvents = true; await using IAsyncEnumerator firstSubscriber = service .StreamEventsAsync(CreateRequest(firstSession.SessionId), CancellationToken.None) .GetAsyncEnumerator(); await using IAsyncEnumerator secondSubscriber = service .StreamEventsAsync(CreateRequest(secondSession.SessionId), CancellationToken.None) .GetAsyncEnumerator(); Assert.True(await firstSubscriber.MoveNextAsync().AsTask().WaitAsync(TestTimeout)); Assert.True(await secondSubscriber.MoveNextAsync().AsTask().WaitAsync(TestTimeout)); await WaitUntilAsync(() => metrics.GetSnapshot().GrpcEventStreamQueueDepth == 4); await firstSubscriber.DisposeAsync(); await WaitUntilAsync(() => metrics.GetSnapshot().GrpcEventStreamQueueDepth == 2); await secondSubscriber.DisposeAsync(); await WaitUntilAsync(() => metrics.GetSnapshot().GrpcEventStreamQueueDepth == 0); } /// /// Re-targeted in Task 5: a per-subscriber channel overflow in the session's /// faults the whole session under the legacy /// single-subscriber FailFast policy (the default, single-subscriber mode) and records /// the overflow + fault metrics. The distributor completes this subscriber's channel /// with the overflow fault, which surfaces here as the same /// the pre-epic per-RPC /// overflow produced. /// [Fact] public async Task StreamEventsAsync_WhenStreamQueueOverflows_FaultsSessionAndReportsOverflow() { FakeWorkerClient workerClient = new(); using GatewayMetrics metrics = new(); GatewaySession session = CreateReadySession( workerClient, queueCapacity: 1, metrics: metrics, backpressurePolicy: EventBackpressurePolicy.FailFast); EventStreamService service = CreateService( new FakeSessionManager(session), metrics, queueCapacity: 1); for (ulong sequence = 1; sequence <= 50; sequence++) { workerClient.Events.Add(CreateWorkerEvent(sequence, MxEventFamily.OnDataChange)); } workerClient.CompleteAfterConfiguredEvents = true; await using IAsyncEnumerator subscriber = service .StreamEventsAsync(CreateRequest(session.SessionId), CancellationToken.None) .GetAsyncEnumerator(); // The pump fans 50 events into a capacity-1 subscriber channel faster than this // single reader drains, so one of the reads observes the terminal overflow fault. SessionManagerException exception = await Assert.ThrowsAsync( async () => { while (await subscriber.MoveNextAsync().AsTask().WaitAsync(TestTimeout)) { } }); Assert.Equal(SessionManagerErrorCode.EventQueueOverflow, exception.ErrorCode); await WaitUntilAsync(() => session.State == SessionState.Faulted); Assert.Equal(SessionState.Faulted, session.State); GatewayMetricsSnapshot snapshot = metrics.GetSnapshot(); Assert.Equal(1, snapshot.QueueOverflows); Assert.Equal(1, snapshot.Faults); // The finally block in StreamEventsAsync calls StreamDisconnected("Detached") on the // overflow+fault path too; pin it here so a regression removing that call is caught. Assert.Equal(1, snapshot.StreamDisconnects); } /// /// Re-targeted in Task 5: under the DisconnectSubscriber policy a per-subscriber /// channel overflow disconnects only that subscriber's stream (terminal /// ) and records the overflow /// metric, but leaves the session and records no /// fault. The session, pump, and any other subscribers are unaffected. /// [Fact] public async Task StreamEventsAsync_WhenStreamQueueOverflowsWithDisconnectPolicy_LeavesSessionReady() { FakeWorkerClient workerClient = new(); using GatewayMetrics metrics = new(); GatewaySession session = CreateReadySession( workerClient, queueCapacity: 1, metrics: metrics, backpressurePolicy: EventBackpressurePolicy.DisconnectSubscriber); EventStreamService service = CreateService( new FakeSessionManager(session), metrics, queueCapacity: 1, backpressurePolicy: EventBackpressurePolicy.DisconnectSubscriber); for (ulong sequence = 1; sequence <= 50; sequence++) { workerClient.Events.Add(CreateWorkerEvent(sequence, MxEventFamily.OnDataChange)); } workerClient.CompleteAfterConfiguredEvents = true; await using IAsyncEnumerator subscriber = service .StreamEventsAsync(CreateRequest(session.SessionId), CancellationToken.None) .GetAsyncEnumerator(); SessionManagerException exception = await Assert.ThrowsAsync( async () => { while (await subscriber.MoveNextAsync().AsTask().WaitAsync(TestTimeout)) { } }); Assert.Equal(SessionManagerErrorCode.EventQueueOverflow, exception.ErrorCode); Assert.Equal(SessionState.Ready, session.State); GatewayMetricsSnapshot snapshot = metrics.GetSnapshot(); Assert.Equal(1, snapshot.QueueOverflows); Assert.Equal(0, snapshot.Faults); Assert.Equal(1, snapshot.StreamDisconnects); } /// Verifies that the event stream does not synthesize OperationComplete events from write completions. [Fact] public async Task StreamEventsAsync_DoesNotSynthesizeOperationComplete() { FakeWorkerClient workerClient = new(); GatewaySession session = CreateReadySession(workerClient); EventStreamService service = CreateService(new FakeSessionManager(session)); workerClient.Events.Add(CreateWorkerEvent(sequence: 10, MxEventFamily.OnWriteComplete)); workerClient.CompleteAfterConfiguredEvents = true; List events = await CollectEventsAsync(service, session.SessionId); MxEvent mxEvent = Assert.Single(events); Assert.Equal(MxEventFamily.OnWriteComplete, mxEvent.Family); Assert.DoesNotContain(events, candidate => candidate.Family == MxEventFamily.OperationComplete); } /// Verifies that a terminal fault from the worker event stream propagates and faults the session. [Fact] public async Task StreamEventsAsync_WhenWorkerEventStreamFaults_PropagatesTerminalFault() { FakeWorkerClient workerClient = new() { TerminalException = new WorkerClientException( WorkerClientErrorCode.WorkerFaulted, "worker terminal fault"), }; GatewaySession session = CreateReadySession(workerClient); using GatewayMetrics metrics = new(); EventStreamService service = CreateService(new FakeSessionManager(session), metrics); await using IAsyncEnumerator subscriber = service .StreamEventsAsync(CreateRequest(session.SessionId), CancellationToken.None) .GetAsyncEnumerator(); WorkerClientException exception = await Assert.ThrowsAsync( async () => await subscriber.MoveNextAsync().AsTask().WaitAsync(TestTimeout)); Assert.Equal(WorkerClientErrorCode.WorkerFaulted, exception.ErrorCode); Assert.Equal(SessionState.Faulted, session.State); Assert.Equal(1, metrics.GetSnapshot().Faults); } private static EventStreamService CreateService( FakeSessionManager sessionManager, GatewayMetrics? metrics = null, int queueCapacity = 8, EventBackpressurePolicy backpressurePolicy = EventBackpressurePolicy.FailFast) { return new EventStreamService( sessionManager, Options.Create(new GatewayOptions { Events = new EventOptions { QueueCapacity = queueCapacity, BackpressurePolicy = backpressurePolicy, }, }), metrics ?? new GatewayMetrics()); } private static async Task> CollectEventsAsync( EventStreamService service, string sessionId) { List events = []; await foreach (MxEvent mxEvent in service .StreamEventsAsync(CreateRequest(sessionId), CancellationToken.None) .WithCancellation(CancellationToken.None)) { events.Add(mxEvent); } return events; } private static StreamEventsRequest CreateRequest(string sessionId) { return new StreamEventsRequest { SessionId = sessionId, }; } private static GatewaySession CreateReadySession( FakeWorkerClient workerClient, string sessionId = "session-events", int queueCapacity = 8, GatewayMetrics? metrics = null, EventBackpressurePolicy backpressurePolicy = EventBackpressurePolicy.FailFast) { // The per-subscriber overflow policy now lives in the session's // SessionEventDistributor, so the session must share the same metrics sink and // backpressure policy the overflow assertions observe. queueCapacity flows into the // distributor's per-subscriber channel bound, which is what overflows. GatewaySession session = new( sessionId, GatewayContractInfo.DefaultBackendName, "pipe", "nonce", "client", ownerKeyId: null, "client-session", "client-correlation", TimeSpan.FromSeconds(30), TimeSpan.FromSeconds(30), TimeSpan.FromSeconds(10), TimeSpan.FromMinutes(30), DateTimeOffset.UtcNow, new SessionEventStreaming( new MxAccessGrpcMapper(), new EventOptions { QueueCapacity = queueCapacity, BackpressurePolicy = backpressurePolicy, }, NullLogger.Instance, TimeProvider.System, metrics ?? new GatewayMetrics())); session.AttachWorkerClient(workerClient); session.MarkReady(); return session; } private static WorkerEvent CreateWorkerEvent( ulong sequence, MxEventFamily family) { MxEvent mxEvent = new() { SessionId = "session-events", Family = family, WorkerSequence = sequence, }; switch (family) { case MxEventFamily.OnDataChange: mxEvent.OnDataChange = new OnDataChangeEvent(); break; case MxEventFamily.OnWriteComplete: mxEvent.OnWriteComplete = new OnWriteCompleteEvent(); break; case MxEventFamily.OperationComplete: mxEvent.OperationComplete = new OperationCompleteEvent(); break; case MxEventFamily.OnBufferedDataChange: mxEvent.OnBufferedDataChange = new OnBufferedDataChangeEvent(); break; } return new WorkerEvent { Event = mxEvent, }; } private static async Task WaitUntilAsync(Func predicate) { using CancellationTokenSource cancellationTokenSource = new(TestTimeout); while (!predicate()) { await Task.Delay(TimeSpan.FromMilliseconds(10), cancellationTokenSource.Token); } } /// Fake session manager for testing event streams. private sealed class FakeSessionManager : ISessionManager { private readonly IReadOnlyDictionary _sessions; /// Initializes a new instance of the FakeSessionManager. /// Sessions to manage. public FakeSessionManager(params GatewaySession[] sessions) { _sessions = sessions.ToDictionary(session => session.SessionId, StringComparer.Ordinal); } /// public Task OpenSessionAsync( SessionOpenRequest request, string? clientIdentity, string? ownerKeyId, CancellationToken cancellationToken) { return Task.FromResult(_sessions.Values.First()); } /// public bool TryGetSession( string sessionId, out GatewaySession gatewaySession) { return _sessions.TryGetValue(sessionId, out gatewaySession!); } /// public Task InvokeAsync( string sessionId, WorkerCommand command, CancellationToken cancellationToken) { return Task.FromResult(new WorkerCommandReply()); } /// public IAsyncEnumerable ReadEventsAsync( string sessionId, CancellationToken cancellationToken) { return _sessions[sessionId].ReadEventsAsync(cancellationToken); } /// public Task CloseSessionAsync( string sessionId, CancellationToken cancellationToken) { return Task.FromResult(new SessionCloseResult(sessionId, SessionState.Closed, AlreadyClosed: false)); } /// public Task KillWorkerAsync( string sessionId, string reason, CancellationToken cancellationToken) { return Task.FromResult(new SessionCloseResult(sessionId, SessionState.Closed, AlreadyClosed: false)); } /// public Task CloseExpiredLeasesAsync( DateTimeOffset now, CancellationToken cancellationToken) { return Task.FromResult(0); } /// public Task ShutdownAsync(CancellationToken cancellationToken) { return Task.CompletedTask; } } /// Fake worker client for testing event streams. private sealed class FakeWorkerClient : IWorkerClient { /// Gets the list of queued worker events. public List Events { get; } = []; /// Gets or sets whether to complete the event stream after configured events are yielded. public bool CompleteAfterConfiguredEvents { get; set; } /// Gets or sets an optional exception to throw as a terminal event stream fault. public Exception? TerminalException { get; init; } /// public string SessionId { get; } = "session-events"; /// public int? ProcessId { get; } = 4321; /// public WorkerClientState State { get; private set; } = WorkerClientState.Ready; /// public DateTimeOffset LastHeartbeatAt { get; } = DateTimeOffset.UtcNow; /// public Task StartAsync(CancellationToken cancellationToken) { return Task.CompletedTask; } /// public Task InvokeAsync( WorkerCommand command, TimeSpan timeout, CancellationToken cancellationToken) { return Task.FromResult(new WorkerCommandReply()); } /// public async IAsyncEnumerable ReadEventsAsync( [EnumeratorCancellation] CancellationToken cancellationToken) { foreach (WorkerEvent workerEvent in Events) { cancellationToken.ThrowIfCancellationRequested(); yield return workerEvent; } if (TerminalException is not null) { throw TerminalException; } if (CompleteAfterConfiguredEvents) { yield break; } await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken); } /// public Task ShutdownAsync( TimeSpan timeout, CancellationToken cancellationToken) { State = WorkerClientState.Closed; return Task.CompletedTask; } /// public void Kill(string reason) { State = WorkerClientState.Faulted; } /// public ValueTask DisposeAsync() { return ValueTask.CompletedTask; } } }