Files
mxaccessgw/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Grpc/EventStreamServiceTests.cs
T

591 lines
24 KiB
C#

using System.Runtime.CompilerServices;
using Grpc.Core;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using ZB.MOM.WW.MxGateway.Contracts;
using ZB.MOM.WW.MxGateway.Contracts.Proto;
using ZB.MOM.WW.MxGateway.Server.Configuration;
using ZB.MOM.WW.MxGateway.Server.Grpc;
using ZB.MOM.WW.MxGateway.Server.Metrics;
using ZB.MOM.WW.MxGateway.Server.Sessions;
using ZB.MOM.WW.MxGateway.Server.Workers;
namespace ZB.MOM.WW.MxGateway.Tests.Gateway.Grpc;
public sealed class EventStreamServiceTests
{
private static readonly TimeSpan TestTimeout = TimeSpan.FromSeconds(5);
/// <summary>Verifies that events from the worker stream maintain their original sequence order.</summary>
[Fact]
public async Task StreamEventsAsync_YieldsEventsInWorkerOrder()
{
FakeWorkerClient workerClient = new();
GatewaySession session = CreateReadySession(workerClient);
FakeSessionManager sessionManager = new(session);
using GatewayMetrics metrics = new();
EventStreamService service = CreateService(sessionManager, metrics: metrics);
workerClient.Events.Add(CreateWorkerEvent(sequence: 10, MxEventFamily.OnDataChange));
workerClient.Events.Add(CreateWorkerEvent(sequence: 11, MxEventFamily.OnWriteComplete));
workerClient.CompleteAfterConfiguredEvents = true;
List<MxEvent> events = await CollectEventsAsync(service, session.SessionId);
Assert.Equal([10UL, 11UL], events.Select(mxEvent => mxEvent.WorkerSequence).ToArray());
Assert.Equal(MxEventFamily.OnDataChange, events[0].Family);
Assert.Equal(MxEventFamily.OnWriteComplete, events[1].Family);
Assert.Equal(1, metrics.GetSnapshot().StreamDisconnects);
}
/// <summary>Verifies that a second event subscriber is rejected when one is already active.</summary>
[Fact]
public async Task StreamEventsAsync_WhenSecondSubscriberStarts_RejectsClearly()
{
FakeWorkerClient workerClient = new();
GatewaySession session = CreateReadySession(workerClient);
EventStreamService service = CreateService(new FakeSessionManager(session));
using CancellationTokenSource firstSubscriberCancellation = new();
await using IAsyncEnumerator<MxEvent> firstSubscriber = service
.StreamEventsAsync(CreateRequest(session.SessionId), firstSubscriberCancellation.Token)
.GetAsyncEnumerator(firstSubscriberCancellation.Token);
Task<bool> firstMoveTask = firstSubscriber.MoveNextAsync().AsTask();
await WaitUntilAsync(() => session.ActiveEventSubscriberCount == 1);
await using IAsyncEnumerator<MxEvent> secondSubscriber = service
.StreamEventsAsync(CreateRequest(session.SessionId), CancellationToken.None)
.GetAsyncEnumerator();
SessionManagerException exception = await Assert.ThrowsAsync<SessionManagerException>(
async () => await secondSubscriber.MoveNextAsync().AsTask().WaitAsync(TestTimeout));
Assert.Equal(SessionManagerErrorCode.EventSubscriberAlreadyActive, exception.ErrorCode);
await firstSubscriberCancellation.CancelAsync();
await Assert.ThrowsAnyAsync<OperationCanceledException>(
async () => await firstMoveTask.WaitAsync(TestTimeout));
await firstSubscriber.DisposeAsync();
await WaitUntilAsync(() => session.ActiveEventSubscriberCount == 0);
}
/// <summary>Verifies that canceling an event stream detaches the subscriber cleanly.</summary>
[Fact]
public async Task StreamEventsAsync_WhenCanceled_DetachesSubscriber()
{
FakeWorkerClient workerClient = new();
GatewaySession session = CreateReadySession(workerClient);
EventStreamService service = CreateService(new FakeSessionManager(session));
using CancellationTokenSource cancellationTokenSource = new();
await using IAsyncEnumerator<MxEvent> subscriber = service
.StreamEventsAsync(CreateRequest(session.SessionId), cancellationTokenSource.Token)
.GetAsyncEnumerator(cancellationTokenSource.Token);
Task<bool> moveTask = subscriber.MoveNextAsync().AsTask();
await WaitUntilAsync(() => session.ActiveEventSubscriberCount == 1);
await cancellationTokenSource.CancelAsync();
await Assert.ThrowsAnyAsync<OperationCanceledException>(
async () => await moveTask.WaitAsync(TestTimeout));
await subscriber.DisposeAsync();
await WaitUntilAsync(() => session.ActiveEventSubscriberCount == 0);
}
/// <summary>Verifies that disposing an event stream with buffered events resets the queue depth metric.</summary>
[Fact]
public async Task StreamEventsAsync_WhenDisposedWithBufferedEvents_ResetsStreamQueueDepth()
{
FakeWorkerClient workerClient = new();
GatewaySession session = CreateReadySession(workerClient);
using GatewayMetrics metrics = new();
EventStreamService service = CreateService(
new FakeSessionManager(session),
metrics,
queueCapacity: 8);
workerClient.Events.Add(CreateWorkerEvent(sequence: 1, MxEventFamily.OnDataChange));
workerClient.Events.Add(CreateWorkerEvent(sequence: 2, MxEventFamily.OnDataChange));
workerClient.Events.Add(CreateWorkerEvent(sequence: 3, MxEventFamily.OnDataChange));
workerClient.CompleteAfterConfiguredEvents = true;
await using IAsyncEnumerator<MxEvent> subscriber = service
.StreamEventsAsync(CreateRequest(session.SessionId), CancellationToken.None)
.GetAsyncEnumerator();
Assert.True(await subscriber.MoveNextAsync().AsTask().WaitAsync(TestTimeout));
await WaitUntilAsync(() => metrics.GetSnapshot().GrpcEventStreamQueueDepth > 0);
await subscriber.DisposeAsync();
await WaitUntilAsync(() => metrics.GetSnapshot().GrpcEventStreamQueueDepth == 0);
}
/// <summary>Verifies that queue depth metrics correctly track concurrent event streams across multiple sessions.</summary>
[Fact]
public async Task StreamEventsAsync_WithConcurrentStreams_TracksAggregateQueueDepth()
{
FakeWorkerClient firstWorkerClient = new();
FakeWorkerClient secondWorkerClient = new();
GatewaySession firstSession = CreateReadySession(firstWorkerClient, "session-events-1");
GatewaySession secondSession = CreateReadySession(secondWorkerClient, "session-events-2");
using GatewayMetrics metrics = new();
EventStreamService service = CreateService(
new FakeSessionManager(firstSession, secondSession),
metrics,
queueCapacity: 8);
for (ulong sequence = 1; sequence <= 3; sequence++)
{
firstWorkerClient.Events.Add(CreateWorkerEvent(sequence, MxEventFamily.OnDataChange));
secondWorkerClient.Events.Add(CreateWorkerEvent(sequence, MxEventFamily.OnDataChange));
}
firstWorkerClient.CompleteAfterConfiguredEvents = true;
secondWorkerClient.CompleteAfterConfiguredEvents = true;
await using IAsyncEnumerator<MxEvent> firstSubscriber = service
.StreamEventsAsync(CreateRequest(firstSession.SessionId), CancellationToken.None)
.GetAsyncEnumerator();
await using IAsyncEnumerator<MxEvent> secondSubscriber = service
.StreamEventsAsync(CreateRequest(secondSession.SessionId), CancellationToken.None)
.GetAsyncEnumerator();
Assert.True(await firstSubscriber.MoveNextAsync().AsTask().WaitAsync(TestTimeout));
Assert.True(await secondSubscriber.MoveNextAsync().AsTask().WaitAsync(TestTimeout));
await WaitUntilAsync(() => metrics.GetSnapshot().GrpcEventStreamQueueDepth == 4);
await firstSubscriber.DisposeAsync();
await WaitUntilAsync(() => metrics.GetSnapshot().GrpcEventStreamQueueDepth == 2);
await secondSubscriber.DisposeAsync();
await WaitUntilAsync(() => metrics.GetSnapshot().GrpcEventStreamQueueDepth == 0);
}
/// <summary>
/// Re-targeted in Task 5: a per-subscriber channel overflow in the session's
/// <see cref="SessionEventDistributor"/> faults the whole session under the legacy
/// single-subscriber FailFast policy (the default, single-subscriber mode) and records
/// the overflow + fault metrics. The distributor completes this subscriber's channel
/// with the overflow fault, which surfaces here as the same
/// <see cref="SessionManagerErrorCode.EventQueueOverflow"/> the pre-epic per-RPC
/// overflow produced.
/// </summary>
[Fact]
public async Task StreamEventsAsync_WhenStreamQueueOverflows_FaultsSessionAndReportsOverflow()
{
FakeWorkerClient workerClient = new();
using GatewayMetrics metrics = new();
GatewaySession session = CreateReadySession(
workerClient,
queueCapacity: 1,
metrics: metrics,
backpressurePolicy: EventBackpressurePolicy.FailFast);
EventStreamService service = CreateService(
new FakeSessionManager(session),
metrics,
queueCapacity: 1);
for (ulong sequence = 1; sequence <= 50; sequence++)
{
workerClient.Events.Add(CreateWorkerEvent(sequence, MxEventFamily.OnDataChange));
}
workerClient.CompleteAfterConfiguredEvents = true;
await using IAsyncEnumerator<MxEvent> subscriber = service
.StreamEventsAsync(CreateRequest(session.SessionId), CancellationToken.None)
.GetAsyncEnumerator();
// The pump fans 50 events into a capacity-1 subscriber channel faster than this
// single reader drains, so one of the reads observes the terminal overflow fault.
SessionManagerException exception = await Assert.ThrowsAsync<SessionManagerException>(
async () =>
{
while (await subscriber.MoveNextAsync().AsTask().WaitAsync(TestTimeout))
{
}
});
Assert.Equal(SessionManagerErrorCode.EventQueueOverflow, exception.ErrorCode);
await WaitUntilAsync(() => session.State == SessionState.Faulted);
Assert.Equal(SessionState.Faulted, session.State);
GatewayMetricsSnapshot snapshot = metrics.GetSnapshot();
Assert.Equal(1, snapshot.QueueOverflows);
Assert.Equal(1, snapshot.Faults);
// The finally block in StreamEventsAsync calls StreamDisconnected("Detached") on the
// overflow+fault path too; pin it here so a regression removing that call is caught.
Assert.Equal(1, snapshot.StreamDisconnects);
}
/// <summary>
/// Re-targeted in Task 5: under the DisconnectSubscriber policy a per-subscriber
/// channel overflow disconnects only that subscriber's stream (terminal
/// <see cref="SessionManagerErrorCode.EventQueueOverflow"/>) and records the overflow
/// metric, but leaves the session <see cref="SessionState.Ready"/> and records no
/// fault. The session, pump, and any other subscribers are unaffected.
/// </summary>
[Fact]
public async Task StreamEventsAsync_WhenStreamQueueOverflowsWithDisconnectPolicy_LeavesSessionReady()
{
FakeWorkerClient workerClient = new();
using GatewayMetrics metrics = new();
GatewaySession session = CreateReadySession(
workerClient,
queueCapacity: 1,
metrics: metrics,
backpressurePolicy: EventBackpressurePolicy.DisconnectSubscriber);
EventStreamService service = CreateService(
new FakeSessionManager(session),
metrics,
queueCapacity: 1,
backpressurePolicy: EventBackpressurePolicy.DisconnectSubscriber);
for (ulong sequence = 1; sequence <= 50; sequence++)
{
workerClient.Events.Add(CreateWorkerEvent(sequence, MxEventFamily.OnDataChange));
}
workerClient.CompleteAfterConfiguredEvents = true;
await using IAsyncEnumerator<MxEvent> subscriber = service
.StreamEventsAsync(CreateRequest(session.SessionId), CancellationToken.None)
.GetAsyncEnumerator();
SessionManagerException exception = await Assert.ThrowsAsync<SessionManagerException>(
async () =>
{
while (await subscriber.MoveNextAsync().AsTask().WaitAsync(TestTimeout))
{
}
});
Assert.Equal(SessionManagerErrorCode.EventQueueOverflow, exception.ErrorCode);
Assert.Equal(SessionState.Ready, session.State);
GatewayMetricsSnapshot snapshot = metrics.GetSnapshot();
Assert.Equal(1, snapshot.QueueOverflows);
Assert.Equal(0, snapshot.Faults);
Assert.Equal(1, snapshot.StreamDisconnects);
}
/// <summary>Verifies that the event stream does not synthesize OperationComplete events from write completions.</summary>
[Fact]
public async Task StreamEventsAsync_DoesNotSynthesizeOperationComplete()
{
FakeWorkerClient workerClient = new();
GatewaySession session = CreateReadySession(workerClient);
EventStreamService service = CreateService(new FakeSessionManager(session));
workerClient.Events.Add(CreateWorkerEvent(sequence: 10, MxEventFamily.OnWriteComplete));
workerClient.CompleteAfterConfiguredEvents = true;
List<MxEvent> events = await CollectEventsAsync(service, session.SessionId);
MxEvent mxEvent = Assert.Single(events);
Assert.Equal(MxEventFamily.OnWriteComplete, mxEvent.Family);
Assert.DoesNotContain(events, candidate => candidate.Family == MxEventFamily.OperationComplete);
}
/// <summary>Verifies that a terminal fault from the worker event stream propagates and faults the session.</summary>
[Fact]
public async Task StreamEventsAsync_WhenWorkerEventStreamFaults_PropagatesTerminalFault()
{
FakeWorkerClient workerClient = new()
{
TerminalException = new WorkerClientException(
WorkerClientErrorCode.WorkerFaulted,
"worker terminal fault"),
};
GatewaySession session = CreateReadySession(workerClient);
using GatewayMetrics metrics = new();
EventStreamService service = CreateService(new FakeSessionManager(session), metrics);
await using IAsyncEnumerator<MxEvent> subscriber = service
.StreamEventsAsync(CreateRequest(session.SessionId), CancellationToken.None)
.GetAsyncEnumerator();
WorkerClientException exception = await Assert.ThrowsAsync<WorkerClientException>(
async () => await subscriber.MoveNextAsync().AsTask().WaitAsync(TestTimeout));
Assert.Equal(WorkerClientErrorCode.WorkerFaulted, exception.ErrorCode);
Assert.Equal(SessionState.Faulted, session.State);
Assert.Equal(1, metrics.GetSnapshot().Faults);
}
private static EventStreamService CreateService(
FakeSessionManager sessionManager,
GatewayMetrics? metrics = null,
int queueCapacity = 8,
EventBackpressurePolicy backpressurePolicy = EventBackpressurePolicy.FailFast)
{
return new EventStreamService(
sessionManager,
Options.Create(new GatewayOptions
{
Events = new EventOptions
{
QueueCapacity = queueCapacity,
BackpressurePolicy = backpressurePolicy,
},
}),
metrics ?? new GatewayMetrics());
}
private static async Task<List<MxEvent>> CollectEventsAsync(
EventStreamService service,
string sessionId)
{
List<MxEvent> events = [];
await foreach (MxEvent mxEvent in service
.StreamEventsAsync(CreateRequest(sessionId), CancellationToken.None)
.WithCancellation(CancellationToken.None))
{
events.Add(mxEvent);
}
return events;
}
private static StreamEventsRequest CreateRequest(string sessionId)
{
return new StreamEventsRequest
{
SessionId = sessionId,
};
}
private static GatewaySession CreateReadySession(
FakeWorkerClient workerClient,
string sessionId = "session-events",
int queueCapacity = 8,
GatewayMetrics? metrics = null,
EventBackpressurePolicy backpressurePolicy = EventBackpressurePolicy.FailFast)
{
// The per-subscriber overflow policy now lives in the session's
// SessionEventDistributor, so the session must share the same metrics sink and
// backpressure policy the overflow assertions observe. queueCapacity flows into the
// distributor's per-subscriber channel bound, which is what overflows.
GatewaySession session = new(
sessionId,
GatewayContractInfo.DefaultBackendName,
"pipe",
"nonce",
"client",
ownerKeyId: null,
"client-session",
"client-correlation",
TimeSpan.FromSeconds(30),
TimeSpan.FromSeconds(30),
TimeSpan.FromSeconds(10),
TimeSpan.FromMinutes(30),
DateTimeOffset.UtcNow,
new SessionEventStreaming(
new MxAccessGrpcMapper(),
new EventOptions
{
QueueCapacity = queueCapacity,
BackpressurePolicy = backpressurePolicy,
},
NullLogger<SessionEventDistributor>.Instance,
TimeProvider.System,
metrics ?? new GatewayMetrics()));
session.AttachWorkerClient(workerClient);
session.MarkReady();
return session;
}
private static WorkerEvent CreateWorkerEvent(
ulong sequence,
MxEventFamily family)
{
MxEvent mxEvent = new()
{
SessionId = "session-events",
Family = family,
WorkerSequence = sequence,
};
switch (family)
{
case MxEventFamily.OnDataChange:
mxEvent.OnDataChange = new OnDataChangeEvent();
break;
case MxEventFamily.OnWriteComplete:
mxEvent.OnWriteComplete = new OnWriteCompleteEvent();
break;
case MxEventFamily.OperationComplete:
mxEvent.OperationComplete = new OperationCompleteEvent();
break;
case MxEventFamily.OnBufferedDataChange:
mxEvent.OnBufferedDataChange = new OnBufferedDataChangeEvent();
break;
}
return new WorkerEvent
{
Event = mxEvent,
};
}
private static async Task WaitUntilAsync(Func<bool> predicate)
{
using CancellationTokenSource cancellationTokenSource = new(TestTimeout);
while (!predicate())
{
await Task.Delay(TimeSpan.FromMilliseconds(10), cancellationTokenSource.Token);
}
}
/// <summary>Fake session manager for testing event streams.</summary>
private sealed class FakeSessionManager : ISessionManager
{
private readonly IReadOnlyDictionary<string, GatewaySession> _sessions;
/// <summary>Initializes a new instance of the FakeSessionManager.</summary>
/// <param name="sessions">Sessions to manage.</param>
public FakeSessionManager(params GatewaySession[] sessions)
{
_sessions = sessions.ToDictionary(session => session.SessionId, StringComparer.Ordinal);
}
/// <inheritdoc />
public Task<GatewaySession> OpenSessionAsync(
SessionOpenRequest request,
string? clientIdentity,
string? ownerKeyId,
CancellationToken cancellationToken)
{
return Task.FromResult(_sessions.Values.First());
}
/// <inheritdoc />
public bool TryGetSession(
string sessionId,
out GatewaySession gatewaySession)
{
return _sessions.TryGetValue(sessionId, out gatewaySession!);
}
/// <inheritdoc />
public Task<WorkerCommandReply> InvokeAsync(
string sessionId,
WorkerCommand command,
CancellationToken cancellationToken)
{
return Task.FromResult(new WorkerCommandReply());
}
/// <inheritdoc />
public IAsyncEnumerable<WorkerEvent> ReadEventsAsync(
string sessionId,
CancellationToken cancellationToken)
{
return _sessions[sessionId].ReadEventsAsync(cancellationToken);
}
/// <inheritdoc />
public Task<SessionCloseResult> CloseSessionAsync(
string sessionId,
CancellationToken cancellationToken)
{
return Task.FromResult(new SessionCloseResult(sessionId, SessionState.Closed, AlreadyClosed: false));
}
/// <inheritdoc />
public Task<SessionCloseResult> KillWorkerAsync(
string sessionId,
string reason,
CancellationToken cancellationToken)
{
return Task.FromResult(new SessionCloseResult(sessionId, SessionState.Closed, AlreadyClosed: false));
}
/// <inheritdoc />
public Task<int> CloseExpiredLeasesAsync(
DateTimeOffset now,
CancellationToken cancellationToken)
{
return Task.FromResult(0);
}
/// <inheritdoc />
public Task ShutdownAsync(CancellationToken cancellationToken)
{
return Task.CompletedTask;
}
}
/// <summary>Fake worker client for testing event streams.</summary>
private sealed class FakeWorkerClient : IWorkerClient
{
/// <summary>Gets the list of queued worker events.</summary>
public List<WorkerEvent> Events { get; } = [];
/// <summary>Gets or sets whether to complete the event stream after configured events are yielded.</summary>
public bool CompleteAfterConfiguredEvents { get; set; }
/// <summary>Gets or sets an optional exception to throw as a terminal event stream fault.</summary>
public Exception? TerminalException { get; init; }
/// <inheritdoc />
public string SessionId { get; } = "session-events";
/// <inheritdoc />
public int? ProcessId { get; } = 4321;
/// <inheritdoc />
public WorkerClientState State { get; private set; } = WorkerClientState.Ready;
/// <inheritdoc />
public DateTimeOffset LastHeartbeatAt { get; } = DateTimeOffset.UtcNow;
/// <inheritdoc />
public Task StartAsync(CancellationToken cancellationToken)
{
return Task.CompletedTask;
}
/// <inheritdoc />
public Task<WorkerCommandReply> InvokeAsync(
WorkerCommand command,
TimeSpan timeout,
CancellationToken cancellationToken)
{
return Task.FromResult(new WorkerCommandReply());
}
/// <inheritdoc />
public async IAsyncEnumerable<WorkerEvent> ReadEventsAsync(
[EnumeratorCancellation] CancellationToken cancellationToken)
{
foreach (WorkerEvent workerEvent in Events)
{
cancellationToken.ThrowIfCancellationRequested();
yield return workerEvent;
}
if (TerminalException is not null)
{
throw TerminalException;
}
if (CompleteAfterConfiguredEvents)
{
yield break;
}
await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken);
}
/// <inheritdoc />
public Task ShutdownAsync(
TimeSpan timeout,
CancellationToken cancellationToken)
{
State = WorkerClientState.Closed;
return Task.CompletedTask;
}
/// <inheritdoc />
public void Kill(string reason)
{
State = WorkerClientState.Faulted;
}
/// <inheritdoc />
public ValueTask DisposeAsync()
{
return ValueTask.CompletedTask;
}
}
}