Fix reliability findings

This commit is contained in:
Joseph Doherty
2026-04-28 06:27:01 -04:00
parent 907aa49aea
commit b0041c5d18
9 changed files with 233 additions and 21 deletions
@@ -85,6 +85,32 @@ public sealed class EventStreamServiceTests
await WaitUntilAsync(() => session.ActiveEventSubscriberCount == 0);
}
[Fact]
public async Task StreamEventsAsync_WhenDisposedWithBufferedEvents_ResetsStreamQueueDepth()
{
FakeWorkerClient workerClient = new();
GatewaySession session = CreateReadySession(workerClient);
using GatewayMetrics metrics = new();
EventStreamService service = CreateService(
new FakeSessionManager(session),
metrics,
queueCapacity: 8);
workerClient.Events.Add(CreateWorkerEvent(sequence: 1, MxEventFamily.OnDataChange));
workerClient.Events.Add(CreateWorkerEvent(sequence: 2, MxEventFamily.OnDataChange));
workerClient.Events.Add(CreateWorkerEvent(sequence: 3, MxEventFamily.OnDataChange));
workerClient.CompleteAfterConfiguredEvents = true;
await using IAsyncEnumerator<MxEvent> subscriber = service
.StreamEventsAsync(CreateRequest(session.SessionId), CancellationToken.None)
.GetAsyncEnumerator();
Assert.True(await subscriber.MoveNextAsync().AsTask().WaitAsync(TestTimeout));
await WaitUntilAsync(() => metrics.GetSnapshot().GrpcEventStreamQueueDepth > 0);
await subscriber.DisposeAsync();
await WaitUntilAsync(() => metrics.GetSnapshot().GrpcEventStreamQueueDepth == 0);
}
[Fact]
public async Task StreamEventsAsync_WhenStreamQueueOverflows_FaultsSessionAndReportsOverflow()
{
@@ -179,6 +179,48 @@ public sealed class SessionManagerTests
Assert.Equal(1, workerClient.KillCount);
}
[Fact]
public async Task CloseSessionAsync_WhenWorkerShutdownFails_RemovesSessionAndReleasesSlot()
{
FakeWorkerClient failingWorkerClient = new()
{
ShutdownException = new WorkerClientException(
WorkerClientErrorCode.ShutdownTimeout,
"Worker shutdown timed out."),
};
FakeWorkerClient replacementWorkerClient = new();
SessionRegistry registry = new();
using GatewayMetrics metrics = new();
SessionManager manager = CreateManager(
new QueueingSessionWorkerClientFactory(failingWorkerClient, replacementWorkerClient),
registry,
metrics,
CreateOptions(maxSessions: 1));
GatewaySession firstSession = await manager.OpenSessionAsync(
CreateOpenRequest(),
"client-1",
CancellationToken.None);
metrics.EventReceived(firstSession.SessionId, MxEventFamily.OnDataChange.ToString());
SessionManagerException exception = await Assert.ThrowsAsync<SessionManagerException>(
async () => await manager.CloseSessionAsync(firstSession.SessionId, CancellationToken.None));
GatewaySession secondSession = await manager.OpenSessionAsync(
CreateOpenRequest(),
"client-2",
CancellationToken.None);
Assert.Equal(SessionManagerErrorCode.CloseFailed, exception.ErrorCode);
Assert.False(manager.TryGetSession(firstSession.SessionId, out _));
Assert.True(manager.TryGetSession(secondSession.SessionId, out _));
Assert.Equal(1, registry.Count);
Assert.Equal(1, failingWorkerClient.KillCount);
Assert.Equal(1, failingWorkerClient.DisposeCount);
GatewayMetricsSnapshot snapshot = metrics.GetSnapshot();
Assert.Equal(0, snapshot.SessionsClosed);
Assert.False(snapshot.EventsBySession.ContainsKey(firstSession.SessionId));
Assert.Equal(1, snapshot.OpenSessions);
}
[Fact]
public async Task OpenSessionAsync_WhenWorkerCreationFails_RemovesSessionFromRegistry()
{
@@ -254,14 +296,14 @@ public sealed class SessionManagerTests
metrics ?? new GatewayMetrics());
}
private static GatewayOptions CreateOptions()
private static GatewayOptions CreateOptions(int maxSessions = 64)
{
return new GatewayOptions
{
Sessions = new SessionOptions
{
DefaultCommandTimeoutSeconds = 30,
MaxSessions = 64,
MaxSessions = maxSessions,
},
Worker = new WorkerOptions
{
@@ -359,6 +401,8 @@ public sealed class SessionManagerTests
public int KillCount { get; private set; }
public int DisposeCount { get; private set; }
public Exception? ShutdownException { get; init; }
public WorkerCommand? LastCommand { get; private set; }
@@ -424,6 +468,7 @@ public sealed class SessionManagerTests
public ValueTask DisposeAsync()
{
DisposeCount++;
return ValueTask.CompletedTask;
}
}