fix(sessions): refuse OpenSession while the gateway is stopping; drain until the registry is empty

ShutdownAsync iterated a single registry snapshot while Kestrel still
accepted OpenSession calls (ApplicationStopping fires before hosted
services stop), so a session opened behind the snapshot had nobody left to
close it and its x86 worker outlived the gateway as an orphan.

SessionManager now observes IHostApplicationLifetime.ApplicationStopping
(and sets the same flag at the start of ShutdownAsync) and rejects
OpenSessionAsync with the new SessionManagerErrorCode.GatewayStopping,
re-checked after registration so the flag write before the drain's
snapshot closes the race. GatewayStopping maps to gRPC Unavailable and is
not a fault. ShutdownAsync re-snapshots until the registry is empty,
bounded at three passes.
This commit is contained in:
Joseph Doherty
2026-10-22 06:23:40 -04:00
parent 09666cc719
commit 37ab629d70
6 changed files with 317 additions and 62 deletions
@@ -1419,6 +1419,84 @@ public sealed class SessionManagerTests
Assert.False(manager.TryGetSession(secondSession.SessionId, out _));
}
/// <summary>
/// Once the drain has started, a new open must be refused with <c>GatewayStopping</c>
/// rather than accepted: a session created behind the drain's snapshot has nobody left to
/// close it, and its worker would outlive the gateway as an orphan. The refusal is not an
/// open failure — no slot is taken, nothing is registered and no fault is recorded.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task OpenSessionAsync_AfterShutdownStarted_ThrowsGatewayStopping()
{
FakeWorkerClient workerClient = new();
SessionRegistry registry = new();
using GatewayMetrics metrics = new();
SessionManager manager = CreateManager(new FakeSessionWorkerClientFactory(workerClient), registry, metrics);
await manager.ShutdownAsync(CancellationToken.None);
SessionManagerException exception = await Assert.ThrowsAsync<SessionManagerException>(
async () => await manager.OpenSessionAsync(CreateOpenRequest(), "client-1", ownerKeyId: null, CancellationToken.None));
Assert.Equal(SessionManagerErrorCode.GatewayStopping, exception.ErrorCode);
Assert.Equal(0, registry.Count);
Assert.Equal(0, metrics.GetSnapshot().SessionsOpened);
Assert.Equal(0, metrics.GetSnapshot().Faults);
}
/// <summary>
/// The host signals <c>ApplicationStopping</c> before hosted services stop, and Kestrel keeps
/// accepting gRPC calls until its own <c>StopAsync</c> runs — so the gate must close on the
/// lifetime signal, not only when <c>ShutdownAsync</c> is finally called.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task OpenSessionAsync_WhenApplicationStopping_ThrowsGatewayStopping()
{
FakeWorkerClient workerClient = new();
using FakeHostApplicationLifetime lifetime = new();
SessionManager manager = new(
new SessionRegistry(),
new FakeSessionWorkerClientFactory(workerClient),
Options.Create(CreateOptions()),
new GatewayMetrics(),
applicationLifetime: lifetime);
GatewaySession beforeStopping = await manager.OpenSessionAsync(CreateOpenRequest(), "client-1", ownerKeyId: null, CancellationToken.None);
Assert.Equal(SessionState.Ready, beforeStopping.State);
lifetime.SignalStopping();
SessionManagerException exception = await Assert.ThrowsAsync<SessionManagerException>(
async () => await manager.OpenSessionAsync(CreateOpenRequest(), "client-2", ownerKeyId: null, CancellationToken.None));
Assert.Equal(SessionManagerErrorCode.GatewayStopping, exception.ErrorCode);
}
/// <summary>
/// An open that passed the stopping gate but registered its session after the drain took
/// its snapshot is still caught: the drain re-snapshots until the registry is empty, so the
/// late session is closed by the drain rather than left running.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task ShutdownAsync_DrainsSessionRegisteredAfterFirstSnapshot()
{
FakeWorkerClient workerClient = new();
LateFirstSnapshotRegistry registry = new();
using GatewayMetrics metrics = new();
SessionManager manager = CreateManager(new FakeSessionWorkerClientFactory(workerClient), registry, metrics);
GatewaySession session = await manager.OpenSessionAsync(CreateOpenRequest(), "client-1", ownerKeyId: null, CancellationToken.None);
await manager.ShutdownAsync(CancellationToken.None);
Assert.True(registry.SnapshotCalls >= 2);
Assert.Equal(SessionState.Closed, session.State);
Assert.Equal(1, workerClient.ShutdownCount);
Assert.Equal(0, registry.Count);
Assert.Equal(0, metrics.GetSnapshot().OpenSessions);
Assert.Equal(1, metrics.GetSnapshot().SessionsClosed);
}
/// <summary>Verifies that shutdown closes all registered sessions.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
@@ -1634,6 +1712,79 @@ public sealed class SessionManagerTests
}
}
/// <summary>
/// Lifetime fake whose <see cref="ApplicationStopping"/> the test fires on demand; the
/// other two tokens never fire.
/// </summary>
private sealed class FakeHostApplicationLifetime : Microsoft.Extensions.Hosting.IHostApplicationLifetime, IDisposable
{
private readonly CancellationTokenSource _stopping = new();
/// <inheritdoc />
public CancellationToken ApplicationStarted => CancellationToken.None;
/// <inheritdoc />
public CancellationToken ApplicationStopping => _stopping.Token;
/// <inheritdoc />
public CancellationToken ApplicationStopped => CancellationToken.None;
/// <summary>Fires <see cref="ApplicationStopping"/>.</summary>
public void SignalStopping()
{
_stopping.Cancel();
}
/// <inheritdoc />
public void StopApplication()
{
SignalStopping();
}
/// <inheritdoc />
public void Dispose()
{
_stopping.Dispose();
}
}
/// <summary>
/// Registry whose first <see cref="Snapshot"/> is empty regardless of its contents,
/// simulating a session whose <c>TryAdd</c> landed just after the drain's first snapshot;
/// every later snapshot is the real one.
/// </summary>
private sealed class LateFirstSnapshotRegistry : ISessionRegistry
{
private readonly SessionRegistry _inner = new();
/// <summary>Gets how many snapshots have been taken.</summary>
public int SnapshotCalls { get; private set; }
/// <inheritdoc />
public int Count => _inner.Count;
/// <inheritdoc />
public int ActiveCount => _inner.ActiveCount;
/// <inheritdoc />
public bool TryAdd(GatewaySession session) => _inner.TryAdd(session);
/// <inheritdoc />
public bool TryGet(string sessionId, [System.Diagnostics.CodeAnalysis.MaybeNullWhen(false)] out GatewaySession session)
=> _inner.TryGet(sessionId, out session);
/// <inheritdoc />
public bool TryRemove(string sessionId, [System.Diagnostics.CodeAnalysis.MaybeNullWhen(false)] out GatewaySession session)
=> _inner.TryRemove(sessionId, out session);
/// <inheritdoc />
public IReadOnlyCollection<GatewaySession> Snapshot()
{
SnapshotCalls++;
return SnapshotCalls == 1 ? [] : _inner.Snapshot();
}
}
private sealed class FakeWorkerClient : IWorkerClient
{
private WorkerClientState _state = WorkerClientState.Ready;