fix(sessions): refuse a dead worker at attach, fault on pump readiness failure, keep cancel/stopping out of OpenFailed

- TryAttachWorkerClientAndMarkReadyAsync returns SessionAttachOutcome and refuses a
  worker client that is already Faulted/Closed; the open discards it and fails OpenFailed.
- MapWorkerEventsAsync calls ObserveWorkerTerminalState when its first readiness check
  fails, so a worker that dies between attach and the first pump read faults the session
  and the faulted sweep reaps it.
- A caller-cancelled open rolls back, records no fault and rethrows the cancellation
  (gRPC Cancelled); an attach refused because the drain closed the session reports
  GatewayStopping.
- docs/Sessions.md updated.
This commit is contained in:
Joseph Doherty
2026-10-22 08:41:02 -04:00
parent 2375df5df3
commit a178d68b60
6 changed files with 256 additions and 31 deletions
@@ -88,6 +88,78 @@ public sealed class GatewaySessionWorkerExitTests
Assert.Equal(0, metrics.GetSnapshot().OpenSessions);
}
/// <summary>
/// A worker that dies right after its handshake (STA loop fault, exit code 7) can reach
/// <c>Faulted</c> before the open attaches it. The attach must refuse such a client: the
/// open fails with <c>OpenFailed</c>, the client is discarded, and no session is left
/// registered reporting <c>Ready</c> over a dead worker until its lease expires.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task OpenSession_WorkerAlreadyFaultedAtAttach_FailsOpenAndDiscardsClient()
{
await using FakeWorkerHarness fakeWorker = await FakeWorkerHarness.CreateConnectedPairAsync();
using GatewayMetrics metrics = new();
WorkerClient client = fakeWorker.CreateClient(metrics: metrics);
await StartClientAsync(fakeWorker, client);
await fakeWorker.DisposeWorkerSideAsync();
await WaitUntilAsync(() => client.State == WorkerClientState.Faulted);
SessionRegistry registry = new();
SessionManager manager = new(
registry,
new SingleClientFactory(client),
Options.Create(new GatewayOptions()),
metrics);
SessionManagerException exception = await Assert.ThrowsAsync<SessionManagerException>(
async () => await OpenSessionAsync(manager));
Assert.Equal(SessionManagerErrorCode.OpenFailed, exception.ErrorCode);
Assert.Contains("Faulted", exception.InnerException?.Message, StringComparison.Ordinal);
Assert.Equal(0, registry.Count);
GatewayMetricsSnapshot snapshot = metrics.GetSnapshot();
Assert.Equal(0, snapshot.OpenSessions);
Assert.Equal(0, snapshot.SessionsOpened);
}
/// <summary>
/// The worker is still <c>Ready</c> when the open attaches it but faults before the event
/// pump's first read (which the dashboard mirror starts at <c>MarkReady</c>). The pump's
/// readiness check then fails without ever reaching the worker event stream, so it is that
/// failure path that must notice the dead worker: the session follows it to
/// <c>Faulted</c> and the faulted-session sweep reaps it, accounting the close once.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task WorkerFaultsBetweenAttachAndFirstPumpRead_SessionFaultsAndSweepReapsItOnce()
{
using GatewayMetrics metrics = new();
ReadyOnceWorkerClient client = new();
SessionManager manager = new(
new SessionRegistry(),
new SingleClientFactory(client),
Options.Create(new GatewayOptions()),
metrics,
dashboardEventBroadcaster: TestSupport.NullDashboardEventBroadcaster.Instance);
GatewaySession session = await OpenSessionAsync(manager);
await WaitUntilAsync(() => session.State == SessionState.Faulted);
Assert.Contains("worker", session.FinalFault, StringComparison.OrdinalIgnoreCase);
Assert.Equal(1, metrics.GetSnapshot().OpenSessions);
int closedCount = await manager.CloseExpiredLeasesAsync(DateTimeOffset.UtcNow, CancellationToken.None);
int secondPass = await manager.CloseExpiredLeasesAsync(DateTimeOffset.UtcNow, CancellationToken.None);
Assert.Equal(1, closedCount);
Assert.Equal(0, secondPass);
Assert.Equal(SessionState.Closed, session.State);
Assert.False(manager.TryGetSession(session.SessionId, out _));
GatewayMetricsSnapshot snapshot = metrics.GetSnapshot();
Assert.Equal(1, snapshot.SessionsClosed);
Assert.Equal(0, snapshot.OpenSessions);
}
private static async Task StartClientAsync(FakeWorkerHarness fakeWorker, WorkerClient client)
{
Task startTask = client.StartAsync(CancellationToken.None);
@@ -148,4 +220,56 @@ public sealed class GatewaySessionWorkerExitTests
return Task.FromResult(client);
}
}
/// <summary>
/// Worker client that reports <c>Ready</c> for its first state read (the open's attach
/// check) and <c>Faulted</c> from then on, reproducing a worker that dies in the window
/// between the attach and the event pump's first readiness check.
/// </summary>
private sealed class ReadyOnceWorkerClient : IWorkerClient
{
private int _stateReads;
/// <inheritdoc />
public string SessionId => "session-ready-once";
/// <inheritdoc />
public int? ProcessId => 4321;
/// <inheritdoc />
public WorkerClientState State
=> Interlocked.Increment(ref _stateReads) == 1 ? WorkerClientState.Ready : WorkerClientState.Faulted;
/// <inheritdoc />
public DateTimeOffset LastHeartbeatAt => DateTimeOffset.UtcNow;
/// <inheritdoc />
public Task StartAsync(CancellationToken cancellationToken) => Task.CompletedTask;
/// <inheritdoc />
public Task<WorkerCommandReply> InvokeAsync(WorkerCommand command, TimeSpan timeout, CancellationToken cancellationToken)
=> throw new InvalidOperationException("The worker is faulted.");
/// <inheritdoc />
public async IAsyncEnumerable<WorkerEvent> ReadEventsAsync(
[System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken)
{
await Task.CompletedTask;
throw new InvalidOperationException("The pump must not reach the event stream of a faulted worker.");
#pragma warning disable CS0162 // Unreachable code: the yield makes this an async iterator.
yield break;
#pragma warning restore CS0162
}
/// <inheritdoc />
public Task ShutdownAsync(TimeSpan timeout, CancellationToken cancellationToken) => Task.CompletedTask;
/// <inheritdoc />
public void Kill(string reason)
{
}
/// <inheritdoc />
public ValueTask DisposeAsync() => ValueTask.CompletedTask;
}
}
@@ -533,13 +533,17 @@ public sealed class SessionManagerTests
// which flaked under CI load) so a regression that waited out the timeout is caught
// while ordinary scheduling jitter never trips it.
const int readyWaitTimeoutMs = 5000;
FakeWorkerClient workerClient = new() { State = WorkerClientState.Faulted };
FakeWorkerClient workerClient = new();
SessionManager manager = CreateManager(
new FakeSessionWorkerClientFactory(workerClient),
options: CreateOptions(workerReadyWaitTimeoutMs: readyWaitTimeoutMs));
GatewaySession session = await manager.OpenSessionAsync(CreateOpenRequest(), "client-1", ownerKeyId: null, CancellationToken.None);
Assert.Equal(SessionState.Ready, session.State);
// The worker faults after the open attached it (an open refuses an already-faulted
// worker). No event pump runs here, so nothing has moved the session off Ready yet.
workerClient.State = WorkerClientState.Faulted;
Stopwatch stopwatch = Stopwatch.StartNew();
SessionManagerException exception = await Assert.ThrowsAsync<SessionManagerException>(
async () => await manager.InvokeAsync(
@@ -1145,7 +1149,9 @@ public sealed class SessionManagerTests
Assert.Equal(0, snapshot.SessionsClosed);
// The slot the killed open held must have been released exactly once: a replacement open
// succeeds against MaxSessions = 1 (the released gate hands the same fake straight back).
// succeeds against MaxSessions = 1 (the released gate hands the same fake straight back,
// so undo the discard's kill first — an open refuses an already-faulted worker).
workerClient.State = WorkerClientState.Ready;
GatewaySession reopened = await manager.OpenSessionAsync(CreateOpenRequest(), "client-2", ownerKeyId: null, CancellationToken.None);
Assert.Equal(SessionState.Ready, reopened.State);
Assert.Equal(1, registry.Count);
@@ -1155,11 +1161,13 @@ public sealed class SessionManagerTests
/// A caller that abandons its open (deadline, disconnect) while the worker is starting
/// must not leave the worker running: whatever fails between the worker arriving and the
/// session becoming Ready — here the caller's own cancellation — the late worker client is
/// killed and disposed before the failure surfaces.
/// killed and disposed before the failure surfaces. The cancellation is not an open
/// failure: it surfaces as the <see cref="OperationCanceledException"/> itself (which the
/// gRPC layer maps to <c>Cancelled</c>) and records no fault.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task OpenSessionAsync_WhenCallerCancelledDuringWorkerStartup_DiscardsWorkerClient()
public async Task OpenSessionAsync_WhenCallerCancelledDuringWorkerStartup_DiscardsWorkerClientAndRethrowsCancellation()
{
FakeWorkerClient workerClient = new();
GatedSessionWorkerClientFactory factory = new(workerClient);
@@ -1173,10 +1181,9 @@ public sealed class SessionManagerTests
await cancellation.CancelAsync();
factory.ReleaseCreate();
SessionManagerException exception = await Assert.ThrowsAsync<SessionManagerException>(async () => await openTask);
await Assert.ThrowsAnyAsync<OperationCanceledException>(async () => await openTask);
Assert.Equal(SessionManagerErrorCode.OpenFailed, exception.ErrorCode);
Assert.IsAssignableFrom<OperationCanceledException>(exception.InnerException);
Assert.Equal(0, metrics.GetSnapshot().Faults);
Assert.Equal(1, workerClient.KillCount);
Assert.Equal(1, workerClient.DisposeCount);
Assert.NotEqual(SessionState.Ready, starting.State);
@@ -1188,11 +1195,13 @@ public sealed class SessionManagerTests
/// <summary>
/// Same window as the kill case, driven by the host drain: a session still starting its
/// worker when <c>ShutdownAsync</c> runs is closed by the drain, and the worker client that
/// arrives afterwards is discarded rather than left running as an orphan.
/// arrives afterwards is discarded rather than left running as an orphan. The refused open
/// reports <c>GatewayStopping</c> (gRPC <c>Unavailable</c>, retry against the restarted
/// gateway), like an open refused at the door, and is not recorded as a fault.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task OpenSessionAsync_WhenShutdownDuringWorkerStartup_FailsOpenAndDiscardsLateWorkerClient()
public async Task OpenSessionAsync_WhenShutdownDuringWorkerStartup_ThrowsGatewayStoppingAndDiscardsLateWorkerClient()
{
FakeWorkerClient workerClient = new();
GatedSessionWorkerClientFactory factory = new(workerClient);
@@ -1207,7 +1216,8 @@ public sealed class SessionManagerTests
factory.ReleaseCreate();
SessionManagerException exception = await Assert.ThrowsAsync<SessionManagerException>(async () => await openTask);
Assert.Equal(SessionManagerErrorCode.OpenFailed, exception.ErrorCode);
Assert.Equal(SessionManagerErrorCode.GatewayStopping, exception.ErrorCode);
Assert.Equal(0, metrics.GetSnapshot().Faults);
Assert.Equal(SessionState.Closed, starting.State);
Assert.Equal(1, workerClient.KillCount);
Assert.Equal(1, workerClient.DisposeCount);