fix(sessions): refuse a dead worker at attach, fault on pump readiness failure, keep cancel/stopping out of OpenFailed

- TryAttachWorkerClientAndMarkReadyAsync returns SessionAttachOutcome and refuses a
  worker client that is already Faulted/Closed; the open discards it and fails OpenFailed.
- MapWorkerEventsAsync calls ObserveWorkerTerminalState when its first readiness check
  fails, so a worker that dies between attach and the first pump read faults the session
  and the faulted sweep reaps it.
- A caller-cancelled open rolls back, records no fault and rethrows the cancellation
  (gRPC Cancelled); an attach refused because the drain closed the session reports
  GatewayStopping.
- docs/Sessions.md updated.
This commit is contained in:
Joseph Doherty
2026-10-22 08:41:02 -04:00
parent 2375df5df3
commit a178d68b60
6 changed files with 256 additions and 31 deletions
@@ -164,11 +164,11 @@ public sealed class SessionManager : ISessionManager
// if a kill or the host drain closed it meanwhile — it must not be returned on a dead
// session; and a caller that abandoned the open while the worker was starting gets
// no session either.
bool ready;
SessionAttachOutcome outcome;
try
{
cancellationToken.ThrowIfCancellationRequested();
ready = await session
outcome = await session
.TryAttachWorkerClientAndMarkReadyAsync(workerClient, _metrics.SessionOpened)
.ConfigureAwait(false);
}
@@ -178,14 +178,25 @@ public sealed class SessionManager : ISessionManager
throw;
}
if (!ready)
if (outcome is SessionAttachOutcome.Attached)
{
await DiscardWorkerClientAsync(session, workerClient).ConfigureAwait(false);
throw new InvalidOperationException(
$"Session {session.SessionId} was closed while its worker was starting.");
return session;
}
return session;
WorkerClientState refusedWorkerState = workerClient.State;
await DiscardWorkerClientAsync(session, workerClient).ConfigureAwait(false);
if (outcome is SessionAttachOutcome.WorkerTerminal)
{
throw new InvalidOperationException(
$"The worker for session {session.SessionId} reached {refusedWorkerState} before it could be attached.");
}
// Refused because the session was closed while its worker was starting. When the host
// drain did that, the caller should retry against the restarted gateway, exactly like
// an open refused at the door, rather than see an open failure.
ThrowIfStopping();
throw new InvalidOperationException(
$"Session {session.SessionId} was closed while its worker was starting.");
}
catch (SessionManagerException exception) when (exception.ErrorCode == SessionManagerErrorCode.GatewayStopping)
{
@@ -194,6 +205,15 @@ public sealed class SessionManager : ISessionManager
await RollBackOpenAsync(session, registered, exception).ConfigureAwait(false);
throw;
}
catch (OperationCanceledException exception) when (cancellationToken.IsCancellationRequested)
{
// The caller abandoned the open (deadline, disconnect). Not an open failure either:
// roll back (a worker that arrived late was already discarded above), record no
// fault, log nothing, and let the cancellation surface so the gRPC layer maps it to
// Cancelled rather than Unavailable.
await RollBackOpenAsync(session, registered, exception).ConfigureAwait(false);
throw;
}
catch (Exception exception)
{
await RollBackOpenAsync(session, registered, exception).ConfigureAwait(false);