fix(sessions): refuse a dead worker at attach, fault on pump readiness failure, keep cancel/stopping out of OpenFailed

- TryAttachWorkerClientAndMarkReadyAsync returns SessionAttachOutcome and refuses a
  worker client that is already Faulted/Closed; the open discards it and fails OpenFailed.
- MapWorkerEventsAsync calls ObserveWorkerTerminalState when its first readiness check
  fails, so a worker that dies between attach and the first pump read faults the session
  and the faulted sweep reaps it.
- A caller-cancelled open rolls back, records no fault and rethrows the cancellation
  (gRPC Cancelled); an attach refused because the drain closed the session reports
  GatewayStopping.
- docs/Sessions.md updated.
This commit is contained in:
Joseph Doherty
2026-10-22 08:41:02 -04:00
parent 2375df5df3
commit a178d68b60
6 changed files with 256 additions and 31 deletions
@@ -475,17 +475,27 @@ public sealed class GatewaySession
/// accounting here, and <see cref="OpenRecorded"/> is set right after it returns.
/// </param>
/// <returns>
/// <see langword="true"/> when the session became Ready; <see langword="false"/> when the
/// attach was refused (see <see cref="AttachWorkerClient"/>), including when the session was
/// disposed before the gate could be taken.
/// <see cref="SessionAttachOutcome.Attached"/> when the session became Ready;
/// <see cref="SessionAttachOutcome.SessionClosed"/> when the session refused the attach
/// (see <see cref="AttachWorkerClient"/>), including when it was disposed before the gate
/// could be taken; <see cref="SessionAttachOutcome.WorkerTerminal"/> when the worker client
/// had already reached <see cref="WorkerClientState.Faulted"/> or
/// <see cref="WorkerClientState.Closed"/>.
/// </returns>
/// <remarks>
/// Not cancellable: the worker client already exists, so the only acceptable outcomes are
/// "attached" or "refused, caller discards it". The gate is held only for the duration of
/// a close or kill, so the wait is bounded by the worker shutdown timeout. The caller
/// still decides what to do about a cancelled open before calling this.
/// <para>
/// A worker that dies right after its handshake (an STA loop fault, exit code 7) can
/// be terminal before it is attached. Attaching it would hand the caller a Ready
/// session over a dead worker that nothing notices until the lease expires, so it is
/// refused here. A worker that dies just after this check is caught by the event
/// pump's readiness failure path instead (see <c>MapWorkerEventsAsync</c>).
/// </para>
/// </remarks>
internal async ValueTask<bool> TryAttachWorkerClientAndMarkReadyAsync(
internal async ValueTask<SessionAttachOutcome> TryAttachWorkerClientAndMarkReadyAsync(
IWorkerClient workerClient,
Action onReady)
{
@@ -499,14 +509,27 @@ public sealed class GatewaySession
catch (ObjectDisposedException)
{
// Disposed while the worker was starting: the session has been torn down.
return false;
return SessionAttachOutcome.SessionClosed;
}
try
{
if (workerClient.State is WorkerClientState.Faulted or WorkerClientState.Closed)
{
// A session closed meanwhile is the stronger reason: it decides whether the open
// reports a stopping gateway rather than a failed worker.
lock (_syncRoot)
{
return _state is SessionState.Closing or SessionState.Closed or SessionState.Faulted
|| _eventDistributorDisposed
? SessionAttachOutcome.SessionClosed
: SessionAttachOutcome.WorkerTerminal;
}
}
if (!AttachWorkerClient(workerClient))
{
return false;
return SessionAttachOutcome.SessionClosed;
}
MarkReady();
@@ -516,7 +539,7 @@ public sealed class GatewaySession
_openRecorded = true;
}
return true;
return SessionAttachOutcome.Attached;
}
finally
{
@@ -1001,7 +1024,22 @@ public sealed class GatewaySession
[EnumeratorCancellation] CancellationToken cancellationToken)
{
MxAccessGrpcMapper mapper = _eventStreaming.Mapper;
IWorkerClient workerClient = await GetReadyWorkerClientAsync(cancellationToken).ConfigureAwait(false);
IWorkerClient workerClient;
try
{
workerClient = await GetReadyWorkerClientAsync(cancellationToken).ConfigureAwait(false);
}
catch (SessionManagerException)
{
// The worker can die between the open's attach and this first read (an STA loop
// fault or exit code 7 right after the handshake). The readiness check then fails
// here and the worker event stream below is never reached, so its finally never
// runs; without this the session would stay Ready over a dead worker until its
// lease expired.
ObserveWorkerTerminalState();
throw;
}
TouchClientActivity(_eventStreaming.TimeProvider.GetUtcNow());
try
@@ -0,0 +1,26 @@
namespace ZB.MOM.WW.MxGateway.Server.Sessions;
/// <summary>
/// Result of <see cref="GatewaySession.TryAttachWorkerClientAndMarkReadyAsync"/>: whether the
/// open's worker client was attached, and if not, which side refused it. The open maps each
/// refusal to a different failure, so the reason has to survive the call.
/// </summary>
internal enum SessionAttachOutcome
{
/// <summary>The worker client was attached and the session is <see cref="SessionState.Ready"/>.</summary>
Attached,
/// <summary>
/// The session was closed, faulted or disposed while its worker was starting (an admin
/// kill or the host drain); the caller discards the client.
/// </summary>
SessionClosed,
/// <summary>
/// The worker client was already <see cref="Workers.WorkerClientState.Faulted"/> or
/// <see cref="Workers.WorkerClientState.Closed"/> — the worker died right after its
/// handshake. Attaching it would leave a Ready session over a dead worker; the caller
/// discards the client and fails the open.
/// </summary>
WorkerTerminal,
}
@@ -164,11 +164,11 @@ public sealed class SessionManager : ISessionManager
// if a kill or the host drain closed it meanwhile — it must not be returned on a dead
// session; and a caller that abandoned the open while the worker was starting gets
// no session either.
bool ready;
SessionAttachOutcome outcome;
try
{
cancellationToken.ThrowIfCancellationRequested();
ready = await session
outcome = await session
.TryAttachWorkerClientAndMarkReadyAsync(workerClient, _metrics.SessionOpened)
.ConfigureAwait(false);
}
@@ -178,14 +178,25 @@ public sealed class SessionManager : ISessionManager
throw;
}
if (!ready)
if (outcome is SessionAttachOutcome.Attached)
{
await DiscardWorkerClientAsync(session, workerClient).ConfigureAwait(false);
throw new InvalidOperationException(
$"Session {session.SessionId} was closed while its worker was starting.");
return session;
}
return session;
WorkerClientState refusedWorkerState = workerClient.State;
await DiscardWorkerClientAsync(session, workerClient).ConfigureAwait(false);
if (outcome is SessionAttachOutcome.WorkerTerminal)
{
throw new InvalidOperationException(
$"The worker for session {session.SessionId} reached {refusedWorkerState} before it could be attached.");
}
// Refused because the session was closed while its worker was starting. When the host
// drain did that, the caller should retry against the restarted gateway, exactly like
// an open refused at the door, rather than see an open failure.
ThrowIfStopping();
throw new InvalidOperationException(
$"Session {session.SessionId} was closed while its worker was starting.");
}
catch (SessionManagerException exception) when (exception.ErrorCode == SessionManagerErrorCode.GatewayStopping)
{
@@ -194,6 +205,15 @@ public sealed class SessionManager : ISessionManager
await RollBackOpenAsync(session, registered, exception).ConfigureAwait(false);
throw;
}
catch (OperationCanceledException exception) when (cancellationToken.IsCancellationRequested)
{
// The caller abandoned the open (deadline, disconnect). Not an open failure either:
// roll back (a worker that arrived late was already discarded above), record no
// fault, log nothing, and let the cancellation surface so the gRPC layer maps it to
// Cancelled rather than Unavailable.
await RollBackOpenAsync(session, registered, exception).ConfigureAwait(false);
throw;
}
catch (Exception exception)
{
await RollBackOpenAsync(session, registered, exception).ConfigureAwait(false);