fix(sessions): refuse a dead worker at attach, fault on pump readiness failure, keep cancel/stopping out of OpenFailed
- TryAttachWorkerClientAndMarkReadyAsync returns SessionAttachOutcome and refuses a worker client that is already Faulted/Closed; the open discards it and fails OpenFailed. - MapWorkerEventsAsync calls ObserveWorkerTerminalState when its first readiness check fails, so a worker that dies between attach and the first pump read faults the session and the faulted sweep reaps it. - A caller-cancelled open rolls back, records no fault and rethrows the cancellation (gRPC Cancelled); an attach refused because the drain closed the session reports GatewayStopping. - docs/Sessions.md updated.
This commit is contained in:
@@ -475,17 +475,27 @@ public sealed class GatewaySession
|
||||
/// accounting here, and <see cref="OpenRecorded"/> is set right after it returns.
|
||||
/// </param>
|
||||
/// <returns>
|
||||
/// <see langword="true"/> when the session became Ready; <see langword="false"/> when the
|
||||
/// attach was refused (see <see cref="AttachWorkerClient"/>), including when the session was
|
||||
/// disposed before the gate could be taken.
|
||||
/// <see cref="SessionAttachOutcome.Attached"/> when the session became Ready;
|
||||
/// <see cref="SessionAttachOutcome.SessionClosed"/> when the session refused the attach
|
||||
/// (see <see cref="AttachWorkerClient"/>), including when it was disposed before the gate
|
||||
/// could be taken; <see cref="SessionAttachOutcome.WorkerTerminal"/> when the worker client
|
||||
/// had already reached <see cref="WorkerClientState.Faulted"/> or
|
||||
/// <see cref="WorkerClientState.Closed"/>.
|
||||
/// </returns>
|
||||
/// <remarks>
|
||||
/// Not cancellable: the worker client already exists, so the only acceptable outcomes are
|
||||
/// "attached" or "refused, caller discards it". The gate is held only for the duration of
|
||||
/// a close or kill, so the wait is bounded by the worker shutdown timeout. The caller
|
||||
/// still decides what to do about a cancelled open before calling this.
|
||||
/// <para>
|
||||
/// A worker that dies right after its handshake (an STA loop fault, exit code 7) can
|
||||
/// be terminal before it is attached. Attaching it would hand the caller a Ready
|
||||
/// session over a dead worker that nothing notices until the lease expires, so it is
|
||||
/// refused here. A worker that dies just after this check is caught by the event
|
||||
/// pump's readiness failure path instead (see <c>MapWorkerEventsAsync</c>).
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
internal async ValueTask<bool> TryAttachWorkerClientAndMarkReadyAsync(
|
||||
internal async ValueTask<SessionAttachOutcome> TryAttachWorkerClientAndMarkReadyAsync(
|
||||
IWorkerClient workerClient,
|
||||
Action onReady)
|
||||
{
|
||||
@@ -499,14 +509,27 @@ public sealed class GatewaySession
|
||||
catch (ObjectDisposedException)
|
||||
{
|
||||
// Disposed while the worker was starting: the session has been torn down.
|
||||
return false;
|
||||
return SessionAttachOutcome.SessionClosed;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
if (workerClient.State is WorkerClientState.Faulted or WorkerClientState.Closed)
|
||||
{
|
||||
// A session closed meanwhile is the stronger reason: it decides whether the open
|
||||
// reports a stopping gateway rather than a failed worker.
|
||||
lock (_syncRoot)
|
||||
{
|
||||
return _state is SessionState.Closing or SessionState.Closed or SessionState.Faulted
|
||||
|| _eventDistributorDisposed
|
||||
? SessionAttachOutcome.SessionClosed
|
||||
: SessionAttachOutcome.WorkerTerminal;
|
||||
}
|
||||
}
|
||||
|
||||
if (!AttachWorkerClient(workerClient))
|
||||
{
|
||||
return false;
|
||||
return SessionAttachOutcome.SessionClosed;
|
||||
}
|
||||
|
||||
MarkReady();
|
||||
@@ -516,7 +539,7 @@ public sealed class GatewaySession
|
||||
_openRecorded = true;
|
||||
}
|
||||
|
||||
return true;
|
||||
return SessionAttachOutcome.Attached;
|
||||
}
|
||||
finally
|
||||
{
|
||||
@@ -1001,7 +1024,22 @@ public sealed class GatewaySession
|
||||
[EnumeratorCancellation] CancellationToken cancellationToken)
|
||||
{
|
||||
MxAccessGrpcMapper mapper = _eventStreaming.Mapper;
|
||||
IWorkerClient workerClient = await GetReadyWorkerClientAsync(cancellationToken).ConfigureAwait(false);
|
||||
IWorkerClient workerClient;
|
||||
try
|
||||
{
|
||||
workerClient = await GetReadyWorkerClientAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (SessionManagerException)
|
||||
{
|
||||
// The worker can die between the open's attach and this first read (an STA loop
|
||||
// fault or exit code 7 right after the handshake). The readiness check then fails
|
||||
// here and the worker event stream below is never reached, so its finally never
|
||||
// runs; without this the session would stay Ready over a dead worker until its
|
||||
// lease expired.
|
||||
ObserveWorkerTerminalState();
|
||||
throw;
|
||||
}
|
||||
|
||||
TouchClientActivity(_eventStreaming.TimeProvider.GetUtcNow());
|
||||
|
||||
try
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
namespace ZB.MOM.WW.MxGateway.Server.Sessions;
|
||||
|
||||
/// <summary>
|
||||
/// Result of <see cref="GatewaySession.TryAttachWorkerClientAndMarkReadyAsync"/>: whether the
|
||||
/// open's worker client was attached, and if not, which side refused it. The open maps each
|
||||
/// refusal to a different failure, so the reason has to survive the call.
|
||||
/// </summary>
|
||||
internal enum SessionAttachOutcome
|
||||
{
|
||||
/// <summary>The worker client was attached and the session is <see cref="SessionState.Ready"/>.</summary>
|
||||
Attached,
|
||||
|
||||
/// <summary>
|
||||
/// The session was closed, faulted or disposed while its worker was starting (an admin
|
||||
/// kill or the host drain); the caller discards the client.
|
||||
/// </summary>
|
||||
SessionClosed,
|
||||
|
||||
/// <summary>
|
||||
/// The worker client was already <see cref="Workers.WorkerClientState.Faulted"/> or
|
||||
/// <see cref="Workers.WorkerClientState.Closed"/> — the worker died right after its
|
||||
/// handshake. Attaching it would leave a Ready session over a dead worker; the caller
|
||||
/// discards the client and fails the open.
|
||||
/// </summary>
|
||||
WorkerTerminal,
|
||||
}
|
||||
@@ -164,11 +164,11 @@ public sealed class SessionManager : ISessionManager
|
||||
// if a kill or the host drain closed it meanwhile — it must not be returned on a dead
|
||||
// session; and a caller that abandoned the open while the worker was starting gets
|
||||
// no session either.
|
||||
bool ready;
|
||||
SessionAttachOutcome outcome;
|
||||
try
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
ready = await session
|
||||
outcome = await session
|
||||
.TryAttachWorkerClientAndMarkReadyAsync(workerClient, _metrics.SessionOpened)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
@@ -178,14 +178,25 @@ public sealed class SessionManager : ISessionManager
|
||||
throw;
|
||||
}
|
||||
|
||||
if (!ready)
|
||||
if (outcome is SessionAttachOutcome.Attached)
|
||||
{
|
||||
await DiscardWorkerClientAsync(session, workerClient).ConfigureAwait(false);
|
||||
throw new InvalidOperationException(
|
||||
$"Session {session.SessionId} was closed while its worker was starting.");
|
||||
return session;
|
||||
}
|
||||
|
||||
return session;
|
||||
WorkerClientState refusedWorkerState = workerClient.State;
|
||||
await DiscardWorkerClientAsync(session, workerClient).ConfigureAwait(false);
|
||||
if (outcome is SessionAttachOutcome.WorkerTerminal)
|
||||
{
|
||||
throw new InvalidOperationException(
|
||||
$"The worker for session {session.SessionId} reached {refusedWorkerState} before it could be attached.");
|
||||
}
|
||||
|
||||
// Refused because the session was closed while its worker was starting. When the host
|
||||
// drain did that, the caller should retry against the restarted gateway, exactly like
|
||||
// an open refused at the door, rather than see an open failure.
|
||||
ThrowIfStopping();
|
||||
throw new InvalidOperationException(
|
||||
$"Session {session.SessionId} was closed while its worker was starting.");
|
||||
}
|
||||
catch (SessionManagerException exception) when (exception.ErrorCode == SessionManagerErrorCode.GatewayStopping)
|
||||
{
|
||||
@@ -194,6 +205,15 @@ public sealed class SessionManager : ISessionManager
|
||||
await RollBackOpenAsync(session, registered, exception).ConfigureAwait(false);
|
||||
throw;
|
||||
}
|
||||
catch (OperationCanceledException exception) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
// The caller abandoned the open (deadline, disconnect). Not an open failure either:
|
||||
// roll back (a worker that arrived late was already discarded above), record no
|
||||
// fault, log nothing, and let the cancellation surface so the gRPC layer maps it to
|
||||
// Cancelled rather than Unavailable.
|
||||
await RollBackOpenAsync(session, registered, exception).ConfigureAwait(false);
|
||||
throw;
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
await RollBackOpenAsync(session, registered, exception).ConfigureAwait(false);
|
||||
|
||||
Reference in New Issue
Block a user