fix(worker): reply WorkerUnavailable to commands that race shutdown (mxaccessgw#182)
TryStartCommandTask returned silently when the session had stopped accepting commands, so a command racing shutdown got no reply and no log and the gateway's correlation wait ran to its timeout. The message loop now answers such a command with a WorkerUnavailable reply (the status the STA dispatcher already uses during shutdown) and logs WorkerCommandRejectedShuttingDown. New test CommandAfterShutdown_RepliesWorkerUnavailable (mxaccessgw#182) Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
@@ -748,7 +748,11 @@ public sealed class WorkerPipeSession
|
||||
return await HandleControlCommandAsync(envelope, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
TryStartCommandTask(envelope, cancellationToken);
|
||||
if (!TryStartCommandTask(envelope, cancellationToken))
|
||||
{
|
||||
await WriteCommandRejectedAsync(envelope, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
return true;
|
||||
case WorkerEnvelope.BodyOneofCase.WorkerShutdown:
|
||||
await ShutdownAsync(envelope.WorkerShutdown, cancellationToken).ConfigureAwait(false);
|
||||
@@ -1230,7 +1234,14 @@ public sealed class WorkerPipeSession
|
||||
}
|
||||
}
|
||||
|
||||
private void TryStartCommandTask(
|
||||
/// <summary>
|
||||
/// Starts the STA command task for <paramref name="envelope"/>, or returns <c>false</c>
|
||||
/// without starting one when the session has stopped accepting commands. A <c>false</c>
|
||||
/// return obliges the caller to answer the correlation (see
|
||||
/// <see cref="WriteCommandRejectedAsync"/>): dropping it silently would leave the gateway's
|
||||
/// correlation wait to run out its full command timeout with no trace of why.
|
||||
/// </summary>
|
||||
private bool TryStartCommandTask(
|
||||
WorkerEnvelope envelope,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
@@ -1239,7 +1250,7 @@ public sealed class WorkerPipeSession
|
||||
{
|
||||
if (!_acceptingCommands)
|
||||
{
|
||||
return;
|
||||
return false;
|
||||
}
|
||||
|
||||
commandTask = ProcessCommandAsync(envelope, cancellationToken);
|
||||
@@ -1254,6 +1265,52 @@ public sealed class WorkerPipeSession
|
||||
// would fault the discarded task with no observer. Close the gap the same way the rest of
|
||||
// the file does.
|
||||
ObserveFaultWhenever(ObserveCommandTaskAsync(commandTask));
|
||||
return true;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Answers a command that arrived after the session stopped accepting commands with a
|
||||
/// <see cref="ProtocolStatusCode.WorkerUnavailable"/> reply — the same status the STA
|
||||
/// dispatcher gives a command that reaches it during shutdown — and logs the rejection.
|
||||
/// A failed write is logged and swallowed: the session is already shutting down, and the
|
||||
/// gateway's own command timeout still bounds the wait.
|
||||
/// </summary>
|
||||
private async Task WriteCommandRejectedAsync(
|
||||
WorkerEnvelope envelope,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
MxCommandKind kind = envelope.WorkerCommand?.Command?.Kind ?? MxCommandKind.Unspecified;
|
||||
const string message = "The worker is shutting down and no longer accepts commands.";
|
||||
_logger?.Information(
|
||||
"WorkerCommandRejectedShuttingDown",
|
||||
new Dictionary<string, object?>
|
||||
{
|
||||
["correlation_id"] = envelope.CorrelationId,
|
||||
["command_kind"] = kind.ToString(),
|
||||
["worker_state"] = _state.ToString(),
|
||||
});
|
||||
|
||||
try
|
||||
{
|
||||
await WriteControlReplyAsync(
|
||||
new MxCommandReply
|
||||
{
|
||||
SessionId = _options.SessionId,
|
||||
CorrelationId = envelope.CorrelationId,
|
||||
Kind = kind,
|
||||
DiagnosticMessage = message,
|
||||
ProtocolStatus = new ProtocolStatus
|
||||
{
|
||||
Code = ProtocolStatusCode.WorkerUnavailable,
|
||||
Message = message,
|
||||
},
|
||||
},
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception exception) when (exception is IOException or ObjectDisposedException or OperationCanceledException)
|
||||
{
|
||||
LogCommandResultDropped(envelope.CorrelationId, kind.ToString());
|
||||
}
|
||||
}
|
||||
|
||||
private async Task ObserveCommandTaskAsync(Task commandTask)
|
||||
@@ -1274,7 +1331,12 @@ public sealed class WorkerPipeSession
|
||||
}
|
||||
}
|
||||
|
||||
private void StopAcceptingCommands()
|
||||
/// <summary>
|
||||
/// Closes the command gate so later STA commands are rejected with
|
||||
/// <see cref="ProtocolStatusCode.WorkerUnavailable"/> instead of being started. Internal so
|
||||
/// tests can open the shutdown race window the message loop otherwise closes immediately.
|
||||
/// </summary>
|
||||
internal void StopAcceptingCommands()
|
||||
{
|
||||
lock (_commandTaskGate)
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user