fix(worker): reply WorkerUnavailable to commands that race shutdown (mxaccessgw#182)

TryStartCommandTask returned silently when the session had stopped
accepting commands, so a command racing shutdown got no reply and no log
and the gateway's correlation wait ran to its timeout. The message loop
now answers such a command with a WorkerUnavailable reply (the status the
STA dispatcher already uses during shutdown) and logs
WorkerCommandRejectedShuttingDown. New test
CommandAfterShutdown_RepliesWorkerUnavailable (mxaccessgw#182)

Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
Joseph Doherty
2026-11-09 12:47:06 -05:00
parent 0609e810f4
commit cf575bfca8
2 changed files with 121 additions and 4 deletions
@@ -748,7 +748,11 @@ public sealed class WorkerPipeSession
return await HandleControlCommandAsync(envelope, cancellationToken).ConfigureAwait(false);
}
TryStartCommandTask(envelope, cancellationToken);
if (!TryStartCommandTask(envelope, cancellationToken))
{
await WriteCommandRejectedAsync(envelope, cancellationToken).ConfigureAwait(false);
}
return true;
case WorkerEnvelope.BodyOneofCase.WorkerShutdown:
await ShutdownAsync(envelope.WorkerShutdown, cancellationToken).ConfigureAwait(false);
@@ -1230,7 +1234,14 @@ public sealed class WorkerPipeSession
}
}
private void TryStartCommandTask(
/// <summary>
/// Starts the STA command task for <paramref name="envelope"/>, or returns <c>false</c>
/// without starting one when the session has stopped accepting commands. A <c>false</c>
/// return obliges the caller to answer the correlation (see
/// <see cref="WriteCommandRejectedAsync"/>): dropping it silently would leave the gateway's
/// correlation wait to run out its full command timeout with no trace of why.
/// </summary>
private bool TryStartCommandTask(
WorkerEnvelope envelope,
CancellationToken cancellationToken)
{
@@ -1239,7 +1250,7 @@ public sealed class WorkerPipeSession
{
if (!_acceptingCommands)
{
return;
return false;
}
commandTask = ProcessCommandAsync(envelope, cancellationToken);
@@ -1254,6 +1265,52 @@ public sealed class WorkerPipeSession
// would fault the discarded task with no observer. Close the gap the same way the rest of
// the file does.
ObserveFaultWhenever(ObserveCommandTaskAsync(commandTask));
return true;
}
/// <summary>
/// Answers a command that arrived after the session stopped accepting commands with a
/// <see cref="ProtocolStatusCode.WorkerUnavailable"/> reply — the same status the STA
/// dispatcher gives a command that reaches it during shutdown — and logs the rejection.
/// A failed write is logged and swallowed: the session is already shutting down, and the
/// gateway's own command timeout still bounds the wait.
/// </summary>
private async Task WriteCommandRejectedAsync(
WorkerEnvelope envelope,
CancellationToken cancellationToken)
{
MxCommandKind kind = envelope.WorkerCommand?.Command?.Kind ?? MxCommandKind.Unspecified;
const string message = "The worker is shutting down and no longer accepts commands.";
_logger?.Information(
"WorkerCommandRejectedShuttingDown",
new Dictionary<string, object?>
{
["correlation_id"] = envelope.CorrelationId,
["command_kind"] = kind.ToString(),
["worker_state"] = _state.ToString(),
});
try
{
await WriteControlReplyAsync(
new MxCommandReply
{
SessionId = _options.SessionId,
CorrelationId = envelope.CorrelationId,
Kind = kind,
DiagnosticMessage = message,
ProtocolStatus = new ProtocolStatus
{
Code = ProtocolStatusCode.WorkerUnavailable,
Message = message,
},
},
cancellationToken).ConfigureAwait(false);
}
catch (Exception exception) when (exception is IOException or ObjectDisposedException or OperationCanceledException)
{
LogCommandResultDropped(envelope.CorrelationId, kind.ToString());
}
}
private async Task ObserveCommandTaskAsync(Task commandTask)
@@ -1274,7 +1331,12 @@ public sealed class WorkerPipeSession
}
}
private void StopAcceptingCommands()
/// <summary>
/// Closes the command gate so later STA commands are rejected with
/// <see cref="ProtocolStatusCode.WorkerUnavailable"/> instead of being started. Internal so
/// tests can open the shutdown race window the message loop otherwise closes immediately.
/// </summary>
internal void StopAcceptingCommands()
{
lock (_commandTaskGate)
{