fix(worker): reply WorkerUnavailable to commands that race shutdown (mxaccessgw#182)

TryStartCommandTask returned silently when the session had stopped
accepting commands, so a command racing shutdown got no reply and no log
and the gateway's correlation wait ran to its timeout. The message loop
now answers such a command with a WorkerUnavailable reply (the status the
STA dispatcher already uses during shutdown) and logs
WorkerCommandRejectedShuttingDown. New test
CommandAfterShutdown_RepliesWorkerUnavailable (mxaccessgw#182)

Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
Joseph Doherty
2026-11-09 12:47:06 -05:00
parent 0609e810f4
commit cf575bfca8
2 changed files with 121 additions and 4 deletions
@@ -1943,6 +1943,61 @@ public sealed class WorkerPipeSessionTests
&& (string?)correlationId == "command-dropped-after-shutdown"); && (string?)correlationId == "command-dropped-after-shutdown");
} }
/// <summary>
/// A command that reaches the message loop after the session stopped accepting commands
/// must be answered with <see cref="ProtocolStatusCode.WorkerUnavailable"/> rather than
/// dropped, so the gateway's correlation wait ends at once instead of at its timeout
/// (mxaccessgw#182).
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task CommandAfterShutdown_RepliesWorkerUnavailable()
{
using CancellationTokenSource cancellation = new(TimeSpan.FromSeconds(10));
using PipePair pipePair = await PipePair.CreateAsync(cancellation.Token);
FakeRuntimeSession runtime = new();
RecordingWorkerLogger logger = new();
WorkerPipeSession session = CreatePipeSession(
pipePair.WorkerStream,
runtime,
new WorkerPipeSessionOptions
{
HeartbeatInterval = TimeSpan.FromSeconds(1),
HeartbeatGrace = TimeSpan.FromSeconds(5),
},
logger);
Task runTask = session.RunAsync(cancellation.Token);
await CompleteGatewayHandshakeAsync(pipePair, cancellation.Token);
// Open the race window the message loop otherwise closes straight away: the command gate
// is shut, but the loop is still reading.
session.StopAcceptingCommands();
await pipePair.GatewayWriter.WriteAsync(
CreateCommandEnvelope("command-after-shutdown"),
cancellation.Token);
using CancellationTokenSource replyWait = CancellationTokenSource
.CreateLinkedTokenSource(cancellation.Token);
replyWait.CancelAfter(TimeSpan.FromSeconds(5));
WorkerEnvelope reply = await ReadUntilAsync(
pipePair.GatewayReader,
WorkerEnvelope.BodyOneofCase.WorkerCommandReply,
envelope => envelope.WorkerCommandReply.Reply.CorrelationId == "command-after-shutdown",
replyWait.Token);
Assert.Equal(ProtocolStatusCode.WorkerUnavailable, reply.WorkerCommandReply.Reply.ProtocolStatus.Code);
Assert.Equal(MxCommandKind.Register, reply.WorkerCommandReply.Reply.Kind);
Assert.Equal(SessionId, reply.WorkerCommandReply.Reply.SessionId);
Assert.False(runtime.DispatchStarted.IsSet);
Assert.Contains(
logger.Events,
entry => entry.EventName == "WorkerCommandRejectedShuttingDown"
&& entry.Fields.TryGetValue("correlation_id", out object? correlationId)
&& (string?)correlationId == "command-after-shutdown");
await SendShutdownAndWaitAsync(pipePair, runTask, cancellation.Token);
}
private static WorkerPipeSession CreateSession( private static WorkerPipeSession CreateSession(
Stream inbound, Stream inbound,
Stream outbound, Stream outbound,
@@ -748,7 +748,11 @@ public sealed class WorkerPipeSession
return await HandleControlCommandAsync(envelope, cancellationToken).ConfigureAwait(false); return await HandleControlCommandAsync(envelope, cancellationToken).ConfigureAwait(false);
} }
TryStartCommandTask(envelope, cancellationToken); if (!TryStartCommandTask(envelope, cancellationToken))
{
await WriteCommandRejectedAsync(envelope, cancellationToken).ConfigureAwait(false);
}
return true; return true;
case WorkerEnvelope.BodyOneofCase.WorkerShutdown: case WorkerEnvelope.BodyOneofCase.WorkerShutdown:
await ShutdownAsync(envelope.WorkerShutdown, cancellationToken).ConfigureAwait(false); await ShutdownAsync(envelope.WorkerShutdown, cancellationToken).ConfigureAwait(false);
@@ -1230,7 +1234,14 @@ public sealed class WorkerPipeSession
} }
} }
private void TryStartCommandTask( /// <summary>
/// Starts the STA command task for <paramref name="envelope"/>, or returns <c>false</c>
/// without starting one when the session has stopped accepting commands. A <c>false</c>
/// return obliges the caller to answer the correlation (see
/// <see cref="WriteCommandRejectedAsync"/>): dropping it silently would leave the gateway's
/// correlation wait to run out its full command timeout with no trace of why.
/// </summary>
private bool TryStartCommandTask(
WorkerEnvelope envelope, WorkerEnvelope envelope,
CancellationToken cancellationToken) CancellationToken cancellationToken)
{ {
@@ -1239,7 +1250,7 @@ public sealed class WorkerPipeSession
{ {
if (!_acceptingCommands) if (!_acceptingCommands)
{ {
return; return false;
} }
commandTask = ProcessCommandAsync(envelope, cancellationToken); commandTask = ProcessCommandAsync(envelope, cancellationToken);
@@ -1254,6 +1265,52 @@ public sealed class WorkerPipeSession
// would fault the discarded task with no observer. Close the gap the same way the rest of // would fault the discarded task with no observer. Close the gap the same way the rest of
// the file does. // the file does.
ObserveFaultWhenever(ObserveCommandTaskAsync(commandTask)); ObserveFaultWhenever(ObserveCommandTaskAsync(commandTask));
return true;
}
/// <summary>
/// Answers a command that arrived after the session stopped accepting commands with a
/// <see cref="ProtocolStatusCode.WorkerUnavailable"/> reply — the same status the STA
/// dispatcher gives a command that reaches it during shutdown — and logs the rejection.
/// A failed write is logged and swallowed: the session is already shutting down, and the
/// gateway's own command timeout still bounds the wait.
/// </summary>
private async Task WriteCommandRejectedAsync(
WorkerEnvelope envelope,
CancellationToken cancellationToken)
{
MxCommandKind kind = envelope.WorkerCommand?.Command?.Kind ?? MxCommandKind.Unspecified;
const string message = "The worker is shutting down and no longer accepts commands.";
_logger?.Information(
"WorkerCommandRejectedShuttingDown",
new Dictionary<string, object?>
{
["correlation_id"] = envelope.CorrelationId,
["command_kind"] = kind.ToString(),
["worker_state"] = _state.ToString(),
});
try
{
await WriteControlReplyAsync(
new MxCommandReply
{
SessionId = _options.SessionId,
CorrelationId = envelope.CorrelationId,
Kind = kind,
DiagnosticMessage = message,
ProtocolStatus = new ProtocolStatus
{
Code = ProtocolStatusCode.WorkerUnavailable,
Message = message,
},
},
cancellationToken).ConfigureAwait(false);
}
catch (Exception exception) when (exception is IOException or ObjectDisposedException or OperationCanceledException)
{
LogCommandResultDropped(envelope.CorrelationId, kind.ToString());
}
} }
private async Task ObserveCommandTaskAsync(Task commandTask) private async Task ObserveCommandTaskAsync(Task commandTask)
@@ -1274,7 +1331,12 @@ public sealed class WorkerPipeSession
} }
} }
private void StopAcceptingCommands() /// <summary>
/// Closes the command gate so later STA commands are rejected with
/// <see cref="ProtocolStatusCode.WorkerUnavailable"/> instead of being started. Internal so
/// tests can open the shutdown race window the message loop otherwise closes immediately.
/// </summary>
internal void StopAcceptingCommands()
{ {
lock (_commandTaskGate) lock (_commandTaskGate)
{ {