fix(worker): reply WorkerUnavailable to commands that race shutdown (mxaccessgw#182)

TryStartCommandTask returned silently when the session had stopped
accepting commands, so a command racing shutdown got no reply and no log
and the gateway's correlation wait ran to its timeout. The message loop
now answers such a command with a WorkerUnavailable reply (the status the
STA dispatcher already uses during shutdown) and logs
WorkerCommandRejectedShuttingDown. New test
CommandAfterShutdown_RepliesWorkerUnavailable (mxaccessgw#182)

Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
Joseph Doherty
2026-11-09 12:47:06 -05:00
parent 0609e810f4
commit cf575bfca8
2 changed files with 121 additions and 4 deletions
@@ -1943,6 +1943,61 @@ public sealed class WorkerPipeSessionTests
&& (string?)correlationId == "command-dropped-after-shutdown");
}
/// <summary>
/// A command that reaches the message loop after the session stopped accepting commands
/// must be answered with <see cref="ProtocolStatusCode.WorkerUnavailable"/> rather than
/// dropped, so the gateway's correlation wait ends at once instead of at its timeout
/// (mxaccessgw#182).
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task CommandAfterShutdown_RepliesWorkerUnavailable()
{
using CancellationTokenSource cancellation = new(TimeSpan.FromSeconds(10));
using PipePair pipePair = await PipePair.CreateAsync(cancellation.Token);
FakeRuntimeSession runtime = new();
RecordingWorkerLogger logger = new();
WorkerPipeSession session = CreatePipeSession(
pipePair.WorkerStream,
runtime,
new WorkerPipeSessionOptions
{
HeartbeatInterval = TimeSpan.FromSeconds(1),
HeartbeatGrace = TimeSpan.FromSeconds(5),
},
logger);
Task runTask = session.RunAsync(cancellation.Token);
await CompleteGatewayHandshakeAsync(pipePair, cancellation.Token);
// Open the race window the message loop otherwise closes straight away: the command gate
// is shut, but the loop is still reading.
session.StopAcceptingCommands();
await pipePair.GatewayWriter.WriteAsync(
CreateCommandEnvelope("command-after-shutdown"),
cancellation.Token);
using CancellationTokenSource replyWait = CancellationTokenSource
.CreateLinkedTokenSource(cancellation.Token);
replyWait.CancelAfter(TimeSpan.FromSeconds(5));
WorkerEnvelope reply = await ReadUntilAsync(
pipePair.GatewayReader,
WorkerEnvelope.BodyOneofCase.WorkerCommandReply,
envelope => envelope.WorkerCommandReply.Reply.CorrelationId == "command-after-shutdown",
replyWait.Token);
Assert.Equal(ProtocolStatusCode.WorkerUnavailable, reply.WorkerCommandReply.Reply.ProtocolStatus.Code);
Assert.Equal(MxCommandKind.Register, reply.WorkerCommandReply.Reply.Kind);
Assert.Equal(SessionId, reply.WorkerCommandReply.Reply.SessionId);
Assert.False(runtime.DispatchStarted.IsSet);
Assert.Contains(
logger.Events,
entry => entry.EventName == "WorkerCommandRejectedShuttingDown"
&& entry.Fields.TryGetValue("correlation_id", out object? correlationId)
&& (string?)correlationId == "command-after-shutdown");
await SendShutdownAndWaitAsync(pipePair, runTask, cancellation.Token);
}
private static WorkerPipeSession CreateSession(
Stream inbound,
Stream outbound,