fix(worker): reply WorkerUnavailable to commands that race shutdown (mxaccessgw#182)
TryStartCommandTask returned silently when the session had stopped accepting commands, so a command racing shutdown got no reply and no log and the gateway's correlation wait ran to its timeout. The message loop now answers such a command with a WorkerUnavailable reply (the status the STA dispatcher already uses during shutdown) and logs WorkerCommandRejectedShuttingDown. New test CommandAfterShutdown_RepliesWorkerUnavailable (mxaccessgw#182) Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
@@ -1943,6 +1943,61 @@ public sealed class WorkerPipeSessionTests
|
||||
&& (string?)correlationId == "command-dropped-after-shutdown");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A command that reaches the message loop after the session stopped accepting commands
|
||||
/// must be answered with <see cref="ProtocolStatusCode.WorkerUnavailable"/> rather than
|
||||
/// dropped, so the gateway's correlation wait ends at once instead of at its timeout
|
||||
/// (mxaccessgw#182).
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task CommandAfterShutdown_RepliesWorkerUnavailable()
|
||||
{
|
||||
using CancellationTokenSource cancellation = new(TimeSpan.FromSeconds(10));
|
||||
using PipePair pipePair = await PipePair.CreateAsync(cancellation.Token);
|
||||
FakeRuntimeSession runtime = new();
|
||||
RecordingWorkerLogger logger = new();
|
||||
WorkerPipeSession session = CreatePipeSession(
|
||||
pipePair.WorkerStream,
|
||||
runtime,
|
||||
new WorkerPipeSessionOptions
|
||||
{
|
||||
HeartbeatInterval = TimeSpan.FromSeconds(1),
|
||||
HeartbeatGrace = TimeSpan.FromSeconds(5),
|
||||
},
|
||||
logger);
|
||||
Task runTask = session.RunAsync(cancellation.Token);
|
||||
await CompleteGatewayHandshakeAsync(pipePair, cancellation.Token);
|
||||
|
||||
// Open the race window the message loop otherwise closes straight away: the command gate
|
||||
// is shut, but the loop is still reading.
|
||||
session.StopAcceptingCommands();
|
||||
await pipePair.GatewayWriter.WriteAsync(
|
||||
CreateCommandEnvelope("command-after-shutdown"),
|
||||
cancellation.Token);
|
||||
|
||||
using CancellationTokenSource replyWait = CancellationTokenSource
|
||||
.CreateLinkedTokenSource(cancellation.Token);
|
||||
replyWait.CancelAfter(TimeSpan.FromSeconds(5));
|
||||
WorkerEnvelope reply = await ReadUntilAsync(
|
||||
pipePair.GatewayReader,
|
||||
WorkerEnvelope.BodyOneofCase.WorkerCommandReply,
|
||||
envelope => envelope.WorkerCommandReply.Reply.CorrelationId == "command-after-shutdown",
|
||||
replyWait.Token);
|
||||
|
||||
Assert.Equal(ProtocolStatusCode.WorkerUnavailable, reply.WorkerCommandReply.Reply.ProtocolStatus.Code);
|
||||
Assert.Equal(MxCommandKind.Register, reply.WorkerCommandReply.Reply.Kind);
|
||||
Assert.Equal(SessionId, reply.WorkerCommandReply.Reply.SessionId);
|
||||
Assert.False(runtime.DispatchStarted.IsSet);
|
||||
Assert.Contains(
|
||||
logger.Events,
|
||||
entry => entry.EventName == "WorkerCommandRejectedShuttingDown"
|
||||
&& entry.Fields.TryGetValue("correlation_id", out object? correlationId)
|
||||
&& (string?)correlationId == "command-after-shutdown");
|
||||
|
||||
await SendShutdownAndWaitAsync(pipePair, runTask, cancellation.Token);
|
||||
}
|
||||
|
||||
private static WorkerPipeSession CreateSession(
|
||||
Stream inbound,
|
||||
Stream outbound,
|
||||
|
||||
Reference in New Issue
Block a user