fix(worker): reply WorkerUnavailable to commands that race shutdown (mxaccessgw#182)
TryStartCommandTask returned silently when the session had stopped accepting commands, so a command racing shutdown got no reply and no log and the gateway's correlation wait ran to its timeout. The message loop now answers such a command with a WorkerUnavailable reply (the status the STA dispatcher already uses during shutdown) and logs WorkerCommandRejectedShuttingDown. New test CommandAfterShutdown_RepliesWorkerUnavailable (mxaccessgw#182) Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
@@ -1943,6 +1943,61 @@ public sealed class WorkerPipeSessionTests
|
|||||||
&& (string?)correlationId == "command-dropped-after-shutdown");
|
&& (string?)correlationId == "command-dropped-after-shutdown");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// A command that reaches the message loop after the session stopped accepting commands
|
||||||
|
/// must be answered with <see cref="ProtocolStatusCode.WorkerUnavailable"/> rather than
|
||||||
|
/// dropped, so the gateway's correlation wait ends at once instead of at its timeout
|
||||||
|
/// (mxaccessgw#182).
|
||||||
|
/// </summary>
|
||||||
|
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||||
|
[Fact]
|
||||||
|
public async Task CommandAfterShutdown_RepliesWorkerUnavailable()
|
||||||
|
{
|
||||||
|
using CancellationTokenSource cancellation = new(TimeSpan.FromSeconds(10));
|
||||||
|
using PipePair pipePair = await PipePair.CreateAsync(cancellation.Token);
|
||||||
|
FakeRuntimeSession runtime = new();
|
||||||
|
RecordingWorkerLogger logger = new();
|
||||||
|
WorkerPipeSession session = CreatePipeSession(
|
||||||
|
pipePair.WorkerStream,
|
||||||
|
runtime,
|
||||||
|
new WorkerPipeSessionOptions
|
||||||
|
{
|
||||||
|
HeartbeatInterval = TimeSpan.FromSeconds(1),
|
||||||
|
HeartbeatGrace = TimeSpan.FromSeconds(5),
|
||||||
|
},
|
||||||
|
logger);
|
||||||
|
Task runTask = session.RunAsync(cancellation.Token);
|
||||||
|
await CompleteGatewayHandshakeAsync(pipePair, cancellation.Token);
|
||||||
|
|
||||||
|
// Open the race window the message loop otherwise closes straight away: the command gate
|
||||||
|
// is shut, but the loop is still reading.
|
||||||
|
session.StopAcceptingCommands();
|
||||||
|
await pipePair.GatewayWriter.WriteAsync(
|
||||||
|
CreateCommandEnvelope("command-after-shutdown"),
|
||||||
|
cancellation.Token);
|
||||||
|
|
||||||
|
using CancellationTokenSource replyWait = CancellationTokenSource
|
||||||
|
.CreateLinkedTokenSource(cancellation.Token);
|
||||||
|
replyWait.CancelAfter(TimeSpan.FromSeconds(5));
|
||||||
|
WorkerEnvelope reply = await ReadUntilAsync(
|
||||||
|
pipePair.GatewayReader,
|
||||||
|
WorkerEnvelope.BodyOneofCase.WorkerCommandReply,
|
||||||
|
envelope => envelope.WorkerCommandReply.Reply.CorrelationId == "command-after-shutdown",
|
||||||
|
replyWait.Token);
|
||||||
|
|
||||||
|
Assert.Equal(ProtocolStatusCode.WorkerUnavailable, reply.WorkerCommandReply.Reply.ProtocolStatus.Code);
|
||||||
|
Assert.Equal(MxCommandKind.Register, reply.WorkerCommandReply.Reply.Kind);
|
||||||
|
Assert.Equal(SessionId, reply.WorkerCommandReply.Reply.SessionId);
|
||||||
|
Assert.False(runtime.DispatchStarted.IsSet);
|
||||||
|
Assert.Contains(
|
||||||
|
logger.Events,
|
||||||
|
entry => entry.EventName == "WorkerCommandRejectedShuttingDown"
|
||||||
|
&& entry.Fields.TryGetValue("correlation_id", out object? correlationId)
|
||||||
|
&& (string?)correlationId == "command-after-shutdown");
|
||||||
|
|
||||||
|
await SendShutdownAndWaitAsync(pipePair, runTask, cancellation.Token);
|
||||||
|
}
|
||||||
|
|
||||||
private static WorkerPipeSession CreateSession(
|
private static WorkerPipeSession CreateSession(
|
||||||
Stream inbound,
|
Stream inbound,
|
||||||
Stream outbound,
|
Stream outbound,
|
||||||
|
|||||||
@@ -748,7 +748,11 @@ public sealed class WorkerPipeSession
|
|||||||
return await HandleControlCommandAsync(envelope, cancellationToken).ConfigureAwait(false);
|
return await HandleControlCommandAsync(envelope, cancellationToken).ConfigureAwait(false);
|
||||||
}
|
}
|
||||||
|
|
||||||
TryStartCommandTask(envelope, cancellationToken);
|
if (!TryStartCommandTask(envelope, cancellationToken))
|
||||||
|
{
|
||||||
|
await WriteCommandRejectedAsync(envelope, cancellationToken).ConfigureAwait(false);
|
||||||
|
}
|
||||||
|
|
||||||
return true;
|
return true;
|
||||||
case WorkerEnvelope.BodyOneofCase.WorkerShutdown:
|
case WorkerEnvelope.BodyOneofCase.WorkerShutdown:
|
||||||
await ShutdownAsync(envelope.WorkerShutdown, cancellationToken).ConfigureAwait(false);
|
await ShutdownAsync(envelope.WorkerShutdown, cancellationToken).ConfigureAwait(false);
|
||||||
@@ -1230,7 +1234,14 @@ public sealed class WorkerPipeSession
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private void TryStartCommandTask(
|
/// <summary>
|
||||||
|
/// Starts the STA command task for <paramref name="envelope"/>, or returns <c>false</c>
|
||||||
|
/// without starting one when the session has stopped accepting commands. A <c>false</c>
|
||||||
|
/// return obliges the caller to answer the correlation (see
|
||||||
|
/// <see cref="WriteCommandRejectedAsync"/>): dropping it silently would leave the gateway's
|
||||||
|
/// correlation wait to run out its full command timeout with no trace of why.
|
||||||
|
/// </summary>
|
||||||
|
private bool TryStartCommandTask(
|
||||||
WorkerEnvelope envelope,
|
WorkerEnvelope envelope,
|
||||||
CancellationToken cancellationToken)
|
CancellationToken cancellationToken)
|
||||||
{
|
{
|
||||||
@@ -1239,7 +1250,7 @@ public sealed class WorkerPipeSession
|
|||||||
{
|
{
|
||||||
if (!_acceptingCommands)
|
if (!_acceptingCommands)
|
||||||
{
|
{
|
||||||
return;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
commandTask = ProcessCommandAsync(envelope, cancellationToken);
|
commandTask = ProcessCommandAsync(envelope, cancellationToken);
|
||||||
@@ -1254,6 +1265,52 @@ public sealed class WorkerPipeSession
|
|||||||
// would fault the discarded task with no observer. Close the gap the same way the rest of
|
// would fault the discarded task with no observer. Close the gap the same way the rest of
|
||||||
// the file does.
|
// the file does.
|
||||||
ObserveFaultWhenever(ObserveCommandTaskAsync(commandTask));
|
ObserveFaultWhenever(ObserveCommandTaskAsync(commandTask));
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Answers a command that arrived after the session stopped accepting commands with a
|
||||||
|
/// <see cref="ProtocolStatusCode.WorkerUnavailable"/> reply — the same status the STA
|
||||||
|
/// dispatcher gives a command that reaches it during shutdown — and logs the rejection.
|
||||||
|
/// A failed write is logged and swallowed: the session is already shutting down, and the
|
||||||
|
/// gateway's own command timeout still bounds the wait.
|
||||||
|
/// </summary>
|
||||||
|
private async Task WriteCommandRejectedAsync(
|
||||||
|
WorkerEnvelope envelope,
|
||||||
|
CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
MxCommandKind kind = envelope.WorkerCommand?.Command?.Kind ?? MxCommandKind.Unspecified;
|
||||||
|
const string message = "The worker is shutting down and no longer accepts commands.";
|
||||||
|
_logger?.Information(
|
||||||
|
"WorkerCommandRejectedShuttingDown",
|
||||||
|
new Dictionary<string, object?>
|
||||||
|
{
|
||||||
|
["correlation_id"] = envelope.CorrelationId,
|
||||||
|
["command_kind"] = kind.ToString(),
|
||||||
|
["worker_state"] = _state.ToString(),
|
||||||
|
});
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
await WriteControlReplyAsync(
|
||||||
|
new MxCommandReply
|
||||||
|
{
|
||||||
|
SessionId = _options.SessionId,
|
||||||
|
CorrelationId = envelope.CorrelationId,
|
||||||
|
Kind = kind,
|
||||||
|
DiagnosticMessage = message,
|
||||||
|
ProtocolStatus = new ProtocolStatus
|
||||||
|
{
|
||||||
|
Code = ProtocolStatusCode.WorkerUnavailable,
|
||||||
|
Message = message,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
cancellationToken).ConfigureAwait(false);
|
||||||
|
}
|
||||||
|
catch (Exception exception) when (exception is IOException or ObjectDisposedException or OperationCanceledException)
|
||||||
|
{
|
||||||
|
LogCommandResultDropped(envelope.CorrelationId, kind.ToString());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private async Task ObserveCommandTaskAsync(Task commandTask)
|
private async Task ObserveCommandTaskAsync(Task commandTask)
|
||||||
@@ -1274,7 +1331,12 @@ public sealed class WorkerPipeSession
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private void StopAcceptingCommands()
|
/// <summary>
|
||||||
|
/// Closes the command gate so later STA commands are rejected with
|
||||||
|
/// <see cref="ProtocolStatusCode.WorkerUnavailable"/> instead of being started. Internal so
|
||||||
|
/// tests can open the shutdown race window the message loop otherwise closes immediately.
|
||||||
|
/// </summary>
|
||||||
|
internal void StopAcceptingCommands()
|
||||||
{
|
{
|
||||||
lock (_commandTaskGate)
|
lock (_commandTaskGate)
|
||||||
{
|
{
|
||||||
|
|||||||
Reference in New Issue
Block a user