From cf575bfca8e5799d2d533e0751f9a77894674ff5 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Mon, 9 Nov 2026 12:47:06 -0500 Subject: [PATCH] fix(worker): reply WorkerUnavailable to commands that race shutdown (mxaccessgw#182) TryStartCommandTask returned silently when the session had stopped accepting commands, so a command racing shutdown got no reply and no log and the gateway's correlation wait ran to its timeout. The message loop now answers such a command with a WorkerUnavailable reply (the status the STA dispatcher already uses during shutdown) and logs WorkerCommandRejectedShuttingDown. New test CommandAfterShutdown_RepliesWorkerUnavailable (mxaccessgw#182) Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV --- .../Ipc/WorkerPipeSessionTests.cs | 55 +++++++++++++++ .../Ipc/WorkerPipeSession.cs | 70 +++++++++++++++++-- 2 files changed, 121 insertions(+), 4 deletions(-) diff --git a/src/ZB.MOM.WW.MxGateway.Worker.Tests/Ipc/WorkerPipeSessionTests.cs b/src/ZB.MOM.WW.MxGateway.Worker.Tests/Ipc/WorkerPipeSessionTests.cs index 033b5c8f..e6fe8a1b 100644 --- a/src/ZB.MOM.WW.MxGateway.Worker.Tests/Ipc/WorkerPipeSessionTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Worker.Tests/Ipc/WorkerPipeSessionTests.cs @@ -1943,6 +1943,61 @@ public sealed class WorkerPipeSessionTests && (string?)correlationId == "command-dropped-after-shutdown"); } + /// + /// A command that reaches the message loop after the session stopped accepting commands + /// must be answered with rather than + /// dropped, so the gateway's correlation wait ends at once instead of at its timeout + /// (mxaccessgw#182). + /// + /// A task that represents the asynchronous operation. + [Fact] + public async Task CommandAfterShutdown_RepliesWorkerUnavailable() + { + using CancellationTokenSource cancellation = new(TimeSpan.FromSeconds(10)); + using PipePair pipePair = await PipePair.CreateAsync(cancellation.Token); + FakeRuntimeSession runtime = new(); + RecordingWorkerLogger logger = new(); + WorkerPipeSession session = CreatePipeSession( + pipePair.WorkerStream, + runtime, + new WorkerPipeSessionOptions + { + HeartbeatInterval = TimeSpan.FromSeconds(1), + HeartbeatGrace = TimeSpan.FromSeconds(5), + }, + logger); + Task runTask = session.RunAsync(cancellation.Token); + await CompleteGatewayHandshakeAsync(pipePair, cancellation.Token); + + // Open the race window the message loop otherwise closes straight away: the command gate + // is shut, but the loop is still reading. + session.StopAcceptingCommands(); + await pipePair.GatewayWriter.WriteAsync( + CreateCommandEnvelope("command-after-shutdown"), + cancellation.Token); + + using CancellationTokenSource replyWait = CancellationTokenSource + .CreateLinkedTokenSource(cancellation.Token); + replyWait.CancelAfter(TimeSpan.FromSeconds(5)); + WorkerEnvelope reply = await ReadUntilAsync( + pipePair.GatewayReader, + WorkerEnvelope.BodyOneofCase.WorkerCommandReply, + envelope => envelope.WorkerCommandReply.Reply.CorrelationId == "command-after-shutdown", + replyWait.Token); + + Assert.Equal(ProtocolStatusCode.WorkerUnavailable, reply.WorkerCommandReply.Reply.ProtocolStatus.Code); + Assert.Equal(MxCommandKind.Register, reply.WorkerCommandReply.Reply.Kind); + Assert.Equal(SessionId, reply.WorkerCommandReply.Reply.SessionId); + Assert.False(runtime.DispatchStarted.IsSet); + Assert.Contains( + logger.Events, + entry => entry.EventName == "WorkerCommandRejectedShuttingDown" + && entry.Fields.TryGetValue("correlation_id", out object? correlationId) + && (string?)correlationId == "command-after-shutdown"); + + await SendShutdownAndWaitAsync(pipePair, runTask, cancellation.Token); + } + private static WorkerPipeSession CreateSession( Stream inbound, Stream outbound, diff --git a/src/ZB.MOM.WW.MxGateway.Worker/Ipc/WorkerPipeSession.cs b/src/ZB.MOM.WW.MxGateway.Worker/Ipc/WorkerPipeSession.cs index f4f2de0b..59f3d7c1 100644 --- a/src/ZB.MOM.WW.MxGateway.Worker/Ipc/WorkerPipeSession.cs +++ b/src/ZB.MOM.WW.MxGateway.Worker/Ipc/WorkerPipeSession.cs @@ -748,7 +748,11 @@ public sealed class WorkerPipeSession return await HandleControlCommandAsync(envelope, cancellationToken).ConfigureAwait(false); } - TryStartCommandTask(envelope, cancellationToken); + if (!TryStartCommandTask(envelope, cancellationToken)) + { + await WriteCommandRejectedAsync(envelope, cancellationToken).ConfigureAwait(false); + } + return true; case WorkerEnvelope.BodyOneofCase.WorkerShutdown: await ShutdownAsync(envelope.WorkerShutdown, cancellationToken).ConfigureAwait(false); @@ -1230,7 +1234,14 @@ public sealed class WorkerPipeSession } } - private void TryStartCommandTask( + /// + /// Starts the STA command task for , or returns false + /// without starting one when the session has stopped accepting commands. A false + /// return obliges the caller to answer the correlation (see + /// ): dropping it silently would leave the gateway's + /// correlation wait to run out its full command timeout with no trace of why. + /// + private bool TryStartCommandTask( WorkerEnvelope envelope, CancellationToken cancellationToken) { @@ -1239,7 +1250,7 @@ public sealed class WorkerPipeSession { if (!_acceptingCommands) { - return; + return false; } commandTask = ProcessCommandAsync(envelope, cancellationToken); @@ -1254,6 +1265,52 @@ public sealed class WorkerPipeSession // would fault the discarded task with no observer. Close the gap the same way the rest of // the file does. ObserveFaultWhenever(ObserveCommandTaskAsync(commandTask)); + return true; + } + + /// + /// Answers a command that arrived after the session stopped accepting commands with a + /// reply — the same status the STA + /// dispatcher gives a command that reaches it during shutdown — and logs the rejection. + /// A failed write is logged and swallowed: the session is already shutting down, and the + /// gateway's own command timeout still bounds the wait. + /// + private async Task WriteCommandRejectedAsync( + WorkerEnvelope envelope, + CancellationToken cancellationToken) + { + MxCommandKind kind = envelope.WorkerCommand?.Command?.Kind ?? MxCommandKind.Unspecified; + const string message = "The worker is shutting down and no longer accepts commands."; + _logger?.Information( + "WorkerCommandRejectedShuttingDown", + new Dictionary + { + ["correlation_id"] = envelope.CorrelationId, + ["command_kind"] = kind.ToString(), + ["worker_state"] = _state.ToString(), + }); + + try + { + await WriteControlReplyAsync( + new MxCommandReply + { + SessionId = _options.SessionId, + CorrelationId = envelope.CorrelationId, + Kind = kind, + DiagnosticMessage = message, + ProtocolStatus = new ProtocolStatus + { + Code = ProtocolStatusCode.WorkerUnavailable, + Message = message, + }, + }, + cancellationToken).ConfigureAwait(false); + } + catch (Exception exception) when (exception is IOException or ObjectDisposedException or OperationCanceledException) + { + LogCommandResultDropped(envelope.CorrelationId, kind.ToString()); + } } private async Task ObserveCommandTaskAsync(Task commandTask) @@ -1274,7 +1331,12 @@ public sealed class WorkerPipeSession } } - private void StopAcceptingCommands() + /// + /// Closes the command gate so later STA commands are rejected with + /// instead of being started. Internal so + /// tests can open the shutdown race window the message loop otherwise closes immediately. + /// + internal void StopAcceptingCommands() { lock (_commandTaskGate) {