fix(worker): exit 5 on gateway death, dedicated exit 8 on event-queue overflow (mxaccessgw#186)

A frame read that hits end of stream means the gateway closed its end of
the pipe (usually because it died). WorkerApplication mapped every
WorkerFrameProtocolException to ProtocolViolation (6); EndOfStream now
exits as PipeConnectionFailed (5), alongside the IOException a broken
pipe raises.

Event-queue overflow exited as the generic UnexpectedFailure (1). The
event drain loop now unwinds with WorkerEventQueueFaultException carrying
the fault category, and QueueOverflow exits with the new
WorkerExitCode.EventQueueOverflow (8). Other categories keep exit 1.

The gateway does not decode worker exit codes (it only logs the number),
so no gateway change is needed. WorkerBootstrap.md and
MxAccessWorkerInstanceDesign.md updated. (mxaccessgw#186)

Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
Joseph Doherty
2026-11-09 12:51:41 -05:00
parent cf575bfca8
commit f49b7a0cc1
9 changed files with 206 additions and 10 deletions
@@ -3,6 +3,7 @@ using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using ZB.MOM.WW.MxGateway.Contracts;
using ZB.MOM.WW.MxGateway.Contracts.Proto;
using ZB.MOM.WW.MxGateway.Worker.Bootstrap;
using ZB.MOM.WW.MxGateway.Worker.Ipc;
@@ -103,6 +104,81 @@ public sealed class WorkerApplicationTests
Assert.Equal("WorkerPipeProtocolFailure", logger.Entries[1].EventName);
}
/// <summary>
/// A frame read that hits end of stream means the gateway closed the pipe (typically it
/// died). That is a lost connection, so it exits as <c>PipeConnectionFailed</c> (5), not
/// <c>ProtocolViolation</c> (6) (mxaccessgw#186).
/// </summary>
[Fact]
public void Run_WhenGatewayClosesPipe_ReturnsPipeConnectionFailed()
{
MemoryWorkerEnvironment environment = CreateEnvironment("nonce-secret");
MemoryWorkerLogger logger = new();
int exitCode = ZB.MOM.WW.MxGateway.Worker.WorkerApplication.Run(
ValidArgs(),
environment,
logger,
new ThrowingPipeClient(new WorkerFrameProtocolException(
WorkerFrameProtocolErrorCode.EndOfStream,
"Worker frame ended before the expected number of bytes were read.")));
Assert.Equal((int)WorkerExitCode.PipeConnectionFailed, exitCode);
Assert.Equal(5, exitCode);
MemoryWorkerLogEntry entry = logger.Entries[1];
Assert.Equal("WorkerPipeConnectionFailed", entry.EventName);
Assert.Equal(WorkerExitCode.PipeConnectionFailed, entry.Fields["exit_code"]);
Assert.Equal(WorkerFrameProtocolErrorCode.EndOfStream, entry.Fields["error_code"]);
}
/// <summary>
/// Event-queue overflow exits with its own code (8) instead of the generic
/// <c>UnexpectedFailure</c> (1), so an overflow is identifiable from the exit code alone
/// (mxaccessgw#186).
/// </summary>
[Fact]
public void Run_WhenEventQueueOverflows_ReturnsEventQueueOverflow()
{
MemoryWorkerEnvironment environment = CreateEnvironment("nonce-secret");
MemoryWorkerLogger logger = new();
int exitCode = ZB.MOM.WW.MxGateway.Worker.WorkerApplication.Run(
ValidArgs(),
environment,
logger,
new ThrowingPipeClient(new WorkerEventQueueFaultException(
WorkerFaultCategory.QueueOverflow,
"MXAccess outbound event queue reached capacity 4.")));
Assert.Equal((int)WorkerExitCode.EventQueueOverflow, exitCode);
Assert.Equal(8, exitCode);
MemoryWorkerLogEntry entry = logger.Entries[1];
Assert.Equal("Error", entry.Level);
Assert.Equal("WorkerEventQueueOverflow", entry.EventName);
Assert.Equal(WorkerExitCode.EventQueueOverflow, entry.Fields["exit_code"]);
}
/// <summary>
/// An event-queue fault of any other category keeps the generic exit code; only overflow
/// has a dedicated one.
/// </summary>
[Fact]
public void Run_WhenEventQueueFaultsForAnotherReason_ReturnsUnexpectedFailure()
{
MemoryWorkerEnvironment environment = CreateEnvironment("nonce-secret");
MemoryWorkerLogger logger = new();
int exitCode = ZB.MOM.WW.MxGateway.Worker.WorkerApplication.Run(
ValidArgs(),
environment,
logger,
new ThrowingPipeClient(new WorkerEventQueueFaultException(
WorkerFaultCategory.MxaccessEventConversionFailed,
"conversion failed")));
Assert.Equal((int)WorkerExitCode.UnexpectedFailure, exitCode);
}
/// <summary>Verifies that unexpected exceptions during bootstrap are logged.</summary>
[Fact]
public void Run_WithUnexpectedBootstrapFailure_ReturnsUnexpectedFailure()
@@ -1998,6 +1998,47 @@ public sealed class WorkerPipeSessionTests
await SendShutdownAndWaitAsync(pipePair, runTask, cancellation.Token);
}
/// <summary>
/// When the event queue records an overflow fault, the drain loop writes the fault frame and
/// unwinds with <see cref="WorkerEventQueueFaultException"/> carrying the category, which is
/// what lets the process boundary exit with the dedicated overflow code (mxaccessgw#186).
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task RunAsync_WhenEventQueueOverflows_WritesFaultAndThrowsTypedException()
{
using CancellationTokenSource cancellation = new(TimeSpan.FromSeconds(10));
using PipePair pipePair = await PipePair.CreateAsync(cancellation.Token);
FakeRuntimeSession runtime = new()
{
PendingFault = new WorkerFault
{
Category = WorkerFaultCategory.QueueOverflow,
DiagnosticMessage = "MXAccess outbound event queue reached capacity 4.",
ProtocolStatus = new ProtocolStatus
{
Code = ProtocolStatusCode.WorkerUnavailable,
Message = "MXAccess outbound event queue reached capacity 4.",
},
},
};
WorkerPipeSession session = CreatePipeSession(pipePair.WorkerStream, runtime);
Task runTask = session.RunAsync(cancellation.Token);
await CompleteGatewayHandshakeAsync(pipePair, cancellation.Token);
WorkerEnvelope faultEnvelope = await ReadUntilAsync(
pipePair.GatewayReader,
WorkerEnvelope.BodyOneofCase.WorkerFault,
cancellation.Token);
Assert.Equal(WorkerFaultCategory.QueueOverflow, faultEnvelope.WorkerFault.Category);
Task completedTask = await Task.WhenAny(runTask, Task.Delay(TimeSpan.FromSeconds(5), cancellation.Token));
Assert.Same(runTask, completedTask);
WorkerEventQueueFaultException exception =
await Assert.ThrowsAsync<WorkerEventQueueFaultException>(async () => await runTask);
Assert.Equal(WorkerFaultCategory.QueueOverflow, exception.Category);
}
private static WorkerPipeSession CreateSession(
Stream inbound,
Stream outbound,
@@ -380,10 +380,21 @@ internal sealed class FakeRuntimeSession : IWorkerRuntimeSession
return eventSignal.WaitAsync(timeout, cancellationToken);
}
/// <summary>
/// Gets or sets a fault the next <see cref="DrainFault"/> call hands back once, standing in
/// for the fault a real event queue self-records on overflow.
/// </summary>
public WorkerFault? PendingFault { get; set; }
/// <inheritdoc />
public WorkerFault? DrainFault()
{
return null;
lock (gate)
{
WorkerFault? fault = PendingFault;
PendingFault = null;
return fault;
}
}
/// <summary>
@@ -7,6 +7,11 @@ public enum WorkerExitCode
InvalidArguments = 2,
InvalidProtocolVersion = 3,
MissingNonce = 4,
/// <summary>
/// The pipe to the gateway could not be connected, or it closed under the worker — an
/// <c>IOException</c>, a connect <c>TimeoutException</c>, or a frame read that hit end of
/// stream because the gateway died or closed its end.
/// </summary>
PipeConnectionFailed = 5,
ProtocolViolation = 6,
@@ -18,4 +23,10 @@ public enum WorkerExitCode
/// exit code.
/// </summary>
UnhandledException = 7,
/// <summary>
/// The outbound MXAccess event queue reached capacity. The worker wrote a
/// <c>QueueOverflow</c> fault and exited rather than drop events (fail-fast backpressure).
/// </summary>
EventQueueOverflow = 8,
}
@@ -0,0 +1,25 @@
using System;
using ZB.MOM.WW.MxGateway.Contracts.Proto;
namespace ZB.MOM.WW.MxGateway.Worker.Ipc;
/// <summary>
/// Raised by the event drain loop after it has written the fault the MXAccess event queue
/// recorded, to unwind the pipe session. It carries the fault category so the process boundary
/// can pick an exit code: <see cref="WorkerFaultCategory.QueueOverflow"/> exits as
/// <c>WorkerExitCode.EventQueueOverflow</c> rather than the generic unexpected-failure code.
/// </summary>
public sealed class WorkerEventQueueFaultException : Exception
{
/// <summary>Initializes a new instance of <see cref="WorkerEventQueueFaultException"/>.</summary>
/// <param name="category">Category of the fault the event queue recorded.</param>
/// <param name="message">Diagnostic message.</param>
public WorkerEventQueueFaultException(WorkerFaultCategory category, string message)
: base(message)
{
Category = category;
}
/// <summary>Gets the category of the fault the event queue recorded.</summary>
public WorkerFaultCategory Category { get; }
}
@@ -592,7 +592,8 @@ public sealed class WorkerPipeSession
{
_state = WorkerState.Faulted;
await TryWriteFaultAsync(fault, cancellationToken).ConfigureAwait(false);
throw new InvalidOperationException(
throw new WorkerEventQueueFaultException(
fault.Category,
string.IsNullOrWhiteSpace(fault.DiagnosticMessage)
? $"MXAccess event queue faulted with category {fault.Category}."
: fault.DiagnosticMessage);
@@ -1,6 +1,7 @@
using System;
using System.Collections.Generic;
using System.IO;
using ZB.MOM.WW.MxGateway.Contracts.Proto;
using ZB.MOM.WW.MxGateway.Worker.Bootstrap;
using ZB.MOM.WW.MxGateway.Worker.Ipc;
@@ -158,6 +159,22 @@ public static class WorkerApplication
return (int)WorkerExitCode.Success;
}
catch (WorkerFrameProtocolException exception)
when (exception.ErrorCode == WorkerFrameProtocolErrorCode.EndOfStream)
{
// A read that hits end of stream means the gateway closed its end of the pipe — most
// often because the gateway process died. That is a lost connection, not a malformed
// frame, so it exits as PipeConnectionFailed alongside the IOException a broken pipe
// raises, rather than as ProtocolViolation.
logger.Error("WorkerPipeConnectionFailed", new Dictionary<string, object?>
{
["exit_code"] = WorkerExitCode.PipeConnectionFailed,
["error_code"] = exception.ErrorCode,
["exception_type"] = exception.GetType().FullName,
});
return (int)WorkerExitCode.PipeConnectionFailed;
}
catch (WorkerFrameProtocolException exception)
{
logger.Error("WorkerPipeProtocolFailure", new Dictionary<string, object?>
@@ -169,6 +186,17 @@ public static class WorkerApplication
return (int)WorkerExitCode.ProtocolViolation;
}
catch (WorkerEventQueueFaultException exception)
when (exception.Category == WorkerFaultCategory.QueueOverflow)
{
logger.Error("WorkerEventQueueOverflow", new Dictionary<string, object?>
{
["exit_code"] = WorkerExitCode.EventQueueOverflow,
["exception_type"] = exception.GetType().FullName,
});
return (int)WorkerExitCode.EventQueueOverflow;
}
catch (Exception exception) when (exception is IOException or TimeoutException)
{
logger.Error("WorkerPipeConnectionFailed", new Dictionary<string, object?>