fix(worker): exit 5 on gateway death, dedicated exit 8 on event-queue overflow (mxaccessgw#186)

A frame read that hits end of stream means the gateway closed its end of
the pipe (usually because it died). WorkerApplication mapped every
WorkerFrameProtocolException to ProtocolViolation (6); EndOfStream now
exits as PipeConnectionFailed (5), alongside the IOException a broken
pipe raises.

Event-queue overflow exited as the generic UnexpectedFailure (1). The
event drain loop now unwinds with WorkerEventQueueFaultException carrying
the fault category, and QueueOverflow exits with the new
WorkerExitCode.EventQueueOverflow (8). Other categories keep exit 1.

The gateway does not decode worker exit codes (it only logs the number),
so no gateway change is needed. WorkerBootstrap.md and
MxAccessWorkerInstanceDesign.md updated. (mxaccessgw#186)

Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
Joseph Doherty
2026-11-09 12:51:41 -05:00
parent cf575bfca8
commit f49b7a0cc1
9 changed files with 206 additions and 10 deletions
@@ -7,6 +7,11 @@ public enum WorkerExitCode
InvalidArguments = 2,
InvalidProtocolVersion = 3,
MissingNonce = 4,
/// <summary>
/// The pipe to the gateway could not be connected, or it closed under the worker — an
/// <c>IOException</c>, a connect <c>TimeoutException</c>, or a frame read that hit end of
/// stream because the gateway died or closed its end.
/// </summary>
PipeConnectionFailed = 5,
ProtocolViolation = 6,
@@ -18,4 +23,10 @@ public enum WorkerExitCode
/// exit code.
/// </summary>
UnhandledException = 7,
/// <summary>
/// The outbound MXAccess event queue reached capacity. The worker wrote a
/// <c>QueueOverflow</c> fault and exited rather than drop events (fail-fast backpressure).
/// </summary>
EventQueueOverflow = 8,
}
@@ -0,0 +1,25 @@
using System;
using ZB.MOM.WW.MxGateway.Contracts.Proto;
namespace ZB.MOM.WW.MxGateway.Worker.Ipc;
/// <summary>
/// Raised by the event drain loop after it has written the fault the MXAccess event queue
/// recorded, to unwind the pipe session. It carries the fault category so the process boundary
/// can pick an exit code: <see cref="WorkerFaultCategory.QueueOverflow"/> exits as
/// <c>WorkerExitCode.EventQueueOverflow</c> rather than the generic unexpected-failure code.
/// </summary>
public sealed class WorkerEventQueueFaultException : Exception
{
/// <summary>Initializes a new instance of <see cref="WorkerEventQueueFaultException"/>.</summary>
/// <param name="category">Category of the fault the event queue recorded.</param>
/// <param name="message">Diagnostic message.</param>
public WorkerEventQueueFaultException(WorkerFaultCategory category, string message)
: base(message)
{
Category = category;
}
/// <summary>Gets the category of the fault the event queue recorded.</summary>
public WorkerFaultCategory Category { get; }
}
@@ -592,7 +592,8 @@ public sealed class WorkerPipeSession
{
_state = WorkerState.Faulted;
await TryWriteFaultAsync(fault, cancellationToken).ConfigureAwait(false);
throw new InvalidOperationException(
throw new WorkerEventQueueFaultException(
fault.Category,
string.IsNullOrWhiteSpace(fault.DiagnosticMessage)
? $"MXAccess event queue faulted with category {fault.Category}."
: fault.DiagnosticMessage);
@@ -1,6 +1,7 @@
using System;
using System.Collections.Generic;
using System.IO;
using ZB.MOM.WW.MxGateway.Contracts.Proto;
using ZB.MOM.WW.MxGateway.Worker.Bootstrap;
using ZB.MOM.WW.MxGateway.Worker.Ipc;
@@ -158,6 +159,22 @@ public static class WorkerApplication
return (int)WorkerExitCode.Success;
}
catch (WorkerFrameProtocolException exception)
when (exception.ErrorCode == WorkerFrameProtocolErrorCode.EndOfStream)
{
// A read that hits end of stream means the gateway closed its end of the pipe — most
// often because the gateway process died. That is a lost connection, not a malformed
// frame, so it exits as PipeConnectionFailed alongside the IOException a broken pipe
// raises, rather than as ProtocolViolation.
logger.Error("WorkerPipeConnectionFailed", new Dictionary<string, object?>
{
["exit_code"] = WorkerExitCode.PipeConnectionFailed,
["error_code"] = exception.ErrorCode,
["exception_type"] = exception.GetType().FullName,
});
return (int)WorkerExitCode.PipeConnectionFailed;
}
catch (WorkerFrameProtocolException exception)
{
logger.Error("WorkerPipeProtocolFailure", new Dictionary<string, object?>
@@ -169,6 +186,17 @@ public static class WorkerApplication
return (int)WorkerExitCode.ProtocolViolation;
}
catch (WorkerEventQueueFaultException exception)
when (exception.Category == WorkerFaultCategory.QueueOverflow)
{
logger.Error("WorkerEventQueueOverflow", new Dictionary<string, object?>
{
["exit_code"] = WorkerExitCode.EventQueueOverflow,
["exception_type"] = exception.GetType().FullName,
});
return (int)WorkerExitCode.EventQueueOverflow;
}
catch (Exception exception) when (exception is IOException or TimeoutException)
{
logger.Error("WorkerPipeConnectionFailed", new Dictionary<string, object?>