fix(worker): exit 5 on gateway death, dedicated exit 8 on event-queue overflow (mxaccessgw#186)
A frame read that hits end of stream means the gateway closed its end of the pipe (usually because it died). WorkerApplication mapped every WorkerFrameProtocolException to ProtocolViolation (6); EndOfStream now exits as PipeConnectionFailed (5), alongside the IOException a broken pipe raises. Event-queue overflow exited as the generic UnexpectedFailure (1). The event drain loop now unwinds with WorkerEventQueueFaultException carrying the fault category, and QueueOverflow exits with the new WorkerExitCode.EventQueueOverflow (8). Other categories keep exit 1. The gateway does not decode worker exit codes (it only logs the number), so no gateway change is needed. WorkerBootstrap.md and MxAccessWorkerInstanceDesign.md updated. (mxaccessgw#186) Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
@@ -7,6 +7,11 @@ public enum WorkerExitCode
|
||||
InvalidArguments = 2,
|
||||
InvalidProtocolVersion = 3,
|
||||
MissingNonce = 4,
|
||||
/// <summary>
|
||||
/// The pipe to the gateway could not be connected, or it closed under the worker — an
|
||||
/// <c>IOException</c>, a connect <c>TimeoutException</c>, or a frame read that hit end of
|
||||
/// stream because the gateway died or closed its end.
|
||||
/// </summary>
|
||||
PipeConnectionFailed = 5,
|
||||
ProtocolViolation = 6,
|
||||
|
||||
@@ -18,4 +23,10 @@ public enum WorkerExitCode
|
||||
/// exit code.
|
||||
/// </summary>
|
||||
UnhandledException = 7,
|
||||
|
||||
/// <summary>
|
||||
/// The outbound MXAccess event queue reached capacity. The worker wrote a
|
||||
/// <c>QueueOverflow</c> fault and exited rather than drop events (fail-fast backpressure).
|
||||
/// </summary>
|
||||
EventQueueOverflow = 8,
|
||||
}
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
using System;
|
||||
using ZB.MOM.WW.MxGateway.Contracts.Proto;
|
||||
|
||||
namespace ZB.MOM.WW.MxGateway.Worker.Ipc;
|
||||
|
||||
/// <summary>
|
||||
/// Raised by the event drain loop after it has written the fault the MXAccess event queue
|
||||
/// recorded, to unwind the pipe session. It carries the fault category so the process boundary
|
||||
/// can pick an exit code: <see cref="WorkerFaultCategory.QueueOverflow"/> exits as
|
||||
/// <c>WorkerExitCode.EventQueueOverflow</c> rather than the generic unexpected-failure code.
|
||||
/// </summary>
|
||||
public sealed class WorkerEventQueueFaultException : Exception
|
||||
{
|
||||
/// <summary>Initializes a new instance of <see cref="WorkerEventQueueFaultException"/>.</summary>
|
||||
/// <param name="category">Category of the fault the event queue recorded.</param>
|
||||
/// <param name="message">Diagnostic message.</param>
|
||||
public WorkerEventQueueFaultException(WorkerFaultCategory category, string message)
|
||||
: base(message)
|
||||
{
|
||||
Category = category;
|
||||
}
|
||||
|
||||
/// <summary>Gets the category of the fault the event queue recorded.</summary>
|
||||
public WorkerFaultCategory Category { get; }
|
||||
}
|
||||
@@ -592,7 +592,8 @@ public sealed class WorkerPipeSession
|
||||
{
|
||||
_state = WorkerState.Faulted;
|
||||
await TryWriteFaultAsync(fault, cancellationToken).ConfigureAwait(false);
|
||||
throw new InvalidOperationException(
|
||||
throw new WorkerEventQueueFaultException(
|
||||
fault.Category,
|
||||
string.IsNullOrWhiteSpace(fault.DiagnosticMessage)
|
||||
? $"MXAccess event queue faulted with category {fault.Category}."
|
||||
: fault.DiagnosticMessage);
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using ZB.MOM.WW.MxGateway.Contracts.Proto;
|
||||
using ZB.MOM.WW.MxGateway.Worker.Bootstrap;
|
||||
using ZB.MOM.WW.MxGateway.Worker.Ipc;
|
||||
|
||||
@@ -158,6 +159,22 @@ public static class WorkerApplication
|
||||
|
||||
return (int)WorkerExitCode.Success;
|
||||
}
|
||||
catch (WorkerFrameProtocolException exception)
|
||||
when (exception.ErrorCode == WorkerFrameProtocolErrorCode.EndOfStream)
|
||||
{
|
||||
// A read that hits end of stream means the gateway closed its end of the pipe — most
|
||||
// often because the gateway process died. That is a lost connection, not a malformed
|
||||
// frame, so it exits as PipeConnectionFailed alongside the IOException a broken pipe
|
||||
// raises, rather than as ProtocolViolation.
|
||||
logger.Error("WorkerPipeConnectionFailed", new Dictionary<string, object?>
|
||||
{
|
||||
["exit_code"] = WorkerExitCode.PipeConnectionFailed,
|
||||
["error_code"] = exception.ErrorCode,
|
||||
["exception_type"] = exception.GetType().FullName,
|
||||
});
|
||||
|
||||
return (int)WorkerExitCode.PipeConnectionFailed;
|
||||
}
|
||||
catch (WorkerFrameProtocolException exception)
|
||||
{
|
||||
logger.Error("WorkerPipeProtocolFailure", new Dictionary<string, object?>
|
||||
@@ -169,6 +186,17 @@ public static class WorkerApplication
|
||||
|
||||
return (int)WorkerExitCode.ProtocolViolation;
|
||||
}
|
||||
catch (WorkerEventQueueFaultException exception)
|
||||
when (exception.Category == WorkerFaultCategory.QueueOverflow)
|
||||
{
|
||||
logger.Error("WorkerEventQueueOverflow", new Dictionary<string, object?>
|
||||
{
|
||||
["exit_code"] = WorkerExitCode.EventQueueOverflow,
|
||||
["exception_type"] = exception.GetType().FullName,
|
||||
});
|
||||
|
||||
return (int)WorkerExitCode.EventQueueOverflow;
|
||||
}
|
||||
catch (Exception exception) when (exception is IOException or TimeoutException)
|
||||
{
|
||||
logger.Error("WorkerPipeConnectionFailed", new Dictionary<string, object?>
|
||||
|
||||
Reference in New Issue
Block a user