diff --git a/docs/MxAccessWorkerInstanceDesign.md b/docs/MxAccessWorkerInstanceDesign.md
index e3785fb5..307b84b9 100644
--- a/docs/MxAccessWorkerInstanceDesign.md
+++ b/docs/MxAccessWorkerInstanceDesign.md
@@ -133,9 +133,10 @@ starts the STA, or touches MXAccess:
| `2` | `InvalidArguments` | Required arguments are missing or unknown arguments are present. |
| `3` | `InvalidProtocolVersion` | `--protocol-version` is not numeric or does not match the supported worker protocol. |
| `4` | `MissingNonce` | `MXGATEWAY_WORKER_NONCE` is absent or empty. |
-| `5` | `PipeConnectionFailed` | An `IOException` or `TimeoutException` escapes the pipe client. |
-| `6` | `ProtocolViolation` | A `WorkerFrameProtocolException` escapes the pipe client. |
+| `5` | `PipeConnectionFailed` | An `IOException` or `TimeoutException` escapes the pipe client, or a frame read hits end of stream because the gateway closed the pipe (gateway death). |
+| `6` | `ProtocolViolation` | Any other `WorkerFrameProtocolException` escapes the pipe client. |
| `7` | `UnhandledException` | An exception reaches the `AppDomain.UnhandledException` hook (see `docs/WorkerBootstrap.md`). |
+| `8` | `EventQueueOverflow` | The outbound MXAccess event queue reached capacity (see the event overflow policy below). |
Bootstrap logs use `WorkerConsoleLogger` key/value output. `WorkerLogRedactor`
redacts fields whose names indicate nonce, secret, password, token,
@@ -868,9 +869,10 @@ category `QueueOverflow`. The event drain loop's next pass observes the fault
through `DrainFault()`, writes the fault frame — a Control-priority frame, so
it is not stuck behind a queued event backlog — and then throws to unwind
`RunAsync`: the worker process exits rather than continuing in a state where
-events are being silently lost. The exit currently uses the generic
-`WorkerExitCode.UnexpectedFailure` rather than a dedicated overflow code; a
-distinct exit code remains open (tracked separately). Do not drop or coalesce
+events are being silently lost. The drain loop unwinds with a
+`WorkerEventQueueFaultException` carrying the fault category, and the process
+boundary maps `QueueOverflow` to the dedicated `WorkerExitCode.EventQueueOverflow`
+(8), so an overflow is identifiable from the worker's exit code alone. Do not drop or coalesce
events to avoid this exit — that is explicitly out of scope for v1. Production
coalescing may be added later, but it must be explicit and tested.
diff --git a/docs/WorkerBootstrap.md b/docs/WorkerBootstrap.md
index 73aa9bda..58557019 100644
--- a/docs/WorkerBootstrap.md
+++ b/docs/WorkerBootstrap.md
@@ -115,9 +115,10 @@ public static WorkerBootstrapResult Failure(WorkerExitCode exitCode, IEnumerable
| `InvalidArguments` | 2 | One or more `--session-id`, `--pipe-name`, or `--protocol-version` errors (missing, empty, unknown flag, or no value). |
| `InvalidProtocolVersion` | 3 | `--protocol-version` is not a `uint` or does not equal `GatewayContractInfo.WorkerProtocolVersion`. |
| `MissingNonce` | 4 | The `MXGATEWAY_WORKER_NONCE` environment variable is null, empty, or whitespace. |
-| `PipeConnectionFailed` | 5 | An `IOException` or `TimeoutException` escapes the pipe client. |
-| `ProtocolViolation` | 6 | A `WorkerFrameProtocolException` escapes the pipe client. |
+| `PipeConnectionFailed` | 5 | An `IOException` or `TimeoutException` escapes the pipe client, or a `WorkerFrameProtocolException` with `EndOfStream` does — the gateway closed its end of the pipe, usually because it died. |
+| `ProtocolViolation` | 6 | Any other `WorkerFrameProtocolException` escapes the pipe client. |
| `UnhandledException` | 7 | An exception reaches `AppDomain.CurrentDomain.UnhandledException` — thrown on a thread outside `WorkerApplication.Run`'s `try/catch` (a background task continuation, a timer, a COM callback thread). |
+| `EventQueueOverflow` | 8 | The outbound MXAccess event queue reached capacity: the event drain loop wrote the `QueueOverflow` fault and unwound with a `WorkerEventQueueFaultException`. Other event-queue fault categories still exit as `UnexpectedFailure`. |
`InvalidArguments`, `InvalidProtocolVersion`, and `MissingNonce` originate in the parser; `UnhandledException` originates in the process-wide hook described below; the others originate in `WorkerApplication.Run`'s `try/catch` around the pipe client.
@@ -219,7 +220,7 @@ if (!result.Succeeded)
}
```
-On success it logs `WorkerBootstrapSucceeded` with the session fields (the `nonce` field is redacted by `WorkerLogRedactor` because of its name), hands the `WorkerOptions` to `IWorkerPipeClient.RunAsync`, and waits synchronously. The `try/catch` around the pipe call maps `WorkerFrameProtocolException` to `ProtocolViolation`, `IOException`/`TimeoutException` to `PipeConnectionFailed`, and any other exception to `UnexpectedFailure`, so every code path through `Run` returns one of the values in `WorkerExitCode`.
+On success it logs `WorkerBootstrapSucceeded` with the session fields (the `nonce` field is redacted by `WorkerLogRedactor` because of its name), hands the `WorkerOptions` to `IWorkerPipeClient.RunAsync`, and waits synchronously. The `try/catch` around the pipe call maps a `WorkerFrameProtocolException` with `EndOfStream` to `PipeConnectionFailed` (a closed pipe is a lost connection, not a malformed frame), any other `WorkerFrameProtocolException` to `ProtocolViolation`, a `WorkerEventQueueFaultException` with category `QueueOverflow` to `EventQueueOverflow`, `IOException`/`TimeoutException` to `PipeConnectionFailed`, and any other exception to `UnexpectedFailure`, so every code path through `Run` returns one of the values in `WorkerExitCode`.
## Related Documentation
diff --git a/src/ZB.MOM.WW.MxGateway.Worker.Tests/Bootstrap/WorkerApplicationTests.cs b/src/ZB.MOM.WW.MxGateway.Worker.Tests/Bootstrap/WorkerApplicationTests.cs
index f29885ee..3d915791 100644
--- a/src/ZB.MOM.WW.MxGateway.Worker.Tests/Bootstrap/WorkerApplicationTests.cs
+++ b/src/ZB.MOM.WW.MxGateway.Worker.Tests/Bootstrap/WorkerApplicationTests.cs
@@ -3,6 +3,7 @@ using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using ZB.MOM.WW.MxGateway.Contracts;
+using ZB.MOM.WW.MxGateway.Contracts.Proto;
using ZB.MOM.WW.MxGateway.Worker.Bootstrap;
using ZB.MOM.WW.MxGateway.Worker.Ipc;
@@ -103,6 +104,81 @@ public sealed class WorkerApplicationTests
Assert.Equal("WorkerPipeProtocolFailure", logger.Entries[1].EventName);
}
+ ///
+ /// A frame read that hits end of stream means the gateway closed the pipe (typically it
+ /// died). That is a lost connection, so it exits as PipeConnectionFailed (5), not
+ /// ProtocolViolation (6) (mxaccessgw#186).
+ ///
+ [Fact]
+ public void Run_WhenGatewayClosesPipe_ReturnsPipeConnectionFailed()
+ {
+ MemoryWorkerEnvironment environment = CreateEnvironment("nonce-secret");
+ MemoryWorkerLogger logger = new();
+
+ int exitCode = ZB.MOM.WW.MxGateway.Worker.WorkerApplication.Run(
+ ValidArgs(),
+ environment,
+ logger,
+ new ThrowingPipeClient(new WorkerFrameProtocolException(
+ WorkerFrameProtocolErrorCode.EndOfStream,
+ "Worker frame ended before the expected number of bytes were read.")));
+
+ Assert.Equal((int)WorkerExitCode.PipeConnectionFailed, exitCode);
+ Assert.Equal(5, exitCode);
+ MemoryWorkerLogEntry entry = logger.Entries[1];
+ Assert.Equal("WorkerPipeConnectionFailed", entry.EventName);
+ Assert.Equal(WorkerExitCode.PipeConnectionFailed, entry.Fields["exit_code"]);
+ Assert.Equal(WorkerFrameProtocolErrorCode.EndOfStream, entry.Fields["error_code"]);
+ }
+
+ ///
+ /// Event-queue overflow exits with its own code (8) instead of the generic
+ /// UnexpectedFailure (1), so an overflow is identifiable from the exit code alone
+ /// (mxaccessgw#186).
+ ///
+ [Fact]
+ public void Run_WhenEventQueueOverflows_ReturnsEventQueueOverflow()
+ {
+ MemoryWorkerEnvironment environment = CreateEnvironment("nonce-secret");
+ MemoryWorkerLogger logger = new();
+
+ int exitCode = ZB.MOM.WW.MxGateway.Worker.WorkerApplication.Run(
+ ValidArgs(),
+ environment,
+ logger,
+ new ThrowingPipeClient(new WorkerEventQueueFaultException(
+ WorkerFaultCategory.QueueOverflow,
+ "MXAccess outbound event queue reached capacity 4.")));
+
+ Assert.Equal((int)WorkerExitCode.EventQueueOverflow, exitCode);
+ Assert.Equal(8, exitCode);
+ MemoryWorkerLogEntry entry = logger.Entries[1];
+ Assert.Equal("Error", entry.Level);
+ Assert.Equal("WorkerEventQueueOverflow", entry.EventName);
+ Assert.Equal(WorkerExitCode.EventQueueOverflow, entry.Fields["exit_code"]);
+ }
+
+ ///
+ /// An event-queue fault of any other category keeps the generic exit code; only overflow
+ /// has a dedicated one.
+ ///
+ [Fact]
+ public void Run_WhenEventQueueFaultsForAnotherReason_ReturnsUnexpectedFailure()
+ {
+ MemoryWorkerEnvironment environment = CreateEnvironment("nonce-secret");
+ MemoryWorkerLogger logger = new();
+
+ int exitCode = ZB.MOM.WW.MxGateway.Worker.WorkerApplication.Run(
+ ValidArgs(),
+ environment,
+ logger,
+ new ThrowingPipeClient(new WorkerEventQueueFaultException(
+ WorkerFaultCategory.MxaccessEventConversionFailed,
+ "conversion failed")));
+
+ Assert.Equal((int)WorkerExitCode.UnexpectedFailure, exitCode);
+ }
+
/// Verifies that unexpected exceptions during bootstrap are logged.
[Fact]
public void Run_WithUnexpectedBootstrapFailure_ReturnsUnexpectedFailure()
diff --git a/src/ZB.MOM.WW.MxGateway.Worker.Tests/Ipc/WorkerPipeSessionTests.cs b/src/ZB.MOM.WW.MxGateway.Worker.Tests/Ipc/WorkerPipeSessionTests.cs
index e6fe8a1b..780d313f 100644
--- a/src/ZB.MOM.WW.MxGateway.Worker.Tests/Ipc/WorkerPipeSessionTests.cs
+++ b/src/ZB.MOM.WW.MxGateway.Worker.Tests/Ipc/WorkerPipeSessionTests.cs
@@ -1998,6 +1998,47 @@ public sealed class WorkerPipeSessionTests
await SendShutdownAndWaitAsync(pipePair, runTask, cancellation.Token);
}
+ ///
+ /// When the event queue records an overflow fault, the drain loop writes the fault frame and
+ /// unwinds with carrying the category, which is
+ /// what lets the process boundary exit with the dedicated overflow code (mxaccessgw#186).
+ ///
+ /// A task that represents the asynchronous operation.
+ [Fact]
+ public async Task RunAsync_WhenEventQueueOverflows_WritesFaultAndThrowsTypedException()
+ {
+ using CancellationTokenSource cancellation = new(TimeSpan.FromSeconds(10));
+ using PipePair pipePair = await PipePair.CreateAsync(cancellation.Token);
+ FakeRuntimeSession runtime = new()
+ {
+ PendingFault = new WorkerFault
+ {
+ Category = WorkerFaultCategory.QueueOverflow,
+ DiagnosticMessage = "MXAccess outbound event queue reached capacity 4.",
+ ProtocolStatus = new ProtocolStatus
+ {
+ Code = ProtocolStatusCode.WorkerUnavailable,
+ Message = "MXAccess outbound event queue reached capacity 4.",
+ },
+ },
+ };
+ WorkerPipeSession session = CreatePipeSession(pipePair.WorkerStream, runtime);
+ Task runTask = session.RunAsync(cancellation.Token);
+ await CompleteGatewayHandshakeAsync(pipePair, cancellation.Token);
+
+ WorkerEnvelope faultEnvelope = await ReadUntilAsync(
+ pipePair.GatewayReader,
+ WorkerEnvelope.BodyOneofCase.WorkerFault,
+ cancellation.Token);
+ Assert.Equal(WorkerFaultCategory.QueueOverflow, faultEnvelope.WorkerFault.Category);
+
+ Task completedTask = await Task.WhenAny(runTask, Task.Delay(TimeSpan.FromSeconds(5), cancellation.Token));
+ Assert.Same(runTask, completedTask);
+ WorkerEventQueueFaultException exception =
+ await Assert.ThrowsAsync(async () => await runTask);
+ Assert.Equal(WorkerFaultCategory.QueueOverflow, exception.Category);
+ }
+
private static WorkerPipeSession CreateSession(
Stream inbound,
Stream outbound,
diff --git a/src/ZB.MOM.WW.MxGateway.Worker.Tests/TestSupport/FakeRuntimeSession.cs b/src/ZB.MOM.WW.MxGateway.Worker.Tests/TestSupport/FakeRuntimeSession.cs
index 016b042c..39c9ed83 100644
--- a/src/ZB.MOM.WW.MxGateway.Worker.Tests/TestSupport/FakeRuntimeSession.cs
+++ b/src/ZB.MOM.WW.MxGateway.Worker.Tests/TestSupport/FakeRuntimeSession.cs
@@ -380,10 +380,21 @@ internal sealed class FakeRuntimeSession : IWorkerRuntimeSession
return eventSignal.WaitAsync(timeout, cancellationToken);
}
+ ///
+ /// Gets or sets a fault the next call hands back once, standing in
+ /// for the fault a real event queue self-records on overflow.
+ ///
+ public WorkerFault? PendingFault { get; set; }
+
///
public WorkerFault? DrainFault()
{
- return null;
+ lock (gate)
+ {
+ WorkerFault? fault = PendingFault;
+ PendingFault = null;
+ return fault;
+ }
}
///
diff --git a/src/ZB.MOM.WW.MxGateway.Worker/Bootstrap/WorkerExitCode.cs b/src/ZB.MOM.WW.MxGateway.Worker/Bootstrap/WorkerExitCode.cs
index 416e6fac..c63b0aeb 100644
--- a/src/ZB.MOM.WW.MxGateway.Worker/Bootstrap/WorkerExitCode.cs
+++ b/src/ZB.MOM.WW.MxGateway.Worker/Bootstrap/WorkerExitCode.cs
@@ -7,6 +7,11 @@ public enum WorkerExitCode
InvalidArguments = 2,
InvalidProtocolVersion = 3,
MissingNonce = 4,
+ ///
+ /// The pipe to the gateway could not be connected, or it closed under the worker — an
+ /// IOException, a connect TimeoutException, or a frame read that hit end of
+ /// stream because the gateway died or closed its end.
+ ///
PipeConnectionFailed = 5,
ProtocolViolation = 6,
@@ -18,4 +23,10 @@ public enum WorkerExitCode
/// exit code.
///
UnhandledException = 7,
+
+ ///
+ /// The outbound MXAccess event queue reached capacity. The worker wrote a
+ /// QueueOverflow fault and exited rather than drop events (fail-fast backpressure).
+ ///
+ EventQueueOverflow = 8,
}
diff --git a/src/ZB.MOM.WW.MxGateway.Worker/Ipc/WorkerEventQueueFaultException.cs b/src/ZB.MOM.WW.MxGateway.Worker/Ipc/WorkerEventQueueFaultException.cs
new file mode 100644
index 00000000..3a21a9fb
--- /dev/null
+++ b/src/ZB.MOM.WW.MxGateway.Worker/Ipc/WorkerEventQueueFaultException.cs
@@ -0,0 +1,25 @@
+using System;
+using ZB.MOM.WW.MxGateway.Contracts.Proto;
+
+namespace ZB.MOM.WW.MxGateway.Worker.Ipc;
+
+///
+/// Raised by the event drain loop after it has written the fault the MXAccess event queue
+/// recorded, to unwind the pipe session. It carries the fault category so the process boundary
+/// can pick an exit code: exits as
+/// WorkerExitCode.EventQueueOverflow rather than the generic unexpected-failure code.
+///
+public sealed class WorkerEventQueueFaultException : Exception
+{
+ /// Initializes a new instance of .
+ /// Category of the fault the event queue recorded.
+ /// Diagnostic message.
+ public WorkerEventQueueFaultException(WorkerFaultCategory category, string message)
+ : base(message)
+ {
+ Category = category;
+ }
+
+ /// Gets the category of the fault the event queue recorded.
+ public WorkerFaultCategory Category { get; }
+}
diff --git a/src/ZB.MOM.WW.MxGateway.Worker/Ipc/WorkerPipeSession.cs b/src/ZB.MOM.WW.MxGateway.Worker/Ipc/WorkerPipeSession.cs
index 59f3d7c1..8b2804f7 100644
--- a/src/ZB.MOM.WW.MxGateway.Worker/Ipc/WorkerPipeSession.cs
+++ b/src/ZB.MOM.WW.MxGateway.Worker/Ipc/WorkerPipeSession.cs
@@ -592,7 +592,8 @@ public sealed class WorkerPipeSession
{
_state = WorkerState.Faulted;
await TryWriteFaultAsync(fault, cancellationToken).ConfigureAwait(false);
- throw new InvalidOperationException(
+ throw new WorkerEventQueueFaultException(
+ fault.Category,
string.IsNullOrWhiteSpace(fault.DiagnosticMessage)
? $"MXAccess event queue faulted with category {fault.Category}."
: fault.DiagnosticMessage);
diff --git a/src/ZB.MOM.WW.MxGateway.Worker/WorkerApplication.cs b/src/ZB.MOM.WW.MxGateway.Worker/WorkerApplication.cs
index 26426072..28e676d5 100644
--- a/src/ZB.MOM.WW.MxGateway.Worker/WorkerApplication.cs
+++ b/src/ZB.MOM.WW.MxGateway.Worker/WorkerApplication.cs
@@ -1,6 +1,7 @@
using System;
using System.Collections.Generic;
using System.IO;
+using ZB.MOM.WW.MxGateway.Contracts.Proto;
using ZB.MOM.WW.MxGateway.Worker.Bootstrap;
using ZB.MOM.WW.MxGateway.Worker.Ipc;
@@ -158,6 +159,22 @@ public static class WorkerApplication
return (int)WorkerExitCode.Success;
}
+ catch (WorkerFrameProtocolException exception)
+ when (exception.ErrorCode == WorkerFrameProtocolErrorCode.EndOfStream)
+ {
+ // A read that hits end of stream means the gateway closed its end of the pipe — most
+ // often because the gateway process died. That is a lost connection, not a malformed
+ // frame, so it exits as PipeConnectionFailed alongside the IOException a broken pipe
+ // raises, rather than as ProtocolViolation.
+ logger.Error("WorkerPipeConnectionFailed", new Dictionary
+ {
+ ["exit_code"] = WorkerExitCode.PipeConnectionFailed,
+ ["error_code"] = exception.ErrorCode,
+ ["exception_type"] = exception.GetType().FullName,
+ });
+
+ return (int)WorkerExitCode.PipeConnectionFailed;
+ }
catch (WorkerFrameProtocolException exception)
{
logger.Error("WorkerPipeProtocolFailure", new Dictionary
@@ -169,6 +186,17 @@ public static class WorkerApplication
return (int)WorkerExitCode.ProtocolViolation;
}
+ catch (WorkerEventQueueFaultException exception)
+ when (exception.Category == WorkerFaultCategory.QueueOverflow)
+ {
+ logger.Error("WorkerEventQueueOverflow", new Dictionary
+ {
+ ["exit_code"] = WorkerExitCode.EventQueueOverflow,
+ ["exception_type"] = exception.GetType().FullName,
+ });
+
+ return (int)WorkerExitCode.EventQueueOverflow;
+ }
catch (Exception exception) when (exception is IOException or TimeoutException)
{
logger.Error("WorkerPipeConnectionFailed", new Dictionary