diff --git a/docs/MxAccessWorkerInstanceDesign.md b/docs/MxAccessWorkerInstanceDesign.md index e3785fb5..307b84b9 100644 --- a/docs/MxAccessWorkerInstanceDesign.md +++ b/docs/MxAccessWorkerInstanceDesign.md @@ -133,9 +133,10 @@ starts the STA, or touches MXAccess: | `2` | `InvalidArguments` | Required arguments are missing or unknown arguments are present. | | `3` | `InvalidProtocolVersion` | `--protocol-version` is not numeric or does not match the supported worker protocol. | | `4` | `MissingNonce` | `MXGATEWAY_WORKER_NONCE` is absent or empty. | -| `5` | `PipeConnectionFailed` | An `IOException` or `TimeoutException` escapes the pipe client. | -| `6` | `ProtocolViolation` | A `WorkerFrameProtocolException` escapes the pipe client. | +| `5` | `PipeConnectionFailed` | An `IOException` or `TimeoutException` escapes the pipe client, or a frame read hits end of stream because the gateway closed the pipe (gateway death). | +| `6` | `ProtocolViolation` | Any other `WorkerFrameProtocolException` escapes the pipe client. | | `7` | `UnhandledException` | An exception reaches the `AppDomain.UnhandledException` hook (see `docs/WorkerBootstrap.md`). | +| `8` | `EventQueueOverflow` | The outbound MXAccess event queue reached capacity (see the event overflow policy below). | Bootstrap logs use `WorkerConsoleLogger` key/value output. `WorkerLogRedactor` redacts fields whose names indicate nonce, secret, password, token, @@ -868,9 +869,10 @@ category `QueueOverflow`. The event drain loop's next pass observes the fault through `DrainFault()`, writes the fault frame — a Control-priority frame, so it is not stuck behind a queued event backlog — and then throws to unwind `RunAsync`: the worker process exits rather than continuing in a state where -events are being silently lost. The exit currently uses the generic -`WorkerExitCode.UnexpectedFailure` rather than a dedicated overflow code; a -distinct exit code remains open (tracked separately). Do not drop or coalesce +events are being silently lost. The drain loop unwinds with a +`WorkerEventQueueFaultException` carrying the fault category, and the process +boundary maps `QueueOverflow` to the dedicated `WorkerExitCode.EventQueueOverflow` +(8), so an overflow is identifiable from the worker's exit code alone. Do not drop or coalesce events to avoid this exit — that is explicitly out of scope for v1. Production coalescing may be added later, but it must be explicit and tested. diff --git a/docs/WorkerBootstrap.md b/docs/WorkerBootstrap.md index 73aa9bda..58557019 100644 --- a/docs/WorkerBootstrap.md +++ b/docs/WorkerBootstrap.md @@ -115,9 +115,10 @@ public static WorkerBootstrapResult Failure(WorkerExitCode exitCode, IEnumerable | `InvalidArguments` | 2 | One or more `--session-id`, `--pipe-name`, or `--protocol-version` errors (missing, empty, unknown flag, or no value). | | `InvalidProtocolVersion` | 3 | `--protocol-version` is not a `uint` or does not equal `GatewayContractInfo.WorkerProtocolVersion`. | | `MissingNonce` | 4 | The `MXGATEWAY_WORKER_NONCE` environment variable is null, empty, or whitespace. | -| `PipeConnectionFailed` | 5 | An `IOException` or `TimeoutException` escapes the pipe client. | -| `ProtocolViolation` | 6 | A `WorkerFrameProtocolException` escapes the pipe client. | +| `PipeConnectionFailed` | 5 | An `IOException` or `TimeoutException` escapes the pipe client, or a `WorkerFrameProtocolException` with `EndOfStream` does — the gateway closed its end of the pipe, usually because it died. | +| `ProtocolViolation` | 6 | Any other `WorkerFrameProtocolException` escapes the pipe client. | | `UnhandledException` | 7 | An exception reaches `AppDomain.CurrentDomain.UnhandledException` — thrown on a thread outside `WorkerApplication.Run`'s `try/catch` (a background task continuation, a timer, a COM callback thread). | +| `EventQueueOverflow` | 8 | The outbound MXAccess event queue reached capacity: the event drain loop wrote the `QueueOverflow` fault and unwound with a `WorkerEventQueueFaultException`. Other event-queue fault categories still exit as `UnexpectedFailure`. | `InvalidArguments`, `InvalidProtocolVersion`, and `MissingNonce` originate in the parser; `UnhandledException` originates in the process-wide hook described below; the others originate in `WorkerApplication.Run`'s `try/catch` around the pipe client. @@ -219,7 +220,7 @@ if (!result.Succeeded) } ``` -On success it logs `WorkerBootstrapSucceeded` with the session fields (the `nonce` field is redacted by `WorkerLogRedactor` because of its name), hands the `WorkerOptions` to `IWorkerPipeClient.RunAsync`, and waits synchronously. The `try/catch` around the pipe call maps `WorkerFrameProtocolException` to `ProtocolViolation`, `IOException`/`TimeoutException` to `PipeConnectionFailed`, and any other exception to `UnexpectedFailure`, so every code path through `Run` returns one of the values in `WorkerExitCode`. +On success it logs `WorkerBootstrapSucceeded` with the session fields (the `nonce` field is redacted by `WorkerLogRedactor` because of its name), hands the `WorkerOptions` to `IWorkerPipeClient.RunAsync`, and waits synchronously. The `try/catch` around the pipe call maps a `WorkerFrameProtocolException` with `EndOfStream` to `PipeConnectionFailed` (a closed pipe is a lost connection, not a malformed frame), any other `WorkerFrameProtocolException` to `ProtocolViolation`, a `WorkerEventQueueFaultException` with category `QueueOverflow` to `EventQueueOverflow`, `IOException`/`TimeoutException` to `PipeConnectionFailed`, and any other exception to `UnexpectedFailure`, so every code path through `Run` returns one of the values in `WorkerExitCode`. ## Related Documentation diff --git a/src/ZB.MOM.WW.MxGateway.Worker.Tests/Bootstrap/WorkerApplicationTests.cs b/src/ZB.MOM.WW.MxGateway.Worker.Tests/Bootstrap/WorkerApplicationTests.cs index f29885ee..3d915791 100644 --- a/src/ZB.MOM.WW.MxGateway.Worker.Tests/Bootstrap/WorkerApplicationTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Worker.Tests/Bootstrap/WorkerApplicationTests.cs @@ -3,6 +3,7 @@ using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; using ZB.MOM.WW.MxGateway.Contracts; +using ZB.MOM.WW.MxGateway.Contracts.Proto; using ZB.MOM.WW.MxGateway.Worker.Bootstrap; using ZB.MOM.WW.MxGateway.Worker.Ipc; @@ -103,6 +104,81 @@ public sealed class WorkerApplicationTests Assert.Equal("WorkerPipeProtocolFailure", logger.Entries[1].EventName); } + /// + /// A frame read that hits end of stream means the gateway closed the pipe (typically it + /// died). That is a lost connection, so it exits as PipeConnectionFailed (5), not + /// ProtocolViolation (6) (mxaccessgw#186). + /// + [Fact] + public void Run_WhenGatewayClosesPipe_ReturnsPipeConnectionFailed() + { + MemoryWorkerEnvironment environment = CreateEnvironment("nonce-secret"); + MemoryWorkerLogger logger = new(); + + int exitCode = ZB.MOM.WW.MxGateway.Worker.WorkerApplication.Run( + ValidArgs(), + environment, + logger, + new ThrowingPipeClient(new WorkerFrameProtocolException( + WorkerFrameProtocolErrorCode.EndOfStream, + "Worker frame ended before the expected number of bytes were read."))); + + Assert.Equal((int)WorkerExitCode.PipeConnectionFailed, exitCode); + Assert.Equal(5, exitCode); + MemoryWorkerLogEntry entry = logger.Entries[1]; + Assert.Equal("WorkerPipeConnectionFailed", entry.EventName); + Assert.Equal(WorkerExitCode.PipeConnectionFailed, entry.Fields["exit_code"]); + Assert.Equal(WorkerFrameProtocolErrorCode.EndOfStream, entry.Fields["error_code"]); + } + + /// + /// Event-queue overflow exits with its own code (8) instead of the generic + /// UnexpectedFailure (1), so an overflow is identifiable from the exit code alone + /// (mxaccessgw#186). + /// + [Fact] + public void Run_WhenEventQueueOverflows_ReturnsEventQueueOverflow() + { + MemoryWorkerEnvironment environment = CreateEnvironment("nonce-secret"); + MemoryWorkerLogger logger = new(); + + int exitCode = ZB.MOM.WW.MxGateway.Worker.WorkerApplication.Run( + ValidArgs(), + environment, + logger, + new ThrowingPipeClient(new WorkerEventQueueFaultException( + WorkerFaultCategory.QueueOverflow, + "MXAccess outbound event queue reached capacity 4."))); + + Assert.Equal((int)WorkerExitCode.EventQueueOverflow, exitCode); + Assert.Equal(8, exitCode); + MemoryWorkerLogEntry entry = logger.Entries[1]; + Assert.Equal("Error", entry.Level); + Assert.Equal("WorkerEventQueueOverflow", entry.EventName); + Assert.Equal(WorkerExitCode.EventQueueOverflow, entry.Fields["exit_code"]); + } + + /// + /// An event-queue fault of any other category keeps the generic exit code; only overflow + /// has a dedicated one. + /// + [Fact] + public void Run_WhenEventQueueFaultsForAnotherReason_ReturnsUnexpectedFailure() + { + MemoryWorkerEnvironment environment = CreateEnvironment("nonce-secret"); + MemoryWorkerLogger logger = new(); + + int exitCode = ZB.MOM.WW.MxGateway.Worker.WorkerApplication.Run( + ValidArgs(), + environment, + logger, + new ThrowingPipeClient(new WorkerEventQueueFaultException( + WorkerFaultCategory.MxaccessEventConversionFailed, + "conversion failed"))); + + Assert.Equal((int)WorkerExitCode.UnexpectedFailure, exitCode); + } + /// Verifies that unexpected exceptions during bootstrap are logged. [Fact] public void Run_WithUnexpectedBootstrapFailure_ReturnsUnexpectedFailure() diff --git a/src/ZB.MOM.WW.MxGateway.Worker.Tests/Ipc/WorkerPipeSessionTests.cs b/src/ZB.MOM.WW.MxGateway.Worker.Tests/Ipc/WorkerPipeSessionTests.cs index e6fe8a1b..780d313f 100644 --- a/src/ZB.MOM.WW.MxGateway.Worker.Tests/Ipc/WorkerPipeSessionTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Worker.Tests/Ipc/WorkerPipeSessionTests.cs @@ -1998,6 +1998,47 @@ public sealed class WorkerPipeSessionTests await SendShutdownAndWaitAsync(pipePair, runTask, cancellation.Token); } + /// + /// When the event queue records an overflow fault, the drain loop writes the fault frame and + /// unwinds with carrying the category, which is + /// what lets the process boundary exit with the dedicated overflow code (mxaccessgw#186). + /// + /// A task that represents the asynchronous operation. + [Fact] + public async Task RunAsync_WhenEventQueueOverflows_WritesFaultAndThrowsTypedException() + { + using CancellationTokenSource cancellation = new(TimeSpan.FromSeconds(10)); + using PipePair pipePair = await PipePair.CreateAsync(cancellation.Token); + FakeRuntimeSession runtime = new() + { + PendingFault = new WorkerFault + { + Category = WorkerFaultCategory.QueueOverflow, + DiagnosticMessage = "MXAccess outbound event queue reached capacity 4.", + ProtocolStatus = new ProtocolStatus + { + Code = ProtocolStatusCode.WorkerUnavailable, + Message = "MXAccess outbound event queue reached capacity 4.", + }, + }, + }; + WorkerPipeSession session = CreatePipeSession(pipePair.WorkerStream, runtime); + Task runTask = session.RunAsync(cancellation.Token); + await CompleteGatewayHandshakeAsync(pipePair, cancellation.Token); + + WorkerEnvelope faultEnvelope = await ReadUntilAsync( + pipePair.GatewayReader, + WorkerEnvelope.BodyOneofCase.WorkerFault, + cancellation.Token); + Assert.Equal(WorkerFaultCategory.QueueOverflow, faultEnvelope.WorkerFault.Category); + + Task completedTask = await Task.WhenAny(runTask, Task.Delay(TimeSpan.FromSeconds(5), cancellation.Token)); + Assert.Same(runTask, completedTask); + WorkerEventQueueFaultException exception = + await Assert.ThrowsAsync(async () => await runTask); + Assert.Equal(WorkerFaultCategory.QueueOverflow, exception.Category); + } + private static WorkerPipeSession CreateSession( Stream inbound, Stream outbound, diff --git a/src/ZB.MOM.WW.MxGateway.Worker.Tests/TestSupport/FakeRuntimeSession.cs b/src/ZB.MOM.WW.MxGateway.Worker.Tests/TestSupport/FakeRuntimeSession.cs index 016b042c..39c9ed83 100644 --- a/src/ZB.MOM.WW.MxGateway.Worker.Tests/TestSupport/FakeRuntimeSession.cs +++ b/src/ZB.MOM.WW.MxGateway.Worker.Tests/TestSupport/FakeRuntimeSession.cs @@ -380,10 +380,21 @@ internal sealed class FakeRuntimeSession : IWorkerRuntimeSession return eventSignal.WaitAsync(timeout, cancellationToken); } + /// + /// Gets or sets a fault the next call hands back once, standing in + /// for the fault a real event queue self-records on overflow. + /// + public WorkerFault? PendingFault { get; set; } + /// public WorkerFault? DrainFault() { - return null; + lock (gate) + { + WorkerFault? fault = PendingFault; + PendingFault = null; + return fault; + } } /// diff --git a/src/ZB.MOM.WW.MxGateway.Worker/Bootstrap/WorkerExitCode.cs b/src/ZB.MOM.WW.MxGateway.Worker/Bootstrap/WorkerExitCode.cs index 416e6fac..c63b0aeb 100644 --- a/src/ZB.MOM.WW.MxGateway.Worker/Bootstrap/WorkerExitCode.cs +++ b/src/ZB.MOM.WW.MxGateway.Worker/Bootstrap/WorkerExitCode.cs @@ -7,6 +7,11 @@ public enum WorkerExitCode InvalidArguments = 2, InvalidProtocolVersion = 3, MissingNonce = 4, + /// + /// The pipe to the gateway could not be connected, or it closed under the worker — an + /// IOException, a connect TimeoutException, or a frame read that hit end of + /// stream because the gateway died or closed its end. + /// PipeConnectionFailed = 5, ProtocolViolation = 6, @@ -18,4 +23,10 @@ public enum WorkerExitCode /// exit code. /// UnhandledException = 7, + + /// + /// The outbound MXAccess event queue reached capacity. The worker wrote a + /// QueueOverflow fault and exited rather than drop events (fail-fast backpressure). + /// + EventQueueOverflow = 8, } diff --git a/src/ZB.MOM.WW.MxGateway.Worker/Ipc/WorkerEventQueueFaultException.cs b/src/ZB.MOM.WW.MxGateway.Worker/Ipc/WorkerEventQueueFaultException.cs new file mode 100644 index 00000000..3a21a9fb --- /dev/null +++ b/src/ZB.MOM.WW.MxGateway.Worker/Ipc/WorkerEventQueueFaultException.cs @@ -0,0 +1,25 @@ +using System; +using ZB.MOM.WW.MxGateway.Contracts.Proto; + +namespace ZB.MOM.WW.MxGateway.Worker.Ipc; + +/// +/// Raised by the event drain loop after it has written the fault the MXAccess event queue +/// recorded, to unwind the pipe session. It carries the fault category so the process boundary +/// can pick an exit code: exits as +/// WorkerExitCode.EventQueueOverflow rather than the generic unexpected-failure code. +/// +public sealed class WorkerEventQueueFaultException : Exception +{ + /// Initializes a new instance of . + /// Category of the fault the event queue recorded. + /// Diagnostic message. + public WorkerEventQueueFaultException(WorkerFaultCategory category, string message) + : base(message) + { + Category = category; + } + + /// Gets the category of the fault the event queue recorded. + public WorkerFaultCategory Category { get; } +} diff --git a/src/ZB.MOM.WW.MxGateway.Worker/Ipc/WorkerPipeSession.cs b/src/ZB.MOM.WW.MxGateway.Worker/Ipc/WorkerPipeSession.cs index 59f3d7c1..8b2804f7 100644 --- a/src/ZB.MOM.WW.MxGateway.Worker/Ipc/WorkerPipeSession.cs +++ b/src/ZB.MOM.WW.MxGateway.Worker/Ipc/WorkerPipeSession.cs @@ -592,7 +592,8 @@ public sealed class WorkerPipeSession { _state = WorkerState.Faulted; await TryWriteFaultAsync(fault, cancellationToken).ConfigureAwait(false); - throw new InvalidOperationException( + throw new WorkerEventQueueFaultException( + fault.Category, string.IsNullOrWhiteSpace(fault.DiagnosticMessage) ? $"MXAccess event queue faulted with category {fault.Category}." : fault.DiagnosticMessage); diff --git a/src/ZB.MOM.WW.MxGateway.Worker/WorkerApplication.cs b/src/ZB.MOM.WW.MxGateway.Worker/WorkerApplication.cs index 26426072..28e676d5 100644 --- a/src/ZB.MOM.WW.MxGateway.Worker/WorkerApplication.cs +++ b/src/ZB.MOM.WW.MxGateway.Worker/WorkerApplication.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.IO; +using ZB.MOM.WW.MxGateway.Contracts.Proto; using ZB.MOM.WW.MxGateway.Worker.Bootstrap; using ZB.MOM.WW.MxGateway.Worker.Ipc; @@ -158,6 +159,22 @@ public static class WorkerApplication return (int)WorkerExitCode.Success; } + catch (WorkerFrameProtocolException exception) + when (exception.ErrorCode == WorkerFrameProtocolErrorCode.EndOfStream) + { + // A read that hits end of stream means the gateway closed its end of the pipe — most + // often because the gateway process died. That is a lost connection, not a malformed + // frame, so it exits as PipeConnectionFailed alongside the IOException a broken pipe + // raises, rather than as ProtocolViolation. + logger.Error("WorkerPipeConnectionFailed", new Dictionary + { + ["exit_code"] = WorkerExitCode.PipeConnectionFailed, + ["error_code"] = exception.ErrorCode, + ["exception_type"] = exception.GetType().FullName, + }); + + return (int)WorkerExitCode.PipeConnectionFailed; + } catch (WorkerFrameProtocolException exception) { logger.Error("WorkerPipeProtocolFailure", new Dictionary @@ -169,6 +186,17 @@ public static class WorkerApplication return (int)WorkerExitCode.ProtocolViolation; } + catch (WorkerEventQueueFaultException exception) + when (exception.Category == WorkerFaultCategory.QueueOverflow) + { + logger.Error("WorkerEventQueueOverflow", new Dictionary + { + ["exit_code"] = WorkerExitCode.EventQueueOverflow, + ["exception_type"] = exception.GetType().FullName, + }); + + return (int)WorkerExitCode.EventQueueOverflow; + } catch (Exception exception) when (exception is IOException or TimeoutException) { logger.Error("WorkerPipeConnectionFailed", new Dictionary