using System; using System.Collections.Generic; using System.IO; using System.IO.Pipes; using System.Runtime.InteropServices; using System.Threading; using System.Threading.Tasks; using Google.Protobuf.WellKnownTypes; using ZB.MOM.WW.MxGateway.Contracts; using ZB.MOM.WW.MxGateway.Contracts.Proto; using ZB.MOM.WW.MxGateway.Worker.Ipc; using ZB.MOM.WW.MxGateway.Worker.MxAccess; using ZB.MOM.WW.MxGateway.Worker.Tests.TestSupport; namespace ZB.MOM.WW.MxGateway.Worker.Tests.Ipc; public sealed class WorkerPipeSessionTests { private const string SessionId = "session-1"; private const string Nonce = "nonce-secret"; // Byte-heavy drain fixture (WRK-21). 10,000 events at ~1.7 KiB each is ~17 MB of queue — far // more than one frame — so DrainEvents must truncate. // // Two limits below are harness accommodations, not properties of the fix. PipePair runs both // ends of a duplex pipe inside one process, with no continuous read pump and with blocking // FlushFileBuffers under every frame write, so it tolerates neither multi-megabyte frames nor // hundreds of large round trips before both ends wedge waiting on each other. Hence a small // negotiated frame maximum, and a smaller queue for the drain-to-empty walk. The byte cap // behaves identically at any frame size; exhaustive no-loss over the full 10,000 events is // covered without a pipe by MxAccessEventQueueTests. private const int ByteHeavyEventCount = 10_000; private const int RepeatedDrainEventCount = 1_000; private const int ByteHeavyEventPayloadBytes = 1_800; private const uint NegotiatedMaxFrameBytes = 128 * 1024; /// Verifies that valid gateway hello triggers worker hello and ready responses. /// A task that represents the asynchronous operation. [Fact] public async Task CompleteStartupHandshakeAsync_WithValidGatewayHello_SendsHelloThenReady() { WorkerFrameProtocolOptions options = CreateOptions(); using MemoryStream inbound = new(); await new WorkerFrameWriter(inbound, options).WriteAsync(CreateGatewayHelloEnvelope()); inbound.Position = 0; using MemoryStream outbound = new(); WorkerPipeSession session = CreateSession(inbound, outbound, options); bool initialized = false; await session.CompleteStartupHandshakeAsync( _ => { initialized = true; return Task.CompletedTask; }); Assert.True(initialized); WorkerEnvelope[] written = ReadWrittenFrames(outbound, options); Assert.Equal(2, written.Length); Assert.Equal(WorkerEnvelope.BodyOneofCase.WorkerHello, written[0].BodyCase); Assert.Equal(WorkerEnvelope.BodyOneofCase.WorkerReady, written[1].BodyCase); Assert.Equal(Nonce, written[0].WorkerHello.Nonce); Assert.Equal(1234, written[1].WorkerReady.WorkerProcessId); Assert.Equal(ZB.MOM.WW.MxGateway.Worker.MxAccess.MxAccessInteropInfo.ProgId, written[1].WorkerReady.MxaccessProgid); Assert.Equal(ZB.MOM.WW.MxGateway.Worker.MxAccess.MxAccessInteropInfo.Clsid, written[1].WorkerReady.MxaccessClsid); Assert.NotNull(written[1].WorkerReady.ReadyTimestamp); } /// Verifies that wrong nonce causes protocol violation fault before initialization. /// A task that represents the asynchronous operation. [Fact] public async Task CompleteStartupHandshakeAsync_WithWrongNonce_FaultsBeforeInitialization() { WorkerFrameProtocolOptions options = CreateOptions(); using MemoryStream inbound = new(); await new WorkerFrameWriter(inbound, options).WriteAsync(CreateGatewayHelloEnvelope(nonce: "wrong")); inbound.Position = 0; using MemoryStream outbound = new(); WorkerPipeSession session = CreateSession(inbound, outbound, options); bool initialized = false; WorkerFrameProtocolException exception = await Assert.ThrowsAsync( async () => await session.CompleteStartupHandshakeAsync( _ => { initialized = true; return Task.CompletedTask; })); Assert.False(initialized); Assert.Equal(WorkerFrameProtocolErrorCode.NonceMismatch, exception.ErrorCode); WorkerEnvelope fault = Assert.Single(ReadWrittenFrames(outbound, options)); Assert.Equal(WorkerEnvelope.BodyOneofCase.WorkerFault, fault.BodyCase); Assert.Equal(WorkerFaultCategory.ProtocolViolation, fault.WorkerFault.Category); } /// Verifies that unsupported protocol version causes mismatch fault before initialization. /// A task that represents the asynchronous operation. [Fact] public async Task CompleteStartupHandshakeAsync_WithWrongProtocol_FaultsBeforeInitialization() { WorkerFrameProtocolOptions options = CreateOptions(); using MemoryStream inbound = new(); await new WorkerFrameWriter(inbound, options).WriteAsync(CreateGatewayHelloEnvelope(supportedProtocolVersion: 999)); inbound.Position = 0; using MemoryStream outbound = new(); WorkerPipeSession session = CreateSession(inbound, outbound, options); bool initialized = false; WorkerFrameProtocolException exception = await Assert.ThrowsAsync( async () => await session.CompleteStartupHandshakeAsync( _ => { initialized = true; return Task.CompletedTask; })); Assert.False(initialized); Assert.Equal(WorkerFrameProtocolErrorCode.ProtocolVersionMismatch, exception.ErrorCode); WorkerEnvelope fault = Assert.Single(ReadWrittenFrames(outbound, options)); Assert.Equal(WorkerFaultCategory.ProtocolMismatch, fault.WorkerFault.Category); } /// Verifies that malformed frame causes protocol violation fault. /// A task that represents the asynchronous operation. [Fact] public async Task CompleteStartupHandshakeAsync_WithMalformedFrame_WritesWorkerFault() { WorkerFrameProtocolOptions options = CreateOptions(); using MemoryStream inbound = new(WorkerFrameTestHelpers.CreateFrame(new byte[] { 0x80 })); using MemoryStream outbound = new(); WorkerPipeSession session = CreateSession(inbound, outbound, options); bool initialized = false; WorkerFrameProtocolException exception = await Assert.ThrowsAsync( async () => await session.CompleteStartupHandshakeAsync( _ => { initialized = true; return Task.CompletedTask; })); Assert.False(initialized); Assert.Equal(WorkerFrameProtocolErrorCode.InvalidEnvelope, exception.ErrorCode); WorkerEnvelope fault = Assert.Single(ReadWrittenFrames(outbound, options)); Assert.Equal(WorkerEnvelope.BodyOneofCase.WorkerFault, fault.BodyCase); Assert.Equal(WorkerFaultCategory.ProtocolViolation, fault.WorkerFault.Category); } /// Verifies that MXAccess COM creation failure produces fault instead of ready. /// A task that represents the asynchronous operation. [Fact] public async Task CompleteStartupHandshakeAsync_WhenMxAccessCreationFails_WritesFaultInsteadOfReady() { const int hresult = unchecked((int)0x80040154); WorkerFrameProtocolOptions options = CreateOptions(); using MemoryStream inbound = new(); await new WorkerFrameWriter(inbound, options).WriteAsync(CreateGatewayHelloEnvelope()); inbound.Position = 0; using MemoryStream outbound = new(); WorkerPipeSession session = CreateSession(inbound, outbound, options); await Assert.ThrowsAsync( async () => await session.CompleteStartupHandshakeAsync( _ => Task.FromException(new COMException("Class not registered.", hresult)))); WorkerEnvelope[] written = ReadWrittenFrames(outbound, options); Assert.Equal(2, written.Length); Assert.Equal(WorkerEnvelope.BodyOneofCase.WorkerHello, written[0].BodyCase); Assert.Equal(WorkerEnvelope.BodyOneofCase.WorkerFault, written[1].BodyCase); Assert.Equal(WorkerFaultCategory.MxaccessCreationFailed, written[1].WorkerFault.Category); Assert.Equal(hresult, written[1].WorkerFault.Hresult); Assert.Equal(typeof(COMException).FullName, written[1].WorkerFault.ExceptionType); Assert.Equal(ProtocolStatusCode.WorkerUnavailable, written[1].WorkerFault.ProtocolStatus.Code); } /// Verifies that heartbeat payload reflects current runtime snapshot. /// A task that represents the asynchronous operation. [Fact] public async Task RunAsync_SendsHeartbeatPayloadFromRuntimeSnapshot() { using CancellationTokenSource cancellation = new(TimeSpan.FromSeconds(5)); using PipePair pipePair = await PipePair.CreateAsync(cancellation.Token); FakeRuntimeSession runtime = new(); runtime.SetSnapshot(new WorkerRuntimeHeartbeatSnapshot( DateTimeOffset.UtcNow, pendingCommandCount: 2, outboundEventQueueDepth: 3, lastEventSequence: 42, currentCommandCorrelationId: "current-command")); WorkerPipeSession session = CreatePipeSession( pipePair.WorkerStream, runtime, new WorkerPipeSessionOptions { HeartbeatInterval = TimeSpan.FromMilliseconds(20), HeartbeatGrace = TimeSpan.FromSeconds(5), }); Task runTask = session.RunAsync(cancellation.Token); await CompleteGatewayHandshakeAsync(pipePair, cancellation.Token); // Deterministic race: read the first heartbeat while watching runTask. // A faulted RunAsync would complete the run task first; if it wins the // race the test fails immediately with the underlying fault instead of // waiting out an arbitrary fixed delay. Task heartbeatTask = ReadUntilAsync( pipePair.GatewayReader, WorkerEnvelope.BodyOneofCase.WorkerHeartbeat, cancellation.Token); Task winner = await Task.WhenAny(runTask, heartbeatTask); if (winner == runTask) { // Surface the RunAsync fault (or assert it did not exit early). await runTask; Assert.Fail("RunAsync completed before the first heartbeat was received."); } WorkerEnvelope heartbeat = await heartbeatTask; Assert.Equal(WorkerState.ExecutingCommand, heartbeat.WorkerHeartbeat.State); Assert.Equal(1234, heartbeat.WorkerHeartbeat.WorkerProcessId); Assert.Equal(2u, heartbeat.WorkerHeartbeat.PendingCommandCount); Assert.Equal(3u, heartbeat.WorkerHeartbeat.OutboundEventQueueDepth); Assert.Equal(42UL, heartbeat.WorkerHeartbeat.LastEventSequence); Assert.Equal("current-command", heartbeat.WorkerHeartbeat.CurrentCommandCorrelationId); await SendShutdownAndWaitAsync(pipePair, runTask, cancellation.Token); } /// Verifies that heartbeat reports current command correlation during execution. /// A task that represents the asynchronous operation. [Fact] public async Task RunAsync_WhenCommandIsExecuting_HeartbeatReportsCurrentCorrelation() { using CancellationTokenSource cancellation = new(TimeSpan.FromSeconds(5)); using PipePair pipePair = await PipePair.CreateAsync(cancellation.Token); FakeRuntimeSession runtime = new() { BlockDispatch = true, }; WorkerPipeSession session = CreatePipeSession( pipePair.WorkerStream, runtime, new WorkerPipeSessionOptions { HeartbeatInterval = TimeSpan.FromMilliseconds(20), HeartbeatGrace = TimeSpan.FromSeconds(5), }); Task runTask = session.RunAsync(cancellation.Token); await CompleteGatewayHandshakeAsync(pipePair, cancellation.Token); await pipePair.GatewayWriter.WriteAsync( CreateCommandEnvelope("command-1"), cancellation.Token); Assert.True(runtime.DispatchStarted.Wait(TimeSpan.FromSeconds(2))); WorkerEnvelope heartbeat = await ReadUntilAsync( pipePair.GatewayReader, WorkerEnvelope.BodyOneofCase.WorkerHeartbeat, envelope => envelope.WorkerHeartbeat.CurrentCommandCorrelationId == "command-1", cancellation.Token); Assert.Equal("command-1", heartbeat.WorkerHeartbeat.CurrentCommandCorrelationId); Assert.Equal(WorkerState.ExecutingCommand, heartbeat.WorkerHeartbeat.State); runtime.ReleaseDispatch(); WorkerEnvelope reply = await ReadUntilAsync( pipePair.GatewayReader, WorkerEnvelope.BodyOneofCase.WorkerCommandReply, cancellation.Token); Assert.Equal("command-1", reply.CorrelationId); Assert.Equal(ProtocolStatusCode.Ok, reply.WorkerCommandReply.Reply.ProtocolStatus.Code); await SendShutdownAndWaitAsync(pipePair, runTask, cancellation.Token); } /// Verifies that worker events are written to the pipe. /// A task that represents the asynchronous operation. [Fact] public async Task RunAsync_WhenRuntimeHasEvents_WritesWorkerEventEnvelope() { using CancellationTokenSource cancellation = new(TimeSpan.FromSeconds(5)); using PipePair pipePair = await PipePair.CreateAsync(cancellation.Token); FakeRuntimeSession runtime = new(); WorkerPipeSession session = CreatePipeSession( pipePair.WorkerStream, runtime, new WorkerPipeSessionOptions { HeartbeatInterval = TimeSpan.FromMilliseconds(100), HeartbeatGrace = TimeSpan.FromSeconds(5), }); Task runTask = session.RunAsync(cancellation.Token); await CompleteGatewayHandshakeAsync(pipePair, cancellation.Token); runtime.EnqueueEvent(CreateWorkerEvent(sequence: 7)); WorkerEnvelope workerEvent = await ReadUntilAsync( pipePair.GatewayReader, WorkerEnvelope.BodyOneofCase.WorkerEvent, cancellation.Token); Assert.Equal(MxEventFamily.OnDataChange, workerEvent.WorkerEvent.Event.Family); Assert.Equal(7UL, workerEvent.WorkerEvent.Event.WorkerSequence); await SendShutdownAndWaitAsync(pipePair, runTask, cancellation.Token); } /// /// Verifies that a Ping control command is answered on the worker side /// (not dispatched to the STA) with an OK reply that echoes the ping /// message into the reply's diagnostic field. /// /// A task that represents the asynchronous operation. [Fact] public async Task RunAsync_PingControlCommand_RepliesOkAndEchoesMessage() { using CancellationTokenSource cancellation = new(TimeSpan.FromSeconds(5)); using PipePair pipePair = await PipePair.CreateAsync(cancellation.Token); FakeRuntimeSession runtime = new(); WorkerPipeSession session = CreatePipeSession(pipePair.WorkerStream, runtime); Task runTask = session.RunAsync(cancellation.Token); await CompleteGatewayHandshakeAsync(pipePair, cancellation.Token); await pipePair.GatewayWriter .WriteAsync(CreatePingCommandEnvelope("ping-1", "hello-worker"), cancellation.Token); WorkerEnvelope replyEnvelope = await ReadUntilAsync( pipePair.GatewayReader, WorkerEnvelope.BodyOneofCase.WorkerCommandReply, cancellation.Token); MxCommandReply reply = replyEnvelope.WorkerCommandReply.Reply; Assert.Equal("ping-1", reply.CorrelationId); Assert.Equal(MxCommandKind.Ping, reply.Kind); Assert.Equal(ProtocolStatusCode.Ok, reply.ProtocolStatus.Code); Assert.Equal("hello-worker", reply.DiagnosticMessage); await SendShutdownAndWaitAsync(pipePair, runTask, cancellation.Token); } /// /// Verifies that GetSessionState reports the worker's lifecycle as the /// proto SessionState — READY while the message loop is serving. /// /// A task that represents the asynchronous operation. [Fact] public async Task RunAsync_GetSessionStateControlCommand_RepliesReady() { using CancellationTokenSource cancellation = new(TimeSpan.FromSeconds(5)); using PipePair pipePair = await PipePair.CreateAsync(cancellation.Token); FakeRuntimeSession runtime = new(); WorkerPipeSession session = CreatePipeSession(pipePair.WorkerStream, runtime); Task runTask = session.RunAsync(cancellation.Token); await CompleteGatewayHandshakeAsync(pipePair, cancellation.Token); await pipePair.GatewayWriter .WriteAsync( CreateControlCommandEnvelope( "state-1", MxCommandKind.GetSessionState, command => command.GetSessionState = new GetSessionStateCommand()), cancellation.Token); WorkerEnvelope replyEnvelope = await ReadUntilAsync( pipePair.GatewayReader, WorkerEnvelope.BodyOneofCase.WorkerCommandReply, cancellation.Token); MxCommandReply reply = replyEnvelope.WorkerCommandReply.Reply; Assert.Equal(MxCommandKind.GetSessionState, reply.Kind); Assert.Equal(ProtocolStatusCode.Ok, reply.ProtocolStatus.Code); Assert.Equal(SessionState.Ready, reply.SessionState.State); await SendShutdownAndWaitAsync(pipePair, runTask, cancellation.Token); } /// /// Verifies that GetWorkerInfo populates the worker process id, version, /// and MXAccess ProgID/CLSID from the worker's own metadata. /// /// A task that represents the asynchronous operation. [Fact] public async Task RunAsync_GetWorkerInfoControlCommand_PopulatesWorkerInfoFields() { using CancellationTokenSource cancellation = new(TimeSpan.FromSeconds(5)); using PipePair pipePair = await PipePair.CreateAsync(cancellation.Token); FakeRuntimeSession runtime = new(); WorkerPipeSession session = CreatePipeSession(pipePair.WorkerStream, runtime); Task runTask = session.RunAsync(cancellation.Token); await CompleteGatewayHandshakeAsync(pipePair, cancellation.Token); await pipePair.GatewayWriter .WriteAsync( CreateControlCommandEnvelope( "info-1", MxCommandKind.GetWorkerInfo, command => command.GetWorkerInfo = new GetWorkerInfoCommand()), cancellation.Token); WorkerEnvelope replyEnvelope = await ReadUntilAsync( pipePair.GatewayReader, WorkerEnvelope.BodyOneofCase.WorkerCommandReply, cancellation.Token); MxCommandReply reply = replyEnvelope.WorkerCommandReply.Reply; Assert.Equal(MxCommandKind.GetWorkerInfo, reply.Kind); Assert.Equal(ProtocolStatusCode.Ok, reply.ProtocolStatus.Code); WorkerInfoReply info = reply.WorkerInfo; Assert.Equal(1234, info.WorkerProcessId); Assert.False(string.IsNullOrEmpty(info.WorkerVersion)); Assert.Equal(MxAccessInteropInfo.ProgId, info.MxaccessProgid); Assert.Equal(MxAccessInteropInfo.Clsid, info.MxaccessClsid); await SendShutdownAndWaitAsync(pipePair, runTask, cancellation.Token); } /// /// Verifies that DrainEvents drains the runtime session's queued events /// into the reply rather than streaming them as WorkerEvent envelopes. /// /// A task that represents the asynchronous operation. [Fact] public async Task RunAsync_DrainEventsControlCommand_ReturnsQueuedEvents() { using CancellationTokenSource cancellation = new(TimeSpan.FromSeconds(5)); using PipePair pipePair = await PipePair.CreateAsync(cancellation.Token); // Suppress the background drain loop's fixed-batch drains so the // queued events survive for the explicit DrainEvents command (which // drains all via max_events == 0). 128 mirrors // WorkerPipeSession.EventDrainBatchSize. FakeRuntimeSession runtime = new() { SuppressDrainForBatchSize = 128 }; WorkerPipeSession session = CreatePipeSession(pipePair.WorkerStream, runtime); runtime.EnqueueEvent(CreateWorkerEvent(sequence: 11)); runtime.EnqueueEvent(CreateWorkerEvent(sequence: 12)); Task runTask = session.RunAsync(cancellation.Token); await CompleteGatewayHandshakeAsync(pipePair, cancellation.Token); await pipePair.GatewayWriter .WriteAsync( CreateControlCommandEnvelope( "drain-1", MxCommandKind.DrainEvents, command => command.DrainEvents = new DrainEventsCommand { MaxEvents = 0 }), cancellation.Token); WorkerEnvelope replyEnvelope = await ReadUntilAsync( pipePair.GatewayReader, WorkerEnvelope.BodyOneofCase.WorkerCommandReply, cancellation.Token); MxCommandReply reply = replyEnvelope.WorkerCommandReply.Reply; Assert.Equal(MxCommandKind.DrainEvents, reply.Kind); Assert.Equal(ProtocolStatusCode.Ok, reply.ProtocolStatus.Code); Assert.Equal(2, reply.DrainEvents.Events.Count); Assert.Contains(reply.DrainEvents.Events, e => e.WorkerSequence == 11UL); Assert.Contains(reply.DrainEvents.Events, e => e.WorkerSequence == 12UL); await SendShutdownAndWaitAsync(pipePair, runTask, cancellation.Token); } /// /// Verifies that a DrainEvents control command with max_events = 0 is bounded by the /// worker rather than draining the entire queue into one reply frame: the session passes a /// capped, non-zero maximum to the runtime session. /// /// A task that represents the asynchronous operation. [Fact] public async Task RunAsync_DrainEventsWithZeroMaxEvents_BoundsTheDrain() { using CancellationTokenSource cancellation = new(TimeSpan.FromSeconds(5)); using PipePair pipePair = await PipePair.CreateAsync(cancellation.Token); FakeRuntimeSession runtime = new() { SuppressDrainForBatchSize = 128 }; WorkerPipeSession session = CreatePipeSession(pipePair.WorkerStream, runtime); runtime.EnqueueEvent(CreateWorkerEvent(sequence: 11)); Task runTask = session.RunAsync(cancellation.Token); await CompleteGatewayHandshakeAsync(pipePair, cancellation.Token); await pipePair.GatewayWriter .WriteAsync( CreateControlCommandEnvelope( "drain-cap-1", MxCommandKind.DrainEvents, command => command.DrainEvents = new DrainEventsCommand { MaxEvents = 0 }), cancellation.Token); await ReadUntilAsync( pipePair.GatewayReader, WorkerEnvelope.BodyOneofCase.WorkerCommandReply, cancellation.Token); // The client asked for "all" (0) but the worker must pass a bounded, non-zero cap so the reply // frame cannot grow without limit. Assert.NotNull(runtime.LastDrainMaxEvents); Assert.NotEqual(0u, runtime.LastDrainMaxEvents!.Value); await SendShutdownAndWaitAsync(pipePair, runTask, cancellation.Token); } /// /// The WRK-21 repro. A queue full of byte-heavy events (large string values — the payload /// profile this gateway exists for) used to make DrainEvents max_events = 0 build a /// reply above the negotiated frame maximum: the writer rejected the frame, the exception /// unwound the session, and the already-dequeued events were gone. The drain is now /// byte-budgeted, so the reply fits, the truncation is reported in the reply's diagnostic /// message, and the session keeps serving. /// /// A task that represents the asynchronous operation. [Fact] public async Task DrainEvents_ByteHeavyQueue_ReplyIsBoundedAndSessionSurvives() { using CancellationTokenSource cancellation = new(TimeSpan.FromSeconds(60)); using PipePair pipePair = await PipePair.CreateAsync(cancellation.Token); FakeRuntimeSession runtime = new() { SuppressDrainForBatchSize = 128, BackingQueue = CreateByteHeavyQueue(ByteHeavyEventCount, ByteHeavyEventPayloadBytes), }; WorkerPipeSession session = CreatePipeSession(pipePair.WorkerStream, runtime); Task runTask = session.RunAsync(cancellation.Token); await CompleteGatewayHandshakeAsync(pipePair, NegotiatedMaxFrameBytes, cancellation.Token); await pipePair.GatewayWriter .WriteAsync( CreateControlCommandEnvelope( "drain-heavy-1", MxCommandKind.DrainEvents, command => command.DrainEvents = new DrainEventsCommand { MaxEvents = 0 }), cancellation.Token); WorkerEnvelope replyEnvelope = await ReadUntilAsync( pipePair.GatewayReader, WorkerEnvelope.BodyOneofCase.WorkerCommandReply, cancellation.Token); MxCommandReply reply = replyEnvelope.WorkerCommandReply.Reply; Assert.Equal(ProtocolStatusCode.Ok, reply.ProtocolStatus.Code); // The whole queue is far larger than one frame, so the reply is a strict subset that fits. Assert.True( replyEnvelope.CalculateSize() <= NegotiatedMaxFrameBytes, $"DrainEvents reply serialized to {replyEnvelope.CalculateSize()} bytes, above the negotiated {NegotiatedMaxFrameBytes}."); Assert.InRange(reply.DrainEvents.Events.Count, 1, ByteHeavyEventCount - 1); Assert.Contains("remain", reply.DiagnosticMessage); Assert.Contains("repeat DrainEvents", reply.DiagnosticMessage); // The session is alive: it still answers a ping, and RunAsync has not unwound. await pipePair.GatewayWriter .WriteAsync(CreatePingCommandEnvelope("ping-after-drain", "still-here"), cancellation.Token); WorkerEnvelope pingReply = await ReadUntilAsync( pipePair.GatewayReader, WorkerEnvelope.BodyOneofCase.WorkerCommandReply, envelope => envelope.WorkerCommandReply.Reply.CorrelationId == "ping-after-drain", cancellation.Token); Assert.Equal("still-here", pingReply.WorkerCommandReply.Reply.DiagnosticMessage); Assert.False(runTask.IsCompleted, "The session must survive a byte-heavy DrainEvents."); await SendShutdownAndWaitAsync(pipePair, runTask, cancellation.Token); } /// /// Verifies the byte-budgeted drain loses nothing: repeating DrainEvents until it comes back /// empty recovers every enqueued event exactly once, in order, across the split replies. The /// pre-fix drain removed events from the queue before sizing the reply, so a rejected frame /// destroyed them — no-loss is the half of the P0 criterion a catch-only fix cannot deliver. /// /// A task that represents the asynchronous operation. [Fact] public async Task DrainEvents_RepeatedCalls_RecoverAllEventsWithoutLoss() { using CancellationTokenSource cancellation = new(TimeSpan.FromSeconds(90)); using PipePair pipePair = await PipePair.CreateAsync(cancellation.Token); FakeRuntimeSession runtime = new() { SuppressDrainForBatchSize = 128, BackingQueue = CreateByteHeavyQueue(RepeatedDrainEventCount, ByteHeavyEventPayloadBytes), }; WorkerPipeSession session = CreatePipeSession(pipePair.WorkerStream, runtime); Task runTask = session.RunAsync(cancellation.Token); await CompleteGatewayHandshakeAsync(pipePair, NegotiatedMaxFrameBytes, cancellation.Token); List recovered = new(); int replyCount = 0; while (true) { string correlationId = $"drain-loop-{replyCount}"; await pipePair.GatewayWriter .WriteAsync( CreateControlCommandEnvelope( correlationId, MxCommandKind.DrainEvents, command => command.DrainEvents = new DrainEventsCommand { MaxEvents = 0 }), cancellation.Token); WorkerEnvelope replyEnvelope = await ReadUntilAsync( pipePair.GatewayReader, WorkerEnvelope.BodyOneofCase.WorkerCommandReply, envelope => envelope.WorkerCommandReply.Reply.CorrelationId == correlationId, cancellation.Token); replyCount++; MxCommandReply reply = replyEnvelope.WorkerCommandReply.Reply; Assert.True( replyEnvelope.CalculateSize() <= NegotiatedMaxFrameBytes, $"DrainEvents reply {replyCount} serialized to {replyEnvelope.CalculateSize()} bytes."); if (reply.DrainEvents.Events.Count == 0) { break; } foreach (MxEvent drained in reply.DrainEvents.Events) { recovered.Add(drained.WorkerSequence); } Assert.True(replyCount < 200, "DrainEvents made no progress across 200 replies."); } // More than one reply proves the drain really split; every event came back exactly once, in // enqueue order. Assert.True(replyCount > 2, $"Expected the byte cap to split the drain, saw {replyCount} replies."); Assert.Equal(RepeatedDrainEventCount, recovered.Count); for (int index = 0; index < recovered.Count; index++) { Assert.Equal((ulong)(index + 1), recovered[index]); } await SendShutdownAndWaitAsync(pipePair, runTask, cancellation.Token); } /// /// Regression for the reserve-boundary budget bug. The gateway validator accepts a Worker /// frame maximum as low as 1024 + 64 KiB, and just above that boundary a naive /// subtract-then-guard budget collapses to ~1024 bytes — too small to move even one /// byte-heavy event, so every drain reports truncation with the same head blocked and the /// drain-until-empty loop never terminates. The budget is now a floor (never below half the /// negotiated maximum), so a byte-heavy queue drains to empty even at the validator floor. /// /// A task that represents the asynchronous operation. [Fact] public async Task DrainEvents_AtValidatorFloorFrameMax_MakesProgressAndTerminates() { // The lowest Worker.MaxMessageBytes GatewayOptionsValidator permits: the public gRPC floor // (1024) plus the 64 KiB envelope-overhead reserve. The naive budget would be exactly 1024 // here; the floored budget is half of the frame max (~33 KiB). const uint validatorFloorFrameMax = 1024 + (64 * 1024); using CancellationTokenSource cancellation = new(TimeSpan.FromSeconds(60)); using PipePair pipePair = await PipePair.CreateAsync(cancellation.Token); FakeRuntimeSession runtime = new() { SuppressDrainForBatchSize = 128, BackingQueue = CreateByteHeavyQueue(200, ByteHeavyEventPayloadBytes), }; WorkerPipeSession session = CreatePipeSession(pipePair.WorkerStream, runtime); Task runTask = session.RunAsync(cancellation.Token); await CompleteGatewayHandshakeAsync(pipePair, validatorFloorFrameMax, cancellation.Token); int recovered = 0; int replyCount = 0; while (true) { string correlationId = $"floor-drain-{replyCount}"; await pipePair.GatewayWriter .WriteAsync( CreateControlCommandEnvelope( correlationId, MxCommandKind.DrainEvents, command => command.DrainEvents = new DrainEventsCommand { MaxEvents = 0 }), cancellation.Token); WorkerEnvelope replyEnvelope = await ReadUntilAsync( pipePair.GatewayReader, WorkerEnvelope.BodyOneofCase.WorkerCommandReply, envelope => envelope.WorkerCommandReply.Reply.CorrelationId == correlationId, cancellation.Token); replyCount++; MxCommandReply reply = replyEnvelope.WorkerCommandReply.Reply; Assert.True( replyEnvelope.CalculateSize() <= validatorFloorFrameMax, $"reply {replyCount} serialized to {replyEnvelope.CalculateSize()} bytes."); int drainedThisReply = reply.DrainEvents.Events.Count; if (drainedThisReply == 0) { break; } // The head is never reported as oversized at this frame max: the ~33 KiB floored budget // comfortably fits the ~1.8 KiB events, so each reply makes real progress. Assert.DoesNotContain("alone exceeds", reply.DiagnosticMessage); recovered += drainedThisReply; Assert.True(replyCount < 200, "DrainEvents made no progress at the validator floor frame max."); } Assert.Equal(200, recovered); await SendShutdownAndWaitAsync(pipePair, runTask, cancellation.Token); } /// /// Verifies the control-reply write seam is not session-fatal on size (WRK-21 backstop). The /// reply builders size their payloads, so this path needs a deliberately budget-blind drain /// to reach — but that is the point: a future command or a sizing bug must degrade to an /// error reply for that correlation, never to a dead session. /// /// A task that represents the asynchronous operation. [Fact] public async Task ControlReplyTooLarge_WritesErrorReplyInsteadOfDying() { const uint tinyMaxFrameBytes = 4096; using CancellationTokenSource cancellation = new(TimeSpan.FromSeconds(15)); using PipePair pipePair = await PipePair.CreateAsync(cancellation.Token); FakeRuntimeSession runtime = new() { SuppressDrainForBatchSize = 128, BackingQueue = CreateByteHeavyQueue(eventCount: 1, payloadBytes: 16 * 1024), IgnoreDrainByteBudget = true, }; WorkerPipeSession session = CreatePipeSession(pipePair.WorkerStream, runtime); Task runTask = session.RunAsync(cancellation.Token); await CompleteGatewayHandshakeAsync(pipePair, tinyMaxFrameBytes, cancellation.Token); await pipePair.GatewayWriter .WriteAsync( CreateControlCommandEnvelope( "drain-too-large", MxCommandKind.DrainEvents, command => command.DrainEvents = new DrainEventsCommand { MaxEvents = 0 }), cancellation.Token); WorkerEnvelope replyEnvelope = await ReadUntilAsync( pipePair.GatewayReader, WorkerEnvelope.BodyOneofCase.WorkerCommandReply, cancellation.Token); MxCommandReply reply = replyEnvelope.WorkerCommandReply.Reply; Assert.Equal("drain-too-large", reply.CorrelationId); Assert.Equal(MxCommandKind.DrainEvents, reply.Kind); Assert.Equal(ProtocolStatusCode.InvalidRequest, reply.ProtocolStatus.Code); Assert.Contains("frame maximum", reply.ProtocolStatus.Message); Assert.False(runTask.IsCompleted, "An oversized control reply must not end the session."); await SendShutdownAndWaitAsync(pipePair, runTask, cancellation.Token); } /// /// Verifies the same backstop on the STA command path: an oversized reply from a dispatched /// command answers its correlation with an error reply instead of falling into the generic /// catch that faults the whole session with MxaccessCommandFailed. /// /// A task that represents the asynchronous operation. [Fact] public async Task CommandReplyTooLarge_WritesErrorReplyInsteadOfFaulting() { const uint tinyMaxFrameBytes = 4096; using CancellationTokenSource cancellation = new(TimeSpan.FromSeconds(15)); using PipePair pipePair = await PipePair.CreateAsync(cancellation.Token); FakeRuntimeSession runtime = new() { DispatchReplyDiagnosticMessage = new string('x', 16 * 1024), }; WorkerPipeSession session = CreatePipeSession(pipePair.WorkerStream, runtime); Task runTask = session.RunAsync(cancellation.Token); await CompleteGatewayHandshakeAsync(pipePair, tinyMaxFrameBytes, cancellation.Token); await pipePair.GatewayWriter .WriteAsync(CreateCommandEnvelope("command-too-large"), cancellation.Token); WorkerEnvelope replyEnvelope = await ReadUntilAsync( pipePair.GatewayReader, WorkerEnvelope.BodyOneofCase.WorkerCommandReply, envelope => envelope.WorkerCommandReply.Reply.CorrelationId == "command-too-large", cancellation.Token); MxCommandReply reply = replyEnvelope.WorkerCommandReply.Reply; Assert.Equal(MxCommandKind.Register, reply.Kind); Assert.Equal(ProtocolStatusCode.InvalidRequest, reply.ProtocolStatus.Code); // No fault, and the session still reports itself Ready rather than Faulted. await pipePair.GatewayWriter .WriteAsync( CreateControlCommandEnvelope( "state-after-too-large", MxCommandKind.GetSessionState, command => command.GetSessionState = new GetSessionStateCommand()), cancellation.Token); WorkerEnvelope stateEnvelope = await ReadUntilAsync( pipePair.GatewayReader, WorkerEnvelope.BodyOneofCase.WorkerCommandReply, envelope => envelope.WorkerCommandReply.Reply.CorrelationId == "state-after-too-large", cancellation.Token); Assert.Equal(SessionState.Ready, stateEnvelope.WorkerCommandReply.Reply.SessionState.State); Assert.False(runTask.IsCompleted, "An oversized command reply must not fault the session."); await SendShutdownAndWaitAsync(pipePair, runTask, cancellation.Token); } /// /// IPC-30. An event above the negotiated frame maximum is undeliverable end to end (the pipe /// maximum sits only an envelope reserve above the public gRPC cap), so the session stays /// fatal by design — but the death must be structured: a WorkerFault naming the event, with /// no value payload in it, before the process exits. Silently dropping the event or /// synthesizing a placeholder were both rejected. /// /// A task that represents the asynchronous operation. [Fact] public async Task RunAsync_EventFrameTooLarge_WritesStructuredFaultThenEndsSession() { const uint tinyMaxFrameBytes = 4096; using CancellationTokenSource cancellation = new(TimeSpan.FromSeconds(15)); using PipePair pipePair = await PipePair.CreateAsync(cancellation.Token); FakeRuntimeSession runtime = new(); RecordingWorkerLogger logger = new(); WorkerPipeSession session = CreatePipeSession( pipePair.WorkerStream, runtime, new WorkerPipeSessionOptions { HeartbeatInterval = TimeSpan.FromMilliseconds(100), HeartbeatGrace = TimeSpan.FromSeconds(5), }, logger); runtime.EnqueueEvent(CreateOversizedWorkerEvent(sequence: 77, payloadBytes: 16 * 1024)); Task runTask = session.RunAsync(cancellation.Token); await CompleteGatewayHandshakeAsync(pipePair, tinyMaxFrameBytes, cancellation.Token); WorkerEnvelope faultEnvelope = await ReadUntilAsync( pipePair.GatewayReader, WorkerEnvelope.BodyOneofCase.WorkerFault, cancellation.Token); WorkerFault fault = faultEnvelope.WorkerFault; Assert.Equal(WorkerFaultCategory.ProtocolViolation, fault.Category); Assert.Equal("EventDrain", fault.CommandMethod); Assert.Contains("77", fault.DiagnosticMessage); Assert.Contains("MaxMessageBytes", fault.DiagnosticMessage); // The identity is reported; the value payload never is. Assert.DoesNotContain(new string('x', 64), fault.DiagnosticMessage); Assert.Contains( logger.Events, entry => entry.EventName == "WorkerEventFrameTooLarge" && entry.Fields.TryGetValue("worker_sequence", out object? sequence) && sequence is ulong sequenceValue && sequenceValue == 77UL); // The session ends, and the fault frame parsed cleanly off the same stream above — the // rejected event never corrupted the wire. Task completedTask = await Task.WhenAny(runTask, Task.Delay(TimeSpan.FromSeconds(5), cancellation.Token)); Assert.Same(runTask, completedTask); await Assert.ThrowsAsync(async () => await runTask); } /// /// Verifies that ShutdownWorker returns its OK reply BEFORE the graceful /// shutdown runs and disposes the runtime session, and that the message /// loop then stops. /// /// A task that represents the asynchronous operation. [Fact] public async Task RunAsync_ShutdownWorkerControlCommand_RepliesOkThenShutsDown() { using CancellationTokenSource cancellation = new(TimeSpan.FromSeconds(5)); using PipePair pipePair = await PipePair.CreateAsync(cancellation.Token); FakeRuntimeSession runtime = new(); WorkerPipeSession session = CreatePipeSession(pipePair.WorkerStream, runtime); Task runTask = session.RunAsync(cancellation.Token); await CompleteGatewayHandshakeAsync(pipePair, cancellation.Token); await pipePair.GatewayWriter .WriteAsync( CreateControlCommandEnvelope( "shutdown-1", MxCommandKind.ShutdownWorker, command => command.ShutdownWorker = new ShutdownWorkerCommand { GracePeriod = Duration.FromTimeSpan(TimeSpan.FromSeconds(1)), }), cancellation.Token); WorkerEnvelope replyEnvelope = await ReadUntilAsync( pipePair.GatewayReader, WorkerEnvelope.BodyOneofCase.WorkerCommandReply, cancellation.Token); MxCommandReply reply = replyEnvelope.WorkerCommandReply.Reply; Assert.Equal("shutdown-1", reply.CorrelationId); Assert.Equal(MxCommandKind.ShutdownWorker, reply.Kind); Assert.Equal(ProtocolStatusCode.Ok, reply.ProtocolStatus.Code); // The OK reply is followed by a shutdown ack, then the loop stops and // the runtime session is disposed. WorkerEnvelope ack = await ReadUntilAsync( pipePair.GatewayReader, WorkerEnvelope.BodyOneofCase.WorkerShutdownAck, cancellation.Token); Assert.Equal(ProtocolStatusCode.Ok, ack.WorkerShutdownAck.Status.Code); Task completedTask = await Task .WhenAny(runTask, Task.Delay(TimeSpan.FromSeconds(5), cancellation.Token)); Assert.Same(runTask, completedTask); await runTask; Assert.True(runtime.Disposed, "ShutdownWorker must dispose the runtime session."); } /// /// Verifies that stale STA activity with no command in flight triggers /// the watchdog StaHung fault. The watchdog skips the fault while a /// command is in flight (the worker is busy executing it, not hung), /// so this test deliberately leaves the current-command correlation /// id empty to assert the genuine-hung path still fires. /// /// A task that represents the asynchronous operation. [Fact] public async Task RunAsync_WhenStaActivityIsStale_WritesWatchdogFault() { using CancellationTokenSource cancellation = new(TimeSpan.FromSeconds(5)); using PipePair pipePair = await PipePair.CreateAsync(cancellation.Token); FakeRuntimeSession runtime = new(); runtime.SetSnapshot(new WorkerRuntimeHeartbeatSnapshot( DateTimeOffset.UtcNow - TimeSpan.FromSeconds(5), pendingCommandCount: 0, outboundEventQueueDepth: 0, lastEventSequence: 0, currentCommandCorrelationId: string.Empty)); WorkerPipeSession session = CreatePipeSession( pipePair.WorkerStream, runtime, new WorkerPipeSessionOptions { HeartbeatInterval = TimeSpan.FromMilliseconds(20), HeartbeatGrace = TimeSpan.FromMilliseconds(50), }); Task runTask = session.RunAsync(cancellation.Token); await CompleteGatewayHandshakeAsync(pipePair, cancellation.Token); WorkerEnvelope fault = await ReadUntilAsync( pipePair.GatewayReader, WorkerEnvelope.BodyOneofCase.WorkerFault, cancellation.Token); Assert.Equal(WorkerFaultCategory.StaHung, fault.WorkerFault.Category); Assert.Contains("STA activity is stale", fault.WorkerFault.DiagnosticMessage); await SendShutdownAndWaitAsync(pipePair, runTask, cancellation.Token); } /// /// While a command is in flight (snapshot's current command /// correlation id is non-empty), stale STA activity must NOT trigger /// the watchdog StaHung fault. The STA is busy executing the command, /// not hung; StaRuntime.ProcessQueuedCommands only calls /// MarkActivity() before and after each work item, so a /// synchronously long-running command (e.g. ReadBulk waiting /// timeout_ms for OnDataChange) legitimately freezes /// LastActivityUtc. The heartbeat already advertises the /// in-flight correlation id so the gateway can apply its own per-command /// timeout. /// /// A task that represents the asynchronous operation. [Fact] public async Task RunAsync_WhenStaActivityIsStaleWithCommandInFlight_DoesNotWriteWatchdogFault() { using CancellationTokenSource cancellation = new(TimeSpan.FromSeconds(10)); using PipePair pipePair = await PipePair.CreateAsync(cancellation.Token); FakeRuntimeSession runtime = new(); runtime.SetSnapshot(new WorkerRuntimeHeartbeatSnapshot( DateTimeOffset.UtcNow - TimeSpan.FromSeconds(5), pendingCommandCount: 0, outboundEventQueueDepth: 0, lastEventSequence: 0, currentCommandCorrelationId: "slow-bulk-read")); WorkerPipeSession session = CreatePipeSession( pipePair.WorkerStream, runtime, new WorkerPipeSessionOptions { HeartbeatInterval = TimeSpan.FromMilliseconds(20), HeartbeatGrace = TimeSpan.FromMilliseconds(50), }); Task runTask = session.RunAsync(cancellation.Token); await CompleteGatewayHandshakeAsync(pipePair, cancellation.Token); // Read several frames over a window much larger than HeartbeatGrace. // None must be a WorkerFault; multiple heartbeats must all carry the // in-flight correlation id. Reading a bounded count of frames keeps // the pipe frame-aligned for the subsequent shutdown handshake. const int framesToInspect = 6; int heartbeatsObserved = 0; for (int index = 0; index < framesToInspect; index++) { WorkerEnvelope envelope = await pipePair.GatewayReader .ReadAsync(cancellation.Token); Assert.NotEqual( WorkerEnvelope.BodyOneofCase.WorkerFault, envelope.BodyCase); if (envelope.BodyCase == WorkerEnvelope.BodyOneofCase.WorkerHeartbeat) { Assert.Equal( "slow-bulk-read", envelope.WorkerHeartbeat.CurrentCommandCorrelationId); heartbeatsObserved++; } } Assert.True( heartbeatsObserved >= 2, $"Expected multiple heartbeats during in-flight command window; observed {heartbeatsObserved}."); await SendShutdownAndWaitAsync(pipePair, runTask, cancellation.Token); } /// /// Once the watchdog reports an StaHung fault, subsequent heartbeats /// must report rather than a /// non-faulted state that contradicts the fault. The snapshot uses an /// empty current-command correlation id so the heartbeat State is /// derived from the session state, not forced to ExecutingCommand. /// /// A task that represents the asynchronous operation. [Fact] public async Task RunAsync_AfterWatchdogFault_HeartbeatReportsFaultedState() { using CancellationTokenSource cancellation = new(TimeSpan.FromSeconds(10)); using PipePair pipePair = await PipePair.CreateAsync(cancellation.Token); FakeRuntimeSession runtime = new(); runtime.SetSnapshot(new WorkerRuntimeHeartbeatSnapshot( DateTimeOffset.UtcNow - TimeSpan.FromSeconds(5), pendingCommandCount: 0, outboundEventQueueDepth: 0, lastEventSequence: 0, currentCommandCorrelationId: string.Empty)); WorkerPipeSession session = CreatePipeSession( pipePair.WorkerStream, runtime, new WorkerPipeSessionOptions { HeartbeatInterval = TimeSpan.FromMilliseconds(20), HeartbeatGrace = TimeSpan.FromMilliseconds(50), }); Task runTask = session.RunAsync(cancellation.Token); await CompleteGatewayHandshakeAsync(pipePair, cancellation.Token); WorkerEnvelope fault = await ReadUntilAsync( pipePair.GatewayReader, WorkerEnvelope.BodyOneofCase.WorkerFault, cancellation.Token); Assert.Equal(WorkerFaultCategory.StaHung, fault.WorkerFault.Category); // The next heartbeat after the fault must agree with it. WorkerEnvelope heartbeat = await ReadUntilAsync( pipePair.GatewayReader, WorkerEnvelope.BodyOneofCase.WorkerHeartbeat, cancellation.Token); Assert.Equal(WorkerState.Faulted, heartbeat.WorkerHeartbeat.State); await SendShutdownAndWaitAsync(pipePair, runTask, cancellation.Token); } /// /// The in-flight-command suppression on the StaHung watchdog is /// bounded by WorkerPipeSessionOptions.HeartbeatStuckCeiling. A /// truly stuck synchronous STA command (e.g. a dead MXAccess provider) /// would otherwise keep CurrentCommandCorrelationId non-empty /// forever and permanently defeat the watchdog. Once /// LastStaActivityUtc has been stale for longer than /// HeartbeatStuckCeiling the watchdog DOES fire StaHung /// even with a command in flight. /// /// A task that represents the asynchronous operation. [Fact] public async Task RunAsync_WhenStaActivityIsStaleBeyondCeilingWithCommandInFlight_WritesWatchdogFault() { using CancellationTokenSource cancellation = new(TimeSpan.FromSeconds(5)); using PipePair pipePair = await PipePair.CreateAsync(cancellation.Token); FakeRuntimeSession runtime = new(); // Stale by 5s, which exceeds the configured 200 ms ceiling — the // watchdog must fire even with a command in flight. runtime.SetSnapshot(new WorkerRuntimeHeartbeatSnapshot( DateTimeOffset.UtcNow - TimeSpan.FromSeconds(5), pendingCommandCount: 0, outboundEventQueueDepth: 0, lastEventSequence: 0, currentCommandCorrelationId: "stuck-command")); WorkerPipeSession session = CreatePipeSession( pipePair.WorkerStream, runtime, new WorkerPipeSessionOptions { HeartbeatInterval = TimeSpan.FromMilliseconds(20), HeartbeatGrace = TimeSpan.FromMilliseconds(50), HeartbeatStuckCeiling = TimeSpan.FromMilliseconds(200), }); Task runTask = session.RunAsync(cancellation.Token); await CompleteGatewayHandshakeAsync(pipePair, cancellation.Token); WorkerEnvelope fault = await ReadUntilAsync( pipePair.GatewayReader, WorkerEnvelope.BodyOneofCase.WorkerFault, cancellation.Token); Assert.Equal(WorkerFaultCategory.StaHung, fault.WorkerFault.Category); Assert.Contains("STA activity is stale", fault.WorkerFault.DiagnosticMessage); await SendShutdownAndWaitAsync(pipePair, runTask, cancellation.Token); } /// /// WRK-27. An STA call outside the command dispatcher (the alarm poll) advertises itself on /// the heartbeat snapshot's StaCallInProgress flag, and the watchdog grants it the same /// grace-to-ceiling suppression as a dispatched command: stale STA activity within the ceiling /// does not fault while the flag is set, but stale activity beyond the ceiling faults anyway. /// This closes the 15 s-vs-75 s asymmetry between polls and commands. /// /// A task that represents the asynchronous operation. [Fact] public async Task Watchdog_StaCallInProgress_SuppressedUntilCeiling() { using CancellationTokenSource cancellation = new(TimeSpan.FromSeconds(10)); // Phase 1 — within the ceiling: stale beyond grace, empty correlation id, StaCallInProgress // set. The default 75 s ceiling is far beyond the 5 s staleness, so the watchdog must suppress. using (PipePair pipePair = await PipePair.CreateAsync(cancellation.Token)) { FakeRuntimeSession runtime = new(); runtime.SetSnapshot(new WorkerRuntimeHeartbeatSnapshot( DateTimeOffset.UtcNow - TimeSpan.FromSeconds(5), pendingCommandCount: 0, outboundEventQueueDepth: 0, lastEventSequence: 0, currentCommandCorrelationId: string.Empty, staCallInProgress: true)); WorkerPipeSession session = CreatePipeSession( pipePair.WorkerStream, runtime, new WorkerPipeSessionOptions { HeartbeatInterval = TimeSpan.FromMilliseconds(20), HeartbeatGrace = TimeSpan.FromMilliseconds(50), }); Task runTask = session.RunAsync(cancellation.Token); await CompleteGatewayHandshakeAsync(pipePair, cancellation.Token); const int framesToInspect = 6; int heartbeatsObserved = 0; for (int index = 0; index < framesToInspect; index++) { WorkerEnvelope envelope = await pipePair.GatewayReader.ReadAsync(cancellation.Token); Assert.NotEqual(WorkerEnvelope.BodyOneofCase.WorkerFault, envelope.BodyCase); if (envelope.BodyCase == WorkerEnvelope.BodyOneofCase.WorkerHeartbeat) { heartbeatsObserved++; } } Assert.True( heartbeatsObserved >= 2, $"Expected multiple heartbeats during the in-progress STA-call window; observed {heartbeatsObserved}."); await SendShutdownAndWaitAsync(pipePair, runTask, cancellation.Token); } // Phase 2 — beyond the ceiling: same StaCallInProgress flag, but staleness (5 s) exceeds the // 200 ms ceiling, so the watchdog must fire even with the poll in progress. using (PipePair pipePair = await PipePair.CreateAsync(cancellation.Token)) { FakeRuntimeSession runtime = new(); runtime.SetSnapshot(new WorkerRuntimeHeartbeatSnapshot( DateTimeOffset.UtcNow - TimeSpan.FromSeconds(5), pendingCommandCount: 0, outboundEventQueueDepth: 0, lastEventSequence: 0, currentCommandCorrelationId: string.Empty, staCallInProgress: true)); WorkerPipeSession session = CreatePipeSession( pipePair.WorkerStream, runtime, new WorkerPipeSessionOptions { HeartbeatInterval = TimeSpan.FromMilliseconds(20), HeartbeatGrace = TimeSpan.FromMilliseconds(50), HeartbeatStuckCeiling = TimeSpan.FromMilliseconds(200), }); Task runTask = session.RunAsync(cancellation.Token); await CompleteGatewayHandshakeAsync(pipePair, cancellation.Token); WorkerEnvelope fault = await ReadUntilAsync( pipePair.GatewayReader, WorkerEnvelope.BodyOneofCase.WorkerFault, cancellation.Token); Assert.Equal(WorkerFaultCategory.StaHung, fault.WorkerFault.Category); await SendShutdownAndWaitAsync(pipePair, runTask, cancellation.Token); } } /// /// WRK-25. The event drain loop submits a whole drained batch through the writer's batch entry /// point, so a burst of 128 events costs one flush, not 128 — the assertion the WRK-12 /// tracking claim needed to actually hold on the event hot path. /// /// A task that represents the asynchronous operation. [Fact] public async Task EventBurst_DrainLoopCoalescesFlushes() { using CancellationTokenSource cancellation = new(TimeSpan.FromSeconds(15)); using PipePair pipePair = await PipePair.CreateAsync(cancellation.Token); FakeRuntimeSession runtime = new(); // A far-off heartbeat interval keeps every heartbeat after the first out of the measurement // window; the first beat is sent immediately on entering the message loop (see // RunHeartbeatLoopAsync) and is closed out explicitly below. FlushCountingPassthroughStream countingStream = new(pipePair.WorkerStream); WorkerPipeSession session = CreatePipeSession( countingStream, runtime, new WorkerPipeSessionOptions { HeartbeatInterval = TimeSpan.FromMinutes(5), HeartbeatGrace = TimeSpan.FromSeconds(30), }); Task runTask = session.RunAsync(cancellation.Token); await CompleteGatewayHandshakeAsync(pipePair, cancellation.Token); // Close the pre-burst window on an explicit signal rather than a sleep. Three frames precede // the burst — WorkerHello, WorkerReady, and the immediate first heartbeat — and each is // flushed only after its bytes are already on the pipe, so having read a frame is no evidence // that its flush has been counted. Read the first beat (the last pre-burst frame), then wait // for the writer's deferred flush before sampling the baseline. A fixed delay left the first // beat's flush free to land after the sample under CI scheduling pressure, where it was // charged to the burst and the assertion below saw two flushes instead of one. await ReadUntilAsync( pipePair.GatewayReader, WorkerEnvelope.BodyOneofCase.WorkerHeartbeat, cancellation.Token); await countingStream.WaitForAllWritesFlushedAsync(cancellation.Token); int baselineFlushes = countingStream.FlushCount; // Enqueue a full 128-event batch atomically so the drain loop sees it as one batch. const int burst = 128; List batch = new(burst); for (int index = 0; index < burst; index++) { batch.Add(CreateWorkerEvent(sequence: (ulong)(index + 1))); } runtime.EnqueueEvents(batch); // Drain all 128 events off the gateway side. for (int index = 0; index < burst; index++) { await ReadUntilAsync( pipePair.GatewayReader, WorkerEnvelope.BodyOneofCase.WorkerEvent, cancellation.Token); } // Take the same edge on the burst's own flush — the 128 frames are on the wire before the // writer flushes them — then assert on the recorded flush shape: exactly one flush beyond the // baseline, and that flush carried every event frame of the burst. Asserting the shape (not // just the count) is what makes the coalescing claim faithful: a split batch would show its // first flush carrying fewer than the whole burst. await countingStream.WaitForAllWritesFlushedAsync(cancellation.Token); IReadOnlyList flushWriteCounts = countingStream.SnapshotFlushWriteCounts(); Assert.Equal(baselineFlushes + 1, flushWriteCounts.Count); Assert.Equal(burst, flushWriteCounts[baselineFlushes]); await SendShutdownAndWaitAsync(pipePair, runTask, cancellation.Token); } /// /// WRK-24. A GatewayHello negotiating a frame maximum below the worker floor faults at the /// handshake with a fault frame rather than being adopted — mirroring the above-ceiling /// handshake behavior — so a nonsensical tiny value never leaves a session that fails every /// later frame. No message loop is entered. /// /// A task that represents the asynchronous operation. [Fact] public async Task Handshake_GatewayHelloWithTinyMaxFrameBytes_FaultsAtHandshake() { WorkerFrameProtocolOptions options = CreateOptions(); using MemoryStream inbound = new(); await new WorkerFrameWriter(inbound, options) .WriteAsync(CreateGatewayHelloEnvelope(maxFrameBytes: 512)); inbound.Position = 0; using MemoryStream outbound = new(); WorkerPipeSession session = CreateSession(inbound, outbound, options); bool initialized = false; WorkerFrameProtocolException exception = await Assert.ThrowsAsync( async () => await session.CompleteStartupHandshakeAsync( _ => { initialized = true; return Task.CompletedTask; })); Assert.False(initialized); Assert.Equal(WorkerFrameProtocolErrorCode.InvalidConfiguration, exception.ErrorCode); WorkerEnvelope fault = Assert.Single(ReadWrittenFrames(outbound, options)); Assert.Equal(WorkerEnvelope.BodyOneofCase.WorkerFault, fault.BodyCase); } /// /// Regression test: a long in-flight STA command that keeps pumping /// must NOT self-fault as StaHung, and its reply must still be /// delivered. The real fix makes StaRuntime.PumpPendingMessages /// refresh LastActivityUtc on every wait iteration, so a healthy /// ReadBulk holding the STA far longer than /// HeartbeatStuckCeiling (75 s in production) keeps its activity /// timestamp fresh. This test compresses the clock — a 100 ms ceiling /// with a command in flight across a window many multiples longer — and /// models the pump refresh by continuously advancing the snapshot's /// LastStaActivityUtc while the command blocks. Contrast /// , /// where a frozen timestamp beyond the ceiling correctly faults; here /// the refreshed timestamp must keep the fault suppressed and let the /// reply through the Ready-state gate. /// /// A task that represents the asynchronous operation. [Fact] public async Task RunAsync_LongInFlightCommandThatKeepsPumping_DoesNotFaultAndDeliversReply() { using CancellationTokenSource cancellation = new(TimeSpan.FromSeconds(20)); using PipePair pipePair = await PipePair.CreateAsync(cancellation.Token); FakeRuntimeSession runtime = new() { BlockDispatch = true, }; WorkerPipeSession session = CreatePipeSession( pipePair.WorkerStream, runtime, new WorkerPipeSessionOptions { HeartbeatInterval = TimeSpan.FromMilliseconds(20), HeartbeatGrace = TimeSpan.FromMilliseconds(50), HeartbeatStuckCeiling = TimeSpan.FromMilliseconds(100), }); Task runTask = session.RunAsync(cancellation.Token); await CompleteGatewayHandshakeAsync(pipePair, cancellation.Token); // Kick off the long command; it blocks in DispatchAsync until released, // so its correlation id stays in flight in the heartbeat snapshot. await pipePair.GatewayWriter .WriteAsync(CreateCommandEnvelope("long-bulk-read"), cancellation.Token); Assert.True( runtime.DispatchStarted.Wait(TimeSpan.FromSeconds(5)), "The long command must reach the runtime and begin dispatch."); // Model the pump refreshing STA activity on each wait iteration: keep // the snapshot's LastStaActivityUtc current while the command is in // flight. using CancellationTokenSource pumpRefresh = new(); Task refreshLoop = Task.Run( async () => { while (!pumpRefresh.IsCancellationRequested) { runtime.SetSnapshot(new WorkerRuntimeHeartbeatSnapshot( DateTimeOffset.UtcNow, pendingCommandCount: 1, outboundEventQueueDepth: 0, lastEventSequence: 0, currentCommandCorrelationId: "long-bulk-read")); await Task.Delay(TimeSpan.FromMilliseconds(20)).ConfigureAwait(false); } }); // Inspect a bounded number of frames over a window many multiples of the // 100 ms ceiling (at least 30 heartbeats at 20 ms ~ 600 ms). None may be // a WorkerFault while activity is continuously refreshed. const int framesToInspect = 30; for (int index = 0; index < framesToInspect; index++) { WorkerEnvelope envelope = await pipePair.GatewayReader .ReadAsync(cancellation.Token); Assert.NotEqual( WorkerEnvelope.BodyOneofCase.WorkerFault, envelope.BodyCase); } // Stop refreshing and release the command; its reply must be delivered // because the session never faulted (state stayed Ready). pumpRefresh.Cancel(); await refreshLoop; runtime.ReleaseDispatch(); WorkerEnvelope reply = await ReadUntilAsync( pipePair.GatewayReader, WorkerEnvelope.BodyOneofCase.WorkerCommandReply, envelope => envelope.CorrelationId == "long-bulk-read", cancellation.Token); Assert.Equal( ProtocolStatusCode.Ok, reply.WorkerCommandReply.Reply.ProtocolStatus.Code); await SendShutdownAndWaitAsync(pipePair, runTask, cancellation.Token); } /// /// RunAsync must throw a diagnostic exception if the /// runtime-session factory returns null, rather than deferring the /// failure to an NRE on the next dereference. /// /// A task that represents the asynchronous operation. [Fact] public async Task RunAsync_WhenRuntimeSessionFactoryReturnsNull_ThrowsDiagnosticException() { using CancellationTokenSource cancellation = new(TimeSpan.FromSeconds(5)); using PipePair pipePair = await PipePair.CreateAsync(cancellation.Token); WorkerFrameProtocolOptions options = CreateOptions(); WorkerPipeSession session = new( new WorkerFrameReader(pipePair.WorkerStream, options), new WorkerFrameWriter(pipePair.WorkerStream, options), options, () => 1234, new WorkerPipeSessionOptions(), () => null!); InvalidOperationException exception = await Assert.ThrowsAsync( () => session.RunAsync(cancellation.Token)); Assert.Contains("factory returned null", exception.Message); } /// /// When graceful shutdown times out, RunAsync must still dispose the /// runtime session in its finally block. Skipping disposal on the /// timed-out path leaked the STA thread and the MXAccess COM object. /// /// A task that represents the asynchronous operation. [Fact] public async Task RunAsync_WhenShutdownTimesOut_StillDisposesRuntimeSession() { using CancellationTokenSource cancellation = new(TimeSpan.FromSeconds(10)); using PipePair pipePair = await PipePair.CreateAsync(cancellation.Token); FakeRuntimeSession runtime = new() { ThrowTimeoutOnShutdown = true, }; WorkerPipeSession session = CreatePipeSession( pipePair.WorkerStream, runtime, new WorkerPipeSessionOptions { HeartbeatInterval = TimeSpan.FromSeconds(1), HeartbeatGrace = TimeSpan.FromSeconds(30), }); Task runTask = session.RunAsync(cancellation.Token); await CompleteGatewayHandshakeAsync(pipePair, cancellation.Token); await pipePair.GatewayWriter .WriteAsync(CreateShutdownEnvelope(), cancellation.Token); // Drain the gateway-side pipe (heartbeats + the shutdown-timeout // fault) so the worker's writes never block on a full pipe buffer. Task drainTask = DrainReaderUntilFaultedAsync(pipePair.GatewayReader, cancellation.Token); // RunAsync must rethrow the TimeoutException and still reach its // finally block, which disposes the runtime session. await Assert.ThrowsAsync(async () => await runTask); Assert.True( runtime.Disposed, "RunAsync must dispose the runtime session even when shutdown times out."); await drainTask; } private static async Task DrainReaderUntilFaultedAsync( WorkerFrameReader reader, CancellationToken cancellationToken) { try { while (!cancellationToken.IsCancellationRequested) { WorkerEnvelope envelope = await reader.ReadAsync(cancellationToken).ConfigureAwait(false); if (envelope.BodyCase == WorkerEnvelope.BodyOneofCase.WorkerFault && envelope.WorkerFault.Category == WorkerFaultCategory.ShutdownTimeout) { return; } } } catch (Exception exception) when ( exception is OperationCanceledException || exception is IOException || exception is WorkerFrameProtocolException) { // The worker pipe closed once RunAsync completed — expected. } } /// Verifies that shutdown drops late replies and sends shutdown ack. /// A task that represents the asynchronous operation. [Fact] public async Task RunAsync_WhenShutdownArrivesDuringCommand_DropsLateReplyAndWritesShutdownAck() { using CancellationTokenSource cancellation = new(TimeSpan.FromSeconds(5)); using PipePair pipePair = await PipePair.CreateAsync(cancellation.Token); FakeRuntimeSession runtime = new() { BlockDispatch = true, }; WorkerPipeSession session = CreatePipeSession( pipePair.WorkerStream, runtime, new WorkerPipeSessionOptions { HeartbeatInterval = TimeSpan.FromSeconds(1), HeartbeatGrace = TimeSpan.FromSeconds(5), }); Task runTask = session.RunAsync(cancellation.Token); await CompleteGatewayHandshakeAsync(pipePair, cancellation.Token); await pipePair.GatewayWriter.WriteAsync( CreateCommandEnvelope("command-during-shutdown"), cancellation.Token); Assert.True(runtime.DispatchStarted.Wait(TimeSpan.FromSeconds(2))); await pipePair.GatewayWriter .WriteAsync(CreateShutdownEnvelope(), cancellation.Token); WorkerEnvelope shutdownAck = await ReadUntilAsync( pipePair.GatewayReader, WorkerEnvelope.BodyOneofCase.WorkerShutdownAck, cancellation.Token); Assert.Equal(ProtocolStatusCode.Ok, shutdownAck.WorkerShutdownAck.Status.Code); Task completedTask = await Task.WhenAny(runTask, Task.Delay(TimeSpan.FromSeconds(2), cancellation.Token)); Assert.Same(runTask, completedTask); await runTask; } /// Verifies that command exceptions after shutdown are dropped before ack. /// A task that represents the asynchronous operation. [Fact] public async Task RunAsync_WhenCommandThrowsAfterShutdown_DropsLateFaultAndWritesShutdownAck() { using CancellationTokenSource cancellation = new(TimeSpan.FromSeconds(5)); using PipePair pipePair = await PipePair.CreateAsync(cancellation.Token); FakeRuntimeSession runtime = new() { BlockDispatch = true, ThrowAfterDispatchReleased = true, }; WorkerPipeSession session = CreatePipeSession( pipePair.WorkerStream, runtime, new WorkerPipeSessionOptions { HeartbeatInterval = TimeSpan.FromSeconds(1), HeartbeatGrace = TimeSpan.FromSeconds(5), }); Task runTask = session.RunAsync(cancellation.Token); await CompleteGatewayHandshakeAsync(pipePair, cancellation.Token); await pipePair.GatewayWriter.WriteAsync( CreateCommandEnvelope("command-fails-during-shutdown"), cancellation.Token); Assert.True(runtime.DispatchStarted.Wait(TimeSpan.FromSeconds(2))); await pipePair.GatewayWriter .WriteAsync(CreateShutdownEnvelope(), cancellation.Token); // The first heartbeat is emitted immediately on entering the loop, // so skip any interleaved heartbeats; the late fault must still be // dropped — no WorkerFault may precede the ack. WorkerEnvelope envelopeAfterShutdown; do { envelopeAfterShutdown = await pipePair.GatewayReader.ReadAsync(cancellation.Token); Assert.NotEqual( WorkerEnvelope.BodyOneofCase.WorkerFault, envelopeAfterShutdown.BodyCase); } while (envelopeAfterShutdown.BodyCase == WorkerEnvelope.BodyOneofCase.WorkerHeartbeat); Assert.Equal(WorkerEnvelope.BodyOneofCase.WorkerShutdownAck, envelopeAfterShutdown.BodyCase); Assert.Equal(ProtocolStatusCode.Ok, envelopeAfterShutdown.WorkerShutdownAck.Status.Code); Task completedTask = await Task.WhenAny(runTask, Task.Delay(TimeSpan.FromSeconds(2), cancellation.Token)); Assert.Same(runTask, completedTask); await runTask; } /// /// The WorkerCancel branch of /// must /// forward the envelope's correlation id to the runtime session via /// and keep the /// message loop running (no fault, no exit). The handler dispatch /// returns true (keep reading), so a subsequent /// WorkerShutdown still produces the normal shutdown ack. /// /// A task that represents the asynchronous operation. [Fact] public async Task RunAsync_WhenGatewaySendsWorkerCancel_ForwardsCorrelationIdToRuntimeSession() { using CancellationTokenSource cancellation = new(TimeSpan.FromSeconds(5)); using PipePair pipePair = await PipePair.CreateAsync(cancellation.Token); FakeRuntimeSession runtime = new(); WorkerPipeSession session = CreatePipeSession( pipePair.WorkerStream, runtime, new WorkerPipeSessionOptions { HeartbeatInterval = TimeSpan.FromSeconds(1), HeartbeatGrace = TimeSpan.FromSeconds(5), }); Task runTask = session.RunAsync(cancellation.Token); await CompleteGatewayHandshakeAsync(pipePair, cancellation.Token); await pipePair.GatewayWriter .WriteAsync(CreateCancelEnvelope("cancel-correlation-1"), cancellation.Token); // The session must remain in its message loop: send a follow-up // shutdown and observe the normal ack. If WorkerCancel had faulted // the pipe or exited the loop, the ack would never arrive. await SendShutdownAndWaitAsync(pipePair, runTask, cancellation.Token); Assert.Contains("cancel-correlation-1", runtime.CancelledCorrelationIds); } /// /// The default: arm of /// must /// throw with /// /// when the gateway sends an envelope body that is invalid /// post-handshake (here a second GatewayHello) and must exit /// the message loop — /// surfaces the exception to the caller. The message loop does not /// emit a fault frame on this path (the handshake catch in /// CompleteStartupHandshakeAsync is what writes faults for /// pre-handshake protocol violations); the contract this test pins /// is the exception type/error-code and message-loop exit. /// /// A task that represents the asynchronous operation. [Fact] public async Task RunAsync_WhenGatewaySendsUnexpectedEnvelopeBodyAfterHandshake_ThrowsAndExitsMessageLoop() { using CancellationTokenSource cancellation = new(TimeSpan.FromSeconds(10)); using PipePair pipePair = await PipePair.CreateAsync(cancellation.Token); FakeRuntimeSession runtime = new(); // Use a long heartbeat interval so no heartbeat frame fires during // the test window. With no heartbeats and no fault frame written on // the unexpected-body path, the gateway pipe receives nothing after // the handshake — no drain task is needed. WorkerPipeSession session = CreatePipeSession( pipePair.WorkerStream, runtime, new WorkerPipeSessionOptions { HeartbeatInterval = TimeSpan.FromSeconds(30), HeartbeatGrace = TimeSpan.FromSeconds(60), }); Task runTask = session.RunAsync(cancellation.Token); await CompleteGatewayHandshakeAsync(pipePair, cancellation.Token); // Send a second GatewayHello — valid envelope, invalid for the // post-handshake state, so DispatchGatewayEnvelopeAsync falls to // the default arm. await pipePair.GatewayWriter .WriteAsync(CreateGatewayHelloEnvelope(), cancellation.Token); WorkerFrameProtocolException exception = await Assert.ThrowsAsync(async () => await runTask); Assert.Equal(WorkerFrameProtocolErrorCode.UnexpectedEnvelopeBody, exception.ErrorCode); } /// /// The first heartbeat must be emitted immediately on entering the /// heartbeat loop, not after a full HeartbeatInterval. A long interval /// is configured so a delay-first loop would fail to deliver a /// heartbeat inside the assertion window. /// /// A task that represents the asynchronous operation. [Fact] public async Task RunAsync_SendsFirstHeartbeatImmediatelyOnEnteringLoop() { using CancellationTokenSource cancellation = new(TimeSpan.FromSeconds(10)); using PipePair pipePair = await PipePair.CreateAsync(cancellation.Token); FakeRuntimeSession runtime = new(); WorkerPipeSession session = CreatePipeSession( pipePair.WorkerStream, runtime, new WorkerPipeSessionOptions { // A deliberately long interval: a delay-before-first-beat // loop would not produce a heartbeat for 30s. HeartbeatInterval = TimeSpan.FromSeconds(30), HeartbeatGrace = TimeSpan.FromSeconds(60), }); Task runTask = session.RunAsync(cancellation.Token); await CompleteGatewayHandshakeAsync(pipePair, cancellation.Token); // The heartbeatWait CTS (5s cancel-after) already enforces the timing bound: // if the first heartbeat is not received within 5s, ReadUntilAsync throws // OperationCanceledException and the test fails. A redundant wall-clock // elapsed < 5s assertion would add the same class of flakiness // corrected elsewhere, so it is omitted here. using CancellationTokenSource heartbeatWait = CancellationTokenSource .CreateLinkedTokenSource(cancellation.Token); heartbeatWait.CancelAfter(TimeSpan.FromSeconds(5)); WorkerEnvelope heartbeat = await ReadUntilAsync( pipePair.GatewayReader, WorkerEnvelope.BodyOneofCase.WorkerHeartbeat, heartbeatWait.Token); Assert.Equal(WorkerEnvelope.BodyOneofCase.WorkerHeartbeat, heartbeat.BodyCase); await SendShutdownAndWaitAsync(pipePair, runTask, cancellation.Token); } /// /// When a command completes after the worker has transitioned out of /// a command-serving state, the dropped reply must be logged with a /// diagnostic rather than discarded silently, so a stuck gateway /// correlation wait can be traced. /// /// A task that represents the asynchronous operation. [Fact] public async Task RunAsync_WhenReplyIsDroppedAfterShutdown_LogsDiagnostic() { using CancellationTokenSource cancellation = new(TimeSpan.FromSeconds(10)); using PipePair pipePair = await PipePair.CreateAsync(cancellation.Token); FakeRuntimeSession runtime = new() { BlockDispatch = true, }; RecordingWorkerLogger logger = new(); WorkerFrameProtocolOptions options = CreateOptions(); WorkerPipeSession session = new( new WorkerFrameReader(pipePair.WorkerStream, options), new WorkerFrameWriter(pipePair.WorkerStream, options), options, () => 1234, new WorkerPipeSessionOptions { HeartbeatInterval = TimeSpan.FromSeconds(1), HeartbeatGrace = TimeSpan.FromSeconds(5), }, () => runtime, logger); Task runTask = session.RunAsync(cancellation.Token); await CompleteGatewayHandshakeAsync(pipePair, cancellation.Token); await pipePair.GatewayWriter.WriteAsync( CreateCommandEnvelope("command-dropped-after-shutdown"), cancellation.Token); Assert.True(runtime.DispatchStarted.Wait(TimeSpan.FromSeconds(2))); await pipePair.GatewayWriter .WriteAsync(CreateShutdownEnvelope(), cancellation.Token); WorkerEnvelope shutdownAck = await ReadUntilAsync( pipePair.GatewayReader, WorkerEnvelope.BodyOneofCase.WorkerShutdownAck, cancellation.Token); Assert.Equal(ProtocolStatusCode.Ok, shutdownAck.WorkerShutdownAck.Status.Code); Task completedTask = await Task.WhenAny(runTask, Task.Delay(TimeSpan.FromSeconds(3), cancellation.Token)); Assert.Same(runTask, completedTask); await runTask; Assert.Contains( logger.Events, entry => entry.EventName == "WorkerCommandResultDropped" && entry.Fields.TryGetValue("correlation_id", out object? correlationId) && (string?)correlationId == "command-dropped-after-shutdown"); } private static WorkerPipeSession CreateSession( Stream inbound, Stream outbound, WorkerFrameProtocolOptions options) { return new WorkerPipeSession( new WorkerFrameReader(inbound, options), new WorkerFrameWriter(outbound, options), options, () => 1234); } private static WorkerPipeSession CreatePipeSession( Stream stream, FakeRuntimeSession runtime) { return CreatePipeSession( stream, runtime, new WorkerPipeSessionOptions { HeartbeatInterval = TimeSpan.FromMilliseconds(100), HeartbeatGrace = TimeSpan.FromSeconds(5), }); } private static WorkerPipeSession CreatePipeSession( Stream stream, FakeRuntimeSession runtime, WorkerPipeSessionOptions sessionOptions) { return CreatePipeSession(stream, runtime, sessionOptions, logger: null); } private static WorkerPipeSession CreatePipeSession( Stream stream, FakeRuntimeSession runtime, WorkerPipeSessionOptions sessionOptions, ZB.MOM.WW.MxGateway.Worker.Bootstrap.IWorkerLogger? logger) { WorkerFrameProtocolOptions options = CreateOptions(); return new WorkerPipeSession( new WorkerFrameReader(stream, options), new WorkerFrameWriter(stream, options), options, () => 1234, sessionOptions, () => runtime, logger); } private static WorkerFrameProtocolOptions CreateOptions() { return new WorkerFrameProtocolOptions( SessionId, GatewayContractInfo.WorkerProtocolVersion, Nonce); } // Inbound-envelope sequence numbers below are documentation-only: the // worker has no inbound monotonicity check, so the literal values do // not affect dispatch. Each helper exposes a sequence parameter // (default = position in the typical Hello/Command/Cancel/Shutdown // ordering) so a multi-frame test that interleaves the helpers can // assign monotonically increasing values and produce a wire trace // that reads in ascending order. private static WorkerEnvelope CreateGatewayHelloEnvelope( string nonce = Nonce, uint supportedProtocolVersion = GatewayContractInfo.WorkerProtocolVersion, ulong sequence = 1, uint maxFrameBytes = 0) { return new WorkerEnvelope { ProtocolVersion = GatewayContractInfo.WorkerProtocolVersion, SessionId = SessionId, Sequence = sequence, GatewayHello = new GatewayHello { SupportedProtocolVersion = supportedProtocolVersion, Nonce = nonce, GatewayVersion = "test-gateway", // 0 leaves the worker on its compile-time default; a non-zero value is adopted // during the handshake and becomes the frame maximum every later assertion is // measured against. MaxFrameBytes = maxFrameBytes, }, }; } // A generic STA-dispatched command used by the dispatch/heartbeat/ // shutdown-race tests. Register is a real MXAccess command kind (not a // worker control command), so it flows through IWorkerRuntimeSession // .DispatchAsync — unlike Ping/GetSessionState/etc., which are answered on // the message-loop thread without touching the STA. private static WorkerEnvelope CreateCommandEnvelope(string correlationId, ulong sequence = 2) { return new WorkerEnvelope { ProtocolVersion = GatewayContractInfo.WorkerProtocolVersion, SessionId = SessionId, Sequence = sequence, CorrelationId = correlationId, WorkerCommand = new WorkerCommand { Command = new MxCommand { Kind = MxCommandKind.Register, Register = new RegisterCommand { ClientName = "test-client", }, }, EnqueueTimestamp = Timestamp.FromDateTimeOffset(DateTimeOffset.UtcNow), }, }; } private static WorkerEnvelope CreatePingCommandEnvelope( string correlationId, string message, ulong sequence = 2) { return CreateControlCommandEnvelope( correlationId, MxCommandKind.Ping, command => command.Ping = new PingCommand { Message = message }, sequence); } private static WorkerEnvelope CreateControlCommandEnvelope( string correlationId, MxCommandKind kind, Action configurePayload, ulong sequence = 2) { MxCommand command = new() { Kind = kind }; configurePayload(command); return new WorkerEnvelope { ProtocolVersion = GatewayContractInfo.WorkerProtocolVersion, SessionId = SessionId, Sequence = sequence, CorrelationId = correlationId, WorkerCommand = new WorkerCommand { Command = command, EnqueueTimestamp = Timestamp.FromDateTimeOffset(DateTimeOffset.UtcNow), }, }; } private static WorkerEnvelope CreateCancelEnvelope(string correlationId, ulong sequence = 2) { return new WorkerEnvelope { ProtocolVersion = GatewayContractInfo.WorkerProtocolVersion, SessionId = SessionId, Sequence = sequence, CorrelationId = correlationId, WorkerCancel = new WorkerCancel { Reason = "test-cancel", }, }; } private static WorkerEnvelope CreateShutdownEnvelope(ulong sequence = 3) { return new WorkerEnvelope { ProtocolVersion = GatewayContractInfo.WorkerProtocolVersion, SessionId = SessionId, Sequence = sequence, WorkerShutdown = new WorkerShutdown { GracePeriod = Duration.FromTimeSpan(TimeSpan.FromSeconds(1)), Reason = "test-complete", }, }; } private static WorkerEvent CreateWorkerEvent(ulong sequence) { return new WorkerEvent { Event = new MxEvent { SessionId = SessionId, Family = MxEventFamily.OnDataChange, WorkerSequence = sequence, OnDataChange = new OnDataChangeEvent(), }, }; } private static WorkerEvent CreateOversizedWorkerEvent(ulong sequence, int payloadBytes) { WorkerEvent workerEvent = CreateWorkerEvent(sequence); workerEvent.Event.ItemHandle = 42; workerEvent.Event.RawStatus = new string('x', payloadBytes); return workerEvent; } /// /// Fills a real event queue with byte-heavy events — a large string field stands in for the /// array/string MxValue payloads that make a count-capped drain overshoot the frame /// maximum. The queue is real (not the fake's plain list) so the production byte-budgeting runs. /// /// Number of events to enqueue. /// Size of each event's raw-status payload string. /// The populated queue. private static MxAccessEventQueue CreateByteHeavyQueue(int eventCount, int payloadBytes) { MxAccessEventQueue queue = new(Math.Max(eventCount, 1)); string payload = new string('x', payloadBytes); for (int index = 0; index < eventCount; index++) { queue.Enqueue(new MxEvent { SessionId = SessionId, Family = MxEventFamily.OnDataChange, ItemHandle = index, RawStatus = payload, OnDataChange = new OnDataChangeEvent(), }); } return queue; } private static Task CompleteGatewayHandshakeAsync( PipePair pipePair, CancellationToken cancellationToken) { return CompleteGatewayHandshakeAsync(pipePair, maxFrameBytes: 0, cancellationToken); } private static async Task CompleteGatewayHandshakeAsync( PipePair pipePair, uint maxFrameBytes, CancellationToken cancellationToken) { await pipePair.GatewayWriter .WriteAsync(CreateGatewayHelloEnvelope(maxFrameBytes: maxFrameBytes), cancellationToken) .ConfigureAwait(false); WorkerEnvelope hello = await pipePair.GatewayReader.ReadAsync(cancellationToken).ConfigureAwait(false); WorkerEnvelope ready = await pipePair.GatewayReader.ReadAsync(cancellationToken).ConfigureAwait(false); Assert.Equal(WorkerEnvelope.BodyOneofCase.WorkerHello, hello.BodyCase); Assert.Equal(WorkerEnvelope.BodyOneofCase.WorkerReady, ready.BodyCase); } private static async Task SendShutdownAndWaitAsync( PipePair pipePair, Task runTask, CancellationToken cancellationToken) { await pipePair.GatewayWriter .WriteAsync(CreateShutdownEnvelope(), cancellationToken) .ConfigureAwait(false); WorkerEnvelope shutdownAck = await ReadUntilAsync( pipePair.GatewayReader, WorkerEnvelope.BodyOneofCase.WorkerShutdownAck, cancellationToken); Assert.Equal(ProtocolStatusCode.Ok, shutdownAck.WorkerShutdownAck.Status.Code); Task completedTask = await Task .WhenAny(runTask, Task.Delay(TimeSpan.FromSeconds(5), cancellationToken)) .ConfigureAwait(false); Assert.Same(runTask, completedTask); await runTask.ConfigureAwait(false); } /// Reads frames until one matching the expected body type is found. /// Frame reader. /// Expected body case. /// Token to cancel the asynchronous operation. /// The matching envelope. private static Task ReadUntilAsync( WorkerFrameReader reader, WorkerEnvelope.BodyOneofCase expectedBody, CancellationToken cancellationToken) { return ReadUntilAsync( reader, expectedBody, _ => true, cancellationToken); } /// Reads frames until one matches the expected body type and predicate. /// Frame reader. /// Expected body case. /// Predicate to match against envelope. /// Token to cancel the asynchronous operation. /// The matching envelope. private static async Task ReadUntilAsync( WorkerFrameReader reader, WorkerEnvelope.BodyOneofCase expectedBody, Func predicate, CancellationToken cancellationToken) { while (true) { WorkerEnvelope envelope = await reader.ReadAsync(cancellationToken).ConfigureAwait(false); if (envelope.BodyCase == expectedBody && predicate(envelope)) { return envelope; } } } private static WorkerEnvelope[] ReadWrittenFrames( MemoryStream stream, WorkerFrameProtocolOptions options) { stream.Position = 0; WorkerFrameReader reader = new(stream, options); List envelopes = new(); while (stream.Position < stream.Length) { envelopes.Add(reader.ReadAsync(CancellationToken.None).GetAwaiter().GetResult()); } return envelopes.ToArray(); } private sealed class RecordingWorkerLogger : ZB.MOM.WW.MxGateway.Worker.Bootstrap.IWorkerLogger { private readonly object gate = new(); private readonly List events = new(); /// Gets a snapshot of the recorded log entries. public IReadOnlyList Events { get { lock (gate) { return new List(events); } } } /// public void Information(string eventName, IReadOnlyDictionary fields) { Record(eventName, fields); } /// public void Error(string eventName, IReadOnlyDictionary fields) { Record(eventName, fields); } private void Record(string eventName, IReadOnlyDictionary fields) { Dictionary copy = new(); foreach (KeyValuePair field in fields) { copy[field.Key] = field.Value; } lock (gate) { events.Add(new LogEntry(eventName, copy)); } } /// A single recorded log entry. public sealed class LogEntry { /// Initializes a recorded log entry. /// The log event name. /// The log event fields. public LogEntry(string eventName, IReadOnlyDictionary fields) { EventName = eventName; Fields = fields; } /// Gets the log event name. public string EventName { get; } /// Gets the log event fields. public IReadOnlyDictionary Fields { get; } } } // Wraps the worker side of the pipe and records the flush shape of the frames the writer emits — // how many stream writes each flush coalesced — so a test can assert the event drain loop turns a // burst into a single flush. Delegates every other operation to the inner stream; does not own the // inner stream's lifetime (PipePair disposes it). // // The writer flushes only after writing a whole drained batch, so the peer can read every frame of // that batch before the flush runs. Neither "I read the frames" nor any fixed sleep is evidence // that a flush has been counted; WaitForAllWritesFlushedAsync is the explicit edge a test must // take before sampling the counters. private sealed class FlushCountingPassthroughStream : Stream { private readonly Stream inner; private readonly object gate = new(); private readonly List flushWriteCounts = new(); private readonly List waiters = new(); private int writesSinceLastFlush; /// Initializes the passthrough over the given inner stream. /// The stream to delegate to. public FlushCountingPassthroughStream(Stream inner) { this.inner = inner; } /// Gets the number of flushes observed so far. public int FlushCount { get { lock (gate) { return flushWriteCounts.Count; } } } /// /// Returns the number of stream writes coalesced into each flush, in flush order. /// /// A snapshot of the per-flush write counts. public IReadOnlyList SnapshotFlushWriteCounts() { lock (gate) { return flushWriteCounts.ToArray(); } } /// /// Completes once every write issued so far has been flushed, giving the caller a /// happens-before edge on the writer's deferred flush instead of a timing guess. /// /// Token to abandon the wait. /// A task that completes when no write is left unflushed. public Task WaitForAllWritesFlushedAsync(CancellationToken cancellationToken) { FlushWaiter waiter; lock (gate) { if (writesSinceLastFlush == 0) { return Task.CompletedTask; } // Any flush drains every pending write, so the next flush is exactly the edge wanted. waiter = new FlushWaiter(flushWriteCounts.Count + 1); waiters.Add(waiter); } return AwaitFlushWaiterAsync(waiter, cancellationToken); } /// public override bool CanRead => inner.CanRead; /// public override bool CanSeek => inner.CanSeek; /// public override bool CanWrite => inner.CanWrite; /// public override long Length => inner.Length; /// public override long Position { get => inner.Position; set => inner.Position = value; } /// public override void Flush() { RecordFlush(); inner.Flush(); } /// public override Task FlushAsync(CancellationToken cancellationToken) { RecordFlush(); return inner.FlushAsync(cancellationToken); } /// public override int Read(byte[] buffer, int offset, int count) => inner.Read(buffer, offset, count); /// public override Task ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) => inner.ReadAsync(buffer, offset, count, cancellationToken); /// public override long Seek(long offset, SeekOrigin origin) => inner.Seek(offset, origin); /// public override void SetLength(long value) => inner.SetLength(value); /// public override void Write(byte[] buffer, int offset, int count) { RecordWrite(); inner.Write(buffer, offset, count); } /// public override Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) { RecordWrite(); return inner.WriteAsync(buffer, offset, count, cancellationToken); } private static async Task AwaitFlushWaiterAsync( FlushWaiter waiter, CancellationToken cancellationToken) { using (cancellationToken.Register(() => waiter.Completion.TrySetCanceled())) { await waiter.Completion.Task.ConfigureAwait(false); } } // Counted at write issue, not completion: the peer can observe the bytes as soon as the write // is issued, so the pending count must already reflect the write by then. private void RecordWrite() { lock (gate) { writesSinceLastFlush++; } } private void RecordFlush() { List? released = null; lock (gate) { flushWriteCounts.Add(writesSinceLastFlush); writesSinceLastFlush = 0; for (int index = waiters.Count - 1; index >= 0; index--) { if (waiters[index].TargetFlushCount <= flushWriteCounts.Count) { released ??= new List(); released.Add(waiters[index]); waiters.RemoveAt(index); } } } if (released is null) { return; } foreach (FlushWaiter waiter in released) { waiter.Completion.TrySetResult(true); } } // A pending WaitForAllWritesFlushedAsync call: completes once the recorded flush count // reaches TargetFlushCount. private sealed class FlushWaiter { /// Initializes a waiter released at the given flush ordinal. /// Flush count that releases the waiter. public FlushWaiter(int targetFlushCount) { TargetFlushCount = targetFlushCount; Completion = new TaskCompletionSource( TaskCreationOptions.RunContinuationsAsynchronously); } /// Gets the flush count at which this waiter completes. public int TargetFlushCount { get; } /// Gets the completion signaled when the target flush count is reached. public TaskCompletionSource Completion { get; } } } private sealed class PipePair : IDisposable { private readonly NamedPipeServerStream gatewayStream; private PipePair( NamedPipeServerStream gatewayStream, NamedPipeClientStream workerStream) { this.gatewayStream = gatewayStream; WorkerStream = workerStream; WorkerFrameProtocolOptions options = CreateOptions(); GatewayReader = new WorkerFrameReader(gatewayStream, options); GatewayWriter = new WorkerFrameWriter(gatewayStream, options); } /// Gets the worker side of the named pipe stream. public Stream WorkerStream { get; } /// Gets the gateway frame reader. public WorkerFrameReader GatewayReader { get; } /// Gets the gateway frame writer. public WorkerFrameWriter GatewayWriter { get; } /// Creates a connected pair of named pipes for testing. /// Cancellation token. /// Connected pipe pair. public static async Task CreateAsync(CancellationToken cancellationToken) { string pipeName = $"mxaccessgw-worker-session-tests-{Guid.NewGuid():N}"; NamedPipeServerStream gatewayStream = TestNamedPipe.CreateServer(pipeName); NamedPipeClientStream workerStream = new( ".", pipeName, PipeDirection.InOut, PipeOptions.Asynchronous); Task waitForConnectionTask = gatewayStream.WaitForConnectionAsync(); await Task .Run(() => workerStream.Connect(5000), cancellationToken) .ConfigureAwait(false); await waitForConnectionTask.ConfigureAwait(false); return new PipePair(gatewayStream, workerStream); } /// public void Dispose() { WorkerStream.Dispose(); gatewayStream.Dispose(); } } }