fix(worker): review 2026-10-21 stream E — cache eviction, phased ReadBulk, poll/STA fault handling
ci / nightly-windev (push) Has been skipped
ci / windows-x86 (push) Failing after 1m3s
ci / java (push) Successful in 2m17s
ci / portable (push) Failing after 4m42s

- RemoveItem/Unregister evict value + write-completion caches (leak under churn)
- ReadBulk snapshot path: add/advise all, one shared deadline, then tear down
- alarm poll faults only after 3 consecutive failures; transient ones logged
- AlarmCommandHandler disposes primary when standby build throws
- StaRuntime pumps between queued work items
- WRK-02: post-start STA loop death surfaces as StaHung WorkerFault
- WRK-09: AppDomain unhandled-exception hook, exit code 7
- RETRIGGER documented as reserved, never synthesized
- TST-05: live facts for Ping/GetSessionState/GetWorkerInfo/DrainEvents/ShutdownWorker
This commit is contained in:
Joseph Doherty
2026-10-22 06:19:32 -04:00
parent 488649ce99
commit 7618cafb6c
22 changed files with 1859 additions and 90 deletions
@@ -1052,6 +1052,208 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
}
}
/// <summary>
/// TST-05: the real worker answers the four non-terminating control commands off-STA in
/// <c>WorkerPipeSession.HandleControlCommandAsync</c>. The default suite only ever sees the
/// FakeWorkerHarness canned replies, so this drives <c>Ping</c>, <c>GetSessionState</c>,
/// <c>GetWorkerInfo</c>, and <c>DrainEvents</c> through the gateway against a live worker and
/// asserts each reply carries real worker state rather than an <c>INVALID_REQUEST</c>.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[LiveMxAccessFact]
public async Task GatewaySession_WithLiveWorker_ControlCommands_ReturnRealWorkerState()
{
string workerExecutablePath = IntegrationTestEnvironment.ResolveLiveMxAccessWorkerExecutablePath();
Assert.True(
File.Exists(workerExecutablePath),
$"Live MXAccess worker executable was not found at {workerExecutablePath}. Build the worker or set {IntegrationTestEnvironment.LiveMxAccessWorkerExecutableVariableName}.");
TestWorkerProcessFactory processFactory = new(output);
await using GatewayServiceFixture fixture = new(workerExecutablePath, processFactory, output);
string? sessionId = null;
try
{
OpenSessionReply openReply = await fixture.Service.OpenSession(
new OpenSessionRequest
{
ClientSessionName = "live-mxaccess-control",
ClientCorrelationId = "live-open-control",
CommandTimeout = Duration.FromTimeSpan(CommandTimeout),
},
new TestServerCallContext()).ConfigureAwait(false);
sessionId = openReply.SessionId;
Assert.Equal(ProtocolStatusCode.Ok, openReply.ProtocolStatus.Code);
Assert.True(openReply.WorkerProcessId > 0);
MxCommandReply pingReply = await fixture.Service.Invoke(
CreateControlRequest(sessionId, "live-ping", new MxCommand
{
Kind = MxCommandKind.Ping,
Ping = new PingCommand { Message = "live-ping-echo" },
}),
new TestServerCallContext()).ConfigureAwait(false);
LogReply("Ping", pingReply);
Assert.Equal(ProtocolStatusCode.Ok, pingReply.ProtocolStatus.Code);
Assert.Equal(MxCommandKind.Ping, pingReply.Kind);
MxCommandReply stateReply = await fixture.Service.Invoke(
CreateControlRequest(sessionId, "live-get-session-state", new MxCommand
{
Kind = MxCommandKind.GetSessionState,
GetSessionState = new GetSessionStateCommand(),
}),
new TestServerCallContext()).ConfigureAwait(false);
LogReply("GetSessionState", stateReply);
Assert.Equal(ProtocolStatusCode.Ok, stateReply.ProtocolStatus.Code);
Assert.Equal(MxCommandKind.GetSessionState, stateReply.Kind);
Assert.NotNull(stateReply.SessionState);
Assert.Equal(SessionState.Ready, stateReply.SessionState.State);
MxCommandReply infoReply = await fixture.Service.Invoke(
CreateControlRequest(sessionId, "live-get-worker-info", new MxCommand
{
Kind = MxCommandKind.GetWorkerInfo,
GetWorkerInfo = new GetWorkerInfoCommand(),
}),
new TestServerCallContext()).ConfigureAwait(false);
LogReply("GetWorkerInfo", infoReply);
Assert.Equal(ProtocolStatusCode.Ok, infoReply.ProtocolStatus.Code);
Assert.Equal(MxCommandKind.GetWorkerInfo, infoReply.Kind);
Assert.NotNull(infoReply.WorkerInfo);
output.WriteLine(
$"GetWorkerInfo pid={infoReply.WorkerInfo.WorkerProcessId} version={infoReply.WorkerInfo.WorkerVersion} "
+ $"progid={infoReply.WorkerInfo.MxaccessProgid} clsid={infoReply.WorkerInfo.MxaccessClsid}");
Assert.Equal(openReply.WorkerProcessId, infoReply.WorkerInfo.WorkerProcessId);
Assert.False(string.IsNullOrWhiteSpace(infoReply.WorkerInfo.WorkerVersion));
Assert.False(string.IsNullOrWhiteSpace(infoReply.WorkerInfo.MxaccessProgid));
Assert.False(string.IsNullOrWhiteSpace(infoReply.WorkerInfo.MxaccessClsid));
// No StreamEvents subscriber is attached, so whatever the worker queued since start
// is still drainable; the reply must carry the DrainEvents payload (possibly empty).
MxCommandReply drainReply = await fixture.Service.Invoke(
CreateControlRequest(sessionId, "live-drain-events", new MxCommand
{
Kind = MxCommandKind.DrainEvents,
DrainEvents = new DrainEventsCommand { MaxEvents = 16 },
}),
new TestServerCallContext()).ConfigureAwait(false);
LogReply("DrainEvents", drainReply);
Assert.Equal(ProtocolStatusCode.Ok, drainReply.ProtocolStatus.Code);
Assert.Equal(MxCommandKind.DrainEvents, drainReply.Kind);
Assert.Equal(MxCommandReply.PayloadOneofCase.DrainEvents, drainReply.PayloadCase);
Assert.True(drainReply.DrainEvents.Events.Count <= 16);
output.WriteLine($"DrainEvents drained={drainReply.DrainEvents.Events.Count}");
// A control command must not disturb the session: it is still Ready afterwards.
Assert.True(fixture.TryGetSession(sessionId, out GatewaySession? session));
Assert.Equal(SessionState.Ready, session!.State);
}
finally
{
await ShutDownAsync(fixture, processFactory, sessionId, streamTask: null).ConfigureAwait(false);
}
}
/// <summary>
/// TST-05: <c>ShutdownWorker</c> writes its OK reply before the worker shuts down, so the
/// reply reaches the caller (a reply written after the pipe closed would surface as a
/// pipe-disconnected failure instead), and the worker process then exits on its own and the
/// session leaves <see cref="SessionState.Ready"/>. Uses its own fixture and worker because it
/// terminates the worker.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[LiveMxAccessFact]
public async Task GatewaySession_WithLiveWorker_ShutdownWorker_RepliesOkThenWorkerExits()
{
string workerExecutablePath = IntegrationTestEnvironment.ResolveLiveMxAccessWorkerExecutablePath();
Assert.True(
File.Exists(workerExecutablePath),
$"Live MXAccess worker executable was not found at {workerExecutablePath}. Build the worker or set {IntegrationTestEnvironment.LiveMxAccessWorkerExecutableVariableName}.");
TestWorkerProcessFactory processFactory = new(output);
await using GatewayServiceFixture fixture = new(workerExecutablePath, processFactory, output);
string? sessionId = null;
try
{
OpenSessionReply openReply = await fixture.Service.OpenSession(
new OpenSessionRequest
{
ClientSessionName = "live-mxaccess-shutdown-worker",
ClientCorrelationId = "live-open-shutdown-worker",
CommandTimeout = Duration.FromTimeSpan(CommandTimeout),
},
new TestServerCallContext()).ConfigureAwait(false);
sessionId = openReply.SessionId;
Assert.Equal(ProtocolStatusCode.Ok, openReply.ProtocolStatus.Code);
MxCommandReply shutdownReply = await fixture.Service.Invoke(
CreateControlRequest(sessionId, "live-shutdown-worker", new MxCommand
{
Kind = MxCommandKind.ShutdownWorker,
ShutdownWorker = new ShutdownWorkerCommand
{
GracePeriod = Duration.FromTimeSpan(TimeSpan.FromSeconds(5)),
},
}),
new TestServerCallContext()).ConfigureAwait(false);
LogReply("ShutdownWorker", shutdownReply);
Assert.Equal(ProtocolStatusCode.Ok, shutdownReply.ProtocolStatus.Code);
Assert.Equal(MxCommandKind.ShutdownWorker, shutdownReply.Kind);
// The worker exits on its own — no CloseSession, no kill. WaitForProcessesAsync
// throws on timeout, which fails the test.
await processFactory.WaitForProcessesAsync(StreamShutdownTimeout).ConfigureAwait(false);
DateTimeOffset waitDeadline = DateTimeOffset.UtcNow + StreamShutdownTimeout;
SessionState observedState = SessionState.Ready;
while (DateTimeOffset.UtcNow < waitDeadline)
{
if (!fixture.TryGetSession(sessionId, out GatewaySession? session))
{
observedState = SessionState.Closed;
break;
}
observedState = session.State;
if (observedState is SessionState.Faulted or SessionState.Closed or SessionState.Closing)
{
break;
}
await Task.Delay(TimeSpan.FromMilliseconds(50)).ConfigureAwait(false);
}
output.WriteLine($"ShutdownWorker observed_state={observedState}");
Assert.True(
observedState is SessionState.Faulted or SessionState.Closed or SessionState.Closing,
$"Expected the session to leave Ready after ShutdownWorker; observed {observedState}.");
}
finally
{
// The worker is gone; a CloseSession round-trip would only log a cleanup failure.
await ShutDownAsync(fixture, processFactory, sessionId: null, streamTask: null).ConfigureAwait(false);
}
}
private static MxCommandRequest CreateControlRequest(
string sessionId,
string correlationId,
MxCommand command)
{
return new MxCommandRequest
{
SessionId = sessionId,
ClientCorrelationId = correlationId,
Command = command,
};
}
/// <summary>
/// Closes the session and drains the event stream / worker processes without letting a
/// cleanup timeout mask the original failure from the test body.