Merge stream E (worker: cache eviction, ReadBulk shared deadline + dedupe, alarm poll threshold, STA loop fault, unhandled-exception hook, live control-command facts)

This commit is contained in:
Joseph Doherty
2026-10-22 08:28:55 -04:00
23 changed files with 2239 additions and 92 deletions
@@ -1052,6 +1052,231 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
}
}
/// <summary>
/// TST-05: the real worker answers the four non-terminating control commands off-STA in
/// <c>WorkerPipeSession.HandleControlCommandAsync</c>. The default suite only ever sees the
/// FakeWorkerHarness canned replies, so this drives <c>Ping</c>, <c>GetSessionState</c>,
/// <c>GetWorkerInfo</c>, and <c>DrainEvents</c> through the gateway against a live worker and
/// asserts each reply carries real worker state rather than an <c>INVALID_REQUEST</c>.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[LiveMxAccessFact]
public async Task GatewaySession_WithLiveWorker_ControlCommands_ReturnRealWorkerState()
{
string workerExecutablePath = IntegrationTestEnvironment.ResolveLiveMxAccessWorkerExecutablePath();
Assert.True(
File.Exists(workerExecutablePath),
$"Live MXAccess worker executable was not found at {workerExecutablePath}. Build the worker or set {IntegrationTestEnvironment.LiveMxAccessWorkerExecutableVariableName}.");
TestWorkerProcessFactory processFactory = new(output);
await using GatewayServiceFixture fixture = new(workerExecutablePath, processFactory, output);
string? sessionId = null;
try
{
OpenSessionReply openReply = await fixture.Service.OpenSession(
new OpenSessionRequest
{
ClientSessionName = "live-mxaccess-control",
ClientCorrelationId = "live-open-control",
CommandTimeout = Duration.FromTimeSpan(CommandTimeout),
},
new TestServerCallContext()).ConfigureAwait(false);
sessionId = openReply.SessionId;
Assert.Equal(ProtocolStatusCode.Ok, openReply.ProtocolStatus.Code);
Assert.True(openReply.WorkerProcessId > 0);
MxCommandReply pingReply = await fixture.Service.Invoke(
CreateControlRequest(sessionId, "live-ping", new MxCommand
{
Kind = MxCommandKind.Ping,
Ping = new PingCommand { Message = "live-ping-echo" },
}),
new TestServerCallContext()).ConfigureAwait(false);
LogReply("Ping", pingReply);
Assert.Equal(ProtocolStatusCode.Ok, pingReply.ProtocolStatus.Code);
Assert.Equal(MxCommandKind.Ping, pingReply.Kind);
MxCommandReply stateReply = await fixture.Service.Invoke(
CreateControlRequest(sessionId, "live-get-session-state", new MxCommand
{
Kind = MxCommandKind.GetSessionState,
GetSessionState = new GetSessionStateCommand(),
}),
new TestServerCallContext()).ConfigureAwait(false);
LogReply("GetSessionState", stateReply);
Assert.Equal(ProtocolStatusCode.Ok, stateReply.ProtocolStatus.Code);
Assert.Equal(MxCommandKind.GetSessionState, stateReply.Kind);
Assert.NotNull(stateReply.SessionState);
Assert.Equal(SessionState.Ready, stateReply.SessionState.State);
MxCommandReply infoReply = await fixture.Service.Invoke(
CreateControlRequest(sessionId, "live-get-worker-info", new MxCommand
{
Kind = MxCommandKind.GetWorkerInfo,
GetWorkerInfo = new GetWorkerInfoCommand(),
}),
new TestServerCallContext()).ConfigureAwait(false);
LogReply("GetWorkerInfo", infoReply);
Assert.Equal(ProtocolStatusCode.Ok, infoReply.ProtocolStatus.Code);
Assert.Equal(MxCommandKind.GetWorkerInfo, infoReply.Kind);
Assert.NotNull(infoReply.WorkerInfo);
output.WriteLine(
$"GetWorkerInfo pid={infoReply.WorkerInfo.WorkerProcessId} version={infoReply.WorkerInfo.WorkerVersion} "
+ $"progid={infoReply.WorkerInfo.MxaccessProgid} clsid={infoReply.WorkerInfo.MxaccessClsid}");
Assert.Equal(openReply.WorkerProcessId, infoReply.WorkerInfo.WorkerProcessId);
Assert.False(string.IsNullOrWhiteSpace(infoReply.WorkerInfo.WorkerVersion));
Assert.False(string.IsNullOrWhiteSpace(infoReply.WorkerInfo.MxaccessProgid));
Assert.False(string.IsNullOrWhiteSpace(infoReply.WorkerInfo.MxaccessClsid));
// No StreamEvents subscriber is attached, so whatever the worker queued since start
// is still drainable; the reply must carry the DrainEvents payload (possibly empty).
MxCommandReply drainReply = await fixture.Service.Invoke(
CreateControlRequest(sessionId, "live-drain-events", new MxCommand
{
Kind = MxCommandKind.DrainEvents,
DrainEvents = new DrainEventsCommand { MaxEvents = 16 },
}),
new TestServerCallContext()).ConfigureAwait(false);
LogReply("DrainEvents", drainReply);
Assert.Equal(ProtocolStatusCode.Ok, drainReply.ProtocolStatus.Code);
Assert.Equal(MxCommandKind.DrainEvents, drainReply.Kind);
Assert.Equal(MxCommandReply.PayloadOneofCase.DrainEvents, drainReply.PayloadCase);
Assert.True(drainReply.DrainEvents.Events.Count <= 16);
output.WriteLine($"DrainEvents drained={drainReply.DrainEvents.Events.Count}");
// A control command must not disturb the session: it is still Ready afterwards.
Assert.True(fixture.TryGetSession(sessionId, out GatewaySession? session));
Assert.Equal(SessionState.Ready, session!.State);
}
finally
{
await ShutDownAsync(fixture, processFactory, sessionId, streamTask: null).ConfigureAwait(false);
}
}
/// <summary>
/// TST-05: <c>ShutdownWorker</c> writes its OK reply before the worker shuts down, so the
/// reply reaches the caller (a reply written after the pipe closed would surface as a
/// pipe-disconnected failure instead), the worker process then exits on its own, and a later
/// command on the session does not succeed. Uses its own fixture and worker because it
/// terminates the worker.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[LiveMxAccessFact]
public async Task GatewaySession_WithLiveWorker_ShutdownWorker_RepliesOkThenWorkerExits()
{
string workerExecutablePath = IntegrationTestEnvironment.ResolveLiveMxAccessWorkerExecutablePath();
Assert.True(
File.Exists(workerExecutablePath),
$"Live MXAccess worker executable was not found at {workerExecutablePath}. Build the worker or set {IntegrationTestEnvironment.LiveMxAccessWorkerExecutableVariableName}.");
TestWorkerProcessFactory processFactory = new(output);
await using GatewayServiceFixture fixture = new(workerExecutablePath, processFactory, output);
using RecordingServerStreamWriter<MxEvent> eventWriter = new();
string? sessionId = null;
Task? streamTask = null;
using CancellationTokenSource streamCancellation = new();
try
{
OpenSessionReply openReply = await fixture.Service.OpenSession(
new OpenSessionRequest
{
ClientSessionName = "live-mxaccess-shutdown-worker",
ClientCorrelationId = "live-open-shutdown-worker",
CommandTimeout = Duration.FromTimeSpan(CommandTimeout),
},
new TestServerCallContext()).ConfigureAwait(false);
sessionId = openReply.SessionId;
Assert.Equal(ProtocolStatusCode.Ok, openReply.ProtocolStatus.Code);
// The gateway observes a worker's pipe closing through the event-stream read loop
// (the same path the abnormal-exit fact relies on); without a subscriber the session
// would sit in Ready until the heartbeat grace expired.
streamTask = fixture.Service.StreamEvents(
new StreamEventsRequest { SessionId = sessionId },
eventWriter,
new TestServerCallContext(streamCancellation.Token));
MxCommandReply shutdownReply = await fixture.Service.Invoke(
CreateControlRequest(sessionId, "live-shutdown-worker", new MxCommand
{
Kind = MxCommandKind.ShutdownWorker,
ShutdownWorker = new ShutdownWorkerCommand
{
GracePeriod = Duration.FromTimeSpan(TimeSpan.FromSeconds(5)),
},
}),
new TestServerCallContext()).ConfigureAwait(false);
LogReply("ShutdownWorker", shutdownReply);
Assert.Equal(ProtocolStatusCode.Ok, shutdownReply.ProtocolStatus.Code);
Assert.Equal(MxCommandKind.ShutdownWorker, shutdownReply.Kind);
// The worker exits on its own — no CloseSession, no kill. WaitForProcessesAsync
// throws on timeout, which fails the test.
await processFactory.WaitForProcessesAsync(StreamShutdownTimeout).ConfigureAwait(false);
// The session can no longer serve commands once its worker is gone: either the
// gateway rejects the call outright or the reply is not Ok.
MxCommandReply? afterReply = null;
Exception? afterException = null;
try
{
afterReply = await fixture.Service.Invoke(
CreateControlRequest(sessionId, "live-ping-after-shutdown", new MxCommand
{
Kind = MxCommandKind.Ping,
Ping = new PingCommand(),
}),
new TestServerCallContext()).ConfigureAwait(false);
LogReply("PingAfterShutdown", afterReply);
}
catch (Exception exception)
{
afterException = exception;
output.WriteLine($"PingAfterShutdown rejected: {exception.GetType().Name}: {exception.Message}");
}
Assert.True(
afterException is not null || afterReply!.ProtocolStatus.Code != ProtocolStatusCode.Ok,
"A command sent after ShutdownWorker must not succeed against the exited worker.");
// Logged, not asserted: the gateway's WorkerClient moves to Closed on the shutdown
// ack, but GatewaySession.State is not driven from that transition today, so the
// public session state still reads Ready here (observed live on windev). That is a
// gateway-side gap, not a worker one — this fact pins the worker half of the
// contract: OK reply first, then a clean exit.
string observedState = fixture.TryGetSession(sessionId, out GatewaySession? session)
? session.State.ToString()
: "<removed>";
output.WriteLine($"ShutdownWorker observed_session_state={observedState}");
}
finally
{
// The worker is gone; a CloseSession round-trip would only log a cleanup failure.
streamCancellation.Cancel();
await ShutDownAsync(fixture, processFactory, sessionId: null, streamTask).ConfigureAwait(false);
}
}
private static MxCommandRequest CreateControlRequest(
string sessionId,
string correlationId,
MxCommand command)
{
return new MxCommandRequest
{
SessionId = sessionId,
ClientCorrelationId = correlationId,
Command = command,
};
}
/// <summary>
/// Closes the session and drains the event stream / worker processes without letting a
/// cleanup timeout mask the original failure from the test body.