feat(worker,gateway): report record-info pin outcome in WorkerReady; drain-thread completion tests
The worker logged its MXSTATUS_PROXY record-info pin outcome only to its own stderr, which the gateway's launcher does not capture, so operators never saw whether the pin took. - Contract (additive, internal IPC): WorkerReady.record_info_pin_status = 5. No protocol version bump; an older peer ignores it or sends it empty. Regenerated Contracts/Generated, the client protoset, the Rust vendored proto, and the Go/Python/Java worker bindings (real delta only). - Worker: the handshake copies MxStatusProxyRecordInfoPin.Status into WorkerReady unless the runtime already set it. - Gateway: WorkerClient records it (IWorkerClient.RecordInfoPinStatus, default null) and logs it at Information when pinned, Warning otherwise; empty is not logged. Shown as "Record-info pin" in the dashboard session details Worker table via DashboardSessionSummary.WorkerRecordInfoPinStatus. - Tests: drain-thread cancellation (task ends Canceled; RunAsync cancels, runtime disposed, thread exits, no EventDrain stop-failure log) and fault (exception on the thread faults RunAsync with the same instance) paths; WorkerReady carries the pin status; gateway logging/recording and the dashboard projection. - Docs: gateway.md STA thread model (drain thread, pin step in COM lifetime), MxAccessWorkerInstanceDesign, GatewayProcessDesign, GatewayDashboardDesign, WorkerFrameProtocol.
This commit is contained in:
@@ -406,6 +406,151 @@ public sealed class WorkerPipeSessionTests
|
||||
await SendShutdownAndWaitAsync(pipePair, runTask, cancellation.Token);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The drain loop's task mirrors the dedicated thread's outcome: cancelling its token ends
|
||||
/// the task Canceled (not faulted, not still running), which is what lets the message loop's
|
||||
/// teardown treat it as an ordinary stop. Exercised directly, before any runtime session is
|
||||
/// attached, so the loop is in its runtime-less poll and only the cancellation can end it.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task RunEventDrainLoopAsync_WhenCancelled_EndsCanceled()
|
||||
{
|
||||
WorkerFrameProtocolOptions options = CreateOptions();
|
||||
using MemoryStream inbound = new();
|
||||
using MemoryStream outbound = new();
|
||||
WorkerPipeSession session = CreateSession(inbound, outbound, options);
|
||||
using CancellationTokenSource stop = new();
|
||||
|
||||
Task drainTask = session.RunEventDrainLoopAsync(stop.Token);
|
||||
Assert.False(drainTask.IsCompleted);
|
||||
stop.Cancel();
|
||||
|
||||
Task completedTask = await Task.WhenAny(drainTask, Task.Delay(TimeSpan.FromSeconds(5)));
|
||||
Assert.Same(drainTask, completedTask);
|
||||
Assert.True(drainTask.IsCanceled, $"Expected Canceled but the drain task ended {drainTask.Status}.");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Cancelling the session while the dedicated drain thread is parked on the event signal
|
||||
/// ends <see cref="WorkerPipeSession.RunAsync"/> as cancelled, disposes the runtime, and lets
|
||||
/// the drain thread exit. Teardown must not log the drain task as faulted or overrunning its
|
||||
/// stop budget: a cancelled drain is a clean stop.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task RunAsync_WhenCancelledWhileDrainThreadParked_EndsCanceledAndDrainThreadExits()
|
||||
{
|
||||
using CancellationTokenSource timeout = new(TimeSpan.FromSeconds(10));
|
||||
using CancellationTokenSource sessionCancellation = new();
|
||||
using PipePair pipePair = await PipePair.CreateAsync(timeout.Token);
|
||||
RecordingWorkerLogger logger = new();
|
||||
FakeRuntimeSession runtime = new()
|
||||
{
|
||||
WaitForEventsOnSignalOnly = true,
|
||||
};
|
||||
WorkerPipeSession session = CreatePipeSession(
|
||||
pipePair.WorkerStream,
|
||||
runtime,
|
||||
new WorkerPipeSessionOptions
|
||||
{
|
||||
HeartbeatInterval = TimeSpan.FromMilliseconds(100),
|
||||
HeartbeatGrace = TimeSpan.FromSeconds(5),
|
||||
},
|
||||
logger);
|
||||
Task runTask = session.RunAsync(sessionCancellation.Token);
|
||||
await CompleteGatewayHandshakeAsync(pipePair, timeout.Token);
|
||||
Thread drainThread = await WaitForDrainThreadAsync(runtime, timeout.Token);
|
||||
|
||||
sessionCancellation.Cancel();
|
||||
|
||||
Task completedTask = await Task.WhenAny(runTask, Task.Delay(TimeSpan.FromSeconds(5), timeout.Token));
|
||||
Assert.Same(runTask, completedTask);
|
||||
await Assert.ThrowsAnyAsync<OperationCanceledException>(async () => await runTask);
|
||||
Assert.True(runtime.Disposed);
|
||||
Assert.True(drainThread.Join(TimeSpan.FromSeconds(5)), "The dedicated drain thread did not exit.");
|
||||
Assert.DoesNotContain(
|
||||
logger.Events,
|
||||
entry => entry.Fields.TryGetValue("task", out object? task) && (task as string) == "EventDrain");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// An unexpected exception inside the loop on the dedicated drain thread faults the drain
|
||||
/// task with that same exception, so the message loop's <c>Task.WhenAny</c> supervision ends
|
||||
/// <see cref="WorkerPipeSession.RunAsync"/> with it exactly as it did when the loop was an
|
||||
/// async method: the session stops, the runtime is disposed, and the thread exits.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task RunAsync_WhenDrainLoopThrows_FaultsSessionWithTheSameException()
|
||||
{
|
||||
using CancellationTokenSource cancellation = new(TimeSpan.FromSeconds(10));
|
||||
using PipePair pipePair = await PipePair.CreateAsync(cancellation.Token);
|
||||
FakeRuntimeSession runtime = new()
|
||||
{
|
||||
WaitForEventsOnSignalOnly = true,
|
||||
};
|
||||
WorkerPipeSession session = CreatePipeSession(pipePair.WorkerStream, runtime);
|
||||
Task runTask = session.RunAsync(cancellation.Token);
|
||||
await CompleteGatewayHandshakeAsync(pipePair, cancellation.Token);
|
||||
Thread drainThread = await WaitForDrainThreadAsync(runtime, cancellation.Token);
|
||||
|
||||
InvalidOperationException scripted = new("scripted drain-loop failure");
|
||||
runtime.DrainEventsException = scripted;
|
||||
// Wake the parked thread; its next pass calls DrainEvents, which now throws.
|
||||
runtime.EnqueueEvent(CreateWorkerEvent(sequence: 4));
|
||||
|
||||
Task completedTask = await Task.WhenAny(runTask, Task.Delay(TimeSpan.FromSeconds(5), cancellation.Token));
|
||||
Assert.Same(runTask, completedTask);
|
||||
InvalidOperationException observed =
|
||||
await Assert.ThrowsAsync<InvalidOperationException>(async () => await runTask);
|
||||
Assert.Same(scripted, observed);
|
||||
Assert.True(runtime.Disposed);
|
||||
Assert.True(drainThread.Join(TimeSpan.FromSeconds(5)), "The dedicated drain thread did not exit.");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The handshake copies the process-wide record-info pin outcome into <c>WorkerReady</c>, so
|
||||
/// the gateway can log it: the worker's own log goes to stderr, which the gateway does not
|
||||
/// capture. Whether the pin succeeds depends on the host, so only the hand-off is asserted.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task CompleteStartupHandshakeAsync_ReportsRecordInfoPinStatusInWorkerReady()
|
||||
{
|
||||
_ = MxStatusProxyRecordInfoPin.EnsurePinned();
|
||||
string? pinStatus = MxStatusProxyRecordInfoPin.Status;
|
||||
Assert.False(string.IsNullOrEmpty(pinStatus));
|
||||
WorkerFrameProtocolOptions options = CreateOptions();
|
||||
using MemoryStream inbound = new();
|
||||
await new WorkerFrameWriter(inbound, options).WriteAsync(CreateGatewayHelloEnvelope());
|
||||
inbound.Position = 0;
|
||||
using MemoryStream outbound = new();
|
||||
WorkerPipeSession session = CreateSession(inbound, outbound, options);
|
||||
|
||||
await session.CompleteStartupHandshakeAsync(
|
||||
_ => Task.FromResult(new WorkerReady { WorkerProcessId = 1234 }));
|
||||
|
||||
WorkerEnvelope[] written = ReadWrittenFrames(outbound, options);
|
||||
Assert.Equal(WorkerEnvelope.BodyOneofCase.WorkerReady, written[1].BodyCase);
|
||||
Assert.Equal(pinStatus, written[1].WorkerReady.RecordInfoPinStatus);
|
||||
}
|
||||
|
||||
// Waits until the drain loop has parked on the fake's event signal at least once and returns the
|
||||
// thread it parked on: the dedicated drain thread.
|
||||
private static async Task<Thread> WaitForDrainThreadAsync(
|
||||
FakeRuntimeSession runtime,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
Thread? drainThread;
|
||||
while ((drainThread = runtime.LastWaitForEventsThread) is null)
|
||||
{
|
||||
await Task.Delay(TimeSpan.FromMilliseconds(5), cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
return drainThread;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies that a Ping control command is answered on the worker side
|
||||
/// (not dispatched to the STA) with an OK reply that echoes the ping
|
||||
|
||||
Reference in New Issue
Block a user