fix(gateway): wake the worker-ready wait on a state-change signal instead of a 25 ms poll (mxaccessgw#189)

GetReadyWorkerClientAsync re-read the session and worker states every 25 ms
until the WorkerReadyWaitTimeoutMs deadline. It now waits on two
TaskCompletionSource signals - one completed by every GatewaySession _state /
_workerClient write, one by every WorkerClient _state write - and re-evaluates
only when one fires or the remaining budget elapses. Both signals are captured
under the lock before the state is evaluated, so a transition between the
evaluation and the await is not lost.

IWorkerClient gains GetStateChangedSignal(), a default interface member that
returns null; a client without a signal (the test fakes) is re-read on the old
25 ms interval, so the fakes need no change. The production WorkerClient always
publishes one. Default behaviour (WorkerReadyWaitTimeoutMs = 0, fail fast) is
unchanged.

Regression tests (SessionManagerTests): ...WakesWithoutPollTimer and
...FailsFastWithoutPollTimer never advance the FakeTimeProvider, so they hang
and fail against the poll loop; WorkerClientTests.GetStateChangedSignal_
CompletesOnEachTransition pins the worker-side signal.

Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
Joseph Doherty
2026-11-09 12:32:43 -05:00
parent 11d8f76c5f
commit 101bda7073
5 changed files with 267 additions and 29 deletions
@@ -519,6 +519,86 @@ public sealed class SessionManagerTests
Assert.Equal(1, workerClient.InvokeCount);
}
/// <summary>
/// mxaccessgw#189: the worker-ready wait is woken by the worker's state-change signal,
/// not by a poll timer. The clock is never advanced here, so a <c>Task.Delay</c>-driven
/// poll on the injected <see cref="TimeProvider"/> would never fire and the invoke would
/// hang until the 5 s test bound.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task InvokeAsync_WhenSignallingWorkerBecomesReady_WakesWithoutPollTimer()
{
FakeTimeProvider clock = new(new DateTimeOffset(2026, 1, 1, 0, 0, 0, TimeSpan.Zero));
FakeWorkerClient workerClient = new() { State = WorkerClientState.Handshaking, SignalsStateChanges = true };
SessionManager manager = CreateManager(
new FakeSessionWorkerClientFactory(workerClient),
options: CreateOptions(workerReadyWaitTimeoutMs: 60_000),
timeProvider: clock);
GatewaySession session = await manager.OpenSessionAsync(CreateOpenRequest(), "client-1", ownerKeyId: null, CancellationToken.None);
TaskCompletionSource handshakingObserved = new(TaskCreationOptions.RunContinuationsAsynchronously);
workerClient.OnStateRead = state =>
{
if (state == WorkerClientState.Handshaking)
{
handshakingObserved.TrySetResult();
}
};
Task<WorkerCommandReply> invoke = manager.InvokeAsync(
session.SessionId,
CreateCommand(MxCommandKind.Ping),
CancellationToken.None);
await handshakingObserved.Task.WaitAsync(TimeSpan.FromSeconds(5));
Assert.False(invoke.IsCompleted);
workerClient.State = WorkerClientState.Ready;
WorkerCommandReply reply = await invoke.WaitAsync(TimeSpan.FromSeconds(5));
Assert.NotNull(reply);
Assert.Equal(1, workerClient.InvokeCount);
}
/// <summary>
/// mxaccessgw#189: a session leaving <c>Ready</c> while an invoke waits for its worker
/// wakes the wait through the session's own state-change signal and fails fast, without
/// waiting for the worker or the clock.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task InvokeAsync_WhenSessionFaultsDuringReadyWait_FailsFastWithoutPollTimer()
{
FakeTimeProvider clock = new(new DateTimeOffset(2026, 1, 1, 0, 0, 0, TimeSpan.Zero));
FakeWorkerClient workerClient = new() { State = WorkerClientState.Handshaking, SignalsStateChanges = true };
SessionManager manager = CreateManager(
new FakeSessionWorkerClientFactory(workerClient),
options: CreateOptions(workerReadyWaitTimeoutMs: 60_000),
timeProvider: clock);
GatewaySession session = await manager.OpenSessionAsync(CreateOpenRequest(), "client-1", ownerKeyId: null, CancellationToken.None);
TaskCompletionSource handshakingObserved = new(TaskCreationOptions.RunContinuationsAsynchronously);
workerClient.OnStateRead = state =>
{
if (state == WorkerClientState.Handshaking)
{
handshakingObserved.TrySetResult();
}
};
Task<WorkerCommandReply> invoke = manager.InvokeAsync(
session.SessionId,
CreateCommand(MxCommandKind.Ping),
CancellationToken.None);
await handshakingObserved.Task.WaitAsync(TimeSpan.FromSeconds(5));
session.MarkFaulted("test-fault");
SessionManagerException exception = await Assert.ThrowsAsync<SessionManagerException>(
async () => await invoke.WaitAsync(TimeSpan.FromSeconds(5)));
Assert.Equal(SessionManagerErrorCode.SessionNotReady, exception.ErrorCode);
Assert.Contains("Session state is Faulted", exception.Message);
Assert.Equal(0, workerClient.InvokeCount);
}
/// <summary>
/// A terminal worker state (<c>Faulted</c>) must fail fast even with a positive
/// worker-ready wait timeout, surfacing both states without burning the timeout.
@@ -2197,7 +2277,16 @@ public sealed class SessionManagerTests
private sealed class FakeWorkerClient : IWorkerClient
{
private readonly object _stateSignalLock = new();
private WorkerClientState _state = WorkerClientState.Ready;
private TaskCompletionSource _stateChanged = new(TaskCreationOptions.RunContinuationsAsynchronously);
/// <summary>
/// Gets a value indicating whether this fake publishes state transitions through
/// <see cref="GetStateChangedSignal"/> like the production <see cref="WorkerClient"/>.
/// Off by default so the unsignalled-client fallback stays covered.
/// </summary>
public bool SignalsStateChanges { get; init; }
/// <inheritdoc />
public string SessionId { get; init; } = "session-1";
@@ -2217,12 +2306,37 @@ public sealed class SessionManagerTests
return _state;
}
set => _state = value;
set
{
TaskCompletionSource changed;
lock (_stateSignalLock)
{
_state = value;
changed = _stateChanged;
_stateChanged = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
}
changed.TrySetResult();
}
}
/// <inheritdoc />
public DateTimeOffset LastHeartbeatAt { get; init; } = DateTimeOffset.UtcNow;
/// <inheritdoc />
public Task? GetStateChangedSignal()
{
if (!SignalsStateChanges)
{
return null;
}
lock (_stateSignalLock)
{
return _stateChanged.Task;
}
}
/// <summary>Gets the number of times invoke was called on the fake worker client.</summary>
public int InvokeCount { get; private set; }
@@ -34,6 +34,34 @@ public sealed class WorkerClientTests
Assert.Equal(WorkerProcessId, client.ProcessId);
}
/// <summary>
/// mxaccessgw#189: every state transition completes the signal captured before it, and a
/// signal captured after the transition is a fresh, pending one. The session's
/// worker-ready wait relies on this instead of polling <see cref="WorkerClient.State"/>.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task GetStateChangedSignal_CompletesOnEachTransition()
{
await using PipePair pipePair = await PipePair.CreateAsync();
await using WorkerClient client = CreateClient(pipePair);
Task createdSignal = client.GetStateChangedSignal();
Assert.False(createdSignal.IsCompleted);
await CompleteHandshakeAsync(client, pipePair);
Assert.Equal(WorkerClientState.Ready, client.State);
await createdSignal.WaitAsync(TestTimeout);
Task readySignal = client.GetStateChangedSignal();
Assert.NotSame(createdSignal, readySignal);
Assert.False(readySignal.IsCompleted);
client.Kill("test");
await readySignal.WaitAsync(TestTimeout);
Assert.NotEqual(WorkerClientState.Ready, client.State);
}
/// <summary>
/// The <c>GatewayHello</c> carries the negotiated worker-frame maximum so the worker adopts the
/// configured limit instead of its own default; a regression to sending <c>0</c> means