fix(gateway): wake the worker-ready wait on a state-change signal instead of a 25 ms poll (mxaccessgw#189)
GetReadyWorkerClientAsync re-read the session and worker states every 25 ms until the WorkerReadyWaitTimeoutMs deadline. It now waits on two TaskCompletionSource signals - one completed by every GatewaySession _state / _workerClient write, one by every WorkerClient _state write - and re-evaluates only when one fires or the remaining budget elapses. Both signals are captured under the lock before the state is evaluated, so a transition between the evaluation and the await is not lost. IWorkerClient gains GetStateChangedSignal(), a default interface member that returns null; a client without a signal (the test fakes) is re-read on the old 25 ms interval, so the fakes need no change. The production WorkerClient always publishes one. Default behaviour (WorkerReadyWaitTimeoutMs = 0, fail fast) is unchanged. Regression tests (SessionManagerTests): ...WakesWithoutPollTimer and ...FailsFastWithoutPollTimer never advance the FakeTimeProvider, so they hang and fail against the poll loop; WorkerClientTests.GetStateChangedSignal_ CompletesOnEachTransition pins the worker-side signal. Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
@@ -17,11 +17,18 @@ public sealed class GatewaySession : IAsyncDisposable
|
|||||||
// cannot be mutated by a cast — the tag set is a visibility grant, not a scratch collection.
|
// cannot be mutated by a cast — the tag set is a visibility grant, not a scratch collection.
|
||||||
private static readonly IReadOnlySet<string> EmptyTags = FrozenSet<string>.Empty;
|
private static readonly IReadOnlySet<string> EmptyTags = FrozenSet<string>.Empty;
|
||||||
|
|
||||||
|
// Re-read interval for a worker client that publishes no state-change signal (test fakes);
|
||||||
|
// see WaitForStateChangeAsync.
|
||||||
|
private static readonly TimeSpan UnsignalledWorkerPollInterval = TimeSpan.FromMilliseconds(25);
|
||||||
|
|
||||||
private readonly object _syncRoot = new();
|
private readonly object _syncRoot = new();
|
||||||
private readonly SemaphoreSlim _closeLock = new(1, 1);
|
private readonly SemaphoreSlim _closeLock = new(1, 1);
|
||||||
private readonly SessionEventStreaming _eventStreaming;
|
private readonly SessionEventStreaming _eventStreaming;
|
||||||
private IWorkerClient? _workerClient;
|
private IWorkerClient? _workerClient;
|
||||||
private SessionState _state = SessionState.Creating;
|
private SessionState _state = SessionState.Creating;
|
||||||
|
// Completed (and replaced) under _syncRoot whenever _state or _workerClient changes, so the
|
||||||
|
// worker-ready wait is woken by the change rather than polling for it (mxaccessgw#189).
|
||||||
|
private TaskCompletionSource _stateChanged = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||||
private string? _finalFault;
|
private string? _finalFault;
|
||||||
private DateTimeOffset _lastClientActivityAt;
|
private DateTimeOffset _lastClientActivityAt;
|
||||||
private DateTimeOffset? _leaseExpiresAt;
|
private DateTimeOffset? _leaseExpiresAt;
|
||||||
@@ -456,6 +463,7 @@ public sealed class GatewaySession : IAsyncDisposable
|
|||||||
}
|
}
|
||||||
|
|
||||||
_workerClient = workerClient;
|
_workerClient = workerClient;
|
||||||
|
SignalStateChangedUnderLock();
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -614,6 +622,8 @@ public sealed class GatewaySession : IAsyncDisposable
|
|||||||
}
|
}
|
||||||
|
|
||||||
_state = nextState;
|
_state = nextState;
|
||||||
|
|
||||||
|
SignalStateChangedUnderLock();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1108,6 +1118,7 @@ public sealed class GatewaySession : IAsyncDisposable
|
|||||||
|
|
||||||
_finalFault = reason;
|
_finalFault = reason;
|
||||||
_state = SessionState.Faulted;
|
_state = SessionState.Faulted;
|
||||||
|
SignalStateChangedUnderLock();
|
||||||
|
|
||||||
// Stamp the fault time once, on the first fault, so the sweeper can apply
|
// Stamp the fault time once, on the first fault, so the sweeper can apply
|
||||||
// FaultedGraceSeconds. A subsequent MarkFaulted (already-faulted session) keeps the
|
// FaultedGraceSeconds. A subsequent MarkFaulted (already-faulted session) keeps the
|
||||||
@@ -1927,6 +1938,7 @@ public sealed class GatewaySession : IAsyncDisposable
|
|||||||
alreadyClosing = _closeStarted;
|
alreadyClosing = _closeStarted;
|
||||||
_closeStarted = true;
|
_closeStarted = true;
|
||||||
_state = SessionState.Closing;
|
_state = SessionState.Closing;
|
||||||
|
SignalStateChangedUnderLock();
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1994,6 +2006,7 @@ public sealed class GatewaySession : IAsyncDisposable
|
|||||||
|
|
||||||
alreadyClosing = _closeStarted;
|
alreadyClosing = _closeStarted;
|
||||||
_state = SessionState.Closing;
|
_state = SessionState.Closing;
|
||||||
|
SignalStateChangedUnderLock();
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -2039,6 +2052,16 @@ public sealed class GatewaySession : IAsyncDisposable
|
|||||||
return now - idleSince >= _unattachedGrace;
|
return now - idleSince >= _unattachedGrace;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Wakes every GetReadyWorkerClientAsync waiter that captured the current signal. Called under
|
||||||
|
// _syncRoot after each _state / _workerClient write; RunContinuationsAsynchronously keeps the
|
||||||
|
// waiters' continuations off this thread, so none of them runs while _syncRoot is held.
|
||||||
|
private void SignalStateChangedUnderLock()
|
||||||
|
{
|
||||||
|
TaskCompletionSource changed = _stateChanged;
|
||||||
|
_stateChanged = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||||
|
changed.TrySetResult();
|
||||||
|
}
|
||||||
|
|
||||||
// Final terminal transition; under _syncRoot to keep _state writes single-lock.
|
// Final terminal transition; under _syncRoot to keep _state writes single-lock.
|
||||||
// Closed is unconditionally terminal — TransitionTo refuses to overwrite it —
|
// Closed is unconditionally terminal — TransitionTo refuses to overwrite it —
|
||||||
// so we don't need to re-check the precondition here.
|
// so we don't need to re-check the precondition here.
|
||||||
@@ -2047,6 +2070,7 @@ public sealed class GatewaySession : IAsyncDisposable
|
|||||||
lock (_syncRoot)
|
lock (_syncRoot)
|
||||||
{
|
{
|
||||||
_state = SessionState.Closed;
|
_state = SessionState.Closed;
|
||||||
|
SignalStateChangedUnderLock();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -2264,9 +2288,11 @@ public sealed class GatewaySession : IAsyncDisposable
|
|||||||
/// session is <see cref="SessionState.Ready"/> but the worker has transiently diverged
|
/// session is <see cref="SessionState.Ready"/> but the worker has transiently diverged
|
||||||
/// to a non-terminal state (<see cref="WorkerClientState.Handshaking"/>/
|
/// to a non-terminal state (<see cref="WorkerClientState.Handshaking"/>/
|
||||||
/// <see cref="WorkerClientState.Created"/>) and the configured worker-ready wait timeout
|
/// <see cref="WorkerClientState.Created"/>) and the configured worker-ready wait timeout
|
||||||
/// is positive, this polls (outside <c>_syncRoot</c>) until the worker reaches
|
/// is positive, this waits (outside <c>_syncRoot</c>) until the worker reaches
|
||||||
/// <see cref="WorkerClientState.Ready"/> or the deadline elapses, re-evaluating the
|
/// <see cref="WorkerClientState.Ready"/> or the deadline elapses. The wait is woken by a
|
||||||
/// fast-path/fail-fast decision under the lock on each poll. Terminal worker states, a
|
/// state-change signal from the session and the worker client rather than a fixed poll,
|
||||||
|
/// and re-evaluates the fast-path/fail-fast decision under the lock on each wake-up, so a
|
||||||
|
/// transition is acted on as soon as it happens (mxaccessgw#189). Terminal worker states, a
|
||||||
/// missing worker, or a non-<c>Ready</c> session fail fast immediately. With the default
|
/// missing worker, or a non-<c>Ready</c> session fail fast immediately. With the default
|
||||||
/// timeout of zero this behaves byte-for-byte like the synchronous fail-fast path: no
|
/// timeout of zero this behaves byte-for-byte like the synchronous fail-fast path: no
|
||||||
/// await, no delay.
|
/// await, no delay.
|
||||||
@@ -2275,11 +2301,15 @@ public sealed class GatewaySession : IAsyncDisposable
|
|||||||
/// <returns>The worker client once both the session and worker are <c>Ready</c>.</returns>
|
/// <returns>The worker client once both the session and worker are <c>Ready</c>.</returns>
|
||||||
private async Task<IWorkerClient> GetReadyWorkerClientAsync(CancellationToken cancellationToken)
|
private async Task<IWorkerClient> GetReadyWorkerClientAsync(CancellationToken cancellationToken)
|
||||||
{
|
{
|
||||||
const int pollIntervalMs = 25;
|
|
||||||
|
|
||||||
string? failureMessage;
|
string? failureMessage;
|
||||||
|
Task sessionChanged;
|
||||||
|
Task? workerChanged;
|
||||||
lock (_syncRoot)
|
lock (_syncRoot)
|
||||||
{
|
{
|
||||||
|
// Capture both signals BEFORE evaluating, so a transition that lands after the
|
||||||
|
// evaluation completes a signal this call is about to wait on (no lost wake-up).
|
||||||
|
sessionChanged = _stateChanged.Task;
|
||||||
|
workerChanged = _workerClient?.GetStateChangedSignal();
|
||||||
IWorkerClient? ready = EvaluateReadyUnderLock(out failureMessage);
|
IWorkerClient? ready = EvaluateReadyUnderLock(out failureMessage);
|
||||||
if (ready is not null)
|
if (ready is not null)
|
||||||
{
|
{
|
||||||
@@ -2300,17 +2330,21 @@ public sealed class GatewaySession : IAsyncDisposable
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
DateTimeOffset deadline = _eventStreaming.TimeProvider.GetUtcNow() + _workerReadyWaitTimeout;
|
TimeProvider timeProvider = _eventStreaming.TimeProvider;
|
||||||
|
DateTimeOffset deadline = timeProvider.GetUtcNow() + _workerReadyWaitTimeout;
|
||||||
while (true)
|
while (true)
|
||||||
{
|
{
|
||||||
await Task.Delay(
|
TimeSpan remaining = deadline - timeProvider.GetUtcNow();
|
||||||
TimeSpan.FromMilliseconds(pollIntervalMs),
|
if (remaining > TimeSpan.Zero)
|
||||||
_eventStreaming.TimeProvider,
|
{
|
||||||
cancellationToken)
|
await WaitForStateChangeAsync(sessionChanged, workerChanged, remaining, timeProvider, cancellationToken)
|
||||||
.ConfigureAwait(false);
|
.ConfigureAwait(false);
|
||||||
|
}
|
||||||
|
|
||||||
lock (_syncRoot)
|
lock (_syncRoot)
|
||||||
{
|
{
|
||||||
|
sessionChanged = _stateChanged.Task;
|
||||||
|
workerChanged = _workerClient?.GetStateChangedSignal();
|
||||||
IWorkerClient? ready = EvaluateReadyUnderLock(out failureMessage);
|
IWorkerClient? ready = EvaluateReadyUnderLock(out failureMessage);
|
||||||
if (ready is not null)
|
if (ready is not null)
|
||||||
{
|
{
|
||||||
@@ -2323,26 +2357,47 @@ public sealed class GatewaySession : IAsyncDisposable
|
|||||||
{
|
{
|
||||||
throw new SessionManagerException(SessionManagerErrorCode.SessionNotReady, failureMessage);
|
throw new SessionManagerException(SessionManagerErrorCode.SessionNotReady, failureMessage);
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
if (_eventStreaming.TimeProvider.GetUtcNow() >= deadline)
|
if (timeProvider.GetUtcNow() >= deadline)
|
||||||
{
|
{
|
||||||
lock (_syncRoot)
|
|
||||||
{
|
|
||||||
IWorkerClient? ready = EvaluateReadyUnderLock(out failureMessage);
|
|
||||||
if (ready is not null)
|
|
||||||
{
|
|
||||||
return ready;
|
|
||||||
}
|
|
||||||
|
|
||||||
throw new SessionManagerException(
|
throw new SessionManagerException(
|
||||||
SessionManagerErrorCode.SessionNotReady,
|
SessionManagerErrorCode.SessionNotReady,
|
||||||
failureMessage ?? BuildNotReadyMessage());
|
BuildNotReadyMessage());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Waits until the session or worker state changes, the remaining ready-wait budget
|
||||||
|
/// elapses, or <paramref name="cancellationToken"/> is cancelled. A worker client that
|
||||||
|
/// publishes no state-change signal (<see cref="IWorkerClient.GetStateChangedSignal"/>
|
||||||
|
/// returned <see langword="null"/>) is re-read on <see cref="UnsignalledWorkerPollInterval"/>
|
||||||
|
/// instead; the production <see cref="WorkerClient"/> always publishes one.
|
||||||
|
/// </summary>
|
||||||
|
private static async Task WaitForStateChangeAsync(
|
||||||
|
Task sessionChanged,
|
||||||
|
Task? workerChanged,
|
||||||
|
TimeSpan remaining,
|
||||||
|
TimeProvider timeProvider,
|
||||||
|
CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
TimeSpan budget = workerChanged is null && remaining > UnsignalledWorkerPollInterval
|
||||||
|
? UnsignalledWorkerPollInterval
|
||||||
|
: remaining;
|
||||||
|
Task changed = workerChanged is null
|
||||||
|
? sessionChanged
|
||||||
|
: Task.WhenAny(sessionChanged, workerChanged);
|
||||||
|
try
|
||||||
|
{
|
||||||
|
await changed.WaitAsync(budget, timeProvider, cancellationToken).ConfigureAwait(false);
|
||||||
|
}
|
||||||
|
catch (TimeoutException)
|
||||||
|
{
|
||||||
|
// Budget elapsed with no transition: the caller re-evaluates and applies the deadline.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Evaluates readiness while the caller already holds <c>_syncRoot</c>. Returns the
|
/// Evaluates readiness while the caller already holds <c>_syncRoot</c>. Returns the
|
||||||
/// worker client when both the session and worker are <see cref="WorkerClientState.Ready"/>
|
/// worker client when both the session and worker are <see cref="WorkerClientState.Ready"/>
|
||||||
|
|||||||
@@ -14,6 +14,19 @@ public interface IWorkerClient : IAsyncDisposable
|
|||||||
/// <summary>Current state of the worker connection.</summary>
|
/// <summary>Current state of the worker connection.</summary>
|
||||||
WorkerClientState State { get; }
|
WorkerClientState State { get; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Returns a task that completes the next time <see cref="State"/> changes. Capture it
|
||||||
|
/// <em>before</em> reading <see cref="State"/>: a transition after the capture completes the
|
||||||
|
/// captured task, so a waiter cannot miss the change it is waiting for. Each call after a
|
||||||
|
/// transition returns a fresh, not-yet-completed task.
|
||||||
|
/// </summary>
|
||||||
|
/// <returns>
|
||||||
|
/// The state-change signal, or <see langword="null"/> when the client does not publish
|
||||||
|
/// transitions. The default implementation returns <see langword="null"/>; callers then fall
|
||||||
|
/// back to re-reading <see cref="State"/> on an interval.
|
||||||
|
/// </returns>
|
||||||
|
Task? GetStateChangedSignal() => null;
|
||||||
|
|
||||||
/// <summary>UTC timestamp of the most recent heartbeat from the worker.</summary>
|
/// <summary>UTC timestamp of the most recent heartbeat from the worker.</summary>
|
||||||
DateTimeOffset LastHeartbeatAt { get; }
|
DateTimeOffset LastHeartbeatAt { get; }
|
||||||
|
|
||||||
|
|||||||
@@ -50,6 +50,9 @@ public sealed class WorkerClient : IWorkerClient
|
|||||||
// interlocking. See WriteLoopAsync for why the stamp happens there rather than at construction.
|
// interlocking. See WriteLoopAsync for why the stamp happens there rather than at construction.
|
||||||
private ulong _nextSequence;
|
private ulong _nextSequence;
|
||||||
private WorkerClientState _state;
|
private WorkerClientState _state;
|
||||||
|
// Completed (and replaced) by every _state write, under _syncRoot, so a waiter that captured the
|
||||||
|
// task before reading State is woken by the next transition instead of polling (mxaccessgw#189).
|
||||||
|
private TaskCompletionSource _stateChanged = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||||
private DateTimeOffset _lastHeartbeatAt;
|
private DateTimeOffset _lastHeartbeatAt;
|
||||||
private int? _processId;
|
private int? _processId;
|
||||||
private int _eventQueueDepth;
|
private int _eventQueueDepth;
|
||||||
@@ -149,6 +152,15 @@ public sealed class WorkerClient : IWorkerClient
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <inheritdoc />
|
||||||
|
public Task GetStateChangedSignal()
|
||||||
|
{
|
||||||
|
lock (_syncRoot)
|
||||||
|
{
|
||||||
|
return _stateChanged.Task;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// <inheritdoc />
|
/// <inheritdoc />
|
||||||
public DateTimeOffset LastHeartbeatAt
|
public DateTimeOffset LastHeartbeatAt
|
||||||
{
|
{
|
||||||
@@ -455,7 +467,7 @@ public sealed class WorkerClient : IWorkerClient
|
|||||||
becameTerminalHere = _state is not (WorkerClientState.Closed or WorkerClientState.Faulted);
|
becameTerminalHere = _state is not (WorkerClientState.Closed or WorkerClientState.Faulted);
|
||||||
if (becameTerminalHere)
|
if (becameTerminalHere)
|
||||||
{
|
{
|
||||||
_state = WorkerClientState.Closed;
|
SetStateUnderLock(WorkerClientState.Closed);
|
||||||
}
|
}
|
||||||
|
|
||||||
UnregisterEventQueueDepthSource();
|
UnregisterEventQueueDepthSource();
|
||||||
@@ -1055,7 +1067,7 @@ public sealed class WorkerClient : IWorkerClient
|
|||||||
? _processId ?? _connection.ProcessHandle?.ProcessId
|
? _processId ?? _connection.ProcessHandle?.ProcessId
|
||||||
: ready.WorkerProcessId;
|
: ready.WorkerProcessId;
|
||||||
_lastHeartbeatAt = _timeProvider.GetUtcNow();
|
_lastHeartbeatAt = _timeProvider.GetUtcNow();
|
||||||
_state = WorkerClientState.Ready;
|
SetStateUnderLock(WorkerClientState.Ready);
|
||||||
_workerStartRecorded = true;
|
_workerStartRecorded = true;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1078,6 +1090,22 @@ public sealed class WorkerClient : IWorkerClient
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Every _state write goes through here, under _syncRoot, so GetStateChangedSignal waiters are
|
||||||
|
// woken exactly once per transition. The completion runs continuations asynchronously, so no
|
||||||
|
// waiter code executes while _syncRoot is held.
|
||||||
|
private void SetStateUnderLock(WorkerClientState state)
|
||||||
|
{
|
||||||
|
if (_state == state)
|
||||||
|
{
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
_state = state;
|
||||||
|
TaskCompletionSource changed = _stateChanged;
|
||||||
|
_stateChanged = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||||
|
changed.TrySetResult();
|
||||||
|
}
|
||||||
|
|
||||||
/// <summary>Transitions to closing state.</summary>
|
/// <summary>Transitions to closing state.</summary>
|
||||||
private void MarkClosing()
|
private void MarkClosing()
|
||||||
{
|
{
|
||||||
@@ -1088,7 +1116,7 @@ public sealed class WorkerClient : IWorkerClient
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
_state = WorkerClientState.Closing;
|
SetStateUnderLock(WorkerClientState.Closing);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1103,7 +1131,7 @@ public sealed class WorkerClient : IWorkerClient
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
_state = WorkerClientState.Closed;
|
SetStateUnderLock(WorkerClientState.Closed);
|
||||||
UnregisterEventQueueDepthSource();
|
UnregisterEventQueueDepthSource();
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1138,7 +1166,7 @@ public sealed class WorkerClient : IWorkerClient
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
_state = WorkerClientState.Faulted;
|
SetStateUnderLock(WorkerClientState.Faulted);
|
||||||
UnregisterEventQueueDepthSource();
|
UnregisterEventQueueDepthSource();
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1246,7 +1274,7 @@ public sealed class WorkerClient : IWorkerClient
|
|||||||
$"Worker client cannot start from state {_state}.");
|
$"Worker client cannot start from state {_state}.");
|
||||||
}
|
}
|
||||||
|
|
||||||
_state = WorkerClientState.Handshaking;
|
SetStateUnderLock(WorkerClientState.Handshaking);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -519,6 +519,86 @@ public sealed class SessionManagerTests
|
|||||||
Assert.Equal(1, workerClient.InvokeCount);
|
Assert.Equal(1, workerClient.InvokeCount);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// mxaccessgw#189: the worker-ready wait is woken by the worker's state-change signal,
|
||||||
|
/// not by a poll timer. The clock is never advanced here, so a <c>Task.Delay</c>-driven
|
||||||
|
/// poll on the injected <see cref="TimeProvider"/> would never fire and the invoke would
|
||||||
|
/// hang until the 5 s test bound.
|
||||||
|
/// </summary>
|
||||||
|
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||||
|
[Fact]
|
||||||
|
public async Task InvokeAsync_WhenSignallingWorkerBecomesReady_WakesWithoutPollTimer()
|
||||||
|
{
|
||||||
|
FakeTimeProvider clock = new(new DateTimeOffset(2026, 1, 1, 0, 0, 0, TimeSpan.Zero));
|
||||||
|
FakeWorkerClient workerClient = new() { State = WorkerClientState.Handshaking, SignalsStateChanges = true };
|
||||||
|
SessionManager manager = CreateManager(
|
||||||
|
new FakeSessionWorkerClientFactory(workerClient),
|
||||||
|
options: CreateOptions(workerReadyWaitTimeoutMs: 60_000),
|
||||||
|
timeProvider: clock);
|
||||||
|
GatewaySession session = await manager.OpenSessionAsync(CreateOpenRequest(), "client-1", ownerKeyId: null, CancellationToken.None);
|
||||||
|
|
||||||
|
TaskCompletionSource handshakingObserved = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||||
|
workerClient.OnStateRead = state =>
|
||||||
|
{
|
||||||
|
if (state == WorkerClientState.Handshaking)
|
||||||
|
{
|
||||||
|
handshakingObserved.TrySetResult();
|
||||||
|
}
|
||||||
|
};
|
||||||
|
Task<WorkerCommandReply> invoke = manager.InvokeAsync(
|
||||||
|
session.SessionId,
|
||||||
|
CreateCommand(MxCommandKind.Ping),
|
||||||
|
CancellationToken.None);
|
||||||
|
await handshakingObserved.Task.WaitAsync(TimeSpan.FromSeconds(5));
|
||||||
|
Assert.False(invoke.IsCompleted);
|
||||||
|
|
||||||
|
workerClient.State = WorkerClientState.Ready;
|
||||||
|
WorkerCommandReply reply = await invoke.WaitAsync(TimeSpan.FromSeconds(5));
|
||||||
|
|
||||||
|
Assert.NotNull(reply);
|
||||||
|
Assert.Equal(1, workerClient.InvokeCount);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// mxaccessgw#189: a session leaving <c>Ready</c> while an invoke waits for its worker
|
||||||
|
/// wakes the wait through the session's own state-change signal and fails fast, without
|
||||||
|
/// waiting for the worker or the clock.
|
||||||
|
/// </summary>
|
||||||
|
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||||
|
[Fact]
|
||||||
|
public async Task InvokeAsync_WhenSessionFaultsDuringReadyWait_FailsFastWithoutPollTimer()
|
||||||
|
{
|
||||||
|
FakeTimeProvider clock = new(new DateTimeOffset(2026, 1, 1, 0, 0, 0, TimeSpan.Zero));
|
||||||
|
FakeWorkerClient workerClient = new() { State = WorkerClientState.Handshaking, SignalsStateChanges = true };
|
||||||
|
SessionManager manager = CreateManager(
|
||||||
|
new FakeSessionWorkerClientFactory(workerClient),
|
||||||
|
options: CreateOptions(workerReadyWaitTimeoutMs: 60_000),
|
||||||
|
timeProvider: clock);
|
||||||
|
GatewaySession session = await manager.OpenSessionAsync(CreateOpenRequest(), "client-1", ownerKeyId: null, CancellationToken.None);
|
||||||
|
|
||||||
|
TaskCompletionSource handshakingObserved = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||||
|
workerClient.OnStateRead = state =>
|
||||||
|
{
|
||||||
|
if (state == WorkerClientState.Handshaking)
|
||||||
|
{
|
||||||
|
handshakingObserved.TrySetResult();
|
||||||
|
}
|
||||||
|
};
|
||||||
|
Task<WorkerCommandReply> invoke = manager.InvokeAsync(
|
||||||
|
session.SessionId,
|
||||||
|
CreateCommand(MxCommandKind.Ping),
|
||||||
|
CancellationToken.None);
|
||||||
|
await handshakingObserved.Task.WaitAsync(TimeSpan.FromSeconds(5));
|
||||||
|
|
||||||
|
session.MarkFaulted("test-fault");
|
||||||
|
|
||||||
|
SessionManagerException exception = await Assert.ThrowsAsync<SessionManagerException>(
|
||||||
|
async () => await invoke.WaitAsync(TimeSpan.FromSeconds(5)));
|
||||||
|
Assert.Equal(SessionManagerErrorCode.SessionNotReady, exception.ErrorCode);
|
||||||
|
Assert.Contains("Session state is Faulted", exception.Message);
|
||||||
|
Assert.Equal(0, workerClient.InvokeCount);
|
||||||
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// A terminal worker state (<c>Faulted</c>) must fail fast even with a positive
|
/// A terminal worker state (<c>Faulted</c>) must fail fast even with a positive
|
||||||
/// worker-ready wait timeout, surfacing both states without burning the timeout.
|
/// worker-ready wait timeout, surfacing both states without burning the timeout.
|
||||||
@@ -2197,7 +2277,16 @@ public sealed class SessionManagerTests
|
|||||||
|
|
||||||
private sealed class FakeWorkerClient : IWorkerClient
|
private sealed class FakeWorkerClient : IWorkerClient
|
||||||
{
|
{
|
||||||
|
private readonly object _stateSignalLock = new();
|
||||||
private WorkerClientState _state = WorkerClientState.Ready;
|
private WorkerClientState _state = WorkerClientState.Ready;
|
||||||
|
private TaskCompletionSource _stateChanged = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets a value indicating whether this fake publishes state transitions through
|
||||||
|
/// <see cref="GetStateChangedSignal"/> like the production <see cref="WorkerClient"/>.
|
||||||
|
/// Off by default so the unsignalled-client fallback stays covered.
|
||||||
|
/// </summary>
|
||||||
|
public bool SignalsStateChanges { get; init; }
|
||||||
|
|
||||||
/// <inheritdoc />
|
/// <inheritdoc />
|
||||||
public string SessionId { get; init; } = "session-1";
|
public string SessionId { get; init; } = "session-1";
|
||||||
@@ -2217,12 +2306,37 @@ public sealed class SessionManagerTests
|
|||||||
return _state;
|
return _state;
|
||||||
}
|
}
|
||||||
|
|
||||||
set => _state = value;
|
set
|
||||||
|
{
|
||||||
|
TaskCompletionSource changed;
|
||||||
|
lock (_stateSignalLock)
|
||||||
|
{
|
||||||
|
_state = value;
|
||||||
|
changed = _stateChanged;
|
||||||
|
_stateChanged = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||||
|
}
|
||||||
|
|
||||||
|
changed.TrySetResult();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <inheritdoc />
|
/// <inheritdoc />
|
||||||
public DateTimeOffset LastHeartbeatAt { get; init; } = DateTimeOffset.UtcNow;
|
public DateTimeOffset LastHeartbeatAt { get; init; } = DateTimeOffset.UtcNow;
|
||||||
|
|
||||||
|
/// <inheritdoc />
|
||||||
|
public Task? GetStateChangedSignal()
|
||||||
|
{
|
||||||
|
if (!SignalsStateChanges)
|
||||||
|
{
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
lock (_stateSignalLock)
|
||||||
|
{
|
||||||
|
return _stateChanged.Task;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// <summary>Gets the number of times invoke was called on the fake worker client.</summary>
|
/// <summary>Gets the number of times invoke was called on the fake worker client.</summary>
|
||||||
public int InvokeCount { get; private set; }
|
public int InvokeCount { get; private set; }
|
||||||
|
|
||||||
|
|||||||
@@ -34,6 +34,34 @@ public sealed class WorkerClientTests
|
|||||||
Assert.Equal(WorkerProcessId, client.ProcessId);
|
Assert.Equal(WorkerProcessId, client.ProcessId);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// mxaccessgw#189: every state transition completes the signal captured before it, and a
|
||||||
|
/// signal captured after the transition is a fresh, pending one. The session's
|
||||||
|
/// worker-ready wait relies on this instead of polling <see cref="WorkerClient.State"/>.
|
||||||
|
/// </summary>
|
||||||
|
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||||
|
[Fact]
|
||||||
|
public async Task GetStateChangedSignal_CompletesOnEachTransition()
|
||||||
|
{
|
||||||
|
await using PipePair pipePair = await PipePair.CreateAsync();
|
||||||
|
await using WorkerClient client = CreateClient(pipePair);
|
||||||
|
|
||||||
|
Task createdSignal = client.GetStateChangedSignal();
|
||||||
|
Assert.False(createdSignal.IsCompleted);
|
||||||
|
|
||||||
|
await CompleteHandshakeAsync(client, pipePair);
|
||||||
|
Assert.Equal(WorkerClientState.Ready, client.State);
|
||||||
|
await createdSignal.WaitAsync(TestTimeout);
|
||||||
|
|
||||||
|
Task readySignal = client.GetStateChangedSignal();
|
||||||
|
Assert.NotSame(createdSignal, readySignal);
|
||||||
|
Assert.False(readySignal.IsCompleted);
|
||||||
|
|
||||||
|
client.Kill("test");
|
||||||
|
await readySignal.WaitAsync(TestTimeout);
|
||||||
|
Assert.NotEqual(WorkerClientState.Ready, client.State);
|
||||||
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// The <c>GatewayHello</c> carries the negotiated worker-frame maximum so the worker adopts the
|
/// The <c>GatewayHello</c> carries the negotiated worker-frame maximum so the worker adopts the
|
||||||
/// configured limit instead of its own default; a regression to sending <c>0</c> means
|
/// configured limit instead of its own default; a regression to sending <c>0</c> means
|
||||||
|
|||||||
Reference in New Issue
Block a user