diff --git a/src/ZB.MOM.WW.MxGateway.Server/Sessions/GatewaySession.cs b/src/ZB.MOM.WW.MxGateway.Server/Sessions/GatewaySession.cs index 29a5653d..aea4ee0f 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Sessions/GatewaySession.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Sessions/GatewaySession.cs @@ -17,11 +17,18 @@ public sealed class GatewaySession : IAsyncDisposable // cannot be mutated by a cast — the tag set is a visibility grant, not a scratch collection. private static readonly IReadOnlySet EmptyTags = FrozenSet.Empty; + // Re-read interval for a worker client that publishes no state-change signal (test fakes); + // see WaitForStateChangeAsync. + private static readonly TimeSpan UnsignalledWorkerPollInterval = TimeSpan.FromMilliseconds(25); + private readonly object _syncRoot = new(); private readonly SemaphoreSlim _closeLock = new(1, 1); private readonly SessionEventStreaming _eventStreaming; private IWorkerClient? _workerClient; private SessionState _state = SessionState.Creating; + // Completed (and replaced) under _syncRoot whenever _state or _workerClient changes, so the + // worker-ready wait is woken by the change rather than polling for it (mxaccessgw#189). + private TaskCompletionSource _stateChanged = new(TaskCreationOptions.RunContinuationsAsynchronously); private string? _finalFault; private DateTimeOffset _lastClientActivityAt; private DateTimeOffset? _leaseExpiresAt; @@ -456,6 +463,7 @@ public sealed class GatewaySession : IAsyncDisposable } _workerClient = workerClient; + SignalStateChangedUnderLock(); return true; } } @@ -614,6 +622,8 @@ public sealed class GatewaySession : IAsyncDisposable } _state = nextState; + + SignalStateChangedUnderLock(); } } @@ -1108,6 +1118,7 @@ public sealed class GatewaySession : IAsyncDisposable _finalFault = reason; _state = SessionState.Faulted; + SignalStateChangedUnderLock(); // Stamp the fault time once, on the first fault, so the sweeper can apply // FaultedGraceSeconds. A subsequent MarkFaulted (already-faulted session) keeps the @@ -1927,6 +1938,7 @@ public sealed class GatewaySession : IAsyncDisposable alreadyClosing = _closeStarted; _closeStarted = true; _state = SessionState.Closing; + SignalStateChangedUnderLock(); return true; } } @@ -1994,6 +2006,7 @@ public sealed class GatewaySession : IAsyncDisposable alreadyClosing = _closeStarted; _state = SessionState.Closing; + SignalStateChangedUnderLock(); return true; } } @@ -2039,6 +2052,16 @@ public sealed class GatewaySession : IAsyncDisposable return now - idleSince >= _unattachedGrace; } + // Wakes every GetReadyWorkerClientAsync waiter that captured the current signal. Called under + // _syncRoot after each _state / _workerClient write; RunContinuationsAsynchronously keeps the + // waiters' continuations off this thread, so none of them runs while _syncRoot is held. + private void SignalStateChangedUnderLock() + { + TaskCompletionSource changed = _stateChanged; + _stateChanged = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + changed.TrySetResult(); + } + // Final terminal transition; under _syncRoot to keep _state writes single-lock. // Closed is unconditionally terminal — TransitionTo refuses to overwrite it — // so we don't need to re-check the precondition here. @@ -2047,6 +2070,7 @@ public sealed class GatewaySession : IAsyncDisposable lock (_syncRoot) { _state = SessionState.Closed; + SignalStateChangedUnderLock(); } } @@ -2264,9 +2288,11 @@ public sealed class GatewaySession : IAsyncDisposable /// session is but the worker has transiently diverged /// to a non-terminal state (/ /// ) and the configured worker-ready wait timeout - /// is positive, this polls (outside _syncRoot) until the worker reaches - /// or the deadline elapses, re-evaluating the - /// fast-path/fail-fast decision under the lock on each poll. Terminal worker states, a + /// is positive, this waits (outside _syncRoot) until the worker reaches + /// or the deadline elapses. The wait is woken by a + /// state-change signal from the session and the worker client rather than a fixed poll, + /// and re-evaluates the fast-path/fail-fast decision under the lock on each wake-up, so a + /// transition is acted on as soon as it happens (mxaccessgw#189). Terminal worker states, a /// missing worker, or a non-Ready session fail fast immediately. With the default /// timeout of zero this behaves byte-for-byte like the synchronous fail-fast path: no /// await, no delay. @@ -2275,11 +2301,15 @@ public sealed class GatewaySession : IAsyncDisposable /// The worker client once both the session and worker are Ready. private async Task GetReadyWorkerClientAsync(CancellationToken cancellationToken) { - const int pollIntervalMs = 25; - string? failureMessage; + Task sessionChanged; + Task? workerChanged; lock (_syncRoot) { + // Capture both signals BEFORE evaluating, so a transition that lands after the + // evaluation completes a signal this call is about to wait on (no lost wake-up). + sessionChanged = _stateChanged.Task; + workerChanged = _workerClient?.GetStateChangedSignal(); IWorkerClient? ready = EvaluateReadyUnderLock(out failureMessage); if (ready is not null) { @@ -2300,17 +2330,21 @@ public sealed class GatewaySession : IAsyncDisposable } } - DateTimeOffset deadline = _eventStreaming.TimeProvider.GetUtcNow() + _workerReadyWaitTimeout; + TimeProvider timeProvider = _eventStreaming.TimeProvider; + DateTimeOffset deadline = timeProvider.GetUtcNow() + _workerReadyWaitTimeout; while (true) { - await Task.Delay( - TimeSpan.FromMilliseconds(pollIntervalMs), - _eventStreaming.TimeProvider, - cancellationToken) - .ConfigureAwait(false); + TimeSpan remaining = deadline - timeProvider.GetUtcNow(); + if (remaining > TimeSpan.Zero) + { + await WaitForStateChangeAsync(sessionChanged, workerChanged, remaining, timeProvider, cancellationToken) + .ConfigureAwait(false); + } lock (_syncRoot) { + sessionChanged = _stateChanged.Task; + workerChanged = _workerClient?.GetStateChangedSignal(); IWorkerClient? ready = EvaluateReadyUnderLock(out failureMessage); if (ready is not null) { @@ -2323,26 +2357,47 @@ public sealed class GatewaySession : IAsyncDisposable { throw new SessionManagerException(SessionManagerErrorCode.SessionNotReady, failureMessage); } - } - if (_eventStreaming.TimeProvider.GetUtcNow() >= deadline) - { - lock (_syncRoot) + if (timeProvider.GetUtcNow() >= deadline) { - IWorkerClient? ready = EvaluateReadyUnderLock(out failureMessage); - if (ready is not null) - { - return ready; - } - throw new SessionManagerException( SessionManagerErrorCode.SessionNotReady, - failureMessage ?? BuildNotReadyMessage()); + BuildNotReadyMessage()); } } } } + /// + /// Waits until the session or worker state changes, the remaining ready-wait budget + /// elapses, or is cancelled. A worker client that + /// publishes no state-change signal ( + /// returned ) is re-read on + /// instead; the production always publishes one. + /// + private static async Task WaitForStateChangeAsync( + Task sessionChanged, + Task? workerChanged, + TimeSpan remaining, + TimeProvider timeProvider, + CancellationToken cancellationToken) + { + TimeSpan budget = workerChanged is null && remaining > UnsignalledWorkerPollInterval + ? UnsignalledWorkerPollInterval + : remaining; + Task changed = workerChanged is null + ? sessionChanged + : Task.WhenAny(sessionChanged, workerChanged); + try + { + await changed.WaitAsync(budget, timeProvider, cancellationToken).ConfigureAwait(false); + } + catch (TimeoutException) + { + // Budget elapsed with no transition: the caller re-evaluates and applies the deadline. + } + } + /// /// Evaluates readiness while the caller already holds _syncRoot. Returns the /// worker client when both the session and worker are diff --git a/src/ZB.MOM.WW.MxGateway.Server/Workers/IWorkerClient.cs b/src/ZB.MOM.WW.MxGateway.Server/Workers/IWorkerClient.cs index 38bada24..f595d84f 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Workers/IWorkerClient.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Workers/IWorkerClient.cs @@ -14,6 +14,19 @@ public interface IWorkerClient : IAsyncDisposable /// Current state of the worker connection. WorkerClientState State { get; } + /// + /// Returns a task that completes the next time changes. Capture it + /// before reading : a transition after the capture completes the + /// captured task, so a waiter cannot miss the change it is waiting for. Each call after a + /// transition returns a fresh, not-yet-completed task. + /// + /// + /// The state-change signal, or when the client does not publish + /// transitions. The default implementation returns ; callers then fall + /// back to re-reading on an interval. + /// + Task? GetStateChangedSignal() => null; + /// UTC timestamp of the most recent heartbeat from the worker. DateTimeOffset LastHeartbeatAt { get; } diff --git a/src/ZB.MOM.WW.MxGateway.Server/Workers/WorkerClient.cs b/src/ZB.MOM.WW.MxGateway.Server/Workers/WorkerClient.cs index cff481d9..8d12dad7 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Workers/WorkerClient.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Workers/WorkerClient.cs @@ -50,6 +50,9 @@ public sealed class WorkerClient : IWorkerClient // interlocking. See WriteLoopAsync for why the stamp happens there rather than at construction. private ulong _nextSequence; private WorkerClientState _state; + // Completed (and replaced) by every _state write, under _syncRoot, so a waiter that captured the + // task before reading State is woken by the next transition instead of polling (mxaccessgw#189). + private TaskCompletionSource _stateChanged = new(TaskCreationOptions.RunContinuationsAsynchronously); private DateTimeOffset _lastHeartbeatAt; private int? _processId; private int _eventQueueDepth; @@ -149,6 +152,15 @@ public sealed class WorkerClient : IWorkerClient } } + /// + public Task GetStateChangedSignal() + { + lock (_syncRoot) + { + return _stateChanged.Task; + } + } + /// public DateTimeOffset LastHeartbeatAt { @@ -455,7 +467,7 @@ public sealed class WorkerClient : IWorkerClient becameTerminalHere = _state is not (WorkerClientState.Closed or WorkerClientState.Faulted); if (becameTerminalHere) { - _state = WorkerClientState.Closed; + SetStateUnderLock(WorkerClientState.Closed); } UnregisterEventQueueDepthSource(); @@ -1055,7 +1067,7 @@ public sealed class WorkerClient : IWorkerClient ? _processId ?? _connection.ProcessHandle?.ProcessId : ready.WorkerProcessId; _lastHeartbeatAt = _timeProvider.GetUtcNow(); - _state = WorkerClientState.Ready; + SetStateUnderLock(WorkerClientState.Ready); _workerStartRecorded = true; } @@ -1078,6 +1090,22 @@ public sealed class WorkerClient : IWorkerClient } } + // Every _state write goes through here, under _syncRoot, so GetStateChangedSignal waiters are + // woken exactly once per transition. The completion runs continuations asynchronously, so no + // waiter code executes while _syncRoot is held. + private void SetStateUnderLock(WorkerClientState state) + { + if (_state == state) + { + return; + } + + _state = state; + TaskCompletionSource changed = _stateChanged; + _stateChanged = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + changed.TrySetResult(); + } + /// Transitions to closing state. private void MarkClosing() { @@ -1088,7 +1116,7 @@ public sealed class WorkerClient : IWorkerClient return; } - _state = WorkerClientState.Closing; + SetStateUnderLock(WorkerClientState.Closing); } } @@ -1103,7 +1131,7 @@ public sealed class WorkerClient : IWorkerClient return; } - _state = WorkerClientState.Closed; + SetStateUnderLock(WorkerClientState.Closed); UnregisterEventQueueDepthSource(); } @@ -1138,7 +1166,7 @@ public sealed class WorkerClient : IWorkerClient return; } - _state = WorkerClientState.Faulted; + SetStateUnderLock(WorkerClientState.Faulted); UnregisterEventQueueDepthSource(); } @@ -1246,7 +1274,7 @@ public sealed class WorkerClient : IWorkerClient $"Worker client cannot start from state {_state}."); } - _state = WorkerClientState.Handshaking; + SetStateUnderLock(WorkerClientState.Handshaking); } } diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Sessions/SessionManagerTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Sessions/SessionManagerTests.cs index dddfb29c..e2596bfe 100644 --- a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Sessions/SessionManagerTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Sessions/SessionManagerTests.cs @@ -519,6 +519,86 @@ public sealed class SessionManagerTests Assert.Equal(1, workerClient.InvokeCount); } + /// + /// mxaccessgw#189: the worker-ready wait is woken by the worker's state-change signal, + /// not by a poll timer. The clock is never advanced here, so a Task.Delay-driven + /// poll on the injected would never fire and the invoke would + /// hang until the 5 s test bound. + /// + /// A task that represents the asynchronous operation. + [Fact] + public async Task InvokeAsync_WhenSignallingWorkerBecomesReady_WakesWithoutPollTimer() + { + FakeTimeProvider clock = new(new DateTimeOffset(2026, 1, 1, 0, 0, 0, TimeSpan.Zero)); + FakeWorkerClient workerClient = new() { State = WorkerClientState.Handshaking, SignalsStateChanges = true }; + SessionManager manager = CreateManager( + new FakeSessionWorkerClientFactory(workerClient), + options: CreateOptions(workerReadyWaitTimeoutMs: 60_000), + timeProvider: clock); + GatewaySession session = await manager.OpenSessionAsync(CreateOpenRequest(), "client-1", ownerKeyId: null, CancellationToken.None); + + TaskCompletionSource handshakingObserved = new(TaskCreationOptions.RunContinuationsAsynchronously); + workerClient.OnStateRead = state => + { + if (state == WorkerClientState.Handshaking) + { + handshakingObserved.TrySetResult(); + } + }; + Task invoke = manager.InvokeAsync( + session.SessionId, + CreateCommand(MxCommandKind.Ping), + CancellationToken.None); + await handshakingObserved.Task.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.False(invoke.IsCompleted); + + workerClient.State = WorkerClientState.Ready; + WorkerCommandReply reply = await invoke.WaitAsync(TimeSpan.FromSeconds(5)); + + Assert.NotNull(reply); + Assert.Equal(1, workerClient.InvokeCount); + } + + /// + /// mxaccessgw#189: a session leaving Ready while an invoke waits for its worker + /// wakes the wait through the session's own state-change signal and fails fast, without + /// waiting for the worker or the clock. + /// + /// A task that represents the asynchronous operation. + [Fact] + public async Task InvokeAsync_WhenSessionFaultsDuringReadyWait_FailsFastWithoutPollTimer() + { + FakeTimeProvider clock = new(new DateTimeOffset(2026, 1, 1, 0, 0, 0, TimeSpan.Zero)); + FakeWorkerClient workerClient = new() { State = WorkerClientState.Handshaking, SignalsStateChanges = true }; + SessionManager manager = CreateManager( + new FakeSessionWorkerClientFactory(workerClient), + options: CreateOptions(workerReadyWaitTimeoutMs: 60_000), + timeProvider: clock); + GatewaySession session = await manager.OpenSessionAsync(CreateOpenRequest(), "client-1", ownerKeyId: null, CancellationToken.None); + + TaskCompletionSource handshakingObserved = new(TaskCreationOptions.RunContinuationsAsynchronously); + workerClient.OnStateRead = state => + { + if (state == WorkerClientState.Handshaking) + { + handshakingObserved.TrySetResult(); + } + }; + Task invoke = manager.InvokeAsync( + session.SessionId, + CreateCommand(MxCommandKind.Ping), + CancellationToken.None); + await handshakingObserved.Task.WaitAsync(TimeSpan.FromSeconds(5)); + + session.MarkFaulted("test-fault"); + + SessionManagerException exception = await Assert.ThrowsAsync( + async () => await invoke.WaitAsync(TimeSpan.FromSeconds(5))); + Assert.Equal(SessionManagerErrorCode.SessionNotReady, exception.ErrorCode); + Assert.Contains("Session state is Faulted", exception.Message); + Assert.Equal(0, workerClient.InvokeCount); + } + /// /// A terminal worker state (Faulted) must fail fast even with a positive /// worker-ready wait timeout, surfacing both states without burning the timeout. @@ -2197,7 +2277,16 @@ public sealed class SessionManagerTests private sealed class FakeWorkerClient : IWorkerClient { + private readonly object _stateSignalLock = new(); private WorkerClientState _state = WorkerClientState.Ready; + private TaskCompletionSource _stateChanged = new(TaskCreationOptions.RunContinuationsAsynchronously); + + /// + /// Gets a value indicating whether this fake publishes state transitions through + /// like the production . + /// Off by default so the unsignalled-client fallback stays covered. + /// + public bool SignalsStateChanges { get; init; } /// public string SessionId { get; init; } = "session-1"; @@ -2217,12 +2306,37 @@ public sealed class SessionManagerTests return _state; } - set => _state = value; + set + { + TaskCompletionSource changed; + lock (_stateSignalLock) + { + _state = value; + changed = _stateChanged; + _stateChanged = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + } + + changed.TrySetResult(); + } } /// public DateTimeOffset LastHeartbeatAt { get; init; } = DateTimeOffset.UtcNow; + /// + public Task? GetStateChangedSignal() + { + if (!SignalsStateChanges) + { + return null; + } + + lock (_stateSignalLock) + { + return _stateChanged.Task; + } + } + /// Gets the number of times invoke was called on the fake worker client. public int InvokeCount { get; private set; } diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Workers/WorkerClientTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Workers/WorkerClientTests.cs index 3585b306..33a0ba9d 100644 --- a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Workers/WorkerClientTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Workers/WorkerClientTests.cs @@ -34,6 +34,34 @@ public sealed class WorkerClientTests Assert.Equal(WorkerProcessId, client.ProcessId); } + /// + /// mxaccessgw#189: every state transition completes the signal captured before it, and a + /// signal captured after the transition is a fresh, pending one. The session's + /// worker-ready wait relies on this instead of polling . + /// + /// A task that represents the asynchronous operation. + [Fact] + public async Task GetStateChangedSignal_CompletesOnEachTransition() + { + await using PipePair pipePair = await PipePair.CreateAsync(); + await using WorkerClient client = CreateClient(pipePair); + + Task createdSignal = client.GetStateChangedSignal(); + Assert.False(createdSignal.IsCompleted); + + await CompleteHandshakeAsync(client, pipePair); + Assert.Equal(WorkerClientState.Ready, client.State); + await createdSignal.WaitAsync(TestTimeout); + + Task readySignal = client.GetStateChangedSignal(); + Assert.NotSame(createdSignal, readySignal); + Assert.False(readySignal.IsCompleted); + + client.Kill("test"); + await readySignal.WaitAsync(TestTimeout); + Assert.NotEqual(WorkerClientState.Ready, client.State); + } + /// /// The GatewayHello carries the negotiated worker-frame maximum so the worker adopts the /// configured limit instead of its own default; a regression to sending 0 means