fix(sessions): review follow-up — never orphan a late worker, exempt internal subscribers from unattached grace, tolerate lost close races

- Open path: the attach gate now waits on CancellationToken.None and the manager discards (kills + disposes) the late worker client on ANY exception out of the attach step, including the caller's own cancellation (checked first, so an abandoned open never yields a session). Previously an OperationCanceledException from the gate wait escaped before the discard and orphaned the x86 worker. The gate Release is guarded against ObjectDisposedException. - Unattached grace: AttachInternalEventSubscriber sets a separate latch so the gateway's own alarm-monitor session (internal subscriber only) is never reaped; the idle clock starts at Ready (MarkReady stamps _readyAtUtc) instead of at open, so a slow worker start does not eat the window. - KillWorkerAsync / CloseSessionCoreAsync: an ObjectDisposedException from the close gate on a session that is no longer registered means a concurrent close already removed, accounted and disposed it; both now return AlreadyClosed with no fault, no second gauge decrement and no sweep failure. The general kill catch only decrements the gauge when the session is still registered. The sweep no longer counts AlreadyClosed results among the sessions it closed.
2026-10-22 06:43:32 -04:00
parent 285ef64160
commit ec9f26400a
6 changed files with 346 additions and 33 deletions
@@ -924,6 +924,71 @@ public sealed class SessionManagerTests
        Assert.True(metrics.GetSnapshot().Faults > 0);
    }

+    /// <summary>
+    ///     A kill whose session lookup succeeded but whose close gate is disposed by the time it
+    ///     waits on it lost the race to a close that already removed, disposed and accounted the
+    ///     session. It must report <c>AlreadyClosed</c> and touch nothing: the old catch recorded a
+    ///     fault, faulted the session and decremented the open gauge a second time (visible here as
+    ///     the other open session's slot disappearing from the gauge).
+    /// </summary>
+    /// <returns>A task that represents the asynchronous operation.</returns>
+    [Fact]
+    public async Task KillWorkerAsync_WhenCloseWonTheRace_ReportsAlreadyClosedWithoutDoubleAccounting()
+    {
+        FakeWorkerClient racedClient = new();
+        FakeWorkerClient otherClient = new();
+        StaleRegistry registry = new();
+        using GatewayMetrics metrics = new();
+        SessionManager manager = CreateManager(new QueueingSessionWorkerClientFactory(racedClient, otherClient), registry, metrics);
+        GatewaySession raced = await manager.OpenSessionAsync(CreateOpenRequest(), "client-1", ownerKeyId: null, CancellationToken.None);
+        await manager.OpenSessionAsync(CreateOpenRequest(), "client-2", ownerKeyId: null, CancellationToken.None);
+
+        await manager.CloseSessionAsync(raced.SessionId, CancellationToken.None);
+        Assert.Equal(1, metrics.GetSnapshot().OpenSessions);
+
+        // The kill looked the session up just before the close removed and disposed it.
+        registry.ServeStaleLookupOnce(raced);
+        SessionCloseResult result = await manager.KillWorkerAsync(raced.SessionId, "admin-kill", CancellationToken.None);
+
+        Assert.True(result.AlreadyClosed);
+        Assert.Equal(SessionState.Closed, result.FinalState);
+        Assert.Equal(0, racedClient.KillCount);
+        GatewayMetricsSnapshot snapshot = metrics.GetSnapshot();
+        Assert.Equal(1, snapshot.OpenSessions);
+        Assert.Equal(1, snapshot.SessionsClosed);
+        Assert.Equal(0, snapshot.Faults);
+    }
+
+    /// <summary>
+    ///     The sweep's counterpart of the kill race: a session that a concurrent close removed and
+    ///     disposed between the snapshot and the close call is simply skipped — it is not a sweep
+    ///     failure to log, and it is not counted as closed by the sweep.
+    /// </summary>
+    /// <returns>A task that represents the asynchronous operation.</returns>
+    [Fact]
+    public async Task CloseExpiredLeasesAsync_WhenCloseWonTheRace_DoesNotReportSweepFailure()
+    {
+        FakeWorkerClient workerClient = new();
+        StaleRegistry registry = new();
+        using GatewayMetrics metrics = new();
+        SessionManager manager = CreateManager(new FakeSessionWorkerClientFactory(workerClient), registry, metrics);
+        GatewaySession session = await manager.OpenSessionAsync(CreateOpenRequest(), "client-1", ownerKeyId: null, CancellationToken.None);
+        DateTimeOffset now = DateTimeOffset.UtcNow;
+        SetLeaseExpiry(session, now.AddSeconds(-1));
+
+        await manager.CloseSessionAsync(session.SessionId, CancellationToken.None);
+        registry.ServeStaleSnapshot(session);
+
+        int closedCount = await manager.CloseExpiredLeasesAsync(now, CancellationToken.None);
+
+        Assert.Equal(0, closedCount);
+        Assert.Equal(1, workerClient.ShutdownCount);
+        GatewayMetricsSnapshot snapshot = metrics.GetSnapshot();
+        Assert.Equal(1, snapshot.SessionsClosed);
+        Assert.Equal(0, snapshot.OpenSessions);
+        Assert.Equal(0, snapshot.Faults);
+    }
+
    /// <summary>
    /// Concurrent kills on the same session must not
    /// double-increment <c>mxgateway.sessions.closed</c>. The first kill wins, the second
@@ -1086,6 +1151,40 @@ public sealed class SessionManagerTests
        Assert.Equal(1, registry.Count);
    }

+    /// <summary>
+    ///     A caller that abandons its open (deadline, disconnect) while the worker is starting
+    ///     must not leave the worker running: whatever fails between the worker arriving and the
+    ///     session becoming Ready — here the caller's own cancellation — the late worker client is
+    ///     killed and disposed before the failure surfaces.
+    /// </summary>
+    /// <returns>A task that represents the asynchronous operation.</returns>
+    [Fact]
+    public async Task OpenSessionAsync_WhenCallerCancelledDuringWorkerStartup_DiscardsWorkerClient()
+    {
+        FakeWorkerClient workerClient = new();
+        GatedSessionWorkerClientFactory factory = new(workerClient);
+        SessionRegistry registry = new();
+        using GatewayMetrics metrics = new();
+        SessionManager manager = CreateManager(factory, registry, metrics);
+        using CancellationTokenSource cancellation = new();
+
+        Task<GatewaySession> openTask = manager.OpenSessionAsync(CreateOpenRequest(), "client-1", ownerKeyId: null, cancellation.Token);
+        GatewaySession starting = await factory.WaitForCreateStartAsync();
+
+        await cancellation.CancelAsync();
+        factory.ReleaseCreate();
+        SessionManagerException exception = await Assert.ThrowsAsync<SessionManagerException>(async () => await openTask);
+
+        Assert.Equal(SessionManagerErrorCode.OpenFailed, exception.ErrorCode);
+        Assert.IsAssignableFrom<OperationCanceledException>(exception.InnerException);
+        Assert.Equal(1, workerClient.KillCount);
+        Assert.Equal(1, workerClient.DisposeCount);
+        Assert.NotEqual(SessionState.Ready, starting.State);
+        Assert.Equal(0, registry.Count);
+        Assert.Equal(0, metrics.GetSnapshot().OpenSessions);
+        Assert.Equal(0, metrics.GetSnapshot().SessionsOpened);
+    }
+
    /// <summary>
    ///     Same window as the kill case, driven by the host drain: a session still starting its
    ///     worker when <c>ShutdownAsync</c> runs is closed by the drain, and the worker client that
@@ -1296,6 +1395,66 @@ public sealed class SessionManagerTests
        subscriber.Dispose();
    }

+    /// <summary>
+    ///     The gateway's own alarm-monitor session only ever attaches an <em>internal</em>
+    ///     (non-counted) subscriber. That must count as "attached" for this rule, or the gateway
+    ///     would reap its own monitor every cycle once the knob is on.
+    /// </summary>
+    /// <returns>A task that represents the asynchronous operation.</returns>
+    [Fact]
+    public async Task CloseExpiredLeasesAsync_UnattachedGrace_DoesNotReapSessionWithInternalSubscriber()
+    {
+        FakeWorkerClient workerClient = new();
+        FakeTimeProvider clock = new(DateTimeOffset.UtcNow);
+        SessionManager manager = CreateManager(
+            new FakeSessionWorkerClientFactory(workerClient),
+            options: CreateOptions(unattachedSessionGraceSeconds: 60),
+            timeProvider: clock);
+        GatewaySession session = await manager.OpenSessionAsync(CreateOpenRequest(), "client-1", ownerKeyId: null, CancellationToken.None);
+        using IEventSubscriberLease internalLease = session.AttachInternalEventSubscriber();
+
+        clock.Advance(TimeSpan.FromSeconds(61));
+        int closedCount = await manager.CloseExpiredLeasesAsync(clock.GetUtcNow(), CancellationToken.None);
+
+        Assert.Equal(0, closedCount);
+        Assert.Equal(SessionState.Ready, session.State);
+        Assert.Equal(0, workerClient.ShutdownCount);
+    }
+
+    /// <summary>
+    ///     The idle clock starts when the session becomes Ready, not when it was opened: a worker
+    ///     that took longer than the window to start must not hand the client a session the very
+    ///     next sweep reaps before it had a chance to subscribe.
+    /// </summary>
+    /// <returns>A task that represents the asynchronous operation.</returns>
+    [Fact]
+    public async Task CloseExpiredLeasesAsync_UnattachedGrace_IdleClockStartsAtReady()
+    {
+        FakeWorkerClient workerClient = new();
+        GatedSessionWorkerClientFactory factory = new(workerClient);
+        FakeTimeProvider clock = new(DateTimeOffset.UtcNow);
+        SessionManager manager = CreateManager(
+            factory,
+            options: CreateOptions(unattachedSessionGraceSeconds: 60),
+            timeProvider: clock);
+
+        Task<GatewaySession> openTask = manager.OpenSessionAsync(CreateOpenRequest(), "client-1", ownerKeyId: null, CancellationToken.None);
+        await factory.WaitForCreateStartAsync();
+        clock.Advance(TimeSpan.FromSeconds(120));
+        factory.ReleaseCreate();
+        GatewaySession session = await openTask;
+
+        int closedRightAfterReady = await manager.CloseExpiredLeasesAsync(clock.GetUtcNow(), CancellationToken.None);
+        Assert.Equal(0, closedRightAfterReady);
+        Assert.Equal(SessionState.Ready, session.State);
+
+        clock.Advance(TimeSpan.FromSeconds(60));
+        int closedAfterIdle = await manager.CloseExpiredLeasesAsync(clock.GetUtcNow(), CancellationToken.None);
+
+        Assert.Equal(1, closedAfterIdle);
+        Assert.Equal(SessionState.Closed, session.State);
+    }
+
    /// <summary>
    ///     The knob is off by default: a never-subscribed idle session lives for its full lease.
    /// </summary>
@@ -1878,12 +2037,81 @@ public sealed class SessionManagerTests
            GatewaySession session,
            CancellationToken cancellationToken)
        {
+            // Deliberately not bound to the caller's token: the tests that cancel the open while
+            // the worker is starting need the worker to still ARRIVE after the cancellation, which
+            // is the orphan-prone case.
            _entered.TrySetResult(session);
-            await _release.Task.WaitAsync(cancellationToken);
+            await _release.Task.WaitAsync(TimeSpan.FromSeconds(5));
            return workerClient;
        }
    }

+    /// <summary>
+    ///     Registry that can replay a race the real registry only produces by timing: a session a
+    ///     concurrent close has already removed and disposed is still returned by one lookup (the
+    ///     kill looked it up just before the removal) or still appears in one snapshot (the sweep
+    ///     snapshotted just before the removal).
+    /// </summary>
+    private sealed class StaleRegistry : ISessionRegistry
+    {
+        private readonly SessionRegistry _inner = new();
+        private GatewaySession? _staleLookup;
+        private GatewaySession? _staleSnapshot;
+
+        /// <summary>Makes the next lookup of <paramref name="session"/>'s id return it although it was removed.</summary>
+        /// <param name="session">Removed session to serve once.</param>
+        public void ServeStaleLookupOnce(GatewaySession session)
+        {
+            _staleLookup = session;
+        }
+
+        /// <summary>Makes the next snapshot include <paramref name="session"/> although it was removed.</summary>
+        /// <param name="session">Removed session to include once.</param>
+        public void ServeStaleSnapshot(GatewaySession session)
+        {
+            _staleSnapshot = session;
+        }
+
+        /// <inheritdoc />
+        public int Count => _inner.Count;
+
+        /// <inheritdoc />
+        public int ActiveCount => _inner.ActiveCount;
+
+        /// <inheritdoc />
+        public bool TryAdd(GatewaySession session) => _inner.TryAdd(session);
+
+        /// <inheritdoc />
+        public bool TryGet(string sessionId, [System.Diagnostics.CodeAnalysis.MaybeNullWhen(false)] out GatewaySession session)
+        {
+            if (_staleLookup is not null && _staleLookup.SessionId == sessionId)
+            {
+                session = _staleLookup;
+                _staleLookup = null;
+                return true;
+            }
+
+            return _inner.TryGet(sessionId, out session);
+        }
+
+        /// <inheritdoc />
+        public bool TryRemove(string sessionId, [System.Diagnostics.CodeAnalysis.MaybeNullWhen(false)] out GatewaySession session)
+            => _inner.TryRemove(sessionId, out session);
+
+        /// <inheritdoc />
+        public IReadOnlyCollection<GatewaySession> Snapshot()
+        {
+            if (_staleSnapshot is null)
+            {
+                return _inner.Snapshot();
+            }
+
+            GatewaySession[] snapshot = [.. _inner.Snapshot(), _staleSnapshot];
+            _staleSnapshot = null;
+            return snapshot;
+        }
+    }
+
    /// <summary>
    ///     Lifetime fake whose <see cref="ApplicationStopping"/> the test fires on demand; the
    ///     other two tokens never fire.