fix(sessions): exception-total shutdown body with non-cancellable kill fallback

src/ZB.MOM.WW.MxGateway.Server/Sessions/ISessionManager.cs

@@ -68,6 +68,12 @@ public interface ISessionManager
     /// <param name="now">The current time to evaluate expiration against.</param>
     /// <param name="cancellationToken">Token to cancel the asynchronous operation.</param>
     /// <returns>The number of sessions closed.</returns>
+    /// <remarks>
+    ///     A close that fails does not abandon the rest of the pass: every session selected by this
+    ///     sweep is attempted, and the first failure is then rethrown so the caller still observes
+    ///     (and logs) that the sweep failed. Which failure surfaces is nondeterministic when several
+    ///     closes fail in the same pass, because the closes run concurrently.
+    /// </remarks>
     Task<int> CloseExpiredLeasesAsync(
         DateTimeOffset now,
         CancellationToken cancellationToken);

src/ZB.MOM.WW.MxGateway.Server/Sessions/SessionManager.cs

@@ -356,18 +356,28 @@ public sealed class SessionManager : ISessionManager
         // tail to the orphan killer. Per-session exclusivity comes from GatewaySession.CloseAsync's
         // own close gate, and each iteration touches only its own session plus thread-safe
         // registry/metrics state.
+        //
+        // The body must be exception-TOTAL. Parallel.ForEachAsync cancels the token it hands the
+        // sibling bodies as soon as one body throws, so a single escaping exception would abort up
+        // to MaxParallelSessionCloses - 1 in-flight graceful shutdowns AND make their kill fallback
+        // throw immediately on the freshly cancelled token — sessions neither closed nor killed,
+        // i.e. leaked x86 workers that nothing reattaches to (a gateway restart terminates orphans
+        // rather than adopting them).
+        //
+        // For the same reason the loop itself is NOT bound to cancellationToken: a cancelled
+        // ParallelOptions token stops dispatching the remaining sessions entirely, whereas the
+        // sequential drain this replaced let every remaining session fail its graceful close fast
+        // and still kill its worker. The token is passed to the graceful close instead, which
+        // preserves that behavior — a host stop deadline turns the drain into a kill sweep rather
+        // than into a leak.
         await Parallel.ForEachAsync(
             _registry.Snapshot(),
-            new ParallelOptions
-            {
-                MaxDegreeOfParallelism = MaxParallelSessionCloses,
-                CancellationToken = cancellationToken,
-            },
-            async (session, closeToken) =>
+            new ParallelOptions { MaxDegreeOfParallelism = MaxParallelSessionCloses },
+            async (session, _) =>
             {
                 try
                 {
-                    await CloseSessionCoreAsync(session, GatewayShutdownReason, closeToken).ConfigureAwait(false);
+                    await CloseSessionCoreAsync(session, GatewayShutdownReason, cancellationToken).ConfigureAwait(false);
                 }
                 catch (Exception exception)
                 {
@@ -376,13 +386,19 @@ public sealed class SessionManager : ISessionManager
                         "Graceful shutdown failed for session {SessionId}; killing worker.",
                         session.SessionId);
 
-                    if (_registry.TryGet(session.SessionId, out _))
+                    if (_registry.TryGet(session.SessionId, out GatewaySession? registeredSession)
+                        && registeredSession is not null)
                     {
                         try
                         {
-                            await KillWorkerAsync(session.SessionId, GatewayShutdownReason, closeToken).ConfigureAwait(false);
+                            // Deliberately NOT the caller's token: the kill is the last-resort orphan
+                            // preventer, so it must still run when the host stop deadline (or a
+                            // sibling body's failure) has already cancelled the drain. It is a
+                            // synchronous Kill plus registry/dispose bookkeeping, not a wait on
+                            // the worker, so it cannot extend the drain meaningfully.
+                            await KillWorkerAsync(session.SessionId, GatewayShutdownReason, CancellationToken.None).ConfigureAwait(false);
                         }
-                        catch (SessionManagerException killException)
+                        catch (Exception killException)
                         {
                             _logger.LogWarning(
                                 killException,

src/ZB.MOM.WW.MxGateway.Tests/Gateway/Sessions/SessionManagerTests.cs

@@ -1171,6 +1171,33 @@ public sealed class SessionManagerTests
         Assert.False(manager.TryGetSession(failingSession.SessionId, out _));
     }
 
+    /// <summary>
+    ///     A drain whose token is already cancelled (the host stop deadline elapsed) must still
+    ///     kill every worker rather than skip the teardown: an unkilled worker is a leaked x86
+    ///     process, and a restarted gateway terminates orphans instead of reattaching to them. This
+    ///     pins both halves of the fix — the parallel loop is not bound to the caller's token, and
+    ///     the kill fallback does not run on it.
+    /// </summary>
+    /// <returns>A task that represents the asynchronous operation.</returns>
+    [Fact]
+    public async Task ShutdownAsync_WhenCancelledBeforeDraining_StillKillsEveryWorker()
+    {
+        FakeWorkerClient firstClient = new();
+        FakeWorkerClient secondClient = new();
+        SessionManager manager = CreateManager(new QueueingSessionWorkerClientFactory(firstClient, secondClient));
+        GatewaySession firstSession = await manager.OpenSessionAsync(CreateOpenRequest(), "client-1", ownerKeyId: null, CancellationToken.None);
+        GatewaySession secondSession = await manager.OpenSessionAsync(CreateOpenRequest(), "client-2", ownerKeyId: null, CancellationToken.None);
+        using CancellationTokenSource cancellation = new();
+        await cancellation.CancelAsync();
+
+        await manager.ShutdownAsync(cancellation.Token);
+
+        Assert.Equal(1, firstClient.KillCount);
+        Assert.Equal(1, secondClient.KillCount);
+        Assert.False(manager.TryGetSession(firstSession.SessionId, out _));
+        Assert.False(manager.TryGetSession(secondSession.SessionId, out _));
+    }
+
     /// <summary>Verifies that shutdown closes all registered sessions.</summary>
     /// <returns>A task that represents the asynchronous operation.</returns>
     [Fact]
@@ -1510,10 +1537,12 @@ public sealed class SessionManagerTests
             {
                 await _reached.Task.WaitAsync(RendezvousTimeout, cancellationToken);
             }
-            catch (TimeoutException)
+            catch (Exception exception) when (exception is TimeoutException or OperationCanceledException)
             {
                 // Sequential teardown: the expected overlap never happens, so let the shutdown
-                // finish and let MaxObservedConcurrency report the (failing) truth.
+                // finish and let MaxObservedConcurrency report the (failing) truth. Cancellation is
+                // swallowed for the same reason — a cancelled rendezvous must not turn into a
+                // second, misleading failure on top of the concurrency assertion.
             }
             finally
             {