Merge stream A (session lifecycle: startup orphan, sweep metrics, stopping gate, kill-fallback Closed, unattached grace, worker-exit faults session)

2026-10-22 08:21:47 -04:00
parent 3f9448f619 cc1e895474
commit 7f0720f956
15 changed files with 1669 additions and 169 deletions
@@ -325,6 +325,10 @@ public sealed class GatewayOptionsValidator : OptionsValidatorBase<GatewayOption
            options.FaultedGraceSeconds,
            "MxGateway:Sessions:FaultedGraceSeconds must be zero or greater (0 reaps a faulted session on the next sweep).",
            builder);
+        AddIfNegative(
+            options.UnattachedSessionGraceSeconds,
+            "MxGateway:Sessions:UnattachedSessionGraceSeconds must be zero or greater (0 disables reaping of never-subscribed sessions).",
+            builder);
        AddIfNegative(
            options.WorkerReadyWaitTimeoutMs,
            "MxGateway:Sessions:WorkerReadyWaitTimeoutMs must be greater than or equal to zero.",
@@ -57,6 +57,28 @@ public sealed class SessionOptions
    /// </summary>
    public int FaultedGraceSeconds { get; init; }

+    /// <summary>
+    ///     Gets the idle window, in seconds, after which the lease monitor reaps a session that
+    ///     has <em>never</em> attached an event-stream subscriber. A client that opens a session
+    ///     and vanishes before it ever streams (for example OtOpcUa's <c>GalaxyDriver</c> failing
+    ///     part-way through its start-up) otherwise pins a session slot and a live x86 worker for
+    ///     the full <see cref="DefaultLeaseSeconds"/> (30 minutes by default). The window is
+    ///     measured from the later of the session becoming Ready and its most recent unary
+    ///     command, so a slow worker start does not eat it and a client that only ever reads and
+    ///     writes is not reaped while it keeps calling; it stops applying the moment the session's
+    ///     first subscriber — external, or the gateway's own internal alarm-monitor subscriber —
+    ///     attaches (from then on <see cref="DetachGraceSeconds"/> and the normal lease govern
+    ///     it). A value of <c>0</c> (the default) disables the sweep. Must be greater than or
+    ///     equal to zero.
+    /// </summary>
+    /// <remarks>
+    ///     Set it comfortably above the time a well-behaved client needs between
+    ///     <c>OpenSession</c> and its first <c>StreamEvents</c>, and remember the close lands on
+    ///     the next sweep after the window elapses (up to <see cref="LeaseSweepIntervalSeconds"/>
+    ///     later).
+    /// </remarks>
+    public int UnattachedSessionGraceSeconds { get; init; }
+
    /// <summary>
    /// Gets a value indicating whether multiple event subscribers are allowed per session.
    /// </summary>
@@ -1104,6 +1104,7 @@ public sealed class MxAccessGatewayService(
            SessionManagerErrorCode.OpenFailed => StatusCode.Unavailable,
            SessionManagerErrorCode.CloseFailed => StatusCode.Unavailable,
            SessionManagerErrorCode.PermissionDenied => StatusCode.PermissionDenied,
+            SessionManagerErrorCode.GatewayStopping => StatusCode.Unavailable,
            _ => StatusCode.Unavailable,
        };

@@ -26,9 +26,11 @@ public sealed class GatewaySession
    private DateTimeOffset _lastClientActivityAt;
    private DateTimeOffset? _leaseExpiresAt;
    private bool _closeStarted;
+    private bool _openRecorded;
    private int _activeEventSubscriberCount;
    private readonly TimeSpan _detachGrace;
    private readonly TimeSpan _faultedGrace;
+    private readonly TimeSpan _unattachedGrace;
    private readonly TimeSpan _workerReadyWaitTimeout;
    private DateTimeOffset? _detachedAtUtc;
    private DateTimeOffset? _faultedAtUtc;
@@ -37,6 +39,12 @@ public sealed class GatewaySession
    // FAILED first attach — which still runs the rollback DetachEventSubscriber from the
    // attach catch path — does not push a never-subscribed session into the grace window.
    private bool _everHadEventSubscriber;
+    // True once an in-process (internal, non-counted) subscriber attached via
+    // AttachInternalEventSubscriber. Consulted only by the unattached-grace sweep rule.
+    private bool _everHadInternalEventSubscriber;
+    // Stamped once, when the session first becomes Ready; the unattached-grace idle clock cannot
+    // start before it.
+    private DateTimeOffset? _readyAtUtc;
    private SessionEventDistributor? _eventDistributor;
    private bool _eventDistributorStarted;
    // Set under _syncRoot when DisposeAsync nulls _eventDistributor, making the
@@ -168,6 +176,12 @@ public sealed class GatewaySession
    ///     The tags come from the owner key, never from the client's wire request, so a client
    ///     cannot label its own session with another tenant's tag.
    /// </param>
+    /// <param name="unattachedGrace">
+    ///     Idle window after which the lease monitor reaps a Ready session that has never attached
+    ///     an event subscriber (see <see cref="IsUnattachedGraceExpired"/>), so a client that
+    ///     opens and vanishes does not pin a worker for the full lease. <see cref="TimeSpan.Zero"/>
+    ///     (the default) disables it.
+    /// </param>
    public GatewaySession(
        string sessionId,
        string backendName,
@@ -187,7 +201,8 @@ public sealed class GatewaySession
        TimeSpan workerReadyWaitTimeout = default,
        ArrayAddressNormalizer? addressNormalizer = null,
        TimeSpan faultedGrace = default,
-        IReadOnlyList<string>? ownerDashboardTags = null)
+        IReadOnlyList<string>? ownerDashboardTags = null,
+        TimeSpan unattachedGrace = default)
    {
        if (string.IsNullOrWhiteSpace(sessionId))
        {
@@ -230,6 +245,7 @@ public sealed class GatewaySession
        _eventStreaming = eventStreaming ?? SessionEventStreaming.Default;
        _detachGrace = detachGrace > TimeSpan.Zero ? detachGrace : TimeSpan.Zero;
        _faultedGrace = faultedGrace > TimeSpan.Zero ? faultedGrace : TimeSpan.Zero;
+        _unattachedGrace = unattachedGrace > TimeSpan.Zero ? unattachedGrace : TimeSpan.Zero;
        _workerReadyWaitTimeout = workerReadyWaitTimeout > TimeSpan.Zero ? workerReadyWaitTimeout : TimeSpan.Zero;
        _addressNormalizer = addressNormalizer;
    }
@@ -413,13 +429,128 @@ public sealed class GatewaySession
    /// Attaches the worker client for this session.
    /// </summary>
    /// <param name="workerClient">Worker client to attach.</param>
-    public void AttachWorkerClient(IWorkerClient workerClient)
+    /// <returns>
+    ///     <see langword="true"/> when the client was attached; <see langword="false"/> when the
+    ///     session is already <see cref="SessionState.Closing"/>, <see cref="SessionState.Closed"/>
+    ///     or <see cref="SessionState.Faulted"/>, or has been disposed — the caller owns the client
+    ///     and must discard it.
+    /// </returns>
+    /// <remarks>
+    ///     The session is registered before its worker is created, so an admin kill or the host
+    ///     drain can close it while the worker is still starting (up to the startup timeout). A
+    ///     worker client arriving after that must not be stored on the dead session: nothing would
+    ///     ever shut it down (the close already ran with no client to stop) and the open would hand
+    ///     a closed session back to the caller as if it had succeeded. The decision is taken under
+    ///     <c>_syncRoot</c> so it cannot interleave with the close's state write.
+    /// </remarks>
+    public bool AttachWorkerClient(IWorkerClient workerClient)
    {
        ArgumentNullException.ThrowIfNull(workerClient);

        lock (_syncRoot)
        {
+            if (_state is SessionState.Closing or SessionState.Closed or SessionState.Faulted
+                || _eventDistributorDisposed)
+            {
+                return false;
+            }
+
            _workerClient = workerClient;
+            return true;
+        }
+    }
+
+    /// <summary>
+    ///     Runs the end of the open sequence — attach the worker client, mark the session
+    ///     <see cref="SessionState.Ready"/>, record it as opened — under the per-session close
+    ///     gate, so a concurrent <see cref="CloseAsync"/> or
+    ///     <see cref="KillWorkerWithCloseGateAsync"/> runs either entirely before it (the attach
+    ///     is refused) or entirely after it (an ordinary close of a Ready session). Without the
+    ///     gate a kill could land between the attach and the open accounting and the two teardown
+    ///     bookkeepings would disagree about whether the session ever counted as open.
+    /// </summary>
+    /// <param name="workerClient">Worker client to attach.</param>
+    /// <param name="onReady">
+    ///     Invoked inside the gate once the session is Ready; the caller records its open-session
+    ///     accounting here, and <see cref="OpenRecorded"/> is set right after it returns.
+    /// </param>
+    /// <returns>
+    ///     <see langword="true"/> when the session became Ready; <see langword="false"/> when the
+    ///     attach was refused (see <see cref="AttachWorkerClient"/>), including when the session was
+    ///     disposed before the gate could be taken.
+    /// </returns>
+    /// <remarks>
+    ///     Not cancellable: the worker client already exists, so the only acceptable outcomes are
+    ///     "attached" or "refused, caller discards it". The gate is held only for the duration of
+    ///     a close or kill, so the wait is bounded by the worker shutdown timeout. The caller
+    ///     still decides what to do about a cancelled open before calling this.
+    /// </remarks>
+    internal async ValueTask<bool> TryAttachWorkerClientAndMarkReadyAsync(
+        IWorkerClient workerClient,
+        Action onReady)
+    {
+        ArgumentNullException.ThrowIfNull(workerClient);
+        ArgumentNullException.ThrowIfNull(onReady);
+
+        try
+        {
+            await _closeLock.WaitAsync(CancellationToken.None).ConfigureAwait(false);
+        }
+        catch (ObjectDisposedException)
+        {
+            // Disposed while the worker was starting: the session has been torn down.
+            return false;
+        }
+
+        try
+        {
+            if (!AttachWorkerClient(workerClient))
+            {
+                return false;
+            }
+
+            MarkReady();
+            onReady();
+            lock (_syncRoot)
+            {
+                _openRecorded = true;
+            }
+
+            return true;
+        }
+        finally
+        {
+            try
+            {
+                _closeLock.Release();
+            }
+            catch (ObjectDisposedException)
+            {
+                // A double-dispose that slipped past DisposeAsync's own gate; the session is torn
+                // down either way and there is no one left to hand the slot to.
+            }
+        }
+    }
+
+    /// <summary>
+    ///     Gets a value indicating whether the open-session accounting
+    ///     (<c>GatewayMetrics.SessionOpened</c>) has been recorded for this session. Every
+    ///     teardown path consults it so a session closed or killed while its worker was still
+    ///     starting — never counted as opened — is not counted as closed either.
+    /// </summary>
+    /// <remarks>
+    ///     Written only by <see cref="TryAttachWorkerClientAndMarkReadyAsync"/>, inside the close
+    ///     gate; read by teardown paths that hold (or just released) the same gate, so the two
+    ///     accountings never observe a half-finished open.
+    /// </remarks>
+    internal bool OpenRecorded
+    {
+        get
+        {
+            lock (_syncRoot)
+            {
+                return _openRecorded;
+            }
        }
    }

@@ -481,6 +612,14 @@ public sealed class GatewaySession
    public void MarkReady()
    {
        TransitionTo(SessionState.Ready);
+        lock (_syncRoot)
+        {
+            // The unattached-grace idle clock starts here, not at open: the client cannot
+            // subscribe before it has the session, and a slow worker start must not hand it a
+            // session the next sweep reaps.
+            _readyAtUtc ??= _eventStreaming.TimeProvider.GetUtcNow();
+        }
+
        StartDashboardMirror();
    }

@@ -634,6 +773,15 @@ public sealed class GatewaySession
        SessionEventDistributor distributor = EnsureDistributorCreated(out bool startNow);
        IEventSubscriberLease lease = distributor.Register(isInternal: true);
        StartPumpIfRequested(distributor, startNow);
+        lock (_syncRoot)
+        {
+            // An in-process consumer (the alarm monitor) is a real subscriber for the
+            // unattached-grace rule: its session is attached, not abandoned. It stays out of
+            // _everHadEventSubscriber on purpose — that latch drives detach-grace, which is keyed
+            // on EXTERNAL subscribers only.
+            _everHadInternalEventSubscriber = true;
+        }
+
        return lease;
    }

@@ -856,12 +1004,54 @@ public sealed class GatewaySession
        IWorkerClient workerClient = await GetReadyWorkerClientAsync(cancellationToken).ConfigureAwait(false);
        TouchClientActivity(_eventStreaming.TimeProvider.GetUtcNow());

-        await foreach (WorkerEvent workerEvent in workerClient
-            .ReadEventsAsync(cancellationToken)
-            .WithCancellation(cancellationToken)
-            .ConfigureAwait(false))
+        try
        {
-            yield return mapper.MapEvent(workerEvent);
+            await foreach (WorkerEvent workerEvent in workerClient
+                .ReadEventsAsync(cancellationToken)
+                .WithCancellation(cancellationToken)
+                .ConfigureAwait(false))
+            {
+                yield return mapper.MapEvent(workerEvent);
+            }
+        }
+        finally
+        {
+            // The worker event stream only ends when the worker client has reached a terminal
+            // state (it enters Closed/Faulted BEFORE completing the channel) or the pump is being
+            // torn down. This is the session's one always-on view of the worker, so it is where
+            // a worker that went away on its own is noticed.
+            ObserveWorkerTerminalState();
+        }
+    }
+
+    // IWorkerClient has no state-change callback, so a worker that reached Closed/Faulted
+    // without the gateway closing it — the client sent ShutdownWorker and the worker exited,
+    // the worker crashed, its pipe broke, a heartbeat timed out — used to leave the session
+    // reporting Ready until its lease expired (live finding). The session follows the worker
+    // into Faulted: it is permanently unusable (every command fails the readiness check), stays
+    // observable via GetSessionStatus with the reason, and the faulted-session sweep reaps it
+    // after FaultedGraceSeconds, accounting SessionClosed exactly once there. Not Closed: Closed
+    // means the gateway closed it, and a Closed session still in the registry would linger until
+    // its lease expired with nothing reaping it. Only a Ready session follows — during a
+    // gateway-initiated close the worker closing is the expected outcome, not a fault.
+    private void ObserveWorkerTerminalState()
+    {
+        lock (_syncRoot)
+        {
+            if (_state is not SessionState.Ready || _workerClient is null)
+            {
+                return;
+            }
+
+            WorkerClientState workerState = _workerClient.State;
+            if (workerState is not (WorkerClientState.Closed or WorkerClientState.Faulted))
+            {
+                return;
+            }
+
+            MarkFaulted(
+                $"Worker for session {SessionId} reached {workerState} outside a gateway-initiated close "
+                + "(worker exited or faulted).");
        }
    }

@@ -901,18 +1091,6 @@ public sealed class GatewaySession
        }
    }

-    /// <summary>
-    /// Extends the session lease to the specified expiration time.
-    /// </summary>
-    /// <param name="leaseExpiresAt">Timestamp when the lease expires.</param>
-    public void ExtendLease(DateTimeOffset leaseExpiresAt)
-    {
-        lock (_syncRoot)
-        {
-            _leaseExpiresAt = leaseExpiresAt;
-        }
-    }
-
    /// <summary>
    /// Determines whether the session lease has expired.
    /// </summary>
@@ -969,6 +1147,25 @@ public sealed class GatewaySession
        }
    }

+    /// <summary>
+    ///     Determines whether a <see cref="SessionState.Ready"/> session that has <em>never</em>
+    ///     attached an event-stream subscriber has been idle — no unary command since open or
+    ///     since its last one — for at least the configured unattached grace. The lease monitor
+    ///     reaps such a session exactly as it reaps an expired lease, so a client that opened and
+    ///     vanished does not pin a worker for the full lease. Always <see langword="false"/> when
+    ///     the grace is disabled, when the session ever had a subscriber (detach-grace and the
+    ///     normal lease govern it from then on) or when one is attached now.
+    /// </summary>
+    /// <param name="now">Current timestamp for comparison.</param>
+    /// <returns><see langword="true"/> if the session is reapable as never-subscribed and idle; otherwise <see langword="false"/>.</returns>
+    public bool IsUnattachedGraceExpired(DateTimeOffset now)
+    {
+        lock (_syncRoot)
+        {
+            return IsUnattachedGraceExpiredCore(now);
+        }
+    }
+
    /// <summary>
    ///     Attaches an event subscriber and returns a lease whose
    ///     <see cref="IEventSubscriberLease.Reader"/> reads the fanned public
@@ -1627,6 +1824,7 @@ public sealed class GatewaySession
                    return new SessionCloseResult(SessionId, SessionState.Closed, AlreadyClosed: true);
                }

+                Exception? workerShutdownFailure = null;
                if (_workerClient is not null)
                {
                    try
@@ -1635,6 +1833,12 @@ public sealed class GatewaySession
                    }
                    catch (Exception exception)
                    {
+                        // The kill is the documented fallback for a worker that will not shut down
+                        // (typically a ShutdownTimeout on a hung worker). When it succeeds the
+                        // worker is gone and this close has done its job, so it completes as a
+                        // normal close carrying the shutdown failure for the caller to log; only a
+                        // kill that ALSO fails is a close failure (SessionCloseStartedException →
+                        // CloseFailed), because then a worker may still be running.
                        try
                        {
                            _workerClient.Kill(reason);
@@ -1646,12 +1850,12 @@ public sealed class GatewaySession
                                new AggregateException(exception, killException));
                        }

-                        throw;
+                        workerShutdownFailure = exception;
                    }
                }

                MarkClosed();
-                return new SessionCloseResult(SessionId, SessionState.Closed, alreadyClosing);
+                return new SessionCloseResult(SessionId, SessionState.Closed, alreadyClosing, workerShutdownFailure);
            }
            catch (Exception exception) when (exception is not SessionCloseStartedException)
            {
@@ -1718,6 +1922,15 @@ public sealed class GatewaySession
    ///         committing to <c>Closing</c>, so a reattach that wins the race leaves the session
    ///         in <c>Ready</c> and usable.
    ///     </para>
+    ///     <para>
+    ///         This is a <em>reservation</em>, not the close itself: it flips the state so no
+    ///         reattach can revive the session and no other sweep pass re-selects it, but it
+    ///         deliberately leaves <c>_closeStarted</c> untouched. The sweep's own
+    ///         <see cref="CloseAsync"/> that follows is the first close to run, and must report
+    ///         <c>AlreadyClosed: false</c> so the manager accounts <c>SessionClosed</c> for it;
+    ///         setting <c>_closeStarted</c> here made that close look like a second, concurrent one
+    ///         and every sweep-driven close went unaccounted.
+    ///     </para>
    /// </remarks>
    internal bool TryBeginCloseIfExpired(DateTimeOffset now, out bool alreadyClosing)
    {
@@ -1731,7 +1944,10 @@ public sealed class GatewaySession

            // Re-verify eligibility atomically. If a subscriber reattached between the sweep's
            // eligibility check and this point, neither condition holds and we decline.
-            bool eligible = IsLeaseExpiredCore(now) || IsFaultedReapableCore(now) || IsDetachGraceExpiredCore(now);
+            bool eligible = IsLeaseExpiredCore(now)
+                || IsFaultedReapableCore(now)
+                || IsDetachGraceExpiredCore(now)
+                || IsUnattachedGraceExpiredCore(now);
            if (!eligible)
            {
                alreadyClosing = false;
@@ -1739,7 +1955,6 @@ public sealed class GatewaySession
            }

            alreadyClosing = _closeStarted;
-            _closeStarted = true;
            _state = SessionState.Closing;
            return true;
        }
@@ -1763,6 +1978,29 @@ public sealed class GatewaySession
                || _faultedAtUtc is null
                || now - _faultedAtUtc.Value >= _faultedGrace);

+    // Ready only: a session still starting its worker is not idle, it is waiting, and reaping it
+    // mid-startup would turn a slow worker start into an OpenFailed for a client that did nothing
+    // wrong. Idle is measured from the later of Ready and the last client activity (last unary
+    // command) rather than from open, so a slow start does not eat the window and a unary-only
+    // client that keeps calling is never reaped by this rule. An internal (in-process) subscriber
+    // — the gateway's own alarm monitor — counts as attached, or the gateway would reap its own
+    // monitor session every cycle.
+    private bool IsUnattachedGraceExpiredCore(DateTimeOffset now)
+    {
+        if (_unattachedGrace <= TimeSpan.Zero
+            || _state is not SessionState.Ready
+            || _readyAtUtc is null
+            || _everHadEventSubscriber
+            || _everHadInternalEventSubscriber
+            || _activeEventSubscriberCount != 0)
+        {
+            return false;
+        }
+
+        DateTimeOffset idleSince = _lastClientActivityAt > _readyAtUtc.Value ? _lastClientActivityAt : _readyAtUtc.Value;
+        return now - idleSince >= _unattachedGrace;
+    }
+
    // Final terminal transition; under _syncRoot to keep _state writes single-lock.
    // Closed is unconditionally terminal — TransitionTo refuses to overwrite it —
    // so we don't need to re-check the precondition here.
@@ -2,7 +2,21 @@ using ZB.MOM.WW.MxGateway.Contracts.Proto;

 namespace ZB.MOM.WW.MxGateway.Server.Sessions;

+/// <summary>Outcome of closing a session.</summary>
+/// <param name="SessionId">Identifier of the closed session.</param>
+/// <param name="FinalState">State the session ended in.</param>
+/// <param name="AlreadyClosed">
+///     <see langword="true"/> when the session was already closed (or being closed by another
+///     caller) when this close acquired the close lock, so this call did no teardown of its own.
+/// </param>
+/// <param name="WorkerShutdownFailure">
+///     The exception the worker's graceful shutdown threw when the close had to fall back to
+///     killing the worker; <see langword="null"/> when the worker shut down cleanly. The close is
+///     still a success — the worker is gone and the session is closed — so this is diagnostic
+///     detail for the caller's log, not an error.
+/// </param>
 public sealed record SessionCloseResult(
    string SessionId,
    SessionState FinalState,
-    bool AlreadyClosed);
+    bool AlreadyClosed,
+    Exception? WorkerShutdownFailure = null);
@@ -2,6 +2,7 @@ using System.Diagnostics.CodeAnalysis;
 using System.Runtime.ExceptionServices;
 using System.Security.Cryptography;
 using Google.Protobuf.WellKnownTypes;
+using Microsoft.Extensions.Hosting;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Logging.Abstractions;
 using Microsoft.Extensions.Options;
@@ -20,6 +21,7 @@ public sealed class SessionManager : ISessionManager
    public const string LeaseExpiredReason = "lease-expired";
    public const string DetachGraceExpiredReason = "detach-grace-expired";
    public const string FaultedReason = "faulted-reaped";
+    public const string UnattachedGraceExpiredReason = "unattached-grace-expired";

    // Bounded so a mass expiry (or a host stop with a full registry) cannot stampede
    // worker-process teardown: every concurrent close is one x86 worker being shut down or
@@ -27,6 +29,13 @@ public sealed class SessionManager : ISessionManager
    // registry down at once.
    private const int MaxParallelSessionCloses = 4;

+    // ShutdownAsync re-snapshots the registry until it is empty so an open that registered its
+    // session just after a snapshot is still drained. The stopping gate in OpenSessionAsync is
+    // what actually closes that window (it is re-checked after registration), so the loop is a
+    // backstop and a small bound is enough; anything still registered after the last pass is
+    // logged and left to the orphan terminator on the next start.
+    private const int MaxShutdownDrainPasses = 3;
+
    private readonly ISessionRegistry _registry;
    private readonly ISessionWorkerClientFactory _workerClientFactory;
    private readonly GatewayMetrics _metrics;
@@ -39,6 +48,12 @@ public sealed class SessionManager : ISessionManager
    private readonly Dashboard.Hubs.IDashboardEventBroadcaster? _dashboardEventBroadcaster;
    private readonly ArrayAddressNormalizer? _addressNormalizer;

+    // 0 = accepting opens, 1 = stopping. Set by the host's ApplicationStopping signal (which fires
+    // before hosted services stop, while Kestrel still accepts gRPC calls) and, as a fallback for
+    // hosts without a lifetime, at the start of ShutdownAsync. Never reset: a manager that has
+    // begun draining does not come back.
+    private int _stopping;
+
    /// <summary>
    /// Initializes a new instance of <see cref="SessionManager"/>.
    /// </summary>
@@ -61,6 +76,12 @@ public sealed class SessionManager : ISessionManager
    ///     metadata; handed to each session so the normalization runs at the outbound choke point.
    ///     Null in unit tests that do not exercise array-write ergonomics.
    /// </param>
+    /// <param name="applicationLifetime">
+    ///     Host lifetime whose <c>ApplicationStopping</c> closes the open gate as soon as the host
+    ///     begins stopping — before hosted services stop and while Kestrel is still accepting gRPC
+    ///     calls — so no session is opened behind the shutdown drain. Null in unit tests; the gate
+    ///     then closes when <see cref="ShutdownAsync"/> starts.
+    /// </param>
    public SessionManager(
        ISessionRegistry registry,
        ISessionWorkerClientFactory workerClientFactory,
@@ -71,7 +92,8 @@ public sealed class SessionManager : ISessionManager
        Grpc.MxAccessGrpcMapper? eventMapper = null,
        ILogger<SessionEventDistributor>? distributorLogger = null,
        Dashboard.Hubs.IDashboardEventBroadcaster? dashboardEventBroadcaster = null,
-        ArrayAddressNormalizer? addressNormalizer = null)
+        ArrayAddressNormalizer? addressNormalizer = null,
+        IHostApplicationLifetime? applicationLifetime = null)
    {
        _registry = registry ?? throw new ArgumentNullException(nameof(registry));
        _workerClientFactory = workerClientFactory ?? throw new ArgumentNullException(nameof(workerClientFactory));
@@ -85,6 +107,10 @@ public sealed class SessionManager : ISessionManager
        _addressNormalizer = addressNormalizer;
        _options = options.Value;
        _sessionSlots = new SemaphoreSlim(_options.Sessions.MaxSessions, _options.Sessions.MaxSessions);
+
+        // The registration is never disposed: the manager is a host singleton that lives exactly
+        // as long as the lifetime it observes.
+        applicationLifetime?.ApplicationStopping.Register(MarkStopping);
    }

    /// <inheritdoc />
@@ -104,10 +130,11 @@ public sealed class SessionManager : ISessionManager
        CancellationToken cancellationToken)
    {
        ArgumentNullException.ThrowIfNull(request);
+        ThrowIfStopping();
        EnsureSessionCapacity();

        GatewaySession? session = null;
-        bool sessionOpenedRecorded = false;
+        bool registered = false;
        try
        {
            session = CreateSession(request, clientIdentity, ownerKeyId, ownerDashboardTags);
@@ -118,36 +145,58 @@ public sealed class SessionManager : ISessionManager
                    $"Session id collision while opening session {session.SessionId}.");
            }

+            registered = true;
+
+            // Re-check after registration. The stopping flag is written before ShutdownAsync takes
+            // its registry snapshot, so an open that does not observe it here registered its
+            // session before that snapshot and will be drained; one that does observe it rolls
+            // back itself. Either way no session is created behind the drain.
+            ThrowIfStopping();
            session.TransitionTo(SessionState.StartingWorker);
            IWorkerClient workerClient = await _workerClientFactory
                .CreateAsync(session, cancellationToken)
                .ConfigureAwait(false);

-            session.AttachWorkerClient(workerClient);
-            session.MarkReady();
-            _metrics.SessionOpened();
-            sessionOpenedRecorded = true;
+            // From here on a live worker exists and nothing but this method knows about it: every
+            // way out of this block other than "attached and Ready" must discard (kill + dispose)
+            // it, or it is an orphan the gateway cannot reattach. The session has been registered
+            // (and killable / drainable) for the whole worker startup, so the attach is refused
+            // if a kill or the host drain closed it meanwhile — it must not be returned on a dead
+            // session; and a caller that abandoned the open while the worker was starting gets
+            // no session either.
+            bool ready;
+            try
+            {
+                cancellationToken.ThrowIfCancellationRequested();
+                ready = await session
+                    .TryAttachWorkerClientAndMarkReadyAsync(workerClient, _metrics.SessionOpened)
+                    .ConfigureAwait(false);
+            }
+            catch
+            {
+                await DiscardWorkerClientAsync(session, workerClient).ConfigureAwait(false);
+                throw;
+            }
+
+            if (!ready)
+            {
+                await DiscardWorkerClientAsync(session, workerClient).ConfigureAwait(false);
+                throw new InvalidOperationException(
+                    $"Session {session.SessionId} was closed while its worker was starting.");
+            }

            return session;
        }
+        catch (SessionManagerException exception) when (exception.ErrorCode == SessionManagerErrorCode.GatewayStopping)
+        {
+            // Not an open failure: the gateway is going away and the caller should retry against
+            // the restarted one, so no fault is recorded and the code is surfaced as-is.
+            await RollBackOpenAsync(session, registered, exception).ConfigureAwait(false);
+            throw;
+        }
        catch (Exception exception)
        {
-            session?.MarkFaulted(exception.Message);
-            if (session is not null)
-            {
-                _registry.TryRemove(session.SessionId, out _);
-                await session.DisposeAsync().ConfigureAwait(false);
-            }
-
-            // If SessionOpened() already incremented the open-session gauge,
-            // a failure after that point (e.g. auto-subscribe rejection) must
-            // decrement it again so mxgateway.sessions.open does not leak.
-            if (sessionOpenedRecorded)
-            {
-                _metrics.SessionRemoved();
-            }
-
-            ReleaseSessionSlot();
+            await RollBackOpenAsync(session, registered, exception).ConfigureAwait(false);
            _metrics.Fault(SessionManagerErrorCode.OpenFailed.ToString());
            _logger.LogWarning(
                exception,
@@ -161,6 +210,62 @@ public sealed class SessionManager : ISessionManager
        }
    }

+    // Order — fault, deregister (dropping per-session event metrics), release the slot, dispose —
+    // matters: releasing the slot before disposal would let the next open race this worker's
+    // teardown on the same machine. A registered session goes through RemoveSessionAsync, the
+    // single removal path, which is a no-op when a concurrent kill or drain already removed it
+    // (that teardown released the slot and disposed the session). The open-session gauge is never
+    // touched here: SessionOpened is recorded inside the attach gate, after which nothing in the
+    // open sequence can fail.
+    private async Task RollBackOpenAsync(GatewaySession? session, bool registered, Exception cause)
+    {
+        if (session is null)
+        {
+            ReleaseSessionSlot();
+            return;
+        }
+
+        session.MarkFaulted(cause.Message);
+        if (registered)
+        {
+            await RemoveSessionAsync(session).ConfigureAwait(false);
+            return;
+        }
+
+        ReleaseSessionSlot();
+        await session.DisposeAsync().ConfigureAwait(false);
+    }
+
+    // Kills and disposes a worker client that arrived after its session was closed. Best-effort:
+    // the client is already orphaned from the session, so a failure here is logged rather than
+    // allowed to mask the open failure the caller is about to receive.
+    private async Task DiscardWorkerClientAsync(GatewaySession session, IWorkerClient workerClient)
+    {
+        try
+        {
+            workerClient.Kill("session-closed-during-startup");
+        }
+        catch (Exception exception)
+        {
+            _logger.LogWarning(
+                exception,
+                "Failed to kill the late worker for closed session {SessionId}.",
+                session.SessionId);
+        }
+
+        try
+        {
+            await workerClient.DisposeAsync().ConfigureAwait(false);
+        }
+        catch (Exception exception)
+        {
+            _logger.LogWarning(
+                exception,
+                "Failed to dispose the late worker client for closed session {SessionId}.",
+                session.SessionId);
+        }
+    }
+
    /// <inheritdoc />
    public bool TryGetSession(
        string sessionId,
@@ -226,16 +331,27 @@ public sealed class SessionManager : ISessionManager
        {
            wasClosed = await session.KillWorkerWithCloseGateAsync(reason, cancellationToken).ConfigureAwait(false);
        }
+        catch (ObjectDisposedException) when (!_registry.TryGet(sessionId, out _))
+        {
+            // Lost the race to a concurrent close: between the lookup above and the gate wait, the
+            // close finished, removed the session, accounted SessionClosed and disposed the gate.
+            // Nothing is left to kill and nothing to account; report it as already closed.
+            return new SessionCloseResult(sessionId, SessionState.Closed, AlreadyClosed: true);
+        }
        catch (Exception exception)
        {
            session.MarkFaulted(exception.Message);
            _metrics.Fault(SessionManagerErrorCode.CloseFailed.ToString());

-            // The open-session gauge was incremented in OpenSessionAsync; every
-            // session reaching KillWorkerAsync had SessionOpened recorded. If the
-            // kill path throws, decrement the gauge here so mxgateway.sessions.open
-            // does not leak — mirroring the equivalent guard in OpenSessionAsync.
-            _metrics.SessionRemoved();
+            // If the open-session gauge was incremented for this session (it was not when the
+            // kill landed while the worker was still starting) and nobody has removed it yet
+            // (a removal accounts the gauge itself), decrement it here so mxgateway.sessions.open
+            // does not leak when the kill path throws.
+            if (session.OpenRecorded && _registry.TryGet(sessionId, out _))
+            {
+                _metrics.SessionRemoved();
+            }
+
            await RemoveSessionAsync(session).ConfigureAwait(false);
            throw new SessionManagerException(
                SessionManagerErrorCode.CloseFailed,
@@ -243,7 +359,7 @@ public sealed class SessionManager : ISessionManager
                exception);
        }

-        if (!wasClosed)
+        if (!wasClosed && session.OpenRecorded)
        {
            _metrics.SessionClosed();
        }
@@ -270,12 +386,14 @@ public sealed class SessionManager : ISessionManager
        {
            // A session is swept when its normal lease has expired, it has FAULTED (a faulted
            // session is permanently unusable yet otherwise pins a session slot and a live x86
-            // worker until its DefaultLeaseSeconds lease expires), OR its detach-grace retention
+            // worker until its DefaultLeaseSeconds lease expires), its detach-grace retention
            // window has elapsed (last external subscriber dropped and no client reconnected
-            // within DetachGraceSeconds). All three are the same teardown; only the reason differs
-            // so operators can tell a fault reap from a short reconnect-window expiry from a long
-            // idle-lease expiry in logs/metrics. Precedence when several fire simultaneously:
-            // lease-expired, then faulted, then detach-grace.
+            // within DetachGraceSeconds), OR it never attached a subscriber and has been idle for
+            // UnattachedSessionGraceSeconds (a client that opened and vanished). All four are the
+            // same teardown; only the reason differs so operators can tell a fault reap from a
+            // short reconnect-window expiry from a never-subscribed reap from a long idle-lease
+            // expiry in logs/metrics. Precedence when several fire simultaneously: lease-expired,
+            // then faulted, then detach-grace, then unattached-grace.
            //
            // TOCTOU note: eligibility is re-verified atomically inside TryBeginCloseIfExpired
            // under _syncRoot, so a client that reattaches a subscriber between the check above
@@ -286,7 +404,9 @@ public sealed class SessionManager : ISessionManager
                    ? FaultedReason
                    : session.IsDetachGraceExpired(now)
                        ? DetachGraceExpiredReason
-                        : null;
+                        : session.IsUnattachedGraceExpired(now)
+                            ? UnattachedGraceExpiredReason
+                            : null;
            if (reason is null)
            {
                continue;
@@ -330,8 +450,11 @@ public sealed class SessionManager : ISessionManager
            {
                try
                {
-                    await CloseSessionCoreAsync(candidate.Session, candidate.Reason, closeToken).ConfigureAwait(false);
-                    Interlocked.Increment(ref closedCount);
+                    SessionCloseResult result = await CloseSessionCoreAsync(candidate.Session, candidate.Reason, closeToken).ConfigureAwait(false);
+                    if (!result.AlreadyClosed)
+                    {
+                        Interlocked.Increment(ref closedCount);
+                    }
                }
                catch (Exception exception)
                {
@@ -381,49 +504,86 @@ public sealed class SessionManager : ISessionManager
        // intentional: that sweep is periodic maintenance whose missed sessions are picked up by
        // the next pass and, ultimately, by this drain. This drain is terminal — nothing runs after
        // it — so it must not be abandoned partway.
-        await Parallel.ForEachAsync(
-            _registry.Snapshot(),
-            new ParallelOptions { MaxDegreeOfParallelism = MaxParallelSessionCloses },
-            async (session, _) =>
+        //
+        // Close the open gate BEFORE the first snapshot (see the re-check in OpenSessionAsync), then
+        // re-snapshot until the registry is empty so an open that registered just before the gate
+        // closed is drained too.
+        MarkStopping();
+        for (int pass = 0; pass < MaxShutdownDrainPasses; pass++)
+        {
+            if (_registry.Count == 0)
            {
-                try
-                {
-                    await CloseSessionCoreAsync(session, GatewayShutdownReason, cancellationToken).ConfigureAwait(false);
-                }
-                catch (Exception exception)
-                {
-                    _logger.LogWarning(
-                        exception,
-                        "Graceful shutdown failed for session {SessionId}; killing worker.",
-                        session.SessionId);
+                return;
+            }

-                    // Defensive fallback: CloseSessionCoreAsync's inner
-                    // SessionCloseStartedException catch normally removes the session
-                    // and accounts the close. This outer fallback only fires for
-                    // sessions still in the registry — route through KillWorkerAsync
-                    // so the bookkeeping is identical to the dashboard kill path.
-                    if (_registry.TryGet(session.SessionId, out GatewaySession? registeredSession)
-                        && registeredSession is not null)
+            await Parallel.ForEachAsync(
+                _registry.Snapshot(),
+                new ParallelOptions { MaxDegreeOfParallelism = MaxParallelSessionCloses },
+                async (session, _) =>
+                {
+                    try
                    {
-                        try
+                        await CloseSessionCoreAsync(session, GatewayShutdownReason, cancellationToken).ConfigureAwait(false);
+                    }
+                    catch (Exception exception)
+                    {
+                        _logger.LogWarning(
+                            exception,
+                            "Graceful shutdown failed for session {SessionId}; killing worker.",
+                            session.SessionId);
+
+                        // Defensive fallback: CloseSessionCoreAsync's inner
+                        // SessionCloseStartedException catch normally removes the session
+                        // and accounts the close. This outer fallback only fires for
+                        // sessions still in the registry — route through KillWorkerAsync
+                        // so the bookkeeping is identical to the dashboard kill path.
+                        if (_registry.TryGet(session.SessionId, out GatewaySession? registeredSession)
+                            && registeredSession is not null)
                        {
-                            // Deliberately NOT the caller's token: the kill is the last-resort orphan
-                            // preventer, so it must still run when the host stop deadline (or a
-                            // sibling body's failure) has already cancelled the drain. It is a
-                            // synchronous Kill plus registry/dispose bookkeeping, not a wait on
-                            // the worker, so it cannot extend the drain meaningfully.
-                            await KillWorkerAsync(session.SessionId, GatewayShutdownReason, CancellationToken.None).ConfigureAwait(false);
-                        }
-                        catch (Exception killException)
-                        {
-                            _logger.LogWarning(
-                                killException,
-                                "Worker kill fallback failed for session {SessionId}.",
-                                session.SessionId);
+                            try
+                            {
+                                // Deliberately NOT the caller's token: the kill is the last-resort orphan
+                                // preventer, so it must still run when the host stop deadline (or a
+                                // sibling body's failure) has already cancelled the drain. It is a
+                                // synchronous Kill plus registry/dispose bookkeeping, not a wait on
+                                // the worker, so it cannot extend the drain meaningfully.
+                                await KillWorkerAsync(session.SessionId, GatewayShutdownReason, CancellationToken.None).ConfigureAwait(false);
+                            }
+                            catch (Exception killException)
+                            {
+                                _logger.LogWarning(
+                                    killException,
+                                    "Worker kill fallback failed for session {SessionId}.",
+                                    session.SessionId);
+                            }
                        }
                    }
-                }
-            }).ConfigureAwait(false);
+                }).ConfigureAwait(false);
+        }
+
+        int remaining = _registry.Count;
+        if (remaining > 0)
+        {
+            _logger.LogWarning(
+                "Session drain finished with {Remaining} session(s) still registered after {Passes} passes; their workers are left to the orphan terminator on the next start.",
+                remaining,
+                MaxShutdownDrainPasses);
+        }
+    }
+
+    private void MarkStopping()
+    {
+        Volatile.Write(ref _stopping, 1);
+    }
+
+    private void ThrowIfStopping()
+    {
+        if (Volatile.Read(ref _stopping) != 0)
+        {
+            throw new SessionManagerException(
+                SessionManagerErrorCode.GatewayStopping,
+                "The gateway is stopping and is not accepting new sessions.");
+        }
    }

    private async Task<SessionCloseResult> CloseSessionCoreAsync(
@@ -435,7 +595,19 @@ public sealed class SessionManager : ISessionManager
        try
        {
            SessionCloseResult result = await session.CloseAsync(reason, cancellationToken).ConfigureAwait(false);
-            if (!wasClosed && !result.AlreadyClosed)
+            if (result.WorkerShutdownFailure is not null)
+            {
+                // A clean close from the session's point of view (the worker was killed and the
+                // session is Closed), but the operator should see that the worker would not stop
+                // gracefully — a run of these is the signal that workers are hanging on shutdown.
+                _logger.LogWarning(
+                    result.WorkerShutdownFailure,
+                    "Worker for session {SessionId} did not shut down gracefully and was killed; reason={Reason}.",
+                    session.SessionId,
+                    reason);
+            }
+
+            if (!wasClosed && !result.AlreadyClosed && session.OpenRecorded)
            {
                _metrics.SessionClosed();
            }
@@ -447,10 +619,17 @@ public sealed class SessionManager : ISessionManager
        {
            throw;
        }
+        catch (ObjectDisposedException) when (!_registry.TryGet(session.SessionId, out _))
+        {
+            // Lost the race to a concurrent close that removed, accounted and disposed the
+            // session after this caller picked it up (a sweep's snapshot, a client close that
+            // looked it up first). Not a failure and nothing to redo.
+            return new SessionCloseResult(session.SessionId, SessionState.Closed, AlreadyClosed: true);
+        }
        catch (SessionCloseStartedException exception)
        {
            session.MarkFaulted(exception.Message);
-            if (!wasClosed)
+            if (!wasClosed && session.OpenRecorded)
            {
                // Account the close as a SessionClosed (decrements the open-session
                // gauge AND increments the sessions.closed counter), not just
@@ -566,7 +745,8 @@ public sealed class SessionManager : ISessionManager
            TimeSpan.FromMilliseconds(Math.Max(0, _options.Sessions.WorkerReadyWaitTimeoutMs)),
            _addressNormalizer,
            TimeSpan.FromSeconds(Math.Max(0, _options.Sessions.FaultedGraceSeconds)),
-            ownerDashboardTags);
+            ownerDashboardTags,
+            TimeSpan.FromSeconds(Math.Max(0, _options.Sessions.UnattachedSessionGraceSeconds)));
    }

    private static string CreateClientCorrelationId(
@@ -17,4 +17,12 @@ public enum SessionManagerErrorCode
    ///     Maps to gRPC <c>PermissionDenied</c>.
    /// </summary>
    PermissionDenied,
+
+    /// <summary>
+    ///     The gateway is stopping, so no new session can be opened: a session created behind the
+    ///     shutdown drain's snapshot would have nobody left to close it and its worker would
+    ///     outlive the gateway as an orphan. Maps to gRPC <c>Unavailable</c> so the client retries
+    ///     against the restarted gateway.
+    /// </summary>
+    GatewayStopping,
 }
@@ -51,6 +51,7 @@
      "DefaultLeaseSeconds": 1800,
      "LeaseSweepIntervalSeconds": 30,
      "DetachGraceSeconds": 30,
+      "UnattachedSessionGraceSeconds": 0,
      "AllowMultipleEventSubscribers": false,
      "MaxEventSubscribersPerSession": 8
    },