feat(sessions): detach-grace retention window for reconnect

src/ZB.MOM.WW.MxGateway.Server/Configuration/GatewayOptionsValidator.cs

@@ -181,6 +181,10 @@ public sealed class GatewayOptionsValidator : OptionsValidatorBase<GatewayOption
             options.MaxEventSubscribersPerSession,
             "MxGateway:Sessions:MaxEventSubscribersPerSession must be greater than zero.",
             builder);
+        AddIfNegative(
+            options.DetachGraceSeconds,
+            "MxGateway:Sessions:DetachGraceSeconds must be zero or greater (0 disables detach-grace retention).",
+            builder);
 
         // NOTE: We intentionally do NOT reject !AllowMultipleEventSubscribers &&
         // MaxEventSubscribersPerSession > 1 as a hard validation error here. The default

src/ZB.MOM.WW.MxGateway.Server/Configuration/SessionOptions.cs

@@ -23,6 +23,21 @@ public sealed class SessionOptions
     /// <summary>Gets the interval for sweeping expired session leases in seconds.</summary>
     public int LeaseSweepIntervalSeconds { get; init; } = 30;
 
+    /// <summary>
+    ///     Gets the detach-grace retention window, in seconds, that a session is kept alive
+    ///     after its last external (gRPC) event-stream subscriber drops, so a client can
+    ///     reconnect to it. While within the window the session stays in
+    ///     <c>Ready</c> and remains usable; if no new external subscriber attaches before the
+    ///     window elapses, the lease monitor closes the session exactly as it closes an
+    ///     expired lease. The gateway-owned internal dashboard subscriber does not count as an
+    ///     external subscriber, so a session whose only remaining subscriber is the dashboard
+    ///     mirror still enters detach-grace. A value of <c>0</c> disables retention: the
+    ///     session reverts to the original behavior of lingering only until its normal lease
+    ///     expires. The reconnect/replay itself is implemented separately (Task 12); this
+    ///     option controls retention and expiry only.
+    /// </summary>
+    public int DetachGraceSeconds { get; init; } = 30;
+
     /// <summary>
     /// Gets a value indicating whether multiple event subscribers are allowed per session.
     /// </summary>

src/ZB.MOM.WW.MxGateway.Server/Sessions/GatewaySession.cs

@@ -22,6 +22,8 @@ public sealed class GatewaySession
     private DateTimeOffset? _leaseExpiresAt;
     private bool _closeStarted;
     private int _activeEventSubscriberCount;
+    private readonly TimeSpan _detachGrace;
+    private DateTimeOffset? _detachedAtUtc;
     private SessionEventDistributor? _eventDistributor;
     private bool _eventDistributorStarted;
     private bool _dashboardMirrorStarted;
@@ -103,6 +105,16 @@ public sealed class GatewaySession
     ///     session directly still get a working distributor. Production passes the
     ///     DI-resolved dependencies.
     /// </param>
+    /// <param name="detachGrace">
+    ///     Retention window kept after the last external (gRPC) event subscriber drops, so a
+    ///     client can reconnect (Task 12). When the window is positive and the active external
+    ///     subscriber count falls to zero, the session stays <see cref="SessionState.Ready"/>
+    ///     and records a detached timestamp; the lease monitor closes it once the window
+    ///     elapses with no subscriber having re-attached. <see cref="TimeSpan.Zero"/> (the
+    ///     default) disables retention and preserves the original lease-only expiry behavior.
+    ///     The clock comes from <paramref name="eventStreaming"/>'s
+    ///     <see cref="SessionEventStreaming.TimeProvider"/> so the timer is unit-testable.
+    /// </param>
     public GatewaySession(
         string sessionId,
         string backendName,
@@ -117,7 +129,8 @@ public sealed class GatewaySession
         TimeSpan shutdownTimeout,
         TimeSpan leaseDuration,
         DateTimeOffset openedAt,
-        SessionEventStreaming? eventStreaming = null)
+        SessionEventStreaming? eventStreaming = null,
+        TimeSpan detachGrace = default)
     {
         if (string.IsNullOrWhiteSpace(sessionId))
         {
@@ -155,6 +168,7 @@ public sealed class GatewaySession
         _lastClientActivityAt = openedAt;
         _leaseExpiresAt = openedAt + leaseDuration;
         _eventStreaming = eventStreaming ?? SessionEventStreaming.Default;
+        _detachGrace = detachGrace > TimeSpan.Zero ? detachGrace : TimeSpan.Zero;
     }
 
     /// <summary>
@@ -300,6 +314,25 @@ public sealed class GatewaySession
         }
     }
 
+    /// <summary>
+    ///     Gets the UTC timestamp at which the session entered its detach-grace retention
+    ///     window (the last external event subscriber dropped while a positive
+    ///     detach-grace was configured), or <see langword="null"/> when the session is not
+    ///     currently within a detach-grace window. Re-attaching an external subscriber clears
+    ///     this. Always <see langword="null"/> when detach-grace is disabled
+    ///     (<c>DetachGraceSeconds == 0</c>).
+    /// </summary>
+    public DateTimeOffset? DetachedAtUtc
+    {
+        get
+        {
+            lock (_syncRoot)
+            {
+                return _detachedAtUtc;
+            }
+        }
+    }
+
     /// <summary>
     /// Attaches the worker client for this session.
     /// </summary>
@@ -679,6 +712,28 @@ public sealed class GatewaySession
         }
     }
 
+    /// <summary>
+    ///     Determines whether the session's detach-grace retention window has elapsed: the
+    ///     session entered detach-grace (its last external event subscriber dropped while a
+    ///     positive detach-grace was configured) and has had no external subscriber re-attach
+    ///     for longer than the configured detach-grace. The lease monitor closes such a
+    ///     session exactly as it closes an expired lease. Always returns <see langword="false"/>
+    ///     when detach-grace is disabled or when an external subscriber is attached (the
+    ///     detached timestamp is cleared on re-attach, so an attached session is never within a
+    ///     window).
+    /// </summary>
+    /// <param name="now">Current timestamp for comparison.</param>
+    public bool IsDetachGraceExpired(DateTimeOffset now)
+    {
+        lock (_syncRoot)
+        {
+            return _detachGrace > TimeSpan.Zero
+                && _activeEventSubscriberCount == 0
+                && _detachedAtUtc is not null
+                && now - _detachedAtUtc.Value >= _detachGrace;
+        }
+    }
+
     /// <summary>
     ///     Attaches an event subscriber and returns a lease whose
     ///     <see cref="IEventSubscriberLease.Reader"/> reads the fanned public
@@ -733,6 +788,12 @@ public sealed class GatewaySession
             }
 
             _activeEventSubscriberCount++;
+
+            // An external subscriber (re)attached: cancel any in-flight detach-grace window so
+            // the lease monitor no longer treats this session as eligible for grace-expiry
+            // close. This is the reattach→grace-cancel transition; it races the sweeper's
+            // IsDetachGraceExpired read, and both run under _syncRoot so they serialize.
+            _detachedAtUtc = null;
         }
 
         // Construct/start the distributor and register this subscriber. Done outside the
@@ -1502,6 +1563,24 @@ public sealed class GatewaySession
             {
                 _activeEventSubscriberCount--;
             }
+
+            // When the LAST external subscriber drops and detach-grace is enabled, retain the
+            // session instead of letting it linger only on the (long) lease: stamp the detached
+            // time so the lease monitor can close it once the grace window elapses. The session
+            // stays in its current (Ready) state and remains usable, so a reconnecting subscriber
+            // (Task 12) re-attaches normally. The gateway-owned internal dashboard subscriber is
+            // NOT counted in _activeEventSubscriberCount (it registers on the distributor with
+            // isInternal: true), so a session whose only remaining subscriber is the dashboard
+            // mirror still enters grace. Only stamp while the session is alive — once
+            // Closing/Closed/Faulted there is nothing to retain. This is the detach→grace-start
+            // transition; it shares _syncRoot with the reattach→grace-cancel write above and the
+            // sweeper's IsDetachGraceExpired read, so the three serialize.
+            if (_detachGrace > TimeSpan.Zero
+                && _activeEventSubscriberCount == 0
+                && _state is not (SessionState.Closing or SessionState.Closed or SessionState.Faulted))
+            {
+                _detachedAtUtc = _eventStreaming.TimeProvider.GetUtcNow();
+            }
         }
     }
 

src/ZB.MOM.WW.MxGateway.Server/Sessions/SessionManager.cs

@@ -17,6 +17,7 @@ public sealed class SessionManager : ISessionManager
     public const string DefaultCloseReason = "client-close";
     public const string GatewayShutdownReason = "gateway-shutdown";
     public const string LeaseExpiredReason = "lease-expired";
+    public const string DetachGraceExpiredReason = "detach-grace-expired";
 
     private readonly ISessionRegistry _registry;
     private readonly ISessionWorkerClientFactory _workerClientFactory;
@@ -295,12 +296,22 @@ public sealed class SessionManager : ISessionManager
         int closedCount = 0;
         foreach (GatewaySession session in _registry.Snapshot())
         {
-            if (!session.IsLeaseExpired(now))
+            // A session is swept when its normal lease has expired OR its detach-grace
+            // retention window has elapsed (last external subscriber dropped and no client
+            // reconnected within DetachGraceSeconds). The detach-grace close is the same
+            // teardown as a lease-expiry close; only the reason differs so operators can tell
+            // a short reconnect-window expiry from a long idle-lease expiry in logs/metrics.
+            string? reason = session.IsLeaseExpired(now)
+                ? LeaseExpiredReason
+                : session.IsDetachGraceExpired(now)
+                    ? DetachGraceExpiredReason
+                    : null;
+            if (reason is null)
             {
                 continue;
             }
 
-            await CloseSessionCoreAsync(session, LeaseExpiredReason, cancellationToken).ConfigureAwait(false);
+            await CloseSessionCoreAsync(session, reason, cancellationToken).ConfigureAwait(false);
             closedCount++;
         }
 
@@ -478,7 +489,8 @@ public sealed class SessionManager : ISessionManager
             shutdownTimeout,
             leaseDuration,
             openedAt,
-            eventStreaming);
+            eventStreaming,
+            TimeSpan.FromSeconds(Math.Max(0, _options.Sessions.DetachGraceSeconds)));
     }
 
     private static string CreateClientCorrelationId(

src/ZB.MOM.WW.MxGateway.Server/appsettings.json

@@ -46,6 +46,7 @@
       "MaxPendingCommandsPerSession": 128,
       "DefaultLeaseSeconds": 1800,
       "LeaseSweepIntervalSeconds": 30,
+      "DetachGraceSeconds": 30,
       "AllowMultipleEventSubscribers": false,
       "MaxEventSubscribersPerSession": 8
     },

src/ZB.MOM.WW.MxGateway.Tests/Configuration/GatewayOptionsTests.cs

@@ -34,6 +34,7 @@ public sealed class GatewayOptionsTests
         Assert.Equal(128, options.Sessions.MaxPendingCommandsPerSession);
         Assert.Equal(1800, options.Sessions.DefaultLeaseSeconds);
         Assert.Equal(30, options.Sessions.LeaseSweepIntervalSeconds);
+        Assert.Equal(30, options.Sessions.DetachGraceSeconds);
         Assert.False(options.Sessions.AllowMultipleEventSubscribers);
         Assert.Equal(8, options.Sessions.MaxEventSubscribersPerSession);
 
@@ -86,6 +87,7 @@ public sealed class GatewayOptionsTests
     [InlineData("MxGateway:Worker:PipeConnectAttemptTimeoutMilliseconds", "0", "MxGateway:Worker:PipeConnectAttemptTimeoutMilliseconds must be greater than zero.")]
     [InlineData("MxGateway:Sessions:DefaultLeaseSeconds", "0", "MxGateway:Sessions:DefaultLeaseSeconds must be greater than zero.")]
     [InlineData("MxGateway:Sessions:LeaseSweepIntervalSeconds", "0", "MxGateway:Sessions:LeaseSweepIntervalSeconds must be greater than zero.")]
+    [InlineData("MxGateway:Sessions:DetachGraceSeconds", "-1", "MxGateway:Sessions:DetachGraceSeconds must be zero or greater (0 disables detach-grace retention).")]
     [InlineData("MxGateway:Events:QueueCapacity", "0", "MxGateway:Events:QueueCapacity must be greater than zero.")]
     [InlineData("MxGateway:Protocol:MaxGrpcMessageBytes", "0", "MxGateway:Protocol:MaxGrpcMessageBytes must be between")]
     [InlineData("MxGateway:Authentication:PepperSecretName", "", "MxGateway:Authentication:PepperSecretName is required")]

src/ZB.MOM.WW.MxGateway.Tests/Gateway/Sessions/GatewaySessionTests.cs

@@ -1,5 +1,6 @@
 using System.Runtime.CompilerServices;
 using Microsoft.Extensions.Logging.Abstractions;
+using Microsoft.Extensions.Time.Testing;
 using ZB.MOM.WW.MxGateway.Contracts.Proto;
 using ZB.MOM.WW.MxGateway.Server.Configuration;
 using ZB.MOM.WW.MxGateway.Server.Dashboard.Hubs;
@@ -432,6 +433,183 @@ public sealed class GatewaySessionTests
         await session.DisposeAsync();
     }
 
+    /// <summary>
+    ///     Task 11. With a positive detach-grace, dropping the last external subscriber must
+    ///     RETAIN the session (it stays <see cref="SessionState.Ready"/>, not Closed/Faulted)
+    ///     and record a detached timestamp so the lease monitor can age it out later.
+    /// </summary>
+    [Fact]
+    public async Task DetachGrace_LastSubscriberDrops_RetainsSessionAndRecordsDetachedTimestamp()
+    {
+        FakeTimeProvider clock = new(DateTimeOffset.UtcNow);
+        FakeWorkerClient workerClient = new();
+        await using GatewaySession session = CreateReadySessionWithDetachGrace(
+            workerClient,
+            clock,
+            detachGrace: TimeSpan.FromSeconds(30));
+
+        IEventSubscriberLease lease = session.AttachEventSubscriber(maxSubscribers: 1);
+        Assert.Null(session.DetachedAtUtc);
+
+        lease.Dispose();
+
+        // Retained, not torn down.
+        Assert.Equal(SessionState.Ready, session.State);
+        Assert.Equal(0, session.ActiveEventSubscriberCount);
+        Assert.Equal(clock.GetUtcNow(), session.DetachedAtUtc);
+        Assert.False(session.IsDetachGraceExpired(clock.GetUtcNow()));
+    }
+
+    /// <summary>
+    ///     Task 11. Advancing the clock past the detach-grace window makes the retained,
+    ///     detached session eligible for close (<see cref="GatewaySession.IsDetachGraceExpired"/>).
+    /// </summary>
+    [Fact]
+    public async Task DetachGrace_ClockAdvancesPastWindow_SessionBecomesEligibleForClose()
+    {
+        FakeTimeProvider clock = new(DateTimeOffset.UtcNow);
+        FakeWorkerClient workerClient = new();
+        await using GatewaySession session = CreateReadySessionWithDetachGrace(
+            workerClient,
+            clock,
+            detachGrace: TimeSpan.FromSeconds(30));
+
+        IEventSubscriberLease lease = session.AttachEventSubscriber(maxSubscribers: 1);
+        lease.Dispose();
+
+        // Just before the window elapses: not yet eligible.
+        clock.Advance(TimeSpan.FromSeconds(29));
+        Assert.False(session.IsDetachGraceExpired(clock.GetUtcNow()));
+
+        // At/after the window: eligible for close.
+        clock.Advance(TimeSpan.FromSeconds(1));
+        Assert.True(session.IsDetachGraceExpired(clock.GetUtcNow()));
+    }
+
+    /// <summary>
+    ///     Task 11. Re-attaching a subscriber before the window elapses cancels the grace:
+    ///     the detached timestamp clears and a subsequent clock advance does NOT make the
+    ///     session eligible for close.
+    /// </summary>
+    [Fact]
+    public async Task DetachGrace_ReattachBeforeExpiry_CancelsGrace()
+    {
+        FakeTimeProvider clock = new(DateTimeOffset.UtcNow);
+        FakeWorkerClient workerClient = new();
+        await using GatewaySession session = CreateReadySessionWithDetachGrace(
+            workerClient,
+            clock,
+            detachGrace: TimeSpan.FromSeconds(30));
+
+        IEventSubscriberLease first = session.AttachEventSubscriber(maxSubscribers: 1);
+        first.Dispose();
+        Assert.NotNull(session.DetachedAtUtc);
+
+        clock.Advance(TimeSpan.FromSeconds(10));
+        IEventSubscriberLease second = session.AttachEventSubscriber(maxSubscribers: 1);
+
+        // Re-attach cancelled the grace window.
+        Assert.Null(session.DetachedAtUtc);
+
+        // Advancing well past what would have been the window does not make it eligible while
+        // a subscriber is attached.
+        clock.Advance(TimeSpan.FromMinutes(5));
+        Assert.False(session.IsDetachGraceExpired(clock.GetUtcNow()));
+
+        second.Dispose();
+    }
+
+    /// <summary>
+    ///     Task 11. With detach-grace disabled (<c>0</c>), dropping the last subscriber must
+    ///     match today's behavior: the session stays <see cref="SessionState.Ready"/> with no
+    ///     detached timestamp and is never eligible for a detach-grace close — it lingers only
+    ///     until its normal lease expires.
+    /// </summary>
+    [Fact]
+    public async Task DetachGrace_Disabled_MatchesTodaysBehavior()
+    {
+        FakeTimeProvider clock = new(DateTimeOffset.UtcNow);
+        FakeWorkerClient workerClient = new();
+        await using GatewaySession session = CreateReadySessionWithDetachGrace(
+            workerClient,
+            clock,
+            detachGrace: TimeSpan.Zero);
+
+        IEventSubscriberLease lease = session.AttachEventSubscriber(maxSubscribers: 1);
+        lease.Dispose();
+
+        Assert.Equal(SessionState.Ready, session.State);
+        Assert.Null(session.DetachedAtUtc);
+
+        clock.Advance(TimeSpan.FromHours(1));
+        Assert.False(session.IsDetachGraceExpired(clock.GetUtcNow()));
+    }
+
+    /// <summary>
+    ///     Task 11. The gateway-owned internal dashboard subscriber must NOT keep a session out
+    ///     of detach-grace: with only the dashboard mirror attached (and no external gRPC
+    ///     subscriber), dropping the last external subscriber still enters grace and the
+    ///     window still expires.
+    /// </summary>
+    [Fact]
+    public async Task DetachGrace_DashboardMirrorAlone_DoesNotPreventGraceEntry()
+    {
+        FakeTimeProvider clock = new(DateTimeOffset.UtcNow);
+        FakeWorkerClient workerClient = new();
+        RecordingDashboardEventBroadcaster broadcaster = new();
+        await using GatewaySession session = CreateReadySessionWithDetachGrace(
+            workerClient,
+            clock,
+            detachGrace: TimeSpan.FromSeconds(30),
+            dashboardBroadcaster: broadcaster);
+
+        // The dashboard mirror is the only subscriber (registered internally at MarkReady).
+        // It is not counted as an external subscriber.
+        Assert.Equal(0, session.ActiveEventSubscriberCount);
+
+        IEventSubscriberLease lease = session.AttachEventSubscriber(maxSubscribers: 1);
+        lease.Dispose();
+
+        // Entered grace despite the dashboard mirror still being attached.
+        Assert.NotNull(session.DetachedAtUtc);
+
+        clock.Advance(TimeSpan.FromSeconds(30));
+        Assert.True(session.IsDetachGraceExpired(clock.GetUtcNow()));
+    }
+
+    private static GatewaySession CreateReadySessionWithDetachGrace(
+        IWorkerClient workerClient,
+        TimeProvider timeProvider,
+        TimeSpan detachGrace,
+        IDashboardEventBroadcaster? dashboardBroadcaster = null)
+    {
+        GatewaySession session = new(
+            sessionId: "session-test-detach-grace",
+            backendName: "mxaccess",
+            pipeName: "mxaccess-gateway-1-session-test-detach-grace",
+            nonce: "nonce",
+            clientIdentity: "client-1",
+            ownerKeyId: null,
+            clientSessionName: "test-session",
+            clientCorrelationId: "client-correlation-1",
+            commandTimeout: TimeSpan.FromSeconds(5),
+            startupTimeout: TimeSpan.FromSeconds(5),
+            shutdownTimeout: TimeSpan.FromSeconds(5),
+            leaseDuration: TimeSpan.FromMinutes(30),
+            openedAt: timeProvider.GetUtcNow(),
+            eventStreaming: new SessionEventStreaming(
+                new MxAccessGrpcMapper(),
+                new EventOptions { QueueCapacity = 8 },
+                NullLogger<SessionEventDistributor>.Instance,
+                timeProvider,
+                new GatewayMetrics(),
+                dashboardBroadcaster),
+            detachGrace: detachGrace);
+        session.AttachWorkerClient(workerClient);
+        session.MarkReady();
+        return session;
+    }
+
     private static GatewaySession CreateReadySession(IWorkerClient workerClient)
     {
         GatewaySession session = new(

src/ZB.MOM.WW.MxGateway.Tests/Gateway/Sessions/SessionManagerTests.cs

@@ -1,5 +1,6 @@
 using Google.Protobuf.WellKnownTypes;
 using Microsoft.Extensions.Options;
+using Microsoft.Extensions.Time.Testing;
 using ZB.MOM.WW.MxGateway.Contracts.Proto;
 using ZB.MOM.WW.MxGateway.Server.Configuration;
 using ZB.MOM.WW.MxGateway.Server.Metrics;
@@ -752,6 +753,42 @@ public sealed class SessionManagerTests
         Assert.Equal(0, workerClient.ShutdownCount);
     }
 
+    /// <summary>
+    ///     Task 11. With detach-grace enabled, a session whose last external subscriber dropped
+    ///     and whose detach-grace window has elapsed is closed by the lease sweep exactly like an
+    ///     expired-lease session — even though its normal lease is still far in the future.
+    /// </summary>
+    [Fact]
+    public async Task CloseExpiredLeasesAsync_ClosesSessionWhoseDetachGraceWindowExpired()
+    {
+        FakeWorkerClient workerClient = new();
+        FakeTimeProvider clock = new(DateTimeOffset.UtcNow);
+        SessionManager manager = CreateManager(
+            new FakeSessionWorkerClientFactory(workerClient),
+            options: CreateOptions(defaultLeaseSeconds: 1800, detachGraceSeconds: 30),
+            timeProvider: clock);
+        GatewaySession session = await manager.OpenSessionAsync(CreateOpenRequest(), "client-1", ownerKeyId: null, CancellationToken.None);
+
+        // Attach and drop an external subscriber to enter detach-grace. The normal lease is
+        // still 30 minutes out, so only the detach-grace window can close this session.
+        IDisposable subscriber = session.AttachEventSubscriber(maxSubscribers: 1);
+        subscriber.Dispose();
+        Assert.NotNull(session.DetachedAtUtc);
+
+        // Before the window elapses: not closed.
+        clock.Advance(TimeSpan.FromSeconds(29));
+        int closedBefore = await manager.CloseExpiredLeasesAsync(clock.GetUtcNow(), CancellationToken.None);
+        Assert.Equal(0, closedBefore);
+        Assert.Equal(SessionState.Ready, session.State);
+
+        // After the window elapses: the sweep closes it.
+        clock.Advance(TimeSpan.FromSeconds(1));
+        int closedAfter = await manager.CloseExpiredLeasesAsync(clock.GetUtcNow(), CancellationToken.None);
+        Assert.Equal(1, closedAfter);
+        Assert.Equal(SessionState.Closed, session.State);
+        Assert.Equal(1, workerClient.ShutdownCount);
+    }
+
     /// <summary>Verifies that shutdown closes all registered sessions.</summary>
     [Fact]
     public async Task ShutdownAsync_ClosesAllRegisteredSessions()
@@ -797,7 +834,8 @@ public sealed class SessionManagerTests
 
     private static GatewayOptions CreateOptions(
         int maxSessions = 64,
-        int defaultLeaseSeconds = 1800)
+        int defaultLeaseSeconds = 1800,
+        int detachGraceSeconds = 0)
     {
         return new GatewayOptions
         {
@@ -806,6 +844,7 @@ public sealed class SessionManagerTests
                 DefaultCommandTimeoutSeconds = 30,
                 MaxSessions = maxSessions,
                 DefaultLeaseSeconds = defaultLeaseSeconds,
+                DetachGraceSeconds = detachGraceSeconds,
             },
             Worker = new WorkerOptions
             {