feat(sessions): detach-grace retention window for reconnect

2026-06-16 06:15:46 -04:00
parent 85e4334bb7
commit db95f8644f
10 changed files with 346 additions and 6 deletions
@@ -34,6 +34,7 @@ public sealed class GatewayOptionsTests
        Assert.Equal(128, options.Sessions.MaxPendingCommandsPerSession);
        Assert.Equal(1800, options.Sessions.DefaultLeaseSeconds);
        Assert.Equal(30, options.Sessions.LeaseSweepIntervalSeconds);
+        Assert.Equal(30, options.Sessions.DetachGraceSeconds);
        Assert.False(options.Sessions.AllowMultipleEventSubscribers);
        Assert.Equal(8, options.Sessions.MaxEventSubscribersPerSession);

@@ -86,6 +87,7 @@ public sealed class GatewayOptionsTests
    [InlineData("MxGateway:Worker:PipeConnectAttemptTimeoutMilliseconds", "0", "MxGateway:Worker:PipeConnectAttemptTimeoutMilliseconds must be greater than zero.")]
    [InlineData("MxGateway:Sessions:DefaultLeaseSeconds", "0", "MxGateway:Sessions:DefaultLeaseSeconds must be greater than zero.")]
    [InlineData("MxGateway:Sessions:LeaseSweepIntervalSeconds", "0", "MxGateway:Sessions:LeaseSweepIntervalSeconds must be greater than zero.")]
+    [InlineData("MxGateway:Sessions:DetachGraceSeconds", "-1", "MxGateway:Sessions:DetachGraceSeconds must be zero or greater (0 disables detach-grace retention).")]
    [InlineData("MxGateway:Events:QueueCapacity", "0", "MxGateway:Events:QueueCapacity must be greater than zero.")]
    [InlineData("MxGateway:Protocol:MaxGrpcMessageBytes", "0", "MxGateway:Protocol:MaxGrpcMessageBytes must be between")]
    [InlineData("MxGateway:Authentication:PepperSecretName", "", "MxGateway:Authentication:PepperSecretName is required")]
@@ -1,5 +1,6 @@
 using System.Runtime.CompilerServices;
 using Microsoft.Extensions.Logging.Abstractions;
+using Microsoft.Extensions.Time.Testing;
 using ZB.MOM.WW.MxGateway.Contracts.Proto;
 using ZB.MOM.WW.MxGateway.Server.Configuration;
 using ZB.MOM.WW.MxGateway.Server.Dashboard.Hubs;
@@ -432,6 +433,183 @@ public sealed class GatewaySessionTests
        await session.DisposeAsync();
    }

+    /// <summary>
+    ///     Task 11. With a positive detach-grace, dropping the last external subscriber must
+    ///     RETAIN the session (it stays <see cref="SessionState.Ready"/>, not Closed/Faulted)
+    ///     and record a detached timestamp so the lease monitor can age it out later.
+    /// </summary>
+    [Fact]
+    public async Task DetachGrace_LastSubscriberDrops_RetainsSessionAndRecordsDetachedTimestamp()
+    {
+        FakeTimeProvider clock = new(DateTimeOffset.UtcNow);
+        FakeWorkerClient workerClient = new();
+        await using GatewaySession session = CreateReadySessionWithDetachGrace(
+            workerClient,
+            clock,
+            detachGrace: TimeSpan.FromSeconds(30));
+
+        IEventSubscriberLease lease = session.AttachEventSubscriber(maxSubscribers: 1);
+        Assert.Null(session.DetachedAtUtc);
+
+        lease.Dispose();
+
+        // Retained, not torn down.
+        Assert.Equal(SessionState.Ready, session.State);
+        Assert.Equal(0, session.ActiveEventSubscriberCount);
+        Assert.Equal(clock.GetUtcNow(), session.DetachedAtUtc);
+        Assert.False(session.IsDetachGraceExpired(clock.GetUtcNow()));
+    }
+
+    /// <summary>
+    ///     Task 11. Advancing the clock past the detach-grace window makes the retained,
+    ///     detached session eligible for close (<see cref="GatewaySession.IsDetachGraceExpired"/>).
+    /// </summary>
+    [Fact]
+    public async Task DetachGrace_ClockAdvancesPastWindow_SessionBecomesEligibleForClose()
+    {
+        FakeTimeProvider clock = new(DateTimeOffset.UtcNow);
+        FakeWorkerClient workerClient = new();
+        await using GatewaySession session = CreateReadySessionWithDetachGrace(
+            workerClient,
+            clock,
+            detachGrace: TimeSpan.FromSeconds(30));
+
+        IEventSubscriberLease lease = session.AttachEventSubscriber(maxSubscribers: 1);
+        lease.Dispose();
+
+        // Just before the window elapses: not yet eligible.
+        clock.Advance(TimeSpan.FromSeconds(29));
+        Assert.False(session.IsDetachGraceExpired(clock.GetUtcNow()));
+
+        // At/after the window: eligible for close.
+        clock.Advance(TimeSpan.FromSeconds(1));
+        Assert.True(session.IsDetachGraceExpired(clock.GetUtcNow()));
+    }
+
+    /// <summary>
+    ///     Task 11. Re-attaching a subscriber before the window elapses cancels the grace:
+    ///     the detached timestamp clears and a subsequent clock advance does NOT make the
+    ///     session eligible for close.
+    /// </summary>
+    [Fact]
+    public async Task DetachGrace_ReattachBeforeExpiry_CancelsGrace()
+    {
+        FakeTimeProvider clock = new(DateTimeOffset.UtcNow);
+        FakeWorkerClient workerClient = new();
+        await using GatewaySession session = CreateReadySessionWithDetachGrace(
+            workerClient,
+            clock,
+            detachGrace: TimeSpan.FromSeconds(30));
+
+        IEventSubscriberLease first = session.AttachEventSubscriber(maxSubscribers: 1);
+        first.Dispose();
+        Assert.NotNull(session.DetachedAtUtc);
+
+        clock.Advance(TimeSpan.FromSeconds(10));
+        IEventSubscriberLease second = session.AttachEventSubscriber(maxSubscribers: 1);
+
+        // Re-attach cancelled the grace window.
+        Assert.Null(session.DetachedAtUtc);
+
+        // Advancing well past what would have been the window does not make it eligible while
+        // a subscriber is attached.
+        clock.Advance(TimeSpan.FromMinutes(5));
+        Assert.False(session.IsDetachGraceExpired(clock.GetUtcNow()));
+
+        second.Dispose();
+    }
+
+    /// <summary>
+    ///     Task 11. With detach-grace disabled (<c>0</c>), dropping the last subscriber must
+    ///     match today's behavior: the session stays <see cref="SessionState.Ready"/> with no
+    ///     detached timestamp and is never eligible for a detach-grace close — it lingers only
+    ///     until its normal lease expires.
+    /// </summary>
+    [Fact]
+    public async Task DetachGrace_Disabled_MatchesTodaysBehavior()
+    {
+        FakeTimeProvider clock = new(DateTimeOffset.UtcNow);
+        FakeWorkerClient workerClient = new();
+        await using GatewaySession session = CreateReadySessionWithDetachGrace(
+            workerClient,
+            clock,
+            detachGrace: TimeSpan.Zero);
+
+        IEventSubscriberLease lease = session.AttachEventSubscriber(maxSubscribers: 1);
+        lease.Dispose();
+
+        Assert.Equal(SessionState.Ready, session.State);
+        Assert.Null(session.DetachedAtUtc);
+
+        clock.Advance(TimeSpan.FromHours(1));
+        Assert.False(session.IsDetachGraceExpired(clock.GetUtcNow()));
+    }
+
+    /// <summary>
+    ///     Task 11. The gateway-owned internal dashboard subscriber must NOT keep a session out
+    ///     of detach-grace: with only the dashboard mirror attached (and no external gRPC
+    ///     subscriber), dropping the last external subscriber still enters grace and the
+    ///     window still expires.
+    /// </summary>
+    [Fact]
+    public async Task DetachGrace_DashboardMirrorAlone_DoesNotPreventGraceEntry()
+    {
+        FakeTimeProvider clock = new(DateTimeOffset.UtcNow);
+        FakeWorkerClient workerClient = new();
+        RecordingDashboardEventBroadcaster broadcaster = new();
+        await using GatewaySession session = CreateReadySessionWithDetachGrace(
+            workerClient,
+            clock,
+            detachGrace: TimeSpan.FromSeconds(30),
+            dashboardBroadcaster: broadcaster);
+
+        // The dashboard mirror is the only subscriber (registered internally at MarkReady).
+        // It is not counted as an external subscriber.
+        Assert.Equal(0, session.ActiveEventSubscriberCount);
+
+        IEventSubscriberLease lease = session.AttachEventSubscriber(maxSubscribers: 1);
+        lease.Dispose();
+
+        // Entered grace despite the dashboard mirror still being attached.
+        Assert.NotNull(session.DetachedAtUtc);
+
+        clock.Advance(TimeSpan.FromSeconds(30));
+        Assert.True(session.IsDetachGraceExpired(clock.GetUtcNow()));
+    }
+
+    private static GatewaySession CreateReadySessionWithDetachGrace(
+        IWorkerClient workerClient,
+        TimeProvider timeProvider,
+        TimeSpan detachGrace,
+        IDashboardEventBroadcaster? dashboardBroadcaster = null)
+    {
+        GatewaySession session = new(
+            sessionId: "session-test-detach-grace",
+            backendName: "mxaccess",
+            pipeName: "mxaccess-gateway-1-session-test-detach-grace",
+            nonce: "nonce",
+            clientIdentity: "client-1",
+            ownerKeyId: null,
+            clientSessionName: "test-session",
+            clientCorrelationId: "client-correlation-1",
+            commandTimeout: TimeSpan.FromSeconds(5),
+            startupTimeout: TimeSpan.FromSeconds(5),
+            shutdownTimeout: TimeSpan.FromSeconds(5),
+            leaseDuration: TimeSpan.FromMinutes(30),
+            openedAt: timeProvider.GetUtcNow(),
+            eventStreaming: new SessionEventStreaming(
+                new MxAccessGrpcMapper(),
+                new EventOptions { QueueCapacity = 8 },
+                NullLogger<SessionEventDistributor>.Instance,
+                timeProvider,
+                new GatewayMetrics(),
+                dashboardBroadcaster),
+            detachGrace: detachGrace);
+        session.AttachWorkerClient(workerClient);
+        session.MarkReady();
+        return session;
+    }
+
    private static GatewaySession CreateReadySession(IWorkerClient workerClient)
    {
        GatewaySession session = new(
@@ -1,5 +1,6 @@
 using Google.Protobuf.WellKnownTypes;
 using Microsoft.Extensions.Options;
+using Microsoft.Extensions.Time.Testing;
 using ZB.MOM.WW.MxGateway.Contracts.Proto;
 using ZB.MOM.WW.MxGateway.Server.Configuration;
 using ZB.MOM.WW.MxGateway.Server.Metrics;
@@ -752,6 +753,42 @@ public sealed class SessionManagerTests
        Assert.Equal(0, workerClient.ShutdownCount);
    }

+    /// <summary>
+    ///     Task 11. With detach-grace enabled, a session whose last external subscriber dropped
+    ///     and whose detach-grace window has elapsed is closed by the lease sweep exactly like an
+    ///     expired-lease session — even though its normal lease is still far in the future.
+    /// </summary>
+    [Fact]
+    public async Task CloseExpiredLeasesAsync_ClosesSessionWhoseDetachGraceWindowExpired()
+    {
+        FakeWorkerClient workerClient = new();
+        FakeTimeProvider clock = new(DateTimeOffset.UtcNow);
+        SessionManager manager = CreateManager(
+            new FakeSessionWorkerClientFactory(workerClient),
+            options: CreateOptions(defaultLeaseSeconds: 1800, detachGraceSeconds: 30),
+            timeProvider: clock);
+        GatewaySession session = await manager.OpenSessionAsync(CreateOpenRequest(), "client-1", ownerKeyId: null, CancellationToken.None);
+
+        // Attach and drop an external subscriber to enter detach-grace. The normal lease is
+        // still 30 minutes out, so only the detach-grace window can close this session.
+        IDisposable subscriber = session.AttachEventSubscriber(maxSubscribers: 1);
+        subscriber.Dispose();
+        Assert.NotNull(session.DetachedAtUtc);
+
+        // Before the window elapses: not closed.
+        clock.Advance(TimeSpan.FromSeconds(29));
+        int closedBefore = await manager.CloseExpiredLeasesAsync(clock.GetUtcNow(), CancellationToken.None);
+        Assert.Equal(0, closedBefore);
+        Assert.Equal(SessionState.Ready, session.State);
+
+        // After the window elapses: the sweep closes it.
+        clock.Advance(TimeSpan.FromSeconds(1));
+        int closedAfter = await manager.CloseExpiredLeasesAsync(clock.GetUtcNow(), CancellationToken.None);
+        Assert.Equal(1, closedAfter);
+        Assert.Equal(SessionState.Closed, session.State);
+        Assert.Equal(1, workerClient.ShutdownCount);
+    }
+
    /// <summary>Verifies that shutdown closes all registered sessions.</summary>
    [Fact]
    public async Task ShutdownAsync_ClosesAllRegisteredSessions()
@@ -797,7 +834,8 @@ public sealed class SessionManagerTests

    private static GatewayOptions CreateOptions(
        int maxSessions = 64,
-        int defaultLeaseSeconds = 1800)
+        int defaultLeaseSeconds = 1800,
+        int detachGraceSeconds = 0)
    {
        return new GatewayOptions
        {
@@ -806,6 +844,7 @@ public sealed class SessionManagerTests
                DefaultCommandTimeoutSeconds = 30,
                MaxSessions = maxSessions,
                DefaultLeaseSeconds = defaultLeaseSeconds,
+                DetachGraceSeconds = detachGraceSeconds,
            },
            Worker = new WorkerOptions
            {