From db95f8644fd2d8052cda5e87a7f116c2963d4ab9 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Tue, 16 Jun 2026 06:15:46 -0400 Subject: [PATCH] feat(sessions): detach-grace retention window for reconnect --- docs/GatewayConfiguration.md | 2 + docs/Sessions.md | 10 +- .../Configuration/GatewayOptionsValidator.cs | 4 + .../Configuration/SessionOptions.cs | 15 ++ .../Sessions/GatewaySession.cs | 81 +++++++- .../Sessions/SessionManager.cs | 18 +- .../appsettings.json | 1 + .../Configuration/GatewayOptionsTests.cs | 2 + .../Gateway/Sessions/GatewaySessionTests.cs | 178 ++++++++++++++++++ .../Gateway/Sessions/SessionManagerTests.cs | 41 +++- 10 files changed, 346 insertions(+), 6 deletions(-) diff --git a/docs/GatewayConfiguration.md b/docs/GatewayConfiguration.md index 6054d6f..7684e7b 100644 --- a/docs/GatewayConfiguration.md +++ b/docs/GatewayConfiguration.md @@ -37,6 +37,7 @@ paths, timeouts, queue sizes, enum values, or protocol values are invalid. "MaxPendingCommandsPerSession": 128, "DefaultLeaseSeconds": 1800, "LeaseSweepIntervalSeconds": 30, + "DetachGraceSeconds": 30, "AllowMultipleEventSubscribers": false, "MaxEventSubscribersPerSession": 8 }, @@ -126,6 +127,7 @@ to avoid accidental large allocations from malformed or oversized frames. | `MxGateway:Sessions:MaxPendingCommandsPerSession` | `128` | Maximum number of pending worker commands for one session. Excess commands fail fast instead of queueing indefinitely. | | `MxGateway:Sessions:DefaultLeaseSeconds` | `1800` | Initial session lease and refresh duration. Unary client activity extends the lease by this duration. | | `MxGateway:Sessions:LeaseSweepIntervalSeconds` | `30` | Hosted monitor interval for closing expired leases. Active event-stream subscribers keep a session from expiring while the stream remains attached. | +| `MxGateway:Sessions:DetachGraceSeconds` | `30` | Detach-grace retention window. When positive, a session whose last external (gRPC) event-stream subscriber drops is retained in `Ready` for this many seconds so a client can reconnect; if no external subscriber re-attaches within the window, the lease monitor closes it with `detach-grace-expired`. The internal dashboard mirror does not count as an external subscriber, so a dashboard-only session still enters detach-grace. `0` disables retention and reverts to closing only on normal lease expiry. Must be zero or greater. Reconnect/replay itself is implemented separately (Task 12); this option controls retention and expiry only. | | `MxGateway:Sessions:AllowMultipleEventSubscribers` | `false` | Controls whether multiple `StreamEvents` subscribers may attach to one session. When `false` the session refuses a second subscriber with `AlreadyExists`. Set to `true` to enable fan-out via the `SessionEventDistributor`. | | `MxGateway:Sessions:MaxEventSubscribersPerSession` | `8` | Maximum number of concurrent `StreamEvents` subscribers per session when `AllowMultipleEventSubscribers` is `true`. Effectively 1 when `AllowMultipleEventSubscribers` is `false`. Must be greater than zero. | diff --git a/docs/Sessions.md b/docs/Sessions.md index 59d381a..884242a 100644 --- a/docs/Sessions.md +++ b/docs/Sessions.md @@ -72,7 +72,7 @@ private void EnsureSessionCapacity() } ``` -`SessionManager` also defines three close-reason constants — `DefaultCloseReason` (`"client-close"`), `GatewayShutdownReason` (`"gateway-shutdown"`), and `LeaseExpiredReason` (`"lease-expired"`) — so that the metrics and worker shutdown paths agree on a fixed vocabulary. +`SessionManager` also defines four close-reason constants — `DefaultCloseReason` (`"client-close"`), `GatewayShutdownReason` (`"gateway-shutdown"`), `LeaseExpiredReason` (`"lease-expired"`), and `DetachGraceExpiredReason` (`"detach-grace-expired"`) — so that the metrics and worker shutdown paths agree on a fixed vocabulary. ### SessionRegistry (ISessionRegistry) @@ -199,6 +199,14 @@ Event streaming uses `AttachEventSubscriber` which returns a disposable lease. W Sessions open with `MxGateway:Sessions:DefaultLeaseSeconds` (default 1800) added to the open timestamp. Unary client activity refreshes the lease by the same duration. `ExtendLease` and `IsLeaseExpired` cooperate with `SessionManager.CloseExpiredLeasesAsync`, which iterates a registry snapshot and closes any session whose lease has expired with `LeaseExpiredReason`. `SessionLeaseMonitorHostedService` runs that sweep every `MxGateway:Sessions:LeaseSweepIntervalSeconds` seconds (default 30). +#### Detach-grace retention + +`MxGateway:Sessions:DetachGraceSeconds` (default 30) is a bounded retention window kept after a session's *last external (gRPC) event-stream subscriber* drops, so a client can reconnect to the same session instead of having it torn down on the first stream disconnect. While the window is open the session stays `Ready` and fully usable — worker commands continue to work and a reconnecting subscriber re-attaches normally. Because retention is keyed on the *external* subscriber count (`_activeEventSubscriberCount`), and the gateway-owned internal dashboard mirror registers directly on the distributor with `isInternal: true` and is therefore *not* counted, a session whose only remaining subscriber is the dashboard mirror still enters detach-grace. + +Mechanically: when the last external subscriber detaches and `DetachGraceSeconds > 0`, `DetachEventSubscriber` stamps `DetachedAtUtc` from the session's `TimeProvider` under `_syncRoot` (the detach→grace-start transition). `AttachEventSubscriber` clears `DetachedAtUtc` under the same lock when a subscriber re-attaches (the reattach→grace-cancel transition), so the two races and the sweeper's read all serialize on `_syncRoot`. `SessionManager.CloseExpiredLeasesAsync` checks `IsDetachGraceExpired(now)` alongside `IsLeaseExpired(now)`: a session detached for at least `DetachGraceSeconds` with no active external subscriber is closed by the same lease sweep, with the distinct `DetachGraceExpiredReason` (`"detach-grace-expired"`) so operators can tell a short reconnect-window expiry from a long idle-lease expiry. Setting `DetachGraceSeconds` to `0` disables retention and reverts to the original behavior: a detached session is retained only until its normal lease expires. + +The reconnect/replay path that re-attaches a dropped client to a retained session is implemented separately (Task 12); `DetachGraceSeconds` controls retention and expiry only. + ### Close `GatewaySession.CloseAsync` is serialized by a per-session `SemaphoreSlim` (`_closeLock`) so only one close runs at a time, but every read/write of `_state` still passes through `_syncRoot` (via `TryBeginClose` and `MarkClosed`). The close path therefore obeys the same lock discipline as `TransitionTo` / `MarkFaulted`: it transitions to `Closing`, asks the worker client to shut down within `ShutdownTimeout`, and on success transitions to `Closed`. `DisposeAsync` waits on `_closeLock` once before disposing the semaphore so an in-flight close's `Release()` cannot race against the dispose. If `WorkerClient.ShutdownAsync` throws, the session falls back to `IWorkerClient.Kill` (forced close): diff --git a/src/ZB.MOM.WW.MxGateway.Server/Configuration/GatewayOptionsValidator.cs b/src/ZB.MOM.WW.MxGateway.Server/Configuration/GatewayOptionsValidator.cs index 53902cd..490f060 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Configuration/GatewayOptionsValidator.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Configuration/GatewayOptionsValidator.cs @@ -181,6 +181,10 @@ public sealed class GatewayOptionsValidator : OptionsValidatorBase 1 as a hard validation error here. The default diff --git a/src/ZB.MOM.WW.MxGateway.Server/Configuration/SessionOptions.cs b/src/ZB.MOM.WW.MxGateway.Server/Configuration/SessionOptions.cs index 59b581a..4decaeb 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Configuration/SessionOptions.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Configuration/SessionOptions.cs @@ -23,6 +23,21 @@ public sealed class SessionOptions /// Gets the interval for sweeping expired session leases in seconds. public int LeaseSweepIntervalSeconds { get; init; } = 30; + /// + /// Gets the detach-grace retention window, in seconds, that a session is kept alive + /// after its last external (gRPC) event-stream subscriber drops, so a client can + /// reconnect to it. While within the window the session stays in + /// Ready and remains usable; if no new external subscriber attaches before the + /// window elapses, the lease monitor closes the session exactly as it closes an + /// expired lease. The gateway-owned internal dashboard subscriber does not count as an + /// external subscriber, so a session whose only remaining subscriber is the dashboard + /// mirror still enters detach-grace. A value of 0 disables retention: the + /// session reverts to the original behavior of lingering only until its normal lease + /// expires. The reconnect/replay itself is implemented separately (Task 12); this + /// option controls retention and expiry only. + /// + public int DetachGraceSeconds { get; init; } = 30; + /// /// Gets a value indicating whether multiple event subscribers are allowed per session. /// diff --git a/src/ZB.MOM.WW.MxGateway.Server/Sessions/GatewaySession.cs b/src/ZB.MOM.WW.MxGateway.Server/Sessions/GatewaySession.cs index 56bf7d1..80f9839 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Sessions/GatewaySession.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Sessions/GatewaySession.cs @@ -22,6 +22,8 @@ public sealed class GatewaySession private DateTimeOffset? _leaseExpiresAt; private bool _closeStarted; private int _activeEventSubscriberCount; + private readonly TimeSpan _detachGrace; + private DateTimeOffset? _detachedAtUtc; private SessionEventDistributor? _eventDistributor; private bool _eventDistributorStarted; private bool _dashboardMirrorStarted; @@ -103,6 +105,16 @@ public sealed class GatewaySession /// session directly still get a working distributor. Production passes the /// DI-resolved dependencies. /// + /// + /// Retention window kept after the last external (gRPC) event subscriber drops, so a + /// client can reconnect (Task 12). When the window is positive and the active external + /// subscriber count falls to zero, the session stays + /// and records a detached timestamp; the lease monitor closes it once the window + /// elapses with no subscriber having re-attached. (the + /// default) disables retention and preserves the original lease-only expiry behavior. + /// The clock comes from 's + /// so the timer is unit-testable. + /// public GatewaySession( string sessionId, string backendName, @@ -117,7 +129,8 @@ public sealed class GatewaySession TimeSpan shutdownTimeout, TimeSpan leaseDuration, DateTimeOffset openedAt, - SessionEventStreaming? eventStreaming = null) + SessionEventStreaming? eventStreaming = null, + TimeSpan detachGrace = default) { if (string.IsNullOrWhiteSpace(sessionId)) { @@ -155,6 +168,7 @@ public sealed class GatewaySession _lastClientActivityAt = openedAt; _leaseExpiresAt = openedAt + leaseDuration; _eventStreaming = eventStreaming ?? SessionEventStreaming.Default; + _detachGrace = detachGrace > TimeSpan.Zero ? detachGrace : TimeSpan.Zero; } /// @@ -300,6 +314,25 @@ public sealed class GatewaySession } } + /// + /// Gets the UTC timestamp at which the session entered its detach-grace retention + /// window (the last external event subscriber dropped while a positive + /// detach-grace was configured), or when the session is not + /// currently within a detach-grace window. Re-attaching an external subscriber clears + /// this. Always when detach-grace is disabled + /// (DetachGraceSeconds == 0). + /// + public DateTimeOffset? DetachedAtUtc + { + get + { + lock (_syncRoot) + { + return _detachedAtUtc; + } + } + } + /// /// Attaches the worker client for this session. /// @@ -679,6 +712,28 @@ public sealed class GatewaySession } } + /// + /// Determines whether the session's detach-grace retention window has elapsed: the + /// session entered detach-grace (its last external event subscriber dropped while a + /// positive detach-grace was configured) and has had no external subscriber re-attach + /// for longer than the configured detach-grace. The lease monitor closes such a + /// session exactly as it closes an expired lease. Always returns + /// when detach-grace is disabled or when an external subscriber is attached (the + /// detached timestamp is cleared on re-attach, so an attached session is never within a + /// window). + /// + /// Current timestamp for comparison. + public bool IsDetachGraceExpired(DateTimeOffset now) + { + lock (_syncRoot) + { + return _detachGrace > TimeSpan.Zero + && _activeEventSubscriberCount == 0 + && _detachedAtUtc is not null + && now - _detachedAtUtc.Value >= _detachGrace; + } + } + /// /// Attaches an event subscriber and returns a lease whose /// reads the fanned public @@ -733,6 +788,12 @@ public sealed class GatewaySession } _activeEventSubscriberCount++; + + // An external subscriber (re)attached: cancel any in-flight detach-grace window so + // the lease monitor no longer treats this session as eligible for grace-expiry + // close. This is the reattach→grace-cancel transition; it races the sweeper's + // IsDetachGraceExpired read, and both run under _syncRoot so they serialize. + _detachedAtUtc = null; } // Construct/start the distributor and register this subscriber. Done outside the @@ -1502,6 +1563,24 @@ public sealed class GatewaySession { _activeEventSubscriberCount--; } + + // When the LAST external subscriber drops and detach-grace is enabled, retain the + // session instead of letting it linger only on the (long) lease: stamp the detached + // time so the lease monitor can close it once the grace window elapses. The session + // stays in its current (Ready) state and remains usable, so a reconnecting subscriber + // (Task 12) re-attaches normally. The gateway-owned internal dashboard subscriber is + // NOT counted in _activeEventSubscriberCount (it registers on the distributor with + // isInternal: true), so a session whose only remaining subscriber is the dashboard + // mirror still enters grace. Only stamp while the session is alive — once + // Closing/Closed/Faulted there is nothing to retain. This is the detach→grace-start + // transition; it shares _syncRoot with the reattach→grace-cancel write above and the + // sweeper's IsDetachGraceExpired read, so the three serialize. + if (_detachGrace > TimeSpan.Zero + && _activeEventSubscriberCount == 0 + && _state is not (SessionState.Closing or SessionState.Closed or SessionState.Faulted)) + { + _detachedAtUtc = _eventStreaming.TimeProvider.GetUtcNow(); + } } } diff --git a/src/ZB.MOM.WW.MxGateway.Server/Sessions/SessionManager.cs b/src/ZB.MOM.WW.MxGateway.Server/Sessions/SessionManager.cs index 22414e3..132c69b 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Sessions/SessionManager.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Sessions/SessionManager.cs @@ -17,6 +17,7 @@ public sealed class SessionManager : ISessionManager public const string DefaultCloseReason = "client-close"; public const string GatewayShutdownReason = "gateway-shutdown"; public const string LeaseExpiredReason = "lease-expired"; + public const string DetachGraceExpiredReason = "detach-grace-expired"; private readonly ISessionRegistry _registry; private readonly ISessionWorkerClientFactory _workerClientFactory; @@ -295,12 +296,22 @@ public sealed class SessionManager : ISessionManager int closedCount = 0; foreach (GatewaySession session in _registry.Snapshot()) { - if (!session.IsLeaseExpired(now)) + // A session is swept when its normal lease has expired OR its detach-grace + // retention window has elapsed (last external subscriber dropped and no client + // reconnected within DetachGraceSeconds). The detach-grace close is the same + // teardown as a lease-expiry close; only the reason differs so operators can tell + // a short reconnect-window expiry from a long idle-lease expiry in logs/metrics. + string? reason = session.IsLeaseExpired(now) + ? LeaseExpiredReason + : session.IsDetachGraceExpired(now) + ? DetachGraceExpiredReason + : null; + if (reason is null) { continue; } - await CloseSessionCoreAsync(session, LeaseExpiredReason, cancellationToken).ConfigureAwait(false); + await CloseSessionCoreAsync(session, reason, cancellationToken).ConfigureAwait(false); closedCount++; } @@ -478,7 +489,8 @@ public sealed class SessionManager : ISessionManager shutdownTimeout, leaseDuration, openedAt, - eventStreaming); + eventStreaming, + TimeSpan.FromSeconds(Math.Max(0, _options.Sessions.DetachGraceSeconds))); } private static string CreateClientCorrelationId( diff --git a/src/ZB.MOM.WW.MxGateway.Server/appsettings.json b/src/ZB.MOM.WW.MxGateway.Server/appsettings.json index db21287..cfe89ea 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/appsettings.json +++ b/src/ZB.MOM.WW.MxGateway.Server/appsettings.json @@ -46,6 +46,7 @@ "MaxPendingCommandsPerSession": 128, "DefaultLeaseSeconds": 1800, "LeaseSweepIntervalSeconds": 30, + "DetachGraceSeconds": 30, "AllowMultipleEventSubscribers": false, "MaxEventSubscribersPerSession": 8 }, diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Configuration/GatewayOptionsTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Configuration/GatewayOptionsTests.cs index 1d164b8..03385a2 100644 --- a/src/ZB.MOM.WW.MxGateway.Tests/Configuration/GatewayOptionsTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Tests/Configuration/GatewayOptionsTests.cs @@ -34,6 +34,7 @@ public sealed class GatewayOptionsTests Assert.Equal(128, options.Sessions.MaxPendingCommandsPerSession); Assert.Equal(1800, options.Sessions.DefaultLeaseSeconds); Assert.Equal(30, options.Sessions.LeaseSweepIntervalSeconds); + Assert.Equal(30, options.Sessions.DetachGraceSeconds); Assert.False(options.Sessions.AllowMultipleEventSubscribers); Assert.Equal(8, options.Sessions.MaxEventSubscribersPerSession); @@ -86,6 +87,7 @@ public sealed class GatewayOptionsTests [InlineData("MxGateway:Worker:PipeConnectAttemptTimeoutMilliseconds", "0", "MxGateway:Worker:PipeConnectAttemptTimeoutMilliseconds must be greater than zero.")] [InlineData("MxGateway:Sessions:DefaultLeaseSeconds", "0", "MxGateway:Sessions:DefaultLeaseSeconds must be greater than zero.")] [InlineData("MxGateway:Sessions:LeaseSweepIntervalSeconds", "0", "MxGateway:Sessions:LeaseSweepIntervalSeconds must be greater than zero.")] + [InlineData("MxGateway:Sessions:DetachGraceSeconds", "-1", "MxGateway:Sessions:DetachGraceSeconds must be zero or greater (0 disables detach-grace retention).")] [InlineData("MxGateway:Events:QueueCapacity", "0", "MxGateway:Events:QueueCapacity must be greater than zero.")] [InlineData("MxGateway:Protocol:MaxGrpcMessageBytes", "0", "MxGateway:Protocol:MaxGrpcMessageBytes must be between")] [InlineData("MxGateway:Authentication:PepperSecretName", "", "MxGateway:Authentication:PepperSecretName is required")] diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Sessions/GatewaySessionTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Sessions/GatewaySessionTests.cs index 33756fc..cb0ec31 100644 --- a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Sessions/GatewaySessionTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Sessions/GatewaySessionTests.cs @@ -1,5 +1,6 @@ using System.Runtime.CompilerServices; using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; using ZB.MOM.WW.MxGateway.Contracts.Proto; using ZB.MOM.WW.MxGateway.Server.Configuration; using ZB.MOM.WW.MxGateway.Server.Dashboard.Hubs; @@ -432,6 +433,183 @@ public sealed class GatewaySessionTests await session.DisposeAsync(); } + /// + /// Task 11. With a positive detach-grace, dropping the last external subscriber must + /// RETAIN the session (it stays , not Closed/Faulted) + /// and record a detached timestamp so the lease monitor can age it out later. + /// + [Fact] + public async Task DetachGrace_LastSubscriberDrops_RetainsSessionAndRecordsDetachedTimestamp() + { + FakeTimeProvider clock = new(DateTimeOffset.UtcNow); + FakeWorkerClient workerClient = new(); + await using GatewaySession session = CreateReadySessionWithDetachGrace( + workerClient, + clock, + detachGrace: TimeSpan.FromSeconds(30)); + + IEventSubscriberLease lease = session.AttachEventSubscriber(maxSubscribers: 1); + Assert.Null(session.DetachedAtUtc); + + lease.Dispose(); + + // Retained, not torn down. + Assert.Equal(SessionState.Ready, session.State); + Assert.Equal(0, session.ActiveEventSubscriberCount); + Assert.Equal(clock.GetUtcNow(), session.DetachedAtUtc); + Assert.False(session.IsDetachGraceExpired(clock.GetUtcNow())); + } + + /// + /// Task 11. Advancing the clock past the detach-grace window makes the retained, + /// detached session eligible for close (). + /// + [Fact] + public async Task DetachGrace_ClockAdvancesPastWindow_SessionBecomesEligibleForClose() + { + FakeTimeProvider clock = new(DateTimeOffset.UtcNow); + FakeWorkerClient workerClient = new(); + await using GatewaySession session = CreateReadySessionWithDetachGrace( + workerClient, + clock, + detachGrace: TimeSpan.FromSeconds(30)); + + IEventSubscriberLease lease = session.AttachEventSubscriber(maxSubscribers: 1); + lease.Dispose(); + + // Just before the window elapses: not yet eligible. + clock.Advance(TimeSpan.FromSeconds(29)); + Assert.False(session.IsDetachGraceExpired(clock.GetUtcNow())); + + // At/after the window: eligible for close. + clock.Advance(TimeSpan.FromSeconds(1)); + Assert.True(session.IsDetachGraceExpired(clock.GetUtcNow())); + } + + /// + /// Task 11. Re-attaching a subscriber before the window elapses cancels the grace: + /// the detached timestamp clears and a subsequent clock advance does NOT make the + /// session eligible for close. + /// + [Fact] + public async Task DetachGrace_ReattachBeforeExpiry_CancelsGrace() + { + FakeTimeProvider clock = new(DateTimeOffset.UtcNow); + FakeWorkerClient workerClient = new(); + await using GatewaySession session = CreateReadySessionWithDetachGrace( + workerClient, + clock, + detachGrace: TimeSpan.FromSeconds(30)); + + IEventSubscriberLease first = session.AttachEventSubscriber(maxSubscribers: 1); + first.Dispose(); + Assert.NotNull(session.DetachedAtUtc); + + clock.Advance(TimeSpan.FromSeconds(10)); + IEventSubscriberLease second = session.AttachEventSubscriber(maxSubscribers: 1); + + // Re-attach cancelled the grace window. + Assert.Null(session.DetachedAtUtc); + + // Advancing well past what would have been the window does not make it eligible while + // a subscriber is attached. + clock.Advance(TimeSpan.FromMinutes(5)); + Assert.False(session.IsDetachGraceExpired(clock.GetUtcNow())); + + second.Dispose(); + } + + /// + /// Task 11. With detach-grace disabled (0), dropping the last subscriber must + /// match today's behavior: the session stays with no + /// detached timestamp and is never eligible for a detach-grace close — it lingers only + /// until its normal lease expires. + /// + [Fact] + public async Task DetachGrace_Disabled_MatchesTodaysBehavior() + { + FakeTimeProvider clock = new(DateTimeOffset.UtcNow); + FakeWorkerClient workerClient = new(); + await using GatewaySession session = CreateReadySessionWithDetachGrace( + workerClient, + clock, + detachGrace: TimeSpan.Zero); + + IEventSubscriberLease lease = session.AttachEventSubscriber(maxSubscribers: 1); + lease.Dispose(); + + Assert.Equal(SessionState.Ready, session.State); + Assert.Null(session.DetachedAtUtc); + + clock.Advance(TimeSpan.FromHours(1)); + Assert.False(session.IsDetachGraceExpired(clock.GetUtcNow())); + } + + /// + /// Task 11. The gateway-owned internal dashboard subscriber must NOT keep a session out + /// of detach-grace: with only the dashboard mirror attached (and no external gRPC + /// subscriber), dropping the last external subscriber still enters grace and the + /// window still expires. + /// + [Fact] + public async Task DetachGrace_DashboardMirrorAlone_DoesNotPreventGraceEntry() + { + FakeTimeProvider clock = new(DateTimeOffset.UtcNow); + FakeWorkerClient workerClient = new(); + RecordingDashboardEventBroadcaster broadcaster = new(); + await using GatewaySession session = CreateReadySessionWithDetachGrace( + workerClient, + clock, + detachGrace: TimeSpan.FromSeconds(30), + dashboardBroadcaster: broadcaster); + + // The dashboard mirror is the only subscriber (registered internally at MarkReady). + // It is not counted as an external subscriber. + Assert.Equal(0, session.ActiveEventSubscriberCount); + + IEventSubscriberLease lease = session.AttachEventSubscriber(maxSubscribers: 1); + lease.Dispose(); + + // Entered grace despite the dashboard mirror still being attached. + Assert.NotNull(session.DetachedAtUtc); + + clock.Advance(TimeSpan.FromSeconds(30)); + Assert.True(session.IsDetachGraceExpired(clock.GetUtcNow())); + } + + private static GatewaySession CreateReadySessionWithDetachGrace( + IWorkerClient workerClient, + TimeProvider timeProvider, + TimeSpan detachGrace, + IDashboardEventBroadcaster? dashboardBroadcaster = null) + { + GatewaySession session = new( + sessionId: "session-test-detach-grace", + backendName: "mxaccess", + pipeName: "mxaccess-gateway-1-session-test-detach-grace", + nonce: "nonce", + clientIdentity: "client-1", + ownerKeyId: null, + clientSessionName: "test-session", + clientCorrelationId: "client-correlation-1", + commandTimeout: TimeSpan.FromSeconds(5), + startupTimeout: TimeSpan.FromSeconds(5), + shutdownTimeout: TimeSpan.FromSeconds(5), + leaseDuration: TimeSpan.FromMinutes(30), + openedAt: timeProvider.GetUtcNow(), + eventStreaming: new SessionEventStreaming( + new MxAccessGrpcMapper(), + new EventOptions { QueueCapacity = 8 }, + NullLogger.Instance, + timeProvider, + new GatewayMetrics(), + dashboardBroadcaster), + detachGrace: detachGrace); + session.AttachWorkerClient(workerClient); + session.MarkReady(); + return session; + } + private static GatewaySession CreateReadySession(IWorkerClient workerClient) { GatewaySession session = new( diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Sessions/SessionManagerTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Sessions/SessionManagerTests.cs index 61dc32e..586bf0f 100644 --- a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Sessions/SessionManagerTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Sessions/SessionManagerTests.cs @@ -1,5 +1,6 @@ using Google.Protobuf.WellKnownTypes; using Microsoft.Extensions.Options; +using Microsoft.Extensions.Time.Testing; using ZB.MOM.WW.MxGateway.Contracts.Proto; using ZB.MOM.WW.MxGateway.Server.Configuration; using ZB.MOM.WW.MxGateway.Server.Metrics; @@ -752,6 +753,42 @@ public sealed class SessionManagerTests Assert.Equal(0, workerClient.ShutdownCount); } + /// + /// Task 11. With detach-grace enabled, a session whose last external subscriber dropped + /// and whose detach-grace window has elapsed is closed by the lease sweep exactly like an + /// expired-lease session — even though its normal lease is still far in the future. + /// + [Fact] + public async Task CloseExpiredLeasesAsync_ClosesSessionWhoseDetachGraceWindowExpired() + { + FakeWorkerClient workerClient = new(); + FakeTimeProvider clock = new(DateTimeOffset.UtcNow); + SessionManager manager = CreateManager( + new FakeSessionWorkerClientFactory(workerClient), + options: CreateOptions(defaultLeaseSeconds: 1800, detachGraceSeconds: 30), + timeProvider: clock); + GatewaySession session = await manager.OpenSessionAsync(CreateOpenRequest(), "client-1", ownerKeyId: null, CancellationToken.None); + + // Attach and drop an external subscriber to enter detach-grace. The normal lease is + // still 30 minutes out, so only the detach-grace window can close this session. + IDisposable subscriber = session.AttachEventSubscriber(maxSubscribers: 1); + subscriber.Dispose(); + Assert.NotNull(session.DetachedAtUtc); + + // Before the window elapses: not closed. + clock.Advance(TimeSpan.FromSeconds(29)); + int closedBefore = await manager.CloseExpiredLeasesAsync(clock.GetUtcNow(), CancellationToken.None); + Assert.Equal(0, closedBefore); + Assert.Equal(SessionState.Ready, session.State); + + // After the window elapses: the sweep closes it. + clock.Advance(TimeSpan.FromSeconds(1)); + int closedAfter = await manager.CloseExpiredLeasesAsync(clock.GetUtcNow(), CancellationToken.None); + Assert.Equal(1, closedAfter); + Assert.Equal(SessionState.Closed, session.State); + Assert.Equal(1, workerClient.ShutdownCount); + } + /// Verifies that shutdown closes all registered sessions. [Fact] public async Task ShutdownAsync_ClosesAllRegisteredSessions() @@ -797,7 +834,8 @@ public sealed class SessionManagerTests private static GatewayOptions CreateOptions( int maxSessions = 64, - int defaultLeaseSeconds = 1800) + int defaultLeaseSeconds = 1800, + int detachGraceSeconds = 0) { return new GatewayOptions { @@ -806,6 +844,7 @@ public sealed class SessionManagerTests DefaultCommandTimeoutSeconds = 30, MaxSessions = maxSessions, DefaultLeaseSeconds = defaultLeaseSeconds, + DetachGraceSeconds = detachGraceSeconds, }, Worker = new WorkerOptions {