feat(sessions): detach-grace retention window for reconnect
This commit is contained in:
@@ -37,6 +37,7 @@ paths, timeouts, queue sizes, enum values, or protocol values are invalid.
|
|||||||
"MaxPendingCommandsPerSession": 128,
|
"MaxPendingCommandsPerSession": 128,
|
||||||
"DefaultLeaseSeconds": 1800,
|
"DefaultLeaseSeconds": 1800,
|
||||||
"LeaseSweepIntervalSeconds": 30,
|
"LeaseSweepIntervalSeconds": 30,
|
||||||
|
"DetachGraceSeconds": 30,
|
||||||
"AllowMultipleEventSubscribers": false,
|
"AllowMultipleEventSubscribers": false,
|
||||||
"MaxEventSubscribersPerSession": 8
|
"MaxEventSubscribersPerSession": 8
|
||||||
},
|
},
|
||||||
@@ -126,6 +127,7 @@ to avoid accidental large allocations from malformed or oversized frames.
|
|||||||
| `MxGateway:Sessions:MaxPendingCommandsPerSession` | `128` | Maximum number of pending worker commands for one session. Excess commands fail fast instead of queueing indefinitely. |
|
| `MxGateway:Sessions:MaxPendingCommandsPerSession` | `128` | Maximum number of pending worker commands for one session. Excess commands fail fast instead of queueing indefinitely. |
|
||||||
| `MxGateway:Sessions:DefaultLeaseSeconds` | `1800` | Initial session lease and refresh duration. Unary client activity extends the lease by this duration. |
|
| `MxGateway:Sessions:DefaultLeaseSeconds` | `1800` | Initial session lease and refresh duration. Unary client activity extends the lease by this duration. |
|
||||||
| `MxGateway:Sessions:LeaseSweepIntervalSeconds` | `30` | Hosted monitor interval for closing expired leases. Active event-stream subscribers keep a session from expiring while the stream remains attached. |
|
| `MxGateway:Sessions:LeaseSweepIntervalSeconds` | `30` | Hosted monitor interval for closing expired leases. Active event-stream subscribers keep a session from expiring while the stream remains attached. |
|
||||||
|
| `MxGateway:Sessions:DetachGraceSeconds` | `30` | Detach-grace retention window. When positive, a session whose last external (gRPC) event-stream subscriber drops is retained in `Ready` for this many seconds so a client can reconnect; if no external subscriber re-attaches within the window, the lease monitor closes it with `detach-grace-expired`. The internal dashboard mirror does not count as an external subscriber, so a dashboard-only session still enters detach-grace. `0` disables retention and reverts to closing only on normal lease expiry. Must be zero or greater. Reconnect/replay itself is implemented separately (Task 12); this option controls retention and expiry only. |
|
||||||
| `MxGateway:Sessions:AllowMultipleEventSubscribers` | `false` | Controls whether multiple `StreamEvents` subscribers may attach to one session. When `false` the session refuses a second subscriber with `AlreadyExists`. Set to `true` to enable fan-out via the `SessionEventDistributor`. |
|
| `MxGateway:Sessions:AllowMultipleEventSubscribers` | `false` | Controls whether multiple `StreamEvents` subscribers may attach to one session. When `false` the session refuses a second subscriber with `AlreadyExists`. Set to `true` to enable fan-out via the `SessionEventDistributor`. |
|
||||||
| `MxGateway:Sessions:MaxEventSubscribersPerSession` | `8` | Maximum number of concurrent `StreamEvents` subscribers per session when `AllowMultipleEventSubscribers` is `true`. Effectively 1 when `AllowMultipleEventSubscribers` is `false`. Must be greater than zero. |
|
| `MxGateway:Sessions:MaxEventSubscribersPerSession` | `8` | Maximum number of concurrent `StreamEvents` subscribers per session when `AllowMultipleEventSubscribers` is `true`. Effectively 1 when `AllowMultipleEventSubscribers` is `false`. Must be greater than zero. |
|
||||||
|
|
||||||
|
|||||||
+9
-1
@@ -72,7 +72,7 @@ private void EnsureSessionCapacity()
|
|||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
`SessionManager` also defines three close-reason constants — `DefaultCloseReason` (`"client-close"`), `GatewayShutdownReason` (`"gateway-shutdown"`), and `LeaseExpiredReason` (`"lease-expired"`) — so that the metrics and worker shutdown paths agree on a fixed vocabulary.
|
`SessionManager` also defines four close-reason constants — `DefaultCloseReason` (`"client-close"`), `GatewayShutdownReason` (`"gateway-shutdown"`), `LeaseExpiredReason` (`"lease-expired"`), and `DetachGraceExpiredReason` (`"detach-grace-expired"`) — so that the metrics and worker shutdown paths agree on a fixed vocabulary.
|
||||||
|
|
||||||
### SessionRegistry (ISessionRegistry)
|
### SessionRegistry (ISessionRegistry)
|
||||||
|
|
||||||
@@ -199,6 +199,14 @@ Event streaming uses `AttachEventSubscriber` which returns a disposable lease. W
|
|||||||
|
|
||||||
Sessions open with `MxGateway:Sessions:DefaultLeaseSeconds` (default 1800) added to the open timestamp. Unary client activity refreshes the lease by the same duration. `ExtendLease` and `IsLeaseExpired` cooperate with `SessionManager.CloseExpiredLeasesAsync`, which iterates a registry snapshot and closes any session whose lease has expired with `LeaseExpiredReason`. `SessionLeaseMonitorHostedService` runs that sweep every `MxGateway:Sessions:LeaseSweepIntervalSeconds` seconds (default 30).
|
Sessions open with `MxGateway:Sessions:DefaultLeaseSeconds` (default 1800) added to the open timestamp. Unary client activity refreshes the lease by the same duration. `ExtendLease` and `IsLeaseExpired` cooperate with `SessionManager.CloseExpiredLeasesAsync`, which iterates a registry snapshot and closes any session whose lease has expired with `LeaseExpiredReason`. `SessionLeaseMonitorHostedService` runs that sweep every `MxGateway:Sessions:LeaseSweepIntervalSeconds` seconds (default 30).
|
||||||
|
|
||||||
|
#### Detach-grace retention
|
||||||
|
|
||||||
|
`MxGateway:Sessions:DetachGraceSeconds` (default 30) is a bounded retention window kept after a session's *last external (gRPC) event-stream subscriber* drops, so a client can reconnect to the same session instead of having it torn down on the first stream disconnect. While the window is open the session stays `Ready` and fully usable — worker commands continue to work and a reconnecting subscriber re-attaches normally. Because retention is keyed on the *external* subscriber count (`_activeEventSubscriberCount`), and the gateway-owned internal dashboard mirror registers directly on the distributor with `isInternal: true` and is therefore *not* counted, a session whose only remaining subscriber is the dashboard mirror still enters detach-grace.
|
||||||
|
|
||||||
|
Mechanically: when the last external subscriber detaches and `DetachGraceSeconds > 0`, `DetachEventSubscriber` stamps `DetachedAtUtc` from the session's `TimeProvider` under `_syncRoot` (the detach→grace-start transition). `AttachEventSubscriber` clears `DetachedAtUtc` under the same lock when a subscriber re-attaches (the reattach→grace-cancel transition), so the two races and the sweeper's read all serialize on `_syncRoot`. `SessionManager.CloseExpiredLeasesAsync` checks `IsDetachGraceExpired(now)` alongside `IsLeaseExpired(now)`: a session detached for at least `DetachGraceSeconds` with no active external subscriber is closed by the same lease sweep, with the distinct `DetachGraceExpiredReason` (`"detach-grace-expired"`) so operators can tell a short reconnect-window expiry from a long idle-lease expiry. Setting `DetachGraceSeconds` to `0` disables retention and reverts to the original behavior: a detached session is retained only until its normal lease expires.
|
||||||
|
|
||||||
|
The reconnect/replay path that re-attaches a dropped client to a retained session is implemented separately (Task 12); `DetachGraceSeconds` controls retention and expiry only.
|
||||||
|
|
||||||
### Close
|
### Close
|
||||||
|
|
||||||
`GatewaySession.CloseAsync` is serialized by a per-session `SemaphoreSlim` (`_closeLock`) so only one close runs at a time, but every read/write of `_state` still passes through `_syncRoot` (via `TryBeginClose` and `MarkClosed`). The close path therefore obeys the same lock discipline as `TransitionTo` / `MarkFaulted`: it transitions to `Closing`, asks the worker client to shut down within `ShutdownTimeout`, and on success transitions to `Closed`. `DisposeAsync` waits on `_closeLock` once before disposing the semaphore so an in-flight close's `Release()` cannot race against the dispose. If `WorkerClient.ShutdownAsync` throws, the session falls back to `IWorkerClient.Kill` (forced close):
|
`GatewaySession.CloseAsync` is serialized by a per-session `SemaphoreSlim` (`_closeLock`) so only one close runs at a time, but every read/write of `_state` still passes through `_syncRoot` (via `TryBeginClose` and `MarkClosed`). The close path therefore obeys the same lock discipline as `TransitionTo` / `MarkFaulted`: it transitions to `Closing`, asks the worker client to shut down within `ShutdownTimeout`, and on success transitions to `Closed`. `DisposeAsync` waits on `_closeLock` once before disposing the semaphore so an in-flight close's `Release()` cannot race against the dispose. If `WorkerClient.ShutdownAsync` throws, the session falls back to `IWorkerClient.Kill` (forced close):
|
||||||
|
|||||||
@@ -181,6 +181,10 @@ public sealed class GatewayOptionsValidator : OptionsValidatorBase<GatewayOption
|
|||||||
options.MaxEventSubscribersPerSession,
|
options.MaxEventSubscribersPerSession,
|
||||||
"MxGateway:Sessions:MaxEventSubscribersPerSession must be greater than zero.",
|
"MxGateway:Sessions:MaxEventSubscribersPerSession must be greater than zero.",
|
||||||
builder);
|
builder);
|
||||||
|
AddIfNegative(
|
||||||
|
options.DetachGraceSeconds,
|
||||||
|
"MxGateway:Sessions:DetachGraceSeconds must be zero or greater (0 disables detach-grace retention).",
|
||||||
|
builder);
|
||||||
|
|
||||||
// NOTE: We intentionally do NOT reject !AllowMultipleEventSubscribers &&
|
// NOTE: We intentionally do NOT reject !AllowMultipleEventSubscribers &&
|
||||||
// MaxEventSubscribersPerSession > 1 as a hard validation error here. The default
|
// MaxEventSubscribersPerSession > 1 as a hard validation error here. The default
|
||||||
|
|||||||
@@ -23,6 +23,21 @@ public sealed class SessionOptions
|
|||||||
/// <summary>Gets the interval for sweeping expired session leases in seconds.</summary>
|
/// <summary>Gets the interval for sweeping expired session leases in seconds.</summary>
|
||||||
public int LeaseSweepIntervalSeconds { get; init; } = 30;
|
public int LeaseSweepIntervalSeconds { get; init; } = 30;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets the detach-grace retention window, in seconds, that a session is kept alive
|
||||||
|
/// after its last external (gRPC) event-stream subscriber drops, so a client can
|
||||||
|
/// reconnect to it. While within the window the session stays in
|
||||||
|
/// <c>Ready</c> and remains usable; if no new external subscriber attaches before the
|
||||||
|
/// window elapses, the lease monitor closes the session exactly as it closes an
|
||||||
|
/// expired lease. The gateway-owned internal dashboard subscriber does not count as an
|
||||||
|
/// external subscriber, so a session whose only remaining subscriber is the dashboard
|
||||||
|
/// mirror still enters detach-grace. A value of <c>0</c> disables retention: the
|
||||||
|
/// session reverts to the original behavior of lingering only until its normal lease
|
||||||
|
/// expires. The reconnect/replay itself is implemented separately (Task 12); this
|
||||||
|
/// option controls retention and expiry only.
|
||||||
|
/// </summary>
|
||||||
|
public int DetachGraceSeconds { get; init; } = 30;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets a value indicating whether multiple event subscribers are allowed per session.
|
/// Gets a value indicating whether multiple event subscribers are allowed per session.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
|
|||||||
@@ -22,6 +22,8 @@ public sealed class GatewaySession
|
|||||||
private DateTimeOffset? _leaseExpiresAt;
|
private DateTimeOffset? _leaseExpiresAt;
|
||||||
private bool _closeStarted;
|
private bool _closeStarted;
|
||||||
private int _activeEventSubscriberCount;
|
private int _activeEventSubscriberCount;
|
||||||
|
private readonly TimeSpan _detachGrace;
|
||||||
|
private DateTimeOffset? _detachedAtUtc;
|
||||||
private SessionEventDistributor? _eventDistributor;
|
private SessionEventDistributor? _eventDistributor;
|
||||||
private bool _eventDistributorStarted;
|
private bool _eventDistributorStarted;
|
||||||
private bool _dashboardMirrorStarted;
|
private bool _dashboardMirrorStarted;
|
||||||
@@ -103,6 +105,16 @@ public sealed class GatewaySession
|
|||||||
/// session directly still get a working distributor. Production passes the
|
/// session directly still get a working distributor. Production passes the
|
||||||
/// DI-resolved dependencies.
|
/// DI-resolved dependencies.
|
||||||
/// </param>
|
/// </param>
|
||||||
|
/// <param name="detachGrace">
|
||||||
|
/// Retention window kept after the last external (gRPC) event subscriber drops, so a
|
||||||
|
/// client can reconnect (Task 12). When the window is positive and the active external
|
||||||
|
/// subscriber count falls to zero, the session stays <see cref="SessionState.Ready"/>
|
||||||
|
/// and records a detached timestamp; the lease monitor closes it once the window
|
||||||
|
/// elapses with no subscriber having re-attached. <see cref="TimeSpan.Zero"/> (the
|
||||||
|
/// default) disables retention and preserves the original lease-only expiry behavior.
|
||||||
|
/// The clock comes from <paramref name="eventStreaming"/>'s
|
||||||
|
/// <see cref="SessionEventStreaming.TimeProvider"/> so the timer is unit-testable.
|
||||||
|
/// </param>
|
||||||
public GatewaySession(
|
public GatewaySession(
|
||||||
string sessionId,
|
string sessionId,
|
||||||
string backendName,
|
string backendName,
|
||||||
@@ -117,7 +129,8 @@ public sealed class GatewaySession
|
|||||||
TimeSpan shutdownTimeout,
|
TimeSpan shutdownTimeout,
|
||||||
TimeSpan leaseDuration,
|
TimeSpan leaseDuration,
|
||||||
DateTimeOffset openedAt,
|
DateTimeOffset openedAt,
|
||||||
SessionEventStreaming? eventStreaming = null)
|
SessionEventStreaming? eventStreaming = null,
|
||||||
|
TimeSpan detachGrace = default)
|
||||||
{
|
{
|
||||||
if (string.IsNullOrWhiteSpace(sessionId))
|
if (string.IsNullOrWhiteSpace(sessionId))
|
||||||
{
|
{
|
||||||
@@ -155,6 +168,7 @@ public sealed class GatewaySession
|
|||||||
_lastClientActivityAt = openedAt;
|
_lastClientActivityAt = openedAt;
|
||||||
_leaseExpiresAt = openedAt + leaseDuration;
|
_leaseExpiresAt = openedAt + leaseDuration;
|
||||||
_eventStreaming = eventStreaming ?? SessionEventStreaming.Default;
|
_eventStreaming = eventStreaming ?? SessionEventStreaming.Default;
|
||||||
|
_detachGrace = detachGrace > TimeSpan.Zero ? detachGrace : TimeSpan.Zero;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
@@ -300,6 +314,25 @@ public sealed class GatewaySession
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets the UTC timestamp at which the session entered its detach-grace retention
|
||||||
|
/// window (the last external event subscriber dropped while a positive
|
||||||
|
/// detach-grace was configured), or <see langword="null"/> when the session is not
|
||||||
|
/// currently within a detach-grace window. Re-attaching an external subscriber clears
|
||||||
|
/// this. Always <see langword="null"/> when detach-grace is disabled
|
||||||
|
/// (<c>DetachGraceSeconds == 0</c>).
|
||||||
|
/// </summary>
|
||||||
|
public DateTimeOffset? DetachedAtUtc
|
||||||
|
{
|
||||||
|
get
|
||||||
|
{
|
||||||
|
lock (_syncRoot)
|
||||||
|
{
|
||||||
|
return _detachedAtUtc;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Attaches the worker client for this session.
|
/// Attaches the worker client for this session.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
@@ -679,6 +712,28 @@ public sealed class GatewaySession
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Determines whether the session's detach-grace retention window has elapsed: the
|
||||||
|
/// session entered detach-grace (its last external event subscriber dropped while a
|
||||||
|
/// positive detach-grace was configured) and has had no external subscriber re-attach
|
||||||
|
/// for longer than the configured detach-grace. The lease monitor closes such a
|
||||||
|
/// session exactly as it closes an expired lease. Always returns <see langword="false"/>
|
||||||
|
/// when detach-grace is disabled or when an external subscriber is attached (the
|
||||||
|
/// detached timestamp is cleared on re-attach, so an attached session is never within a
|
||||||
|
/// window).
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="now">Current timestamp for comparison.</param>
|
||||||
|
public bool IsDetachGraceExpired(DateTimeOffset now)
|
||||||
|
{
|
||||||
|
lock (_syncRoot)
|
||||||
|
{
|
||||||
|
return _detachGrace > TimeSpan.Zero
|
||||||
|
&& _activeEventSubscriberCount == 0
|
||||||
|
&& _detachedAtUtc is not null
|
||||||
|
&& now - _detachedAtUtc.Value >= _detachGrace;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Attaches an event subscriber and returns a lease whose
|
/// Attaches an event subscriber and returns a lease whose
|
||||||
/// <see cref="IEventSubscriberLease.Reader"/> reads the fanned public
|
/// <see cref="IEventSubscriberLease.Reader"/> reads the fanned public
|
||||||
@@ -733,6 +788,12 @@ public sealed class GatewaySession
|
|||||||
}
|
}
|
||||||
|
|
||||||
_activeEventSubscriberCount++;
|
_activeEventSubscriberCount++;
|
||||||
|
|
||||||
|
// An external subscriber (re)attached: cancel any in-flight detach-grace window so
|
||||||
|
// the lease monitor no longer treats this session as eligible for grace-expiry
|
||||||
|
// close. This is the reattach→grace-cancel transition; it races the sweeper's
|
||||||
|
// IsDetachGraceExpired read, and both run under _syncRoot so they serialize.
|
||||||
|
_detachedAtUtc = null;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Construct/start the distributor and register this subscriber. Done outside the
|
// Construct/start the distributor and register this subscriber. Done outside the
|
||||||
@@ -1502,6 +1563,24 @@ public sealed class GatewaySession
|
|||||||
{
|
{
|
||||||
_activeEventSubscriberCount--;
|
_activeEventSubscriberCount--;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// When the LAST external subscriber drops and detach-grace is enabled, retain the
|
||||||
|
// session instead of letting it linger only on the (long) lease: stamp the detached
|
||||||
|
// time so the lease monitor can close it once the grace window elapses. The session
|
||||||
|
// stays in its current (Ready) state and remains usable, so a reconnecting subscriber
|
||||||
|
// (Task 12) re-attaches normally. The gateway-owned internal dashboard subscriber is
|
||||||
|
// NOT counted in _activeEventSubscriberCount (it registers on the distributor with
|
||||||
|
// isInternal: true), so a session whose only remaining subscriber is the dashboard
|
||||||
|
// mirror still enters grace. Only stamp while the session is alive — once
|
||||||
|
// Closing/Closed/Faulted there is nothing to retain. This is the detach→grace-start
|
||||||
|
// transition; it shares _syncRoot with the reattach→grace-cancel write above and the
|
||||||
|
// sweeper's IsDetachGraceExpired read, so the three serialize.
|
||||||
|
if (_detachGrace > TimeSpan.Zero
|
||||||
|
&& _activeEventSubscriberCount == 0
|
||||||
|
&& _state is not (SessionState.Closing or SessionState.Closed or SessionState.Faulted))
|
||||||
|
{
|
||||||
|
_detachedAtUtc = _eventStreaming.TimeProvider.GetUtcNow();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -17,6 +17,7 @@ public sealed class SessionManager : ISessionManager
|
|||||||
public const string DefaultCloseReason = "client-close";
|
public const string DefaultCloseReason = "client-close";
|
||||||
public const string GatewayShutdownReason = "gateway-shutdown";
|
public const string GatewayShutdownReason = "gateway-shutdown";
|
||||||
public const string LeaseExpiredReason = "lease-expired";
|
public const string LeaseExpiredReason = "lease-expired";
|
||||||
|
public const string DetachGraceExpiredReason = "detach-grace-expired";
|
||||||
|
|
||||||
private readonly ISessionRegistry _registry;
|
private readonly ISessionRegistry _registry;
|
||||||
private readonly ISessionWorkerClientFactory _workerClientFactory;
|
private readonly ISessionWorkerClientFactory _workerClientFactory;
|
||||||
@@ -295,12 +296,22 @@ public sealed class SessionManager : ISessionManager
|
|||||||
int closedCount = 0;
|
int closedCount = 0;
|
||||||
foreach (GatewaySession session in _registry.Snapshot())
|
foreach (GatewaySession session in _registry.Snapshot())
|
||||||
{
|
{
|
||||||
if (!session.IsLeaseExpired(now))
|
// A session is swept when its normal lease has expired OR its detach-grace
|
||||||
|
// retention window has elapsed (last external subscriber dropped and no client
|
||||||
|
// reconnected within DetachGraceSeconds). The detach-grace close is the same
|
||||||
|
// teardown as a lease-expiry close; only the reason differs so operators can tell
|
||||||
|
// a short reconnect-window expiry from a long idle-lease expiry in logs/metrics.
|
||||||
|
string? reason = session.IsLeaseExpired(now)
|
||||||
|
? LeaseExpiredReason
|
||||||
|
: session.IsDetachGraceExpired(now)
|
||||||
|
? DetachGraceExpiredReason
|
||||||
|
: null;
|
||||||
|
if (reason is null)
|
||||||
{
|
{
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
await CloseSessionCoreAsync(session, LeaseExpiredReason, cancellationToken).ConfigureAwait(false);
|
await CloseSessionCoreAsync(session, reason, cancellationToken).ConfigureAwait(false);
|
||||||
closedCount++;
|
closedCount++;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -478,7 +489,8 @@ public sealed class SessionManager : ISessionManager
|
|||||||
shutdownTimeout,
|
shutdownTimeout,
|
||||||
leaseDuration,
|
leaseDuration,
|
||||||
openedAt,
|
openedAt,
|
||||||
eventStreaming);
|
eventStreaming,
|
||||||
|
TimeSpan.FromSeconds(Math.Max(0, _options.Sessions.DetachGraceSeconds)));
|
||||||
}
|
}
|
||||||
|
|
||||||
private static string CreateClientCorrelationId(
|
private static string CreateClientCorrelationId(
|
||||||
|
|||||||
@@ -46,6 +46,7 @@
|
|||||||
"MaxPendingCommandsPerSession": 128,
|
"MaxPendingCommandsPerSession": 128,
|
||||||
"DefaultLeaseSeconds": 1800,
|
"DefaultLeaseSeconds": 1800,
|
||||||
"LeaseSweepIntervalSeconds": 30,
|
"LeaseSweepIntervalSeconds": 30,
|
||||||
|
"DetachGraceSeconds": 30,
|
||||||
"AllowMultipleEventSubscribers": false,
|
"AllowMultipleEventSubscribers": false,
|
||||||
"MaxEventSubscribersPerSession": 8
|
"MaxEventSubscribersPerSession": 8
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -34,6 +34,7 @@ public sealed class GatewayOptionsTests
|
|||||||
Assert.Equal(128, options.Sessions.MaxPendingCommandsPerSession);
|
Assert.Equal(128, options.Sessions.MaxPendingCommandsPerSession);
|
||||||
Assert.Equal(1800, options.Sessions.DefaultLeaseSeconds);
|
Assert.Equal(1800, options.Sessions.DefaultLeaseSeconds);
|
||||||
Assert.Equal(30, options.Sessions.LeaseSweepIntervalSeconds);
|
Assert.Equal(30, options.Sessions.LeaseSweepIntervalSeconds);
|
||||||
|
Assert.Equal(30, options.Sessions.DetachGraceSeconds);
|
||||||
Assert.False(options.Sessions.AllowMultipleEventSubscribers);
|
Assert.False(options.Sessions.AllowMultipleEventSubscribers);
|
||||||
Assert.Equal(8, options.Sessions.MaxEventSubscribersPerSession);
|
Assert.Equal(8, options.Sessions.MaxEventSubscribersPerSession);
|
||||||
|
|
||||||
@@ -86,6 +87,7 @@ public sealed class GatewayOptionsTests
|
|||||||
[InlineData("MxGateway:Worker:PipeConnectAttemptTimeoutMilliseconds", "0", "MxGateway:Worker:PipeConnectAttemptTimeoutMilliseconds must be greater than zero.")]
|
[InlineData("MxGateway:Worker:PipeConnectAttemptTimeoutMilliseconds", "0", "MxGateway:Worker:PipeConnectAttemptTimeoutMilliseconds must be greater than zero.")]
|
||||||
[InlineData("MxGateway:Sessions:DefaultLeaseSeconds", "0", "MxGateway:Sessions:DefaultLeaseSeconds must be greater than zero.")]
|
[InlineData("MxGateway:Sessions:DefaultLeaseSeconds", "0", "MxGateway:Sessions:DefaultLeaseSeconds must be greater than zero.")]
|
||||||
[InlineData("MxGateway:Sessions:LeaseSweepIntervalSeconds", "0", "MxGateway:Sessions:LeaseSweepIntervalSeconds must be greater than zero.")]
|
[InlineData("MxGateway:Sessions:LeaseSweepIntervalSeconds", "0", "MxGateway:Sessions:LeaseSweepIntervalSeconds must be greater than zero.")]
|
||||||
|
[InlineData("MxGateway:Sessions:DetachGraceSeconds", "-1", "MxGateway:Sessions:DetachGraceSeconds must be zero or greater (0 disables detach-grace retention).")]
|
||||||
[InlineData("MxGateway:Events:QueueCapacity", "0", "MxGateway:Events:QueueCapacity must be greater than zero.")]
|
[InlineData("MxGateway:Events:QueueCapacity", "0", "MxGateway:Events:QueueCapacity must be greater than zero.")]
|
||||||
[InlineData("MxGateway:Protocol:MaxGrpcMessageBytes", "0", "MxGateway:Protocol:MaxGrpcMessageBytes must be between")]
|
[InlineData("MxGateway:Protocol:MaxGrpcMessageBytes", "0", "MxGateway:Protocol:MaxGrpcMessageBytes must be between")]
|
||||||
[InlineData("MxGateway:Authentication:PepperSecretName", "", "MxGateway:Authentication:PepperSecretName is required")]
|
[InlineData("MxGateway:Authentication:PepperSecretName", "", "MxGateway:Authentication:PepperSecretName is required")]
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
using System.Runtime.CompilerServices;
|
using System.Runtime.CompilerServices;
|
||||||
using Microsoft.Extensions.Logging.Abstractions;
|
using Microsoft.Extensions.Logging.Abstractions;
|
||||||
|
using Microsoft.Extensions.Time.Testing;
|
||||||
using ZB.MOM.WW.MxGateway.Contracts.Proto;
|
using ZB.MOM.WW.MxGateway.Contracts.Proto;
|
||||||
using ZB.MOM.WW.MxGateway.Server.Configuration;
|
using ZB.MOM.WW.MxGateway.Server.Configuration;
|
||||||
using ZB.MOM.WW.MxGateway.Server.Dashboard.Hubs;
|
using ZB.MOM.WW.MxGateway.Server.Dashboard.Hubs;
|
||||||
@@ -432,6 +433,183 @@ public sealed class GatewaySessionTests
|
|||||||
await session.DisposeAsync();
|
await session.DisposeAsync();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Task 11. With a positive detach-grace, dropping the last external subscriber must
|
||||||
|
/// RETAIN the session (it stays <see cref="SessionState.Ready"/>, not Closed/Faulted)
|
||||||
|
/// and record a detached timestamp so the lease monitor can age it out later.
|
||||||
|
/// </summary>
|
||||||
|
[Fact]
|
||||||
|
public async Task DetachGrace_LastSubscriberDrops_RetainsSessionAndRecordsDetachedTimestamp()
|
||||||
|
{
|
||||||
|
FakeTimeProvider clock = new(DateTimeOffset.UtcNow);
|
||||||
|
FakeWorkerClient workerClient = new();
|
||||||
|
await using GatewaySession session = CreateReadySessionWithDetachGrace(
|
||||||
|
workerClient,
|
||||||
|
clock,
|
||||||
|
detachGrace: TimeSpan.FromSeconds(30));
|
||||||
|
|
||||||
|
IEventSubscriberLease lease = session.AttachEventSubscriber(maxSubscribers: 1);
|
||||||
|
Assert.Null(session.DetachedAtUtc);
|
||||||
|
|
||||||
|
lease.Dispose();
|
||||||
|
|
||||||
|
// Retained, not torn down.
|
||||||
|
Assert.Equal(SessionState.Ready, session.State);
|
||||||
|
Assert.Equal(0, session.ActiveEventSubscriberCount);
|
||||||
|
Assert.Equal(clock.GetUtcNow(), session.DetachedAtUtc);
|
||||||
|
Assert.False(session.IsDetachGraceExpired(clock.GetUtcNow()));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Task 11. Advancing the clock past the detach-grace window makes the retained,
|
||||||
|
/// detached session eligible for close (<see cref="GatewaySession.IsDetachGraceExpired"/>).
|
||||||
|
/// </summary>
|
||||||
|
[Fact]
|
||||||
|
public async Task DetachGrace_ClockAdvancesPastWindow_SessionBecomesEligibleForClose()
|
||||||
|
{
|
||||||
|
FakeTimeProvider clock = new(DateTimeOffset.UtcNow);
|
||||||
|
FakeWorkerClient workerClient = new();
|
||||||
|
await using GatewaySession session = CreateReadySessionWithDetachGrace(
|
||||||
|
workerClient,
|
||||||
|
clock,
|
||||||
|
detachGrace: TimeSpan.FromSeconds(30));
|
||||||
|
|
||||||
|
IEventSubscriberLease lease = session.AttachEventSubscriber(maxSubscribers: 1);
|
||||||
|
lease.Dispose();
|
||||||
|
|
||||||
|
// Just before the window elapses: not yet eligible.
|
||||||
|
clock.Advance(TimeSpan.FromSeconds(29));
|
||||||
|
Assert.False(session.IsDetachGraceExpired(clock.GetUtcNow()));
|
||||||
|
|
||||||
|
// At/after the window: eligible for close.
|
||||||
|
clock.Advance(TimeSpan.FromSeconds(1));
|
||||||
|
Assert.True(session.IsDetachGraceExpired(clock.GetUtcNow()));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Task 11. Re-attaching a subscriber before the window elapses cancels the grace:
|
||||||
|
/// the detached timestamp clears and a subsequent clock advance does NOT make the
|
||||||
|
/// session eligible for close.
|
||||||
|
/// </summary>
|
||||||
|
[Fact]
|
||||||
|
public async Task DetachGrace_ReattachBeforeExpiry_CancelsGrace()
|
||||||
|
{
|
||||||
|
FakeTimeProvider clock = new(DateTimeOffset.UtcNow);
|
||||||
|
FakeWorkerClient workerClient = new();
|
||||||
|
await using GatewaySession session = CreateReadySessionWithDetachGrace(
|
||||||
|
workerClient,
|
||||||
|
clock,
|
||||||
|
detachGrace: TimeSpan.FromSeconds(30));
|
||||||
|
|
||||||
|
IEventSubscriberLease first = session.AttachEventSubscriber(maxSubscribers: 1);
|
||||||
|
first.Dispose();
|
||||||
|
Assert.NotNull(session.DetachedAtUtc);
|
||||||
|
|
||||||
|
clock.Advance(TimeSpan.FromSeconds(10));
|
||||||
|
IEventSubscriberLease second = session.AttachEventSubscriber(maxSubscribers: 1);
|
||||||
|
|
||||||
|
// Re-attach cancelled the grace window.
|
||||||
|
Assert.Null(session.DetachedAtUtc);
|
||||||
|
|
||||||
|
// Advancing well past what would have been the window does not make it eligible while
|
||||||
|
// a subscriber is attached.
|
||||||
|
clock.Advance(TimeSpan.FromMinutes(5));
|
||||||
|
Assert.False(session.IsDetachGraceExpired(clock.GetUtcNow()));
|
||||||
|
|
||||||
|
second.Dispose();
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Task 11. With detach-grace disabled (<c>0</c>), dropping the last subscriber must
|
||||||
|
/// match today's behavior: the session stays <see cref="SessionState.Ready"/> with no
|
||||||
|
/// detached timestamp and is never eligible for a detach-grace close — it lingers only
|
||||||
|
/// until its normal lease expires.
|
||||||
|
/// </summary>
|
||||||
|
[Fact]
|
||||||
|
public async Task DetachGrace_Disabled_MatchesTodaysBehavior()
|
||||||
|
{
|
||||||
|
FakeTimeProvider clock = new(DateTimeOffset.UtcNow);
|
||||||
|
FakeWorkerClient workerClient = new();
|
||||||
|
await using GatewaySession session = CreateReadySessionWithDetachGrace(
|
||||||
|
workerClient,
|
||||||
|
clock,
|
||||||
|
detachGrace: TimeSpan.Zero);
|
||||||
|
|
||||||
|
IEventSubscriberLease lease = session.AttachEventSubscriber(maxSubscribers: 1);
|
||||||
|
lease.Dispose();
|
||||||
|
|
||||||
|
Assert.Equal(SessionState.Ready, session.State);
|
||||||
|
Assert.Null(session.DetachedAtUtc);
|
||||||
|
|
||||||
|
clock.Advance(TimeSpan.FromHours(1));
|
||||||
|
Assert.False(session.IsDetachGraceExpired(clock.GetUtcNow()));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Task 11. The gateway-owned internal dashboard subscriber must NOT keep a session out
|
||||||
|
/// of detach-grace: with only the dashboard mirror attached (and no external gRPC
|
||||||
|
/// subscriber), dropping the last external subscriber still enters grace and the
|
||||||
|
/// window still expires.
|
||||||
|
/// </summary>
|
||||||
|
[Fact]
|
||||||
|
public async Task DetachGrace_DashboardMirrorAlone_DoesNotPreventGraceEntry()
|
||||||
|
{
|
||||||
|
FakeTimeProvider clock = new(DateTimeOffset.UtcNow);
|
||||||
|
FakeWorkerClient workerClient = new();
|
||||||
|
RecordingDashboardEventBroadcaster broadcaster = new();
|
||||||
|
await using GatewaySession session = CreateReadySessionWithDetachGrace(
|
||||||
|
workerClient,
|
||||||
|
clock,
|
||||||
|
detachGrace: TimeSpan.FromSeconds(30),
|
||||||
|
dashboardBroadcaster: broadcaster);
|
||||||
|
|
||||||
|
// The dashboard mirror is the only subscriber (registered internally at MarkReady).
|
||||||
|
// It is not counted as an external subscriber.
|
||||||
|
Assert.Equal(0, session.ActiveEventSubscriberCount);
|
||||||
|
|
||||||
|
IEventSubscriberLease lease = session.AttachEventSubscriber(maxSubscribers: 1);
|
||||||
|
lease.Dispose();
|
||||||
|
|
||||||
|
// Entered grace despite the dashboard mirror still being attached.
|
||||||
|
Assert.NotNull(session.DetachedAtUtc);
|
||||||
|
|
||||||
|
clock.Advance(TimeSpan.FromSeconds(30));
|
||||||
|
Assert.True(session.IsDetachGraceExpired(clock.GetUtcNow()));
|
||||||
|
}
|
||||||
|
|
||||||
|
private static GatewaySession CreateReadySessionWithDetachGrace(
|
||||||
|
IWorkerClient workerClient,
|
||||||
|
TimeProvider timeProvider,
|
||||||
|
TimeSpan detachGrace,
|
||||||
|
IDashboardEventBroadcaster? dashboardBroadcaster = null)
|
||||||
|
{
|
||||||
|
GatewaySession session = new(
|
||||||
|
sessionId: "session-test-detach-grace",
|
||||||
|
backendName: "mxaccess",
|
||||||
|
pipeName: "mxaccess-gateway-1-session-test-detach-grace",
|
||||||
|
nonce: "nonce",
|
||||||
|
clientIdentity: "client-1",
|
||||||
|
ownerKeyId: null,
|
||||||
|
clientSessionName: "test-session",
|
||||||
|
clientCorrelationId: "client-correlation-1",
|
||||||
|
commandTimeout: TimeSpan.FromSeconds(5),
|
||||||
|
startupTimeout: TimeSpan.FromSeconds(5),
|
||||||
|
shutdownTimeout: TimeSpan.FromSeconds(5),
|
||||||
|
leaseDuration: TimeSpan.FromMinutes(30),
|
||||||
|
openedAt: timeProvider.GetUtcNow(),
|
||||||
|
eventStreaming: new SessionEventStreaming(
|
||||||
|
new MxAccessGrpcMapper(),
|
||||||
|
new EventOptions { QueueCapacity = 8 },
|
||||||
|
NullLogger<SessionEventDistributor>.Instance,
|
||||||
|
timeProvider,
|
||||||
|
new GatewayMetrics(),
|
||||||
|
dashboardBroadcaster),
|
||||||
|
detachGrace: detachGrace);
|
||||||
|
session.AttachWorkerClient(workerClient);
|
||||||
|
session.MarkReady();
|
||||||
|
return session;
|
||||||
|
}
|
||||||
|
|
||||||
private static GatewaySession CreateReadySession(IWorkerClient workerClient)
|
private static GatewaySession CreateReadySession(IWorkerClient workerClient)
|
||||||
{
|
{
|
||||||
GatewaySession session = new(
|
GatewaySession session = new(
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
using Google.Protobuf.WellKnownTypes;
|
using Google.Protobuf.WellKnownTypes;
|
||||||
using Microsoft.Extensions.Options;
|
using Microsoft.Extensions.Options;
|
||||||
|
using Microsoft.Extensions.Time.Testing;
|
||||||
using ZB.MOM.WW.MxGateway.Contracts.Proto;
|
using ZB.MOM.WW.MxGateway.Contracts.Proto;
|
||||||
using ZB.MOM.WW.MxGateway.Server.Configuration;
|
using ZB.MOM.WW.MxGateway.Server.Configuration;
|
||||||
using ZB.MOM.WW.MxGateway.Server.Metrics;
|
using ZB.MOM.WW.MxGateway.Server.Metrics;
|
||||||
@@ -752,6 +753,42 @@ public sealed class SessionManagerTests
|
|||||||
Assert.Equal(0, workerClient.ShutdownCount);
|
Assert.Equal(0, workerClient.ShutdownCount);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Task 11. With detach-grace enabled, a session whose last external subscriber dropped
|
||||||
|
/// and whose detach-grace window has elapsed is closed by the lease sweep exactly like an
|
||||||
|
/// expired-lease session — even though its normal lease is still far in the future.
|
||||||
|
/// </summary>
|
||||||
|
[Fact]
|
||||||
|
public async Task CloseExpiredLeasesAsync_ClosesSessionWhoseDetachGraceWindowExpired()
|
||||||
|
{
|
||||||
|
FakeWorkerClient workerClient = new();
|
||||||
|
FakeTimeProvider clock = new(DateTimeOffset.UtcNow);
|
||||||
|
SessionManager manager = CreateManager(
|
||||||
|
new FakeSessionWorkerClientFactory(workerClient),
|
||||||
|
options: CreateOptions(defaultLeaseSeconds: 1800, detachGraceSeconds: 30),
|
||||||
|
timeProvider: clock);
|
||||||
|
GatewaySession session = await manager.OpenSessionAsync(CreateOpenRequest(), "client-1", ownerKeyId: null, CancellationToken.None);
|
||||||
|
|
||||||
|
// Attach and drop an external subscriber to enter detach-grace. The normal lease is
|
||||||
|
// still 30 minutes out, so only the detach-grace window can close this session.
|
||||||
|
IDisposable subscriber = session.AttachEventSubscriber(maxSubscribers: 1);
|
||||||
|
subscriber.Dispose();
|
||||||
|
Assert.NotNull(session.DetachedAtUtc);
|
||||||
|
|
||||||
|
// Before the window elapses: not closed.
|
||||||
|
clock.Advance(TimeSpan.FromSeconds(29));
|
||||||
|
int closedBefore = await manager.CloseExpiredLeasesAsync(clock.GetUtcNow(), CancellationToken.None);
|
||||||
|
Assert.Equal(0, closedBefore);
|
||||||
|
Assert.Equal(SessionState.Ready, session.State);
|
||||||
|
|
||||||
|
// After the window elapses: the sweep closes it.
|
||||||
|
clock.Advance(TimeSpan.FromSeconds(1));
|
||||||
|
int closedAfter = await manager.CloseExpiredLeasesAsync(clock.GetUtcNow(), CancellationToken.None);
|
||||||
|
Assert.Equal(1, closedAfter);
|
||||||
|
Assert.Equal(SessionState.Closed, session.State);
|
||||||
|
Assert.Equal(1, workerClient.ShutdownCount);
|
||||||
|
}
|
||||||
|
|
||||||
/// <summary>Verifies that shutdown closes all registered sessions.</summary>
|
/// <summary>Verifies that shutdown closes all registered sessions.</summary>
|
||||||
[Fact]
|
[Fact]
|
||||||
public async Task ShutdownAsync_ClosesAllRegisteredSessions()
|
public async Task ShutdownAsync_ClosesAllRegisteredSessions()
|
||||||
@@ -797,7 +834,8 @@ public sealed class SessionManagerTests
|
|||||||
|
|
||||||
private static GatewayOptions CreateOptions(
|
private static GatewayOptions CreateOptions(
|
||||||
int maxSessions = 64,
|
int maxSessions = 64,
|
||||||
int defaultLeaseSeconds = 1800)
|
int defaultLeaseSeconds = 1800,
|
||||||
|
int detachGraceSeconds = 0)
|
||||||
{
|
{
|
||||||
return new GatewayOptions
|
return new GatewayOptions
|
||||||
{
|
{
|
||||||
@@ -806,6 +844,7 @@ public sealed class SessionManagerTests
|
|||||||
DefaultCommandTimeoutSeconds = 30,
|
DefaultCommandTimeoutSeconds = 30,
|
||||||
MaxSessions = maxSessions,
|
MaxSessions = maxSessions,
|
||||||
DefaultLeaseSeconds = defaultLeaseSeconds,
|
DefaultLeaseSeconds = defaultLeaseSeconds,
|
||||||
|
DetachGraceSeconds = detachGraceSeconds,
|
||||||
},
|
},
|
||||||
Worker = new WorkerOptions
|
Worker = new WorkerOptions
|
||||||
{
|
{
|
||||||
|
|||||||
Reference in New Issue
Block a user