Merge stream A (session lifecycle: startup orphan, sweep metrics, stopping gate, kill-fallback Closed, unattached grace, worker-exit faults session)
This commit is contained in:
@@ -38,6 +38,7 @@ paths, timeouts, queue sizes, enum values, or protocol values are invalid.
|
||||
"LeaseSweepIntervalSeconds": 30,
|
||||
"DetachGraceSeconds": 30,
|
||||
"FaultedGraceSeconds": 0,
|
||||
"UnattachedSessionGraceSeconds": 0,
|
||||
"AllowMultipleEventSubscribers": false,
|
||||
"MaxEventSubscribersPerSession": 8,
|
||||
"WorkerReadyWaitTimeoutMs": 0
|
||||
@@ -153,6 +154,7 @@ validation fails fast when either half of the headroom rule is violated.
|
||||
| `MxGateway:Sessions:LeaseSweepIntervalSeconds` | `30` | Hosted monitor interval for closing expired leases. Active event-stream subscribers keep a session from expiring while the stream remains attached. |
|
||||
| `MxGateway:Sessions:DetachGraceSeconds` | `30` | Detach-grace retention window. When positive, a session whose last external (gRPC) event-stream subscriber drops is retained in `Ready` for this many seconds so a client can reconnect; if no external subscriber re-attaches within the window, the lease monitor closes it with `detach-grace-expired`. The internal dashboard mirror does not count as an external subscriber, so a dashboard-only session still enters detach-grace. `0` disables retention and reverts to closing only on normal lease expiry. Must be zero or greater. Reconnect/replay itself is implemented separately (Task 12); this option controls retention and expiry only. The effective close happens within the next sweep cycle after the window elapses — up to `LeaseSweepIntervalSeconds` after expiry. Operators wanting a firm minimum retention bound should set `DetachGraceSeconds` greater than `LeaseSweepIntervalSeconds`. |
|
||||
| `MxGateway:Sessions:FaultedGraceSeconds` | `0` | Grace window before the lease monitor reaps a faulted session (killing its worker and freeing the slot). A faulted session is permanently unusable yet otherwise pins a session slot and a live x86 worker until its normal lease expires. `0` (the default) reaps it on the next sweep cycle, bounding blast radius; a positive value keeps the faulted session observable via `GetSessionStatus` for that window before it is reclaimed, with the close reason `faulted-reaped`. Must be zero or greater. |
|
||||
| `MxGateway:Sessions:UnattachedSessionGraceSeconds` | `0` | Idle window after which the lease monitor reaps a `Ready` session that has **never** attached a `StreamEvents` subscriber, with the close reason `unattached-grace-expired`. Exists because a client that opens a session and vanishes before it ever streams — OtOpcUa's `GalaxyDriver` failing part-way through start-up, for example — otherwise pins a session slot and a live x86 worker for the full `DefaultLeaseSeconds` (30 minutes). The window is measured from the later of the session becoming `Ready` and its last unary command, so a slow worker start does not eat it and a unary-only client that keeps calling is never reaped by this rule; it stops applying once the session's first subscriber attaches (`DetachGraceSeconds` and the normal lease govern it from then on), and the gateway's own alarm-monitor session, whose subscriber is internal, is exempt the same way. `0` (the default) disables it. Must be zero or greater. Set it comfortably above the time a well-behaved client needs between `OpenSession` and its first `StreamEvents`; the close lands on the next sweep after the window elapses, up to `LeaseSweepIntervalSeconds` later. |
|
||||
| `MxGateway:Sessions:AllowMultipleEventSubscribers` | `false` | Controls whether multiple `StreamEvents` subscribers may attach to one session. When `false` the session refuses a second subscriber with `AlreadyExists`. Set to `true` to enable fan-out via the `SessionEventDistributor`. |
|
||||
| `MxGateway:Sessions:MaxEventSubscribersPerSession` | `8` | Maximum number of concurrent `StreamEvents` subscribers per session when `AllowMultipleEventSubscribers` is `true`. Effectively 1 when `AllowMultipleEventSubscribers` is `false`. Must be greater than zero. |
|
||||
| `MxGateway:Sessions:WorkerReadyWaitTimeoutMs` | `0` | Bounded time, in milliseconds, the gateway will wait for a worker to reach `Ready` when the session is already `Ready` but the worker state has transiently diverged (e.g. `Handshaking` after a heartbeat blip). Applies only to transient worker states; terminal states (`Faulted`/`Closing`/`Closed`/no worker) fail fast immediately regardless of this setting. `0` (the default) disables the wait and preserves the original fail-fast behavior. Must be greater than or equal to zero. |
|
||||
|
||||
@@ -314,6 +314,8 @@ StatusCode statusCode = exception.ErrorCode switch
|
||||
SessionManagerErrorCode.SessionLimitExceeded => StatusCode.ResourceExhausted,
|
||||
SessionManagerErrorCode.OpenFailed => StatusCode.Unavailable,
|
||||
SessionManagerErrorCode.CloseFailed => StatusCode.Unavailable,
|
||||
SessionManagerErrorCode.PermissionDenied => StatusCode.PermissionDenied,
|
||||
SessionManagerErrorCode.GatewayStopping => StatusCode.Unavailable,
|
||||
_ => StatusCode.Unavailable,
|
||||
};
|
||||
```
|
||||
|
||||
+87
-44
@@ -80,7 +80,7 @@ private void EnsureSessionCapacity()
|
||||
}
|
||||
```
|
||||
|
||||
`SessionManager` also defines five close-reason constants — `DefaultCloseReason` (`"client-close"`), `GatewayShutdownReason` (`"gateway-shutdown"`), `LeaseExpiredReason` (`"lease-expired"`), `FaultedReason` (`"faulted-reaped"`), and `DetachGraceExpiredReason` (`"detach-grace-expired"`) — so that the metrics and worker shutdown paths agree on a fixed vocabulary.
|
||||
`SessionManager` also defines six close-reason constants — `DefaultCloseReason` (`"client-close"`), `GatewayShutdownReason` (`"gateway-shutdown"`), `LeaseExpiredReason` (`"lease-expired"`), `FaultedReason` (`"faulted-reaped"`), `DetachGraceExpiredReason` (`"detach-grace-expired"`), and `UnattachedGraceExpiredReason` (`"unattached-grace-expired"`) — so that the metrics and worker shutdown paths agree on a fixed vocabulary.
|
||||
|
||||
### SessionRegistry (ISessionRegistry)
|
||||
|
||||
@@ -137,18 +137,19 @@ public sealed record SessionOpenRequest(
|
||||
|
||||
### SessionCloseResult
|
||||
|
||||
`SessionCloseResult` is the record returned from a successful close. `AlreadyClosed` distinguishes "this call closed the session" from "the session was already closed when we acquired the close lock", which the metrics layer uses to avoid double-counting.
|
||||
`SessionCloseResult` is the record returned from a successful close. `AlreadyClosed` distinguishes "this call closed the session" from "the session was already closed when we acquired the close lock", which the metrics layer uses to avoid double-counting. `WorkerShutdownFailure` is set when the worker refused a graceful shutdown and was killed instead — still a successful close (see [Close](#close)); the manager logs it and the gRPC reply does not expose it.
|
||||
|
||||
```csharp
|
||||
public sealed record SessionCloseResult(
|
||||
string SessionId,
|
||||
SessionState FinalState,
|
||||
bool AlreadyClosed);
|
||||
bool AlreadyClosed,
|
||||
Exception? WorkerShutdownFailure = null);
|
||||
```
|
||||
|
||||
### SessionCloseStartedException
|
||||
|
||||
`SessionCloseStartedException` is `internal` and is only thrown from inside `GatewaySession.CloseAsync` when the close path has already begun mutating worker state and a subsequent step fails. `SessionManager.CloseSessionCoreAsync` catches it, marks the session faulted, increments the close-failed metric, removes the session from the registry, and rethrows it wrapped as `SessionManagerException` with `CloseFailed`. The intermediate type exists so the public API surface only ever exposes `SessionManagerException`.
|
||||
`SessionCloseStartedException` is `internal` and is only thrown from inside `GatewaySession.CloseAsync` when the close path has already begun mutating worker state and a subsequent step fails — in practice, when the worker's graceful shutdown failed *and* the kill fall-back failed too, so a worker may still be running. `SessionManager.CloseSessionCoreAsync` catches it, marks the session faulted, increments the close-failed metric, removes the session from the registry, and rethrows it wrapped as `SessionManagerException` with `CloseFailed`. The intermediate type exists so the public API surface only ever exposes `SessionManagerException`.
|
||||
|
||||
### SessionManagerException and SessionManagerErrorCode
|
||||
|
||||
@@ -163,7 +164,9 @@ public sealed record SessionCloseResult(
|
||||
| `EventQueueOverflow` | Reserved for the worker event channel overflow path. |
|
||||
| `SessionLimitExceeded` | `MaxSessions` is in use. |
|
||||
| `OpenFailed` | `OpenSessionAsync` failed; the inner exception carries the cause. |
|
||||
| `CloseFailed` | A close started but did not complete cleanly; the session is removed and faulted. |
|
||||
| `CloseFailed` | A close started but did not complete cleanly — the worker's graceful shutdown and the kill fall-back both failed (or a kill threw); the session is removed and faulted. A shutdown failure whose kill succeeded is a successful close, not this. |
|
||||
| `PermissionDenied` | The caller may not perform the operation on this session (for example, attaching `StreamEvents` to a session opened by a different API key). |
|
||||
| `GatewayStopping` | The gateway is stopping and refuses new sessions (see [Shutdown Coordination](#shutdown-coordination)). Maps to gRPC `Unavailable`; not recorded as a fault. |
|
||||
|
||||
## Lifecycle
|
||||
|
||||
@@ -174,14 +177,7 @@ public sealed record SessionCloseResult(
|
||||
```csharp
|
||||
catch (Exception exception)
|
||||
{
|
||||
session?.MarkFaulted(exception.Message);
|
||||
if (session is not null)
|
||||
{
|
||||
_registry.TryRemove(session.SessionId, out _);
|
||||
await session.DisposeAsync().ConfigureAwait(false);
|
||||
}
|
||||
|
||||
ReleaseSessionSlot();
|
||||
await RollBackOpenAsync(session, registered, exception).ConfigureAwait(false);
|
||||
_metrics.Fault(SessionManagerErrorCode.OpenFailed.ToString());
|
||||
_logger.LogWarning(
|
||||
exception,
|
||||
@@ -195,7 +191,17 @@ catch (Exception exception)
|
||||
}
|
||||
```
|
||||
|
||||
The order — fault, deregister, dispose, release slot, record metric, log, rethrow — matters because releasing the semaphore before disposal would let the next open race the worker process tear-down on the same machine.
|
||||
`RollBackOpenAsync` faults the session and, when it had been registered, routes it through `RemoveSessionAsync` — the single removal path every teardown uses, which deregisters, drops the session's per-session event metrics, releases the slot and disposes, in that order. The order matters because releasing the semaphore before disposal would let the next open race the worker process tear-down on the same machine. A session that was never registered (id collision) releases the slot and disposes directly. The open-session gauge is not touched by the rollback: `SessionOpened` is recorded inside the attach gate described next, after which nothing in the open sequence can fail.
|
||||
|
||||
#### Close or kill during worker startup
|
||||
|
||||
The session is registered — and therefore visible to the dashboard's Kill button, the lease sweep and the host drain — for the whole worker startup, which can last up to `MxGateway:Worker:StartupTimeoutSeconds`. A kill or drain that lands in that window finds no worker client to stop, flips the session to `Closed`, removes it and disposes it. The worker client that `SessionWorkerClientFactory.CreateAsync` then returns must not be stored on that dead session and handed back to the caller as a successful open: nothing would ever shut the worker down (the close already ran) and the client would hold a `Closed` session.
|
||||
|
||||
`GatewaySession.AttachWorkerClient` therefore returns `false`, under `_syncRoot`, when the session is already `Closing`, `Closed` or `Faulted` or has been disposed. The manager runs the end of the open sequence — attach, `MarkReady`, `SessionOpened` — through `TryAttachWorkerClientAndMarkReadyAsync`, which holds the per-session close gate (`_closeLock`) so a concurrent `CloseAsync` / `KillWorkerWithCloseGateAsync` runs either entirely before it (the attach is refused) or entirely after it (an ordinary close of a Ready session). On a refused attach the manager kills and disposes the late worker client and the open fails with `OpenFailed`; the registry is already empty and the slot was released by whichever teardown closed the session.
|
||||
|
||||
Once `CreateAsync` has returned, a live worker exists that only the open path knows about, so the gate wait is **not cancellable** (it waits on `CancellationToken.None`; the gate is only ever held for the duration of one close or kill) and the manager discards the client on *any* exception out of the attach step — the caller's own cancellation (checked before the attach, so an abandoned open never yields a session), a disposed gate, anything thrown while becoming Ready. The failure then surfaces as `OpenFailed` with the cause as inner exception. An exception that escaped before the discard would be an orphaned x86 worker, which a restarted gateway terminates rather than adopts.
|
||||
|
||||
Because such a session was never counted as opened, it must not be counted as closed either. `GatewaySession.OpenRecorded` is set inside the same gate right after `SessionOpened` is recorded, and every teardown path (`KillWorkerAsync`, `CloseSessionCoreAsync`, the close-failed branch) records `SessionClosed` / `SessionRemoved` only when it is set — so `sessions.opened`, `sessions.closed` and the `sessions.open` gauge stay consistent across the startup window.
|
||||
|
||||
### Run
|
||||
|
||||
@@ -215,11 +221,13 @@ The repair transitions the monitor's reconcile broadcasts on the alarm feed (Rai
|
||||
|
||||
The same one-distributor-per-session rule is enforced at the other end of the lifecycle. Session disposal disposes the distributor and latches the fact, so an attach that passed its readiness gate and then lost the CPU until the session was torn down throws `SessionNotReady` rather than constructing a replacement distributor. A replacement can never be pumped — the pump-start flag is a one-way latch — so its subscriber would block on an empty channel that is never completed until the client's own deadline expired, with no error, log or metric to explain it. Every attach entry point (`AttachEventSubscriber`, `AttachEventSubscriberWithReplay`, `AttachInternalEventSubscriber`) fails this way, and each rolls back the subscriber-count slot it took, so a caller that loses the race to teardown sees the same failure it would have seen a moment later from the readiness gate.
|
||||
|
||||
Sessions open with `MxGateway:Sessions:DefaultLeaseSeconds` (default 1800) added to the open timestamp. Unary client activity refreshes the lease by the same duration. `ExtendLease` and `IsLeaseExpired` cooperate with `SessionManager.CloseExpiredLeasesAsync`, which iterates a registry snapshot and closes any session whose lease has expired with `LeaseExpiredReason`. `SessionLeaseMonitorHostedService` runs that sweep every `MxGateway:Sessions:LeaseSweepIntervalSeconds` seconds (default 30).
|
||||
Sessions open with `MxGateway:Sessions:DefaultLeaseSeconds` (default 1800) added to the open timestamp. Unary client activity refreshes the lease by the same duration (`TouchClientActivity` is the only path that moves the lease). `IsLeaseExpired` cooperates with `SessionManager.CloseExpiredLeasesAsync`, which iterates a registry snapshot and closes any session whose lease has expired with `LeaseExpiredReason`. `SessionLeaseMonitorHostedService` runs that sweep every `MxGateway:Sessions:LeaseSweepIntervalSeconds` seconds (default 30).
|
||||
|
||||
#### Teardown parallelism
|
||||
|
||||
A sweep pass is two phases. *Selection* stays a single sequential pass over the snapshot, because that is what gives the precedence rule (lease-expiry, then faulted, then detach-grace) and the `TryBeginCloseIfExpired` TOCTOU re-check their meaning. *Closing* then runs over the already-selected set with `Parallel.ForEachAsync` at `MaxParallelSessionCloses`, a compile-time constant of `4` in `SessionManager`. Each close is bounded by `MxGateway:Worker:ShutdownTimeoutSeconds` (default 10), so a one-at-a-time sweep lets a few hung workers serialize reaping and starve session slots for the rest. Parallel closing is safe because `TryBeginCloseIfExpired` already flipped each selected session to `Closing` under its own lock — that idempotent begin-close is the per-session exclusivity invariant, so no two teardowns can ever run against one session. The degree is a fixed constant rather than an option, and bounded rather than unlimited, because every concurrent close is one x86 worker process being shut down or killed; the fan-out exists to hide a few hung workers, not to tear the whole registry down at once.
|
||||
A sweep pass is two phases. *Selection* stays a single sequential pass over the snapshot, because that is what gives the precedence rule (lease-expiry, then faulted, then detach-grace, then unattached-grace) and the `TryBeginCloseIfExpired` TOCTOU re-check their meaning. *Closing* then runs over the already-selected set with `Parallel.ForEachAsync` at `MaxParallelSessionCloses`, a compile-time constant of `4` in `SessionManager`. Each close is bounded by `MxGateway:Worker:ShutdownTimeoutSeconds` (default 10), so a one-at-a-time sweep lets a few hung workers serialize reaping and starve session slots for the rest. Parallel closing is safe because `TryBeginCloseIfExpired` already flipped each selected session to `Closing` under its own lock — that idempotent begin-close is the per-session exclusivity invariant, so no two teardowns can ever run against one session. The degree is a fixed constant rather than an option, and bounded rather than unlimited, because every concurrent close is one x86 worker process being shut down or killed; the fan-out exists to hide a few hung workers, not to tear the whole registry down at once.
|
||||
|
||||
`TryBeginCloseIfExpired` is a *reservation*, not the close: it writes `Closing` but leaves the session's close-started flag alone, so the `CloseAsync` the sweep issues next is the first close to run and reports `AlreadyClosed: false`. That is what lets `CloseSessionCoreAsync` account `SessionClosed` for lease-expired, faulted-reaped and detach-grace-expired closes exactly once; a reservation that also set the flag made every sweep-driven close look like a second concurrent close, and swept sessions never left the `sessions.open` gauge. A client close that races the reservation still counts once: whichever `CloseAsync` takes the close lock first does the accounting, and the other finds the session `Closed`.
|
||||
|
||||
Splitting the phases moves the TOCTOU re-check earlier, and that is an accepted trade rather than an unchanged behavior: selection now flips **every** chosen session to `Closing` up front, before any teardown runs, whereas the sequential sweep re-checked session *N* only after sessions *1..N-1* had finished closing. A client that re-attaches a subscriber while the close phase is running therefore loses a race it could previously win — the eligibility snapshot is taken at one instant for the whole pass. Expiry evaluation itself is unaffected, because `now` is a parameter and is not re-read per session.
|
||||
|
||||
@@ -239,9 +247,21 @@ Mechanically: when the last external subscriber detaches and `DetachGraceSeconds
|
||||
|
||||
`DetachGraceSeconds` controls retention and expiry only; the reconnect/replay path that re-attaches a dropped client to a retained session is described in [Reconnect and replay](#reconnect-and-replay).
|
||||
|
||||
#### Worker exit outside a gateway close
|
||||
|
||||
`IWorkerClient` has no state-change callback, so a worker that reached `Closed` or `Faulted` *without the gateway closing it* — the client sent the `ShutdownWorker` control command and the worker exited cleanly, the worker crashed, its pipe broke, a heartbeat timed out — used to leave the session reporting `Ready` until its lease expired (a live finding: `WorkerClient` was `Closed` on the shutdown ack, the session still `Ready`, even with an event stream attached). The session now observes it from its own worker-event pump source (`MapWorkerEventsAsync`): the single distributor pump is the session's one always-on view of the worker (it runs from `MarkReady` via the dashboard mirror), and `WorkerClient` enters its terminal state *before* it completes the event channel, so when the enumeration ends the terminal state is visible. `ObserveWorkerTerminalState` then faults a `Ready` session whose worker is `Closed`/`Faulted`.
|
||||
|
||||
The choice is **`Faulted`, not `Closed`**: `Closed` means the gateway closed the session, and a `Closed` session left in the registry would linger until its lease expired with nothing reaping it. A faulted session is permanently unusable (every command fails the readiness check, as the worker is gone), stays observable via `GetSessionStatus` with `FinalFault` naming the worker state, and is reaped by the faulted-session sweep below after `FaultedGraceSeconds` — which is where `SessionClosed` is accounted, exactly once, through the ordinary `CloseSessionCoreAsync` path (`WorkerClient.ShutdownAsync` on a `Closed` client returns immediately, so no kill fallback or warning). Only a `Ready` session follows its worker: during a gateway-initiated close the worker closing is the expected outcome, not a fault.
|
||||
|
||||
#### Faulted-session reaping
|
||||
|
||||
A session that faults (for example, a slow single-subscriber client that overflows its event queue under the `FailFast` backpressure policy) is left permanently unusable — every command fails `EvaluateReadyUnderLock` — but its worker keeps running and it still pins one of `MxGateway:Sessions:MaxSessions` slots. Rather than waiting up to `DefaultLeaseSeconds` for the lease to expire, `SessionManager.CloseExpiredLeasesAsync` also checks `IsFaultedReapable(now)` and reaps a faulted session through the same `TryBeginCloseIfExpired` → `CloseSessionCoreAsync` teardown, with the distinct `FaultedReason` (`"faulted-reaped"`). `MarkFaulted` stamps a fault timestamp from the session's `TimeProvider`; `MxGateway:Sessions:FaultedGraceSeconds` (default `0`) bounds how long the faulted session is retained before the next sweep reaps it — `0` reaps it on the next sweep, a positive value keeps it observable via `GetSessionStatus` for that window first. Sweep precedence when several conditions fire at once is lease-expiry, then faulted, then detach-grace.
|
||||
A session that faults (for example, a slow single-subscriber client that overflows its event queue under the `FailFast` backpressure policy) is left permanently unusable — every command fails `EvaluateReadyUnderLock` — but its worker keeps running and it still pins one of `MxGateway:Sessions:MaxSessions` slots. Rather than waiting up to `DefaultLeaseSeconds` for the lease to expire, `SessionManager.CloseExpiredLeasesAsync` also checks `IsFaultedReapable(now)` and reaps a faulted session through the same `TryBeginCloseIfExpired` → `CloseSessionCoreAsync` teardown, with the distinct `FaultedReason` (`"faulted-reaped"`). `MarkFaulted` stamps a fault timestamp from the session's `TimeProvider`; `MxGateway:Sessions:FaultedGraceSeconds` (default `0`) bounds how long the faulted session is retained before the next sweep reaps it — `0` reaps it on the next sweep, a positive value keeps it observable via `GetSessionStatus` for that window first. Sweep precedence when several conditions fire at once is lease-expiry, then faulted, then detach-grace, then unattached-grace.
|
||||
|
||||
#### Never-subscribed session reaping
|
||||
|
||||
Detach-grace only helps a session that *had* a subscriber. A client that opens a session and then disappears before it ever calls `StreamEvents` — OtOpcUa's `GalaxyDriver` failing part-way through its start-up is the motivating case — leaves a `Ready` session with no subscriber, no detach timestamp and a lease 30 minutes out, pinning one of `MaxSessions` and a live x86 worker for all of `DefaultLeaseSeconds`. `MxGateway:Sessions:UnattachedSessionGraceSeconds` (default `0`, off) closes that gap: when positive, `CloseExpiredLeasesAsync` also checks `IsUnattachedGraceExpired(now)` and reaps, with the distinct `UnattachedGraceExpiredReason` (`"unattached-grace-expired"`), any `Ready` session that has never had a subscriber of either kind and has been idle for the window.
|
||||
|
||||
Four deliberate limits keep the rule from catching legitimate sessions. It applies only to `Ready` sessions, and its idle clock starts at `Ready` (`MarkReady` stamps `_readyAtUtc`), so a slow worker start neither reaps the session mid-startup nor eats the window before the client even has the session id. Idle is measured from the later of that Ready stamp and the session's *last client activity* (its most recent unary command), so a unary-only client (reads and writes, never streams) is not reaped while it keeps calling; the rule is an idle timeout for sessions that never streamed, not a deadline to stream. It stops applying the moment the first subscriber attaches, after which detach-grace and the normal lease govern the session. And an *internal* in-process subscriber counts as attached for this rule (`AttachInternalEventSubscriber` sets a separate `_everHadInternalEventSubscriber` latch): the gateway's own `GatewayAlarmMonitor` session only ever attaches that way, and it must not be reaped by its own gateway every cycle. The latch is separate from `_everHadEventSubscriber` because that one drives detach-grace, which is keyed on external subscribers only. The dashboard mirror, which every session registers at `MarkReady`, is deliberately *not* a subscriber for this rule. Like the other sweep conditions it is re-verified inside `TryBeginCloseIfExpired`, so a subscriber that attaches between selection and the close wins the race. Operators should set the window comfortably above a well-behaved client's open-to-first-`StreamEvents` time; the close lands on the next sweep after the window elapses, up to `LeaseSweepIntervalSeconds` later.
|
||||
|
||||
#### Reconnect and replay
|
||||
|
||||
@@ -270,6 +290,7 @@ The ReplayGap sentinel is emitted **only** on the `StreamEvents` server stream a
|
||||
`GatewaySession.CloseAsync` is serialized by a per-session `SemaphoreSlim` (`_closeLock`) so only one close runs at a time, but every read/write of `_state` still passes through `_syncRoot` (via `TryBeginClose` and `MarkClosed`). The close path therefore obeys the same lock discipline as `TransitionTo` / `MarkFaulted`: it transitions to `Closing`, asks the worker client to shut down within `ShutdownTimeout`, and on success transitions to `Closed`. `DisposeAsync` waits on `_closeLock` once before disposing the semaphore so an in-flight close's `Release()` cannot race against the dispose. `WorkerClient.ShutdownAsync` is bounded by `ShutdownTimeout` end to end — queuing the `WorkerShutdown` envelope included, since a worker that stopped reading the pipe can leave the bounded outbound channel full with no watchdog left to clear it once the client is `Closing` — so a close never holds `_closeLock` longer than the timeout before reaching the fallback. If `WorkerClient.ShutdownAsync` throws, the session falls back to `IWorkerClient.Kill` (forced close):
|
||||
|
||||
```csharp
|
||||
Exception? workerShutdownFailure = null;
|
||||
if (_workerClient is not null)
|
||||
{
|
||||
try
|
||||
@@ -289,54 +310,76 @@ if (_workerClient is not null)
|
||||
new AggregateException(exception, killException));
|
||||
}
|
||||
|
||||
throw;
|
||||
workerShutdownFailure = exception;
|
||||
}
|
||||
}
|
||||
|
||||
MarkClosed();
|
||||
return new SessionCloseResult(SessionId, SessionState.Closed, alreadyClosing, workerShutdownFailure);
|
||||
```
|
||||
|
||||
If both graceful shutdown and the kill fall-back fail, the original and kill exceptions are bundled into an `AggregateException` and surfaced as `SessionCloseStartedException`. `SessionManager.CloseSessionCoreAsync` then translates that into a `SessionManagerException` with `CloseFailed` and removes the session.
|
||||
A kill that succeeds completes the close **successfully**: the worker is gone and the session is `Closed`, exactly as if the shutdown had worked, and the caller's `CloseSession` reply says so. The graceful failure is not lost — it travels back as `SessionCloseResult.WorkerShutdownFailure`, and `SessionManager.CloseSessionCoreAsync` logs it at warning level ("did not shut down gracefully and was killed"), which is the operator signal that workers are hanging on shutdown. The kill itself is counted by `WorkerClient.Kill` (`mxgateway.workers.killed`). Nothing is faulted and no `CloseFailed` fault is recorded, because nothing failed from the session's point of view; reporting the common hung-worker case that way had made every such close look like a teardown failure.
|
||||
|
||||
`GatewaySession.KillWorker` is the unconditional forced-close path used by shutdown when graceful close itself throws, and also by `SessionManager.KillWorkerAsync` — the explicit kill path that the dashboard's admin Kill button invokes. `KillWorkerAsync` skips `WorkerClient.ShutdownAsync` entirely, so `KillCount` increments while `ShutdownCount` does not; the session is then removed from the registry and the open-session slot is released, identical to the cleanup that follows a successful `CloseSessionAsync`.
|
||||
Only if both the graceful shutdown and the kill fall-back fail are the original and kill exceptions bundled into an `AggregateException` and surfaced as `SessionCloseStartedException`. `SessionManager.CloseSessionCoreAsync` then translates that into a `SessionManagerException` with `CloseFailed`, faults the session, records the fault metric and removes the session — a worker may still be running, which is what `CloseFailed` is reserved for.
|
||||
|
||||
**Losing a race to a concurrent close.** A caller that looked a session up (a kill, a client close, a sweep's snapshot) and then found its close gate already disposed — because a concurrent close finished, removed the session, accounted `SessionClosed` and disposed it in between — sees `ObjectDisposedException` from the gate. Both `KillWorkerAsync` and `CloseSessionCoreAsync` catch that case *when the session is no longer registered* and return `AlreadyClosed: true` with no accounting, no fault and no log: the close that won already did all of it. (The old kill catch recorded a fault, faulted the session and decremented the open gauge a second time; the sweep rethrew it as a sweep failure.) A sweep does not count such a session among the sessions it closed. A disposed gate on a session that *is* still registered is a real failure and takes the normal `CloseFailed` path.
|
||||
|
||||
`SessionManager.KillWorkerAsync` is the explicit kill path that the dashboard's admin Kill button invokes and that `ShutdownAsync` falls back to when a graceful close throws. It goes through `GatewaySession.KillWorkerWithCloseGateAsync`, which skips `WorkerClient.ShutdownAsync` entirely, so `KillCount` increments while `ShutdownCount` does not; the session is then removed from the registry and the open-session slot is released, identical to the cleanup that follows a successful `CloseSessionAsync`.
|
||||
|
||||
## Shutdown Coordination
|
||||
|
||||
`SessionShutdownHostedService.StopAsync` calls `SessionManager.ShutdownAsync`, which closes every registered session with `GatewayShutdownReason`. Sessions are drained with the same bounded fan-out the lease sweep uses (`MaxParallelSessionCloses`), because a one-at-a-time drain of a full registry at a worst-case worker shutdown timeout each outruns any host stop-timeout and leaves the tail to the orphan killer. Each iteration catches its own exceptions — *every* exception, including from the fallback — calls `KillWorkerAsync` on an uncancellable token, and removes the session, so that neither one stuck worker nor one failing teardown can block or abort the rest of the host's drain:
|
||||
|
||||
```csharp
|
||||
await Parallel.ForEachAsync(
|
||||
_registry.Snapshot(),
|
||||
new ParallelOptions { MaxDegreeOfParallelism = MaxParallelSessionCloses },
|
||||
async (session, _) =>
|
||||
MarkStopping();
|
||||
for (int pass = 0; pass < MaxShutdownDrainPasses; pass++)
|
||||
{
|
||||
if (_registry.Count == 0)
|
||||
{
|
||||
try
|
||||
return;
|
||||
}
|
||||
|
||||
await Parallel.ForEachAsync(
|
||||
_registry.Snapshot(),
|
||||
new ParallelOptions { MaxDegreeOfParallelism = MaxParallelSessionCloses },
|
||||
async (session, _) =>
|
||||
{
|
||||
await CloseSessionCoreAsync(session, GatewayShutdownReason, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
_logger.LogWarning(
|
||||
exception,
|
||||
"Graceful shutdown failed for session {SessionId}; killing worker.",
|
||||
session.SessionId);
|
||||
if (_registry.TryGet(session.SessionId, out GatewaySession? registeredSession)
|
||||
&& registeredSession is not null)
|
||||
try
|
||||
{
|
||||
try
|
||||
await CloseSessionCoreAsync(session, GatewayShutdownReason, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
_logger.LogWarning(
|
||||
exception,
|
||||
"Graceful shutdown failed for session {SessionId}; killing worker.",
|
||||
session.SessionId);
|
||||
if (_registry.TryGet(session.SessionId, out GatewaySession? registeredSession)
|
||||
&& registeredSession is not null)
|
||||
{
|
||||
// Not the caller's token: the kill is the last-resort orphan preventer.
|
||||
await KillWorkerAsync(session.SessionId, GatewayShutdownReason, CancellationToken.None).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception killException)
|
||||
{
|
||||
_logger.LogWarning(killException, "Worker kill fallback failed for session {SessionId}.", session.SessionId);
|
||||
try
|
||||
{
|
||||
// Not the caller's token: the kill is the last-resort orphan preventer.
|
||||
await KillWorkerAsync(session.SessionId, GatewayShutdownReason, CancellationToken.None).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception killException)
|
||||
{
|
||||
_logger.LogWarning(killException, "Worker kill fallback failed for session {SessionId}.", session.SessionId);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}).ConfigureAwait(false);
|
||||
}).ConfigureAwait(false);
|
||||
}
|
||||
```
|
||||
|
||||
Iterating over `Snapshot` rather than the live dictionary lets `RemoveSessionAsync` mutate the registry from inside the loop without throwing, and gives the parallel drain a stable, already-materialized source.
|
||||
|
||||
### Stopping gate
|
||||
|
||||
A drain that only ever looked at one snapshot could be outrun by `OpenSession`: the host raises `ApplicationStopping` *before* hosted services stop, and Kestrel keeps accepting gRPC calls until its own `StopAsync` runs, so a session opened behind the snapshot would have nobody left to close it and its x86 worker would survive the gateway as an orphan. `SessionManager` therefore keeps a stopping flag that is set by the host's `IHostApplicationLifetime.ApplicationStopping` (the manager registers for it in its constructor when the lifetime is injected) and, as a fallback for hosts without one, at the start of `ShutdownAsync`. Once set it is never cleared. `OpenSessionAsync` checks it twice: on entry, before a slot is taken, and again right after the session is registered. The second check is what closes the race — the flag is written before the drain takes its registry snapshot, so an open that does not observe it there registered its session before that snapshot and is drained, while one that does observe it rolls its own session back. Either way the caller receives `SessionManagerException` with `GatewayStopping`, which the gRPC layer maps to `Unavailable` so the client retries against the restarted gateway; it is not an open failure and records no fault metric.
|
||||
|
||||
The drain loop above is the backstop to that gate: it re-snapshots until the registry is empty, bounded at `MaxShutdownDrainPasses` (`3`) so a session that somehow cannot be removed cannot hold the host stop forever. Anything still registered after the last pass is logged and left to the orphan terminator on the next start.
|
||||
|
||||
## Dependency Injection
|
||||
|
||||
`SessionServiceCollectionExtensions.AddGatewaySessions` registers the four singletons and the hosted service:
|
||||
|
||||
@@ -325,6 +325,10 @@ public sealed class GatewayOptionsValidator : OptionsValidatorBase<GatewayOption
|
||||
options.FaultedGraceSeconds,
|
||||
"MxGateway:Sessions:FaultedGraceSeconds must be zero or greater (0 reaps a faulted session on the next sweep).",
|
||||
builder);
|
||||
AddIfNegative(
|
||||
options.UnattachedSessionGraceSeconds,
|
||||
"MxGateway:Sessions:UnattachedSessionGraceSeconds must be zero or greater (0 disables reaping of never-subscribed sessions).",
|
||||
builder);
|
||||
AddIfNegative(
|
||||
options.WorkerReadyWaitTimeoutMs,
|
||||
"MxGateway:Sessions:WorkerReadyWaitTimeoutMs must be greater than or equal to zero.",
|
||||
|
||||
@@ -57,6 +57,28 @@ public sealed class SessionOptions
|
||||
/// </summary>
|
||||
public int FaultedGraceSeconds { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets the idle window, in seconds, after which the lease monitor reaps a session that
|
||||
/// has <em>never</em> attached an event-stream subscriber. A client that opens a session
|
||||
/// and vanishes before it ever streams (for example OtOpcUa's <c>GalaxyDriver</c> failing
|
||||
/// part-way through its start-up) otherwise pins a session slot and a live x86 worker for
|
||||
/// the full <see cref="DefaultLeaseSeconds"/> (30 minutes by default). The window is
|
||||
/// measured from the later of the session becoming Ready and its most recent unary
|
||||
/// command, so a slow worker start does not eat it and a client that only ever reads and
|
||||
/// writes is not reaped while it keeps calling; it stops applying the moment the session's
|
||||
/// first subscriber — external, or the gateway's own internal alarm-monitor subscriber —
|
||||
/// attaches (from then on <see cref="DetachGraceSeconds"/> and the normal lease govern
|
||||
/// it). A value of <c>0</c> (the default) disables the sweep. Must be greater than or
|
||||
/// equal to zero.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Set it comfortably above the time a well-behaved client needs between
|
||||
/// <c>OpenSession</c> and its first <c>StreamEvents</c>, and remember the close lands on
|
||||
/// the next sweep after the window elapses (up to <see cref="LeaseSweepIntervalSeconds"/>
|
||||
/// later).
|
||||
/// </remarks>
|
||||
public int UnattachedSessionGraceSeconds { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets a value indicating whether multiple event subscribers are allowed per session.
|
||||
/// </summary>
|
||||
|
||||
@@ -1104,6 +1104,7 @@ public sealed class MxAccessGatewayService(
|
||||
SessionManagerErrorCode.OpenFailed => StatusCode.Unavailable,
|
||||
SessionManagerErrorCode.CloseFailed => StatusCode.Unavailable,
|
||||
SessionManagerErrorCode.PermissionDenied => StatusCode.PermissionDenied,
|
||||
SessionManagerErrorCode.GatewayStopping => StatusCode.Unavailable,
|
||||
_ => StatusCode.Unavailable,
|
||||
};
|
||||
|
||||
|
||||
@@ -26,9 +26,11 @@ public sealed class GatewaySession
|
||||
private DateTimeOffset _lastClientActivityAt;
|
||||
private DateTimeOffset? _leaseExpiresAt;
|
||||
private bool _closeStarted;
|
||||
private bool _openRecorded;
|
||||
private int _activeEventSubscriberCount;
|
||||
private readonly TimeSpan _detachGrace;
|
||||
private readonly TimeSpan _faultedGrace;
|
||||
private readonly TimeSpan _unattachedGrace;
|
||||
private readonly TimeSpan _workerReadyWaitTimeout;
|
||||
private DateTimeOffset? _detachedAtUtc;
|
||||
private DateTimeOffset? _faultedAtUtc;
|
||||
@@ -37,6 +39,12 @@ public sealed class GatewaySession
|
||||
// FAILED first attach — which still runs the rollback DetachEventSubscriber from the
|
||||
// attach catch path — does not push a never-subscribed session into the grace window.
|
||||
private bool _everHadEventSubscriber;
|
||||
// True once an in-process (internal, non-counted) subscriber attached via
|
||||
// AttachInternalEventSubscriber. Consulted only by the unattached-grace sweep rule.
|
||||
private bool _everHadInternalEventSubscriber;
|
||||
// Stamped once, when the session first becomes Ready; the unattached-grace idle clock cannot
|
||||
// start before it.
|
||||
private DateTimeOffset? _readyAtUtc;
|
||||
private SessionEventDistributor? _eventDistributor;
|
||||
private bool _eventDistributorStarted;
|
||||
// Set under _syncRoot when DisposeAsync nulls _eventDistributor, making the
|
||||
@@ -168,6 +176,12 @@ public sealed class GatewaySession
|
||||
/// The tags come from the owner key, never from the client's wire request, so a client
|
||||
/// cannot label its own session with another tenant's tag.
|
||||
/// </param>
|
||||
/// <param name="unattachedGrace">
|
||||
/// Idle window after which the lease monitor reaps a Ready session that has never attached
|
||||
/// an event subscriber (see <see cref="IsUnattachedGraceExpired"/>), so a client that
|
||||
/// opens and vanishes does not pin a worker for the full lease. <see cref="TimeSpan.Zero"/>
|
||||
/// (the default) disables it.
|
||||
/// </param>
|
||||
public GatewaySession(
|
||||
string sessionId,
|
||||
string backendName,
|
||||
@@ -187,7 +201,8 @@ public sealed class GatewaySession
|
||||
TimeSpan workerReadyWaitTimeout = default,
|
||||
ArrayAddressNormalizer? addressNormalizer = null,
|
||||
TimeSpan faultedGrace = default,
|
||||
IReadOnlyList<string>? ownerDashboardTags = null)
|
||||
IReadOnlyList<string>? ownerDashboardTags = null,
|
||||
TimeSpan unattachedGrace = default)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(sessionId))
|
||||
{
|
||||
@@ -230,6 +245,7 @@ public sealed class GatewaySession
|
||||
_eventStreaming = eventStreaming ?? SessionEventStreaming.Default;
|
||||
_detachGrace = detachGrace > TimeSpan.Zero ? detachGrace : TimeSpan.Zero;
|
||||
_faultedGrace = faultedGrace > TimeSpan.Zero ? faultedGrace : TimeSpan.Zero;
|
||||
_unattachedGrace = unattachedGrace > TimeSpan.Zero ? unattachedGrace : TimeSpan.Zero;
|
||||
_workerReadyWaitTimeout = workerReadyWaitTimeout > TimeSpan.Zero ? workerReadyWaitTimeout : TimeSpan.Zero;
|
||||
_addressNormalizer = addressNormalizer;
|
||||
}
|
||||
@@ -413,13 +429,128 @@ public sealed class GatewaySession
|
||||
/// Attaches the worker client for this session.
|
||||
/// </summary>
|
||||
/// <param name="workerClient">Worker client to attach.</param>
|
||||
public void AttachWorkerClient(IWorkerClient workerClient)
|
||||
/// <returns>
|
||||
/// <see langword="true"/> when the client was attached; <see langword="false"/> when the
|
||||
/// session is already <see cref="SessionState.Closing"/>, <see cref="SessionState.Closed"/>
|
||||
/// or <see cref="SessionState.Faulted"/>, or has been disposed — the caller owns the client
|
||||
/// and must discard it.
|
||||
/// </returns>
|
||||
/// <remarks>
|
||||
/// The session is registered before its worker is created, so an admin kill or the host
|
||||
/// drain can close it while the worker is still starting (up to the startup timeout). A
|
||||
/// worker client arriving after that must not be stored on the dead session: nothing would
|
||||
/// ever shut it down (the close already ran with no client to stop) and the open would hand
|
||||
/// a closed session back to the caller as if it had succeeded. The decision is taken under
|
||||
/// <c>_syncRoot</c> so it cannot interleave with the close's state write.
|
||||
/// </remarks>
|
||||
public bool AttachWorkerClient(IWorkerClient workerClient)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(workerClient);
|
||||
|
||||
lock (_syncRoot)
|
||||
{
|
||||
if (_state is SessionState.Closing or SessionState.Closed or SessionState.Faulted
|
||||
|| _eventDistributorDisposed)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
_workerClient = workerClient;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Runs the end of the open sequence — attach the worker client, mark the session
|
||||
/// <see cref="SessionState.Ready"/>, record it as opened — under the per-session close
|
||||
/// gate, so a concurrent <see cref="CloseAsync"/> or
|
||||
/// <see cref="KillWorkerWithCloseGateAsync"/> runs either entirely before it (the attach
|
||||
/// is refused) or entirely after it (an ordinary close of a Ready session). Without the
|
||||
/// gate a kill could land between the attach and the open accounting and the two teardown
|
||||
/// bookkeepings would disagree about whether the session ever counted as open.
|
||||
/// </summary>
|
||||
/// <param name="workerClient">Worker client to attach.</param>
|
||||
/// <param name="onReady">
|
||||
/// Invoked inside the gate once the session is Ready; the caller records its open-session
|
||||
/// accounting here, and <see cref="OpenRecorded"/> is set right after it returns.
|
||||
/// </param>
|
||||
/// <returns>
|
||||
/// <see langword="true"/> when the session became Ready; <see langword="false"/> when the
|
||||
/// attach was refused (see <see cref="AttachWorkerClient"/>), including when the session was
|
||||
/// disposed before the gate could be taken.
|
||||
/// </returns>
|
||||
/// <remarks>
|
||||
/// Not cancellable: the worker client already exists, so the only acceptable outcomes are
|
||||
/// "attached" or "refused, caller discards it". The gate is held only for the duration of
|
||||
/// a close or kill, so the wait is bounded by the worker shutdown timeout. The caller
|
||||
/// still decides what to do about a cancelled open before calling this.
|
||||
/// </remarks>
|
||||
internal async ValueTask<bool> TryAttachWorkerClientAndMarkReadyAsync(
|
||||
IWorkerClient workerClient,
|
||||
Action onReady)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(workerClient);
|
||||
ArgumentNullException.ThrowIfNull(onReady);
|
||||
|
||||
try
|
||||
{
|
||||
await _closeLock.WaitAsync(CancellationToken.None).ConfigureAwait(false);
|
||||
}
|
||||
catch (ObjectDisposedException)
|
||||
{
|
||||
// Disposed while the worker was starting: the session has been torn down.
|
||||
return false;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
if (!AttachWorkerClient(workerClient))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
MarkReady();
|
||||
onReady();
|
||||
lock (_syncRoot)
|
||||
{
|
||||
_openRecorded = true;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
finally
|
||||
{
|
||||
try
|
||||
{
|
||||
_closeLock.Release();
|
||||
}
|
||||
catch (ObjectDisposedException)
|
||||
{
|
||||
// A double-dispose that slipped past DisposeAsync's own gate; the session is torn
|
||||
// down either way and there is no one left to hand the slot to.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets a value indicating whether the open-session accounting
|
||||
/// (<c>GatewayMetrics.SessionOpened</c>) has been recorded for this session. Every
|
||||
/// teardown path consults it so a session closed or killed while its worker was still
|
||||
/// starting — never counted as opened — is not counted as closed either.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Written only by <see cref="TryAttachWorkerClientAndMarkReadyAsync"/>, inside the close
|
||||
/// gate; read by teardown paths that hold (or just released) the same gate, so the two
|
||||
/// accountings never observe a half-finished open.
|
||||
/// </remarks>
|
||||
internal bool OpenRecorded
|
||||
{
|
||||
get
|
||||
{
|
||||
lock (_syncRoot)
|
||||
{
|
||||
return _openRecorded;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -481,6 +612,14 @@ public sealed class GatewaySession
|
||||
public void MarkReady()
|
||||
{
|
||||
TransitionTo(SessionState.Ready);
|
||||
lock (_syncRoot)
|
||||
{
|
||||
// The unattached-grace idle clock starts here, not at open: the client cannot
|
||||
// subscribe before it has the session, and a slow worker start must not hand it a
|
||||
// session the next sweep reaps.
|
||||
_readyAtUtc ??= _eventStreaming.TimeProvider.GetUtcNow();
|
||||
}
|
||||
|
||||
StartDashboardMirror();
|
||||
}
|
||||
|
||||
@@ -634,6 +773,15 @@ public sealed class GatewaySession
|
||||
SessionEventDistributor distributor = EnsureDistributorCreated(out bool startNow);
|
||||
IEventSubscriberLease lease = distributor.Register(isInternal: true);
|
||||
StartPumpIfRequested(distributor, startNow);
|
||||
lock (_syncRoot)
|
||||
{
|
||||
// An in-process consumer (the alarm monitor) is a real subscriber for the
|
||||
// unattached-grace rule: its session is attached, not abandoned. It stays out of
|
||||
// _everHadEventSubscriber on purpose — that latch drives detach-grace, which is keyed
|
||||
// on EXTERNAL subscribers only.
|
||||
_everHadInternalEventSubscriber = true;
|
||||
}
|
||||
|
||||
return lease;
|
||||
}
|
||||
|
||||
@@ -856,12 +1004,54 @@ public sealed class GatewaySession
|
||||
IWorkerClient workerClient = await GetReadyWorkerClientAsync(cancellationToken).ConfigureAwait(false);
|
||||
TouchClientActivity(_eventStreaming.TimeProvider.GetUtcNow());
|
||||
|
||||
await foreach (WorkerEvent workerEvent in workerClient
|
||||
.ReadEventsAsync(cancellationToken)
|
||||
.WithCancellation(cancellationToken)
|
||||
.ConfigureAwait(false))
|
||||
try
|
||||
{
|
||||
yield return mapper.MapEvent(workerEvent);
|
||||
await foreach (WorkerEvent workerEvent in workerClient
|
||||
.ReadEventsAsync(cancellationToken)
|
||||
.WithCancellation(cancellationToken)
|
||||
.ConfigureAwait(false))
|
||||
{
|
||||
yield return mapper.MapEvent(workerEvent);
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
// The worker event stream only ends when the worker client has reached a terminal
|
||||
// state (it enters Closed/Faulted BEFORE completing the channel) or the pump is being
|
||||
// torn down. This is the session's one always-on view of the worker, so it is where
|
||||
// a worker that went away on its own is noticed.
|
||||
ObserveWorkerTerminalState();
|
||||
}
|
||||
}
|
||||
|
||||
// IWorkerClient has no state-change callback, so a worker that reached Closed/Faulted
|
||||
// without the gateway closing it — the client sent ShutdownWorker and the worker exited,
|
||||
// the worker crashed, its pipe broke, a heartbeat timed out — used to leave the session
|
||||
// reporting Ready until its lease expired (live finding). The session follows the worker
|
||||
// into Faulted: it is permanently unusable (every command fails the readiness check), stays
|
||||
// observable via GetSessionStatus with the reason, and the faulted-session sweep reaps it
|
||||
// after FaultedGraceSeconds, accounting SessionClosed exactly once there. Not Closed: Closed
|
||||
// means the gateway closed it, and a Closed session still in the registry would linger until
|
||||
// its lease expired with nothing reaping it. Only a Ready session follows — during a
|
||||
// gateway-initiated close the worker closing is the expected outcome, not a fault.
|
||||
private void ObserveWorkerTerminalState()
|
||||
{
|
||||
lock (_syncRoot)
|
||||
{
|
||||
if (_state is not SessionState.Ready || _workerClient is null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
WorkerClientState workerState = _workerClient.State;
|
||||
if (workerState is not (WorkerClientState.Closed or WorkerClientState.Faulted))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
MarkFaulted(
|
||||
$"Worker for session {SessionId} reached {workerState} outside a gateway-initiated close "
|
||||
+ "(worker exited or faulted).");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -901,18 +1091,6 @@ public sealed class GatewaySession
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Extends the session lease to the specified expiration time.
|
||||
/// </summary>
|
||||
/// <param name="leaseExpiresAt">Timestamp when the lease expires.</param>
|
||||
public void ExtendLease(DateTimeOffset leaseExpiresAt)
|
||||
{
|
||||
lock (_syncRoot)
|
||||
{
|
||||
_leaseExpiresAt = leaseExpiresAt;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Determines whether the session lease has expired.
|
||||
/// </summary>
|
||||
@@ -969,6 +1147,25 @@ public sealed class GatewaySession
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Determines whether a <see cref="SessionState.Ready"/> session that has <em>never</em>
|
||||
/// attached an event-stream subscriber has been idle — no unary command since open or
|
||||
/// since its last one — for at least the configured unattached grace. The lease monitor
|
||||
/// reaps such a session exactly as it reaps an expired lease, so a client that opened and
|
||||
/// vanished does not pin a worker for the full lease. Always <see langword="false"/> when
|
||||
/// the grace is disabled, when the session ever had a subscriber (detach-grace and the
|
||||
/// normal lease govern it from then on) or when one is attached now.
|
||||
/// </summary>
|
||||
/// <param name="now">Current timestamp for comparison.</param>
|
||||
/// <returns><see langword="true"/> if the session is reapable as never-subscribed and idle; otherwise <see langword="false"/>.</returns>
|
||||
public bool IsUnattachedGraceExpired(DateTimeOffset now)
|
||||
{
|
||||
lock (_syncRoot)
|
||||
{
|
||||
return IsUnattachedGraceExpiredCore(now);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Attaches an event subscriber and returns a lease whose
|
||||
/// <see cref="IEventSubscriberLease.Reader"/> reads the fanned public
|
||||
@@ -1627,6 +1824,7 @@ public sealed class GatewaySession
|
||||
return new SessionCloseResult(SessionId, SessionState.Closed, AlreadyClosed: true);
|
||||
}
|
||||
|
||||
Exception? workerShutdownFailure = null;
|
||||
if (_workerClient is not null)
|
||||
{
|
||||
try
|
||||
@@ -1635,6 +1833,12 @@ public sealed class GatewaySession
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
// The kill is the documented fallback for a worker that will not shut down
|
||||
// (typically a ShutdownTimeout on a hung worker). When it succeeds the
|
||||
// worker is gone and this close has done its job, so it completes as a
|
||||
// normal close carrying the shutdown failure for the caller to log; only a
|
||||
// kill that ALSO fails is a close failure (SessionCloseStartedException →
|
||||
// CloseFailed), because then a worker may still be running.
|
||||
try
|
||||
{
|
||||
_workerClient.Kill(reason);
|
||||
@@ -1646,12 +1850,12 @@ public sealed class GatewaySession
|
||||
new AggregateException(exception, killException));
|
||||
}
|
||||
|
||||
throw;
|
||||
workerShutdownFailure = exception;
|
||||
}
|
||||
}
|
||||
|
||||
MarkClosed();
|
||||
return new SessionCloseResult(SessionId, SessionState.Closed, alreadyClosing);
|
||||
return new SessionCloseResult(SessionId, SessionState.Closed, alreadyClosing, workerShutdownFailure);
|
||||
}
|
||||
catch (Exception exception) when (exception is not SessionCloseStartedException)
|
||||
{
|
||||
@@ -1718,6 +1922,15 @@ public sealed class GatewaySession
|
||||
/// committing to <c>Closing</c>, so a reattach that wins the race leaves the session
|
||||
/// in <c>Ready</c> and usable.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// This is a <em>reservation</em>, not the close itself: it flips the state so no
|
||||
/// reattach can revive the session and no other sweep pass re-selects it, but it
|
||||
/// deliberately leaves <c>_closeStarted</c> untouched. The sweep's own
|
||||
/// <see cref="CloseAsync"/> that follows is the first close to run, and must report
|
||||
/// <c>AlreadyClosed: false</c> so the manager accounts <c>SessionClosed</c> for it;
|
||||
/// setting <c>_closeStarted</c> here made that close look like a second, concurrent one
|
||||
/// and every sweep-driven close went unaccounted.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
internal bool TryBeginCloseIfExpired(DateTimeOffset now, out bool alreadyClosing)
|
||||
{
|
||||
@@ -1731,7 +1944,10 @@ public sealed class GatewaySession
|
||||
|
||||
// Re-verify eligibility atomically. If a subscriber reattached between the sweep's
|
||||
// eligibility check and this point, neither condition holds and we decline.
|
||||
bool eligible = IsLeaseExpiredCore(now) || IsFaultedReapableCore(now) || IsDetachGraceExpiredCore(now);
|
||||
bool eligible = IsLeaseExpiredCore(now)
|
||||
|| IsFaultedReapableCore(now)
|
||||
|| IsDetachGraceExpiredCore(now)
|
||||
|| IsUnattachedGraceExpiredCore(now);
|
||||
if (!eligible)
|
||||
{
|
||||
alreadyClosing = false;
|
||||
@@ -1739,7 +1955,6 @@ public sealed class GatewaySession
|
||||
}
|
||||
|
||||
alreadyClosing = _closeStarted;
|
||||
_closeStarted = true;
|
||||
_state = SessionState.Closing;
|
||||
return true;
|
||||
}
|
||||
@@ -1763,6 +1978,29 @@ public sealed class GatewaySession
|
||||
|| _faultedAtUtc is null
|
||||
|| now - _faultedAtUtc.Value >= _faultedGrace);
|
||||
|
||||
// Ready only: a session still starting its worker is not idle, it is waiting, and reaping it
|
||||
// mid-startup would turn a slow worker start into an OpenFailed for a client that did nothing
|
||||
// wrong. Idle is measured from the later of Ready and the last client activity (last unary
|
||||
// command) rather than from open, so a slow start does not eat the window and a unary-only
|
||||
// client that keeps calling is never reaped by this rule. An internal (in-process) subscriber
|
||||
// — the gateway's own alarm monitor — counts as attached, or the gateway would reap its own
|
||||
// monitor session every cycle.
|
||||
private bool IsUnattachedGraceExpiredCore(DateTimeOffset now)
|
||||
{
|
||||
if (_unattachedGrace <= TimeSpan.Zero
|
||||
|| _state is not SessionState.Ready
|
||||
|| _readyAtUtc is null
|
||||
|| _everHadEventSubscriber
|
||||
|| _everHadInternalEventSubscriber
|
||||
|| _activeEventSubscriberCount != 0)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
DateTimeOffset idleSince = _lastClientActivityAt > _readyAtUtc.Value ? _lastClientActivityAt : _readyAtUtc.Value;
|
||||
return now - idleSince >= _unattachedGrace;
|
||||
}
|
||||
|
||||
// Final terminal transition; under _syncRoot to keep _state writes single-lock.
|
||||
// Closed is unconditionally terminal — TransitionTo refuses to overwrite it —
|
||||
// so we don't need to re-check the precondition here.
|
||||
|
||||
@@ -2,7 +2,21 @@ using ZB.MOM.WW.MxGateway.Contracts.Proto;
|
||||
|
||||
namespace ZB.MOM.WW.MxGateway.Server.Sessions;
|
||||
|
||||
/// <summary>Outcome of closing a session.</summary>
|
||||
/// <param name="SessionId">Identifier of the closed session.</param>
|
||||
/// <param name="FinalState">State the session ended in.</param>
|
||||
/// <param name="AlreadyClosed">
|
||||
/// <see langword="true"/> when the session was already closed (or being closed by another
|
||||
/// caller) when this close acquired the close lock, so this call did no teardown of its own.
|
||||
/// </param>
|
||||
/// <param name="WorkerShutdownFailure">
|
||||
/// The exception the worker's graceful shutdown threw when the close had to fall back to
|
||||
/// killing the worker; <see langword="null"/> when the worker shut down cleanly. The close is
|
||||
/// still a success — the worker is gone and the session is closed — so this is diagnostic
|
||||
/// detail for the caller's log, not an error.
|
||||
/// </param>
|
||||
public sealed record SessionCloseResult(
|
||||
string SessionId,
|
||||
SessionState FinalState,
|
||||
bool AlreadyClosed);
|
||||
bool AlreadyClosed,
|
||||
Exception? WorkerShutdownFailure = null);
|
||||
|
||||
@@ -2,6 +2,7 @@ using System.Diagnostics.CodeAnalysis;
|
||||
using System.Runtime.ExceptionServices;
|
||||
using System.Security.Cryptography;
|
||||
using Google.Protobuf.WellKnownTypes;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
@@ -20,6 +21,7 @@ public sealed class SessionManager : ISessionManager
|
||||
public const string LeaseExpiredReason = "lease-expired";
|
||||
public const string DetachGraceExpiredReason = "detach-grace-expired";
|
||||
public const string FaultedReason = "faulted-reaped";
|
||||
public const string UnattachedGraceExpiredReason = "unattached-grace-expired";
|
||||
|
||||
// Bounded so a mass expiry (or a host stop with a full registry) cannot stampede
|
||||
// worker-process teardown: every concurrent close is one x86 worker being shut down or
|
||||
@@ -27,6 +29,13 @@ public sealed class SessionManager : ISessionManager
|
||||
// registry down at once.
|
||||
private const int MaxParallelSessionCloses = 4;
|
||||
|
||||
// ShutdownAsync re-snapshots the registry until it is empty so an open that registered its
|
||||
// session just after a snapshot is still drained. The stopping gate in OpenSessionAsync is
|
||||
// what actually closes that window (it is re-checked after registration), so the loop is a
|
||||
// backstop and a small bound is enough; anything still registered after the last pass is
|
||||
// logged and left to the orphan terminator on the next start.
|
||||
private const int MaxShutdownDrainPasses = 3;
|
||||
|
||||
private readonly ISessionRegistry _registry;
|
||||
private readonly ISessionWorkerClientFactory _workerClientFactory;
|
||||
private readonly GatewayMetrics _metrics;
|
||||
@@ -39,6 +48,12 @@ public sealed class SessionManager : ISessionManager
|
||||
private readonly Dashboard.Hubs.IDashboardEventBroadcaster? _dashboardEventBroadcaster;
|
||||
private readonly ArrayAddressNormalizer? _addressNormalizer;
|
||||
|
||||
// 0 = accepting opens, 1 = stopping. Set by the host's ApplicationStopping signal (which fires
|
||||
// before hosted services stop, while Kestrel still accepts gRPC calls) and, as a fallback for
|
||||
// hosts without a lifetime, at the start of ShutdownAsync. Never reset: a manager that has
|
||||
// begun draining does not come back.
|
||||
private int _stopping;
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of <see cref="SessionManager"/>.
|
||||
/// </summary>
|
||||
@@ -61,6 +76,12 @@ public sealed class SessionManager : ISessionManager
|
||||
/// metadata; handed to each session so the normalization runs at the outbound choke point.
|
||||
/// Null in unit tests that do not exercise array-write ergonomics.
|
||||
/// </param>
|
||||
/// <param name="applicationLifetime">
|
||||
/// Host lifetime whose <c>ApplicationStopping</c> closes the open gate as soon as the host
|
||||
/// begins stopping — before hosted services stop and while Kestrel is still accepting gRPC
|
||||
/// calls — so no session is opened behind the shutdown drain. Null in unit tests; the gate
|
||||
/// then closes when <see cref="ShutdownAsync"/> starts.
|
||||
/// </param>
|
||||
public SessionManager(
|
||||
ISessionRegistry registry,
|
||||
ISessionWorkerClientFactory workerClientFactory,
|
||||
@@ -71,7 +92,8 @@ public sealed class SessionManager : ISessionManager
|
||||
Grpc.MxAccessGrpcMapper? eventMapper = null,
|
||||
ILogger<SessionEventDistributor>? distributorLogger = null,
|
||||
Dashboard.Hubs.IDashboardEventBroadcaster? dashboardEventBroadcaster = null,
|
||||
ArrayAddressNormalizer? addressNormalizer = null)
|
||||
ArrayAddressNormalizer? addressNormalizer = null,
|
||||
IHostApplicationLifetime? applicationLifetime = null)
|
||||
{
|
||||
_registry = registry ?? throw new ArgumentNullException(nameof(registry));
|
||||
_workerClientFactory = workerClientFactory ?? throw new ArgumentNullException(nameof(workerClientFactory));
|
||||
@@ -85,6 +107,10 @@ public sealed class SessionManager : ISessionManager
|
||||
_addressNormalizer = addressNormalizer;
|
||||
_options = options.Value;
|
||||
_sessionSlots = new SemaphoreSlim(_options.Sessions.MaxSessions, _options.Sessions.MaxSessions);
|
||||
|
||||
// The registration is never disposed: the manager is a host singleton that lives exactly
|
||||
// as long as the lifetime it observes.
|
||||
applicationLifetime?.ApplicationStopping.Register(MarkStopping);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
@@ -104,10 +130,11 @@ public sealed class SessionManager : ISessionManager
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(request);
|
||||
ThrowIfStopping();
|
||||
EnsureSessionCapacity();
|
||||
|
||||
GatewaySession? session = null;
|
||||
bool sessionOpenedRecorded = false;
|
||||
bool registered = false;
|
||||
try
|
||||
{
|
||||
session = CreateSession(request, clientIdentity, ownerKeyId, ownerDashboardTags);
|
||||
@@ -118,36 +145,58 @@ public sealed class SessionManager : ISessionManager
|
||||
$"Session id collision while opening session {session.SessionId}.");
|
||||
}
|
||||
|
||||
registered = true;
|
||||
|
||||
// Re-check after registration. The stopping flag is written before ShutdownAsync takes
|
||||
// its registry snapshot, so an open that does not observe it here registered its
|
||||
// session before that snapshot and will be drained; one that does observe it rolls
|
||||
// back itself. Either way no session is created behind the drain.
|
||||
ThrowIfStopping();
|
||||
session.TransitionTo(SessionState.StartingWorker);
|
||||
IWorkerClient workerClient = await _workerClientFactory
|
||||
.CreateAsync(session, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
session.AttachWorkerClient(workerClient);
|
||||
session.MarkReady();
|
||||
_metrics.SessionOpened();
|
||||
sessionOpenedRecorded = true;
|
||||
// From here on a live worker exists and nothing but this method knows about it: every
|
||||
// way out of this block other than "attached and Ready" must discard (kill + dispose)
|
||||
// it, or it is an orphan the gateway cannot reattach. The session has been registered
|
||||
// (and killable / drainable) for the whole worker startup, so the attach is refused
|
||||
// if a kill or the host drain closed it meanwhile — it must not be returned on a dead
|
||||
// session; and a caller that abandoned the open while the worker was starting gets
|
||||
// no session either.
|
||||
bool ready;
|
||||
try
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
ready = await session
|
||||
.TryAttachWorkerClientAndMarkReadyAsync(workerClient, _metrics.SessionOpened)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
catch
|
||||
{
|
||||
await DiscardWorkerClientAsync(session, workerClient).ConfigureAwait(false);
|
||||
throw;
|
||||
}
|
||||
|
||||
if (!ready)
|
||||
{
|
||||
await DiscardWorkerClientAsync(session, workerClient).ConfigureAwait(false);
|
||||
throw new InvalidOperationException(
|
||||
$"Session {session.SessionId} was closed while its worker was starting.");
|
||||
}
|
||||
|
||||
return session;
|
||||
}
|
||||
catch (SessionManagerException exception) when (exception.ErrorCode == SessionManagerErrorCode.GatewayStopping)
|
||||
{
|
||||
// Not an open failure: the gateway is going away and the caller should retry against
|
||||
// the restarted one, so no fault is recorded and the code is surfaced as-is.
|
||||
await RollBackOpenAsync(session, registered, exception).ConfigureAwait(false);
|
||||
throw;
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
session?.MarkFaulted(exception.Message);
|
||||
if (session is not null)
|
||||
{
|
||||
_registry.TryRemove(session.SessionId, out _);
|
||||
await session.DisposeAsync().ConfigureAwait(false);
|
||||
}
|
||||
|
||||
// If SessionOpened() already incremented the open-session gauge,
|
||||
// a failure after that point (e.g. auto-subscribe rejection) must
|
||||
// decrement it again so mxgateway.sessions.open does not leak.
|
||||
if (sessionOpenedRecorded)
|
||||
{
|
||||
_metrics.SessionRemoved();
|
||||
}
|
||||
|
||||
ReleaseSessionSlot();
|
||||
await RollBackOpenAsync(session, registered, exception).ConfigureAwait(false);
|
||||
_metrics.Fault(SessionManagerErrorCode.OpenFailed.ToString());
|
||||
_logger.LogWarning(
|
||||
exception,
|
||||
@@ -161,6 +210,62 @@ public sealed class SessionManager : ISessionManager
|
||||
}
|
||||
}
|
||||
|
||||
// Order — fault, deregister (dropping per-session event metrics), release the slot, dispose —
|
||||
// matters: releasing the slot before disposal would let the next open race this worker's
|
||||
// teardown on the same machine. A registered session goes through RemoveSessionAsync, the
|
||||
// single removal path, which is a no-op when a concurrent kill or drain already removed it
|
||||
// (that teardown released the slot and disposed the session). The open-session gauge is never
|
||||
// touched here: SessionOpened is recorded inside the attach gate, after which nothing in the
|
||||
// open sequence can fail.
|
||||
private async Task RollBackOpenAsync(GatewaySession? session, bool registered, Exception cause)
|
||||
{
|
||||
if (session is null)
|
||||
{
|
||||
ReleaseSessionSlot();
|
||||
return;
|
||||
}
|
||||
|
||||
session.MarkFaulted(cause.Message);
|
||||
if (registered)
|
||||
{
|
||||
await RemoveSessionAsync(session).ConfigureAwait(false);
|
||||
return;
|
||||
}
|
||||
|
||||
ReleaseSessionSlot();
|
||||
await session.DisposeAsync().ConfigureAwait(false);
|
||||
}
|
||||
|
||||
// Kills and disposes a worker client that arrived after its session was closed. Best-effort:
|
||||
// the client is already orphaned from the session, so a failure here is logged rather than
|
||||
// allowed to mask the open failure the caller is about to receive.
|
||||
private async Task DiscardWorkerClientAsync(GatewaySession session, IWorkerClient workerClient)
|
||||
{
|
||||
try
|
||||
{
|
||||
workerClient.Kill("session-closed-during-startup");
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
_logger.LogWarning(
|
||||
exception,
|
||||
"Failed to kill the late worker for closed session {SessionId}.",
|
||||
session.SessionId);
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
await workerClient.DisposeAsync().ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
_logger.LogWarning(
|
||||
exception,
|
||||
"Failed to dispose the late worker client for closed session {SessionId}.",
|
||||
session.SessionId);
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public bool TryGetSession(
|
||||
string sessionId,
|
||||
@@ -226,16 +331,27 @@ public sealed class SessionManager : ISessionManager
|
||||
{
|
||||
wasClosed = await session.KillWorkerWithCloseGateAsync(reason, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (ObjectDisposedException) when (!_registry.TryGet(sessionId, out _))
|
||||
{
|
||||
// Lost the race to a concurrent close: between the lookup above and the gate wait, the
|
||||
// close finished, removed the session, accounted SessionClosed and disposed the gate.
|
||||
// Nothing is left to kill and nothing to account; report it as already closed.
|
||||
return new SessionCloseResult(sessionId, SessionState.Closed, AlreadyClosed: true);
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
session.MarkFaulted(exception.Message);
|
||||
_metrics.Fault(SessionManagerErrorCode.CloseFailed.ToString());
|
||||
|
||||
// The open-session gauge was incremented in OpenSessionAsync; every
|
||||
// session reaching KillWorkerAsync had SessionOpened recorded. If the
|
||||
// kill path throws, decrement the gauge here so mxgateway.sessions.open
|
||||
// does not leak — mirroring the equivalent guard in OpenSessionAsync.
|
||||
_metrics.SessionRemoved();
|
||||
// If the open-session gauge was incremented for this session (it was not when the
|
||||
// kill landed while the worker was still starting) and nobody has removed it yet
|
||||
// (a removal accounts the gauge itself), decrement it here so mxgateway.sessions.open
|
||||
// does not leak when the kill path throws.
|
||||
if (session.OpenRecorded && _registry.TryGet(sessionId, out _))
|
||||
{
|
||||
_metrics.SessionRemoved();
|
||||
}
|
||||
|
||||
await RemoveSessionAsync(session).ConfigureAwait(false);
|
||||
throw new SessionManagerException(
|
||||
SessionManagerErrorCode.CloseFailed,
|
||||
@@ -243,7 +359,7 @@ public sealed class SessionManager : ISessionManager
|
||||
exception);
|
||||
}
|
||||
|
||||
if (!wasClosed)
|
||||
if (!wasClosed && session.OpenRecorded)
|
||||
{
|
||||
_metrics.SessionClosed();
|
||||
}
|
||||
@@ -270,12 +386,14 @@ public sealed class SessionManager : ISessionManager
|
||||
{
|
||||
// A session is swept when its normal lease has expired, it has FAULTED (a faulted
|
||||
// session is permanently unusable yet otherwise pins a session slot and a live x86
|
||||
// worker until its DefaultLeaseSeconds lease expires), OR its detach-grace retention
|
||||
// worker until its DefaultLeaseSeconds lease expires), its detach-grace retention
|
||||
// window has elapsed (last external subscriber dropped and no client reconnected
|
||||
// within DetachGraceSeconds). All three are the same teardown; only the reason differs
|
||||
// so operators can tell a fault reap from a short reconnect-window expiry from a long
|
||||
// idle-lease expiry in logs/metrics. Precedence when several fire simultaneously:
|
||||
// lease-expired, then faulted, then detach-grace.
|
||||
// within DetachGraceSeconds), OR it never attached a subscriber and has been idle for
|
||||
// UnattachedSessionGraceSeconds (a client that opened and vanished). All four are the
|
||||
// same teardown; only the reason differs so operators can tell a fault reap from a
|
||||
// short reconnect-window expiry from a never-subscribed reap from a long idle-lease
|
||||
// expiry in logs/metrics. Precedence when several fire simultaneously: lease-expired,
|
||||
// then faulted, then detach-grace, then unattached-grace.
|
||||
//
|
||||
// TOCTOU note: eligibility is re-verified atomically inside TryBeginCloseIfExpired
|
||||
// under _syncRoot, so a client that reattaches a subscriber between the check above
|
||||
@@ -286,7 +404,9 @@ public sealed class SessionManager : ISessionManager
|
||||
? FaultedReason
|
||||
: session.IsDetachGraceExpired(now)
|
||||
? DetachGraceExpiredReason
|
||||
: null;
|
||||
: session.IsUnattachedGraceExpired(now)
|
||||
? UnattachedGraceExpiredReason
|
||||
: null;
|
||||
if (reason is null)
|
||||
{
|
||||
continue;
|
||||
@@ -330,8 +450,11 @@ public sealed class SessionManager : ISessionManager
|
||||
{
|
||||
try
|
||||
{
|
||||
await CloseSessionCoreAsync(candidate.Session, candidate.Reason, closeToken).ConfigureAwait(false);
|
||||
Interlocked.Increment(ref closedCount);
|
||||
SessionCloseResult result = await CloseSessionCoreAsync(candidate.Session, candidate.Reason, closeToken).ConfigureAwait(false);
|
||||
if (!result.AlreadyClosed)
|
||||
{
|
||||
Interlocked.Increment(ref closedCount);
|
||||
}
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
@@ -381,49 +504,86 @@ public sealed class SessionManager : ISessionManager
|
||||
// intentional: that sweep is periodic maintenance whose missed sessions are picked up by
|
||||
// the next pass and, ultimately, by this drain. This drain is terminal — nothing runs after
|
||||
// it — so it must not be abandoned partway.
|
||||
await Parallel.ForEachAsync(
|
||||
_registry.Snapshot(),
|
||||
new ParallelOptions { MaxDegreeOfParallelism = MaxParallelSessionCloses },
|
||||
async (session, _) =>
|
||||
//
|
||||
// Close the open gate BEFORE the first snapshot (see the re-check in OpenSessionAsync), then
|
||||
// re-snapshot until the registry is empty so an open that registered just before the gate
|
||||
// closed is drained too.
|
||||
MarkStopping();
|
||||
for (int pass = 0; pass < MaxShutdownDrainPasses; pass++)
|
||||
{
|
||||
if (_registry.Count == 0)
|
||||
{
|
||||
try
|
||||
{
|
||||
await CloseSessionCoreAsync(session, GatewayShutdownReason, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
_logger.LogWarning(
|
||||
exception,
|
||||
"Graceful shutdown failed for session {SessionId}; killing worker.",
|
||||
session.SessionId);
|
||||
return;
|
||||
}
|
||||
|
||||
// Defensive fallback: CloseSessionCoreAsync's inner
|
||||
// SessionCloseStartedException catch normally removes the session
|
||||
// and accounts the close. This outer fallback only fires for
|
||||
// sessions still in the registry — route through KillWorkerAsync
|
||||
// so the bookkeeping is identical to the dashboard kill path.
|
||||
if (_registry.TryGet(session.SessionId, out GatewaySession? registeredSession)
|
||||
&& registeredSession is not null)
|
||||
await Parallel.ForEachAsync(
|
||||
_registry.Snapshot(),
|
||||
new ParallelOptions { MaxDegreeOfParallelism = MaxParallelSessionCloses },
|
||||
async (session, _) =>
|
||||
{
|
||||
try
|
||||
{
|
||||
try
|
||||
await CloseSessionCoreAsync(session, GatewayShutdownReason, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
_logger.LogWarning(
|
||||
exception,
|
||||
"Graceful shutdown failed for session {SessionId}; killing worker.",
|
||||
session.SessionId);
|
||||
|
||||
// Defensive fallback: CloseSessionCoreAsync's inner
|
||||
// SessionCloseStartedException catch normally removes the session
|
||||
// and accounts the close. This outer fallback only fires for
|
||||
// sessions still in the registry — route through KillWorkerAsync
|
||||
// so the bookkeeping is identical to the dashboard kill path.
|
||||
if (_registry.TryGet(session.SessionId, out GatewaySession? registeredSession)
|
||||
&& registeredSession is not null)
|
||||
{
|
||||
// Deliberately NOT the caller's token: the kill is the last-resort orphan
|
||||
// preventer, so it must still run when the host stop deadline (or a
|
||||
// sibling body's failure) has already cancelled the drain. It is a
|
||||
// synchronous Kill plus registry/dispose bookkeeping, not a wait on
|
||||
// the worker, so it cannot extend the drain meaningfully.
|
||||
await KillWorkerAsync(session.SessionId, GatewayShutdownReason, CancellationToken.None).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception killException)
|
||||
{
|
||||
_logger.LogWarning(
|
||||
killException,
|
||||
"Worker kill fallback failed for session {SessionId}.",
|
||||
session.SessionId);
|
||||
try
|
||||
{
|
||||
// Deliberately NOT the caller's token: the kill is the last-resort orphan
|
||||
// preventer, so it must still run when the host stop deadline (or a
|
||||
// sibling body's failure) has already cancelled the drain. It is a
|
||||
// synchronous Kill plus registry/dispose bookkeeping, not a wait on
|
||||
// the worker, so it cannot extend the drain meaningfully.
|
||||
await KillWorkerAsync(session.SessionId, GatewayShutdownReason, CancellationToken.None).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception killException)
|
||||
{
|
||||
_logger.LogWarning(
|
||||
killException,
|
||||
"Worker kill fallback failed for session {SessionId}.",
|
||||
session.SessionId);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}).ConfigureAwait(false);
|
||||
}).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
int remaining = _registry.Count;
|
||||
if (remaining > 0)
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"Session drain finished with {Remaining} session(s) still registered after {Passes} passes; their workers are left to the orphan terminator on the next start.",
|
||||
remaining,
|
||||
MaxShutdownDrainPasses);
|
||||
}
|
||||
}
|
||||
|
||||
private void MarkStopping()
|
||||
{
|
||||
Volatile.Write(ref _stopping, 1);
|
||||
}
|
||||
|
||||
private void ThrowIfStopping()
|
||||
{
|
||||
if (Volatile.Read(ref _stopping) != 0)
|
||||
{
|
||||
throw new SessionManagerException(
|
||||
SessionManagerErrorCode.GatewayStopping,
|
||||
"The gateway is stopping and is not accepting new sessions.");
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<SessionCloseResult> CloseSessionCoreAsync(
|
||||
@@ -435,7 +595,19 @@ public sealed class SessionManager : ISessionManager
|
||||
try
|
||||
{
|
||||
SessionCloseResult result = await session.CloseAsync(reason, cancellationToken).ConfigureAwait(false);
|
||||
if (!wasClosed && !result.AlreadyClosed)
|
||||
if (result.WorkerShutdownFailure is not null)
|
||||
{
|
||||
// A clean close from the session's point of view (the worker was killed and the
|
||||
// session is Closed), but the operator should see that the worker would not stop
|
||||
// gracefully — a run of these is the signal that workers are hanging on shutdown.
|
||||
_logger.LogWarning(
|
||||
result.WorkerShutdownFailure,
|
||||
"Worker for session {SessionId} did not shut down gracefully and was killed; reason={Reason}.",
|
||||
session.SessionId,
|
||||
reason);
|
||||
}
|
||||
|
||||
if (!wasClosed && !result.AlreadyClosed && session.OpenRecorded)
|
||||
{
|
||||
_metrics.SessionClosed();
|
||||
}
|
||||
@@ -447,10 +619,17 @@ public sealed class SessionManager : ISessionManager
|
||||
{
|
||||
throw;
|
||||
}
|
||||
catch (ObjectDisposedException) when (!_registry.TryGet(session.SessionId, out _))
|
||||
{
|
||||
// Lost the race to a concurrent close that removed, accounted and disposed the
|
||||
// session after this caller picked it up (a sweep's snapshot, a client close that
|
||||
// looked it up first). Not a failure and nothing to redo.
|
||||
return new SessionCloseResult(session.SessionId, SessionState.Closed, AlreadyClosed: true);
|
||||
}
|
||||
catch (SessionCloseStartedException exception)
|
||||
{
|
||||
session.MarkFaulted(exception.Message);
|
||||
if (!wasClosed)
|
||||
if (!wasClosed && session.OpenRecorded)
|
||||
{
|
||||
// Account the close as a SessionClosed (decrements the open-session
|
||||
// gauge AND increments the sessions.closed counter), not just
|
||||
@@ -566,7 +745,8 @@ public sealed class SessionManager : ISessionManager
|
||||
TimeSpan.FromMilliseconds(Math.Max(0, _options.Sessions.WorkerReadyWaitTimeoutMs)),
|
||||
_addressNormalizer,
|
||||
TimeSpan.FromSeconds(Math.Max(0, _options.Sessions.FaultedGraceSeconds)),
|
||||
ownerDashboardTags);
|
||||
ownerDashboardTags,
|
||||
TimeSpan.FromSeconds(Math.Max(0, _options.Sessions.UnattachedSessionGraceSeconds)));
|
||||
}
|
||||
|
||||
private static string CreateClientCorrelationId(
|
||||
|
||||
@@ -17,4 +17,12 @@ public enum SessionManagerErrorCode
|
||||
/// Maps to gRPC <c>PermissionDenied</c>.
|
||||
/// </summary>
|
||||
PermissionDenied,
|
||||
|
||||
/// <summary>
|
||||
/// The gateway is stopping, so no new session can be opened: a session created behind the
|
||||
/// shutdown drain's snapshot would have nobody left to close it and its worker would
|
||||
/// outlive the gateway as an orphan. Maps to gRPC <c>Unavailable</c> so the client retries
|
||||
/// against the restarted gateway.
|
||||
/// </summary>
|
||||
GatewayStopping,
|
||||
}
|
||||
|
||||
@@ -51,6 +51,7 @@
|
||||
"DefaultLeaseSeconds": 1800,
|
||||
"LeaseSweepIntervalSeconds": 30,
|
||||
"DetachGraceSeconds": 30,
|
||||
"UnattachedSessionGraceSeconds": 0,
|
||||
"AllowMultipleEventSubscribers": false,
|
||||
"MaxEventSubscribersPerSession": 8
|
||||
},
|
||||
|
||||
@@ -619,6 +619,34 @@ public sealed class GatewayOptionsValidatorTests
|
||||
Assert.True(result.Succeeded);
|
||||
}
|
||||
|
||||
/// <summary>Verifies a negative <see cref="SessionOptions.UnattachedSessionGraceSeconds"/> fails validation.</summary>
|
||||
[Fact]
|
||||
public void Validate_Fails_WhenUnattachedSessionGraceSecondsIsNegative()
|
||||
{
|
||||
GatewayOptions options = CloneWithSessions(
|
||||
ValidOptions(),
|
||||
new SessionOptions { UnattachedSessionGraceSeconds = -1 });
|
||||
ValidateOptionsResult result = new GatewayOptionsValidator().Validate(null, options);
|
||||
Assert.True(result.Failed);
|
||||
Assert.Contains(
|
||||
result.Failures!,
|
||||
f => f.Contains("MxGateway:Sessions:UnattachedSessionGraceSeconds"));
|
||||
}
|
||||
|
||||
/// <summary>Verifies a zero (off) and a positive <see cref="SessionOptions.UnattachedSessionGraceSeconds"/> pass validation.</summary>
|
||||
/// <param name="seconds">Grace window under test.</param>
|
||||
[Theory]
|
||||
[InlineData(0)]
|
||||
[InlineData(120)]
|
||||
public void Validate_Succeeds_WhenUnattachedSessionGraceSecondsIsZeroOrPositive(int seconds)
|
||||
{
|
||||
GatewayOptions options = CloneWithSessions(
|
||||
ValidOptions(),
|
||||
new SessionOptions { UnattachedSessionGraceSeconds = seconds });
|
||||
ValidateOptionsResult result = new GatewayOptionsValidator().Validate(null, options);
|
||||
Assert.True(result.Succeeded);
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// ReplayBufferCapacity / ReplayRetentionSeconds validation
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
@@ -76,6 +76,49 @@ public sealed class GatewaySessionTests
|
||||
await session.DisposeAsync();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A worker client that arrives after the session has already been closed (an admin kill
|
||||
/// or the host drain landed while the worker was still starting) must be refused:
|
||||
/// <see cref="GatewaySession.AttachWorkerClient"/> returns <see langword="false"/> and
|
||||
/// leaves <see cref="GatewaySession.WorkerClient"/> unset, so the caller can discard the
|
||||
/// client instead of handing a dead session back as an opened one.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task AttachWorkerClient_AfterClose_RefusesLateWorkerClient()
|
||||
{
|
||||
GatewaySession session = CreateSession();
|
||||
session.TransitionTo(SessionState.StartingWorker);
|
||||
|
||||
await session.CloseAsync("admin-kill", CancellationToken.None);
|
||||
bool attached = session.AttachWorkerClient(new BlockingShutdownWorkerClient());
|
||||
|
||||
Assert.False(attached);
|
||||
Assert.Null(session.WorkerClient);
|
||||
Assert.Equal(SessionState.Closed, session.State);
|
||||
|
||||
await session.DisposeAsync();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Disposal is the other end of the same window: once the session's distributor has been
|
||||
/// torn down nothing can ever pump a late worker's events, so the attach is refused there
|
||||
/// too rather than storing a client that is never read.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task AttachWorkerClient_AfterDispose_RefusesLateWorkerClient()
|
||||
{
|
||||
GatewaySession session = CreateSession();
|
||||
session.TransitionTo(SessionState.StartingWorker);
|
||||
|
||||
await session.DisposeAsync();
|
||||
bool attached = session.AttachWorkerClient(new BlockingShutdownWorkerClient());
|
||||
|
||||
Assert.False(attached);
|
||||
Assert.Null(session.WorkerClient);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A <see cref="GatewaySession.MarkFaulted"/> issued
|
||||
/// while <see cref="GatewaySession.CloseAsync"/> is parked between its
|
||||
|
||||
@@ -0,0 +1,151 @@
|
||||
using Microsoft.Extensions.Options;
|
||||
using ZB.MOM.WW.MxGateway.Contracts.Proto;
|
||||
using ZB.MOM.WW.MxGateway.Server.Configuration;
|
||||
using ZB.MOM.WW.MxGateway.Server.Metrics;
|
||||
using ZB.MOM.WW.MxGateway.Server.Sessions;
|
||||
using ZB.MOM.WW.MxGateway.Server.Workers;
|
||||
using ZB.MOM.WW.MxGateway.Tests.Gateway.Workers.Fakes;
|
||||
|
||||
namespace ZB.MOM.WW.MxGateway.Tests.Gateway.Sessions;
|
||||
|
||||
/// <summary>
|
||||
/// A worker that goes away on its own — the client sent the <c>ShutdownWorker</c> control
|
||||
/// command and the worker exited cleanly, or the worker crashed / its pipe broke — must be
|
||||
/// noticed by its <see cref="GatewaySession"/>. The real <see cref="WorkerClient"/> reaches
|
||||
/// <c>Closed</c> / <c>Faulted</c>, but nothing used to tell the session, which kept reporting
|
||||
/// <c>Ready</c> (live windev finding) until its lease expired.
|
||||
/// </summary>
|
||||
public sealed class GatewaySessionWorkerExitTests
|
||||
{
|
||||
private static readonly TimeSpan TestTimeout = TimeSpan.FromSeconds(5);
|
||||
|
||||
/// <summary>
|
||||
/// After the client's <c>ShutdownWorker</c> command the fake worker replies OK, sends the
|
||||
/// shutdown ack and the worker client closes. The session must fault (the worker is gone
|
||||
/// outside a gateway-initiated close), stay observable with a fault reason, and then be
|
||||
/// reaped by the ordinary faulted-session sweep — which is where the session is accounted
|
||||
/// closed, exactly once.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task ClientShutdownWorker_WorkerExits_SessionFaultsAndSweepReapsItOnce()
|
||||
{
|
||||
await using FakeWorkerHarness fakeWorker = await FakeWorkerHarness.CreateConnectedPairAsync();
|
||||
using GatewayMetrics metrics = new();
|
||||
WorkerClient client = fakeWorker.CreateClient(metrics: metrics);
|
||||
await StartClientAsync(fakeWorker, client);
|
||||
SessionManager manager = CreateManager(client, metrics);
|
||||
GatewaySession session = await OpenSessionAsync(manager);
|
||||
using IEventSubscriberLease stream = session.AttachEventSubscriber(maxSubscribers: 1);
|
||||
|
||||
Task<WorkerCommandReply> invokeTask = manager.InvokeAsync(session.SessionId, CreateShutdownWorkerCommand(), CancellationToken.None);
|
||||
await fakeWorker.RespondToControlCommandAsync().WaitAsync(TestTimeout);
|
||||
WorkerCommandReply reply = await invokeTask.WaitAsync(TestTimeout);
|
||||
Assert.Equal(ProtocolStatusCode.Ok, reply.Reply.ProtocolStatus.Code);
|
||||
|
||||
await WaitUntilAsync(() => session.State == SessionState.Faulted);
|
||||
|
||||
Assert.Equal(WorkerClientState.Closed, client.State);
|
||||
Assert.Contains("worker", session.FinalFault, StringComparison.OrdinalIgnoreCase);
|
||||
Assert.True(manager.TryGetSession(session.SessionId, out _));
|
||||
Assert.Equal(1, metrics.GetSnapshot().OpenSessions);
|
||||
Assert.Equal(0, metrics.GetSnapshot().SessionsClosed);
|
||||
|
||||
int closedCount = await manager.CloseExpiredLeasesAsync(DateTimeOffset.UtcNow, CancellationToken.None);
|
||||
|
||||
Assert.Equal(1, closedCount);
|
||||
Assert.Equal(SessionState.Closed, session.State);
|
||||
Assert.False(manager.TryGetSession(session.SessionId, out _));
|
||||
GatewayMetricsSnapshot snapshot = metrics.GetSnapshot();
|
||||
Assert.Equal(1, snapshot.SessionsClosed);
|
||||
Assert.Equal(0, snapshot.OpenSessions);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The other way a worker goes away: its pipe breaks (crash). The worker client faults, and
|
||||
/// the session must follow it to <c>Faulted</c> rather than stay <c>Ready</c>.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task WorkerPipeBreaks_SessionFaults()
|
||||
{
|
||||
await using FakeWorkerHarness fakeWorker = await FakeWorkerHarness.CreateConnectedPairAsync();
|
||||
using GatewayMetrics metrics = new();
|
||||
WorkerClient client = fakeWorker.CreateClient(metrics: metrics);
|
||||
await StartClientAsync(fakeWorker, client);
|
||||
SessionManager manager = CreateManager(client, metrics);
|
||||
GatewaySession session = await OpenSessionAsync(manager);
|
||||
using IEventSubscriberLease stream = session.AttachEventSubscriber(maxSubscribers: 1);
|
||||
|
||||
await fakeWorker.DisposeWorkerSideAsync();
|
||||
|
||||
await WaitUntilAsync(() => session.State == SessionState.Faulted);
|
||||
Assert.Equal(WorkerClientState.Faulted, client.State);
|
||||
|
||||
await manager.CloseExpiredLeasesAsync(DateTimeOffset.UtcNow, CancellationToken.None);
|
||||
Assert.False(manager.TryGetSession(session.SessionId, out _));
|
||||
Assert.Equal(1, metrics.GetSnapshot().SessionsClosed);
|
||||
Assert.Equal(0, metrics.GetSnapshot().OpenSessions);
|
||||
}
|
||||
|
||||
private static async Task StartClientAsync(FakeWorkerHarness fakeWorker, WorkerClient client)
|
||||
{
|
||||
Task startTask = client.StartAsync(CancellationToken.None);
|
||||
await fakeWorker.CompleteStartupAsync();
|
||||
await startTask.WaitAsync(TestTimeout);
|
||||
Assert.Equal(WorkerClientState.Ready, client.State);
|
||||
}
|
||||
|
||||
private static SessionManager CreateManager(IWorkerClient client, GatewayMetrics metrics)
|
||||
{
|
||||
return new SessionManager(
|
||||
new SessionRegistry(),
|
||||
new SingleClientFactory(client),
|
||||
Options.Create(new GatewayOptions()),
|
||||
metrics);
|
||||
}
|
||||
|
||||
private static Task<GatewaySession> OpenSessionAsync(SessionManager manager)
|
||||
{
|
||||
return manager.OpenSessionAsync(
|
||||
new SessionOpenRequest(RequestedBackend: null, ClientSessionName: "worker-exit", ClientCorrelationId: null, CommandTimeout: null),
|
||||
"client-1",
|
||||
ownerKeyId: null,
|
||||
CancellationToken.None);
|
||||
}
|
||||
|
||||
private static WorkerCommand CreateShutdownWorkerCommand()
|
||||
{
|
||||
return new WorkerCommand
|
||||
{
|
||||
Command = new MxCommand
|
||||
{
|
||||
Kind = MxCommandKind.ShutdownWorker,
|
||||
ShutdownWorker = new ShutdownWorkerCommand(),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
private static async Task WaitUntilAsync(Func<bool> condition)
|
||||
{
|
||||
DateTime deadline = DateTime.UtcNow + TestTimeout;
|
||||
while (!condition())
|
||||
{
|
||||
if (DateTime.UtcNow > deadline)
|
||||
{
|
||||
Assert.Fail("Condition was not met within the test timeout.");
|
||||
}
|
||||
|
||||
await Task.Delay(10);
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class SingleClientFactory(IWorkerClient client) : ISessionWorkerClientFactory
|
||||
{
|
||||
/// <inheritdoc />
|
||||
public Task<IWorkerClient> CreateAsync(GatewaySession session, CancellationToken cancellationToken)
|
||||
{
|
||||
return Task.FromResult(client);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -641,10 +641,16 @@ public sealed class SessionManagerTests
|
||||
Assert.Equal(0, metrics.GetSnapshot().OpenSessions);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that closing a session kills the worker when shutdown fails.</summary>
|
||||
/// <summary>
|
||||
/// A worker that does not shut down gracefully is killed, and that is a <em>successful</em>
|
||||
/// close: the kill is the documented fallback, the worker is gone and the session is
|
||||
/// <c>Closed</c>. Reporting it as <c>CloseFailed</c> + <c>Faulted</c> + a fault metric
|
||||
/// misreported every hung-worker close as a teardown failure. <c>CloseFailed</c> is reserved
|
||||
/// for both the shutdown and the kill failing.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task CloseSessionAsync_WhenWorkerShutdownFails_KillsWorker()
|
||||
public async Task CloseSessionAsync_WhenWorkerShutdownFails_KillsWorkerAndReportsClosed()
|
||||
{
|
||||
FakeWorkerClient workerClient = new()
|
||||
{
|
||||
@@ -652,15 +658,24 @@ public sealed class SessionManagerTests
|
||||
WorkerClientErrorCode.ShutdownTimeout,
|
||||
"Worker shutdown timed out."),
|
||||
};
|
||||
SessionManager manager = CreateManager(new FakeSessionWorkerClientFactory(workerClient));
|
||||
using GatewayMetrics metrics = new();
|
||||
SessionManager manager = CreateManager(new FakeSessionWorkerClientFactory(workerClient), metrics: metrics);
|
||||
GatewaySession session = await manager.OpenSessionAsync(CreateOpenRequest(), "client-1", ownerKeyId: null, CancellationToken.None);
|
||||
|
||||
SessionManagerException exception = await Assert.ThrowsAsync<SessionManagerException>(
|
||||
async () => await manager.CloseSessionAsync(session.SessionId, CancellationToken.None));
|
||||
SessionCloseResult result = await manager.CloseSessionAsync(session.SessionId, CancellationToken.None);
|
||||
|
||||
Assert.Equal(SessionManagerErrorCode.CloseFailed, exception.ErrorCode);
|
||||
Assert.Equal(SessionState.Closed, result.FinalState);
|
||||
Assert.False(result.AlreadyClosed);
|
||||
Assert.Same(workerClient.ShutdownException, result.WorkerShutdownFailure);
|
||||
Assert.Equal(SessionState.Closed, session.State);
|
||||
Assert.Null(session.FinalFault);
|
||||
Assert.Equal(1, workerClient.ShutdownCount);
|
||||
Assert.Equal(1, workerClient.KillCount);
|
||||
Assert.False(manager.TryGetSession(session.SessionId, out _));
|
||||
GatewayMetricsSnapshot snapshot = metrics.GetSnapshot();
|
||||
Assert.Equal(0, snapshot.Faults);
|
||||
Assert.Equal(1, snapshot.SessionsClosed);
|
||||
Assert.Equal(0, snapshot.OpenSessions);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that when worker shutdown fails, the session is removed and the slot is released.</summary>
|
||||
@@ -689,31 +704,59 @@ public sealed class SessionManagerTests
|
||||
CancellationToken.None);
|
||||
metrics.EventReceived(firstSession.SessionId, MxEventFamily.OnDataChange.ToString());
|
||||
|
||||
SessionManagerException exception = await Assert.ThrowsAsync<SessionManagerException>(
|
||||
async () => await manager.CloseSessionAsync(firstSession.SessionId, CancellationToken.None));
|
||||
SessionCloseResult result = await manager.CloseSessionAsync(firstSession.SessionId, CancellationToken.None);
|
||||
GatewaySession secondSession = await manager.OpenSessionAsync(
|
||||
CreateOpenRequest(),
|
||||
"client-2",
|
||||
ownerKeyId: null,
|
||||
CancellationToken.None);
|
||||
|
||||
Assert.Equal(SessionManagerErrorCode.CloseFailed, exception.ErrorCode);
|
||||
Assert.Equal(SessionState.Closed, result.FinalState);
|
||||
Assert.False(manager.TryGetSession(firstSession.SessionId, out _));
|
||||
Assert.True(manager.TryGetSession(secondSession.SessionId, out _));
|
||||
Assert.Equal(1, registry.Count);
|
||||
Assert.Equal(1, failingWorkerClient.KillCount);
|
||||
Assert.Equal(1, failingWorkerClient.DisposeCount);
|
||||
GatewayMetricsSnapshot snapshot = metrics.GetSnapshot();
|
||||
|
||||
// A close-that-failed accounts as SessionClosed (counter += 1) rather
|
||||
// than SessionRemoved (gauge -= 1, counter unchanged). The session is
|
||||
// being removed from the registry on this path, so it must show up in
|
||||
// the closed count.
|
||||
Assert.Equal(1, snapshot.SessionsClosed);
|
||||
Assert.False(snapshot.EventsBySession.ContainsKey(firstSession.SessionId));
|
||||
Assert.Equal(1, snapshot.OpenSessions);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Only when the graceful shutdown AND the kill fallback both fail is the close a failure:
|
||||
/// the session is faulted, removed, counted as closed (it is leaving the registry) and the
|
||||
/// caller gets <c>CloseFailed</c>.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task CloseSessionAsync_WhenShutdownAndKillBothFail_ThrowsCloseFailed()
|
||||
{
|
||||
FakeWorkerClient workerClient = new()
|
||||
{
|
||||
ShutdownException = new WorkerClientException(
|
||||
WorkerClientErrorCode.ShutdownTimeout,
|
||||
"Worker shutdown timed out."),
|
||||
KillException = new InvalidOperationException("worker kill failed"),
|
||||
};
|
||||
using GatewayMetrics metrics = new();
|
||||
SessionManager manager = CreateManager(new FakeSessionWorkerClientFactory(workerClient), metrics: metrics);
|
||||
GatewaySession session = await manager.OpenSessionAsync(CreateOpenRequest(), "client-1", ownerKeyId: null, CancellationToken.None);
|
||||
|
||||
SessionManagerException exception = await Assert.ThrowsAsync<SessionManagerException>(
|
||||
async () => await manager.CloseSessionAsync(session.SessionId, CancellationToken.None));
|
||||
|
||||
Assert.Equal(SessionManagerErrorCode.CloseFailed, exception.ErrorCode);
|
||||
Assert.Equal(SessionState.Faulted, session.State);
|
||||
Assert.Equal(1, workerClient.ShutdownCount);
|
||||
Assert.Equal(1, workerClient.KillCount);
|
||||
Assert.False(manager.TryGetSession(session.SessionId, out _));
|
||||
GatewayMetricsSnapshot snapshot = metrics.GetSnapshot();
|
||||
Assert.Equal(1, snapshot.Faults);
|
||||
Assert.Equal(1, snapshot.SessionsClosed);
|
||||
Assert.Equal(0, snapshot.OpenSessions);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that when the second close is canceled, the session is not removed if owned by the first close.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
@@ -881,6 +924,71 @@ public sealed class SessionManagerTests
|
||||
Assert.True(metrics.GetSnapshot().Faults > 0);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A kill whose session lookup succeeded but whose close gate is disposed by the time it
|
||||
/// waits on it lost the race to a close that already removed, disposed and accounted the
|
||||
/// session. It must report <c>AlreadyClosed</c> and touch nothing: the old catch recorded a
|
||||
/// fault, faulted the session and decremented the open gauge a second time (visible here as
|
||||
/// the other open session's slot disappearing from the gauge).
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task KillWorkerAsync_WhenCloseWonTheRace_ReportsAlreadyClosedWithoutDoubleAccounting()
|
||||
{
|
||||
FakeWorkerClient racedClient = new();
|
||||
FakeWorkerClient otherClient = new();
|
||||
StaleRegistry registry = new();
|
||||
using GatewayMetrics metrics = new();
|
||||
SessionManager manager = CreateManager(new QueueingSessionWorkerClientFactory(racedClient, otherClient), registry, metrics);
|
||||
GatewaySession raced = await manager.OpenSessionAsync(CreateOpenRequest(), "client-1", ownerKeyId: null, CancellationToken.None);
|
||||
await manager.OpenSessionAsync(CreateOpenRequest(), "client-2", ownerKeyId: null, CancellationToken.None);
|
||||
|
||||
await manager.CloseSessionAsync(raced.SessionId, CancellationToken.None);
|
||||
Assert.Equal(1, metrics.GetSnapshot().OpenSessions);
|
||||
|
||||
// The kill looked the session up just before the close removed and disposed it.
|
||||
registry.ServeStaleLookupOnce(raced);
|
||||
SessionCloseResult result = await manager.KillWorkerAsync(raced.SessionId, "admin-kill", CancellationToken.None);
|
||||
|
||||
Assert.True(result.AlreadyClosed);
|
||||
Assert.Equal(SessionState.Closed, result.FinalState);
|
||||
Assert.Equal(0, racedClient.KillCount);
|
||||
GatewayMetricsSnapshot snapshot = metrics.GetSnapshot();
|
||||
Assert.Equal(1, snapshot.OpenSessions);
|
||||
Assert.Equal(1, snapshot.SessionsClosed);
|
||||
Assert.Equal(0, snapshot.Faults);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The sweep's counterpart of the kill race: a session that a concurrent close removed and
|
||||
/// disposed between the snapshot and the close call is simply skipped — it is not a sweep
|
||||
/// failure to log, and it is not counted as closed by the sweep.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task CloseExpiredLeasesAsync_WhenCloseWonTheRace_DoesNotReportSweepFailure()
|
||||
{
|
||||
FakeWorkerClient workerClient = new();
|
||||
StaleRegistry registry = new();
|
||||
using GatewayMetrics metrics = new();
|
||||
SessionManager manager = CreateManager(new FakeSessionWorkerClientFactory(workerClient), registry, metrics);
|
||||
GatewaySession session = await manager.OpenSessionAsync(CreateOpenRequest(), "client-1", ownerKeyId: null, CancellationToken.None);
|
||||
DateTimeOffset now = DateTimeOffset.UtcNow;
|
||||
SetLeaseExpiry(session, now.AddSeconds(-1));
|
||||
|
||||
await manager.CloseSessionAsync(session.SessionId, CancellationToken.None);
|
||||
registry.ServeStaleSnapshot(session);
|
||||
|
||||
int closedCount = await manager.CloseExpiredLeasesAsync(now, CancellationToken.None);
|
||||
|
||||
Assert.Equal(0, closedCount);
|
||||
Assert.Equal(1, workerClient.ShutdownCount);
|
||||
GatewayMetricsSnapshot snapshot = metrics.GetSnapshot();
|
||||
Assert.Equal(1, snapshot.SessionsClosed);
|
||||
Assert.Equal(0, snapshot.OpenSessions);
|
||||
Assert.Equal(0, snapshot.Faults);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Concurrent kills on the same session must not
|
||||
/// double-increment <c>mxgateway.sessions.closed</c>. The first kill wins, the second
|
||||
@@ -926,7 +1034,9 @@ public sealed class SessionManagerTests
|
||||
/// <c>KillWorker</c> + <c>RemoveSessionAsync</c> when <c>CloseSessionCoreAsync</c> throws)
|
||||
/// must still account a successful close: both the open-session gauge must drop to zero AND
|
||||
/// the <c>mxgateway.sessions.closed</c> counter must increment. Without the fix, the
|
||||
/// graceful-close failure path under-counts the closed counter.
|
||||
/// graceful-close failure path under-counts the closed counter. Both the shutdown and the
|
||||
/// in-close kill fail here, because a kill that succeeds is a clean close and never reaches
|
||||
/// the fallback.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
@@ -935,6 +1045,7 @@ public sealed class SessionManagerTests
|
||||
FakeWorkerClient throwingClient = new()
|
||||
{
|
||||
ShutdownException = new InvalidOperationException("worker shutdown failed"),
|
||||
KillException = new InvalidOperationException("worker kill failed"),
|
||||
};
|
||||
using GatewayMetrics metrics = new();
|
||||
SessionManager manager = CreateManager(
|
||||
@@ -978,6 +1089,133 @@ public sealed class SessionManagerTests
|
||||
Assert.Equal(1, metrics.GetSnapshot().Faults);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A failed open must leave no per-session event metrics behind: the rollback goes through
|
||||
/// the same removal path every other teardown uses, so <c>EventsBySession</c> is cleared
|
||||
/// for the session that never opened.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task OpenSessionAsync_WhenWorkerCreationFails_RemovesSessionEventMetrics()
|
||||
{
|
||||
using GatewayMetrics metrics = new();
|
||||
EventRecordingFailingSessionWorkerClientFactory factory = new(metrics);
|
||||
SessionManager manager = CreateManager(factory, metrics: metrics);
|
||||
|
||||
await Assert.ThrowsAsync<SessionManagerException>(
|
||||
async () => await manager.OpenSessionAsync(CreateOpenRequest(), "client-1", ownerKeyId: null, CancellationToken.None));
|
||||
|
||||
Assert.NotNull(factory.StartedSessionId);
|
||||
Assert.False(metrics.GetSnapshot().EventsBySession.ContainsKey(factory.StartedSessionId!));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The session is registered before its worker is created, so an admin Kill can land while
|
||||
/// the worker is still starting (up to <c>StartupTimeoutSeconds</c>). The late worker client
|
||||
/// must not be attached to the already-closed session and handed back to the caller as if
|
||||
/// the open had succeeded: it is killed and disposed, the open fails with
|
||||
/// <c>OpenFailed</c>, and nothing is left in the registry or the metrics.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task OpenSessionAsync_WhenKilledDuringWorkerStartup_FailsOpenAndDiscardsLateWorkerClient()
|
||||
{
|
||||
FakeWorkerClient workerClient = new();
|
||||
GatedSessionWorkerClientFactory factory = new(workerClient);
|
||||
SessionRegistry registry = new();
|
||||
using GatewayMetrics metrics = new();
|
||||
SessionManager manager = CreateManager(factory, registry, metrics, CreateOptions(maxSessions: 1));
|
||||
|
||||
Task<GatewaySession> openTask = manager.OpenSessionAsync(CreateOpenRequest(), "client-1", ownerKeyId: null, CancellationToken.None);
|
||||
GatewaySession starting = await factory.WaitForCreateStartAsync();
|
||||
|
||||
SessionCloseResult kill = await manager.KillWorkerAsync(starting.SessionId, "admin-kill", CancellationToken.None);
|
||||
factory.ReleaseCreate();
|
||||
SessionManagerException exception = await Assert.ThrowsAsync<SessionManagerException>(async () => await openTask);
|
||||
|
||||
Assert.Equal(SessionState.Closed, kill.FinalState);
|
||||
Assert.Equal(SessionManagerErrorCode.OpenFailed, exception.ErrorCode);
|
||||
Assert.Equal(1, workerClient.KillCount);
|
||||
Assert.Equal(1, workerClient.DisposeCount);
|
||||
Assert.Null(starting.WorkerClient);
|
||||
Assert.Equal(0, registry.Count);
|
||||
GatewayMetricsSnapshot snapshot = metrics.GetSnapshot();
|
||||
Assert.Equal(0, snapshot.OpenSessions);
|
||||
Assert.Equal(0, snapshot.SessionsOpened);
|
||||
Assert.Equal(0, snapshot.SessionsClosed);
|
||||
|
||||
// The slot the killed open held must have been released exactly once: a replacement open
|
||||
// succeeds against MaxSessions = 1 (the released gate hands the same fake straight back).
|
||||
GatewaySession reopened = await manager.OpenSessionAsync(CreateOpenRequest(), "client-2", ownerKeyId: null, CancellationToken.None);
|
||||
Assert.Equal(SessionState.Ready, reopened.State);
|
||||
Assert.Equal(1, registry.Count);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A caller that abandons its open (deadline, disconnect) while the worker is starting
|
||||
/// must not leave the worker running: whatever fails between the worker arriving and the
|
||||
/// session becoming Ready — here the caller's own cancellation — the late worker client is
|
||||
/// killed and disposed before the failure surfaces.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task OpenSessionAsync_WhenCallerCancelledDuringWorkerStartup_DiscardsWorkerClient()
|
||||
{
|
||||
FakeWorkerClient workerClient = new();
|
||||
GatedSessionWorkerClientFactory factory = new(workerClient);
|
||||
SessionRegistry registry = new();
|
||||
using GatewayMetrics metrics = new();
|
||||
SessionManager manager = CreateManager(factory, registry, metrics);
|
||||
using CancellationTokenSource cancellation = new();
|
||||
|
||||
Task<GatewaySession> openTask = manager.OpenSessionAsync(CreateOpenRequest(), "client-1", ownerKeyId: null, cancellation.Token);
|
||||
GatewaySession starting = await factory.WaitForCreateStartAsync();
|
||||
|
||||
await cancellation.CancelAsync();
|
||||
factory.ReleaseCreate();
|
||||
SessionManagerException exception = await Assert.ThrowsAsync<SessionManagerException>(async () => await openTask);
|
||||
|
||||
Assert.Equal(SessionManagerErrorCode.OpenFailed, exception.ErrorCode);
|
||||
Assert.IsAssignableFrom<OperationCanceledException>(exception.InnerException);
|
||||
Assert.Equal(1, workerClient.KillCount);
|
||||
Assert.Equal(1, workerClient.DisposeCount);
|
||||
Assert.NotEqual(SessionState.Ready, starting.State);
|
||||
Assert.Equal(0, registry.Count);
|
||||
Assert.Equal(0, metrics.GetSnapshot().OpenSessions);
|
||||
Assert.Equal(0, metrics.GetSnapshot().SessionsOpened);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Same window as the kill case, driven by the host drain: a session still starting its
|
||||
/// worker when <c>ShutdownAsync</c> runs is closed by the drain, and the worker client that
|
||||
/// arrives afterwards is discarded rather than left running as an orphan.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task OpenSessionAsync_WhenShutdownDuringWorkerStartup_FailsOpenAndDiscardsLateWorkerClient()
|
||||
{
|
||||
FakeWorkerClient workerClient = new();
|
||||
GatedSessionWorkerClientFactory factory = new(workerClient);
|
||||
SessionRegistry registry = new();
|
||||
using GatewayMetrics metrics = new();
|
||||
SessionManager manager = CreateManager(factory, registry, metrics);
|
||||
|
||||
Task<GatewaySession> openTask = manager.OpenSessionAsync(CreateOpenRequest(), "client-1", ownerKeyId: null, CancellationToken.None);
|
||||
GatewaySession starting = await factory.WaitForCreateStartAsync();
|
||||
|
||||
await manager.ShutdownAsync(CancellationToken.None);
|
||||
factory.ReleaseCreate();
|
||||
SessionManagerException exception = await Assert.ThrowsAsync<SessionManagerException>(async () => await openTask);
|
||||
|
||||
Assert.Equal(SessionManagerErrorCode.OpenFailed, exception.ErrorCode);
|
||||
Assert.Equal(SessionState.Closed, starting.State);
|
||||
Assert.Equal(1, workerClient.KillCount);
|
||||
Assert.Equal(1, workerClient.DisposeCount);
|
||||
Assert.Equal(0, registry.Count);
|
||||
Assert.Equal(0, metrics.GetSnapshot().OpenSessions);
|
||||
Assert.Equal(0, metrics.GetSnapshot().SessionsClosed);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that closing expired leases only closes expired sessions.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
@@ -990,8 +1228,8 @@ public sealed class SessionManagerTests
|
||||
GatewaySession expiredSession = await manager.OpenSessionAsync(CreateOpenRequest(), "client-1", ownerKeyId: null, CancellationToken.None);
|
||||
GatewaySession activeSession = await manager.OpenSessionAsync(CreateOpenRequest(), "client-2", ownerKeyId: null, CancellationToken.None);
|
||||
DateTimeOffset now = DateTimeOffset.UtcNow;
|
||||
expiredSession.ExtendLease(now.AddSeconds(-1));
|
||||
activeSession.ExtendLease(now.AddMinutes(5));
|
||||
SetLeaseExpiry(expiredSession, now.AddSeconds(-1));
|
||||
SetLeaseExpiry(activeSession, now.AddMinutes(5));
|
||||
|
||||
int closedCount = await manager.CloseExpiredLeasesAsync(now, CancellationToken.None);
|
||||
|
||||
@@ -1002,6 +1240,243 @@ public sealed class SessionManagerTests
|
||||
Assert.Equal(0, activeClient.ShutdownCount);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Every sweep-driven close — lease expiry, faulted reap and detach-grace expiry — must
|
||||
/// account <c>SessionClosed</c> exactly once. The sweep begins the close itself
|
||||
/// (<c>TryBeginCloseIfExpired</c>) before delegating to the shared close path, and that
|
||||
/// reservation must not make the delegated close look like a second, already-running close
|
||||
/// whose accounting is skipped: the open-session gauge would then never drop for swept
|
||||
/// sessions and the closed counter would under-count them.
|
||||
/// </summary>
|
||||
/// <param name="sweepKind">Which sweep condition selects the session.</param>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Theory]
|
||||
[InlineData(SessionManager.LeaseExpiredReason)]
|
||||
[InlineData(SessionManager.FaultedReason)]
|
||||
[InlineData(SessionManager.DetachGraceExpiredReason)]
|
||||
public async Task CloseExpiredLeasesAsync_SweptClose_AccountsSessionClosedExactlyOnce(string sweepKind)
|
||||
{
|
||||
FakeWorkerClient workerClient = new();
|
||||
FakeTimeProvider clock = new(DateTimeOffset.UtcNow);
|
||||
using GatewayMetrics metrics = new();
|
||||
SessionManager manager = CreateManager(
|
||||
new FakeSessionWorkerClientFactory(workerClient),
|
||||
metrics: metrics,
|
||||
options: CreateOptions(detachGraceSeconds: 30),
|
||||
timeProvider: clock);
|
||||
GatewaySession session = await manager.OpenSessionAsync(CreateOpenRequest(), "client-1", ownerKeyId: null, CancellationToken.None);
|
||||
Assert.Equal(1, metrics.GetSnapshot().OpenSessions);
|
||||
|
||||
switch (sweepKind)
|
||||
{
|
||||
case SessionManager.LeaseExpiredReason:
|
||||
SetLeaseExpiry(session, clock.GetUtcNow().AddSeconds(-1));
|
||||
break;
|
||||
case SessionManager.FaultedReason:
|
||||
session.MarkFaulted("test-fault");
|
||||
break;
|
||||
case SessionManager.DetachGraceExpiredReason:
|
||||
session.AttachEventSubscriber(maxSubscribers: 1).Dispose();
|
||||
clock.Advance(TimeSpan.FromSeconds(31));
|
||||
break;
|
||||
default:
|
||||
throw new ArgumentOutOfRangeException(nameof(sweepKind), sweepKind, "Unknown sweep kind.");
|
||||
}
|
||||
|
||||
int closedCount = await manager.CloseExpiredLeasesAsync(clock.GetUtcNow(), CancellationToken.None);
|
||||
|
||||
Assert.Equal(1, closedCount);
|
||||
Assert.Equal(SessionState.Closed, session.State);
|
||||
Assert.False(manager.TryGetSession(session.SessionId, out _));
|
||||
GatewayMetricsSnapshot snapshot = metrics.GetSnapshot();
|
||||
Assert.Equal(1, snapshot.SessionsClosed);
|
||||
Assert.Equal(0, snapshot.OpenSessions);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// With <c>UnattachedSessionGraceSeconds</c> set, a session that never attached an event
|
||||
/// subscriber and has been idle for the window is reaped by the ordinary lease sweep even
|
||||
/// though its 30-minute lease is nowhere near expiry. This is the knob for clients that
|
||||
/// open a session and vanish (OtOpcUa's GalaxyDriver on a failed start): without it each
|
||||
/// such open pins an x86 worker for <c>DefaultLeaseSeconds</c>.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task CloseExpiredLeasesAsync_ReapsNeverSubscribedSessionAfterUnattachedGrace()
|
||||
{
|
||||
FakeWorkerClient workerClient = new();
|
||||
FakeTimeProvider clock = new(DateTimeOffset.UtcNow);
|
||||
using GatewayMetrics metrics = new();
|
||||
SessionManager manager = CreateManager(
|
||||
new FakeSessionWorkerClientFactory(workerClient),
|
||||
metrics: metrics,
|
||||
options: CreateOptions(unattachedSessionGraceSeconds: 60),
|
||||
timeProvider: clock);
|
||||
GatewaySession session = await manager.OpenSessionAsync(CreateOpenRequest(), "client-1", ownerKeyId: null, CancellationToken.None);
|
||||
|
||||
clock.Advance(TimeSpan.FromSeconds(59));
|
||||
int closedBefore = await manager.CloseExpiredLeasesAsync(clock.GetUtcNow(), CancellationToken.None);
|
||||
Assert.Equal(0, closedBefore);
|
||||
Assert.Equal(SessionState.Ready, session.State);
|
||||
|
||||
clock.Advance(TimeSpan.FromSeconds(1));
|
||||
int closedAfter = await manager.CloseExpiredLeasesAsync(clock.GetUtcNow(), CancellationToken.None);
|
||||
|
||||
Assert.Equal(1, closedAfter);
|
||||
Assert.Equal(SessionState.Closed, session.State);
|
||||
Assert.Equal(1, workerClient.ShutdownCount);
|
||||
Assert.False(manager.TryGetSession(session.SessionId, out _));
|
||||
Assert.Equal(1, metrics.GetSnapshot().SessionsClosed);
|
||||
Assert.Equal(0, metrics.GetSnapshot().OpenSessions);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The window is an idle timeout, not a hard deadline from open: unary activity restarts
|
||||
/// it, so a client that only ever reads and writes (never streams) is not reaped while it
|
||||
/// keeps calling.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task CloseExpiredLeasesAsync_UnattachedGrace_IsRestartedByUnaryActivity()
|
||||
{
|
||||
FakeWorkerClient workerClient = new();
|
||||
FakeTimeProvider clock = new(DateTimeOffset.UtcNow);
|
||||
SessionManager manager = CreateManager(
|
||||
new FakeSessionWorkerClientFactory(workerClient),
|
||||
options: CreateOptions(unattachedSessionGraceSeconds: 60),
|
||||
timeProvider: clock);
|
||||
GatewaySession session = await manager.OpenSessionAsync(CreateOpenRequest(), "client-1", ownerKeyId: null, CancellationToken.None);
|
||||
|
||||
clock.Advance(TimeSpan.FromSeconds(59));
|
||||
await manager.InvokeAsync(session.SessionId, CreateCommand(MxCommandKind.Ping), CancellationToken.None);
|
||||
clock.Advance(TimeSpan.FromSeconds(59));
|
||||
int closedWhileActive = await manager.CloseExpiredLeasesAsync(clock.GetUtcNow(), CancellationToken.None);
|
||||
Assert.Equal(0, closedWhileActive);
|
||||
Assert.Equal(SessionState.Ready, session.State);
|
||||
|
||||
clock.Advance(TimeSpan.FromSeconds(1));
|
||||
int closedAfterIdle = await manager.CloseExpiredLeasesAsync(clock.GetUtcNow(), CancellationToken.None);
|
||||
|
||||
Assert.Equal(1, closedAfterIdle);
|
||||
Assert.Equal(SessionState.Closed, session.State);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A session that ever had an event subscriber is outside this knob's remit — once it has
|
||||
/// streamed, detach-grace and the normal lease govern it — and a session with a subscriber
|
||||
/// attached right now is never idle.
|
||||
/// </summary>
|
||||
/// <param name="keepSubscriberAttached">Whether the subscriber is still attached at sweep time.</param>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Theory]
|
||||
[InlineData(true)]
|
||||
[InlineData(false)]
|
||||
public async Task CloseExpiredLeasesAsync_UnattachedGrace_DoesNotReapSessionThatSubscribed(bool keepSubscriberAttached)
|
||||
{
|
||||
FakeWorkerClient workerClient = new();
|
||||
FakeTimeProvider clock = new(DateTimeOffset.UtcNow);
|
||||
SessionManager manager = CreateManager(
|
||||
new FakeSessionWorkerClientFactory(workerClient),
|
||||
options: CreateOptions(detachGraceSeconds: 0, unattachedSessionGraceSeconds: 60),
|
||||
timeProvider: clock);
|
||||
GatewaySession session = await manager.OpenSessionAsync(CreateOpenRequest(), "client-1", ownerKeyId: null, CancellationToken.None);
|
||||
IDisposable subscriber = session.AttachEventSubscriber(maxSubscribers: 1);
|
||||
if (!keepSubscriberAttached)
|
||||
{
|
||||
subscriber.Dispose();
|
||||
}
|
||||
|
||||
clock.Advance(TimeSpan.FromSeconds(61));
|
||||
int closedCount = await manager.CloseExpiredLeasesAsync(clock.GetUtcNow(), CancellationToken.None);
|
||||
|
||||
Assert.Equal(0, closedCount);
|
||||
Assert.Equal(SessionState.Ready, session.State);
|
||||
Assert.Equal(0, workerClient.ShutdownCount);
|
||||
subscriber.Dispose();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The gateway's own alarm-monitor session only ever attaches an <em>internal</em>
|
||||
/// (non-counted) subscriber. That must count as "attached" for this rule, or the gateway
|
||||
/// would reap its own monitor every cycle once the knob is on.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task CloseExpiredLeasesAsync_UnattachedGrace_DoesNotReapSessionWithInternalSubscriber()
|
||||
{
|
||||
FakeWorkerClient workerClient = new();
|
||||
FakeTimeProvider clock = new(DateTimeOffset.UtcNow);
|
||||
SessionManager manager = CreateManager(
|
||||
new FakeSessionWorkerClientFactory(workerClient),
|
||||
options: CreateOptions(unattachedSessionGraceSeconds: 60),
|
||||
timeProvider: clock);
|
||||
GatewaySession session = await manager.OpenSessionAsync(CreateOpenRequest(), "client-1", ownerKeyId: null, CancellationToken.None);
|
||||
using IEventSubscriberLease internalLease = session.AttachInternalEventSubscriber();
|
||||
|
||||
clock.Advance(TimeSpan.FromSeconds(61));
|
||||
int closedCount = await manager.CloseExpiredLeasesAsync(clock.GetUtcNow(), CancellationToken.None);
|
||||
|
||||
Assert.Equal(0, closedCount);
|
||||
Assert.Equal(SessionState.Ready, session.State);
|
||||
Assert.Equal(0, workerClient.ShutdownCount);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The idle clock starts when the session becomes Ready, not when it was opened: a worker
|
||||
/// that took longer than the window to start must not hand the client a session the very
|
||||
/// next sweep reaps before it had a chance to subscribe.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task CloseExpiredLeasesAsync_UnattachedGrace_IdleClockStartsAtReady()
|
||||
{
|
||||
FakeWorkerClient workerClient = new();
|
||||
GatedSessionWorkerClientFactory factory = new(workerClient);
|
||||
FakeTimeProvider clock = new(DateTimeOffset.UtcNow);
|
||||
SessionManager manager = CreateManager(
|
||||
factory,
|
||||
options: CreateOptions(unattachedSessionGraceSeconds: 60),
|
||||
timeProvider: clock);
|
||||
|
||||
Task<GatewaySession> openTask = manager.OpenSessionAsync(CreateOpenRequest(), "client-1", ownerKeyId: null, CancellationToken.None);
|
||||
await factory.WaitForCreateStartAsync();
|
||||
clock.Advance(TimeSpan.FromSeconds(120));
|
||||
factory.ReleaseCreate();
|
||||
GatewaySession session = await openTask;
|
||||
|
||||
int closedRightAfterReady = await manager.CloseExpiredLeasesAsync(clock.GetUtcNow(), CancellationToken.None);
|
||||
Assert.Equal(0, closedRightAfterReady);
|
||||
Assert.Equal(SessionState.Ready, session.State);
|
||||
|
||||
clock.Advance(TimeSpan.FromSeconds(60));
|
||||
int closedAfterIdle = await manager.CloseExpiredLeasesAsync(clock.GetUtcNow(), CancellationToken.None);
|
||||
|
||||
Assert.Equal(1, closedAfterIdle);
|
||||
Assert.Equal(SessionState.Closed, session.State);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The knob is off by default: a never-subscribed idle session lives for its full lease.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task CloseExpiredLeasesAsync_UnattachedGrace_IsDisabledByDefault()
|
||||
{
|
||||
FakeWorkerClient workerClient = new();
|
||||
FakeTimeProvider clock = new(DateTimeOffset.UtcNow);
|
||||
SessionManager manager = CreateManager(
|
||||
new FakeSessionWorkerClientFactory(workerClient),
|
||||
options: CreateOptions(),
|
||||
timeProvider: clock);
|
||||
GatewaySession session = await manager.OpenSessionAsync(CreateOpenRequest(), "client-1", ownerKeyId: null, CancellationToken.None);
|
||||
|
||||
clock.Advance(TimeSpan.FromMinutes(20));
|
||||
int closedCount = await manager.CloseExpiredLeasesAsync(clock.GetUtcNow(), CancellationToken.None);
|
||||
|
||||
Assert.Equal(0, closedCount);
|
||||
Assert.Equal(SessionState.Ready, session.State);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that an expired-lease sweep leaves a session with an active event subscriber open.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
@@ -1011,7 +1486,7 @@ public sealed class SessionManagerTests
|
||||
SessionManager manager = CreateManager(new FakeSessionWorkerClientFactory(workerClient));
|
||||
GatewaySession session = await manager.OpenSessionAsync(CreateOpenRequest(), "client-1", ownerKeyId: null, CancellationToken.None);
|
||||
DateTimeOffset now = DateTimeOffset.UtcNow;
|
||||
session.ExtendLease(now.AddSeconds(-1));
|
||||
SetLeaseExpiry(session, now.AddSeconds(-1));
|
||||
using IDisposable eventSubscriber = session.AttachEventSubscriber(maxSubscribers: 1);
|
||||
|
||||
int closedCount = await manager.CloseExpiredLeasesAsync(now, CancellationToken.None);
|
||||
@@ -1118,8 +1593,8 @@ public sealed class SessionManagerTests
|
||||
DateTimeOffset now = DateTimeOffset.UtcNow;
|
||||
|
||||
// Both leases are far in the future, so only the fault can reap the faulted session.
|
||||
faultedSession.ExtendLease(now.AddMinutes(30));
|
||||
healthySession.ExtendLease(now.AddMinutes(30));
|
||||
SetLeaseExpiry(faultedSession, now.AddMinutes(30));
|
||||
SetLeaseExpiry(healthySession, now.AddMinutes(30));
|
||||
faultedSession.MarkFaulted("test fault");
|
||||
|
||||
int closedCount = await manager.CloseExpiredLeasesAsync(now, CancellationToken.None);
|
||||
@@ -1180,8 +1655,8 @@ public sealed class SessionManagerTests
|
||||
GatewaySession firstSession = await manager.OpenSessionAsync(CreateOpenRequest(), "client-1", ownerKeyId: null, CancellationToken.None);
|
||||
GatewaySession secondSession = await manager.OpenSessionAsync(CreateOpenRequest(), "client-2", ownerKeyId: null, CancellationToken.None);
|
||||
DateTimeOffset now = DateTimeOffset.UtcNow;
|
||||
firstSession.ExtendLease(now.AddSeconds(-1));
|
||||
secondSession.ExtendLease(now.AddSeconds(-1));
|
||||
SetLeaseExpiry(firstSession, now.AddSeconds(-1));
|
||||
SetLeaseExpiry(secondSession, now.AddSeconds(-1));
|
||||
|
||||
int closedCount = await manager.CloseExpiredLeasesAsync(now, CancellationToken.None);
|
||||
|
||||
@@ -1233,8 +1708,8 @@ public sealed class SessionManagerTests
|
||||
GatewaySession failingSession = await manager.OpenSessionAsync(CreateOpenRequest(), "client-1", ownerKeyId: null, CancellationToken.None);
|
||||
GatewaySession healthySession = await manager.OpenSessionAsync(CreateOpenRequest(), "client-2", ownerKeyId: null, CancellationToken.None);
|
||||
DateTimeOffset now = DateTimeOffset.UtcNow;
|
||||
failingSession.ExtendLease(now.AddSeconds(-1));
|
||||
healthySession.ExtendLease(now.AddSeconds(-1));
|
||||
SetLeaseExpiry(failingSession, now.AddSeconds(-1));
|
||||
SetLeaseExpiry(healthySession, now.AddSeconds(-1));
|
||||
|
||||
SessionManagerException exception = await Assert.ThrowsAsync<SessionManagerException>(
|
||||
async () => await manager.CloseExpiredLeasesAsync(now, CancellationToken.None));
|
||||
@@ -1273,6 +1748,84 @@ public sealed class SessionManagerTests
|
||||
Assert.False(manager.TryGetSession(secondSession.SessionId, out _));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Once the drain has started, a new open must be refused with <c>GatewayStopping</c>
|
||||
/// rather than accepted: a session created behind the drain's snapshot has nobody left to
|
||||
/// close it, and its worker would outlive the gateway as an orphan. The refusal is not an
|
||||
/// open failure — no slot is taken, nothing is registered and no fault is recorded.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task OpenSessionAsync_AfterShutdownStarted_ThrowsGatewayStopping()
|
||||
{
|
||||
FakeWorkerClient workerClient = new();
|
||||
SessionRegistry registry = new();
|
||||
using GatewayMetrics metrics = new();
|
||||
SessionManager manager = CreateManager(new FakeSessionWorkerClientFactory(workerClient), registry, metrics);
|
||||
await manager.ShutdownAsync(CancellationToken.None);
|
||||
|
||||
SessionManagerException exception = await Assert.ThrowsAsync<SessionManagerException>(
|
||||
async () => await manager.OpenSessionAsync(CreateOpenRequest(), "client-1", ownerKeyId: null, CancellationToken.None));
|
||||
|
||||
Assert.Equal(SessionManagerErrorCode.GatewayStopping, exception.ErrorCode);
|
||||
Assert.Equal(0, registry.Count);
|
||||
Assert.Equal(0, metrics.GetSnapshot().SessionsOpened);
|
||||
Assert.Equal(0, metrics.GetSnapshot().Faults);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The host signals <c>ApplicationStopping</c> before hosted services stop, and Kestrel keeps
|
||||
/// accepting gRPC calls until its own <c>StopAsync</c> runs — so the gate must close on the
|
||||
/// lifetime signal, not only when <c>ShutdownAsync</c> is finally called.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task OpenSessionAsync_WhenApplicationStopping_ThrowsGatewayStopping()
|
||||
{
|
||||
FakeWorkerClient workerClient = new();
|
||||
using FakeHostApplicationLifetime lifetime = new();
|
||||
SessionManager manager = new(
|
||||
new SessionRegistry(),
|
||||
new FakeSessionWorkerClientFactory(workerClient),
|
||||
Options.Create(CreateOptions()),
|
||||
new GatewayMetrics(),
|
||||
applicationLifetime: lifetime);
|
||||
|
||||
GatewaySession beforeStopping = await manager.OpenSessionAsync(CreateOpenRequest(), "client-1", ownerKeyId: null, CancellationToken.None);
|
||||
Assert.Equal(SessionState.Ready, beforeStopping.State);
|
||||
|
||||
lifetime.SignalStopping();
|
||||
SessionManagerException exception = await Assert.ThrowsAsync<SessionManagerException>(
|
||||
async () => await manager.OpenSessionAsync(CreateOpenRequest(), "client-2", ownerKeyId: null, CancellationToken.None));
|
||||
|
||||
Assert.Equal(SessionManagerErrorCode.GatewayStopping, exception.ErrorCode);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// An open that passed the stopping gate but registered its session after the drain took
|
||||
/// its snapshot is still caught: the drain re-snapshots until the registry is empty, so the
|
||||
/// late session is closed by the drain rather than left running.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task ShutdownAsync_DrainsSessionRegisteredAfterFirstSnapshot()
|
||||
{
|
||||
FakeWorkerClient workerClient = new();
|
||||
LateFirstSnapshotRegistry registry = new();
|
||||
using GatewayMetrics metrics = new();
|
||||
SessionManager manager = CreateManager(new FakeSessionWorkerClientFactory(workerClient), registry, metrics);
|
||||
GatewaySession session = await manager.OpenSessionAsync(CreateOpenRequest(), "client-1", ownerKeyId: null, CancellationToken.None);
|
||||
|
||||
await manager.ShutdownAsync(CancellationToken.None);
|
||||
|
||||
Assert.True(registry.SnapshotCalls >= 2);
|
||||
Assert.Equal(SessionState.Closed, session.State);
|
||||
Assert.Equal(1, workerClient.ShutdownCount);
|
||||
Assert.Equal(0, registry.Count);
|
||||
Assert.Equal(0, metrics.GetSnapshot().OpenSessions);
|
||||
Assert.Equal(1, metrics.GetSnapshot().SessionsClosed);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that shutdown closes all registered sessions.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
@@ -1322,7 +1875,8 @@ public sealed class SessionManagerTests
|
||||
int defaultLeaseSeconds = 1800,
|
||||
int detachGraceSeconds = 0,
|
||||
int workerReadyWaitTimeoutMs = 0,
|
||||
int faultedGraceSeconds = 0)
|
||||
int faultedGraceSeconds = 0,
|
||||
int unattachedSessionGraceSeconds = 0)
|
||||
{
|
||||
return new GatewayOptions
|
||||
{
|
||||
@@ -1334,6 +1888,7 @@ public sealed class SessionManagerTests
|
||||
DetachGraceSeconds = detachGraceSeconds,
|
||||
WorkerReadyWaitTimeoutMs = workerReadyWaitTimeoutMs,
|
||||
FaultedGraceSeconds = faultedGraceSeconds,
|
||||
UnattachedSessionGraceSeconds = unattachedSessionGraceSeconds,
|
||||
},
|
||||
Worker = new WorkerOptions
|
||||
{
|
||||
@@ -1343,6 +1898,17 @@ public sealed class SessionManagerTests
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Moves a session's lease expiry by back-dating its last client activity, which is the only
|
||||
/// production path that moves the lease (<c>_leaseExpiresAt = activityAt + LeaseDuration</c>).
|
||||
/// </summary>
|
||||
/// <param name="session">Session whose lease to move.</param>
|
||||
/// <param name="leaseExpiresAt">Wanted lease expiry.</param>
|
||||
private static void SetLeaseExpiry(GatewaySession session, DateTimeOffset leaseExpiresAt)
|
||||
{
|
||||
session.TouchClientActivity(leaseExpiresAt - session.LeaseDuration);
|
||||
}
|
||||
|
||||
private static SessionOpenRequest CreateOpenRequest()
|
||||
{
|
||||
return new SessionOpenRequest(
|
||||
@@ -1422,6 +1988,203 @@ public sealed class SessionManagerTests
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Factory that records a per-session event metric for the session it is starting and
|
||||
/// then fails, so a test can check the open rollback clears that metric.
|
||||
/// </summary>
|
||||
private sealed class EventRecordingFailingSessionWorkerClientFactory(GatewayMetrics metrics) : ISessionWorkerClientFactory
|
||||
{
|
||||
/// <summary>Gets the id of the session whose worker creation was attempted.</summary>
|
||||
public string? StartedSessionId { get; private set; }
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task<IWorkerClient> CreateAsync(
|
||||
GatewaySession session,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
StartedSessionId = session.SessionId;
|
||||
metrics.EventReceived(session.SessionId, MxEventFamily.OnDataChange.ToString());
|
||||
throw new InvalidOperationException("worker startup failed");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Factory whose <see cref="CreateAsync"/> parks until the test releases it, so the test can
|
||||
/// act on the registered-but-still-starting session (kill it, drain the manager) inside the
|
||||
/// window between registry insertion and worker attach. Once released it hands the same
|
||||
/// client to every later call.
|
||||
/// </summary>
|
||||
private sealed class GatedSessionWorkerClientFactory(IWorkerClient workerClient) : ISessionWorkerClientFactory
|
||||
{
|
||||
private readonly TaskCompletionSource<GatewaySession> _entered = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
private readonly TaskCompletionSource _release = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
|
||||
/// <summary>Waits until the manager has asked for a worker and returns the starting session.</summary>
|
||||
/// <returns>The session whose worker is being created.</returns>
|
||||
public Task<GatewaySession> WaitForCreateStartAsync()
|
||||
{
|
||||
return _entered.Task.WaitAsync(TimeSpan.FromSeconds(5));
|
||||
}
|
||||
|
||||
/// <summary>Lets the parked worker creation complete.</summary>
|
||||
public void ReleaseCreate()
|
||||
{
|
||||
_release.TrySetResult();
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<IWorkerClient> CreateAsync(
|
||||
GatewaySession session,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
// Deliberately not bound to the caller's token: the tests that cancel the open while
|
||||
// the worker is starting need the worker to still ARRIVE after the cancellation, which
|
||||
// is the orphan-prone case.
|
||||
_entered.TrySetResult(session);
|
||||
await _release.Task.WaitAsync(TimeSpan.FromSeconds(5));
|
||||
return workerClient;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Registry that can replay a race the real registry only produces by timing: a session a
|
||||
/// concurrent close has already removed and disposed is still returned by one lookup (the
|
||||
/// kill looked it up just before the removal) or still appears in one snapshot (the sweep
|
||||
/// snapshotted just before the removal).
|
||||
/// </summary>
|
||||
private sealed class StaleRegistry : ISessionRegistry
|
||||
{
|
||||
private readonly SessionRegistry _inner = new();
|
||||
private GatewaySession? _staleLookup;
|
||||
private GatewaySession? _staleSnapshot;
|
||||
|
||||
/// <summary>Makes the next lookup of <paramref name="session"/>'s id return it although it was removed.</summary>
|
||||
/// <param name="session">Removed session to serve once.</param>
|
||||
public void ServeStaleLookupOnce(GatewaySession session)
|
||||
{
|
||||
_staleLookup = session;
|
||||
}
|
||||
|
||||
/// <summary>Makes the next snapshot include <paramref name="session"/> although it was removed.</summary>
|
||||
/// <param name="session">Removed session to include once.</param>
|
||||
public void ServeStaleSnapshot(GatewaySession session)
|
||||
{
|
||||
_staleSnapshot = session;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public int Count => _inner.Count;
|
||||
|
||||
/// <inheritdoc />
|
||||
public int ActiveCount => _inner.ActiveCount;
|
||||
|
||||
/// <inheritdoc />
|
||||
public bool TryAdd(GatewaySession session) => _inner.TryAdd(session);
|
||||
|
||||
/// <inheritdoc />
|
||||
public bool TryGet(string sessionId, [System.Diagnostics.CodeAnalysis.MaybeNullWhen(false)] out GatewaySession session)
|
||||
{
|
||||
if (_staleLookup is not null && _staleLookup.SessionId == sessionId)
|
||||
{
|
||||
session = _staleLookup;
|
||||
_staleLookup = null;
|
||||
return true;
|
||||
}
|
||||
|
||||
return _inner.TryGet(sessionId, out session);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public bool TryRemove(string sessionId, [System.Diagnostics.CodeAnalysis.MaybeNullWhen(false)] out GatewaySession session)
|
||||
=> _inner.TryRemove(sessionId, out session);
|
||||
|
||||
/// <inheritdoc />
|
||||
public IReadOnlyCollection<GatewaySession> Snapshot()
|
||||
{
|
||||
if (_staleSnapshot is null)
|
||||
{
|
||||
return _inner.Snapshot();
|
||||
}
|
||||
|
||||
GatewaySession[] snapshot = [.. _inner.Snapshot(), _staleSnapshot];
|
||||
_staleSnapshot = null;
|
||||
return snapshot;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Lifetime fake whose <see cref="ApplicationStopping"/> the test fires on demand; the
|
||||
/// other two tokens never fire.
|
||||
/// </summary>
|
||||
private sealed class FakeHostApplicationLifetime : Microsoft.Extensions.Hosting.IHostApplicationLifetime, IDisposable
|
||||
{
|
||||
private readonly CancellationTokenSource _stopping = new();
|
||||
|
||||
/// <inheritdoc />
|
||||
public CancellationToken ApplicationStarted => CancellationToken.None;
|
||||
|
||||
/// <inheritdoc />
|
||||
public CancellationToken ApplicationStopping => _stopping.Token;
|
||||
|
||||
/// <inheritdoc />
|
||||
public CancellationToken ApplicationStopped => CancellationToken.None;
|
||||
|
||||
/// <summary>Fires <see cref="ApplicationStopping"/>.</summary>
|
||||
public void SignalStopping()
|
||||
{
|
||||
_stopping.Cancel();
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public void StopApplication()
|
||||
{
|
||||
SignalStopping();
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public void Dispose()
|
||||
{
|
||||
_stopping.Dispose();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Registry whose first <see cref="Snapshot"/> is empty regardless of its contents,
|
||||
/// simulating a session whose <c>TryAdd</c> landed just after the drain's first snapshot;
|
||||
/// every later snapshot is the real one.
|
||||
/// </summary>
|
||||
private sealed class LateFirstSnapshotRegistry : ISessionRegistry
|
||||
{
|
||||
private readonly SessionRegistry _inner = new();
|
||||
|
||||
/// <summary>Gets how many snapshots have been taken.</summary>
|
||||
public int SnapshotCalls { get; private set; }
|
||||
|
||||
/// <inheritdoc />
|
||||
public int Count => _inner.Count;
|
||||
|
||||
/// <inheritdoc />
|
||||
public int ActiveCount => _inner.ActiveCount;
|
||||
|
||||
/// <inheritdoc />
|
||||
public bool TryAdd(GatewaySession session) => _inner.TryAdd(session);
|
||||
|
||||
/// <inheritdoc />
|
||||
public bool TryGet(string sessionId, [System.Diagnostics.CodeAnalysis.MaybeNullWhen(false)] out GatewaySession session)
|
||||
=> _inner.TryGet(sessionId, out session);
|
||||
|
||||
/// <inheritdoc />
|
||||
public bool TryRemove(string sessionId, [System.Diagnostics.CodeAnalysis.MaybeNullWhen(false)] out GatewaySession session)
|
||||
=> _inner.TryRemove(sessionId, out session);
|
||||
|
||||
/// <inheritdoc />
|
||||
public IReadOnlyCollection<GatewaySession> Snapshot()
|
||||
{
|
||||
SnapshotCalls++;
|
||||
return SnapshotCalls == 1 ? [] : _inner.Snapshot();
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class FakeWorkerClient : IWorkerClient
|
||||
{
|
||||
private WorkerClientState _state = WorkerClientState.Ready;
|
||||
|
||||
Reference in New Issue
Block a user