fix(sessions): exception-total shutdown body with non-cancellable kill fallback
This commit is contained in:
@@ -68,6 +68,12 @@ public interface ISessionManager
|
||||
/// <param name="now">The current time to evaluate expiration against.</param>
|
||||
/// <param name="cancellationToken">Token to cancel the asynchronous operation.</param>
|
||||
/// <returns>The number of sessions closed.</returns>
|
||||
/// <remarks>
|
||||
/// A close that fails does not abandon the rest of the pass: every session selected by this
|
||||
/// sweep is attempted, and the first failure is then rethrown so the caller still observes
|
||||
/// (and logs) that the sweep failed. Which failure surfaces is nondeterministic when several
|
||||
/// closes fail in the same pass, because the closes run concurrently.
|
||||
/// </remarks>
|
||||
Task<int> CloseExpiredLeasesAsync(
|
||||
DateTimeOffset now,
|
||||
CancellationToken cancellationToken);
|
||||
|
||||
@@ -356,18 +356,28 @@ public sealed class SessionManager : ISessionManager
|
||||
// tail to the orphan killer. Per-session exclusivity comes from GatewaySession.CloseAsync's
|
||||
// own close gate, and each iteration touches only its own session plus thread-safe
|
||||
// registry/metrics state.
|
||||
//
|
||||
// The body must be exception-TOTAL. Parallel.ForEachAsync cancels the token it hands the
|
||||
// sibling bodies as soon as one body throws, so a single escaping exception would abort up
|
||||
// to MaxParallelSessionCloses - 1 in-flight graceful shutdowns AND make their kill fallback
|
||||
// throw immediately on the freshly cancelled token — sessions neither closed nor killed,
|
||||
// i.e. leaked x86 workers that nothing reattaches to (a gateway restart terminates orphans
|
||||
// rather than adopting them).
|
||||
//
|
||||
// For the same reason the loop itself is NOT bound to cancellationToken: a cancelled
|
||||
// ParallelOptions token stops dispatching the remaining sessions entirely, whereas the
|
||||
// sequential drain this replaced let every remaining session fail its graceful close fast
|
||||
// and still kill its worker. The token is passed to the graceful close instead, which
|
||||
// preserves that behavior — a host stop deadline turns the drain into a kill sweep rather
|
||||
// than into a leak.
|
||||
await Parallel.ForEachAsync(
|
||||
_registry.Snapshot(),
|
||||
new ParallelOptions
|
||||
{
|
||||
MaxDegreeOfParallelism = MaxParallelSessionCloses,
|
||||
CancellationToken = cancellationToken,
|
||||
},
|
||||
async (session, closeToken) =>
|
||||
new ParallelOptions { MaxDegreeOfParallelism = MaxParallelSessionCloses },
|
||||
async (session, _) =>
|
||||
{
|
||||
try
|
||||
{
|
||||
await CloseSessionCoreAsync(session, GatewayShutdownReason, closeToken).ConfigureAwait(false);
|
||||
await CloseSessionCoreAsync(session, GatewayShutdownReason, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
@@ -376,13 +386,19 @@ public sealed class SessionManager : ISessionManager
|
||||
"Graceful shutdown failed for session {SessionId}; killing worker.",
|
||||
session.SessionId);
|
||||
|
||||
if (_registry.TryGet(session.SessionId, out _))
|
||||
if (_registry.TryGet(session.SessionId, out GatewaySession? registeredSession)
|
||||
&& registeredSession is not null)
|
||||
{
|
||||
try
|
||||
{
|
||||
await KillWorkerAsync(session.SessionId, GatewayShutdownReason, closeToken).ConfigureAwait(false);
|
||||
// Deliberately NOT the caller's token: the kill is the last-resort orphan
|
||||
// preventer, so it must still run when the host stop deadline (or a
|
||||
// sibling body's failure) has already cancelled the drain. It is a
|
||||
// synchronous Kill plus registry/dispose bookkeeping, not a wait on
|
||||
// the worker, so it cannot extend the drain meaningfully.
|
||||
await KillWorkerAsync(session.SessionId, GatewayShutdownReason, CancellationToken.None).ConfigureAwait(false);
|
||||
}
|
||||
catch (SessionManagerException killException)
|
||||
catch (Exception killException)
|
||||
{
|
||||
_logger.LogWarning(
|
||||
killException,
|
||||
|
||||
Reference in New Issue
Block a user