fix(sessions): exception-total shutdown body with non-cancellable kill fallback

This commit is contained in:
Joseph Doherty
2026-08-15 13:19:08 -04:00
parent 0bc13b5292
commit 95f8ba918d
4 changed files with 86 additions and 24 deletions
@@ -68,6 +68,12 @@ public interface ISessionManager
/// <param name="now">The current time to evaluate expiration against.</param>
/// <param name="cancellationToken">Token to cancel the asynchronous operation.</param>
/// <returns>The number of sessions closed.</returns>
/// <remarks>
/// A close that fails does not abandon the rest of the pass: every session selected by this
/// sweep is attempted, and the first failure is then rethrown so the caller still observes
/// (and logs) that the sweep failed. Which failure surfaces is nondeterministic when several
/// closes fail in the same pass, because the closes run concurrently.
/// </remarks>
Task<int> CloseExpiredLeasesAsync(
DateTimeOffset now,
CancellationToken cancellationToken);
@@ -356,18 +356,28 @@ public sealed class SessionManager : ISessionManager
// tail to the orphan killer. Per-session exclusivity comes from GatewaySession.CloseAsync's
// own close gate, and each iteration touches only its own session plus thread-safe
// registry/metrics state.
//
// The body must be exception-TOTAL. Parallel.ForEachAsync cancels the token it hands the
// sibling bodies as soon as one body throws, so a single escaping exception would abort up
// to MaxParallelSessionCloses - 1 in-flight graceful shutdowns AND make their kill fallback
// throw immediately on the freshly cancelled token — sessions neither closed nor killed,
// i.e. leaked x86 workers that nothing reattaches to (a gateway restart terminates orphans
// rather than adopting them).
//
// For the same reason the loop itself is NOT bound to cancellationToken: a cancelled
// ParallelOptions token stops dispatching the remaining sessions entirely, whereas the
// sequential drain this replaced let every remaining session fail its graceful close fast
// and still kill its worker. The token is passed to the graceful close instead, which
// preserves that behavior — a host stop deadline turns the drain into a kill sweep rather
// than into a leak.
await Parallel.ForEachAsync(
_registry.Snapshot(),
new ParallelOptions
{
MaxDegreeOfParallelism = MaxParallelSessionCloses,
CancellationToken = cancellationToken,
},
async (session, closeToken) =>
new ParallelOptions { MaxDegreeOfParallelism = MaxParallelSessionCloses },
async (session, _) =>
{
try
{
await CloseSessionCoreAsync(session, GatewayShutdownReason, closeToken).ConfigureAwait(false);
await CloseSessionCoreAsync(session, GatewayShutdownReason, cancellationToken).ConfigureAwait(false);
}
catch (Exception exception)
{
@@ -376,13 +386,19 @@ public sealed class SessionManager : ISessionManager
"Graceful shutdown failed for session {SessionId}; killing worker.",
session.SessionId);
if (_registry.TryGet(session.SessionId, out _))
if (_registry.TryGet(session.SessionId, out GatewaySession? registeredSession)
&& registeredSession is not null)
{
try
{
await KillWorkerAsync(session.SessionId, GatewayShutdownReason, closeToken).ConfigureAwait(false);
// Deliberately NOT the caller's token: the kill is the last-resort orphan
// preventer, so it must still run when the host stop deadline (or a
// sibling body's failure) has already cancelled the drain. It is a
// synchronous Kill plus registry/dispose bookkeeping, not a wait on
// the worker, so it cannot extend the drain meaningfully.
await KillWorkerAsync(session.SessionId, GatewayShutdownReason, CancellationToken.None).ConfigureAwait(false);
}
catch (SessionManagerException killException)
catch (Exception killException)
{
_logger.LogWarning(
killException,