fix(sessions): report a close whose kill fallback succeeded as Closed, not CloseFailed

CloseAsync rethrew the graceful-shutdown exception after a successful
Kill, so the manager faulted the session, recorded a CloseFailed fault and
surfaced CloseFailed to the caller for the ordinary hung-worker case.
docs/Sessions.md reserves CloseFailed for both shutdown and kill failing.

A successful kill now completes the close as Closed; the shutdown failure
rides back on SessionCloseResult.WorkerShutdownFailure and the manager
logs it at warning level. Both-fail still raises CloseFailed.
This commit is contained in:
Joseph Doherty
2026-10-22 06:26:33 -04:00
parent 37ab629d70
commit 2b165cd3a4
5 changed files with 111 additions and 25 deletions
@@ -1731,6 +1731,7 @@ public sealed class GatewaySession
return new SessionCloseResult(SessionId, SessionState.Closed, AlreadyClosed: true);
}
Exception? workerShutdownFailure = null;
if (_workerClient is not null)
{
try
@@ -1739,6 +1740,12 @@ public sealed class GatewaySession
}
catch (Exception exception)
{
// The kill is the documented fallback for a worker that will not shut down
// (typically a ShutdownTimeout on a hung worker). When it succeeds the
// worker is gone and this close has done its job, so it completes as a
// normal close carrying the shutdown failure for the caller to log; only a
// kill that ALSO fails is a close failure (SessionCloseStartedException →
// CloseFailed), because then a worker may still be running.
try
{
_workerClient.Kill(reason);
@@ -1750,12 +1757,12 @@ public sealed class GatewaySession
new AggregateException(exception, killException));
}
throw;
workerShutdownFailure = exception;
}
}
MarkClosed();
return new SessionCloseResult(SessionId, SessionState.Closed, alreadyClosing);
return new SessionCloseResult(SessionId, SessionState.Closed, alreadyClosing, workerShutdownFailure);
}
catch (Exception exception) when (exception is not SessionCloseStartedException)
{
@@ -2,7 +2,21 @@ using ZB.MOM.WW.MxGateway.Contracts.Proto;
namespace ZB.MOM.WW.MxGateway.Server.Sessions;
/// <summary>Outcome of closing a session.</summary>
/// <param name="SessionId">Identifier of the closed session.</param>
/// <param name="FinalState">State the session ended in.</param>
/// <param name="AlreadyClosed">
/// <see langword="true"/> when the session was already closed (or being closed by another
/// caller) when this close acquired the close lock, so this call did no teardown of its own.
/// </param>
/// <param name="WorkerShutdownFailure">
/// The exception the worker's graceful shutdown threw when the close had to fall back to
/// killing the worker; <see langword="null"/> when the worker shut down cleanly. The close is
/// still a success — the worker is gone and the session is closed — so this is diagnostic
/// detail for the caller's log, not an error.
/// </param>
public sealed record SessionCloseResult(
string SessionId,
SessionState FinalState,
bool AlreadyClosed);
bool AlreadyClosed,
Exception? WorkerShutdownFailure = null);
@@ -565,6 +565,18 @@ public sealed class SessionManager : ISessionManager
try
{
SessionCloseResult result = await session.CloseAsync(reason, cancellationToken).ConfigureAwait(false);
if (result.WorkerShutdownFailure is not null)
{
// A clean close from the session's point of view (the worker was killed and the
// session is Closed), but the operator should see that the worker would not stop
// gracefully — a run of these is the signal that workers are hanging on shutdown.
_logger.LogWarning(
result.WorkerShutdownFailure,
"Worker for session {SessionId} did not shut down gracefully and was killed; reason={Reason}.",
session.SessionId,
reason);
}
if (!wasClosed && !result.AlreadyClosed && session.OpenRecorded)
{
_metrics.SessionClosed();