dohertj2/mxaccessgw
1
0
Fork 0
Code Issues 3 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

fix(sessions): report a close whose kill fallback succeeded as Closed, not CloseFailed

Browse Source 
CloseAsync rethrew the graceful-shutdown exception after a successful
Kill, so the manager faulted the session, recorded a CloseFailed fault and
surfaced CloseFailed to the caller for the ordinary hung-worker case.
docs/Sessions.md reserves CloseFailed for both shutdown and kill failing.

A successful kill now completes the close as Closed; the shutdown failure
rides back on SessionCloseResult.WorkerShutdownFailure and the manager
logs it at warning level. Both-fail still raises CloseFailed.
This commit is contained in:
Joseph Doherty
2026-10-22 06:26:33 -04:00
parent 37ab629d70
commit 2b165cd3a4
5 changed files with 111 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ZB.MOM.WW.MxGateway.Tests/Gateway/Sessions/SessionManagerTests.cs
+61 -15
View File
@@ -641,10 +641,16 @@ public sealed class SessionManagerTests
Assert.Equal(0, metrics.GetSnapshot().OpenSessions);
}
/// <summary>Verifies that closing a session kills the worker when shutdown fails.</summary>
/// <summary>
/// A worker that does not shut down gracefully is killed, and that is a <em>successful</em>
/// close: the kill is the documented fallback, the worker is gone and the session is
/// <c>Closed</c>. Reporting it as <c>CloseFailed</c> + <c>Faulted</c> + a fault metric
/// misreported every hung-worker close as a teardown failure. <c>CloseFailed</c> is reserved
/// for both the shutdown and the kill failing.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task CloseSessionAsync_WhenWorkerShutdownFails_KillsWorker()
public async Task CloseSessionAsync_WhenWorkerShutdownFails_KillsWorkerAndReportsClosed()
{
FakeWorkerClient workerClient = new()
{
@@ -652,15 +658,24 @@ public sealed class SessionManagerTests
WorkerClientErrorCode.ShutdownTimeout,
"Worker shutdown timed out."),
};
SessionManager manager = CreateManager(new FakeSessionWorkerClientFactory(workerClient));
using GatewayMetrics metrics = new();
SessionManager manager = CreateManager(new FakeSessionWorkerClientFactory(workerClient), metrics: metrics);
GatewaySession session = await manager.OpenSessionAsync(CreateOpenRequest(), "client-1", ownerKeyId: null, CancellationToken.None);
SessionManagerException exception = await Assert.ThrowsAsync<SessionManagerException>(
async () => await manager.CloseSessionAsync(session.SessionId, CancellationToken.None));
SessionCloseResult result = await manager.CloseSessionAsync(session.SessionId, CancellationToken.None);
Assert.Equal(SessionManagerErrorCode.CloseFailed, exception.ErrorCode);
Assert.Equal(SessionState.Closed, result.FinalState);
Assert.False(result.AlreadyClosed);
Assert.Same(workerClient.ShutdownException, result.WorkerShutdownFailure);
Assert.Equal(SessionState.Closed, session.State);
Assert.Null(session.FinalFault);
Assert.Equal(1, workerClient.ShutdownCount);
Assert.Equal(1, workerClient.KillCount);
Assert.False(manager.TryGetSession(session.SessionId, out _));
GatewayMetricsSnapshot snapshot = metrics.GetSnapshot();
Assert.Equal(0, snapshot.Faults);
Assert.Equal(1, snapshot.SessionsClosed);
Assert.Equal(0, snapshot.OpenSessions);
}
/// <summary>Verifies that when worker shutdown fails, the session is removed and the slot is released.</summary>
@@ -689,31 +704,59 @@ public sealed class SessionManagerTests
CancellationToken.None);
metrics.EventReceived(firstSession.SessionId, MxEventFamily.OnDataChange.ToString());
SessionManagerException exception = await Assert.ThrowsAsync<SessionManagerException>(
async () => await manager.CloseSessionAsync(firstSession.SessionId, CancellationToken.None));
SessionCloseResult result = await manager.CloseSessionAsync(firstSession.SessionId, CancellationToken.None);
GatewaySession secondSession = await manager.OpenSessionAsync(
CreateOpenRequest(),
"client-2",
ownerKeyId: null,
CancellationToken.None);
Assert.Equal(SessionManagerErrorCode.CloseFailed, exception.ErrorCode);
Assert.Equal(SessionState.Closed, result.FinalState);
Assert.False(manager.TryGetSession(firstSession.SessionId, out _));
Assert.True(manager.TryGetSession(secondSession.SessionId, out _));
Assert.Equal(1, registry.Count);
Assert.Equal(1, failingWorkerClient.KillCount);
Assert.Equal(1, failingWorkerClient.DisposeCount);
GatewayMetricsSnapshot snapshot = metrics.GetSnapshot();
// A close-that-failed accounts as SessionClosed (counter += 1) rather
// than SessionRemoved (gauge -= 1, counter unchanged). The session is
// being removed from the registry on this path, so it must show up in
// the closed count.
Assert.Equal(1, snapshot.SessionsClosed);
Assert.False(snapshot.EventsBySession.ContainsKey(firstSession.SessionId));
Assert.Equal(1, snapshot.OpenSessions);
}
/// <summary>
/// Only when the graceful shutdown AND the kill fallback both fail is the close a failure:
/// the session is faulted, removed, counted as closed (it is leaving the registry) and the
/// caller gets <c>CloseFailed</c>.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task CloseSessionAsync_WhenShutdownAndKillBothFail_ThrowsCloseFailed()
{
FakeWorkerClient workerClient = new()
{
ShutdownException = new WorkerClientException(
WorkerClientErrorCode.ShutdownTimeout,
"Worker shutdown timed out."),
KillException = new InvalidOperationException("worker kill failed"),
};
using GatewayMetrics metrics = new();
SessionManager manager = CreateManager(new FakeSessionWorkerClientFactory(workerClient), metrics: metrics);
GatewaySession session = await manager.OpenSessionAsync(CreateOpenRequest(), "client-1", ownerKeyId: null, CancellationToken.None);
SessionManagerException exception = await Assert.ThrowsAsync<SessionManagerException>(
async () => await manager.CloseSessionAsync(session.SessionId, CancellationToken.None));
Assert.Equal(SessionManagerErrorCode.CloseFailed, exception.ErrorCode);
Assert.Equal(SessionState.Faulted, session.State);
Assert.Equal(1, workerClient.ShutdownCount);
Assert.Equal(1, workerClient.KillCount);
Assert.False(manager.TryGetSession(session.SessionId, out _));
GatewayMetricsSnapshot snapshot = metrics.GetSnapshot();
Assert.Equal(1, snapshot.Faults);
Assert.Equal(1, snapshot.SessionsClosed);
Assert.Equal(0, snapshot.OpenSessions);
}
/// <summary>Verifies that when the second close is canceled, the session is not removed if owned by the first close.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
@@ -926,7 +969,9 @@ public sealed class SessionManagerTests
/// <c>KillWorker</c> + <c>RemoveSessionAsync</c> when <c>CloseSessionCoreAsync</c> throws)
/// must still account a successful close: both the open-session gauge must drop to zero AND
/// the <c>mxgateway.sessions.closed</c> counter must increment. Without the fix, the
/// graceful-close failure path under-counts the closed counter.
/// graceful-close failure path under-counts the closed counter. Both the shutdown and the
/// in-close kill fail here, because a kill that succeeds is a clean close and never reaches
/// the fallback.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
@@ -935,6 +980,7 @@ public sealed class SessionManagerTests
FakeWorkerClient throwingClient = new()
{
ShutdownException = new InvalidOperationException("worker shutdown failed"),
KillException = new InvalidOperationException("worker kill failed"),
};
using GatewayMetrics metrics = new();
SessionManager manager = CreateManager(