|
|
|
@@ -641,10 +641,16 @@ public sealed class SessionManagerTests
|
|
|
|
|
Assert.Equal(0, metrics.GetSnapshot().OpenSessions);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// <summary>Verifies that closing a session kills the worker when shutdown fails.</summary>
|
|
|
|
|
/// <summary>
|
|
|
|
|
/// A worker that does not shut down gracefully is killed, and that is a <em>successful</em>
|
|
|
|
|
/// close: the kill is the documented fallback, the worker is gone and the session is
|
|
|
|
|
/// <c>Closed</c>. Reporting it as <c>CloseFailed</c> + <c>Faulted</c> + a fault metric
|
|
|
|
|
/// misreported every hung-worker close as a teardown failure. <c>CloseFailed</c> is reserved
|
|
|
|
|
/// for both the shutdown and the kill failing.
|
|
|
|
|
/// </summary>
|
|
|
|
|
/// <returns>A task that represents the asynchronous operation.</returns>
|
|
|
|
|
[Fact]
|
|
|
|
|
public async Task CloseSessionAsync_WhenWorkerShutdownFails_KillsWorker()
|
|
|
|
|
public async Task CloseSessionAsync_WhenWorkerShutdownFails_KillsWorkerAndReportsClosed()
|
|
|
|
|
{
|
|
|
|
|
FakeWorkerClient workerClient = new()
|
|
|
|
|
{
|
|
|
|
@@ -652,15 +658,24 @@ public sealed class SessionManagerTests
|
|
|
|
|
WorkerClientErrorCode.ShutdownTimeout,
|
|
|
|
|
"Worker shutdown timed out."),
|
|
|
|
|
};
|
|
|
|
|
SessionManager manager = CreateManager(new FakeSessionWorkerClientFactory(workerClient));
|
|
|
|
|
using GatewayMetrics metrics = new();
|
|
|
|
|
SessionManager manager = CreateManager(new FakeSessionWorkerClientFactory(workerClient), metrics: metrics);
|
|
|
|
|
GatewaySession session = await manager.OpenSessionAsync(CreateOpenRequest(), "client-1", ownerKeyId: null, CancellationToken.None);
|
|
|
|
|
|
|
|
|
|
SessionManagerException exception = await Assert.ThrowsAsync<SessionManagerException>(
|
|
|
|
|
async () => await manager.CloseSessionAsync(session.SessionId, CancellationToken.None));
|
|
|
|
|
SessionCloseResult result = await manager.CloseSessionAsync(session.SessionId, CancellationToken.None);
|
|
|
|
|
|
|
|
|
|
Assert.Equal(SessionManagerErrorCode.CloseFailed, exception.ErrorCode);
|
|
|
|
|
Assert.Equal(SessionState.Closed, result.FinalState);
|
|
|
|
|
Assert.False(result.AlreadyClosed);
|
|
|
|
|
Assert.Same(workerClient.ShutdownException, result.WorkerShutdownFailure);
|
|
|
|
|
Assert.Equal(SessionState.Closed, session.State);
|
|
|
|
|
Assert.Null(session.FinalFault);
|
|
|
|
|
Assert.Equal(1, workerClient.ShutdownCount);
|
|
|
|
|
Assert.Equal(1, workerClient.KillCount);
|
|
|
|
|
Assert.False(manager.TryGetSession(session.SessionId, out _));
|
|
|
|
|
GatewayMetricsSnapshot snapshot = metrics.GetSnapshot();
|
|
|
|
|
Assert.Equal(0, snapshot.Faults);
|
|
|
|
|
Assert.Equal(1, snapshot.SessionsClosed);
|
|
|
|
|
Assert.Equal(0, snapshot.OpenSessions);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// <summary>Verifies that when worker shutdown fails, the session is removed and the slot is released.</summary>
|
|
|
|
@@ -689,31 +704,59 @@ public sealed class SessionManagerTests
|
|
|
|
|
CancellationToken.None);
|
|
|
|
|
metrics.EventReceived(firstSession.SessionId, MxEventFamily.OnDataChange.ToString());
|
|
|
|
|
|
|
|
|
|
SessionManagerException exception = await Assert.ThrowsAsync<SessionManagerException>(
|
|
|
|
|
async () => await manager.CloseSessionAsync(firstSession.SessionId, CancellationToken.None));
|
|
|
|
|
SessionCloseResult result = await manager.CloseSessionAsync(firstSession.SessionId, CancellationToken.None);
|
|
|
|
|
GatewaySession secondSession = await manager.OpenSessionAsync(
|
|
|
|
|
CreateOpenRequest(),
|
|
|
|
|
"client-2",
|
|
|
|
|
ownerKeyId: null,
|
|
|
|
|
CancellationToken.None);
|
|
|
|
|
|
|
|
|
|
Assert.Equal(SessionManagerErrorCode.CloseFailed, exception.ErrorCode);
|
|
|
|
|
Assert.Equal(SessionState.Closed, result.FinalState);
|
|
|
|
|
Assert.False(manager.TryGetSession(firstSession.SessionId, out _));
|
|
|
|
|
Assert.True(manager.TryGetSession(secondSession.SessionId, out _));
|
|
|
|
|
Assert.Equal(1, registry.Count);
|
|
|
|
|
Assert.Equal(1, failingWorkerClient.KillCount);
|
|
|
|
|
Assert.Equal(1, failingWorkerClient.DisposeCount);
|
|
|
|
|
GatewayMetricsSnapshot snapshot = metrics.GetSnapshot();
|
|
|
|
|
|
|
|
|
|
// A close-that-failed accounts as SessionClosed (counter += 1) rather
|
|
|
|
|
// than SessionRemoved (gauge -= 1, counter unchanged). The session is
|
|
|
|
|
// being removed from the registry on this path, so it must show up in
|
|
|
|
|
// the closed count.
|
|
|
|
|
Assert.Equal(1, snapshot.SessionsClosed);
|
|
|
|
|
Assert.False(snapshot.EventsBySession.ContainsKey(firstSession.SessionId));
|
|
|
|
|
Assert.Equal(1, snapshot.OpenSessions);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// <summary>
|
|
|
|
|
/// Only when the graceful shutdown AND the kill fallback both fail is the close a failure:
|
|
|
|
|
/// the session is faulted, removed, counted as closed (it is leaving the registry) and the
|
|
|
|
|
/// caller gets <c>CloseFailed</c>.
|
|
|
|
|
/// </summary>
|
|
|
|
|
/// <returns>A task that represents the asynchronous operation.</returns>
|
|
|
|
|
[Fact]
|
|
|
|
|
public async Task CloseSessionAsync_WhenShutdownAndKillBothFail_ThrowsCloseFailed()
|
|
|
|
|
{
|
|
|
|
|
FakeWorkerClient workerClient = new()
|
|
|
|
|
{
|
|
|
|
|
ShutdownException = new WorkerClientException(
|
|
|
|
|
WorkerClientErrorCode.ShutdownTimeout,
|
|
|
|
|
"Worker shutdown timed out."),
|
|
|
|
|
KillException = new InvalidOperationException("worker kill failed"),
|
|
|
|
|
};
|
|
|
|
|
using GatewayMetrics metrics = new();
|
|
|
|
|
SessionManager manager = CreateManager(new FakeSessionWorkerClientFactory(workerClient), metrics: metrics);
|
|
|
|
|
GatewaySession session = await manager.OpenSessionAsync(CreateOpenRequest(), "client-1", ownerKeyId: null, CancellationToken.None);
|
|
|
|
|
|
|
|
|
|
SessionManagerException exception = await Assert.ThrowsAsync<SessionManagerException>(
|
|
|
|
|
async () => await manager.CloseSessionAsync(session.SessionId, CancellationToken.None));
|
|
|
|
|
|
|
|
|
|
Assert.Equal(SessionManagerErrorCode.CloseFailed, exception.ErrorCode);
|
|
|
|
|
Assert.Equal(SessionState.Faulted, session.State);
|
|
|
|
|
Assert.Equal(1, workerClient.ShutdownCount);
|
|
|
|
|
Assert.Equal(1, workerClient.KillCount);
|
|
|
|
|
Assert.False(manager.TryGetSession(session.SessionId, out _));
|
|
|
|
|
GatewayMetricsSnapshot snapshot = metrics.GetSnapshot();
|
|
|
|
|
Assert.Equal(1, snapshot.Faults);
|
|
|
|
|
Assert.Equal(1, snapshot.SessionsClosed);
|
|
|
|
|
Assert.Equal(0, snapshot.OpenSessions);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// <summary>Verifies that when the second close is canceled, the session is not removed if owned by the first close.</summary>
|
|
|
|
|
/// <returns>A task that represents the asynchronous operation.</returns>
|
|
|
|
|
[Fact]
|
|
|
|
@@ -881,6 +924,71 @@ public sealed class SessionManagerTests
|
|
|
|
|
Assert.True(metrics.GetSnapshot().Faults > 0);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// <summary>
|
|
|
|
|
/// A kill whose session lookup succeeded but whose close gate is disposed by the time it
|
|
|
|
|
/// waits on it lost the race to a close that already removed, disposed and accounted the
|
|
|
|
|
/// session. It must report <c>AlreadyClosed</c> and touch nothing: the old catch recorded a
|
|
|
|
|
/// fault, faulted the session and decremented the open gauge a second time (visible here as
|
|
|
|
|
/// the other open session's slot disappearing from the gauge).
|
|
|
|
|
/// </summary>
|
|
|
|
|
/// <returns>A task that represents the asynchronous operation.</returns>
|
|
|
|
|
[Fact]
|
|
|
|
|
public async Task KillWorkerAsync_WhenCloseWonTheRace_ReportsAlreadyClosedWithoutDoubleAccounting()
|
|
|
|
|
{
|
|
|
|
|
FakeWorkerClient racedClient = new();
|
|
|
|
|
FakeWorkerClient otherClient = new();
|
|
|
|
|
StaleRegistry registry = new();
|
|
|
|
|
using GatewayMetrics metrics = new();
|
|
|
|
|
SessionManager manager = CreateManager(new QueueingSessionWorkerClientFactory(racedClient, otherClient), registry, metrics);
|
|
|
|
|
GatewaySession raced = await manager.OpenSessionAsync(CreateOpenRequest(), "client-1", ownerKeyId: null, CancellationToken.None);
|
|
|
|
|
await manager.OpenSessionAsync(CreateOpenRequest(), "client-2", ownerKeyId: null, CancellationToken.None);
|
|
|
|
|
|
|
|
|
|
await manager.CloseSessionAsync(raced.SessionId, CancellationToken.None);
|
|
|
|
|
Assert.Equal(1, metrics.GetSnapshot().OpenSessions);
|
|
|
|
|
|
|
|
|
|
// The kill looked the session up just before the close removed and disposed it.
|
|
|
|
|
registry.ServeStaleLookupOnce(raced);
|
|
|
|
|
SessionCloseResult result = await manager.KillWorkerAsync(raced.SessionId, "admin-kill", CancellationToken.None);
|
|
|
|
|
|
|
|
|
|
Assert.True(result.AlreadyClosed);
|
|
|
|
|
Assert.Equal(SessionState.Closed, result.FinalState);
|
|
|
|
|
Assert.Equal(0, racedClient.KillCount);
|
|
|
|
|
GatewayMetricsSnapshot snapshot = metrics.GetSnapshot();
|
|
|
|
|
Assert.Equal(1, snapshot.OpenSessions);
|
|
|
|
|
Assert.Equal(1, snapshot.SessionsClosed);
|
|
|
|
|
Assert.Equal(0, snapshot.Faults);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// <summary>
|
|
|
|
|
/// The sweep's counterpart of the kill race: a session that a concurrent close removed and
|
|
|
|
|
/// disposed between the snapshot and the close call is simply skipped — it is not a sweep
|
|
|
|
|
/// failure to log, and it is not counted as closed by the sweep.
|
|
|
|
|
/// </summary>
|
|
|
|
|
/// <returns>A task that represents the asynchronous operation.</returns>
|
|
|
|
|
[Fact]
|
|
|
|
|
public async Task CloseExpiredLeasesAsync_WhenCloseWonTheRace_DoesNotReportSweepFailure()
|
|
|
|
|
{
|
|
|
|
|
FakeWorkerClient workerClient = new();
|
|
|
|
|
StaleRegistry registry = new();
|
|
|
|
|
using GatewayMetrics metrics = new();
|
|
|
|
|
SessionManager manager = CreateManager(new FakeSessionWorkerClientFactory(workerClient), registry, metrics);
|
|
|
|
|
GatewaySession session = await manager.OpenSessionAsync(CreateOpenRequest(), "client-1", ownerKeyId: null, CancellationToken.None);
|
|
|
|
|
DateTimeOffset now = DateTimeOffset.UtcNow;
|
|
|
|
|
SetLeaseExpiry(session, now.AddSeconds(-1));
|
|
|
|
|
|
|
|
|
|
await manager.CloseSessionAsync(session.SessionId, CancellationToken.None);
|
|
|
|
|
registry.ServeStaleSnapshot(session);
|
|
|
|
|
|
|
|
|
|
int closedCount = await manager.CloseExpiredLeasesAsync(now, CancellationToken.None);
|
|
|
|
|
|
|
|
|
|
Assert.Equal(0, closedCount);
|
|
|
|
|
Assert.Equal(1, workerClient.ShutdownCount);
|
|
|
|
|
GatewayMetricsSnapshot snapshot = metrics.GetSnapshot();
|
|
|
|
|
Assert.Equal(1, snapshot.SessionsClosed);
|
|
|
|
|
Assert.Equal(0, snapshot.OpenSessions);
|
|
|
|
|
Assert.Equal(0, snapshot.Faults);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// <summary>
|
|
|
|
|
/// Concurrent kills on the same session must not
|
|
|
|
|
/// double-increment <c>mxgateway.sessions.closed</c>. The first kill wins, the second
|
|
|
|
@@ -926,7 +1034,9 @@ public sealed class SessionManagerTests
|
|
|
|
|
/// <c>KillWorker</c> + <c>RemoveSessionAsync</c> when <c>CloseSessionCoreAsync</c> throws)
|
|
|
|
|
/// must still account a successful close: both the open-session gauge must drop to zero AND
|
|
|
|
|
/// the <c>mxgateway.sessions.closed</c> counter must increment. Without the fix, the
|
|
|
|
|
/// graceful-close failure path under-counts the closed counter.
|
|
|
|
|
/// graceful-close failure path under-counts the closed counter. Both the shutdown and the
|
|
|
|
|
/// in-close kill fail here, because a kill that succeeds is a clean close and never reaches
|
|
|
|
|
/// the fallback.
|
|
|
|
|
/// </summary>
|
|
|
|
|
/// <returns>A task that represents the asynchronous operation.</returns>
|
|
|
|
|
[Fact]
|
|
|
|
@@ -935,6 +1045,7 @@ public sealed class SessionManagerTests
|
|
|
|
|
FakeWorkerClient throwingClient = new()
|
|
|
|
|
{
|
|
|
|
|
ShutdownException = new InvalidOperationException("worker shutdown failed"),
|
|
|
|
|
KillException = new InvalidOperationException("worker kill failed"),
|
|
|
|
|
};
|
|
|
|
|
using GatewayMetrics metrics = new();
|
|
|
|
|
SessionManager manager = CreateManager(
|
|
|
|
@@ -978,6 +1089,133 @@ public sealed class SessionManagerTests
|
|
|
|
|
Assert.Equal(1, metrics.GetSnapshot().Faults);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// <summary>
|
|
|
|
|
/// A failed open must leave no per-session event metrics behind: the rollback goes through
|
|
|
|
|
/// the same removal path every other teardown uses, so <c>EventsBySession</c> is cleared
|
|
|
|
|
/// for the session that never opened.
|
|
|
|
|
/// </summary>
|
|
|
|
|
/// <returns>A task that represents the asynchronous operation.</returns>
|
|
|
|
|
[Fact]
|
|
|
|
|
public async Task OpenSessionAsync_WhenWorkerCreationFails_RemovesSessionEventMetrics()
|
|
|
|
|
{
|
|
|
|
|
using GatewayMetrics metrics = new();
|
|
|
|
|
EventRecordingFailingSessionWorkerClientFactory factory = new(metrics);
|
|
|
|
|
SessionManager manager = CreateManager(factory, metrics: metrics);
|
|
|
|
|
|
|
|
|
|
await Assert.ThrowsAsync<SessionManagerException>(
|
|
|
|
|
async () => await manager.OpenSessionAsync(CreateOpenRequest(), "client-1", ownerKeyId: null, CancellationToken.None));
|
|
|
|
|
|
|
|
|
|
Assert.NotNull(factory.StartedSessionId);
|
|
|
|
|
Assert.False(metrics.GetSnapshot().EventsBySession.ContainsKey(factory.StartedSessionId!));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// <summary>
|
|
|
|
|
/// The session is registered before its worker is created, so an admin Kill can land while
|
|
|
|
|
/// the worker is still starting (up to <c>StartupTimeoutSeconds</c>). The late worker client
|
|
|
|
|
/// must not be attached to the already-closed session and handed back to the caller as if
|
|
|
|
|
/// the open had succeeded: it is killed and disposed, the open fails with
|
|
|
|
|
/// <c>OpenFailed</c>, and nothing is left in the registry or the metrics.
|
|
|
|
|
/// </summary>
|
|
|
|
|
/// <returns>A task that represents the asynchronous operation.</returns>
|
|
|
|
|
[Fact]
|
|
|
|
|
public async Task OpenSessionAsync_WhenKilledDuringWorkerStartup_FailsOpenAndDiscardsLateWorkerClient()
|
|
|
|
|
{
|
|
|
|
|
FakeWorkerClient workerClient = new();
|
|
|
|
|
GatedSessionWorkerClientFactory factory = new(workerClient);
|
|
|
|
|
SessionRegistry registry = new();
|
|
|
|
|
using GatewayMetrics metrics = new();
|
|
|
|
|
SessionManager manager = CreateManager(factory, registry, metrics, CreateOptions(maxSessions: 1));
|
|
|
|
|
|
|
|
|
|
Task<GatewaySession> openTask = manager.OpenSessionAsync(CreateOpenRequest(), "client-1", ownerKeyId: null, CancellationToken.None);
|
|
|
|
|
GatewaySession starting = await factory.WaitForCreateStartAsync();
|
|
|
|
|
|
|
|
|
|
SessionCloseResult kill = await manager.KillWorkerAsync(starting.SessionId, "admin-kill", CancellationToken.None);
|
|
|
|
|
factory.ReleaseCreate();
|
|
|
|
|
SessionManagerException exception = await Assert.ThrowsAsync<SessionManagerException>(async () => await openTask);
|
|
|
|
|
|
|
|
|
|
Assert.Equal(SessionState.Closed, kill.FinalState);
|
|
|
|
|
Assert.Equal(SessionManagerErrorCode.OpenFailed, exception.ErrorCode);
|
|
|
|
|
Assert.Equal(1, workerClient.KillCount);
|
|
|
|
|
Assert.Equal(1, workerClient.DisposeCount);
|
|
|
|
|
Assert.Null(starting.WorkerClient);
|
|
|
|
|
Assert.Equal(0, registry.Count);
|
|
|
|
|
GatewayMetricsSnapshot snapshot = metrics.GetSnapshot();
|
|
|
|
|
Assert.Equal(0, snapshot.OpenSessions);
|
|
|
|
|
Assert.Equal(0, snapshot.SessionsOpened);
|
|
|
|
|
Assert.Equal(0, snapshot.SessionsClosed);
|
|
|
|
|
|
|
|
|
|
// The slot the killed open held must have been released exactly once: a replacement open
|
|
|
|
|
// succeeds against MaxSessions = 1 (the released gate hands the same fake straight back).
|
|
|
|
|
GatewaySession reopened = await manager.OpenSessionAsync(CreateOpenRequest(), "client-2", ownerKeyId: null, CancellationToken.None);
|
|
|
|
|
Assert.Equal(SessionState.Ready, reopened.State);
|
|
|
|
|
Assert.Equal(1, registry.Count);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// <summary>
|
|
|
|
|
/// A caller that abandons its open (deadline, disconnect) while the worker is starting
|
|
|
|
|
/// must not leave the worker running: whatever fails between the worker arriving and the
|
|
|
|
|
/// session becoming Ready — here the caller's own cancellation — the late worker client is
|
|
|
|
|
/// killed and disposed before the failure surfaces.
|
|
|
|
|
/// </summary>
|
|
|
|
|
/// <returns>A task that represents the asynchronous operation.</returns>
|
|
|
|
|
[Fact]
|
|
|
|
|
public async Task OpenSessionAsync_WhenCallerCancelledDuringWorkerStartup_DiscardsWorkerClient()
|
|
|
|
|
{
|
|
|
|
|
FakeWorkerClient workerClient = new();
|
|
|
|
|
GatedSessionWorkerClientFactory factory = new(workerClient);
|
|
|
|
|
SessionRegistry registry = new();
|
|
|
|
|
using GatewayMetrics metrics = new();
|
|
|
|
|
SessionManager manager = CreateManager(factory, registry, metrics);
|
|
|
|
|
using CancellationTokenSource cancellation = new();
|
|
|
|
|
|
|
|
|
|
Task<GatewaySession> openTask = manager.OpenSessionAsync(CreateOpenRequest(), "client-1", ownerKeyId: null, cancellation.Token);
|
|
|
|
|
GatewaySession starting = await factory.WaitForCreateStartAsync();
|
|
|
|
|
|
|
|
|
|
await cancellation.CancelAsync();
|
|
|
|
|
factory.ReleaseCreate();
|
|
|
|
|
SessionManagerException exception = await Assert.ThrowsAsync<SessionManagerException>(async () => await openTask);
|
|
|
|
|
|
|
|
|
|
Assert.Equal(SessionManagerErrorCode.OpenFailed, exception.ErrorCode);
|
|
|
|
|
Assert.IsAssignableFrom<OperationCanceledException>(exception.InnerException);
|
|
|
|
|
Assert.Equal(1, workerClient.KillCount);
|
|
|
|
|
Assert.Equal(1, workerClient.DisposeCount);
|
|
|
|
|
Assert.NotEqual(SessionState.Ready, starting.State);
|
|
|
|
|
Assert.Equal(0, registry.Count);
|
|
|
|
|
Assert.Equal(0, metrics.GetSnapshot().OpenSessions);
|
|
|
|
|
Assert.Equal(0, metrics.GetSnapshot().SessionsOpened);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// <summary>
|
|
|
|
|
/// Same window as the kill case, driven by the host drain: a session still starting its
|
|
|
|
|
/// worker when <c>ShutdownAsync</c> runs is closed by the drain, and the worker client that
|
|
|
|
|
/// arrives afterwards is discarded rather than left running as an orphan.
|
|
|
|
|
/// </summary>
|
|
|
|
|
/// <returns>A task that represents the asynchronous operation.</returns>
|
|
|
|
|
[Fact]
|
|
|
|
|
public async Task OpenSessionAsync_WhenShutdownDuringWorkerStartup_FailsOpenAndDiscardsLateWorkerClient()
|
|
|
|
|
{
|
|
|
|
|
FakeWorkerClient workerClient = new();
|
|
|
|
|
GatedSessionWorkerClientFactory factory = new(workerClient);
|
|
|
|
|
SessionRegistry registry = new();
|
|
|
|
|
using GatewayMetrics metrics = new();
|
|
|
|
|
SessionManager manager = CreateManager(factory, registry, metrics);
|
|
|
|
|
|
|
|
|
|
Task<GatewaySession> openTask = manager.OpenSessionAsync(CreateOpenRequest(), "client-1", ownerKeyId: null, CancellationToken.None);
|
|
|
|
|
GatewaySession starting = await factory.WaitForCreateStartAsync();
|
|
|
|
|
|
|
|
|
|
await manager.ShutdownAsync(CancellationToken.None);
|
|
|
|
|
factory.ReleaseCreate();
|
|
|
|
|
SessionManagerException exception = await Assert.ThrowsAsync<SessionManagerException>(async () => await openTask);
|
|
|
|
|
|
|
|
|
|
Assert.Equal(SessionManagerErrorCode.OpenFailed, exception.ErrorCode);
|
|
|
|
|
Assert.Equal(SessionState.Closed, starting.State);
|
|
|
|
|
Assert.Equal(1, workerClient.KillCount);
|
|
|
|
|
Assert.Equal(1, workerClient.DisposeCount);
|
|
|
|
|
Assert.Equal(0, registry.Count);
|
|
|
|
|
Assert.Equal(0, metrics.GetSnapshot().OpenSessions);
|
|
|
|
|
Assert.Equal(0, metrics.GetSnapshot().SessionsClosed);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// <summary>Verifies that closing expired leases only closes expired sessions.</summary>
|
|
|
|
|
/// <returns>A task that represents the asynchronous operation.</returns>
|
|
|
|
|
[Fact]
|
|
|
|
@@ -990,8 +1228,8 @@ public sealed class SessionManagerTests
|
|
|
|
|
GatewaySession expiredSession = await manager.OpenSessionAsync(CreateOpenRequest(), "client-1", ownerKeyId: null, CancellationToken.None);
|
|
|
|
|
GatewaySession activeSession = await manager.OpenSessionAsync(CreateOpenRequest(), "client-2", ownerKeyId: null, CancellationToken.None);
|
|
|
|
|
DateTimeOffset now = DateTimeOffset.UtcNow;
|
|
|
|
|
expiredSession.ExtendLease(now.AddSeconds(-1));
|
|
|
|
|
activeSession.ExtendLease(now.AddMinutes(5));
|
|
|
|
|
SetLeaseExpiry(expiredSession, now.AddSeconds(-1));
|
|
|
|
|
SetLeaseExpiry(activeSession, now.AddMinutes(5));
|
|
|
|
|
|
|
|
|
|
int closedCount = await manager.CloseExpiredLeasesAsync(now, CancellationToken.None);
|
|
|
|
|
|
|
|
|
@@ -1002,6 +1240,243 @@ public sealed class SessionManagerTests
|
|
|
|
|
Assert.Equal(0, activeClient.ShutdownCount);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// <summary>
|
|
|
|
|
/// Every sweep-driven close — lease expiry, faulted reap and detach-grace expiry — must
|
|
|
|
|
/// account <c>SessionClosed</c> exactly once. The sweep begins the close itself
|
|
|
|
|
/// (<c>TryBeginCloseIfExpired</c>) before delegating to the shared close path, and that
|
|
|
|
|
/// reservation must not make the delegated close look like a second, already-running close
|
|
|
|
|
/// whose accounting is skipped: the open-session gauge would then never drop for swept
|
|
|
|
|
/// sessions and the closed counter would under-count them.
|
|
|
|
|
/// </summary>
|
|
|
|
|
/// <param name="sweepKind">Which sweep condition selects the session.</param>
|
|
|
|
|
/// <returns>A task that represents the asynchronous operation.</returns>
|
|
|
|
|
[Theory]
|
|
|
|
|
[InlineData(SessionManager.LeaseExpiredReason)]
|
|
|
|
|
[InlineData(SessionManager.FaultedReason)]
|
|
|
|
|
[InlineData(SessionManager.DetachGraceExpiredReason)]
|
|
|
|
|
public async Task CloseExpiredLeasesAsync_SweptClose_AccountsSessionClosedExactlyOnce(string sweepKind)
|
|
|
|
|
{
|
|
|
|
|
FakeWorkerClient workerClient = new();
|
|
|
|
|
FakeTimeProvider clock = new(DateTimeOffset.UtcNow);
|
|
|
|
|
using GatewayMetrics metrics = new();
|
|
|
|
|
SessionManager manager = CreateManager(
|
|
|
|
|
new FakeSessionWorkerClientFactory(workerClient),
|
|
|
|
|
metrics: metrics,
|
|
|
|
|
options: CreateOptions(detachGraceSeconds: 30),
|
|
|
|
|
timeProvider: clock);
|
|
|
|
|
GatewaySession session = await manager.OpenSessionAsync(CreateOpenRequest(), "client-1", ownerKeyId: null, CancellationToken.None);
|
|
|
|
|
Assert.Equal(1, metrics.GetSnapshot().OpenSessions);
|
|
|
|
|
|
|
|
|
|
switch (sweepKind)
|
|
|
|
|
{
|
|
|
|
|
case SessionManager.LeaseExpiredReason:
|
|
|
|
|
SetLeaseExpiry(session, clock.GetUtcNow().AddSeconds(-1));
|
|
|
|
|
break;
|
|
|
|
|
case SessionManager.FaultedReason:
|
|
|
|
|
session.MarkFaulted("test-fault");
|
|
|
|
|
break;
|
|
|
|
|
case SessionManager.DetachGraceExpiredReason:
|
|
|
|
|
session.AttachEventSubscriber(maxSubscribers: 1).Dispose();
|
|
|
|
|
clock.Advance(TimeSpan.FromSeconds(31));
|
|
|
|
|
break;
|
|
|
|
|
default:
|
|
|
|
|
throw new ArgumentOutOfRangeException(nameof(sweepKind), sweepKind, "Unknown sweep kind.");
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
int closedCount = await manager.CloseExpiredLeasesAsync(clock.GetUtcNow(), CancellationToken.None);
|
|
|
|
|
|
|
|
|
|
Assert.Equal(1, closedCount);
|
|
|
|
|
Assert.Equal(SessionState.Closed, session.State);
|
|
|
|
|
Assert.False(manager.TryGetSession(session.SessionId, out _));
|
|
|
|
|
GatewayMetricsSnapshot snapshot = metrics.GetSnapshot();
|
|
|
|
|
Assert.Equal(1, snapshot.SessionsClosed);
|
|
|
|
|
Assert.Equal(0, snapshot.OpenSessions);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// <summary>
|
|
|
|
|
/// With <c>UnattachedSessionGraceSeconds</c> set, a session that never attached an event
|
|
|
|
|
/// subscriber and has been idle for the window is reaped by the ordinary lease sweep even
|
|
|
|
|
/// though its 30-minute lease is nowhere near expiry. This is the knob for clients that
|
|
|
|
|
/// open a session and vanish (OtOpcUa's GalaxyDriver on a failed start): without it each
|
|
|
|
|
/// such open pins an x86 worker for <c>DefaultLeaseSeconds</c>.
|
|
|
|
|
/// </summary>
|
|
|
|
|
/// <returns>A task that represents the asynchronous operation.</returns>
|
|
|
|
|
[Fact]
|
|
|
|
|
public async Task CloseExpiredLeasesAsync_ReapsNeverSubscribedSessionAfterUnattachedGrace()
|
|
|
|
|
{
|
|
|
|
|
FakeWorkerClient workerClient = new();
|
|
|
|
|
FakeTimeProvider clock = new(DateTimeOffset.UtcNow);
|
|
|
|
|
using GatewayMetrics metrics = new();
|
|
|
|
|
SessionManager manager = CreateManager(
|
|
|
|
|
new FakeSessionWorkerClientFactory(workerClient),
|
|
|
|
|
metrics: metrics,
|
|
|
|
|
options: CreateOptions(unattachedSessionGraceSeconds: 60),
|
|
|
|
|
timeProvider: clock);
|
|
|
|
|
GatewaySession session = await manager.OpenSessionAsync(CreateOpenRequest(), "client-1", ownerKeyId: null, CancellationToken.None);
|
|
|
|
|
|
|
|
|
|
clock.Advance(TimeSpan.FromSeconds(59));
|
|
|
|
|
int closedBefore = await manager.CloseExpiredLeasesAsync(clock.GetUtcNow(), CancellationToken.None);
|
|
|
|
|
Assert.Equal(0, closedBefore);
|
|
|
|
|
Assert.Equal(SessionState.Ready, session.State);
|
|
|
|
|
|
|
|
|
|
clock.Advance(TimeSpan.FromSeconds(1));
|
|
|
|
|
int closedAfter = await manager.CloseExpiredLeasesAsync(clock.GetUtcNow(), CancellationToken.None);
|
|
|
|
|
|
|
|
|
|
Assert.Equal(1, closedAfter);
|
|
|
|
|
Assert.Equal(SessionState.Closed, session.State);
|
|
|
|
|
Assert.Equal(1, workerClient.ShutdownCount);
|
|
|
|
|
Assert.False(manager.TryGetSession(session.SessionId, out _));
|
|
|
|
|
Assert.Equal(1, metrics.GetSnapshot().SessionsClosed);
|
|
|
|
|
Assert.Equal(0, metrics.GetSnapshot().OpenSessions);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// <summary>
|
|
|
|
|
/// The window is an idle timeout, not a hard deadline from open: unary activity restarts
|
|
|
|
|
/// it, so a client that only ever reads and writes (never streams) is not reaped while it
|
|
|
|
|
/// keeps calling.
|
|
|
|
|
/// </summary>
|
|
|
|
|
/// <returns>A task that represents the asynchronous operation.</returns>
|
|
|
|
|
[Fact]
|
|
|
|
|
public async Task CloseExpiredLeasesAsync_UnattachedGrace_IsRestartedByUnaryActivity()
|
|
|
|
|
{
|
|
|
|
|
FakeWorkerClient workerClient = new();
|
|
|
|
|
FakeTimeProvider clock = new(DateTimeOffset.UtcNow);
|
|
|
|
|
SessionManager manager = CreateManager(
|
|
|
|
|
new FakeSessionWorkerClientFactory(workerClient),
|
|
|
|
|
options: CreateOptions(unattachedSessionGraceSeconds: 60),
|
|
|
|
|
timeProvider: clock);
|
|
|
|
|
GatewaySession session = await manager.OpenSessionAsync(CreateOpenRequest(), "client-1", ownerKeyId: null, CancellationToken.None);
|
|
|
|
|
|
|
|
|
|
clock.Advance(TimeSpan.FromSeconds(59));
|
|
|
|
|
await manager.InvokeAsync(session.SessionId, CreateCommand(MxCommandKind.Ping), CancellationToken.None);
|
|
|
|
|
clock.Advance(TimeSpan.FromSeconds(59));
|
|
|
|
|
int closedWhileActive = await manager.CloseExpiredLeasesAsync(clock.GetUtcNow(), CancellationToken.None);
|
|
|
|
|
Assert.Equal(0, closedWhileActive);
|
|
|
|
|
Assert.Equal(SessionState.Ready, session.State);
|
|
|
|
|
|
|
|
|
|
clock.Advance(TimeSpan.FromSeconds(1));
|
|
|
|
|
int closedAfterIdle = await manager.CloseExpiredLeasesAsync(clock.GetUtcNow(), CancellationToken.None);
|
|
|
|
|
|
|
|
|
|
Assert.Equal(1, closedAfterIdle);
|
|
|
|
|
Assert.Equal(SessionState.Closed, session.State);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// <summary>
|
|
|
|
|
/// A session that ever had an event subscriber is outside this knob's remit — once it has
|
|
|
|
|
/// streamed, detach-grace and the normal lease govern it — and a session with a subscriber
|
|
|
|
|
/// attached right now is never idle.
|
|
|
|
|
/// </summary>
|
|
|
|
|
/// <param name="keepSubscriberAttached">Whether the subscriber is still attached at sweep time.</param>
|
|
|
|
|
/// <returns>A task that represents the asynchronous operation.</returns>
|
|
|
|
|
[Theory]
|
|
|
|
|
[InlineData(true)]
|
|
|
|
|
[InlineData(false)]
|
|
|
|
|
public async Task CloseExpiredLeasesAsync_UnattachedGrace_DoesNotReapSessionThatSubscribed(bool keepSubscriberAttached)
|
|
|
|
|
{
|
|
|
|
|
FakeWorkerClient workerClient = new();
|
|
|
|
|
FakeTimeProvider clock = new(DateTimeOffset.UtcNow);
|
|
|
|
|
SessionManager manager = CreateManager(
|
|
|
|
|
new FakeSessionWorkerClientFactory(workerClient),
|
|
|
|
|
options: CreateOptions(detachGraceSeconds: 0, unattachedSessionGraceSeconds: 60),
|
|
|
|
|
timeProvider: clock);
|
|
|
|
|
GatewaySession session = await manager.OpenSessionAsync(CreateOpenRequest(), "client-1", ownerKeyId: null, CancellationToken.None);
|
|
|
|
|
IDisposable subscriber = session.AttachEventSubscriber(maxSubscribers: 1);
|
|
|
|
|
if (!keepSubscriberAttached)
|
|
|
|
|
{
|
|
|
|
|
subscriber.Dispose();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
clock.Advance(TimeSpan.FromSeconds(61));
|
|
|
|
|
int closedCount = await manager.CloseExpiredLeasesAsync(clock.GetUtcNow(), CancellationToken.None);
|
|
|
|
|
|
|
|
|
|
Assert.Equal(0, closedCount);
|
|
|
|
|
Assert.Equal(SessionState.Ready, session.State);
|
|
|
|
|
Assert.Equal(0, workerClient.ShutdownCount);
|
|
|
|
|
subscriber.Dispose();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// <summary>
|
|
|
|
|
/// The gateway's own alarm-monitor session only ever attaches an <em>internal</em>
|
|
|
|
|
/// (non-counted) subscriber. That must count as "attached" for this rule, or the gateway
|
|
|
|
|
/// would reap its own monitor every cycle once the knob is on.
|
|
|
|
|
/// </summary>
|
|
|
|
|
/// <returns>A task that represents the asynchronous operation.</returns>
|
|
|
|
|
[Fact]
|
|
|
|
|
public async Task CloseExpiredLeasesAsync_UnattachedGrace_DoesNotReapSessionWithInternalSubscriber()
|
|
|
|
|
{
|
|
|
|
|
FakeWorkerClient workerClient = new();
|
|
|
|
|
FakeTimeProvider clock = new(DateTimeOffset.UtcNow);
|
|
|
|
|
SessionManager manager = CreateManager(
|
|
|
|
|
new FakeSessionWorkerClientFactory(workerClient),
|
|
|
|
|
options: CreateOptions(unattachedSessionGraceSeconds: 60),
|
|
|
|
|
timeProvider: clock);
|
|
|
|
|
GatewaySession session = await manager.OpenSessionAsync(CreateOpenRequest(), "client-1", ownerKeyId: null, CancellationToken.None);
|
|
|
|
|
using IEventSubscriberLease internalLease = session.AttachInternalEventSubscriber();
|
|
|
|
|
|
|
|
|
|
clock.Advance(TimeSpan.FromSeconds(61));
|
|
|
|
|
int closedCount = await manager.CloseExpiredLeasesAsync(clock.GetUtcNow(), CancellationToken.None);
|
|
|
|
|
|
|
|
|
|
Assert.Equal(0, closedCount);
|
|
|
|
|
Assert.Equal(SessionState.Ready, session.State);
|
|
|
|
|
Assert.Equal(0, workerClient.ShutdownCount);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// <summary>
|
|
|
|
|
/// The idle clock starts when the session becomes Ready, not when it was opened: a worker
|
|
|
|
|
/// that took longer than the window to start must not hand the client a session the very
|
|
|
|
|
/// next sweep reaps before it had a chance to subscribe.
|
|
|
|
|
/// </summary>
|
|
|
|
|
/// <returns>A task that represents the asynchronous operation.</returns>
|
|
|
|
|
[Fact]
|
|
|
|
|
public async Task CloseExpiredLeasesAsync_UnattachedGrace_IdleClockStartsAtReady()
|
|
|
|
|
{
|
|
|
|
|
FakeWorkerClient workerClient = new();
|
|
|
|
|
GatedSessionWorkerClientFactory factory = new(workerClient);
|
|
|
|
|
FakeTimeProvider clock = new(DateTimeOffset.UtcNow);
|
|
|
|
|
SessionManager manager = CreateManager(
|
|
|
|
|
factory,
|
|
|
|
|
options: CreateOptions(unattachedSessionGraceSeconds: 60),
|
|
|
|
|
timeProvider: clock);
|
|
|
|
|
|
|
|
|
|
Task<GatewaySession> openTask = manager.OpenSessionAsync(CreateOpenRequest(), "client-1", ownerKeyId: null, CancellationToken.None);
|
|
|
|
|
await factory.WaitForCreateStartAsync();
|
|
|
|
|
clock.Advance(TimeSpan.FromSeconds(120));
|
|
|
|
|
factory.ReleaseCreate();
|
|
|
|
|
GatewaySession session = await openTask;
|
|
|
|
|
|
|
|
|
|
int closedRightAfterReady = await manager.CloseExpiredLeasesAsync(clock.GetUtcNow(), CancellationToken.None);
|
|
|
|
|
Assert.Equal(0, closedRightAfterReady);
|
|
|
|
|
Assert.Equal(SessionState.Ready, session.State);
|
|
|
|
|
|
|
|
|
|
clock.Advance(TimeSpan.FromSeconds(60));
|
|
|
|
|
int closedAfterIdle = await manager.CloseExpiredLeasesAsync(clock.GetUtcNow(), CancellationToken.None);
|
|
|
|
|
|
|
|
|
|
Assert.Equal(1, closedAfterIdle);
|
|
|
|
|
Assert.Equal(SessionState.Closed, session.State);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// <summary>
|
|
|
|
|
/// The knob is off by default: a never-subscribed idle session lives for its full lease.
|
|
|
|
|
/// </summary>
|
|
|
|
|
/// <returns>A task that represents the asynchronous operation.</returns>
|
|
|
|
|
[Fact]
|
|
|
|
|
public async Task CloseExpiredLeasesAsync_UnattachedGrace_IsDisabledByDefault()
|
|
|
|
|
{
|
|
|
|
|
FakeWorkerClient workerClient = new();
|
|
|
|
|
FakeTimeProvider clock = new(DateTimeOffset.UtcNow);
|
|
|
|
|
SessionManager manager = CreateManager(
|
|
|
|
|
new FakeSessionWorkerClientFactory(workerClient),
|
|
|
|
|
options: CreateOptions(),
|
|
|
|
|
timeProvider: clock);
|
|
|
|
|
GatewaySession session = await manager.OpenSessionAsync(CreateOpenRequest(), "client-1", ownerKeyId: null, CancellationToken.None);
|
|
|
|
|
|
|
|
|
|
clock.Advance(TimeSpan.FromMinutes(20));
|
|
|
|
|
int closedCount = await manager.CloseExpiredLeasesAsync(clock.GetUtcNow(), CancellationToken.None);
|
|
|
|
|
|
|
|
|
|
Assert.Equal(0, closedCount);
|
|
|
|
|
Assert.Equal(SessionState.Ready, session.State);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// <summary>Verifies that an expired-lease sweep leaves a session with an active event subscriber open.</summary>
|
|
|
|
|
/// <returns>A task that represents the asynchronous operation.</returns>
|
|
|
|
|
[Fact]
|
|
|
|
@@ -1011,7 +1486,7 @@ public sealed class SessionManagerTests
|
|
|
|
|
SessionManager manager = CreateManager(new FakeSessionWorkerClientFactory(workerClient));
|
|
|
|
|
GatewaySession session = await manager.OpenSessionAsync(CreateOpenRequest(), "client-1", ownerKeyId: null, CancellationToken.None);
|
|
|
|
|
DateTimeOffset now = DateTimeOffset.UtcNow;
|
|
|
|
|
session.ExtendLease(now.AddSeconds(-1));
|
|
|
|
|
SetLeaseExpiry(session, now.AddSeconds(-1));
|
|
|
|
|
using IDisposable eventSubscriber = session.AttachEventSubscriber(maxSubscribers: 1);
|
|
|
|
|
|
|
|
|
|
int closedCount = await manager.CloseExpiredLeasesAsync(now, CancellationToken.None);
|
|
|
|
@@ -1118,8 +1593,8 @@ public sealed class SessionManagerTests
|
|
|
|
|
DateTimeOffset now = DateTimeOffset.UtcNow;
|
|
|
|
|
|
|
|
|
|
// Both leases are far in the future, so only the fault can reap the faulted session.
|
|
|
|
|
faultedSession.ExtendLease(now.AddMinutes(30));
|
|
|
|
|
healthySession.ExtendLease(now.AddMinutes(30));
|
|
|
|
|
SetLeaseExpiry(faultedSession, now.AddMinutes(30));
|
|
|
|
|
SetLeaseExpiry(healthySession, now.AddMinutes(30));
|
|
|
|
|
faultedSession.MarkFaulted("test fault");
|
|
|
|
|
|
|
|
|
|
int closedCount = await manager.CloseExpiredLeasesAsync(now, CancellationToken.None);
|
|
|
|
@@ -1180,8 +1655,8 @@ public sealed class SessionManagerTests
|
|
|
|
|
GatewaySession firstSession = await manager.OpenSessionAsync(CreateOpenRequest(), "client-1", ownerKeyId: null, CancellationToken.None);
|
|
|
|
|
GatewaySession secondSession = await manager.OpenSessionAsync(CreateOpenRequest(), "client-2", ownerKeyId: null, CancellationToken.None);
|
|
|
|
|
DateTimeOffset now = DateTimeOffset.UtcNow;
|
|
|
|
|
firstSession.ExtendLease(now.AddSeconds(-1));
|
|
|
|
|
secondSession.ExtendLease(now.AddSeconds(-1));
|
|
|
|
|
SetLeaseExpiry(firstSession, now.AddSeconds(-1));
|
|
|
|
|
SetLeaseExpiry(secondSession, now.AddSeconds(-1));
|
|
|
|
|
|
|
|
|
|
int closedCount = await manager.CloseExpiredLeasesAsync(now, CancellationToken.None);
|
|
|
|
|
|
|
|
|
@@ -1233,8 +1708,8 @@ public sealed class SessionManagerTests
|
|
|
|
|
GatewaySession failingSession = await manager.OpenSessionAsync(CreateOpenRequest(), "client-1", ownerKeyId: null, CancellationToken.None);
|
|
|
|
|
GatewaySession healthySession = await manager.OpenSessionAsync(CreateOpenRequest(), "client-2", ownerKeyId: null, CancellationToken.None);
|
|
|
|
|
DateTimeOffset now = DateTimeOffset.UtcNow;
|
|
|
|
|
failingSession.ExtendLease(now.AddSeconds(-1));
|
|
|
|
|
healthySession.ExtendLease(now.AddSeconds(-1));
|
|
|
|
|
SetLeaseExpiry(failingSession, now.AddSeconds(-1));
|
|
|
|
|
SetLeaseExpiry(healthySession, now.AddSeconds(-1));
|
|
|
|
|
|
|
|
|
|
SessionManagerException exception = await Assert.ThrowsAsync<SessionManagerException>(
|
|
|
|
|
async () => await manager.CloseExpiredLeasesAsync(now, CancellationToken.None));
|
|
|
|
@@ -1273,6 +1748,84 @@ public sealed class SessionManagerTests
|
|
|
|
|
Assert.False(manager.TryGetSession(secondSession.SessionId, out _));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// <summary>
|
|
|
|
|
/// Once the drain has started, a new open must be refused with <c>GatewayStopping</c>
|
|
|
|
|
/// rather than accepted: a session created behind the drain's snapshot has nobody left to
|
|
|
|
|
/// close it, and its worker would outlive the gateway as an orphan. The refusal is not an
|
|
|
|
|
/// open failure — no slot is taken, nothing is registered and no fault is recorded.
|
|
|
|
|
/// </summary>
|
|
|
|
|
/// <returns>A task that represents the asynchronous operation.</returns>
|
|
|
|
|
[Fact]
|
|
|
|
|
public async Task OpenSessionAsync_AfterShutdownStarted_ThrowsGatewayStopping()
|
|
|
|
|
{
|
|
|
|
|
FakeWorkerClient workerClient = new();
|
|
|
|
|
SessionRegistry registry = new();
|
|
|
|
|
using GatewayMetrics metrics = new();
|
|
|
|
|
SessionManager manager = CreateManager(new FakeSessionWorkerClientFactory(workerClient), registry, metrics);
|
|
|
|
|
await manager.ShutdownAsync(CancellationToken.None);
|
|
|
|
|
|
|
|
|
|
SessionManagerException exception = await Assert.ThrowsAsync<SessionManagerException>(
|
|
|
|
|
async () => await manager.OpenSessionAsync(CreateOpenRequest(), "client-1", ownerKeyId: null, CancellationToken.None));
|
|
|
|
|
|
|
|
|
|
Assert.Equal(SessionManagerErrorCode.GatewayStopping, exception.ErrorCode);
|
|
|
|
|
Assert.Equal(0, registry.Count);
|
|
|
|
|
Assert.Equal(0, metrics.GetSnapshot().SessionsOpened);
|
|
|
|
|
Assert.Equal(0, metrics.GetSnapshot().Faults);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// <summary>
|
|
|
|
|
/// The host signals <c>ApplicationStopping</c> before hosted services stop, and Kestrel keeps
|
|
|
|
|
/// accepting gRPC calls until its own <c>StopAsync</c> runs — so the gate must close on the
|
|
|
|
|
/// lifetime signal, not only when <c>ShutdownAsync</c> is finally called.
|
|
|
|
|
/// </summary>
|
|
|
|
|
/// <returns>A task that represents the asynchronous operation.</returns>
|
|
|
|
|
[Fact]
|
|
|
|
|
public async Task OpenSessionAsync_WhenApplicationStopping_ThrowsGatewayStopping()
|
|
|
|
|
{
|
|
|
|
|
FakeWorkerClient workerClient = new();
|
|
|
|
|
using FakeHostApplicationLifetime lifetime = new();
|
|
|
|
|
SessionManager manager = new(
|
|
|
|
|
new SessionRegistry(),
|
|
|
|
|
new FakeSessionWorkerClientFactory(workerClient),
|
|
|
|
|
Options.Create(CreateOptions()),
|
|
|
|
|
new GatewayMetrics(),
|
|
|
|
|
applicationLifetime: lifetime);
|
|
|
|
|
|
|
|
|
|
GatewaySession beforeStopping = await manager.OpenSessionAsync(CreateOpenRequest(), "client-1", ownerKeyId: null, CancellationToken.None);
|
|
|
|
|
Assert.Equal(SessionState.Ready, beforeStopping.State);
|
|
|
|
|
|
|
|
|
|
lifetime.SignalStopping();
|
|
|
|
|
SessionManagerException exception = await Assert.ThrowsAsync<SessionManagerException>(
|
|
|
|
|
async () => await manager.OpenSessionAsync(CreateOpenRequest(), "client-2", ownerKeyId: null, CancellationToken.None));
|
|
|
|
|
|
|
|
|
|
Assert.Equal(SessionManagerErrorCode.GatewayStopping, exception.ErrorCode);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// <summary>
|
|
|
|
|
/// An open that passed the stopping gate but registered its session after the drain took
|
|
|
|
|
/// its snapshot is still caught: the drain re-snapshots until the registry is empty, so the
|
|
|
|
|
/// late session is closed by the drain rather than left running.
|
|
|
|
|
/// </summary>
|
|
|
|
|
/// <returns>A task that represents the asynchronous operation.</returns>
|
|
|
|
|
[Fact]
|
|
|
|
|
public async Task ShutdownAsync_DrainsSessionRegisteredAfterFirstSnapshot()
|
|
|
|
|
{
|
|
|
|
|
FakeWorkerClient workerClient = new();
|
|
|
|
|
LateFirstSnapshotRegistry registry = new();
|
|
|
|
|
using GatewayMetrics metrics = new();
|
|
|
|
|
SessionManager manager = CreateManager(new FakeSessionWorkerClientFactory(workerClient), registry, metrics);
|
|
|
|
|
GatewaySession session = await manager.OpenSessionAsync(CreateOpenRequest(), "client-1", ownerKeyId: null, CancellationToken.None);
|
|
|
|
|
|
|
|
|
|
await manager.ShutdownAsync(CancellationToken.None);
|
|
|
|
|
|
|
|
|
|
Assert.True(registry.SnapshotCalls >= 2);
|
|
|
|
|
Assert.Equal(SessionState.Closed, session.State);
|
|
|
|
|
Assert.Equal(1, workerClient.ShutdownCount);
|
|
|
|
|
Assert.Equal(0, registry.Count);
|
|
|
|
|
Assert.Equal(0, metrics.GetSnapshot().OpenSessions);
|
|
|
|
|
Assert.Equal(1, metrics.GetSnapshot().SessionsClosed);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// <summary>Verifies that shutdown closes all registered sessions.</summary>
|
|
|
|
|
/// <returns>A task that represents the asynchronous operation.</returns>
|
|
|
|
|
[Fact]
|
|
|
|
@@ -1322,7 +1875,8 @@ public sealed class SessionManagerTests
|
|
|
|
|
int defaultLeaseSeconds = 1800,
|
|
|
|
|
int detachGraceSeconds = 0,
|
|
|
|
|
int workerReadyWaitTimeoutMs = 0,
|
|
|
|
|
int faultedGraceSeconds = 0)
|
|
|
|
|
int faultedGraceSeconds = 0,
|
|
|
|
|
int unattachedSessionGraceSeconds = 0)
|
|
|
|
|
{
|
|
|
|
|
return new GatewayOptions
|
|
|
|
|
{
|
|
|
|
@@ -1334,6 +1888,7 @@ public sealed class SessionManagerTests
|
|
|
|
|
DetachGraceSeconds = detachGraceSeconds,
|
|
|
|
|
WorkerReadyWaitTimeoutMs = workerReadyWaitTimeoutMs,
|
|
|
|
|
FaultedGraceSeconds = faultedGraceSeconds,
|
|
|
|
|
UnattachedSessionGraceSeconds = unattachedSessionGraceSeconds,
|
|
|
|
|
},
|
|
|
|
|
Worker = new WorkerOptions
|
|
|
|
|
{
|
|
|
|
@@ -1343,6 +1898,17 @@ public sealed class SessionManagerTests
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// <summary>
|
|
|
|
|
/// Moves a session's lease expiry by back-dating its last client activity, which is the only
|
|
|
|
|
/// production path that moves the lease (<c>_leaseExpiresAt = activityAt + LeaseDuration</c>).
|
|
|
|
|
/// </summary>
|
|
|
|
|
/// <param name="session">Session whose lease to move.</param>
|
|
|
|
|
/// <param name="leaseExpiresAt">Wanted lease expiry.</param>
|
|
|
|
|
private static void SetLeaseExpiry(GatewaySession session, DateTimeOffset leaseExpiresAt)
|
|
|
|
|
{
|
|
|
|
|
session.TouchClientActivity(leaseExpiresAt - session.LeaseDuration);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private static SessionOpenRequest CreateOpenRequest()
|
|
|
|
|
{
|
|
|
|
|
return new SessionOpenRequest(
|
|
|
|
@@ -1422,6 +1988,203 @@ public sealed class SessionManagerTests
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// <summary>
|
|
|
|
|
/// Factory that records a per-session event metric for the session it is starting and
|
|
|
|
|
/// then fails, so a test can check the open rollback clears that metric.
|
|
|
|
|
/// </summary>
|
|
|
|
|
private sealed class EventRecordingFailingSessionWorkerClientFactory(GatewayMetrics metrics) : ISessionWorkerClientFactory
|
|
|
|
|
{
|
|
|
|
|
/// <summary>Gets the id of the session whose worker creation was attempted.</summary>
|
|
|
|
|
public string? StartedSessionId { get; private set; }
|
|
|
|
|
|
|
|
|
|
/// <inheritdoc />
|
|
|
|
|
public Task<IWorkerClient> CreateAsync(
|
|
|
|
|
GatewaySession session,
|
|
|
|
|
CancellationToken cancellationToken)
|
|
|
|
|
{
|
|
|
|
|
StartedSessionId = session.SessionId;
|
|
|
|
|
metrics.EventReceived(session.SessionId, MxEventFamily.OnDataChange.ToString());
|
|
|
|
|
throw new InvalidOperationException("worker startup failed");
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// <summary>
|
|
|
|
|
/// Factory whose <see cref="CreateAsync"/> parks until the test releases it, so the test can
|
|
|
|
|
/// act on the registered-but-still-starting session (kill it, drain the manager) inside the
|
|
|
|
|
/// window between registry insertion and worker attach. Once released it hands the same
|
|
|
|
|
/// client to every later call.
|
|
|
|
|
/// </summary>
|
|
|
|
|
private sealed class GatedSessionWorkerClientFactory(IWorkerClient workerClient) : ISessionWorkerClientFactory
|
|
|
|
|
{
|
|
|
|
|
private readonly TaskCompletionSource<GatewaySession> _entered = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
|
|
|
|
private readonly TaskCompletionSource _release = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
|
|
|
|
|
|
|
|
|
/// <summary>Waits until the manager has asked for a worker and returns the starting session.</summary>
|
|
|
|
|
/// <returns>The session whose worker is being created.</returns>
|
|
|
|
|
public Task<GatewaySession> WaitForCreateStartAsync()
|
|
|
|
|
{
|
|
|
|
|
return _entered.Task.WaitAsync(TimeSpan.FromSeconds(5));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// <summary>Lets the parked worker creation complete.</summary>
|
|
|
|
|
public void ReleaseCreate()
|
|
|
|
|
{
|
|
|
|
|
_release.TrySetResult();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// <inheritdoc />
|
|
|
|
|
public async Task<IWorkerClient> CreateAsync(
|
|
|
|
|
GatewaySession session,
|
|
|
|
|
CancellationToken cancellationToken)
|
|
|
|
|
{
|
|
|
|
|
// Deliberately not bound to the caller's token: the tests that cancel the open while
|
|
|
|
|
// the worker is starting need the worker to still ARRIVE after the cancellation, which
|
|
|
|
|
// is the orphan-prone case.
|
|
|
|
|
_entered.TrySetResult(session);
|
|
|
|
|
await _release.Task.WaitAsync(TimeSpan.FromSeconds(5));
|
|
|
|
|
return workerClient;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// <summary>
|
|
|
|
|
/// Registry that can replay a race the real registry only produces by timing: a session a
|
|
|
|
|
/// concurrent close has already removed and disposed is still returned by one lookup (the
|
|
|
|
|
/// kill looked it up just before the removal) or still appears in one snapshot (the sweep
|
|
|
|
|
/// snapshotted just before the removal).
|
|
|
|
|
/// </summary>
|
|
|
|
|
private sealed class StaleRegistry : ISessionRegistry
|
|
|
|
|
{
|
|
|
|
|
private readonly SessionRegistry _inner = new();
|
|
|
|
|
private GatewaySession? _staleLookup;
|
|
|
|
|
private GatewaySession? _staleSnapshot;
|
|
|
|
|
|
|
|
|
|
/// <summary>Makes the next lookup of <paramref name="session"/>'s id return it although it was removed.</summary>
|
|
|
|
|
/// <param name="session">Removed session to serve once.</param>
|
|
|
|
|
public void ServeStaleLookupOnce(GatewaySession session)
|
|
|
|
|
{
|
|
|
|
|
_staleLookup = session;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// <summary>Makes the next snapshot include <paramref name="session"/> although it was removed.</summary>
|
|
|
|
|
/// <param name="session">Removed session to include once.</param>
|
|
|
|
|
public void ServeStaleSnapshot(GatewaySession session)
|
|
|
|
|
{
|
|
|
|
|
_staleSnapshot = session;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// <inheritdoc />
|
|
|
|
|
public int Count => _inner.Count;
|
|
|
|
|
|
|
|
|
|
/// <inheritdoc />
|
|
|
|
|
public int ActiveCount => _inner.ActiveCount;
|
|
|
|
|
|
|
|
|
|
/// <inheritdoc />
|
|
|
|
|
public bool TryAdd(GatewaySession session) => _inner.TryAdd(session);
|
|
|
|
|
|
|
|
|
|
/// <inheritdoc />
|
|
|
|
|
public bool TryGet(string sessionId, [System.Diagnostics.CodeAnalysis.MaybeNullWhen(false)] out GatewaySession session)
|
|
|
|
|
{
|
|
|
|
|
if (_staleLookup is not null && _staleLookup.SessionId == sessionId)
|
|
|
|
|
{
|
|
|
|
|
session = _staleLookup;
|
|
|
|
|
_staleLookup = null;
|
|
|
|
|
return true;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return _inner.TryGet(sessionId, out session);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// <inheritdoc />
|
|
|
|
|
public bool TryRemove(string sessionId, [System.Diagnostics.CodeAnalysis.MaybeNullWhen(false)] out GatewaySession session)
|
|
|
|
|
=> _inner.TryRemove(sessionId, out session);
|
|
|
|
|
|
|
|
|
|
/// <inheritdoc />
|
|
|
|
|
public IReadOnlyCollection<GatewaySession> Snapshot()
|
|
|
|
|
{
|
|
|
|
|
if (_staleSnapshot is null)
|
|
|
|
|
{
|
|
|
|
|
return _inner.Snapshot();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
GatewaySession[] snapshot = [.. _inner.Snapshot(), _staleSnapshot];
|
|
|
|
|
_staleSnapshot = null;
|
|
|
|
|
return snapshot;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// <summary>
|
|
|
|
|
/// Lifetime fake whose <see cref="ApplicationStopping"/> the test fires on demand; the
|
|
|
|
|
/// other two tokens never fire.
|
|
|
|
|
/// </summary>
|
|
|
|
|
private sealed class FakeHostApplicationLifetime : Microsoft.Extensions.Hosting.IHostApplicationLifetime, IDisposable
|
|
|
|
|
{
|
|
|
|
|
private readonly CancellationTokenSource _stopping = new();
|
|
|
|
|
|
|
|
|
|
/// <inheritdoc />
|
|
|
|
|
public CancellationToken ApplicationStarted => CancellationToken.None;
|
|
|
|
|
|
|
|
|
|
/// <inheritdoc />
|
|
|
|
|
public CancellationToken ApplicationStopping => _stopping.Token;
|
|
|
|
|
|
|
|
|
|
/// <inheritdoc />
|
|
|
|
|
public CancellationToken ApplicationStopped => CancellationToken.None;
|
|
|
|
|
|
|
|
|
|
/// <summary>Fires <see cref="ApplicationStopping"/>.</summary>
|
|
|
|
|
public void SignalStopping()
|
|
|
|
|
{
|
|
|
|
|
_stopping.Cancel();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// <inheritdoc />
|
|
|
|
|
public void StopApplication()
|
|
|
|
|
{
|
|
|
|
|
SignalStopping();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// <inheritdoc />
|
|
|
|
|
public void Dispose()
|
|
|
|
|
{
|
|
|
|
|
_stopping.Dispose();
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// <summary>
|
|
|
|
|
/// Registry whose first <see cref="Snapshot"/> is empty regardless of its contents,
|
|
|
|
|
/// simulating a session whose <c>TryAdd</c> landed just after the drain's first snapshot;
|
|
|
|
|
/// every later snapshot is the real one.
|
|
|
|
|
/// </summary>
|
|
|
|
|
private sealed class LateFirstSnapshotRegistry : ISessionRegistry
|
|
|
|
|
{
|
|
|
|
|
private readonly SessionRegistry _inner = new();
|
|
|
|
|
|
|
|
|
|
/// <summary>Gets how many snapshots have been taken.</summary>
|
|
|
|
|
public int SnapshotCalls { get; private set; }
|
|
|
|
|
|
|
|
|
|
/// <inheritdoc />
|
|
|
|
|
public int Count => _inner.Count;
|
|
|
|
|
|
|
|
|
|
/// <inheritdoc />
|
|
|
|
|
public int ActiveCount => _inner.ActiveCount;
|
|
|
|
|
|
|
|
|
|
/// <inheritdoc />
|
|
|
|
|
public bool TryAdd(GatewaySession session) => _inner.TryAdd(session);
|
|
|
|
|
|
|
|
|
|
/// <inheritdoc />
|
|
|
|
|
public bool TryGet(string sessionId, [System.Diagnostics.CodeAnalysis.MaybeNullWhen(false)] out GatewaySession session)
|
|
|
|
|
=> _inner.TryGet(sessionId, out session);
|
|
|
|
|
|
|
|
|
|
/// <inheritdoc />
|
|
|
|
|
public bool TryRemove(string sessionId, [System.Diagnostics.CodeAnalysis.MaybeNullWhen(false)] out GatewaySession session)
|
|
|
|
|
=> _inner.TryRemove(sessionId, out session);
|
|
|
|
|
|
|
|
|
|
/// <inheritdoc />
|
|
|
|
|
public IReadOnlyCollection<GatewaySession> Snapshot()
|
|
|
|
|
{
|
|
|
|
|
SnapshotCalls++;
|
|
|
|
|
return SnapshotCalls == 1 ? [] : _inner.Snapshot();
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private sealed class FakeWorkerClient : IWorkerClient
|
|
|
|
|
{
|
|
|
|
|
private WorkerClientState _state = WorkerClientState.Ready;
|
|
|
|
|