Merge stream A (session lifecycle: startup orphan, sweep metrics, stopping gate, kill-fallback Closed, unattached grace, worker-exit faults session)

This commit is contained in:
Joseph Doherty
2026-10-22 08:21:47 -04:00
15 changed files with 1669 additions and 169 deletions
@@ -619,6 +619,34 @@ public sealed class GatewayOptionsValidatorTests
Assert.True(result.Succeeded);
}
/// <summary>Verifies a negative <see cref="SessionOptions.UnattachedSessionGraceSeconds"/> fails validation.</summary>
[Fact]
public void Validate_Fails_WhenUnattachedSessionGraceSecondsIsNegative()
{
GatewayOptions options = CloneWithSessions(
ValidOptions(),
new SessionOptions { UnattachedSessionGraceSeconds = -1 });
ValidateOptionsResult result = new GatewayOptionsValidator().Validate(null, options);
Assert.True(result.Failed);
Assert.Contains(
result.Failures!,
f => f.Contains("MxGateway:Sessions:UnattachedSessionGraceSeconds"));
}
/// <summary>Verifies a zero (off) and a positive <see cref="SessionOptions.UnattachedSessionGraceSeconds"/> pass validation.</summary>
/// <param name="seconds">Grace window under test.</param>
[Theory]
[InlineData(0)]
[InlineData(120)]
public void Validate_Succeeds_WhenUnattachedSessionGraceSecondsIsZeroOrPositive(int seconds)
{
GatewayOptions options = CloneWithSessions(
ValidOptions(),
new SessionOptions { UnattachedSessionGraceSeconds = seconds });
ValidateOptionsResult result = new GatewayOptionsValidator().Validate(null, options);
Assert.True(result.Succeeded);
}
// -------------------------------------------------------------------------
// ReplayBufferCapacity / ReplayRetentionSeconds validation
// -------------------------------------------------------------------------
@@ -76,6 +76,49 @@ public sealed class GatewaySessionTests
await session.DisposeAsync();
}
/// <summary>
/// A worker client that arrives after the session has already been closed (an admin kill
/// or the host drain landed while the worker was still starting) must be refused:
/// <see cref="GatewaySession.AttachWorkerClient"/> returns <see langword="false"/> and
/// leaves <see cref="GatewaySession.WorkerClient"/> unset, so the caller can discard the
/// client instead of handing a dead session back as an opened one.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task AttachWorkerClient_AfterClose_RefusesLateWorkerClient()
{
GatewaySession session = CreateSession();
session.TransitionTo(SessionState.StartingWorker);
await session.CloseAsync("admin-kill", CancellationToken.None);
bool attached = session.AttachWorkerClient(new BlockingShutdownWorkerClient());
Assert.False(attached);
Assert.Null(session.WorkerClient);
Assert.Equal(SessionState.Closed, session.State);
await session.DisposeAsync();
}
/// <summary>
/// Disposal is the other end of the same window: once the session's distributor has been
/// torn down nothing can ever pump a late worker's events, so the attach is refused there
/// too rather than storing a client that is never read.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task AttachWorkerClient_AfterDispose_RefusesLateWorkerClient()
{
GatewaySession session = CreateSession();
session.TransitionTo(SessionState.StartingWorker);
await session.DisposeAsync();
bool attached = session.AttachWorkerClient(new BlockingShutdownWorkerClient());
Assert.False(attached);
Assert.Null(session.WorkerClient);
}
/// <summary>
/// A <see cref="GatewaySession.MarkFaulted"/> issued
/// while <see cref="GatewaySession.CloseAsync"/> is parked between its
@@ -0,0 +1,151 @@
using Microsoft.Extensions.Options;
using ZB.MOM.WW.MxGateway.Contracts.Proto;
using ZB.MOM.WW.MxGateway.Server.Configuration;
using ZB.MOM.WW.MxGateway.Server.Metrics;
using ZB.MOM.WW.MxGateway.Server.Sessions;
using ZB.MOM.WW.MxGateway.Server.Workers;
using ZB.MOM.WW.MxGateway.Tests.Gateway.Workers.Fakes;
namespace ZB.MOM.WW.MxGateway.Tests.Gateway.Sessions;
/// <summary>
/// A worker that goes away on its own — the client sent the <c>ShutdownWorker</c> control
/// command and the worker exited cleanly, or the worker crashed / its pipe broke — must be
/// noticed by its <see cref="GatewaySession"/>. The real <see cref="WorkerClient"/> reaches
/// <c>Closed</c> / <c>Faulted</c>, but nothing used to tell the session, which kept reporting
/// <c>Ready</c> (live windev finding) until its lease expired.
/// </summary>
public sealed class GatewaySessionWorkerExitTests
{
private static readonly TimeSpan TestTimeout = TimeSpan.FromSeconds(5);
/// <summary>
/// After the client's <c>ShutdownWorker</c> command the fake worker replies OK, sends the
/// shutdown ack and the worker client closes. The session must fault (the worker is gone
/// outside a gateway-initiated close), stay observable with a fault reason, and then be
/// reaped by the ordinary faulted-session sweep — which is where the session is accounted
/// closed, exactly once.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task ClientShutdownWorker_WorkerExits_SessionFaultsAndSweepReapsItOnce()
{
await using FakeWorkerHarness fakeWorker = await FakeWorkerHarness.CreateConnectedPairAsync();
using GatewayMetrics metrics = new();
WorkerClient client = fakeWorker.CreateClient(metrics: metrics);
await StartClientAsync(fakeWorker, client);
SessionManager manager = CreateManager(client, metrics);
GatewaySession session = await OpenSessionAsync(manager);
using IEventSubscriberLease stream = session.AttachEventSubscriber(maxSubscribers: 1);
Task<WorkerCommandReply> invokeTask = manager.InvokeAsync(session.SessionId, CreateShutdownWorkerCommand(), CancellationToken.None);
await fakeWorker.RespondToControlCommandAsync().WaitAsync(TestTimeout);
WorkerCommandReply reply = await invokeTask.WaitAsync(TestTimeout);
Assert.Equal(ProtocolStatusCode.Ok, reply.Reply.ProtocolStatus.Code);
await WaitUntilAsync(() => session.State == SessionState.Faulted);
Assert.Equal(WorkerClientState.Closed, client.State);
Assert.Contains("worker", session.FinalFault, StringComparison.OrdinalIgnoreCase);
Assert.True(manager.TryGetSession(session.SessionId, out _));
Assert.Equal(1, metrics.GetSnapshot().OpenSessions);
Assert.Equal(0, metrics.GetSnapshot().SessionsClosed);
int closedCount = await manager.CloseExpiredLeasesAsync(DateTimeOffset.UtcNow, CancellationToken.None);
Assert.Equal(1, closedCount);
Assert.Equal(SessionState.Closed, session.State);
Assert.False(manager.TryGetSession(session.SessionId, out _));
GatewayMetricsSnapshot snapshot = metrics.GetSnapshot();
Assert.Equal(1, snapshot.SessionsClosed);
Assert.Equal(0, snapshot.OpenSessions);
}
/// <summary>
/// The other way a worker goes away: its pipe breaks (crash). The worker client faults, and
/// the session must follow it to <c>Faulted</c> rather than stay <c>Ready</c>.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task WorkerPipeBreaks_SessionFaults()
{
await using FakeWorkerHarness fakeWorker = await FakeWorkerHarness.CreateConnectedPairAsync();
using GatewayMetrics metrics = new();
WorkerClient client = fakeWorker.CreateClient(metrics: metrics);
await StartClientAsync(fakeWorker, client);
SessionManager manager = CreateManager(client, metrics);
GatewaySession session = await OpenSessionAsync(manager);
using IEventSubscriberLease stream = session.AttachEventSubscriber(maxSubscribers: 1);
await fakeWorker.DisposeWorkerSideAsync();
await WaitUntilAsync(() => session.State == SessionState.Faulted);
Assert.Equal(WorkerClientState.Faulted, client.State);
await manager.CloseExpiredLeasesAsync(DateTimeOffset.UtcNow, CancellationToken.None);
Assert.False(manager.TryGetSession(session.SessionId, out _));
Assert.Equal(1, metrics.GetSnapshot().SessionsClosed);
Assert.Equal(0, metrics.GetSnapshot().OpenSessions);
}
private static async Task StartClientAsync(FakeWorkerHarness fakeWorker, WorkerClient client)
{
Task startTask = client.StartAsync(CancellationToken.None);
await fakeWorker.CompleteStartupAsync();
await startTask.WaitAsync(TestTimeout);
Assert.Equal(WorkerClientState.Ready, client.State);
}
private static SessionManager CreateManager(IWorkerClient client, GatewayMetrics metrics)
{
return new SessionManager(
new SessionRegistry(),
new SingleClientFactory(client),
Options.Create(new GatewayOptions()),
metrics);
}
private static Task<GatewaySession> OpenSessionAsync(SessionManager manager)
{
return manager.OpenSessionAsync(
new SessionOpenRequest(RequestedBackend: null, ClientSessionName: "worker-exit", ClientCorrelationId: null, CommandTimeout: null),
"client-1",
ownerKeyId: null,
CancellationToken.None);
}
private static WorkerCommand CreateShutdownWorkerCommand()
{
return new WorkerCommand
{
Command = new MxCommand
{
Kind = MxCommandKind.ShutdownWorker,
ShutdownWorker = new ShutdownWorkerCommand(),
},
};
}
private static async Task WaitUntilAsync(Func<bool> condition)
{
DateTime deadline = DateTime.UtcNow + TestTimeout;
while (!condition())
{
if (DateTime.UtcNow > deadline)
{
Assert.Fail("Condition was not met within the test timeout.");
}
await Task.Delay(10);
}
}
private sealed class SingleClientFactory(IWorkerClient client) : ISessionWorkerClientFactory
{
/// <inheritdoc />
public Task<IWorkerClient> CreateAsync(GatewaySession session, CancellationToken cancellationToken)
{
return Task.FromResult(client);
}
}
}
@@ -641,10 +641,16 @@ public sealed class SessionManagerTests
Assert.Equal(0, metrics.GetSnapshot().OpenSessions);
}
/// <summary>Verifies that closing a session kills the worker when shutdown fails.</summary>
/// <summary>
/// A worker that does not shut down gracefully is killed, and that is a <em>successful</em>
/// close: the kill is the documented fallback, the worker is gone and the session is
/// <c>Closed</c>. Reporting it as <c>CloseFailed</c> + <c>Faulted</c> + a fault metric
/// misreported every hung-worker close as a teardown failure. <c>CloseFailed</c> is reserved
/// for both the shutdown and the kill failing.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task CloseSessionAsync_WhenWorkerShutdownFails_KillsWorker()
public async Task CloseSessionAsync_WhenWorkerShutdownFails_KillsWorkerAndReportsClosed()
{
FakeWorkerClient workerClient = new()
{
@@ -652,15 +658,24 @@ public sealed class SessionManagerTests
WorkerClientErrorCode.ShutdownTimeout,
"Worker shutdown timed out."),
};
SessionManager manager = CreateManager(new FakeSessionWorkerClientFactory(workerClient));
using GatewayMetrics metrics = new();
SessionManager manager = CreateManager(new FakeSessionWorkerClientFactory(workerClient), metrics: metrics);
GatewaySession session = await manager.OpenSessionAsync(CreateOpenRequest(), "client-1", ownerKeyId: null, CancellationToken.None);
SessionManagerException exception = await Assert.ThrowsAsync<SessionManagerException>(
async () => await manager.CloseSessionAsync(session.SessionId, CancellationToken.None));
SessionCloseResult result = await manager.CloseSessionAsync(session.SessionId, CancellationToken.None);
Assert.Equal(SessionManagerErrorCode.CloseFailed, exception.ErrorCode);
Assert.Equal(SessionState.Closed, result.FinalState);
Assert.False(result.AlreadyClosed);
Assert.Same(workerClient.ShutdownException, result.WorkerShutdownFailure);
Assert.Equal(SessionState.Closed, session.State);
Assert.Null(session.FinalFault);
Assert.Equal(1, workerClient.ShutdownCount);
Assert.Equal(1, workerClient.KillCount);
Assert.False(manager.TryGetSession(session.SessionId, out _));
GatewayMetricsSnapshot snapshot = metrics.GetSnapshot();
Assert.Equal(0, snapshot.Faults);
Assert.Equal(1, snapshot.SessionsClosed);
Assert.Equal(0, snapshot.OpenSessions);
}
/// <summary>Verifies that when worker shutdown fails, the session is removed and the slot is released.</summary>
@@ -689,31 +704,59 @@ public sealed class SessionManagerTests
CancellationToken.None);
metrics.EventReceived(firstSession.SessionId, MxEventFamily.OnDataChange.ToString());
SessionManagerException exception = await Assert.ThrowsAsync<SessionManagerException>(
async () => await manager.CloseSessionAsync(firstSession.SessionId, CancellationToken.None));
SessionCloseResult result = await manager.CloseSessionAsync(firstSession.SessionId, CancellationToken.None);
GatewaySession secondSession = await manager.OpenSessionAsync(
CreateOpenRequest(),
"client-2",
ownerKeyId: null,
CancellationToken.None);
Assert.Equal(SessionManagerErrorCode.CloseFailed, exception.ErrorCode);
Assert.Equal(SessionState.Closed, result.FinalState);
Assert.False(manager.TryGetSession(firstSession.SessionId, out _));
Assert.True(manager.TryGetSession(secondSession.SessionId, out _));
Assert.Equal(1, registry.Count);
Assert.Equal(1, failingWorkerClient.KillCount);
Assert.Equal(1, failingWorkerClient.DisposeCount);
GatewayMetricsSnapshot snapshot = metrics.GetSnapshot();
// A close-that-failed accounts as SessionClosed (counter += 1) rather
// than SessionRemoved (gauge -= 1, counter unchanged). The session is
// being removed from the registry on this path, so it must show up in
// the closed count.
Assert.Equal(1, snapshot.SessionsClosed);
Assert.False(snapshot.EventsBySession.ContainsKey(firstSession.SessionId));
Assert.Equal(1, snapshot.OpenSessions);
}
/// <summary>
/// Only when the graceful shutdown AND the kill fallback both fail is the close a failure:
/// the session is faulted, removed, counted as closed (it is leaving the registry) and the
/// caller gets <c>CloseFailed</c>.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task CloseSessionAsync_WhenShutdownAndKillBothFail_ThrowsCloseFailed()
{
FakeWorkerClient workerClient = new()
{
ShutdownException = new WorkerClientException(
WorkerClientErrorCode.ShutdownTimeout,
"Worker shutdown timed out."),
KillException = new InvalidOperationException("worker kill failed"),
};
using GatewayMetrics metrics = new();
SessionManager manager = CreateManager(new FakeSessionWorkerClientFactory(workerClient), metrics: metrics);
GatewaySession session = await manager.OpenSessionAsync(CreateOpenRequest(), "client-1", ownerKeyId: null, CancellationToken.None);
SessionManagerException exception = await Assert.ThrowsAsync<SessionManagerException>(
async () => await manager.CloseSessionAsync(session.SessionId, CancellationToken.None));
Assert.Equal(SessionManagerErrorCode.CloseFailed, exception.ErrorCode);
Assert.Equal(SessionState.Faulted, session.State);
Assert.Equal(1, workerClient.ShutdownCount);
Assert.Equal(1, workerClient.KillCount);
Assert.False(manager.TryGetSession(session.SessionId, out _));
GatewayMetricsSnapshot snapshot = metrics.GetSnapshot();
Assert.Equal(1, snapshot.Faults);
Assert.Equal(1, snapshot.SessionsClosed);
Assert.Equal(0, snapshot.OpenSessions);
}
/// <summary>Verifies that when the second close is canceled, the session is not removed if owned by the first close.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
@@ -881,6 +924,71 @@ public sealed class SessionManagerTests
Assert.True(metrics.GetSnapshot().Faults > 0);
}
/// <summary>
/// A kill whose session lookup succeeded but whose close gate is disposed by the time it
/// waits on it lost the race to a close that already removed, disposed and accounted the
/// session. It must report <c>AlreadyClosed</c> and touch nothing: the old catch recorded a
/// fault, faulted the session and decremented the open gauge a second time (visible here as
/// the other open session's slot disappearing from the gauge).
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task KillWorkerAsync_WhenCloseWonTheRace_ReportsAlreadyClosedWithoutDoubleAccounting()
{
FakeWorkerClient racedClient = new();
FakeWorkerClient otherClient = new();
StaleRegistry registry = new();
using GatewayMetrics metrics = new();
SessionManager manager = CreateManager(new QueueingSessionWorkerClientFactory(racedClient, otherClient), registry, metrics);
GatewaySession raced = await manager.OpenSessionAsync(CreateOpenRequest(), "client-1", ownerKeyId: null, CancellationToken.None);
await manager.OpenSessionAsync(CreateOpenRequest(), "client-2", ownerKeyId: null, CancellationToken.None);
await manager.CloseSessionAsync(raced.SessionId, CancellationToken.None);
Assert.Equal(1, metrics.GetSnapshot().OpenSessions);
// The kill looked the session up just before the close removed and disposed it.
registry.ServeStaleLookupOnce(raced);
SessionCloseResult result = await manager.KillWorkerAsync(raced.SessionId, "admin-kill", CancellationToken.None);
Assert.True(result.AlreadyClosed);
Assert.Equal(SessionState.Closed, result.FinalState);
Assert.Equal(0, racedClient.KillCount);
GatewayMetricsSnapshot snapshot = metrics.GetSnapshot();
Assert.Equal(1, snapshot.OpenSessions);
Assert.Equal(1, snapshot.SessionsClosed);
Assert.Equal(0, snapshot.Faults);
}
/// <summary>
/// The sweep's counterpart of the kill race: a session that a concurrent close removed and
/// disposed between the snapshot and the close call is simply skipped — it is not a sweep
/// failure to log, and it is not counted as closed by the sweep.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task CloseExpiredLeasesAsync_WhenCloseWonTheRace_DoesNotReportSweepFailure()
{
FakeWorkerClient workerClient = new();
StaleRegistry registry = new();
using GatewayMetrics metrics = new();
SessionManager manager = CreateManager(new FakeSessionWorkerClientFactory(workerClient), registry, metrics);
GatewaySession session = await manager.OpenSessionAsync(CreateOpenRequest(), "client-1", ownerKeyId: null, CancellationToken.None);
DateTimeOffset now = DateTimeOffset.UtcNow;
SetLeaseExpiry(session, now.AddSeconds(-1));
await manager.CloseSessionAsync(session.SessionId, CancellationToken.None);
registry.ServeStaleSnapshot(session);
int closedCount = await manager.CloseExpiredLeasesAsync(now, CancellationToken.None);
Assert.Equal(0, closedCount);
Assert.Equal(1, workerClient.ShutdownCount);
GatewayMetricsSnapshot snapshot = metrics.GetSnapshot();
Assert.Equal(1, snapshot.SessionsClosed);
Assert.Equal(0, snapshot.OpenSessions);
Assert.Equal(0, snapshot.Faults);
}
/// <summary>
/// Concurrent kills on the same session must not
/// double-increment <c>mxgateway.sessions.closed</c>. The first kill wins, the second
@@ -926,7 +1034,9 @@ public sealed class SessionManagerTests
/// <c>KillWorker</c> + <c>RemoveSessionAsync</c> when <c>CloseSessionCoreAsync</c> throws)
/// must still account a successful close: both the open-session gauge must drop to zero AND
/// the <c>mxgateway.sessions.closed</c> counter must increment. Without the fix, the
/// graceful-close failure path under-counts the closed counter.
/// graceful-close failure path under-counts the closed counter. Both the shutdown and the
/// in-close kill fail here, because a kill that succeeds is a clean close and never reaches
/// the fallback.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
@@ -935,6 +1045,7 @@ public sealed class SessionManagerTests
FakeWorkerClient throwingClient = new()
{
ShutdownException = new InvalidOperationException("worker shutdown failed"),
KillException = new InvalidOperationException("worker kill failed"),
};
using GatewayMetrics metrics = new();
SessionManager manager = CreateManager(
@@ -978,6 +1089,133 @@ public sealed class SessionManagerTests
Assert.Equal(1, metrics.GetSnapshot().Faults);
}
/// <summary>
/// A failed open must leave no per-session event metrics behind: the rollback goes through
/// the same removal path every other teardown uses, so <c>EventsBySession</c> is cleared
/// for the session that never opened.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task OpenSessionAsync_WhenWorkerCreationFails_RemovesSessionEventMetrics()
{
using GatewayMetrics metrics = new();
EventRecordingFailingSessionWorkerClientFactory factory = new(metrics);
SessionManager manager = CreateManager(factory, metrics: metrics);
await Assert.ThrowsAsync<SessionManagerException>(
async () => await manager.OpenSessionAsync(CreateOpenRequest(), "client-1", ownerKeyId: null, CancellationToken.None));
Assert.NotNull(factory.StartedSessionId);
Assert.False(metrics.GetSnapshot().EventsBySession.ContainsKey(factory.StartedSessionId!));
}
/// <summary>
/// The session is registered before its worker is created, so an admin Kill can land while
/// the worker is still starting (up to <c>StartupTimeoutSeconds</c>). The late worker client
/// must not be attached to the already-closed session and handed back to the caller as if
/// the open had succeeded: it is killed and disposed, the open fails with
/// <c>OpenFailed</c>, and nothing is left in the registry or the metrics.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task OpenSessionAsync_WhenKilledDuringWorkerStartup_FailsOpenAndDiscardsLateWorkerClient()
{
FakeWorkerClient workerClient = new();
GatedSessionWorkerClientFactory factory = new(workerClient);
SessionRegistry registry = new();
using GatewayMetrics metrics = new();
SessionManager manager = CreateManager(factory, registry, metrics, CreateOptions(maxSessions: 1));
Task<GatewaySession> openTask = manager.OpenSessionAsync(CreateOpenRequest(), "client-1", ownerKeyId: null, CancellationToken.None);
GatewaySession starting = await factory.WaitForCreateStartAsync();
SessionCloseResult kill = await manager.KillWorkerAsync(starting.SessionId, "admin-kill", CancellationToken.None);
factory.ReleaseCreate();
SessionManagerException exception = await Assert.ThrowsAsync<SessionManagerException>(async () => await openTask);
Assert.Equal(SessionState.Closed, kill.FinalState);
Assert.Equal(SessionManagerErrorCode.OpenFailed, exception.ErrorCode);
Assert.Equal(1, workerClient.KillCount);
Assert.Equal(1, workerClient.DisposeCount);
Assert.Null(starting.WorkerClient);
Assert.Equal(0, registry.Count);
GatewayMetricsSnapshot snapshot = metrics.GetSnapshot();
Assert.Equal(0, snapshot.OpenSessions);
Assert.Equal(0, snapshot.SessionsOpened);
Assert.Equal(0, snapshot.SessionsClosed);
// The slot the killed open held must have been released exactly once: a replacement open
// succeeds against MaxSessions = 1 (the released gate hands the same fake straight back).
GatewaySession reopened = await manager.OpenSessionAsync(CreateOpenRequest(), "client-2", ownerKeyId: null, CancellationToken.None);
Assert.Equal(SessionState.Ready, reopened.State);
Assert.Equal(1, registry.Count);
}
/// <summary>
/// A caller that abandons its open (deadline, disconnect) while the worker is starting
/// must not leave the worker running: whatever fails between the worker arriving and the
/// session becoming Ready — here the caller's own cancellation — the late worker client is
/// killed and disposed before the failure surfaces.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task OpenSessionAsync_WhenCallerCancelledDuringWorkerStartup_DiscardsWorkerClient()
{
FakeWorkerClient workerClient = new();
GatedSessionWorkerClientFactory factory = new(workerClient);
SessionRegistry registry = new();
using GatewayMetrics metrics = new();
SessionManager manager = CreateManager(factory, registry, metrics);
using CancellationTokenSource cancellation = new();
Task<GatewaySession> openTask = manager.OpenSessionAsync(CreateOpenRequest(), "client-1", ownerKeyId: null, cancellation.Token);
GatewaySession starting = await factory.WaitForCreateStartAsync();
await cancellation.CancelAsync();
factory.ReleaseCreate();
SessionManagerException exception = await Assert.ThrowsAsync<SessionManagerException>(async () => await openTask);
Assert.Equal(SessionManagerErrorCode.OpenFailed, exception.ErrorCode);
Assert.IsAssignableFrom<OperationCanceledException>(exception.InnerException);
Assert.Equal(1, workerClient.KillCount);
Assert.Equal(1, workerClient.DisposeCount);
Assert.NotEqual(SessionState.Ready, starting.State);
Assert.Equal(0, registry.Count);
Assert.Equal(0, metrics.GetSnapshot().OpenSessions);
Assert.Equal(0, metrics.GetSnapshot().SessionsOpened);
}
/// <summary>
/// Same window as the kill case, driven by the host drain: a session still starting its
/// worker when <c>ShutdownAsync</c> runs is closed by the drain, and the worker client that
/// arrives afterwards is discarded rather than left running as an orphan.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task OpenSessionAsync_WhenShutdownDuringWorkerStartup_FailsOpenAndDiscardsLateWorkerClient()
{
FakeWorkerClient workerClient = new();
GatedSessionWorkerClientFactory factory = new(workerClient);
SessionRegistry registry = new();
using GatewayMetrics metrics = new();
SessionManager manager = CreateManager(factory, registry, metrics);
Task<GatewaySession> openTask = manager.OpenSessionAsync(CreateOpenRequest(), "client-1", ownerKeyId: null, CancellationToken.None);
GatewaySession starting = await factory.WaitForCreateStartAsync();
await manager.ShutdownAsync(CancellationToken.None);
factory.ReleaseCreate();
SessionManagerException exception = await Assert.ThrowsAsync<SessionManagerException>(async () => await openTask);
Assert.Equal(SessionManagerErrorCode.OpenFailed, exception.ErrorCode);
Assert.Equal(SessionState.Closed, starting.State);
Assert.Equal(1, workerClient.KillCount);
Assert.Equal(1, workerClient.DisposeCount);
Assert.Equal(0, registry.Count);
Assert.Equal(0, metrics.GetSnapshot().OpenSessions);
Assert.Equal(0, metrics.GetSnapshot().SessionsClosed);
}
/// <summary>Verifies that closing expired leases only closes expired sessions.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
@@ -990,8 +1228,8 @@ public sealed class SessionManagerTests
GatewaySession expiredSession = await manager.OpenSessionAsync(CreateOpenRequest(), "client-1", ownerKeyId: null, CancellationToken.None);
GatewaySession activeSession = await manager.OpenSessionAsync(CreateOpenRequest(), "client-2", ownerKeyId: null, CancellationToken.None);
DateTimeOffset now = DateTimeOffset.UtcNow;
expiredSession.ExtendLease(now.AddSeconds(-1));
activeSession.ExtendLease(now.AddMinutes(5));
SetLeaseExpiry(expiredSession, now.AddSeconds(-1));
SetLeaseExpiry(activeSession, now.AddMinutes(5));
int closedCount = await manager.CloseExpiredLeasesAsync(now, CancellationToken.None);
@@ -1002,6 +1240,243 @@ public sealed class SessionManagerTests
Assert.Equal(0, activeClient.ShutdownCount);
}
/// <summary>
/// Every sweep-driven close — lease expiry, faulted reap and detach-grace expiry — must
/// account <c>SessionClosed</c> exactly once. The sweep begins the close itself
/// (<c>TryBeginCloseIfExpired</c>) before delegating to the shared close path, and that
/// reservation must not make the delegated close look like a second, already-running close
/// whose accounting is skipped: the open-session gauge would then never drop for swept
/// sessions and the closed counter would under-count them.
/// </summary>
/// <param name="sweepKind">Which sweep condition selects the session.</param>
/// <returns>A task that represents the asynchronous operation.</returns>
[Theory]
[InlineData(SessionManager.LeaseExpiredReason)]
[InlineData(SessionManager.FaultedReason)]
[InlineData(SessionManager.DetachGraceExpiredReason)]
public async Task CloseExpiredLeasesAsync_SweptClose_AccountsSessionClosedExactlyOnce(string sweepKind)
{
FakeWorkerClient workerClient = new();
FakeTimeProvider clock = new(DateTimeOffset.UtcNow);
using GatewayMetrics metrics = new();
SessionManager manager = CreateManager(
new FakeSessionWorkerClientFactory(workerClient),
metrics: metrics,
options: CreateOptions(detachGraceSeconds: 30),
timeProvider: clock);
GatewaySession session = await manager.OpenSessionAsync(CreateOpenRequest(), "client-1", ownerKeyId: null, CancellationToken.None);
Assert.Equal(1, metrics.GetSnapshot().OpenSessions);
switch (sweepKind)
{
case SessionManager.LeaseExpiredReason:
SetLeaseExpiry(session, clock.GetUtcNow().AddSeconds(-1));
break;
case SessionManager.FaultedReason:
session.MarkFaulted("test-fault");
break;
case SessionManager.DetachGraceExpiredReason:
session.AttachEventSubscriber(maxSubscribers: 1).Dispose();
clock.Advance(TimeSpan.FromSeconds(31));
break;
default:
throw new ArgumentOutOfRangeException(nameof(sweepKind), sweepKind, "Unknown sweep kind.");
}
int closedCount = await manager.CloseExpiredLeasesAsync(clock.GetUtcNow(), CancellationToken.None);
Assert.Equal(1, closedCount);
Assert.Equal(SessionState.Closed, session.State);
Assert.False(manager.TryGetSession(session.SessionId, out _));
GatewayMetricsSnapshot snapshot = metrics.GetSnapshot();
Assert.Equal(1, snapshot.SessionsClosed);
Assert.Equal(0, snapshot.OpenSessions);
}
/// <summary>
/// With <c>UnattachedSessionGraceSeconds</c> set, a session that never attached an event
/// subscriber and has been idle for the window is reaped by the ordinary lease sweep even
/// though its 30-minute lease is nowhere near expiry. This is the knob for clients that
/// open a session and vanish (OtOpcUa's GalaxyDriver on a failed start): without it each
/// such open pins an x86 worker for <c>DefaultLeaseSeconds</c>.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task CloseExpiredLeasesAsync_ReapsNeverSubscribedSessionAfterUnattachedGrace()
{
FakeWorkerClient workerClient = new();
FakeTimeProvider clock = new(DateTimeOffset.UtcNow);
using GatewayMetrics metrics = new();
SessionManager manager = CreateManager(
new FakeSessionWorkerClientFactory(workerClient),
metrics: metrics,
options: CreateOptions(unattachedSessionGraceSeconds: 60),
timeProvider: clock);
GatewaySession session = await manager.OpenSessionAsync(CreateOpenRequest(), "client-1", ownerKeyId: null, CancellationToken.None);
clock.Advance(TimeSpan.FromSeconds(59));
int closedBefore = await manager.CloseExpiredLeasesAsync(clock.GetUtcNow(), CancellationToken.None);
Assert.Equal(0, closedBefore);
Assert.Equal(SessionState.Ready, session.State);
clock.Advance(TimeSpan.FromSeconds(1));
int closedAfter = await manager.CloseExpiredLeasesAsync(clock.GetUtcNow(), CancellationToken.None);
Assert.Equal(1, closedAfter);
Assert.Equal(SessionState.Closed, session.State);
Assert.Equal(1, workerClient.ShutdownCount);
Assert.False(manager.TryGetSession(session.SessionId, out _));
Assert.Equal(1, metrics.GetSnapshot().SessionsClosed);
Assert.Equal(0, metrics.GetSnapshot().OpenSessions);
}
/// <summary>
/// The window is an idle timeout, not a hard deadline from open: unary activity restarts
/// it, so a client that only ever reads and writes (never streams) is not reaped while it
/// keeps calling.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task CloseExpiredLeasesAsync_UnattachedGrace_IsRestartedByUnaryActivity()
{
FakeWorkerClient workerClient = new();
FakeTimeProvider clock = new(DateTimeOffset.UtcNow);
SessionManager manager = CreateManager(
new FakeSessionWorkerClientFactory(workerClient),
options: CreateOptions(unattachedSessionGraceSeconds: 60),
timeProvider: clock);
GatewaySession session = await manager.OpenSessionAsync(CreateOpenRequest(), "client-1", ownerKeyId: null, CancellationToken.None);
clock.Advance(TimeSpan.FromSeconds(59));
await manager.InvokeAsync(session.SessionId, CreateCommand(MxCommandKind.Ping), CancellationToken.None);
clock.Advance(TimeSpan.FromSeconds(59));
int closedWhileActive = await manager.CloseExpiredLeasesAsync(clock.GetUtcNow(), CancellationToken.None);
Assert.Equal(0, closedWhileActive);
Assert.Equal(SessionState.Ready, session.State);
clock.Advance(TimeSpan.FromSeconds(1));
int closedAfterIdle = await manager.CloseExpiredLeasesAsync(clock.GetUtcNow(), CancellationToken.None);
Assert.Equal(1, closedAfterIdle);
Assert.Equal(SessionState.Closed, session.State);
}
/// <summary>
/// A session that ever had an event subscriber is outside this knob's remit — once it has
/// streamed, detach-grace and the normal lease govern it — and a session with a subscriber
/// attached right now is never idle.
/// </summary>
/// <param name="keepSubscriberAttached">Whether the subscriber is still attached at sweep time.</param>
/// <returns>A task that represents the asynchronous operation.</returns>
[Theory]
[InlineData(true)]
[InlineData(false)]
public async Task CloseExpiredLeasesAsync_UnattachedGrace_DoesNotReapSessionThatSubscribed(bool keepSubscriberAttached)
{
FakeWorkerClient workerClient = new();
FakeTimeProvider clock = new(DateTimeOffset.UtcNow);
SessionManager manager = CreateManager(
new FakeSessionWorkerClientFactory(workerClient),
options: CreateOptions(detachGraceSeconds: 0, unattachedSessionGraceSeconds: 60),
timeProvider: clock);
GatewaySession session = await manager.OpenSessionAsync(CreateOpenRequest(), "client-1", ownerKeyId: null, CancellationToken.None);
IDisposable subscriber = session.AttachEventSubscriber(maxSubscribers: 1);
if (!keepSubscriberAttached)
{
subscriber.Dispose();
}
clock.Advance(TimeSpan.FromSeconds(61));
int closedCount = await manager.CloseExpiredLeasesAsync(clock.GetUtcNow(), CancellationToken.None);
Assert.Equal(0, closedCount);
Assert.Equal(SessionState.Ready, session.State);
Assert.Equal(0, workerClient.ShutdownCount);
subscriber.Dispose();
}
/// <summary>
/// The gateway's own alarm-monitor session only ever attaches an <em>internal</em>
/// (non-counted) subscriber. That must count as "attached" for this rule, or the gateway
/// would reap its own monitor every cycle once the knob is on.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task CloseExpiredLeasesAsync_UnattachedGrace_DoesNotReapSessionWithInternalSubscriber()
{
FakeWorkerClient workerClient = new();
FakeTimeProvider clock = new(DateTimeOffset.UtcNow);
SessionManager manager = CreateManager(
new FakeSessionWorkerClientFactory(workerClient),
options: CreateOptions(unattachedSessionGraceSeconds: 60),
timeProvider: clock);
GatewaySession session = await manager.OpenSessionAsync(CreateOpenRequest(), "client-1", ownerKeyId: null, CancellationToken.None);
using IEventSubscriberLease internalLease = session.AttachInternalEventSubscriber();
clock.Advance(TimeSpan.FromSeconds(61));
int closedCount = await manager.CloseExpiredLeasesAsync(clock.GetUtcNow(), CancellationToken.None);
Assert.Equal(0, closedCount);
Assert.Equal(SessionState.Ready, session.State);
Assert.Equal(0, workerClient.ShutdownCount);
}
/// <summary>
/// The idle clock starts when the session becomes Ready, not when it was opened: a worker
/// that took longer than the window to start must not hand the client a session the very
/// next sweep reaps before it had a chance to subscribe.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task CloseExpiredLeasesAsync_UnattachedGrace_IdleClockStartsAtReady()
{
FakeWorkerClient workerClient = new();
GatedSessionWorkerClientFactory factory = new(workerClient);
FakeTimeProvider clock = new(DateTimeOffset.UtcNow);
SessionManager manager = CreateManager(
factory,
options: CreateOptions(unattachedSessionGraceSeconds: 60),
timeProvider: clock);
Task<GatewaySession> openTask = manager.OpenSessionAsync(CreateOpenRequest(), "client-1", ownerKeyId: null, CancellationToken.None);
await factory.WaitForCreateStartAsync();
clock.Advance(TimeSpan.FromSeconds(120));
factory.ReleaseCreate();
GatewaySession session = await openTask;
int closedRightAfterReady = await manager.CloseExpiredLeasesAsync(clock.GetUtcNow(), CancellationToken.None);
Assert.Equal(0, closedRightAfterReady);
Assert.Equal(SessionState.Ready, session.State);
clock.Advance(TimeSpan.FromSeconds(60));
int closedAfterIdle = await manager.CloseExpiredLeasesAsync(clock.GetUtcNow(), CancellationToken.None);
Assert.Equal(1, closedAfterIdle);
Assert.Equal(SessionState.Closed, session.State);
}
/// <summary>
/// The knob is off by default: a never-subscribed idle session lives for its full lease.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task CloseExpiredLeasesAsync_UnattachedGrace_IsDisabledByDefault()
{
FakeWorkerClient workerClient = new();
FakeTimeProvider clock = new(DateTimeOffset.UtcNow);
SessionManager manager = CreateManager(
new FakeSessionWorkerClientFactory(workerClient),
options: CreateOptions(),
timeProvider: clock);
GatewaySession session = await manager.OpenSessionAsync(CreateOpenRequest(), "client-1", ownerKeyId: null, CancellationToken.None);
clock.Advance(TimeSpan.FromMinutes(20));
int closedCount = await manager.CloseExpiredLeasesAsync(clock.GetUtcNow(), CancellationToken.None);
Assert.Equal(0, closedCount);
Assert.Equal(SessionState.Ready, session.State);
}
/// <summary>Verifies that an expired-lease sweep leaves a session with an active event subscriber open.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
@@ -1011,7 +1486,7 @@ public sealed class SessionManagerTests
SessionManager manager = CreateManager(new FakeSessionWorkerClientFactory(workerClient));
GatewaySession session = await manager.OpenSessionAsync(CreateOpenRequest(), "client-1", ownerKeyId: null, CancellationToken.None);
DateTimeOffset now = DateTimeOffset.UtcNow;
session.ExtendLease(now.AddSeconds(-1));
SetLeaseExpiry(session, now.AddSeconds(-1));
using IDisposable eventSubscriber = session.AttachEventSubscriber(maxSubscribers: 1);
int closedCount = await manager.CloseExpiredLeasesAsync(now, CancellationToken.None);
@@ -1118,8 +1593,8 @@ public sealed class SessionManagerTests
DateTimeOffset now = DateTimeOffset.UtcNow;
// Both leases are far in the future, so only the fault can reap the faulted session.
faultedSession.ExtendLease(now.AddMinutes(30));
healthySession.ExtendLease(now.AddMinutes(30));
SetLeaseExpiry(faultedSession, now.AddMinutes(30));
SetLeaseExpiry(healthySession, now.AddMinutes(30));
faultedSession.MarkFaulted("test fault");
int closedCount = await manager.CloseExpiredLeasesAsync(now, CancellationToken.None);
@@ -1180,8 +1655,8 @@ public sealed class SessionManagerTests
GatewaySession firstSession = await manager.OpenSessionAsync(CreateOpenRequest(), "client-1", ownerKeyId: null, CancellationToken.None);
GatewaySession secondSession = await manager.OpenSessionAsync(CreateOpenRequest(), "client-2", ownerKeyId: null, CancellationToken.None);
DateTimeOffset now = DateTimeOffset.UtcNow;
firstSession.ExtendLease(now.AddSeconds(-1));
secondSession.ExtendLease(now.AddSeconds(-1));
SetLeaseExpiry(firstSession, now.AddSeconds(-1));
SetLeaseExpiry(secondSession, now.AddSeconds(-1));
int closedCount = await manager.CloseExpiredLeasesAsync(now, CancellationToken.None);
@@ -1233,8 +1708,8 @@ public sealed class SessionManagerTests
GatewaySession failingSession = await manager.OpenSessionAsync(CreateOpenRequest(), "client-1", ownerKeyId: null, CancellationToken.None);
GatewaySession healthySession = await manager.OpenSessionAsync(CreateOpenRequest(), "client-2", ownerKeyId: null, CancellationToken.None);
DateTimeOffset now = DateTimeOffset.UtcNow;
failingSession.ExtendLease(now.AddSeconds(-1));
healthySession.ExtendLease(now.AddSeconds(-1));
SetLeaseExpiry(failingSession, now.AddSeconds(-1));
SetLeaseExpiry(healthySession, now.AddSeconds(-1));
SessionManagerException exception = await Assert.ThrowsAsync<SessionManagerException>(
async () => await manager.CloseExpiredLeasesAsync(now, CancellationToken.None));
@@ -1273,6 +1748,84 @@ public sealed class SessionManagerTests
Assert.False(manager.TryGetSession(secondSession.SessionId, out _));
}
/// <summary>
/// Once the drain has started, a new open must be refused with <c>GatewayStopping</c>
/// rather than accepted: a session created behind the drain's snapshot has nobody left to
/// close it, and its worker would outlive the gateway as an orphan. The refusal is not an
/// open failure — no slot is taken, nothing is registered and no fault is recorded.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task OpenSessionAsync_AfterShutdownStarted_ThrowsGatewayStopping()
{
FakeWorkerClient workerClient = new();
SessionRegistry registry = new();
using GatewayMetrics metrics = new();
SessionManager manager = CreateManager(new FakeSessionWorkerClientFactory(workerClient), registry, metrics);
await manager.ShutdownAsync(CancellationToken.None);
SessionManagerException exception = await Assert.ThrowsAsync<SessionManagerException>(
async () => await manager.OpenSessionAsync(CreateOpenRequest(), "client-1", ownerKeyId: null, CancellationToken.None));
Assert.Equal(SessionManagerErrorCode.GatewayStopping, exception.ErrorCode);
Assert.Equal(0, registry.Count);
Assert.Equal(0, metrics.GetSnapshot().SessionsOpened);
Assert.Equal(0, metrics.GetSnapshot().Faults);
}
/// <summary>
/// The host signals <c>ApplicationStopping</c> before hosted services stop, and Kestrel keeps
/// accepting gRPC calls until its own <c>StopAsync</c> runs — so the gate must close on the
/// lifetime signal, not only when <c>ShutdownAsync</c> is finally called.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task OpenSessionAsync_WhenApplicationStopping_ThrowsGatewayStopping()
{
FakeWorkerClient workerClient = new();
using FakeHostApplicationLifetime lifetime = new();
SessionManager manager = new(
new SessionRegistry(),
new FakeSessionWorkerClientFactory(workerClient),
Options.Create(CreateOptions()),
new GatewayMetrics(),
applicationLifetime: lifetime);
GatewaySession beforeStopping = await manager.OpenSessionAsync(CreateOpenRequest(), "client-1", ownerKeyId: null, CancellationToken.None);
Assert.Equal(SessionState.Ready, beforeStopping.State);
lifetime.SignalStopping();
SessionManagerException exception = await Assert.ThrowsAsync<SessionManagerException>(
async () => await manager.OpenSessionAsync(CreateOpenRequest(), "client-2", ownerKeyId: null, CancellationToken.None));
Assert.Equal(SessionManagerErrorCode.GatewayStopping, exception.ErrorCode);
}
/// <summary>
/// An open that passed the stopping gate but registered its session after the drain took
/// its snapshot is still caught: the drain re-snapshots until the registry is empty, so the
/// late session is closed by the drain rather than left running.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task ShutdownAsync_DrainsSessionRegisteredAfterFirstSnapshot()
{
FakeWorkerClient workerClient = new();
LateFirstSnapshotRegistry registry = new();
using GatewayMetrics metrics = new();
SessionManager manager = CreateManager(new FakeSessionWorkerClientFactory(workerClient), registry, metrics);
GatewaySession session = await manager.OpenSessionAsync(CreateOpenRequest(), "client-1", ownerKeyId: null, CancellationToken.None);
await manager.ShutdownAsync(CancellationToken.None);
Assert.True(registry.SnapshotCalls >= 2);
Assert.Equal(SessionState.Closed, session.State);
Assert.Equal(1, workerClient.ShutdownCount);
Assert.Equal(0, registry.Count);
Assert.Equal(0, metrics.GetSnapshot().OpenSessions);
Assert.Equal(1, metrics.GetSnapshot().SessionsClosed);
}
/// <summary>Verifies that shutdown closes all registered sessions.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
@@ -1322,7 +1875,8 @@ public sealed class SessionManagerTests
int defaultLeaseSeconds = 1800,
int detachGraceSeconds = 0,
int workerReadyWaitTimeoutMs = 0,
int faultedGraceSeconds = 0)
int faultedGraceSeconds = 0,
int unattachedSessionGraceSeconds = 0)
{
return new GatewayOptions
{
@@ -1334,6 +1888,7 @@ public sealed class SessionManagerTests
DetachGraceSeconds = detachGraceSeconds,
WorkerReadyWaitTimeoutMs = workerReadyWaitTimeoutMs,
FaultedGraceSeconds = faultedGraceSeconds,
UnattachedSessionGraceSeconds = unattachedSessionGraceSeconds,
},
Worker = new WorkerOptions
{
@@ -1343,6 +1898,17 @@ public sealed class SessionManagerTests
};
}
/// <summary>
/// Moves a session's lease expiry by back-dating its last client activity, which is the only
/// production path that moves the lease (<c>_leaseExpiresAt = activityAt + LeaseDuration</c>).
/// </summary>
/// <param name="session">Session whose lease to move.</param>
/// <param name="leaseExpiresAt">Wanted lease expiry.</param>
private static void SetLeaseExpiry(GatewaySession session, DateTimeOffset leaseExpiresAt)
{
session.TouchClientActivity(leaseExpiresAt - session.LeaseDuration);
}
private static SessionOpenRequest CreateOpenRequest()
{
return new SessionOpenRequest(
@@ -1422,6 +1988,203 @@ public sealed class SessionManagerTests
}
}
/// <summary>
/// Factory that records a per-session event metric for the session it is starting and
/// then fails, so a test can check the open rollback clears that metric.
/// </summary>
private sealed class EventRecordingFailingSessionWorkerClientFactory(GatewayMetrics metrics) : ISessionWorkerClientFactory
{
/// <summary>Gets the id of the session whose worker creation was attempted.</summary>
public string? StartedSessionId { get; private set; }
/// <inheritdoc />
public Task<IWorkerClient> CreateAsync(
GatewaySession session,
CancellationToken cancellationToken)
{
StartedSessionId = session.SessionId;
metrics.EventReceived(session.SessionId, MxEventFamily.OnDataChange.ToString());
throw new InvalidOperationException("worker startup failed");
}
}
/// <summary>
/// Factory whose <see cref="CreateAsync"/> parks until the test releases it, so the test can
/// act on the registered-but-still-starting session (kill it, drain the manager) inside the
/// window between registry insertion and worker attach. Once released it hands the same
/// client to every later call.
/// </summary>
private sealed class GatedSessionWorkerClientFactory(IWorkerClient workerClient) : ISessionWorkerClientFactory
{
private readonly TaskCompletionSource<GatewaySession> _entered = new(TaskCreationOptions.RunContinuationsAsynchronously);
private readonly TaskCompletionSource _release = new(TaskCreationOptions.RunContinuationsAsynchronously);
/// <summary>Waits until the manager has asked for a worker and returns the starting session.</summary>
/// <returns>The session whose worker is being created.</returns>
public Task<GatewaySession> WaitForCreateStartAsync()
{
return _entered.Task.WaitAsync(TimeSpan.FromSeconds(5));
}
/// <summary>Lets the parked worker creation complete.</summary>
public void ReleaseCreate()
{
_release.TrySetResult();
}
/// <inheritdoc />
public async Task<IWorkerClient> CreateAsync(
GatewaySession session,
CancellationToken cancellationToken)
{
// Deliberately not bound to the caller's token: the tests that cancel the open while
// the worker is starting need the worker to still ARRIVE after the cancellation, which
// is the orphan-prone case.
_entered.TrySetResult(session);
await _release.Task.WaitAsync(TimeSpan.FromSeconds(5));
return workerClient;
}
}
/// <summary>
/// Registry that can replay a race the real registry only produces by timing: a session a
/// concurrent close has already removed and disposed is still returned by one lookup (the
/// kill looked it up just before the removal) or still appears in one snapshot (the sweep
/// snapshotted just before the removal).
/// </summary>
private sealed class StaleRegistry : ISessionRegistry
{
private readonly SessionRegistry _inner = new();
private GatewaySession? _staleLookup;
private GatewaySession? _staleSnapshot;
/// <summary>Makes the next lookup of <paramref name="session"/>'s id return it although it was removed.</summary>
/// <param name="session">Removed session to serve once.</param>
public void ServeStaleLookupOnce(GatewaySession session)
{
_staleLookup = session;
}
/// <summary>Makes the next snapshot include <paramref name="session"/> although it was removed.</summary>
/// <param name="session">Removed session to include once.</param>
public void ServeStaleSnapshot(GatewaySession session)
{
_staleSnapshot = session;
}
/// <inheritdoc />
public int Count => _inner.Count;
/// <inheritdoc />
public int ActiveCount => _inner.ActiveCount;
/// <inheritdoc />
public bool TryAdd(GatewaySession session) => _inner.TryAdd(session);
/// <inheritdoc />
public bool TryGet(string sessionId, [System.Diagnostics.CodeAnalysis.MaybeNullWhen(false)] out GatewaySession session)
{
if (_staleLookup is not null && _staleLookup.SessionId == sessionId)
{
session = _staleLookup;
_staleLookup = null;
return true;
}
return _inner.TryGet(sessionId, out session);
}
/// <inheritdoc />
public bool TryRemove(string sessionId, [System.Diagnostics.CodeAnalysis.MaybeNullWhen(false)] out GatewaySession session)
=> _inner.TryRemove(sessionId, out session);
/// <inheritdoc />
public IReadOnlyCollection<GatewaySession> Snapshot()
{
if (_staleSnapshot is null)
{
return _inner.Snapshot();
}
GatewaySession[] snapshot = [.. _inner.Snapshot(), _staleSnapshot];
_staleSnapshot = null;
return snapshot;
}
}
/// <summary>
/// Lifetime fake whose <see cref="ApplicationStopping"/> the test fires on demand; the
/// other two tokens never fire.
/// </summary>
private sealed class FakeHostApplicationLifetime : Microsoft.Extensions.Hosting.IHostApplicationLifetime, IDisposable
{
private readonly CancellationTokenSource _stopping = new();
/// <inheritdoc />
public CancellationToken ApplicationStarted => CancellationToken.None;
/// <inheritdoc />
public CancellationToken ApplicationStopping => _stopping.Token;
/// <inheritdoc />
public CancellationToken ApplicationStopped => CancellationToken.None;
/// <summary>Fires <see cref="ApplicationStopping"/>.</summary>
public void SignalStopping()
{
_stopping.Cancel();
}
/// <inheritdoc />
public void StopApplication()
{
SignalStopping();
}
/// <inheritdoc />
public void Dispose()
{
_stopping.Dispose();
}
}
/// <summary>
/// Registry whose first <see cref="Snapshot"/> is empty regardless of its contents,
/// simulating a session whose <c>TryAdd</c> landed just after the drain's first snapshot;
/// every later snapshot is the real one.
/// </summary>
private sealed class LateFirstSnapshotRegistry : ISessionRegistry
{
private readonly SessionRegistry _inner = new();
/// <summary>Gets how many snapshots have been taken.</summary>
public int SnapshotCalls { get; private set; }
/// <inheritdoc />
public int Count => _inner.Count;
/// <inheritdoc />
public int ActiveCount => _inner.ActiveCount;
/// <inheritdoc />
public bool TryAdd(GatewaySession session) => _inner.TryAdd(session);
/// <inheritdoc />
public bool TryGet(string sessionId, [System.Diagnostics.CodeAnalysis.MaybeNullWhen(false)] out GatewaySession session)
=> _inner.TryGet(sessionId, out session);
/// <inheritdoc />
public bool TryRemove(string sessionId, [System.Diagnostics.CodeAnalysis.MaybeNullWhen(false)] out GatewaySession session)
=> _inner.TryRemove(sessionId, out session);
/// <inheritdoc />
public IReadOnlyCollection<GatewaySession> Snapshot()
{
SnapshotCalls++;
return SnapshotCalls == 1 ? [] : _inner.Snapshot();
}
}
private sealed class FakeWorkerClient : IWorkerClient
{
private WorkerClientState _state = WorkerClientState.Ready;