fix(sessions): refuse a dead worker at attach, fault on pump readiness failure, keep cancel/stopping out of OpenFailed
- TryAttachWorkerClientAndMarkReadyAsync returns SessionAttachOutcome and refuses a worker client that is already Faulted/Closed; the open discards it and fails OpenFailed. - MapWorkerEventsAsync calls ObserveWorkerTerminalState when its first readiness check fails, so a worker that dies between attach and the first pump read faults the session and the faulted sweep reaps it. - A caller-cancelled open rolls back, records no fault and rethrows the cancellation (gRPC Cancelled); an attach refused because the drain closed the session reports GatewayStopping. - docs/Sessions.md updated.
This commit is contained in:
@@ -88,6 +88,78 @@ public sealed class GatewaySessionWorkerExitTests
|
||||
Assert.Equal(0, metrics.GetSnapshot().OpenSessions);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A worker that dies right after its handshake (STA loop fault, exit code 7) can reach
|
||||
/// <c>Faulted</c> before the open attaches it. The attach must refuse such a client: the
|
||||
/// open fails with <c>OpenFailed</c>, the client is discarded, and no session is left
|
||||
/// registered reporting <c>Ready</c> over a dead worker until its lease expires.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task OpenSession_WorkerAlreadyFaultedAtAttach_FailsOpenAndDiscardsClient()
|
||||
{
|
||||
await using FakeWorkerHarness fakeWorker = await FakeWorkerHarness.CreateConnectedPairAsync();
|
||||
using GatewayMetrics metrics = new();
|
||||
WorkerClient client = fakeWorker.CreateClient(metrics: metrics);
|
||||
await StartClientAsync(fakeWorker, client);
|
||||
await fakeWorker.DisposeWorkerSideAsync();
|
||||
await WaitUntilAsync(() => client.State == WorkerClientState.Faulted);
|
||||
SessionRegistry registry = new();
|
||||
SessionManager manager = new(
|
||||
registry,
|
||||
new SingleClientFactory(client),
|
||||
Options.Create(new GatewayOptions()),
|
||||
metrics);
|
||||
|
||||
SessionManagerException exception = await Assert.ThrowsAsync<SessionManagerException>(
|
||||
async () => await OpenSessionAsync(manager));
|
||||
|
||||
Assert.Equal(SessionManagerErrorCode.OpenFailed, exception.ErrorCode);
|
||||
Assert.Contains("Faulted", exception.InnerException?.Message, StringComparison.Ordinal);
|
||||
Assert.Equal(0, registry.Count);
|
||||
GatewayMetricsSnapshot snapshot = metrics.GetSnapshot();
|
||||
Assert.Equal(0, snapshot.OpenSessions);
|
||||
Assert.Equal(0, snapshot.SessionsOpened);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The worker is still <c>Ready</c> when the open attaches it but faults before the event
|
||||
/// pump's first read (which the dashboard mirror starts at <c>MarkReady</c>). The pump's
|
||||
/// readiness check then fails without ever reaching the worker event stream, so it is that
|
||||
/// failure path that must notice the dead worker: the session follows it to
|
||||
/// <c>Faulted</c> and the faulted-session sweep reaps it, accounting the close once.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task WorkerFaultsBetweenAttachAndFirstPumpRead_SessionFaultsAndSweepReapsItOnce()
|
||||
{
|
||||
using GatewayMetrics metrics = new();
|
||||
ReadyOnceWorkerClient client = new();
|
||||
SessionManager manager = new(
|
||||
new SessionRegistry(),
|
||||
new SingleClientFactory(client),
|
||||
Options.Create(new GatewayOptions()),
|
||||
metrics,
|
||||
dashboardEventBroadcaster: TestSupport.NullDashboardEventBroadcaster.Instance);
|
||||
|
||||
GatewaySession session = await OpenSessionAsync(manager);
|
||||
|
||||
await WaitUntilAsync(() => session.State == SessionState.Faulted);
|
||||
Assert.Contains("worker", session.FinalFault, StringComparison.OrdinalIgnoreCase);
|
||||
Assert.Equal(1, metrics.GetSnapshot().OpenSessions);
|
||||
|
||||
int closedCount = await manager.CloseExpiredLeasesAsync(DateTimeOffset.UtcNow, CancellationToken.None);
|
||||
int secondPass = await manager.CloseExpiredLeasesAsync(DateTimeOffset.UtcNow, CancellationToken.None);
|
||||
|
||||
Assert.Equal(1, closedCount);
|
||||
Assert.Equal(0, secondPass);
|
||||
Assert.Equal(SessionState.Closed, session.State);
|
||||
Assert.False(manager.TryGetSession(session.SessionId, out _));
|
||||
GatewayMetricsSnapshot snapshot = metrics.GetSnapshot();
|
||||
Assert.Equal(1, snapshot.SessionsClosed);
|
||||
Assert.Equal(0, snapshot.OpenSessions);
|
||||
}
|
||||
|
||||
private static async Task StartClientAsync(FakeWorkerHarness fakeWorker, WorkerClient client)
|
||||
{
|
||||
Task startTask = client.StartAsync(CancellationToken.None);
|
||||
@@ -148,4 +220,56 @@ public sealed class GatewaySessionWorkerExitTests
|
||||
return Task.FromResult(client);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Worker client that reports <c>Ready</c> for its first state read (the open's attach
|
||||
/// check) and <c>Faulted</c> from then on, reproducing a worker that dies in the window
|
||||
/// between the attach and the event pump's first readiness check.
|
||||
/// </summary>
|
||||
private sealed class ReadyOnceWorkerClient : IWorkerClient
|
||||
{
|
||||
private int _stateReads;
|
||||
|
||||
/// <inheritdoc />
|
||||
public string SessionId => "session-ready-once";
|
||||
|
||||
/// <inheritdoc />
|
||||
public int? ProcessId => 4321;
|
||||
|
||||
/// <inheritdoc />
|
||||
public WorkerClientState State
|
||||
=> Interlocked.Increment(ref _stateReads) == 1 ? WorkerClientState.Ready : WorkerClientState.Faulted;
|
||||
|
||||
/// <inheritdoc />
|
||||
public DateTimeOffset LastHeartbeatAt => DateTimeOffset.UtcNow;
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task StartAsync(CancellationToken cancellationToken) => Task.CompletedTask;
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task<WorkerCommandReply> InvokeAsync(WorkerCommand command, TimeSpan timeout, CancellationToken cancellationToken)
|
||||
=> throw new InvalidOperationException("The worker is faulted.");
|
||||
|
||||
/// <inheritdoc />
|
||||
public async IAsyncEnumerable<WorkerEvent> ReadEventsAsync(
|
||||
[System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken)
|
||||
{
|
||||
await Task.CompletedTask;
|
||||
throw new InvalidOperationException("The pump must not reach the event stream of a faulted worker.");
|
||||
#pragma warning disable CS0162 // Unreachable code: the yield makes this an async iterator.
|
||||
yield break;
|
||||
#pragma warning restore CS0162
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task ShutdownAsync(TimeSpan timeout, CancellationToken cancellationToken) => Task.CompletedTask;
|
||||
|
||||
/// <inheritdoc />
|
||||
public void Kill(string reason)
|
||||
{
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public ValueTask DisposeAsync() => ValueTask.CompletedTask;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -533,13 +533,17 @@ public sealed class SessionManagerTests
|
||||
// which flaked under CI load) so a regression that waited out the timeout is caught
|
||||
// while ordinary scheduling jitter never trips it.
|
||||
const int readyWaitTimeoutMs = 5000;
|
||||
FakeWorkerClient workerClient = new() { State = WorkerClientState.Faulted };
|
||||
FakeWorkerClient workerClient = new();
|
||||
SessionManager manager = CreateManager(
|
||||
new FakeSessionWorkerClientFactory(workerClient),
|
||||
options: CreateOptions(workerReadyWaitTimeoutMs: readyWaitTimeoutMs));
|
||||
GatewaySession session = await manager.OpenSessionAsync(CreateOpenRequest(), "client-1", ownerKeyId: null, CancellationToken.None);
|
||||
Assert.Equal(SessionState.Ready, session.State);
|
||||
|
||||
// The worker faults after the open attached it (an open refuses an already-faulted
|
||||
// worker). No event pump runs here, so nothing has moved the session off Ready yet.
|
||||
workerClient.State = WorkerClientState.Faulted;
|
||||
|
||||
Stopwatch stopwatch = Stopwatch.StartNew();
|
||||
SessionManagerException exception = await Assert.ThrowsAsync<SessionManagerException>(
|
||||
async () => await manager.InvokeAsync(
|
||||
@@ -1145,7 +1149,9 @@ public sealed class SessionManagerTests
|
||||
Assert.Equal(0, snapshot.SessionsClosed);
|
||||
|
||||
// The slot the killed open held must have been released exactly once: a replacement open
|
||||
// succeeds against MaxSessions = 1 (the released gate hands the same fake straight back).
|
||||
// succeeds against MaxSessions = 1 (the released gate hands the same fake straight back,
|
||||
// so undo the discard's kill first — an open refuses an already-faulted worker).
|
||||
workerClient.State = WorkerClientState.Ready;
|
||||
GatewaySession reopened = await manager.OpenSessionAsync(CreateOpenRequest(), "client-2", ownerKeyId: null, CancellationToken.None);
|
||||
Assert.Equal(SessionState.Ready, reopened.State);
|
||||
Assert.Equal(1, registry.Count);
|
||||
@@ -1155,11 +1161,13 @@ public sealed class SessionManagerTests
|
||||
/// A caller that abandons its open (deadline, disconnect) while the worker is starting
|
||||
/// must not leave the worker running: whatever fails between the worker arriving and the
|
||||
/// session becoming Ready — here the caller's own cancellation — the late worker client is
|
||||
/// killed and disposed before the failure surfaces.
|
||||
/// killed and disposed before the failure surfaces. The cancellation is not an open
|
||||
/// failure: it surfaces as the <see cref="OperationCanceledException"/> itself (which the
|
||||
/// gRPC layer maps to <c>Cancelled</c>) and records no fault.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task OpenSessionAsync_WhenCallerCancelledDuringWorkerStartup_DiscardsWorkerClient()
|
||||
public async Task OpenSessionAsync_WhenCallerCancelledDuringWorkerStartup_DiscardsWorkerClientAndRethrowsCancellation()
|
||||
{
|
||||
FakeWorkerClient workerClient = new();
|
||||
GatedSessionWorkerClientFactory factory = new(workerClient);
|
||||
@@ -1173,10 +1181,9 @@ public sealed class SessionManagerTests
|
||||
|
||||
await cancellation.CancelAsync();
|
||||
factory.ReleaseCreate();
|
||||
SessionManagerException exception = await Assert.ThrowsAsync<SessionManagerException>(async () => await openTask);
|
||||
await Assert.ThrowsAnyAsync<OperationCanceledException>(async () => await openTask);
|
||||
|
||||
Assert.Equal(SessionManagerErrorCode.OpenFailed, exception.ErrorCode);
|
||||
Assert.IsAssignableFrom<OperationCanceledException>(exception.InnerException);
|
||||
Assert.Equal(0, metrics.GetSnapshot().Faults);
|
||||
Assert.Equal(1, workerClient.KillCount);
|
||||
Assert.Equal(1, workerClient.DisposeCount);
|
||||
Assert.NotEqual(SessionState.Ready, starting.State);
|
||||
@@ -1188,11 +1195,13 @@ public sealed class SessionManagerTests
|
||||
/// <summary>
|
||||
/// Same window as the kill case, driven by the host drain: a session still starting its
|
||||
/// worker when <c>ShutdownAsync</c> runs is closed by the drain, and the worker client that
|
||||
/// arrives afterwards is discarded rather than left running as an orphan.
|
||||
/// arrives afterwards is discarded rather than left running as an orphan. The refused open
|
||||
/// reports <c>GatewayStopping</c> (gRPC <c>Unavailable</c>, retry against the restarted
|
||||
/// gateway), like an open refused at the door, and is not recorded as a fault.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task OpenSessionAsync_WhenShutdownDuringWorkerStartup_FailsOpenAndDiscardsLateWorkerClient()
|
||||
public async Task OpenSessionAsync_WhenShutdownDuringWorkerStartup_ThrowsGatewayStoppingAndDiscardsLateWorkerClient()
|
||||
{
|
||||
FakeWorkerClient workerClient = new();
|
||||
GatedSessionWorkerClientFactory factory = new(workerClient);
|
||||
@@ -1207,7 +1216,8 @@ public sealed class SessionManagerTests
|
||||
factory.ReleaseCreate();
|
||||
SessionManagerException exception = await Assert.ThrowsAsync<SessionManagerException>(async () => await openTask);
|
||||
|
||||
Assert.Equal(SessionManagerErrorCode.OpenFailed, exception.ErrorCode);
|
||||
Assert.Equal(SessionManagerErrorCode.GatewayStopping, exception.ErrorCode);
|
||||
Assert.Equal(0, metrics.GetSnapshot().Faults);
|
||||
Assert.Equal(SessionState.Closed, starting.State);
|
||||
Assert.Equal(1, workerClient.KillCount);
|
||||
Assert.Equal(1, workerClient.DisposeCount);
|
||||
|
||||
Reference in New Issue
Block a user