Merge fix/c3-session-lifecycle: GWC-32 WorkerClient dispose is terminal, GWC-34 attach-after-dispose throws SessionNotReady, GWC-37 subtag watch-list coverage graded on provider status + re-resolved at failover edge
This commit is contained in:
+6
-3
@@ -179,8 +179,10 @@ public sealed class DashboardBrowseAndAlarmModelTests
|
||||
|
||||
/// <summary>
|
||||
/// An explicitly-degraded status whose mode is still Alarmmgr (the
|
||||
/// <c>Degraded || Mode==Subtag</c> guard's second, independent branch) must still
|
||||
/// map to the degraded amber badge.
|
||||
/// <c>Degraded || Mode==Subtag</c> guard's second, independent branch) must map to the
|
||||
/// amber badge, but not to the failover label: GWC-37 gives that combination a meaning —
|
||||
/// the alarm manager is serving and the subtag standby behind it covers nothing — so
|
||||
/// reporting "Subtag monitoring (degraded)" would name the wrong live provider.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void FromProviderStatus_Alarmmgr_DegradedFlagSet_WarningBadge()
|
||||
@@ -195,8 +197,9 @@ public sealed class DashboardBrowseAndAlarmModelTests
|
||||
DashboardAlarmProviderStatus model = DashboardAlarmProviderStatus.FromProviderStatus(status);
|
||||
|
||||
Assert.True(model.IsDegraded);
|
||||
Assert.Equal(DashboardAlarmProviderStatus.DegradedLabel, model.Label);
|
||||
Assert.Equal(DashboardAlarmProviderStatus.NoSubtagStandbyLabel, model.Label);
|
||||
Assert.Contains("bg-warning", model.BadgeCssClass, StringComparison.Ordinal);
|
||||
Assert.Equal(AlarmProviderMode.Alarmmgr, model.Mode);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
||||
@@ -719,6 +719,47 @@ public sealed class GatewaySessionTests
|
||||
Assert.Equal([1UL, 2UL], received.Select(mxEvent => mxEvent.WorkerSequence).ToArray());
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// GWC-34, the disposal-side twin of the GWC-27 premature-attach poison: an attach that
|
||||
/// passed its readiness gate and then lost the CPU until the session was torn down used to
|
||||
/// find <c>_eventDistributor</c> null, read that as "not created yet" and build a
|
||||
/// REPLACEMENT distributor. Its pump could never start (the started flag is a one-way latch)
|
||||
/// and its channel was never completed, so the subscriber registered on it and blocked in
|
||||
/// <c>WaitToReadAsync</c> until its own deadline — no error, no log, no metric. Disposal now
|
||||
/// latches the null-out, so every attach entry point fails immediately and loudly instead.
|
||||
/// Attaching twice pins the rollback: the second call must still report
|
||||
/// <c>SessionNotReady</c>, not <c>EventSubscriberAlreadyActive</c>, which it only can if the
|
||||
/// first failure released the subscriber slot it took.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task AttachAfterDisposeThrowsSessionNotReadyInsteadOfRegisteringOnANeverPumpedDistributor()
|
||||
{
|
||||
FakeWorkerClient workerClient = new();
|
||||
GatewaySession session = CreateReadySession(workerClient);
|
||||
|
||||
// Disposal leaves the session state Ready (only CloseAsync transitions it), so the attach
|
||||
// paths' readiness gates pass and the calls below genuinely reach EnsureDistributorCreated —
|
||||
// the exact position the racing attach in the finding occupies.
|
||||
await session.DisposeAsync();
|
||||
|
||||
SessionManagerException attachException = Assert.Throws<SessionManagerException>(
|
||||
() => session.AttachEventSubscriber(maxSubscribers: 1));
|
||||
Assert.Equal(SessionManagerErrorCode.SessionNotReady, attachException.ErrorCode);
|
||||
|
||||
SessionManagerException secondAttachException = Assert.Throws<SessionManagerException>(
|
||||
() => session.AttachEventSubscriber(maxSubscribers: 1));
|
||||
Assert.Equal(SessionManagerErrorCode.SessionNotReady, secondAttachException.ErrorCode);
|
||||
|
||||
SessionManagerException replayException = Assert.Throws<SessionManagerException>(
|
||||
() => session.AttachEventSubscriberWithReplay(maxSubscribers: 1, afterSequence: 0));
|
||||
Assert.Equal(SessionManagerErrorCode.SessionNotReady, replayException.ErrorCode);
|
||||
|
||||
SessionManagerException internalException = Assert.Throws<SessionManagerException>(
|
||||
() => session.AttachInternalEventSubscriber());
|
||||
Assert.Equal(SessionManagerErrorCode.SessionNotReady, internalException.ErrorCode);
|
||||
}
|
||||
|
||||
private static GatewaySession CreateSession()
|
||||
{
|
||||
return new GatewaySession(
|
||||
|
||||
@@ -14,6 +14,10 @@ public sealed class WorkerClientTests
|
||||
private const string SessionId = "session-worker-client";
|
||||
private const string Nonce = "nonce-worker-client";
|
||||
private const int WorkerProcessId = 4321;
|
||||
|
||||
/// <summary>Upper bound on the events <see cref="EmitEventsUntilStoppedAsync"/> writes.</summary>
|
||||
private const ulong EmittedEventCeiling = 2000;
|
||||
|
||||
private static readonly TimeSpan TestTimeout = TimeSpan.FromSeconds(5);
|
||||
|
||||
/// <summary>Verifies that StartAsync enters ready state after receiving worker hello and ready messages.</summary>
|
||||
@@ -705,6 +709,96 @@ public sealed class WorkerClientTests
|
||||
Assert.True(process.Disposed);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// GWC-32: disposing a still-<c>Ready</c> client while the worker is emitting events must be
|
||||
/// an ordinary teardown, not a fault. <c>DisposeAsync</c> completes the same three channels
|
||||
/// <c>MarkClosed</c> and <c>SetFaulted</c> do, so it must enter a terminal state first;
|
||||
/// otherwise the read loop — already inside <c>DispatchEnvelope</c> when the staging channel
|
||||
/// is completed, or reading a stream that is torn down underneath it — misdiagnoses the
|
||||
/// completed channel as the GWC-24 sustained-backpressure overflow and raises a spurious
|
||||
/// <c>ProtocolViolation</c> fault, a <c>worker-event-staging</c> overflow metric and a
|
||||
/// redundant kill during an intentional dispose. The event channel is sized far above the
|
||||
/// burst so a genuine overflow is impossible, and <c>DisposeAsync</c> awaits its background
|
||||
/// tasks, so the metrics read afterwards is not racing a late fault.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task DisposeAsync_WhileWorkerIsEmittingEvents_ClosesWithoutFaultingOrOverflowing()
|
||||
{
|
||||
// The burst is capped at EmittedEventCeiling and the staging channel holds 2 x capacity, so
|
||||
// even if every emitted event were staged and none consumed the GWC-24 bound is unreachable:
|
||||
// an overflow metric in this test can only be the dispose-race misdiagnosis.
|
||||
const int capacity = 2048;
|
||||
using GatewayMetrics metrics = new();
|
||||
await using PipePair pipePair = await PipePair.CreateAsync();
|
||||
WorkerClient client = CreateClient(
|
||||
pipePair,
|
||||
new WorkerClientOptions
|
||||
{
|
||||
EventChannelCapacity = capacity,
|
||||
EventChannelFullModeTimeout = TimeSpan.FromMinutes(5),
|
||||
HeartbeatGrace = TimeSpan.FromSeconds(30),
|
||||
HeartbeatCheckInterval = TimeSpan.FromSeconds(30),
|
||||
},
|
||||
metrics: metrics);
|
||||
await CompleteHandshakeAsync(client, pipePair);
|
||||
|
||||
using CancellationTokenSource emitCts = new(TestTimeout);
|
||||
Task emitTask = EmitEventsUntilStoppedAsync(pipePair, emitCts.Token);
|
||||
|
||||
// Proves the read loop is mid-stream — not parked before its first event — when dispose runs.
|
||||
await WaitUntilAsync(() => metrics.GetSnapshot().WorkerEventQueueDepth > 0, TestTimeout);
|
||||
|
||||
await client.DisposeAsync().AsTask().WaitAsync(TestTimeout);
|
||||
await emitCts.CancelAsync();
|
||||
await emitTask.WaitAsync(TestTimeout);
|
||||
|
||||
Assert.Equal(WorkerClientState.Closed, client.State);
|
||||
|
||||
GatewayMetricsSnapshot snapshot = metrics.GetSnapshot();
|
||||
Assert.Equal(0L, snapshot.Faults);
|
||||
Assert.Equal(0L, snapshot.QueueOverflows);
|
||||
Assert.Equal(0L, snapshot.WorkerKills);
|
||||
|
||||
// Disposal is now a terminal transition, so the worker that reached Ready is recorded as
|
||||
// stopped exactly once — the running-worker gauge no longer leaks on this path.
|
||||
Assert.Equal(0, snapshot.WorkersRunning);
|
||||
Assert.Equal(1L, snapshot.WorkerExits);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Writes worker events from the fake worker side until cancelled or until
|
||||
/// <see cref="EmittedEventCeiling"/> events have been written, so the gateway read loop has a
|
||||
/// continuous stream to dispatch while a test tears the client down. The ceiling keeps the
|
||||
/// burst structurally below the staging bound the caller configures. Every failure is
|
||||
/// swallowed: disposal closes the gateway end of the pipe underneath this writer, which is
|
||||
/// the situation under test rather than a test failure.
|
||||
/// </summary>
|
||||
/// <param name="pipePair">The connected pipe pair whose worker side emits the events.</param>
|
||||
/// <param name="cancellationToken">Stops the emission.</param>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
private static async Task EmitEventsUntilStoppedAsync(
|
||||
PipePair pipePair,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
await Task.Yield();
|
||||
try
|
||||
{
|
||||
for (ulong sequence = 1;
|
||||
sequence <= EmittedEventCeiling && !cancellationToken.IsCancellationRequested;
|
||||
sequence++)
|
||||
{
|
||||
await pipePair.WorkerWriter.WriteAsync(
|
||||
CreateEventEnvelope(sequence, MxEventFamily.OnDataChange),
|
||||
cancellationToken);
|
||||
}
|
||||
}
|
||||
catch (Exception)
|
||||
{
|
||||
// Broken pipe, disposed stream or cancellation — all expected once dispose starts.
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies that a heartbeat envelope updates the last-heartbeat timestamp and worker
|
||||
/// process id. Uses a <see cref="ManualTimeProvider"/> so the timestamp advance is
|
||||
|
||||
Reference in New Issue
Block a user