Merge fix/c3-session-lifecycle: GWC-32 WorkerClient dispose is terminal, GWC-34 attach-after-dispose throws SessionNotReady, GWC-37 subtag watch-list coverage graded on provider status + re-resolved at failover edge

2026-08-20 09:39:53 -04:00
parent 06cd8b844c 193221773d
commit c6650c156b
21 changed files with 864 additions and 50 deletions
@@ -420,12 +420,32 @@ public sealed class WorkerClient : IWorkerClient
    /// <returns>A task that represents the asynchronous operation.</returns>
    public async ValueTask DisposeAsync()
    {
-        if (_disposed)
+        // GWC-32: disposal is the third channel-completing path, and it must enter a terminal state
+        // like the other two (MarkClosed, SetFaulted) BEFORE the channels are completed. Otherwise a
+        // client disposed while still Ready keeps reporting IsTerminalState() == false, and a read
+        // loop already inside DispatchEnvelope (the _stopCts.Cancel below only interrupts the NEXT
+        // ReadAsync) takes StageWorkerEvent's overflow arm on the completed staging channel: a
+        // spurious ProtocolViolation fault, a worker-event-staging overflow metric and a redundant
+        // kill, all during an intentional teardown. MarkClosed is deliberately not reused here: it
+        // would overwrite Faulted with Closed and it re-runs the whole teardown sequence this method
+        // performs itself. The transition is skipped when the state is already terminal, so a
+        // dispose-after-fault keeps Faulted and a double dispose changes nothing.
+        bool becameTerminalHere;
+        lock (_syncRoot)
        {
-            return;
+            if (_disposed)
+            {
+                return;
+            }
+
+            _disposed = true;
+            becameTerminalHere = _state is not (WorkerClientState.Closed or WorkerClientState.Faulted);
+            if (becameTerminalHere)
+            {
+                _state = WorkerClientState.Closed;
+            }
        }

-        _disposed = true;
        // Drop out of the worker queue-depth gauge before teardown: whatever this client still holds
        // is about to be discarded, and the sum must not keep counting a client that is going away.
        _eventQueueDepthRegistration?.Dispose();
@@ -434,6 +454,16 @@ public sealed class WorkerClient : IWorkerClient
        _outboundEnvelopes.Writer.TryComplete();
        _eventStaging.Writer.TryComplete();
        _events.Writer.TryComplete();
+        if (becameTerminalHere)
+        {
+            // A worker that reached Ready and is now being disposed without a prior Close or fault
+            // (SessionManager.OpenSessionAsync's failure arm does exactly this) stopped, so the
+            // running-worker gauge must come down. RecordWorkerStoppedOnce is latched on
+            // _workerStartRecorded, so a client that already recorded its stop through MarkClosed or
+            // SetFaulted cannot record a second one here.
+            RecordWorkerStoppedOnce("dispose");
+        }
+
        CompletePendingCommands(
            new WorkerClientException(
                WorkerClientErrorCode.GatewayShutdown,
@@ -647,8 +677,10 @@ public sealed class WorkerClient : IWorkerClient
    ///     Hands a received worker event to the dedicated event writer without blocking the read loop.
    ///     <c>TryWrite</c> is non-blocking, so the read loop keeps dispatching replies, heartbeats and
    ///     faults regardless of how backed up the event path is. It returns <c>false</c> in two cases:
-    ///     the staging channel has been completed during shutdown (the event is safely dropped because
-    ///     the client is already terminal), or the channel is full at its
+    ///     the staging channel has been completed by one of the three completing paths —
+    ///     <see cref="MarkClosed"/>, <see cref="SetFaulted"/> or <see cref="DisposeAsync"/>, each of
+    ///     which enters a terminal state before completing the channels (GWC-32), so the event is
+    ///     safely dropped because the client is already terminal — or the channel is full at its
    ///     2 × <see cref="WorkerClientOptions.EventChannelCapacity"/> bound. The latter means the
    ///     consumer has been draining slower than the worker produces for the whole time it took the
    ///     worker to emit that many further events, so the client is faulted immediately (GWC-24) —