perf(metrics): pull-model worker queue gauge (fixes last-writer-wins), Interlocked command counters

src/ZB.MOM.WW.MxGateway.Server/Workers/WorkerClient.cs

@@ -40,6 +40,12 @@ public sealed class WorkerClient : IWorkerClient
     private readonly ConcurrentDictionary<string, PendingCommand> _pendingCommands = new(StringComparer.Ordinal);
     private readonly SemaphoreSlim _pendingCommandSlots;
     private readonly CancellationTokenSource _stopCts = new();
+
+    // GWC-30: this client's contribution to the gateway-wide worker queue-depth gauge. Registered
+    // once here and read only when the gauge is scraped, so staging and consuming an event cost an
+    // Interlocked on _eventQueueDepth and nothing else — previously each of those two hot-path steps
+    // called into GatewayMetrics and took its process-wide lock. Null when metrics are disabled.
+    private readonly IDisposable? _eventQueueDepthRegistration;
     // Touched only by WriteLoopAsync — the single consumer of _outboundEnvelopes — so it needs no
     // interlocking. See WriteLoopAsync for why the stamp happens there rather than at construction.
     private ulong _nextSequence;
@@ -111,6 +117,8 @@ public sealed class WorkerClient : IWorkerClient
                 FullMode = BoundedChannelFullMode.Wait,
                 AllowSynchronousContinuations = false,
             });
+        _eventQueueDepthRegistration = _metrics?.RegisterWorkerEventQueueDepthSource(
+            () => Volatile.Read(ref _eventQueueDepth));
         _lastHeartbeatAt = _timeProvider.GetUtcNow();
     }
 
@@ -299,8 +307,8 @@ public sealed class WorkerClient : IWorkerClient
     {
         await foreach (WorkerEvent workerEvent in _events.Reader.ReadAllAsync(cancellationToken).ConfigureAwait(false))
         {
-            int queueDepth = Math.Max(0, Interlocked.Decrement(ref _eventQueueDepth));
-            _metrics?.SetWorkerEventQueueDepth(queueDepth);
+            // No metrics call on the hot path: the gauge pulls _eventQueueDepth when scraped (GWC-30).
+            Interlocked.Decrement(ref _eventQueueDepth);
             yield return workerEvent;
         }
     }
@@ -371,6 +379,9 @@ public sealed class WorkerClient : IWorkerClient
         }
 
         _disposed = true;
+        // Drop out of the worker queue-depth gauge before teardown: whatever this client still holds
+        // is about to be discarded, and the sum must not keep counting a client that is going away.
+        _eventQueueDepthRegistration?.Dispose();
         KillOwnedProcess("Dispose");
         _stopCts.Cancel();
         _outboundEnvelopes.Writer.TryComplete();
@@ -598,8 +609,7 @@ public sealed class WorkerClient : IWorkerClient
         {
             // Counted here rather than at the _events write so the single gauge reports total
             // undelivered events (staged + queued). ReadEventsCoreAsync decrements on consumer read.
-            int queueDepth = Interlocked.Increment(ref _eventQueueDepth);
-            _metrics?.SetWorkerEventQueueDepth(queueDepth);
+            Interlocked.Increment(ref _eventQueueDepth);
             return;
         }