perf(metrics): pull-model worker queue gauge (fixes last-writer-wins), Interlocked command counters

src/ZB.MOM.WW.MxGateway.Tests/Metrics/GatewayMetricsTests.cs

@@ -19,7 +19,10 @@ public sealed class GatewayMetricsTests
         metrics.CommandFailed("WriteSecured", "AuthorizationFailed", TimeSpan.FromMilliseconds(12));
         metrics.EventReceived("session-1", "OnDataChange");
         metrics.EventReceived("session-1", "OnDataChange");
-        metrics.SetWorkerEventQueueDepth(7);
+        // GWC-30: the worker queue-depth gauge sums one live source per worker client, so the two
+        // registrations below stand in for two concurrent sessions holding 3 and 4 events.
+        using IDisposable workerDepthSourceA = metrics.RegisterWorkerEventQueueDepthSource(static () => 3);
+        using IDisposable workerDepthSourceB = metrics.RegisterWorkerEventQueueDepthSource(static () => 4);
         // GWC-15: the gRPC stream queue-depth gauge sums live backlog sources at collection time
         // rather than tracking a pushed running total. Register a source reporting 3.
         using IDisposable backlogSource = metrics.RegisterEventStreamBacklogSource(static () => 3);
@@ -54,16 +57,111 @@ public sealed class GatewayMetricsTests
         Assert.Equal(2, snapshot.EventsBySession["session-1"]);
     }
 
-    /// <summary>Verifies that negative queue depth is rejected.</summary>
+    /// <summary>
+    ///     GWC-30: the worker queue-depth gauge sums every registered source rather than holding a
+    ///     single pushed scalar, so concurrent sessions add up instead of overwriting one another,
+    ///     and a disposed registration (a worker client going away) drops out of the sum. Disposal
+    ///     is idempotent because a client's dispose path can run twice.
+    /// </summary>
     [Fact]
-    public void SetEventQueueDepth_RejectsNegativeDepth()
+    public void WorkerEventQueueDepthSources_SumAcrossRegistrationsAndDropOnDispose()
     {
         using GatewayMetrics metrics = new();
 
-        ArgumentOutOfRangeException exception = Assert.Throws<ArgumentOutOfRangeException>(
-            () => metrics.SetWorkerEventQueueDepth(-1));
+        IDisposable firstSource = metrics.RegisterWorkerEventQueueDepthSource(static () => 3);
+        using IDisposable secondSource = metrics.RegisterWorkerEventQueueDepthSource(static () => 4);
 
-        Assert.Equal("depth", exception.ParamName);
+        Assert.Equal(7, metrics.GetSnapshot().WorkerEventQueueDepth);
+
+        firstSource.Dispose();
+        Assert.Equal(4, metrics.GetSnapshot().WorkerEventQueueDepth);
+
+        firstSource.Dispose();
+        Assert.Equal(4, metrics.GetSnapshot().WorkerEventQueueDepth);
+    }
+
+    /// <summary>
+    ///     A depth source reads a lock-free counter that a racing decrement can momentarily push
+    ///     below zero, so the sum clamps each reading instead of rejecting it — the pull model has
+    ///     no caller to throw back at.
+    /// </summary>
+    [Fact]
+    public void WorkerEventQueueDepthSources_ClampNegativeReadingsToZero()
+    {
+        using GatewayMetrics metrics = new();
+
+        using IDisposable negativeSource = metrics.RegisterWorkerEventQueueDepthSource(static () => -5);
+        using IDisposable positiveSource = metrics.RegisterWorkerEventQueueDepthSource(static () => 4);
+
+        Assert.Equal(4, metrics.GetSnapshot().WorkerEventQueueDepth);
+    }
+
+    /// <summary>
+    ///     Verifies the exported gauge keeps the name <c>mxgateway.events.worker_queue.depth</c> and
+    ///     reports the summed sources, so the pull-model rework is invisible to exporters.
+    /// </summary>
+    [Fact]
+    public void WorkerEventQueueDepthGauge_ReportsSummedSources()
+    {
+        using GatewayMetrics metrics = new();
+        using MeterListener listener = new();
+
+        int? capturedDepth = null;
+
+        listener.InstrumentPublished = (instrument, meterListener) =>
+        {
+            if (ReferenceEquals(instrument.Meter, metrics.Meter)
+                && instrument.Name == "mxgateway.events.worker_queue.depth")
+            {
+                meterListener.EnableMeasurementEvents(instrument);
+            }
+        };
+        listener.SetMeasurementEventCallback<int>(
+            (instrument, measurement, _, _) =>
+            {
+                if (ReferenceEquals(instrument.Meter, metrics.Meter)
+                    && instrument.Name == "mxgateway.events.worker_queue.depth")
+                {
+                    capturedDepth = measurement;
+                }
+            });
+        listener.Start();
+
+        using IDisposable firstSource = metrics.RegisterWorkerEventQueueDepthSource(static () => 5);
+        using IDisposable secondSource = metrics.RegisterWorkerEventQueueDepthSource(static () => 6);
+        listener.RecordObservableInstruments();
+
+        Assert.Equal(11, capturedDepth);
+    }
+
+    /// <summary>
+    ///     The command counters are incremented with <see cref="Interlocked"/> rather than under the
+    ///     process-wide metrics lock, so this asserts no increment is lost when every gRPC thread
+    ///     records at once.
+    /// </summary>
+    [Fact]
+    public void CommandCounters_CountEveryConcurrentInvocation()
+    {
+        const int workers = 8;
+        const int perWorker = 500;
+        using GatewayMetrics metrics = new();
+
+        Parallel.For(0, workers, _ =>
+        {
+            for (int index = 0; index < perWorker; index++)
+            {
+                metrics.CommandStarted("Register");
+                metrics.CommandSucceeded("Register", TimeSpan.FromMilliseconds(1));
+                metrics.CommandFailed("WriteSecured", "AuthorizationFailed", TimeSpan.FromMilliseconds(1));
+            }
+        });
+
+        GatewayMetricsSnapshot snapshot = metrics.GetSnapshot();
+
+        Assert.Equal(workers * perWorker, snapshot.CommandsStarted);
+        Assert.Equal(workers * perWorker, snapshot.CommandsSucceeded);
+        Assert.Equal(workers * perWorker, snapshot.CommandsFailed);
+        Assert.Equal(workers * perWorker, snapshot.CommandFailuresByMethod["WriteSecured"]);
     }
 
     /// <summary>