dohertj2/mxaccessgw
1
0
Fork 0
Code Issues 14 Pull Requests 2 Actions Packages Projects Releases Wiki Activity

perf(metrics): pull-model worker queue gauge (fixes last-writer-wins), Interlocked command counters

Browse Source
This commit is contained in:
Joseph Doherty
2026-08-15 12:21:51 -04:00
parent 77c5731b7b
commit 9735ac3b7c
4 changed files with 178 additions and 77 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ZB.MOM.WW.MxGateway.Server/Metrics/GatewayMetrics.cs
+57 -64
View File
@@ -28,7 +28,9 @@ public sealed class GatewayMetrics : IDisposable
private readonly Histogram<double> _workerStartupLatencyHistogram;
private readonly Histogram<double> _commandLatencyHistogram;
private readonly Histogram<double> _eventStreamSendLatencyHistogram;
private readonly Dictionary<string, long> _commandFailuresByMethod = new(StringComparer.OrdinalIgnoreCase);
// Concurrent (not Dictionary + _syncRoot) because CommandFailed runs on every failing gRPC call:
// the command counters are recorded outside the lock, so their breakdown map must be too.
private readonly ConcurrentDictionary<string, long> _commandFailuresByMethod = new(StringComparer.OrdinalIgnoreCase);
private readonly ConcurrentDictionary<string, long> _eventsByFamily = new(StringComparer.OrdinalIgnoreCase);
private readonly ConcurrentDictionary<string, long> _eventsBySession = new(StringComparer.Ordinal);
private readonly Dictionary<string, long> _retryAttemptsByArea = new(StringComparer.OrdinalIgnoreCase);
@@ -41,9 +43,16 @@ public sealed class GatewayMetrics : IDisposable
private readonly ConcurrentDictionary<long, Func<int>> _eventStreamBacklogSources = new();
private long _nextEventStreamBacklogSourceId;
// GWC-30: the same pull model for the worker event queue depth. It replaces a pushed scalar that
// every WorkerClient wrote twice per event (staged, consumed) under _syncRoot — a process-wide
// lock on the hottest path, and last-writer-wins across sessions, so the gauge reported one
// arbitrary session's backlog instead of the gateway's. Each client registers a source returning
// its own undelivered depth; the gauge sums them at collection time only.
private readonly ConcurrentDictionary<long, Func<int>> _workerEventQueueDepthSources = new();
private long _nextWorkerEventQueueDepthSourceId;
private int _openSessions;
private int _workersRunning;
private int _workerEventQueueDepth;
private int _alarmProviderMode;
private long _sessionsOpened;
private long _sessionsClosed;
@@ -201,10 +210,10 @@ public sealed class GatewayMetrics : IDisposable
/// <param name="method">Name of the command method.</param>
public void CommandStarted(string method)
{
lock (_syncRoot)
{
_commandsStarted++;
}
// GWC-30: the three command counters run two-to-three times per gRPC call, so they use
// Interlocked rather than _syncRoot — the same idiom as EventReceived. Nothing here needs a
// consistent multi-field view; GetSnapshot reads each with Interlocked.Read.
Interlocked.Increment(ref _commandsStarted);
_commandsStartedCounter.Add(1, new KeyValuePair<string, object?>("method", method));
}
@@ -216,10 +225,7 @@ public sealed class GatewayMetrics : IDisposable
/// <param name="duration">Elapsed time to complete the command.</param>
public void CommandSucceeded(string method, TimeSpan duration)
{
lock (_syncRoot)
{
_commandsSucceeded++;
}
Interlocked.Increment(ref _commandsSucceeded);
KeyValuePair<string, object?> methodTag = new("method", method);
_commandsSucceededCounter.Add(1, methodTag);
@@ -234,11 +240,8 @@ public sealed class GatewayMetrics : IDisposable
/// <param name="duration">Elapsed time before command failed.</param>
public void CommandFailed(string method, string category, TimeSpan duration)
{
lock (_syncRoot)
{
_commandsFailed++;
Increment(_commandFailuresByMethod, method);
}
Interlocked.Increment(ref _commandsFailed);
Increment(_commandFailuresByMethod, method);
KeyValuePair<string, object?> methodTag = new("method", method);
KeyValuePair<string, object?> categoryTag = new("category", category);
@@ -275,29 +278,24 @@ public sealed class GatewayMetrics : IDisposable
}
/// <summary>
/// Sets the worker event queue depth; delegates to SetWorkerEventQueueDepth.
/// Registers a live depth source for the worker event queue-depth gauge and returns a handle
/// that removes it when disposed. Each <c>WorkerClient</c> registers once and reports its own
/// undelivered (staged + queued) event count, so the gauge is the gateway-wide sum instead of
/// the last value any one session happened to push (GWC-30).
/// </summary>
/// <param name="depth">Queue depth value.</param>
public void SetEventQueueDepth(int depth)
/// <param name="depth">
/// Returns this worker client's current undelivered event count. Invoked only at collection
/// time; must be cheap and non-blocking (a <see cref="Volatile.Read(ref int)"/> of an
/// interlocked counter). Negative readings — which a racing decrement can produce — are
/// clamped to zero when summed.
/// </param>
/// <returns>A handle whose disposal unregisters the source. Safe to dispose more than once.</returns>
public IDisposable RegisterWorkerEventQueueDepthSource(Func<int> depth)
{
SetWorkerEventQueueDepth(depth);
}
/// <summary>
/// Sets the worker event queue depth to the given value.
/// </summary>
/// <param name="depth">Queue depth value.</param>
public void SetWorkerEventQueueDepth(int depth)
{
if (depth < 0)
{
throw new ArgumentOutOfRangeException(nameof(depth), depth, "Queue depth cannot be negative.");
}
lock (_syncRoot)
{
_workerEventQueueDepth = depth;
}
ArgumentNullException.ThrowIfNull(depth);
long id = Interlocked.Increment(ref _nextWorkerEventQueueDepthSourceId);
_workerEventQueueDepthSources[id] = depth;
return new GaugeSourceRegistration(_workerEventQueueDepthSources, id);
}
/// <summary>
@@ -318,7 +316,7 @@ public sealed class GatewayMetrics : IDisposable
ArgumentNullException.ThrowIfNull(backlog);
long id = Interlocked.Increment(ref _nextEventStreamBacklogSourceId);
_eventStreamBacklogSources[id] = backlog;
return new EventStreamBacklogRegistration(this, id);
return new GaugeSourceRegistration(_eventStreamBacklogSources, id);
}
/// <summary>
@@ -460,21 +458,23 @@ public sealed class GatewayMetrics : IDisposable
/// <returns>The current metrics snapshot.</returns>
public GatewayMetricsSnapshot GetSnapshot()
{
// Compute the live gRPC stream backlog outside _syncRoot: the sources are the subscriber
// channels' Count (their own locks) and must not run under this lock. GWC-15.
// Compute the live queue depths outside _syncRoot: the sources are the subscriber channels'
// Count (their own locks) and the worker clients' interlocked counters, neither of which may
// run under this lock. GWC-15, GWC-30.
int workerEventQueueDepth = GetWorkerEventQueueDepth();
int grpcEventStreamQueueDepth = GetGrpcEventStreamQueueDepth();
lock (_syncRoot)
{
return new GatewayMetricsSnapshot(
OpenSessions: _openSessions,
WorkersRunning: _workersRunning,
WorkerEventQueueDepth: _workerEventQueueDepth,
WorkerEventQueueDepth: workerEventQueueDepth,
GrpcEventStreamQueueDepth: grpcEventStreamQueueDepth,
SessionsOpened: _sessionsOpened,
SessionsClosed: _sessionsClosed,
CommandsStarted: _commandsStarted,
CommandsSucceeded: _commandsSucceeded,
CommandsFailed: _commandsFailed,
CommandsStarted: Interlocked.Read(ref _commandsStarted),
CommandsSucceeded: Interlocked.Read(ref _commandsSucceeded),
CommandsFailed: Interlocked.Read(ref _commandsFailed),
EventsReceived: Interlocked.Read(ref _eventsReceived),
QueueOverflows: _queueOverflows,
Faults: _faults,
@@ -521,22 +521,19 @@ public sealed class GatewayMetrics : IDisposable
}
}
private int GetWorkerEventQueueDepth()
{
lock (_syncRoot)
{
return _workerEventQueueDepth;
}
}
// Sums the undelivered event backlog across every live worker client (GWC-30).
private int GetWorkerEventQueueDepth() => SumSources(_workerEventQueueDepthSources);
// Sums the live backlog across every registered event-stream subscriber. Runs at collection
// time (ObservableGauge scrape) or when GetSnapshot projects the value — never on the
// per-event path. Enumerating ConcurrentDictionary.Values never throws on concurrent
// Sums the live backlog across every registered event-stream subscriber.
private int GetGrpcEventStreamQueueDepth() => SumSources(_eventStreamBacklogSources);
// Runs at collection time (ObservableGauge scrape) or when GetSnapshot projects the value —
// never on a per-event path. Enumerating ConcurrentDictionary.Values never throws on concurrent
// register/unregister; a source removed mid-enumeration simply drops from this sample.
private int GetGrpcEventStreamQueueDepth()
private static int SumSources(ConcurrentDictionary<long, Func<int>> sources)
{
int total = 0;
foreach (Func<int> source in _eventStreamBacklogSources.Values)
foreach (Func<int> source in sources.Values)
{
int value = source();
if (value > 0)
@@ -548,11 +545,6 @@ public sealed class GatewayMetrics : IDisposable
return total;
}
private void UnregisterEventStreamBacklogSource(long id)
{
_eventStreamBacklogSources.TryRemove(id, out _);
}
private int GetAlarmProviderMode()
{
lock (_syncRoot)
@@ -572,9 +564,10 @@ public sealed class GatewayMetrics : IDisposable
values.AddOrUpdate(key, 1, static (_, currentValue) => currentValue + 1);
}
// Handle returned by RegisterEventStreamBacklogSource. Disposal (once) removes the source
// from the gauge's live sum. Idempotent so a double dispose from a stream teardown is safe.
private sealed class EventStreamBacklogRegistration(GatewayMetrics metrics, long id) : IDisposable
// Handle returned by the pull-model gauge registrations. Disposal (once) removes the source from
// that gauge's live sum. Idempotent so a double dispose from a stream or worker-client teardown
// is safe, and shared by both gauges so the two registrations cannot drift apart.
private sealed class GaugeSourceRegistration(ConcurrentDictionary<long, Func<int>> sources, long id) : IDisposable
{
private int _disposed;
@@ -582,7 +575,7 @@ public sealed class GatewayMetrics : IDisposable
{
if (Interlocked.Exchange(ref _disposed, 1) == 0)
{
metrics.UnregisterEventStreamBacklogSource(id);
sources.TryRemove(id, out _);
}
}
}