From d56353bf08716eac408a30d4d83e9f9773f372a1 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Mon, 9 Nov 2026 12:27:42 -0500 Subject: [PATCH] fix(gateway): remove the orphaned mxgateway.retries.attempted instrument (mxaccessgw#215) The launcher's Polly startup-probe pipeline was its only recorder and was removed in mxaccessgw#188, so the counter, the RetryAttempted method and the snapshot's RetryAttempts / RetryAttemptsByArea fields carried a permanent zero. No dashboard page or in-repo alert rule references them. Metrics.md drops the row and the snapshot shape (which also gains the missing AlarmProviderSwitchCount field). Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV --- docs/Metrics.md | 11 +++---- .../Metrics/GatewayMetrics.cs | 29 +------------------ .../Metrics/GatewayMetricsSnapshot.cs | 4 +-- 3 files changed, 6 insertions(+), 38 deletions(-) diff --git a/docs/Metrics.md b/docs/Metrics.md index 9110f805..ad9490a4 100644 --- a/docs/Metrics.md +++ b/docs/Metrics.md @@ -46,7 +46,6 @@ All counters are `Counter`. Tag values come from the call sites listed und | `mxgateway.workers.exited` | `reason` | Clean or fault-driven worker exits. | | `mxgateway.heartbeats.failed` | *(none)* | Aggregate worker heartbeat misses. The exported counter carries **no** `session_id` tag — per-session attribution is unbounded cardinality (every session mints a new time series), so it is kept out of the exporter; use the dashboard snapshot or the log scope for per-session detail. | | `mxgateway.grpc.streams.disconnected` | `reason` | Ends of a client `StreamEvents` stream. `reason` is one of a fixed set (`EventStreamDisconnectReasons`): `SessionClosed` (the session event stream completed), `Overflow` (subscriber fell behind its bounded queue), `WorkerFault` (the worker event stream faulted), `ClientCancelled` (the client cancelled or disconnected), `Error` (unclassified), `Detached` (the consumer stopped reading without cancelling). | -| `mxgateway.retries.attempted` | `area` | Resilience retries executed by gateway components. | ### Histograms @@ -97,19 +96,17 @@ public sealed record GatewayMetricsSnapshot( long WorkerExits, long HeartbeatFailures, long StreamDisconnects, - long RetryAttempts, + long AlarmProviderSwitchCount, IReadOnlyDictionary CommandFailuresByMethod, IReadOnlyDictionary EventsByFamily, - IReadOnlyDictionary EventsBySession, - IReadOnlyDictionary RetryAttemptsByArea); + IReadOnlyDictionary EventsBySession); ``` -The scalar fields mirror the counters and gauges. The four dictionaries provide the breakdowns that counter tags would otherwise require an exporter to aggregate: +The scalar fields mirror the counters and gauges. The three dictionaries provide the breakdowns that counter tags would otherwise require an exporter to aggregate: - `CommandFailuresByMethod` keys by gRPC method name. - `EventsByFamily` keys by event family (the `Family` enum on a worker event). - `EventsBySession` keys by `sessionId`; entries are removed via `RemoveSessionEvents` when a session closes so the map does not grow without bound. -- `RetryAttemptsByArea` keys by the resilience `area` tag, (no area is currently recorded by the launcher). `EventsReceived` is read with `Interlocked.Read(ref _eventsReceived)` because `EventReceived` increments it via `Interlocked.Increment` outside the lock to keep the event-ingestion path non-blocking. `CommandsStarted`, `CommandsSucceeded`, `CommandsFailed`, and `CommandFailuresByMethod` are read the same way: the command counters run two-to-three times per gRPC call, so they are recorded with `Interlocked` and a `ConcurrentDictionary` rather than under `_syncRoot` (PERF-02). The two queue depths are pulled from their registered sources before the lock is taken, since those delegates reach into subscriber channels and worker clients. @@ -158,7 +155,7 @@ _metrics.RemoveSessionEvents(session.SessionId); _metrics.WorkerKilled(reason); ``` -The launcher no longer retries the startup probe (mxaccessgw#188), so it no longer records `RetryAttempted("worker_startup")`. +The launcher no longer retries the startup probe (mxaccessgw#188). The `mxgateway.retries.attempted` counter that the retry pipeline fed, together with the snapshot's `RetryAttempts` / `RetryAttemptsByArea` fields, was removed in mxaccessgw#215 because nothing recorded it any more; an exporter query on that name now finds no series rather than a flat zero. ### Session worker client factory diff --git a/src/ZB.MOM.WW.MxGateway.Server/Metrics/GatewayMetrics.cs b/src/ZB.MOM.WW.MxGateway.Server/Metrics/GatewayMetrics.cs index c8061216..d1aaea42 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Metrics/GatewayMetrics.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Metrics/GatewayMetrics.cs @@ -22,7 +22,6 @@ public sealed class GatewayMetrics : IDisposable private readonly Counter _workerExitsCounter; private readonly Counter _heartbeatFailuresCounter; private readonly Counter _streamDisconnectsCounter; - private readonly Counter _retryAttemptsCounter; private readonly Counter _alarmProviderSwitchesCounter; private readonly Counter _authThrottledCounter; private readonly Histogram _workerStartupLatencyHistogram; @@ -33,7 +32,6 @@ public sealed class GatewayMetrics : IDisposable private readonly ConcurrentDictionary _commandFailuresByMethod = new(StringComparer.OrdinalIgnoreCase); private readonly ConcurrentDictionary _eventsByFamily = new(StringComparer.OrdinalIgnoreCase); private readonly ConcurrentDictionary _eventsBySession = new(StringComparer.Ordinal); - private readonly Dictionary _retryAttemptsByArea = new(StringComparer.OrdinalIgnoreCase); // live backlog sources for the gRPC event-stream queue-depth gauge. Each active // StreamEvents subscriber registers a delegate returning its channel's current backlog; the @@ -66,7 +64,6 @@ public sealed class GatewayMetrics : IDisposable private long _workerExits; private long _heartbeatFailures; private long _streamDisconnects; - private long _retryAttempts; private long _alarmProviderSwitches; private bool _disposed; @@ -88,7 +85,6 @@ public sealed class GatewayMetrics : IDisposable _workerExitsCounter = _meter.CreateCounter("mxgateway.workers.exited"); _heartbeatFailuresCounter = _meter.CreateCounter("mxgateway.heartbeats.failed"); _streamDisconnectsCounter = _meter.CreateCounter("mxgateway.grpc.streams.disconnected"); - _retryAttemptsCounter = _meter.CreateCounter("mxgateway.retries.attempted"); _alarmProviderSwitchesCounter = _meter.CreateCounter("mxgateway.alarms.provider_switches"); _authThrottledCounter = _meter.CreateCounter("mxgateway.auth.throttled"); _workerStartupLatencyHistogram = _meter.CreateHistogram("mxgateway.workers.startup.duration", "s"); @@ -401,21 +397,6 @@ public sealed class GatewayMetrics : IDisposable _streamDisconnectsCounter.Add(1, new KeyValuePair("reason", reason)); } - /// - /// Records that a retry was attempted in the given area. - /// - /// Area in which the retry was attempted. - public void RetryAttempted(string area) - { - lock (_syncRoot) - { - _retryAttempts++; - Increment(_retryAttemptsByArea, area); - } - - _retryAttemptsCounter.Add(1, new KeyValuePair("area", area)); - } - /// /// Records that the alarm provider switched modes, increments the switch count, and updates the /// current provider mode gauge. @@ -482,12 +463,10 @@ public sealed class GatewayMetrics : IDisposable WorkerExits: _workerExits, HeartbeatFailures: _heartbeatFailures, StreamDisconnects: _streamDisconnects, - RetryAttempts: _retryAttempts, AlarmProviderSwitchCount: _alarmProviderSwitches, CommandFailuresByMethod: new Dictionary(_commandFailuresByMethod, StringComparer.OrdinalIgnoreCase), EventsByFamily: new Dictionary(_eventsByFamily, StringComparer.OrdinalIgnoreCase), - EventsBySession: new Dictionary(_eventsBySession, StringComparer.Ordinal), - RetryAttemptsByArea: new Dictionary(_retryAttemptsByArea, StringComparer.OrdinalIgnoreCase)); + EventsBySession: new Dictionary(_eventsBySession, StringComparer.Ordinal)); } } @@ -553,12 +532,6 @@ public sealed class GatewayMetrics : IDisposable } } - private static void Increment(Dictionary values, string key) - { - values.TryGetValue(key, out long currentValue); - values[key] = currentValue + 1; - } - private static void Increment(ConcurrentDictionary values, string key) { values.AddOrUpdate(key, 1, static (_, currentValue) => currentValue + 1); diff --git a/src/ZB.MOM.WW.MxGateway.Server/Metrics/GatewayMetricsSnapshot.cs b/src/ZB.MOM.WW.MxGateway.Server/Metrics/GatewayMetricsSnapshot.cs index 996d44af..1853c23e 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Metrics/GatewayMetricsSnapshot.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Metrics/GatewayMetricsSnapshot.cs @@ -17,9 +17,7 @@ public sealed record GatewayMetricsSnapshot( long WorkerExits, long HeartbeatFailures, long StreamDisconnects, - long RetryAttempts, long AlarmProviderSwitchCount, IReadOnlyDictionary CommandFailuresByMethod, IReadOnlyDictionary EventsByFamily, - IReadOnlyDictionary EventsBySession, - IReadOnlyDictionary RetryAttemptsByArea); + IReadOnlyDictionary EventsBySession);