using System.Collections.Concurrent; using System.Diagnostics.Metrics; using System.Globalization; namespace ZB.MOM.WW.MxGateway.Server.Metrics; public sealed class GatewayMetrics : IDisposable { public const string MeterName = "ZB.MOM.WW.MxGateway"; private readonly object _syncRoot = new(); private readonly Meter _meter; private readonly Counter _sessionsOpenedCounter; private readonly Counter _sessionsClosedCounter; private readonly Counter _commandsStartedCounter; private readonly Counter _commandsSucceededCounter; private readonly Counter _commandsFailedCounter; private readonly Counter _eventsReceivedCounter; private readonly Counter _queueOverflowsCounter; private readonly Counter _faultsCounter; private readonly Counter _workerKillsCounter; private readonly Counter _workerExitsCounter; private readonly Counter _heartbeatFailuresCounter; private readonly Counter _streamDisconnectsCounter; private readonly Counter _retryAttemptsCounter; private readonly Counter _alarmProviderSwitchesCounter; private readonly Histogram _workerStartupLatencyHistogram; private readonly Histogram _commandLatencyHistogram; private readonly Histogram _eventStreamSendLatencyHistogram; private readonly Dictionary _commandFailuresByMethod = new(StringComparer.OrdinalIgnoreCase); private readonly ConcurrentDictionary _eventsByFamily = new(StringComparer.OrdinalIgnoreCase); private readonly ConcurrentDictionary _eventsBySession = new(StringComparer.Ordinal); private readonly Dictionary _retryAttemptsByArea = new(StringComparer.OrdinalIgnoreCase); // GWC-15: live backlog sources for the gRPC event-stream queue-depth gauge. Each active // StreamEvents subscriber registers a delegate returning its channel's current backlog; the // gauge sums them ONLY when scraped or projected (GetGrpcEventStreamQueueDepth), so nothing is // read on the per-event streaming hot path. Concurrent because subscribers register/unregister // from arbitrary gRPC threads while the gauge callback / GetSnapshot enumerate. private readonly ConcurrentDictionary> _eventStreamBacklogSources = new(); private long _nextEventStreamBacklogSourceId; private int _openSessions; private int _workersRunning; private int _workerEventQueueDepth; private int _alarmProviderMode; private long _sessionsOpened; private long _sessionsClosed; private long _commandsStarted; private long _commandsSucceeded; private long _commandsFailed; private long _eventsReceived; private long _queueOverflows; private long _faults; private long _workerKills; private long _workerExits; private long _heartbeatFailures; private long _streamDisconnects; private long _retryAttempts; private long _alarmProviderSwitches; private bool _disposed; /// /// Initializes the gateway metrics with OpenTelemetry counters and histograms. /// public GatewayMetrics() { _meter = new Meter(MeterName, typeof(GatewayMetrics).Assembly.GetName().Version?.ToString()); _sessionsOpenedCounter = _meter.CreateCounter("mxgateway.sessions.opened"); _sessionsClosedCounter = _meter.CreateCounter("mxgateway.sessions.closed"); _commandsStartedCounter = _meter.CreateCounter("mxgateway.commands.started"); _commandsSucceededCounter = _meter.CreateCounter("mxgateway.commands.succeeded"); _commandsFailedCounter = _meter.CreateCounter("mxgateway.commands.failed"); _eventsReceivedCounter = _meter.CreateCounter("mxgateway.events.received"); _queueOverflowsCounter = _meter.CreateCounter("mxgateway.queues.overflows"); _faultsCounter = _meter.CreateCounter("mxgateway.faults"); _workerKillsCounter = _meter.CreateCounter("mxgateway.workers.killed"); _workerExitsCounter = _meter.CreateCounter("mxgateway.workers.exited"); _heartbeatFailuresCounter = _meter.CreateCounter("mxgateway.heartbeats.failed"); _streamDisconnectsCounter = _meter.CreateCounter("mxgateway.grpc.streams.disconnected"); _retryAttemptsCounter = _meter.CreateCounter("mxgateway.retries.attempted"); _alarmProviderSwitchesCounter = _meter.CreateCounter("mxgateway.alarms.provider_switches"); _workerStartupLatencyHistogram = _meter.CreateHistogram("mxgateway.workers.startup.duration", "s"); _commandLatencyHistogram = _meter.CreateHistogram("mxgateway.commands.duration", "s"); _eventStreamSendLatencyHistogram = _meter.CreateHistogram("mxgateway.events.stream_send.duration", "s"); _meter.CreateObservableGauge("mxgateway.sessions.open", GetOpenSessions); _meter.CreateObservableGauge("mxgateway.workers.running", GetWorkersRunning); _meter.CreateObservableGauge("mxgateway.events.worker_queue.depth", GetWorkerEventQueueDepth); _meter.CreateObservableGauge("mxgateway.events.grpc_stream_queue.depth", GetGrpcEventStreamQueueDepth); _meter.CreateObservableGauge("mxgateway.alarms.provider_mode", GetAlarmProviderMode); } /// /// Gets the underlying instance backing this metrics object. Exposed to tests /// (via InternalsVisibleTo) so a can filter measurements by /// against this specific instance rather than by the /// process-shared , which would cross-talk between parallel tests that /// each build their own . /// internal Meter Meter => _meter; /// /// Records that a session has been opened. /// public void SessionOpened() { lock (_syncRoot) { _openSessions++; _sessionsOpened++; } _sessionsOpenedCounter.Add(1); } /// /// Records that a session has been closed. /// public void SessionClosed() { lock (_syncRoot) { if (_openSessions > 0) { _openSessions--; } _sessionsClosed++; } _sessionsClosedCounter.Add(1); } /// /// Records that a session has been removed from registry. /// public void SessionRemoved() { lock (_syncRoot) { if (_openSessions > 0) { _openSessions--; } } } /// /// Records that a worker process has started and its startup latency. /// /// Duration elapsed while starting the worker. public void WorkerStarted(TimeSpan startupDuration) { lock (_syncRoot) { _workersRunning++; } _workerStartupLatencyHistogram.Record(startupDuration.TotalSeconds); } /// /// Records that a worker process has stopped with the given reason. /// /// Cause of the worker stopping. public void WorkerStopped(string reason) { lock (_syncRoot) { if (_workersRunning > 0) { _workersRunning--; } _workerExits++; } _workerExitsCounter.Add(1, new KeyValuePair("reason", reason)); } /// /// Records that a worker process was killed with the given reason. /// /// Cause of the worker termination. public void WorkerKilled(string reason) { lock (_syncRoot) { _workerKills++; } _workerKillsCounter.Add(1, new KeyValuePair("reason", reason)); } /// /// Records that a command has started for the given method. /// /// Name of the command method. public void CommandStarted(string method) { lock (_syncRoot) { _commandsStarted++; } _commandsStartedCounter.Add(1, new KeyValuePair("method", method)); } /// /// Records that a command succeeded for the given method and duration. /// /// Name of the command method. /// Elapsed time to complete the command. public void CommandSucceeded(string method, TimeSpan duration) { lock (_syncRoot) { _commandsSucceeded++; } KeyValuePair methodTag = new("method", method); _commandsSucceededCounter.Add(1, methodTag); _commandLatencyHistogram.Record(duration.TotalSeconds, methodTag); } /// /// Records that a command failed for the given method, category, and duration. /// /// Name of the command method. /// Classification of the failure. /// Elapsed time before command failed. public void CommandFailed(string method, string category, TimeSpan duration) { lock (_syncRoot) { _commandsFailed++; Increment(_commandFailuresByMethod, method); } KeyValuePair methodTag = new("method", method); KeyValuePair categoryTag = new("category", category); _commandsFailedCounter.Add(1, methodTag, categoryTag); _commandLatencyHistogram.Record(duration.TotalSeconds, methodTag, categoryTag); } /// /// Records that an event was received for the given session and family. /// /// Identifier of the session receiving the event. /// Event family classification. public void EventReceived(string sessionId, string family) { Interlocked.Increment(ref _eventsReceived); Increment(_eventsByFamily, family); Increment(_eventsBySession, sessionId); _eventsReceivedCounter.Add( 1, new KeyValuePair("family", family)); } /// /// Records the latency of sending an event to a client stream. /// /// Event family name. /// Time taken to send the event. public void RecordEventStreamSend(string family, TimeSpan duration) { _eventStreamSendLatencyHistogram.Record( duration.TotalSeconds, new KeyValuePair("family", family)); } /// /// Sets the worker event queue depth; delegates to SetWorkerEventQueueDepth. /// /// Queue depth value. public void SetEventQueueDepth(int depth) { SetWorkerEventQueueDepth(depth); } /// /// Sets the worker event queue depth to the given value. /// /// Queue depth value. public void SetWorkerEventQueueDepth(int depth) { if (depth < 0) { throw new ArgumentOutOfRangeException(nameof(depth), depth, "Queue depth cannot be negative."); } lock (_syncRoot) { _workerEventQueueDepth = depth; } } /// /// Registers a live backlog source for the gRPC event-stream queue-depth gauge and /// returns a handle that removes it when disposed. The gauge sums every registered /// source's current value on demand (when scraped or via ), /// rather than tracking a pushed running total, so the streaming hot path does no /// per-event gauge bookkeeping (GWC-15). /// /// /// Returns this subscriber's current channel backlog. Invoked only at collection time; /// must be cheap and non-blocking (a bounded channel's Count). Negative returns /// are clamped to zero when summed. /// /// A handle whose disposal unregisters the source. Safe to dispose more than once. public IDisposable RegisterEventStreamBacklogSource(Func backlog) { ArgumentNullException.ThrowIfNull(backlog); long id = Interlocked.Increment(ref _nextEventStreamBacklogSourceId); _eventStreamBacklogSources[id] = backlog; return new EventStreamBacklogRegistration(this, id); } /// /// Removes event counters for the given session. /// /// Identifier of the session. public void RemoveSessionEvents(string sessionId) { _eventsBySession.TryRemove(sessionId, out _); } /// /// Records that a queue overflow occurred for the given queue name. /// /// Name of the queue that overflowed. public void QueueOverflow(string queueName) { lock (_syncRoot) { _queueOverflows++; } _queueOverflowsCounter.Add(1, new KeyValuePair("queue", queueName)); } /// /// Records that a fault occurred in the given category. /// /// Category of the fault. public void Fault(string category) { lock (_syncRoot) { _faults++; } _faultsCounter.Add(1, new KeyValuePair("category", category)); } /// /// Records that a heartbeat failed for the given session. /// /// Identifier of the session. public void HeartbeatFailed(string sessionId) { lock (_syncRoot) { _heartbeatFailures++; } // Exported without a session_id tag: per-session attribution is unbounded cardinality // (every session ever opened mints a new exporter time series, bloating Prometheus/OTLP // storage on long-running gateways). Per-session detail stays available via the dashboard // snapshot and the log scope; the exported counter tracks only the aggregate. _heartbeatFailuresCounter.Add(1); } /// /// Records that an event stream was disconnected with the given reason. /// /// Reason for the disconnection. public void StreamDisconnected(string reason) { lock (_syncRoot) { _streamDisconnects++; } _streamDisconnectsCounter.Add(1, new KeyValuePair("reason", reason)); } /// /// Records that a retry was attempted in the given area. /// /// Area in which the retry was attempted. public void RetryAttempted(string area) { lock (_syncRoot) { _retryAttempts++; Increment(_retryAttemptsByArea, area); } _retryAttemptsCounter.Add(1, new KeyValuePair("area", area)); } /// /// Records that the alarm provider switched modes, increments the switch count, and updates the /// current provider mode gauge. /// /// Provider mode before the switch (1=alarmmgr, 2=subtag, 0=unknown). /// Provider mode after the switch (1=alarmmgr, 2=subtag, 0=unknown). /// Bounded switch classification used as the counter's reason tag. public void AlarmProviderSwitched(int fromMode, int toMode, AlarmProviderSwitchReason reason) { lock (_syncRoot) { _alarmProviderMode = toMode; _alarmProviderSwitches++; } _alarmProviderSwitchesCounter.Add( 1, new KeyValuePair("from", fromMode.ToString(CultureInfo.InvariantCulture)), new KeyValuePair("to", toMode.ToString(CultureInfo.InvariantCulture)), new KeyValuePair("reason", ReasonTag(reason))); } private static string ReasonTag(AlarmProviderSwitchReason reason) => reason switch { AlarmProviderSwitchReason.Failover => "failover", AlarmProviderSwitchReason.Failback => "failback", _ => "unknown", }; /// Sets the current alarm provider-mode gauge without recording a switch (e.g. startup baseline). /// The alarm provider mode value to record. public void SetAlarmProviderMode(int mode) { lock (_syncRoot) { _alarmProviderMode = mode; } } /// /// Returns a snapshot of all current metric values. /// /// The current metrics snapshot. public GatewayMetricsSnapshot GetSnapshot() { // Compute the live gRPC stream backlog outside _syncRoot: the sources are the subscriber // channels' Count (their own locks) and must not run under this lock. GWC-15. int grpcEventStreamQueueDepth = GetGrpcEventStreamQueueDepth(); lock (_syncRoot) { return new GatewayMetricsSnapshot( OpenSessions: _openSessions, WorkersRunning: _workersRunning, WorkerEventQueueDepth: _workerEventQueueDepth, GrpcEventStreamQueueDepth: grpcEventStreamQueueDepth, SessionsOpened: _sessionsOpened, SessionsClosed: _sessionsClosed, CommandsStarted: _commandsStarted, CommandsSucceeded: _commandsSucceeded, CommandsFailed: _commandsFailed, EventsReceived: Interlocked.Read(ref _eventsReceived), QueueOverflows: _queueOverflows, Faults: _faults, WorkerKills: _workerKills, WorkerExits: _workerExits, HeartbeatFailures: _heartbeatFailures, StreamDisconnects: _streamDisconnects, RetryAttempts: _retryAttempts, AlarmProviderSwitchCount: _alarmProviderSwitches, CommandFailuresByMethod: new Dictionary(_commandFailuresByMethod, StringComparer.OrdinalIgnoreCase), EventsByFamily: new Dictionary(_eventsByFamily, StringComparer.OrdinalIgnoreCase), EventsBySession: new Dictionary(_eventsBySession, StringComparer.Ordinal), RetryAttemptsByArea: new Dictionary(_retryAttemptsByArea, StringComparer.OrdinalIgnoreCase)); } } /// /// Disposes the underlying OpenTelemetry meter. /// public void Dispose() { if (_disposed) { return; } _meter.Dispose(); _disposed = true; } private int GetOpenSessions() { lock (_syncRoot) { return _openSessions; } } private int GetWorkersRunning() { lock (_syncRoot) { return _workersRunning; } } private int GetWorkerEventQueueDepth() { lock (_syncRoot) { return _workerEventQueueDepth; } } // Sums the live backlog across every registered event-stream subscriber. Runs at collection // time (ObservableGauge scrape) or when GetSnapshot projects the value — never on the // per-event path. Enumerating ConcurrentDictionary.Values never throws on concurrent // register/unregister; a source removed mid-enumeration simply drops from this sample. private int GetGrpcEventStreamQueueDepth() { int total = 0; foreach (Func source in _eventStreamBacklogSources.Values) { int value = source(); if (value > 0) { total += value; } } return total; } private void UnregisterEventStreamBacklogSource(long id) { _eventStreamBacklogSources.TryRemove(id, out _); } private int GetAlarmProviderMode() { lock (_syncRoot) { return _alarmProviderMode; } } private static void Increment(Dictionary values, string key) { values.TryGetValue(key, out long currentValue); values[key] = currentValue + 1; } private static void Increment(ConcurrentDictionary values, string key) { values.AddOrUpdate(key, 1, static (_, currentValue) => currentValue + 1); } // Handle returned by RegisterEventStreamBacklogSource. Disposal (once) removes the source // from the gauge's live sum. Idempotent so a double dispose from a stream teardown is safe. private sealed class EventStreamBacklogRegistration(GatewayMetrics metrics, long id) : IDisposable { private int _disposed; public void Dispose() { if (Interlocked.Exchange(ref _disposed, 1) == 0) { metrics.UnregisterEventStreamBacklogSource(id); } } } }