using System.Diagnostics.Metrics; using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums; namespace ZB.MOM.WW.ScadaBridge.Commons.Observability; /// /// Central + instrument definitions for ScadaBridge's application /// telemetry, modelled on OtOpcUa's OtOpcUaTelemetry. Modules emit through these /// pre-created instruments so a single OpenTelemetry / Prometheus binding in /// Host (registered via AddZbTelemetry with this meter named in /// ZbTelemetryOptions.Meters) catches everything. No exporter is required — /// instruments are no-op until a listener attaches, so tests and dev hosts pay nothing /// for instrumentation that nobody scrapes. /// /// Instrument names follow the OpenTelemetry semantic convention pattern /// scadabridge.<subsystem>.<event>. This task defines the instruments and /// their emit helpers; four later tasks wire the actual emit points. Until those land the /// helpers are dormant but inert — calling them is safe and simply records against a meter /// that nothing observes. /// public static class ScadaBridgeTelemetry { /// The meter name registered with OTel via ZbTelemetryOptions.Meters. public const string MeterName = "ZB.MOM.WW.ScadaBridge"; /// Singleton all instruments hang off. private static readonly Meter Meter = new(MeterName); // ---------------- Counters ---------------- /// Incremented each time a deployment is successfully applied. private static readonly Counter _deploymentsApplied = Meter.CreateCounter("scadabridge.deployments.applied", unit: "1", description: "Deployments applied."); /// /// Incremented for each deployment that ends in a terminal failure (ScadaBridge#81), tagged /// reason: timeout (the site round-trip timed out), error (any other fault /// in the send or post-send persistence path) or site_failed (the site answered with a /// non-Success status). Pairs with scadabridge.deployments.applied to give the deploy /// success ratio; the rate-limited log lines cannot be alerted on. /// private static readonly Counter _deploymentsFailed = Meter.CreateCounter("scadabridge.deployments.failed", unit: "1", description: "Deployments that ended in a terminal failure, tagged by reason (timeout|error|site_failed)."); /// /// Incremented for each store-and-forward lifecycle event (ScadaBridge#81), tagged action /// (Queued, Delivered, Retried, Parked, Retry, Discard) /// and category. A sustained Parked rate is the alertable signal that messages are /// exhausting retries; the queue-depth gauge alone cannot distinguish a drain from a park. /// private static readonly Counter _sfActivity = Meter.CreateCounter("scadabridge.store_and_forward.activity", unit: "1", description: "Store-and-forward lifecycle events, tagged by action and category."); /// /// Incremented for each finished site script run (ScadaBridge#81), tagged outcome /// (completed | failed | timed_out). Deliberately NOT tagged by instance or /// script name: that cardinality is unbounded across a site's instances. /// private static readonly Counter _scriptRuns = Meter.CreateCounter("scadabridge.site.script.runs", unit: "1", description: "Finished site script runs, tagged by outcome (completed|failed|timed_out)."); /// /// Incremented each time a data connection actually CHANGES state (ScadaBridge#81), tagged /// state (the new ). The data_connections gauges show the /// current picture; this shows flapping that a scrape interval would otherwise average away. /// private static readonly Counter _dataConnectionTransitions = Meter.CreateCounter("scadabridge.site.data_connections.transitions", unit: "1", description: "Data connection state changes, tagged by the new state."); /// Incremented for each inbound API request, tagged with the API method. private static readonly Counter _inboundApiRequests = Meter.CreateCounter("scadabridge.inbound_api.requests", unit: "1", description: "Inbound API requests, tagged by method."); /// Incremented each time an inbound execution is abandoned (handler outlived its request), tagged with the API method. private static readonly Counter _inboundAbandonedExecutions = Meter.CreateCounter("scadabridge.inbound_api.abandoned_executions", unit: "1", description: "Inbound API executions abandoned after timeout/abort (handler still running), tagged by method."); /// Incremented each time an S&F buffer replication op fails to dispatch/deliver to the peer. private static readonly Counter _replicationFailures = Meter.CreateCounter("scadabridge.store_and_forward.replication.failures", unit: "1", description: "S&F buffer replication operations that failed to dispatch/deliver to the peer node"); private static readonly Counter _sfResyncCompleted = Meter.CreateCounter("scadabridge.store_and_forward.resync.completed", unit: "1", description: "S&F anti-entropy resyncs the standby acknowledged as applied"); private static readonly Counter _sfResyncAckMissing = Meter.CreateCounter("scadabridge.store_and_forward.resync.ack_missing", unit: "1", description: "S&F resyncs answered by the active node but never acknowledged by the standby within the ack window (lost chunks / dead peer)"); /// /// Incremented each time a per-site live-alarm aggregator re-establishes its site-wide /// gRPC stream — a NodeA↔NodeB failover flip or a reconcile-driven reopen after the /// stream was given up. A sustained climb signals a flapping site link. /// private static readonly Counter _liveAlarmStreamReconnects = Meter.CreateCounter("scadabridge.site.alarm_cache.reconnects", unit: "1", description: "Live-alarm aggregator site-wide gRPC stream reconnects (NodeA↔NodeB flip or reconcile-driven reopen)."); /// /// Incremented for each live delta evicted from a per-site live-alarm aggregator's /// bounded pre-seed buffer (drop-oldest). Non-zero means a seed/reconcile /// fan-out ran long enough for the delta storm behind it to exceed the cap — the /// dropped transitions are recovered by the fan-out's authoritative snapshot, but a /// sustained climb points at a slow site. /// private static readonly Counter _liveAlarmBufferDrops = Meter.CreateCounter("scadabridge.site.alarm_cache.buffer_dropped", unit: "1", description: "Live deltas evicted from a live-alarm aggregator's bounded pre-seed buffer (drop-oldest)."); /// /// Incremented by the number of entries the site script compile cache evicts when it crosses its /// bound (ScadaBridge#68). Eviction does not unload the Roslyn assembly, so each recompile of an /// evicted body loads another permanent copy: a sustained climb means compile-cache thrash and /// monotonic working-set growth - raise ScadaBridge:SiteRuntime:ScriptCompileCacheMaxEntries. /// Alert on this instead of the rate-limited Warning log. /// private static readonly Counter _scriptCompileCacheEvictions = Meter.CreateCounter("scadabridge.site.script_compile_cache.evictions", unit: "1", description: "Entries evicted from the site script compile cache at its bound (each recompile of an evicted body loads another permanent Roslyn assembly)."); /// /// Incremented for each debug event evicted from a central debug session's bounded /// pre-snapshot buffer (drop-oldest). The Debug View is lossy-under-backpressure /// by design; this makes the loss measurable instead of unbounded memory growth. /// private static readonly Counter _debugPreSnapshotDrops = Meter.CreateCounter("scadabridge.central.debug_view.presnapshot_dropped", unit: "1", description: "Debug events evicted from a central debug session's bounded pre-snapshot buffer (drop-oldest)."); /// /// Incremented for each event evicted from a site-hosted gRPC stream's bounded send /// channel, tagged by stream kind (instance = Debug View, site-alarms = /// the site-wide alarm feed behind the operator Alarm Summary). /// private static readonly Counter _siteStreamEventDrops = Meter.CreateCounter("scadabridge.site.stream.events_dropped", unit: "1", description: "Events evicted from a site gRPC stream's bounded send channel, tagged by stream kind."); /// /// Register row 66: incremented each time BundleSessionStore.Open refuses a /// bundle-import session because TransportOptions.MaxConcurrentImportSessions /// (default 8) is genuinely exhausted — i.e. the store was still at the cap AFTER /// expired sessions were reclaimed. Reclaim-then-succeed and re-opening an already /// open session id are deliberately NOT counted: neither turns an operator away. /// /// Row 53 shipped the companion open-sessions gauge and a Warning log, but a log line /// is not alertable. The pair is what distinguishes the two failure modes: rejections /// climbing while the gauge oscillates below the cap is genuine concurrency (raise the /// cap), whereas rejections climbing while the gauge sits pinned at the cap is a /// session leak like register row 43's (find the leak — raising the cap only defers it). /// private static readonly Counter _transportImportSessionsRejected = Meter.CreateCounter("scadabridge.central.transport.import_sessions.rejected", unit: "1", description: "Bundle-import sessions rejected because the concurrent-import-session cap was exhausted."); /// /// Register row 34: incremented once per site-event-log purge tick that stopped at /// SiteEventLogOptions.MaxPurgedRowsPerTick with expired rows still to delete. /// /// A single hit is routine (a backlog is being worked through on accelerated catch-up /// ticks) — what the series is for is the SHAPE. A short burst of hits that stops is a /// backlog converging; hits that never stop mean the event log is being written faster /// than one paced tick can purge, i.e. the ceiling or the catch-up interval is now /// mis-sized for this site's event rate. The paired log line is rate-limited and so /// cannot answer that question on its own. /// private static readonly Counter _siteEventLogPurgeCeilingHits = Meter.CreateCounter("scadabridge.site.event_log.purge_ceiling_hits", unit: "1", description: "Site event log purge ticks that stopped at the per-tick delete ceiling with work remaining."); // ---------------- Histograms ---------------- /// /// Distribution of how many events each site→central stream frame carried (R2 — gRPC /// event batching), tagged by stream kind. Recorded ONLY for subscriptions that /// negotiated batching, so the series' very existence says "this central speaks the /// batched wire". A distribution pinned at 1 means the coalescing window never sees a /// backlog (the site is quiet, or the window is too small to be earning anything); /// mass at the size cap means the cap, not the window, is the binding constraint. /// private static readonly Histogram _siteStreamBatchSize = Meter.CreateHistogram("scadabridge.site.stream.batch_size", unit: "1", description: "Events per site gRPC stream frame (1 = unbatched frame), tagged by stream kind."); /// Duration of finished site script runs in milliseconds (ScadaBridge#81), tagged by outcome. private static readonly Histogram _scriptRunDuration = Meter.CreateHistogram("scadabridge.site.script.duration", unit: "ms", description: "Site script run duration in milliseconds, tagged by outcome (completed|failed|timed_out)."); // ---------------- Observable gauges ---------------- /// /// Current count of open central→site gRPC subscriber streams, mutated via /// . See 's gauge for /// why this is NOT a data-connection count. /// private static long _siteStreamSubscriberCount; /// /// Register row 54: last-known state of every data connection on this site node, keyed by /// connection name. Written by the DCL at its four state-transition points and read by the /// data-connection gauges below. /// /// /// A static registry, matching 's precedent, rather /// than a DI-injected holder: the DCL actors already receive ISiteHealthCollector /// through their constructors, and threading a second dependency through both actor /// constructors would ripple into every test that builds them. A site node hosts exactly /// one DCL, so a process-global registry and a per-node one are the same thing here. /// private static readonly System.Collections.Concurrent.ConcurrentDictionary _dataConnectionStates = new(StringComparer.Ordinal); /// Current count of running per-site live-alarm aggregators, mutated via . private static long _liveAlarmAggregatorsActive; /// Provider that yields the live StoreAndForward queue depth; set by a later task. private static Func? _queueDepthProvider; /// /// Register row 53: provider that yields the number of currently open (unexpired) /// Transport bundle-import sessions, registered by the Transport component's /// session-eviction hosted service. Null on a site node (Transport is central-only), /// where the gauge simply reports 0. /// private static Func? _importSessionCountProvider; #pragma warning disable IDE0052 // Held to keep the observable gauges alive for the meter's lifetime. /// /// Gauge reporting the number of open central→site gRPC subscriber streams. /// /// /// Register row 54 RENAMED this from scadabridge.site.connection.up /// (scadabridge_site_connection_up in Prometheus). The old name read as a /// data-connection health signal, but it has only ever counted open subscriber streams /// on SiteStreamGrpcServer — Debug View and the site-wide alarm feed. It therefore /// sits at 0 on a perfectly healthy site that nobody is watching, and an alert written /// against the old name fired on an idle operator console rather than a broken device. /// For real data-connection health use the scadabridge.site.data_connections.* /// gauges below. Counts both stream kinds together (no stream tag). /// private static readonly ObservableGauge _siteStreamSubscribersActive = Meter.CreateObservableGauge("scadabridge.site.stream.subscribers.active", () => Interlocked.Read(ref _siteStreamSubscriberCount), unit: "1", description: "Open central→site gRPC subscriber streams (Debug View + site-wide alarm feed). NOT data connections."); /// /// Register row 54: gauge reporting how many of this site node's data connections are /// currently in — the real data-connection /// signal the misnamed stream-subscriber gauge was being mistaken for. /// /// /// Read together with : connected < total is the /// alertable condition, and neither number alone distinguishes "one device down" from /// "nothing configured". Expect **0 on the standby node** — connections are opened by the /// deployment path on the active node only, so a standby reporting 0 is healthy, not a /// fault. Alert on the active node, or on max() across the pair. /// private static readonly ObservableGauge _dataConnectionsConnected = Meter.CreateObservableGauge("scadabridge.site.data_connections.connected", CountDataConnections, unit: "1", description: "Data connections on this site node currently in the Connected state."); /// /// Register row 54: gauge reporting how many data connections this site node knows about, /// in any state. The denominator for . /// private static readonly ObservableGauge _dataConnectionsTotal = Meter.CreateObservableGauge("scadabridge.site.data_connections.total", () => (long)_dataConnectionStates.Count, unit: "1", description: "Data connections known to this site node, in any state."); /// Gauge reporting the current StoreAndForward queue depth via the registered provider. private static readonly ObservableGauge _storeAndForwardQueueDepth = Meter.CreateObservableGauge("scadabridge.store_and_forward.queue.depth", () => Volatile.Read(ref _queueDepthProvider) is { } p ? p() : 0L, unit: "items", description: "Current StoreAndForward queue depth."); /// Gauge reporting the number of currently running per-site live-alarm aggregators. private static readonly ObservableGauge _liveAlarmAggregatorsActiveGauge = Meter.CreateObservableGauge("scadabridge.site.alarm_cache.aggregators.active", () => Interlocked.Read(ref _liveAlarmAggregatorsActive), unit: "1", description: "Number of per-site live-alarm aggregators currently running on the active central node."); /// /// Register row 53: gauge reporting the number of open Transport bundle-import /// sessions on this central node. Each open session pins a fully decrypted bundle /// in memory and consumes one of TransportOptions.MaxConcurrentImportSessions /// (default 8), so a series that plateaus at the cap is the signature of leaked /// sessions — the failure mode register row 43 could only be inferred from rotating /// test failures because nothing counted them. /// private static readonly ObservableGauge _transportImportSessionsOpen = Meter.CreateObservableGauge("scadabridge.central.transport.import_sessions.open", () => Volatile.Read(ref _importSessionCountProvider) is { } p ? p() : 0L, unit: "1", description: "Number of currently open bundle-import sessions on this central node."); #pragma warning restore IDE0052 // ---------------- Emit helpers ---------------- /// Records that a deployment was applied. public static void RecordDeploymentApplied() => _deploymentsApplied.Add(1); /// Records one deployment that ended in a terminal failure (ScadaBridge#81). /// One of timeout, error, site_failed. public static void RecordDeploymentFailed(string reason) => _deploymentsFailed.Add(1, new KeyValuePair("reason", reason)); /// Records one store-and-forward lifecycle event (ScadaBridge#81). /// The activity action, e.g. Parked. /// The message category name. public static void RecordStoreAndForwardActivity(string action, string category) => _sfActivity.Add(1, new KeyValuePair("action", action), new KeyValuePair("category", category)); /// Records one finished site script run and its duration (ScadaBridge#81). /// One of completed, failed, timed_out. /// The run duration in milliseconds. public static void RecordScriptRun(string outcome, long durationMs) { var tag = new KeyValuePair("outcome", outcome); _scriptRuns.Add(1, tag); _scriptRunDuration.Record(durationMs < 0 ? 0 : durationMs, tag); } /// Records an inbound API request for the given . /// The API method the request targeted. public static void RecordInboundApiRequest(string method) => _inboundApiRequests.Add(1, new KeyValuePair("method", method)); /// Records that an inbound execution was abandoned (handler outlived its request) for the given . /// The API method whose execution was abandoned. public static void RecordInboundAbandonedExecution(string method) => _inboundAbandonedExecutions.Add(1, new KeyValuePair("method", method)); /// Records one failed S&F replication dispatch. public static void RecordReplicationFailure() => _replicationFailures.Add(1); /// Records one acknowledged (applied) S&F buffer resync. public static void RecordSfResyncCompleted() => _sfResyncCompleted.Add(1); /// Records one resync whose ack window expired. public static void RecordSfResyncAckMissing() => _sfResyncAckMissing.Add(1); /// /// Records one bundle-import session rejected on a genuine concurrent-session cap hit /// (register row 66). Call ONLY from the post-eviction recheck in /// BundleSessionStore.Open — counting the pre-eviction check would inflate the /// series with reclaim-then-succeed cases that never turned an operator away. /// public static void RecordImportSessionRejected() => _transportImportSessionsRejected.Add(1); /// /// Records one site-event-log purge tick that stopped at its per-tick delete ceiling /// (register row 34). Call ONLY when rows genuinely remain to be purged — a tick that /// finished the work and happened to land exactly on the ceiling has nothing to report. /// public static void RecordSiteEventLogPurgeCeilingHit() => _siteEventLogPurgeCeilingHits.Add(1); /// /// Records that a central→site gRPC subscriber stream opened (increments /// scadabridge.site.stream.subscribers.active). Renamed from /// SiteConnectionOpened by register row 54 — it never counted data connections. /// public static void SiteStreamSubscriberOpened() => Interlocked.Increment(ref _siteStreamSubscriberCount); /// /// Records that a central→site gRPC subscriber stream closed (decrements /// scadabridge.site.stream.subscribers.active). Renamed from /// SiteConnectionClosed by register row 54. /// public static void SiteStreamSubscriberClosed() => Interlocked.Decrement(ref _siteStreamSubscriberCount); /// /// Records the current state of one data connection, feeding the /// scadabridge.site.data_connections.* gauges (register row 54). Called by the DCL /// at each connection state transition; safe to call repeatedly with the same value. /// /// The data connection's name (unique per site node). /// The connection's new state. public static void RecordDataConnectionState(string connectionName, ConnectionHealth state) { if (string.IsNullOrEmpty(connectionName)) { return; } var previous = _dataConnectionStates.TryGetValue(connectionName, out var prior) ? prior : (ConnectionHealth?)null; _dataConnectionStates[connectionName] = state; if (previous != state) { _dataConnectionTransitions.Add(1, new KeyValuePair("state", state.ToString())); } } /// /// Drops a data connection from the gauges (register row 54). Called when the DCL removes /// a connection, so an undeployed connection stops counting toward the total rather than /// lingering as a permanently-not-connected row that would alert forever. /// /// The data connection's name. public static void RemoveDataConnection(string connectionName) { if (string.IsNullOrEmpty(connectionName)) { return; } _dataConnectionStates.TryRemove(connectionName, out _); } /// Counts data connections currently in . /// The connected count. private static long CountDataConnections() { long connected = 0; foreach (var state in _dataConnectionStates.Values) { if (state == ConnectionHealth.Connected) { connected++; } } return connected; } /// Records that a per-site live-alarm aggregator started (increments the active-aggregator gauge). public static void LiveAlarmAggregatorStarted() => Interlocked.Increment(ref _liveAlarmAggregatorsActive); /// Records that a per-site live-alarm aggregator stopped (decrements the active-aggregator gauge). public static void LiveAlarmAggregatorStopped() => Interlocked.Decrement(ref _liveAlarmAggregatorsActive); /// Records that a per-site live-alarm aggregator re-established its site-wide gRPC stream. public static void RecordLiveAlarmStreamReconnect() => _liveAlarmStreamReconnects.Add(1); /// Records live deltas evicted from a live-alarm aggregator's bounded pre-seed buffer. /// Number of deltas evicted. public static void RecordLiveAlarmBufferDrop(long count = 1) => _liveAlarmBufferDrops.Add(count); /// Records entries evicted from the site script compile cache. /// Number of entries evicted by one sweep. public static void RecordScriptCompileCacheEvictions(long count) => _scriptCompileCacheEvictions.Add(count); /// Records debug events evicted from a debug session's bounded pre-snapshot buffer. /// Number of events evicted. public static void RecordDebugPreSnapshotDrop(long count = 1) => _debugPreSnapshotDrops.Add(count); /// Records an event evicted from a site gRPC stream's bounded send channel. /// Stream kind tag (instance or site-alarms). public static void RecordSiteStreamEventDropped(string streamKind) => _siteStreamEventDrops.Add(1, new KeyValuePair("stream", streamKind)); /// /// Records how many events one site gRPC stream frame carried. Called once per emitted /// frame on a batching-negotiated subscription (never on an un-negotiated one, where it /// would degenerate into a per-event instrument on the hottest path in the product). /// /// Stream kind tag (instance or site-alarms). /// Events packed into the frame; 1 for a plain unbatched frame. public static void RecordSiteStreamBatchSize(string streamKind, int events) => _siteStreamBatchSize.Record(events, new KeyValuePair("stream", streamKind)); /// /// Registers the provider the StoreAndForward queue-depth gauge reads on each observation. /// A later task supplies a provider that reads the real StoreAndForward depth. A null /// provider is ignored so the gauge falls back to reporting 0. /// /// A callback returning the current queue depth. public static void SetQueueDepthProvider(Func provider) { if (provider is null) { return; } Volatile.Write(ref _queueDepthProvider, provider); } /// /// Clears the StoreAndForward queue-depth provider, but only if the currently /// registered provider is the exact delegate passed in /// (reference-equal compare-and-clear). This lets a StoreAndForward service deregister /// its own provider on graceful stop without stomping a newer instance that already /// re-registered into the process-global slot: if a late stop of the old instance /// passes its (now-superseded) delegate, the identity check fails and the newer /// provider is preserved. After a successful clear the gauge falls back to reporting 0. /// Mirrors 's signature and /// access pattern. /// /// The provider delegate to remove; ignored unless it is the /// one currently registered. public static void ClearQueueDepthProvider(Func provider) { if (provider is null) { return; } // Compare-and-clear: only null the slot when it still holds the caller's // delegate, so a stale stop cannot clobber a successor's provider. Interlocked.CompareExchange(ref _queueDepthProvider, null, provider); } /// /// Register row 53: registers the provider the open-import-sessions gauge reads on /// each observation. Called by the Transport component's session-eviction hosted /// service on start. A null provider is ignored so the gauge falls back to 0. /// The callback is invoked synchronously on the metrics collection thread, so it /// must stay cheap — the Transport implementation reads a /// it /// already owns. /// /// A callback returning the current open-session count. public static void SetImportSessionCountProvider(Func provider) { if (provider is null) { return; } Volatile.Write(ref _importSessionCountProvider, provider); } /// /// Clears the open-import-sessions provider, but only if the currently registered /// provider is the exact delegate passed in /// (reference-equal compare-and-clear), so a late stop of a superseded instance /// cannot stomp a newer registrant. Mirrors . /// /// The provider delegate to remove; ignored unless it is the /// one currently registered. public static void ClearImportSessionCountProvider(Func provider) { if (provider is null) { return; } Interlocked.CompareExchange(ref _importSessionCountProvider, null, provider); } }