using System.Diagnostics.Metrics; namespace ZB.MOM.WW.ScadaBridge.Commons.Observability; /// /// Central + instrument definitions for ScadaBridge's application /// telemetry, modelled on OtOpcUa's OtOpcUaTelemetry. Modules emit through these /// pre-created instruments so a single OpenTelemetry / Prometheus binding in /// Host (registered via AddZbTelemetry with this meter named in /// ZbTelemetryOptions.Meters) catches everything. No exporter is required — /// instruments are no-op until a listener attaches, so tests and dev hosts pay nothing /// for instrumentation that nobody scrapes. /// /// Instrument names follow the OpenTelemetry semantic convention pattern /// scadabridge.<subsystem>.<event>. This task defines the instruments and /// their emit helpers; four later tasks wire the actual emit points. Until those land the /// helpers are dormant but inert — calling them is safe and simply records against a meter /// that nothing observes. /// public static class ScadaBridgeTelemetry { /// The meter name registered with OTel via ZbTelemetryOptions.Meters. public const string MeterName = "ZB.MOM.WW.ScadaBridge"; /// Singleton all instruments hang off. private static readonly Meter Meter = new(MeterName); // ---------------- Counters ---------------- /// Incremented each time a deployment is successfully applied. private static readonly Counter _deploymentsApplied = Meter.CreateCounter("scadabridge.deployments.applied", unit: "1", description: "Deployments applied."); /// Incremented for each inbound API request, tagged with the API method. private static readonly Counter _inboundApiRequests = Meter.CreateCounter("scadabridge.inbound_api.requests", unit: "1", description: "Inbound API requests, tagged by method."); /// Incremented each time an inbound execution is abandoned (handler outlived its request), tagged with the API method. private static readonly Counter _inboundAbandonedExecutions = Meter.CreateCounter("scadabridge.inbound_api.abandoned_executions", unit: "1", description: "Inbound API executions abandoned after timeout/abort (handler still running), tagged by method."); /// Incremented each time an S&F buffer replication op fails to dispatch/deliver to the peer. private static readonly Counter _replicationFailures = Meter.CreateCounter("scadabridge.store_and_forward.replication.failures", unit: "1", description: "S&F buffer replication operations that failed to dispatch/deliver to the peer node"); private static readonly Counter _sfResyncCompleted = Meter.CreateCounter("scadabridge.store_and_forward.resync.completed", unit: "1", description: "S&F anti-entropy resyncs the standby acknowledged as applied"); private static readonly Counter _sfResyncAckMissing = Meter.CreateCounter("scadabridge.store_and_forward.resync.ack_missing", unit: "1", description: "S&F resyncs answered by the active node but never acknowledged by the standby within the ack window (lost chunks / dead peer)"); /// /// Incremented each time a per-site live-alarm aggregator re-establishes its site-wide /// gRPC stream — a NodeA↔NodeB failover flip or a reconcile-driven reopen after the /// stream was given up (plan #10, Task 6). A sustained climb signals a flapping site link. /// private static readonly Counter _liveAlarmStreamReconnects = Meter.CreateCounter("scadabridge.site.alarm_cache.reconnects", unit: "1", description: "Live-alarm aggregator site-wide gRPC stream reconnects (NodeA↔NodeB flip or reconcile-driven reopen)."); /// /// Incremented for each live delta evicted from a per-site live-alarm aggregator's /// bounded pre-seed buffer (drop-oldest, WP2.3). Non-zero means a seed/reconcile /// fan-out ran long enough for the delta storm behind it to exceed the cap — the /// dropped transitions are recovered by the fan-out's authoritative snapshot, but a /// sustained climb points at a slow site. /// private static readonly Counter _liveAlarmBufferDrops = Meter.CreateCounter("scadabridge.site.alarm_cache.buffer_dropped", unit: "1", description: "Live deltas evicted from a live-alarm aggregator's bounded pre-seed buffer (drop-oldest)."); /// /// Incremented for each debug event evicted from a central debug session's bounded /// pre-snapshot buffer (drop-oldest, WP2.3). The Debug View is lossy-under-backpressure /// by design; this makes the loss measurable instead of unbounded memory growth. /// private static readonly Counter _debugPreSnapshotDrops = Meter.CreateCounter("scadabridge.central.debug_view.presnapshot_dropped", unit: "1", description: "Debug events evicted from a central debug session's bounded pre-snapshot buffer (drop-oldest)."); /// /// Incremented for each event evicted from a site-hosted gRPC stream's bounded send /// channel, tagged by stream kind (instance = Debug View, site-alarms = /// the site-wide alarm feed behind the operator Alarm Summary). /// private static readonly Counter _siteStreamEventDrops = Meter.CreateCounter("scadabridge.site.stream.events_dropped", unit: "1", description: "Events evicted from a site gRPC stream's bounded send channel, tagged by stream kind."); // ---------------- Observable gauges ---------------- /// Current count of open site connections, mutated via . private static long _siteConnectionsUp; /// Current count of running per-site live-alarm aggregators, mutated via . private static long _liveAlarmAggregatorsActive; /// Provider that yields the live StoreAndForward queue depth; set by a later task. private static Func? _queueDepthProvider; #pragma warning disable IDE0052 // Held to keep the observable gauges alive for the meter's lifetime. /// Gauge reporting the number of currently open site connections. private static readonly ObservableGauge _siteConnectionUp = Meter.CreateObservableGauge("scadabridge.site.connection.up", () => Interlocked.Read(ref _siteConnectionsUp), description: "Number of currently open site connections."); /// Gauge reporting the current StoreAndForward queue depth via the registered provider. private static readonly ObservableGauge _storeAndForwardQueueDepth = Meter.CreateObservableGauge("scadabridge.store_and_forward.queue.depth", () => Volatile.Read(ref _queueDepthProvider) is { } p ? p() : 0L, unit: "items", description: "Current StoreAndForward queue depth."); /// Gauge reporting the number of currently running per-site live-alarm aggregators. private static readonly ObservableGauge _liveAlarmAggregatorsActiveGauge = Meter.CreateObservableGauge("scadabridge.site.alarm_cache.aggregators.active", () => Interlocked.Read(ref _liveAlarmAggregatorsActive), unit: "1", description: "Number of per-site live-alarm aggregators currently running on the active central node."); #pragma warning restore IDE0052 // ---------------- Emit helpers ---------------- /// Records that a deployment was applied. public static void RecordDeploymentApplied() => _deploymentsApplied.Add(1); /// Records an inbound API request for the given . /// The API method the request targeted. public static void RecordInboundApiRequest(string method) => _inboundApiRequests.Add(1, new KeyValuePair("method", method)); /// Records that an inbound execution was abandoned (handler outlived its request) for the given . /// The API method whose execution was abandoned. public static void RecordInboundAbandonedExecution(string method) => _inboundAbandonedExecutions.Add(1, new KeyValuePair("method", method)); /// Records one failed S&F replication dispatch. public static void RecordReplicationFailure() => _replicationFailures.Add(1); /// Records one acknowledged (applied) S&F buffer resync. public static void RecordSfResyncCompleted() => _sfResyncCompleted.Add(1); /// Records one resync whose ack window expired. public static void RecordSfResyncAckMissing() => _sfResyncAckMissing.Add(1); /// Records that a site connection opened (increments the up-count gauge). public static void SiteConnectionOpened() => Interlocked.Increment(ref _siteConnectionsUp); /// Records that a site connection closed (decrements the up-count gauge). public static void SiteConnectionClosed() => Interlocked.Decrement(ref _siteConnectionsUp); /// Records that a per-site live-alarm aggregator started (increments the active-aggregator gauge). public static void LiveAlarmAggregatorStarted() => Interlocked.Increment(ref _liveAlarmAggregatorsActive); /// Records that a per-site live-alarm aggregator stopped (decrements the active-aggregator gauge). public static void LiveAlarmAggregatorStopped() => Interlocked.Decrement(ref _liveAlarmAggregatorsActive); /// Records that a per-site live-alarm aggregator re-established its site-wide gRPC stream. public static void RecordLiveAlarmStreamReconnect() => _liveAlarmStreamReconnects.Add(1); /// Records live deltas evicted from a live-alarm aggregator's bounded pre-seed buffer. /// Number of deltas evicted. public static void RecordLiveAlarmBufferDrop(long count = 1) => _liveAlarmBufferDrops.Add(count); /// Records debug events evicted from a debug session's bounded pre-snapshot buffer. /// Number of events evicted. public static void RecordDebugPreSnapshotDrop(long count = 1) => _debugPreSnapshotDrops.Add(count); /// Records an event evicted from a site gRPC stream's bounded send channel. /// Stream kind tag (instance or site-alarms). public static void RecordSiteStreamEventDropped(string streamKind) => _siteStreamEventDrops.Add(1, new KeyValuePair("stream", streamKind)); /// /// Registers the provider the StoreAndForward queue-depth gauge reads on each observation. /// A later task supplies a provider that reads the real StoreAndForward depth. A null /// provider is ignored so the gauge falls back to reporting 0. /// /// A callback returning the current queue depth. public static void SetQueueDepthProvider(Func provider) { if (provider is null) { return; } Volatile.Write(ref _queueDepthProvider, provider); } /// /// Clears the StoreAndForward queue-depth provider, but only if the currently /// registered provider is the exact delegate passed in /// (reference-equal compare-and-clear). This lets a StoreAndForward service deregister /// its own provider on graceful stop without stomping a newer instance that already /// re-registered into the process-global slot: if a late stop of the old instance /// passes its (now-superseded) delegate, the identity check fails and the newer /// provider is preserved. After a successful clear the gauge falls back to reporting 0. /// Mirrors 's signature and /// access pattern. /// /// The provider delegate to remove; ignored unless it is the /// one currently registered. public static void ClearQueueDepthProvider(Func provider) { if (provider is null) { return; } // Compare-and-clear: only null the slot when it still holds the caller's // delegate, so a stale stop cannot clobber a successor's provider. Interlocked.CompareExchange(ref _queueDepthProvider, null, provider); } }