using System.Diagnostics.Metrics;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
namespace ZB.MOM.WW.ScadaBridge.Commons.Observability;
///
/// Central + instrument definitions for ScadaBridge's application
/// telemetry, modelled on OtOpcUa's OtOpcUaTelemetry. Modules emit through these
/// pre-created instruments so a single OpenTelemetry / Prometheus binding in
/// Host (registered via AddZbTelemetry with this meter named in
/// ZbTelemetryOptions.Meters) catches everything. No exporter is required —
/// instruments are no-op until a listener attaches, so tests and dev hosts pay nothing
/// for instrumentation that nobody scrapes.
///
/// Instrument names follow the OpenTelemetry semantic convention pattern
/// scadabridge.<subsystem>.<event>. This task defines the instruments and
/// their emit helpers; four later tasks wire the actual emit points. Until those land the
/// helpers are dormant but inert — calling them is safe and simply records against a meter
/// that nothing observes.
///
public static class ScadaBridgeTelemetry
{
/// The meter name registered with OTel via ZbTelemetryOptions.Meters.
public const string MeterName = "ZB.MOM.WW.ScadaBridge";
/// Singleton all instruments hang off.
private static readonly Meter Meter = new(MeterName);
// ---------------- Counters ----------------
/// Incremented each time a deployment is successfully applied.
private static readonly Counter _deploymentsApplied =
Meter.CreateCounter("scadabridge.deployments.applied", unit: "1",
description: "Deployments applied.");
///
/// Incremented for each deployment that ends in a terminal failure (ScadaBridge#81), tagged
/// reason: timeout (the site round-trip timed out), error (any other fault
/// in the send or post-send persistence path) or site_failed (the site answered with a
/// non-Success status). Pairs with scadabridge.deployments.applied to give the deploy
/// success ratio; the rate-limited log lines cannot be alerted on.
///
private static readonly Counter _deploymentsFailed =
Meter.CreateCounter("scadabridge.deployments.failed", unit: "1",
description: "Deployments that ended in a terminal failure, tagged by reason (timeout|error|site_failed).");
///
/// Incremented for each store-and-forward lifecycle event (ScadaBridge#81), tagged action
/// (Queued, Delivered, Retried, Parked, Retry, Discard)
/// and category. A sustained Parked rate is the alertable signal that messages are
/// exhausting retries; the queue-depth gauge alone cannot distinguish a drain from a park.
///
private static readonly Counter _sfActivity =
Meter.CreateCounter("scadabridge.store_and_forward.activity", unit: "1",
description: "Store-and-forward lifecycle events, tagged by action and category.");
///
/// Incremented for each finished site script run (ScadaBridge#81), tagged outcome
/// (completed | failed | timed_out). Deliberately NOT tagged by instance or
/// script name: that cardinality is unbounded across a site's instances.
///
private static readonly Counter _scriptRuns =
Meter.CreateCounter("scadabridge.site.script.runs", unit: "1",
description: "Finished site script runs, tagged by outcome (completed|failed|timed_out).");
///
/// Incremented each time a data connection actually CHANGES state (ScadaBridge#81), tagged
/// state (the new ). The data_connections gauges show the
/// current picture; this shows flapping that a scrape interval would otherwise average away.
///
private static readonly Counter _dataConnectionTransitions =
Meter.CreateCounter("scadabridge.site.data_connections.transitions", unit: "1",
description: "Data connection state changes, tagged by the new state.");
/// Incremented for each inbound API request, tagged with the API method.
private static readonly Counter _inboundApiRequests =
Meter.CreateCounter("scadabridge.inbound_api.requests", unit: "1",
description: "Inbound API requests, tagged by method.");
/// Incremented each time an inbound execution is abandoned (handler outlived its request), tagged with the API method.
private static readonly Counter _inboundAbandonedExecutions =
Meter.CreateCounter("scadabridge.inbound_api.abandoned_executions", unit: "1",
description: "Inbound API executions abandoned after timeout/abort (handler still running), tagged by method.");
/// Incremented each time an S&F buffer replication op fails to dispatch/deliver to the peer.
private static readonly Counter _replicationFailures =
Meter.CreateCounter("scadabridge.store_and_forward.replication.failures", unit: "1",
description: "S&F buffer replication operations that failed to dispatch/deliver to the peer node");
private static readonly Counter _sfResyncCompleted =
Meter.CreateCounter("scadabridge.store_and_forward.resync.completed", unit: "1",
description: "S&F anti-entropy resyncs the standby acknowledged as applied");
private static readonly Counter _sfResyncAckMissing =
Meter.CreateCounter("scadabridge.store_and_forward.resync.ack_missing", unit: "1",
description: "S&F resyncs answered by the active node but never acknowledged by the standby within the ack window (lost chunks / dead peer)");
///
/// Incremented each time a per-site live-alarm aggregator re-establishes its site-wide
/// gRPC stream — a NodeA↔NodeB failover flip or a reconcile-driven reopen after the
/// stream was given up. A sustained climb signals a flapping site link.
///
private static readonly Counter _liveAlarmStreamReconnects =
Meter.CreateCounter("scadabridge.site.alarm_cache.reconnects", unit: "1",
description: "Live-alarm aggregator site-wide gRPC stream reconnects (NodeA↔NodeB flip or reconcile-driven reopen).");
///
/// Incremented for each live delta evicted from a per-site live-alarm aggregator's
/// bounded pre-seed buffer (drop-oldest). Non-zero means a seed/reconcile
/// fan-out ran long enough for the delta storm behind it to exceed the cap — the
/// dropped transitions are recovered by the fan-out's authoritative snapshot, but a
/// sustained climb points at a slow site.
///
private static readonly Counter _liveAlarmBufferDrops =
Meter.CreateCounter("scadabridge.site.alarm_cache.buffer_dropped", unit: "1",
description: "Live deltas evicted from a live-alarm aggregator's bounded pre-seed buffer (drop-oldest).");
///
/// Incremented by the number of entries the site script compile cache evicts when it crosses its
/// bound (ScadaBridge#68). Eviction does not unload the Roslyn assembly, so each recompile of an
/// evicted body loads another permanent copy: a sustained climb means compile-cache thrash and
/// monotonic working-set growth - raise ScadaBridge:SiteRuntime:ScriptCompileCacheMaxEntries.
/// Alert on this instead of the rate-limited Warning log.
///
private static readonly Counter _scriptCompileCacheEvictions =
Meter.CreateCounter("scadabridge.site.script_compile_cache.evictions", unit: "1",
description: "Entries evicted from the site script compile cache at its bound (each recompile of an evicted body loads another permanent Roslyn assembly).");
///
/// Incremented for each debug event evicted from a central debug session's bounded
/// pre-snapshot buffer (drop-oldest). The Debug View is lossy-under-backpressure
/// by design; this makes the loss measurable instead of unbounded memory growth.
///
private static readonly Counter _debugPreSnapshotDrops =
Meter.CreateCounter("scadabridge.central.debug_view.presnapshot_dropped", unit: "1",
description: "Debug events evicted from a central debug session's bounded pre-snapshot buffer (drop-oldest).");
///
/// Incremented for each event evicted from a site-hosted gRPC stream's bounded send
/// channel, tagged by stream kind (instance = Debug View, site-alarms =
/// the site-wide alarm feed behind the operator Alarm Summary).
///
private static readonly Counter _siteStreamEventDrops =
Meter.CreateCounter("scadabridge.site.stream.events_dropped", unit: "1",
description: "Events evicted from a site gRPC stream's bounded send channel, tagged by stream kind.");
///
/// Register row 66: incremented each time BundleSessionStore.Open refuses a
/// bundle-import session because TransportOptions.MaxConcurrentImportSessions
/// (default 8) is genuinely exhausted — i.e. the store was still at the cap AFTER
/// expired sessions were reclaimed. Reclaim-then-succeed and re-opening an already
/// open session id are deliberately NOT counted: neither turns an operator away.
///
/// Row 53 shipped the companion open-sessions gauge and a Warning log, but a log line
/// is not alertable. The pair is what distinguishes the two failure modes: rejections
/// climbing while the gauge oscillates below the cap is genuine concurrency (raise the
/// cap), whereas rejections climbing while the gauge sits pinned at the cap is a
/// session leak like register row 43's (find the leak — raising the cap only defers it).
///
private static readonly Counter _transportImportSessionsRejected =
Meter.CreateCounter("scadabridge.central.transport.import_sessions.rejected", unit: "1",
description: "Bundle-import sessions rejected because the concurrent-import-session cap was exhausted.");
///
/// Register row 34: incremented once per site-event-log purge tick that stopped at
/// SiteEventLogOptions.MaxPurgedRowsPerTick with expired rows still to delete.
///
/// A single hit is routine (a backlog is being worked through on accelerated catch-up
/// ticks) — what the series is for is the SHAPE. A short burst of hits that stops is a
/// backlog converging; hits that never stop mean the event log is being written faster
/// than one paced tick can purge, i.e. the ceiling or the catch-up interval is now
/// mis-sized for this site's event rate. The paired log line is rate-limited and so
/// cannot answer that question on its own.
///
private static readonly Counter _siteEventLogPurgeCeilingHits =
Meter.CreateCounter("scadabridge.site.event_log.purge_ceiling_hits", unit: "1",
description: "Site event log purge ticks that stopped at the per-tick delete ceiling with work remaining.");
// ---------------- Histograms ----------------
///
/// Distribution of how many events each site→central stream frame carried (R2 — gRPC
/// event batching), tagged by stream kind. Recorded ONLY for subscriptions that
/// negotiated batching, so the series' very existence says "this central speaks the
/// batched wire". A distribution pinned at 1 means the coalescing window never sees a
/// backlog (the site is quiet, or the window is too small to be earning anything);
/// mass at the size cap means the cap, not the window, is the binding constraint.
///
private static readonly Histogram _siteStreamBatchSize =
Meter.CreateHistogram("scadabridge.site.stream.batch_size", unit: "1",
description: "Events per site gRPC stream frame (1 = unbatched frame), tagged by stream kind.");
/// Duration of finished site script runs in milliseconds (ScadaBridge#81), tagged by outcome.
private static readonly Histogram _scriptRunDuration =
Meter.CreateHistogram("scadabridge.site.script.duration", unit: "ms",
description: "Site script run duration in milliseconds, tagged by outcome (completed|failed|timed_out).");
// ---------------- Observable gauges ----------------
///
/// Current count of open central→site gRPC subscriber streams, mutated via
/// . See 's gauge for
/// why this is NOT a data-connection count.
///
private static long _siteStreamSubscriberCount;
///
/// Register row 54: last-known state of every data connection on this site node, keyed by
/// connection name. Written by the DCL at its four state-transition points and read by the
/// data-connection gauges below.
///
///
/// A static registry, matching 's precedent, rather
/// than a DI-injected holder: the DCL actors already receive ISiteHealthCollector
/// through their constructors, and threading a second dependency through both actor
/// constructors would ripple into every test that builds them. A site node hosts exactly
/// one DCL, so a process-global registry and a per-node one are the same thing here.
///
private static readonly System.Collections.Concurrent.ConcurrentDictionary
_dataConnectionStates = new(StringComparer.Ordinal);
/// Current count of running per-site live-alarm aggregators, mutated via .
private static long _liveAlarmAggregatorsActive;
/// Provider that yields the live StoreAndForward queue depth; set by a later task.
private static Func? _queueDepthProvider;
///
/// Register row 53: provider that yields the number of currently open (unexpired)
/// Transport bundle-import sessions, registered by the Transport component's
/// session-eviction hosted service. Null on a site node (Transport is central-only),
/// where the gauge simply reports 0.
///
private static Func? _importSessionCountProvider;
#pragma warning disable IDE0052 // Held to keep the observable gauges alive for the meter's lifetime.
///
/// Gauge reporting the number of open central→site gRPC subscriber streams.
///
///
/// Register row 54 RENAMED this from scadabridge.site.connection.up
/// (scadabridge_site_connection_up in Prometheus). The old name read as a
/// data-connection health signal, but it has only ever counted open subscriber streams
/// on SiteStreamGrpcServer — Debug View and the site-wide alarm feed. It therefore
/// sits at 0 on a perfectly healthy site that nobody is watching, and an alert written
/// against the old name fired on an idle operator console rather than a broken device.
/// For real data-connection health use the scadabridge.site.data_connections.*
/// gauges below. Counts both stream kinds together (no stream tag).
///
private static readonly ObservableGauge _siteStreamSubscribersActive =
Meter.CreateObservableGauge("scadabridge.site.stream.subscribers.active",
() => Interlocked.Read(ref _siteStreamSubscriberCount),
unit: "1",
description: "Open central→site gRPC subscriber streams (Debug View + site-wide alarm feed). NOT data connections.");
///
/// Register row 54: gauge reporting how many of this site node's data connections are
/// currently in — the real data-connection
/// signal the misnamed stream-subscriber gauge was being mistaken for.
///
///
/// Read together with : connected < total is the
/// alertable condition, and neither number alone distinguishes "one device down" from
/// "nothing configured". Expect **0 on the standby node** — connections are opened by the
/// deployment path on the active node only, so a standby reporting 0 is healthy, not a
/// fault. Alert on the active node, or on max() across the pair.
///
private static readonly ObservableGauge _dataConnectionsConnected =
Meter.CreateObservableGauge("scadabridge.site.data_connections.connected",
CountDataConnections,
unit: "1",
description: "Data connections on this site node currently in the Connected state.");
///
/// Register row 54: gauge reporting how many data connections this site node knows about,
/// in any state. The denominator for .
///
private static readonly ObservableGauge _dataConnectionsTotal =
Meter.CreateObservableGauge("scadabridge.site.data_connections.total",
() => (long)_dataConnectionStates.Count,
unit: "1",
description: "Data connections known to this site node, in any state.");
/// Gauge reporting the current StoreAndForward queue depth via the registered provider.
private static readonly ObservableGauge _storeAndForwardQueueDepth =
Meter.CreateObservableGauge("scadabridge.store_and_forward.queue.depth",
() => Volatile.Read(ref _queueDepthProvider) is { } p ? p() : 0L,
unit: "items",
description: "Current StoreAndForward queue depth.");
/// Gauge reporting the number of currently running per-site live-alarm aggregators.
private static readonly ObservableGauge _liveAlarmAggregatorsActiveGauge =
Meter.CreateObservableGauge("scadabridge.site.alarm_cache.aggregators.active",
() => Interlocked.Read(ref _liveAlarmAggregatorsActive),
unit: "1",
description: "Number of per-site live-alarm aggregators currently running on the active central node.");
///
/// Register row 53: gauge reporting the number of open Transport bundle-import
/// sessions on this central node. Each open session pins a fully decrypted bundle
/// in memory and consumes one of TransportOptions.MaxConcurrentImportSessions
/// (default 8), so a series that plateaus at the cap is the signature of leaked
/// sessions — the failure mode register row 43 could only be inferred from rotating
/// test failures because nothing counted them.
///
private static readonly ObservableGauge _transportImportSessionsOpen =
Meter.CreateObservableGauge("scadabridge.central.transport.import_sessions.open",
() => Volatile.Read(ref _importSessionCountProvider) is { } p ? p() : 0L,
unit: "1",
description: "Number of currently open bundle-import sessions on this central node.");
#pragma warning restore IDE0052
// ---------------- Emit helpers ----------------
/// Records that a deployment was applied.
public static void RecordDeploymentApplied() => _deploymentsApplied.Add(1);
/// Records one deployment that ended in a terminal failure (ScadaBridge#81).
/// One of timeout, error, site_failed.
public static void RecordDeploymentFailed(string reason) =>
_deploymentsFailed.Add(1, new KeyValuePair("reason", reason));
/// Records one store-and-forward lifecycle event (ScadaBridge#81).
/// The activity action, e.g. Parked.
/// The message category name.
public static void RecordStoreAndForwardActivity(string action, string category) =>
_sfActivity.Add(1,
new KeyValuePair("action", action),
new KeyValuePair("category", category));
/// Records one finished site script run and its duration (ScadaBridge#81).
/// One of completed, failed, timed_out.
/// The run duration in milliseconds.
public static void RecordScriptRun(string outcome, long durationMs)
{
var tag = new KeyValuePair("outcome", outcome);
_scriptRuns.Add(1, tag);
_scriptRunDuration.Record(durationMs < 0 ? 0 : durationMs, tag);
}
/// Records an inbound API request for the given .
/// The API method the request targeted.
public static void RecordInboundApiRequest(string method) =>
_inboundApiRequests.Add(1, new KeyValuePair("method", method));
/// Records that an inbound execution was abandoned (handler outlived its request) for the given .
/// The API method whose execution was abandoned.
public static void RecordInboundAbandonedExecution(string method) =>
_inboundAbandonedExecutions.Add(1, new KeyValuePair("method", method));
/// Records one failed S&F replication dispatch.
public static void RecordReplicationFailure() => _replicationFailures.Add(1);
/// Records one acknowledged (applied) S&F buffer resync.
public static void RecordSfResyncCompleted() => _sfResyncCompleted.Add(1);
/// Records one resync whose ack window expired.
public static void RecordSfResyncAckMissing() => _sfResyncAckMissing.Add(1);
///
/// Records one bundle-import session rejected on a genuine concurrent-session cap hit
/// (register row 66). Call ONLY from the post-eviction recheck in
/// BundleSessionStore.Open — counting the pre-eviction check would inflate the
/// series with reclaim-then-succeed cases that never turned an operator away.
///
public static void RecordImportSessionRejected() => _transportImportSessionsRejected.Add(1);
///
/// Records one site-event-log purge tick that stopped at its per-tick delete ceiling
/// (register row 34). Call ONLY when rows genuinely remain to be purged — a tick that
/// finished the work and happened to land exactly on the ceiling has nothing to report.
///
public static void RecordSiteEventLogPurgeCeilingHit() => _siteEventLogPurgeCeilingHits.Add(1);
///
/// Records that a central→site gRPC subscriber stream opened (increments
/// scadabridge.site.stream.subscribers.active). Renamed from
/// SiteConnectionOpened by register row 54 — it never counted data connections.
///
public static void SiteStreamSubscriberOpened() => Interlocked.Increment(ref _siteStreamSubscriberCount);
///
/// Records that a central→site gRPC subscriber stream closed (decrements
/// scadabridge.site.stream.subscribers.active). Renamed from
/// SiteConnectionClosed by register row 54.
///
public static void SiteStreamSubscriberClosed() => Interlocked.Decrement(ref _siteStreamSubscriberCount);
///
/// Records the current state of one data connection, feeding the
/// scadabridge.site.data_connections.* gauges (register row 54). Called by the DCL
/// at each connection state transition; safe to call repeatedly with the same value.
///
/// The data connection's name (unique per site node).
/// The connection's new state.
public static void RecordDataConnectionState(string connectionName, ConnectionHealth state)
{
if (string.IsNullOrEmpty(connectionName))
{
return;
}
var previous = _dataConnectionStates.TryGetValue(connectionName, out var prior) ? prior : (ConnectionHealth?)null;
_dataConnectionStates[connectionName] = state;
if (previous != state)
{
_dataConnectionTransitions.Add(1, new KeyValuePair("state", state.ToString()));
}
}
///
/// Drops a data connection from the gauges (register row 54). Called when the DCL removes
/// a connection, so an undeployed connection stops counting toward the total rather than
/// lingering as a permanently-not-connected row that would alert forever.
///
/// The data connection's name.
public static void RemoveDataConnection(string connectionName)
{
if (string.IsNullOrEmpty(connectionName))
{
return;
}
_dataConnectionStates.TryRemove(connectionName, out _);
}
/// Counts data connections currently in .
/// The connected count.
private static long CountDataConnections()
{
long connected = 0;
foreach (var state in _dataConnectionStates.Values)
{
if (state == ConnectionHealth.Connected)
{
connected++;
}
}
return connected;
}
/// Records that a per-site live-alarm aggregator started (increments the active-aggregator gauge).
public static void LiveAlarmAggregatorStarted() => Interlocked.Increment(ref _liveAlarmAggregatorsActive);
/// Records that a per-site live-alarm aggregator stopped (decrements the active-aggregator gauge).
public static void LiveAlarmAggregatorStopped() => Interlocked.Decrement(ref _liveAlarmAggregatorsActive);
/// Records that a per-site live-alarm aggregator re-established its site-wide gRPC stream.
public static void RecordLiveAlarmStreamReconnect() => _liveAlarmStreamReconnects.Add(1);
/// Records live deltas evicted from a live-alarm aggregator's bounded pre-seed buffer.
/// Number of deltas evicted.
public static void RecordLiveAlarmBufferDrop(long count = 1) => _liveAlarmBufferDrops.Add(count);
/// Records entries evicted from the site script compile cache.
/// Number of entries evicted by one sweep.
public static void RecordScriptCompileCacheEvictions(long count) => _scriptCompileCacheEvictions.Add(count);
/// Records debug events evicted from a debug session's bounded pre-snapshot buffer.
/// Number of events evicted.
public static void RecordDebugPreSnapshotDrop(long count = 1) => _debugPreSnapshotDrops.Add(count);
/// Records an event evicted from a site gRPC stream's bounded send channel.
/// Stream kind tag (instance or site-alarms).
public static void RecordSiteStreamEventDropped(string streamKind) =>
_siteStreamEventDrops.Add(1, new KeyValuePair("stream", streamKind));
///
/// Records how many events one site gRPC stream frame carried. Called once per emitted
/// frame on a batching-negotiated subscription (never on an un-negotiated one, where it
/// would degenerate into a per-event instrument on the hottest path in the product).
///
/// Stream kind tag (instance or site-alarms).
/// Events packed into the frame; 1 for a plain unbatched frame.
public static void RecordSiteStreamBatchSize(string streamKind, int events) =>
_siteStreamBatchSize.Record(events, new KeyValuePair("stream", streamKind));
///
/// Registers the provider the StoreAndForward queue-depth gauge reads on each observation.
/// A later task supplies a provider that reads the real StoreAndForward depth. A null
/// provider is ignored so the gauge falls back to reporting 0.
///
/// A callback returning the current queue depth.
public static void SetQueueDepthProvider(Func provider)
{
if (provider is null)
{
return;
}
Volatile.Write(ref _queueDepthProvider, provider);
}
///
/// Clears the StoreAndForward queue-depth provider, but only if the currently
/// registered provider is the exact delegate passed in
/// (reference-equal compare-and-clear). This lets a StoreAndForward service deregister
/// its own provider on graceful stop without stomping a newer instance that already
/// re-registered into the process-global slot: if a late stop of the old instance
/// passes its (now-superseded) delegate, the identity check fails and the newer
/// provider is preserved. After a successful clear the gauge falls back to reporting 0.
/// Mirrors 's signature and
/// access pattern.
///
/// The provider delegate to remove; ignored unless it is the
/// one currently registered.
public static void ClearQueueDepthProvider(Func provider)
{
if (provider is null)
{
return;
}
// Compare-and-clear: only null the slot when it still holds the caller's
// delegate, so a stale stop cannot clobber a successor's provider.
Interlocked.CompareExchange(ref _queueDepthProvider, null, provider);
}
///
/// Register row 53: registers the provider the open-import-sessions gauge reads on
/// each observation. Called by the Transport component's session-eviction hosted
/// service on start. A null provider is ignored so the gauge falls back to 0.
/// The callback is invoked synchronously on the metrics collection thread, so it
/// must stay cheap — the Transport implementation reads a
/// it
/// already owns.
///
/// A callback returning the current open-session count.
public static void SetImportSessionCountProvider(Func provider)
{
if (provider is null)
{
return;
}
Volatile.Write(ref _importSessionCountProvider, provider);
}
///
/// Clears the open-import-sessions provider, but only if the currently registered
/// provider is the exact delegate passed in
/// (reference-equal compare-and-clear), so a late stop of a superseded instance
/// cannot stomp a newer registrant. Mirrors .
///
/// The provider delegate to remove; ignored unless it is the
/// one currently registered.
public static void ClearImportSessionCountProvider(Func provider)
{
if (provider is null)
{
return;
}
Interlocked.CompareExchange(ref _importSessionCountProvider, null, provider);
}
}