Joseph Doherty
222 lines
13 KiB
C#
222 lines
13 KiB
C#
using System.Diagnostics.Metrics;
|
|
|
|
namespace ZB.MOM.WW.ScadaBridge.Commons.Observability;
|
|
|
|
/// <summary>
|
|
/// Central <see cref="Meter"/> + instrument definitions for ScadaBridge's application
|
|
/// telemetry, modelled on OtOpcUa's <c>OtOpcUaTelemetry</c>. Modules emit through these
|
|
/// pre-created instruments so a single OpenTelemetry / Prometheus binding in
|
|
/// <c>Host</c> (registered via <c>AddZbTelemetry</c> with this meter named in
|
|
/// <c>ZbTelemetryOptions.Meters</c>) catches everything. No exporter is required —
|
|
/// instruments are no-op until a listener attaches, so tests and dev hosts pay nothing
|
|
/// for instrumentation that nobody scrapes.
|
|
///
|
|
/// Instrument names follow the OpenTelemetry semantic convention pattern
|
|
/// <c>scadabridge.<subsystem>.<event></c>. This task defines the instruments and
|
|
/// their emit helpers; four later tasks wire the actual emit points. Until those land the
|
|
/// helpers are dormant but inert — calling them is safe and simply records against a meter
|
|
/// that nothing observes.
|
|
/// </summary>
|
|
public static class ScadaBridgeTelemetry
|
|
{
|
|
/// <summary>The meter name registered with OTel via <c>ZbTelemetryOptions.Meters</c>.</summary>
|
|
public const string MeterName = "ZB.MOM.WW.ScadaBridge";
|
|
|
|
/// <summary>Singleton <see cref="Meter"/> all instruments hang off.</summary>
|
|
private static readonly Meter Meter = new(MeterName);
|
|
|
|
// ---------------- Counters ----------------
|
|
|
|
/// <summary>Incremented each time a deployment is successfully applied.</summary>
|
|
private static readonly Counter<long> _deploymentsApplied =
|
|
Meter.CreateCounter<long>("scadabridge.deployments.applied", unit: "1",
|
|
description: "Deployments applied.");
|
|
|
|
/// <summary>Incremented for each inbound API request, tagged with the API method.</summary>
|
|
private static readonly Counter<long> _inboundApiRequests =
|
|
Meter.CreateCounter<long>("scadabridge.inbound_api.requests", unit: "1",
|
|
description: "Inbound API requests, tagged by method.");
|
|
|
|
/// <summary>Incremented each time an inbound execution is abandoned (handler outlived its request), tagged with the API method.</summary>
|
|
private static readonly Counter<long> _inboundAbandonedExecutions =
|
|
Meter.CreateCounter<long>("scadabridge.inbound_api.abandoned_executions", unit: "1",
|
|
description: "Inbound API executions abandoned after timeout/abort (handler still running), tagged by method.");
|
|
|
|
/// <summary>Incremented each time an S&F buffer replication op fails to dispatch/deliver to the peer.</summary>
|
|
private static readonly Counter<long> _replicationFailures =
|
|
Meter.CreateCounter<long>("scadabridge.store_and_forward.replication.failures", unit: "1",
|
|
description: "S&F buffer replication operations that failed to dispatch/deliver to the peer node");
|
|
|
|
private static readonly Counter<long> _sfResyncCompleted =
|
|
Meter.CreateCounter<long>("scadabridge.store_and_forward.resync.completed", unit: "1",
|
|
description: "S&F anti-entropy resyncs the standby acknowledged as applied");
|
|
|
|
private static readonly Counter<long> _sfResyncAckMissing =
|
|
Meter.CreateCounter<long>("scadabridge.store_and_forward.resync.ack_missing", unit: "1",
|
|
description: "S&F resyncs answered by the active node but never acknowledged by the standby within the ack window (lost chunks / dead peer)");
|
|
|
|
/// <summary>
|
|
/// Incremented each time a per-site live-alarm aggregator re-establishes its site-wide
|
|
/// gRPC stream — a NodeA↔NodeB failover flip or a reconcile-driven reopen after the
|
|
/// stream was given up (plan #10, Task 6). A sustained climb signals a flapping site link.
|
|
/// </summary>
|
|
private static readonly Counter<long> _liveAlarmStreamReconnects =
|
|
Meter.CreateCounter<long>("scadabridge.site.alarm_cache.reconnects", unit: "1",
|
|
description: "Live-alarm aggregator site-wide gRPC stream reconnects (NodeA↔NodeB flip or reconcile-driven reopen).");
|
|
|
|
/// <summary>
|
|
/// Incremented for each live delta evicted from a per-site live-alarm aggregator's
|
|
/// bounded pre-seed buffer (drop-oldest, WP2.3). Non-zero means a seed/reconcile
|
|
/// fan-out ran long enough for the delta storm behind it to exceed the cap — the
|
|
/// dropped transitions are recovered by the fan-out's authoritative snapshot, but a
|
|
/// sustained climb points at a slow site.
|
|
/// </summary>
|
|
private static readonly Counter<long> _liveAlarmBufferDrops =
|
|
Meter.CreateCounter<long>("scadabridge.site.alarm_cache.buffer_dropped", unit: "1",
|
|
description: "Live deltas evicted from a live-alarm aggregator's bounded pre-seed buffer (drop-oldest).");
|
|
|
|
/// <summary>
|
|
/// Incremented for each debug event evicted from a central debug session's bounded
|
|
/// pre-snapshot buffer (drop-oldest, WP2.3). The Debug View is lossy-under-backpressure
|
|
/// by design; this makes the loss measurable instead of unbounded memory growth.
|
|
/// </summary>
|
|
private static readonly Counter<long> _debugPreSnapshotDrops =
|
|
Meter.CreateCounter<long>("scadabridge.central.debug_view.presnapshot_dropped", unit: "1",
|
|
description: "Debug events evicted from a central debug session's bounded pre-snapshot buffer (drop-oldest).");
|
|
|
|
/// <summary>
|
|
/// Incremented for each event evicted from a site-hosted gRPC stream's bounded send
|
|
/// channel, tagged by stream kind (<c>instance</c> = Debug View, <c>site-alarms</c> =
|
|
/// the site-wide alarm feed behind the operator Alarm Summary).
|
|
/// </summary>
|
|
private static readonly Counter<long> _siteStreamEventDrops =
|
|
Meter.CreateCounter<long>("scadabridge.site.stream.events_dropped", unit: "1",
|
|
description: "Events evicted from a site gRPC stream's bounded send channel, tagged by stream kind.");
|
|
|
|
// ---------------- Observable gauges ----------------
|
|
|
|
/// <summary>Current count of open site connections, mutated via <see cref="Interlocked"/>.</summary>
|
|
private static long _siteConnectionsUp;
|
|
|
|
/// <summary>Current count of running per-site live-alarm aggregators, mutated via <see cref="Interlocked"/>.</summary>
|
|
private static long _liveAlarmAggregatorsActive;
|
|
|
|
/// <summary>Provider that yields the live StoreAndForward queue depth; set by a later task.</summary>
|
|
private static Func<long>? _queueDepthProvider;
|
|
|
|
#pragma warning disable IDE0052 // Held to keep the observable gauges alive for the meter's lifetime.
|
|
/// <summary>Gauge reporting the number of currently open site connections.</summary>
|
|
private static readonly ObservableGauge<long> _siteConnectionUp =
|
|
Meter.CreateObservableGauge<long>("scadabridge.site.connection.up",
|
|
() => Interlocked.Read(ref _siteConnectionsUp),
|
|
description: "Number of currently open site connections.");
|
|
|
|
/// <summary>Gauge reporting the current StoreAndForward queue depth via the registered provider.</summary>
|
|
private static readonly ObservableGauge<long> _storeAndForwardQueueDepth =
|
|
Meter.CreateObservableGauge<long>("scadabridge.store_and_forward.queue.depth",
|
|
() => Volatile.Read(ref _queueDepthProvider) is { } p ? p() : 0L,
|
|
unit: "items",
|
|
description: "Current StoreAndForward queue depth.");
|
|
|
|
/// <summary>Gauge reporting the number of currently running per-site live-alarm aggregators.</summary>
|
|
private static readonly ObservableGauge<long> _liveAlarmAggregatorsActiveGauge =
|
|
Meter.CreateObservableGauge<long>("scadabridge.site.alarm_cache.aggregators.active",
|
|
() => Interlocked.Read(ref _liveAlarmAggregatorsActive),
|
|
unit: "1",
|
|
description: "Number of per-site live-alarm aggregators currently running on the active central node.");
|
|
#pragma warning restore IDE0052
|
|
|
|
// ---------------- Emit helpers ----------------
|
|
|
|
/// <summary>Records that a deployment was applied.</summary>
|
|
public static void RecordDeploymentApplied() => _deploymentsApplied.Add(1);
|
|
|
|
/// <summary>Records an inbound API request for the given <paramref name="method"/>.</summary>
|
|
/// <param name="method">The API method the request targeted.</param>
|
|
public static void RecordInboundApiRequest(string method) =>
|
|
_inboundApiRequests.Add(1, new KeyValuePair<string, object?>("method", method));
|
|
|
|
/// <summary>Records that an inbound execution was abandoned (handler outlived its request) for the given <paramref name="method"/>.</summary>
|
|
/// <param name="method">The API method whose execution was abandoned.</param>
|
|
public static void RecordInboundAbandonedExecution(string method) =>
|
|
_inboundAbandonedExecutions.Add(1, new KeyValuePair<string, object?>("method", method));
|
|
|
|
/// <summary>Records one failed S&F replication dispatch.</summary>
|
|
public static void RecordReplicationFailure() => _replicationFailures.Add(1);
|
|
|
|
/// <summary>Records one acknowledged (applied) S&F buffer resync.</summary>
|
|
public static void RecordSfResyncCompleted() => _sfResyncCompleted.Add(1);
|
|
|
|
/// <summary>Records one resync whose ack window expired.</summary>
|
|
public static void RecordSfResyncAckMissing() => _sfResyncAckMissing.Add(1);
|
|
|
|
/// <summary>Records that a site connection opened (increments the up-count gauge).</summary>
|
|
public static void SiteConnectionOpened() => Interlocked.Increment(ref _siteConnectionsUp);
|
|
|
|
/// <summary>Records that a site connection closed (decrements the up-count gauge).</summary>
|
|
public static void SiteConnectionClosed() => Interlocked.Decrement(ref _siteConnectionsUp);
|
|
|
|
/// <summary>Records that a per-site live-alarm aggregator started (increments the active-aggregator gauge).</summary>
|
|
public static void LiveAlarmAggregatorStarted() => Interlocked.Increment(ref _liveAlarmAggregatorsActive);
|
|
|
|
/// <summary>Records that a per-site live-alarm aggregator stopped (decrements the active-aggregator gauge).</summary>
|
|
public static void LiveAlarmAggregatorStopped() => Interlocked.Decrement(ref _liveAlarmAggregatorsActive);
|
|
|
|
/// <summary>Records that a per-site live-alarm aggregator re-established its site-wide gRPC stream.</summary>
|
|
public static void RecordLiveAlarmStreamReconnect() => _liveAlarmStreamReconnects.Add(1);
|
|
|
|
/// <summary>Records live deltas evicted from a live-alarm aggregator's bounded pre-seed buffer.</summary>
|
|
/// <param name="count">Number of deltas evicted.</param>
|
|
public static void RecordLiveAlarmBufferDrop(long count = 1) => _liveAlarmBufferDrops.Add(count);
|
|
|
|
/// <summary>Records debug events evicted from a debug session's bounded pre-snapshot buffer.</summary>
|
|
/// <param name="count">Number of events evicted.</param>
|
|
public static void RecordDebugPreSnapshotDrop(long count = 1) => _debugPreSnapshotDrops.Add(count);
|
|
|
|
/// <summary>Records an event evicted from a site gRPC stream's bounded send channel.</summary>
|
|
/// <param name="streamKind">Stream kind tag (<c>instance</c> or <c>site-alarms</c>).</param>
|
|
public static void RecordSiteStreamEventDropped(string streamKind) =>
|
|
_siteStreamEventDrops.Add(1, new KeyValuePair<string, object?>("stream", streamKind));
|
|
|
|
/// <summary>
|
|
/// Registers the provider the StoreAndForward queue-depth gauge reads on each observation.
|
|
/// A later task supplies a provider that reads the real StoreAndForward depth. A null
|
|
/// provider is ignored so the gauge falls back to reporting 0.
|
|
/// </summary>
|
|
/// <param name="provider">A callback returning the current queue depth.</param>
|
|
public static void SetQueueDepthProvider(Func<long> provider)
|
|
{
|
|
if (provider is null)
|
|
{
|
|
return;
|
|
}
|
|
|
|
Volatile.Write(ref _queueDepthProvider, provider);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Clears the StoreAndForward queue-depth provider, but only if the currently
|
|
/// registered provider is the exact <paramref name="provider"/> delegate passed in
|
|
/// (reference-equal compare-and-clear). This lets a StoreAndForward service deregister
|
|
/// its own provider on graceful stop without stomping a newer instance that already
|
|
/// re-registered into the process-global slot: if a late stop of the old instance
|
|
/// passes its (now-superseded) delegate, the identity check fails and the newer
|
|
/// provider is preserved. After a successful clear the gauge falls back to reporting 0.
|
|
/// Mirrors <see cref="SetQueueDepthProvider"/>'s signature and <see cref="Volatile"/>
|
|
/// access pattern.
|
|
/// </summary>
|
|
/// <param name="provider">The provider delegate to remove; ignored unless it is the
|
|
/// one currently registered.</param>
|
|
public static void ClearQueueDepthProvider(Func<long> provider)
|
|
{
|
|
if (provider is null)
|
|
{
|
|
return;
|
|
}
|
|
|
|
// Compare-and-clear: only null the slot when it still holds the caller's
|
|
// delegate, so a stale stop cannot clobber a successor's provider.
|
|
Interlocked.CompareExchange(ref _queueDepthProvider, null, provider);
|
|
}
|
|
}
|