ScadaBridge/src/ZB.MOM.WW.ScadaBridge.Commons/Observability/ScadaBridgeTelemetry.cs

using System.Diagnostics.Metrics;

namespace ZB.MOM.WW.ScadaBridge.Commons.Observability;

/// <summary>
/// Central <see cref="Meter"/> + instrument definitions for ScadaBridge's application
/// telemetry, modelled on OtOpcUa's <c>OtOpcUaTelemetry</c>. Modules emit through these
/// pre-created instruments so a single OpenTelemetry / Prometheus binding in
/// <c>Host</c> (registered via <c>AddZbTelemetry</c> with this meter named in
/// <c>ZbTelemetryOptions.Meters</c>) catches everything. No exporter is required —
/// instruments are no-op until a listener attaches, so tests and dev hosts pay nothing
/// for instrumentation that nobody scrapes.
///
/// Instrument names follow the OpenTelemetry semantic convention pattern
/// <c>scadabridge.&lt;subsystem&gt;.&lt;event&gt;</c>. This task defines the instruments and
/// their emit helpers; four later tasks wire the actual emit points. Until those land the
/// helpers are dormant but inert — calling them is safe and simply records against a meter
/// that nothing observes.
/// </summary>
public static class ScadaBridgeTelemetry
{
    /// <summary>The meter name registered with OTel via <c>ZbTelemetryOptions.Meters</c>.</summary>
    public const string MeterName = "ZB.MOM.WW.ScadaBridge";

    /// <summary>Singleton <see cref="Meter"/> all instruments hang off.</summary>
    private static readonly Meter Meter = new(MeterName);

    // ---------------- Counters ----------------

    /// <summary>Incremented each time a deployment is successfully applied.</summary>
    private static readonly Counter<long> _deploymentsApplied =
        Meter.CreateCounter<long>("scadabridge.deployments.applied", unit: "1",
            description: "Deployments applied.");

    /// <summary>Incremented for each inbound API request, tagged with the API method.</summary>
    private static readonly Counter<long> _inboundApiRequests =
        Meter.CreateCounter<long>("scadabridge.inbound_api.requests", unit: "1",
            description: "Inbound API requests, tagged by method.");

    /// <summary>Incremented each time an inbound execution is abandoned (handler outlived its request), tagged with the API method.</summary>
    private static readonly Counter<long> _inboundAbandonedExecutions =
        Meter.CreateCounter<long>("scadabridge.inbound_api.abandoned_executions", unit: "1",
            description: "Inbound API executions abandoned after timeout/abort (handler still running), tagged by method.");

    /// <summary>Incremented each time an S&amp;F buffer replication op fails to dispatch/deliver to the peer.</summary>
    private static readonly Counter<long> _replicationFailures =
        Meter.CreateCounter<long>("scadabridge.store_and_forward.replication.failures", unit: "1",
            description: "S&F buffer replication operations that failed to dispatch/deliver to the peer node");

    private static readonly Counter<long> _sfResyncCompleted =
        Meter.CreateCounter<long>("scadabridge.store_and_forward.resync.completed", unit: "1",
            description: "S&F anti-entropy resyncs the standby acknowledged as applied");

    private static readonly Counter<long> _sfResyncAckMissing =
        Meter.CreateCounter<long>("scadabridge.store_and_forward.resync.ack_missing", unit: "1",
            description: "S&F resyncs answered by the active node but never acknowledged by the standby within the ack window (lost chunks / dead peer)");

    /// <summary>
    /// Incremented each time a per-site live-alarm aggregator re-establishes its site-wide
    /// gRPC stream — a NodeA↔NodeB failover flip or a reconcile-driven reopen after the
    /// stream was given up (plan #10, Task 6). A sustained climb signals a flapping site link.
    /// </summary>
    private static readonly Counter<long> _liveAlarmStreamReconnects =
        Meter.CreateCounter<long>("scadabridge.site.alarm_cache.reconnects", unit: "1",
            description: "Live-alarm aggregator site-wide gRPC stream reconnects (NodeA↔NodeB flip or reconcile-driven reopen).");

    /// <summary>
    /// Incremented for each live delta evicted from a per-site live-alarm aggregator's
    /// bounded pre-seed buffer (drop-oldest, WP2.3). Non-zero means a seed/reconcile
    /// fan-out ran long enough for the delta storm behind it to exceed the cap — the
    /// dropped transitions are recovered by the fan-out's authoritative snapshot, but a
    /// sustained climb points at a slow site.
    /// </summary>
    private static readonly Counter<long> _liveAlarmBufferDrops =
        Meter.CreateCounter<long>("scadabridge.site.alarm_cache.buffer_dropped", unit: "1",
            description: "Live deltas evicted from a live-alarm aggregator's bounded pre-seed buffer (drop-oldest).");

    /// <summary>
    /// Incremented for each debug event evicted from a central debug session's bounded
    /// pre-snapshot buffer (drop-oldest, WP2.3). The Debug View is lossy-under-backpressure
    /// by design; this makes the loss measurable instead of unbounded memory growth.
    /// </summary>
    private static readonly Counter<long> _debugPreSnapshotDrops =
        Meter.CreateCounter<long>("scadabridge.central.debug_view.presnapshot_dropped", unit: "1",
            description: "Debug events evicted from a central debug session's bounded pre-snapshot buffer (drop-oldest).");

    /// <summary>
    /// Incremented for each event evicted from a site-hosted gRPC stream's bounded send
    /// channel, tagged by stream kind (<c>instance</c> = Debug View, <c>site-alarms</c> =
    /// the site-wide alarm feed behind the operator Alarm Summary).
    /// </summary>
    private static readonly Counter<long> _siteStreamEventDrops =
        Meter.CreateCounter<long>("scadabridge.site.stream.events_dropped", unit: "1",
            description: "Events evicted from a site gRPC stream's bounded send channel, tagged by stream kind.");

    // ---------------- Histograms ----------------

    /// <summary>
    /// Distribution of how many events each site→central stream frame carried (R2 — gRPC
    /// event batching), tagged by stream kind. Recorded ONLY for subscriptions that
    /// negotiated batching, so the series' very existence says "this central speaks the
    /// batched wire". A distribution pinned at 1 means the coalescing window never sees a
    /// backlog (the site is quiet, or the window is too small to be earning anything);
    /// mass at the size cap means the cap, not the window, is the binding constraint.
    /// </summary>
    private static readonly Histogram<int> _siteStreamBatchSize =
        Meter.CreateHistogram<int>("scadabridge.site.stream.batch_size", unit: "1",
            description: "Events per site gRPC stream frame (1 = unbatched frame), tagged by stream kind.");

    // ---------------- Observable gauges ----------------

    /// <summary>Current count of open site connections, mutated via <see cref="Interlocked"/>.</summary>
    private static long _siteConnectionsUp;

    /// <summary>Current count of running per-site live-alarm aggregators, mutated via <see cref="Interlocked"/>.</summary>
    private static long _liveAlarmAggregatorsActive;

    /// <summary>Provider that yields the live StoreAndForward queue depth; set by a later task.</summary>
    private static Func<long>? _queueDepthProvider;

    /// <summary>
    /// Register row 53: provider that yields the number of currently open (unexpired)
    /// Transport bundle-import sessions, registered by the Transport component's
    /// session-eviction hosted service. Null on a site node (Transport is central-only),
    /// where the gauge simply reports 0.
    /// </summary>
    private static Func<long>? _importSessionCountProvider;

#pragma warning disable IDE0052 // Held to keep the observable gauges alive for the meter's lifetime.
    /// <summary>Gauge reporting the number of currently open site connections.</summary>
    private static readonly ObservableGauge<long> _siteConnectionUp =
        Meter.CreateObservableGauge<long>("scadabridge.site.connection.up",
            () => Interlocked.Read(ref _siteConnectionsUp),
            description: "Number of currently open site connections.");

    /// <summary>Gauge reporting the current StoreAndForward queue depth via the registered provider.</summary>
    private static readonly ObservableGauge<long> _storeAndForwardQueueDepth =
        Meter.CreateObservableGauge<long>("scadabridge.store_and_forward.queue.depth",
            () => Volatile.Read(ref _queueDepthProvider) is { } p ? p() : 0L,
            unit: "items",
            description: "Current StoreAndForward queue depth.");

    /// <summary>Gauge reporting the number of currently running per-site live-alarm aggregators.</summary>
    private static readonly ObservableGauge<long> _liveAlarmAggregatorsActiveGauge =
        Meter.CreateObservableGauge<long>("scadabridge.site.alarm_cache.aggregators.active",
            () => Interlocked.Read(ref _liveAlarmAggregatorsActive),
            unit: "1",
            description: "Number of per-site live-alarm aggregators currently running on the active central node.");

    /// <summary>
    /// Register row 53: gauge reporting the number of open Transport bundle-import
    /// sessions on this central node. Each open session pins a fully decrypted bundle
    /// in memory and consumes one of <c>TransportOptions.MaxConcurrentImportSessions</c>
    /// (default 8), so a series that plateaus at the cap is the signature of leaked
    /// sessions — the failure mode register row 43 could only be inferred from rotating
    /// test failures because nothing counted them.
    /// </summary>
    private static readonly ObservableGauge<long> _transportImportSessionsOpen =
        Meter.CreateObservableGauge<long>("scadabridge.central.transport.import_sessions.open",
            () => Volatile.Read(ref _importSessionCountProvider) is { } p ? p() : 0L,
            unit: "1",
            description: "Number of currently open bundle-import sessions on this central node.");
#pragma warning restore IDE0052

    // ---------------- Emit helpers ----------------

    /// <summary>Records that a deployment was applied.</summary>
    public static void RecordDeploymentApplied() => _deploymentsApplied.Add(1);

    /// <summary>Records an inbound API request for the given <paramref name="method"/>.</summary>
    /// <param name="method">The API method the request targeted.</param>
    public static void RecordInboundApiRequest(string method) =>
        _inboundApiRequests.Add(1, new KeyValuePair<string, object?>("method", method));

    /// <summary>Records that an inbound execution was abandoned (handler outlived its request) for the given <paramref name="method"/>.</summary>
    /// <param name="method">The API method whose execution was abandoned.</param>
    public static void RecordInboundAbandonedExecution(string method) =>
        _inboundAbandonedExecutions.Add(1, new KeyValuePair<string, object?>("method", method));

    /// <summary>Records one failed S&amp;F replication dispatch.</summary>
    public static void RecordReplicationFailure() => _replicationFailures.Add(1);

    /// <summary>Records one acknowledged (applied) S&amp;F buffer resync.</summary>
    public static void RecordSfResyncCompleted() => _sfResyncCompleted.Add(1);

    /// <summary>Records one resync whose ack window expired.</summary>
    public static void RecordSfResyncAckMissing() => _sfResyncAckMissing.Add(1);

    /// <summary>Records that a site connection opened (increments the up-count gauge).</summary>
    public static void SiteConnectionOpened() => Interlocked.Increment(ref _siteConnectionsUp);

    /// <summary>Records that a site connection closed (decrements the up-count gauge).</summary>
    public static void SiteConnectionClosed() => Interlocked.Decrement(ref _siteConnectionsUp);

    /// <summary>Records that a per-site live-alarm aggregator started (increments the active-aggregator gauge).</summary>
    public static void LiveAlarmAggregatorStarted() => Interlocked.Increment(ref _liveAlarmAggregatorsActive);

    /// <summary>Records that a per-site live-alarm aggregator stopped (decrements the active-aggregator gauge).</summary>
    public static void LiveAlarmAggregatorStopped() => Interlocked.Decrement(ref _liveAlarmAggregatorsActive);

    /// <summary>Records that a per-site live-alarm aggregator re-established its site-wide gRPC stream.</summary>
    public static void RecordLiveAlarmStreamReconnect() => _liveAlarmStreamReconnects.Add(1);

    /// <summary>Records live deltas evicted from a live-alarm aggregator's bounded pre-seed buffer.</summary>
    /// <param name="count">Number of deltas evicted.</param>
    public static void RecordLiveAlarmBufferDrop(long count = 1) => _liveAlarmBufferDrops.Add(count);

    /// <summary>Records debug events evicted from a debug session's bounded pre-snapshot buffer.</summary>
    /// <param name="count">Number of events evicted.</param>
    public static void RecordDebugPreSnapshotDrop(long count = 1) => _debugPreSnapshotDrops.Add(count);

    /// <summary>Records an event evicted from a site gRPC stream's bounded send channel.</summary>
    /// <param name="streamKind">Stream kind tag (<c>instance</c> or <c>site-alarms</c>).</param>
    public static void RecordSiteStreamEventDropped(string streamKind) =>
        _siteStreamEventDrops.Add(1, new KeyValuePair<string, object?>("stream", streamKind));

    /// <summary>
    /// Records how many events one site gRPC stream frame carried. Called once per emitted
    /// frame on a batching-negotiated subscription (never on an un-negotiated one, where it
    /// would degenerate into a per-event instrument on the hottest path in the product).
    /// </summary>
    /// <param name="streamKind">Stream kind tag (<c>instance</c> or <c>site-alarms</c>).</param>
    /// <param name="events">Events packed into the frame; 1 for a plain unbatched frame.</param>
    public static void RecordSiteStreamBatchSize(string streamKind, int events) =>
        _siteStreamBatchSize.Record(events, new KeyValuePair<string, object?>("stream", streamKind));

    /// <summary>
    /// Registers the provider the StoreAndForward queue-depth gauge reads on each observation.
    /// A later task supplies a provider that reads the real StoreAndForward depth. A null
    /// provider is ignored so the gauge falls back to reporting 0.
    /// </summary>
    /// <param name="provider">A callback returning the current queue depth.</param>
    public static void SetQueueDepthProvider(Func<long> provider)
    {
        if (provider is null)
        {
            return;
        }

        Volatile.Write(ref _queueDepthProvider, provider);
    }

    /// <summary>
    /// Clears the StoreAndForward queue-depth provider, but only if the currently
    /// registered provider is the exact <paramref name="provider"/> delegate passed in
    /// (reference-equal compare-and-clear). This lets a StoreAndForward service deregister
    /// its own provider on graceful stop without stomping a newer instance that already
    /// re-registered into the process-global slot: if a late stop of the old instance
    /// passes its (now-superseded) delegate, the identity check fails and the newer
    /// provider is preserved. After a successful clear the gauge falls back to reporting 0.
    /// Mirrors <see cref="SetQueueDepthProvider"/>'s signature and <see cref="Volatile"/>
    /// access pattern.
    /// </summary>
    /// <param name="provider">The provider delegate to remove; ignored unless it is the
    /// one currently registered.</param>
    public static void ClearQueueDepthProvider(Func<long> provider)
    {
        if (provider is null)
        {
            return;
        }

        // Compare-and-clear: only null the slot when it still holds the caller's
        // delegate, so a stale stop cannot clobber a successor's provider.
        Interlocked.CompareExchange(ref _queueDepthProvider, null, provider);
    }

    /// <summary>
    /// Register row 53: registers the provider the open-import-sessions gauge reads on
    /// each observation. Called by the Transport component's session-eviction hosted
    /// service on start. A null provider is ignored so the gauge falls back to 0.
    /// The callback is invoked synchronously on the metrics collection thread, so it
    /// must stay cheap — the Transport implementation reads a
    /// <see cref="System.Collections.Concurrent.ConcurrentDictionary{TKey,TValue}"/> it
    /// already owns.
    /// </summary>
    /// <param name="provider">A callback returning the current open-session count.</param>
    public static void SetImportSessionCountProvider(Func<long> provider)
    {
        if (provider is null)
        {
            return;
        }

        Volatile.Write(ref _importSessionCountProvider, provider);
    }

    /// <summary>
    /// Clears the open-import-sessions provider, but only if the currently registered
    /// provider is the exact <paramref name="provider"/> delegate passed in
    /// (reference-equal compare-and-clear), so a late stop of a superseded instance
    /// cannot stomp a newer registrant. Mirrors <see cref="ClearQueueDepthProvider"/>.
    /// </summary>
    /// <param name="provider">The provider delegate to remove; ignored unless it is the
    /// one currently registered.</param>
    public static void ClearImportSessionCountProvider(Func<long> provider)
    {
        if (provider is null)
        {
            return;
        }

        Interlocked.CompareExchange(ref _importSessionCountProvider, null, provider);
    }
}