lmxopcua/src/Core/ZB.MOM.WW.OtOpcUa.Commons/Observability/OtOpcUaTelemetry.cs

using System.Diagnostics;
using System.Diagnostics.Metrics;

namespace ZB.MOM.WW.OtOpcUa.Commons.Observability;

/// <summary>
/// Central <see cref="Meter"/> + <see cref="ActivitySource"/> definitions for OtOpcUa.
/// All Akka actors, the OPC UA publish path, and the deploy coordinator emit through these
/// pre-created instruments so a single OpenTelemetry / Prometheus binding in <c>Host</c>
/// catches everything. No exporter is required — instruments are no-op until a listener
/// attaches, so tests and dev hosts pay nothing for instrumentation that nobody scrapes.
///
/// Instrument names follow the OpenTelemetry semantic convention pattern
/// <c>otopcua.&lt;subsystem&gt;.&lt;event&gt;</c>. Subsystem is one of: deploy, driver,
/// virtualtag, scriptedalarm, opcua, redundancy.
/// </summary>
public static class OtOpcUaTelemetry
{
    public const string MeterName = "ZB.MOM.WW.OtOpcUa";
    public const string ActivitySourceName = "ZB.MOM.WW.OtOpcUa";

    /// <summary>Singleton <see cref="Meter"/> all counters/histograms hang off.</summary>
    public static readonly Meter Meter = new(MeterName);

    /// <summary>Singleton <see cref="ActivitySource"/> used to start spans wrapping deploy/apply/rebuild.</summary>
    public static readonly ActivitySource ActivitySource = new(ActivitySourceName);

    // ---------------- Deployment / driver-host coordination ----------------

    /// <summary>Incremented every time DriverHostActor finishes applying a deployment (Ack or Reject).</summary>
    public static readonly Counter<long> DeploymentApplied =
        Meter.CreateCounter<long>("otopcua.deploy.applied", unit: "{deployment}",
            description: "Deployments applied by a driver-role node (outcome=ack|reject).");

    /// <summary>Time from DriverHostActor receiving DispatchDeployment to emitting the ack/reject.</summary>
    public static readonly Histogram<double> DeploymentApplyDurationSec =
        Meter.CreateHistogram<double>("otopcua.deploy.apply.duration", unit: "s",
            description: "Driver-role apply latency from DispatchDeployment → Ack/Reject.");

    /// <summary>DriverInstanceActor spawn count (added=new instance; stop=disposed).</summary>
    public static readonly Counter<long> DriverInstanceLifecycle =
        Meter.CreateCounter<long>("otopcua.driver.lifecycle", unit: "{event}",
            description: "DriverInstanceActor lifecycle transitions (event=spawn|stop|fault).");

    /// <summary>
    /// Poll cycles that ran longer than their subscription's publishing interval, as counted by
    /// <c>PollGroupEngine.OverrunCount</c>. <c>Core.Abstractions</c> deliberately carries no metrics
    /// dependency, so each poll driver wires this counter to the engine's <c>onOverrun</c> callback;
    /// this is the production consumer that surface was always missing. Tagged
    /// <see cref="PollOverrunDriverTypeTag"/> (<c>driver_type</c>, a <c>DriverTypeNames</c> constant)
    /// + <see cref="PollOverrunDriverInstanceTag"/> (<c>driver_instance</c>) — snake_case, per the
    /// convention those two consts document.
    /// A non-zero and climbing rate means the device cannot sustain its configured poll rate — the
    /// loop is already running flat out.
    /// </summary>
    public static readonly Counter<long> PollOverruns =
        Meter.CreateCounter<long>("otopcua.driver.poll.overrun", unit: "{overrun}",
            description: "Poll cycles that ran longer than the poll interval (slow success and slow failure both count — see PollGroupEngine.NoteOverrun).");

    /// <summary>
    /// Tag key carrying the <c>DriverTypeNames</c> constant on <see cref="PollOverruns"/>.
    /// <para><b>The snake_case spelling is load-bearing, not a style choice.</b> It is the same
    /// literal key <see cref="DriverInstanceLifecycle"/> emits (<c>DriverInstanceActor</c>), and
    /// every other instrument on this <see cref="Meter"/> uses bare/snake_case keys
    /// (<c>event</c>, <c>kind</c>, <c>outcome</c>, <c>site</c>, <c>reason</c>). A dotted
    /// <c>driver.type</c> here would silently split a dashboard's "group by driver type" across
    /// two key names that no single query can join. The seven poll drivers reference this const
    /// rather than a literal so that can never drift back.</para>
    /// </summary>
    public const string PollOverrunDriverTypeTag = "driver_type";

    /// <summary>
    /// Tag key carrying the emitting driver's instance id on <see cref="PollOverruns"/>. Cardinality
    /// is bounded by the number of configured driver instances on the node. Snake_case for the same
    /// convention reason as <see cref="PollOverrunDriverTypeTag"/>.
    /// </summary>
    public const string PollOverrunDriverInstanceTag = "driver_instance";

    /// <summary>
    /// Session/publish-level events of an OPC UA Client driver instance (#276): the operator's view of a
    /// flapping upstream. Tagged <see cref="PollOverrunDriverInstanceTag"/> (<c>driver_instance</c>) +
    /// <c>event</c> = <c>publish_response|notification|data_change|bad_status_value|publish_error|
    /// keepalive_miss|keepalive_recovered|stale_notification_dropped</c>.
    /// </summary>
    public static readonly Counter<long> OpcUaClientEvents =
        Meter.CreateCounter<long>("otopcua.driver.opcuaclient.events", unit: "{event}",
            description: "OPC UA Client driver session/publish events (event=publish_response|notification|data_change|bad_status_value|publish_error|keepalive_miss|keepalive_recovered|stale_notification_dropped).");

    // ---------------- VirtualTag / ScriptedAlarm engines ----------------

    public static readonly Counter<long> VirtualTagEval =
        Meter.CreateCounter<long>("otopcua.virtualtag.eval", unit: "{eval}",
            description: "Virtual-tag evaluations attempted (outcome=ok|fail|skip).");

    public static readonly Counter<long> ScriptedAlarmTransition =
        Meter.CreateCounter<long>("otopcua.scriptedalarm.transition", unit: "{transition}",
            description: "Scripted-alarm state transitions (state=active|acknowledged|inactive).");

    // ---------------- OPC UA address-space + redundancy ----------------

    public static readonly Counter<long> OpcUaSinkWrite =
        Meter.CreateCounter<long>("otopcua.opcua.sink.write", unit: "{write}",
            description: "Writes that landed in IOpcUaAddressSpaceSink (kind=value|alarm|rebuild).");

    /// <summary>Address-space apply/materialise passes that swallowed at least one sink failure —
    /// a failed rebuild (kind=rebuild) or per-node materialise failures (kind=nodes). A non-zero rate
    /// means the running server holds a stale or partially-materialised address space despite a
    /// reported-successful deploy (archreview 01/S-1).</summary>
    public static readonly Counter<long> OpcUaApplyFailed =
        Meter.CreateCounter<long>("otopcua.opcua.apply.failed", unit: "{apply}",
            description: "Apply/materialise passes with swallowed sink failures (kind=rebuild|nodes).");

    public static readonly Counter<long> ServiceLevelChange =
        Meter.CreateCounter<long>("otopcua.redundancy.service_level_change", unit: "{change}",
            description: "OPC UA Server.ServiceLevel transitions emitted by the redundancy state.");

    /// <summary>Archreview 03/S15 — tag refs a driver left <c>BadNoCommunication</c> after it reconnected:
    /// the host marked them Bad on the outage, and the driver never re-published them. Non-zero means a
    /// driver that reconnects without a full re-publish (a push driver missing its initial push, or a
    /// value-only dedup swallowing an unchanged value), which leaves those nodes Bad indefinitely.
    /// Tagged driver + driver_type.</summary>
    public static readonly Counter<long> DriverCommsLatchedRefs =
        Meter.CreateCounter<long>("otopcua.driver.comms_latched_refs", unit: "{ref}",
            description: "Refs still BadNoCommunication after their driver reconnected (driver, driver_type).");

    /// <summary>Inbound operations denied by the Primary data-plane gate (archreview 03/S4) —
    /// site=write|ack|alarm-emit, reason=secondary|detached|role-unknown.</summary>
    public static readonly Counter<long> PrimaryGateDenied =
        Meter.CreateCounter<long>("otopcua.redundancy.primary_gate_denied", unit: "{denial}",
            description: "Operations denied by the Primary gate (site=write|ack|alarm-emit, reason=secondary|detached|role-unknown).");

    /// <summary>
    ///     API-key store lookups the 03/S18 bound intervened on (outcome=timeout|cache_hit).
    /// </summary>
    /// <remarks>
    ///     One counter with an <c>outcome</c> tag rather than two: <c>timeout</c> is the alarm signal that
    ///     the ConfigDb is not answering the OPC UA impersonation callback, and <c>cache_hit</c> is the
    ///     ratio that says whether the TTL is doing anything. A missing <c>cache_hit</c> stream on a busy
    ///     fused central is how you notice <c>Security:ApiKeys:StoreCacheSeconds</c> was set to 0 by
    ///     accident. A plain read-through meters neither — the counter names interventions, not traffic.
    /// </remarks>
    public static readonly Counter<long> ApiKeyStoreBounded =
        Meter.CreateCounter<long>("otopcua.opcua.apikey_store.bounded", unit: "{lookup}",
            description: "API-key store lookups the bound intervened on (outcome=timeout|cache_hit).");

    // ---------------- Convenience helpers ----------------

    /// <summary>
    /// Starts a deploy span tagged with the deployment id. Caller disposes to close. Returns
    /// null when no listener is attached so the call site stays cheap on undecorated builds.
    /// </summary>
    /// <param name="deploymentId">The deployment identifier to tag the span with.</param>
    /// <returns>The started activity, or <c>null</c> when no listener is attached.</returns>
    public static Activity? StartDeployApplySpan(string deploymentId)
    {
        var activity = ActivitySource.StartActivity("otopcua.deploy.apply", ActivityKind.Internal);
        activity?.SetTag("otopcua.deployment_id", deploymentId);
        return activity;
    }

    /// <summary>Span wrapping a full OPC UA address-space rebuild (AddressSpace plan → apply).</summary>
    /// <returns>The started activity, or <c>null</c> when no listener is attached.</returns>
    public static Activity? StartAddressSpaceRebuildSpan()
        => ActivitySource.StartActivity("otopcua.opcua.address_space_rebuild", ActivityKind.Internal);
}