using System.Diagnostics; using System.Diagnostics.Metrics; namespace ZB.MOM.WW.OtOpcUa.Commons.Observability; /// /// Central + definitions for OtOpcUa. /// All Akka actors, the OPC UA publish path, and the deploy coordinator emit through these /// pre-created instruments so a single OpenTelemetry / Prometheus binding in Host /// catches everything. No exporter is required — instruments are no-op until a listener /// attaches, so tests and dev hosts pay nothing for instrumentation that nobody scrapes. /// /// Instrument names follow the OpenTelemetry semantic convention pattern /// otopcua.<subsystem>.<event>. Subsystem is one of: deploy, driver, /// virtualtag, scriptedalarm, opcua, redundancy. /// public static class OtOpcUaTelemetry { public const string MeterName = "ZB.MOM.WW.OtOpcUa"; public const string ActivitySourceName = "ZB.MOM.WW.OtOpcUa"; /// Singleton all counters/histograms hang off. public static readonly Meter Meter = new(MeterName); /// Singleton used to start spans wrapping deploy/apply/rebuild. public static readonly ActivitySource ActivitySource = new(ActivitySourceName); // ---------------- Deployment / driver-host coordination ---------------- /// Incremented every time DriverHostActor finishes applying a deployment (Ack or Reject). public static readonly Counter DeploymentApplied = Meter.CreateCounter("otopcua.deploy.applied", unit: "{deployment}", description: "Deployments applied by a driver-role node (outcome=ack|reject)."); /// Time from DriverHostActor receiving DispatchDeployment to emitting the ack/reject. public static readonly Histogram DeploymentApplyDurationSec = Meter.CreateHistogram("otopcua.deploy.apply.duration", unit: "s", description: "Driver-role apply latency from DispatchDeployment → Ack/Reject."); /// DriverInstanceActor spawn count (added=new instance; stop=disposed). public static readonly Counter DriverInstanceLifecycle = Meter.CreateCounter("otopcua.driver.lifecycle", unit: "{event}", description: "DriverInstanceActor lifecycle transitions (event=spawn|stop|fault)."); /// /// Poll cycles that ran longer than their subscription's publishing interval, as counted by /// PollGroupEngine.OverrunCount. Core.Abstractions deliberately carries no metrics /// dependency, so each poll driver wires this counter to the engine's onOverrun callback; /// this is the production consumer that surface was always missing. Tagged /// (driver_type, a DriverTypeNames constant) /// + (driver_instance) — snake_case, per the /// convention those two consts document. /// A non-zero and climbing rate means the device cannot sustain its configured poll rate — the /// loop is already running flat out. /// public static readonly Counter PollOverruns = Meter.CreateCounter("otopcua.driver.poll.overrun", unit: "{overrun}", description: "Poll cycles that ran longer than the poll interval (slow success and slow failure both count — see PollGroupEngine.NoteOverrun)."); /// /// Tag key carrying the DriverTypeNames constant on . /// The snake_case spelling is load-bearing, not a style choice. It is the same /// literal key emits (DriverInstanceActor), and /// every other instrument on this uses bare/snake_case keys /// (event, kind, outcome, site, reason). A dotted /// driver.type here would silently split a dashboard's "group by driver type" across /// two key names that no single query can join. The seven poll drivers reference this const /// rather than a literal so that can never drift back. /// public const string PollOverrunDriverTypeTag = "driver_type"; /// /// Tag key carrying the emitting driver's instance id on . Cardinality /// is bounded by the number of configured driver instances on the node. Snake_case for the same /// convention reason as . /// public const string PollOverrunDriverInstanceTag = "driver_instance"; /// /// Session/publish-level events of an OPC UA Client driver instance: the operator's view of a /// flapping upstream. Tagged (driver_instance) + /// event = publish_response|notification|data_change|bad_status_value|publish_error| /// keepalive_miss|keepalive_recovered|stale_notification_dropped. /// public static readonly Counter OpcUaClientEvents = Meter.CreateCounter("otopcua.driver.opcuaclient.events", unit: "{event}", description: "OPC UA Client driver session/publish events (event=publish_response|notification|data_change|bad_status_value|publish_error|keepalive_miss|keepalive_recovered|stale_notification_dropped)."); /// /// Controller alarms a native-alarm driver saw that NO configured alarm selector on that device claims, so no /// Part 9 condition shows them (lmxopcua#839 — Heidenhain ALARM: selectors without */OTHER). /// Counted once per alarm occurrence (an alarm entering the unclaimed set), not once per poll. Tagged /// + . A non-zero rate means /// operators are not seeing some controller alarms — add a selector (or an OTHER catch-all). /// public static readonly Counter DriverAlarmsUnmatched = Meter.CreateCounter("otopcua.driver.alarms.unmatched", unit: "{alarm}", description: "Controller alarm occurrences no configured native-alarm selector claims (shown by no condition)."); // ---------------- VirtualTag / ScriptedAlarm engines ---------------- public static readonly Counter VirtualTagEval = Meter.CreateCounter("otopcua.virtualtag.eval", unit: "{eval}", description: "Virtual-tag evaluations attempted (outcome=ok|fail|skip)."); public static readonly Counter ScriptedAlarmTransition = Meter.CreateCounter("otopcua.scriptedalarm.transition", unit: "{transition}", description: "Scripted-alarm state transitions (state=active|acknowledged|inactive)."); // ---------------- OPC UA address-space + redundancy ---------------- public static readonly Counter OpcUaSinkWrite = Meter.CreateCounter("otopcua.opcua.sink.write", unit: "{write}", description: "Writes that landed in IOpcUaAddressSpaceSink (kind=value|alarm|rebuild)."); /// Address-space apply/materialise passes that swallowed at least one sink failure — /// a failed rebuild (kind=rebuild) or per-node materialise failures (kind=nodes). A non-zero rate /// means the running server holds a stale or partially-materialised address space despite a /// reported-successful deploy (archreview 01/S-1). public static readonly Counter OpcUaApplyFailed = Meter.CreateCounter("otopcua.opcua.apply.failed", unit: "{apply}", description: "Apply/materialise passes with swallowed sink failures (kind=rebuild|nodes)."); public static readonly Counter ServiceLevelChange = Meter.CreateCounter("otopcua.redundancy.service_level_change", unit: "{change}", description: "OPC UA Server.ServiceLevel transitions emitted by the redundancy state."); /// Archreview 03/S15 — tag refs a driver left BadNoCommunication after it reconnected: /// the host marked them Bad on the outage, and the driver never re-published them. Non-zero means a /// driver that reconnects without a full re-publish (a push driver missing its initial push, or a /// value-only dedup swallowing an unchanged value), which leaves those nodes Bad indefinitely. /// Tagged driver + driver_type. public static readonly Counter DriverCommsLatchedRefs = Meter.CreateCounter("otopcua.driver.comms_latched_refs", unit: "{ref}", description: "Refs still BadNoCommunication after their driver reconnected (driver, driver_type)."); /// Inbound operations denied by the Primary data-plane gate (archreview 03/S4) — /// site=write|ack|alarm-emit, reason=secondary|detached|role-unknown. public static readonly Counter PrimaryGateDenied = Meter.CreateCounter("otopcua.redundancy.primary_gate_denied", unit: "{denial}", description: "Operations denied by the Primary gate (site=write|ack|alarm-emit, reason=secondary|detached|role-unknown)."); /// /// API-key store lookups the 03/S18 bound intervened on (outcome=timeout|cache_hit). /// /// /// One counter with an outcome tag rather than two: timeout is the alarm signal that /// the ConfigDb is not answering the OPC UA impersonation callback, and cache_hit is the /// ratio that says whether the TTL is doing anything. A missing cache_hit stream on a busy /// fused central is how you notice Security:ApiKeys:StoreCacheSeconds was set to 0 by /// accident. A plain read-through meters neither — the counter names interventions, not traffic. /// public static readonly Counter ApiKeyStoreBounded = Meter.CreateCounter("otopcua.opcua.apikey_store.bounded", unit: "{lookup}", description: "API-key store lookups the bound intervened on (outcome=timeout|cache_hit)."); /// /// Part 9 condition events whose delivery is currently deferred because an event notifier's SDK delivery /// channel was full (Gitea #838). Zero in steady state; non-zero means some folder's event subscribers are /// not keeping up with the alarm event rate (or the node manager lock is held across a large burst). /// public static readonly UpDownCounter ConditionEventReportsDeferred = Meter.CreateUpDownCounter("otopcua.opcua.condition_events.deferred", unit: "{event}", description: "Condition events waiting for room in an event notifier's delivery channel."); /// /// Deferred condition events discarded before delivery (Gitea #838): reason=coalesced — the per-condition /// deferral budget was exceeded and the condition's oldest not-yet-started event gave way to its newest; /// reason=not_live — the condition was removed or replaced before its deferred event was delivered. /// public static readonly Counter ConditionEventReportsDiscarded = Meter.CreateCounter("otopcua.opcua.condition_events.discarded", unit: "{event}", description: "Deferred condition events discarded before delivery (reason=coalesced|not_live)."); /// /// Deferred condition events withheld from one Server-object (i=2253) item because a ConditionRefresh of that /// item had already replayed a newer state (Gitea #854) — one per item and event. Every other subscriber still /// receives them; this only stops a refresh being overtaken by older events. /// /// /// Event filters given a private projection group because the server-wide registry of distinct live filter shapes is /// full (Gitea #855 perf). Delivery is unchanged; only the per-report sharing of where-clause / field projection /// stops for those filters, so a steadily rising count means more per-item work under the node-manager lock. /// public static readonly Counter ConditionEventFilterGroupsUnshared = Meter.CreateCounter("otopcua.opcua.condition_events.filter_groups_unshared", unit: "{filter}", description: "Event filters not shared because the registry of live filter shapes is full."); public static readonly Counter ConditionEventDeliveriesSuperseded = Meter.CreateCounter("otopcua.opcua.condition_events.superseded", unit: "{delivery}", description: "Deferred condition events withheld from a Server-object item whose ConditionRefresh already replayed a newer state."); // ---------------- Warm redundancy (Gitea #717, spec §7) ---------------- private static int _warmRole = -1; /// Records this process's Warm role for : 1 Active, 0 Standby, −1 not Warm. /// The role value. public static void SetWarmRole(int value) => Volatile.Write(ref _warmRole, value); /// The Warm role of this node: 1 = Active, 0 = Standby, −1 = not Warm. public static readonly ObservableGauge WarmRole = Meter.CreateObservableGauge("otopcua.redundancy.warm_role", () => Volatile.Read(ref _warmRole), unit: "{role}", description: "Warm redundancy role of this node: 1 Active, 0 Standby, -1 not Warm."); /// Warm role transitions (direction=promote|demote, reason=membership|snapshot|endpoint|boot|unreachable-primary|agreed). public static readonly Counter WarmTransition = Meter.CreateCounter("otopcua.redundancy.warm_transition", unit: "{transition}", description: "Warm role transitions (direction=promote|demote, reason)."); /// Redundancy snapshots not accepted by a subscriber (reason=older_revision|unrevisioned). public static readonly Counter SnapshotRejected = Meter.CreateCounter("otopcua.redundancy.snapshot_rejected", unit: "{snapshot}", description: "Redundancy snapshots dropped by a subscriber (reason=older_revision|unrevisioned)."); /// The artifact's ServerCluster.RedundancyMode disagrees with the local OpcUa:Redundancy:Mode. public static readonly Counter RedundancyModeMismatch = Meter.CreateCounter("otopcua.redundancy.mode_mismatch", unit: "{apply}", description: "Applied artifacts whose cluster RedundancyMode disagrees with the local OpcUa:Redundancy:Mode."); /// Ordinary-tag notifications rewritten BadOutOfService at a Warm Standby's Publish/Republish (D4). public static readonly Counter StandbyProjectedNotifications = Meter.CreateCounter("otopcua.opcua.standby.projected_notifications", unit: "{notification}", description: "Ordinary-tag notifications rewritten BadOutOfService at a Warm standby's first delivery."); /// New alarm condition events not delivered on a Warm Standby (still stamped). public static readonly Counter StandbySuppressedAlarmEvents = Meter.CreateCounter("otopcua.opcua.standby.suppressed_alarm_events", unit: "{event}", description: "New alarm condition events suppressed on a Warm standby."); // ---------------- Convenience helpers ---------------- /// /// Starts a deploy span tagged with the deployment id. Caller disposes to close. Returns /// null when no listener is attached so the call site stays cheap on undecorated builds. /// /// The deployment identifier to tag the span with. /// The started activity, or null when no listener is attached. public static Activity? StartDeployApplySpan(string deploymentId) { var activity = ActivitySource.StartActivity("otopcua.deploy.apply", ActivityKind.Internal); activity?.SetTag("otopcua.deployment_id", deploymentId); return activity; } /// Span wrapping a full OPC UA address-space rebuild (AddressSpace plan → apply). /// The started activity, or null when no listener is attached. public static Activity? StartAddressSpaceRebuildSpan() => ActivitySource.StartActivity("otopcua.opcua.address_space_rebuild", ActivityKind.Internal); }