Files
lmxopcua/src/Core/ZB.MOM.WW.OtOpcUa.Commons/Observability/OtOpcUaTelemetry.cs
T
Joseph Doherty 7829b937d1 feat(717): Warm non-transparent redundancy, default-off (T1-T10)
OpcUa:Redundancy:Mode (Unset default = today's server exactly; None; Warm; Hot and unknown values fail start).
- T1 options + validator + effective-URI check + boot banner; Warm needs the driver role.
- T2 metadata in OnNodeManagerStartedAsync: Warm -> NonTransparentRedundancyType, read-only
  ServerUriArray [self, peers], RedundancySupport=Warm, boot ServiceLevel 1; None -> RedundancySupport=None
  and the SDK's orphan i=11314 deleted; Unset untouched.
- T3 RedundancyStateChanged.Revision (up-number, incarnation tick, sequence); subscribers drop older
  revisions under Warm (OpcUaPublishActor, DriverHostActor); unrevisioned never makes a node Active.
- T4 WarmRoleAuthority + ComputeWarm bands (Standby 90/50, Active 250/200/100), PromoteWhilePrimaryUnreachable.
- T5 IStandbyProjection + DeferredStandbyProjection; OpcUaPublishActor runs promotion/demotion in the spec
  4.7 order (Active level last and retried; Standby level first, data degraded regardless).
- T6 EquipmentVariableState read projection (BadOutOfService/null); Warm Standby refuses writes BadInvalidState.
- T7 D4 mechanism adopted after T0: ordinary-tag marker on MonitoredItemNotification.Message validated by
  (ClientHandle, DataValue); first-delivery decision keyed by (SubscriptionId, SequenceNumber); the SDK's
  cached copy is never mutated.
- T8 queue current/BadOutOfService to live ordinary-tag items ignoring filters.
- T9 alarm stamp/deliver split (suppressed on Standby), Part 9 methods refused on Standby, promotion
  refresh via ISubscription.ConditionRefresh2Async on every live event item.
- T10 deploy-time RedundancyMode cross-check + six instruments.

Claude-Session: https://claude.ai/code/session_01GT67ACmpMR7b9LGRPLijMi
2026-11-13 03:39:50 -05:00

251 lines
16 KiB
C#
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
using System.Diagnostics;
using System.Diagnostics.Metrics;
namespace ZB.MOM.WW.OtOpcUa.Commons.Observability;
/// <summary>
/// Central <see cref="Meter"/> + <see cref="ActivitySource"/> definitions for OtOpcUa.
/// All Akka actors, the OPC UA publish path, and the deploy coordinator emit through these
/// pre-created instruments so a single OpenTelemetry / Prometheus binding in <c>Host</c>
/// catches everything. No exporter is required — instruments are no-op until a listener
/// attaches, so tests and dev hosts pay nothing for instrumentation that nobody scrapes.
///
/// Instrument names follow the OpenTelemetry semantic convention pattern
/// <c>otopcua.&lt;subsystem&gt;.&lt;event&gt;</c>. Subsystem is one of: deploy, driver,
/// virtualtag, scriptedalarm, opcua, redundancy.
/// </summary>
public static class OtOpcUaTelemetry
{
public const string MeterName = "ZB.MOM.WW.OtOpcUa";
public const string ActivitySourceName = "ZB.MOM.WW.OtOpcUa";
/// <summary>Singleton <see cref="Meter"/> all counters/histograms hang off.</summary>
public static readonly Meter Meter = new(MeterName);
/// <summary>Singleton <see cref="ActivitySource"/> used to start spans wrapping deploy/apply/rebuild.</summary>
public static readonly ActivitySource ActivitySource = new(ActivitySourceName);
// ---------------- Deployment / driver-host coordination ----------------
/// <summary>Incremented every time DriverHostActor finishes applying a deployment (Ack or Reject).</summary>
public static readonly Counter<long> DeploymentApplied =
Meter.CreateCounter<long>("otopcua.deploy.applied", unit: "{deployment}",
description: "Deployments applied by a driver-role node (outcome=ack|reject).");
/// <summary>Time from DriverHostActor receiving DispatchDeployment to emitting the ack/reject.</summary>
public static readonly Histogram<double> DeploymentApplyDurationSec =
Meter.CreateHistogram<double>("otopcua.deploy.apply.duration", unit: "s",
description: "Driver-role apply latency from DispatchDeployment → Ack/Reject.");
/// <summary>DriverInstanceActor spawn count (added=new instance; stop=disposed).</summary>
public static readonly Counter<long> DriverInstanceLifecycle =
Meter.CreateCounter<long>("otopcua.driver.lifecycle", unit: "{event}",
description: "DriverInstanceActor lifecycle transitions (event=spawn|stop|fault).");
/// <summary>
/// Poll cycles that ran longer than their subscription's publishing interval, as counted by
/// <c>PollGroupEngine.OverrunCount</c>. <c>Core.Abstractions</c> deliberately carries no metrics
/// dependency, so each poll driver wires this counter to the engine's <c>onOverrun</c> callback;
/// this is the production consumer that surface was always missing. Tagged
/// <see cref="PollOverrunDriverTypeTag"/> (<c>driver_type</c>, a <c>DriverTypeNames</c> constant)
/// + <see cref="PollOverrunDriverInstanceTag"/> (<c>driver_instance</c>) — snake_case, per the
/// convention those two consts document.
/// A non-zero and climbing rate means the device cannot sustain its configured poll rate — the
/// loop is already running flat out.
/// </summary>
public static readonly Counter<long> PollOverruns =
Meter.CreateCounter<long>("otopcua.driver.poll.overrun", unit: "{overrun}",
description: "Poll cycles that ran longer than the poll interval (slow success and slow failure both count — see PollGroupEngine.NoteOverrun).");
/// <summary>
/// Tag key carrying the <c>DriverTypeNames</c> constant on <see cref="PollOverruns"/>.
/// <para><b>The snake_case spelling is load-bearing, not a style choice.</b> It is the same
/// literal key <see cref="DriverInstanceLifecycle"/> emits (<c>DriverInstanceActor</c>), and
/// every other instrument on this <see cref="Meter"/> uses bare/snake_case keys
/// (<c>event</c>, <c>kind</c>, <c>outcome</c>, <c>site</c>, <c>reason</c>). A dotted
/// <c>driver.type</c> here would silently split a dashboard's "group by driver type" across
/// two key names that no single query can join. The seven poll drivers reference this const
/// rather than a literal so that can never drift back.</para>
/// </summary>
public const string PollOverrunDriverTypeTag = "driver_type";
/// <summary>
/// Tag key carrying the emitting driver's instance id on <see cref="PollOverruns"/>. Cardinality
/// is bounded by the number of configured driver instances on the node. Snake_case for the same
/// convention reason as <see cref="PollOverrunDriverTypeTag"/>.
/// </summary>
public const string PollOverrunDriverInstanceTag = "driver_instance";
/// <summary>
/// Session/publish-level events of an OPC UA Client driver instance: the operator's view of a
/// flapping upstream. Tagged <see cref="PollOverrunDriverInstanceTag"/> (<c>driver_instance</c>) +
/// <c>event</c> = <c>publish_response|notification|data_change|bad_status_value|publish_error|
/// keepalive_miss|keepalive_recovered|stale_notification_dropped</c>.
/// </summary>
public static readonly Counter<long> OpcUaClientEvents =
Meter.CreateCounter<long>("otopcua.driver.opcuaclient.events", unit: "{event}",
description: "OPC UA Client driver session/publish events (event=publish_response|notification|data_change|bad_status_value|publish_error|keepalive_miss|keepalive_recovered|stale_notification_dropped).");
/// <summary>
/// Controller alarms a native-alarm driver saw that NO configured alarm selector on that device claims, so no
/// Part 9 condition shows them (lmxopcua#839 — Heidenhain <c>ALARM:</c> selectors without <c>*</c>/<c>OTHER</c>).
/// Counted once per alarm occurrence (an alarm entering the unclaimed set), not once per poll. Tagged
/// <see cref="PollOverrunDriverTypeTag"/> + <see cref="PollOverrunDriverInstanceTag"/>. A non-zero rate means
/// operators are not seeing some controller alarms — add a selector (or an <c>OTHER</c> catch-all).
/// </summary>
public static readonly Counter<long> DriverAlarmsUnmatched =
Meter.CreateCounter<long>("otopcua.driver.alarms.unmatched", unit: "{alarm}",
description: "Controller alarm occurrences no configured native-alarm selector claims (shown by no condition).");
// ---------------- VirtualTag / ScriptedAlarm engines ----------------
public static readonly Counter<long> VirtualTagEval =
Meter.CreateCounter<long>("otopcua.virtualtag.eval", unit: "{eval}",
description: "Virtual-tag evaluations attempted (outcome=ok|fail|skip).");
public static readonly Counter<long> ScriptedAlarmTransition =
Meter.CreateCounter<long>("otopcua.scriptedalarm.transition", unit: "{transition}",
description: "Scripted-alarm state transitions (state=active|acknowledged|inactive).");
// ---------------- OPC UA address-space + redundancy ----------------
public static readonly Counter<long> OpcUaSinkWrite =
Meter.CreateCounter<long>("otopcua.opcua.sink.write", unit: "{write}",
description: "Writes that landed in IOpcUaAddressSpaceSink (kind=value|alarm|rebuild).");
/// <summary>Address-space apply/materialise passes that swallowed at least one sink failure —
/// a failed rebuild (kind=rebuild) or per-node materialise failures (kind=nodes). A non-zero rate
/// means the running server holds a stale or partially-materialised address space despite a
/// reported-successful deploy (archreview 01/S-1).</summary>
public static readonly Counter<long> OpcUaApplyFailed =
Meter.CreateCounter<long>("otopcua.opcua.apply.failed", unit: "{apply}",
description: "Apply/materialise passes with swallowed sink failures (kind=rebuild|nodes).");
public static readonly Counter<long> ServiceLevelChange =
Meter.CreateCounter<long>("otopcua.redundancy.service_level_change", unit: "{change}",
description: "OPC UA Server.ServiceLevel transitions emitted by the redundancy state.");
/// <summary>Archreview 03/S15 — tag refs a driver left <c>BadNoCommunication</c> after it reconnected:
/// the host marked them Bad on the outage, and the driver never re-published them. Non-zero means a
/// driver that reconnects without a full re-publish (a push driver missing its initial push, or a
/// value-only dedup swallowing an unchanged value), which leaves those nodes Bad indefinitely.
/// Tagged driver + driver_type.</summary>
public static readonly Counter<long> DriverCommsLatchedRefs =
Meter.CreateCounter<long>("otopcua.driver.comms_latched_refs", unit: "{ref}",
description: "Refs still BadNoCommunication after their driver reconnected (driver, driver_type).");
/// <summary>Inbound operations denied by the Primary data-plane gate (archreview 03/S4) —
/// site=write|ack|alarm-emit, reason=secondary|detached|role-unknown.</summary>
public static readonly Counter<long> PrimaryGateDenied =
Meter.CreateCounter<long>("otopcua.redundancy.primary_gate_denied", unit: "{denial}",
description: "Operations denied by the Primary gate (site=write|ack|alarm-emit, reason=secondary|detached|role-unknown).");
/// <summary>
/// API-key store lookups the 03/S18 bound intervened on (outcome=timeout|cache_hit).
/// </summary>
/// <remarks>
/// One counter with an <c>outcome</c> tag rather than two: <c>timeout</c> is the alarm signal that
/// the ConfigDb is not answering the OPC UA impersonation callback, and <c>cache_hit</c> is the
/// ratio that says whether the TTL is doing anything. A missing <c>cache_hit</c> stream on a busy
/// fused central is how you notice <c>Security:ApiKeys:StoreCacheSeconds</c> was set to 0 by
/// accident. A plain read-through meters neither — the counter names interventions, not traffic.
/// </remarks>
public static readonly Counter<long> ApiKeyStoreBounded =
Meter.CreateCounter<long>("otopcua.opcua.apikey_store.bounded", unit: "{lookup}",
description: "API-key store lookups the bound intervened on (outcome=timeout|cache_hit).");
/// <summary>
/// Part 9 condition events whose delivery is currently deferred because an event notifier's SDK delivery
/// channel was full (Gitea #838). Zero in steady state; non-zero means some folder's event subscribers are
/// not keeping up with the alarm event rate (or the node manager lock is held across a large burst).
/// </summary>
public static readonly UpDownCounter<long> ConditionEventReportsDeferred =
Meter.CreateUpDownCounter<long>("otopcua.opcua.condition_events.deferred", unit: "{event}",
description: "Condition events waiting for room in an event notifier's delivery channel.");
/// <summary>
/// Deferred condition events discarded before delivery (Gitea #838): <c>reason=coalesced</c> — the per-condition
/// deferral budget was exceeded and the condition's oldest not-yet-started event gave way to its newest;
/// <c>reason=not_live</c> — the condition was removed or replaced before its deferred event was delivered.
/// </summary>
public static readonly Counter<long> ConditionEventReportsDiscarded =
Meter.CreateCounter<long>("otopcua.opcua.condition_events.discarded", unit: "{event}",
description: "Deferred condition events discarded before delivery (reason=coalesced|not_live).");
/// <summary>
/// Deferred condition events withheld from one Server-object (<c>i=2253</c>) item because a ConditionRefresh of that
/// item had already replayed a newer state (Gitea #854) — one per item and event. Every other subscriber still
/// receives them; this only stops a refresh being overtaken by older events.
/// </summary>
/// <summary>
/// Event filters given a private projection group because the server-wide registry of distinct live filter shapes is
/// full (Gitea #855 perf). Delivery is unchanged; only the per-report sharing of where-clause / field projection
/// stops for those filters, so a steadily rising count means more per-item work under the node-manager lock.
/// </summary>
public static readonly Counter<long> ConditionEventFilterGroupsUnshared =
Meter.CreateCounter<long>("otopcua.opcua.condition_events.filter_groups_unshared", unit: "{filter}",
description: "Event filters not shared because the registry of live filter shapes is full.");
public static readonly Counter<long> ConditionEventDeliveriesSuperseded =
Meter.CreateCounter<long>("otopcua.opcua.condition_events.superseded", unit: "{delivery}",
description: "Deferred condition events withheld from a Server-object item whose ConditionRefresh already replayed a newer state.");
// ---------------- Warm redundancy (Gitea #717, spec §7) ----------------
private static int _warmRole = -1;
/// <summary>Records this process's Warm role for <see cref="WarmRole"/>: 1 Active, 0 Standby, −1 not Warm.</summary>
/// <param name="value">The role value.</param>
public static void SetWarmRole(int value) => Volatile.Write(ref _warmRole, value);
/// <summary>The Warm role of this node: 1 = Active, 0 = Standby, −1 = not Warm.</summary>
public static readonly ObservableGauge<int> WarmRole =
Meter.CreateObservableGauge("otopcua.redundancy.warm_role", () => Volatile.Read(ref _warmRole), unit: "{role}",
description: "Warm redundancy role of this node: 1 Active, 0 Standby, -1 not Warm.");
/// <summary>Warm role transitions (direction=promote|demote, reason=membership|snapshot|endpoint|boot|unreachable-primary|agreed).</summary>
public static readonly Counter<long> WarmTransition =
Meter.CreateCounter<long>("otopcua.redundancy.warm_transition", unit: "{transition}",
description: "Warm role transitions (direction=promote|demote, reason).");
/// <summary>Redundancy snapshots not accepted by a subscriber (reason=older_revision|unrevisioned).</summary>
public static readonly Counter<long> SnapshotRejected =
Meter.CreateCounter<long>("otopcua.redundancy.snapshot_rejected", unit: "{snapshot}",
description: "Redundancy snapshots dropped by a subscriber (reason=older_revision|unrevisioned).");
/// <summary>The artifact's ServerCluster.RedundancyMode disagrees with the local OpcUa:Redundancy:Mode.</summary>
public static readonly Counter<long> RedundancyModeMismatch =
Meter.CreateCounter<long>("otopcua.redundancy.mode_mismatch", unit: "{apply}",
description: "Applied artifacts whose cluster RedundancyMode disagrees with the local OpcUa:Redundancy:Mode.");
/// <summary>Ordinary-tag notifications rewritten BadOutOfService at a Warm Standby's Publish/Republish (D4).</summary>
public static readonly Counter<long> StandbyProjectedNotifications =
Meter.CreateCounter<long>("otopcua.opcua.standby.projected_notifications", unit: "{notification}",
description: "Ordinary-tag notifications rewritten BadOutOfService at a Warm standby's first delivery.");
/// <summary>New alarm condition events not delivered on a Warm Standby (still stamped).</summary>
public static readonly Counter<long> StandbySuppressedAlarmEvents =
Meter.CreateCounter<long>("otopcua.opcua.standby.suppressed_alarm_events", unit: "{event}",
description: "New alarm condition events suppressed on a Warm standby.");
// ---------------- Convenience helpers ----------------
/// <summary>
/// Starts a deploy span tagged with the deployment id. Caller disposes to close. Returns
/// null when no listener is attached so the call site stays cheap on undecorated builds.
/// </summary>
/// <param name="deploymentId">The deployment identifier to tag the span with.</param>
/// <returns>The started activity, or <c>null</c> when no listener is attached.</returns>
public static Activity? StartDeployApplySpan(string deploymentId)
{
var activity = ActivitySource.StartActivity("otopcua.deploy.apply", ActivityKind.Internal);
activity?.SetTag("otopcua.deployment_id", deploymentId);
return activity;
}
/// <summary>Span wrapping a full OPC UA address-space rebuild (AddressSpace plan → apply).</summary>
/// <returns>The started activity, or <c>null</c> when no listener is attached.</returns>
public static Activity? StartAddressSpaceRebuildSpan()
=> ActivitySource.StartActivity("otopcua.opcua.address_space_rebuild", ActivityKind.Internal);
}