7829b937d1
OpcUa:Redundancy:Mode (Unset default = today's server exactly; None; Warm; Hot and unknown values fail start). - T1 options + validator + effective-URI check + boot banner; Warm needs the driver role. - T2 metadata in OnNodeManagerStartedAsync: Warm -> NonTransparentRedundancyType, read-only ServerUriArray [self, peers], RedundancySupport=Warm, boot ServiceLevel 1; None -> RedundancySupport=None and the SDK's orphan i=11314 deleted; Unset untouched. - T3 RedundancyStateChanged.Revision (up-number, incarnation tick, sequence); subscribers drop older revisions under Warm (OpcUaPublishActor, DriverHostActor); unrevisioned never makes a node Active. - T4 WarmRoleAuthority + ComputeWarm bands (Standby 90/50, Active 250/200/100), PromoteWhilePrimaryUnreachable. - T5 IStandbyProjection + DeferredStandbyProjection; OpcUaPublishActor runs promotion/demotion in the spec 4.7 order (Active level last and retried; Standby level first, data degraded regardless). - T6 EquipmentVariableState read projection (BadOutOfService/null); Warm Standby refuses writes BadInvalidState. - T7 D4 mechanism adopted after T0: ordinary-tag marker on MonitoredItemNotification.Message validated by (ClientHandle, DataValue); first-delivery decision keyed by (SubscriptionId, SequenceNumber); the SDK's cached copy is never mutated. - T8 queue current/BadOutOfService to live ordinary-tag items ignoring filters. - T9 alarm stamp/deliver split (suppressed on Standby), Part 9 methods refused on Standby, promotion refresh via ISubscription.ConditionRefresh2Async on every live event item. - T10 deploy-time RedundancyMode cross-check + six instruments. Claude-Session: https://claude.ai/code/session_01GT67ACmpMR7b9LGRPLijMi
251 lines
16 KiB
C#
251 lines
16 KiB
C#
using System.Diagnostics;
|
||
using System.Diagnostics.Metrics;
|
||
|
||
namespace ZB.MOM.WW.OtOpcUa.Commons.Observability;
|
||
|
||
/// <summary>
|
||
/// Central <see cref="Meter"/> + <see cref="ActivitySource"/> definitions for OtOpcUa.
|
||
/// All Akka actors, the OPC UA publish path, and the deploy coordinator emit through these
|
||
/// pre-created instruments so a single OpenTelemetry / Prometheus binding in <c>Host</c>
|
||
/// catches everything. No exporter is required — instruments are no-op until a listener
|
||
/// attaches, so tests and dev hosts pay nothing for instrumentation that nobody scrapes.
|
||
///
|
||
/// Instrument names follow the OpenTelemetry semantic convention pattern
|
||
/// <c>otopcua.<subsystem>.<event></c>. Subsystem is one of: deploy, driver,
|
||
/// virtualtag, scriptedalarm, opcua, redundancy.
|
||
/// </summary>
|
||
public static class OtOpcUaTelemetry
|
||
{
|
||
public const string MeterName = "ZB.MOM.WW.OtOpcUa";
|
||
public const string ActivitySourceName = "ZB.MOM.WW.OtOpcUa";
|
||
|
||
/// <summary>Singleton <see cref="Meter"/> all counters/histograms hang off.</summary>
|
||
public static readonly Meter Meter = new(MeterName);
|
||
|
||
/// <summary>Singleton <see cref="ActivitySource"/> used to start spans wrapping deploy/apply/rebuild.</summary>
|
||
public static readonly ActivitySource ActivitySource = new(ActivitySourceName);
|
||
|
||
// ---------------- Deployment / driver-host coordination ----------------
|
||
|
||
/// <summary>Incremented every time DriverHostActor finishes applying a deployment (Ack or Reject).</summary>
|
||
public static readonly Counter<long> DeploymentApplied =
|
||
Meter.CreateCounter<long>("otopcua.deploy.applied", unit: "{deployment}",
|
||
description: "Deployments applied by a driver-role node (outcome=ack|reject).");
|
||
|
||
/// <summary>Time from DriverHostActor receiving DispatchDeployment to emitting the ack/reject.</summary>
|
||
public static readonly Histogram<double> DeploymentApplyDurationSec =
|
||
Meter.CreateHistogram<double>("otopcua.deploy.apply.duration", unit: "s",
|
||
description: "Driver-role apply latency from DispatchDeployment → Ack/Reject.");
|
||
|
||
/// <summary>DriverInstanceActor spawn count (added=new instance; stop=disposed).</summary>
|
||
public static readonly Counter<long> DriverInstanceLifecycle =
|
||
Meter.CreateCounter<long>("otopcua.driver.lifecycle", unit: "{event}",
|
||
description: "DriverInstanceActor lifecycle transitions (event=spawn|stop|fault).");
|
||
|
||
/// <summary>
|
||
/// Poll cycles that ran longer than their subscription's publishing interval, as counted by
|
||
/// <c>PollGroupEngine.OverrunCount</c>. <c>Core.Abstractions</c> deliberately carries no metrics
|
||
/// dependency, so each poll driver wires this counter to the engine's <c>onOverrun</c> callback;
|
||
/// this is the production consumer that surface was always missing. Tagged
|
||
/// <see cref="PollOverrunDriverTypeTag"/> (<c>driver_type</c>, a <c>DriverTypeNames</c> constant)
|
||
/// + <see cref="PollOverrunDriverInstanceTag"/> (<c>driver_instance</c>) — snake_case, per the
|
||
/// convention those two consts document.
|
||
/// A non-zero and climbing rate means the device cannot sustain its configured poll rate — the
|
||
/// loop is already running flat out.
|
||
/// </summary>
|
||
public static readonly Counter<long> PollOverruns =
|
||
Meter.CreateCounter<long>("otopcua.driver.poll.overrun", unit: "{overrun}",
|
||
description: "Poll cycles that ran longer than the poll interval (slow success and slow failure both count — see PollGroupEngine.NoteOverrun).");
|
||
|
||
/// <summary>
|
||
/// Tag key carrying the <c>DriverTypeNames</c> constant on <see cref="PollOverruns"/>.
|
||
/// <para><b>The snake_case spelling is load-bearing, not a style choice.</b> It is the same
|
||
/// literal key <see cref="DriverInstanceLifecycle"/> emits (<c>DriverInstanceActor</c>), and
|
||
/// every other instrument on this <see cref="Meter"/> uses bare/snake_case keys
|
||
/// (<c>event</c>, <c>kind</c>, <c>outcome</c>, <c>site</c>, <c>reason</c>). A dotted
|
||
/// <c>driver.type</c> here would silently split a dashboard's "group by driver type" across
|
||
/// two key names that no single query can join. The seven poll drivers reference this const
|
||
/// rather than a literal so that can never drift back.</para>
|
||
/// </summary>
|
||
public const string PollOverrunDriverTypeTag = "driver_type";
|
||
|
||
/// <summary>
|
||
/// Tag key carrying the emitting driver's instance id on <see cref="PollOverruns"/>. Cardinality
|
||
/// is bounded by the number of configured driver instances on the node. Snake_case for the same
|
||
/// convention reason as <see cref="PollOverrunDriverTypeTag"/>.
|
||
/// </summary>
|
||
public const string PollOverrunDriverInstanceTag = "driver_instance";
|
||
|
||
/// <summary>
|
||
/// Session/publish-level events of an OPC UA Client driver instance: the operator's view of a
|
||
/// flapping upstream. Tagged <see cref="PollOverrunDriverInstanceTag"/> (<c>driver_instance</c>) +
|
||
/// <c>event</c> = <c>publish_response|notification|data_change|bad_status_value|publish_error|
|
||
/// keepalive_miss|keepalive_recovered|stale_notification_dropped</c>.
|
||
/// </summary>
|
||
public static readonly Counter<long> OpcUaClientEvents =
|
||
Meter.CreateCounter<long>("otopcua.driver.opcuaclient.events", unit: "{event}",
|
||
description: "OPC UA Client driver session/publish events (event=publish_response|notification|data_change|bad_status_value|publish_error|keepalive_miss|keepalive_recovered|stale_notification_dropped).");
|
||
|
||
/// <summary>
|
||
/// Controller alarms a native-alarm driver saw that NO configured alarm selector on that device claims, so no
|
||
/// Part 9 condition shows them (lmxopcua#839 — Heidenhain <c>ALARM:</c> selectors without <c>*</c>/<c>OTHER</c>).
|
||
/// Counted once per alarm occurrence (an alarm entering the unclaimed set), not once per poll. Tagged
|
||
/// <see cref="PollOverrunDriverTypeTag"/> + <see cref="PollOverrunDriverInstanceTag"/>. A non-zero rate means
|
||
/// operators are not seeing some controller alarms — add a selector (or an <c>OTHER</c> catch-all).
|
||
/// </summary>
|
||
public static readonly Counter<long> DriverAlarmsUnmatched =
|
||
Meter.CreateCounter<long>("otopcua.driver.alarms.unmatched", unit: "{alarm}",
|
||
description: "Controller alarm occurrences no configured native-alarm selector claims (shown by no condition).");
|
||
|
||
// ---------------- VirtualTag / ScriptedAlarm engines ----------------
|
||
|
||
public static readonly Counter<long> VirtualTagEval =
|
||
Meter.CreateCounter<long>("otopcua.virtualtag.eval", unit: "{eval}",
|
||
description: "Virtual-tag evaluations attempted (outcome=ok|fail|skip).");
|
||
|
||
public static readonly Counter<long> ScriptedAlarmTransition =
|
||
Meter.CreateCounter<long>("otopcua.scriptedalarm.transition", unit: "{transition}",
|
||
description: "Scripted-alarm state transitions (state=active|acknowledged|inactive).");
|
||
|
||
// ---------------- OPC UA address-space + redundancy ----------------
|
||
|
||
public static readonly Counter<long> OpcUaSinkWrite =
|
||
Meter.CreateCounter<long>("otopcua.opcua.sink.write", unit: "{write}",
|
||
description: "Writes that landed in IOpcUaAddressSpaceSink (kind=value|alarm|rebuild).");
|
||
|
||
/// <summary>Address-space apply/materialise passes that swallowed at least one sink failure —
|
||
/// a failed rebuild (kind=rebuild) or per-node materialise failures (kind=nodes). A non-zero rate
|
||
/// means the running server holds a stale or partially-materialised address space despite a
|
||
/// reported-successful deploy (archreview 01/S-1).</summary>
|
||
public static readonly Counter<long> OpcUaApplyFailed =
|
||
Meter.CreateCounter<long>("otopcua.opcua.apply.failed", unit: "{apply}",
|
||
description: "Apply/materialise passes with swallowed sink failures (kind=rebuild|nodes).");
|
||
|
||
public static readonly Counter<long> ServiceLevelChange =
|
||
Meter.CreateCounter<long>("otopcua.redundancy.service_level_change", unit: "{change}",
|
||
description: "OPC UA Server.ServiceLevel transitions emitted by the redundancy state.");
|
||
|
||
/// <summary>Archreview 03/S15 — tag refs a driver left <c>BadNoCommunication</c> after it reconnected:
|
||
/// the host marked them Bad on the outage, and the driver never re-published them. Non-zero means a
|
||
/// driver that reconnects without a full re-publish (a push driver missing its initial push, or a
|
||
/// value-only dedup swallowing an unchanged value), which leaves those nodes Bad indefinitely.
|
||
/// Tagged driver + driver_type.</summary>
|
||
public static readonly Counter<long> DriverCommsLatchedRefs =
|
||
Meter.CreateCounter<long>("otopcua.driver.comms_latched_refs", unit: "{ref}",
|
||
description: "Refs still BadNoCommunication after their driver reconnected (driver, driver_type).");
|
||
|
||
/// <summary>Inbound operations denied by the Primary data-plane gate (archreview 03/S4) —
|
||
/// site=write|ack|alarm-emit, reason=secondary|detached|role-unknown.</summary>
|
||
public static readonly Counter<long> PrimaryGateDenied =
|
||
Meter.CreateCounter<long>("otopcua.redundancy.primary_gate_denied", unit: "{denial}",
|
||
description: "Operations denied by the Primary gate (site=write|ack|alarm-emit, reason=secondary|detached|role-unknown).");
|
||
|
||
/// <summary>
|
||
/// API-key store lookups the 03/S18 bound intervened on (outcome=timeout|cache_hit).
|
||
/// </summary>
|
||
/// <remarks>
|
||
/// One counter with an <c>outcome</c> tag rather than two: <c>timeout</c> is the alarm signal that
|
||
/// the ConfigDb is not answering the OPC UA impersonation callback, and <c>cache_hit</c> is the
|
||
/// ratio that says whether the TTL is doing anything. A missing <c>cache_hit</c> stream on a busy
|
||
/// fused central is how you notice <c>Security:ApiKeys:StoreCacheSeconds</c> was set to 0 by
|
||
/// accident. A plain read-through meters neither — the counter names interventions, not traffic.
|
||
/// </remarks>
|
||
public static readonly Counter<long> ApiKeyStoreBounded =
|
||
Meter.CreateCounter<long>("otopcua.opcua.apikey_store.bounded", unit: "{lookup}",
|
||
description: "API-key store lookups the bound intervened on (outcome=timeout|cache_hit).");
|
||
|
||
/// <summary>
|
||
/// Part 9 condition events whose delivery is currently deferred because an event notifier's SDK delivery
|
||
/// channel was full (Gitea #838). Zero in steady state; non-zero means some folder's event subscribers are
|
||
/// not keeping up with the alarm event rate (or the node manager lock is held across a large burst).
|
||
/// </summary>
|
||
public static readonly UpDownCounter<long> ConditionEventReportsDeferred =
|
||
Meter.CreateUpDownCounter<long>("otopcua.opcua.condition_events.deferred", unit: "{event}",
|
||
description: "Condition events waiting for room in an event notifier's delivery channel.");
|
||
|
||
/// <summary>
|
||
/// Deferred condition events discarded before delivery (Gitea #838): <c>reason=coalesced</c> — the per-condition
|
||
/// deferral budget was exceeded and the condition's oldest not-yet-started event gave way to its newest;
|
||
/// <c>reason=not_live</c> — the condition was removed or replaced before its deferred event was delivered.
|
||
/// </summary>
|
||
public static readonly Counter<long> ConditionEventReportsDiscarded =
|
||
Meter.CreateCounter<long>("otopcua.opcua.condition_events.discarded", unit: "{event}",
|
||
description: "Deferred condition events discarded before delivery (reason=coalesced|not_live).");
|
||
|
||
/// <summary>
|
||
/// Deferred condition events withheld from one Server-object (<c>i=2253</c>) item because a ConditionRefresh of that
|
||
/// item had already replayed a newer state (Gitea #854) — one per item and event. Every other subscriber still
|
||
/// receives them; this only stops a refresh being overtaken by older events.
|
||
/// </summary>
|
||
/// <summary>
|
||
/// Event filters given a private projection group because the server-wide registry of distinct live filter shapes is
|
||
/// full (Gitea #855 perf). Delivery is unchanged; only the per-report sharing of where-clause / field projection
|
||
/// stops for those filters, so a steadily rising count means more per-item work under the node-manager lock.
|
||
/// </summary>
|
||
public static readonly Counter<long> ConditionEventFilterGroupsUnshared =
|
||
Meter.CreateCounter<long>("otopcua.opcua.condition_events.filter_groups_unshared", unit: "{filter}",
|
||
description: "Event filters not shared because the registry of live filter shapes is full.");
|
||
|
||
public static readonly Counter<long> ConditionEventDeliveriesSuperseded =
|
||
Meter.CreateCounter<long>("otopcua.opcua.condition_events.superseded", unit: "{delivery}",
|
||
description: "Deferred condition events withheld from a Server-object item whose ConditionRefresh already replayed a newer state.");
|
||
|
||
// ---------------- Warm redundancy (Gitea #717, spec §7) ----------------
|
||
|
||
private static int _warmRole = -1;
|
||
|
||
/// <summary>Records this process's Warm role for <see cref="WarmRole"/>: 1 Active, 0 Standby, −1 not Warm.</summary>
|
||
/// <param name="value">The role value.</param>
|
||
public static void SetWarmRole(int value) => Volatile.Write(ref _warmRole, value);
|
||
|
||
/// <summary>The Warm role of this node: 1 = Active, 0 = Standby, −1 = not Warm.</summary>
|
||
public static readonly ObservableGauge<int> WarmRole =
|
||
Meter.CreateObservableGauge("otopcua.redundancy.warm_role", () => Volatile.Read(ref _warmRole), unit: "{role}",
|
||
description: "Warm redundancy role of this node: 1 Active, 0 Standby, -1 not Warm.");
|
||
|
||
/// <summary>Warm role transitions (direction=promote|demote, reason=membership|snapshot|endpoint|boot|unreachable-primary|agreed).</summary>
|
||
public static readonly Counter<long> WarmTransition =
|
||
Meter.CreateCounter<long>("otopcua.redundancy.warm_transition", unit: "{transition}",
|
||
description: "Warm role transitions (direction=promote|demote, reason).");
|
||
|
||
/// <summary>Redundancy snapshots not accepted by a subscriber (reason=older_revision|unrevisioned).</summary>
|
||
public static readonly Counter<long> SnapshotRejected =
|
||
Meter.CreateCounter<long>("otopcua.redundancy.snapshot_rejected", unit: "{snapshot}",
|
||
description: "Redundancy snapshots dropped by a subscriber (reason=older_revision|unrevisioned).");
|
||
|
||
/// <summary>The artifact's ServerCluster.RedundancyMode disagrees with the local OpcUa:Redundancy:Mode.</summary>
|
||
public static readonly Counter<long> RedundancyModeMismatch =
|
||
Meter.CreateCounter<long>("otopcua.redundancy.mode_mismatch", unit: "{apply}",
|
||
description: "Applied artifacts whose cluster RedundancyMode disagrees with the local OpcUa:Redundancy:Mode.");
|
||
|
||
/// <summary>Ordinary-tag notifications rewritten BadOutOfService at a Warm Standby's Publish/Republish (D4).</summary>
|
||
public static readonly Counter<long> StandbyProjectedNotifications =
|
||
Meter.CreateCounter<long>("otopcua.opcua.standby.projected_notifications", unit: "{notification}",
|
||
description: "Ordinary-tag notifications rewritten BadOutOfService at a Warm standby's first delivery.");
|
||
|
||
/// <summary>New alarm condition events not delivered on a Warm Standby (still stamped).</summary>
|
||
public static readonly Counter<long> StandbySuppressedAlarmEvents =
|
||
Meter.CreateCounter<long>("otopcua.opcua.standby.suppressed_alarm_events", unit: "{event}",
|
||
description: "New alarm condition events suppressed on a Warm standby.");
|
||
|
||
// ---------------- Convenience helpers ----------------
|
||
|
||
/// <summary>
|
||
/// Starts a deploy span tagged with the deployment id. Caller disposes to close. Returns
|
||
/// null when no listener is attached so the call site stays cheap on undecorated builds.
|
||
/// </summary>
|
||
/// <param name="deploymentId">The deployment identifier to tag the span with.</param>
|
||
/// <returns>The started activity, or <c>null</c> when no listener is attached.</returns>
|
||
public static Activity? StartDeployApplySpan(string deploymentId)
|
||
{
|
||
var activity = ActivitySource.StartActivity("otopcua.deploy.apply", ActivityKind.Internal);
|
||
activity?.SetTag("otopcua.deployment_id", deploymentId);
|
||
return activity;
|
||
}
|
||
|
||
/// <summary>Span wrapping a full OPC UA address-space rebuild (AddressSpace plan → apply).</summary>
|
||
/// <returns>The started activity, or <c>null</c> when no listener is attached.</returns>
|
||
public static Activity? StartAddressSpaceRebuildSpan()
|
||
=> ActivitySource.StartActivity("otopcua.opcua.address_space_rebuild", ActivityKind.Internal);
|
||
}
|