From baf89b3df77d33c7418524d65261be2a6d13fda0 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Mon, 9 Nov 2026 14:50:42 -0500 Subject: [PATCH] test+docs: serialise data-connection gauge tests, add metrics instrument reference (ScadaBridge#99) DataConnectionGaugeTests and ExtendedInstrumentTests both mutate the process-global data connection registry; put them in a non-parallel xUnit collection. Add docs/operations/metrics-reference.md. Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV --- docs/operations/metrics-reference.md | 64 +++++++++++++++++++ .../DataConnectionGaugeCollection.cs | 15 +++++ .../Observability/DataConnectionGaugeTests.cs | 5 +- .../Observability/ExtendedInstrumentTests.cs | 1 + 4 files changed, 84 insertions(+), 1 deletion(-) create mode 100644 docs/operations/metrics-reference.md create mode 100644 tests/ZB.MOM.WW.ScadaBridge.Commons.Tests/Observability/DataConnectionGaugeCollection.cs diff --git a/docs/operations/metrics-reference.md b/docs/operations/metrics-reference.md new file mode 100644 index 00000000..8de48f62 --- /dev/null +++ b/docs/operations/metrics-reference.md @@ -0,0 +1,64 @@ +# Metrics instrument reference + +Every application metric ScadaBridge emits, derived from the code (Gitea #99). Source of truth: +`src/ZB.MOM.WW.ScadaBridge.Commons/Observability/ScadaBridgeTelemetry.cs` plus +`src/ZB.MOM.WW.ScadaBridge.AuditLog/Site/SiteAuditTelemetryMetrics.cs`. To regenerate this list: + +```bash +grep -rnE "Create(Counter|Histogram|UpDownCounter|Observable[A-Za-z]+)<" src --include='*.cs' +``` + +## Meter + +All instruments below live on one meter, `ZB.MOM.WW.ScadaBridge` (`ScadaBridgeTelemetry.MeterName`). The +meter is exported only because it is in the `ZbTelemetryOptions.Meters` allowlist +(`SiteServiceRegistration.ObservedMeters`); a new meter that is not added there is silently never +scraped. That list also carries the LocalDb replication meter (`LocalDbMetrics.MeterName`), which is +owned by the `ZB.MOM.WW.LocalDb` library and not catalogued here. + +Instrument names are dotted (`scadabridge.site.script.runs`); Prometheus exposes them with underscores +(`scadabridge_site_script_runs_total` for counters, plus unit suffixes). Instruments are no-ops until +a listener attaches. Tag values listed are the ones emitted by the code today. + +## Counters + +| Instrument | Unit | Tags | Meaning / alert guidance | +|---|---|---|---| +| `scadabridge.deployments.applied` | 1 | none | Deployments successfully applied. Pair with `deployments.failed` for the success ratio. | +| `scadabridge.deployments.failed` | 1 | `reason` = `timeout` \| `error` \| `site_failed` | Deployments that ended in terminal failure. Alert on any sustained rate. | +| `scadabridge.store_and_forward.activity` | 1 | `action` (`Queued`, `Delivered`, `Retried`, `Parked`, `Retry`, `Discard`), `category` | S&F lifecycle events. A sustained `Parked` rate means messages are exhausting retries (the queue-depth gauge cannot tell drain from park). | +| `scadabridge.site.script.runs` | 1 | `outcome` = `completed` \| `failed` \| `timed_out` | Finished site script runs. Deliberately not tagged by instance/script (unbounded cardinality). | +| `scadabridge.site.data_connections.transitions` | 1 | `state` (new `ConnectionHealth`) | Real data-connection state changes only (not repeats). Shows flapping the gauges average away. | +| `scadabridge.inbound_api.requests` | 1 | `method` | Inbound API requests per API method. | +| `scadabridge.inbound_api.abandoned_executions` | 1 | `method` | Inbound executions abandoned after timeout/abort while the handler still ran. | +| `scadabridge.store_and_forward.replication.failures` | 1 | none | S&F buffer replication ops that failed to reach the peer node. | +| `scadabridge.store_and_forward.resync.completed` | 1 | none | Anti-entropy resyncs the standby acknowledged as applied. | +| `scadabridge.store_and_forward.resync.ack_missing` | 1 | none | Resyncs answered by the active node but not acknowledged by the standby in the ack window (lost chunks / dead peer). | +| `scadabridge.site.alarm_cache.reconnects` | 1 | none | Live-alarm aggregator site-wide gRPC stream reconnects. A sustained climb = flapping site link. | +| `scadabridge.site.alarm_cache.buffer_dropped` | 1 | none | Live deltas evicted from an aggregator's bounded pre-seed buffer (drop-oldest). | +| `scadabridge.site.script_compile_cache.evictions` | 1 | none | Entries evicted from the script compile cache at its bound (#68). A sustained climb = cache thrash and working-set growth; raise `ScadaBridge:SiteRuntime:ScriptCompileCacheMaxEntries`. | +| `scadabridge.central.debug_view.presnapshot_dropped` | 1 | none | Debug events evicted from a central debug session's pre-snapshot buffer. | +| `scadabridge.site.stream.events_dropped` | 1 | `stream` = `instance` \| `site-alarms` | Events evicted from a site gRPC stream's bounded send channel. | +| `scadabridge.central.transport.import_sessions.rejected` | 1 | none | Bundle-import sessions refused because `MaxConcurrentImportSessions` was exhausted. Rising while the open gauge sits at the cap = session leak; rising while it oscillates below = real concurrency (raise the cap). | +| `scadabridge.site.event_log.purge_ceiling_hits` | 1 | none | Event-log purge ticks that stopped at `MaxPurgedRowsPerTick` with work remaining. A burst that stops is a converging backlog; hits that never stop mean the ceiling/interval is mis-sized. | +| `scadabridge.site.audit.rejected_rows` | 1 | none | Site audit rows central permanently refused `MaxRejectedPushAttempts` times; moved to Rejected (still served by the reconciliation pull). | + +## Histograms + +| Instrument | Unit | Tags | Meaning | +|---|---|---|---| +| `scadabridge.site.stream.batch_size` | 1 (events) | `stream` | Events per site gRPC stream frame; recorded only for subscriptions that negotiated batching. Pinned at 1 = window never sees a backlog. | +| `scadabridge.site.script.duration` | ms | `outcome` = `completed` \| `failed` \| `timed_out` | Site script run duration. | + +## Observable gauges + +| Instrument | Unit | Tags | Meaning / alert guidance | +|---|---|---|---| +| `scadabridge.site.stream.subscribers.active` | 1 | none | Open central-to-site gRPC subscriber streams (Debug View + site-wide alarm feed). **Not** data connections; 0 on a healthy unwatched site. (Renamed from `scadabridge.site.connection.up`.) | +| `scadabridge.site.data_connections.connected` | 1 | none | Data connections on this node currently `Connected`. | +| `scadabridge.site.data_connections.total` | 1 | none | Data connections known to this node in any state. Alert on `connected < total` on the active node; the standby legitimately reports 0, so alert on the active node or `max()` across the pair. | +| `scadabridge.store_and_forward.queue.depth` | items | none | Current S&F queue depth. | +| `scadabridge.site.alarm_cache.aggregators.active` | 1 | none | Running per-site live-alarm aggregators on the active central node. | +| `scadabridge.central.transport.import_sessions.open` | 1 | none | Open bundle-import sessions on this central node (each pins a decrypted bundle in memory). 0 on sites. A plateau at the cap indicates leaked sessions. | + +No UpDownCounters or observable counters are defined. diff --git a/tests/ZB.MOM.WW.ScadaBridge.Commons.Tests/Observability/DataConnectionGaugeCollection.cs b/tests/ZB.MOM.WW.ScadaBridge.Commons.Tests/Observability/DataConnectionGaugeCollection.cs new file mode 100644 index 00000000..a547ec4c --- /dev/null +++ b/tests/ZB.MOM.WW.ScadaBridge.Commons.Tests/Observability/DataConnectionGaugeCollection.cs @@ -0,0 +1,15 @@ +namespace ZB.MOM.WW.ScadaBridge.Commons.Tests.Observability; + +/// +/// Serialises every test class that mutates the process-global data-connection gauge +/// registry in ScadaBridgeTelemetry (Gitea #99). The delta-against-baseline assertions in +/// DataConnectionGaugeTests are only valid if no other test adds or removes a connection +/// between the baseline read and the assertion; DisableParallelization makes this +/// collection run alone, not concurrently with any other collection in the assembly. +/// +[CollectionDefinition(Name, DisableParallelization = true)] +public sealed class DataConnectionGaugeCollection +{ + /// Collection name shared by the classes that touch the gauge registry. + public const string Name = "DataConnectionGauge"; +} diff --git a/tests/ZB.MOM.WW.ScadaBridge.Commons.Tests/Observability/DataConnectionGaugeTests.cs b/tests/ZB.MOM.WW.ScadaBridge.Commons.Tests/Observability/DataConnectionGaugeTests.cs index 1f047fef..2375c132 100644 --- a/tests/ZB.MOM.WW.ScadaBridge.Commons.Tests/Observability/DataConnectionGaugeTests.cs +++ b/tests/ZB.MOM.WW.ScadaBridge.Commons.Tests/Observability/DataConnectionGaugeTests.cs @@ -16,8 +16,11 @@ namespace ZB.MOM.WW.ScadaBridge.Commons.Tests.Observability; /// gauges is a process-global static, so absolute assertions would inherit the register /// row 32 flake that makes QueueDepthGaugeTests pass alone and fail under the full /// concurrent suite. Each test also removes its own connections in a finally so it -/// leaves the shared registry exactly as it found it. +/// leaves the shared registry exactly as it found it. The class is also in the +/// non-parallel DataConnectionGauge collection (Gitea #99), because the deltas can still be +/// corrupted by another class mutating the registry between baseline and assertion. /// +[Collection(DataConnectionGaugeCollection.Name)] public sealed class DataConnectionGaugeTests { private const string ConnectedGauge = "scadabridge.site.data_connections.connected"; diff --git a/tests/ZB.MOM.WW.ScadaBridge.Commons.Tests/Observability/ExtendedInstrumentTests.cs b/tests/ZB.MOM.WW.ScadaBridge.Commons.Tests/Observability/ExtendedInstrumentTests.cs index f8497437..233a1f6d 100644 --- a/tests/ZB.MOM.WW.ScadaBridge.Commons.Tests/Observability/ExtendedInstrumentTests.cs +++ b/tests/ZB.MOM.WW.ScadaBridge.Commons.Tests/Observability/ExtendedInstrumentTests.cs @@ -9,6 +9,7 @@ namespace ZB.MOM.WW.ScadaBridge.Commons.Tests.Observability; /// filtered to the emitting (test) thread, because the meter is /// process-global and other tests emit on it concurrently. /// +[Collection(DataConnectionGaugeCollection.Name)] public sealed class ExtendedInstrumentTests { private static List<(long Value, Dictionary Tags)> Capture(string instrument, Action act)