dohertj2/ScadaBridge
1
0
Fork 0
Code Issues 15 Pull Requests Actions Packages Projects Releases Wiki Activity

feat(telemetry): deploy-failure, S&F activity, script-run and DCL-transition instruments (ScadaBridge#81)

Browse Source 
Adds four instruments on the already-allow-listed ZB.MOM.WW.ScadaBridge meter:
scadabridge.deployments.failed{reason}, scadabridge.store_and_forward.activity{action,category},
scadabridge.site.script.runs{outcome} + scadabridge.site.script.duration{outcome} (ms),
scadabridge.site.data_connections.transitions{state}. Queue depth, DCL connection state and
deploy-applied already existed. No new Meter, so ZbTelemetryOptions.Meters needs no change.

Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
Joseph Doherty
2026-11-09 14:17:53 -05:00
parent 7892c622cf
commit 5ef27fd9a3
8 changed files with 301 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ZB.MOM.WW.ScadaBridge.Commons/Observability/ScadaBridgeTelemetry.cs
+73
View File
@@ -33,6 +33,45 @@ public static class ScadaBridgeTelemetry
Meter.CreateCounter<long>("scadabridge.deployments.applied", unit: "1",
description: "Deployments applied.");
/// <summary>
/// Incremented for each deployment that ends in a terminal failure (ScadaBridge#81), tagged
/// <c>reason</c>: <c>timeout</c> (the site round-trip timed out), <c>error</c> (any other fault
/// in the send or post-send persistence path) or <c>site_failed</c> (the site answered with a
/// non-Success status). Pairs with <c>scadabridge.deployments.applied</c> to give the deploy
/// success ratio; the rate-limited log lines cannot be alerted on.
/// </summary>
private static readonly Counter<long> _deploymentsFailed =
Meter.CreateCounter<long>("scadabridge.deployments.failed", unit: "1",
description: "Deployments that ended in a terminal failure, tagged by reason (timeout|error|site_failed).");
/// <summary>
/// Incremented for each store-and-forward lifecycle event (ScadaBridge#81), tagged <c>action</c>
/// (<c>Queued</c>, <c>Delivered</c>, <c>Retried</c>, <c>Parked</c>, <c>Retry</c>, <c>Discard</c>)
/// and <c>category</c>. A sustained <c>Parked</c> rate is the alertable signal that messages are
/// exhausting retries; the queue-depth gauge alone cannot distinguish a drain from a park.
/// </summary>
private static readonly Counter<long> _sfActivity =
Meter.CreateCounter<long>("scadabridge.store_and_forward.activity", unit: "1",
description: "Store-and-forward lifecycle events, tagged by action and category.");
/// <summary>
/// Incremented for each finished site script run (ScadaBridge#81), tagged <c>outcome</c>
/// (<c>completed</c> | <c>failed</c> | <c>timed_out</c>). Deliberately NOT tagged by instance or
/// script name: that cardinality is unbounded across a site's instances.
/// </summary>
private static readonly Counter<long> _scriptRuns =
Meter.CreateCounter<long>("scadabridge.site.script.runs", unit: "1",
description: "Finished site script runs, tagged by outcome (completed|failed|timed_out).");
/// <summary>
/// Incremented each time a data connection actually CHANGES state (ScadaBridge#81), tagged
/// <c>state</c> (the new <see cref="ConnectionHealth"/>). The data_connections gauges show the
/// current picture; this shows flapping that a scrape interval would otherwise average away.
/// </summary>
private static readonly Counter<long> _dataConnectionTransitions =
Meter.CreateCounter<long>("scadabridge.site.data_connections.transitions", unit: "1",
description: "Data connection state changes, tagged by the new state.");
/// <summary>Incremented for each inbound API request, tagged with the API method.</summary>
private static readonly Counter<long> _inboundApiRequests =
Meter.CreateCounter<long>("scadabridge.inbound_api.requests", unit: "1",
@@ -151,6 +190,11 @@ public static class ScadaBridgeTelemetry
Meter.CreateHistogram<int>("scadabridge.site.stream.batch_size", unit: "1",
description: "Events per site gRPC stream frame (1 = unbatched frame), tagged by stream kind.");
/// <summary>Duration of finished site script runs in milliseconds (ScadaBridge#81), tagged by outcome.</summary>
private static readonly Histogram<long> _scriptRunDuration =
Meter.CreateHistogram<long>("scadabridge.site.script.duration", unit: "ms",
description: "Site script run duration in milliseconds, tagged by outcome (completed|failed|timed_out).");
// ---------------- Observable gauges ----------------
/// <summary>
@@ -271,6 +315,29 @@ public static class ScadaBridgeTelemetry
/// <summary>Records that a deployment was applied.</summary>
public static void RecordDeploymentApplied() => _deploymentsApplied.Add(1);
/// <summary>Records one deployment that ended in a terminal failure (ScadaBridge#81).</summary>
/// <param name="reason">One of <c>timeout</c>, <c>error</c>, <c>site_failed</c>.</param>
public static void RecordDeploymentFailed(string reason) =>
_deploymentsFailed.Add(1, new KeyValuePair<string, object?>("reason", reason));
/// <summary>Records one store-and-forward lifecycle event (ScadaBridge#81).</summary>
/// <param name="action">The activity action, e.g. <c>Parked</c>.</param>
/// <param name="category">The message category name.</param>
public static void RecordStoreAndForwardActivity(string action, string category) =>
_sfActivity.Add(1,
new KeyValuePair<string, object?>("action", action),
new KeyValuePair<string, object?>("category", category));
/// <summary>Records one finished site script run and its duration (ScadaBridge#81).</summary>
/// <param name="outcome">One of <c>completed</c>, <c>failed</c>, <c>timed_out</c>.</param>
/// <param name="durationMs">The run duration in milliseconds.</param>
public static void RecordScriptRun(string outcome, long durationMs)
{
var tag = new KeyValuePair<string, object?>("outcome", outcome);
_scriptRuns.Add(1, tag);
_scriptRunDuration.Record(durationMs < 0 ? 0 : durationMs, tag);
}
/// <summary>Records an inbound API request for the given <paramref name="method"/>.</summary>
/// <param name="method">The API method the request targeted.</param>
public static void RecordInboundApiRequest(string method) =>
@@ -333,7 +400,13 @@ public static class ScadaBridgeTelemetry
return;
}
var previous = _dataConnectionStates.TryGetValue(connectionName, out var prior) ? prior : (ConnectionHealth?)null;
_dataConnectionStates[connectionName] = state;
if (previous != state)
{
_dataConnectionTransitions.Add(1, new KeyValuePair<string, object?>("state", state.ToString()));
}
}
/// <summary>