feat(telemetry): deploy-failure, S&F activity, script-run and DCL-transition instruments (ScadaBridge#81)

Adds four instruments on the already-allow-listed ZB.MOM.WW.ScadaBridge meter:
scadabridge.deployments.failed{reason}, scadabridge.store_and_forward.activity{action,category},
scadabridge.site.script.runs{outcome} + scadabridge.site.script.duration{outcome} (ms),
scadabridge.site.data_connections.transitions{state}. Queue depth, DCL connection state and
deploy-applied already existed. No new Meter, so ZbTelemetryOptions.Meters needs no change.

Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV

src/ZB.MOM.WW.ScadaBridge.Commons/Observability/ScadaBridgeTelemetry.cs

@@ -33,6 +33,45 @@ public static class ScadaBridgeTelemetry
         Meter.CreateCounter<long>("scadabridge.deployments.applied", unit: "1",
             description: "Deployments applied.");
 
+    /// <summary>
+    /// Incremented for each deployment that ends in a terminal failure (ScadaBridge#81), tagged
+    /// <c>reason</c>: <c>timeout</c> (the site round-trip timed out), <c>error</c> (any other fault
+    /// in the send or post-send persistence path) or <c>site_failed</c> (the site answered with a
+    /// non-Success status). Pairs with <c>scadabridge.deployments.applied</c> to give the deploy
+    /// success ratio; the rate-limited log lines cannot be alerted on.
+    /// </summary>
+    private static readonly Counter<long> _deploymentsFailed =
+        Meter.CreateCounter<long>("scadabridge.deployments.failed", unit: "1",
+            description: "Deployments that ended in a terminal failure, tagged by reason (timeout|error|site_failed).");
+
+    /// <summary>
+    /// Incremented for each store-and-forward lifecycle event (ScadaBridge#81), tagged <c>action</c>
+    /// (<c>Queued</c>, <c>Delivered</c>, <c>Retried</c>, <c>Parked</c>, <c>Retry</c>, <c>Discard</c>)
+    /// and <c>category</c>. A sustained <c>Parked</c> rate is the alertable signal that messages are
+    /// exhausting retries; the queue-depth gauge alone cannot distinguish a drain from a park.
+    /// </summary>
+    private static readonly Counter<long> _sfActivity =
+        Meter.CreateCounter<long>("scadabridge.store_and_forward.activity", unit: "1",
+            description: "Store-and-forward lifecycle events, tagged by action and category.");
+
+    /// <summary>
+    /// Incremented for each finished site script run (ScadaBridge#81), tagged <c>outcome</c>
+    /// (<c>completed</c> | <c>failed</c> | <c>timed_out</c>). Deliberately NOT tagged by instance or
+    /// script name: that cardinality is unbounded across a site's instances.
+    /// </summary>
+    private static readonly Counter<long> _scriptRuns =
+        Meter.CreateCounter<long>("scadabridge.site.script.runs", unit: "1",
+            description: "Finished site script runs, tagged by outcome (completed|failed|timed_out).");
+
+    /// <summary>
+    /// Incremented each time a data connection actually CHANGES state (ScadaBridge#81), tagged
+    /// <c>state</c> (the new <see cref="ConnectionHealth"/>). The data_connections gauges show the
+    /// current picture; this shows flapping that a scrape interval would otherwise average away.
+    /// </summary>
+    private static readonly Counter<long> _dataConnectionTransitions =
+        Meter.CreateCounter<long>("scadabridge.site.data_connections.transitions", unit: "1",
+            description: "Data connection state changes, tagged by the new state.");
+
     /// <summary>Incremented for each inbound API request, tagged with the API method.</summary>
     private static readonly Counter<long> _inboundApiRequests =
         Meter.CreateCounter<long>("scadabridge.inbound_api.requests", unit: "1",
@@ -151,6 +190,11 @@ public static class ScadaBridgeTelemetry
         Meter.CreateHistogram<int>("scadabridge.site.stream.batch_size", unit: "1",
             description: "Events per site gRPC stream frame (1 = unbatched frame), tagged by stream kind.");
 
+    /// <summary>Duration of finished site script runs in milliseconds (ScadaBridge#81), tagged by outcome.</summary>
+    private static readonly Histogram<long> _scriptRunDuration =
+        Meter.CreateHistogram<long>("scadabridge.site.script.duration", unit: "ms",
+            description: "Site script run duration in milliseconds, tagged by outcome (completed|failed|timed_out).");
+
     // ---------------- Observable gauges ----------------
 
     /// <summary>
@@ -271,6 +315,29 @@ public static class ScadaBridgeTelemetry
     /// <summary>Records that a deployment was applied.</summary>
     public static void RecordDeploymentApplied() => _deploymentsApplied.Add(1);
 
+    /// <summary>Records one deployment that ended in a terminal failure (ScadaBridge#81).</summary>
+    /// <param name="reason">One of <c>timeout</c>, <c>error</c>, <c>site_failed</c>.</param>
+    public static void RecordDeploymentFailed(string reason) =>
+        _deploymentsFailed.Add(1, new KeyValuePair<string, object?>("reason", reason));
+
+    /// <summary>Records one store-and-forward lifecycle event (ScadaBridge#81).</summary>
+    /// <param name="action">The activity action, e.g. <c>Parked</c>.</param>
+    /// <param name="category">The message category name.</param>
+    public static void RecordStoreAndForwardActivity(string action, string category) =>
+        _sfActivity.Add(1,
+            new KeyValuePair<string, object?>("action", action),
+            new KeyValuePair<string, object?>("category", category));
+
+    /// <summary>Records one finished site script run and its duration (ScadaBridge#81).</summary>
+    /// <param name="outcome">One of <c>completed</c>, <c>failed</c>, <c>timed_out</c>.</param>
+    /// <param name="durationMs">The run duration in milliseconds.</param>
+    public static void RecordScriptRun(string outcome, long durationMs)
+    {
+        var tag = new KeyValuePair<string, object?>("outcome", outcome);
+        _scriptRuns.Add(1, tag);
+        _scriptRunDuration.Record(durationMs < 0 ? 0 : durationMs, tag);
+    }
+
     /// <summary>Records an inbound API request for the given <paramref name="method"/>.</summary>
     /// <param name="method">The API method the request targeted.</param>
     public static void RecordInboundApiRequest(string method) =>
@@ -333,7 +400,13 @@ public static class ScadaBridgeTelemetry
             return;
         }
 
+        var previous = _dataConnectionStates.TryGetValue(connectionName, out var prior) ? prior : (ConnectionHealth?)null;
         _dataConnectionStates[connectionName] = state;
+
+        if (previous != state)
+        {
+            _dataConnectionTransitions.Add(1, new KeyValuePair<string, object?>("state", state.ToString()));
+        }
     }
 
     /// <summary>

src/ZB.MOM.WW.ScadaBridge.DeploymentManager/DeploymentService.cs

@@ -631,6 +631,11 @@ public class DeploymentService
             await _repository.SaveChangesAsync(cancellationToken);
             NotifyStatusChange(record);
 
+            if (response.Status != DeploymentStatus.Success)
+            {
+                ScadaBridgeTelemetry.RecordDeploymentFailed("site_failed");
+            }
+
             if (response.Status == DeploymentStatus.Success)
             {
                 // Telemetry: one instance deployment successfully applied to a
@@ -721,6 +726,7 @@ public class DeploymentService
         string user,
         Exception cause)
     {
+        ScadaBridgeTelemetry.RecordDeploymentFailed(IsTimeoutFault(cause) ? "timeout" : "error");
         record.Status = DeploymentStatus.Failed;
         record.ErrorMessage = IsTimeoutFault(cause)
             ? $"{TimeoutFailurePrefix} {cause.Message}"

src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Scripts/ScriptRunSummaryRecorder.cs

@@ -1,6 +1,7 @@
 using System.Collections.Concurrent;
 using System.Globalization;
 using System.Text.Json;
+using ZB.MOM.WW.ScadaBridge.Commons.Observability;
 using ZB.MOM.WW.ScadaBridge.SiteEventLogging;
 
 namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts;
@@ -70,6 +71,7 @@ public sealed class ScriptRunSummaryRecorder
     {
         var counters = GetOrAddCounters(instanceName, scriptName);
         Interlocked.Increment(ref counters.Completed);
+        ScadaBridgeTelemetry.RecordScriptRun("completed", durationMs);
         RecordDuration(counters, durationMs);
     }
 
@@ -81,6 +83,7 @@ public sealed class ScriptRunSummaryRecorder
     {
         var counters = GetOrAddCounters(instanceName, scriptName);
         Interlocked.Increment(ref counters.Failed);
+        ScadaBridgeTelemetry.RecordScriptRun("failed", durationMs);
         RecordDuration(counters, durationMs);
     }
 
@@ -95,6 +98,7 @@ public sealed class ScriptRunSummaryRecorder
     {
         var counters = GetOrAddCounters(instanceName, scriptName);
         Interlocked.Increment(ref counters.TimedOut);
+        ScadaBridgeTelemetry.RecordScriptRun("timed_out", durationMs);
         RecordDuration(counters, durationMs);
     }
 

src/ZB.MOM.WW.ScadaBridge.StoreAndForward/StoreAndForwardService.cs

@@ -1347,6 +1347,8 @@ public class StoreAndForwardService
     /// </summary>
     private void RaiseActivity(string action, StoreAndForwardCategory category, string detail)
     {
+        ScadaBridgeTelemetry.RecordStoreAndForwardActivity(action, category.ToString());
+
         var handlers = OnActivity;
         if (handlers == null) return;