feat(telemetry): deploy-failure, S&F activity, script-run and DCL-transition instruments (ScadaBridge#81)

Adds four instruments on the already-allow-listed ZB.MOM.WW.ScadaBridge meter: scadabridge.deployments.failed{reason}, scadabridge.store_and_forward.activity{action,category}, scadabridge.site.script.runs{outcome} + scadabridge.site.script.duration{outcome} (ms), scadabridge.site.data_connections.transitions{state}. Queue depth, DCL connection state and deploy-applied already existed. No new Meter, so ZbTelemetryOptions.Meters needs no change. Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
2026-11-09 14:17:53 -05:00
parent 7892c622cf
commit 5ef27fd9a3
8 changed files with 301 additions and 0 deletions
@@ -0,0 +1,98 @@
+using System.Diagnostics.Metrics;
+using ZB.MOM.WW.ScadaBridge.Commons.Observability;
+using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
+
+namespace ZB.MOM.WW.ScadaBridge.Commons.Tests.Observability;
+
+/// <summary>
+/// ScadaBridge#81: the instruments added beyond the first set. Each is observed through a
+/// <see cref="MeterListener"/> filtered to the emitting (test) thread, because the meter is
+/// process-global and other tests emit on it concurrently.
+/// </summary>
+public sealed class ExtendedInstrumentTests
+{
+    private static List<(long Value, Dictionary<string, object?> Tags)> Capture(string instrument, Action act)
+    {
+        var tid = Environment.CurrentManagedThreadId;
+        var got = new List<(long, Dictionary<string, object?>)>();
+        using var listener = new MeterListener
+        {
+            InstrumentPublished = (i, l) =>
+            {
+                if (i.Meter.Name == ScadaBridgeTelemetry.MeterName && i.Name == instrument)
+                    l.EnableMeasurementEvents(i);
+            }
+        };
+        listener.SetMeasurementEventCallback<long>((_, m, tags, _) =>
+        {
+            if (Environment.CurrentManagedThreadId != tid) return;
+            var d = new Dictionary<string, object?>();
+            foreach (var t in tags) d[t.Key] = t.Value;
+            got.Add((m, d));
+        });
+        listener.Start();
+        act();
+        return got;
+    }
+
+    /// <summary>A state change emits one transition tagged with the new state; repeating the same state emits none.</summary>
+    [Fact]
+    public void DataConnectionTransitions_EmitOnlyOnRealStateChange()
+    {
+        var name = "xfer-" + Guid.NewGuid().ToString("N");
+        try
+        {
+            var got = Capture("scadabridge.site.data_connections.transitions", () =>
+            {
+                ScadaBridgeTelemetry.RecordDataConnectionState(name, ConnectionHealth.Connecting);
+                ScadaBridgeTelemetry.RecordDataConnectionState(name, ConnectionHealth.Connected);
+                ScadaBridgeTelemetry.RecordDataConnectionState(name, ConnectionHealth.Connected);
+                ScadaBridgeTelemetry.RecordDataConnectionState(name, ConnectionHealth.Disconnected);
+            });
+
+            Assert.Equal(
+                new object?[] { "Connecting", "Connected", "Disconnected" },
+                got.Select(g => g.Tags["state"]).ToArray());
+        }
+        finally { ScadaBridgeTelemetry.RemoveDataConnection(name); }
+    }
+
+    /// <summary>Store-and-forward activity is counted by action and category.</summary>
+    [Fact]
+    public void StoreAndForwardActivity_IsCountedByActionAndCategory()
+    {
+        var got = Capture("scadabridge.store_and_forward.activity",
+            () => ScadaBridgeTelemetry.RecordStoreAndForwardActivity("Parked", "ExternalSystem"));
+
+        var m = Assert.Single(got);
+        Assert.Equal(1, m.Value);
+        Assert.Equal("Parked", m.Tags["action"]);
+        Assert.Equal("ExternalSystem", m.Tags["category"]);
+    }
+
+    /// <summary>Deployment failures are counted by reason.</summary>
+    [Fact]
+    public void DeploymentsFailed_IsCountedByReason()
+    {
+        var got = Capture("scadabridge.deployments.failed",
+            () => ScadaBridgeTelemetry.RecordDeploymentFailed("timeout"));
+
+        var m = Assert.Single(got);
+        Assert.Equal("timeout", m.Tags["reason"]);
+    }
+
+    /// <summary>A script run emits both the run counter and the duration histogram, clamping negatives.</summary>
+    [Fact]
+    public void ScriptRun_EmitsCounterAndDuration()
+    {
+        var runs = Capture("scadabridge.site.script.runs",
+            () => ScadaBridgeTelemetry.RecordScriptRun("failed", 5));
+        Assert.Equal("failed", Assert.Single(runs).Tags["outcome"]);
+
+        var durations = Capture("scadabridge.site.script.duration",
+            () => ScadaBridgeTelemetry.RecordScriptRun("completed", -3));
+        var d = Assert.Single(durations);
+        Assert.Equal(0, d.Value);
+        Assert.Equal("completed", d.Tags["outcome"]);
+    }
+}
@@ -1063,6 +1063,45 @@ public class DeploymentServiceTests : TestKit
        Assert.Equal(1, Interlocked.Read(ref applied));
    }

+    /// <summary>Verifies a faulted deployment emits the deployments failed counter tagged reason=error (ScadaBridge#81).</summary>
+    /// <returns>A task that represents the asynchronous operation.</returns>
+    [Fact]
+    public async Task DeployInstanceAsync_Fault_EmitsDeploymentsFailedCounter()
+    {
+        var instance = new Instance("FailMetricInst") { Id = 1, SiteId = 1, State = InstanceState.NotDeployed };
+        _repo.GetInstanceByIdAsync(1).Returns(instance);
+        var config = new FlattenedConfiguration { InstanceUniqueName = "FailMetricInst" };
+        _pipeline.FlattenAndValidateAsync(1, Arg.Any<CancellationToken>())
+            .Returns(Result<FlatteningPipelineResult>.Success(
+                new FlatteningPipelineResult(config, "sha256:abc", ValidationResult.Success())));
+
+        var reasons = new System.Collections.Concurrent.ConcurrentQueue<string?>();
+        using var listener = new MeterListener
+        {
+            InstrumentPublished = (instrument, l) =>
+            {
+                if (instrument.Meter.Name == ScadaBridgeTelemetry.MeterName
+                    && instrument.Name == "scadabridge.deployments.failed")
+                {
+                    l.EnableMeasurementEvents(instrument);
+                }
+            }
+        };
+        listener.SetMeasurementEventCallback<long>((_, _, tags, _) =>
+        {
+            foreach (var t in tags)
+                if (t.Key == "reason") reasons.Enqueue(t.Value as string);
+        });
+        listener.Start();
+
+        // _comms has no actor set -> non-timeout fault -> MarkDeploymentFailedAsync.
+        var result = await _service.DeployInstanceAsync(1, "admin");
+        listener.Dispose();
+
+        Assert.True(result.IsFailure);
+        Assert.Contains("error", reasons);
+    }
+
    /// <summary>Verifies that deploy instance async reconciled does not emit deployments applied counter.</summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [Fact]
@@ -12,6 +12,48 @@ namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Scripts;
 /// </summary>
 public class ScriptRunSummaryRecorderTests
 {
+    /// <summary>Verifies each finished run emits the script runs counter and duration histogram with its outcome (ScadaBridge#81).</summary>
+    [Fact]
+    public void FinishedRuns_EmitRunCounterAndDurationHistogram_TaggedByOutcome()
+    {
+        // MeterListener callbacks run synchronously on the emitting thread, so filtering on the
+        // test thread isolates this test from other tests emitting on the same process-wide meter.
+        var tid = Environment.CurrentManagedThreadId;
+        var runs = new List<string?>();
+        var durations = new List<(string? Outcome, long Ms)>();
+        using var listener = new System.Diagnostics.Metrics.MeterListener
+        {
+            InstrumentPublished = (i, l) =>
+            {
+                if (i.Meter.Name == Commons.Observability.ScadaBridgeTelemetry.MeterName
+                    && (i.Name == "scadabridge.site.script.runs" || i.Name == "scadabridge.site.script.duration"))
+                    l.EnableMeasurementEvents(i);
+            }
+        };
+        static string? Outcome(ReadOnlySpan<KeyValuePair<string, object?>> tags)
+        {
+            foreach (var t in tags) if (t.Key == "outcome") return t.Value as string;
+            return null;
+        }
+        listener.SetMeasurementEventCallback<long>((inst, m, tags, _) =>
+        {
+            if (Environment.CurrentManagedThreadId != tid) return;
+            if (inst.Name == "scadabridge.site.script.runs") runs.Add(Outcome(tags));
+            else durations.Add((Outcome(tags), m));
+        });
+        listener.Start();
+
+        var recorder = new ScriptRunSummaryRecorder();
+        recorder.RecordCompleted("i", "s", 10);
+        recorder.RecordFailed("i", "s", 20);
+        recorder.RecordTimedOut("i", "s", 30);
+
+        Assert.Equal(new string?[] { "completed", "failed", "timed_out" }, runs);
+        Assert.Equal(
+            new (string?, long)[] { ("completed", 10), ("failed", 20), ("timed_out", 30) },
+            durations);
+    }
+
    /// <summary>Verifies that flushing with no activity emits no row.</summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [Fact]
@@ -97,6 +97,43 @@ public class QueueDepthGaugeTests : IAsyncLifetime, IDisposable
        return observed;
    }

+    /// <summary>
+    /// ScadaBridge#81: the service emits <c>scadabridge.store_and_forward.activity</c> for a buffered
+    /// message (Queued) and for a message that exhausts its retries (Parked), through the real service.
+    /// </summary>
+    /// <returns>A task that represents the asynchronous operation.</returns>
+    [Fact]
+    public async Task ActivityCounter_IsEmitted_ForQueuedAndParked()
+    {
+        var seen = new System.Collections.Concurrent.ConcurrentQueue<string>();
+        using var listener = new MeterListener
+        {
+            InstrumentPublished = (i, l) =>
+            {
+                if (i.Meter.Name == ScadaBridgeTelemetry.MeterName
+                    && i.Name == "scadabridge.store_and_forward.activity")
+                    l.EnableMeasurementEvents(i);
+            }
+        };
+        listener.SetMeasurementEventCallback<long>((_, _, tags, _) =>
+        {
+            foreach (var t in tags)
+                if (t.Key == "action" && t.Value is string a) seen.Enqueue(a);
+        });
+        listener.Start();
+
+        _service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem,
+            _ => throw new HttpRequestException("transient"));
+
+        var r = await _service.EnqueueAsync(
+            StoreAndForwardCategory.ExternalSystem, "api-activity", """{}""", maxRetries: 1);
+        Assert.True(r.WasBuffered);
+        await _service.RetryPendingMessagesAsync();
+
+        Assert.Contains("Queued", seen);
+        Assert.Contains("Parked", seen);
+    }
+
    /// <summary>
    /// Review 2026-10-21 02/C3: rows that arrive without passing through this node's counter
    /// paths (LocalDb CDC on a standby) are picked up by the sweep's periodic re-seed.