feat(sitestream): validate live-alarm-cache options + active-aggregator/reconnect telemetry (plan #10 T6)

Extend CommunicationOptionsValidator with eager bounds for the four T4 live-alarm-cache options (linger >= 0, reconcile > 0, seed concurrency 1..64, subscribers-per-site >= 1). Enforce the per-site viewer cap fail-safe in SiteAlarmLiveCacheService.Subscribe (reject excess viewers with a no-op disposable rather than growing the list or throwing into the Blazor render path). Surface two telemetry instruments on the existing ScadaBridgeTelemetry meter: an active-aggregator observable gauge and a reconnect counter, wired from the aggregator actor's PreStart/PostStop and its NodeA<->NodeB flip / reconcile-driven reopen. Claude-Session: https://claude.ai/code/session_01MtdgwpEeCUn6cUA5f1LMPj
2026-07-10 12:33:40 -04:00
parent b91ed3c840
commit c4f97d0e87
8 changed files with 263 additions and 5 deletions
@@ -36,4 +36,54 @@ public class CommunicationOptionsValidatorTests
        Assert.True(result.Failed);
        Assert.Contains("GrpcMaxConcurrentStreams", result.FailureMessage);
    }
+
+    // ── Aggregated live alarm cache options (plan #10, Task 6) ───────────────────
+
+    [Fact]
+    public void ZeroLiveAlarmCacheLinger_IsValid()
+    {
+        // Zero linger = stop the aggregator immediately when the last viewer leaves.
+        var result = Validate(new CommunicationOptions { LiveAlarmCacheLinger = TimeSpan.Zero });
+        Assert.True(result.Succeeded, result.FailureMessage);
+    }
+
+    [Fact]
+    public void NegativeLiveAlarmCacheLinger_IsRejected()
+    {
+        var result = Validate(new CommunicationOptions { LiveAlarmCacheLinger = TimeSpan.FromSeconds(-1) });
+        Assert.True(result.Failed);
+        Assert.Contains("LiveAlarmCacheLinger", result.FailureMessage);
+    }
+
+    [Fact]
+    public void ZeroLiveAlarmCacheReconcileInterval_IsRejected()
+    {
+        var result = Validate(new CommunicationOptions { LiveAlarmCacheReconcileInterval = TimeSpan.Zero });
+        Assert.True(result.Failed);
+        Assert.Contains("LiveAlarmCacheReconcileInterval", result.FailureMessage);
+    }
+
+    [Fact]
+    public void ZeroLiveAlarmCacheSeedConcurrency_IsRejected()
+    {
+        var result = Validate(new CommunicationOptions { LiveAlarmCacheSeedConcurrency = 0 });
+        Assert.True(result.Failed);
+        Assert.Contains("LiveAlarmCacheSeedConcurrency", result.FailureMessage);
+    }
+
+    [Fact]
+    public void ExcessiveLiveAlarmCacheSeedConcurrency_IsRejected()
+    {
+        var result = Validate(new CommunicationOptions { LiveAlarmCacheSeedConcurrency = 65 });
+        Assert.True(result.Failed);
+        Assert.Contains("LiveAlarmCacheSeedConcurrency", result.FailureMessage);
+    }
+
+    [Fact]
+    public void ZeroLiveAlarmCacheMaxSubscribersPerSite_IsRejected()
+    {
+        var result = Validate(new CommunicationOptions { LiveAlarmCacheMaxSubscribersPerSite = 0 });
+        Assert.True(result.Failed);
+        Assert.Contains("LiveAlarmCacheMaxSubscribersPerSite", result.FailureMessage);
+    }
 }
@@ -54,7 +54,8 @@ public class SiteAlarmLiveCacheServiceTests : TestKit
        public override SiteStreamGrpcClient? TryGet(string siteIdentifier, string grpcEndpoint) => _client;
    }

-    private SiteAlarmLiveCacheService CreateService(TimeSpan linger, out CountingFactory factory)
+    private SiteAlarmLiveCacheService CreateService(TimeSpan linger, out CountingFactory factory,
+        int maxSubscribersPerSite = 200)
    {
        // Site with gRPC addresses, and NO enabled instances → the seed fan-out returns
        // empty immediately (so IsLive flips true fast without any snapshot Asks).
@@ -80,7 +81,8 @@ public class SiteAlarmLiveCacheServiceTests : TestKit
        var options = Options.Create(new CommunicationOptions
        {
            LiveAlarmCacheLinger = linger,
-            LiveAlarmCacheReconcileInterval = TimeSpan.FromMinutes(10)
+            LiveAlarmCacheReconcileInterval = TimeSpan.FromMinutes(10),
+            LiveAlarmCacheMaxSubscribersPerSite = maxSubscribersPerSite
        });

        var comm = new CommunicationService(Options.Create(new CommunicationOptions()),
@@ -214,6 +216,38 @@ public class SiteAlarmLiveCacheServiceTests : TestKit
        AwaitCondition(() => service.IsLive(SiteId), TimeSpan.FromSeconds(5));
    }

+    [Fact]
+    public void Viewer_Over_The_Per_Site_Cap_Is_Rejected_Safely_And_Not_Counted()
+    {
+        // Cap of 2: the first two viewers register; a third is rejected fail-safe with a
+        // no-op handle (never throws into the render path, never grows the list). Proof it
+        // was NOT counted: disposing only the two real viewers tears the aggregator down
+        // after the linger — if the rejected viewer had been registered, one subscriber
+        // would remain and the site would stay live forever.
+        var service = CreateService(TimeSpan.FromMilliseconds(200), out _, maxSubscribersPerSite: 2);
+
+        var sub1 = service.Subscribe(SiteId, () => { });
+        var sub2 = service.Subscribe(SiteId, () => { });
+        AwaitCondition(() => service.IsLive(SiteId), TimeSpan.FromSeconds(5));
+
+        // Third viewer is over the cap.
+        var overflow = service.Subscribe(SiteId, () => { });
+        Assert.NotNull(overflow);
+        // Disposing the rejected handle is a genuine no-op — safe, idempotent, touches nothing.
+        overflow.Dispose();
+        overflow.Dispose();
+
+        // Still live with the two real viewers present.
+        Assert.True(service.IsLive(SiteId));
+
+        sub1.Dispose();
+        sub2.Dispose();
+
+        // Both real viewers gone → after the linger the aggregator stops. This only holds
+        // if the overflow viewer was never added to the subscriber list.
+        AwaitCondition(() => !service.IsLive(SiteId), TimeSpan.FromSeconds(3));
+    }
+
    [Fact]
    public void Unknown_Site_Registers_Viewer_But_Never_Goes_Live()
    {
@@ -0,0 +1,94 @@
+using System.Diagnostics.Metrics;
+using ZB.MOM.WW.ScadaBridge.Commons.Observability;
+
+namespace ZB.MOM.WW.ScadaBridge.Communication.Tests;
+
+/// <summary>
+/// Telemetry guards for the aggregated live alarm cache (plan #10, Task 6): the
+/// active-aggregator observable gauge reflects Started/Stopped, and the reconnect
+/// counter increments on each recorded reconnect. Mirrors the MeterListener read
+/// pattern used by <c>SiteStreamGrpcServerTests.SiteConnectionUpGauge_*</c>; reads are
+/// relative to a baseline so the process-wide static instruments are robust to any
+/// parallel test interleaving.
+/// </summary>
+public class SiteAlarmLiveCacheTelemetryTests
+{
+    private static long ReadGauge(string instrumentName)
+    {
+        long observed = 0;
+        using var listener = new MeterListener();
+        listener.InstrumentPublished = (instrument, l) =>
+        {
+            if (instrument.Meter.Name == ScadaBridgeTelemetry.MeterName &&
+                instrument.Name == instrumentName)
+            {
+                l.EnableMeasurementEvents(instrument);
+            }
+        };
+        listener.SetMeasurementEventCallback<long>((_, measurement, _, _) => observed = measurement);
+        listener.Start();
+        listener.RecordObservableInstruments();
+        return observed;
+    }
+
+    private static long ReadCounter(string instrumentName, Action act)
+    {
+        long delta = 0;
+        using var listener = new MeterListener();
+        listener.InstrumentPublished = (instrument, l) =>
+        {
+            if (instrument.Meter.Name == ScadaBridgeTelemetry.MeterName &&
+                instrument.Name == instrumentName)
+            {
+                l.EnableMeasurementEvents(instrument);
+            }
+        };
+        listener.SetMeasurementEventCallback<long>((_, measurement, _, _) => delta += measurement);
+        listener.Start();
+        act();
+        return delta;
+    }
+
+    [Fact]
+    public void ActiveAggregatorGauge_ReflectsStartAndStop()
+    {
+        const string gauge = "scadabridge.site.alarm_cache.aggregators.active";
+        var baseline = ReadGauge(gauge);
+
+        ScadaBridgeTelemetry.LiveAlarmAggregatorStarted();
+        try
+        {
+            Assert.Equal(baseline + 1, ReadGauge(gauge));
+
+            ScadaBridgeTelemetry.LiveAlarmAggregatorStarted();
+            try
+            {
+                Assert.Equal(baseline + 2, ReadGauge(gauge));
+            }
+            finally
+            {
+                ScadaBridgeTelemetry.LiveAlarmAggregatorStopped();
+            }
+
+            Assert.Equal(baseline + 1, ReadGauge(gauge));
+        }
+        finally
+        {
+            ScadaBridgeTelemetry.LiveAlarmAggregatorStopped();
+        }
+
+        Assert.Equal(baseline, ReadGauge(gauge));
+    }
+
+    [Fact]
+    public void ReconnectCounter_IncrementsOnEachRecordedReconnect()
+    {
+        var delta = ReadCounter("scadabridge.site.alarm_cache.reconnects", () =>
+        {
+            ScadaBridgeTelemetry.RecordLiveAlarmStreamReconnect();
+            ScadaBridgeTelemetry.RecordLiveAlarmStreamReconnect();
+        });
+
+        Assert.Equal(2, delta);
+    }
+}