perf(misc): cached hot-path lookups, bounded observer queue, alarm-priority stream path

WP2.6 (arch-review remediation, cross-cutting misc): - SiteExternalSystemRepository: name/ID-indexed ExternalSystemDefinitionCache replaces the fetch-all + reverse-map scan on every by-ID/method lookup; loaded once per redeploy, invalidated by DeploymentManagerActor after HandleDeployArtifacts applies external-system changes. Static JsonSerializerOptions for method-list parsing. - Inbound API: short-TTL ApiMethodCache fronts the per-request ApiMethod repository fetch; invalidated by name via the existing ScriptArtifactChangeSubscriber/ IScriptArtifactChangeBus pipeline, self-healing via TTL for changes the bus doesn't cover (e.g. Management API edits). - StoreAndForward: the cached-call audit-observer queue — the one unbounded channel left in the system — is now bounded (ObserverQueueCapacity, default 10,000) with DropOldest overflow and a dropped-notification counter. - SiteStreamManager: alarm state changes now travel a dedicated publish source/broadcast hub, isolated from the (far higher-volume) attribute path, so an attribute storm can no longer evict a pending alarm transition; the alarm hand-off queue is bounded with a drop counter surfaced on the site health report (SiteStreamAlarmDropCount via the new SiteStreamAlarmDropReporter), and publishing is skipped entirely at zero subscribers on either path. - CLI ManagementHttpClient: explicit 30s HttpClient.Timeout on the shared construction (was the 100s framework default), overridable via SCADABRIDGE_HTTP_TIMEOUT_SECONDS. Deviation: the failback-probe heartbeat item is NOT included — its only viable surface (CentralChannelProvider.cs / heartbeat consumers) lives entirely in the Communication project, explicitly off-limits to this work package this phase. Tests: SiteRuntime.Tests (550), InboundAPI.Tests (278), StoreAndForward.Tests (133), CLI.Tests (390), HealthMonitoring.Tests (97) — all green after full solution build.
2026-08-14 20:59:43 -04:00
parent ee193cd2bb
commit a212283104
32 changed files with 1361 additions and 111 deletions
@@ -0,0 +1,57 @@
+using Akka.TestKit.Xunit2;
+using Microsoft.Extensions.Logging.Abstractions;
+using ZB.MOM.WW.ScadaBridge.HealthMonitoring;
+using ZB.MOM.WW.ScadaBridge.SiteRuntime.Streaming;
+
+namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Streaming;
+
+/// <summary>
+/// WP2.6d: the hosted reporter must lift <see cref="SiteStreamManager.AlarmPublishDroppedCount"/>
+/// onto the site health report via <see cref="ISiteHealthCollector"/>. Uses the real
+/// collector (mirrors <c>ScriptSchedulerStatsReporterTests</c> — NSubstitute is not
+/// referenced by this test project). The queue's own drop-counting mechanism is verified
+/// deterministically and separately by
+/// <see cref="Streaming.SiteStreamManagerTests.CreateAlarmPublishQueue_OverCapacity_DropsOldestAndInvokesCallback"/>;
+/// this test proves the reporter's poll-and-push wiring runs correctly.
+/// </summary>
+public class SiteStreamAlarmDropReporterTests : TestKit, IDisposable
+{
+    void IDisposable.Dispose() => Shutdown();
+
+    [Fact]
+    public async Task Reporter_PushesAlarmDropCountToCollector()
+    {
+        var options = new SiteRuntimeOptions { StreamBufferSize = 100 };
+        var collector = new SiteHealthCollector();
+        var streamManager = new SiteStreamManager(options, NullLogger<SiteStreamManager>.Instance);
+        streamManager.Initialize(Sys);
+
+        using var reporter = new SiteStreamAlarmDropReporter(
+            collector, streamManager, NullLogger<SiteStreamAlarmDropReporter>.Instance,
+            pollInterval: TimeSpan.FromMilliseconds(50));
+
+        await reporter.StartAsync(CancellationToken.None);
+        try
+        {
+            // The immediate first probe (no drops yet) must reach the report as 0 —
+            // proving the reporter actually ran and pushed a value, not that the field
+            // simply defaulted.
+            await WaitUntilAsync(() =>
+                collector.CollectReport("site-1").SiteStreamAlarmDropCount == streamManager.AlarmPublishDroppedCount);
+
+            var report = collector.CollectReport("site-1");
+            Assert.Equal(streamManager.AlarmPublishDroppedCount, report.SiteStreamAlarmDropCount);
+        }
+        finally
+        {
+            await reporter.StopAsync(CancellationToken.None);
+        }
+    }
+
+    private static async Task WaitUntilAsync(Func<bool> condition)
+    {
+        for (var i = 0; i < 100 && !condition(); i++)
+            await Task.Delay(50);
+        Assert.True(condition(), "condition not met within timeout");
+    }
+}
@@ -159,4 +159,86 @@ public class SiteStreamManagerTests : TestKit, IDisposable
        _streamManager.RemoveSubscriber(probe.Ref);
        Assert.Equal(0, _streamManager.SubscriptionCount);
    }
+
+    // ── WP2.6d: separate alarm publish path, drop counter, skip-at-zero-subscribers ──
+
+    /// <summary>
+    /// WP2.6d: alarm state changes now travel a dedicated publish source, isolated from
+    /// the (far higher-volume) attribute path — a burst of attribute events for OTHER
+    /// instances, interleaved with an alarm-only subscriber's events, must not cause any
+    /// alarm to be lost.
+    /// </summary>
+    [Fact]
+    public void PublishAlarmStateChanged_SurvivesConcurrentAttributeStorm_ForUnrelatedInstances()
+    {
+        var alarmProbe = CreateTestProbe();
+        _streamManager.SubscribeSiteAlarms(alarmProbe.Ref);
+
+        // A storm of attribute events for a DIFFERENT instance — none of which the
+        // alarm subscriber is even listening to — interleaved with alarm events. Before
+        // WP2.6d these shared one upstream buffer; now they are fully separate sources.
+        for (var i = 0; i < 500; i++)
+        {
+            _streamManager.PublishAttributeValueChanged(new AttributeValueChanged(
+                "NoisyPump", "Temperature", "Temperature", i.ToString(), "Good", DateTimeOffset.UtcNow));
+        }
+
+        _streamManager.PublishAlarmStateChanged(new AlarmStateChanged(
+            "Pump1", "HighTemp", AlarmState.Active, 1, DateTimeOffset.UtcNow));
+
+        var received = alarmProbe.ExpectMsg<AlarmStateChanged>(TimeSpan.FromSeconds(3));
+        Assert.Equal("Pump1", received.InstanceUniqueName);
+    }
+
+    /// <summary>
+    /// WP2.6d: with zero subscribers of any kind, PublishAlarmStateChanged/
+    /// PublishAttributeValueChanged must be no-ops — no exception, and (for alarms) the
+    /// event never reaches the bounded hand-off queue, so
+    /// <see cref="SiteStreamManager.AlarmPublishDroppedCount"/> stays at zero rather than
+    /// counting events nobody could ever have received anyway.
+    /// </summary>
+    [Fact]
+    public void Publish_WithNoSubscribers_IsNoOp_AndDoesNotCountAsDropped()
+    {
+        _streamManager.PublishAlarmStateChanged(new AlarmStateChanged(
+            "Pump1", "HighTemp", AlarmState.Active, 1, DateTimeOffset.UtcNow));
+        _streamManager.PublishAttributeValueChanged(new AttributeValueChanged(
+            "Pump1", "Temperature", "Temperature", "1", "Good", DateTimeOffset.UtcNow));
+
+        Assert.Equal(0, _streamManager.AlarmPublishDroppedCount);
+
+        // Publishing resumes working normally once a subscriber exists.
+        var probe = CreateTestProbe();
+        _streamManager.SubscribeSiteAlarms(probe.Ref);
+        _streamManager.PublishAlarmStateChanged(new AlarmStateChanged(
+            "Pump1", "HighTemp", AlarmState.Active, 2, DateTimeOffset.UtcNow));
+
+        probe.ExpectMsg<AlarmStateChanged>(TimeSpan.FromSeconds(3));
+    }
+
+    /// <summary>
+    /// WP2.6d: once the bounded alarm hand-off queue (the exact factory
+    /// <see cref="SiteStreamManager.Initialize"/> wires into the drop-count callback) is
+    /// full, a further write evicts the oldest queued item and fires the drop callback.
+    /// Exercised directly against the queue factory — deterministic, no actor
+    /// system/async pump race involved (see the factory's own doc comment).
+    /// </summary>
+    [Fact]
+    public void CreateAlarmPublishQueue_OverCapacity_DropsOldestAndInvokesCallback()
+    {
+        var dropped = 0;
+        var queue = SiteStreamManager.CreateAlarmPublishQueue(capacity: 2, () => Interlocked.Increment(ref dropped));
+
+        // Nothing reads from this queue, so all five writes stay purely upstream —
+        // capacity 2 means the 3rd/4th/5th writes must each evict the oldest.
+        for (var i = 0; i < 5; i++)
+        {
+            var wrote = queue.Writer.TryWrite(new AlarmStateChanged(
+                "Pump1", "HighTemp", AlarmState.Active, i, DateTimeOffset.UtcNow));
+            Assert.True(wrote); // DropOldest TryWrite always succeeds once capacity > 0
+        }
+
+        Assert.Equal(3, dropped);
+        Assert.Equal(2, queue.Reader.Count);
+    }
 }