perf(misc): cached hot-path lookups, bounded observer queue, alarm-priority stream path

WP2.6 (arch-review remediation, cross-cutting misc): - SiteExternalSystemRepository: name/ID-indexed ExternalSystemDefinitionCache replaces the fetch-all + reverse-map scan on every by-ID/method lookup; loaded once per redeploy, invalidated by DeploymentManagerActor after HandleDeployArtifacts applies external-system changes. Static JsonSerializerOptions for method-list parsing. - Inbound API: short-TTL ApiMethodCache fronts the per-request ApiMethod repository fetch; invalidated by name via the existing ScriptArtifactChangeSubscriber/ IScriptArtifactChangeBus pipeline, self-healing via TTL for changes the bus doesn't cover (e.g. Management API edits). - StoreAndForward: the cached-call audit-observer queue — the one unbounded channel left in the system — is now bounded (ObserverQueueCapacity, default 10,000) with DropOldest overflow and a dropped-notification counter. - SiteStreamManager: alarm state changes now travel a dedicated publish source/broadcast hub, isolated from the (far higher-volume) attribute path, so an attribute storm can no longer evict a pending alarm transition; the alarm hand-off queue is bounded with a drop counter surfaced on the site health report (SiteStreamAlarmDropCount via the new SiteStreamAlarmDropReporter), and publishing is skipped entirely at zero subscribers on either path. - CLI ManagementHttpClient: explicit 30s HttpClient.Timeout on the shared construction (was the 100s framework default), overridable via SCADABRIDGE_HTTP_TIMEOUT_SECONDS. Deviation: the failback-probe heartbeat item is NOT included — its only viable surface (CentralChannelProvider.cs / heartbeat consumers) lives entirely in the Communication project, explicitly off-limits to this work package this phase. Tests: SiteRuntime.Tests (550), InboundAPI.Tests (278), StoreAndForward.Tests (133), CLI.Tests (390), HealthMonitoring.Tests (97) — all green after full solution build.
2026-08-14 20:59:43 -04:00
parent ee193cd2bb
commit a212283104
32 changed files with 1361 additions and 111 deletions
@@ -1,4 +1,6 @@
 using Microsoft.Extensions.Logging.Abstractions;
+using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
+using ZB.MOM.WW.ScadaBridge.Commons.Types;
 using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
 using ZB.MOM.WW.ScadaBridge.TestSupport;

@@ -885,4 +887,82 @@ public class StoreAndForwardServiceTests : IAsyncLifetime, IDisposable
        release.TrySetResult();
        await stop.WaitAsync(TimeSpan.FromSeconds(5)); // drains the real sweep promptly once released
    }
+
+    // ── WP2.6c: bounded, DropOldest observer queue + drop counter ──
+
+    private sealed class BlockingObserver : ICachedCallLifecycleObserver
+    {
+        private readonly TaskCompletionSource _gate;
+        public BlockingObserver(TaskCompletionSource gate) => _gate = gate;
+        public async Task OnAttemptCompletedAsync(CachedCallAttemptContext context, CancellationToken ct = default)
+            => await _gate.Task;
+    }
+
+    /// <summary>
+    /// WP2.6c: the cached-call audit-observer queue is bounded — once the single-reader
+    /// pump is stuck awaiting a slow/stuck observer, further posted notifications must
+    /// evict the oldest queued one (DropOldest) instead of growing without bound, and
+    /// every eviction must increment <see cref="StoreAndForwardService.ObserverQueueDroppedCount"/>.
+    /// </summary>
+    [Fact]
+    public async Task ObserverQueue_BoundedCapacity_DropsOldestAndCountsDrops()
+    {
+        var gate = new TaskCompletionSource();
+        var observer = new BlockingObserver(gate);
+
+        var localDb = TestLocalDb.CreateTemp("ObsQueueBound");
+        var storage = new StoreAndForwardStorage(localDb.Db, NullLogger<StoreAndForwardStorage>.Instance);
+        await storage.InitializeAsync();
+        var service = new StoreAndForwardService(
+            storage,
+            new StoreAndForwardOptions
+            {
+                DefaultRetryInterval = TimeSpan.Zero,
+                DefaultMaxRetries = 5,
+                RetryTimerInterval = TimeSpan.FromHours(1), // timer never fires in-test
+                ObserverQueueCapacity = 2,
+            },
+            NullLogger<StoreAndForwardService>.Instance,
+            cachedCallObserver: observer,
+            siteId: "site-77");
+
+        await service.StartAsync();
+        try
+        {
+            service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem,
+                _ => throw new HttpRequestException("transient"));
+
+            // Enqueue more messages than the bounded capacity (2) — the pump dequeues
+            // the first notification and blocks on the observer gate, so every
+            // subsequent notification posted during this sweep queues (and, past
+            // capacity, evicts the oldest still-queued one) rather than being
+            // processed.
+            for (var i = 0; i < 6; i++)
+            {
+                await service.EnqueueAsync(
+                    StoreAndForwardCategory.ExternalSystem, $"t{i}", "{}",
+                    attemptImmediateDelivery: false, retryInterval: TimeSpan.Zero,
+                    messageId: TrackedOperationId.New().ToString());
+            }
+
+            await service.RetryPendingMessagesAsync();
+
+            // Give the bounded channel a moment to have absorbed/evicted every post
+            // (the pump itself stays blocked on the gate throughout).
+            var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(5);
+            while (DateTime.UtcNow < deadline && service.ObserverQueueDroppedCount == 0)
+                await Task.Delay(10);
+
+            Assert.True(service.ObserverQueueDroppedCount > 0,
+                "expected at least one notification to be dropped once the bounded queue filled");
+        }
+        finally
+        {
+            gate.TrySetResult();
+            await service.StopAsync();
+            var path = localDb.Path;
+            localDb.Dispose();
+            TestLocalDb.DeleteFiles(path);
+        }
+    }
 }