feat(sitestream): central transient per-site live alarm cache w/ seed-then-stream + failover (plan #10 T4)

Adds the active-central-node, in-memory, reference-counted per-site live alarm cache backing the operator Alarm Summary page. No persisted central alarm store ([PERM]) — no EF entity/table/migration/DbSet. - ISiteAlarmLiveCache (new): singleton seam — Subscribe(siteId, onChanged) -> IDisposable (ref-counted, linger stop on last-out), GetCurrentAlarms, IsLive. - SiteAlarmAggregatorActor (new): one per site. Seed-then-stream ordering copied from DebugStreamBridgeActor — open the site-wide alarm-only gRPC stream first, buffer live deltas while the snapshot fan-out runs, flush with per-key dedup (AlarmKey = InstanceUniqueName|AlarmName|SourceReference), then live pass-through into an in-memory dict. Placeholder rows seeded from the snapshot and never expected on the stream; a real-alarm delta (distinct key) never wipes them. NodeA<->NodeB reconnect (retry budget + stability window), reconnect re-seeds, periodic reconcile (authoritative clear-and-rebuild) corrects instance-set drift, and a budget-exhausted stream self-heals on the next reconcile tick. - SiteAlarmLiveCacheService (new): DI singleton facade — viewer reference-counting, linger-delayed last-out stop (version + TryRemove(ref) race guards), the snapshot fan-out seed (RequestDebugSnapshotAsync per Enabled instance, capped), bounded start-retry self-heal on transient start failure, and the immutable published-snapshot store the page reads. Cache mutated only on the actor thread; viewer callbacks invoked outside the lock. - CommunicationOptions: LiveAlarmCacheLinger (30s), LiveAlarmCacheReconcileInterval (60s), LiveAlarmCacheSeedConcurrency (8), LiveAlarmCacheMaxSubscribersPerSite (200). Task 6 formalizes eager validation + telemetry. - DI registration + AkkaHostedService SetActorSystem wiring on the active central node. - Tests: 14 actor (seed/stream ordering, dedup, native-alarm parity, placeholder coherence, reconnect re-seed, reconcile replace, self-heal, stop) + 6 service (shared start, linger stop, resubscribe cancels stop, idempotent dispose, unknown site, transient-start self-heal). Code-reviewer pass: no persistence, no Critical; both Important findings addressed. Claude-Session: https://claude.ai/code/session_01MtdgwpEeCUn6cUA5f1LMPj
2026-07-10 12:18:02 -04:00
parent 6d5ecc92a5
commit 696a4ffea2
8 changed files with 1739 additions and 0 deletions
@@ -0,0 +1,227 @@
+using Akka.TestKit.Xunit2;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Logging.Abstractions;
+using Microsoft.Extensions.Options;
+using NSubstitute;
+using ZB.MOM.WW.ScadaBridge.Commons.Entities.Instances;
+using ZB.MOM.WW.ScadaBridge.Commons.Entities.Sites;
+using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
+using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
+using ZB.MOM.WW.ScadaBridge.Communication;
+using ZB.MOM.WW.ScadaBridge.Communication.Grpc;
+
+namespace ZB.MOM.WW.ScadaBridge.Communication.Tests;
+
+/// <summary>
+/// Tests for <see cref="SiteAlarmLiveCacheService"/> (plan #10, Task 4) — the DI singleton
+/// that reference-counts Alarm Summary viewers over one shared per-site aggregator. Covers
+/// shared start (one aggregator for many viewers), last-out linger stop, and re-subscribe
+/// cancelling a pending stop. Uses a real (in-memory) ActorSystem and a mock gRPC factory
+/// whose site-wide stream never faults, so no reconnect noise.
+/// </summary>
+public class SiteAlarmLiveCacheServiceTests : TestKit
+{
+    private const int SiteId = 7;
+
+    // A mock site-wide alarm stream client whose subscription hangs until cancelled.
+    private sealed class HangingClient : SiteStreamGrpcClient
+    {
+        public HangingClient() : base() { }
+        public override Task SubscribeSiteAsync(
+            string correlationId, Action<Commons.Messages.Streaming.AlarmStateChanged> onAlarmEvent,
+            Action<Exception> onError, CancellationToken ct)
+        {
+            var tcs = new TaskCompletionSource();
+            ct.Register(() => tcs.TrySetResult());
+            return tcs.Task;
+        }
+        public override void Unsubscribe(string correlationId) { }
+    }
+
+    private sealed class CountingFactory : SiteStreamGrpcClientFactory
+    {
+        private readonly HangingClient _client = new();
+        public int GetOrCreateCount;
+
+        public CountingFactory() : base(NullLoggerFactory.Instance) { }
+
+        public override SiteStreamGrpcClient GetOrCreate(string siteIdentifier, string grpcEndpoint)
+        {
+            Interlocked.Increment(ref GetOrCreateCount);
+            return _client;
+        }
+
+        public override SiteStreamGrpcClient? TryGet(string siteIdentifier, string grpcEndpoint) => _client;
+    }
+
+    private SiteAlarmLiveCacheService CreateService(TimeSpan linger, out CountingFactory factory)
+    {
+        // Site with gRPC addresses, and NO enabled instances → the seed fan-out returns
+        // empty immediately (so IsLive flips true fast without any snapshot Asks).
+        var site = new Site("Alpha", "site-alpha")
+        {
+            Id = SiteId,
+            GrpcNodeAAddress = "http://localhost:5100",
+            GrpcNodeBAddress = "http://localhost:5200"
+        };
+
+        var siteRepo = Substitute.For<ISiteRepository>();
+        siteRepo.GetSiteByIdAsync(SiteId, Arg.Any<CancellationToken>()).Returns(site);
+
+        var instanceRepo = Substitute.For<ITemplateEngineRepository>();
+        instanceRepo.GetInstancesBySiteIdAsync(SiteId, Arg.Any<CancellationToken>())
+            .Returns(new List<Instance>());
+
+        var services = new ServiceCollection();
+        services.AddScoped(_ => siteRepo);
+        services.AddScoped(_ => instanceRepo);
+        var provider = services.BuildServiceProvider();
+
+        var options = Options.Create(new CommunicationOptions
+        {
+            LiveAlarmCacheLinger = linger,
+            LiveAlarmCacheReconcileInterval = TimeSpan.FromMinutes(10)
+        });
+
+        var comm = new CommunicationService(Options.Create(new CommunicationOptions()),
+            NullLogger<CommunicationService>.Instance);
+
+        factory = new CountingFactory();
+        var service = new SiteAlarmLiveCacheService(
+            provider, comm, factory, options, NullLogger<SiteAlarmLiveCacheService>.Instance);
+        service.SetActorSystem(Sys);
+        return service;
+    }
+
+    [Fact]
+    public void First_Subscriber_Starts_One_Aggregator_Shared_By_Multiple_Viewers()
+    {
+        var service = CreateService(TimeSpan.FromMilliseconds(200), out var factory);
+
+        var changes1 = 0;
+        var changes2 = 0;
+        using var sub1 = service.Subscribe(SiteId, () => Interlocked.Increment(ref changes1));
+        using var sub2 = service.Subscribe(SiteId, () => Interlocked.Increment(ref changes2));
+
+        // The (empty) seed completes → aggregator publishes → live.
+        AwaitCondition(() => service.IsLive(SiteId), TimeSpan.FromSeconds(5));
+
+        // Exactly ONE gRPC stream opened for the site regardless of viewer count.
+        Assert.Equal(1, factory.GetOrCreateCount);
+        // Empty seed → empty current snapshot.
+        Assert.Empty(service.GetCurrentAlarms(SiteId));
+        // Both viewers were notified of the initial publish.
+        AwaitCondition(() => changes1 >= 1 && changes2 >= 1, TimeSpan.FromSeconds(3));
+    }
+
+    [Fact]
+    public void Last_Viewer_Leaving_Stops_Aggregator_After_Linger()
+    {
+        var service = CreateService(TimeSpan.FromMilliseconds(200), out _);
+
+        var sub1 = service.Subscribe(SiteId, () => { });
+        var sub2 = service.Subscribe(SiteId, () => { });
+        AwaitCondition(() => service.IsLive(SiteId), TimeSpan.FromSeconds(5));
+
+        sub1.Dispose();
+        // One viewer remains — still live, not stopped.
+        Thread.Sleep(300);
+        Assert.True(service.IsLive(SiteId));
+
+        sub2.Dispose();
+        // Last viewer left → after the linger the aggregator is torn down and the entry cleared.
+        AwaitCondition(() => !service.IsLive(SiteId), TimeSpan.FromSeconds(3));
+        Assert.Empty(service.GetCurrentAlarms(SiteId));
+    }
+
+    [Fact]
+    public void Resubscribe_Within_Linger_Cancels_The_Pending_Stop()
+    {
+        var service = CreateService(TimeSpan.FromMilliseconds(400), out var factory);
+
+        var sub1 = service.Subscribe(SiteId, () => { });
+        AwaitCondition(() => service.IsLive(SiteId), TimeSpan.FromSeconds(5));
+
+        sub1.Dispose();                 // schedules linger stop
+        var sub2 = service.Subscribe(SiteId, () => { }); // returns within the linger window → cancels it
+
+        // Wait comfortably past the original linger; the aggregator must still be alive.
+        Thread.Sleep(700);
+        Assert.True(service.IsLive(SiteId));
+        // No second aggregator/stream was opened — the same one was kept warm.
+        Assert.Equal(1, factory.GetOrCreateCount);
+
+        sub2.Dispose();
+    }
+
+    [Fact]
+    public void Dispose_Is_Idempotent()
+    {
+        var service = CreateService(TimeSpan.FromMilliseconds(200), out _);
+        var sub = service.Subscribe(SiteId, () => { });
+        AwaitCondition(() => service.IsLive(SiteId), TimeSpan.FromSeconds(5));
+
+        sub.Dispose();
+        sub.Dispose(); // must not throw or double-decrement
+
+        AwaitCondition(() => !service.IsLive(SiteId), TimeSpan.FromSeconds(3));
+    }
+
+    [Fact]
+    public void Transient_Start_Failure_Self_Heals_On_Retry_For_A_Stable_Viewer_Cohort()
+    {
+        // A transient failure resolving the site at first-subscribe must NOT leave the
+        // viewer cohort polling forever — a bounded retry re-attempts and self-heals.
+        var site = new Site("Alpha", "site-alpha")
+        {
+            Id = SiteId,
+            GrpcNodeAAddress = "http://localhost:5100",
+            GrpcNodeBAddress = "http://localhost:5200"
+        };
+
+        var siteRepo = Substitute.For<ISiteRepository>();
+        // First resolve returns null (transient blip); every resolve thereafter succeeds.
+        siteRepo.GetSiteByIdAsync(SiteId, Arg.Any<CancellationToken>()).Returns((Site?)null, site);
+
+        var instanceRepo = Substitute.For<ITemplateEngineRepository>();
+        instanceRepo.GetInstancesBySiteIdAsync(SiteId, Arg.Any<CancellationToken>())
+            .Returns(new List<Instance>());
+
+        var services = new ServiceCollection();
+        services.AddScoped(_ => siteRepo);
+        services.AddScoped(_ => instanceRepo);
+        var provider = services.BuildServiceProvider();
+
+        var options = Options.Create(new CommunicationOptions
+        {
+            LiveAlarmCacheLinger = TimeSpan.FromSeconds(30),
+            // Retry cadence reuses the reconcile interval — keep it short for the test.
+            LiveAlarmCacheReconcileInterval = TimeSpan.FromMilliseconds(300)
+        });
+        var comm = new CommunicationService(Options.Create(new CommunicationOptions()),
+            NullLogger<CommunicationService>.Instance);
+        var factory = new CountingFactory();
+
+        var service = new SiteAlarmLiveCacheService(
+            provider, comm, factory, options, NullLogger<SiteAlarmLiveCacheService>.Instance);
+        service.SetActorSystem(Sys);
+
+        using var sub = service.Subscribe(SiteId, () => { });
+
+        // First start failed (site not found); a stable single viewer stays subscribed.
+        // The retry eventually resolves the site and the aggregator goes live — without
+        // any further Subscribe call.
+        AwaitCondition(() => service.IsLive(SiteId), TimeSpan.FromSeconds(5));
+    }
+
+    [Fact]
+    public void Unknown_Site_Registers_Viewer_But_Never_Goes_Live()
+    {
+        var service = CreateService(TimeSpan.FromMilliseconds(200), out _);
+
+        using var sub = service.Subscribe(999, () => { }); // no such site in the repo
+        Thread.Sleep(300);
+        Assert.False(service.IsLive(999));
+        Assert.Empty(service.GetCurrentAlarms(999));
+    }
+}