using Akka.TestKit.Xunit2; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Options; using NSubstitute; using ZB.MOM.WW.ScadaBridge.Commons.Entities.Instances; using ZB.MOM.WW.ScadaBridge.Commons.Entities.Sites; using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories; using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums; using ZB.MOM.WW.ScadaBridge.Communication; using ZB.MOM.WW.ScadaBridge.Communication.Grpc; namespace ZB.MOM.WW.ScadaBridge.Communication.Tests; /// /// Tests for (plan #10, Task 4) — the DI singleton /// that reference-counts Alarm Summary viewers over one shared per-site aggregator. Covers /// shared start (one aggregator for many viewers), last-out linger stop, and re-subscribe /// cancelling a pending stop. Uses a real (in-memory) ActorSystem and a mock gRPC factory /// whose site-wide stream never faults, so no reconnect noise. /// public class SiteAlarmLiveCacheServiceTests : TestKit { private const int SiteId = 7; // A mock site-wide alarm stream client whose subscription hangs until cancelled. private sealed class HangingClient : SiteStreamGrpcClient { private readonly object _lock = new(); private readonly List _connects = new(); private readonly List _completions = new(); public HangingClient() : base() { } /// When false the site never accepts the subscription (no connect signal). public bool AutoConnect { get; set; } = true; /// Connect callbacks captured from every subscribe, for manual firing. public List Connects { get { lock (_lock) { return _connects.ToList(); } } } /// Graceful-completion callbacks captured from every subscribe. public List Completions { get { lock (_lock) { return _completions.ToList(); } } } public override Task SubscribeSiteAsync( string correlationId, Action onAlarmEvent, Action onError, Action onCompleted, CancellationToken ct, Action? onConnected = null) { lock (_lock) { if (onConnected is not null) _connects.Add(onConnected); _completions.Add(onCompleted); } // A healthy site accepts the subscription immediately (headers flushed on // subscribe), which is what makes the aggregator report the stream live. if (AutoConnect) onConnected?.Invoke(); var tcs = new TaskCompletionSource(); ct.Register(() => tcs.TrySetResult()); return tcs.Task; } public override void Unsubscribe(string correlationId) { } } private sealed class CountingFactory : SiteStreamGrpcClientFactory { private readonly HangingClient _client = new(); public int GetOrCreateCount; /// The single client this factory hands out, for connect/complete control. public HangingClient Client => _client; public CountingFactory() : base(NullLoggerFactory.Instance) { } public override SiteStreamGrpcClient GetOrCreate(string siteIdentifier, string grpcEndpoint) { Interlocked.Increment(ref GetOrCreateCount); return _client; } public override SiteStreamGrpcClient? TryGet(string siteIdentifier, string grpcEndpoint) => _client; } private SiteAlarmLiveCacheService CreateService(TimeSpan linger, out CountingFactory factory, int maxSubscribersPerSite = 200, IReadOnlyList? enabledInstances = null) { // Site with gRPC addresses, and NO enabled instances → the seed fan-out returns // empty immediately (so IsLive flips true fast without any snapshot Asks). var site = new Site("Alpha", "site-alpha") { Id = SiteId, GrpcNodeAAddress = "http://localhost:5100", GrpcNodeBAddress = "http://localhost:5200" }; var siteRepo = Substitute.For(); siteRepo.GetSiteByIdAsync(SiteId, Arg.Any()).Returns(site); var instanceRepo = Substitute.For(); instanceRepo.GetInstancesBySiteIdAsync(SiteId, Arg.Any()) .Returns((IReadOnlyList)(enabledInstances ?? new List())); var services = new ServiceCollection(); services.AddScoped(_ => siteRepo); services.AddScoped(_ => instanceRepo); var provider = services.BuildServiceProvider(); var options = Options.Create(new CommunicationOptions { LiveAlarmCacheLinger = linger, LiveAlarmCacheReconcileInterval = TimeSpan.FromMinutes(10), LiveAlarmCacheMaxSubscribersPerSite = maxSubscribersPerSite }); var comm = new CommunicationService(Options.Create(new CommunicationOptions()), NullLogger.Instance); factory = new CountingFactory(); var service = new SiteAlarmLiveCacheService( provider, comm, factory, options, NullLogger.Instance); service.SetActorSystem(Sys); return service; } [Fact] public void Seed_FanOut_Publishes_The_Instances_That_Failed_To_Answer() { // Arch-review phase-2 residual #4: the seed/reconcile fan-out already knows which // Enabled instances failed to answer and used to discard it, forcing the Alarm Summary // page to run a SECOND identical fan-out purely to rebuild that list. It is now // published alongside the snapshot. var instances = new List { new("inst-b") { Id = 2, SiteId = SiteId, State = InstanceState.Enabled }, new("inst-a") { Id = 1, SiteId = SiteId, State = InstanceState.Enabled }, // Disabled instances are never fanned out, so they can never be "not reporting". new("inst-off") { Id = 3, SiteId = SiteId, State = InstanceState.Disabled }, }; // The CommunicationService has no site actor wired, so every snapshot Ask faults — // which is exactly the "instance did not answer" case. var service = CreateService(TimeSpan.FromMilliseconds(200), out _, enabledInstances: instances); using var sub = service.Subscribe(SiteId, () => { }); AwaitCondition(() => service.IsLive(SiteId), TimeSpan.FromSeconds(5)); AwaitCondition( () => service.GetNotReportingInstances(SiteId).Count == 2, TimeSpan.FromSeconds(5)); // Ordered by name (ordinal-ignore-case), matching AlarmSummaryService's poll output so // the page renders identically whichever source supplied the list. Assert.Equal(new[] { "inst-a", "inst-b" }, service.GetNotReportingInstances(SiteId)); } [Fact] public void NotReporting_Is_Empty_For_An_Unknown_Site() { var service = CreateService(TimeSpan.FromMilliseconds(200), out _); Assert.Empty(service.GetNotReportingInstances(SiteId)); } [Fact] public void First_Subscriber_Starts_One_Aggregator_Shared_By_Multiple_Viewers() { var service = CreateService(TimeSpan.FromMilliseconds(200), out var factory); var changes1 = 0; var changes2 = 0; using var sub1 = service.Subscribe(SiteId, () => Interlocked.Increment(ref changes1)); using var sub2 = service.Subscribe(SiteId, () => Interlocked.Increment(ref changes2)); // The (empty) seed completes → aggregator publishes → live. AwaitCondition(() => service.IsLive(SiteId), TimeSpan.FromSeconds(5)); // Exactly ONE gRPC stream opened for the site regardless of viewer count. Assert.Equal(1, factory.GetOrCreateCount); // Empty seed → empty current snapshot. Assert.Empty(service.GetCurrentAlarms(SiteId)); // Both viewers were notified of the initial publish. AwaitCondition(() => changes1 >= 1 && changes2 >= 1, TimeSpan.FromSeconds(3)); } [Fact] public void Last_Viewer_Leaving_Stops_Aggregator_After_Linger() { var service = CreateService(TimeSpan.FromMilliseconds(200), out _); var sub1 = service.Subscribe(SiteId, () => { }); var sub2 = service.Subscribe(SiteId, () => { }); AwaitCondition(() => service.IsLive(SiteId), TimeSpan.FromSeconds(5)); sub1.Dispose(); // One viewer remains — still live, not stopped. Thread.Sleep(300); Assert.True(service.IsLive(SiteId)); sub2.Dispose(); // Last viewer left → after the linger the aggregator is torn down and the entry cleared. AwaitCondition(() => !service.IsLive(SiteId), TimeSpan.FromSeconds(3)); Assert.Empty(service.GetCurrentAlarms(SiteId)); } [Fact] public void Resubscribe_Within_Linger_Cancels_The_Pending_Stop() { var service = CreateService(TimeSpan.FromMilliseconds(400), out var factory); var sub1 = service.Subscribe(SiteId, () => { }); AwaitCondition(() => service.IsLive(SiteId), TimeSpan.FromSeconds(5)); sub1.Dispose(); // schedules linger stop var sub2 = service.Subscribe(SiteId, () => { }); // returns within the linger window → cancels it // Wait comfortably past the original linger; the aggregator must still be alive. Thread.Sleep(700); Assert.True(service.IsLive(SiteId)); // No second aggregator/stream was opened — the same one was kept warm. Assert.Equal(1, factory.GetOrCreateCount); sub2.Dispose(); } [Fact] public void Dispose_Is_Idempotent() { var service = CreateService(TimeSpan.FromMilliseconds(200), out _); var sub = service.Subscribe(SiteId, () => { }); AwaitCondition(() => service.IsLive(SiteId), TimeSpan.FromSeconds(5)); sub.Dispose(); sub.Dispose(); // must not throw or double-decrement AwaitCondition(() => !service.IsLive(SiteId), TimeSpan.FromSeconds(3)); } [Fact] public void Transient_Start_Failure_Self_Heals_On_Retry_For_A_Stable_Viewer_Cohort() { // A transient failure resolving the site at first-subscribe must NOT leave the // viewer cohort polling forever — a bounded retry re-attempts and self-heals. var site = new Site("Alpha", "site-alpha") { Id = SiteId, GrpcNodeAAddress = "http://localhost:5100", GrpcNodeBAddress = "http://localhost:5200" }; var siteRepo = Substitute.For(); // First resolve returns null (transient blip); every resolve thereafter succeeds. siteRepo.GetSiteByIdAsync(SiteId, Arg.Any()).Returns((Site?)null, site); var instanceRepo = Substitute.For(); instanceRepo.GetInstancesBySiteIdAsync(SiteId, Arg.Any()) .Returns(new List()); var services = new ServiceCollection(); services.AddScoped(_ => siteRepo); services.AddScoped(_ => instanceRepo); var provider = services.BuildServiceProvider(); var options = Options.Create(new CommunicationOptions { LiveAlarmCacheLinger = TimeSpan.FromSeconds(30), // Retry cadence reuses the reconcile interval — keep it short for the test. LiveAlarmCacheReconcileInterval = TimeSpan.FromMilliseconds(300) }); var comm = new CommunicationService(Options.Create(new CommunicationOptions()), NullLogger.Instance); var factory = new CountingFactory(); var service = new SiteAlarmLiveCacheService( provider, comm, factory, options, NullLogger.Instance); service.SetActorSystem(Sys); using var sub = service.Subscribe(SiteId, () => { }); // First start failed (site not found); a stable single viewer stays subscribed. // The retry eventually resolves the site and the aggregator goes live — without // any further Subscribe call. AwaitCondition(() => service.IsLive(SiteId), TimeSpan.FromSeconds(5)); } [Fact] public void Viewer_Over_The_Per_Site_Cap_Is_Rejected_Safely_And_Not_Counted() { // Cap of 2: the first two viewers register; a third is rejected fail-safe with a // no-op handle (never throws into the render path, never grows the list). Proof it // was NOT counted: disposing only the two real viewers tears the aggregator down // after the linger — if the rejected viewer had been registered, one subscriber // would remain and the site would stay live forever. var service = CreateService(TimeSpan.FromMilliseconds(200), out _, maxSubscribersPerSite: 2); var sub1 = service.Subscribe(SiteId, () => { }); var sub2 = service.Subscribe(SiteId, () => { }); AwaitCondition(() => service.IsLive(SiteId), TimeSpan.FromSeconds(5)); // Third viewer is over the cap. var overflow = service.Subscribe(SiteId, () => { }); Assert.NotNull(overflow); // Disposing the rejected handle is a genuine no-op — safe, idempotent, touches nothing. overflow.Dispose(); overflow.Dispose(); // Still live with the two real viewers present. Assert.True(service.IsLive(SiteId)); sub1.Dispose(); sub2.Dispose(); // Both real viewers gone → after the linger the aggregator stops. This only holds // if the overflow viewer was never added to the subscriber list. AwaitCondition(() => !service.IsLive(SiteId), TimeSpan.FromSeconds(3)); } [Fact] public void Unknown_Site_Registers_Viewer_But_Never_Goes_Live() { var service = CreateService(TimeSpan.FromMilliseconds(200), out _); using var sub = service.Subscribe(999, () => { }); // no such site in the repo Thread.Sleep(300); Assert.False(service.IsLive(999)); Assert.Empty(service.GetCurrentAlarms(999)); } // ── R2 T15: single-endpoint sites can go live (N9) ── [Fact] public void SiteWithOnlyNodeAEndpoint_StartsAnAggregator_AgainstThatEndpoint() { var site = new Site("Three", "site-3") { Id = 3, GrpcNodeAAddress = "http://only-node:8083", GrpcNodeBAddress = null, // single-endpoint site — pre-fix: never goes live }; var siteRepo = Substitute.For(); siteRepo.GetSiteByIdAsync(3, Arg.Any()).Returns(site); var instanceRepo = Substitute.For(); instanceRepo.GetInstancesBySiteIdAsync(3, Arg.Any()) .Returns(new List()); var services = new ServiceCollection(); services.AddScoped(_ => siteRepo); services.AddScoped(_ => instanceRepo); var provider = services.BuildServiceProvider(); var options = Options.Create(new CommunicationOptions { LiveAlarmCacheLinger = TimeSpan.FromSeconds(30), LiveAlarmCacheReconcileInterval = TimeSpan.FromMinutes(10), }); var comm = new CommunicationService(Options.Create(new CommunicationOptions()), NullLogger.Instance); var factory = new CountingFactory(); var service = new SiteAlarmLiveCacheService( provider, comm, factory, options, NullLogger.Instance); service.SetActorSystem(Sys); using var sub = service.Subscribe(3, () => { }); // Pre-fix: ResolveSiteAsync required BOTH endpoints and bailed → never live, no client. AwaitCondition(() => service.IsLive(3), TimeSpan.FromSeconds(5)); Assert.True(factory.GetOrCreateCount >= 1); // a client was created for the single endpoint } // ── R2 T10: deathwatch un-sticks IsLive on aggregator death (N6) ── [Fact] public async Task Aggregator_Death_Resets_IsLive_And_Clears_The_Snapshot() { var service = CreateService(TimeSpan.FromMinutes(5), out _); using var sub = service.Subscribe(SiteId, () => { }); AwaitAssert(() => Assert.True(service.IsLive(SiteId)), TimeSpan.FromSeconds(10)); // Kill the aggregator out from under the service (crash-death, NOT a linger stop). var aggregator = await Sys.ActorSelection($"/user/site-alarm-aggregator-{SiteId}-*") .ResolveOne(TimeSpan.FromSeconds(5)); Sys.Stop(aggregator); AwaitAssert(() => { Assert.False(service.IsLive(SiteId)); // sticky no more (R2 N6) Assert.Empty(service.GetCurrentAlarms(SiteId)); // frozen snapshot never grafted }, TimeSpan.FromSeconds(10)); } [Fact] public void Linger_Stop_Does_Not_Resurrect_The_Aggregator() { // Last viewer leaves, linger fires, entry removed: the deathwatch on the // deliberately-stopped actor must be a no-op (entry gone), not a restart. var service = CreateService(TimeSpan.FromMilliseconds(50), out var factory); var sub = service.Subscribe(SiteId, () => { }); AwaitAssert(() => Assert.True(service.IsLive(SiteId)), TimeSpan.FromSeconds(10)); sub.Dispose(); AwaitAssert(() => Assert.False(service.IsLive(SiteId)), TimeSpan.FromSeconds(10)); var startsAfterStop = factory.GetOrCreateCount; Thread.Sleep(300); Assert.Equal(startsAfterStop, factory.GetOrCreateCount); // no self-heal restart fired } // ── WP2.3: IsLive tracks the STREAM, not merely "a snapshot was published once" ── [Fact] public void IsLive_StaysFalse_UntilTheSiteAcceptsTheStream() { var service = CreateService(TimeSpan.FromMilliseconds(200), out var factory); factory.Client.AutoConnect = false; // site never answers the subscription using var sub = service.Subscribe(SiteId, () => { }); // The (empty) seed completes and publishes, but with no accepted stream behind it // the cache is not live — the page must keep polling. Pre-fix IsLive flipped true // here and the page grafted a never-updating snapshot over fresh poll data. AwaitCondition(() => service.GetCurrentAlarms(SiteId) is not null, TimeSpan.FromSeconds(5)); Thread.Sleep(400); Assert.False(service.IsLive(SiteId)); // The site accepts it → live. AwaitCondition(() => factory.Client.Connects.Count >= 1, TimeSpan.FromSeconds(5)); factory.Client.Connects[0](); AwaitCondition(() => service.IsLive(SiteId), TimeSpan.FromSeconds(5)); } [Fact] public void IsLive_DropsWhenTheStreamCompletes() { // The carried residual from Phase 1: a stream that ended gracefully (the site's 4h // max lifetime) left IsLive true until the next reconcile publish. var service = CreateService(TimeSpan.FromMilliseconds(200), out var factory); using var sub = service.Subscribe(SiteId, () => { }); AwaitCondition(() => service.IsLive(SiteId), TimeSpan.FromSeconds(5)); AwaitCondition(() => factory.Client.Completions.Count >= 1, TimeSpan.FromSeconds(5)); factory.Client.Completions[0](); AwaitCondition(() => !service.IsLive(SiteId), TimeSpan.FromSeconds(5)); } }