ScadaBridge/tests/ZB.MOM.WW.ScadaBridge.Communication.Tests/SiteAlarmLiveCacheServiceTests.cs

using Akka.TestKit.Xunit2;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using NSubstitute;
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Instances;
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Sites;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
using ZB.MOM.WW.ScadaBridge.Communication;
using ZB.MOM.WW.ScadaBridge.Communication.Grpc;

namespace ZB.MOM.WW.ScadaBridge.Communication.Tests;

/// <summary>
/// Tests for <see cref="SiteAlarmLiveCacheService"/> (plan #10, Task 4) — the DI singleton
/// that reference-counts Alarm Summary viewers over one shared per-site aggregator. Covers
/// shared start (one aggregator for many viewers), last-out linger stop, and re-subscribe
/// cancelling a pending stop. Uses a real (in-memory) ActorSystem and a mock gRPC factory
/// whose site-wide stream never faults, so no reconnect noise.
/// </summary>
public class SiteAlarmLiveCacheServiceTests : TestKit
{
    private const int SiteId = 7;

    // A mock site-wide alarm stream client whose subscription hangs until cancelled.
    private sealed class HangingClient : SiteStreamGrpcClient
    {
        private readonly object _lock = new();
        private readonly List<Action> _connects = new();
        private readonly List<Action> _completions = new();

        public HangingClient() : base() { }

        /// <summary>When false the site never accepts the subscription (no connect signal).</summary>
        public bool AutoConnect { get; set; } = true;

        /// <summary>Connect callbacks captured from every subscribe, for manual firing.</summary>
        public List<Action> Connects { get { lock (_lock) { return _connects.ToList(); } } }

        /// <summary>Graceful-completion callbacks captured from every subscribe.</summary>
        public List<Action> Completions { get { lock (_lock) { return _completions.ToList(); } } }

        private int _streamEnds;

        /// <summary>
        /// Register row 40: how many site-wide subscriptions have been torn down (their
        /// cancellation token fired). This is the aggregator-stop observable — the linger
        /// stop cancels the aggregator's token, which ends the hanging subscribe below.
        /// Tests that must show an aggregator was NOT stopped assert on this rather than
        /// sampling <c>IsLive</c> once after a fixed sleep.
        /// </summary>
        public int StreamEndCount => Volatile.Read(ref _streamEnds);

        public override Task SubscribeSiteAsync(
            string correlationId, Action<Commons.Messages.Streaming.AlarmStateChanged> onAlarmEvent,
            Action<Exception> onError, Action onCompleted, CancellationToken ct,
            Action? onConnected = null)
        {
            lock (_lock)
            {
                if (onConnected is not null) _connects.Add(onConnected);
                _completions.Add(onCompleted);
            }
            // A healthy site accepts the subscription immediately (headers flushed on
            // subscribe), which is what makes the aggregator report the stream live.
            if (AutoConnect) onConnected?.Invoke();
            var tcs = new TaskCompletionSource();
            ct.Register(() =>
            {
                Interlocked.Increment(ref _streamEnds);
                tcs.TrySetResult();
            });
            return tcs.Task;
        }
        public override void Unsubscribe(string correlationId) { }
    }

    private sealed class CountingFactory : SiteStreamGrpcClientFactory
    {
        private readonly HangingClient _client = new();
        public int GetOrCreateCount;

        /// <summary>The single client this factory hands out, for connect/complete control.</summary>
        public HangingClient Client => _client;

        public CountingFactory() : base(NullLoggerFactory.Instance) { }

        public override SiteStreamGrpcClient GetOrCreate(string siteIdentifier, string grpcEndpoint)
        {
            Interlocked.Increment(ref GetOrCreateCount);
            return _client;
        }

        public override SiteStreamGrpcClient? TryGet(string siteIdentifier, string grpcEndpoint) => _client;
    }

    private SiteAlarmLiveCacheService CreateService(TimeSpan linger, out CountingFactory factory,
        int maxSubscribersPerSite = 200, IReadOnlyList<Instance>? enabledInstances = null)
    {
        // Site with gRPC addresses, and NO enabled instances → the seed fan-out returns
        // empty immediately (so IsLive flips true fast without any snapshot Asks).
        var site = new Site("Alpha", "site-alpha")
        {
            Id = SiteId,
            GrpcNodeAAddress = "http://localhost:5100",
            GrpcNodeBAddress = "http://localhost:5200"
        };

        var siteRepo = Substitute.For<ISiteRepository>();
        siteRepo.GetSiteByIdAsync(SiteId, Arg.Any<CancellationToken>()).Returns(site);

        var instanceRepo = Substitute.For<ITemplateEngineRepository>();
        instanceRepo.GetInstancesBySiteIdAsync(SiteId, Arg.Any<CancellationToken>())
            .Returns((IReadOnlyList<Instance>)(enabledInstances ?? new List<Instance>()));

        var services = new ServiceCollection();
        services.AddScoped(_ => siteRepo);
        services.AddScoped(_ => instanceRepo);
        var provider = services.BuildServiceProvider();

        var options = Options.Create(new CommunicationOptions
        {
            LiveAlarmCacheLinger = linger,
            LiveAlarmCacheReconcileInterval = TimeSpan.FromMinutes(10),
            LiveAlarmCacheMaxSubscribersPerSite = maxSubscribersPerSite
        });

        var comm = new CommunicationService(Options.Create(new CommunicationOptions()),
            NullLogger<CommunicationService>.Instance);

        factory = new CountingFactory();
        var service = new SiteAlarmLiveCacheService(
            provider, comm, factory, options, NullLogger<SiteAlarmLiveCacheService>.Instance);
        service.SetActorSystem(Sys);
        return service;
    }

    [Fact]
    public void Seed_FanOut_Publishes_The_Instances_That_Failed_To_Answer()
    {
        // Arch-review phase-2 residual #4: the seed/reconcile fan-out already knows which
        // Enabled instances failed to answer and used to discard it, forcing the Alarm Summary
        // page to run a SECOND identical fan-out purely to rebuild that list. It is now
        // published alongside the snapshot.
        var instances = new List<Instance>
        {
            new("inst-b") { Id = 2, SiteId = SiteId, State = InstanceState.Enabled },
            new("inst-a") { Id = 1, SiteId = SiteId, State = InstanceState.Enabled },
            // Disabled instances are never fanned out, so they can never be "not reporting".
            new("inst-off") { Id = 3, SiteId = SiteId, State = InstanceState.Disabled },
        };

        // The CommunicationService has no site actor wired, so every snapshot Ask faults —
        // which is exactly the "instance did not answer" case.
        var service = CreateService(TimeSpan.FromMilliseconds(200), out _, enabledInstances: instances);

        using var sub = service.Subscribe(SiteId, () => { });
        AwaitCondition(() => service.IsLive(SiteId), TimeSpan.FromSeconds(5));

        AwaitCondition(
            () => service.GetNotReportingInstances(SiteId).Count == 2,
            TimeSpan.FromSeconds(5));

        // Ordered by name (ordinal-ignore-case), matching AlarmSummaryService's poll output so
        // the page renders identically whichever source supplied the list.
        Assert.Equal(new[] { "inst-a", "inst-b" }, service.GetNotReportingInstances(SiteId));
    }

    [Fact]
    public void NotReporting_Is_Empty_For_An_Unknown_Site()
    {
        var service = CreateService(TimeSpan.FromMilliseconds(200), out _);

        Assert.Empty(service.GetNotReportingInstances(SiteId));
    }

    [Fact]
    public void First_Subscriber_Starts_One_Aggregator_Shared_By_Multiple_Viewers()
    {
        var service = CreateService(TimeSpan.FromMilliseconds(200), out var factory);

        var changes1 = 0;
        var changes2 = 0;
        using var sub1 = service.Subscribe(SiteId, () => Interlocked.Increment(ref changes1));
        using var sub2 = service.Subscribe(SiteId, () => Interlocked.Increment(ref changes2));

        // The (empty) seed completes → aggregator publishes → live.
        AwaitCondition(() => service.IsLive(SiteId), TimeSpan.FromSeconds(5));

        // Exactly ONE gRPC stream opened for the site regardless of viewer count.
        Assert.Equal(1, factory.GetOrCreateCount);
        // Empty seed → empty current snapshot.
        Assert.Empty(service.GetCurrentAlarms(SiteId));
        // Both viewers were notified of the initial publish.
        AwaitCondition(() => changes1 >= 1 && changes2 >= 1, TimeSpan.FromSeconds(3));
    }

    [Fact]
    public void Last_Viewer_Leaving_Stops_Aggregator_After_Linger()
    {
        var linger = TimeSpan.FromMilliseconds(200);
        var service = CreateService(linger, out var factory);

        var sub1 = service.Subscribe(SiteId, () => { });
        var sub2 = service.Subscribe(SiteId, () => { });
        AwaitCondition(() => service.IsLive(SiteId), TimeSpan.FromSeconds(5));

        sub1.Dispose();

        // Register row 40: one viewer remains, so the aggregator must NOT be stopped. The old
        // test slept 300 ms and sampled IsLive once — a teardown that happened at any other
        // instant (or one that stopped and restarted the stream) was invisible to it. Poll
        // CONTINUOUSLY across three linger windows instead, bailing out the moment either
        // signal says the aggregator went away.
        var held = System.Diagnostics.Stopwatch.StartNew();
        AwaitCondition(
            () => held.Elapsed > linger * 3
                  || !service.IsLive(SiteId)
                  || factory.Client.StreamEndCount > 0,
            TimeSpan.FromSeconds(5), TimeSpan.FromMilliseconds(10));
        Assert.True(service.IsLive(SiteId),
            "the aggregator was reported not-live while a viewer was still subscribed");
        Assert.Equal(0, factory.Client.StreamEndCount);
        Assert.Equal(1, factory.GetOrCreateCount);

        sub2.Dispose();
        // Last viewer left → after the linger the aggregator is torn down and the entry cleared.
        AwaitCondition(() => !service.IsLive(SiteId), TimeSpan.FromSeconds(3));

        // Positive control on the SAME observable the negative above rests on: the site-wide
        // stream really is torn down now, so the zero above was a live signal rather than a
        // counter that never moves.
        AwaitCondition(() => factory.Client.StreamEndCount > 0, TimeSpan.FromSeconds(3));
        Assert.Empty(service.GetCurrentAlarms(SiteId));
    }

    [Fact]
    public void Resubscribe_Within_Linger_Cancels_The_Pending_Stop()
    {
        var linger = TimeSpan.FromMilliseconds(400);
        var service = CreateService(linger, out var factory);

        var sub1 = service.Subscribe(SiteId, () => { });
        AwaitCondition(() => service.IsLive(SiteId), TimeSpan.FromSeconds(5));

        sub1.Dispose();                 // schedules linger stop
        var sub2 = service.Subscribe(SiteId, () => { }); // returns within the linger window → cancels it

        // Register row 40: the pending stop must be CANCELLED, not merely outrun. The old test
        // slept 700 ms and sampled once — it could not see a stream that was torn down and
        // silently replaced inside that window, because GetOrCreateCount is checked only at the
        // end and the mock hands back the same client object either way. Poll continuously past
        // the original linger on the teardown observable itself.
        var kept = System.Diagnostics.Stopwatch.StartNew();
        AwaitCondition(
            () => kept.Elapsed > linger * 3
                  || !service.IsLive(SiteId)
                  || factory.Client.StreamEndCount > 0,
            TimeSpan.FromSeconds(5), TimeSpan.FromMilliseconds(10));
        Assert.True(service.IsLive(SiteId),
            "the re-subscribe did not cancel the pending linger stop");
        Assert.Equal(0, factory.Client.StreamEndCount);
        // No second aggregator/stream was opened — the same one was kept warm.
        Assert.Equal(1, factory.GetOrCreateCount);

        sub2.Dispose();

        // Positive control: with the last viewer gone the linger stop is allowed to run, so the
        // teardown counter this test's zero rests on does in fact move.
        AwaitCondition(() => factory.Client.StreamEndCount > 0, TimeSpan.FromSeconds(5));
    }

    [Fact]
    public void Dispose_Is_Idempotent()
    {
        var service = CreateService(TimeSpan.FromMilliseconds(200), out _);
        var sub = service.Subscribe(SiteId, () => { });
        AwaitCondition(() => service.IsLive(SiteId), TimeSpan.FromSeconds(5));

        sub.Dispose();
        sub.Dispose(); // must not throw or double-decrement

        AwaitCondition(() => !service.IsLive(SiteId), TimeSpan.FromSeconds(3));
    }

    [Fact]
    public void Transient_Start_Failure_Self_Heals_On_Retry_For_A_Stable_Viewer_Cohort()
    {
        // A transient failure resolving the site at first-subscribe must NOT leave the
        // viewer cohort polling forever — a bounded retry re-attempts and self-heals.
        var site = new Site("Alpha", "site-alpha")
        {
            Id = SiteId,
            GrpcNodeAAddress = "http://localhost:5100",
            GrpcNodeBAddress = "http://localhost:5200"
        };

        var siteRepo = Substitute.For<ISiteRepository>();
        // First resolve returns null (transient blip); every resolve thereafter succeeds.
        siteRepo.GetSiteByIdAsync(SiteId, Arg.Any<CancellationToken>()).Returns((Site?)null, site);

        var instanceRepo = Substitute.For<ITemplateEngineRepository>();
        instanceRepo.GetInstancesBySiteIdAsync(SiteId, Arg.Any<CancellationToken>())
            .Returns(new List<Instance>());

        var services = new ServiceCollection();
        services.AddScoped(_ => siteRepo);
        services.AddScoped(_ => instanceRepo);
        var provider = services.BuildServiceProvider();

        var options = Options.Create(new CommunicationOptions
        {
            LiveAlarmCacheLinger = TimeSpan.FromSeconds(30),
            // Retry cadence reuses the reconcile interval — keep it short for the test.
            LiveAlarmCacheReconcileInterval = TimeSpan.FromMilliseconds(300)
        });
        var comm = new CommunicationService(Options.Create(new CommunicationOptions()),
            NullLogger<CommunicationService>.Instance);
        var factory = new CountingFactory();

        var service = new SiteAlarmLiveCacheService(
            provider, comm, factory, options, NullLogger<SiteAlarmLiveCacheService>.Instance);
        service.SetActorSystem(Sys);

        using var sub = service.Subscribe(SiteId, () => { });

        // First start failed (site not found); a stable single viewer stays subscribed.
        // The retry eventually resolves the site and the aggregator goes live — without
        // any further Subscribe call.
        AwaitCondition(() => service.IsLive(SiteId), TimeSpan.FromSeconds(5));
    }

    [Fact]
    public void Viewer_Over_The_Per_Site_Cap_Is_Rejected_Safely_And_Not_Counted()
    {
        // Cap of 2: the first two viewers register; a third is rejected fail-safe with a
        // no-op handle (never throws into the render path, never grows the list). Proof it
        // was NOT counted: disposing only the two real viewers tears the aggregator down
        // after the linger — if the rejected viewer had been registered, one subscriber
        // would remain and the site would stay live forever.
        var service = CreateService(TimeSpan.FromMilliseconds(200), out _, maxSubscribersPerSite: 2);

        var sub1 = service.Subscribe(SiteId, () => { });
        var sub2 = service.Subscribe(SiteId, () => { });
        AwaitCondition(() => service.IsLive(SiteId), TimeSpan.FromSeconds(5));

        // Third viewer is over the cap.
        var overflow = service.Subscribe(SiteId, () => { });
        Assert.NotNull(overflow);
        // Disposing the rejected handle is a genuine no-op — safe, idempotent, touches nothing.
        overflow.Dispose();
        overflow.Dispose();

        // Still live with the two real viewers present.
        Assert.True(service.IsLive(SiteId));

        sub1.Dispose();
        sub2.Dispose();

        // Both real viewers gone → after the linger the aggregator stops. This only holds
        // if the overflow viewer was never added to the subscriber list.
        AwaitCondition(() => !service.IsLive(SiteId), TimeSpan.FromSeconds(3));
    }

    [Fact]
    public void Unknown_Site_Registers_Viewer_But_Never_Goes_Live()
    {
        var service = CreateService(TimeSpan.FromMilliseconds(200), out _);

        using var sub = service.Subscribe(999, () => { }); // no such site in the repo
        Thread.Sleep(300);
        Assert.False(service.IsLive(999));
        Assert.Empty(service.GetCurrentAlarms(999));
    }

    // ── R2 T15: single-endpoint sites can go live (N9) ──

    [Fact]
    public void SiteWithOnlyNodeAEndpoint_StartsAnAggregator_AgainstThatEndpoint()
    {
        var site = new Site("Three", "site-3")
        {
            Id = 3,
            GrpcNodeAAddress = "http://only-node:8083",
            GrpcNodeBAddress = null, // single-endpoint site — pre-fix: never goes live
        };

        var siteRepo = Substitute.For<ISiteRepository>();
        siteRepo.GetSiteByIdAsync(3, Arg.Any<CancellationToken>()).Returns(site);

        var instanceRepo = Substitute.For<ITemplateEngineRepository>();
        instanceRepo.GetInstancesBySiteIdAsync(3, Arg.Any<CancellationToken>())
            .Returns(new List<Instance>());

        var services = new ServiceCollection();
        services.AddScoped(_ => siteRepo);
        services.AddScoped(_ => instanceRepo);
        var provider = services.BuildServiceProvider();

        var options = Options.Create(new CommunicationOptions
        {
            LiveAlarmCacheLinger = TimeSpan.FromSeconds(30),
            LiveAlarmCacheReconcileInterval = TimeSpan.FromMinutes(10),
        });
        var comm = new CommunicationService(Options.Create(new CommunicationOptions()),
            NullLogger<CommunicationService>.Instance);
        var factory = new CountingFactory();

        var service = new SiteAlarmLiveCacheService(
            provider, comm, factory, options, NullLogger<SiteAlarmLiveCacheService>.Instance);
        service.SetActorSystem(Sys);

        using var sub = service.Subscribe(3, () => { });

        // Pre-fix: ResolveSiteAsync required BOTH endpoints and bailed → never live, no client.
        AwaitCondition(() => service.IsLive(3), TimeSpan.FromSeconds(5));
        Assert.True(factory.GetOrCreateCount >= 1); // a client was created for the single endpoint
    }

    // ── R2 T10: deathwatch un-sticks IsLive on aggregator death (N6) ──

    [Fact]
    public async Task Aggregator_Death_Resets_IsLive_And_Clears_The_Snapshot()
    {
        var service = CreateService(TimeSpan.FromMinutes(5), out _);
        using var sub = service.Subscribe(SiteId, () => { });
        AwaitAssert(() => Assert.True(service.IsLive(SiteId)), TimeSpan.FromSeconds(10));

        // Kill the aggregator out from under the service (crash-death, NOT a linger stop).
        var aggregator = await Sys.ActorSelection($"/user/site-alarm-aggregator-{SiteId}-*")
            .ResolveOne(TimeSpan.FromSeconds(5));
        Sys.Stop(aggregator);

        AwaitAssert(() =>
        {
            Assert.False(service.IsLive(SiteId));          // sticky no more (R2 N6)
            Assert.Empty(service.GetCurrentAlarms(SiteId)); // frozen snapshot never grafted
        }, TimeSpan.FromSeconds(10));
    }

    [Fact]
    public void Linger_Stop_Does_Not_Resurrect_The_Aggregator()
    {
        // Last viewer leaves, linger fires, entry removed: the deathwatch on the
        // deliberately-stopped actor must be a no-op (entry gone), not a restart.
        var service = CreateService(TimeSpan.FromMilliseconds(50), out var factory);
        var sub = service.Subscribe(SiteId, () => { });
        AwaitAssert(() => Assert.True(service.IsLive(SiteId)), TimeSpan.FromSeconds(10));
        sub.Dispose();
        AwaitAssert(() => Assert.False(service.IsLive(SiteId)), TimeSpan.FromSeconds(10));
        var startsAfterStop = factory.GetOrCreateCount;
        Thread.Sleep(300);
        Assert.Equal(startsAfterStop, factory.GetOrCreateCount); // no self-heal restart fired
    }

    // ── WP2.3: IsLive tracks the STREAM, not merely "a snapshot was published once" ──

    [Fact]
    public void IsLive_StaysFalse_UntilTheSiteAcceptsTheStream()
    {
        var service = CreateService(TimeSpan.FromMilliseconds(200), out var factory);
        factory.Client.AutoConnect = false; // site never answers the subscription

        using var sub = service.Subscribe(SiteId, () => { });

        // The (empty) seed completes and publishes, but with no accepted stream behind it
        // the cache is not live — the page must keep polling. Pre-fix IsLive flipped true
        // here and the page grafted a never-updating snapshot over fresh poll data.
        AwaitCondition(() => service.GetCurrentAlarms(SiteId) is not null, TimeSpan.FromSeconds(5));
        Thread.Sleep(400);
        Assert.False(service.IsLive(SiteId));

        // The site accepts it → live.
        AwaitCondition(() => factory.Client.Connects.Count >= 1, TimeSpan.FromSeconds(5));
        factory.Client.Connects[0]();
        AwaitCondition(() => service.IsLive(SiteId), TimeSpan.FromSeconds(5));
    }

    [Fact]
    public void IsLive_DropsWhenTheStreamCompletes()
    {
        // The carried residual from Phase 1: a stream that ended gracefully (the site's 4h
        // max lifetime) left IsLive true until the next reconcile publish.
        var service = CreateService(TimeSpan.FromMilliseconds(200), out var factory);

        using var sub = service.Subscribe(SiteId, () => { });
        AwaitCondition(() => service.IsLive(SiteId), TimeSpan.FromSeconds(5));

        AwaitCondition(() => factory.Client.Completions.Count >= 1, TimeSpan.FromSeconds(5));
        factory.Client.Completions[0]();

        AwaitCondition(() => !service.IsLive(SiteId), TimeSpan.FromSeconds(5));
    }
}