ScadaBridge/tests/ZB.MOM.WW.ScadaBridge.CentralUI.Tests/Pages/HealthPageTests.cs

using System.Security.Claims;
using ZB.MOM.WW.ScadaBridge.Security;
using Akka.Actor;
using Bunit;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Components;
using Microsoft.AspNetCore.Components.Authorization;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using NSubstitute;
using ZB.MOM.WW.ScadaBridge.CentralUI.Components.Shared;
using ZB.MOM.WW.ScadaBridge.CentralUI.Services;
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Sites;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Audit;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Notification;
using ZB.MOM.WW.ScadaBridge.Commons.Types;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Kpi;
using ZB.MOM.WW.ScadaBridge.Communication;
using ZB.MOM.WW.ScadaBridge.HealthMonitoring;
using HealthPage = ZB.MOM.WW.ScadaBridge.CentralUI.Components.Pages.Monitoring.Health;

namespace ZB.MOM.WW.ScadaBridge.CentralUI.Tests.Pages;

/// <summary>
/// bUnit rendering tests for the Health Monitoring dashboard (Task 24).
///
/// Scope: the Notification Outbox KPI tile row added to the Health dashboard.
/// <see cref="ICentralHealthAggregator"/> is an interface (mockable), but
/// <see cref="CommunicationService"/> is a concrete class whose outbox calls
/// route through an injected notification-outbox <see cref="IActorRef"/>; the
/// tests reuse the scripted-actor seam established by the Notification Report
/// page tests (see <c>NotificationReportPageTests</c>).
/// </summary>
public class HealthPageTests : BunitContext
{
    private readonly ActorSystem _system = ActorSystem.Create("health-page-tests");
    private readonly CommunicationService _comms;

    // Mutable scripted reply — individual tests can override before rendering.
    private NotificationKpiResponse _kpiReply =
        new("k", true, null, QueueDepth: 12, StuckCount: 4, ParkedCount: 3,
            DeliveredLastInterval: 88, OldestPendingAge: TimeSpan.FromMinutes(6));

    // Site Call Audit (#22) Task 7 — mutable scripted Site Call KPI reply. Tests
    // that target the Site Call tiles override this before rendering.
    private SiteCallKpiResponse _siteCallKpiReply =
        new("k", true, null, BufferedCount: 9, ParkedCount: 2, FailedLastInterval: 1,
            DeliveredLastInterval: 40, OldestPendingAge: TimeSpan.FromMinutes(3),
            StuckCount: 5);

    public HealthPageTests()
    {
        _comms = new CommunicationService(
            Options.Create(new CommunicationOptions()),
            NullLogger<CommunicationService>.Instance);

        var outbox = _system.ActorOf(Props.Create(() => new ScriptedOutboxActor(this)));
        _comms.SetNotificationOutbox(outbox);

        var siteCallAudit = _system.ActorOf(Props.Create(() => new ScriptedSiteCallAuditActor(this)));
        _comms.SetSiteCallAudit(siteCallAudit);
        Services.AddSingleton(_comms);

        // arch-review WP2.4 — the page no longer queries CommunicationService /
        // IAuditLogQueryService per circuit; it reads the process-level memoized
        // KPI cache. The real cache is registered here (not a substitute) so the
        // scripted-actor seam above is still what actually answers.
        Services.AddSingleton<IKpiSnapshotCache>(sp => new KpiSnapshotCache(
            sp.GetRequiredService<CommunicationService>(),
            sp.GetRequiredService<IServiceScopeFactory>()));

        var aggregator = Substitute.For<ICentralHealthAggregator>();
        aggregator.GetAllSiteStates()
            .Returns(new Dictionary<string, SiteHealthState>());
        Services.AddSingleton(aggregator);

        // M6 K16 — the Health page now injects IKpiHistoryQueryService to feed the
        // per-site Site Health Trends panel. Stub it with a known non-empty series
        // so the page resolves the dependency and the trend charts have data; the
        // dedicated trend tests below seed sites / override behaviour.
        var kpiHistory = Substitute.For<IKpiHistoryQueryService>();
        kpiHistory.GetSeriesAsync(
                Arg.Any<string>(), Arg.Any<string>(), Arg.Any<string>(), Arg.Any<string?>(),
                Arg.Any<DateTime>(), Arg.Any<DateTime>(), Arg.Any<int?>(), Arg.Any<CancellationToken>())
            .Returns(Task.FromResult<IReadOnlyList<KpiSeriesPoint>>(SampleSeries()));
        Services.AddSingleton(kpiHistory);

        var siteRepo = Substitute.For<ISiteRepository>();
        siteRepo.GetAllSitesAsync(Arg.Any<CancellationToken>())
            .Returns(Task.FromResult<IReadOnlyList<Site>>(new List<Site>()));
        Services.AddSingleton(siteRepo);

        // PLAN-01 Task 14 — the Health page now shows the metrics-stale badge with
        // a tooltip naming the configured MetricsStaleTimeout, so it injects the
        // health-monitoring options.
        Services.AddSingleton(Options.Create(new HealthMonitoringOptions()));

        // Audit Log (#23) M7 Bundle E — the Health page now also fetches the
        // Audit KPI snapshot. Stub it with an empty point-in-time reading so
        // the existing assertions (Notification Outbox tiles, Online/Offline
        // counts) keep passing; tests that target the Audit tiles set their
        // own substitute.
        var auditService = Substitute.For<IAuditLogQueryService>();
        auditService.GetKpiSnapshotAsync(Arg.Any<CancellationToken>())
            .Returns(Task.FromResult(new AuditLogKpiSnapshot(0, 0, 0, DateTime.UtcNow)));
        Services.AddSingleton(auditService);

        var claims = new[]
        {
            new Claim(JwtTokenService.UsernameClaimType, "tester"),
            new Claim(JwtTokenService.RoleClaimType, "Administrator"),
        };
        var user = new ClaimsPrincipal(new ClaimsIdentity(claims, "TestAuth"));
        Services.AddSingleton<AuthenticationStateProvider>(new TestAuthStateProvider(user));
        Services.AddAuthorizationCore();
        // The failover control's AuthorizeView evaluates the named RequireAdmin policy, so the
        // real policy set has to be registered or the page throws on any card that renders it.
        AuthorizationPolicies.AddScadaBridgeAuthorization(Services);
        // BunitContext pre-registers a placeholder IAuthorizationService that throws when a
        // policy is evaluated; force the real one (same note as NavMenuTests).
        Services.AddSingleton<IAuthorizationService, DefaultAuthorizationService>();

        // The cluster cards embed CentralFailoverControl (Task 10), which injects these two.
        // The page's DI graph genuinely requires them in production, so the test supplies
        // them rather than the component making its dependencies optional. Behaviour of the
        // control itself is covered by HealthFailoverButtonTests.
        Services.AddSingleton(Substitute.For<IManualFailoverService>());
        Services.AddSingleton(Substitute.For<IDialogService>());
    }

    /// <summary>
    /// Renders the dashboard the way the app does — inside a
    /// <see cref="CascadingAuthenticationState"/>, which the real layout supplies. The
    /// cluster cards contain an AuthorizeView (the Task 10 failover control), and
    /// AuthorizeView throws without that cascading value.
    /// </summary>
    private IRenderedComponent<HealthPage> RenderHealthPage()
    {
        var host = Render<CascadingAuthenticationState>(parameters => parameters
            .Add(p => p.ChildContent, (RenderFragment)(builder =>
            {
                builder.OpenComponent<HealthPage>(0);
                builder.CloseComponent();
            })));

        return host.FindComponent<HealthPage>();
    }

    [Fact]
    public void Renders_OutboxKpiTiles_WithValues()
    {
        var cut = RenderHealthPage();

        // KPI data arrives via an async actor Ask after first render.
        cut.WaitForAssertion(() =>
        {
            Assert.Contains("Notification Outbox", cut.Markup);
            Assert.Contains("Queue Depth", cut.Markup);
            Assert.Contains("Stuck", cut.Markup);
            Assert.Contains("Parked", cut.Markup);
            // KPI numeric values surface in the tiles.
            Assert.Contains(">12<", cut.Markup);  // QueueDepth
            Assert.Contains(">4<", cut.Markup);   // StuckCount
            Assert.Contains(">3<", cut.Markup);   // ParkedCount
        });
    }

    [Fact]
    public void RendersLinkToTheNotificationKpisPage()
    {
        var cut = RenderHealthPage();
        var link = cut.Find("a[href='/notifications/kpis']");
        Assert.Contains("View details", link.TextContent);
    }

    [Fact]
    public void Renders_AuditKpiTiles_WithValues()
    {
        // Override the default empty snapshot — this test wants concrete values
        // to land in the three Audit tiles.
        var auditService = Substitute.For<IAuditLogQueryService>();
        auditService.GetKpiSnapshotAsync(Arg.Any<CancellationToken>())
            .Returns(Task.FromResult(new AuditLogKpiSnapshot(
                TotalEventsLastHour: 250,
                ErrorEventsLastHour: 5,
                BacklogTotal: 17,
                AsOfUtc: DateTime.UtcNow)));
        Services.AddSingleton(auditService);

        var cut = RenderHealthPage();

        cut.WaitForAssertion(() =>
        {
            // The three audit tiles render at the documented data-test selectors.
            Assert.Contains("data-test=\"audit-kpi-volume\"", cut.Markup);
            Assert.Contains("data-test=\"audit-kpi-error-rate\"", cut.Markup);
            Assert.Contains("data-test=\"audit-kpi-backlog\"", cut.Markup);
            // Volume shows the formatted thousand-separator value.
            Assert.Contains("250", cut.Markup);
            // Backlog renders 17.
            Assert.Contains("17", cut.Markup);
        });
    }

    [Fact]
    public void Renders_SiteCallKpiTiles_WithValues()
    {
        var cut = RenderHealthPage();

        // KPI data arrives via an async actor Ask after first render.
        cut.WaitForAssertion(() =>
        {
            Assert.Contains("Site Calls", cut.Markup);
            // The three Site Call tiles render at the documented data-test selectors.
            Assert.Contains("data-test=\"site-call-kpi-buffered\"", cut.Markup);
            Assert.Contains("data-test=\"site-call-kpi-stuck\"", cut.Markup);
            Assert.Contains("data-test=\"site-call-kpi-parked\"", cut.Markup);
            // KPI numeric values surface in the tiles.
            Assert.Contains(">9<", cut.Markup);  // BufferedCount
            Assert.Contains(">5<", cut.Markup);  // StuckCount
            Assert.Contains(">2<", cut.Markup);  // ParkedCount
        });
    }

    [Fact]
    public void RendersLinkToTheSiteCallsReportPage()
    {
        var cut = RenderHealthPage();
        var link = cut.Find("a[href='/site-calls/report']");
        Assert.Contains("View details", link.TextContent);
    }

    [Fact]
    public void SiteCallKpiFailure_ShowsGracefulFallback()
    {
        _siteCallKpiReply = new SiteCallKpiResponse(
            "k", false, "site call repository unavailable", 0, 0, 0, 0, null, 0);

        var cut = RenderHealthPage();

        cut.WaitForAssertion(() =>
        {
            // Failure must not crash the page; tiles fall back to a dash and the
            // inline error message surfaces.
            Assert.Contains("Site Calls", cut.Markup);
            Assert.Contains("Site Call KPIs unavailable", cut.Markup);
            Assert.Contains("site call repository unavailable", cut.Markup);
            Assert.Contains(">—<", cut.Markup);
        });
    }

    [Fact]
    public void OutboxKpiFailure_ShowsGracefulFallback()
    {
        _kpiReply = new NotificationKpiResponse(
            "k", false, "outbox repository unavailable", 0, 0, 0, 0, null);

        var cut = RenderHealthPage();

        cut.WaitForAssertion(() =>
        {
            // Failure must not crash the page; tiles fall back to a dash.
            Assert.Contains("Notification Outbox", cut.Markup);
            Assert.Contains("Queue Depth", cut.Markup);
            Assert.Contains(">—<", cut.Markup);
        });
    }

    [Fact]
    public void Renders_SiteHealthTrends_PanelAndChart_ForSelectedSite()
    {
        // Seed one site so the trend panel's selector has an option and the
        // default-site load produces charts.
        SeedSites("site-a");

        var cut = RenderHealthPage();

        cut.WaitForAssertion(() =>
        {
            // The panel + its site selector render at the documented hooks.
            Assert.Contains("data-test=\"site-health-trends\"", cut.Markup);
            Assert.Contains("data-test=\"site-health-trends-site\"", cut.Markup);
            // The four metric charts render (the shared KpiTrendChart slug hook),
            // and the seeded non-empty series draws a polyline. The "S&F Buffer
            // Depth" title slugifies to "s-f-buffer-depth" (the & and the spaces
            // each collapse to a dash) — see KpiTrendChart.Slugify.
            Assert.Contains("kpi-trend-connections-down", cut.Markup);
            Assert.Contains("kpi-trend-dead-letters", cut.Markup);
            Assert.Contains("kpi-trend-script-errors", cut.Markup);
            Assert.Contains("kpi-trend-s-f-buffer-depth", cut.Markup);
            Assert.Contains("<polyline", cut.Markup);
        });
    }

    [Fact]
    public void SiteHealthTrendsFailure_DoesNotBreakDashboard()
    {
        SeedSites("site-a");

        // The KPI-history service throws on every query — the trend load is
        // best-effort, so the dashboard (and its tiles) must still render.
        var faulting = Substitute.For<IKpiHistoryQueryService>();
        faulting.GetSeriesAsync(
                Arg.Any<string>(), Arg.Any<string>(), Arg.Any<string>(), Arg.Any<string?>(),
                Arg.Any<DateTime>(), Arg.Any<DateTime>(), Arg.Any<int?>(), Arg.Any<CancellationToken>())
            .Returns<Task<IReadOnlyList<KpiSeriesPoint>>>(_ =>
                throw new InvalidOperationException("kpi history unavailable"));
        Services.AddSingleton(faulting);

        var cut = RenderHealthPage();

        cut.WaitForAssertion(() =>
        {
            // No unhandled exception: the core dashboard tiles still render, and
            // the panel falls back to the per-chart unavailable placeholder.
            Assert.Contains("Notification Outbox", cut.Markup);
            Assert.Contains("data-test=\"site-health-trends\"", cut.Markup);
            Assert.Contains("Trend data unavailable.", cut.Markup);
        });
    }

    [Fact]
    public void Renders_MetricsStaleBadge_ForOnlineButStaleSite()
    {
        // PLAN-01 Task 14: a site that is online (heartbeating) but whose report
        // pipeline died shows a distinct "Metrics stale" warning badge.
        SeedStates(new SiteHealthState
        {
            SiteId = "site-a",
            IsOnline = true,
            IsMetricsStale = true,
            LastHeartbeatAt = DateTimeOffset.UtcNow,
            LastReportReceivedAt = DateTimeOffset.UtcNow.AddMinutes(-5),
        });

        var cut = RenderHealthPage();

        cut.WaitForAssertion(() =>
        {
            Assert.Contains("Metrics stale", cut.Markup);
            // The badge uses the page's Bootstrap warning style.
            Assert.Contains("bg-warning", cut.Markup);
        });
    }

    // ------------------------------------------------------------------
    // Sustained LocalDb replication backlog badge (register row 33)
    // ------------------------------------------------------------------

    [Fact]
    public void Renders_ReplicationBacklogBadge_WhenBacklogSustainedPastTheDwell()
    {
        SeedStates(new SiteHealthState
        {
            SiteId = "site-a",
            IsOnline = true,
            LastHeartbeatAt = DateTimeOffset.UtcNow,
            // Default dwell is 15 minutes; 30 minutes above threshold is well past it.
            LocalDbBacklogAboveSince = DateTimeOffset.UtcNow.AddMinutes(-30),
        });

        var cut = RenderHealthPage();

        cut.WaitForAssertion(() =>
        {
            Assert.Contains("data-test=\"localdb-backlog-badge\"", cut.Markup);
            Assert.Contains("Replication backlog", cut.Markup);
        });
    }

    [Fact]
    public void DoesNotRender_ReplicationBacklogBadge_DuringAPurgeBurst()
    {
        // THE false-positive this badge is shaped to avoid. The daily site_events purge is
        // CDC-captured, so the backlog spikes and then drains. Two minutes above the
        // threshold is a purge burst, not a fault, and must stay silent — otherwise the
        // badge fires on every replicated site every day and gets ignored.
        SeedStates(new SiteHealthState
        {
            SiteId = "site-a",
            IsOnline = true,
            LastHeartbeatAt = DateTimeOffset.UtcNow,
            LocalDbBacklogAboveSince = DateTimeOffset.UtcNow.AddMinutes(-2),
        });

        var cut = RenderHealthPage();

        cut.WaitForAssertion(() => Assert.Contains("site-a", cut.Markup));
        Assert.DoesNotContain("data-test=\"localdb-backlog-badge\"", cut.Markup);
    }

    [Fact]
    public void DoesNotRender_ReplicationBacklogBadge_ForAnUnreplicatedSite()
    {
        // Rig site-b/site-c report a null backlog forever, so the aggregator never gives them
        // an anchor. A deliberately-unreplicated site must never be badged.
        SeedStates(new SiteHealthState
        {
            SiteId = "site-b",
            IsOnline = true,
            LastHeartbeatAt = DateTimeOffset.UtcNow,
            LocalDbBacklogAboveSince = null,
        });

        var cut = RenderHealthPage();

        cut.WaitForAssertion(() => Assert.Contains("site-b", cut.Markup));
        Assert.DoesNotContain("data-test=\"localdb-backlog-badge\"", cut.Markup);
    }

    [Fact]
    public void DoesNotRender_ReplicationBacklogBadge_ForAnOfflineSite()
    {
        // An offline site's last-known anchor is stale by definition; the Offline badge is the
        // signal that matters, and stacking a backlog warning on top is noise.
        SeedStates(new SiteHealthState
        {
            SiteId = "site-a",
            IsOnline = false,
            LastHeartbeatAt = DateTimeOffset.UtcNow.AddMinutes(-30),
            LocalDbBacklogAboveSince = DateTimeOffset.UtcNow.AddMinutes(-30),
        });

        var cut = RenderHealthPage();

        cut.WaitForAssertion(() => Assert.Contains("site-a", cut.Markup));
        Assert.DoesNotContain("data-test=\"localdb-backlog-badge\"", cut.Markup);
    }

    [Fact]
    public void Renders_LocalDbBacklogTrendChart()
    {
        // The retrospective half of row 33: a point-in-time tile can never answer "did that
        // spike drain?", which is the only question that distinguishes healthy from faulty.
        SeedSites("site-a");

        var cut = RenderHealthPage();

        cut.WaitForAssertion(() =>
            Assert.Contains("LocalDb Replication Backlog", cut.Markup));
    }

    [Fact]
    public void Renders_OfflineSince_ForOfflineSiteWithStatusChange()
    {
        var changedAt = new DateTimeOffset(2026, 7, 8, 12, 34, 56, TimeSpan.Zero);
        SeedStates(new SiteHealthState
        {
            SiteId = "site-a",
            IsOnline = false,
            LastHeartbeatAt = changedAt,
            LastStatusChangeAt = changedAt,
        });

        var cut = RenderHealthPage();

        cut.WaitForAssertion(() =>
        {
            Assert.Contains("offline since", cut.Markup);
            // The "u" (universal sortable) format the page renders the flip with.
            Assert.Contains(changedAt.ToString("u"), cut.Markup);
        });
    }

    // Re-seeds the aggregator substitute with fully-specified states (as opposed
    // to SeedSites' minimal online placeholders).
    private void SeedStates(params SiteHealthState[] states)
    {
        var aggregator = Substitute.For<ICentralHealthAggregator>();
        aggregator.GetAllSiteStates()
            .Returns(states.ToDictionary(s => s.SiteId));
        Services.AddSingleton(aggregator);
    }

    // Re-seeds the aggregator substitute so the trend panel's site selector has
    // options. Each site id maps to a minimal online SiteHealthState (a null
    // report is fine — the trend panel keys off the site ids, not the report).
    private void SeedSites(params string[] siteIds)
    {
        var aggregator = Substitute.For<ICentralHealthAggregator>();
        var states = siteIds.ToDictionary(
            id => id,
            id => new SiteHealthState
            {
                SiteId = id,
                IsOnline = true,
                LastHeartbeatAt = DateTimeOffset.UtcNow,
            });
        aggregator.GetAllSiteStates()
            .Returns(new Dictionary<string, SiteHealthState>(states));
        Services.AddSingleton(aggregator);
    }

    // A known non-empty (≥2-point) series so KpiTrendChart renders a polyline
    // rather than the single-sample / unavailable placeholder.
    private static IReadOnlyList<KpiSeriesPoint> SampleSeries()
    {
        var baseUtc = DateTime.UtcNow.AddHours(-24);
        return new List<KpiSeriesPoint>
        {
            new(baseUtc, 1),
            new(baseUtc.AddHours(6), 3),
            new(baseUtc.AddHours(12), 2),
            new(baseUtc.AddHours(18), 5),
        };
    }

    protected override void Dispose(bool disposing)
    {
        if (disposing)
        {
            _system.Terminate().Wait(TimeSpan.FromSeconds(5));
        }
        base.Dispose(disposing);
    }

    /// <summary>
    /// Stand-in for the notification-outbox actor. Replies to the KPI request
    /// with the test's currently-scripted response.
    /// </summary>
    private sealed class ScriptedOutboxActor : ReceiveActor
    {
        public ScriptedOutboxActor(HealthPageTests test)
        {
            Receive<NotificationKpiRequest>(_ => Sender.Tell(test._kpiReply));
        }
    }

    /// <summary>
    /// Stand-in for the Site Call Audit actor. Replies to the KPI request with
    /// the test's currently-scripted response. Also handles the per-node KPI
    /// request (T6: M5.2) with an empty-nodes success reply so the Health page
    /// can complete initialization without a 30-second Ask timeout.
    /// </summary>
    private sealed class ScriptedSiteCallAuditActor : ReceiveActor
    {
        public ScriptedSiteCallAuditActor(HealthPageTests test)
        {
            Receive<SiteCallKpiRequest>(_ => Sender.Tell(test._siteCallKpiReply));
            Receive<PerNodeSiteCallKpiRequest>(req => Sender.Tell(
                new PerNodeSiteCallKpiResponse(req.CorrelationId, Success: true, ErrorMessage: null,
                    Nodes: Array.Empty<SiteCallNodeKpiSnapshot>())));
        }
    }
}