@page "/monitoring/health"
@attribute [Authorize]
@using ZB.MOM.WW.ScadaBridge.CentralUI.Components.Health
@using ZB.MOM.WW.ScadaBridge.CentralUI.Services
@using ZB.MOM.WW.ScadaBridge.Commons.Types
@using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums
@using ZB.MOM.WW.ScadaBridge.Commons.Entities.Sites
@using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories
@using ZB.MOM.WW.ScadaBridge.HealthMonitoring
@using ZB.MOM.WW.ScadaBridge.Commons.Messages.Notification
@using ZB.MOM.WW.ScadaBridge.Commons.Messages.Audit
@using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit
@using ZB.MOM.WW.ScadaBridge.Commons.Types.Kpi
@using ZB.MOM.WW.ScadaBridge.Communication
@implements IDisposable
@inject ICentralHealthAggregator HealthAggregator
@inject ISiteRepository SiteRepository
@inject IKpiSnapshotCache KpiSnapshots
@inject IKpiHistoryQueryService KpiHistory
@inject Microsoft.Extensions.Options.IOptions<HealthMonitoringOptions> HealthOptions

<div class="container-fluid mt-3">
    <div class="d-flex justify-content-between align-items-center mb-3">
        <h4 class="mb-0">Health Dashboard</h4>
        <div>
            <span class="text-muted small me-2">Auto-refresh: @(_autoRefreshSeconds)s</span>
            <button class="btn btn-outline-secondary btn-sm" @onclick="RefreshNow">Refresh Now</button>
        </div>
    </div>

    @* Headline KPI groups. Each is a self-contained panel, so each gets its own card
       boundary — the same treatment the Site Health Trends group below already uses,
       including tiles-that-are-cards nested in a card body (Trends does exactly that
       with KpiTrendChart). The two component-rendered groups own their heading and
       "View details" link internally, so the Notification Outbox group keeps its
       heading inside the card body too rather than promoting it to a card-header:
       uniformity across the panels beats matching the Trends card, whose card-header
       exists to carry the site selector and window toggle. card-body pb-0 lets each
       tile row's own mb-3 supply the bottom padding instead of doubling it. *@

    @* Notification Outbox headline KPIs — a central concern, shown regardless of site reports *@
    <div class="card mb-3">
        <div class="card-body pb-0">
            <div class="d-flex justify-content-between align-items-center mb-2">
                <h6 class="text-muted mb-0">Notification Outbox</h6>
                <a class="small" href="/notifications/kpis">View details &rarr;</a>
            </div>
            <div class="row g-3 mb-3">
                <div class="col-lg-4 col-md-6 col-12">
                    <div class="card h-100">
                        <div class="card-body text-center">
                            <h3 class="mb-0">@OutboxTileValue(_outboxKpi.QueueDepth)</h3>
                            <small class="text-muted">Queue Depth</small>
                        </div>
                    </div>
                </div>
                <div class="col-lg-4 col-md-6 col-12">
                    <div class="card h-100 @(_outboxKpiAvailable && _outboxKpi.StuckCount > 0 ? "border-warning" : "")">
                        <div class="card-body text-center">
                            <h3 class="mb-0 @(_outboxKpiAvailable && _outboxKpi.StuckCount > 0 ? "text-warning" : "")">@OutboxTileValue(_outboxKpi.StuckCount)</h3>
                            <small class="text-muted">Stuck</small>
                        </div>
                    </div>
                </div>
                <div class="col-lg-4 col-md-6 col-12">
                    <div class="card h-100 @(_outboxKpiAvailable && _outboxKpi.ParkedCount > 0 ? "border-danger" : "")">
                        <div class="card-body text-center">
                            <h3 class="mb-0 @(_outboxKpiAvailable && _outboxKpi.ParkedCount > 0 ? "text-danger" : "")">@OutboxTileValue(_outboxKpi.ParkedCount)</h3>
                            <small class="text-muted">Parked</small>
                        </div>
                    </div>
                </div>
            </div>
            @if (!_outboxKpiAvailable && _outboxKpiError != null)
            {
                <div class="text-muted small mb-3">Notification Outbox KPIs unavailable: @_outboxKpiError</div>
            }
        </div>
    </div>

    @* Site Call Audit (#22) Task 7 — three KPI tiles for the Site Call channel
       (buffered / stuck / parked). Refreshed alongside the site states. *@
    <div class="card mb-3">
        <div class="card-body pb-0">
            <SiteCallKpiTiles Snapshot="@_siteCallKpi"
                              IsAvailable="@_siteCallKpiAvailable"
                              ErrorMessage="@_siteCallKpiError"
                              PerNodeSnapshots="@_siteCallNodeKpis"
                              PerNodeAvailable="@_siteCallNodeKpiAvailable" />
        </div>
    </div>

    @* Three KPI tiles for the Audit channel
       (volume / error rate / backlog). Refreshed alongside the site states. *@
    <div class="card mb-3">
        <div class="card-body pb-0">
            <AuditKpiTiles Snapshot="@_auditKpi"
                           IsAvailable="@_auditKpiAvailable"
                           ErrorMessage="@_auditKpiError" />
        </div>
    </div>

    @* Site Health Trends (M6) — per-site Site Health KPI history. Loads on a
       separate path from the 10s tile-refresh timer so a trend-query fault can
       never disturb the live dashboard or its polling loop. The site selector
       reuses the site keys already loaded into _siteStates; the window toggle
       drives the time range. Both re-query independently. *@
    <div class="card mb-3" data-test="site-health-trends">
        <div class="card-header d-flex justify-content-between align-items-center py-2">
            <div class="d-flex align-items-center">
                <h6 class="text-muted mb-0 me-3">Site Health Trends</h6>
                @if (_trendSiteKeys.Count > 0)
                {
                    @* OnTrendSiteChangedAsync is async — Blazor wraps it in an EventCallback so the returned Task IS awaited; keep the Async signature, do not change it to a void/fire-and-forget handler. This comment MUST stay OUTSIDE the <select> start tag: a Razor comment placed between attributes renders as an invalid attribute name and crashes the SignalR circuit (InvalidCharacterError on setAttribute). *@
                    <select class="form-select form-select-sm" style="width:auto"
                            data-test="site-health-trends-site"
                            value="@_trendSiteId"
                            @onchange="OnTrendSiteChangedAsync">
                        @foreach (var key in _trendSiteKeys)
                        {
                            <option value="@key">@TrendSiteLabel(key)</option>
                        }
                    </select>
                }
            </div>
            <div class="btn-group btn-group-sm" role="group" aria-label="Trend window">
                <button type="button"
                        class="btn @(_trendWindowHours == 24 ? "btn-primary" : "btn-outline-secondary")"
                        @onclick="() => SetTrendWindowAsync(24)" disabled="@_trendsLoading">24h</button>
                <button type="button"
                        class="btn @(_trendWindowHours == 168 ? "btn-primary" : "btn-outline-secondary")"
                        @onclick="() => SetTrendWindowAsync(168)" disabled="@_trendsLoading">7d</button>
                <button type="button"
                        class="btn @(_trendWindowHours == 720 ? "btn-primary" : "btn-outline-secondary")"
                        @onclick="() => SetTrendWindowAsync(720)" disabled="@_trendsLoading">30d</button>
                <button type="button"
                        class="btn @(_trendWindowHours == 2160 ? "btn-primary" : "btn-outline-secondary")"
                        @onclick="() => SetTrendWindowAsync(2160)" disabled="@_trendsLoading">90d</button>
            </div>
        </div>
        <div class="card-body p-3">
            @if (_trendSiteKeys.Count == 0)
            {
                <span class="text-muted small">No sites available for trends yet.</span>
            }
            else
            {
                <div class="row g-3">
                    <div class="col-lg-3 col-md-6">
                        <KpiTrendChart Title="Connections Down"
                                       Points="@_connectionsDownSeries"
                                       IsAvailable="@_connectionsDownAvailable"
                                       ErrorMessage="@_connectionsDownError" />
                    </div>
                    <div class="col-lg-3 col-md-6">
                        <KpiTrendChart Title="Dead Letters"
                                       Points="@_deadLettersSeries"
                                       IsAvailable="@_deadLettersAvailable"
                                       ErrorMessage="@_deadLettersError" />
                    </div>
                    <div class="col-lg-3 col-md-6">
                        <KpiTrendChart Title="Script Errors"
                                       Points="@_scriptErrorsSeries"
                                       IsAvailable="@_scriptErrorsAvailable"
                                       ErrorMessage="@_scriptErrorsError" />
                    </div>
                    <div class="col-lg-3 col-md-6">
                        <KpiTrendChart Title="S&amp;F Buffer Depth"
                                       Points="@_sfBufferDepthSeries"
                                       IsAvailable="@_sfBufferDepthAvailable"
                                       ErrorMessage="@_sfBufferDepthError" />
                    </div>
                    @* Register row 33. The point of charting the backlog is retrospective:
                       the daily purge burst is expected, so what an operator needs is to see
                       whether a past spike DRAINED — which a point-in-time tile can never
                       answer. An unreplicated site emits no samples, so this chart is
                       legitimately empty there rather than a flat, misleading zero line. *@
                    <div class="col-lg-3 col-md-6">
                        <KpiTrendChart Title="LocalDb Replication Backlog"
                                       Points="@_localDbBacklogSeries"
                                       IsAvailable="@_localDbBacklogAvailable"
                                       ErrorMessage="@_localDbBacklogError" />
                    </div>
                </div>
            }
        </div>
    </div>

    @if (_siteStates.Count == 0)
    {
        <div class="alert alert-info">No site health reports received yet.</div>
    }
    else
    {
        @* Overview cards — the fourth headline tile row, so it takes the same card
           boundary as the three above it. It was also the only tile row on the page
           with no heading at all; it now carries one for the same reason. *@
        <div class="card mb-3">
            <div class="card-body pb-0">
                <div class="d-flex justify-content-between align-items-center mb-2">
                    <h6 class="text-muted mb-0">Sites</h6>
                </div>
                <div class="row g-3 mb-3">
                    <div class="col-lg-4 col-md-6 col-12">
                        <div class="card border-success h-100">
                            <div class="card-body text-center">
                                <h3 class="mb-0 text-success">@_siteStates.Values.Count(s => s.IsOnline)</h3>
                                <small class="text-muted">Sites Online</small>
                            </div>
                        </div>
                    </div>
                    <div class="col-lg-4 col-md-6 col-12">
                        <div class="card border-danger h-100">
                            <div class="card-body text-center">
                                <h3 class="mb-0 text-danger">@_siteStates.Values.Count(s => !s.IsOnline)</h3>
                                <small class="text-muted">Sites Offline</small>
                            </div>
                        </div>
                    </div>
                    <div class="col-lg-4 col-md-6 col-12">
                        <div class="card border-warning h-100">
                            <div class="card-body text-center">
                                <h3 class="mb-0 text-warning">@_siteStates.Values.Count(SiteHasActiveErrors)</h3>
                                <small class="text-muted">Sites with active errors</small>
                            </div>
                        </div>
                    </div>
                </div>
            </div>
        </div>

        @* Per-site detail cards — central cluster pinned to the top, then sites alphabetically *@
        @foreach (var (siteId, state) in _siteStates.OrderBy(s => s.Key == CentralHealthReportLoop.CentralSiteId ? 0 : 1).ThenBy(s => s.Key))
        {
            var isCentral = siteId == CentralHealthReportLoop.CentralSiteId;
            var siteName = isCentral ? "Central Cluster" : GetSiteName(siteId);
            var detailsCollapseId = $"site-details-{siteId}";
            <div class="card mb-3">
                <div class="card-header d-flex justify-content-between align-items-center py-2">
                    @* Display name and machine id are two different things and are read
                       for two different reasons, so they get two lines: chips + name on
                       the first, the site identifier on its own beneath it. Reference
                       shape: Pages/Admin/Sites.razor (card title + monospace identifier). *@
                    <div>
                        <div class="d-flex align-items-center flex-wrap gap-2">
                            @if (state.IsOnline)
                            {
                                <span class="badge bg-success" aria-label="State: Online">@OnlineGlyph Online</span>
                            }
                            else
                            {
                                <span class="badge bg-danger" aria-label="State: Offline">@OfflineGlyph Offline</span>
                            }
                            <strong class="fs-5">@siteName</strong>
                            @if (state.IsOnline && state.IsMetricsStale)
                            {
                                <span class="badge bg-warning text-dark"
                                      title="Site is heartbeating but has sent no health report for over @StaleTimeoutDisplay — the metrics pipeline may be dead.">
                                    Metrics stale
                                </span>
                            }
                            @if (IsLocalDbBacklogSustained(state))
                            {
                                <span class="badge bg-warning text-dark"
                                      data-test="localdb-backlog-badge"
                                      title="LocalDb replication oplog backlog has stayed above @HealthOptions.Value.LocalDbBacklogThreshold for over @BacklogSustainedDisplay. A short daily spike at the site_events purge is expected and is NOT this — the peer has stopped draining.">
                                    Replication backlog
                                </span>
                            }
                            @if (IsStartupReconcilerMissing(state))
                            {
                                <span class="badge bg-warning text-dark"
                                      data-test="startup-reconciler-badge"
                                      title="This site reported that a node booted without its SiteReconciliationActor (no IDeploymentConfigFetcher), so the startup self-heal is disabled: a deploy that lands while the node is DOWN will not be recovered on restart. Config still reaches the node via LocalDb replication while it is up. Check the node's configuration.">
                                    No self-heal
                                </span>
                            }
                            @if (HasSecretMirrorFailures(state))
                            {
                                <span class="badge bg-warning text-dark"
                                      data-test="secret-mirror-badge"
                                      title="A site node could not apply and/or decrypt one or more deployed secrets. A standby node's failures surface here before a failover would expose them — see the details in the card body.">
                                    Secret mirror failing
                                </span>
                            }
                            @if (!state.IsOnline && state.LastStatusChangeAt is { } changedAt)
                            {
                                <small class="text-muted">offline since @changedAt.ToString("u")</small>
                            }
                        </div>
                        @if (!isCentral)
                        {
                            <div class="small text-muted font-monospace">@siteId</div>
                        }
                    </div>
                    <div class="d-flex align-items-center gap-3">
                        @* Admin-only; the control gates itself, and disables when the pair has
                           no online standby to take over. Central acts on the local cluster;
                           a site is asked over the command/control relay. *@
                        <CentralFailoverControl OnlineCentralNodeCount="@OnlineNodeCount(state)"
                                                SiteId="@(isCentral ? null : siteId)" />
                        <small class="text-muted">
                            Last report: <TimestampDisplay Value="@state.LastReportReceivedAt" Format="HH:mm:ss" NullText="awaiting first report" />
                            | Last heartbeat: <TimestampDisplay Value="@state.LastHeartbeatAt" Format="HH:mm:ss" />
                            | Seq: @state.LastSequenceNumber
                        </small>
                    </div>
                </div>
                <div class="card-body p-3">
                    @if (state.LatestReport != null)
                    {
                        var report = state.LatestReport;
                        @if (HasSecretMirrorFailures(state))
                        {
                            @* Review finding M1: every site node's deployed-secrets mirror failures,
                               the standby's included (carried on the active node's report). *@
                            <div class="alert alert-warning py-2 px-3 mb-3 small" data-test="secret-mirror-failures" role="alert">
                                <strong>Deployed secrets failing on a site node</strong>
                                <ul class="mb-0 ps-3">
                                    @foreach (var node in report.SecretMirrorStatuses!.Where(n => n.Failures.Count > 0))
                                    {
                                        @foreach (var failure in node.Failures)
                                        {
                                            <li>
                                                <span class="font-monospace">@node.NodeName</span> &middot;
                                                <span class="font-monospace">@failure.SecretName</span>: @failure.Message
                                            </li>
                                        }
                                    }
                                </ul>
                            </div>
                        }
                        <div class="row g-3">
                            @* Each logical group below sits in its own card, with the group's
                               head promoted to a real card-header, instead of h6 rules stacked
                               inside one undifferentiated body. Columns 3 and 4 keep their
                               existing single collapse region (and its id) spanning both of
                               their sub-sections: the disclosure toggle IS the card-header and
                               the two sub-heads live inside the one card body. *@

                            @* Column 1: Nodes *@
                            <div class="col-md-6">
                                <div class="card h-100">
                                    <div class="card-header py-2">
                                        <h6 class="text-muted mb-0">Nodes (@NodeRowCount(state))</h6>
                                    </div>
                                    <div class="card-body p-2">
                                        <table class="table table-sm table-borderless mb-0">
                                            <tbody>
                                                @if (report.ClusterNodes is { Count: > 0 })
                                                {
                                                    @foreach (var node in report.ClusterNodes)
                                                    {
                                                        <tr>
                                                            <td class="small">@node.Hostname</td>
                                                            <td>
                                                                <span class="badge @(node.IsOnline ? "bg-success" : "bg-danger")"
                                                                      aria-label="State: @(node.IsOnline ? "Online" : "Offline")">
                                                                    @(node.IsOnline ? OnlineGlyph : OfflineGlyph) @(node.IsOnline ? "Online" : "Offline")
                                                                </span>
                                                            </td>
                                                            <td>
                                                                <span class="badge @(node.Role == "Primary" ? "bg-primary" : "bg-secondary")"
                                                                      aria-label="State: @node.Role">
                                                                    @(node.Role == "Primary" ? PrimaryGlyph : StandbyGlyph) @node.Role
                                                                </span>
                                                            </td>
                                                        </tr>
                                                    }
                                                }
                                                else
                                                {
                                                    <tr>
                                                        <td class="small">@(report.NodeHostname != "" ? report.NodeHostname : "Node")</td>
                                                        <td>
                                                            <span class="badge @(state.IsOnline ? "bg-success" : "bg-danger")"
                                                                  aria-label="State: @(state.IsOnline ? "Online" : "Offline")">
                                                                @(state.IsOnline ? OnlineGlyph : OfflineGlyph) @(state.IsOnline ? "Online" : "Offline")
                                                            </span>
                                                        </td>
                                                        <td>
                                                            @{
                                                                var roleLabel = report.NodeRole == "Active" ? "Primary" : "Standby";
                                                            }
                                                            <span class="badge @(report.NodeRole == "Active" ? "bg-primary" : "bg-secondary")"
                                                                  aria-label="State: @roleLabel">
                                                                @(roleLabel == "Primary" ? PrimaryGlyph : StandbyGlyph) @roleLabel
                                                            </span>
                                                        </td>
                                                    </tr>
                                                }
                                            </tbody>
                                        </table>
                                    </div>
                                </div>
                            </div>

                            @* Column 2: Data Connections (collapsible) *@
                            <div class="col-md-6">
                                <div class="card h-100">
                                    <div class="card-header py-2">
                                        <button class="btn btn-link btn-sm p-0 text-decoration-none"
                                                data-bs-toggle="collapse"
                                                data-bs-target="@($"#{detailsCollapseId}-conns")"
                                                aria-expanded="false">
                                            Data Connections (@report.DataConnectionStatuses.Count)
                                        </button>
                                    </div>
                                    <div class="collapse" id="@($"{detailsCollapseId}-conns")">
                                        <div class="card-body p-2">
                                            @if (report.DataConnectionStatuses.Count == 0)
                                            {
                                                <span class="text-muted small">None</span>
                                            }
                                            else
                                            {
                                                @foreach (var (connName, health) in report.DataConnectionStatuses)
                                                {
                                                    var endpoint = report.DataConnectionEndpoints?.GetValueOrDefault(connName);
                                                    var quality = report.DataConnectionTagQuality?.GetValueOrDefault(connName);
                                                    var endpointText = endpoint ?? health.ToString();
                                                    <div class="mb-2">
                                                        @* Both halves are site-supplied strings — a long connection
                                                           name or endpoint URL would otherwise wrap this flex row
                                                           into several lines. Clip both, full value on the title. *@
                                                        <div class="d-flex justify-content-between gap-2">
                                                            <strong class="small cell-clip cell-clip-sm" title="@connName">@connName</strong>
                                                            <span class="small cell-clip cell-clip-sm text-end" title="@endpointText">@endpointText</span>
                                                        </div>
                                                        @if (quality != null)
                                                        {
                                                            <table class="table table-sm table-borderless mb-0">
                                                                <tbody>
                                                                    <tr>
                                                                        <td class="small text-muted py-0">Tags good</td>
                                                                        <td class="small text-end py-0">@quality.Good.ToString("N0")</td>
                                                                    </tr>
                                                                    <tr>
                                                                        <td class="small text-muted py-0">Tags bad</td>
                                                                        <td class="small text-end py-0">@quality.Bad.ToString("N0")</td>
                                                                    </tr>
                                                                    <tr>
                                                                        <td class="small text-muted py-0">Tags uncertain</td>
                                                                        <td class="small text-end py-0">@quality.Uncertain.ToString("N0")</td>
                                                                    </tr>
                                                                </tbody>
                                                            </table>
                                                        }
                                                    </div>
                                                }
                                            }
                                        </div>
                                    </div>
                                </div>
                            </div>

                            @* Column 3: Instances + Store-and-Forward (collapsible) *@
                            <div class="col-md-6">
                                <div class="card h-100">
                                    <div class="card-header py-2">
                                        <button class="btn btn-link btn-sm p-0 text-decoration-none"
                                                data-bs-toggle="collapse"
                                                data-bs-target="@($"#{detailsCollapseId}-queues")"
                                                aria-expanded="false">
                                            Instances &amp; Queues
                                        </button>
                                    </div>
                                    <div class="collapse" id="@($"{detailsCollapseId}-queues")">
                                        <div class="card-body p-2">
                                            <h6 class="text-muted mb-2 border-bottom pb-1">Instances</h6>
                                            <table class="table table-sm table-borderless mb-0">
                                                <tbody>
                                                    <tr>
                                                        <td class="small">Deployed</td>
                                                        <td class="text-end">@report.DeployedInstanceCount</td>
                                                    </tr>
                                                    <tr>
                                                        <td class="small">Enabled</td>
                                                        <td class="text-end text-success">@report.EnabledInstanceCount</td>
                                                    </tr>
                                                    <tr>
                                                        <td class="small">Disabled</td>
                                                        <td class="text-end">@report.DisabledInstanceCount</td>
                                                    </tr>
                                                </tbody>
                                            </table>

                                            <h6 class="text-muted mb-2 mt-3 border-bottom pb-1">
                                                Store-and-Forward Buffers (@report.StoreAndForwardBufferDepths.Count)
                                            </h6>
                                            @if (report.StoreAndForwardBufferDepths.Count == 0)
                                            {
                                                <span class="text-muted small">Empty</span>
                                            }
                                            else
                                            {
                                                @foreach (var (category, depth) in report.StoreAndForwardBufferDepths)
                                                {
                                                    <div class="d-flex justify-content-between mb-1">
                                                        <span class="small">@category</span>
                                                        <span class="badge @(depth > 0 ? "bg-warning text-dark" : "bg-light text-dark")">@depth</span>
                                                    </div>
                                                }
                                            }
                                        </div>
                                    </div>
                                </div>
                            </div>

                            @* Column 4: Error Counts + Parked Messages (collapsible) *@
                            <div class="col-md-6">
                                <div class="card h-100">
                                    <div class="card-header py-2">
                                        <button class="btn btn-link btn-sm p-0 text-decoration-none"
                                                data-bs-toggle="collapse"
                                                data-bs-target="@($"#{detailsCollapseId}-errors")"
                                                aria-expanded="false">
                                            Errors &amp; Parked Messages
                                        </button>
                                    </div>
                                    <div class="collapse" id="@($"{detailsCollapseId}-errors")">
                                        <div class="card-body p-2">
                                            <h6 class="text-muted mb-2 border-bottom pb-1">Error Counts</h6>
                                            <table class="table table-sm table-borderless mb-0">
                                                <tbody>
                                                    <tr>
                                                        <td class="small">Script Errors</td>
                                                        <td class="text-end">
                                                            <span class="@(report.ScriptErrorCount > 0 ? "text-danger fw-bold" : "")">@report.ScriptErrorCount</span>
                                                        </td>
                                                    </tr>
                                                    <tr>
                                                        <td class="small">Alarm Eval Errors</td>
                                                        <td class="text-end">
                                                            <span class="@(report.AlarmEvaluationErrorCount > 0 ? "text-warning fw-bold" : "")">@report.AlarmEvaluationErrorCount</span>
                                                        </td>
                                                    </tr>
                                                    <tr>
                                                        <td class="small">Dead Letters</td>
                                                        <td class="text-end">
                                                            <span class="@(report.DeadLetterCount > 0 ? "text-danger fw-bold" : "")">@report.DeadLetterCount</span>
                                                        </td>
                                                    </tr>
                                                </tbody>
                                            </table>

                                            <h6 class="text-muted mb-2 mt-3 border-bottom pb-1">Parked Messages</h6>
                                            @if (report.ParkedMessageCount == 0)
                                            {
                                                <span class="text-muted small">Empty</span>
                                            }
                                            else
                                            {
                                                <span class="badge bg-warning text-dark">@report.ParkedMessageCount</span>
                                            }
                                        </div>
                                    </div>
                                </div>
                            </div>
                        </div>
                    }
                    else
                    {
                        <span class="text-muted">No report data available.</span>
                    }
                </div>
            </div>
        }
    }
</div>

@code {
    // Shape-coded status glyphs to pair with badge colour.
    private const string OnlineGlyph = "●";   // ●
    private const string OfflineGlyph = "○";  // ○
    private const string PrimaryGlyph = "▲";  // ▲
    private const string StandbyGlyph = "△";  // △

    // Human-friendly rendering of the configured metrics-stale window for the
    // "Metrics stale" badge tooltip (e.g. "2 minutes").
    private string StaleTimeoutDisplay =>
        FormatDuration(HealthOptions.Value.MetricsStaleTimeout);

    // Human-friendly rendering of the configured sustained-backlog dwell for the
    // "Replication backlog" badge tooltip (e.g. "15 minutes").
    private string BacklogSustainedDisplay =>
        FormatDuration(HealthOptions.Value.LocalDbBacklogSustainedFor);

    // Register row 33: the sustained LocalDb replication backlog badge.
    //
    // Reads the anchor the aggregator maintains (LocalDbBacklogAboveSince) rather than the
    // instantaneous backlog on the report, because a bare "backlog > threshold" badge would
    // fire on EVERY replicated site EVERY day: the site_events retention purge is CDC-captured
    // like any other write, so the backlog jumps by the deleted batch size and then drains.
    // Healthy is "the backlog comes back down"; only a dwell can express that.
    //
    // A site that reports no backlog at all (unreplicated — rig site-b/site-c — or a failed
    // poll) never gets an anchor, so it can never be badged. That is the whole null-semantics
    // requirement: null means "no reading", not zero.
    private bool IsLocalDbBacklogSustained(SiteHealthState state) =>
        state.IsOnline
        && state.LocalDbBacklogAboveSince is { } since
        && DateTimeOffset.UtcNow - since >= HealthOptions.Value.LocalDbBacklogSustainedFor;

    // Register row 73: the "No self-heal" badge.
    //
    // Warning-level treatment, deliberately not an error and never an offline-style state:
    // the node still receives config through LocalDb replication while it is up, so what it
    // has lost is recovery from a deploy that landed while it was DOWN. That is degraded,
    // not down — which is exactly what a warning badge says.
    //
    // Tri-state, and only the explicit false is badged. Null means "not reported": the
    // central self-report row (where a site reconciler is meaningless) and any site node on
    // a pre-row-73 build both send null, and badging those would fire on every site the
    // moment central is upgraded ahead of the sites. Offline sites are skipped for the same
    // reason the backlog badge skips them — the Offline badge is the signal that matters.
    private bool IsStartupReconcilerMissing(SiteHealthState state) =>
        state.IsOnline
        && state.LatestReport?.StartupReconcilerActive == false;

    // Review finding M1: warn when any site node (the standby included) reports deployed
    // secrets it cannot apply/decrypt. Null ("no data": central self-report, pre-M1 site) and
    // nodes with an empty failure list are healthy. Offline sites are skipped like the other
    // warning badges — the Offline badge is the signal that matters there.
    private static bool HasSecretMirrorFailures(SiteHealthState state) =>
        state.IsOnline
        && state.LatestReport?.SecretMirrorStatuses is { } statuses
        && statuses.Any(n => n.Failures.Count > 0);

    // Online central nodes, from the same ClusterNodes list the Nodes column renders.
    // Fewer than two means no standby, which disables the manual-failover control. This is
    // a display-side guard only — AkkaManualFailoverService re-checks against live cluster
    // membership, which is the authoritative answer.
    private static int OnlineNodeCount(SiteHealthState state) =>
        state.LatestReport?.ClusterNodes is { Count: > 0 } nodes
            ? nodes.Count(n => n.IsOnline)
            : (state.IsOnline ? 1 : 0);

    // Rows the Nodes group actually renders: the reported cluster-node list when the
    // site sends one, otherwise the single synthesized fallback row. Shown in that
    // group's card-header so the count is legible without reading the table — the same
    // metadata-in-the-header treatment the Data Connections group already had inline.
    private static int NodeRowCount(SiteHealthState state) =>
        state.LatestReport?.ClusterNodes is { Count: > 0 } nodes ? nodes.Count : 1;

    private static string FormatDuration(TimeSpan span) =>
        span.TotalMinutes >= 1 && span == TimeSpan.FromMinutes(Math.Round(span.TotalMinutes))
            ? $"{span.TotalMinutes:0} minute{(span.TotalMinutes == 1 ? "" : "s")}"
            : $"{span.TotalSeconds:0} seconds";

    private IReadOnlyDictionary<string, SiteHealthState> _siteStates = new Dictionary<string, SiteHealthState>();
    private Dictionary<string, string> _siteNames = new();
    private Timer? _refreshTimer;
    private readonly ZB.MOM.WW.ScadaBridge.CentralUI.Services.PollGate _pollGate = new();
    private int _autoRefreshSeconds = 10;

    // Notification Outbox headline KPIs, refreshed alongside the site states.
    private NotificationKpiResponse _outboxKpi =
        new(
            CorrelationId: string.Empty,
            Success: false,
            ErrorMessage: null,
            QueueDepth: 0,
            StuckCount: 0,
            ParkedCount: 0,
            DeliveredLastInterval: 0,
            OldestPendingAge: null);
    private bool _outboxKpiAvailable;
    private string? _outboxKpiError;

    // Audit KPI tiles. Volume + error rate come
    // from a 1h aggregate over the central AuditLog table; backlog sums the
    // per-site SiteAuditBacklog.PendingCount via the health aggregator.
    private AuditLogKpiSnapshot? _auditKpi;
    private bool _auditKpiAvailable;
    private string? _auditKpiError;

    // Site Call Audit (#22) Task 7 — Site Call KPI tiles. Point-in-time counts
    // from the central SiteCalls table, fetched alongside the site states. The
    // SiteCallKpiResponse message doubles as the snapshot the tile takes.
    private SiteCallKpiResponse? _siteCallKpi;
    private bool _siteCallKpiAvailable;
    private string? _siteCallKpiError;

    // Per-node Site Call KPI breakdown (M5.2 per-node stuck-count KPIs).
    // Passed to SiteCallKpiTiles as an optional sub-table.
    private IReadOnlyList<SiteCallNodeKpiSnapshot> _siteCallNodeKpis =
        Array.Empty<SiteCallNodeKpiSnapshot>();
    private bool _siteCallNodeKpiAvailable;

    // ── Site Health Trends (M6) ───────────────────────────────────────────────
    // Per-site Site Health KPI history, loaded on a path entirely separate from
    // the 10s tile-refresh timer (LoadSiteHealthTrendsAsync, never called from
    // the timer tick). The site keys are a snapshot of the dashboard's site set,
    // captured each time trends load so the selector mirrors the live cards.
    // Window in hours: 24h (default) or 168h (7d). Changing the selected site OR
    // the window re-queries. Each metric chart carries its own availability +
    // error so one failed GetSeriesAsync degrades a single chart, never the
    // dashboard.
    private IReadOnlyList<string> _trendSiteKeys = Array.Empty<string>();
    private string? _trendSiteId;
    private int _trendWindowHours = 24;
    private bool _trendsLoading;

    private IReadOnlyList<KpiSeriesPoint>? _connectionsDownSeries;
    private bool _connectionsDownAvailable = true;
    private string? _connectionsDownError;

    private IReadOnlyList<KpiSeriesPoint>? _deadLettersSeries;
    private bool _deadLettersAvailable = true;
    private string? _deadLettersError;

    private IReadOnlyList<KpiSeriesPoint>? _scriptErrorsSeries;
    private bool _scriptErrorsAvailable = true;
    private string? _scriptErrorsError;

    private IReadOnlyList<KpiSeriesPoint>? _sfBufferDepthSeries;
    private bool _sfBufferDepthAvailable = true;
    private string? _sfBufferDepthError;
    private IReadOnlyList<KpiSeriesPoint>? _localDbBacklogSeries;
    private bool _localDbBacklogAvailable = true;
    private string? _localDbBacklogError;

    private static bool SiteHasActiveErrors(SiteHealthState state)
    {
        var report = state.LatestReport;
        if (report == null) return false;
        return report.ScriptErrorCount > 0
            || report.AlarmEvaluationErrorCount > 0
            || report.DeadLetterCount > 0;
    }

    protected override async Task OnInitializedAsync()
    {
        // Load site names for display
        try
        {
            var sites = await SiteRepository.GetAllSitesAsync();
            _siteNames = sites.ToDictionary(s => s.SiteIdentifier, s => s.Name);
        }
        catch
        {
            // Non-fatal — fall back to showing siteId only
        }

        // First load rides whatever the shared KPI cache already has fresh — a second
        // operator opening the dashboard inside the window costs no extra query.
        await RefreshAllAsync(forceRefresh: false);

        // Site Health Trends (M6) load on their own path — never from the
        // timer tick below — so a trend-query fault can't disturb the live tile
        // refresh. Seed the selector from the sites just loaded into _siteStates
        // and query the default site.
        await LoadSiteHealthTrendsAsync();

        _refreshTimer = new Timer(_ =>
        {
            _ = _pollGate.RunAsync(() => InvokeAsync(async () =>
            {
                if (_disposed) return;
                await RefreshAllAsync(forceRefresh: false);
                StateHasChanged();
            }), error => System.Diagnostics.Trace.TraceWarning("Health refresh failed: {0}", error));
        }, null, TimeSpan.FromSeconds(_autoRefreshSeconds), TimeSpan.FromSeconds(_autoRefreshSeconds));
    }

    // Re-query when the operator picks a different site. Best-effort: the load
    // itself swallows faults per-chart.
    private async Task OnTrendSiteChangedAsync(ChangeEventArgs e)
    {
        var selected = e.Value?.ToString();
        if (string.IsNullOrEmpty(selected) || selected == _trendSiteId)
        {
            return;
        }

        _trendSiteId = selected;
        await LoadSiteHealthTrendsAsync(refreshSiteKeys: false);
    }

    // Re-query when the window toggle changes (24h ↔ 7d).
    private async Task SetTrendWindowAsync(int windowHours)
    {
        if (_trendWindowHours == windowHours)
        {
            return;
        }

        _trendWindowHours = windowHours;
        await LoadSiteHealthTrendsAsync(refreshSiteKeys: false);
    }

    // Loads the four Site Health trend series for the selected site over the
    // selected window. Deliberately decoupled from RefreshNow / the 10s timer:
    // a fault here degrades the affected chart(s) only and never propagates to
    // the tile-refresh loop.
    //
    // refreshSiteKeys re-snapshots the dashboard's site set into the selector
    // (true on init); the site-change / window-toggle paths pass false so a
    // mid-interaction site addition/removal can't yank the operator's choice.
    private async Task LoadSiteHealthTrendsAsync(bool refreshSiteKeys = true)
    {
        if (refreshSiteKeys)
        {
            // Mirror the dashboard ordering: central cluster pinned first, then
            // sites alphabetically — the same comparer the detail cards use.
            _trendSiteKeys = _siteStates.Keys
                .OrderBy(k => k == CentralHealthReportLoop.CentralSiteId ? 0 : 1)
                .ThenBy(k => k)
                .ToList();

            // Default to the first site (or keep a still-valid prior selection).
            if (_trendSiteId == null || !_trendSiteKeys.Contains(_trendSiteId))
            {
                _trendSiteId = _trendSiteKeys.FirstOrDefault();
            }
        }

        if (string.IsNullOrEmpty(_trendSiteId))
        {
            return;
        }

        _trendsLoading = true;
        try
        {
            var toUtc = DateTime.UtcNow;
            var fromUtc = toUtc - TimeSpan.FromHours(_trendWindowHours);
            var siteId = _trendSiteId;

            (_connectionsDownSeries, _connectionsDownAvailable, _connectionsDownError) =
                await LoadTrendSeriesAsync(KpiMetrics.SiteHealth.ConnectionsDown, siteId, fromUtc, toUtc);
            (_deadLettersSeries, _deadLettersAvailable, _deadLettersError) =
                await LoadTrendSeriesAsync(KpiMetrics.SiteHealth.DeadLetters, siteId, fromUtc, toUtc);
            (_scriptErrorsSeries, _scriptErrorsAvailable, _scriptErrorsError) =
                await LoadTrendSeriesAsync(KpiMetrics.SiteHealth.ScriptErrors, siteId, fromUtc, toUtc);
            (_sfBufferDepthSeries, _sfBufferDepthAvailable, _sfBufferDepthError) =
                await LoadTrendSeriesAsync(KpiMetrics.SiteHealth.SfBufferDepth, siteId, fromUtc, toUtc);
            (_localDbBacklogSeries, _localDbBacklogAvailable, _localDbBacklogError) =
                await LoadTrendSeriesAsync(KpiMetrics.SiteHealth.LocalDbOplogBacklog, siteId, fromUtc, toUtc);
        }
        finally
        {
            _trendsLoading = false;
        }
    }

    // Single best-effort series fetch. Site Health metrics are Site-scoped, so
    // scope = KpiScopes.Site and scopeKey = the selected site id. On any fault
    // the chart falls back to the unavailable placeholder — a failure here must
    // NEVER break the dashboard.
    private async Task<(IReadOnlyList<KpiSeriesPoint>?, bool, string?)> LoadTrendSeriesAsync(
        string metric, string siteId, DateTime fromUtc, DateTime toUtc)
    {
        try
        {
            var series = await KpiHistory.GetSeriesAsync(
                KpiSources.SiteHealth, metric, KpiScopes.Site, siteId, fromUtc, toUtc);
            return (series, true, null);
        }
        catch
        {
            return (null, false, "Trend data unavailable.");
        }
    }

    private string TrendSiteLabel(string siteKey) =>
        siteKey == CentralHealthReportLoop.CentralSiteId
            ? "Central Cluster"
            : $"{GetSiteName(siteKey)} ({siteKey})";

    // Per-tile budget for one refresh tick. The underlying KPI Asks carry the
    // 30s CommunicationOptions.QueryTimeout, so without this a single hung
    // singleton stalled the WHOLE tick (and, with a 10s timer, stacked ticks)
    // rather than degrading its own tile. Cancelling here abandons only this
    // circuit's wait — the shared KPI flight continues and its result lands in
    // the process-level cache for the next tick.
    private static readonly TimeSpan TileTimeout = TimeSpan.FromSeconds(5);

    // The operator's explicit "Refresh Now" bypasses the shared cache's freshness
    // window; the timer tick and the first load ride whatever is already fresh
    // (which is the whole point — N dashboards share one query per window).
    private Task RefreshNow() => RefreshAllAsync(forceRefresh: true);

    // Every tile loads concurrently, each with its own short budget, so one slow
    // KPI degrades one tile instead of the tick. Every loader swallows its own
    // faults, so Task.WhenAll here can never observe an exception.
    private async Task RefreshAllAsync(bool forceRefresh)
    {
        _siteStates = HealthAggregator.GetAllSiteStates();
        await Task.WhenAll(
            LoadOutboxKpis(forceRefresh),
            LoadSiteCallKpis(forceRefresh),
            LoadSiteCallNodeKpis(forceRefresh),
            LoadAuditKpis(forceRefresh));
    }

    private async Task LoadOutboxKpis(bool forceRefresh)
    {
        try
        {
            using var timeout = new CancellationTokenSource(TileTimeout);
            var response = await KpiSnapshots.GetNotificationKpisAsync(forceRefresh, timeout.Token);
            if (response.Success)
            {
                _outboxKpi = response;
                _outboxKpiAvailable = true;
                _outboxKpiError = null;
            }
            else
            {
                _outboxKpiAvailable = false;
                _outboxKpiError = response.ErrorMessage ?? "KPI query failed.";
            }
        }
        catch (OperationCanceledException)
        {
            _outboxKpiAvailable = false;
            _outboxKpiError = "KPI query timed out.";
        }
        catch (Exception ex)
        {
            _outboxKpiAvailable = false;
            _outboxKpiError = $"KPI query failed: {ex.Message}";
        }
    }

    // Site Call KPI loader: wraps the service call so a transient fault degrades
    // the three Site Call tiles to em dashes with an inline error rather than
    // killing the dashboard. Mirrors LoadOutboxKpis's error handling shape — a
    // response with Success == false (repository fault) and an Ask that threw
    // (transport fault) both collapse to "unavailable".
    private async Task LoadSiteCallKpis(bool forceRefresh)
    {
        try
        {
            using var timeout = new CancellationTokenSource(TileTimeout);
            var response = await KpiSnapshots.GetSiteCallKpisAsync(forceRefresh, timeout.Token);
            if (response.Success)
            {
                _siteCallKpi = response;
                _siteCallKpiAvailable = true;
                _siteCallKpiError = null;
            }
            else
            {
                _siteCallKpiAvailable = false;
                _siteCallKpiError = response.ErrorMessage ?? "KPI query failed.";
            }
        }
        catch (OperationCanceledException)
        {
            _siteCallKpiAvailable = false;
            _siteCallKpiError = "KPI query timed out.";
        }
        catch (Exception ex)
        {
            _siteCallKpiAvailable = false;
            _siteCallKpiError = $"KPI query failed: {ex.Message}";
        }
    }

    // Per-node site-call KPI loader (M5.2). Best-effort; a fault silently
    // suppresses the per-node sub-table rather than degrading the dashboard.
    private async Task LoadSiteCallNodeKpis(bool forceRefresh)
    {
        try
        {
            using var timeout = new CancellationTokenSource(TileTimeout);
            var response = await KpiSnapshots.GetPerNodeSiteCallKpisAsync(forceRefresh, timeout.Token);
            if (response.Success)
            {
                _siteCallNodeKpis = response.Nodes;
                _siteCallNodeKpiAvailable = true;
            }
            else
            {
                _siteCallNodeKpiAvailable = false;
            }
        }
        catch
        {
            _siteCallNodeKpiAvailable = false;
        }
    }

    // Tiles show the numeric KPI when available, or an em dash when the outbox
    // KPI query failed — matching how the page renders other unavailable data.
    private string OutboxTileValue(int value) =>
        _outboxKpiAvailable ? value.ToString() : "—";

    // Audit KPI loader: wraps the service call so a transient DB outage degrades
    // the three tiles to em dashes with an inline error rather than killing the
    // dashboard. Mirrors LoadOutboxKpis's error handling shape.
    private async Task LoadAuditKpis(bool forceRefresh)
    {
        try
        {
            using var timeout = new CancellationTokenSource(TileTimeout);
            _auditKpi = await KpiSnapshots.GetAuditKpiSnapshotAsync(forceRefresh, timeout.Token);
            _auditKpiAvailable = true;
            _auditKpiError = null;
        }
        catch (OperationCanceledException)
        {
            _auditKpiAvailable = false;
            _auditKpiError = "KPI query timed out.";
        }
        catch (Exception ex)
        {
            _auditKpiAvailable = false;
            _auditKpiError = $"KPI query failed: {ex.Message}";
        }
    }

    private string GetSiteName(string siteId)
    {
        return _siteNames.GetValueOrDefault(siteId, siteId);
    }

    private static string GetConnectionHealthBadge(ConnectionHealth health) => health switch
    {
        ConnectionHealth.Connected => "bg-success",
        ConnectionHealth.Connecting => "bg-warning text-dark",
        ConnectionHealth.Disconnected => "bg-danger",
        _ => "bg-secondary"
    };

    private volatile bool _disposed;

    public void Dispose()
    {
        _disposed = true;
        _refreshTimer?.Dispose();
    }
}