ScadaBridge/src/ZB.MOM.WW.ScadaBridge.HealthMonitoring/CentralHealthAggregator.cs

using System.Collections.Concurrent;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Health;

namespace ZB.MOM.WW.ScadaBridge.HealthMonitoring;

/// <summary>
/// Central-side aggregator that receives health reports from all sites,
/// tracks latest metrics in memory, and detects offline sites.
/// No persistence — display-only for Central UI consumption.
/// </summary>
public class CentralHealthAggregator : BackgroundService, ICentralHealthAggregator
{
    private readonly ConcurrentDictionary<string, SiteHealthState> _siteStates = new();
    private readonly HealthMonitoringOptions _options;
    private readonly ILogger<CentralHealthAggregator> _logger;
    private readonly TimeProvider _timeProvider;

    /// <summary>Initializes a new instance of <see cref="CentralHealthAggregator"/>.</summary>
    /// <param name="options">Health monitoring configuration.</param>
    /// <param name="logger">Logger for aggregator diagnostics.</param>
    /// <param name="timeProvider">Optional time provider; defaults to <see cref="TimeProvider.System"/>.</param>
    public CentralHealthAggregator(
        IOptions<HealthMonitoringOptions> options,
        ILogger<CentralHealthAggregator> logger,
        TimeProvider? timeProvider = null)
    {
        _options = options.Value;
        _logger = logger;
        _timeProvider = timeProvider ?? TimeProvider.System;
    }

    /// <inheritdoc />
    public void ProcessReport(SiteHealthReport report)
    {
        var now = _timeProvider.GetUtcNow();

        while (true)
        {
            if (!_siteStates.TryGetValue(report.SiteId, out var existing))
            {
                var registered = new SiteHealthState
                {
                    SiteId = report.SiteId,
                    LatestReport = report,
                    LastReportReceivedAt = now,
                    LastHeartbeatAt = now,
                    FirstSeenAt = now,
                    LastSequenceNumber = report.SequenceNumber,
                    IsOnline = true,
                    IsMetricsStale = false,
                    LocalDbBacklogAboveSince = NextBacklogAboveSince(null, report, now)
                    // LastStatusChangeAt stays null: first observation is not a flip.
                };

                if (_siteStates.TryAdd(report.SiteId, registered))
                {
                    _logger.LogInformation(
                        "Site {SiteId} registered with sequence #{Seq}", report.SiteId, report.SequenceNumber);
                    return;
                }

                // Lost the race — another thread registered first; retry as an update.
                continue;
            }

            if (report.SequenceNumber <= existing.LastSequenceNumber)
            {
                _logger.LogDebug(
                    "Rejecting stale report from site {SiteId}: seq {Incoming} <= {Last}",
                    report.SiteId, report.SequenceNumber, existing.LastSequenceNumber);
                return;
            }

            var updated = existing with
            {
                LatestReport = report,
                LastReportReceivedAt = now,
                LastHeartbeatAt = now,
                LastSequenceNumber = report.SequenceNumber,
                IsOnline = true,
                // A fresh report is proof the metrics pipeline is alive again.
                IsMetricsStale = false,
                // Stamp a flip only when this report brings the site back online;
                // a report from an already-online site leaves the timestamp be.
                LastStatusChangeAt = existing.IsOnline ? existing.LastStatusChangeAt : now,
                LocalDbBacklogAboveSince =
                    NextBacklogAboveSince(existing.LocalDbBacklogAboveSince, report, now)
            };

            if (_siteStates.TryUpdate(report.SiteId, updated, existing))
            {
                if (!existing.IsOnline)
                {
                    _logger.LogInformation(
                        "Site {SiteId} is back online (seq #{Seq})", report.SiteId, report.SequenceNumber);
                }
                return;
            }

            // CAS lost — the entry changed under us; retry with the fresh value.
        }
    }

    /// <inheritdoc />
    public void MarkHeartbeat(string siteId, DateTimeOffset receivedAt)
    {
        while (true)
        {
            if (!_siteStates.TryGetValue(siteId, out var existing))
            {
                // Unknown site — register it as online, awaiting its first
                // full report. LatestReport and LastReportReceivedAt both stay
                // null until ProcessReport runs — "no report yet" is an explicit
                // nullable state, not a DateTime.MinValue sentinel the UI must special-case.
                var registered = new SiteHealthState
                {
                    SiteId = siteId,
                    LatestReport = null,
                    LastReportReceivedAt = null,
                    LastHeartbeatAt = receivedAt,
                    FirstSeenAt = receivedAt,
                    LastSequenceNumber = 0,
                    IsOnline = true
                };

                if (_siteStates.TryAdd(siteId, registered))
                {
                    _logger.LogInformation(
                        "Site {SiteId} registered online via heartbeat (awaiting first report)", siteId);
                    return;
                }

                // Lost the race — another thread registered first; retry as an update.
                continue;
            }

            // When an offline→online transition is being
            // applied, the heartbeat timestamp must reflect a fresh observation,
            // not the prior stored value. If receivedAt is older than the stored
            // LastHeartbeatAt (clock skew, an out-of-order heartbeat arriving
            // after an earlier one already advanced the field), promoting the
            // site back to online while leaving LastHeartbeatAt stale would let
            // CheckForOfflineSites flap it straight back to offline on the next
            // tick. Anchor the heartbeat to the current time provider instead,
            // so an offline-to-online transition is always backed by an
            // up-to-date heartbeat.
            DateTimeOffset newHeartbeat;
            if (!existing.IsOnline)
            {
                var now = _timeProvider.GetUtcNow();
                newHeartbeat = receivedAt > now ? receivedAt : now;
            }
            else
            {
                newHeartbeat = receivedAt > existing.LastHeartbeatAt
                    ? receivedAt
                    : existing.LastHeartbeatAt;
            }

            // Nothing to change — avoid a needless swap.
            if (newHeartbeat == existing.LastHeartbeatAt && existing.IsOnline)
                return;

            var updated = existing with
            {
                LastHeartbeatAt = newHeartbeat,
                IsOnline = true,
                // Stamp the flip when this heartbeat promotes an offline site back
                // online. IsMetricsStale is deliberately left untouched — a
                // heartbeat says nothing about whether the metrics pipeline
                // recovered; only a full report clears staleness.
                LastStatusChangeAt = existing.IsOnline
                    ? existing.LastStatusChangeAt
                    : _timeProvider.GetUtcNow()
            };

            if (_siteStates.TryUpdate(siteId, updated, existing))
            {
                if (!existing.IsOnline)
                    _logger.LogInformation("Site {SiteId} is back online (heartbeat)", siteId);
                return;
            }

            // CAS lost — retry with the fresh value.
        }
    }

    /// <inheritdoc />
    public IReadOnlyDictionary<string, SiteHealthState> GetAllSiteStates()
    {
        return new Dictionary<string, SiteHealthState>(_siteStates);
    }

    /// <inheritdoc />
    public SiteHealthState? GetSiteState(string siteId)
    {
        _siteStates.TryGetValue(siteId, out var state);
        return state;
    }

    /// <inheritdoc />
    public void PruneUnknownSites(IReadOnlyCollection<string> knownSiteIds)
    {
        var known = new HashSet<string>(knownSiteIds, StringComparer.Ordinal);
        foreach (var siteId in _siteStates.Keys)
        {
            if (siteId == CentralHealthReportLoop.CentralSiteId || known.Contains(siteId))
                continue;
            if (_siteStates.TryRemove(siteId, out _))
                _logger.LogInformation(
                    "Site {SiteId} evicted from health aggregator (no longer configured)", siteId);
        }
    }

    /// <inheritdoc />
    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
    {
        _logger.LogInformation(
            "Central health aggregator started, offline timeout {Timeout}s (central {CentralTimeout}s)",
            _options.OfflineTimeout.TotalSeconds, _options.CentralOfflineTimeout.TotalSeconds);

        // Check at half the shorter of the two offline timeouts so detection is
        // timely for whichever site class (real or "central") has the tighter
        // window — see ComputeCheckInterval.
        using var timer = new PeriodicTimer(ComputeCheckInterval(_options));

        while (await timer.WaitForNextTickAsync(stoppingToken).ConfigureAwait(false))
        {
            CheckForOfflineSites();
        }
    }

    /// <summary>
    /// Computes the offline-check timer cadence: half of the <em>shorter</em> of
    /// <see cref="HealthMonitoringOptions.OfflineTimeout"/> and
    /// <see cref="HealthMonitoringOptions.CentralOfflineTimeout"/>. Deriving it
    /// from the shorter timeout guarantees that whichever site class has the
    /// tighter window is still polled at least twice within it — so if an
    /// operator configures <c>CentralOfflineTimeout</c> smaller than
    /// <c>OfflineTimeout</c>, central offline detection is not delayed by up to a
    /// full <c>OfflineTimeout / 2</c>.
    /// </summary>
    /// <param name="options">The health monitoring options to derive the interval from.</param>
    /// <returns>Half the shorter of the two configured offline timeouts.</returns>
    internal static TimeSpan ComputeCheckInterval(HealthMonitoringOptions options)
    {
        var shorter = options.OfflineTimeout < options.CentralOfflineTimeout
            ? options.OfflineTimeout
            : options.CentralOfflineTimeout;
        return TimeSpan.FromMilliseconds(shorter.TotalMilliseconds / 2);
    }

    /// <summary>Iterates all tracked sites and marks any that have exceeded their offline timeout as offline.</summary>
    internal void CheckForOfflineSites()
    {
        var now = _timeProvider.GetUtcNow();

        foreach (var kvp in _siteStates)
        {
            var state = kvp.Value;
            if (!state.IsOnline) continue;

            // Use LastHeartbeatAt — heartbeats arrive every ~5s from any
            // healthy site node (cadence owned by Cluster Infrastructure /
            // SiteCommunicationActor — CommunicationOptions.TransportHeartbeatInterval),
            // so the 60s OfflineTimeout tolerates several missed heartbeats and
            // only fires when no node can reach central, not during single-node
            // failovers.
            //
            // The synthetic "central" site has no heartbeat source — its only
            // signal is the 30s CentralHealthReportLoop self-report — so it gets
            // a longer grace window (CentralOfflineTimeout) to survive a single
            // skipped/late self-report.
            var timeout = kvp.Key == CentralHealthReportLoop.CentralSiteId
                ? _options.CentralOfflineTimeout
                : _options.OfflineTimeout;

            var elapsed = now - state.LastHeartbeatAt;
            if (elapsed > timeout)
            {
                // Atomically swap to an offline copy, stamping the flip. If the
                // CAS loses to a concurrent report/heartbeat the site was just
                // heard from, so leaving it online is correct — no retry needed.
                var offline = state with { IsOnline = false, LastStatusChangeAt = now };
                if (_siteStates.TryUpdate(kvp.Key, offline, state))
                {
                    _logger.LogWarning(
                        "Site {SiteId} marked offline — no signal for {Elapsed}s (timeout: {Timeout}s)",
                        state.SiteId, elapsed.TotalSeconds, timeout.TotalSeconds);
                }

                // Whether the CAS won or lost, do not also evaluate staleness on
                // this stale snapshot — offline supersedes metrics-stale.
                continue;
            }

            // Second signal: the site is still live (heartbeats within timeout)
            // but its full-report pipeline may have died. Flag it metrics-stale
            // when no report has arrived within MetricsStaleTimeout. Distinct from
            // offline — a site whose HealthReportSender crashed keeps heartbeating
            // and would otherwise show "online with frozen metrics forever".
            // A site that has NEVER reported (heartbeat-only registration,
            // LastReportReceivedAt == null) anchors on FirstSeenAt instead —
            // otherwise a pipeline dead from first boot is never flagged (N3).
            var reportAnchor = state.LastReportReceivedAt ?? state.FirstSeenAt;
            if (reportAnchor is { } anchor
                && now - anchor > _options.MetricsStaleTimeout
                && !state.IsMetricsStale)
            {
                var stale = state with { IsMetricsStale = true };
                if (_siteStates.TryUpdate(kvp.Key, stale, state))
                {
                    _logger.LogWarning(
                        "Site {SiteId} metrics are stale — online (heartbeats) but no report for {Elapsed}s{NeverNote} (timeout: {Timeout}s)",
                        state.SiteId, (now - anchor).TotalSeconds,
                        state.LastReportReceivedAt is null ? " (never reported since first contact)" : string.Empty,
                        _options.MetricsStaleTimeout.TotalSeconds);
                }
                // CAS loss ⇒ a fresh report/heartbeat swapped in ⇒ correct to skip.
            }
        }
    }

    /// <summary>
    /// Advances the sustained-backlog anchor for one incoming report (register row 33).
    /// </summary>
    /// <remarks>
    /// Three cases, and the null one is the subtle one:
    /// <list type="bullet">
    /// <item><description>
    /// Backlog <c>null</c> — no reading. An unreplicated site (rig site-b/site-c) reports this
    /// forever, and a replicated site reports it when a poll fails. Carry the previous anchor
    /// through UNCHANGED: null is not zero, so it is neither evidence of a drain (which would
    /// justify clearing) nor of a backlog (which would justify starting one). An unreplicated
    /// site therefore never accumulates an anchor and can never be badged.
    /// </description></item>
    /// <item><description>
    /// Backlog above the threshold — start the clock if it is not already running. Keeping the
    /// ORIGINAL start instant is what makes the window sustained rather than sliding.
    /// </description></item>
    /// <item><description>
    /// Backlog at or below the threshold — clear. This is the daily purge burst's exit path:
    /// the spike drains within a few reports and the anchor is discarded long before the dwell
    /// elapses, so the expected burst never badges the site.
    /// </description></item>
    /// </list>
    /// </remarks>
    /// <param name="current">The anchor currently held for this site, if any.</param>
    /// <param name="report">The incoming health report.</param>
    /// <param name="now">The current time.</param>
    /// <returns>The anchor to store.</returns>
    private DateTimeOffset? NextBacklogAboveSince(
        DateTimeOffset? current, SiteHealthReport report, DateTimeOffset now)
    {
        if (report.LocalDbOplogBacklog is not { } backlog)
        {
            return current;
        }

        return backlog > _options.LocalDbBacklogThreshold ? current ?? now : null;
    }
}