ScadaBridge/src/ZB.MOM.WW.ScadaBridge.HealthMonitoring/CentralHealthAggregator.cs

using System.Collections.Concurrent;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Health;

namespace ZB.MOM.WW.ScadaBridge.HealthMonitoring;

/// <summary>
/// Central-side aggregator that receives health reports from all sites,
/// tracks latest metrics in memory, and detects offline sites.
/// No persistence — display-only for Central UI consumption.
/// </summary>
public class CentralHealthAggregator : BackgroundService, ICentralHealthAggregator
{
    private readonly ConcurrentDictionary<string, SiteHealthState> _siteStates = new();
    private readonly HealthMonitoringOptions _options;
    private readonly ILogger<CentralHealthAggregator> _logger;
    private readonly TimeProvider _timeProvider;

    /// <summary>Initializes a new instance of <see cref="CentralHealthAggregator"/>.</summary>
    /// <param name="options">Health monitoring configuration.</param>
    /// <param name="logger">Logger for aggregator diagnostics.</param>
    /// <param name="timeProvider">Optional time provider; defaults to <see cref="TimeProvider.System"/>.</param>
    public CentralHealthAggregator(
        IOptions<HealthMonitoringOptions> options,
        ILogger<CentralHealthAggregator> logger,
        TimeProvider? timeProvider = null)
    {
        _options = options.Value;
        _logger = logger;
        _timeProvider = timeProvider ?? TimeProvider.System;
    }

    /// <inheritdoc />
    public void ProcessReport(SiteHealthReport report)
    {
        var now = _timeProvider.GetUtcNow();

        while (true)
        {
            if (!_siteStates.TryGetValue(report.SiteId, out var existing))
            {
                var registered = new SiteHealthState
                {
                    SiteId = report.SiteId,
                    LatestReport = report,
                    LastReportReceivedAt = now,
                    LastHeartbeatAt = now,
                    LastSequenceNumber = report.SequenceNumber,
                    IsOnline = true
                };

                if (_siteStates.TryAdd(report.SiteId, registered))
                {
                    _logger.LogInformation(
                        "Site {SiteId} registered with sequence #{Seq}", report.SiteId, report.SequenceNumber);
                    return;
                }

                // Lost the race — another thread registered first; retry as an update.
                continue;
            }

            if (report.SequenceNumber <= existing.LastSequenceNumber)
            {
                _logger.LogDebug(
                    "Rejecting stale report from site {SiteId}: seq {Incoming} <= {Last}",
                    report.SiteId, report.SequenceNumber, existing.LastSequenceNumber);
                return;
            }

            var updated = existing with
            {
                LatestReport = report,
                LastReportReceivedAt = now,
                LastHeartbeatAt = now,
                LastSequenceNumber = report.SequenceNumber,
                IsOnline = true
            };

            if (_siteStates.TryUpdate(report.SiteId, updated, existing))
            {
                if (!existing.IsOnline)
                {
                    _logger.LogInformation(
                        "Site {SiteId} is back online (seq #{Seq})", report.SiteId, report.SequenceNumber);
                }
                return;
            }

            // CAS lost — the entry changed under us; retry with the fresh value.
        }
    }

    /// <inheritdoc />
    public void MarkHeartbeat(string siteId, DateTimeOffset receivedAt)
    {
        while (true)
        {
            if (!_siteStates.TryGetValue(siteId, out var existing))
            {
                // Unknown site — register it as online, awaiting its first
                // full report. LatestReport and LastReportReceivedAt both stay
                // null until ProcessReport runs — "no report yet" is an explicit
                // nullable state, not a year-0001 sentinel the UI must special-case.
                var registered = new SiteHealthState
                {
                    SiteId = siteId,
                    LatestReport = null,
                    LastReportReceivedAt = null,
                    LastHeartbeatAt = receivedAt,
                    LastSequenceNumber = 0,
                    IsOnline = true
                };

                if (_siteStates.TryAdd(siteId, registered))
                {
                    _logger.LogInformation(
                        "Site {SiteId} registered online via heartbeat (awaiting first report)", siteId);
                    return;
                }

                // Lost the race — another thread registered first; retry as an update.
                continue;
            }

            // When an offline→online transition is being
            // applied, the heartbeat timestamp must reflect a fresh observation,
            // not the prior stored value. If receivedAt is older than the stored
            // LastHeartbeatAt (clock skew, an out-of-order heartbeat arriving
            // after an earlier one already advanced the field), promoting the
            // site back to online while leaving LastHeartbeatAt stale would let
            // CheckForOfflineSites flap it straight back to offline on the next
            // tick. Anchor the heartbeat to the current time provider instead,
            // so an offline-to-online transition is always backed by an
            // up-to-date heartbeat.
            DateTimeOffset newHeartbeat;
            if (!existing.IsOnline)
            {
                var now = _timeProvider.GetUtcNow();
                newHeartbeat = receivedAt > now ? receivedAt : now;
            }
            else
            {
                newHeartbeat = receivedAt > existing.LastHeartbeatAt
                    ? receivedAt
                    : existing.LastHeartbeatAt;
            }

            // Nothing to change — avoid a needless swap.
            if (newHeartbeat == existing.LastHeartbeatAt && existing.IsOnline)
                return;

            var updated = existing with
            {
                LastHeartbeatAt = newHeartbeat,
                IsOnline = true
            };

            if (_siteStates.TryUpdate(siteId, updated, existing))
            {
                if (!existing.IsOnline)
                    _logger.LogInformation("Site {SiteId} is back online (heartbeat)", siteId);
                return;
            }

            // CAS lost — retry with the fresh value.
        }
    }

    /// <inheritdoc />
    public IReadOnlyDictionary<string, SiteHealthState> GetAllSiteStates()
    {
        return new Dictionary<string, SiteHealthState>(_siteStates);
    }

    /// <inheritdoc />
    public SiteHealthState? GetSiteState(string siteId)
    {
        _siteStates.TryGetValue(siteId, out var state);
        return state;
    }

    /// <inheritdoc />
    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
    {
        _logger.LogInformation(
            "Central health aggregator started, offline timeout {Timeout}s (central {CentralTimeout}s)",
            _options.OfflineTimeout.TotalSeconds, _options.CentralOfflineTimeout.TotalSeconds);

        // Check at half the shorter of the two offline timeouts so detection is
        // timely for whichever site class (real or "central") has the tighter
        // window — see ComputeCheckInterval.
        using var timer = new PeriodicTimer(ComputeCheckInterval(_options));

        while (await timer.WaitForNextTickAsync(stoppingToken).ConfigureAwait(false))
        {
            CheckForOfflineSites();
        }
    }

    /// <summary>
    /// Computes the offline-check timer cadence: half of the <em>shorter</em> of
    /// <see cref="HealthMonitoringOptions.OfflineTimeout"/> and
    /// <see cref="HealthMonitoringOptions.CentralOfflineTimeout"/>. Deriving it
    /// from the shorter timeout guarantees that whichever site class has the
    /// tighter window is still polled at least twice within it — so if an
    /// operator configures <c>CentralOfflineTimeout</c> smaller than
    /// <c>OfflineTimeout</c>, central offline detection is not delayed by up to a
    /// full <c>OfflineTimeout / 2</c>.
    /// </summary>
    /// <param name="options">The health monitoring options to derive the interval from.</param>
    /// <returns>Half the shorter of the two configured offline timeouts.</returns>
    internal static TimeSpan ComputeCheckInterval(HealthMonitoringOptions options)
    {
        var shorter = options.OfflineTimeout < options.CentralOfflineTimeout
            ? options.OfflineTimeout
            : options.CentralOfflineTimeout;
        return TimeSpan.FromMilliseconds(shorter.TotalMilliseconds / 2);
    }

    /// <summary>Iterates all tracked sites and marks any that have exceeded their offline timeout as offline.</summary>
    internal void CheckForOfflineSites()
    {
        var now = _timeProvider.GetUtcNow();

        foreach (var kvp in _siteStates)
        {
            var state = kvp.Value;
            if (!state.IsOnline) continue;

            // Use LastHeartbeatAt — heartbeats arrive every ~5s from any
            // healthy site node (cadence owned by Cluster Infrastructure /
            // SiteCommunicationActor — CommunicationOptions.TransportHeartbeatInterval),
            // so the 60s OfflineTimeout tolerates several missed heartbeats and
            // only fires when no node can reach central, not during single-node
            // failovers.
            //
            // The synthetic "central" site has no heartbeat source — its only
            // signal is the 30s CentralHealthReportLoop self-report — so it gets
            // a longer grace window (CentralOfflineTimeout) to survive a single
            // skipped/late self-report.
            var timeout = kvp.Key == CentralHealthReportLoop.CentralSiteId
                ? _options.CentralOfflineTimeout
                : _options.OfflineTimeout;

            var elapsed = now - state.LastHeartbeatAt;
            if (elapsed <= timeout)
                continue;

            // Atomically swap to an offline copy. If the CAS loses to a
            // concurrent report/heartbeat the site was just heard from, so
            // leaving it online is the correct outcome — no retry needed.
            var offline = state with { IsOnline = false };
            if (_siteStates.TryUpdate(kvp.Key, offline, state))
            {
                _logger.LogWarning(
                    "Site {SiteId} marked offline — no signal for {Elapsed}s (timeout: {Timeout}s)",
                    state.SiteId, elapsed.TotalSeconds, timeout.TotalSeconds);
            }
        }
    }
}