using System.Collections.Concurrent; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using ZB.MOM.WW.ScadaBridge.Commons.Messages.Health; namespace ZB.MOM.WW.ScadaBridge.HealthMonitoring; /// /// Central-side aggregator that receives health reports from all sites, /// tracks latest metrics in memory, and detects offline sites. /// No persistence — display-only for Central UI consumption. /// public class CentralHealthAggregator : BackgroundService, ICentralHealthAggregator { private readonly ConcurrentDictionary _siteStates = new(); private readonly HealthMonitoringOptions _options; private readonly ILogger _logger; private readonly TimeProvider _timeProvider; /// Initializes a new instance of . /// Health monitoring configuration. /// Logger for aggregator diagnostics. /// Optional time provider; defaults to . public CentralHealthAggregator( IOptions options, ILogger logger, TimeProvider? timeProvider = null) { _options = options.Value; _logger = logger; _timeProvider = timeProvider ?? TimeProvider.System; } /// public void ProcessReport(SiteHealthReport report) { var now = _timeProvider.GetUtcNow(); while (true) { if (!_siteStates.TryGetValue(report.SiteId, out var existing)) { var registered = new SiteHealthState { SiteId = report.SiteId, LatestReport = report, LastReportReceivedAt = now, LastHeartbeatAt = now, FirstSeenAt = now, LastSequenceNumber = report.SequenceNumber, IsOnline = true, IsMetricsStale = false // LastStatusChangeAt stays null: first observation is not a flip. }; if (_siteStates.TryAdd(report.SiteId, registered)) { _logger.LogInformation( "Site {SiteId} registered with sequence #{Seq}", report.SiteId, report.SequenceNumber); return; } // Lost the race — another thread registered first; retry as an update. continue; } if (report.SequenceNumber <= existing.LastSequenceNumber) { _logger.LogDebug( "Rejecting stale report from site {SiteId}: seq {Incoming} <= {Last}", report.SiteId, report.SequenceNumber, existing.LastSequenceNumber); return; } var updated = existing with { LatestReport = report, LastReportReceivedAt = now, LastHeartbeatAt = now, LastSequenceNumber = report.SequenceNumber, IsOnline = true, // A fresh report is proof the metrics pipeline is alive again. IsMetricsStale = false, // Stamp a flip only when this report brings the site back online; // a report from an already-online site leaves the timestamp be. LastStatusChangeAt = existing.IsOnline ? existing.LastStatusChangeAt : now }; if (_siteStates.TryUpdate(report.SiteId, updated, existing)) { if (!existing.IsOnline) { _logger.LogInformation( "Site {SiteId} is back online (seq #{Seq})", report.SiteId, report.SequenceNumber); } return; } // CAS lost — the entry changed under us; retry with the fresh value. } } /// public void MarkHeartbeat(string siteId, DateTimeOffset receivedAt) { while (true) { if (!_siteStates.TryGetValue(siteId, out var existing)) { // Unknown site — register it as online, awaiting its first // full report. LatestReport and LastReportReceivedAt both stay // null until ProcessReport runs — "no report yet" is an explicit // nullable state, not a DateTime.MinValue sentinel the UI must special-case. var registered = new SiteHealthState { SiteId = siteId, LatestReport = null, LastReportReceivedAt = null, LastHeartbeatAt = receivedAt, FirstSeenAt = receivedAt, LastSequenceNumber = 0, IsOnline = true }; if (_siteStates.TryAdd(siteId, registered)) { _logger.LogInformation( "Site {SiteId} registered online via heartbeat (awaiting first report)", siteId); return; } // Lost the race — another thread registered first; retry as an update. continue; } // When an offline→online transition is being // applied, the heartbeat timestamp must reflect a fresh observation, // not the prior stored value. If receivedAt is older than the stored // LastHeartbeatAt (clock skew, an out-of-order heartbeat arriving // after an earlier one already advanced the field), promoting the // site back to online while leaving LastHeartbeatAt stale would let // CheckForOfflineSites flap it straight back to offline on the next // tick. Anchor the heartbeat to the current time provider instead, // so an offline-to-online transition is always backed by an // up-to-date heartbeat. DateTimeOffset newHeartbeat; if (!existing.IsOnline) { var now = _timeProvider.GetUtcNow(); newHeartbeat = receivedAt > now ? receivedAt : now; } else { newHeartbeat = receivedAt > existing.LastHeartbeatAt ? receivedAt : existing.LastHeartbeatAt; } // Nothing to change — avoid a needless swap. if (newHeartbeat == existing.LastHeartbeatAt && existing.IsOnline) return; var updated = existing with { LastHeartbeatAt = newHeartbeat, IsOnline = true, // Stamp the flip when this heartbeat promotes an offline site back // online. IsMetricsStale is deliberately left untouched — a // heartbeat says nothing about whether the metrics pipeline // recovered; only a full report clears staleness. LastStatusChangeAt = existing.IsOnline ? existing.LastStatusChangeAt : _timeProvider.GetUtcNow() }; if (_siteStates.TryUpdate(siteId, updated, existing)) { if (!existing.IsOnline) _logger.LogInformation("Site {SiteId} is back online (heartbeat)", siteId); return; } // CAS lost — retry with the fresh value. } } /// public IReadOnlyDictionary GetAllSiteStates() { return new Dictionary(_siteStates); } /// public SiteHealthState? GetSiteState(string siteId) { _siteStates.TryGetValue(siteId, out var state); return state; } /// public void PruneUnknownSites(IReadOnlyCollection knownSiteIds) { var known = new HashSet(knownSiteIds, StringComparer.Ordinal); foreach (var siteId in _siteStates.Keys) { if (siteId == CentralHealthReportLoop.CentralSiteId || known.Contains(siteId)) continue; if (_siteStates.TryRemove(siteId, out _)) _logger.LogInformation( "Site {SiteId} evicted from health aggregator (no longer configured)", siteId); } } /// protected override async Task ExecuteAsync(CancellationToken stoppingToken) { _logger.LogInformation( "Central health aggregator started, offline timeout {Timeout}s (central {CentralTimeout}s)", _options.OfflineTimeout.TotalSeconds, _options.CentralOfflineTimeout.TotalSeconds); // Check at half the shorter of the two offline timeouts so detection is // timely for whichever site class (real or "central") has the tighter // window — see ComputeCheckInterval. using var timer = new PeriodicTimer(ComputeCheckInterval(_options)); while (await timer.WaitForNextTickAsync(stoppingToken).ConfigureAwait(false)) { CheckForOfflineSites(); } } /// /// Computes the offline-check timer cadence: half of the shorter of /// and /// . Deriving it /// from the shorter timeout guarantees that whichever site class has the /// tighter window is still polled at least twice within it — so if an /// operator configures CentralOfflineTimeout smaller than /// OfflineTimeout, central offline detection is not delayed by up to a /// full OfflineTimeout / 2. /// /// The health monitoring options to derive the interval from. /// Half the shorter of the two configured offline timeouts. internal static TimeSpan ComputeCheckInterval(HealthMonitoringOptions options) { var shorter = options.OfflineTimeout < options.CentralOfflineTimeout ? options.OfflineTimeout : options.CentralOfflineTimeout; return TimeSpan.FromMilliseconds(shorter.TotalMilliseconds / 2); } /// Iterates all tracked sites and marks any that have exceeded their offline timeout as offline. internal void CheckForOfflineSites() { var now = _timeProvider.GetUtcNow(); foreach (var kvp in _siteStates) { var state = kvp.Value; if (!state.IsOnline) continue; // Use LastHeartbeatAt — heartbeats arrive every ~5s from any // healthy site node (cadence owned by Cluster Infrastructure / // SiteCommunicationActor — CommunicationOptions.TransportHeartbeatInterval), // so the 60s OfflineTimeout tolerates several missed heartbeats and // only fires when no node can reach central, not during single-node // failovers. // // The synthetic "central" site has no heartbeat source — its only // signal is the 30s CentralHealthReportLoop self-report — so it gets // a longer grace window (CentralOfflineTimeout) to survive a single // skipped/late self-report. var timeout = kvp.Key == CentralHealthReportLoop.CentralSiteId ? _options.CentralOfflineTimeout : _options.OfflineTimeout; var elapsed = now - state.LastHeartbeatAt; if (elapsed > timeout) { // Atomically swap to an offline copy, stamping the flip. If the // CAS loses to a concurrent report/heartbeat the site was just // heard from, so leaving it online is correct — no retry needed. var offline = state with { IsOnline = false, LastStatusChangeAt = now }; if (_siteStates.TryUpdate(kvp.Key, offline, state)) { _logger.LogWarning( "Site {SiteId} marked offline — no signal for {Elapsed}s (timeout: {Timeout}s)", state.SiteId, elapsed.TotalSeconds, timeout.TotalSeconds); } // Whether the CAS won or lost, do not also evaluate staleness on // this stale snapshot — offline supersedes metrics-stale. continue; } // Second signal: the site is still live (heartbeats within timeout) // but its full-report pipeline may have died. Flag it metrics-stale // when no report has arrived within MetricsStaleTimeout. Distinct from // offline — a site whose HealthReportSender crashed keeps heartbeating // and would otherwise show "online with frozen metrics forever". // A site that has NEVER reported (heartbeat-only registration, // LastReportReceivedAt == null) anchors on FirstSeenAt instead — // otherwise a pipeline dead from first boot is never flagged (N3). var reportAnchor = state.LastReportReceivedAt ?? state.FirstSeenAt; if (reportAnchor is { } anchor && now - anchor > _options.MetricsStaleTimeout && !state.IsMetricsStale) { var stale = state with { IsMetricsStale = true }; if (_siteStates.TryUpdate(kvp.Key, stale, state)) { _logger.LogWarning( "Site {SiteId} metrics are stale — online (heartbeats) but no report for {Elapsed}s{NeverNote} (timeout: {Timeout}s)", state.SiteId, (now - anchor).TotalSeconds, state.LastReportReceivedAt is null ? " (never reported since first contact)" : string.Empty, _options.MetricsStaleTimeout.TotalSeconds); } // CAS loss ⇒ a fresh report/heartbeat swapped in ⇒ correct to skip. } } } }