using System.Collections.Concurrent; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using ZB.MOM.WW.ScadaBridge.Commons.Messages.Health; namespace ZB.MOM.WW.ScadaBridge.HealthMonitoring; /// /// Central-side aggregator that receives health reports from all sites, /// tracks latest metrics in memory, and detects offline sites. /// No persistence — display-only for Central UI consumption. /// public class CentralHealthAggregator : BackgroundService, ICentralHealthAggregator { private readonly ConcurrentDictionary _siteStates = new(); private readonly HealthMonitoringOptions _options; private readonly ILogger _logger; private readonly TimeProvider _timeProvider; /// Initializes a new instance of . /// Health monitoring configuration. /// Logger for aggregator diagnostics. /// Optional time provider; defaults to . public CentralHealthAggregator( IOptions options, ILogger logger, TimeProvider? timeProvider = null) { _options = options.Value; _logger = logger; _timeProvider = timeProvider ?? TimeProvider.System; } /// public void ProcessReport(SiteHealthReport report) { var now = _timeProvider.GetUtcNow(); while (true) { if (!_siteStates.TryGetValue(report.SiteId, out var existing)) { var registered = new SiteHealthState { SiteId = report.SiteId, LatestReport = report, LastReportReceivedAt = now, LastHeartbeatAt = now, LastSequenceNumber = report.SequenceNumber, IsOnline = true }; if (_siteStates.TryAdd(report.SiteId, registered)) { _logger.LogInformation( "Site {SiteId} registered with sequence #{Seq}", report.SiteId, report.SequenceNumber); return; } // Lost the race — another thread registered first; retry as an update. continue; } if (report.SequenceNumber <= existing.LastSequenceNumber) { _logger.LogDebug( "Rejecting stale report from site {SiteId}: seq {Incoming} <= {Last}", report.SiteId, report.SequenceNumber, existing.LastSequenceNumber); return; } var updated = existing with { LatestReport = report, LastReportReceivedAt = now, LastHeartbeatAt = now, LastSequenceNumber = report.SequenceNumber, IsOnline = true }; if (_siteStates.TryUpdate(report.SiteId, updated, existing)) { if (!existing.IsOnline) { _logger.LogInformation( "Site {SiteId} is back online (seq #{Seq})", report.SiteId, report.SequenceNumber); } return; } // CAS lost — the entry changed under us; retry with the fresh value. } } /// public void MarkHeartbeat(string siteId, DateTimeOffset receivedAt) { while (true) { if (!_siteStates.TryGetValue(siteId, out var existing)) { // Unknown site — register it as online, awaiting its first // full report. LatestReport and LastReportReceivedAt both stay // null until ProcessReport runs — "no report yet" is an explicit // nullable state, not a year-0001 sentinel the UI must special-case. var registered = new SiteHealthState { SiteId = siteId, LatestReport = null, LastReportReceivedAt = null, LastHeartbeatAt = receivedAt, LastSequenceNumber = 0, IsOnline = true }; if (_siteStates.TryAdd(siteId, registered)) { _logger.LogInformation( "Site {SiteId} registered online via heartbeat (awaiting first report)", siteId); return; } // Lost the race — another thread registered first; retry as an update. continue; } // When an offline→online transition is being // applied, the heartbeat timestamp must reflect a fresh observation, // not the prior stored value. If receivedAt is older than the stored // LastHeartbeatAt (clock skew, an out-of-order heartbeat arriving // after an earlier one already advanced the field), promoting the // site back to online while leaving LastHeartbeatAt stale would let // CheckForOfflineSites flap it straight back to offline on the next // tick. Anchor the heartbeat to the current time provider instead, // so an offline-to-online transition is always backed by an // up-to-date heartbeat. DateTimeOffset newHeartbeat; if (!existing.IsOnline) { var now = _timeProvider.GetUtcNow(); newHeartbeat = receivedAt > now ? receivedAt : now; } else { newHeartbeat = receivedAt > existing.LastHeartbeatAt ? receivedAt : existing.LastHeartbeatAt; } // Nothing to change — avoid a needless swap. if (newHeartbeat == existing.LastHeartbeatAt && existing.IsOnline) return; var updated = existing with { LastHeartbeatAt = newHeartbeat, IsOnline = true }; if (_siteStates.TryUpdate(siteId, updated, existing)) { if (!existing.IsOnline) _logger.LogInformation("Site {SiteId} is back online (heartbeat)", siteId); return; } // CAS lost — retry with the fresh value. } } /// public IReadOnlyDictionary GetAllSiteStates() { return new Dictionary(_siteStates); } /// public SiteHealthState? GetSiteState(string siteId) { _siteStates.TryGetValue(siteId, out var state); return state; } /// protected override async Task ExecuteAsync(CancellationToken stoppingToken) { _logger.LogInformation( "Central health aggregator started, offline timeout {Timeout}s (central {CentralTimeout}s)", _options.OfflineTimeout.TotalSeconds, _options.CentralOfflineTimeout.TotalSeconds); // Check at half the shorter of the two offline timeouts so detection is // timely for whichever site class (real or "central") has the tighter // window — see ComputeCheckInterval. using var timer = new PeriodicTimer(ComputeCheckInterval(_options)); while (await timer.WaitForNextTickAsync(stoppingToken).ConfigureAwait(false)) { CheckForOfflineSites(); } } /// /// Computes the offline-check timer cadence: half of the shorter of /// and /// . Deriving it /// from the shorter timeout guarantees that whichever site class has the /// tighter window is still polled at least twice within it — so if an /// operator configures CentralOfflineTimeout smaller than /// OfflineTimeout, central offline detection is not delayed by up to a /// full OfflineTimeout / 2. /// /// The health monitoring options to derive the interval from. /// Half the shorter of the two configured offline timeouts. internal static TimeSpan ComputeCheckInterval(HealthMonitoringOptions options) { var shorter = options.OfflineTimeout < options.CentralOfflineTimeout ? options.OfflineTimeout : options.CentralOfflineTimeout; return TimeSpan.FromMilliseconds(shorter.TotalMilliseconds / 2); } /// Iterates all tracked sites and marks any that have exceeded their offline timeout as offline. internal void CheckForOfflineSites() { var now = _timeProvider.GetUtcNow(); foreach (var kvp in _siteStates) { var state = kvp.Value; if (!state.IsOnline) continue; // Use LastHeartbeatAt — heartbeats arrive every ~5s from any // healthy site node (cadence owned by Cluster Infrastructure / // SiteCommunicationActor — CommunicationOptions.TransportHeartbeatInterval), // so the 60s OfflineTimeout tolerates several missed heartbeats and // only fires when no node can reach central, not during single-node // failovers. // // The synthetic "central" site has no heartbeat source — its only // signal is the 30s CentralHealthReportLoop self-report — so it gets // a longer grace window (CentralOfflineTimeout) to survive a single // skipped/late self-report. var timeout = kvp.Key == CentralHealthReportLoop.CentralSiteId ? _options.CentralOfflineTimeout : _options.OfflineTimeout; var elapsed = now - state.LastHeartbeatAt; if (elapsed <= timeout) continue; // Atomically swap to an offline copy. If the CAS loses to a // concurrent report/heartbeat the site was just heard from, so // leaving it online is the correct outcome — no retry needed. var offline = state with { IsOnline = false }; if (_siteStates.TryUpdate(kvp.Key, offline, state)) { _logger.LogWarning( "Site {SiteId} marked offline — no signal for {Elapsed}s (timeout: {Timeout}s)", state.SiteId, elapsed.TotalSeconds, timeout.TotalSeconds); } } } }