using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using ZB.MOM.WW.ScadaBridge.Commons.Messages.Health; using ZB.MOM.WW.ScadaBridge.StoreAndForward; namespace ZB.MOM.WW.ScadaBridge.HealthMonitoring; /// /// Periodically collects a SiteHealthReport and sends it to central via Akka remoting. /// Sequence numbers are monotonic and reset on service restart. They are not /// zero/one-based: the per-process counter is seeded with the current Unix epoch /// (milliseconds) at construction so that, after a failover, reports from a /// freshly-active node always sort after reports from any prior active node for the /// same site — otherwise the central aggregator's sequence-number guard would /// silently reject the new active's first reports as stale. /// public class HealthReportSender : BackgroundService { private readonly ISiteHealthCollector _collector; private readonly IHealthReportTransport _transport; private readonly HealthMonitoringOptions _options; private readonly ILogger _logger; private readonly string _siteId; private readonly StoreAndForwardStorage? _sfStorage; private readonly IClusterNodeProvider? _clusterNodeProvider; // Seeded with Unix-ms at construction so reports from a freshly-active // node always sort after reports from any prior active node for the same // site. Without this seeding, failover would silently drop the new // active's first reports because their per-process counter starts below // the prior active's last sequence number. The clock is read through the // injected TimeProvider so the seeding is deterministically testable. private long _sequenceNumber; /// Initializes the sender, seeds the monotonic sequence number from the current Unix timestamp. /// Site health metric collector supplying the report payload. /// Transport used to send the health report to central. /// Health monitoring options including the report interval. /// Logger instance. /// Provides the site identifier embedded in each report. /// Optional store-and-forward storage for queue depth metrics. /// Optional cluster node provider for active-node detection. /// Optional time provider; defaults to . public HealthReportSender( ISiteHealthCollector collector, IHealthReportTransport transport, IOptions options, ILogger logger, ISiteIdentityProvider siteIdentityProvider, StoreAndForwardStorage? sfStorage = null, IClusterNodeProvider? clusterNodeProvider = null, TimeProvider? timeProvider = null) { _collector = collector; _transport = transport; _options = options.Value; _logger = logger; _siteId = siteIdentityProvider.SiteId; _sfStorage = sfStorage; _clusterNodeProvider = clusterNodeProvider; _sequenceNumber = (timeProvider ?? TimeProvider.System).GetUtcNow().ToUnixTimeMilliseconds(); } /// /// Current sequence number (for testing). /// public long CurrentSequenceNumber => Interlocked.Read(ref _sequenceNumber); /// protected override async Task ExecuteAsync(CancellationToken stoppingToken) { _logger.LogInformation( "Health report sender starting for site {SiteId}, interval {Interval}s", _siteId, _options.ReportInterval.TotalSeconds); using var timer = new PeriodicTimer(_options.ReportInterval); while (await timer.WaitForNextTickAsync(stoppingToken).ConfigureAwait(false)) { try { // Only the active node (running the DeploymentManager singleton) sends health reports. // The standby node has no instance/connection data and would overwrite the active's report. if (!_collector.IsActiveNode) continue; if (_clusterNodeProvider != null) { try { _collector.SetClusterNodes(_clusterNodeProvider.GetClusterNodes()); } catch (Exception ex) { // Non-fatal — the report ships with the previous cluster // node list. Logged so a persistent failure is diagnosable. _logger.LogWarning(ex, "Failed to refresh cluster nodes for health report (site {SiteId}); using stale list", _siteId); } } if (_sfStorage != null) { try { var parkedCount = await _sfStorage.GetParkedMessageCountAsync(); _collector.SetParkedMessageCount(parkedCount); } catch (Exception ex) { // Non-fatal — parked count will be 0 in this report. _logger.LogWarning(ex, "Failed to query parked message count for health report (site {SiteId})", _siteId); } try { // Per-category pending-message buffer depths (the documented // "store-and-forward buffer depth" triage metric). Keyed by // StoreAndForwardCategory name so the central dashboard can // render external/notification/DB-write depths separately. var depthsByCategory = await _sfStorage.GetBufferDepthByCategoryAsync(); var depths = depthsByCategory.ToDictionary( kvp => kvp.Key.ToString(), kvp => kvp.Value); _collector.SetStoreAndForwardDepths(depths); } catch (Exception ex) { // Non-fatal — buffer depths will be empty in this report. _logger.LogWarning(ex, "Failed to query store-and-forward buffer depths for health report (site {SiteId})", _siteId); } } var seq = Interlocked.Increment(ref _sequenceNumber); // HealthMonitoring-017: CollectReport atomically read-and-resets // the per-interval error counters via Interlocked.Exchange. If // the Send below throws, those counts are otherwise lost // forever — neither in the un-sent report nor in the now-zeroed // collector. Snapshot the freshly-collected report so that on a // transport failure we can atomically restore the counts back // into the collector via Interlocked.Add, so the next // successful report includes them. Concurrent increments // arriving during the Send are preserved on the counter (they // accumulate against zero); the restore Add safely sums with // any such concurrent increments. var report = _collector.CollectReport(_siteId); // Replace the placeholder sequence number with our monotonic one var reportWithSeq = report with { SequenceNumber = seq }; try { _transport.Send(reportWithSeq); } catch { // Restore the captured per-interval counters atomically so // they roll forward into the next report — see // HealthMonitoring-017. Any concurrent increment that // arrived during the failed Send remains on the counter; // Interlocked.Add sums correctly with it. _collector.AddIntervalCounters( scriptErrors: report.ScriptErrorCount, alarmErrors: report.AlarmEvaluationErrorCount, deadLetters: report.DeadLetterCount, siteAuditWriteFailures: report.SiteAuditWriteFailures, auditRedactionFailures: report.AuditRedactionFailure); throw; } _logger.LogInformation("Sent health report #{Seq} for site {SiteId}", seq, _siteId); } catch (Exception ex) { _logger.LogError(ex, "Failed to send health report for site {SiteId}", _siteId); // Continue sending — don't let a single failure stop reporting } } } }