using System.Collections.Concurrent; using ZB.MOM.WW.ScadaBridge.Commons.Messages.Health; using ZB.MOM.WW.ScadaBridge.Commons.Types; using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums; namespace ZB.MOM.WW.ScadaBridge.HealthMonitoring; /// /// Collects health metrics from all site subsystems. /// Thread-safe: counters use Interlocked operations, connection/tag data uses ConcurrentDictionary. /// public class SiteHealthCollector : ISiteHealthCollector { private int _scriptErrorCount; private int _alarmErrorCount; private int _deadLetterCount; private int _siteAuditWriteFailures; private int _auditRedactionFailures; private volatile SiteAuditBacklogSnapshot? _siteAuditBacklog; private long _siteEventLogWriteFailures; // One volatile tuple rather than two independent fields: a torn read that paired a // fresh Connected with a stale backlog would be indistinguishable from a real state. // Null = the reporter has not yet run. Once it HAS run, an unreplicated node latches // (null, null) — "no reading" — rather than the inert engine's (false, 0); see // LocalDbReplicationStatusReporter and register row 80. private volatile Tuple? _localDbReplicationStatus; // Latched once at boot by the Host (register row 73). A tri-state int rather than a // bool? because `volatile` cannot be applied to Nullable (a struct) and this is // written on the startup thread but read on the report-sender thread: // 0 = never told -> report null ("not reported": central self-report, or a test host) // 1 = reconciler spawned // -1 = reconciler SKIPPED (no IDeploymentConfigFetcher) // Null must stay distinct from -1: "we don't know" is not "self-heal is missing". private volatile int _startupReconcilerActive; private readonly ConcurrentDictionary _connectionStatuses = new(); private readonly ConcurrentDictionary _tagResolutionCounts = new(); private readonly ConcurrentDictionary _connectionEndpoints = new(); private readonly ConcurrentDictionary _tagQualityCounts = new(); private IReadOnlyDictionary _sfBufferDepths = new Dictionary(); private int _deployedInstanceCount, _enabledInstanceCount, _disabledInstanceCount; private int _parkedMessageCount; // Age (seconds) of the oldest parked message, stored as raw double bits so it can // be read/written atomically via Interlocked (double? is neither atomic nor // volatile-able). NaN is the "no parked rows" sentinel (ages are always >= 0). private long _oldestParkedAgeBits = BitConverter.DoubleToInt64Bits(double.NaN); // Script-execution-scheduler gauges (S2/UA5), point-in-time, refreshed by the // ScriptSchedulerStatsReporter. Oldest-busy age stored as double bits like the // parked-age above; NaN = idle (no in-flight script). private int _scriptQueueDepth; private int _scriptBusyThreads; private long _scriptOldestBusyAgeBits = BitConverter.DoubleToInt64Bits(double.NaN); // WP3.1: workers detached and replaced by the stuck-script watchdog and not yet exited // (point-in-time, from the same reporter tick as the gauges above), and the per-interval // count of runs shed by the per-script in-flight cap (reset on collect like the error // counters, and restored by AddIntervalCounters when a report fails to send). private int _scriptDetachedThreads; private int _scriptRunShedCount; // WP2.6d: cumulative alarm-publish-queue drop count, refreshed by // SiteStreamAlarmDropReporter. Point-in-time (not reset on collect), like the // scheduler gauges above. private long _siteStreamAlarmDropCount; private volatile string _nodeHostname = ""; private volatile IReadOnlyList? _clusterNodes; private volatile bool _isActiveNode; private readonly TimeProvider _timeProvider; /// /// Creates a collector. The stamps each /// report's timestamp; it defaults to and /// is injectable so the report timestamp is deterministically testable — /// consistent with the rest of the module's time-dependent classes. /// /// Optional custom time provider; defaults to system time. public SiteHealthCollector(TimeProvider? timeProvider = null) { _timeProvider = timeProvider ?? TimeProvider.System; } /// public void IncrementScriptError() { Interlocked.Increment(ref _scriptErrorCount); } /// public void IncrementAlarmError() { Interlocked.Increment(ref _alarmErrorCount); } /// public void IncrementDeadLetter() { Interlocked.Increment(ref _deadLetterCount); } /// public void IncrementSiteAuditWriteFailures() { Interlocked.Increment(ref _siteAuditWriteFailures); } /// public void IncrementAuditRedactionFailure() { Interlocked.Increment(ref _auditRedactionFailures); } /// public void UpdateSiteAuditBacklog(SiteAuditBacklogSnapshot snapshot) { _siteAuditBacklog = snapshot ?? throw new ArgumentNullException(nameof(snapshot)); } /// public void SetSiteEventLogWriteFailures(long count) { Interlocked.Exchange(ref _siteEventLogWriteFailures, count); } /// public void SetLocalDbReplicationStatus(bool? connected, long? oplogBacklog) { _localDbReplicationStatus = Tuple.Create(connected, oplogBacklog); } /// public void SetStartupReconcilerActive(bool active) { _startupReconcilerActive = active ? 1 : -1; } /// public void UpdateConnectionHealth(string connectionName, ConnectionHealth health) { _connectionStatuses[connectionName] = health; } /// public void RemoveConnection(string connectionName) { _connectionStatuses.TryRemove(connectionName, out _); _tagResolutionCounts.TryRemove(connectionName, out _); _connectionEndpoints.TryRemove(connectionName, out _); _tagQualityCounts.TryRemove(connectionName, out _); } /// public void UpdateTagResolution(string connectionName, int totalSubscribed, int successfullyResolved) { _tagResolutionCounts[connectionName] = new TagResolutionStatus(totalSubscribed, successfullyResolved); } /// public void UpdateConnectionEndpoint(string connectionName, string endpoint) { _connectionEndpoints[connectionName] = endpoint; } /// public void UpdateTagQuality(string connectionName, int good, int bad, int uncertain) { _tagQualityCounts[connectionName] = new TagQualityCounts(good, bad, uncertain); } /// public void SetParkedMessageCount(int count) { Interlocked.Exchange(ref _parkedMessageCount, count); } /// public void SetOldestParkedMessageAgeSeconds(double? ageSeconds) { Interlocked.Exchange(ref _oldestParkedAgeBits, BitConverter.DoubleToInt64Bits(ageSeconds ?? double.NaN)); } /// Reads the atomically-stored oldest-parked age, mapping the NaN sentinel back to null. private double? ReadOldestParkedAgeSeconds() { var value = BitConverter.Int64BitsToDouble(Interlocked.Read(ref _oldestParkedAgeBits)); return double.IsNaN(value) ? null : value; } /// public void SetScriptSchedulerStats( int queueDepth, int busyThreads, double? oldestBusyAgeSeconds, int detachedThreads = 0) { Interlocked.Exchange(ref _scriptQueueDepth, queueDepth); Interlocked.Exchange(ref _scriptBusyThreads, busyThreads); Interlocked.Exchange(ref _scriptOldestBusyAgeBits, BitConverter.DoubleToInt64Bits(oldestBusyAgeSeconds ?? double.NaN)); Interlocked.Exchange(ref _scriptDetachedThreads, detachedThreads); } /// public void IncrementScriptRunShed() { Interlocked.Increment(ref _scriptRunShedCount); } /// Reads the atomically-stored oldest-busy script age, mapping the NaN sentinel back to null. private double? ReadScriptOldestBusyAgeSeconds() { var value = BitConverter.Int64BitsToDouble(Interlocked.Read(ref _scriptOldestBusyAgeBits)); return double.IsNaN(value) ? null : value; } /// public void SetSiteStreamAlarmDropCount(long count) { Interlocked.Exchange(ref _siteStreamAlarmDropCount, count); } /// public void SetNodeHostname(string hostname) => _nodeHostname = hostname; /// public void SetClusterNodes(IReadOnlyList nodes) => _clusterNodes = nodes; /// public void SetStoreAndForwardDepths(IReadOnlyDictionary depths) { _sfBufferDepths = depths; } /// public void SetInstanceCounts(int deployed, int enabled, int disabled) { Interlocked.Exchange(ref _deployedInstanceCount, deployed); Interlocked.Exchange(ref _enabledInstanceCount, enabled); Interlocked.Exchange(ref _disabledInstanceCount, disabled); } /// public void SetActiveNode(bool isActive) => _isActiveNode = isActive; /// public bool IsActiveNode => _isActiveNode; /// public void AddIntervalCounters( int scriptErrors, int alarmErrors, int deadLetters, int siteAuditWriteFailures, int auditRedactionFailures, int scriptRunShedCount = 0) { // Each counter is restored atomically via // Interlocked.Add so an increment that arrived during the failed Send // (and therefore accumulated against the zero left by CollectReport's // Exchange) is correctly summed with the values being put back. No // ordering between the six Adds is required — they target independent // fields. scriptRunShedCount joins this restore path (arch-review // 2026-08-19 P8-1) because it is drained by CollectReport the same way // as the other five; DetachedScriptThreads and SiteStreamAlarmDropCount // are point-in-time gauges (CompareExchange/Read, never zeroed on // collect) so a failed Send loses nothing for them to restore. if (scriptErrors != 0) Interlocked.Add(ref _scriptErrorCount, scriptErrors); if (alarmErrors != 0) Interlocked.Add(ref _alarmErrorCount, alarmErrors); if (deadLetters != 0) Interlocked.Add(ref _deadLetterCount, deadLetters); if (siteAuditWriteFailures != 0) Interlocked.Add(ref _siteAuditWriteFailures, siteAuditWriteFailures); if (auditRedactionFailures != 0) Interlocked.Add(ref _auditRedactionFailures, auditRedactionFailures); if (scriptRunShedCount != 0) Interlocked.Add(ref _scriptRunShedCount, scriptRunShedCount); } /// public SiteHealthReport CollectReport(string siteId) { // Atomically read and reset the counters var scriptErrors = Interlocked.Exchange(ref _scriptErrorCount, 0); var alarmErrors = Interlocked.Exchange(ref _alarmErrorCount, 0); var deadLetters = Interlocked.Exchange(ref _deadLetterCount, 0); var siteAuditWriteFailures = Interlocked.Exchange(ref _siteAuditWriteFailures, 0); var auditRedactionFailures = Interlocked.Exchange(ref _auditRedactionFailures, 0); // Single read of the volatile tuple — two reads could straddle a reporter tick and // pair a fresh Connected with a stale backlog. var localDbReplication = _localDbReplicationStatus; var startupReconciler = _startupReconcilerActive; // Snapshot current connection and tag resolution state var connectionStatuses = new Dictionary(_connectionStatuses); var tagResolution = new Dictionary(_tagResolutionCounts); var connectionEndpoints = new Dictionary(_connectionEndpoints); var tagQuality = new Dictionary(_tagQualityCounts); // Snapshot current S&F buffer depths var sfBufferDepths = new Dictionary(_sfBufferDepths); // Determine node role from active/standby state var nodeRole = _isActiveNode ? "Active" : "Standby"; return new SiteHealthReport( SiteId: siteId, SequenceNumber: 0, // Caller (HealthReportSender) assigns the sequence number ReportTimestamp: _timeProvider.GetUtcNow(), DataConnectionStatuses: connectionStatuses, TagResolutionCounts: tagResolution, ScriptErrorCount: scriptErrors, AlarmEvaluationErrorCount: alarmErrors, StoreAndForwardBufferDepths: sfBufferDepths, DeadLetterCount: deadLetters, DeployedInstanceCount: _deployedInstanceCount, EnabledInstanceCount: _enabledInstanceCount, DisabledInstanceCount: _disabledInstanceCount, NodeRole: nodeRole, NodeHostname: _nodeHostname, DataConnectionEndpoints: connectionEndpoints, DataConnectionTagQuality: tagQuality, ParkedMessageCount: Interlocked.CompareExchange(ref _parkedMessageCount, 0, 0), ClusterNodes: _clusterNodes?.ToList(), SiteAuditWriteFailures: siteAuditWriteFailures, AuditRedactionFailure: auditRedactionFailures, SiteAuditBacklog: _siteAuditBacklog, SiteEventLogWriteFailures: Interlocked.Read(ref _siteEventLogWriteFailures), OldestParkedMessageAgeSeconds: ReadOldestParkedAgeSeconds()) { ScriptQueueDepth = Interlocked.CompareExchange(ref _scriptQueueDepth, 0, 0), ScriptBusyThreads = Interlocked.CompareExchange(ref _scriptBusyThreads, 0, 0), ScriptOldestBusyAgeSeconds = ReadScriptOldestBusyAgeSeconds(), // WP3.1: point-in-time (like the three gauges above), so a CompareExchange read // rather than an Exchange reset. DetachedScriptThreads = Interlocked.CompareExchange(ref _scriptDetachedThreads, 0, 0), // WP3.1: per-interval, so drained on collect like the error counters. ScriptRunShedCount = Interlocked.Exchange(ref _scriptRunShedCount, 0), // Both fields come from the ONE snapshot read above. Null (the reporter has // not run) leaves both report fields null — "no data", not "disconnected // with an empty backlog". LocalDbReplicationConnected = localDbReplication?.Item1, LocalDbOplogBacklog = localDbReplication?.Item2, SiteStreamAlarmDropCount = Interlocked.Read(ref _siteStreamAlarmDropCount), // Tri-state: 0 stays null ("not reported"), which must never collapse into // false ("self-heal is missing"). Register row 73. StartupReconcilerActive = startupReconciler == 0 ? null : startupReconciler > 0 }; } }