Joseph Doherty
340 lines
15 KiB
C#
340 lines
15 KiB
C#
using System.Collections.Concurrent;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Health;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Types;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
|
|
|
|
namespace ZB.MOM.WW.ScadaBridge.HealthMonitoring;
|
|
|
|
/// <summary>
|
|
/// Collects health metrics from all site subsystems.
|
|
/// Thread-safe: counters use Interlocked operations, connection/tag data uses ConcurrentDictionary.
|
|
/// </summary>
|
|
public class SiteHealthCollector : ISiteHealthCollector
|
|
{
|
|
private int _scriptErrorCount;
|
|
private int _alarmErrorCount;
|
|
private int _deadLetterCount;
|
|
private int _siteAuditWriteFailures;
|
|
private int _auditRedactionFailures;
|
|
private volatile SiteAuditBacklogSnapshot? _siteAuditBacklog;
|
|
private long _siteEventLogWriteFailures;
|
|
// One volatile tuple rather than two independent fields: a torn read that paired a
|
|
// fresh Connected with a stale backlog would be indistinguishable from a real state.
|
|
// Null = the reporter has not yet run. Once it HAS run, an unreplicated node latches
|
|
// (null, null) — "no reading" — rather than the inert engine's (false, 0); see
|
|
// LocalDbReplicationStatusReporter and register row 80.
|
|
private volatile Tuple<bool?, long?>? _localDbReplicationStatus;
|
|
// Latched once at boot by the Host (register row 73). A tri-state int rather than a
|
|
// bool? because `volatile` cannot be applied to Nullable<bool> (a struct) and this is
|
|
// written on the startup thread but read on the report-sender thread:
|
|
// 0 = never told -> report null ("not reported": central self-report, or a test host)
|
|
// 1 = reconciler spawned
|
|
// -1 = reconciler SKIPPED (no IDeploymentConfigFetcher)
|
|
// Null must stay distinct from -1: "we don't know" is not "self-heal is missing".
|
|
private volatile int _startupReconcilerActive;
|
|
private readonly ConcurrentDictionary<string, ConnectionHealth> _connectionStatuses = new();
|
|
private readonly ConcurrentDictionary<string, TagResolutionStatus> _tagResolutionCounts = new();
|
|
private readonly ConcurrentDictionary<string, string> _connectionEndpoints = new();
|
|
private readonly ConcurrentDictionary<string, TagQualityCounts> _tagQualityCounts = new();
|
|
private IReadOnlyDictionary<string, int> _sfBufferDepths = new Dictionary<string, int>();
|
|
private int _deployedInstanceCount, _enabledInstanceCount, _disabledInstanceCount;
|
|
private int _parkedMessageCount;
|
|
// Age (seconds) of the oldest parked message, stored as raw double bits so it can
|
|
// be read/written atomically via Interlocked (double? is neither atomic nor
|
|
// volatile-able). NaN is the "no parked rows" sentinel (ages are always >= 0).
|
|
private long _oldestParkedAgeBits = BitConverter.DoubleToInt64Bits(double.NaN);
|
|
// Script-execution-scheduler gauges (S2/UA5), point-in-time, refreshed by the
|
|
// ScriptSchedulerStatsReporter. Oldest-busy age stored as double bits like the
|
|
// parked-age above; NaN = idle (no in-flight script).
|
|
private int _scriptQueueDepth;
|
|
private int _scriptBusyThreads;
|
|
private long _scriptOldestBusyAgeBits = BitConverter.DoubleToInt64Bits(double.NaN);
|
|
// WP3.1: workers detached and replaced by the stuck-script watchdog and not yet exited
|
|
// (point-in-time, from the same reporter tick as the gauges above), and the per-interval
|
|
// count of runs shed by the per-script in-flight cap (reset on collect like the error
|
|
// counters, and restored by AddIntervalCounters when a report fails to send).
|
|
private int _scriptDetachedThreads;
|
|
private int _scriptRunShedCount;
|
|
// WP2.6d: cumulative alarm-publish-queue drop count, refreshed by
|
|
// SiteStreamAlarmDropReporter. Point-in-time (not reset on collect), like the
|
|
// scheduler gauges above.
|
|
private long _siteStreamAlarmDropCount;
|
|
private volatile string _nodeHostname = "";
|
|
private volatile IReadOnlyList<Commons.Messages.Health.NodeStatus>? _clusterNodes;
|
|
private volatile bool _isActiveNode;
|
|
private readonly TimeProvider _timeProvider;
|
|
|
|
/// <summary>
|
|
/// Creates a collector. The <paramref name="timeProvider"/> stamps each
|
|
/// report's timestamp; it defaults to <see cref="TimeProvider.System"/> and
|
|
/// is injectable so the report timestamp is deterministically testable —
|
|
/// consistent with the rest of the module's time-dependent classes.
|
|
/// </summary>
|
|
/// <param name="timeProvider">Optional custom time provider; defaults to system time.</param>
|
|
public SiteHealthCollector(TimeProvider? timeProvider = null)
|
|
{
|
|
_timeProvider = timeProvider ?? TimeProvider.System;
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public void IncrementScriptError()
|
|
{
|
|
Interlocked.Increment(ref _scriptErrorCount);
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public void IncrementAlarmError()
|
|
{
|
|
Interlocked.Increment(ref _alarmErrorCount);
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public void IncrementDeadLetter()
|
|
{
|
|
Interlocked.Increment(ref _deadLetterCount);
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public void IncrementSiteAuditWriteFailures()
|
|
{
|
|
Interlocked.Increment(ref _siteAuditWriteFailures);
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public void IncrementAuditRedactionFailure()
|
|
{
|
|
Interlocked.Increment(ref _auditRedactionFailures);
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public void UpdateSiteAuditBacklog(SiteAuditBacklogSnapshot snapshot)
|
|
{
|
|
_siteAuditBacklog = snapshot ?? throw new ArgumentNullException(nameof(snapshot));
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public void SetSiteEventLogWriteFailures(long count)
|
|
{
|
|
Interlocked.Exchange(ref _siteEventLogWriteFailures, count);
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public void SetLocalDbReplicationStatus(bool? connected, long? oplogBacklog)
|
|
{
|
|
_localDbReplicationStatus = Tuple.Create(connected, oplogBacklog);
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public void SetStartupReconcilerActive(bool active)
|
|
{
|
|
_startupReconcilerActive = active ? 1 : -1;
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public void UpdateConnectionHealth(string connectionName, ConnectionHealth health)
|
|
{
|
|
_connectionStatuses[connectionName] = health;
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public void RemoveConnection(string connectionName)
|
|
{
|
|
_connectionStatuses.TryRemove(connectionName, out _);
|
|
_tagResolutionCounts.TryRemove(connectionName, out _);
|
|
_connectionEndpoints.TryRemove(connectionName, out _);
|
|
_tagQualityCounts.TryRemove(connectionName, out _);
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public void UpdateTagResolution(string connectionName, int totalSubscribed, int successfullyResolved)
|
|
{
|
|
_tagResolutionCounts[connectionName] = new TagResolutionStatus(totalSubscribed, successfullyResolved);
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public void UpdateConnectionEndpoint(string connectionName, string endpoint)
|
|
{
|
|
_connectionEndpoints[connectionName] = endpoint;
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public void UpdateTagQuality(string connectionName, int good, int bad, int uncertain)
|
|
{
|
|
_tagQualityCounts[connectionName] = new TagQualityCounts(good, bad, uncertain);
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public void SetParkedMessageCount(int count)
|
|
{
|
|
Interlocked.Exchange(ref _parkedMessageCount, count);
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public void SetOldestParkedMessageAgeSeconds(double? ageSeconds)
|
|
{
|
|
Interlocked.Exchange(ref _oldestParkedAgeBits,
|
|
BitConverter.DoubleToInt64Bits(ageSeconds ?? double.NaN));
|
|
}
|
|
|
|
/// <summary>Reads the atomically-stored oldest-parked age, mapping the NaN sentinel back to null.</summary>
|
|
private double? ReadOldestParkedAgeSeconds()
|
|
{
|
|
var value = BitConverter.Int64BitsToDouble(Interlocked.Read(ref _oldestParkedAgeBits));
|
|
return double.IsNaN(value) ? null : value;
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public void SetScriptSchedulerStats(
|
|
int queueDepth, int busyThreads, double? oldestBusyAgeSeconds, int detachedThreads = 0)
|
|
{
|
|
Interlocked.Exchange(ref _scriptQueueDepth, queueDepth);
|
|
Interlocked.Exchange(ref _scriptBusyThreads, busyThreads);
|
|
Interlocked.Exchange(ref _scriptOldestBusyAgeBits,
|
|
BitConverter.DoubleToInt64Bits(oldestBusyAgeSeconds ?? double.NaN));
|
|
Interlocked.Exchange(ref _scriptDetachedThreads, detachedThreads);
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public void IncrementScriptRunShed()
|
|
{
|
|
Interlocked.Increment(ref _scriptRunShedCount);
|
|
}
|
|
|
|
/// <summary>Reads the atomically-stored oldest-busy script age, mapping the NaN sentinel back to null.</summary>
|
|
private double? ReadScriptOldestBusyAgeSeconds()
|
|
{
|
|
var value = BitConverter.Int64BitsToDouble(Interlocked.Read(ref _scriptOldestBusyAgeBits));
|
|
return double.IsNaN(value) ? null : value;
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public void SetSiteStreamAlarmDropCount(long count)
|
|
{
|
|
Interlocked.Exchange(ref _siteStreamAlarmDropCount, count);
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public void SetNodeHostname(string hostname) => _nodeHostname = hostname;
|
|
|
|
/// <inheritdoc />
|
|
public void SetClusterNodes(IReadOnlyList<Commons.Messages.Health.NodeStatus> nodes) => _clusterNodes = nodes;
|
|
|
|
/// <inheritdoc />
|
|
public void SetStoreAndForwardDepths(IReadOnlyDictionary<string, int> depths)
|
|
{
|
|
_sfBufferDepths = depths;
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public void SetInstanceCounts(int deployed, int enabled, int disabled)
|
|
{
|
|
Interlocked.Exchange(ref _deployedInstanceCount, deployed);
|
|
Interlocked.Exchange(ref _enabledInstanceCount, enabled);
|
|
Interlocked.Exchange(ref _disabledInstanceCount, disabled);
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public void SetActiveNode(bool isActive) => _isActiveNode = isActive;
|
|
|
|
/// <inheritdoc />
|
|
public bool IsActiveNode => _isActiveNode;
|
|
|
|
/// <inheritdoc />
|
|
public void AddIntervalCounters(
|
|
int scriptErrors,
|
|
int alarmErrors,
|
|
int deadLetters,
|
|
int siteAuditWriteFailures,
|
|
int auditRedactionFailures,
|
|
int scriptRunShedCount = 0)
|
|
{
|
|
// Each counter is restored atomically via
|
|
// Interlocked.Add so an increment that arrived during the failed Send
|
|
// (and therefore accumulated against the zero left by CollectReport's
|
|
// Exchange) is correctly summed with the values being put back. No
|
|
// ordering between the six Adds is required — they target independent
|
|
// fields. scriptRunShedCount joins this restore path (arch-review
|
|
// 2026-08-19 P8-1) because it is drained by CollectReport the same way
|
|
// as the other five; DetachedScriptThreads and SiteStreamAlarmDropCount
|
|
// are point-in-time gauges (CompareExchange/Read, never zeroed on
|
|
// collect) so a failed Send loses nothing for them to restore.
|
|
if (scriptErrors != 0) Interlocked.Add(ref _scriptErrorCount, scriptErrors);
|
|
if (alarmErrors != 0) Interlocked.Add(ref _alarmErrorCount, alarmErrors);
|
|
if (deadLetters != 0) Interlocked.Add(ref _deadLetterCount, deadLetters);
|
|
if (siteAuditWriteFailures != 0) Interlocked.Add(ref _siteAuditWriteFailures, siteAuditWriteFailures);
|
|
if (auditRedactionFailures != 0) Interlocked.Add(ref _auditRedactionFailures, auditRedactionFailures);
|
|
if (scriptRunShedCount != 0) Interlocked.Add(ref _scriptRunShedCount, scriptRunShedCount);
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public SiteHealthReport CollectReport(string siteId)
|
|
{
|
|
// Atomically read and reset the counters
|
|
var scriptErrors = Interlocked.Exchange(ref _scriptErrorCount, 0);
|
|
var alarmErrors = Interlocked.Exchange(ref _alarmErrorCount, 0);
|
|
var deadLetters = Interlocked.Exchange(ref _deadLetterCount, 0);
|
|
var siteAuditWriteFailures = Interlocked.Exchange(ref _siteAuditWriteFailures, 0);
|
|
var auditRedactionFailures = Interlocked.Exchange(ref _auditRedactionFailures, 0);
|
|
// Single read of the volatile tuple — two reads could straddle a reporter tick and
|
|
// pair a fresh Connected with a stale backlog.
|
|
var localDbReplication = _localDbReplicationStatus;
|
|
var startupReconciler = _startupReconcilerActive;
|
|
|
|
// Snapshot current connection and tag resolution state
|
|
var connectionStatuses = new Dictionary<string, ConnectionHealth>(_connectionStatuses);
|
|
var tagResolution = new Dictionary<string, TagResolutionStatus>(_tagResolutionCounts);
|
|
var connectionEndpoints = new Dictionary<string, string>(_connectionEndpoints);
|
|
var tagQuality = new Dictionary<string, TagQualityCounts>(_tagQualityCounts);
|
|
|
|
// Snapshot current S&F buffer depths
|
|
var sfBufferDepths = new Dictionary<string, int>(_sfBufferDepths);
|
|
|
|
// Determine node role from active/standby state
|
|
var nodeRole = _isActiveNode ? "Active" : "Standby";
|
|
|
|
return new SiteHealthReport(
|
|
SiteId: siteId,
|
|
SequenceNumber: 0, // Caller (HealthReportSender) assigns the sequence number
|
|
ReportTimestamp: _timeProvider.GetUtcNow(),
|
|
DataConnectionStatuses: connectionStatuses,
|
|
TagResolutionCounts: tagResolution,
|
|
ScriptErrorCount: scriptErrors,
|
|
AlarmEvaluationErrorCount: alarmErrors,
|
|
StoreAndForwardBufferDepths: sfBufferDepths,
|
|
DeadLetterCount: deadLetters,
|
|
DeployedInstanceCount: _deployedInstanceCount,
|
|
EnabledInstanceCount: _enabledInstanceCount,
|
|
DisabledInstanceCount: _disabledInstanceCount,
|
|
NodeRole: nodeRole,
|
|
NodeHostname: _nodeHostname,
|
|
DataConnectionEndpoints: connectionEndpoints,
|
|
DataConnectionTagQuality: tagQuality,
|
|
ParkedMessageCount: Interlocked.CompareExchange(ref _parkedMessageCount, 0, 0),
|
|
ClusterNodes: _clusterNodes?.ToList(),
|
|
SiteAuditWriteFailures: siteAuditWriteFailures,
|
|
AuditRedactionFailure: auditRedactionFailures,
|
|
SiteAuditBacklog: _siteAuditBacklog,
|
|
SiteEventLogWriteFailures: Interlocked.Read(ref _siteEventLogWriteFailures),
|
|
OldestParkedMessageAgeSeconds: ReadOldestParkedAgeSeconds())
|
|
{
|
|
ScriptQueueDepth = Interlocked.CompareExchange(ref _scriptQueueDepth, 0, 0),
|
|
ScriptBusyThreads = Interlocked.CompareExchange(ref _scriptBusyThreads, 0, 0),
|
|
ScriptOldestBusyAgeSeconds = ReadScriptOldestBusyAgeSeconds(),
|
|
// WP3.1: point-in-time (like the three gauges above), so a CompareExchange read
|
|
// rather than an Exchange reset.
|
|
DetachedScriptThreads = Interlocked.CompareExchange(ref _scriptDetachedThreads, 0, 0),
|
|
// WP3.1: per-interval, so drained on collect like the error counters.
|
|
ScriptRunShedCount = Interlocked.Exchange(ref _scriptRunShedCount, 0),
|
|
// Both fields come from the ONE snapshot read above. Null (the reporter has
|
|
// not run) leaves both report fields null — "no data", not "disconnected
|
|
// with an empty backlog".
|
|
LocalDbReplicationConnected = localDbReplication?.Item1,
|
|
LocalDbOplogBacklog = localDbReplication?.Item2,
|
|
SiteStreamAlarmDropCount = Interlocked.Read(ref _siteStreamAlarmDropCount),
|
|
// Tri-state: 0 stays null ("not reported"), which must never collapse into
|
|
// false ("self-heal is missing"). Register row 73.
|
|
StartupReconcilerActive = startupReconciler == 0 ? null : startupReconciler > 0
|
|
};
|
|
}
|
|
}
|