From 7d50555fd306375af09fb73c205f992a342f62c9 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Wed, 8 Jul 2026 23:37:50 -0400 Subject: [PATCH] feat(health): additive script-scheduler stats on ISiteHealthCollector + SiteHealthReport (S2/UA5) --- .../Messages/Health/SiteHealthReport.cs | 21 +++++++++++++- .../ISiteHealthCollector.cs | 18 ++++++++++++ .../SiteHealthCollector.cs | 29 ++++++++++++++++++- .../SiteHealthCollectorTests.cs | 19 ++++++++++++ 4 files changed, 85 insertions(+), 2 deletions(-) diff --git a/src/ZB.MOM.WW.ScadaBridge.Commons/Messages/Health/SiteHealthReport.cs b/src/ZB.MOM.WW.ScadaBridge.Commons/Messages/Health/SiteHealthReport.cs index a6a2dc89..216bd656 100644 --- a/src/ZB.MOM.WW.ScadaBridge.Commons/Messages/Health/SiteHealthReport.cs +++ b/src/ZB.MOM.WW.ScadaBridge.Commons/Messages/Health/SiteHealthReport.cs @@ -54,7 +54,26 @@ public record SiteHealthReport( // signal surfaces a forgotten site's stale backlog (count alone hides age). // Additive-only trailing member — defaults to null for producers / tests that // don't populate it. - double? OldestParkedMessageAgeSeconds = null); + double? OldestParkedMessageAgeSeconds = null) +{ + // Script-execution-scheduler gauges (S2/UA5), surfaced additively as init + // properties (rather than positional parameters) so the constructor signature + // is untouched. Point-in-time, refreshed on the site by ScriptSchedulerStatsReporter; + // safe defaults for producers / tests that don't populate them. + + /// Script tasks waiting to run on the dedicated script-execution scheduler. + public int ScriptQueueDepth { get; init; } + + /// Script-execution worker threads currently running a script body. + public int ScriptBusyThreads { get; init; } + + /// + /// Age (seconds) of the oldest in-flight script across the scheduler's worker + /// threads, or null when the pool is idle. A large value with all threads busy + /// indicates a stuck/blocked script holding a dedicated thread. + /// + public double? ScriptOldestBusyAgeSeconds { get; init; } +} /// /// Broadcast wrapper used between central nodes to keep per-node diff --git a/src/ZB.MOM.WW.ScadaBridge.HealthMonitoring/ISiteHealthCollector.cs b/src/ZB.MOM.WW.ScadaBridge.HealthMonitoring/ISiteHealthCollector.cs index afc12ba6..3f9ab84d 100644 --- a/src/ZB.MOM.WW.ScadaBridge.HealthMonitoring/ISiteHealthCollector.cs +++ b/src/ZB.MOM.WW.ScadaBridge.HealthMonitoring/ISiteHealthCollector.cs @@ -141,6 +141,24 @@ public interface ISiteHealthCollector // SiteHealthCollector overrides this with the Interlocked.Exchange store. } + /// + /// Replace the latest script-execution-scheduler gauges (queue depth, busy + /// thread count, and age in seconds of the oldest in-flight script) used by + /// the next call. Refreshed periodically by the + /// ScriptSchedulerStatsReporter hosted service. Point-in-time: the + /// values are NOT reset on . S2/UA5 — surfaces a + /// saturated or stuck script-execution pool on the site health report. + /// Default interface implementation is a no-op so existing test fakes continue + /// to compile without per-fake updates. + /// + /// Script tasks waiting to run. + /// Worker threads currently executing a script. + /// Age (seconds) of the oldest in-flight script, or null when idle. + void SetScriptSchedulerStats(int queueDepth, int busyThreads, double? oldestBusyAgeSeconds) + { + // Default no-op so test fakes do not need to be updated. + } + /// /// Sets the hostname of this node. /// diff --git a/src/ZB.MOM.WW.ScadaBridge.HealthMonitoring/SiteHealthCollector.cs b/src/ZB.MOM.WW.ScadaBridge.HealthMonitoring/SiteHealthCollector.cs index 7e978331..51f22a7e 100644 --- a/src/ZB.MOM.WW.ScadaBridge.HealthMonitoring/SiteHealthCollector.cs +++ b/src/ZB.MOM.WW.ScadaBridge.HealthMonitoring/SiteHealthCollector.cs @@ -29,6 +29,12 @@ public class SiteHealthCollector : ISiteHealthCollector // be read/written atomically via Interlocked (double? is neither atomic nor // volatile-able). NaN is the "no parked rows" sentinel (ages are always >= 0). private long _oldestParkedAgeBits = BitConverter.DoubleToInt64Bits(double.NaN); + // Script-execution-scheduler gauges (S2/UA5), point-in-time, refreshed by the + // ScriptSchedulerStatsReporter. Oldest-busy age stored as double bits like the + // parked-age above; NaN = idle (no in-flight script). + private int _scriptQueueDepth; + private int _scriptBusyThreads; + private long _scriptOldestBusyAgeBits = BitConverter.DoubleToInt64Bits(double.NaN); private volatile string _nodeHostname = ""; private volatile IReadOnlyList? _clusterNodes; private volatile bool _isActiveNode; @@ -141,6 +147,22 @@ public class SiteHealthCollector : ISiteHealthCollector return double.IsNaN(value) ? null : value; } + /// + public void SetScriptSchedulerStats(int queueDepth, int busyThreads, double? oldestBusyAgeSeconds) + { + Interlocked.Exchange(ref _scriptQueueDepth, queueDepth); + Interlocked.Exchange(ref _scriptBusyThreads, busyThreads); + Interlocked.Exchange(ref _scriptOldestBusyAgeBits, + BitConverter.DoubleToInt64Bits(oldestBusyAgeSeconds ?? double.NaN)); + } + + /// Reads the atomically-stored oldest-busy script age, mapping the NaN sentinel back to null. + private double? ReadScriptOldestBusyAgeSeconds() + { + var value = BitConverter.Int64BitsToDouble(Interlocked.Read(ref _scriptOldestBusyAgeBits)); + return double.IsNaN(value) ? null : value; + } + /// public void SetNodeHostname(string hostname) => _nodeHostname = hostname; @@ -233,6 +255,11 @@ public class SiteHealthCollector : ISiteHealthCollector AuditRedactionFailure: auditRedactionFailures, SiteAuditBacklog: _siteAuditBacklog, SiteEventLogWriteFailures: Interlocked.Read(ref _siteEventLogWriteFailures), - OldestParkedMessageAgeSeconds: ReadOldestParkedAgeSeconds()); + OldestParkedMessageAgeSeconds: ReadOldestParkedAgeSeconds()) + { + ScriptQueueDepth = Interlocked.CompareExchange(ref _scriptQueueDepth, 0, 0), + ScriptBusyThreads = Interlocked.CompareExchange(ref _scriptBusyThreads, 0, 0), + ScriptOldestBusyAgeSeconds = ReadScriptOldestBusyAgeSeconds() + }; } } diff --git a/tests/ZB.MOM.WW.ScadaBridge.HealthMonitoring.Tests/SiteHealthCollectorTests.cs b/tests/ZB.MOM.WW.ScadaBridge.HealthMonitoring.Tests/SiteHealthCollectorTests.cs index d4b0c2f9..67e248bc 100644 --- a/tests/ZB.MOM.WW.ScadaBridge.HealthMonitoring.Tests/SiteHealthCollectorTests.cs +++ b/tests/ZB.MOM.WW.ScadaBridge.HealthMonitoring.Tests/SiteHealthCollectorTests.cs @@ -38,6 +38,25 @@ public class SiteHealthCollectorTests Assert.Equal(2, report.AlarmEvaluationErrorCount); } + [Fact] + public void CollectReport_CarriesScriptSchedulerStats() + { + _collector.SetScriptSchedulerStats(queueDepth: 7, busyThreads: 3, oldestBusyAgeSeconds: 42.5); + var report = _collector.CollectReport("site-1"); + Assert.Equal(7, report.ScriptQueueDepth); + Assert.Equal(3, report.ScriptBusyThreads); + Assert.Equal(42.5, report.ScriptOldestBusyAgeSeconds); + } + + [Fact] + public void CollectReport_ScriptSchedulerStats_DefaultToZeroAndNull() + { + var report = _collector.CollectReport("site-1"); + Assert.Equal(0, report.ScriptQueueDepth); + Assert.Equal(0, report.ScriptBusyThreads); + Assert.Null(report.ScriptOldestBusyAgeSeconds); + } + [Fact] public void IncrementDeadLetter_AccumulatesBetweenReports() {