feat(health): additive script-scheduler stats on ISiteHealthCollector + SiteHealthReport (S2/UA5)

This commit is contained in:
Joseph Doherty
2026-07-08 23:37:50 -04:00
parent 474ec8a221
commit 7d50555fd3
4 changed files with 85 additions and 2 deletions
@@ -54,7 +54,26 @@ public record SiteHealthReport(
// signal surfaces a forgotten site's stale backlog (count alone hides age).
// Additive-only trailing member — defaults to null for producers / tests that
// don't populate it.
double? OldestParkedMessageAgeSeconds = null);
double? OldestParkedMessageAgeSeconds = null)
{
// Script-execution-scheduler gauges (S2/UA5), surfaced additively as init
// properties (rather than positional parameters) so the constructor signature
// is untouched. Point-in-time, refreshed on the site by ScriptSchedulerStatsReporter;
// safe defaults for producers / tests that don't populate them.
/// <summary>Script tasks waiting to run on the dedicated script-execution scheduler.</summary>
public int ScriptQueueDepth { get; init; }
/// <summary>Script-execution worker threads currently running a script body.</summary>
public int ScriptBusyThreads { get; init; }
/// <summary>
/// Age (seconds) of the oldest in-flight script across the scheduler's worker
/// threads, or null when the pool is idle. A large value with all threads busy
/// indicates a stuck/blocked script holding a dedicated thread.
/// </summary>
public double? ScriptOldestBusyAgeSeconds { get; init; }
}
/// <summary>
/// Broadcast wrapper used between central nodes to keep per-node
@@ -141,6 +141,24 @@ public interface ISiteHealthCollector
// SiteHealthCollector overrides this with the Interlocked.Exchange store.
}
/// <summary>
/// Replace the latest script-execution-scheduler gauges (queue depth, busy
/// thread count, and age in seconds of the oldest in-flight script) used by
/// the next <see cref="CollectReport"/> call. Refreshed periodically by the
/// <c>ScriptSchedulerStatsReporter</c> hosted service. Point-in-time: the
/// values are NOT reset on <see cref="CollectReport"/>. S2/UA5 — surfaces a
/// saturated or stuck script-execution pool on the site health report.
/// Default interface implementation is a no-op so existing test fakes continue
/// to compile without per-fake updates.
/// </summary>
/// <param name="queueDepth">Script tasks waiting to run.</param>
/// <param name="busyThreads">Worker threads currently executing a script.</param>
/// <param name="oldestBusyAgeSeconds">Age (seconds) of the oldest in-flight script, or <c>null</c> when idle.</param>
void SetScriptSchedulerStats(int queueDepth, int busyThreads, double? oldestBusyAgeSeconds)
{
// Default no-op so test fakes do not need to be updated.
}
/// <summary>
/// Sets the hostname of this node.
/// </summary>
@@ -29,6 +29,12 @@ public class SiteHealthCollector : ISiteHealthCollector
// be read/written atomically via Interlocked (double? is neither atomic nor
// volatile-able). NaN is the "no parked rows" sentinel (ages are always >= 0).
private long _oldestParkedAgeBits = BitConverter.DoubleToInt64Bits(double.NaN);
// Script-execution-scheduler gauges (S2/UA5), point-in-time, refreshed by the
// ScriptSchedulerStatsReporter. Oldest-busy age stored as double bits like the
// parked-age above; NaN = idle (no in-flight script).
private int _scriptQueueDepth;
private int _scriptBusyThreads;
private long _scriptOldestBusyAgeBits = BitConverter.DoubleToInt64Bits(double.NaN);
private volatile string _nodeHostname = "";
private volatile IReadOnlyList<Commons.Messages.Health.NodeStatus>? _clusterNodes;
private volatile bool _isActiveNode;
@@ -141,6 +147,22 @@ public class SiteHealthCollector : ISiteHealthCollector
return double.IsNaN(value) ? null : value;
}
/// <inheritdoc />
public void SetScriptSchedulerStats(int queueDepth, int busyThreads, double? oldestBusyAgeSeconds)
{
Interlocked.Exchange(ref _scriptQueueDepth, queueDepth);
Interlocked.Exchange(ref _scriptBusyThreads, busyThreads);
Interlocked.Exchange(ref _scriptOldestBusyAgeBits,
BitConverter.DoubleToInt64Bits(oldestBusyAgeSeconds ?? double.NaN));
}
/// <summary>Reads the atomically-stored oldest-busy script age, mapping the NaN sentinel back to null.</summary>
private double? ReadScriptOldestBusyAgeSeconds()
{
var value = BitConverter.Int64BitsToDouble(Interlocked.Read(ref _scriptOldestBusyAgeBits));
return double.IsNaN(value) ? null : value;
}
/// <inheritdoc />
public void SetNodeHostname(string hostname) => _nodeHostname = hostname;
@@ -233,6 +255,11 @@ public class SiteHealthCollector : ISiteHealthCollector
AuditRedactionFailure: auditRedactionFailures,
SiteAuditBacklog: _siteAuditBacklog,
SiteEventLogWriteFailures: Interlocked.Read(ref _siteEventLogWriteFailures),
OldestParkedMessageAgeSeconds: ReadOldestParkedAgeSeconds());
OldestParkedMessageAgeSeconds: ReadOldestParkedAgeSeconds())
{
ScriptQueueDepth = Interlocked.CompareExchange(ref _scriptQueueDepth, 0, 0),
ScriptBusyThreads = Interlocked.CompareExchange(ref _scriptBusyThreads, 0, 0),
ScriptOldestBusyAgeSeconds = ReadScriptOldestBusyAgeSeconds()
};
}
}
@@ -38,6 +38,25 @@ public class SiteHealthCollectorTests
Assert.Equal(2, report.AlarmEvaluationErrorCount);
}
[Fact]
public void CollectReport_CarriesScriptSchedulerStats()
{
_collector.SetScriptSchedulerStats(queueDepth: 7, busyThreads: 3, oldestBusyAgeSeconds: 42.5);
var report = _collector.CollectReport("site-1");
Assert.Equal(7, report.ScriptQueueDepth);
Assert.Equal(3, report.ScriptBusyThreads);
Assert.Equal(42.5, report.ScriptOldestBusyAgeSeconds);
}
[Fact]
public void CollectReport_ScriptSchedulerStats_DefaultToZeroAndNull()
{
var report = _collector.CollectReport("site-1");
Assert.Equal(0, report.ScriptQueueDepth);
Assert.Equal(0, report.ScriptBusyThreads);
Assert.Null(report.ScriptOldestBusyAgeSeconds);
}
[Fact]
public void IncrementDeadLetter_AccumulatesBetweenReports()
{