feat(health): additive script-scheduler stats on ISiteHealthCollector + SiteHealthReport (S2/UA5)
This commit is contained in:
@@ -54,7 +54,26 @@ public record SiteHealthReport(
|
||||
// signal surfaces a forgotten site's stale backlog (count alone hides age).
|
||||
// Additive-only trailing member — defaults to null for producers / tests that
|
||||
// don't populate it.
|
||||
double? OldestParkedMessageAgeSeconds = null);
|
||||
double? OldestParkedMessageAgeSeconds = null)
|
||||
{
|
||||
// Script-execution-scheduler gauges (S2/UA5), surfaced additively as init
|
||||
// properties (rather than positional parameters) so the constructor signature
|
||||
// is untouched. Point-in-time, refreshed on the site by ScriptSchedulerStatsReporter;
|
||||
// safe defaults for producers / tests that don't populate them.
|
||||
|
||||
/// <summary>Script tasks waiting to run on the dedicated script-execution scheduler.</summary>
|
||||
public int ScriptQueueDepth { get; init; }
|
||||
|
||||
/// <summary>Script-execution worker threads currently running a script body.</summary>
|
||||
public int ScriptBusyThreads { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Age (seconds) of the oldest in-flight script across the scheduler's worker
|
||||
/// threads, or null when the pool is idle. A large value with all threads busy
|
||||
/// indicates a stuck/blocked script holding a dedicated thread.
|
||||
/// </summary>
|
||||
public double? ScriptOldestBusyAgeSeconds { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Broadcast wrapper used between central nodes to keep per-node
|
||||
|
||||
@@ -141,6 +141,24 @@ public interface ISiteHealthCollector
|
||||
// SiteHealthCollector overrides this with the Interlocked.Exchange store.
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Replace the latest script-execution-scheduler gauges (queue depth, busy
|
||||
/// thread count, and age in seconds of the oldest in-flight script) used by
|
||||
/// the next <see cref="CollectReport"/> call. Refreshed periodically by the
|
||||
/// <c>ScriptSchedulerStatsReporter</c> hosted service. Point-in-time: the
|
||||
/// values are NOT reset on <see cref="CollectReport"/>. S2/UA5 — surfaces a
|
||||
/// saturated or stuck script-execution pool on the site health report.
|
||||
/// Default interface implementation is a no-op so existing test fakes continue
|
||||
/// to compile without per-fake updates.
|
||||
/// </summary>
|
||||
/// <param name="queueDepth">Script tasks waiting to run.</param>
|
||||
/// <param name="busyThreads">Worker threads currently executing a script.</param>
|
||||
/// <param name="oldestBusyAgeSeconds">Age (seconds) of the oldest in-flight script, or <c>null</c> when idle.</param>
|
||||
void SetScriptSchedulerStats(int queueDepth, int busyThreads, double? oldestBusyAgeSeconds)
|
||||
{
|
||||
// Default no-op so test fakes do not need to be updated.
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Sets the hostname of this node.
|
||||
/// </summary>
|
||||
|
||||
@@ -29,6 +29,12 @@ public class SiteHealthCollector : ISiteHealthCollector
|
||||
// be read/written atomically via Interlocked (double? is neither atomic nor
|
||||
// volatile-able). NaN is the "no parked rows" sentinel (ages are always >= 0).
|
||||
private long _oldestParkedAgeBits = BitConverter.DoubleToInt64Bits(double.NaN);
|
||||
// Script-execution-scheduler gauges (S2/UA5), point-in-time, refreshed by the
|
||||
// ScriptSchedulerStatsReporter. Oldest-busy age stored as double bits like the
|
||||
// parked-age above; NaN = idle (no in-flight script).
|
||||
private int _scriptQueueDepth;
|
||||
private int _scriptBusyThreads;
|
||||
private long _scriptOldestBusyAgeBits = BitConverter.DoubleToInt64Bits(double.NaN);
|
||||
private volatile string _nodeHostname = "";
|
||||
private volatile IReadOnlyList<Commons.Messages.Health.NodeStatus>? _clusterNodes;
|
||||
private volatile bool _isActiveNode;
|
||||
@@ -141,6 +147,22 @@ public class SiteHealthCollector : ISiteHealthCollector
|
||||
return double.IsNaN(value) ? null : value;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public void SetScriptSchedulerStats(int queueDepth, int busyThreads, double? oldestBusyAgeSeconds)
|
||||
{
|
||||
Interlocked.Exchange(ref _scriptQueueDepth, queueDepth);
|
||||
Interlocked.Exchange(ref _scriptBusyThreads, busyThreads);
|
||||
Interlocked.Exchange(ref _scriptOldestBusyAgeBits,
|
||||
BitConverter.DoubleToInt64Bits(oldestBusyAgeSeconds ?? double.NaN));
|
||||
}
|
||||
|
||||
/// <summary>Reads the atomically-stored oldest-busy script age, mapping the NaN sentinel back to null.</summary>
|
||||
private double? ReadScriptOldestBusyAgeSeconds()
|
||||
{
|
||||
var value = BitConverter.Int64BitsToDouble(Interlocked.Read(ref _scriptOldestBusyAgeBits));
|
||||
return double.IsNaN(value) ? null : value;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public void SetNodeHostname(string hostname) => _nodeHostname = hostname;
|
||||
|
||||
@@ -233,6 +255,11 @@ public class SiteHealthCollector : ISiteHealthCollector
|
||||
AuditRedactionFailure: auditRedactionFailures,
|
||||
SiteAuditBacklog: _siteAuditBacklog,
|
||||
SiteEventLogWriteFailures: Interlocked.Read(ref _siteEventLogWriteFailures),
|
||||
OldestParkedMessageAgeSeconds: ReadOldestParkedAgeSeconds());
|
||||
OldestParkedMessageAgeSeconds: ReadOldestParkedAgeSeconds())
|
||||
{
|
||||
ScriptQueueDepth = Interlocked.CompareExchange(ref _scriptQueueDepth, 0, 0),
|
||||
ScriptBusyThreads = Interlocked.CompareExchange(ref _scriptBusyThreads, 0, 0),
|
||||
ScriptOldestBusyAgeSeconds = ReadScriptOldestBusyAgeSeconds()
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
@@ -38,6 +38,25 @@ public class SiteHealthCollectorTests
|
||||
Assert.Equal(2, report.AlarmEvaluationErrorCount);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CollectReport_CarriesScriptSchedulerStats()
|
||||
{
|
||||
_collector.SetScriptSchedulerStats(queueDepth: 7, busyThreads: 3, oldestBusyAgeSeconds: 42.5);
|
||||
var report = _collector.CollectReport("site-1");
|
||||
Assert.Equal(7, report.ScriptQueueDepth);
|
||||
Assert.Equal(3, report.ScriptBusyThreads);
|
||||
Assert.Equal(42.5, report.ScriptOldestBusyAgeSeconds);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CollectReport_ScriptSchedulerStats_DefaultToZeroAndNull()
|
||||
{
|
||||
var report = _collector.CollectReport("site-1");
|
||||
Assert.Equal(0, report.ScriptQueueDepth);
|
||||
Assert.Equal(0, report.ScriptBusyThreads);
|
||||
Assert.Null(report.ScriptOldestBusyAgeSeconds);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void IncrementDeadLetter_AccumulatesBetweenReports()
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user