From 474ec8a22152cc3950a4d1a129ef43835dca57bc Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Wed, 8 Jul 2026 23:35:09 -0400 Subject: [PATCH] =?UTF-8?q?feat(site-runtime):=20script=20scheduler=20gaug?= =?UTF-8?q?es=20=E2=80=94=20queue=20depth,=20busy=20threads,=20oldest-busy?= =?UTF-8?q?=20age=20(S2/UA5)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Scripts/ScriptExecutionScheduler.cs | 58 ++++++++++++++++++- .../Scripts/ScriptExecutionSchedulerTests.cs | 30 ++++++++++ 2 files changed, 85 insertions(+), 3 deletions(-) diff --git a/src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Scripts/ScriptExecutionScheduler.cs b/src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Scripts/ScriptExecutionScheduler.cs index b64cb2db..1cc102ba 100644 --- a/src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Scripts/ScriptExecutionScheduler.cs +++ b/src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Scripts/ScriptExecutionScheduler.cs @@ -22,6 +22,12 @@ public sealed class ScriptExecutionScheduler : TaskScheduler, IDisposable private readonly List _threads; private int _disposed; + // Per-worker "busy since" timestamp (Environment.TickCount64 ms) while a task + // is executing on that worker, 0 when idle. Written by the worker thread, + // read (lock-free) by the observability gauges below. S2/UA5: makes a + // saturated or stuck script-execution pool visible on the site health report. + private readonly long[] _busySinceTicks; + private static volatile ScriptExecutionScheduler? _shared; private static readonly object SharedLock = new(); @@ -52,10 +58,12 @@ public sealed class ScriptExecutionScheduler : TaskScheduler, IDisposable if (threadCount < 1) threadCount = 1; + _busySinceTicks = new long[threadCount]; _threads = new List(threadCount); for (var i = 0; i < threadCount; i++) { - var thread = new Thread(WorkerLoop) + var index = i; // capture per-worker slot index + var thread = new Thread(() => WorkerLoop(index)) { IsBackground = true, Name = $"script-execution-{i}" @@ -68,13 +76,57 @@ public sealed class ScriptExecutionScheduler : TaskScheduler, IDisposable /// public override int MaximumConcurrencyLevel => _threads.Count; - private void WorkerLoop() + /// Number of tasks waiting in the queue (not counting those currently executing). + public int QueueDepth => _queue.Count; + + /// Number of worker threads currently executing a task. + public int BusyThreadCount + { + get + { + var count = 0; + for (var i = 0; i < _busySinceTicks.Length; i++) + if (Volatile.Read(ref _busySinceTicks[i]) != 0) count++; + return count; + } + } + + /// + /// Age of the longest-running in-flight task across all workers, or null when + /// the pool is idle. A large value with a full + /// indicates a stuck/blocked script holding a dedicated thread. + /// + public TimeSpan? OldestBusyAge + { + get + { + var now = Environment.TickCount64; + long oldestSince = 0; + for (var i = 0; i < _busySinceTicks.Length; i++) + { + var since = Volatile.Read(ref _busySinceTicks[i]); + if (since != 0 && (oldestSince == 0 || since < oldestSince)) + oldestSince = since; + } + return oldestSince == 0 ? null : TimeSpan.FromMilliseconds(Math.Max(0, now - oldestSince)); + } + } + + private void WorkerLoop(int index) { try { foreach (var task in _queue.GetConsumingEnumerable()) { - TryExecuteTask(task); + Volatile.Write(ref _busySinceTicks[index], Environment.TickCount64); + try + { + TryExecuteTask(task); + } + finally + { + Volatile.Write(ref _busySinceTicks[index], 0); + } } } catch (ObjectDisposedException) diff --git a/tests/ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests/Scripts/ScriptExecutionSchedulerTests.cs b/tests/ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests/Scripts/ScriptExecutionSchedulerTests.cs index 8321daf0..27997918 100644 --- a/tests/ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests/Scripts/ScriptExecutionSchedulerTests.cs +++ b/tests/ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests/Scripts/ScriptExecutionSchedulerTests.cs @@ -44,4 +44,34 @@ public class ScriptExecutionSchedulerTests var b = ScriptExecutionScheduler.Shared(options); Assert.Same(a, b); } + + // SiteRuntime-S2/UA5: the scheduler exposes observability gauges so a stuck / + // saturated script-execution pool is visible to operators via site health. + [Fact] + public async Task Gauges_ReflectBusyThreadAndQueueDepth() + { + using var scheduler = new ScriptExecutionScheduler(1); + using var gate = new ManualResetEventSlim(false); + + var blocking = Task.Factory.StartNew(() => gate.Wait(), + CancellationToken.None, TaskCreationOptions.None, scheduler); + var queued = Task.Factory.StartNew(() => { }, + CancellationToken.None, TaskCreationOptions.None, scheduler); + + await WaitUntilAsync(() => scheduler.BusyThreadCount == 1); + Assert.Equal(1, scheduler.QueueDepth); // second task waiting + Assert.True(scheduler.OldestBusyAge > TimeSpan.Zero); + + gate.Set(); + await Task.WhenAll(blocking, queued); + await WaitUntilAsync(() => scheduler.BusyThreadCount == 0); + Assert.Null(scheduler.OldestBusyAge); + } + + private static async Task WaitUntilAsync(Func condition) + { + for (var i = 0; i < 100 && !condition(); i++) + await Task.Delay(50); + Assert.True(condition(), "condition not met within timeout"); + } }