diff --git a/src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Scripts/ScriptExecutionScheduler.cs b/src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Scripts/ScriptExecutionScheduler.cs index b64cb2db..1cc102ba 100644 --- a/src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Scripts/ScriptExecutionScheduler.cs +++ b/src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Scripts/ScriptExecutionScheduler.cs @@ -22,6 +22,12 @@ public sealed class ScriptExecutionScheduler : TaskScheduler, IDisposable private readonly List _threads; private int _disposed; + // Per-worker "busy since" timestamp (Environment.TickCount64 ms) while a task + // is executing on that worker, 0 when idle. Written by the worker thread, + // read (lock-free) by the observability gauges below. S2/UA5: makes a + // saturated or stuck script-execution pool visible on the site health report. + private readonly long[] _busySinceTicks; + private static volatile ScriptExecutionScheduler? _shared; private static readonly object SharedLock = new(); @@ -52,10 +58,12 @@ public sealed class ScriptExecutionScheduler : TaskScheduler, IDisposable if (threadCount < 1) threadCount = 1; + _busySinceTicks = new long[threadCount]; _threads = new List(threadCount); for (var i = 0; i < threadCount; i++) { - var thread = new Thread(WorkerLoop) + var index = i; // capture per-worker slot index + var thread = new Thread(() => WorkerLoop(index)) { IsBackground = true, Name = $"script-execution-{i}" @@ -68,13 +76,57 @@ public sealed class ScriptExecutionScheduler : TaskScheduler, IDisposable /// public override int MaximumConcurrencyLevel => _threads.Count; - private void WorkerLoop() + /// Number of tasks waiting in the queue (not counting those currently executing). + public int QueueDepth => _queue.Count; + + /// Number of worker threads currently executing a task. + public int BusyThreadCount + { + get + { + var count = 0; + for (var i = 0; i < _busySinceTicks.Length; i++) + if (Volatile.Read(ref _busySinceTicks[i]) != 0) count++; + return count; + } + } + + /// + /// Age of the longest-running in-flight task across all workers, or null when + /// the pool is idle. A large value with a full + /// indicates a stuck/blocked script holding a dedicated thread. + /// + public TimeSpan? OldestBusyAge + { + get + { + var now = Environment.TickCount64; + long oldestSince = 0; + for (var i = 0; i < _busySinceTicks.Length; i++) + { + var since = Volatile.Read(ref _busySinceTicks[i]); + if (since != 0 && (oldestSince == 0 || since < oldestSince)) + oldestSince = since; + } + return oldestSince == 0 ? null : TimeSpan.FromMilliseconds(Math.Max(0, now - oldestSince)); + } + } + + private void WorkerLoop(int index) { try { foreach (var task in _queue.GetConsumingEnumerable()) { - TryExecuteTask(task); + Volatile.Write(ref _busySinceTicks[index], Environment.TickCount64); + try + { + TryExecuteTask(task); + } + finally + { + Volatile.Write(ref _busySinceTicks[index], 0); + } } } catch (ObjectDisposedException) diff --git a/tests/ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests/Scripts/ScriptExecutionSchedulerTests.cs b/tests/ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests/Scripts/ScriptExecutionSchedulerTests.cs index 8321daf0..27997918 100644 --- a/tests/ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests/Scripts/ScriptExecutionSchedulerTests.cs +++ b/tests/ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests/Scripts/ScriptExecutionSchedulerTests.cs @@ -44,4 +44,34 @@ public class ScriptExecutionSchedulerTests var b = ScriptExecutionScheduler.Shared(options); Assert.Same(a, b); } + + // SiteRuntime-S2/UA5: the scheduler exposes observability gauges so a stuck / + // saturated script-execution pool is visible to operators via site health. + [Fact] + public async Task Gauges_ReflectBusyThreadAndQueueDepth() + { + using var scheduler = new ScriptExecutionScheduler(1); + using var gate = new ManualResetEventSlim(false); + + var blocking = Task.Factory.StartNew(() => gate.Wait(), + CancellationToken.None, TaskCreationOptions.None, scheduler); + var queued = Task.Factory.StartNew(() => { }, + CancellationToken.None, TaskCreationOptions.None, scheduler); + + await WaitUntilAsync(() => scheduler.BusyThreadCount == 1); + Assert.Equal(1, scheduler.QueueDepth); // second task waiting + Assert.True(scheduler.OldestBusyAge > TimeSpan.Zero); + + gate.Set(); + await Task.WhenAll(blocking, queued); + await WaitUntilAsync(() => scheduler.BusyThreadCount == 0); + Assert.Null(scheduler.OldestBusyAge); + } + + private static async Task WaitUntilAsync(Func condition) + { + for (var i = 0; i < 100 && !condition(); i++) + await Task.Delay(50); + Assert.True(condition(), "condition not met within timeout"); + } }