feat(site-runtime): periodic scheduler-stats reporter feeding site health (S2/UA5)

2026-07-08 23:43:09 -04:00
parent d44b6a0e52
commit dddafc025e
3 changed files with 147 additions and 0 deletions
@@ -0,0 +1,83 @@
+using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Logging;
+using ZB.MOM.WW.ScadaBridge.HealthMonitoring;
+
+namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts;
+
+/// <summary>
+/// Site-side hosted service (S2/UA5) that periodically reads the process-wide
+/// <see cref="ScriptExecutionScheduler"/> gauges (queue depth, busy thread count,
+/// oldest in-flight script age) and pushes them into
+/// <see cref="ISiteHealthCollector.SetScriptSchedulerStats"/> so the next
+/// <see cref="ISiteHealthCollector.CollectReport"/> carries a fresh snapshot on
+/// the site health report. Mirrors <c>SiteEventLogFailureCountReporter</c>:
+/// immediate first probe, fixed cadence, exceptions logged and swallowed so the
+/// loop survives every probe failure.
+/// </summary>
+public sealed class ScriptSchedulerStatsReporter : BackgroundService
+{
+    /// <summary>Default poll cadence (10 s) — coarse enough to amortise across health reports.</summary>
+    internal static readonly TimeSpan DefaultPollInterval = TimeSpan.FromSeconds(10);
+
+    private readonly ISiteHealthCollector _collector;
+    private readonly SiteRuntimeOptions _options;
+    private readonly ILogger<ScriptSchedulerStatsReporter> _logger;
+    private readonly TimeSpan _pollInterval;
+
+    /// <summary>Initializes a new instance of <see cref="ScriptSchedulerStatsReporter"/>.</summary>
+    /// <param name="collector">The site health collector that receives the scheduler gauges.</param>
+    /// <param name="options">Site runtime options supplying the shared scheduler's thread count.</param>
+    /// <param name="logger">Logger instance.</param>
+    /// <param name="pollInterval">Poll interval override; defaults to <see cref="DefaultPollInterval"/> (10 s).</param>
+    public ScriptSchedulerStatsReporter(
+        ISiteHealthCollector collector,
+        SiteRuntimeOptions options,
+        ILogger<ScriptSchedulerStatsReporter> logger,
+        TimeSpan? pollInterval = null)
+    {
+        _collector = collector ?? throw new ArgumentNullException(nameof(collector));
+        _options = options ?? throw new ArgumentNullException(nameof(options));
+        _logger = logger ?? throw new ArgumentNullException(nameof(logger));
+        _pollInterval = pollInterval ?? DefaultPollInterval;
+    }
+
+    /// <inheritdoc />
+    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
+    {
+        // Immediate first probe so the first health report after start carries a
+        // real snapshot instead of zeros.
+        Probe();
+
+        while (!stoppingToken.IsCancellationRequested)
+        {
+            try
+            {
+                await Task.Delay(_pollInterval, stoppingToken).ConfigureAwait(false);
+            }
+            catch (OperationCanceledException)
+            {
+                break;
+            }
+
+            Probe();
+        }
+    }
+
+    private void Probe()
+    {
+        try
+        {
+            var scheduler = ScriptExecutionScheduler.Shared(_options);
+            _collector.SetScriptSchedulerStats(
+                scheduler.QueueDepth,
+                scheduler.BusyThreadCount,
+                scheduler.OldestBusyAge?.TotalSeconds);
+        }
+        catch (Exception ex)
+        {
+            // Catch-all is deliberate: the hosted service must survive every class
+            // of probe failure so the next tick gets a chance.
+            _logger.LogWarning(ex, "ScriptSchedulerStatsReporter probe failed; next tick will retry.");
+        }
+    }
+}
@@ -70,6 +70,15 @@ public static class ServiceCollectionExtensions
                c.Timeout = TimeSpan.FromSeconds(
                    sp.GetRequiredService<IOptions<SiteRuntimeOptions>>().Value.ConfigFetchTimeoutSeconds));

+        // S2/UA5: periodically lift the shared script-execution scheduler gauges
+        // (queue depth, busy threads, oldest-busy age) onto the site health report.
+        // Registered via factory-lambda so ISiteHealthCollector (registered by the
+        // Host's AddSiteHealthMonitoring) is resolved lazily at host start.
+        services.AddHostedService(sp => new ScriptSchedulerStatsReporter(
+            sp.GetRequiredService<HealthMonitoring.ISiteHealthCollector>(),
+            sp.GetRequiredService<IOptions<SiteRuntimeOptions>>().Value,
+            sp.GetRequiredService<ILogger<ScriptSchedulerStatsReporter>>()));
+
        return services;
    }

@@ -0,0 +1,55 @@
+using Microsoft.Extensions.Logging.Abstractions;
+using ZB.MOM.WW.ScadaBridge.HealthMonitoring;
+using ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts;
+
+namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Scripts;
+
+/// <summary>
+/// SiteRuntime-S2/UA5: the hosted reporter must lift the process-wide
+/// <see cref="ScriptExecutionScheduler"/> gauges onto the site health report via
+/// <see cref="ISiteHealthCollector"/>. Uses the real collector (NSubstitute is not
+/// referenced by this test project) and drives the shared scheduler busy so the
+/// pushed value is distinguishable from the idle default.
+/// </summary>
+public class ScriptSchedulerStatsReporterTests
+{
+    [Fact]
+    public async Task Reporter_PushesSchedulerStatsToCollector()
+    {
+        var options = new SiteRuntimeOptions { ScriptExecutionThreadCount = 1 };
+        var collector = new SiteHealthCollector();
+        var scheduler = ScriptExecutionScheduler.Shared(options);
+
+        using var gate = new ManualResetEventSlim(false);
+        // Occupy a worker thread so BusyThreadCount is a non-zero, distinguishable value.
+        var blocking = Task.Factory.StartNew(() => gate.Wait(),
+            CancellationToken.None, TaskCreationOptions.None, scheduler);
+        await WaitUntilAsync(() => scheduler.BusyThreadCount >= 1);
+
+        using var reporter = new ScriptSchedulerStatsReporter(
+            collector, options, NullLogger<ScriptSchedulerStatsReporter>.Instance,
+            pollInterval: TimeSpan.FromMilliseconds(50));
+
+        await reporter.StartAsync(CancellationToken.None);
+        try
+        {
+            await WaitUntilAsync(() => collector.CollectReport("site-1").ScriptBusyThreads >= 1);
+            var report = collector.CollectReport("site-1");
+            Assert.True(report.ScriptBusyThreads >= 1);
+            Assert.NotNull(report.ScriptOldestBusyAgeSeconds); // a script is in flight
+        }
+        finally
+        {
+            gate.Set();
+            await blocking;
+            await reporter.StopAsync(CancellationToken.None);
+        }
+    }
+
+    private static async Task WaitUntilAsync(Func<bool> condition)
+    {
+        for (var i = 0; i < 100 && !condition(); i++)
+            await Task.Delay(50);
+        Assert.True(condition(), "condition not met within timeout");
+    }
+}