perf(runtime): split trigger evals from the blocking pool; bounded, deadline-aware execution
This commit is contained in:
Joseph Doherty
@@ -0,0 +1,195 @@
|
||||
using Akka.Actor;
|
||||
using Akka.TestKit.Xunit2;
|
||||
using Microsoft.CodeAnalysis.CSharp.Scripting;
|
||||
using Microsoft.CodeAnalysis.Scripting;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.ScriptExecution;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Flattening;
|
||||
using ZB.MOM.WW.ScadaBridge.HealthMonitoring;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Actors;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.TestSupport;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Actors;
|
||||
|
||||
/// <summary>
|
||||
/// WP3.1 test group 4 — the stuck-script watchdog now REPLACES the thread a wedged script is
|
||||
/// holding, not just names it.
|
||||
///
|
||||
/// <para>Before WP3.1 the watchdog was observability only: a script blocked in synchronous,
|
||||
/// uninterruptible I/O never observes the cooperative cancellation the timeout requests, so
|
||||
/// the pool simply lost that dedicated thread — permanently, and silently apart from one log
|
||||
/// line. Eight such scripts left the site with no script execution at all. The watchdog now
|
||||
/// detaches the worker (it exits when its body finally returns) and starts a replacement, and
|
||||
/// the count of live detached workers is surfaced on the site health report as
|
||||
/// <c>DetachedScriptThreads</c>. Replacement is capped at the pool size, because bounded
|
||||
/// starvation beats unbounded thread growth when scripts wedge en masse.</para>
|
||||
/// </summary>
|
||||
public class StuckScriptWatchdogTests : TestKit, IDisposable
|
||||
{
|
||||
private readonly SharedScriptLibrary _sharedLibrary;
|
||||
|
||||
public StuckScriptWatchdogTests()
|
||||
{
|
||||
var compilationService = new ScriptCompilationService(
|
||||
NullLogger<ScriptCompilationService>.Instance);
|
||||
_sharedLibrary = new SharedScriptLibrary(
|
||||
compilationService, NullLogger<SharedScriptLibrary>.Instance);
|
||||
WatchdogHooks.Gate = new SemaphoreSlim(0);
|
||||
}
|
||||
|
||||
void IDisposable.Dispose()
|
||||
{
|
||||
// Free any still-wedged worker before tearing down, so no test leaves a blocked thread.
|
||||
WatchdogHooks.Gate.Release(8);
|
||||
Shutdown();
|
||||
}
|
||||
|
||||
private static Script<object?> CompileRaw(string code)
|
||||
{
|
||||
var scriptOptions = ScriptOptions.Default
|
||||
.WithReferences(typeof(object).Assembly, typeof(Enumerable).Assembly,
|
||||
typeof(WatchdogHooks).Assembly)
|
||||
.WithImports("System", "System.Collections.Generic", "System.Linq", "System.Threading.Tasks");
|
||||
var script = CSharpScript.Create<object?>(code, scriptOptions, typeof(ScriptGlobals));
|
||||
script.Compile();
|
||||
return script;
|
||||
}
|
||||
|
||||
/// <summary>A body that blocks its worker thread outright — cooperative cancellation is never observed.</summary>
|
||||
private static Script<object?> WedgeScript() => CompileRaw(
|
||||
"ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Actors.WatchdogHooks.Gate.Wait(); return null;");
|
||||
|
||||
private IActorRef BuildScriptActor(
|
||||
string name,
|
||||
Script<object?> compiled,
|
||||
SiteRuntimeOptions options,
|
||||
ScriptExecutionScheduler scheduler,
|
||||
IServiceProvider? serviceProvider)
|
||||
{
|
||||
var instance = CreateTestProbe().Ref;
|
||||
var config = new ResolvedScript { CanonicalName = name, TriggerType = "Call" };
|
||||
return ActorOf(Props.Create(() => new ScriptActor(
|
||||
name, "Inst1", instance, compiled, config, _sharedLibrary, options,
|
||||
NullLogger<ScriptActor>.Instance, null, null, null, serviceProvider, scheduler, null)));
|
||||
}
|
||||
|
||||
private static SiteRuntimeOptions WatchdogOptions() => new()
|
||||
{
|
||||
ScriptExecutionTimeoutSeconds = 1,
|
||||
StuckScriptGraceMs = 200,
|
||||
MaxScriptCallDepth = 10
|
||||
};
|
||||
|
||||
[Fact]
|
||||
public void WedgedScript_DetachesItsWorker_StartsAReplacement_AndDrainsWhenFreed()
|
||||
{
|
||||
using var scheduler = new ScriptExecutionScheduler(1);
|
||||
var siteLog = new FakeSiteEventLogger();
|
||||
var options = WatchdogOptions();
|
||||
|
||||
var wedged = BuildScriptActor("Wedged", WedgeScript(), options, scheduler,
|
||||
new SingleServiceProvider(siteLog));
|
||||
wedged.Tell(new ScriptCallRequest("Wedged", null, 0, "corr-wedge"), ActorRefs.NoSender);
|
||||
|
||||
// Timeout (1 s) + grace (200 ms) later the watchdog fires: the worker is detached and
|
||||
// a replacement thread starts.
|
||||
AwaitAssert(
|
||||
() => Assert.Equal(1, scheduler.DetachedThreadCount),
|
||||
TimeSpan.FromSeconds(15));
|
||||
|
||||
AwaitAssert(
|
||||
() => Assert.Contains(siteLog.OfType("script"), r =>
|
||||
r.Severity == "Error" &&
|
||||
r.Message.Contains("still executing", StringComparison.OrdinalIgnoreCase) &&
|
||||
r.Message.Contains("Wedged") &&
|
||||
r.Message.Contains("DETACHED")),
|
||||
TimeSpan.FromSeconds(5));
|
||||
|
||||
// The replacement worker is live: a fresh script runs even though the pool's only
|
||||
// original thread is still blocked. Before WP3.1 this reply never came.
|
||||
var healthy = BuildScriptActor("Healthy", CompileRaw("return 7;"), options, scheduler, null);
|
||||
var caller = CreateTestProbe();
|
||||
healthy.Tell(new ScriptCallRequest("Healthy", null, 0, "corr-healthy"), caller.Ref);
|
||||
var result = caller.ExpectMsg<ScriptCallResult>(TimeSpan.FromSeconds(15));
|
||||
Assert.True(result.Success, result.ErrorMessage);
|
||||
Assert.Equal(7, result.ReturnValue);
|
||||
|
||||
// Freeing the wedged body lets the detached worker exit rather than pull more work,
|
||||
// so the pool does not silently end up double-sized.
|
||||
WatchdogHooks.Gate.Release();
|
||||
AwaitAssert(
|
||||
() => Assert.Equal(0, scheduler.DetachedThreadCount),
|
||||
TimeSpan.FromSeconds(15));
|
||||
Assert.Equal(1, scheduler.MaximumConcurrencyLevel);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void AtTheDetachCap_NoFurtherReplacementIsStarted_AndAnErrorEventIsEmitted()
|
||||
{
|
||||
// A one-thread pool caps live detached workers at one, so the SECOND wedge cannot be
|
||||
// replaced — the deliberate "bounded starvation beats unbounded thread growth" trade.
|
||||
using var scheduler = new ScriptExecutionScheduler(1);
|
||||
var siteLog = new FakeSiteEventLogger();
|
||||
var options = WatchdogOptions();
|
||||
|
||||
var first = BuildScriptActor("WedgeOne", WedgeScript(), options, scheduler,
|
||||
new SingleServiceProvider(siteLog));
|
||||
first.Tell(new ScriptCallRequest("WedgeOne", null, 0, "corr-w1"), ActorRefs.NoSender);
|
||||
AwaitAssert(() => Assert.Equal(1, scheduler.DetachedThreadCount), TimeSpan.FromSeconds(15));
|
||||
|
||||
var second = BuildScriptActor("WedgeTwo", WedgeScript(), options, scheduler,
|
||||
new SingleServiceProvider(siteLog));
|
||||
second.Tell(new ScriptCallRequest("WedgeTwo", null, 0, "corr-w2"), ActorRefs.NoSender);
|
||||
|
||||
AwaitAssert(
|
||||
() => Assert.Contains(siteLog.OfType("script"), r =>
|
||||
r.Severity == "Error" &&
|
||||
r.Message.Contains("WedgeTwo") &&
|
||||
r.Message.Contains("at cap", StringComparison.OrdinalIgnoreCase)),
|
||||
TimeSpan.FromSeconds(20));
|
||||
|
||||
// Still exactly one detached worker: the second wedge was NOT replaced.
|
||||
Assert.Equal(1, scheduler.DetachedThreadCount);
|
||||
|
||||
WatchdogHooks.Gate.Release(2);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task DetachedThreadCount_IsSurfacedOnTheSiteHealthReport()
|
||||
{
|
||||
using var scheduler = new ScriptExecutionScheduler(1);
|
||||
var collector = new SiteHealthCollector();
|
||||
|
||||
var wedged = BuildScriptActor("GaugeWedge", WedgeScript(), WatchdogOptions(), scheduler, null);
|
||||
wedged.Tell(new ScriptCallRequest("GaugeWedge", null, 0, "corr-gauge"), ActorRefs.NoSender);
|
||||
AwaitAssert(() => Assert.Equal(1, scheduler.DetachedThreadCount), TimeSpan.FromSeconds(15));
|
||||
|
||||
using var reporter = new ScriptSchedulerStatsReporter(
|
||||
collector, WatchdogOptions(), NullLogger<ScriptSchedulerStatsReporter>.Instance,
|
||||
pollInterval: TimeSpan.FromMilliseconds(50), scheduler: scheduler);
|
||||
await reporter.StartAsync(CancellationToken.None);
|
||||
try
|
||||
{
|
||||
AwaitAssert(
|
||||
() => Assert.Equal(1, collector.CollectReport("site-1").DetachedScriptThreads),
|
||||
TimeSpan.FromSeconds(10));
|
||||
}
|
||||
finally
|
||||
{
|
||||
await reporter.StopAsync(CancellationToken.None);
|
||||
WatchdogHooks.Gate.Release();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Test hook the stuck-script watchdog tests use to block a script-execution thread
|
||||
/// deterministically: the compiled body waits on <see cref="Gate"/>, which the test releases
|
||||
/// once it has observed the detach.
|
||||
/// </summary>
|
||||
public static class WatchdogHooks
|
||||
{
|
||||
/// <summary>Gate a blocking test script waits on; reset per test.</summary>
|
||||
public static SemaphoreSlim Gate = new(0);
|
||||
}
|
||||
Reference in New Issue
Block a user