test(scriptworker): tight-heartbeat tests warm the worker before arming the 3-interval silence kill (#827 audit)
A_burst_of_cold_compiles_does_not_starve_the_heartbeat (50 ms heartbeat) and A_caller_that_blocks_after_its_result_does_not_starve_the_reader (100 ms) launched their worker under the tight silence kill, so its cold start (launch, first compile, Roslyn JIT) had to stay within 150/300 ms of silence. Under 28 busy processes on 14 cores that failed 2 in 10 runs in the WARM-UP, killed for ~165 ms of silence, and replacement workers were killed the same way. New test seam: WorkerLaunchSpec.HeartbeatSilenceLimit (runner internal ctor heartbeatSilenceLimit) and WorkerConnection.SilenceLimit. The tests launch with a 30 s limit, warm, then tighten to three intervals for the phase they are about; the assertions on that phase are unchanged.
This commit is contained in:
@@ -31,6 +31,27 @@ public sealed class SupervisionTests
|
||||
|
||||
private static CancellationToken Ct => TestContext.Current.CancellationToken;
|
||||
|
||||
/// <summary>
|
||||
/// A runner whose workers heartbeat every <paramref name="heartbeat"/> but start with a generous silence limit:
|
||||
/// the cold start (launch, first compile, Roslyn's JIT) is not what the tight-heartbeat tests are about, and on a
|
||||
/// loaded box it can pause the worker for longer than three short intervals (lmxopcua#827 audit: 2 in 10 runs
|
||||
/// under 28 busy processes on 14 cores were killed for 160 ms of silence during the warm-up compile, and the
|
||||
/// replacement workers were too). <see cref="WarmTight"/> then tightens the limit to three intervals for the
|
||||
/// phase under test.
|
||||
/// </summary>
|
||||
private static OutOfProcessScriptRunner TightHeartbeatRunner(TimeSpan heartbeat, TimeSpan loadTimeout, CapturingSink sink)
|
||||
=> new(Scripts.Options(), sink.CreateLogger(), null, heartbeatInterval: heartbeat, loadTimeout: loadTimeout,
|
||||
heartbeatSilenceLimit: TimeSpan.FromSeconds(30));
|
||||
|
||||
/// <summary>Warms the shard's worker with <paramref name="key"/>, then arms the three-interval silence kill.</summary>
|
||||
private static WorkerConnection WarmTight(OutOfProcessScriptRunner runner, ScriptRunOutcome warm, TimeSpan heartbeat, CapturingSink sink)
|
||||
{
|
||||
warm.Value.ShouldBe(7, $"{warm.Status} {warm.Reason}\n{string.Join(Environment.NewLine, sink.Rendered)}");
|
||||
var worker = runner.Shards[0].Current.ShouldNotBeNull();
|
||||
worker.SilenceLimit = heartbeat * 3;
|
||||
return worker;
|
||||
}
|
||||
|
||||
private static string VtKey(string key) => InProcessScriptRunner.ScopedKey(ScriptContextKind.VirtualTag, key);
|
||||
|
||||
// ---------------------------------------------------------------- the quarantine race
|
||||
@@ -112,10 +133,9 @@ public sealed class SupervisionTests
|
||||
public async Task A_burst_of_cold_compiles_does_not_starve_the_heartbeat()
|
||||
{
|
||||
var sink = new CapturingSink();
|
||||
await using var runner = new OutOfProcessScriptRunner(Scripts.Options(), sink.CreateLogger(), null,
|
||||
heartbeatInterval: TimeSpan.FromMilliseconds(50), loadTimeout: TimeSpan.FromSeconds(120));
|
||||
(await runner.RunAsync(Scripts.Vt("first", Healthy), Ct)).Value.ShouldBe(7);
|
||||
var worker = runner.Shards[0].Current.ShouldNotBeNull();
|
||||
var heartbeat = TimeSpan.FromMilliseconds(50);
|
||||
await using var runner = TightHeartbeatRunner(heartbeat, TimeSpan.FromSeconds(120), sink);
|
||||
var worker = WarmTight(runner, await runner.RunAsync(Scripts.Vt("first", Healthy), Ct), heartbeat, sink);
|
||||
|
||||
const int count = 1500;
|
||||
var outcomes = await Task.WhenAll(Enumerable.Range(0, count).Select(i => runner.RunAsync(
|
||||
@@ -514,10 +534,9 @@ public sealed class SupervisionTests
|
||||
public async Task A_caller_that_blocks_after_its_result_does_not_starve_the_reader()
|
||||
{
|
||||
var sink = new CapturingSink();
|
||||
await using var runner = new OutOfProcessScriptRunner(Scripts.Options(), sink.CreateLogger(), null,
|
||||
heartbeatInterval: TimeSpan.FromMilliseconds(100), loadTimeout: TimeSpan.FromSeconds(60));
|
||||
(await runner.RunAsync(Scripts.Vt("warm", Healthy), Ct)).Value.ShouldBe(7);
|
||||
var worker = runner.Shards[0].Current.ShouldNotBeNull();
|
||||
var heartbeat = TimeSpan.FromMilliseconds(100);
|
||||
await using var runner = TightHeartbeatRunner(heartbeat, TimeSpan.FromSeconds(60), sink);
|
||||
var worker = WarmTight(runner, await runner.RunAsync(Scripts.Vt("warm", Healthy), Ct), heartbeat, sink);
|
||||
|
||||
string? resumedOn = null;
|
||||
await runner.RunAsync(Scripts.Vt("warm", Healthy), Ct).ContinueWith(t =>
|
||||
|
||||
Reference in New Issue
Block a user