Joseph Doherty
3f5fdd1d90
With the worker soft clock the host's per-run timer is only the transport deadline and the kill G is armed by the worker's soft notice, so a worker whose watchdog never sends one (the design treats the worker as untrusted) held the run, the key's slot and the tag's BadResourceUnavailable forever; a deferred run had no host clock at all. WorkerConnection now tracks evaluation progress (any Results frame, or the submit that made the worker busy again). The heartbeat monitor kills the worker when evaluations are outstanding and nothing progressed for 3 x the largest outstanding T + G plus the silence limit: WorkerExitReason.KilledUnresponsive (metric reason killed_unresponsive), no culprit, attributed like a heartbeat kill (suspects, no strike on a shard). A legitimate run has answered or been killed by T + G. Test seams: ProgressCeilingOverride, DropSoftNoticesForTest. Tests: a worker whose notices are dropped (G = 60 s, so its own fail-fast cannot end it) is killed KilledUnresponsive after the 1.5 s ceiling and the run is released (with the check disabled the test times out); positive control: 20 queued 150 ms runs behind one thread under a 1 s ceiling are all answered.
478 lines
27 KiB
C#
478 lines
27 KiB
C#
using System.Diagnostics;
|
|
using System.Runtime.InteropServices;
|
|
using System.Security.Cryptography;
|
|
using System.Text;
|
|
using Serilog;
|
|
using Shouldly;
|
|
using Xunit;
|
|
using ZB.MOM.WW.OtOpcUa.Core.Scripting;
|
|
using ZB.MOM.WW.OtOpcUa.Core.Scripting.OutOfProcess;
|
|
using ZB.MOM.WW.OtOpcUa.Core.Scripting.Protocol.V1;
|
|
|
|
namespace ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests;
|
|
|
|
/// <summary>
|
|
/// lmxopcua#827: the soft deadline T measures the script's OWN execution — the worker runs it from the moment an
|
|
/// evaluation thread starts the run — so the pipe round trip and the time a starved worker waits to be scheduled
|
|
/// no longer count against the script. The host keeps a transport deadline (T + G from submit) and reports passing
|
|
/// it as a runner fault (<see cref="ScriptRunOutcome.WorkerUnresponsiveReason"/>, BadResourceUnavailable), never as a
|
|
/// script timeout. The starved worker is made real here: its process is suspended (SIGSTOP / NtSuspendProcess)
|
|
/// while a trivial script is submitted.
|
|
/// </summary>
|
|
[Collection(ScriptWorkerCollection.Name)]
|
|
[Trait("Category", "ScriptWorker")]
|
|
public sealed class WorkerSoftClockTests
|
|
{
|
|
private const string Trivial = "return 7;";
|
|
private const string Runaway = "while (true) { }";
|
|
|
|
private static CancellationToken Ct => TestContext.Current.CancellationToken;
|
|
|
|
/// <summary>
|
|
/// A runner whose worker heartbeats every 2 s, so the host's silence kill (3 intervals) stays well clear of the
|
|
/// suspensions below: what is measured is the deadline, not the heartbeat monitor.
|
|
/// </summary>
|
|
private static OutOfProcessScriptRunner Runner(TimeSpan hardGrace, ILogger? log = null)
|
|
=> new(Scripts.Options(hardGrace: hardGrace, quarantineAfterKills: 1), log ?? new LoggerConfiguration().CreateLogger(), null,
|
|
heartbeatInterval: TimeSpan.FromSeconds(2), loadTimeout: TimeSpan.FromSeconds(60));
|
|
|
|
private static async Task<WorkerConnection> WarmAsync(OutOfProcessScriptRunner runner, string key)
|
|
{
|
|
var warm = await runner.RunAsync(Scripts.Vt(key, Trivial, TimeSpan.FromSeconds(30)), Ct);
|
|
warm.Status.ShouldBe(ScriptRunStatus.Ok, warm.Reason);
|
|
var worker = runner.Shards[0].Current.ShouldNotBeNull();
|
|
worker.WorkerSoftClock.ShouldBeTrue("the worker negotiated the soft clock");
|
|
return worker;
|
|
}
|
|
|
|
/// <summary>
|
|
/// The issue's headline: a worker that cannot be scheduled for longer than T, but shorter than the transport
|
|
/// deadline, answers a trivial script with its value. Before #827 the host's clock, started at submit, answered
|
|
/// <c>script timed out after 0.3s</c> for <c>return 7;</c>.
|
|
/// </summary>
|
|
[Fact]
|
|
public async Task A_worker_stalled_for_longer_than_T_still_answers_a_trivial_script_with_its_value()
|
|
{
|
|
var softT = TimeSpan.FromMilliseconds(300);
|
|
await using var runner = Runner(hardGrace: TimeSpan.FromSeconds(3));
|
|
var worker = await WarmAsync(runner, "trivial");
|
|
|
|
Task<ScriptRunOutcome> run;
|
|
using (ProcessFreezer.Freeze(worker.Pid))
|
|
{
|
|
run = runner.RunAsync(Scripts.Vt("trivial", Trivial, softT), Ct);
|
|
await Task.Delay(softT * 4, Ct); // 1.2 s: well past T on the host's clock
|
|
run.IsCompleted.ShouldBeFalse("nothing can answer while the worker is suspended");
|
|
}
|
|
|
|
var outcome = await run.WaitAsync(TimeSpan.FromSeconds(30), Ct);
|
|
outcome.Status.ShouldBe(ScriptRunStatus.Ok, outcome.Reason);
|
|
outcome.Value.ShouldBe(7);
|
|
runner.Shards[0].Current.ShouldNotBeNull().Pid.ShouldBe(worker.Pid, "a stall is not a deadline kill");
|
|
runner.QuarantinedKeys(ScriptContextKind.VirtualTag).ShouldBeEmpty();
|
|
}
|
|
|
|
/// <summary>
|
|
/// A stall past the transport deadline (T + G) is the runner's fault: the caller gets
|
|
/// <see cref="ScriptRunOutcome.WorkerUnresponsiveReason"/> as <see cref="ScriptRunStatus.Unavailable"/> (published
|
|
/// BadResourceUnavailable) with a recovery that completes when the worker answers; a re-trigger meanwhile is the
|
|
/// runner's fault too (not "previous run still running"); the worker is not killed and the key is not struck.
|
|
/// Before #827 this was a script timeout at T and a deadline kill at T + G that struck the key — with
|
|
/// QuarantineAfterKills = 1, a quarantine for <c>return 7;</c>.
|
|
/// </summary>
|
|
[Fact]
|
|
public async Task A_worker_stalled_past_the_transport_deadline_is_a_runner_fault_and_nothing_is_struck()
|
|
{
|
|
var softT = TimeSpan.FromMilliseconds(200);
|
|
var grace = TimeSpan.FromMilliseconds(400);
|
|
await using var runner = Runner(grace);
|
|
var worker = await WarmAsync(runner, "trivial");
|
|
|
|
ScriptRunOutcome stalled, retriggered;
|
|
using (ProcessFreezer.Freeze(worker.Pid))
|
|
{
|
|
stalled = await runner.RunAsync(Scripts.Vt("trivial", Trivial, softT), Ct).WaitAsync(TimeSpan.FromSeconds(10), Ct);
|
|
retriggered = await runner.RunAsync(Scripts.Vt("trivial", Trivial, softT), Ct).WaitAsync(TimeSpan.FromSeconds(10), Ct);
|
|
stalled.Recovery.ShouldNotBeNull().IsCompleted.ShouldBeFalse("the worker has not answered yet");
|
|
}
|
|
|
|
stalled.Status.ShouldBe(ScriptRunStatus.Unavailable, stalled.Reason);
|
|
stalled.Reason.ShouldBe(ScriptRunOutcome.WorkerUnresponsiveReason);
|
|
retriggered.Status.ShouldBe(ScriptRunStatus.Unavailable, retriggered.Reason);
|
|
retriggered.Reason.ShouldBe(ScriptRunOutcome.WorkerUnresponsiveReason);
|
|
stalled.ToVirtualTagEvalResult().StatusCode.ShouldBe(Commons.OpcUa.OpcUaStatus.BadResourceUnavailable);
|
|
|
|
await stalled.Recovery!.WaitAsync(TimeSpan.FromSeconds(30), Ct);
|
|
var next = await runner.RunAsync(Scripts.Vt("trivial", Trivial, softT), Ct);
|
|
next.Status.ShouldBe(ScriptRunStatus.Ok, next.Reason);
|
|
next.Value.ShouldBe(7);
|
|
runner.Shards[0].Current.ShouldNotBeNull().Pid.ShouldBe(worker.Pid, "a stall is not a deadline kill");
|
|
runner.QuarantinedKeys(ScriptContextKind.VirtualTag).ShouldBeEmpty();
|
|
}
|
|
|
|
/// <summary>
|
|
/// The other side of the contract is unchanged: a script that really runs past T is answered at T from its own
|
|
/// start and killed G after that, even with the worker stalled before it started. Positive control for the two
|
|
/// tests above — they would also pass if the runner simply never timed anything out.
|
|
/// </summary>
|
|
[Fact]
|
|
public async Task A_runaway_that_starts_late_is_still_timed_out_at_T_from_its_start_and_killed_at_G()
|
|
{
|
|
var softT = TimeSpan.FromMilliseconds(300);
|
|
var grace = TimeSpan.FromMilliseconds(600);
|
|
var stall = TimeSpan.FromMilliseconds(500);
|
|
await using var runner = Runner(grace);
|
|
var worker = await WarmAsync(runner, "warm");
|
|
var load = await runner.RunAsync(Scripts.Vt("runaway", "return 0;", TimeSpan.FromSeconds(30)), Ct);
|
|
load.Status.ShouldBe(ScriptRunStatus.Ok, load.Reason);
|
|
|
|
var sw = Stopwatch.StartNew();
|
|
Task<ScriptRunOutcome> run;
|
|
using (ProcessFreezer.Freeze(worker.Pid))
|
|
{
|
|
run = runner.RunAsync(Scripts.Vt("runaway", Runaway, softT), Ct);
|
|
await Task.Delay(stall, Ct);
|
|
}
|
|
var thawed = sw.Elapsed;
|
|
var outcome = await run.WaitAsync(TimeSpan.FromSeconds(30), Ct);
|
|
var answered = sw.Elapsed;
|
|
|
|
outcome.Status.ShouldBe(ScriptRunStatus.Failed);
|
|
outcome.Reason.ShouldNotBeNull().ShouldStartWith("script timed out after ");
|
|
(answered - thawed).ShouldBeGreaterThanOrEqualTo(softT - TimeSpan.FromMilliseconds(50), "T counts from the run's start, after the stall");
|
|
|
|
var exit = await worker.Exited.WaitAsync(TimeSpan.FromSeconds(30), Ct);
|
|
exit.Reason.ShouldBe(WorkerExitReason.KilledDeadline);
|
|
exit.CulpritKey.ShouldBe(InProcessScriptRunner.ScopedKey(ScriptContextKind.VirtualTag, "runaway"));
|
|
(sw.Elapsed - thawed).ShouldBeGreaterThanOrEqualTo(softT + grace - TimeSpan.FromMilliseconds(50), "G counts from the soft deadline");
|
|
}
|
|
|
|
/// <summary>
|
|
/// Shadow (lmxopcua#780 P2) compares the shadow pool with the published in-process result; a starved worker used to
|
|
/// show up there as a script mismatch (published Ok, shadow "timed out") and skew the census for the OutOfProcess
|
|
/// flip. Now a stall shorter than the transport deadline is a match, and a longer one is <c>shadow_unavailable</c>
|
|
/// — never a mismatch — including the re-trigger that finds the stalled run still holding its slot.
|
|
/// </summary>
|
|
[Fact]
|
|
public async Task In_Shadow_a_stalled_worker_is_a_match_or_shadow_unavailable_never_a_mismatch()
|
|
{
|
|
var softT = TimeSpan.FromMilliseconds(300);
|
|
var grace = TimeSpan.FromSeconds(1);
|
|
var options = Scripts.Options(hardGrace: grace, quarantineAfterKills: 1);
|
|
options.Runner = ScriptRunnerMode.Shadow;
|
|
await using var pool = Runner(grace);
|
|
using var shadow = new ShadowScriptRunner(pool, options, new LoggerConfiguration().CreateLogger());
|
|
var worker = await WarmAsync(pool, "trivial");
|
|
var published = ScriptRunOutcome.Ok(7);
|
|
|
|
// Stall past T, short of T + G: compared, and a match.
|
|
using (ProcessFreezer.Freeze(worker.Pid))
|
|
{
|
|
shadow.Mirror(Scripts.Vt("trivial", Trivial, softT), published);
|
|
await Task.Delay(TimeSpan.FromMilliseconds(900), Ct);
|
|
}
|
|
(await shadow.WaitIdleAsync(TimeSpan.FromSeconds(30))).ShouldBeTrue();
|
|
shadow.Stats().Matched.ShouldBe(1, Summary(shadow));
|
|
|
|
// Stall past T + G: not compared. The second mirror arrives while the first still holds the key's slot.
|
|
using (ProcessFreezer.Freeze(worker.Pid))
|
|
{
|
|
shadow.Mirror(Scripts.Vt("trivial", Trivial, softT), published);
|
|
await Scripts.Eventually(() => shadow.Stats().ShadowUnavailable == 1, TimeSpan.FromSeconds(10), "the transport deadline");
|
|
shadow.Mirror(Scripts.Vt("trivial", Trivial, softT), published);
|
|
await Scripts.Eventually(() => shadow.Stats().ShadowUnavailable == 2, TimeSpan.FromSeconds(10), "the stalled re-trigger");
|
|
}
|
|
(await shadow.WaitIdleAsync(TimeSpan.FromSeconds(30))).ShouldBeTrue();
|
|
|
|
var stats = shadow.Stats();
|
|
stats.Mismatched.ShouldBe(0, Summary(shadow));
|
|
stats.ShadowUnavailable.ShouldBe(2, Summary(shadow));
|
|
stats.Matched.ShouldBe(1);
|
|
pool.Shards[0].Current.ShouldNotBeNull().Pid.ShouldBe(worker.Pid);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Protocol compatibility: the soft clock is negotiated (<c>Hello.worker_soft_clock</c> → <c>Ready.worker_soft_clock</c>).
|
|
/// A host that does not ask for it keeps the pre-#827 contract — the soft deadline is the host's, from submit,
|
|
/// and the worker sends no soft notices — and a runaway is still answered at T and killed at T + G.
|
|
/// </summary>
|
|
[Theory]
|
|
[InlineData(true)]
|
|
[InlineData(false)]
|
|
public async Task The_soft_clock_is_negotiated_and_both_clocks_enforce_T_and_G(bool workerSoftClock)
|
|
{
|
|
var spec = new WorkerLaunchSpec("t", OutOfProcessScriptRunner.ResolveWorkerPath(new ScriptWorkerOptions { ExecutablePath = Scripts.WorkerUnderTest }),
|
|
256L * 1024 * 1024, EvalThreads: 2, HardGrace: TimeSpan.FromSeconds(1), HeartbeatInterval: TimeSpan.FromSeconds(1),
|
|
HandshakeTimeout: TimeSpan.FromSeconds(15), WorkerSoftClock: workerSoftClock);
|
|
var connection = await WorkerConnection.StartAsync(spec, new LoggerConfiguration().CreateLogger(), _ => { }, _ => { }, Ct);
|
|
await using var _ = connection;
|
|
connection.WorkerSoftClock.ShouldBe(workerSoftClock);
|
|
|
|
var entry = await connection.EnsureLoadedAsync("k", ContextKind.VirtualTag, Runaway,
|
|
SHA256.HashData(Encoding.UTF8.GetBytes(Runaway)), []).WaitAsync(TimeSpan.FromSeconds(60), Ct);
|
|
entry.Outcome.ShouldNotBeNull().Ok.ShouldBeTrue();
|
|
|
|
var softT = TimeSpan.FromMilliseconds(300);
|
|
var grace = TimeSpan.FromMilliseconds(500);
|
|
var sw = Stopwatch.StartNew();
|
|
var pending = connection.Submit("k",
|
|
new EvalRequest { Key = "k", SoftTimeoutMs = (uint)softT.TotalMilliseconds, HardGraceMs = (uint)grace.TotalMilliseconds, NowUtcTicks = DateTime.UtcNow.Ticks },
|
|
softT, grace, null);
|
|
(await pending.Outcome.WaitAsync(TimeSpan.FromSeconds(10), Ct)).ShouldBeSameAs(PendingEval.SoftTimedOut);
|
|
sw.Elapsed.ShouldBeGreaterThanOrEqualTo(softT - TimeSpan.FromMilliseconds(20));
|
|
connection.Cancel(pending);
|
|
|
|
var exit = await connection.Exited.WaitAsync(TimeSpan.FromSeconds(10), Ct);
|
|
exit.Reason.ShouldBe(WorkerExitReason.KilledDeadline);
|
|
sw.Elapsed.ShouldBeGreaterThanOrEqualTo(softT + grace - TimeSpan.FromMilliseconds(50));
|
|
}
|
|
|
|
// ---------------------------------------------------------------- review findings (#827)
|
|
|
|
/// <summary>
|
|
/// Review F1: the soft notice is handled on the pipe READER, and the caller's resumption is the caller's code. It
|
|
/// must go through the result dispatcher like a result does: a continuation that blocks for 1.5 s must not hold up
|
|
/// the reader, or the next heartbeats sit unread and a healthy worker is killed for silence (300 ms here). Before
|
|
/// the fix the continuation ran inline on the reader and the worker was killed (KilledHeartbeat).
|
|
/// </summary>
|
|
[Fact]
|
|
public async Task A_slow_continuation_of_a_soft_timeout_does_not_stall_the_reader()
|
|
{
|
|
var heartbeat = TimeSpan.FromMilliseconds(100);
|
|
var spec = new WorkerLaunchSpec("t", OutOfProcessScriptRunner.ResolveWorkerPath(new ScriptWorkerOptions { ExecutablePath = Scripts.WorkerUnderTest }),
|
|
256L * 1024 * 1024, EvalThreads: 2, HardGrace: TimeSpan.FromSeconds(60), HeartbeatInterval: heartbeat,
|
|
HandshakeTimeout: TimeSpan.FromSeconds(15), HeartbeatSilenceLimit: TimeSpan.FromSeconds(30));
|
|
var connection = await WorkerConnection.StartAsync(spec, new LoggerConfiguration().CreateLogger(), _ => { }, _ => { }, Ct);
|
|
await using var _ = connection;
|
|
var entry = await connection.EnsureLoadedAsync("k", ContextKind.VirtualTag, Runaway,
|
|
SHA256.HashData(Encoding.UTF8.GetBytes(Runaway)), []).WaitAsync(TimeSpan.FromSeconds(60), Ct);
|
|
entry.Outcome.ShouldNotBeNull().Ok.ShouldBeTrue();
|
|
connection.SilenceLimit = heartbeat * 3; // armed only once the cold start is over (see SupervisionTests)
|
|
|
|
var softT = TimeSpan.FromMilliseconds(200);
|
|
var pending = connection.Submit("k",
|
|
new EvalRequest { Key = "k", SoftTimeoutMs = (uint)softT.TotalMilliseconds, HardGraceMs = 60_000, NowUtcTicks = DateTime.UtcNow.Ticks },
|
|
softT, TimeSpan.FromSeconds(60), null);
|
|
string? resumedOn = null;
|
|
var slowConsumer = pending.Outcome.ContinueWith(t =>
|
|
{
|
|
resumedOn = Thread.CurrentThread.Name;
|
|
t.Result.ShouldBeSameAs(PendingEval.SoftTimedOut);
|
|
Thread.Sleep(1_500);
|
|
}, CancellationToken.None, TaskContinuationOptions.ExecuteSynchronously, TaskScheduler.Default);
|
|
|
|
await slowConsumer.WaitAsync(TimeSpan.FromSeconds(10), Ct);
|
|
resumedOn.ShouldBe("ScriptWorker[t].Results", "the caller resumes on the dispatcher, never on the reader");
|
|
connection.Exited.IsCompleted.ShouldBeFalse("the reader kept reading heartbeats while the consumer was slow");
|
|
connection.IsAlive.ShouldBeTrue();
|
|
}
|
|
|
|
/// <summary>
|
|
/// Review F2: a stall past T + G answers "worker unresponsive"; when the worker then runs the script and it really
|
|
/// overruns T, the late soft notice must still cancel its token — a polling script ends itself instead of the whole
|
|
/// worker being killed G later — and the run counts as a timeout (QuarantineAfterConsecutiveTimeouts = 1 here).
|
|
/// Before the fix nobody sent the Cancel: the worker was killed at notice + G (KilledDeadline, a kill strike).
|
|
/// </summary>
|
|
[Fact]
|
|
public async Task A_slow_script_whose_notice_arrives_after_the_transport_deadline_is_cancelled_and_counted_not_killed()
|
|
{
|
|
var softT = TimeSpan.FromMilliseconds(200);
|
|
var grace = TimeSpan.FromMilliseconds(400);
|
|
var options = Scripts.Options(hardGrace: grace, quarantineAfterKills: 1);
|
|
options.QuarantineAfterConsecutiveTimeouts = 1;
|
|
await using var runner = new OutOfProcessScriptRunner(options, new LoggerConfiguration().CreateLogger(), null,
|
|
heartbeatInterval: TimeSpan.FromSeconds(2), loadTimeout: TimeSpan.FromSeconds(60));
|
|
const string polite = "if ((int)ctx.GetTag(\"spin\").Value == 1) { while (!ctx.CancellationToken.IsCancellationRequested) { } } return 1;";
|
|
(await runner.RunAsync(Scripts.Vt("polite", polite, TimeSpan.FromSeconds(30), Scripts.Input("spin", 0)), Ct)).Value.ShouldBe(1);
|
|
var worker = runner.Shards[0].Current.ShouldNotBeNull();
|
|
|
|
ScriptRunOutcome stalled;
|
|
using (ProcessFreezer.Freeze(worker.Pid))
|
|
{
|
|
stalled = await runner.RunAsync(Scripts.Vt("polite", polite, softT, Scripts.Input("spin", 1)), Ct).WaitAsync(TimeSpan.FromSeconds(10), Ct);
|
|
}
|
|
stalled.Reason.ShouldBe(ScriptRunOutcome.WorkerUnresponsiveReason);
|
|
|
|
// Thawed: the script starts, spins past T, the late notice cancels it and it returns. The worker survives G.
|
|
await stalled.Recovery.ShouldNotBeNull().WaitAsync(TimeSpan.FromSeconds(10), Ct);
|
|
await Task.Delay(softT + grace + TimeSpan.FromMilliseconds(500), Ct);
|
|
worker.IsAlive.ShouldBeTrue("a cancelled polling script must not cost its worker");
|
|
runner.Shards[0].Current.ShouldNotBeNull().Pid.ShouldBe(worker.Pid);
|
|
|
|
// …and it counted as the timeout it was (threshold 1), not as a kill.
|
|
await Scripts.Eventually(() => runner.QuarantinedKeys(ScriptContextKind.VirtualTag).Count == 1, TimeSpan.FromSeconds(5), "the timeout quarantine");
|
|
var refused = await runner.RunAsync(Scripts.Vt("polite", polite, softT, Scripts.Input("spin", 0)), Ct);
|
|
refused.Reason.ShouldNotBeNull().ShouldContain("timed out");
|
|
refused.Reason.ShouldNotContain("hard deadline");
|
|
}
|
|
|
|
/// <summary>
|
|
/// Review F3: the host keeps its own bound. A worker that heartbeats but never sends a soft notice (a bug, or a
|
|
/// compromised worker — simulated by the host dropping its notices) used to hold the run, the key's slot and the
|
|
/// tag's BadResourceUnavailable forever: the transport deadline answers the caller once and the kill G is armed
|
|
/// only by a notice. Now no progress for the ceiling, with evaluations outstanding, kills the worker
|
|
/// (<see cref="WorkerExitReason.KilledUnresponsive"/>, no culprit) and the run completes. G is 60 s here, so the
|
|
/// worker's own fail-fast watchdog (notice + G + 0.5 s) cannot be what ends it.
|
|
/// </summary>
|
|
[Fact]
|
|
public async Task A_worker_that_heartbeats_but_never_reports_progress_is_killed_at_the_host_ceiling()
|
|
{
|
|
var spec = new WorkerLaunchSpec("t", OutOfProcessScriptRunner.ResolveWorkerPath(new ScriptWorkerOptions { ExecutablePath = Scripts.WorkerUnderTest }),
|
|
256L * 1024 * 1024, EvalThreads: 2, HardGrace: TimeSpan.FromSeconds(60), HeartbeatInterval: TimeSpan.FromMilliseconds(200),
|
|
HandshakeTimeout: TimeSpan.FromSeconds(15), HeartbeatSilenceLimit: TimeSpan.FromSeconds(30));
|
|
var connection = await WorkerConnection.StartAsync(spec, new LoggerConfiguration().CreateLogger(), _ => { }, _ => { }, Ct);
|
|
await using var _ = connection;
|
|
var entry = await connection.EnsureLoadedAsync("k", ContextKind.VirtualTag, Runaway,
|
|
SHA256.HashData(Encoding.UTF8.GetBytes(Runaway)), []).WaitAsync(TimeSpan.FromSeconds(60), Ct);
|
|
entry.Outcome.ShouldNotBeNull().Ok.ShouldBeTrue();
|
|
connection.DropSoftNoticesForTest = true;
|
|
connection.ProgressCeilingOverride = TimeSpan.FromSeconds(1.5);
|
|
|
|
var softT = TimeSpan.FromMilliseconds(200);
|
|
var sw = Stopwatch.StartNew();
|
|
var pending = connection.Submit("k",
|
|
new EvalRequest { Key = "k", SoftTimeoutMs = (uint)softT.TotalMilliseconds, HardGraceMs = 60_000, NowUtcTicks = DateTime.UtcNow.Ticks },
|
|
softT, TimeSpan.FromSeconds(60), null);
|
|
|
|
var exit = await connection.Exited.WaitAsync(TimeSpan.FromSeconds(15), Ct);
|
|
exit.Reason.ShouldBe(WorkerExitReason.KilledUnresponsive);
|
|
exit.CulpritKey.ShouldBeNull("nothing says which script, so nothing is struck");
|
|
exit.InFlightKeys.ShouldBe(["k"]);
|
|
sw.Elapsed.ShouldBeGreaterThanOrEqualTo(TimeSpan.FromSeconds(1.4), "not before the ceiling");
|
|
(await pending.Completion.WaitAsync(TimeSpan.FromSeconds(5), Ct)).ShouldBeNull("the run is released");
|
|
}
|
|
|
|
/// <summary>Positive control for the ceiling: the default rule leaves a busy, answering worker alone.</summary>
|
|
[Fact]
|
|
public async Task The_progress_ceiling_leaves_a_worker_that_keeps_answering_alone()
|
|
{
|
|
var spec = new WorkerLaunchSpec("t", OutOfProcessScriptRunner.ResolveWorkerPath(new ScriptWorkerOptions { ExecutablePath = Scripts.WorkerUnderTest }),
|
|
256L * 1024 * 1024, EvalThreads: 1, HardGrace: TimeSpan.FromSeconds(60), HeartbeatInterval: TimeSpan.FromMilliseconds(100),
|
|
HandshakeTimeout: TimeSpan.FromSeconds(15), HeartbeatSilenceLimit: TimeSpan.FromSeconds(30));
|
|
var connection = await WorkerConnection.StartAsync(spec, new LoggerConfiguration().CreateLogger(), _ => { }, _ => { }, Ct);
|
|
await using var _ = connection;
|
|
const string spin = "var end = ctx.Now.AddMilliseconds(150); while (ctx.Now < end) { } return 1;";
|
|
var entry = await connection.EnsureLoadedAsync("k", ContextKind.VirtualTag, spin,
|
|
SHA256.HashData(Encoding.UTF8.GetBytes(spin)), []).WaitAsync(TimeSpan.FromSeconds(60), Ct);
|
|
entry.Outcome.ShouldNotBeNull().Ok.ShouldBeTrue();
|
|
// Ceiling 1 s; 20 runs of 150 ms queued behind one evaluation thread take ~3 s, but each answers.
|
|
connection.ProgressCeilingOverride = TimeSpan.FromSeconds(1);
|
|
var soft = TimeSpan.FromSeconds(30);
|
|
var pendings = connection.SubmitBatch(Enumerable.Range(0, 20).Select(_ => new BatchSubmission("k",
|
|
new EvalRequest { Key = "k", SoftTimeoutMs = 30_000, HardGraceMs = 60_000, NowUtcTicks = DateTime.UtcNow.Ticks },
|
|
soft, TimeSpan.FromSeconds(60), null)).ToArray());
|
|
foreach (var p in pendings) (await p.Completion.WaitAsync(TimeSpan.FromSeconds(30), Ct)).ShouldNotBeNull();
|
|
connection.IsAlive.ShouldBeTrue();
|
|
}
|
|
|
|
// ---------------------------------------------------------------- PendingEval, no process
|
|
|
|
/// <summary>The host side of the worker clock, deterministic: no notice means the transport deadline at T + G.</summary>
|
|
[Fact]
|
|
public async Task Worker_clock_with_no_word_from_the_worker_answers_transport_timed_out_at_T_plus_G_and_never_kills()
|
|
{
|
|
var killed = 0;
|
|
var pending = new PendingEval("k", 1);
|
|
var sw = Stopwatch.StartNew();
|
|
pending.Arm(TimeSpan.FromMilliseconds(100), TimeSpan.FromMilliseconds(200), _ => Interlocked.Increment(ref killed), workerClock: true);
|
|
|
|
(await pending.Outcome.WaitAsync(TimeSpan.FromSeconds(10), Ct)).ShouldBeSameAs(PendingEval.TransportTimedOut);
|
|
sw.Elapsed.ShouldBeGreaterThanOrEqualTo(TimeSpan.FromMilliseconds(280));
|
|
await Task.Delay(500, Ct);
|
|
killed.ShouldBe(0, "a late worker is not the script's fault");
|
|
|
|
// A soft notice after the transport answer still arms the kill: the script itself ran for T.
|
|
pending.SoftExpired();
|
|
await Scripts.Eventually(() => Volatile.Read(ref killed) == 1, TimeSpan.FromSeconds(10), "the kill G after the soft notice");
|
|
}
|
|
|
|
/// <summary>The soft notice answers the caller and arms the kill G later; a result before then cancels it.</summary>
|
|
[Fact]
|
|
public async Task Worker_clock_soft_notice_answers_soft_timed_out_and_arms_the_kill_G_later()
|
|
{
|
|
var killed = 0;
|
|
var pending = new PendingEval("k", 1);
|
|
pending.Arm(TimeSpan.FromSeconds(30), TimeSpan.FromMilliseconds(300), _ => Interlocked.Increment(ref killed), workerClock: true);
|
|
pending.Outcome.IsCompleted.ShouldBeFalse();
|
|
|
|
var sw = Stopwatch.StartNew();
|
|
pending.SoftExpired();
|
|
(await pending.Outcome.WaitAsync(TimeSpan.FromSeconds(1), Ct)).ShouldBeSameAs(PendingEval.SoftTimedOut);
|
|
await Scripts.Eventually(() => Volatile.Read(ref killed) == 1, TimeSpan.FromSeconds(10), "the kill");
|
|
sw.Elapsed.ShouldBeGreaterThanOrEqualTo(TimeSpan.FromMilliseconds(280));
|
|
|
|
var answered = new PendingEval("k2", 2);
|
|
answered.Arm(TimeSpan.FromSeconds(30), TimeSpan.FromMilliseconds(100), _ => Interlocked.Increment(ref killed), workerClock: true);
|
|
answered.SoftExpired();
|
|
answered.Complete(new EvalResult { EvalId = 2 });
|
|
await Task.Delay(400, Ct);
|
|
killed.ShouldBe(1, "a run that answered inside G is not killed");
|
|
}
|
|
|
|
/// <summary>Under the host clock a soft notice means nothing (a worker that did not negotiate it cannot send one).</summary>
|
|
[Fact]
|
|
public void Host_clock_ignores_a_soft_notice()
|
|
{
|
|
var pending = new PendingEval("k", 1);
|
|
pending.Arm(TimeSpan.FromSeconds(30), TimeSpan.FromSeconds(30), _ => { }, workerClock: false);
|
|
pending.SoftExpired();
|
|
pending.Outcome.IsCompleted.ShouldBeFalse();
|
|
pending.Complete(null);
|
|
}
|
|
|
|
private static string Summary(ShadowScriptRunner shadow)
|
|
=> $"{shadow.Stats()}; " + string.Join("; ", shadow.Mismatches.Select(m => $"{m.Key}: {m.LastReason} — {m.LastSummary}"));
|
|
}
|
|
|
|
/// <summary>Suspends a process (SIGSTOP, or NtSuspendProcess on Windows) until disposed.</summary>
|
|
internal sealed class ProcessFreezer : IDisposable
|
|
{
|
|
private readonly int _pid;
|
|
private int _thawed;
|
|
|
|
private ProcessFreezer(int pid)
|
|
{
|
|
_pid = pid;
|
|
Signal(pid, stop: true);
|
|
}
|
|
|
|
public static ProcessFreezer Freeze(int pid) => new(pid);
|
|
|
|
public void Dispose()
|
|
{
|
|
if (Interlocked.Exchange(ref _thawed, 1) == 0) Signal(_pid, stop: false);
|
|
}
|
|
|
|
private static void Signal(int pid, bool stop)
|
|
{
|
|
if (OperatingSystem.IsWindows())
|
|
{
|
|
using var process = Process.GetProcessById(pid);
|
|
var status = stop ? NtSuspendProcess(process.Handle) : NtResumeProcess(process.Handle);
|
|
status.ShouldBe(0, $"Nt{(stop ? "Suspend" : "Resume")}Process");
|
|
return;
|
|
}
|
|
// libc kill(2), not the kill(1) command: a child process's exit is observed through the same SIGCHLD
|
|
// handling that the suspended worker (another child of this process) just disturbed, and WaitForExit was
|
|
// seen to take seconds.
|
|
var signal = stop ? SigStop : SigCont;
|
|
Kill(pid, signal).ShouldBe(0, $"kill({pid}, {signal}): errno {Marshal.GetLastPInvokeError()}");
|
|
}
|
|
|
|
/// <summary>SIGSTOP / SIGCONT: 17 / 19 on macOS (BSD), 19 / 18 on Linux.</summary>
|
|
private static int SigStop => OperatingSystem.IsMacOS() ? 17 : 19;
|
|
|
|
private static int SigCont => OperatingSystem.IsMacOS() ? 19 : 18;
|
|
|
|
#pragma warning disable SYSLIB1054 // DllImport kept: LibraryImport would need AllowUnsafeBlocks for three test-only calls.
|
|
[DllImport("libc", EntryPoint = "kill", SetLastError = true)]
|
|
private static extern int Kill(int pid, int signal);
|
|
|
|
[DllImport("ntdll.dll")]
|
|
private static extern int NtSuspendProcess(IntPtr processHandle);
|
|
|
|
[DllImport("ntdll.dll")]
|
|
private static extern int NtResumeProcess(IntPtr processHandle);
|
|
#pragma warning restore SYSLIB1054
|
|
}
|