fix(scripting): host progress ceiling kills a worker that heartbeats but never answers (#827 review F3)
With the worker soft clock the host's per-run timer is only the transport deadline and the kill G is armed by the worker's soft notice, so a worker whose watchdog never sends one (the design treats the worker as untrusted) held the run, the key's slot and the tag's BadResourceUnavailable forever; a deferred run had no host clock at all. WorkerConnection now tracks evaluation progress (any Results frame, or the submit that made the worker busy again). The heartbeat monitor kills the worker when evaluations are outstanding and nothing progressed for 3 x the largest outstanding T + G plus the silence limit: WorkerExitReason.KilledUnresponsive (metric reason killed_unresponsive), no culprit, attributed like a heartbeat kill (suspects, no strike on a shard). A legitimate run has answered or been killed by T + G. Test seams: ProgressCeilingOverride, DropSoftNoticesForTest. Tests: a worker whose notices are dropped (G = 60 s, so its own fail-fast cannot end it) is killed KilledUnresponsive after the 1.5 s ceiling and the run is released (with the check disabled the test times out); positive control: 20 queued 150 ms runs behind one thread under a 1 s ceiling are all answered.
This commit is contained in:
@@ -306,6 +306,65 @@ public sealed class WorkerSoftClockTests
|
||||
refused.Reason.ShouldNotContain("hard deadline");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Review F3: the host keeps its own bound. A worker that heartbeats but never sends a soft notice (a bug, or a
|
||||
/// compromised worker — simulated by the host dropping its notices) used to hold the run, the key's slot and the
|
||||
/// tag's BadResourceUnavailable forever: the transport deadline answers the caller once and the kill G is armed
|
||||
/// only by a notice. Now no progress for the ceiling, with evaluations outstanding, kills the worker
|
||||
/// (<see cref="WorkerExitReason.KilledUnresponsive"/>, no culprit) and the run completes. G is 60 s here, so the
|
||||
/// worker's own fail-fast watchdog (notice + G + 0.5 s) cannot be what ends it.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task A_worker_that_heartbeats_but_never_reports_progress_is_killed_at_the_host_ceiling()
|
||||
{
|
||||
var spec = new WorkerLaunchSpec("t", OutOfProcessScriptRunner.ResolveWorkerPath(new ScriptWorkerOptions { ExecutablePath = Scripts.WorkerUnderTest }),
|
||||
256L * 1024 * 1024, EvalThreads: 2, HardGrace: TimeSpan.FromSeconds(60), HeartbeatInterval: TimeSpan.FromMilliseconds(200),
|
||||
HandshakeTimeout: TimeSpan.FromSeconds(15), HeartbeatSilenceLimit: TimeSpan.FromSeconds(30));
|
||||
var connection = await WorkerConnection.StartAsync(spec, new LoggerConfiguration().CreateLogger(), _ => { }, _ => { }, Ct);
|
||||
await using var _ = connection;
|
||||
var entry = await connection.EnsureLoadedAsync("k", ContextKind.VirtualTag, Runaway,
|
||||
SHA256.HashData(Encoding.UTF8.GetBytes(Runaway)), []).WaitAsync(TimeSpan.FromSeconds(60), Ct);
|
||||
entry.Outcome.ShouldNotBeNull().Ok.ShouldBeTrue();
|
||||
connection.DropSoftNoticesForTest = true;
|
||||
connection.ProgressCeilingOverride = TimeSpan.FromSeconds(1.5);
|
||||
|
||||
var softT = TimeSpan.FromMilliseconds(200);
|
||||
var sw = Stopwatch.StartNew();
|
||||
var pending = connection.Submit("k",
|
||||
new EvalRequest { Key = "k", SoftTimeoutMs = (uint)softT.TotalMilliseconds, HardGraceMs = 60_000, NowUtcTicks = DateTime.UtcNow.Ticks },
|
||||
softT, TimeSpan.FromSeconds(60), null);
|
||||
|
||||
var exit = await connection.Exited.WaitAsync(TimeSpan.FromSeconds(15), Ct);
|
||||
exit.Reason.ShouldBe(WorkerExitReason.KilledUnresponsive);
|
||||
exit.CulpritKey.ShouldBeNull("nothing says which script, so nothing is struck");
|
||||
exit.InFlightKeys.ShouldBe(["k"]);
|
||||
sw.Elapsed.ShouldBeGreaterThanOrEqualTo(TimeSpan.FromSeconds(1.4), "not before the ceiling");
|
||||
(await pending.Completion.WaitAsync(TimeSpan.FromSeconds(5), Ct)).ShouldBeNull("the run is released");
|
||||
}
|
||||
|
||||
/// <summary>Positive control for the ceiling: the default rule leaves a busy, answering worker alone.</summary>
|
||||
[Fact]
|
||||
public async Task The_progress_ceiling_leaves_a_worker_that_keeps_answering_alone()
|
||||
{
|
||||
var spec = new WorkerLaunchSpec("t", OutOfProcessScriptRunner.ResolveWorkerPath(new ScriptWorkerOptions { ExecutablePath = Scripts.WorkerUnderTest }),
|
||||
256L * 1024 * 1024, EvalThreads: 1, HardGrace: TimeSpan.FromSeconds(60), HeartbeatInterval: TimeSpan.FromMilliseconds(100),
|
||||
HandshakeTimeout: TimeSpan.FromSeconds(15), HeartbeatSilenceLimit: TimeSpan.FromSeconds(30));
|
||||
var connection = await WorkerConnection.StartAsync(spec, new LoggerConfiguration().CreateLogger(), _ => { }, _ => { }, Ct);
|
||||
await using var _ = connection;
|
||||
const string spin = "var end = ctx.Now.AddMilliseconds(150); while (ctx.Now < end) { } return 1;";
|
||||
var entry = await connection.EnsureLoadedAsync("k", ContextKind.VirtualTag, spin,
|
||||
SHA256.HashData(Encoding.UTF8.GetBytes(spin)), []).WaitAsync(TimeSpan.FromSeconds(60), Ct);
|
||||
entry.Outcome.ShouldNotBeNull().Ok.ShouldBeTrue();
|
||||
// Ceiling 1 s; 20 runs of 150 ms queued behind one evaluation thread take ~3 s, but each answers.
|
||||
connection.ProgressCeilingOverride = TimeSpan.FromSeconds(1);
|
||||
var soft = TimeSpan.FromSeconds(30);
|
||||
var pendings = connection.SubmitBatch(Enumerable.Range(0, 20).Select(_ => new BatchSubmission("k",
|
||||
new EvalRequest { Key = "k", SoftTimeoutMs = 30_000, HardGraceMs = 60_000, NowUtcTicks = DateTime.UtcNow.Ticks },
|
||||
soft, TimeSpan.FromSeconds(60), null)).ToArray());
|
||||
foreach (var p in pendings) (await p.Completion.WaitAsync(TimeSpan.FromSeconds(30), Ct)).ShouldNotBeNull();
|
||||
connection.IsAlive.ShouldBeTrue();
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- PendingEval, no process
|
||||
|
||||
/// <summary>The host side of the worker clock, deterministic: no notice means the transport deadline at T + G.</summary>
|
||||
|
||||
Reference in New Issue
Block a user