Merge #843 #840 #841 load flakes, ScriptRunGuard slot-release race, DurationGood conformance, CI QuestDB pin
v2-ci / unit-tests (push) Failing after 1h12m19s

This commit is contained in:
Joseph Doherty
2026-11-11 05:10:00 -05:00
8 changed files with 272 additions and 51 deletions
@@ -85,7 +85,7 @@ public sealed class EquipmentNamespaceMaterializationTests
return true;
}
return false;
}, TwoNodeClusterHarness.DeploySealTimeout);
}, TwoNodeClusterHarness.DeploySealTimeout, () => DescribeDeploymentAsync(harness, deploymentId));
var composition = DeploymentArtifact.ParseComposition(artifact);
@@ -182,7 +182,7 @@ public sealed class EquipmentNamespaceMaterializationTests
return true;
}
return false;
}, TwoNodeClusterHarness.DeploySealTimeout);
}, TwoNodeClusterHarness.DeploySealTimeout, () => DescribeDeploymentAsync(harness, deploymentId));
await using (var db = await CreateDbAsync(harness))
{
@@ -261,7 +261,8 @@ public sealed class EquipmentNamespaceMaterializationTests
return await factory.CreateDbContextAsync();
}
private static async Task WaitForAsync(Func<Task<bool>> condition, TimeSpan timeout)
private static async Task WaitForAsync(Func<Task<bool>> condition, TimeSpan timeout,
Func<Task<string>>? diagnose = null)
{
var deadline = DateTime.UtcNow + timeout;
while (DateTime.UtcNow < deadline)
@@ -269,6 +270,29 @@ public sealed class EquipmentNamespaceMaterializationTests
if (await condition()) return;
await Task.Delay(200);
}
throw new TimeoutException($"Condition not met within {timeout}");
throw new TimeoutException($"Condition not met within {timeout}"
+ (diagnose is null ? "" : $"; state at timeout: {await diagnose()}"));
}
/// <summary>
/// lmxopcua#843: the seal wait used to time out with a bare "Condition not met within 45s" (seen once at load
/// average 140–280, 48 s against a normal 3 s). Report where the deploy stood — the deployment row, each node's
/// per-node state, and both nodes' cluster views — so the next failure says whether it was a slow apply, a node
/// that never received the dispatch (no ack, still Applying), or a failed apply.
/// </summary>
private static async Task<string> DescribeDeploymentAsync(TwoNodeClusterHarness harness, Guid deploymentId)
{
await using var db = await CreateDbAsync(harness);
var d = await db.Deployments.AsNoTracking().FirstOrDefaultAsync(x => x.DeploymentId == deploymentId);
var nodes = await db.NodeDeploymentStates.AsNoTracking()
.Where(s => s.DeploymentId == deploymentId).ToListAsync();
static string View(Akka.Actor.ActorSystem s) => string.Join(",",
Akka.Cluster.Cluster.Get(s).State.Members.Select(m => $"{m.Address.Port}:{m.Status}"))
+ $" unreachable={Akka.Cluster.Cluster.Get(s).State.Unreachable.Count}";
return $"deployment={d?.Status.ToString() ?? "<missing>"} artifact={d?.ArtifactBlob.Length ?? 0}B "
+ $"failure='{d?.FailureReason}' created={d?.CreatedAtUtc:O} now={DateTime.UtcNow:O}; "
+ $"nodes=[{string.Join("; ", nodes.Select(n => $"{n.NodeId} {n.Status} started={n.StartedAtUtc:O} applied={n.AppliedAtUtc:O} failure='{n.FailureReason}'"))}] "
+ $"(expected {harness.NodeANodeId}, {harness.NodeBNodeId}); "
+ $"A view=[{View(harness.NodeASystem)}] B view=[{View(harness.NodeBSystem)}]";
}
}
@@ -466,6 +466,19 @@ public sealed class RoslynVirtualTagEvaluatorTests
/// for the same reason: a handful of queued-too-long evaluations is the environment, but 10% of them
/// missing a 5 s budget means the evaluator stopped evaluating. If a loaded box ever does trip EITHER
/// bound, the rate itself is the finding: record it before widening the number.</para></summary>
/// <remarks>
/// <b>lmxopcua#843 — this test found a product bug, not a timing assumption.</b> Under the full Host
/// integration suite plus 28 CPU burners (load ~300) it failed 1 run in 3 with 24 of 4,800 evaluations
/// refused <c>"previous evaluation still running"</c> — impossible for one sequential loop per id unless the
/// id's previous run still held its <see cref="ScriptRunGuard"/> slot after its caller had resumed. It did:
/// the slot was released by a continuation on the run task, and a task is observably complete before its
/// continuations run (worst for the evaluator's immediate retry after a raced
/// <see cref="ObjectDisposedException"/>). The release now happens inside the run, before it completes
/// (<c>ScriptRunGuardTests.A_caller_that_awaited_its_run_is_never_refused_by_it</c> pins the ordering). A
/// refusal stays in the "leaked" bucket on purpose. The doc above predates P5: the guard's budget clock
/// starts when the pool starts the run, so a queued evaluation no longer lands in the timed-out bucket.
/// Every assertion now prints the run's shape (elapsed, evaluations, clears, pool state, failure histogram).
/// </remarks>
[Fact]
public async Task Evaluate_racing_ClearCompiledScripts_never_leaks_a_raw_disposed_failure()
{
@@ -511,19 +524,37 @@ public sealed class RoslynVirtualTagEvaluatorTests
Interlocked.Decrement(ref remaining);
}, TestContext.Current.CancellationToken)).ToArray();
var clears = 0L;
var clearLoop = Task.Run(() =>
{
while (Volatile.Read(ref remaining) > 0)
{
sut.ClearCompiledScripts();
Interlocked.Increment(ref clears);
Thread.SpinWait(4000); // ~tens of µs between clears — frequent, but not back-to-back
}
}, TestContext.Current.CancellationToken);
// lmxopcua#843: every assertion below carries the run's shape, so a failure under load says which bucket
// moved and how starved the box was (it passed 5/5 on rerun with nothing recorded).
var elapsed = System.Diagnostics.Stopwatch.StartNew();
string Diagnostics() =>
$" [elapsed {elapsed.ElapsedMilliseconds} ms, {Volatile.Read(ref evaluated)} evaluations, " +
$"{Interlocked.Read(ref clears)} clears, loops still running {Volatile.Read(ref remaining)}, " +
$"pool threads {ThreadPool.ThreadCount} / pending {ThreadPool.PendingWorkItemCount}; failures: " +
string.Join(", ", failures.GroupBy(r => r.Length > 80 ? r[..80] : r).Select(g => $"{g.Count()}× \"{g.Key}\"")) + "]";
// Hang detector only: the loops self-terminate at RaceWindow, so firing this means something
// wedged, not that the box was slow.
await Task.WhenAll(evalLoops.Append(clearLoop))
.WaitAsync(TimeSpan.FromSeconds(45), TestContext.Current.CancellationToken);
try
{
await Task.WhenAll(evalLoops.Append(clearLoop))
.WaitAsync(TimeSpan.FromSeconds(45), TestContext.Current.CancellationToken);
}
catch (TimeoutException ex)
{
throw new TimeoutException("the racing loops did not finish within 45 s" + Diagnostics(), ex);
}
var totalEvaluations = Volatile.Read(ref evaluated);
// Loose by design, with an absolute floor so a starved run (few evaluations landed) cannot trip on
@@ -544,20 +575,20 @@ public sealed class RoslynVirtualTagEvaluatorTests
leaked.ShouldBeEmpty(
"a racing ClearCompiledScripts must never surface a failure other than " +
$"\"{ScriptEvalFailureReasons.EvaluatorDisposedDuringEvaluation}\" or " +
$"\"{ScriptEvalFailureReasons.TimedOutPrefix}…\"; observed: {string.Join(" | ", leaked)}");
$"\"{ScriptEvalFailureReasons.TimedOutPrefix}…\"; observed: {string.Join(" | ", leaked)}" + Diagnostics());
// (2) Non-vacuity: the retry must actually be absorbing the race, not merely relabelling it.
exhausted.ShouldBeLessThan(
rateBound,
"the one-shot retry must absorb the overwhelming majority of raced clears; " +
$"{exhausted}/{totalEvaluations} evaluations exhausted it");
$"{exhausted}/{totalEvaluations} evaluations exhausted it" + Diagnostics());
// (3) The starvation bucket is tolerated but not unbounded — at 10% the evaluator has stopped
// evaluating rather than the box being busy.
starved.ShouldBeLessThan(
rateBound,
"queue-starvation timeouts are expected under this stress but must stay marginal; " +
$"{starved}/{totalEvaluations} evaluations missed the 5s budget");
$"{starved}/{totalEvaluations} evaluations missed the 5s budget" + Diagnostics());
}
// ── input quality reaches the script (archreview 02/S14) ────────────────────────────────
@@ -125,9 +125,14 @@ public sealed class SupervisionTests
/// A burst of cold compiles must not starve the worker's heartbeat. Found on windev by the §8 perf harness: 500
/// loads at once, each compiled synchronously on its own thread-pool work item, queued the heartbeat timer and
/// the pipe writer behind them; the host saw no frame for 3 s, killed the shard (KilledHeartbeat) and made all 500
/// keys suspects. Here a 100 ms heartbeat (kill after 300 ms of silence) and 400 distinct cold scripts at once:
/// every script answers and the shard's worker is the one that started. Mutation check: compile on the thread
/// pool again and the worker is killed for silence.
/// keys suspects. Here a 50 ms heartbeat (kill after 3 s of silence, production's limit) and 1,500 distinct cold scripts at once:
/// every script answers and the shard's worker is the one that started.
/// <para>⚠️ The old "mutation check: compile on the thread pool again and the worker is killed for silence" does
/// NOT hold any more (re-checked for lmxopcua#843 on a 14-core Mac: the mutated worker passed at both the 150 ms
/// and the 1 s limit). The host's silence clock counts ANY frame, and since lmxopcua#789 a compile writes its own
/// <c>LoadStarted</c>/<c>LoadResult</c> frames inline — so pool-queued compiles keep the pipe busy themselves (the 3 s silence
/// windev saw presumably also needed the pre-#789 pool-scheduled writer stuck behind them). What this still pins is the
/// behaviour: a cold burst under a tight heartbeat is answered without a kill.</para>
/// </summary>
[Fact]
public async Task A_burst_of_cold_compiles_does_not_starve_the_heartbeat()
@@ -136,13 +141,29 @@ public sealed class SupervisionTests
var heartbeat = TimeSpan.FromMilliseconds(50);
await using var runner = TightHeartbeatRunner(heartbeat, TimeSpan.FromSeconds(120), sink);
var worker = WarmTight(runner, await runner.RunAsync(Scripts.Vt("first", Healthy), Ct), heartbeat, sink);
// lmxopcua#843: three 50 ms intervals (150 ms) is a limit the OS scheduler alone breaks on an overloaded box — at
// load average 100–280 the worker or the host's reader waited 150–310 ms for a core (10/10 runs killed for silence
// under 28 CPU burners, every compile on its dedicated thread), and at ~300 even a 1 s limit was hit once in 10
// (1,012 ms). The limit is now PRODUCTION's (3 × the default 1 s heartbeat): the test kills only where a real node
// would, and the windev defect (no frame for 3 s, the whole burst) is still the size it catches. The 50 ms
// heartbeat keeps frames flowing for the silence clock.
worker.SilenceLimit = TimeSpan.FromSeconds(3);
const int count = 1500;
var burst = Stopwatch.StartNew();
var outcomes = await Task.WhenAll(Enumerable.Range(0, count).Select(i => runner.RunAsync(
Scripts.Vt("burst-" + i, $"return (int)ctx.GetTag(\"x\").Value * {i} + 1;", TimeSpan.FromSeconds(60),
Scripts.Input("x", 2)), Ct)));
burst.Stop();
var log = string.Join(Environment.NewLine, sink.Rendered.Where(l => l.Contains("ScriptWorker", StringComparison.Ordinal)).Take(5));
// The kill line ("sent nothing for N ms") against the burst's length is the diagnosis: N close to the burst is
// the pool-starvation defect, N just over the limit is a starved box.
var log = $"burst took {burst.ElapsedMilliseconds} ms, silence limit {worker.SilenceLimit.TotalMilliseconds} ms\n"
+ string.Join(Environment.NewLine, sink.Rendered
.Where(l => l.Contains("ScriptWorker", StringComparison.Ordinal))
.Where(l => l.Contains("sent nothing", StringComparison.Ordinal) || l.Contains("kill", StringComparison.OrdinalIgnoreCase)
|| l.Contains("exit", StringComparison.OrdinalIgnoreCase))
.Take(10));
for (var i = 0; i < count; i++) outcomes[i].Value.ShouldBe(2 * i + 1, $"{outcomes[i].Status} {outcomes[i].Reason}\n{log}");
runner.Shards[0].Current.ShouldNotBeNull().Pid.ShouldBe(worker.Pid, log);
worker.Exited.IsCompleted.ShouldBeFalse(log);