Merge #843 #840 #841 load flakes, ScriptRunGuard slot-release race, DurationGood conformance, CI QuestDB pin
v2-ci / unit-tests (push) Failing after 1h12m19s
v2-ci / unit-tests (push) Failing after 1h12m19s
This commit is contained in:
+28
-4
@@ -85,7 +85,7 @@ public sealed class EquipmentNamespaceMaterializationTests
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}, TwoNodeClusterHarness.DeploySealTimeout);
|
||||
}, TwoNodeClusterHarness.DeploySealTimeout, () => DescribeDeploymentAsync(harness, deploymentId));
|
||||
|
||||
var composition = DeploymentArtifact.ParseComposition(artifact);
|
||||
|
||||
@@ -182,7 +182,7 @@ public sealed class EquipmentNamespaceMaterializationTests
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}, TwoNodeClusterHarness.DeploySealTimeout);
|
||||
}, TwoNodeClusterHarness.DeploySealTimeout, () => DescribeDeploymentAsync(harness, deploymentId));
|
||||
|
||||
await using (var db = await CreateDbAsync(harness))
|
||||
{
|
||||
@@ -261,7 +261,8 @@ public sealed class EquipmentNamespaceMaterializationTests
|
||||
return await factory.CreateDbContextAsync();
|
||||
}
|
||||
|
||||
private static async Task WaitForAsync(Func<Task<bool>> condition, TimeSpan timeout)
|
||||
private static async Task WaitForAsync(Func<Task<bool>> condition, TimeSpan timeout,
|
||||
Func<Task<string>>? diagnose = null)
|
||||
{
|
||||
var deadline = DateTime.UtcNow + timeout;
|
||||
while (DateTime.UtcNow < deadline)
|
||||
@@ -269,6 +270,29 @@ public sealed class EquipmentNamespaceMaterializationTests
|
||||
if (await condition()) return;
|
||||
await Task.Delay(200);
|
||||
}
|
||||
throw new TimeoutException($"Condition not met within {timeout}");
|
||||
throw new TimeoutException($"Condition not met within {timeout}"
|
||||
+ (diagnose is null ? "" : $"; state at timeout: {await diagnose()}"));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// lmxopcua#843: the seal wait used to time out with a bare "Condition not met within 45s" (seen once at load
|
||||
/// average 140–280, 48 s against a normal 3 s). Report where the deploy stood — the deployment row, each node's
|
||||
/// per-node state, and both nodes' cluster views — so the next failure says whether it was a slow apply, a node
|
||||
/// that never received the dispatch (no ack, still Applying), or a failed apply.
|
||||
/// </summary>
|
||||
private static async Task<string> DescribeDeploymentAsync(TwoNodeClusterHarness harness, Guid deploymentId)
|
||||
{
|
||||
await using var db = await CreateDbAsync(harness);
|
||||
var d = await db.Deployments.AsNoTracking().FirstOrDefaultAsync(x => x.DeploymentId == deploymentId);
|
||||
var nodes = await db.NodeDeploymentStates.AsNoTracking()
|
||||
.Where(s => s.DeploymentId == deploymentId).ToListAsync();
|
||||
static string View(Akka.Actor.ActorSystem s) => string.Join(",",
|
||||
Akka.Cluster.Cluster.Get(s).State.Members.Select(m => $"{m.Address.Port}:{m.Status}"))
|
||||
+ $" unreachable={Akka.Cluster.Cluster.Get(s).State.Unreachable.Count}";
|
||||
return $"deployment={d?.Status.ToString() ?? "<missing>"} artifact={d?.ArtifactBlob.Length ?? 0}B "
|
||||
+ $"failure='{d?.FailureReason}' created={d?.CreatedAtUtc:O} now={DateTime.UtcNow:O}; "
|
||||
+ $"nodes=[{string.Join("; ", nodes.Select(n => $"{n.NodeId} {n.Status} started={n.StartedAtUtc:O} applied={n.AppliedAtUtc:O} failure='{n.FailureReason}'"))}] "
|
||||
+ $"(expected {harness.NodeANodeId}, {harness.NodeBNodeId}); "
|
||||
+ $"A view=[{View(harness.NodeASystem)}] B view=[{View(harness.NodeBSystem)}]";
|
||||
}
|
||||
}
|
||||
|
||||
+36
-5
@@ -466,6 +466,19 @@ public sealed class RoslynVirtualTagEvaluatorTests
|
||||
/// for the same reason: a handful of queued-too-long evaluations is the environment, but 10% of them
|
||||
/// missing a 5 s budget means the evaluator stopped evaluating. If a loaded box ever does trip EITHER
|
||||
/// bound, the rate itself is the finding: record it before widening the number.</para></summary>
|
||||
/// <remarks>
|
||||
/// <b>lmxopcua#843 — this test found a product bug, not a timing assumption.</b> Under the full Host
|
||||
/// integration suite plus 28 CPU burners (load ~300) it failed 1 run in 3 with 24 of 4,800 evaluations
|
||||
/// refused <c>"previous evaluation still running"</c> — impossible for one sequential loop per id unless the
|
||||
/// id's previous run still held its <see cref="ScriptRunGuard"/> slot after its caller had resumed. It did:
|
||||
/// the slot was released by a continuation on the run task, and a task is observably complete before its
|
||||
/// continuations run (worst for the evaluator's immediate retry after a raced
|
||||
/// <see cref="ObjectDisposedException"/>). The release now happens inside the run, before it completes
|
||||
/// (<c>ScriptRunGuardTests.A_caller_that_awaited_its_run_is_never_refused_by_it</c> pins the ordering). A
|
||||
/// refusal stays in the "leaked" bucket on purpose. The doc above predates P5: the guard's budget clock
|
||||
/// starts when the pool starts the run, so a queued evaluation no longer lands in the timed-out bucket.
|
||||
/// Every assertion now prints the run's shape (elapsed, evaluations, clears, pool state, failure histogram).
|
||||
/// </remarks>
|
||||
[Fact]
|
||||
public async Task Evaluate_racing_ClearCompiledScripts_never_leaks_a_raw_disposed_failure()
|
||||
{
|
||||
@@ -511,19 +524,37 @@ public sealed class RoslynVirtualTagEvaluatorTests
|
||||
Interlocked.Decrement(ref remaining);
|
||||
}, TestContext.Current.CancellationToken)).ToArray();
|
||||
|
||||
var clears = 0L;
|
||||
var clearLoop = Task.Run(() =>
|
||||
{
|
||||
while (Volatile.Read(ref remaining) > 0)
|
||||
{
|
||||
sut.ClearCompiledScripts();
|
||||
Interlocked.Increment(ref clears);
|
||||
Thread.SpinWait(4000); // ~tens of µs between clears — frequent, but not back-to-back
|
||||
}
|
||||
}, TestContext.Current.CancellationToken);
|
||||
|
||||
// lmxopcua#843: every assertion below carries the run's shape, so a failure under load says which bucket
|
||||
// moved and how starved the box was (it passed 5/5 on rerun with nothing recorded).
|
||||
var elapsed = System.Diagnostics.Stopwatch.StartNew();
|
||||
string Diagnostics() =>
|
||||
$" [elapsed {elapsed.ElapsedMilliseconds} ms, {Volatile.Read(ref evaluated)} evaluations, " +
|
||||
$"{Interlocked.Read(ref clears)} clears, loops still running {Volatile.Read(ref remaining)}, " +
|
||||
$"pool threads {ThreadPool.ThreadCount} / pending {ThreadPool.PendingWorkItemCount}; failures: " +
|
||||
string.Join(", ", failures.GroupBy(r => r.Length > 80 ? r[..80] : r).Select(g => $"{g.Count()}× \"{g.Key}\"")) + "]";
|
||||
|
||||
// Hang detector only: the loops self-terminate at RaceWindow, so firing this means something
|
||||
// wedged, not that the box was slow.
|
||||
await Task.WhenAll(evalLoops.Append(clearLoop))
|
||||
.WaitAsync(TimeSpan.FromSeconds(45), TestContext.Current.CancellationToken);
|
||||
try
|
||||
{
|
||||
await Task.WhenAll(evalLoops.Append(clearLoop))
|
||||
.WaitAsync(TimeSpan.FromSeconds(45), TestContext.Current.CancellationToken);
|
||||
}
|
||||
catch (TimeoutException ex)
|
||||
{
|
||||
throw new TimeoutException("the racing loops did not finish within 45 s" + Diagnostics(), ex);
|
||||
}
|
||||
|
||||
var totalEvaluations = Volatile.Read(ref evaluated);
|
||||
// Loose by design, with an absolute floor so a starved run (few evaluations landed) cannot trip on
|
||||
@@ -544,20 +575,20 @@ public sealed class RoslynVirtualTagEvaluatorTests
|
||||
leaked.ShouldBeEmpty(
|
||||
"a racing ClearCompiledScripts must never surface a failure other than " +
|
||||
$"\"{ScriptEvalFailureReasons.EvaluatorDisposedDuringEvaluation}\" or " +
|
||||
$"\"{ScriptEvalFailureReasons.TimedOutPrefix}…\"; observed: {string.Join(" | ", leaked)}");
|
||||
$"\"{ScriptEvalFailureReasons.TimedOutPrefix}…\"; observed: {string.Join(" | ", leaked)}" + Diagnostics());
|
||||
|
||||
// (2) Non-vacuity: the retry must actually be absorbing the race, not merely relabelling it.
|
||||
exhausted.ShouldBeLessThan(
|
||||
rateBound,
|
||||
"the one-shot retry must absorb the overwhelming majority of raced clears; " +
|
||||
$"{exhausted}/{totalEvaluations} evaluations exhausted it");
|
||||
$"{exhausted}/{totalEvaluations} evaluations exhausted it" + Diagnostics());
|
||||
|
||||
// (3) The starvation bucket is tolerated but not unbounded — at 10% the evaluator has stopped
|
||||
// evaluating rather than the box being busy.
|
||||
starved.ShouldBeLessThan(
|
||||
rateBound,
|
||||
"queue-starvation timeouts are expected under this stress but must stay marginal; " +
|
||||
$"{starved}/{totalEvaluations} evaluations missed the 5s budget");
|
||||
$"{starved}/{totalEvaluations} evaluations missed the 5s budget" + Diagnostics());
|
||||
}
|
||||
|
||||
// ── input quality reaches the script (archreview 02/S14) ────────────────────────────────
|
||||
|
||||
@@ -125,9 +125,14 @@ public sealed class SupervisionTests
|
||||
/// A burst of cold compiles must not starve the worker's heartbeat. Found on windev by the §8 perf harness: 500
|
||||
/// loads at once, each compiled synchronously on its own thread-pool work item, queued the heartbeat timer and
|
||||
/// the pipe writer behind them; the host saw no frame for 3 s, killed the shard (KilledHeartbeat) and made all 500
|
||||
/// keys suspects. Here a 100 ms heartbeat (kill after 300 ms of silence) and 400 distinct cold scripts at once:
|
||||
/// every script answers and the shard's worker is the one that started. Mutation check: compile on the thread
|
||||
/// pool again and the worker is killed for silence.
|
||||
/// keys suspects. Here a 50 ms heartbeat (kill after 3 s of silence, production's limit) and 1,500 distinct cold scripts at once:
|
||||
/// every script answers and the shard's worker is the one that started.
|
||||
/// <para>⚠️ The old "mutation check: compile on the thread pool again and the worker is killed for silence" does
|
||||
/// NOT hold any more (re-checked for lmxopcua#843 on a 14-core Mac: the mutated worker passed at both the 150 ms
|
||||
/// and the 1 s limit). The host's silence clock counts ANY frame, and since lmxopcua#789 a compile writes its own
|
||||
/// <c>LoadStarted</c>/<c>LoadResult</c> frames inline — so pool-queued compiles keep the pipe busy themselves (the 3 s silence
|
||||
/// windev saw presumably also needed the pre-#789 pool-scheduled writer stuck behind them). What this still pins is the
|
||||
/// behaviour: a cold burst under a tight heartbeat is answered without a kill.</para>
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task A_burst_of_cold_compiles_does_not_starve_the_heartbeat()
|
||||
@@ -136,13 +141,29 @@ public sealed class SupervisionTests
|
||||
var heartbeat = TimeSpan.FromMilliseconds(50);
|
||||
await using var runner = TightHeartbeatRunner(heartbeat, TimeSpan.FromSeconds(120), sink);
|
||||
var worker = WarmTight(runner, await runner.RunAsync(Scripts.Vt("first", Healthy), Ct), heartbeat, sink);
|
||||
// lmxopcua#843: three 50 ms intervals (150 ms) is a limit the OS scheduler alone breaks on an overloaded box — at
|
||||
// load average 100–280 the worker or the host's reader waited 150–310 ms for a core (10/10 runs killed for silence
|
||||
// under 28 CPU burners, every compile on its dedicated thread), and at ~300 even a 1 s limit was hit once in 10
|
||||
// (1,012 ms). The limit is now PRODUCTION's (3 × the default 1 s heartbeat): the test kills only where a real node
|
||||
// would, and the windev defect (no frame for 3 s, the whole burst) is still the size it catches. The 50 ms
|
||||
// heartbeat keeps frames flowing for the silence clock.
|
||||
worker.SilenceLimit = TimeSpan.FromSeconds(3);
|
||||
|
||||
const int count = 1500;
|
||||
var burst = Stopwatch.StartNew();
|
||||
var outcomes = await Task.WhenAll(Enumerable.Range(0, count).Select(i => runner.RunAsync(
|
||||
Scripts.Vt("burst-" + i, $"return (int)ctx.GetTag(\"x\").Value * {i} + 1;", TimeSpan.FromSeconds(60),
|
||||
Scripts.Input("x", 2)), Ct)));
|
||||
burst.Stop();
|
||||
|
||||
var log = string.Join(Environment.NewLine, sink.Rendered.Where(l => l.Contains("ScriptWorker", StringComparison.Ordinal)).Take(5));
|
||||
// The kill line ("sent nothing for N ms") against the burst's length is the diagnosis: N close to the burst is
|
||||
// the pool-starvation defect, N just over the limit is a starved box.
|
||||
var log = $"burst took {burst.ElapsedMilliseconds} ms, silence limit {worker.SilenceLimit.TotalMilliseconds} ms\n"
|
||||
+ string.Join(Environment.NewLine, sink.Rendered
|
||||
.Where(l => l.Contains("ScriptWorker", StringComparison.Ordinal))
|
||||
.Where(l => l.Contains("sent nothing", StringComparison.Ordinal) || l.Contains("kill", StringComparison.OrdinalIgnoreCase)
|
||||
|| l.Contains("exit", StringComparison.OrdinalIgnoreCase))
|
||||
.Take(10));
|
||||
for (var i = 0; i < count; i++) outcomes[i].Value.ShouldBe(2 * i + 1, $"{outcomes[i].Status} {outcomes[i].Reason}\n{log}");
|
||||
runner.Shards[0].Current.ShouldNotBeNull().Pid.ShouldBe(worker.Pid, log);
|
||||
worker.Exited.IsCompleted.ShouldBeFalse(log);
|
||||
|
||||
Reference in New Issue
Block a user