ci: concurrent-write test's held write hit the fixed 5 s write cap under CI timefactor
v2-ci / unit-tests (push) Has been cancelled

A_queued_write_older_than_the_budget_is_failed_BadTimeout_instead_of_written_late
failed hosted runs 2215 (7ec5b95d) and 2260 (8ef5adc6) at
"(await inFlight).Success should be True but was False", after 7 s each time.
It is not load: it fails on an idle machine with AKKA_TEST_TIMEFACTOR=3, the
value v2-ci sets, and passes with 1.

Akka's AwaitCondition with no interval polls every max/10 of the dilated
budget: PresenceBudget 15 s x 3 = 45 s, so a 4.5 s sleep after a first poll
that runs before the pool has entered the write. The held write is bounded by
DriverInstanceActor.RunWriteAsync's fixed, undilated 5 s
CancellationTokenSource, so 4.5 s of polling plus the test's 600 ms settle
cancelled it ("write timeout") before ReleaseWrites(). The sibling tests that
hold a write took 7 s for the same reason and passed only by margin.

All eight AwaitCondition calls in the class now go through
AwaitWritesEntered, which polls every 20 ms. The budget (PresenceBudget)
and every assertion are unchanged.

Local, AKKA_TEST_TIMEFACTOR=3: before 6/7 (this test failing, 7 s); after
7/7 in three runs, test 3 s.

Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
Joseph Doherty
2026-11-09 14:17:50 -05:00
parent 84e293d2ff
commit 342a825d69
@@ -22,6 +22,18 @@ public sealed class DriverInstanceActorConcurrentWriteTests : RuntimeActorTestBa
{
private static readonly TimeSpan WriteLatency = TimeSpan.FromMilliseconds(1200);
/// <summary>
/// Polls for <paramref name="count"/> writes inside the driver with an EXPLICIT short interval. Akka's
/// <c>AwaitCondition</c> without one polls every <c>max / 10</c> of the DILATED budget — 1.5 s locally, 4.5 s
/// under CI's <c>AKKA_TEST_TIMEFACTOR=3</c>. The write under test is already held inside the driver by then,
/// and <c>DriverInstanceActor.RunWriteAsync</c> bounds it with a fixed (undilated) 5 s token, so one missed
/// first poll plus the test's own settle window cancelled it ("write timeout") before
/// <c>ReleaseWrites()</c>: <c>A_queued_write_older_than_the_budget…</c> failed every CI run that hit it
/// (hosted runs 2215 and 2260), and fails locally at timefactor 3.
/// </summary>
private void AwaitWritesEntered(SlowWritableDriver driver, int count) =>
AwaitCondition(() => driver.WritesEntered == count, PresenceBudget, TimeSpan.FromMilliseconds(20));
private (IActorRef Actor, Akka.TestKit.TestProbe Parent) Connect(
IDriver driver, int? maxWritesInFlight = null, TimeSpan? queuedWriteBudget = null, TimeSpan? writeTimeout = null)
{
@@ -67,7 +79,7 @@ public sealed class DriverInstanceActorConcurrentWriteTests : RuntimeActorTestBa
var pending = actor.Ask<DriverInstanceActor.WriteAttributeResult>(
new DriverInstanceActor.WriteAttribute("tag-a", 7), PresenceBudget);
AwaitCondition(() => driver.WritesEntered == 1, PresenceBudget);
AwaitWritesEntered(driver, 1);
driver.FireDataChange("tag-a", 3.5);
var published = parent.ExpectMsg<DriverInstanceActor.AttributeValueBatchPublished>(PresenceBudget);
@@ -90,7 +102,7 @@ public sealed class DriverInstanceActorConcurrentWriteTests : RuntimeActorTestBa
new DriverInstanceActor.WriteAttribute($"tag-{i}", i), PresenceBudget))
.ToArray();
AwaitCondition(() => driver.WritesEntered == 2, PresenceBudget);
AwaitWritesEntered(driver, 2);
// Settle: a third write entering the driver would show up here.
await Task.Delay(300);
driver.WritesEntered.ShouldBe(2, "the cap let a third write into the driver");
@@ -119,13 +131,13 @@ public sealed class DriverInstanceActorConcurrentWriteTests : RuntimeActorTestBa
var first = actor.Ask<DriverInstanceActor.WriteAttributeResult>(
new DriverInstanceActor.WriteAttribute("tag-a", 1), PresenceBudget);
AwaitCondition(() => driver.WritesEntered == 1, PresenceBudget);
AwaitWritesEntered(driver, 1);
var second = actor.Ask<DriverInstanceActor.WriteAttributeResult>(
new DriverInstanceActor.WriteAttribute("tag-a", 2), PresenceBudget);
// Positive control: a DIFFERENT tag is not held behind tag-a.
var other = actor.Ask<DriverInstanceActor.WriteAttributeResult>(
new DriverInstanceActor.WriteAttribute("tag-b", 9), PresenceBudget);
AwaitCondition(() => driver.WritesEntered == 2, PresenceBudget);
AwaitWritesEntered(driver, 2);
driver.EnteredRefs.ShouldBe(new[] { "tag-a", "tag-b" });
await Task.Delay(300);
@@ -149,7 +161,7 @@ public sealed class DriverInstanceActorConcurrentWriteTests : RuntimeActorTestBa
var inFlight = actor.Ask<DriverInstanceActor.WriteAttributeResult>(
new DriverInstanceActor.WriteAttribute("tag-0", 0), PresenceBudget);
AwaitCondition(() => driver.WritesEntered == 1, PresenceBudget);
AwaitWritesEntered(driver, 1);
var stale = actor.Ask<DriverInstanceActor.WriteAttributeResult>(
new DriverInstanceActor.WriteAttribute("tag-1", 1), PresenceBudget);
@@ -181,7 +193,7 @@ public sealed class DriverInstanceActorConcurrentWriteTests : RuntimeActorTestBa
var hung = actor.Ask<DriverInstanceActor.WriteAttributeResult>(
new DriverInstanceActor.WriteAttribute("tag-a", 1), PresenceBudget);
AwaitCondition(() => driver.WritesEntered == 1, PresenceBudget);
AwaitWritesEntered(driver, 1);
var hungResult = await hung;
hungResult.Success.ShouldBeFalse();
@@ -193,7 +205,7 @@ public sealed class DriverInstanceActorConcurrentWriteTests : RuntimeActorTestBa
// abandoned one is still inside it.
var later = actor.Ask<DriverInstanceActor.WriteAttributeResult>(
new DriverInstanceActor.WriteAttribute("tag-a", 2), PresenceBudget);
AwaitCondition(() => driver.WritesEntered == 2, PresenceBudget);
AwaitWritesEntered(driver, 2);
driver.EnteredRefs.ShouldBe(new[] { "tag-a", "tag-a" });
var laterResult = await later;
laterResult.Success.ShouldBeTrue(laterResult.Reason);
@@ -209,7 +221,7 @@ public sealed class DriverInstanceActorConcurrentWriteTests : RuntimeActorTestBa
var inFlight = actor.Ask<DriverInstanceActor.WriteAttributeResult>(
new DriverInstanceActor.WriteAttribute("tag-0", 0), PresenceBudget);
AwaitCondition(() => driver.WritesEntered == 1, PresenceBudget);
AwaitWritesEntered(driver, 1);
var queued = actor.Ask<DriverInstanceActor.WriteAttributeResult>(
new DriverInstanceActor.WriteAttribute("tag-1", 1), PresenceBudget);