perf(historian): one QuestDB WAL wait per drain pass, with an exponential poll (lmxopcua#746)

The QuestDB live writer waited for WAL visibility after every tag group, and
the wait polled wal_tables() every 10 ms. With DrainConcurrency = 8 that was
up to eight pollers at ~100 queries/s each against a database whose WAL
apply was already behind.

New Runtime seam IHistorianPassWriter (WriteLiveValuesInPassAsync +
FlushAsync). The recorder writes every group of a pass through it and calls
FlushAsync once after all groups report, before the ack, whenever any group
wrote (an un-acked pass is redelivered and its insert-if-absent check must
see this pass's rows). A flush failure does not fail the pass. The QuestDB
writer implements it: in-pass writes only mark a flush as owed; FlushAsync
waits once. The poll now backs off 10 -> 20 -> ... -> 200 ms, never past the
timeout. Direct WriteLiveValuesAsync callers still wait per call.

Also updates the QuestDB cancelled-write integration test to the rethrow
contract from the previous commit.

Rebased from the unpushed remed/lane-H2 branch (originally authored 2026-10-22
in session_01Ub9hGmYLQvn9XETzw7CMsT) onto current master.

Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
Joseph Doherty
2026-10-22 16:48:40 -04:00
parent 7c5cc9c7e5
commit ee314d1b9e
9 changed files with 395 additions and 13 deletions
@@ -85,6 +85,40 @@ public sealed class WriteTests(QuestDbFixture fx)
}
}
[Fact]
public async Task A_drain_pass_waits_for_wal_visibility_once_not_once_per_tag_group()
{
fx.SkipIfUnavailable();
var o = fx.Options(fx.UniquePrefix());
var c = new QuestDbConnection(o);
var b = new QuestDbSchemaBootstrapper(c, o);
var catalogWal = new QuestDbWalVisibility(c, o.VisibilityTimeout);
var sampleWal = new QuestDbWalVisibility(c, o.VisibilityTimeout); // the writer's own: counts only sample waits
var w = new QuestDbHistorianValueWriter(c, new QuestDbTagCatalog(c, b, catalogWal), sampleWal, utcNow: () => ServerNow);
foreach (var tag in new[] { "pass.a", "pass.b", "pass.c" })
(await w.WriteLiveValuesInPassAsync(tag, [new(T0, HistorianValue.FromDouble(1), 0)], CancellationToken.None)).ShouldBeTrue();
sampleWal.Waits.ShouldBe(0, "in-pass writes defer the visibility wait");
await w.FlushAsync(CancellationToken.None);
sampleWal.Waits.ShouldBe(1, "one wait for the whole pass");
foreach (var tag in new[] { "pass.a", "pass.b", "pass.c" })
(await RowsAsync(c, tag)).Count.ShouldBe(1, $"{tag} is visible after the flush");
// A redelivery of the pass (the insert-if-absent check reads what the flush made visible) keeps one row each —
// and, having inserted nothing, owes no wait.
foreach (var tag in new[] { "pass.a", "pass.b", "pass.c" })
(await w.WriteLiveValuesInPassAsync(tag, [new(T0, HistorianValue.FromDouble(2), 0)], CancellationToken.None)).ShouldBeTrue();
await w.FlushAsync(CancellationToken.None);
foreach (var tag in new[] { "pass.a", "pass.b", "pass.c" })
(await RowsAsync(c, tag)).Single().N.ShouldBe(1d);
sampleWal.Waits.ShouldBe(1);
// The plain per-call path still waits per call (positive control).
(await w.WriteLiveValuesAsync("pass.d", [new(T0, HistorianValue.FromDouble(4), 0)], CancellationToken.None)).ShouldBeTrue();
sampleWal.Waits.ShouldBe(2);
}
[Fact]
public async Task A_redelivered_batch_leaves_one_row_and_the_first_value()
{
@@ -210,13 +244,16 @@ public sealed class WriteTests(QuestDbFixture fx)
}
[Fact]
public async Task A_cancelled_write_returns_false_without_throwing()
public async Task A_cancelled_write_rethrows_so_the_recorder_does_not_book_a_tag_failure()
{
// The recorder's contract: a cancelled drain pass is not this tag's failure (it used to return false, which the
// recorder counted toward the tag's quarantine budget).
fx.SkipIfUnavailable();
var (w, _) = Make();
using var cts = new CancellationTokenSource();
cts.Cancel();
(await w.WriteLiveValuesAsync("x", [new(T0, HistorianValue.FromDouble(1), 0)], cts.Token)).ShouldBeFalse();
await Should.ThrowAsync<OperationCanceledException>(
() => w.WriteLiveValuesAsync("x", [new(T0, HistorianValue.FromDouble(1), 0)], cts.Token));
}
private static async Task SeedRowAsync(QuestDbConnection c, QuestDbWalVisibility wal, string tag, DateTime t, double v)
@@ -30,4 +30,12 @@ public sealed class QuestDbWalVisibilityTests
[InlineData(4, 5, false)]
public void Reached_is_writerTxn_at_or_beyond_the_target(long writer, long target, bool expected) =>
QuestDbWalVisibility.Reached(writer, target).ShouldBe(expected);
[Fact]
public void The_poll_backs_off_exponentially_from_10_ms_to_a_200_ms_cap()
{
var delays = new List<TimeSpan> { QuestDbWalVisibility.FirstPollInterval };
while (delays.Count < 8) delays.Add(QuestDbWalVisibility.NextPollInterval(delays[^1]));
delays.Select(d => (int)d.TotalMilliseconds).ShouldBe([10, 20, 40, 80, 160, 200, 200, 200]);
}
}