perf(historian): one QuestDB WAL wait per drain pass, with an exponential poll (lmxopcua#746)

The QuestDB live writer waited for WAL visibility after every tag group, and
the wait polled wal_tables() every 10 ms. With DrainConcurrency = 8 that was
up to eight pollers at ~100 queries/s each against a database whose WAL
apply was already behind.

New Runtime seam IHistorianPassWriter (WriteLiveValuesInPassAsync +
FlushAsync). The recorder writes every group of a pass through it and calls
FlushAsync once after all groups report, before the ack, whenever any group
wrote (an un-acked pass is redelivered and its insert-if-absent check must
see this pass's rows). A flush failure does not fail the pass. The QuestDB
writer implements it: in-pass writes only mark a flush as owed; FlushAsync
waits once. The poll now backs off 10 -> 20 -> ... -> 200 ms, never past the
timeout. Direct WriteLiveValuesAsync callers still wait per call.

Also updates the QuestDB cancelled-write integration test to the rethrow
contract from the previous commit.

Rebased from the unpushed remed/lane-H2 branch (originally authored 2026-10-22
in session_01Ub9hGmYLQvn9XETzw7CMsT) onto current master.

Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV

src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb/QuestDbHistorianValueWriter.cs

@@ -6,6 +6,7 @@ using Npgsql;
 using NpgsqlTypes;
 using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
 using ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common;
+using ZB.MOM.WW.OtOpcUa.Runtime.Historian;
 
 namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb;
 
@@ -22,10 +23,14 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb;
 ///     + Bad, an in-batch µs collision keeps the first). <b>RC1:</b> a sample whose time is before 1970 cannot be a
 ///     QuestDB designated timestamp and one such row would fail the whole multi-row INSERT, so those samples are dropped
 ///     (Warning at most once per tag per <see cref="Pre1970WarningInterval"/>; every one counted in <see cref="DroppedPre1970"/>) and the call still succeeds for the rest.
-///     <b>R5:</b> after inserting, the writer waits for WAL visibility so an immediate redelivery sees the rows.
+///     <b>R5:</b> after inserting, the writer waits for WAL visibility so an immediate redelivery sees the rows — per call
+///     through <see cref="WriteLiveValuesAsync"/>, but ONCE per drain pass through the recorder's
+///     <see cref="IHistorianPassWriter"/> path (<see cref="WriteLiveValuesInPassAsync"/> + <see cref="FlushAsync"/>): with
+///     <c>DrainConcurrency</c> groups per pass each running its own poll, a lagging WAL apply was polled by up to eight
+///     waiters at once, amplifying load on a database that was already behind.
 ///     Inserts are chunked at <see cref="BatchRows"/> rows (8 bound parameters per row, far under the 32,767 limit).
 /// </remarks>
-internal sealed class QuestDbHistorianValueWriter : IHistorianValueWriter
+internal sealed class QuestDbHistorianValueWriter : IHistorianPassWriter
 {
     /// <summary>Rows per multi-row INSERT (T0 Q2: 1,000 rows is reliable).</summary>
     internal const int BatchRows = 1000;
@@ -42,6 +47,7 @@ internal sealed class QuestDbHistorianValueWriter : IHistorianValueWriter
     private readonly ILogger _log;
     private readonly Func<DateTime> _utcNow;
     private long _droppedPre1970;
+    private int _flushPending;
     private readonly ConcurrentDictionary<string, DateTime> _pre1970WarnedUtc = new(StringComparer.Ordinal);
 
     /// <summary>Creates the writer.</summary>
@@ -64,7 +70,32 @@ internal sealed class QuestDbHistorianValueWriter : IHistorianValueWriter
     internal long DroppedPre1970 => Interlocked.Read(ref _droppedPre1970);
 
     /// <inheritdoc/>
-    public async Task<bool> WriteLiveValuesAsync(string tag, IReadOnlyList<HistorizationValue> values, CancellationToken ct)
+    public Task<bool> WriteLiveValuesAsync(string tag, IReadOnlyList<HistorizationValue> values, CancellationToken ct) =>
+        WriteAsync(tag, values, waitForVisibility: true, ct);
+
+    /// <inheritdoc/>
+    public Task<bool> WriteLiveValuesInPassAsync(string tag, IReadOnlyList<HistorizationValue> values, CancellationToken ct) =>
+        WriteAsync(tag, values, waitForVisibility: false, ct);
+
+    /// <inheritdoc/>
+    // One WAL visibility wait for every in-pass insert since the last flush; the wait's timeout / a suspended table does
+    // not change the outcome (the rows are committed and become visible later).
+    public async Task FlushAsync(CancellationToken ct)
+    {
+        if (Interlocked.Exchange(ref _flushPending, 0) == 0) return;
+        try
+        {
+            await _wal.WaitAsync(_conn.SampleTable, ct).ConfigureAwait(false);
+        }
+        catch
+        {
+            Volatile.Write(ref _flushPending, 1); // still owed: the next flush retries it
+            throw;
+        }
+    }
+
+    private async Task<bool> WriteAsync(string tag, IReadOnlyList<HistorizationValue> values, bool waitForVisibility,
+        CancellationToken ct)
     {
         try
         {
@@ -99,7 +130,10 @@ internal sealed class QuestDbHistorianValueWriter : IHistorianValueWriter
             for (var i = 0; i < fresh.Count; i += BatchRows)
                 await InsertAsync(tag, fresh.GetRange(i, Math.Min(BatchRows, fresh.Count - i)), serverTs, ct).ConfigureAwait(false);
 
-            await _wal.WaitAsync(_conn.SampleTable, ct).ConfigureAwait(false); // timeout/suspension does not change the result (R5)
+            if (waitForVisibility)
+                await _wal.WaitAsync(_conn.SampleTable, ct).ConfigureAwait(false); // timeout/suspension does not change the result
+            else
+                Volatile.Write(ref _flushPending, 1); // the pass's FlushAsync waits once for every group
             return true;
         }
         catch (OperationCanceledException) when (ct.IsCancellationRequested)

src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb/QuestDbWalVisibility.cs

@@ -8,12 +8,29 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb;
 ///     The read-your-writes wait (R5, T0 Q4). QuestDB applies WAL commits asynchronously, so a row just inserted may
 ///     not be visible to the next query. After its INSERT(s) a writer calls <see cref="WaitAsync"/>, which captures
 ///     <c>sequencerTxn</c> from <c>wal_tables()</c> (every autocommitted INSERT is one sequencer transaction; a concurrent
-///     writer can only raise the target) and polls <c>writerTxn</c> every ~10 ms until it reaches the target, the table
-///     is suspended, or <see cref="QuestDbHistorianOptions.VisibilityTimeout"/> elapses.
+///     writer can only raise the target) and polls <c>writerTxn</c> — first after 10 ms, then backing off exponentially
+///     (doubling, capped at <see cref="MaxPollInterval"/>) so a lagging WAL apply is not hammered with ~100 queries/s per
+///     waiter — until it reaches the target, the table is suspended, or <see cref="QuestDbHistorianOptions.VisibilityTimeout"/>
+///     elapses.
 /// </summary>
 internal sealed class QuestDbWalVisibility
 {
-    private static readonly TimeSpan PollInterval = TimeSpan.FromMilliseconds(10);
+    /// <summary>The first poll delay.</summary>
+    internal static readonly TimeSpan FirstPollInterval = TimeSpan.FromMilliseconds(10);
+
+    /// <summary>The longest poll delay the backoff reaches.</summary>
+    internal static readonly TimeSpan MaxPollInterval = TimeSpan.FromMilliseconds(200);
+
+    /// <summary>The poll delay after <paramref name="current"/>: doubled, capped at <see cref="MaxPollInterval"/>.</summary>
+    /// <param name="current">The delay just waited.</param>
+    /// <returns>The next delay.</returns>
+    internal static TimeSpan NextPollInterval(TimeSpan current) =>
+        current >= MaxPollInterval / 2 ? MaxPollInterval : current * 2;
+
+    private int _waits;
+
+    /// <summary>Waits started so far (lifetime) — test/diagnostic.</summary>
+    internal int Waits => Volatile.Read(ref _waits);
 
     private readonly QuestDbConnection _conn;
     private readonly TimeSpan _timeout;
@@ -49,7 +66,9 @@ internal sealed class QuestDbWalVisibility
     public async Task<bool> WaitAsync(string table, CancellationToken ct)
     {
         if (Disabled) return true;
+        Interlocked.Increment(ref _waits);
         var sw = Stopwatch.StartNew();
+        var delay = FirstPollInterval;
         await using var conn = await _conn.OpenAsync(ct).ConfigureAwait(false);
 
         long target;
@@ -86,7 +105,11 @@ internal sealed class QuestDbWalVisibility
                     table, _timeout.TotalMilliseconds, writer, target);
                 return false;
             }
-            await Task.Delay(PollInterval, ct).ConfigureAwait(false);
+            var remaining = _timeout - sw.Elapsed;
+            // Never sleep past the timeout (and never a negative delay if it elapsed in between).
+            await Task.Delay(remaining <= TimeSpan.Zero ? TimeSpan.Zero : delay < remaining ? delay : remaining, ct)
+                .ConfigureAwait(false);
+            delay = NextPollInterval(delay);
         }
     }
 }