perf(historian): one QuestDB WAL wait per drain pass, with an exponential poll (lmxopcua#746)

The QuestDB live writer waited for WAL visibility after every tag group, and
the wait polled wal_tables() every 10 ms. With DrainConcurrency = 8 that was
up to eight pollers at ~100 queries/s each against a database whose WAL
apply was already behind.

New Runtime seam IHistorianPassWriter (WriteLiveValuesInPassAsync +
FlushAsync). The recorder writes every group of a pass through it and calls
FlushAsync once after all groups report, before the ack, whenever any group
wrote (an un-acked pass is redelivered and its insert-if-absent check must
see this pass's rows). A flush failure does not fail the pass. The QuestDB
writer implements it: in-pass writes only mark a flush as owed; FlushAsync
waits once. The poll now backs off 10 -> 20 -> ... -> 200 ms, never past the
timeout. Direct WriteLiveValuesAsync callers still wait per call.

Also updates the QuestDB cancelled-write integration test to the rethrow
contract from the previous commit.

Rebased from the unpushed remed/lane-H2 branch (originally authored 2026-10-22
in session_01Ub9hGmYLQvn9XETzw7CMsT) onto current master.

Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV

src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb/QuestDbHistorianValueWriter.cs

@@ -6,6 +6,7 @@ using Npgsql;
 using NpgsqlTypes;
 using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
 using ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common;
+using ZB.MOM.WW.OtOpcUa.Runtime.Historian;
 
 namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb;
 
@@ -22,10 +23,14 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb;
 ///     + Bad, an in-batch µs collision keeps the first). <b>RC1:</b> a sample whose time is before 1970 cannot be a
 ///     QuestDB designated timestamp and one such row would fail the whole multi-row INSERT, so those samples are dropped
 ///     (Warning at most once per tag per <see cref="Pre1970WarningInterval"/>; every one counted in <see cref="DroppedPre1970"/>) and the call still succeeds for the rest.
-///     <b>R5:</b> after inserting, the writer waits for WAL visibility so an immediate redelivery sees the rows.
+///     <b>R5:</b> after inserting, the writer waits for WAL visibility so an immediate redelivery sees the rows — per call
+///     through <see cref="WriteLiveValuesAsync"/>, but ONCE per drain pass through the recorder's
+///     <see cref="IHistorianPassWriter"/> path (<see cref="WriteLiveValuesInPassAsync"/> + <see cref="FlushAsync"/>): with
+///     <c>DrainConcurrency</c> groups per pass each running its own poll, a lagging WAL apply was polled by up to eight
+///     waiters at once, amplifying load on a database that was already behind.
 ///     Inserts are chunked at <see cref="BatchRows"/> rows (8 bound parameters per row, far under the 32,767 limit).
 /// </remarks>
-internal sealed class QuestDbHistorianValueWriter : IHistorianValueWriter
+internal sealed class QuestDbHistorianValueWriter : IHistorianPassWriter
 {
     /// <summary>Rows per multi-row INSERT (T0 Q2: 1,000 rows is reliable).</summary>
     internal const int BatchRows = 1000;
@@ -42,6 +47,7 @@ internal sealed class QuestDbHistorianValueWriter : IHistorianValueWriter
     private readonly ILogger _log;
     private readonly Func<DateTime> _utcNow;
     private long _droppedPre1970;
+    private int _flushPending;
     private readonly ConcurrentDictionary<string, DateTime> _pre1970WarnedUtc = new(StringComparer.Ordinal);
 
     /// <summary>Creates the writer.</summary>
@@ -64,7 +70,32 @@ internal sealed class QuestDbHistorianValueWriter : IHistorianValueWriter
     internal long DroppedPre1970 => Interlocked.Read(ref _droppedPre1970);
 
     /// <inheritdoc/>
-    public async Task<bool> WriteLiveValuesAsync(string tag, IReadOnlyList<HistorizationValue> values, CancellationToken ct)
+    public Task<bool> WriteLiveValuesAsync(string tag, IReadOnlyList<HistorizationValue> values, CancellationToken ct) =>
+        WriteAsync(tag, values, waitForVisibility: true, ct);
+
+    /// <inheritdoc/>
+    public Task<bool> WriteLiveValuesInPassAsync(string tag, IReadOnlyList<HistorizationValue> values, CancellationToken ct) =>
+        WriteAsync(tag, values, waitForVisibility: false, ct);
+
+    /// <inheritdoc/>
+    // One WAL visibility wait for every in-pass insert since the last flush; the wait's timeout / a suspended table does
+    // not change the outcome (the rows are committed and become visible later).
+    public async Task FlushAsync(CancellationToken ct)
+    {
+        if (Interlocked.Exchange(ref _flushPending, 0) == 0) return;
+        try
+        {
+            await _wal.WaitAsync(_conn.SampleTable, ct).ConfigureAwait(false);
+        }
+        catch
+        {
+            Volatile.Write(ref _flushPending, 1); // still owed: the next flush retries it
+            throw;
+        }
+    }
+
+    private async Task<bool> WriteAsync(string tag, IReadOnlyList<HistorizationValue> values, bool waitForVisibility,
+        CancellationToken ct)
     {
         try
         {
@@ -99,7 +130,10 @@ internal sealed class QuestDbHistorianValueWriter : IHistorianValueWriter
             for (var i = 0; i < fresh.Count; i += BatchRows)
                 await InsertAsync(tag, fresh.GetRange(i, Math.Min(BatchRows, fresh.Count - i)), serverTs, ct).ConfigureAwait(false);
 
-            await _wal.WaitAsync(_conn.SampleTable, ct).ConfigureAwait(false); // timeout/suspension does not change the result (R5)
+            if (waitForVisibility)
+                await _wal.WaitAsync(_conn.SampleTable, ct).ConfigureAwait(false); // timeout/suspension does not change the result
+            else
+                Volatile.Write(ref _flushPending, 1); // the pass's FlushAsync waits once for every group
             return true;
         }
         catch (OperationCanceledException) when (ct.IsCancellationRequested)

src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb/QuestDbWalVisibility.cs

@@ -8,12 +8,29 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb;
 ///     The read-your-writes wait (R5, T0 Q4). QuestDB applies WAL commits asynchronously, so a row just inserted may
 ///     not be visible to the next query. After its INSERT(s) a writer calls <see cref="WaitAsync"/>, which captures
 ///     <c>sequencerTxn</c> from <c>wal_tables()</c> (every autocommitted INSERT is one sequencer transaction; a concurrent
-///     writer can only raise the target) and polls <c>writerTxn</c> every ~10 ms until it reaches the target, the table
-///     is suspended, or <see cref="QuestDbHistorianOptions.VisibilityTimeout"/> elapses.
+///     writer can only raise the target) and polls <c>writerTxn</c> — first after 10 ms, then backing off exponentially
+///     (doubling, capped at <see cref="MaxPollInterval"/>) so a lagging WAL apply is not hammered with ~100 queries/s per
+///     waiter — until it reaches the target, the table is suspended, or <see cref="QuestDbHistorianOptions.VisibilityTimeout"/>
+///     elapses.
 /// </summary>
 internal sealed class QuestDbWalVisibility
 {
-    private static readonly TimeSpan PollInterval = TimeSpan.FromMilliseconds(10);
+    /// <summary>The first poll delay.</summary>
+    internal static readonly TimeSpan FirstPollInterval = TimeSpan.FromMilliseconds(10);
+
+    /// <summary>The longest poll delay the backoff reaches.</summary>
+    internal static readonly TimeSpan MaxPollInterval = TimeSpan.FromMilliseconds(200);
+
+    /// <summary>The poll delay after <paramref name="current"/>: doubled, capped at <see cref="MaxPollInterval"/>.</summary>
+    /// <param name="current">The delay just waited.</param>
+    /// <returns>The next delay.</returns>
+    internal static TimeSpan NextPollInterval(TimeSpan current) =>
+        current >= MaxPollInterval / 2 ? MaxPollInterval : current * 2;
+
+    private int _waits;
+
+    /// <summary>Waits started so far (lifetime) — test/diagnostic.</summary>
+    internal int Waits => Volatile.Read(ref _waits);
 
     private readonly QuestDbConnection _conn;
     private readonly TimeSpan _timeout;
@@ -49,7 +66,9 @@ internal sealed class QuestDbWalVisibility
     public async Task<bool> WaitAsync(string table, CancellationToken ct)
     {
         if (Disabled) return true;
+        Interlocked.Increment(ref _waits);
         var sw = Stopwatch.StartNew();
+        var delay = FirstPollInterval;
         await using var conn = await _conn.OpenAsync(ct).ConfigureAwait(false);
 
         long target;
@@ -86,7 +105,11 @@ internal sealed class QuestDbWalVisibility
                     table, _timeout.TotalMilliseconds, writer, target);
                 return false;
             }
-            await Task.Delay(PollInterval, ct).ConfigureAwait(false);
+            var remaining = _timeout - sw.Elapsed;
+            // Never sleep past the timeout (and never a negative delay if it elapsed in between).
+            await Task.Delay(remaining <= TimeSpan.Zero ? TimeSpan.Zero : delay < remaining ? delay : remaining, ct)
+                .ConfigureAwait(false);
+            delay = NextPollInterval(delay);
         }
     }
 }

src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Historian/ContinuousHistorizationRecorder.cs

@@ -141,6 +141,9 @@ public sealed class ContinuousHistorizationRecorder : ReceiveActor, IWithTimers
 
     private readonly IActorRef _dependencyMux;
     private readonly IHistorianValueWriter _writer;
+
+    /// <summary><see cref="_writer"/> as a pass-aware writer (one flush per pass), or null for a plain writer.</summary>
+    private readonly IHistorianPassWriter? _passWriter;
     private readonly IHistorizationOutbox _outbox;
 
     /// <summary>The tracked historized tags, keyed by mux ref (<see cref="HistorizedTagRef.MuxRef"/> — the
@@ -279,6 +282,7 @@ public sealed class ContinuousHistorizationRecorder : ReceiveActor, IWithTimers
     {
         _dependencyMux = dependencyMux ?? throw new ArgumentNullException(nameof(dependencyMux));
         _writer = writer ?? throw new ArgumentNullException(nameof(writer));
+        _passWriter = writer as IHistorianPassWriter;
         _outbox = outbox ?? throw new ArgumentNullException(nameof(outbox));
         ArgumentNullException.ThrowIfNull(historizedRefs);
         // The ctor seed is the no-override identity (mux ref == historian name); production seeds an EMPTY
@@ -740,7 +744,9 @@ public sealed class ContinuousHistorizationRecorder : ReceiveActor, IWithTimers
                     bool ok;
                     try
                     {
-                        ok = await _writer.WriteLiveValuesAsync(g.Key, g.Values, ct).ConfigureAwait(false);
+                        ok = _passWriter is { } passWriter
+                            ? await passWriter.WriteLiveValuesInPassAsync(g.Key, g.Values, ct).ConfigureAwait(false)
+                            : await _writer.WriteLiveValuesAsync(g.Key, g.Values, ct).ConfigureAwait(false);
                     }
                     catch (OperationCanceledException) when (ct.IsCancellationRequested)
                     {
@@ -776,6 +782,27 @@ public sealed class ContinuousHistorizationRecorder : ReceiveActor, IWithTimers
             // where the not-yet-entered tasks fault out of WaitAsync without reaching the finally.
             (string Tag, int Count, bool Ok)[] outcomes = await Task.WhenAll(tasks).ConfigureAwait(false);
 
+            // A pass-aware writer completes the pass ONCE (QuestDB: one WAL visibility wait for every group, instead of
+            // one per group) — whenever any group wrote, acked or not, because an un-acked pass is redelivered and the
+            // redelivery's insert-if-absent check must see what this pass wrote. Before the ack. A flush failure does not
+            // fail the pass: the rows are committed, only their visibility is unproven.
+            if (_passWriter is { } flushWriter && outcomes.Any(o => o.Ok))
+            {
+                try
+                {
+                    await flushWriter.FlushAsync(ct).ConfigureAwait(false);
+                }
+                catch (OperationCanceledException) when (ct.IsCancellationRequested)
+                {
+                    throw;
+                }
+                catch (Exception ex)
+                {
+                    _log.Warning("ContinuousHistorization: pass flush failed ({Exception}); the pass's writes are committed.",
+                        ex.GetType().Name);
+                }
+            }
+
             // Fold the outcomes on the single logical drain flow — NEVER inside the parallel lambdas.
             // _tagWriteFailures is actor state (kept single-flight by _draining); mutating it from N
             // concurrent tasks would corrupt the failure counters. Nothing below observes completion

src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Historian/IHistorianPassWriter.cs

@@ -0,0 +1,36 @@
+using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
+
+namespace ZB.MOM.WW.OtOpcUa.Runtime.Historian;
+
+/// <summary>
+///     An <see cref="IHistorianValueWriter"/> whose per-write completion work can be done ONCE per drain pass instead of
+///     once per tag group — QuestDB's WAL visibility wait, which a backend needs so that an immediate redelivery's
+///     insert-if-absent check sees the rows just written. The continuous-historization recorder writes every group of a
+///     pass with <see cref="WriteLiveValuesInPassAsync"/> and then calls <see cref="FlushAsync"/> once, after every group
+///     has reported and before it acks the pass.
+/// </summary>
+/// <remarks>
+///     The cancellation contract is the recorder's for both members: a cancellation of the caller's token (the pass) is
+///     rethrown; any other failure of a write is <c>false</c>. A <see cref="FlushAsync"/> failure does not fail the pass —
+///     the rows are committed; only their visibility to the next pass is unproven.
+/// </remarks>
+public interface IHistorianPassWriter : IHistorianValueWriter
+{
+    /// <summary>
+    ///     Writes a batch of live values for a single tag as part of a drain pass, deferring the per-write completion
+    ///     work to <see cref="FlushAsync"/>. Same result contract as <see cref="IHistorianValueWriter.WriteLiveValuesAsync"/>.
+    /// </summary>
+    /// <param name="tag">Fully-qualified historian tag the values are recorded against.</param>
+    /// <param name="values">The samples to write, in append order.</param>
+    /// <param name="ct">The pass's cancellation token.</param>
+    /// <returns><c>true</c> when the backend stored the batch; <c>false</c> on a retryable failure.</returns>
+    Task<bool> WriteLiveValuesInPassAsync(string tag, IReadOnlyList<HistorizationValue> values, CancellationToken ct);
+
+    /// <summary>
+    ///     Completes the pass: does the deferred completion work for every write since the last flush (QuestDB: one WAL
+    ///     visibility wait). A no-op when nothing was written.
+    /// </summary>
+    /// <param name="ct">The pass's cancellation token.</param>
+    /// <returns>A task that completes when the pass's writes are visible (or the wait gave up).</returns>
+    Task FlushAsync(CancellationToken ct);
+}