perf(historian): one QuestDB WAL wait per drain pass, with an exponential poll (lmxopcua#746)

The QuestDB live writer waited for WAL visibility after every tag group, and the wait polled wal_tables() every 10 ms. With DrainConcurrency = 8 that was up to eight pollers at ~100 queries/s each against a database whose WAL apply was already behind. New Runtime seam IHistorianPassWriter (WriteLiveValuesInPassAsync + FlushAsync). The recorder writes every group of a pass through it and calls FlushAsync once after all groups report, before the ack, whenever any group wrote (an un-acked pass is redelivered and its insert-if-absent check must see this pass's rows). A flush failure does not fail the pass. The QuestDB writer implements it: in-pass writes only mark a flush as owed; FlushAsync waits once. The poll now backs off 10 -> 20 -> ... -> 200 ms, never past the timeout. Direct WriteLiveValuesAsync callers still wait per call. Also updates the QuestDB cancelled-write integration test to the rethrow contract from the previous commit. Rebased from the unpushed remed/lane-H2 branch (originally authored 2026-10-22 in session_01Ub9hGmYLQvn9XETzw7CMsT) onto current master. Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
2026-10-22 16:48:40 -04:00
parent 7c5cc9c7e5
commit ee314d1b9e
9 changed files with 395 additions and 13 deletions
@@ -141,6 +141,9 @@ public sealed class ContinuousHistorizationRecorder : ReceiveActor, IWithTimers

    private readonly IActorRef _dependencyMux;
    private readonly IHistorianValueWriter _writer;
+
+    /// <summary><see cref="_writer"/> as a pass-aware writer (one flush per pass), or null for a plain writer.</summary>
+    private readonly IHistorianPassWriter? _passWriter;
    private readonly IHistorizationOutbox _outbox;

    /// <summary>The tracked historized tags, keyed by mux ref (<see cref="HistorizedTagRef.MuxRef"/> — the
@@ -279,6 +282,7 @@ public sealed class ContinuousHistorizationRecorder : ReceiveActor, IWithTimers
    {
        _dependencyMux = dependencyMux ?? throw new ArgumentNullException(nameof(dependencyMux));
        _writer = writer ?? throw new ArgumentNullException(nameof(writer));
+        _passWriter = writer as IHistorianPassWriter;
        _outbox = outbox ?? throw new ArgumentNullException(nameof(outbox));
        ArgumentNullException.ThrowIfNull(historizedRefs);
        // The ctor seed is the no-override identity (mux ref == historian name); production seeds an EMPTY
@@ -740,7 +744,9 @@ public sealed class ContinuousHistorizationRecorder : ReceiveActor, IWithTimers
                    bool ok;
                    try
                    {
-                        ok = await _writer.WriteLiveValuesAsync(g.Key, g.Values, ct).ConfigureAwait(false);
+                        ok = _passWriter is { } passWriter
+                            ? await passWriter.WriteLiveValuesInPassAsync(g.Key, g.Values, ct).ConfigureAwait(false)
+                            : await _writer.WriteLiveValuesAsync(g.Key, g.Values, ct).ConfigureAwait(false);
                    }
                    catch (OperationCanceledException) when (ct.IsCancellationRequested)
                    {
@@ -776,6 +782,27 @@ public sealed class ContinuousHistorizationRecorder : ReceiveActor, IWithTimers
            // where the not-yet-entered tasks fault out of WaitAsync without reaching the finally.
            (string Tag, int Count, bool Ok)[] outcomes = await Task.WhenAll(tasks).ConfigureAwait(false);

+            // A pass-aware writer completes the pass ONCE (QuestDB: one WAL visibility wait for every group, instead of
+            // one per group) — whenever any group wrote, acked or not, because an un-acked pass is redelivered and the
+            // redelivery's insert-if-absent check must see what this pass wrote. Before the ack. A flush failure does not
+            // fail the pass: the rows are committed, only their visibility is unproven.
+            if (_passWriter is { } flushWriter && outcomes.Any(o => o.Ok))
+            {
+                try
+                {
+                    await flushWriter.FlushAsync(ct).ConfigureAwait(false);
+                }
+                catch (OperationCanceledException) when (ct.IsCancellationRequested)
+                {
+                    throw;
+                }
+                catch (Exception ex)
+                {
+                    _log.Warning("ContinuousHistorization: pass flush failed ({Exception}); the pass's writes are committed.",
+                        ex.GetType().Name);
+                }
+            }
+
            // Fold the outcomes on the single logical drain flow — NEVER inside the parallel lambdas.
            // _tagWriteFailures is actor state (kept single-flight by _draining); mutating it from N
            // concurrent tasks would corrupt the failure counters. Nothing below observes completion
@@ -0,0 +1,36 @@
+using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
+
+namespace ZB.MOM.WW.OtOpcUa.Runtime.Historian;
+
+/// <summary>
+///     An <see cref="IHistorianValueWriter"/> whose per-write completion work can be done ONCE per drain pass instead of
+///     once per tag group — QuestDB's WAL visibility wait, which a backend needs so that an immediate redelivery's
+///     insert-if-absent check sees the rows just written. The continuous-historization recorder writes every group of a
+///     pass with <see cref="WriteLiveValuesInPassAsync"/> and then calls <see cref="FlushAsync"/> once, after every group
+///     has reported and before it acks the pass.
+/// </summary>
+/// <remarks>
+///     The cancellation contract is the recorder's for both members: a cancellation of the caller's token (the pass) is
+///     rethrown; any other failure of a write is <c>false</c>. A <see cref="FlushAsync"/> failure does not fail the pass —
+///     the rows are committed; only their visibility to the next pass is unproven.
+/// </remarks>
+public interface IHistorianPassWriter : IHistorianValueWriter
+{
+    /// <summary>
+    ///     Writes a batch of live values for a single tag as part of a drain pass, deferring the per-write completion
+    ///     work to <see cref="FlushAsync"/>. Same result contract as <see cref="IHistorianValueWriter.WriteLiveValuesAsync"/>.
+    /// </summary>
+    /// <param name="tag">Fully-qualified historian tag the values are recorded against.</param>
+    /// <param name="values">The samples to write, in append order.</param>
+    /// <param name="ct">The pass's cancellation token.</param>
+    /// <returns><c>true</c> when the backend stored the batch; <c>false</c> on a retryable failure.</returns>
+    Task<bool> WriteLiveValuesInPassAsync(string tag, IReadOnlyList<HistorizationValue> values, CancellationToken ct);
+
+    /// <summary>
+    ///     Completes the pass: does the deferred completion work for every write since the last flush (QuestDB: one WAL
+    ///     visibility wait). A no-op when nothing was written.
+    /// </summary>
+    /// <param name="ct">The pass's cancellation token.</param>
+    /// <returns>A task that completes when the pass's writes are visible (or the wait gave up).</returns>
+    Task FlushAsync(CancellationToken ct);
+}