fix(historian): #694 page raw reads at a strict gateway's learned ceiling; opt-in EventReadHorizon

Item 1: IHistorianDataSource.MaxRawValuesPerRead (Gateway: learned ceiling - 16 = L; 0 elsewhere, guard-tested). RawPageLimit plans every raw page on top of #692's lookahead: with L > 0 the read (page P + boundary ties k + lookahead) is <= L, P = min(N, L - k - 1) (uncapped: L - k - 1); no room => straight to the tie-cluster over-fetch (never an uncapped or underflowing request). One re-plan + re-read when the read taught a new limit or was refused for crowding pre-window rows (HistoryReadLimitExceededException.PreWindowRows): the page shrinks by them. Over-fetch bound min(MaxTieClusterOverfetch, L - 1); slice size never 0. Review fixes: F1 a window starting at/before the epoch counts AVEVA's synthetic row as pre-window (a ceiling-truncated read was reported complete); F2 the page path disables ReverseRawReader's own re-run (cold end-only ceiling 20/N=3000 cost 3,023 reads, now <= 40); F3 crowded look-back re-read around, not refused; F4 the #687 crowding re-read room at least doubles (up to 8 re-reads), so a cluster of hundreds just before a resume point no longer fails on a default gateway; a crowding refusal that leaves no room is BadTooManyOperations, never a tick-by-tick cursor. Item 2: ServerHistorian:EventReadHorizon (default 00:00:00 = exact, unchanged). When set, every fresh HistoryReadEvents window's earlier bound is raised to now - horizon (both directions, all backends). Lossy by design; documented. Item 3 (native TimeAverage/Total re-check) stays blocked on historiangw#27.
2026-10-13 00:52:12 -04:00
parent a5a0496176
commit 3c3513ea3e
23 changed files with 1461 additions and 79 deletions
@@ -51,6 +51,9 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
    /// <summary>#693: AtTime from raw reads (true) or AVEVA's own interpolation (the kill switch).</summary>
    internal bool AtTimeFromRaw => _atTimeFromRaw;

+    /// <summary>The most #687 crowding re-reads one raw read makes (#694 F4): each at least doubles the pre-window room.</summary>
+    private const int MaxCrowdedReReads = 8;
+
    /// <summary>A strict gateway's row ceiling, learned from its first refusal (#691.3).</summary>
    private readonly GatewayRowCeiling _ceiling = new();

@@ -191,29 +194,46 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
            throw new HistoryReadLimitExceededException(cap,
                $"The raw read of '{fullReference}' holds more than the HistorianGateway's row ceiling ({all.Budget}, its " +
                "Historian:ReadLimits:MaxValuesCeiling with AllowStreamingBeyondCeiling=false); set NumValuesPerNode to page, " +
-                "or narrow the window.");
+                "or narrow the window.")
+            { PreWindowRows = Crowding(all.Result.PreWindowRows) };
        }

        var asked = (int)Math.Min(int.MaxValue, (long)cap + RawAggregateReader.PreWindowAllowance);
-        var first = await PassAsync(asked).ConfigureAwait(false);
-        if (first.Result.Complete) return first.Result.Samples;
+        var pass = await PassAsync(asked).ConfigureAwait(false);
+        if (pass.Result.Complete) return pass.Result.Samples;

-        // A strict gateway's ceiling already bounded that read (#691.3): a larger budget would only be refused again.
-        if (first.Budget < asked)
+        // A strict gateway's ceiling already bounded that read (#691.3): a larger budget would only be refused again. The
+        // refusal carries the pre-window rows it saw (#694): the server re-reads the page that many values smaller.
+        if (pass.Budget < asked)
            throw new HistoryReadLimitExceededException(cap,
                $"The raw read of '{fullReference}' could not be read completely within the HistorianGateway's row ceiling " +
-                $"({first.Budget}, its Historian:ReadLimits:MaxValuesCeiling with AllowStreamingBeyondCeiling=false); lower " +
-                "NumValuesPerNode or narrow the window.");
+                $"({pass.Budget}, its Historian:ReadLimits:MaxValuesCeiling with AllowStreamingBeyondCeiling=false); lower " +
+                "NumValuesPerNode or narrow the window.")
+            { PreWindowRows = Crowding(pass.Result.PreWindowRows) };

-        // The margin crowded the window out: one bounded re-read with room for the pre-window rows already seen.
-        var second = await PassAsync(
-            (int)Math.Min(int.MaxValue, (long)cap + first.Result.PreWindowRows + RawAggregateReader.PreWindowAllowance))
-            .ConfigureAwait(false);
-        if (second.Result.Complete) return second.Result.Samples;
+        // The margin crowded the window out: re-read with room for the pre-window rows. A read that was itself cut short saw
+        // only some of them (#694 F4: a tie cluster of hundreds just before a resume point used to fail even with no
+        // ceiling), so the room at least doubles per re-read — bounded by MaxCrowdedReReads, and never past a known ceiling.
+        var preWindow = pass.Result.PreWindowRows;
+        var room = Math.Max(1, preWindow);
+        for (var attempt = 0; attempt < MaxCrowdedReReads; attempt++)
+        {
+            var budget = (int)Math.Min(int.MaxValue, (long)cap + room + RawAggregateReader.PreWindowAllowance);
+            if (_ceiling.Learned is var known and > 0 && budget > known) break; // over a strict ceiling: refused anyway
+            pass = await PassAsync(budget).ConfigureAwait(false);
+            if (pass.Result.Complete) return pass.Result.Samples;
+            preWindow = Math.Max(preWindow, pass.Result.PreWindowRows);
+            if (pass.Budget < budget) break; // bounded by a ceiling learned meanwhile
+            room = (int)Math.Min(int.MaxValue / 2, Math.Max(preWindow, 2L * room));
+        }

        throw new HistoryReadLimitExceededException(cap,
-            $"The raw read of '{fullReference}' could not be read completely: more than {second.Result.PreWindowRows} rows lie in " +
-            $"the {RawAggregateReader.BoundaryMargin.TotalMilliseconds} ms before the window and exhausted the row budget.");
+            $"The raw read of '{fullReference}' could not be read completely: more than {pass.Result.PreWindowRows} rows lie in " +
+            $"the {RawAggregateReader.BoundaryMargin.TotalMilliseconds} ms before the window and exhausted the row budget.")
+        { PreWindowRows = preWindow };
+
+        // Pre-window rows the allowance did not cover are what crowded the read; fewer are not the cause (#694).
+        static int Crowding(int preWindowRows) => preWindowRows > RawAggregateReader.PreWindowAllowance ? preWindowRows : 0;

        // One drained pass; a strict gateway's refusal of the budget is retried once at the ceiling it names (#691.3).
        Task<((List<HistorianSample> Samples, int PreWindowRows, bool Complete) Result, int Budget)> PassAsync(int budget) =>
@@ -240,7 +260,11 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
            {
                if (cap == 0 || samples.Count < cap) samples.Add(sample);
            }
-            else if (sample.Timestamp is { } ts && ts.ToDateTime() < startUtc) preWindow++;
+            // A row before the window — or, for a window starting at/before the epoch (no look-back), AVEVA's synthetic row at
+            // the query start, which is no sample either (#694 F1: it used to set reachedEnd, so a ceiling-truncated read
+            // passed as complete) — is a pre-window row.
+            else if (sample.Timestamp is { } ts
+                     && (ts.ToDateTime() < startUtc || !RawAggregateReader.IsAfterFloor(ts.ToDateTime()))) preWindow++;
            else if (sample.Timestamp is not null) reachedEnd = true;
        }

@@ -453,6 +477,17 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
    /// </remarks>
    public bool SupportsReverseRaw => false;

+    /// <inheritdoc />
+    /// <remarks>
+    ///     Gitea #694: <c>0</c> until a strict gateway (<c>AllowStreamingBeyondCeiling=false</c>) has refused a budget and
+    ///     named its ceiling (#691.3, <see cref="GatewayRowCeiling"/>); then that ceiling less the
+    ///     <see cref="RawAggregateReader.PreWindowAllowance"/> a capped read adds for AVEVA's boundary row — so a page of this
+    ///     size asks for exactly the ceiling. Back to <c>0</c> when a re-probe finds the gateway streaming again.
+    /// </remarks>
+    public int MaxRawValuesPerRead => _ceiling.Learned is var c and > 0
+        ? Math.Max(1, c - RawAggregateReader.PreWindowAllowance)
+        : 0;
+
    /// <inheritdoc />
    /// <remarks>
    ///     <c>false</c>: the gateway's <c>ReadEvents</c> streams ascending only (Gitea #675). The server never sends it a
@@ -359,6 +359,10 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource
    /// <remarks>Reverse raw reads are served natively (<c>(end, start]</c>, <c>ORDER BY … DESC</c>; #669/#686).</remarks>
    public bool SupportsReverseRaw => true;

+    /// <inheritdoc />
+    /// <remarks><c>0</c>: a capped raw read is one bounded statement — no backend row ceiling (Gitea #694).</remarks>
+    public int MaxRawValuesPerRead => 0;
+
    /// <inheritdoc />
    /// <remarks>Reverse events reads are served natively (<c>ORDER BY … DESC</c>, Gitea #675).</remarks>
    public bool SupportsReverseEvents => true;
@@ -316,6 +316,10 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
    /// <remarks>Reverse raw reads are served natively (<c>(end, start]</c>, <c>ORDER BY … DESC</c>; #669/#686).</remarks>
    public bool SupportsReverseRaw => true;

+    /// <inheritdoc />
+    /// <remarks><c>0</c>: a capped raw read is one bounded statement — no backend row ceiling (Gitea #694).</remarks>
+    public int MaxRawValuesPerRead => 0;
+
    /// <inheritdoc />
    /// <remarks>Reverse events reads are served natively (<c>ORDER BY … DESC</c>, Gitea #675).</remarks>
    public bool SupportsReverseEvents => true;