fix(historian): #694 page raw reads at a strict gateway's learned ceiling; opt-in EventReadHorizon

Item 1: IHistorianDataSource.MaxRawValuesPerRead (Gateway: learned ceiling - 16 = L; 0 elsewhere, guard-tested). RawPageLimit plans every raw page on top of #692's lookahead: with L > 0 the read (page P + boundary ties k + lookahead) is <= L, P = min(N, L - k - 1) (uncapped: L - k - 1); no room => straight to the tie-cluster over-fetch (never an uncapped or underflowing request). One re-plan + re-read when the read taught a new limit or was refused for crowding pre-window rows (HistoryReadLimitExceededException.PreWindowRows): the page shrinks by them. Over-fetch bound min(MaxTieClusterOverfetch, L - 1); slice size never 0. Review fixes: F1 a window starting at/before the epoch counts AVEVA's synthetic row as pre-window (a ceiling-truncated read was reported complete); F2 the page path disables ReverseRawReader's own re-run (cold end-only ceiling 20/N=3000 cost 3,023 reads, now <= 40); F3 crowded look-back re-read around, not refused; F4 the #687 crowding re-read room at least doubles (up to 8 re-reads), so a cluster of hundreds just before a resume point no longer fails on a default gateway; a crowding refusal that leaves no room is BadTooManyOperations, never a tick-by-tick cursor. Item 2: ServerHistorian:EventReadHorizon (default 00:00:00 = exact, unchanged). When set, every fresh HistoryReadEvents window's earlier bound is raised to now - horizon (both directions, all backends). Lossy by design; documented. Item 3 (native TimeAverage/Total re-check) stays blocked on historiangw#27.
2026-10-13 00:52:12 -04:00
parent a5a0496176
commit 3c3513ea3e
23 changed files with 1461 additions and 79 deletions
@@ -58,25 +58,72 @@ internal static class ReverseRawReader
    /// <param name="ct">Cancellation.</param>
    /// <returns>The page, newest first, and the hint for the next page.</returns>
    /// <exception cref="HistoryReadLimitExceededException">One timestamp holds more than the tie budget.</exception>
+    /// <remarks>
+    ///     Gitea #694: every segment read fits the backend's <see cref="IHistorianDataSource.MaxRawValuesPerRead"/> (a strict
+    ///     gateway's learned ceiling) — the walk's tie budget is bounded by it so its segments are sized for it, and a
+    ///     crowded re-read asking more is cut to it (and counts as truncated, never as complete). A walk that TEACHES the
+    ///     backend its limit (the first read after start-up) and is refused for it is re-run once with segments sized for the
+    ///     new limit — unless <paramref name="retryOnLearnedLimit"/> is false: the HistoryRead page path passes false, so the
+    ///     refusal reaches <see cref="RawPageLimit"/>, which shrinks the PAGE to the limit as well (re-running the walk for a
+    ///     page of N far above the limit cost thousands of segment reads). The bound lookups (a page of one) keep the re-run.
+    ///     The page size itself is never cut here: a short answer would read as the last page. A segment refused because rows
+    ///     just before it crowded the backend's budget (<see cref="HistoryReadLimitExceededException.PreWindowRows"/>) is
+    ///     re-read once that many rows smaller and counts as truncated when full. An uncapped walk (one forward read of the
+    ///     whole window) is never cut: it is answered in full or refused.
+    /// </remarks>
+    /// <param name="retryOnLearnedLimit">Re-run the walk once when it taught the backend its limit (see the remarks).</param>
    public static async Task<ReverseRawPage> ReadAsync(
        IHistorianDataSource source, string tagname, DateTime hi, DateTime lo, uint maxValues, int maxTieCluster,
-        DateTime nowUtc, ReverseWalkHint hint, CancellationToken ct)
+        DateTime nowUtc, ReverseWalkHint hint, CancellationToken ct, bool retryOnLearnedLimit = true)
    {
-        var budget = Math.Max(1, maxTieCluster);
-        var (rows, next) = await ReverseWalk.WalkAsync(
-                async (start, end, cap, token) =>
-                {
-                    var samples = (await source.ReadRawAsync(tagname, start, end, (uint)cap, token).ConfigureAwait(false)).Samples;
-                    return new ForwardSegmentRead<DataValueSnapshot>(samples, cap > 0 && samples.Count >= cap);
-                },
-                Ticks, tieOrder: null, hi, lo, maxValues, budget, source.TimestampResolutionTicks, nowUtc, hint,
-                at => new HistoryReadLimitExceededException(budget,
-                    $"A reverse raw read of '{tagname}' at {at:O} found more than {budget} samples within one millisecond; the " +
-                    "backend reads forward only, so the latest of them cannot be selected. Increase " +
-                    "ServerHistorian:MaxTieClusterOverfetch."),
-                ct)
-            .ConfigureAwait(false);
-        return new ReverseRawPage(rows, next.WidthTicks) { Hint = next };
+        var before = source.MaxRawValuesPerRead;
+        try
+        {
+            return await WalkAsync(before).ConfigureAwait(false);
+        }
+        catch (HistoryReadLimitExceededException) when (retryOnLearnedLimit && maxValues > 0
+                                                        && source.MaxRawValuesPerRead is var now and > 0 && now != before)
+        {
+            // fall through (outside the catch): the walk taught the backend its limit — re-run it sized for that limit once.
+            // Not an uncapped walk: it is one read of the whole window, which the limit does not change.
+        }
+
+        return await WalkAsync(source.MaxRawValuesPerRead).ConfigureAwait(false);
+
+        async Task<ReverseRawPage> WalkAsync(int limit)
+        {
+            // A segment asks for at most its budget + 2 rows (ReverseWalk), so the budget is bounded 2 under the limit.
+            var budget = Math.Max(1, maxTieCluster);
+            if (limit > 0) budget = Math.Min(budget, Math.Max(1, limit - 2));
+            var (rows, next) = await ReverseWalk.WalkAsync(
+                    async (start, end, cap, token) =>
+                    {
+                        var asked = limit > 0 && cap > limit ? limit : cap; // cap 0 (the whole window) is never cut
+                        IReadOnlyList<DataValueSnapshot> samples;
+                        try
+                        {
+                            samples = (await source.ReadRawAsync(tagname, start, end, (uint)asked, token).ConfigureAwait(false)).Samples;
+                        }
+                        catch (HistoryReadLimitExceededException ex) when (asked > 0 && ex.PreWindowRows > 0 && asked - ex.PreWindowRows >= 1)
+                        {
+                            // #694: rows just before the segment crowded the backend's budget — read it smaller; a full answer
+                            // then counts as truncated (the walk narrows), never as complete.
+                            asked -= ex.PreWindowRows;
+                            samples = (await source.ReadRawAsync(tagname, start, end, (uint)asked, token).ConfigureAwait(false)).Samples;
+                        }
+                        return new ForwardSegmentRead<DataValueSnapshot>(samples, asked > 0 && samples.Count >= asked);
+                    },
+                    Ticks, tieOrder: null, hi, lo, maxValues, budget, source.TimestampResolutionTicks, nowUtc, hint,
+                    at => new HistoryReadLimitExceededException(budget,
+                        $"A reverse raw read of '{tagname}' at {at:O} found more than {budget} samples within one millisecond; the " +
+                        "backend reads forward only, so the latest of them cannot be selected. Increase " +
+                        (budget < Math.Max(1, maxTieCluster)
+                            ? "the backend's row ceiling (HistorianGateway Historian:ReadLimits:MaxValuesCeiling)."
+                            : "ServerHistorian:MaxTieClusterOverfetch.")),
+                    ct)
+                .ConfigureAwait(false);
+            return new ReverseRawPage(rows, next.WidthTicks) { Hint = next };
+        }
    }

    private static long Ticks(DataValueSnapshot s) => (s.SourceTimestampUtc ?? DateTime.MinValue).Ticks;