fix(historian): #689 size reverse-walk segments from the page, not the tie budget

ReverseRawReader aimed every segment at half of MaxTieClusterOverfetch (~32k rows by default) and carried that width page to page, so a paged 'latest N' read on a dense Gateway tag fetched ~32,770 rows per page (2-5 s live) to return N. The walk now lives in a generic ReverseWalk core: segments are aimed at ~N+8 rows (page budget 2N+16), a discovery segment (first without a hint / after an empty one) may read 1,024, and the tie budget is only the ceiling a single-timestamp cluster escalates to. A truncated read never re-enters the stretch below its newest row until the gap above it is bisected. Tests: per-page row counts on a dense tag (node manager + walk), randomized exactness fuzz (walk and node-manager paging), dense-tag-below-a-long-gap read bound.
2026-10-12 16:25:07 -04:00
parent 71d031fcd0
commit d33992034d
8 changed files with 663 additions and 183 deletions
@@ -650,6 +650,95 @@ public sealed class NodeManagerHistoryReadPagingTests : IDisposable
        await host.DisposeAsync();
    }

+    /// <summary>
+    ///     #689: a paged end-only read on a dense tag sizes each segment from the PAGE (N), not from
+    ///     <c>MaxTieClusterOverfetch</c>. Live (AVEVA, 1 Hz, N = 3): from about page 5 every page read 32,770 rows and took
+    ///     2–5 s to return 2 rows, because the walk re-aimed its width at half the 65,536-row tie budget and carried it.
+    /// </summary>
+    [Fact]
+    public async Task Raw_reverse_paging_on_a_dense_forward_only_tag_reads_about_the_page_size_per_page()
+    {
+        // 100,000 samples 100 ms apart (~2.8 h): far more than the default tie budget would ever let a page read.
+        var series = Enumerable.Range(0, 100_000)
+            .Select(i => { var t = Epoch.AddMilliseconds(100L * i); return new DataValueSnapshot((double)i, StatusCodes.Good, t, t); })
+            .ToArray();
+        var source = new ForwardOnlyHistorianDataSource(series);
+        var (host, nm, nodeId) = await BootForwardOnlyAsync(source, "FwdDense689");
+        nm.MaxTieClusterOverfetch.ShouldBe(65536, "the default budget — the one the live gate ran with");
+        var end = Epoch.AddMilliseconds(100L * 99_999);
+
+        byte[]? cp = null;
+        var expected = 99_999;
+        for (var page = 1; page <= 20; page++)
+        {
+            source.ResetCounts();
+            var (values, error, next) = ReadRaw(nm, nodeId, DateTime.MinValue, end, max: 3, inboundCp: cp);
+            error.StatusCode.Code.ShouldBe(StatusCodes.Good);
+            values.ShouldNotBeEmpty();
+            values[^1].ShouldBe((double)(expected - 2), $"page {page} continues exactly");
+            expected = (int)values[^1];
+            next.ShouldNotBeNull();
+            cp = next;
+
+            // The discriminating counts: rows read and the largest cap asked for, per page. Page 1 has no density yet and
+            // may read one discovery segment (a minute here, 600 rows); every page after it is sized from N.
+            var rowsBound = page == 1 ? ReverseWalk.DiscoveryRows + 2 : 64;
+            source.RowsReturned.ShouldBeLessThanOrEqualTo(rowsBound, $"page {page} read {source.RowsReturned} rows to return {values.Count}");
+            source.MaxCapRequested.ShouldBeLessThanOrEqualTo((uint)rowsBound, $"page {page} asked for {source.MaxCapRequested} rows");
+            source.ReadCount.ShouldBeLessThanOrEqualTo(3, $"page {page}");
+        }
+
+        await host.DisposeAsync();
+    }
+
+    /// <summary>
+    ///     #689/#691: randomized end-to-end paging of end-only reads through the node manager on a forward-only backend —
+    ///     bursts, gaps, sub-millisecond stamps and tie clusters (some larger than the page) — the union of the pages is
+    ///     every sample exactly once, newest first, whatever N and the tie budget.
+    /// </summary>
+    [Fact]
+    public async Task Raw_reverse_paging_on_a_forward_only_backend_is_exact_on_random_data()
+    {
+        var (host, nm, nodeId) = await BootForwardOnlyAsync(new ForwardOnlyHistorianDataSource([]), "FwdFuzz");
+        for (var seed = 0; seed < 150; seed++)
+        {
+            var rng = new Random(seed);
+            var series = new List<DataValueSnapshot>();
+            var t = Epoch.Ticks;
+            var v = 0;
+            var count = rng.Next(1, 300);
+            while (series.Count < count)
+            {
+                t += rng.Next(5) switch
+                {
+                    0 => rng.Next(1, 10),
+                    1 => rng.Next(1, 5) * TimeSpan.TicksPerMillisecond,
+                    2 => rng.Next(1, 90) * TimeSpan.TicksPerMinute,
+                    _ => rng.Next(1, 2000) * TimeSpan.TicksPerMillisecond,
+                };
+                var ties = rng.Next(6) == 0 ? rng.Next(2, 7) : 1;
+                for (var k = 0; k < ties; k++)
+                {
+                    var ts = new DateTime(t, DateTimeKind.Utc);
+                    series.Add(new DataValueSnapshot((double)v++, StatusCodes.Good, ts, ts));
+                }
+            }
+
+            // No fabricated start row: the Gateway source drops it (#687) — the node manager's tie-cluster point read relies on
+            // that. The walk itself is fuzzed WITH the row in ReverseWalkTests.
+            nm.HistorianDataSource = new ForwardOnlyHistorianDataSource(series, emitBoundaryRow: false);
+            nm.MaxTieClusterOverfetch = rng.Next(2) == 0 ? 8 : 65536;
+            var max = (uint)rng.Next(1, 12);
+            var end = new DateTime(t + rng.Next(0, 3) * TimeSpan.TicksPerSecond, DateTimeKind.Utc);
+
+            var collected = await DrainAsync(nm, nodeId, DateTime.MinValue, end, max, maxPages: 2000);
+
+            collected.ShouldBe(Enumerable.Range(0, series.Count).Reverse().Select(i => (double)i), $"seed {seed}, N = {max}");
+        }
+
+        await host.DisposeAsync();
+    }
+
    /// <summary>#686 review: a reverse read from <c>StartTime = DateTime.MaxValue</c> walks back from now + 1 day, not from
    /// year 9999.</summary>
    [Fact]
@@ -978,9 +1067,15 @@ public sealed class NodeManagerHistoryReadPagingTests : IDisposable
        /// <summary>Reads whose row count equalled the cap (a full, possibly truncated, read).</summary>
        public int FullReads { get; private set; }

+        /// <summary>Rows returned across every read since the last <see cref="ResetCounts"/> (#689).</summary>
+        public long RowsReturned { get; private set; }
+
+        /// <summary>The largest <c>maxValuesPerNode</c> asked for since the last <see cref="ResetCounts"/> (#689).</summary>
+        public uint MaxCapRequested { get; private set; }
+
        public void ResetCounts()
        {
-            lock (_gate) FullReads = 0;
+            lock (_gate) { FullReads = 0; RowsReturned = 0; MaxCapRequested = 0; }
            Interlocked.Exchange(ref _readCount, 0);
        }

@@ -1016,8 +1111,12 @@ public sealed class NodeManagerHistoryReadPagingTests : IDisposable
                    : new DataValueSnapshot(prior.Value, prior.StatusCode, stamp, stamp));
            rows.AddRange(series.Where(s => Ts(s) >= startUtc && Ts(s) <= endUtc));
            var page = maxValuesPerNode == 0 ? rows : rows.Take((int)maxValuesPerNode).ToList();
-            if (maxValuesPerNode > 0 && page.Count == maxValuesPerNode)
-                lock (_gate) FullReads++;
+            lock (_gate)
+            {
+                if (maxValuesPerNode > 0 && page.Count == maxValuesPerNode) FullReads++;
+                RowsReturned += page.Count;
+                if (maxValuesPerNode > MaxCapRequested) MaxCapRequested = maxValuesPerNode;
+            }
            int seen;
            while (page.Count > (seen = Volatile.Read(ref _maxRowsReturned))
                   && Interlocked.CompareExchange(ref _maxRowsReturned, page.Count, seen) != seen) { }