fix(historian): #692 every HistoryRead-Raw page is full — no empty page between values
A resumed raw page asked the backend for only N rows from the boundary timestamp (inclusive), then trimmed the boundary ties already emitted: it netted N - skip values, and at N = 1 nothing — the empty "full" page was taken for a stalled tie cluster and a MaxTieClusterOverfetch + 1 (65,537-row) over-fetch stepped one tick, returning an EMPTY page with a continuation point between every value (8 values = 17 pages / 25 reads), forward and reverse, with or without ReturnBounds, on every backend. Each page's read now asks for N + boundarySkip + 1 rows: the ties it trims, the page and one lookahead row. The lookahead row is the 'more data' signal, so an exact multiple ends on its last full page (no trailing empty page). The over-fetch fires only when every row past the trimmed ties is still on the boundary (a genuine cluster larger than the page); a cursor left inside a cluster is flagged WithinTieCluster and over-fetches directly. A page that never leaves the boundary adds the prior skip to its cursor. Low items: BoundedReadHasAnythingAsync carries its walk hint and, when there is no data, the end bound it found (HistoryEndBound.Resolved + DataExhausted), so the next page reads nothing; HistoryBounds.IsPlaceholder is an explicit marker (ConditionalWeakTable over the NotFound instances) instead of status+null shape, so a stored Bad_BoundNotFound row is data. Tests: NodeManagerHistoryReadPageFillTests (N=1 every shape/direction/bounds, tie clusters straddling boundaries at N=1/2/3, exact-multiple, carried end bound + walk hint, stored placeholder-shaped row). Existing paging tests updated where they encoded the short pages.
This commit is contained in:
+19
-13
@@ -29,8 +29,8 @@ public sealed class NodeManagerHistoryReadPagingTests : IDisposable
|
||||
|
||||
/// <summary>A full first page emits a continuation point; resuming returns subsequent pages; the
|
||||
/// final short page clears the continuation point — and the union of pages is the complete series with
|
||||
/// no duplicate or skipped sample. (Resumed pages net slightly fewer than the cap because each resume
|
||||
/// re-reads + trims the boundary sample to stay tie-safe — see the design; the union is what matters.)</summary>
|
||||
/// no duplicate or skipped sample. (#692: every page is full — a resume re-reads the boundary ties it trims plus one
|
||||
/// lookahead row, so 250 at 100 is exactly three pages of 100, 100 and 50.)</summary>
|
||||
[Fact]
|
||||
public async Task Raw_pages_full_series_across_continuation_points()
|
||||
{
|
||||
@@ -64,7 +64,7 @@ public sealed class NodeManagerHistoryReadPagingTests : IDisposable
|
||||
while (cp is not null);
|
||||
|
||||
sawCp.ShouldBeTrue("a 250-sample series capped at 100 must page");
|
||||
pageCount.ShouldBeGreaterThan(1);
|
||||
pageCount.ShouldBe(3, "100 + 100 + 50 — every page full (#692)");
|
||||
|
||||
// The union is the exact series — no duplicates, no skips, in order.
|
||||
collected.Count.ShouldBe(250);
|
||||
@@ -132,11 +132,11 @@ public sealed class NodeManagerHistoryReadPagingTests : IDisposable
|
||||
await host.DisposeAsync();
|
||||
}
|
||||
|
||||
/// <summary>An exactly-full series whose total is a multiple of the cap pages to a trailing
|
||||
/// EMPTY page (GoodNoData, no CP) — the server can't know the prior page was the last until it reads
|
||||
/// past the end.</summary>
|
||||
/// <summary>An exactly-full series whose total is a multiple of the cap ends on its last full page: each read asks for
|
||||
/// one lookahead row past the page (#692), so the server knows the page is the last and emits no continuation point —
|
||||
/// there is no trailing EMPTY page any more. A stale point resumed anyway still terminates with GoodNoData.</summary>
|
||||
[Fact]
|
||||
public async Task Raw_exact_multiple_terminates_with_empty_final_page()
|
||||
public async Task Raw_exact_multiple_ends_on_its_last_full_page()
|
||||
{
|
||||
var (host, server) = await BootAsync();
|
||||
var nm = server.NodeManager!;
|
||||
@@ -149,12 +149,17 @@ public sealed class NodeManagerHistoryReadPagingTests : IDisposable
|
||||
|
||||
var (r1, _, cp1) = ReadRaw(nm, nodeId, Epoch, Epoch.AddHours(1), max: 100, inboundCp: null);
|
||||
r1.Count.ShouldBe(100);
|
||||
cp1.ShouldNotBeNull("a full page emits a CP even when it happens to be the last");
|
||||
cp1.ShouldBeNull("the lookahead row found nothing past the page — it is the last");
|
||||
|
||||
var (r2, e2, cp2) = ReadRaw(nm, nodeId, Epoch, Epoch.AddHours(1), max: 100, inboundCp: cp1);
|
||||
r2.Count.ShouldBe(0);
|
||||
// Positive control: one more sample past the page ⇒ the same full page DOES carry a continuation point.
|
||||
nm.HistorianDataSource = new SeriesHistorianDataSource(MakeSeries(count: 101, stepSeconds: 1));
|
||||
var (r2, e2, cp2) = ReadRaw(nm, nodeId, Epoch, Epoch.AddHours(1), max: 100, inboundCp: null);
|
||||
r2.Count.ShouldBe(100);
|
||||
e2.StatusCode.Code.ShouldBe(StatusCodes.Good);
|
||||
cp2.ShouldBeNull("the empty trailing page terminates paging");
|
||||
cp2.ShouldNotBeNull();
|
||||
var (r3, _, cp3) = ReadRaw(nm, nodeId, Epoch, Epoch.AddHours(1), max: 100, inboundCp: cp2);
|
||||
r3.ShouldBe([100d]);
|
||||
cp3.ShouldBeNull();
|
||||
|
||||
await host.DisposeAsync();
|
||||
}
|
||||
@@ -675,8 +680,9 @@ public sealed class NodeManagerHistoryReadPagingTests : IDisposable
|
||||
var (values, error, next) = ReadRaw(nm, nodeId, DateTime.MinValue, end, max: 3, inboundCp: cp);
|
||||
error.StatusCode.Code.ShouldBe(StatusCodes.Good);
|
||||
values.ShouldNotBeEmpty();
|
||||
values[^1].ShouldBe((double)(expected - 2), $"page {page} continues exactly");
|
||||
expected = (int)values[^1];
|
||||
// #692: every page is full (N = 3) — a resumed page used to net N − 1 after trimming its boundary sample.
|
||||
values.ShouldBe([expected, expected - 1d, expected - 2d], $"page {page} continues exactly");
|
||||
expected -= 3;
|
||||
next.ShouldNotBeNull();
|
||||
cp = next;
|
||||
|
||||
|
||||
Reference in New Issue
Block a user