fix(historian): #692 every HistoryRead-Raw page is full — no empty page between values

A resumed raw page asked the backend for only N rows from the boundary timestamp
(inclusive), then trimmed the boundary ties already emitted: it netted N - skip values,
and at N = 1 nothing — the empty "full" page was taken for a stalled tie cluster and a
MaxTieClusterOverfetch + 1 (65,537-row) over-fetch stepped one tick, returning an EMPTY
page with a continuation point between every value (8 values = 17 pages / 25 reads),
forward and reverse, with or without ReturnBounds, on every backend.

Each page's read now asks for N + boundarySkip + 1 rows: the ties it trims, the page and
one lookahead row. The lookahead row is the 'more data' signal, so an exact multiple ends
on its last full page (no trailing empty page). The over-fetch fires only when every row
past the trimmed ties is still on the boundary (a genuine cluster larger than the page);
a cursor left inside a cluster is flagged WithinTieCluster and over-fetches directly.
A page that never leaves the boundary adds the prior skip to its cursor.

Low items: BoundedReadHasAnythingAsync carries its walk hint and, when there is no data,
the end bound it found (HistoryEndBound.Resolved + DataExhausted), so the next page reads
nothing; HistoryBounds.IsPlaceholder is an explicit marker (ConditionalWeakTable over the
NotFound instances) instead of status+null shape, so a stored Bad_BoundNotFound row is data.

Tests: NodeManagerHistoryReadPageFillTests (N=1 every shape/direction/bounds, tie clusters
straddling boundaries at N=1/2/3, exact-multiple, carried end bound + walk hint, stored
placeholder-shaped row). Existing paging tests updated where they encoded the short pages.
This commit is contained in:
Joseph Doherty
2026-10-13 00:42:13 -04:00
parent 15ccd9b722
commit 8dfa165110
6 changed files with 554 additions and 91 deletions
@@ -17,8 +17,13 @@ namespace ZB.MOM.WW.OtOpcUa.OpcUaServer;
/// <param name="Open">The end time was not given (a one-sided read's sentinel): there is no end bound to look up.</param>
/// <param name="DataExhausted">The data is drained and the end bound did not fit on the previous page.</param>
/// <param name="LastReturnedUtc">The last value's time returned on an earlier page (stamps an open bound's placeholder).</param>
/// <param name="Resolved">
/// Gitea #692: the end bound an earlier page already looked up (a stored sample — never a placeholder), so the page that
/// returns it does not read the backend again. <c>null</c> = not looked up yet.
/// </param>
internal sealed record HistoryEndBound(
DateTime AtUtc, bool Reverse, bool Inclusive, bool Open, bool DataExhausted = false, DateTime? LastReturnedUtc = null);
DateTime AtUtc, bool Reverse, bool Inclusive, bool Open, bool DataExhausted = false, DateTime? LastReturnedUtc = null,
DataValueSnapshot? Resolved = null);
/// <summary>
/// Gitea #691: OPC UA Part 11 bounding values for HistoryRead-Raw (<c>ReadRawModifiedDetails.ReturnBounds</c>), composed
@@ -59,6 +64,15 @@ internal static class HistoryBounds
{
private static readonly TimeSpan OpenBoundStep = TimeSpan.FromSeconds(1);
/// <summary>
/// Gitea #692: the placeholders this server synthesised (<see cref="NotFound"/>), by reference. A placeholder is known by
/// this marker, never by its shape: a stored row can legitimately carry <c>Bad_BoundNotFound</c> with a null value (a
/// client may write one through HistoryUpdate), and that row is data.
/// </summary>
private static readonly System.Runtime.CompilerServices.ConditionalWeakTable<DataValueSnapshot, object> Placeholders = new();
private static readonly object PlaceholderMark = new();
/// <summary>The end bound owed by a fresh read over the normalised window.</summary>
/// <param name="startUtc">The (normalised) start: the later time of a reverse read.</param>
/// <param name="endUtc">The (normalised) end; a one-sided read's sentinel (tick 0 / MaxValue) is an open bound.</param>
@@ -66,11 +80,13 @@ internal static class HistoryBounds
public static HistoryEndBound For(DateTime startUtc, DateTime endUtc) =>
new(endUtc, Reverse: startUtc > endUtc, Inclusive: startUtc != endUtc, Open: IsOpen(endUtc));
/// <summary>Whether a returned value is a <see cref="NotFound"/> placeholder rather than a stored sample.</summary>
/// <summary>
/// Whether a returned value is a <see cref="NotFound"/> placeholder this server synthesised rather than a stored sample
/// — by an explicit marker (#692), so a stored row whose status happens to be <c>Bad_BoundNotFound</c> is not taken for one.
/// </summary>
/// <param name="value">The value.</param>
/// <returns><c>true</c> for a <c>Bad_BoundNotFound</c> placeholder.</returns>
public static bool IsPlaceholder(DataValueSnapshot value) =>
value.StatusCode == StatusCodes.BadBoundNotFound && value.Value is null;
/// <returns><c>true</c> only for an instance <see cref="NotFound"/> produced.</returns>
public static bool IsPlaceholder(DataValueSnapshot value) => Placeholders.TryGetValue(value, out _);
/// <summary>The pending end bound to store behind a continuation point, remembering the page's last returned time.</summary>
/// <param name="bound">The pending end bound (null = bounds not requested).</param>
@@ -84,8 +100,12 @@ internal static class HistoryBounds
/// <summary>The <c>Bad_BoundNotFound</c> placeholder (null value) stamped at <paramref name="atUtc"/>.</summary>
/// <param name="atUtc">The bound's time.</param>
/// <returns>The placeholder.</returns>
public static DataValueSnapshot NotFound(DateTime atUtc) =>
new(null, StatusCodes.BadBoundNotFound, atUtc, atUtc);
public static DataValueSnapshot NotFound(DateTime atUtc)
{
var placeholder = new DataValueSnapshot(null, StatusCodes.BadBoundNotFound, atUtc, atUtc);
Placeholders.AddOrUpdate(placeholder, PlaceholderMark);
return placeholder;
}
/// <summary>
/// The start bound to return ahead of the data, or <c>null</c> when a stored sample sits on StartTime (that sample is
@@ -123,6 +143,8 @@ internal static class HistoryBounds
return NotFound(new DateTime(ticks, DateTimeKind.Utc));
}
if (bound.Resolved is { } resolved) return resolved; // #692: an earlier page already looked it up
DataValueSnapshot? found;
if (bound.Reverse)
found = await LatestAtOrBeforeAsync(source, tagname, bound.AtUtc, maxTieCluster, nowUtc, ct).ConfigureAwait(false);