dohertj2/lmxopcua
1
0
Fork 0
Code Issues 32 Pull Requests Actions Packages Projects Releases Wiki Activity

fix(historian): #691.4 review — search the gap above a truncated read by gallop then bisect, never below 1 ms

Browse Source 
Bisecting the gap above a truncated read halved below the 1 ms request
granularity (requests are ms-rounded, so it re-sent identical requests)
and bisected the WHOLE gap even when the burst ended right above the
row it saw: a 1,026-row burst ending 100 days ago took 46 reads. After
an empty top-anchored segment the walk now gallops up from the known
row ((known + d, top], d x4, following each truncated read), then
bisects the bracket, with halving floored at the granularity (the Min
kept; a crowded gallop falls back to top-anchored halving). Burst: 18
reads, 0 repeated requests; dense tag stopped 2 days ago: 26 (gallop
alone crawled: 94). Walk fuzzed at 4,000 seeds locally (400 in-tree).
Docs: measured counts; CLAUDE.md gateway prerequisites list
RuntimeDb:EventReadMaxRows.
This commit is contained in:
Joseph Doherty
2026-10-12 17:37:32 -04:00
parent ccda27d220
commit d28e3e64f1
4 changed files with 119 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/ReverseWalk.cs
+70 -7
View File
@@ -156,10 +156,19 @@ internal static class ReverseWalk
var discoveryBudget = Math.Min(budget, Math.Max(pageBudget, DiscoveryRows));
// `known` > 0: a truncated read saw rows up to this timestamp (below segHi), and nothing is known of (known, segHi].
// Segments stay above it (bisecting the gap) until that stretch is read; then `knownWidth` (the density the
// truncated read showed) sizes the segment that ends on it.
// Segments stay above it until that stretch is read; then `knownWidth` (the density the truncated read showed)
// sizes the segment that ends on it. Searching the gap: first a top-anchored segment at that density (data running
// right up to segHi); if it is empty, the data stops somewhere above `known`, so GALLOP up from it — read
// (known + lowerBy, segHi] with lowerBy growing ×4, `known` following what each truncated read saw — until a read
// comes back empty (the top is now bracketed below its segLo), then BISECT that bracket. Any complete read holding
// rows ends the search. A burst that ended long before segHi is found in a few reads instead of bisecting the whole
// gap down to the millisecond, and a long dense stretch is crossed geometrically, not a budget at a time (#691.4
// review). `noGallop` falls back to plain top-anchored halving if a galloping read is crowded by rows before it.
var known = 0L;
var knownWidth = 0L;
var lowerBy = 0L;
var bisecting = false;
var noGallop = false;
var segHi = hiTicks;
// #691: the remainder probe (once per page, after two empty segments) and the empty stretches.
@@ -178,7 +187,20 @@ internal static class ReverseWalk
var probe = !probed && known == 0 && emptyRun >= 2 && segHi - stopTicks > width;
var w = probe ? segHi - stopTicks : width;
if (known > 0) w = Math.Min(w, Math.Max(1, (segHi - known) / 2));
var lowered = false;
var bisected = false;
if (known > 0)
{
var gap = segHi - known;
lowered = lowerBy > 0 && lowerBy < gap;
bisected = !lowered && (bisecting || lowerBy > 0) && gap > 2 * Math.Max(granularity, knownWidth);
// Never narrower than the request granularity when halving: requests are rounded outward to it, so a
// narrower half re-sends the same request (#691.4 review). Keep the Min: a truncated read crowded by rows
// before the segment must still shrink the segment.
w = lowered ? gap - lowerBy
: bisected ? gap / 2
: Math.Min(w, gap <= 2 * granularity ? gap : gap / 2);
}
var segLo = segHi - stopTicks <= w ? stopTicks : segHi - w;
var start = Utc(AlignDown(segLo, granularity));
var end = Utc(AlignUp(segHi + 1, granularity));
@@ -244,6 +266,14 @@ internal static class ReverseWalk
if (inner > 0 && lastInner == segHi)
{
width = resolution; // a tie cluster at the top of the segment: read that one timestamp
lowerBy = 0;
bisecting = false;
}
else if (inner > 0 && (lowered || bisected))
{
// Data reaches past this segment's bottom: move `known` up to what this read saw; a gallop goes further.
known = lastInner;
if (lowered) lowerBy = lowerBy > maxWidth / 4 ? maxWidth : lowerBy * 4;
}
else if (inner > 0)
{
@@ -255,10 +285,17 @@ internal static class ReverseWalk
known = lastInner;
knownWidth = Math.Max(resolution, aimed);
width = Math.Max(resolution, Math.Min(aimed, segHi - segLo - 1));
lowerBy = 0;
bisecting = false;
}
else
{
width = Math.Max(resolution, (segHi - segLo) / 4); // only rows before the segment: shrink it
// Only rows before the segment: shrink it — top-anchored from here on (a gallop would re-send the same
// crowded read).
width = Math.Max(resolution, Math.Min(width, segHi - segLo) / 4);
if (lowered || bisected) noGallop = true;
lowerBy = 0;
bisecting = false;
}
discovering = false;
@@ -283,9 +320,29 @@ internal static class ReverseWalk
var grown = width > maxWidth / 4 ? maxWidth : width * 4;
if (count == 0)
{
// ×4 growth may overshoot into dense data; above a known row, bisect the rest of the gap instead.
width = known > 0 ? maxWidth : grown;
discovering = true;
discovering = true; // ×4 growth may overshoot into dense data
if (known > 0)
{
// An empty top-anchored segment: gallop up from `known`. An empty galloping read: the top is bracketed
// in (known, segLo] — bisect it (an empty bisecting read keeps bisecting).
if (lowered || bisected)
{
lowerBy = 0;
bisecting = true;
}
else if (noGallop)
{
width = maxWidth; // halve the gap, top-anchored
}
else
{
lowerBy = Math.Max(knownWidth, granularity);
}
}
else
{
width = grown;
}
}
else
{
@@ -293,6 +350,9 @@ internal static class ReverseWalk
width = (long)Math.Clamp((double)(segHi - segLo) * Math.Max(1.0, pageBudget / 2.0) / count, resolution, grown);
discovering = false;
known = 0;
lowerBy = 0;
bisecting = false;
noGallop = false;
}
segHi = segLo;
@@ -300,6 +360,9 @@ internal static class ReverseWalk
{
// Everything above the newest row the truncated read saw is read: the next segment ends on it, at its density.
known = 0;
lowerBy = 0;
bisecting = false;
noGallop = false;
width = knownWidth;
discovering = false;
}