dohertj2/lmxopcua
1
0
Fork 0
Code Issues 14 Pull Requests Actions Packages Projects Releases Wiki Activity

fix(historian): #691.5 a reverse walk proves the rest empty once and carries it page to page

Browse Source 
The last pages of a deep reverse page-through on a forward-only backend
each walked down to 1970 one x4 growth step at a time (~14-16 reads,
~2.5 s live). After two empty segments the walk now asks once per page
for the whole remainder at the page budget (a truncated answer is
discarded and the walk carries on), and the stretch the page's last
complete read proved empty rides the continuation point
(HistoryContinuationState.EmptyFromTicks/EmptyToTicks, session-held, so
no versioning) so the next page steps over it. Fuzz now pages with the
walk's own hints and with true random empty stretches.
This commit is contained in:
Joseph Doherty
2026-10-12 16:32:00 -04:00
parent d33992034d
commit 186b7d5ea5
7 changed files with 222 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.Tests/NodeManagerHistoryReadPagingTests.cs
+32
View File
@@ -739,6 +739,38 @@ public sealed class NodeManagerHistoryReadPagingTests : IDisposable
await host.DisposeAsync();
}
/// <summary>
/// #691: the last pages of a deep reverse page-through no longer each walk down to 1970 (live: ~14–16 reads, ~2.5 s
/// per page). The stretch a page proved empty below its oldest row rides the continuation point, and a page that runs
/// out of rows asks once for the whole remainder — so every page here is a couple of reads.
/// </summary>
[Fact]
public async Task Raw_reverse_page_through_on_a_forward_only_backend_does_not_rewalk_to_the_epoch_on_each_last_page()
{
var source = new ForwardOnlyHistorianDataSource(MakeSeries(count: 25, stepSeconds: 1));
var (host, nm, nodeId) = await BootForwardOnlyAsync(source, "FwdDeep691");
nm.Clock = new FixedClock(Epoch.AddHours(1));
var collected = new List<double>();
byte[]? cp = null;
var page = 0;
do
{
source.ResetCounts();
var (values, error, next) = ReadRaw(nm, nodeId, DateTime.MinValue, Epoch.AddHours(1), max: 10, inboundCp: cp);
error.StatusCode.Code.ShouldBe(StatusCodes.Good);
collected.AddRange(values);
cp = next;
page++;
source.ReadCount.ShouldBeLessThanOrEqualTo(page == 1 ? 4 : 1, $"page {page}");
}
while (cp is not null && page < 10);
collected.ShouldBe(Enumerable.Range(0, 25).Reverse().Select(i => (double)i));
source.MinRequestedStart.ShouldBeGreaterThanOrEqualTo(DateTime.UnixEpoch);
await host.DisposeAsync();
}
/// <summary>#686 review: a reverse read from <c>StartTime = DateTime.MaxValue</c> walks back from now + 1 day, not from
/// year 9999.</summary>
[Fact]
tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.Tests/ReverseWalkTests.cs
+83 -14
View File
@@ -46,23 +46,44 @@ public sealed class ReverseWalkTests
1 => new ReverseWalkHint(rng.Next(1, 10)),
_ => new ReverseWalkHint((long)(rng.NextDouble() * TimeSpan.TicksPerHour)),
};
var expected = series.Where(s => Ts(s) > Math.Max(lo, DateTime.UnixEpoch.Ticks) && Ts(s) <= hi)
.Select((s, i) => (s, i)).OrderByDescending(x => Ts(x.s)).ThenByDescending(x => x.i)
.Take((int)want).Select(x => (double)x.s.Value!).ToList();
try
if (rng.Next(2) == 0)
{
var page = await ReverseRawReader.ReadAsync(source, "T", new DateTime(hi, DateTimeKind.Utc),
new DateTime(lo, DateTimeKind.Utc), want, tieBudget, T0.AddDays(1), hint, Ct);
var got = page.Samples.Select(s => (double)s.Value!).ToList();
if (!got.SequenceEqual(expected))
failures.Add($"seed {seed}: got [{string.Join(",", got)}] expected [{string.Join(",", expected)}]");
if (page.Hint.WidthTicks <= 0) failures.Add($"seed {seed}: no width carried");
// A TRUE empty stretch (#691): between two adjacent distinct timestamps, or from the floor to the first row.
var k = rng.Next(series.Count);
var from = k == 0 ? (rng.Next(2) == 0 ? 0 : DateTime.UnixEpoch.Ticks) : Ts(series[k - 1]);
var to = Ts(series[k]);
if (to > from) hint = hint with { EmptyFromTicks = from, EmptyToTicks = to };
}
catch (HistoryReadLimitExceededException)
// Page on: each next page resumes just below the previous page's oldest timestamp with the hint it returned —
// so the hints the walk itself produces (width, proven-empty stretch) are exercised too.
for (var pageNo = 0; pageNo < 6 && hi > Math.Max(lo, DateTime.UnixEpoch.Ticks); pageNo++)
{
if (maxCluster <= tieBudget) failures.Add($"seed {seed}: refused with no cluster over {tieBudget} (max {maxCluster})");
var expected = series.Where(s => Ts(s) > Math.Max(lo, DateTime.UnixEpoch.Ticks) && Ts(s) <= hi)
.Select((s, i) => (s, i)).OrderByDescending(x => Ts(x.s)).ThenByDescending(x => x.i)
.Take((int)want).Select(x => (double)x.s.Value!).ToList();
try
{
var page = await ReverseRawReader.ReadAsync(source, "T", new DateTime(hi, DateTimeKind.Utc),
new DateTime(lo, DateTimeKind.Utc), want, tieBudget, T0.AddDays(1), hint, Ct);
var got = page.Samples.Select(s => (double)s.Value!).ToList();
if (!got.SequenceEqual(expected))
{
failures.Add($"seed {seed} page {pageNo}: got [{string.Join(",", got)}] expected [{string.Join(",", expected)}]");
break;
}
if (page.Hint.WidthTicks <= 0) failures.Add($"seed {seed}: no width carried");
if (got.Count < want) break;
hint = page.Hint;
hi = Ts(page.Samples[^1]) - 1;
}
catch (HistoryReadLimitExceededException)
{
if (maxCluster <= tieBudget) failures.Add($"seed {seed}: refused with no cluster over {tieBudget} (max {maxCluster})");
break;
}
}
if (source.MaxCap > 2L * Math.Max(Math.Max(tieBudget, ReverseWalk.DiscoveryRows), 2 * want + 16) + 4)
@@ -116,6 +137,54 @@ public sealed class ReverseWalkTests
source.RowsReturned.ShouldBeLessThanOrEqualTo(25 * (ReverseWalk.DiscoveryRows + 2));
}
/// <summary>
/// #691: the end of a walk proves the rest of the window empty in about one read — after two empty segments the walk
/// asks once for the whole remainder down to the floor — instead of one read per ×4 growth step to 1970 (~14 reads).
/// </summary>
[Fact]
public async Task Running_out_of_rows_proves_the_rest_of_the_window_empty_in_a_few_reads()
{
var series = Enumerable.Range(0, 5).Select(i => At(T0.Ticks + i * TimeSpan.TicksPerSecond, i)).ToList();
var source = new AvevaShapedSource(series);
var page = await ReverseRawReader.ReadAsync(source, "T", T0.AddSeconds(4), DateTime.MinValue, 10, 65536, T0,
default(ReverseWalkHint), Ct);
page.Samples.Select(s => (double)s.Value!).ShouldBe([4d, 3d, 2d, 1d, 0d]);
source.Reads.ShouldBeLessThanOrEqualTo(4, "one segment holding the rows, two empty ones, then the remainder at once");
}
/// <summary>
/// #691: a page's last complete read proves the stretch below its oldest row empty, and the continuation carries it:
/// the next page steps straight over it to the floor instead of walking it again.
/// </summary>
[Fact]
public async Task A_resumed_page_steps_over_the_stretch_the_previous_page_proved_empty()
{
var series = Enumerable.Range(0, 25).Select(i => At(T0.Ticks + i * TimeSpan.TicksPerSecond, i)).ToList();
var source = new AvevaShapedSource(series);
var now = T0.AddHours(1);
// Page 1: an hour of nothing above the rows; the remainder probe reads all 25 and proves (1970, row 0) empty.
var p1 = await ReverseRawReader.ReadAsync(source, "T", now, DateTime.MinValue, 10, 65536, now, default(ReverseWalkHint), Ct);
p1.Samples.Select(s => (double)s.Value!).ShouldBe(Enumerable.Range(15, 10).Reverse().Select(i => (double)i));
p1.Hint.HasEmptyStretch.ShouldBeTrue();
p1.Hint.EmptyFromTicks.ShouldBeLessThanOrEqualTo(DateTime.UnixEpoch.Ticks);
p1.Hint.EmptyToTicks.ShouldBe(Ts(series[0]));
var hint = p1.Hint;
var hi = p1.Samples[^1].SourceTimestampUtc!.Value.AddTicks(-1);
for (var page = 2; page <= 3; page++)
{
source.Reset();
var p = await ReverseRawReader.ReadAsync(source, "T", hi, DateTime.MinValue, 10, 65536, now, hint, Ct);
p.Samples.Count.ShouldBe(page == 2 ? 10 : 5);
source.Reads.ShouldBe(1, $"page {page} reads its rows and steps over the proven-empty stretch to the floor");
hint = p.Hint;
hi = p.Samples[^1].SourceTimestampUtc!.Value.AddTicks(-1);
}
}
private static long Ts(DataValueSnapshot s) => s.SourceTimestampUtc!.Value.Ticks;
/// <summary>Random ascending series: bursts, gaps, sub-millisecond stamps and tie clusters.</summary>