Files
lmxopcua/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.Tests/NodeManagerHistoryReadPageFillTests.cs
T
Joseph Doherty e04e1bf127 fix(historian): #692 review — a drained tie cluster never resumes onto the exclusive EndTime
F1 (pre-existing): SliceTieCluster continued a forward read while next <= endUtc, but the
forward window is end-exclusive; an oversized cluster one step below EndTime resumed AT
EndTime, the resume became a start == end point read and returned the EndTime samples as
data (microsecond: EndTime 8.1 us truncated to 8.0). Continue only while next is strictly
before EndTime truncated to the backend resolution.

Tests: A_tie_cluster_just_below_the_end_time_never_resumes_onto_the_end_time (native +
microsecond; both fail on the old rule) and
A_cursor_inside_a_tie_cluster_over_fetches_without_a_normal_read_first (pins the
WithinTieCluster shortcut by per-page read count; fails with the shortcut disabled).
Docs: the oversized-cluster exception to 'every page full / no lookahead => last' (F2),
and the N + skip + 1 strict-ceiling edge note referencing #694 (F4).
2026-10-13 01:02:12 -04:00

439 lines
21 KiB
C#

using Opc.Ua;
using Opc.Ua.Server;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using HistorianRead = ZB.MOM.WW.OtOpcUa.Core.Abstractions.HistoryReadResult;
using SdkHistoryReadResult = Opc.Ua.HistoryReadResult;
namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.Tests;
/// <summary>
/// Gitea #692: a paged HistoryRead-Raw fills every page. With <c>NumValuesPerNode = 1</c> a resumed page used to re-read
/// only the boundary sample, trim it, take the empty page for a stalled tie cluster and over-fetch
/// <c>MaxTieClusterOverfetch + 1</c> rows to step one tick — an EMPTY page (GoodNoData + continuation point) between every
/// value (8 values = 17 pages / 25 backend reads), on every backend, forward and reverse, with or without ReturnBounds.
/// Larger pages lost one value per resumed page the same way. Every page must now carry exactly N values (bounds count),
/// the over-fetch must fire only for a genuine tie cluster, and the read must not end with an empty trailing page when the
/// last data page already knew it was the last.
/// </summary>
public sealed class NodeManagerHistoryReadPageFillTests(BoundsServerFixture fx) : IClassFixture<BoundsServerFixture>
{
private static readonly DateTime T0 = new(2026, 10, 12, 19, 0, 0, DateTimeKind.Utc);
public enum Shape { Native, ForwardOnly, Microsecond }
public enum Direction { Forward, Reverse, EndOnly }
/// <summary>The live-gate repro (#692): 8 values one second apart, read over [19:00:00.1, 19:00:10.5).</summary>
private static DataValueSnapshot[] EightValues() =>
Enumerable.Range(1, 8).Select(i => Sample(i, T0.AddSeconds(i))).ToArray();
private static DataValueSnapshot Sample(double v, DateTime t) => new(v, StatusCodes.Good, t, t);
private static (DateTime Start, DateTime End) Window(Direction direction) => direction switch
{
Direction.Forward => (T0.AddMilliseconds(100), T0.AddMilliseconds(10_500)),
Direction.Reverse => (T0.AddMilliseconds(10_500), T0.AddMilliseconds(100)),
_ => (DateTime.MinValue, T0.AddMilliseconds(10_500)),
};
public static TheoryData<Shape, Direction, bool> AllShapes()
{
var data = new TheoryData<Shape, Direction, bool>();
foreach (var s in Enum.GetValues<Shape>())
foreach (var d in Enum.GetValues<Direction>())
foreach (var b in new[] { false, true })
data.Add(s, d, b);
return data;
}
/// <summary>
/// N = 1 paged to completion: every page carries exactly one value, the pages concatenate to the one-page answer, the
/// read never asks the backend for a tie-cluster over-fetch, and the backend read count stays proportional to the
/// pages (native backends: one data read per page plus the bound lookups).
/// </summary>
[Theory]
[MemberData(nameof(AllShapes))]
public async Task One_value_per_page_returns_one_value_on_every_page(Shape shape, Direction direction, bool bounds)
{
var source = Source(shape, EightValues());
var (start, end) = Window(direction);
var expected = await SinglePageAsync(source, start, end, bounds);
expected.Count.ShouldBeGreaterThanOrEqualTo(8);
source.ResetCounts();
var pages = await DrainPagesAsync(source, start, end, 1, bounds);
Render(pages.SelectMany(p => p)).ShouldBe(Render(expected));
pages.Select(p => p.Count).ShouldAllBe(c => c == 1, $"pages: {string.Join(",", pages.Select(p => p.Count))}");
pages.Count.ShouldBe(expected.Count);
source.MaxCapRequested.ShouldBeLessThanOrEqualTo((uint)(ReverseWalk.DiscoveryRows + 2),
"no tie-cluster over-fetch for a series without ties");
if (shape != Shape.ForwardOnly)
source.Reads.ShouldBeLessThanOrEqualTo(pages.Count + 4, $"{source.Reads} backend reads for {pages.Count} pages");
}
/// <summary>The live-gate numbers, pinned: 8 values at N = 1 on a native backend is 8 pages and 8 backend reads (was 17 / 25).</summary>
[Fact]
public async Task Eight_values_at_one_per_page_cost_eight_pages_and_eight_reads()
{
var source = Source(Shape.Native, EightValues());
var (start, end) = Window(Direction.Forward);
var pages = await DrainPagesAsync(source, start, end, 1, bounds: false);
pages.Count.ShouldBe(8, $"{source.Reads} reads, {source.RequestedRows} rows requested");
source.Reads.ShouldBe(8);
source.RequestedRows.ShouldBeLessThanOrEqualTo(8 * 3);
}
/// <summary>A series whose tie clusters straddle the page boundaries at every page size.</summary>
private static DataValueSnapshot[] TiedValues()
{
var seconds = new[] { 1, 2, 2, 3, 3, 3, 4, 5, 5, 6, 7, 7, 7, 8 };
return seconds.Select((s, i) => Sample(i, T0.AddSeconds(s))).ToArray();
}
public static TheoryData<Shape, Direction, bool, uint> TiedShapes()
{
var data = new TheoryData<Shape, Direction, bool, uint>();
foreach (var s in Enum.GetValues<Shape>())
foreach (var d in Enum.GetValues<Direction>())
foreach (var b in new[] { false, true })
foreach (var n in new uint[] { 1, 2, 3 })
data.Add(s, d, b, n);
return data;
}
/// <summary>
/// #669 tie semantics at the page boundary are kept: ties that straddle a page boundary come back once each, in the
/// read's order, at N = 1/2/3 — and every page that has a continuation point is full, no page is empty.
/// </summary>
[Theory]
[MemberData(nameof(TiedShapes))]
public async Task Tie_clusters_straddling_page_boundaries_page_without_loss_duplication_or_short_pages(
Shape shape, Direction direction, bool bounds, uint n)
{
var source = Source(shape, TiedValues());
var (start, end) = Window(direction);
var expected = await SinglePageAsync(source, start, end, bounds);
var pages = await DrainPagesAsync(source, start, end, n, bounds);
Render(pages.SelectMany(p => p)).ShouldBe(Render(expected));
pages.ShouldAllBe(p => p.Count > 0, "no empty page");
for (var i = 0; i < pages.Count - 1; i++)
pages[i].Count.ShouldBe((int)n, $"page {i} of {pages.Count}: {string.Join(",", pages.Select(p => p.Count))}");
}
/// <summary>
/// A read whose total is an exact multiple of the page size ends on its last full page: the lookahead row tells the
/// server there is nothing more, so there is no trailing 0-value page (the #692 gate's reverse N = 2 observation).
/// </summary>
[Theory]
[InlineData(Shape.Native, Direction.Forward)]
[InlineData(Shape.Native, Direction.Reverse)]
[InlineData(Shape.ForwardOnly, Direction.Forward)]
[InlineData(Shape.ForwardOnly, Direction.Reverse)]
[InlineData(Shape.Microsecond, Direction.Reverse)]
public async Task An_exact_multiple_ends_on_its_last_full_page(Shape shape, Direction direction)
{
var (start, end) = Window(direction);
var pages = await DrainPagesAsync(Source(shape, EightValues()), start, end, 2, bounds: false);
pages.Select(p => p.Count).ShouldBe([2, 2, 2, 2]);
}
/// <summary>
/// #692 review F1: an oversized tie cluster one step below EndTime used to resume AT EndTime (the drain check was
/// <c>next &lt;= end</c>, but the forward window is end-exclusive), so the next read was a <c>start == end</c> point
/// read that returned the samples AT EndTime as data. Paged must equal unpaged — neither holds the EndTime samples.
/// The microsecond shape truncates EndTime (8.1 µs ⇒ 8.0 µs) before the same thing happens.
/// </summary>
[Theory]
[InlineData(Shape.Native, 1L, 8L)]
[InlineData(Shape.Microsecond, 10L, 81L)]
public async Task A_tie_cluster_just_below_the_end_time_never_resumes_onto_the_end_time(Shape shape, long unit, long endTicks)
{
var series = new List<DataValueSnapshot>();
var v = 0;
foreach (var (at, count) in new[] { (5L, 5), (6L, 2), (7L, 3), (8L, 3) })
for (var i = 0; i < count; i++)
series.Add(Sample(v++, T0.AddTicks(at * unit)));
var source = Source(shape, series);
var (start, end) = (T0.AddSeconds(-3), T0.AddTicks(endTicks));
var unpaged = await SinglePageAsync(source, start, end, bounds: false);
unpaged.Count.ShouldBe(10, "the samples at the (exclusive) end time are not data");
var pages = await DrainPagesAsync(source, start, end, 1, bounds: false);
Render(pages.SelectMany(p => p)).ShouldBe(Render(unpaged));
}
/// <summary>
/// #692: a cursor left inside an oversized tie cluster is flagged, and its next page goes straight to the cluster
/// over-fetch — one backend read — instead of first re-reading <c>N + skip + 1</c> rows it knows are ties.
/// </summary>
[Fact]
public async Task A_cursor_inside_a_tie_cluster_over_fetches_without_a_normal_read_first()
{
// a @1s, five ties @2s, b @3s; N = 1.
var series = new List<DataValueSnapshot> { Sample(0, T0.AddSeconds(1)) };
for (var i = 1; i <= 5; i++) series.Add(Sample(i, T0.AddSeconds(2)));
series.Add(Sample(6, T0.AddSeconds(3)));
var source = Source(Shape.Native, series);
var (start, end) = (T0, T0.AddSeconds(10));
var readsPerPage = new List<int>();
var all = new List<DataValue>();
byte[]? cp = null;
do
{
source.ResetCounts();
var (values, error, next) = await ReadAsync(source, start, end, 1, bounds: false, cp);
error.StatusCode.Code.ShouldBe(StatusCodes.Good);
values.Count.ShouldBe(1);
all.AddRange(values);
readsPerPage.Add(source.Reads);
cp = next;
readsPerPage.Count.ShouldBeLessThan(20);
}
while (cp is not null);
all.Select(x => (double)x.Value).ShouldBe([0d, 1d, 2d, 3d, 4d, 5d, 6d]);
// Page 1 [a], page 2 [t1]: one read each. Page 3 [t2]: the read stalls on the cluster, then the over-fetch (2).
// Pages 4 [t3], 5 [t4], 6 [t5]: the flagged cursor over-fetches directly (1 each). Page 7 [b]: one read.
readsPerPage.ShouldBe([1, 1, 2, 1, 1, 1, 1]);
}
/// <summary>
/// #692 (low): a one-value bounded read whose start bound is not found asks whether the read holds anything. When the
/// answer is "only the end bound", the bound it just found rides the continuation point — the next page is served
/// without reading the backend again.
/// </summary>
[Theory]
[InlineData(Shape.Native, Direction.Forward)]
[InlineData(Shape.Native, Direction.Reverse)]
[InlineData(Shape.ForwardOnly, Direction.Forward)]
[InlineData(Shape.ForwardOnly, Direction.Reverse)]
[InlineData(Shape.Microsecond, Direction.Forward)]
public async Task A_start_bound_only_page_carries_the_end_bound_it_found(Shape shape, Direction direction)
{
// Forward [5s, 7s): nothing at/before 5s, nothing inside, the end bound at 9s. Reverse (5s ← 7s]: mirrored.
var series = direction == Direction.Forward ? new[] { Sample(9, T0.AddSeconds(9)) } : [Sample(3, T0.AddSeconds(3))];
var source = Source(shape, series);
var (start, end) = direction == Direction.Forward ? (T0.AddSeconds(5), T0.AddSeconds(7)) : (T0.AddSeconds(7), T0.AddSeconds(5));
var (first, _, cp) = await ReadAsync(source, start, end, 1, bounds: true, cp: null);
first.Single().StatusCode.Code.ShouldBe(StatusCodes.BadBoundNotFound);
cp.ShouldNotBeNull();
source.ResetCounts();
var (second, _, cp2) = await ReadAsync(source, start, end, 1, bounds: true, cp: cp);
second.Single().Value.ShouldBe(direction == Direction.Forward ? 9d : 3d);
cp2.ShouldBeNull();
source.Reads.ShouldBe(0, "the end bound was found by the first page and carried");
}
/// <summary>
/// #692 (low): a one-value bounded reverse read on a forward-only backend walks back to answer "is there any data";
/// the walk hint that walk computed rides the continuation point (it used to be dropped).
/// </summary>
[Fact]
public async Task A_start_bound_only_page_carries_the_walk_hint_it_computed()
{
var source = Source(Shape.ForwardOnly, EightValues());
var store = new RecordingStore();
await fx.Gate.WaitAsync(TestContext.Current.CancellationToken);
try
{
fx.Nm.HistoryContinuationStore = store;
fx.Nm.HistorianDataSource = source;
// Reverse (0.1s ← 30s]: no sample at or after 30s ⇒ the start bound is a placeholder; the data is below.
var (values, _, cp) = Read(T0.AddSeconds(30), T0.AddMilliseconds(100), 1, bounds: true, cp: null);
values.Single().StatusCode.Code.ShouldBe(StatusCodes.BadBoundNotFound);
cp.ShouldNotBeNull();
store.Saved.Last().WalkHint.WidthTicks.ShouldBeGreaterThan(0);
}
finally
{
fx.Nm.HistoryContinuationStore = new InMemoryHistoryContinuationStore();
fx.Gate.Release();
}
}
/// <summary>
/// #692 (low): only a SERVER-synthesised bound is a placeholder. A stored row that happens to carry
/// <c>Bad_BoundNotFound</c> with a null value (a client can write one through HistoryUpdate) is data — it used to make
/// the read look "all placeholders" and come back GoodNoData with no values.
/// </summary>
[Fact]
public async Task A_stored_row_with_the_bound_not_found_status_is_data_not_a_placeholder()
{
var stored = new DataValueSnapshot(null, StatusCodes.BadBoundNotFound, T0.AddSeconds(6), T0.AddSeconds(6));
HistoryBounds.IsPlaceholder(stored).ShouldBeFalse();
HistoryBounds.IsPlaceholder(HistoryBounds.NotFound(T0)).ShouldBeTrue();
var (values, _, cp) = await ReadAsync(Source(Shape.Native, [stored]), T0.AddSeconds(5), T0.AddSeconds(7), 0, bounds: true, cp: null);
cp.ShouldBeNull();
values.Count.ShouldBe(3, "start placeholder, the stored row, end placeholder");
values[1].SourceTimestamp.ShouldBe(T0.AddSeconds(6));
}
// --- helpers --------------------------------------------------------------------------------
private static string Render(IEnumerable<DataValue> values) =>
string.Join(" | ", values.Select(v => $"{v.SourceTimestamp:HH:mm:ss.fffffff}={v.Value ?? "null"}/{v.StatusCode.Code:X8}"));
private async Task<List<DataValue>> SinglePageAsync(CountingSource source, DateTime start, DateTime end, bool bounds)
{
var (values, error, cp) = await ReadAsync(source, start, end, 10_000, bounds, cp: null);
error.StatusCode.Code.ShouldBe(StatusCodes.Good);
cp.ShouldBeNull();
return values.ToList();
}
private async Task<List<IReadOnlyList<DataValue>>> DrainPagesAsync(
CountingSource source, DateTime start, DateTime end, uint max, bool bounds)
{
var pages = new List<IReadOnlyList<DataValue>>();
byte[]? cp = null;
do
{
var (values, error, next) = await ReadAsync(source, start, end, max, bounds, cp);
error.StatusCode.Code.ShouldBe(StatusCodes.Good);
pages.Add(values);
cp = next;
pages.Count.ShouldBeLessThan(80, "paging must terminate");
}
while (cp is not null);
return pages;
}
private async Task<(IReadOnlyList<DataValue> Values, ServiceResult Error, byte[]? Cp)> ReadAsync(
IHistorianDataSource source, DateTime start, DateTime end, uint max, bool bounds, byte[]? cp)
{
await fx.Gate.WaitAsync(TestContext.Current.CancellationToken);
try
{
fx.Nm.HistorianDataSource = source;
return Read(start, end, max, bounds, cp);
}
finally { fx.Gate.Release(); }
}
private (IReadOnlyList<DataValue> Values, ServiceResult Error, byte[]? Cp) Read(
DateTime start, DateTime end, uint max, bool bounds, byte[]? cp)
{
var context = new OperationContext(new RequestHeader(), secureChannelContext: null, RequestType.HistoryRead, identity: null);
var details = new ReadRawModifiedDetails
{
StartTime = start, EndTime = end, NumValuesPerNode = max, ReturnBounds = bounds, IsReadModified = false,
};
var nodesToRead = new List<HistoryReadValueId> { new() { NodeId = fx.NodeId, ContinuationPoint = cp } };
var results = new List<SdkHistoryReadResult> { null! };
var errors = new List<ServiceResult> { null! };
fx.Nm.HistoryRead(context, details, TimestampsToReturn.Both, releaseContinuationPoints: false, nodesToRead, results, errors);
var values = new List<DataValue>();
if (results[0]?.HistoryData is { } ho && !ExtensionObject.IsNull(ho))
values.AddRange(((HistoryData)ExtensionObject.ToEncodeable(ho)).DataValues);
return (values, errors[0], results[0]?.ContinuationPoint);
}
private static CountingSource Source(Shape shape, IReadOnlyList<DataValueSnapshot> series) => shape switch
{
Shape.Native => new CountingSource(series, reverse: true, resolution: 1),
Shape.ForwardOnly => new CountingSource(series, reverse: false, resolution: 1),
_ => new CountingSource(series, reverse: true, resolution: 10),
};
/// <summary>Records every saved cursor (to inspect what rides a continuation point).</summary>
private sealed class RecordingStore : IHistoryContinuationStore
{
private readonly InMemoryHistoryContinuationStore _inner = new();
public List<HistoryContinuationState> Saved { get; } = [];
public byte[]? Save(ISession? session, HistoryContinuationState state)
{
Saved.Add(state);
return _inner.Save(session, state);
}
public HistoryContinuationState? TryTake(ISession? session, byte[] continuationPoint) => _inner.TryTake(session, continuationPoint);
public void Release(ISession? session, byte[] continuationPoint) => _inner.Release(session, continuationPoint);
}
/// <summary>
/// A series-backed historian that counts what it is asked for. <c>reverse</c>: serves <c>start &gt; end</c> newest-first
/// (SqlServer/QuestDB); otherwise refuses it (the Gateway — the server walks back with forward reads). Forward reads
/// are <c>[start, end)</c>, a point read the instant; <c>resolution</c> &gt; 1 truncates the window bounds (QuestDB µs).
/// </summary>
private sealed class CountingSource(IReadOnlyList<DataValueSnapshot> series, bool reverse, long resolution) : IHistorianDataSource
{
private readonly Lock _gate = new();
public int Reads { get; private set; }
public long RequestedRows { get; private set; }
public uint MaxCapRequested { get; private set; }
public void ResetCounts()
{
lock (_gate) { Reads = 0; RequestedRows = 0; MaxCapRequested = 0; }
}
public bool SupportsReverseRaw => reverse;
public long TimestampResolutionTicks => resolution;
private DateTime Trunc(DateTime t) => new(t.Ticks - t.Ticks % resolution, DateTimeKind.Utc);
public Task<HistorianRead> ReadRawAsync(
string fullReference, DateTime startUtc, DateTime endUtc, uint maxValuesPerNode, CancellationToken cancellationToken)
{
lock (_gate)
{
Reads++;
RequestedRows += maxValuesPerNode;
if (maxValuesPerNode > MaxCapRequested) MaxCapRequested = maxValuesPerNode;
}
static DateTime Ts(DataValueSnapshot s) => s.SourceTimestampUtc ?? DateTime.MinValue;
startUtc = Trunc(startUtc);
endUtc = Trunc(endUtc);
List<DataValueSnapshot> window;
if (startUtc > endUtc)
{
if (!reverse) throw new InvalidOperationException("start_time must be less than or equal to end_time. (InvalidArgument)");
window = series.Where(s => Ts(s) > endUtc && Ts(s) <= startUtc).Reverse().ToList();
}
else if (startUtc == endUtc) window = series.Where(s => Ts(s) == startUtc).ToList();
else window = series.Where(s => Ts(s) >= startUtc && Ts(s) < endUtc).ToList();
var page = maxValuesPerNode == 0 ? window : window.Take((int)maxValuesPerNode).ToList();
return Task.FromResult(new HistorianRead(page, null));
}
public Task<HistorianRead> ReadProcessedAsync(
string fullReference, DateTime startUtc, DateTime endUtc, TimeSpan interval,
HistoryAggregateType aggregate, CancellationToken cancellationToken) =>
Task.FromResult(new HistorianRead(Array.Empty<DataValueSnapshot>(), null));
public Task<HistorianRead> ReadAtTimeAsync(
string fullReference, IReadOnlyList<DateTime> timestampsUtc, CancellationToken cancellationToken) =>
Task.FromResult(new HistorianRead(Array.Empty<DataValueSnapshot>(), null));
public Task<HistoricalEventsResult> ReadEventsAsync(
string? sourceName, DateTime startUtc, DateTime endUtc, int maxEvents, CancellationToken cancellationToken) =>
Task.FromResult(new HistoricalEventsResult(Array.Empty<HistoricalEvent>(), null));
public HistorianHealthSnapshot GetHealthSnapshot() => NullHistorianDataSource.Instance.GetHealthSnapshot();
public void Dispose() { }
}
}