using Opc.Ua; using Shouldly; using Xunit; using ZB.MOM.WW.OtOpcUa.Core.Abstractions; using HistorianRead = ZB.MOM.WW.OtOpcUa.Core.Abstractions.HistoryReadResult; namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.Tests; /// /// Gitea #689 / #691: the reverse walk-back ( through ) driven /// directly, without a server: a randomized exactness check against a brute-force answer, and the sizing properties. /// public sealed class ReverseWalkTests { private static CancellationToken Ct => TestContext.Current.CancellationToken; private static readonly DateTime T0 = new(2026, 10, 12, 18, 0, 0, DateTimeKind.Utc); private static DataValueSnapshot At(long ticks, double v) => new(v, StatusCodes.Good, new DateTime(ticks, DateTimeKind.Utc), new DateTime(ticks, DateTimeKind.Utc)); /// /// A randomized walk over AVEVA-shaped forward reads (inclusive end, a fabricated row at the query start — optionally /// stamped at the start rounded UP to the millisecond — counted against the cap): every page is exactly the brute-force /// newest-first answer over (lo, hi], ties in the backend's forward order reversed, whatever the carried width /// — and a timestamp over the tie budget is the only refusal. /// [Fact] public async Task A_walked_page_is_exactly_the_newest_N_over_the_window_on_random_data() { var failures = new List(); for (var seed = 0; seed < 400; seed++) { var rng = new Random(seed); var series = RandomSeries(rng, out _); // The refusal unit is one millisecond (#691.4 re-review): the most rows inside any 1 ms window. var maxCluster = 0; for (int i = 0, j = 0; i < series.Count; i++) { while (Ts(series[i]) - Ts(series[j]) >= TimeSpan.TicksPerMillisecond) j++; maxCluster = Math.Max(maxCluster, i - j + 1); } var source = new AvevaShapedSource(series, roundUp: rng.Next(2) == 0); var tieBudget = rng.Next(4) switch { 0 => 1, 1 => 3, 2 => 8, _ => 65536 }; var want = (uint)rng.Next(1, 40); var span = Ts(series[^1]) - Ts(series[0]); var hi = Ts(series[0]) + (long)(rng.NextDouble() * 1.2 * span) + rng.Next(0, 3); var lo = Ts(series[0]) - TimeSpan.TicksPerSecond + (long)(rng.NextDouble() * (hi - Ts(series[0])) * 0.8); if (rng.Next(5) == 0) lo = 0; // an end-only read var hint = rng.Next(3) switch { 0 => default, 1 => new ReverseWalkHint(rng.Next(1, 10)), _ => new ReverseWalkHint((long)(rng.NextDouble() * TimeSpan.TicksPerHour)), }; if (rng.Next(2) == 0) { // A TRUE empty stretch (#691): between two adjacent distinct timestamps, or from the floor to the first row. var k = rng.Next(series.Count); var from = k == 0 ? (rng.Next(2) == 0 ? 0 : DateTime.UnixEpoch.Ticks) : Ts(series[k - 1]); var to = Ts(series[k]); if (to > from) hint = hint with { EmptyFromTicks = from, EmptyToTicks = to }; } // Page on: each next page resumes just below the previous page's oldest timestamp with the hint it returned — // so the hints the walk itself produces (width, proven-empty stretch) are exercised too. for (var pageNo = 0; pageNo < 6 && hi > Math.Max(lo, DateTime.UnixEpoch.Ticks); pageNo++) { var expected = series.Where(s => Ts(s) > Math.Max(lo, DateTime.UnixEpoch.Ticks) && Ts(s) <= hi) .Select((s, i) => (s, i)).OrderByDescending(x => Ts(x.s)).ThenByDescending(x => x.i) .Take((int)want).Select(x => (double)x.s.Value!).ToList(); try { var page = await ReverseRawReader.ReadAsync(source, "T", new DateTime(hi, DateTimeKind.Utc), new DateTime(lo, DateTimeKind.Utc), want, tieBudget, T0.AddDays(1), hint, Ct); var got = page.Samples.Select(s => (double)s.Value!).ToList(); if (!got.SequenceEqual(expected)) { failures.Add($"seed {seed} page {pageNo}: got [{string.Join(",", got)}] expected [{string.Join(",", expected)}]"); break; } if (page.Hint.WidthTicks <= 0) failures.Add($"seed {seed}: no width carried"); if (got.Count < want) break; hint = page.Hint; hi = Ts(page.Samples[^1]) - 1; } catch (HistoryReadLimitExceededException) { if (maxCluster <= tieBudget) failures.Add($"seed {seed}: refused with no millisecond over {tieBudget} rows (max {maxCluster})"); break; } } if (source.MaxCap > 2L * Math.Max(Math.Max(tieBudget, ReverseWalk.DiscoveryRows), 2 * want + 16) + 4) failures.Add($"seed {seed}: asked for {source.MaxCap} rows"); if (source.BadRequests > 0) failures.Add($"seed {seed}: {source.BadRequests} invalid requests"); } failures.ShouldBeEmpty(); } /// /// #689: the carried width sizes a resumed page from N, not from the tie budget — the live symptom was 32,770 rows /// per page for N = 3 on a 1 Hz tag once the width had grown over a few pages. /// [Fact] public async Task A_resumed_page_on_a_dense_tag_reads_about_N_rows_with_the_default_tie_budget() { var series = Enumerable.Range(0, 200_000).Select(i => At(T0.Ticks + i * TimeSpan.TicksPerSecond, i)).ToList(); var source = new AvevaShapedSource(series); var hi = new DateTime(Ts(series[^1]), DateTimeKind.Utc); var hint = default(ReverseWalkHint); for (var page = 1; page <= 30; page++) { source.Reset(); var result = await ReverseRawReader.ReadAsync(source, "T", hi, DateTime.MinValue, 3, 65536, hi.AddHours(1), hint, Ct); result.Samples.Count.ShouldBe(3); hint = result.Hint; hi = result.Samples[^1].SourceTimestampUtc!.Value.AddTicks(-1); if (page > 1) source.RowsReturned.ShouldBeLessThanOrEqualTo(30, $"page {page}"); } } /// /// #689: a dense tag that stopped long ago. The walk's ×4 growth across the empty stretch overshoots into far more /// rows than one read takes; the truncated read only shows the OLDEST of them, so the walk must not keep re-entering /// that stretch — it searches the gap above the newest row it saw (gallop up, then bisect). Measured: 26 reads. /// [Fact] public async Task A_dense_tag_below_a_long_empty_stretch_is_found_in_a_bounded_number_of_reads() { // 10 Hz for 3 hours, ending two days before "now". var start = T0.AddDays(-2).AddHours(-3); var series = Enumerable.Range(0, 108_000).Select(i => At(start.Ticks + i * TimeSpan.TicksPerMillisecond * 100, i)).ToList(); var source = new AvevaShapedSource(series); var page = await ReverseRawReader.ReadAsync(source, "T", T0, DateTime.MinValue, 3, 65536, T0, default(ReverseWalkHint), Ct); page.Samples.Select(s => (double)s.Value!).ShouldBe([107_999d, 107_998d, 107_997d]); source.Reads.ShouldBeLessThanOrEqualTo(30); source.RowsReturned.ShouldBeLessThanOrEqualTo(30 * (ReverseWalk.DiscoveryRows + 2)); } /// /// #691: the end of a walk proves the rest of the window empty in about one read — after two empty segments the walk /// asks once for the whole remainder down to the floor — instead of one read per ×4 growth step to 1970 (~14 reads). /// [Fact] public async Task Running_out_of_rows_proves_the_rest_of_the_window_empty_in_a_few_reads() { var series = Enumerable.Range(0, 5).Select(i => At(T0.Ticks + i * TimeSpan.TicksPerSecond, i)).ToList(); var source = new AvevaShapedSource(series); var page = await ReverseRawReader.ReadAsync(source, "T", T0.AddSeconds(4), DateTime.MinValue, 10, 65536, T0, default(ReverseWalkHint), Ct); page.Samples.Select(s => (double)s.Value!).ShouldBe([4d, 3d, 2d, 1d, 0d]); source.Reads.ShouldBeLessThanOrEqualTo(4, "one segment holding the rows, two empty ones, then the remainder at once"); } /// /// #691: a page's last complete read proves the stretch below its oldest row empty, and the continuation carries it: /// the next page steps straight over it to the floor instead of walking it again. /// [Fact] public async Task A_resumed_page_steps_over_the_stretch_the_previous_page_proved_empty() { var series = Enumerable.Range(0, 25).Select(i => At(T0.Ticks + i * TimeSpan.TicksPerSecond, i)).ToList(); var source = new AvevaShapedSource(series); var now = T0.AddHours(1); // Page 1: an hour of nothing above the rows; the remainder probe reads all 25 and proves (1970, row 0) empty. var p1 = await ReverseRawReader.ReadAsync(source, "T", now, DateTime.MinValue, 10, 65536, now, default(ReverseWalkHint), Ct); p1.Samples.Select(s => (double)s.Value!).ShouldBe(Enumerable.Range(15, 10).Reverse().Select(i => (double)i)); p1.Hint.HasEmptyStretch.ShouldBeTrue(); p1.Hint.EmptyFromTicks.ShouldBeLessThanOrEqualTo(DateTime.UnixEpoch.Ticks); p1.Hint.EmptyToTicks.ShouldBe(Ts(series[0])); var hint = p1.Hint; var hi = p1.Samples[^1].SourceTimestampUtc!.Value.AddTicks(-1); for (var page = 2; page <= 3; page++) { source.Reset(); var p = await ReverseRawReader.ReadAsync(source, "T", hi, DateTime.MinValue, 10, 65536, now, hint, Ct); p.Samples.Count.ShouldBe(page == 2 ? 10 : 5); source.Reads.ShouldBe(1, $"page {page} reads its rows and steps over the proven-empty stretch to the floor"); hint = p.Hint; hi = p.Samples[^1].SourceTimestampUtc!.Value.AddTicks(-1); } } /// /// #691.4 review: a burst just over the discovery read, sub-millisecond spaced, ending 100 days ago. The search for /// the top of the burst above the truncated read never re-sends an identical request (halving below the 1 ms request /// granularity did), and gallops up from the truncated read's newest row instead of bisecting the whole 100-day gap /// down to the millisecond (was 46 reads). /// [Fact] public async Task Bisecting_above_a_truncated_read_never_resends_an_identical_request() { var end = T0.AddDays(-100); var count = ReverseWalk.DiscoveryRows + 2; var series = Enumerable.Range(0, count).Select(i => At(end.Ticks - (count - 1 - i) * 1_000, i)).ToList(); // 0.1 ms apart var source = new AvevaShapedSource(series); var page = await ReverseRawReader.ReadAsync(source, "T", T0, DateTime.MinValue, 3, 65536, T0, default(ReverseWalkHint), Ct); page.Samples.Select(s => (double)s.Value!).ShouldBe([count - 1d, count - 2d, count - 3d]); source.RepeatedRequests.ShouldBe(0, string.Join("\n", source.Log)); source.Reads.ShouldBeLessThanOrEqualTo(22, string.Join("\n", source.Log)); } /// /// #691.4 re-review (S8): a tie cluster under a carried width. 30 rows at one whole-millisecond timestamp 0.3 s below /// hi, nothing above, N = 1, a carried 1 s width. The truncated read of the cluster left the width at one tick; /// once the gap above it was under two milliseconds nothing floored it, so segHi stepped down one tick per read — /// 10,006 reads of the same millisecond (28 before the gallop). /// [Theory] [InlineData(30, 1, 0.3, 1.0)] [InlineData(34, 1, 0.52, 0.48)] [InlineData(25, 1, 1.2, 1.0)] [InlineData(40, 3, 3.5, 2.0)] public async Task A_tie_cluster_under_a_carried_width_is_read_in_a_bounded_number_of_reads( int ties, int want, double secondsBelow, double hintSeconds) { var at = T0.Ticks - (long)(secondsBelow * TimeSpan.TicksPerSecond); at -= at % TimeSpan.TicksPerMillisecond; var series = Enumerable.Range(0, ties).Select(i => At(at, i)).ToList(); var source = new AvevaShapedSource(series); var page = await ReverseRawReader.ReadAsync(source, "T", T0, DateTime.MinValue, (uint)want, 65536, T0, new ReverseWalkHint((long)(hintSeconds * TimeSpan.TicksPerSecond)), Ct); page.Samples.Select(s => (double)s.Value!).ShouldBe(Enumerable.Range(ties - want, want).Reverse().Select(i => (double)i)); source.Reads.ShouldBeLessThanOrEqualTo(30, string.Join("\n", source.Log.Take(40))); source.RepeatedRequests.ShouldBe(0); } private static long Ts(DataValueSnapshot s) => s.SourceTimestampUtc!.Value.Ticks; /// Random ascending series: bursts, gaps, sub-millisecond stamps and tie clusters. private static List RandomSeries(Random rng, out int maxCluster) { var list = new List(); var t = T0.Ticks - rng.Next(0, 3) * TimeSpan.TicksPerDay; var count = rng.Next(1, 400); maxCluster = 1; var v = 0; while (list.Count < count) { t += rng.Next(6) switch { 0 => rng.Next(1, 10), // sub-millisecond 1 => rng.Next(1, 5) * TimeSpan.TicksPerMillisecond, // dense 2 => rng.Next(1, 3) * TimeSpan.TicksPerSecond, 3 => rng.Next(1, 120) * TimeSpan.TicksPerMinute, // a gap _ => rng.Next(1, 1000) * TimeSpan.TicksPerMillisecond, }; var ties = rng.Next(8) == 0 ? rng.Next(2, 12) : 1; maxCluster = Math.Max(maxCluster, ties); for (var k = 0; k < ties; k++) list.Add(At(t, v++)); } return list; } /// AVEVA-shaped forward-only raw source (see the fuzz test's summary). private sealed class AvevaShapedSource(IReadOnlyList series, bool roundUp = false) : IHistorianDataSource { public long MaxCap { get; private set; } public long RowsReturned { get; private set; } public int Reads { get; private set; } public int BadRequests { get; private set; } /// Requests that repeated an earlier request exactly (start, end, cap) — read-only, so pure waste. public int RepeatedRequests { get; private set; } private readonly HashSet<(DateTime, DateTime, uint)> _seen = []; /// Every request, for failure messages. public List Log { get; } = []; public void Reset() { MaxCap = 0; RowsReturned = 0; Reads = 0; RepeatedRequests = 0; _seen.Clear(); Log.Clear(); } public bool SupportsReverseRaw => false; public Task ReadRawAsync(string fullReference, DateTime startUtc, DateTime endUtc, uint maxValuesPerNode, CancellationToken cancellationToken) { Reads++; if (!_seen.Add((startUtc, endUtc, maxValuesPerNode))) RepeatedRequests++; Log.Add($"[{startUtc:O} .. {endUtc:O}] cap {maxValuesPerNode}"); if (startUtc > endUtc || maxValuesPerNode == 0 || startUtc < DateTime.UnixEpoch) BadRequests++; MaxCap = Math.Max(MaxCap, maxValuesPerNode); var stamp = roundUp && startUtc.Ticks % TimeSpan.TicksPerMillisecond != 0 ? new DateTime(startUtc.Ticks - startUtc.Ticks % TimeSpan.TicksPerMillisecond + TimeSpan.TicksPerMillisecond, DateTimeKind.Utc) : startUtc; var prior = series.LastOrDefault(s => s.SourceTimestampUtc <= startUtc); var rows = new List { prior is null ? new DataValueSnapshot(-1d, StatusCodes.BadNoData, stamp, stamp) : new DataValueSnapshot(-2d, prior.StatusCode, stamp, stamp), }; rows.AddRange(series.Where(s => s.SourceTimestampUtc >= startUtc && s.SourceTimestampUtc <= endUtc)); var page = rows.Take((int)Math.Min(int.MaxValue, maxValuesPerNode)).ToList(); RowsReturned += page.Count; return Task.FromResult(new HistorianRead(page, null)); } public Task ReadProcessedAsync(string fullReference, DateTime startUtc, DateTime endUtc, TimeSpan interval, HistoryAggregateType aggregate, CancellationToken cancellationToken) => throw new NotSupportedException(); public Task ReadAtTimeAsync(string fullReference, IReadOnlyList timestampsUtc, CancellationToken cancellationToken) => throw new NotSupportedException(); public Task ReadEventsAsync(string? sourceName, DateTime startUtc, DateTime endUtc, int maxEvents, CancellationToken cancellationToken) => throw new NotSupportedException(); public HistorianHealthSnapshot GetHealthSnapshot() => NullHistorianDataSource.Instance.GetHealthSnapshot(); public void Dispose() { } } }