Joseph Doherty
186b7d5ea5
The last pages of a deep reverse page-through on a forward-only backend each walked down to 1970 one x4 growth step at a time (~14-16 reads, ~2.5 s live). After two empty segments the walk now asks once per page for the whole remainder at the page budget (a truncated answer is discarded and the walk carries on), and the stretch the page's last complete read proved empty rides the continuation point (HistoryContinuationState.EmptyFromTicks/EmptyToTicks, session-held, so no versioning) so the next page steps over it. Fuzz now pages with the walk's own hints and with true random empty stretches.
264 lines
14 KiB
C#
264 lines
14 KiB
C#
using Opc.Ua;
|
||
using Shouldly;
|
||
using Xunit;
|
||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||
using HistorianRead = ZB.MOM.WW.OtOpcUa.Core.Abstractions.HistoryReadResult;
|
||
|
||
namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.Tests;
|
||
|
||
/// <summary>
|
||
/// Gitea #689 / #691: the reverse walk-back (<see cref="ReverseWalk"/> through <see cref="ReverseRawReader"/>) driven
|
||
/// directly, without a server: a randomized exactness check against a brute-force answer, and the sizing properties.
|
||
/// </summary>
|
||
public sealed class ReverseWalkTests
|
||
{
|
||
private static CancellationToken Ct => TestContext.Current.CancellationToken;
|
||
|
||
private static readonly DateTime T0 = new(2026, 10, 12, 18, 0, 0, DateTimeKind.Utc);
|
||
|
||
private static DataValueSnapshot At(long ticks, double v) =>
|
||
new(v, StatusCodes.Good, new DateTime(ticks, DateTimeKind.Utc), new DateTime(ticks, DateTimeKind.Utc));
|
||
|
||
/// <summary>
|
||
/// A randomized walk over AVEVA-shaped forward reads (inclusive end, a fabricated row at the query start — optionally
|
||
/// stamped at the start rounded UP to the millisecond — counted against the cap): every page is exactly the brute-force
|
||
/// newest-first answer over <c>(lo, hi]</c>, ties in the backend's forward order reversed, whatever the carried width
|
||
/// — and a timestamp over the tie budget is the only refusal.
|
||
/// </summary>
|
||
[Fact]
|
||
public async Task A_walked_page_is_exactly_the_newest_N_over_the_window_on_random_data()
|
||
{
|
||
var failures = new List<string>();
|
||
for (var seed = 0; seed < 400; seed++)
|
||
{
|
||
var rng = new Random(seed);
|
||
var series = RandomSeries(rng, out var maxCluster);
|
||
var source = new AvevaShapedSource(series, roundUp: rng.Next(2) == 0);
|
||
var tieBudget = rng.Next(4) switch { 0 => 1, 1 => 3, 2 => 8, _ => 65536 };
|
||
var want = (uint)rng.Next(1, 40);
|
||
var span = Ts(series[^1]) - Ts(series[0]);
|
||
var hi = Ts(series[0]) + (long)(rng.NextDouble() * 1.2 * span) + rng.Next(0, 3);
|
||
var lo = Ts(series[0]) - TimeSpan.TicksPerSecond + (long)(rng.NextDouble() * (hi - Ts(series[0])) * 0.8);
|
||
if (rng.Next(5) == 0) lo = 0; // an end-only read
|
||
var hint = rng.Next(3) switch
|
||
{
|
||
0 => default,
|
||
1 => new ReverseWalkHint(rng.Next(1, 10)),
|
||
_ => new ReverseWalkHint((long)(rng.NextDouble() * TimeSpan.TicksPerHour)),
|
||
};
|
||
if (rng.Next(2) == 0)
|
||
{
|
||
// A TRUE empty stretch (#691): between two adjacent distinct timestamps, or from the floor to the first row.
|
||
var k = rng.Next(series.Count);
|
||
var from = k == 0 ? (rng.Next(2) == 0 ? 0 : DateTime.UnixEpoch.Ticks) : Ts(series[k - 1]);
|
||
var to = Ts(series[k]);
|
||
if (to > from) hint = hint with { EmptyFromTicks = from, EmptyToTicks = to };
|
||
}
|
||
|
||
// Page on: each next page resumes just below the previous page's oldest timestamp with the hint it returned —
|
||
// so the hints the walk itself produces (width, proven-empty stretch) are exercised too.
|
||
for (var pageNo = 0; pageNo < 6 && hi > Math.Max(lo, DateTime.UnixEpoch.Ticks); pageNo++)
|
||
{
|
||
var expected = series.Where(s => Ts(s) > Math.Max(lo, DateTime.UnixEpoch.Ticks) && Ts(s) <= hi)
|
||
.Select((s, i) => (s, i)).OrderByDescending(x => Ts(x.s)).ThenByDescending(x => x.i)
|
||
.Take((int)want).Select(x => (double)x.s.Value!).ToList();
|
||
|
||
try
|
||
{
|
||
var page = await ReverseRawReader.ReadAsync(source, "T", new DateTime(hi, DateTimeKind.Utc),
|
||
new DateTime(lo, DateTimeKind.Utc), want, tieBudget, T0.AddDays(1), hint, Ct);
|
||
var got = page.Samples.Select(s => (double)s.Value!).ToList();
|
||
if (!got.SequenceEqual(expected))
|
||
{
|
||
failures.Add($"seed {seed} page {pageNo}: got [{string.Join(",", got)}] expected [{string.Join(",", expected)}]");
|
||
break;
|
||
}
|
||
|
||
if (page.Hint.WidthTicks <= 0) failures.Add($"seed {seed}: no width carried");
|
||
if (got.Count < want) break;
|
||
hint = page.Hint;
|
||
hi = Ts(page.Samples[^1]) - 1;
|
||
}
|
||
catch (HistoryReadLimitExceededException)
|
||
{
|
||
if (maxCluster <= tieBudget) failures.Add($"seed {seed}: refused with no cluster over {tieBudget} (max {maxCluster})");
|
||
break;
|
||
}
|
||
}
|
||
|
||
if (source.MaxCap > 2L * Math.Max(Math.Max(tieBudget, ReverseWalk.DiscoveryRows), 2 * want + 16) + 4)
|
||
failures.Add($"seed {seed}: asked for {source.MaxCap} rows");
|
||
if (source.BadRequests > 0) failures.Add($"seed {seed}: {source.BadRequests} invalid requests");
|
||
}
|
||
|
||
failures.ShouldBeEmpty();
|
||
}
|
||
|
||
/// <summary>
|
||
/// #689: the carried width sizes a resumed page from N, not from the tie budget — the live symptom was 32,770 rows
|
||
/// per page for N = 3 on a 1 Hz tag once the width had grown over a few pages.
|
||
/// </summary>
|
||
[Fact]
|
||
public async Task A_resumed_page_on_a_dense_tag_reads_about_N_rows_with_the_default_tie_budget()
|
||
{
|
||
var series = Enumerable.Range(0, 200_000).Select(i => At(T0.Ticks + i * TimeSpan.TicksPerSecond, i)).ToList();
|
||
var source = new AvevaShapedSource(series);
|
||
var hi = new DateTime(Ts(series[^1]), DateTimeKind.Utc);
|
||
var hint = default(ReverseWalkHint);
|
||
|
||
for (var page = 1; page <= 30; page++)
|
||
{
|
||
source.Reset();
|
||
var result = await ReverseRawReader.ReadAsync(source, "T", hi, DateTime.MinValue, 3, 65536, hi.AddHours(1), hint, Ct);
|
||
result.Samples.Count.ShouldBe(3);
|
||
hint = result.Hint;
|
||
hi = result.Samples[^1].SourceTimestampUtc!.Value.AddTicks(-1);
|
||
if (page > 1) source.RowsReturned.ShouldBeLessThanOrEqualTo(30, $"page {page}");
|
||
}
|
||
}
|
||
|
||
/// <summary>
|
||
/// #689: a dense tag that stopped long ago. The walk's ×4 growth across the empty stretch overshoots into far more
|
||
/// rows than one read takes; the truncated read only shows the OLDEST of them, so the walk must not keep re-entering
|
||
/// that stretch — it bisects the gap above the newest row it saw.
|
||
/// </summary>
|
||
[Fact]
|
||
public async Task A_dense_tag_below_a_long_empty_stretch_is_found_in_a_bounded_number_of_reads()
|
||
{
|
||
// 10 Hz for 3 hours, ending two days before "now".
|
||
var start = T0.AddDays(-2).AddHours(-3);
|
||
var series = Enumerable.Range(0, 108_000).Select(i => At(start.Ticks + i * TimeSpan.TicksPerMillisecond * 100, i)).ToList();
|
||
var source = new AvevaShapedSource(series);
|
||
|
||
var page = await ReverseRawReader.ReadAsync(source, "T", T0, DateTime.MinValue, 3, 65536, T0, default(ReverseWalkHint), Ct);
|
||
|
||
page.Samples.Select(s => (double)s.Value!).ShouldBe([107_999d, 107_998d, 107_997d]);
|
||
source.Reads.ShouldBeLessThanOrEqualTo(25);
|
||
source.RowsReturned.ShouldBeLessThanOrEqualTo(25 * (ReverseWalk.DiscoveryRows + 2));
|
||
}
|
||
|
||
/// <summary>
|
||
/// #691: the end of a walk proves the rest of the window empty in about one read — after two empty segments the walk
|
||
/// asks once for the whole remainder down to the floor — instead of one read per ×4 growth step to 1970 (~14 reads).
|
||
/// </summary>
|
||
[Fact]
|
||
public async Task Running_out_of_rows_proves_the_rest_of_the_window_empty_in_a_few_reads()
|
||
{
|
||
var series = Enumerable.Range(0, 5).Select(i => At(T0.Ticks + i * TimeSpan.TicksPerSecond, i)).ToList();
|
||
var source = new AvevaShapedSource(series);
|
||
|
||
var page = await ReverseRawReader.ReadAsync(source, "T", T0.AddSeconds(4), DateTime.MinValue, 10, 65536, T0,
|
||
default(ReverseWalkHint), Ct);
|
||
|
||
page.Samples.Select(s => (double)s.Value!).ShouldBe([4d, 3d, 2d, 1d, 0d]);
|
||
source.Reads.ShouldBeLessThanOrEqualTo(4, "one segment holding the rows, two empty ones, then the remainder at once");
|
||
}
|
||
|
||
/// <summary>
|
||
/// #691: a page's last complete read proves the stretch below its oldest row empty, and the continuation carries it:
|
||
/// the next page steps straight over it to the floor instead of walking it again.
|
||
/// </summary>
|
||
[Fact]
|
||
public async Task A_resumed_page_steps_over_the_stretch_the_previous_page_proved_empty()
|
||
{
|
||
var series = Enumerable.Range(0, 25).Select(i => At(T0.Ticks + i * TimeSpan.TicksPerSecond, i)).ToList();
|
||
var source = new AvevaShapedSource(series);
|
||
var now = T0.AddHours(1);
|
||
|
||
// Page 1: an hour of nothing above the rows; the remainder probe reads all 25 and proves (1970, row 0) empty.
|
||
var p1 = await ReverseRawReader.ReadAsync(source, "T", now, DateTime.MinValue, 10, 65536, now, default(ReverseWalkHint), Ct);
|
||
p1.Samples.Select(s => (double)s.Value!).ShouldBe(Enumerable.Range(15, 10).Reverse().Select(i => (double)i));
|
||
p1.Hint.HasEmptyStretch.ShouldBeTrue();
|
||
p1.Hint.EmptyFromTicks.ShouldBeLessThanOrEqualTo(DateTime.UnixEpoch.Ticks);
|
||
p1.Hint.EmptyToTicks.ShouldBe(Ts(series[0]));
|
||
|
||
var hint = p1.Hint;
|
||
var hi = p1.Samples[^1].SourceTimestampUtc!.Value.AddTicks(-1);
|
||
for (var page = 2; page <= 3; page++)
|
||
{
|
||
source.Reset();
|
||
var p = await ReverseRawReader.ReadAsync(source, "T", hi, DateTime.MinValue, 10, 65536, now, hint, Ct);
|
||
p.Samples.Count.ShouldBe(page == 2 ? 10 : 5);
|
||
source.Reads.ShouldBe(1, $"page {page} reads its rows and steps over the proven-empty stretch to the floor");
|
||
hint = p.Hint;
|
||
hi = p.Samples[^1].SourceTimestampUtc!.Value.AddTicks(-1);
|
||
}
|
||
}
|
||
|
||
private static long Ts(DataValueSnapshot s) => s.SourceTimestampUtc!.Value.Ticks;
|
||
|
||
/// <summary>Random ascending series: bursts, gaps, sub-millisecond stamps and tie clusters.</summary>
|
||
private static List<DataValueSnapshot> RandomSeries(Random rng, out int maxCluster)
|
||
{
|
||
var list = new List<DataValueSnapshot>();
|
||
var t = T0.Ticks - rng.Next(0, 3) * TimeSpan.TicksPerDay;
|
||
var count = rng.Next(1, 400);
|
||
maxCluster = 1;
|
||
var v = 0;
|
||
while (list.Count < count)
|
||
{
|
||
t += rng.Next(6) switch
|
||
{
|
||
0 => rng.Next(1, 10), // sub-millisecond
|
||
1 => rng.Next(1, 5) * TimeSpan.TicksPerMillisecond, // dense
|
||
2 => rng.Next(1, 3) * TimeSpan.TicksPerSecond,
|
||
3 => rng.Next(1, 120) * TimeSpan.TicksPerMinute, // a gap
|
||
_ => rng.Next(1, 1000) * TimeSpan.TicksPerMillisecond,
|
||
};
|
||
var ties = rng.Next(8) == 0 ? rng.Next(2, 12) : 1;
|
||
maxCluster = Math.Max(maxCluster, ties);
|
||
for (var k = 0; k < ties; k++) list.Add(At(t, v++));
|
||
}
|
||
|
||
return list;
|
||
}
|
||
|
||
/// <summary>AVEVA-shaped forward-only raw source (see the fuzz test's summary).</summary>
|
||
private sealed class AvevaShapedSource(IReadOnlyList<DataValueSnapshot> series, bool roundUp = false) : IHistorianDataSource
|
||
{
|
||
public long MaxCap { get; private set; }
|
||
public long RowsReturned { get; private set; }
|
||
public int Reads { get; private set; }
|
||
public int BadRequests { get; private set; }
|
||
|
||
public void Reset() { MaxCap = 0; RowsReturned = 0; Reads = 0; }
|
||
|
||
public bool SupportsReverseRaw => false;
|
||
|
||
public Task<HistorianRead> ReadRawAsync(string fullReference, DateTime startUtc, DateTime endUtc, uint maxValuesPerNode,
|
||
CancellationToken cancellationToken)
|
||
{
|
||
Reads++;
|
||
if (startUtc > endUtc || maxValuesPerNode == 0 || startUtc < DateTime.UnixEpoch) BadRequests++;
|
||
MaxCap = Math.Max(MaxCap, maxValuesPerNode);
|
||
var stamp = roundUp && startUtc.Ticks % TimeSpan.TicksPerMillisecond != 0
|
||
? new DateTime(startUtc.Ticks - startUtc.Ticks % TimeSpan.TicksPerMillisecond + TimeSpan.TicksPerMillisecond, DateTimeKind.Utc)
|
||
: startUtc;
|
||
var prior = series.LastOrDefault(s => s.SourceTimestampUtc <= startUtc);
|
||
var rows = new List<DataValueSnapshot>
|
||
{
|
||
prior is null
|
||
? new DataValueSnapshot(-1d, StatusCodes.BadNoData, stamp, stamp)
|
||
: new DataValueSnapshot(-2d, prior.StatusCode, stamp, stamp),
|
||
};
|
||
rows.AddRange(series.Where(s => s.SourceTimestampUtc >= startUtc && s.SourceTimestampUtc <= endUtc));
|
||
var page = rows.Take((int)Math.Min(int.MaxValue, maxValuesPerNode)).ToList();
|
||
RowsReturned += page.Count;
|
||
return Task.FromResult(new HistorianRead(page, null));
|
||
}
|
||
|
||
public Task<HistorianRead> ReadProcessedAsync(string fullReference, DateTime startUtc, DateTime endUtc, TimeSpan interval,
|
||
HistoryAggregateType aggregate, CancellationToken cancellationToken) => throw new NotSupportedException();
|
||
|
||
public Task<HistorianRead> ReadAtTimeAsync(string fullReference, IReadOnlyList<DateTime> timestampsUtc,
|
||
CancellationToken cancellationToken) => throw new NotSupportedException();
|
||
|
||
public Task<HistoricalEventsResult> ReadEventsAsync(string? sourceName, DateTime startUtc, DateTime endUtc, int maxEvents,
|
||
CancellationToken cancellationToken) => throw new NotSupportedException();
|
||
|
||
public HistorianHealthSnapshot GetHealthSnapshot() => NullHistorianDataSource.Instance.GetHealthSnapshot();
|
||
|
||
public void Dispose() { }
|
||
}
|
||
}
|