fix(historian): #694 page raw reads at a strict gateway's learned ceiling; opt-in EventReadHorizon
Item 1: IHistorianDataSource.MaxRawValuesPerRead (Gateway: learned ceiling - 16 = L; 0 elsewhere, guard-tested). RawPageLimit plans every raw page on top of #692's lookahead: with L > 0 the read (page P + boundary ties k + lookahead) is <= L, P = min(N, L - k - 1) (uncapped: L - k - 1); no room => straight to the tie-cluster over-fetch (never an uncapped or underflowing request). One re-plan + re-read when the read taught a new limit or was refused for crowding pre-window rows (HistoryReadLimitExceededException.PreWindowRows): the page shrinks by them. Over-fetch bound min(MaxTieClusterOverfetch, L - 1); slice size never 0. Review fixes: F1 a window starting at/before the epoch counts AVEVA's synthetic row as pre-window (a ceiling-truncated read was reported complete); F2 the page path disables ReverseRawReader's own re-run (cold end-only ceiling 20/N=3000 cost 3,023 reads, now <= 40); F3 crowded look-back re-read around, not refused; F4 the #687 crowding re-read room at least doubles (up to 8 re-reads), so a cluster of hundreds just before a resume point no longer fails on a default gateway; a crowding refusal that leaves no room is BadTooManyOperations, never a tick-by-tick cursor. Item 2: ServerHistorian:EventReadHorizon (default 00:00:00 = exact, unchanged). When set, every fresh HistoryReadEvents window's earlier bound is raised to now - horizon (both directions, all backends). Lossy by design; documented. Item 3 (native TimeAverage/Total re-check) stays blocked on historiangw#27.
This commit is contained in:
Joseph Doherty
+499
@@ -0,0 +1,499 @@
|
||||
using Google.Protobuf.WellKnownTypes;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Opc.Ua;
|
||||
using Opc.Ua.Server;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.HistorianGateway.Contracts.Grpc;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.OpcUaServer;
|
||||
using SdkHistoryReadResult = Opc.Ua.HistoryReadResult;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #694 item 1 (live gate F3): a strict gateway (<c>AllowStreamingBeyondCeiling=false</c>) with a LOWERED
|
||||
/// <c>MaxValuesCeiling</c> (1000) refused ordinary page sizes — <c>NumValuesPerNode = 5000</c>, or an uncapped read over
|
||||
/// 3,600 rows, came back <c>BadTooManyOperations</c>. The server now sizes the page to the learned ceiling
|
||||
/// (<see cref="IHistorianDataSource.MaxRawValuesPerRead"/> L = ceiling − 16): the read — page P + boundary ties k + the
|
||||
/// #692 lookahead row — is at most L, so a first page holds L − 1 values and a resumed one L − k − 1, each with a
|
||||
/// continuation point. Driven end to end: a REAL SDK server's node manager → the REAL
|
||||
/// <see cref="GatewayHistorianDataSource"/> → an AVEVA-shaped fake gateway client that refuses an over-ceiling
|
||||
/// <c>max_values</c> exactly as the real gateway does (<c>InvalidArgument</c> "max_values exceeds the server ceiling of N",
|
||||
/// wrapped by the client package) and rejects <c>max_values <= 0</c> as the client package does.
|
||||
/// </summary>
|
||||
public sealed class GatewayCeilingPagingTests : IDisposable
|
||||
{
|
||||
private static CancellationToken Ct => TestContext.Current.CancellationToken;
|
||||
|
||||
private static readonly DateTime T0 = new(2026, 10, 12, 10, 0, 0, DateTimeKind.Utc);
|
||||
|
||||
private const int Ceiling = 1000;
|
||||
private const int L = Ceiling - RawAggregateReader.PreWindowAllowance; // 984
|
||||
private const int FirstPage = L - 1; // 983: page + lookahead fills a read
|
||||
private const int NextPage = L - 2; // 982: + the one boundary tie a resumed page re-reads (distinct timestamps)
|
||||
|
||||
private readonly string _pkiRoot = Path.Combine(Path.GetTempPath(), $"otopcua-gwceiling-{Guid.NewGuid():N}");
|
||||
|
||||
private static DateTime At(int seconds) => T0.AddSeconds(seconds);
|
||||
|
||||
private static HistorianSample Row(DateTime t, double v, uint quality = 0) =>
|
||||
new() { Tag = "T", Timestamp = Timestamp.FromDateTime(t), NumericValue = v, OpcQuality = 192, Quality = quality };
|
||||
|
||||
/// <summary>
|
||||
/// AVEVA-shaped ReadRaw over <paramref name="stored"/>: a boundary row at the query start (the value in effect, or a
|
||||
/// NULL row before the first sample), then <c>[start, end]</c> inclusive ascending, capped at <c>max_values</c>.
|
||||
/// </summary>
|
||||
private static FakeHistorianGatewayClient Gateway(IReadOnlyList<(DateTime T, double V)> stored, int? ceiling) => new()
|
||||
{
|
||||
MaxValuesCeiling = ceiling,
|
||||
RawSource = (start, end, max) =>
|
||||
{
|
||||
if (max <= 0) throw new ArgumentOutOfRangeException(nameof(max), "max_values must be positive"); // the client package
|
||||
var rows = new List<HistorianSample>();
|
||||
if (stored.All(s => s.T != start))
|
||||
{
|
||||
var prior = stored.LastOrDefault(s => s.T < start);
|
||||
rows.Add(prior.T == default
|
||||
? new HistorianSample { Tag = "T", Timestamp = Timestamp.FromDateTime(start), NumericValue = 0, OpcQuality = 0, Quality = 1 }
|
||||
: Row(start, prior.V, 133));
|
||||
}
|
||||
rows.AddRange(stored.Where(s => s.T >= start && s.T <= end).Select(s => Row(s.T, s.V)));
|
||||
return rows.Take(max);
|
||||
},
|
||||
};
|
||||
|
||||
/// <summary>One sample per second, value = seconds, at +0 … +(count − 1).</summary>
|
||||
private static List<(DateTime T, double V)> PerSecond(int count) =>
|
||||
Enumerable.Range(0, count).Select(i => (At(i), (double)i)).ToList();
|
||||
|
||||
private static int Refusals(FakeHistorianGatewayClient fake) =>
|
||||
fake.RawCalls.Count(c => fake.MaxValuesCeiling is { } n && c.MaxValues > n);
|
||||
|
||||
private static void EveryRequestFits(FakeHistorianGatewayClient fake) =>
|
||||
fake.RawCalls.ShouldAllBe(c => c.MaxValues > 0 && c.MaxValues <= Ceiling); // never uncapped, never over the ceiling
|
||||
|
||||
// ---- the data source's page limit ------------------------------------------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public async Task The_page_limit_is_unknown_until_a_refusal_names_the_ceiling_and_forgotten_when_the_gateway_streams()
|
||||
{
|
||||
var fake = Gateway(PerSecond(3600), Ceiling);
|
||||
var ds = new GatewayHistorianDataSource(fake, NullLogger<GatewayHistorianDataSource>.Instance);
|
||||
ds.MaxRawValuesPerRead.ShouldBe(0);
|
||||
|
||||
await ds.ReadRawAsync("T", At(0), At(10), 0, Ct);
|
||||
ds.MaxRawValuesPerRead.ShouldBe(L); // a read of this size asks for exactly the ceiling
|
||||
|
||||
(await ds.ReadRawAsync("T", At(0), At(3600), (uint)L, Ct)).Samples.Count.ShouldBe(L);
|
||||
fake.RawCalls[^1].MaxValues.ShouldBe(Ceiling);
|
||||
Refusals(fake).ShouldBe(1);
|
||||
|
||||
// The operator enabled AllowStreamingBeyondCeiling: a ceiling-full uncapped read re-probes (#691.3) and forgets it.
|
||||
fake.MaxValuesCeiling = null;
|
||||
(await ds.ReadRawAsync("T", At(0), At(3600), 0, Ct)).Samples.Count.ShouldBe(3600);
|
||||
ds.MaxRawValuesPerRead.ShouldBe(0);
|
||||
}
|
||||
|
||||
// ---- through the node manager ----------------------------------------------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public async Task The_cold_first_read_costs_one_refusal_and_its_page_fits_the_learned_ceiling()
|
||||
{
|
||||
var fake = Gateway(PerSecond(3600), Ceiling);
|
||||
var (host, nm, nodeId) = await BootAsync(fake);
|
||||
try
|
||||
{
|
||||
var first = ReadRaw(nm, nodeId, At(0), At(3600), 5000, null);
|
||||
first.Error.StatusCode.Code.ShouldBe(StatusCodes.Good);
|
||||
first.Values.Count.ShouldBe(FirstPage);
|
||||
first.Cp.ShouldNotBeNull();
|
||||
// The budget the client asked for (+ lookahead + allowance, refused naming 1000), the ceiling re-read that cannot
|
||||
// hold the window, then the page sized to the learned ceiling.
|
||||
fake.RawCalls.Select(c => c.MaxValues).ShouldBe([5000 + 1 + RawAggregateReader.PreWindowAllowance, Ceiling, Ceiling]);
|
||||
|
||||
var (rest, _, status) = DrainFrom(nm, nodeId, At(0), At(3600), 5000, first.Cp);
|
||||
status.Code.ShouldBe(StatusCodes.Good);
|
||||
first.Values.Concat(rest).ShouldBe(PerSecond(3600).Select(s => s.Item2));
|
||||
Refusals(fake).ShouldBe(1);
|
||||
}
|
||||
finally { await host.DisposeAsync(); }
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(5000u, false)]
|
||||
[InlineData(0u, false)]
|
||||
[InlineData(5000u, true)]
|
||||
[InlineData(0u, true)]
|
||||
public async Task With_the_ceiling_known_every_page_is_exactly_the_ceiling_page_and_nothing_is_refused(uint n, bool reverse)
|
||||
{
|
||||
var stored = PerSecond(3600);
|
||||
var fake = Gateway(stored, Ceiling);
|
||||
var (host, nm, nodeId) = await BootAsync(fake);
|
||||
try
|
||||
{
|
||||
ReadRaw(nm, nodeId, At(0), At(3600), 5000, null); // learns the ceiling
|
||||
fake.RawCalls.Clear();
|
||||
|
||||
var (values, pages, status) = reverse
|
||||
? DrainFrom(nm, nodeId, At(3600), At(-1), n, null)
|
||||
: DrainFrom(nm, nodeId, At(0), At(3600), n, null);
|
||||
|
||||
status.Code.ShouldBe(StatusCodes.Good);
|
||||
var expected = stored.Select(s => s.Item2);
|
||||
values.ShouldBe(reverse ? expected.Reverse() : expected);
|
||||
pages[0].ShouldBe(FirstPage);
|
||||
pages.Skip(1).SkipLast(1).ShouldAllBe(p => p == NextPage);
|
||||
pages[^1].ShouldBeGreaterThan(0); // no empty trailing page
|
||||
Refusals(fake).ShouldBe(0);
|
||||
EveryRequestFits(fake);
|
||||
if (!reverse) fake.RawCalls.Count.ShouldBe(pages.Count); // one read per page
|
||||
}
|
||||
finally { await host.DisposeAsync(); }
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task An_uncapped_read_that_fits_the_ceiling_is_still_one_page_without_a_continuation_point()
|
||||
{
|
||||
var fake = Gateway(PerSecond(3600), Ceiling);
|
||||
var (host, nm, nodeId) = await BootAsync(fake);
|
||||
try
|
||||
{
|
||||
var (values, pages, _) = DrainFrom(nm, nodeId, At(0), At(10), 0, null);
|
||||
values.Count.ShouldBe(10);
|
||||
pages.Count.ShouldBe(1);
|
||||
fake.RawCalls.Select(c => c.MaxValues).ShouldBe([int.MaxValue, Ceiling]); // #691.3 unchanged
|
||||
|
||||
var again = DrainFrom(nm, nodeId, At(0), At(10), 0, null);
|
||||
again.Pages.Count.ShouldBe(1);
|
||||
fake.RawCalls[^1].MaxValues.ShouldBe(Ceiling); // the learned page size, one read
|
||||
}
|
||||
finally { await host.DisposeAsync(); }
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_default_gateway_is_unchanged_one_read_and_no_continuation_point()
|
||||
{
|
||||
var fake = Gateway(PerSecond(3600), ceiling: null);
|
||||
var (host, nm, nodeId) = await BootAsync(fake);
|
||||
try
|
||||
{
|
||||
var (values, pages, _) = DrainFrom(nm, nodeId, At(0), At(3600), 5000, null);
|
||||
values.Count.ShouldBe(3600);
|
||||
pages.Count.ShouldBe(1);
|
||||
fake.RawCalls.Select(c => c.MaxValues).ShouldBe([5000 + 1 + RawAggregateReader.PreWindowAllowance]);
|
||||
}
|
||||
finally { await host.DisposeAsync(); }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A tie cluster straddling a page boundary, at the sizes where the ceiling arithmetic turns: a read can carry L rows;
|
||||
/// the over-fetch that pages within a cluster reads at most L − 1 (+1 to detect more). Every outcome is exact, or
|
||||
/// <c>BadTooManyOperations</c> after an exact prefix — never an uncapped or over-ceiling request, never an endless read.
|
||||
/// </summary>
|
||||
[Theory]
|
||||
[InlineData(L - 2, 5000u)]
|
||||
[InlineData(L - 1, 5000u)]
|
||||
[InlineData(L, 5000u)]
|
||||
[InlineData(L + 1, 5000u)]
|
||||
[InlineData(Ceiling - 17, 5000u)]
|
||||
[InlineData(Ceiling - 16, 5000u)]
|
||||
[InlineData(Ceiling + 1, 5000u)]
|
||||
[InlineData(L - 2, 0u)]
|
||||
[InlineData(L - 1, 0u)]
|
||||
[InlineData(L + 1, 0u)]
|
||||
[InlineData(Ceiling + 1, 0u)]
|
||||
public async Task A_tie_cluster_at_a_page_boundary_is_exact_or_refused_never_truncated_or_endless(int size, uint n)
|
||||
{
|
||||
var stored = PerSecond(500)
|
||||
.Concat(Enumerable.Range(0, size).Select(i => (At(600), 10_000.0 + i)))
|
||||
.Concat(Enumerable.Range(0, 1500).Select(i => (At(700 + i), 20_000.0 + i)))
|
||||
.ToList();
|
||||
var fake = Gateway(stored, Ceiling);
|
||||
var (host, nm, nodeId) = await BootAsync(fake);
|
||||
try
|
||||
{
|
||||
ReadRaw(nm, nodeId, At(0), At(5), 5000, null); // learns the ceiling
|
||||
fake.RawCalls.Clear();
|
||||
|
||||
var (values, pages, status) = DrainFrom(nm, nodeId, At(0), At(5000), n, null, maxPages: 60);
|
||||
var expected = stored.Select(s => s.Item2).ToList();
|
||||
if (status.Code == StatusCodes.Good) values.ShouldBe(expected);
|
||||
else
|
||||
{
|
||||
status.Code.ShouldBe(StatusCodes.BadTooManyOperations);
|
||||
values.ShouldBe(expected.Take(values.Count)); // an exact prefix
|
||||
size.ShouldBeGreaterThan(L - 2, "a cluster that fits the over-fetch is always served");
|
||||
}
|
||||
pages.Count.ShouldBeLessThan(60);
|
||||
EveryRequestFits(fake);
|
||||
}
|
||||
finally { await host.DisposeAsync(); }
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task An_uncapped_cursor_inside_a_tie_cluster_finishes_exactly_when_the_gateway_stops_being_strict()
|
||||
{
|
||||
// 100 samples, then 983 ties: the first page ends inside the cluster. Then the operator enables streaming and the
|
||||
// data source forgets the ceiling (L back to 0) — the uncapped cursor must still finish exactly, not loop.
|
||||
var stored = PerSecond(100)
|
||||
.Concat(Enumerable.Range(0, L - 1).Select(i => (At(200), 10_000.0 + i)))
|
||||
.Concat(Enumerable.Range(0, 300).Select(i => (At(300 + i), 20_000.0 + i)))
|
||||
.ToList();
|
||||
var fake = Gateway(stored, Ceiling);
|
||||
var ds = new GatewayHistorianDataSource(fake, NullLogger<GatewayHistorianDataSource>.Instance);
|
||||
var (host, nm, nodeId) = await BootAsync(fake, ds);
|
||||
try
|
||||
{
|
||||
ReadRaw(nm, nodeId, At(0), At(5), 5000, null); // learns the ceiling
|
||||
var first = ReadRaw(nm, nodeId, At(0), At(5000), 0, null);
|
||||
first.Values.Count.ShouldBe(FirstPage);
|
||||
first.Cp.ShouldNotBeNull();
|
||||
|
||||
fake.MaxValuesCeiling = null;
|
||||
await ds.ReadRawAsync("T", At(0), At(5000), 0, Ct); // a ceiling-full read re-probes and forgets
|
||||
ds.MaxRawValuesPerRead.ShouldBe(0);
|
||||
|
||||
var (rest, pages, status) = DrainFrom(nm, nodeId, At(0), At(5000), 0, first.Cp, maxPages: 20);
|
||||
status.Code.ShouldBe(StatusCodes.Good);
|
||||
first.Values.Concat(rest).ShouldBe(stored.Select(s => s.Item2));
|
||||
pages.Count.ShouldBeLessThan(20);
|
||||
}
|
||||
finally { await host.DisposeAsync(); }
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_ceiling_lowered_mid_page_through_is_re_learned_and_the_read_stays_exact()
|
||||
{
|
||||
var stored = PerSecond(3600);
|
||||
var fake = Gateway(stored, Ceiling);
|
||||
var (host, nm, nodeId) = await BootAsync(fake);
|
||||
try
|
||||
{
|
||||
var got = new List<double>();
|
||||
var p1 = ReadRaw(nm, nodeId, At(0), At(3600), 5000, null);
|
||||
got.AddRange(p1.Values);
|
||||
var p2 = ReadRaw(nm, nodeId, At(0), At(3600), 5000, p1.Cp);
|
||||
got.AddRange(p2.Values);
|
||||
|
||||
fake.MaxValuesCeiling = 500; // the operator lowered it
|
||||
fake.RawCalls.Clear();
|
||||
var (rest, pages, status) = DrainFrom(nm, nodeId, At(0), At(3600), 5000, p2.Cp, maxPages: 20);
|
||||
status.Code.ShouldBe(StatusCodes.Good);
|
||||
got.AddRange(rest);
|
||||
got.ShouldBe(stored.Select(s => s.Item2));
|
||||
pages.ShouldAllBe(p => p <= 500 - RawAggregateReader.PreWindowAllowance);
|
||||
Refusals(fake).ShouldBe(1); // the one that names the new ceiling
|
||||
}
|
||||
finally { await host.DisposeAsync(); }
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Bounding_values_page_within_the_ceiling()
|
||||
{
|
||||
var stored = PerSecond(3600);
|
||||
var fake = Gateway(stored, Ceiling);
|
||||
var (host, nm, nodeId) = await BootAsync(fake);
|
||||
try
|
||||
{
|
||||
ReadRaw(nm, nodeId, At(0), At(5), 5000, null); // learns the ceiling
|
||||
fake.RawCalls.Clear();
|
||||
|
||||
// Start bound = the sample at +10 (before StartTime), end bound = the sample at +3001 (after EndTime).
|
||||
var start = At(10).AddMilliseconds(500);
|
||||
var end = At(3000).AddMilliseconds(500);
|
||||
var (values, pages, status) = DrainFrom(nm, nodeId, start, end, 5000, null, returnBounds: true);
|
||||
status.Code.ShouldBe(StatusCodes.Good);
|
||||
values.ShouldBe(Enumerable.Range(10, 2992).Select(i => (double)i));
|
||||
pages.ShouldAllBe(p => p <= L);
|
||||
Refusals(fake).ShouldBe(0);
|
||||
}
|
||||
finally { await host.DisposeAsync(); }
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(1000, 100_000u, 10)]
|
||||
[InlineData(1000, 5000u, 10)]
|
||||
[InlineData(20, 3000u, 40)]
|
||||
public async Task A_cold_end_only_read_with_a_huge_page_shrinks_the_page_not_the_walk(int ceiling, uint n, int maxReads)
|
||||
{
|
||||
// Review F2: the walk used to re-run itself at the learned ceiling while keeping the page at N — a cold end-only read
|
||||
// with ceiling 20 and N = 3000 cost 3,023 gateway reads in ONE HistoryRead (ceiling 1000, N = 100,000: 207). The
|
||||
// refusal now reaches the page planner, which shrinks the page to the ceiling.
|
||||
var dense = Enumerable.Range(0, 200_000).Select(i => (T0.AddMilliseconds(i), (double)i)).ToList();
|
||||
var fake = Gateway(dense, ceiling);
|
||||
var (host, nm, nodeId) = await BootAsync(fake);
|
||||
try
|
||||
{
|
||||
var r = ReadRaw(nm, nodeId, DateTime.MinValue, T0.AddMilliseconds(200_000), n, null);
|
||||
r.Error.StatusCode.Code.ShouldBe(StatusCodes.Good);
|
||||
r.Cp.ShouldNotBeNull();
|
||||
var page = ceiling - RawAggregateReader.PreWindowAllowance - 1;
|
||||
r.Values.ShouldBe(Enumerable.Range(200_000 - r.Values.Count, r.Values.Count).Reverse().Select(i => (double)i));
|
||||
r.Values.Count.ShouldBeLessThanOrEqualTo(page);
|
||||
fake.RawCalls.Count.ShouldBeLessThanOrEqualTo(maxReads);
|
||||
}
|
||||
finally { await host.DisposeAsync(); }
|
||||
}
|
||||
|
||||
// ---- F1: a window starting at/before the epoch gets no look-back (review of #694) -------------------------------------
|
||||
|
||||
private static List<(DateTime T, double V)> FromEpoch(int count) =>
|
||||
Enumerable.Range(1, count).Select(i => (DateTime.UnixEpoch.AddSeconds(i), (double)i)).ToList();
|
||||
|
||||
[Fact]
|
||||
public async Task A_ceiling_truncated_read_from_the_epoch_is_never_reported_complete()
|
||||
{
|
||||
// AVEVA's synthetic row lands AT the epoch start (no 1 ms look-back there). It used to count as "past the window
|
||||
// end", so a ceiling-truncated uncapped read answered its first 999 rows, Good, as if that were everything.
|
||||
var fake = Gateway(FromEpoch(3000), Ceiling);
|
||||
var ds = new GatewayHistorianDataSource(fake, NullLogger<GatewayHistorianDataSource>.Instance);
|
||||
await Should.ThrowAsync<HistoryReadLimitExceededException>(() =>
|
||||
ds.ReadRawAsync("T", DateTime.UnixEpoch, DateTime.UnixEpoch.AddDays(1), 0, Ct));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task An_uncapped_read_from_the_epoch_pages_every_sample()
|
||||
{
|
||||
var stored = FromEpoch(3000);
|
||||
var fake = Gateway(stored, Ceiling);
|
||||
var (host, nm, nodeId) = await BootAsync(fake);
|
||||
try
|
||||
{
|
||||
var (values, pages, status) = DrainFrom(nm, nodeId, DateTime.UnixEpoch, DateTime.UnixEpoch.AddDays(1), 0, null);
|
||||
status.Code.ShouldBe(StatusCodes.Good);
|
||||
values.ShouldBe(stored.Select(s => s.Item2));
|
||||
pages.Count.ShouldBeGreaterThan(1);
|
||||
}
|
||||
finally { await host.DisposeAsync(); }
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task An_end_only_read_far_past_epoch_data_returns_the_latest_values()
|
||||
{
|
||||
var stored = FromEpoch(3000);
|
||||
var fake = Gateway(stored, Ceiling);
|
||||
var (host, nm, nodeId) = await BootAsync(fake);
|
||||
try
|
||||
{
|
||||
ReadRaw(nm, nodeId, DateTime.UnixEpoch, DateTime.UnixEpoch.AddSeconds(5), 5000, null); // learns the ceiling
|
||||
var r = ReadRaw(nm, nodeId, DateTime.MinValue, DateTime.UnixEpoch.AddDays(30), 10, null);
|
||||
r.Error.StatusCode.Code.ShouldBe(StatusCodes.Good);
|
||||
r.Values.ShouldBe(Enumerable.Range(2991, 10).Reverse().Select(i => (double)i)); // the LATEST ten, newest first
|
||||
}
|
||||
finally { await host.DisposeAsync(); }
|
||||
}
|
||||
|
||||
// ---- F3: rows crowding the look-back before a page start ---------------------------------------------------------------
|
||||
|
||||
[Theory]
|
||||
[InlineData(5000u)]
|
||||
[InlineData(0u)]
|
||||
[InlineData((uint)L)]
|
||||
[InlineData(900u)]
|
||||
public async Task Ties_just_before_a_page_start_are_read_around_not_refused(uint n)
|
||||
{
|
||||
// 962 samples, 20 ties at X, one sample at X + 1 ms: the first page (983) ends exactly at X + 1 ms. The next page's
|
||||
// 1 ms look-back holds the 20 ties — more than the 16-row allowance, and a ceiling-full page leaves no room for the
|
||||
// #687 re-read, so it used to fail BadTooManyOperations. The source now names the crowding rows and the page is
|
||||
// re-read that much smaller.
|
||||
var x = At(1000);
|
||||
var stored = Enumerable.Range(0, 962).Select(i => (At(i), (double)i))
|
||||
.Concat(Enumerable.Range(0, 20).Select(i => (x, 962.0 + i)))
|
||||
.Append((x.AddMilliseconds(1), 982.0))
|
||||
.Concat(Enumerable.Range(0, 3000).Select(i => (x.AddSeconds(1 + i), 983.0 + i)))
|
||||
.ToList();
|
||||
var fake = Gateway(stored, Ceiling);
|
||||
var (host, nm, nodeId) = await BootAsync(fake);
|
||||
try
|
||||
{
|
||||
ReadRaw(nm, nodeId, At(0), At(5), 5000, null); // learns the ceiling
|
||||
fake.RawCalls.Clear();
|
||||
|
||||
var (values, _, status) = DrainFrom(nm, nodeId, At(0), x.AddDays(1), n, null);
|
||||
status.Code.ShouldBe(StatusCodes.Good);
|
||||
values.ShouldBe(stored.Select(s => s.Item2));
|
||||
Refusals(fake).ShouldBe(0);
|
||||
EveryRequestFits(fake);
|
||||
}
|
||||
finally { await host.DisposeAsync(); }
|
||||
}
|
||||
|
||||
// ---- helpers ---------------------------------------------------------------------------------------------------------
|
||||
|
||||
private async Task<(OpcUaApplicationHost Host, OtOpcUaNodeManager Nm, NodeId NodeId)> BootAsync(
|
||||
FakeHistorianGatewayClient fake, GatewayHistorianDataSource? ds = null)
|
||||
{
|
||||
var started = await OpcUaHostStart.StartAsync(
|
||||
port => new OpcUaApplicationHost(
|
||||
new OpcUaApplicationHostOptions
|
||||
{
|
||||
ApplicationName = "OtOpcUa.GatewayCeilingPagingTest",
|
||||
ApplicationUri = $"urn:OtOpcUa.GatewayCeilingPagingTest:{Guid.NewGuid():N}",
|
||||
OpcUaPort = port,
|
||||
PublicHostname = "localhost",
|
||||
PkiStoreRoot = _pkiRoot,
|
||||
},
|
||||
NullLogger<OpcUaApplicationHost>.Instance),
|
||||
() => new OtOpcUaSdkServer(), Ct);
|
||||
var nm = started.Server.NodeManager!;
|
||||
nm.HistoryContinuationStore = new InMemoryHistoryContinuationStore();
|
||||
nm.HistorianDataSource = ds ?? new GatewayHistorianDataSource(fake, NullLogger<GatewayHistorianDataSource>.Instance);
|
||||
nm.EnsureVariable("eq-1/pv", parentFolderNodeId: null, displayName: "PV", dataType: "Double",
|
||||
writable: false, historianTagname: "T", realm: AddressSpaceRealm.Uns);
|
||||
return (started.Host, nm, nm.TryGetVariable("eq-1/pv")!.NodeId);
|
||||
}
|
||||
|
||||
/// <summary>Pages until no continuation point or a Bad status; the page sizes and the final status.</summary>
|
||||
private static (List<double> Values, List<int> Pages, StatusCode Status) DrainFrom(
|
||||
OtOpcUaNodeManager nm, NodeId nodeId, DateTime start, DateTime end, uint max, byte[]? cp, int maxPages = 20,
|
||||
bool returnBounds = false)
|
||||
{
|
||||
var values = new List<double>();
|
||||
var pages = new List<int>();
|
||||
ServiceResult error;
|
||||
do
|
||||
{
|
||||
(var page, error, cp) = ReadRaw(nm, nodeId, start, end, max, cp, returnBounds);
|
||||
values.AddRange(page);
|
||||
pages.Add(page.Count);
|
||||
pages.Count.ShouldBeLessThanOrEqualTo(maxPages, "paging must terminate");
|
||||
}
|
||||
while (cp is not null && StatusCode.IsGood(error.StatusCode));
|
||||
return (values, pages, error.StatusCode);
|
||||
}
|
||||
|
||||
private static (IReadOnlyList<double> Values, ServiceResult Error, byte[]? Cp) ReadRaw(
|
||||
OtOpcUaNodeManager nm, NodeId nodeId, DateTime start, DateTime end, uint max, byte[]? inboundCp, bool returnBounds = false)
|
||||
{
|
||||
var context = new OperationContext(new RequestHeader(), secureChannelContext: null, RequestType.HistoryRead, identity: null);
|
||||
var details = new ReadRawModifiedDetails
|
||||
{
|
||||
ReturnBounds = returnBounds,
|
||||
IsReadModified = false,
|
||||
StartTime = start,
|
||||
EndTime = end,
|
||||
NumValuesPerNode = max,
|
||||
};
|
||||
var nodesToRead = new List<HistoryReadValueId> { new() { NodeId = nodeId, ContinuationPoint = inboundCp } };
|
||||
var results = new List<SdkHistoryReadResult> { null! };
|
||||
var errors = new List<ServiceResult> { null! };
|
||||
nm.HistoryRead(context, details, TimestampsToReturn.Both, releaseContinuationPoints: false, nodesToRead, results, errors);
|
||||
|
||||
var values = new List<double>();
|
||||
if (results[0]?.HistoryData is { } ho && !ExtensionObject.IsNull(ho))
|
||||
foreach (var dv in ((HistoryData)ExtensionObject.ToEncodeable(ho)).DataValues)
|
||||
values.Add(Convert.ToDouble(dv.Value));
|
||||
return (values, errors[0], results[0]?.ContinuationPoint);
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
if (Directory.Exists(_pkiRoot))
|
||||
{
|
||||
try { Directory.Delete(_pkiRoot, recursive: true); }
|
||||
catch { /* best-effort */ }
|
||||
}
|
||||
}
|
||||
}
|
||||
+24
-5
@@ -222,17 +222,36 @@ public sealed class GatewayRawBoundaryRowTests
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_margin_still_crowding_the_re_read_fails_the_node_as_a_request_limit()
|
||||
public async Task A_margin_crowding_a_truncated_read_is_re_read_with_growing_room()
|
||||
{
|
||||
// 200 rows in the margin: the single bounded re-read (5 + 21 + 16) is still short of the cap because of them.
|
||||
// #694 F4: 200 rows in the margin. The first read (5 + 16) saw only 21 of them, so a single re-read sized from that
|
||||
// count (5 + 21 + 16) was still short and the node failed — with no ceiling at all. The room now at least doubles per
|
||||
// re-read (21, 42, 84, 168, 336), so the window is served in a bounded number of reads.
|
||||
var margin = Enumerable.Range(0, 200).Select(i => (At(30).AddTicks(-1 - i), (double)i, false));
|
||||
var fake = Aveva(margin.Append((At(35), 1d, false)).ToList());
|
||||
var ds = Source(fake);
|
||||
|
||||
await Should.ThrowAsync<HistoryReadLimitExceededException>(() => ds.ReadRawAsync("T", At(30), At(40), 5, Ct));
|
||||
var r = await ds.ReadRawAsync("T", At(30), At(40), 5, Ct);
|
||||
|
||||
fake.ReadRawCallCount.ShouldBe(2); // one bounded re-read, never more
|
||||
fake.RawStreamsCompleted.ShouldBe(2);
|
||||
r.Samples.Select(s => s.Value).ShouldBe([1d]);
|
||||
fake.ReadRawCallCount.ShouldBe(6);
|
||||
fake.RawStreamsCompleted.ShouldBe(6);
|
||||
ds.GetHealthSnapshot().TotalFailures.ShouldBe(0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_margin_still_crowding_the_bounded_re_reads_fails_the_node_as_a_request_limit()
|
||||
{
|
||||
// 20,000 rows in the margin: past the bounded re-reads (21 · 2^8 ≈ 5,400 rows of room).
|
||||
var margin = Enumerable.Range(0, 20_000).Select(i => (At(30).AddTicks(-1 - i), (double)i, false));
|
||||
var fake = Aveva(margin.Append((At(35), 1d, false)).ToList());
|
||||
var ds = Source(fake);
|
||||
|
||||
var ex = await Should.ThrowAsync<HistoryReadLimitExceededException>(() => ds.ReadRawAsync("T", At(30), At(40), 5, Ct));
|
||||
|
||||
ex.PreWindowRows.ShouldBeGreaterThan(0);
|
||||
fake.ReadRawCallCount.ShouldBe(9); // the read + MaxCrowdedReReads, never more
|
||||
fake.RawStreamsCompleted.ShouldBe(9);
|
||||
ds.GetHealthSnapshot().TotalFailures.ShouldBe(0);
|
||||
}
|
||||
|
||||
|
||||
+5
@@ -31,4 +31,9 @@
|
||||
<ProjectReference Include="..\..\..\src\Drivers\ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway\ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.csproj"/>
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<!-- #694: the node manager's HistoryRead paging over the REAL gateway data source boots a real SDK server. -->
|
||||
<Compile Include="..\..\Server\ZB.MOM.WW.OtOpcUa.OpcUaServer.Tests\OpcUaHostStart.cs" Link="OpcUaHostStart.cs"/>
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
|
||||
Reference in New Issue
Block a user