Files
lmxopcua/tests/Drivers/Historian.Conformance/RawReadConformance.cs
T
Joseph Doherty 6669fae7e5 feat(historian): ServerHistorian:MaxRawValuesPerRead caps raw pages and backend reads
Review R08:1.1: an uncapped HistoryRead-Raw (NumValuesPerNode = 0) read a
tag's whole window into memory on every backend (SQL TOP(int.MaxValue),
QuestDB LIMIT, Gateway max_values = int.MaxValue + a second snapshot copy).

New ServerHistorian:MaxRawValuesPerRead (default 10 000; <= 0 unbounded with
a Validate() warning; 1 raised to 2). Each backend data source takes it,
reports it as IHistorianDataSource.MaxRawValuesPerRead (the Gateway: min
with a strict gateway's learned ceiling), so the node manager's existing
RawPageLimit paging serves N = 0 and N > cap as pages of at most the cap
with a continuation point. Independently, a backend raw read asking for 0
or more than the cap reads cap + 1 rows and is REFUSED
(HistoryReadLimitExceededException => BadTooManyOperations) when the window
holds more - never answered truncated, since the server's paging reads a
short answer as the last page. Bound through the Host registry, the
SqlServer/QuestDb providers and GatewayHistorian.CreateDataSource.

Tests: options (default/binding/warnings), provider binding, Gateway unit +
through a real SDK node manager (uncapped -> 999/998/998/605 + CPs), and a
conformance case for the SQL backends.

Refs #736

Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT
2026-10-22 06:29:10 -04:00

196 lines
9.7 KiB
C#

using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Conformance;
/// <summary>Raw-read behaviour over the fixture tag A (see <see cref="ConformanceData.SeedAAsync"/>).</summary>
public abstract class RawReadConformance : ConformanceBase
{
private static readonly CancellationToken Ct = CancellationToken.None;
private static double?[] Values(HistoryReadResult r) => r.Samples.Select(s => (double?)s.Value).ToArray();
[Fact]
public async Task Forward_read_is_start_inclusive_end_exclusive_ascending_with_exact_ticks()
{
await ConformanceData.SeedAAsync(B);
var r = await B.CreateDataSource().ReadRawAsync("A", At(10), At(40), 0, Ct);
Values(r).ShouldBe([20d, 30d, 40d]); // +10 included, +40 (== end) excluded
r.Samples.Select(s => s.SourceTimestampUtc!.Value.Ticks).ShouldBe([B.Round(At(10)).Ticks, B.Round(At(20)).Ticks, B.Round(At(30)).Ticks]);
r.Samples.Select(s => s.StatusCode).ShouldBe([Good, Bad, Good]);
r.Samples[0].SourceTimestampUtc!.Value.Kind.ShouldBe(DateTimeKind.Utc);
r.Samples[0].ServerTimestampUtc.Ticks.ShouldBe(B.Round(At(11)).Ticks); // stored ServerTimeUtc, not "now"
r.ContinuationPoint.ShouldBeNull();
}
/// <summary>
/// R08:1.1 (#736): the server raw cap bounds the backend's own <c>TOP</c> / <c>LIMIT</c>. A read asking for everything
/// (or for more than the cap) reads at most cap + 1 rows and is refused when the window holds more — never answered
/// truncated (a short answer is what the server's paging reads as "the last page") — and is served in full when it fits.
/// A read at or under the cap is unchanged. The refusal is a client-request limit, not a backend fault.
/// </summary>
[Fact]
public async Task The_server_raw_cap_bounds_every_read_and_refuses_an_over_cap_window_rather_than_truncating()
{
await ConformanceData.SeedAAsync(B); // five samples at +0, +10, +20, +30, +40
var src = B.CreateDataSource(maxTimeWeightedRawRows: 1_000_000, maxRawValuesPerRead: 3);
src.MaxRawValuesPerRead.ShouldBe(3);
// Five rows under a cap of three: everything / above the cap ⇒ refused; at the cap ⇒ the first three; under ⇒ as asked.
var refused = await Should.ThrowAsync<HistoryReadLimitExceededException>(() => src.ReadRawAsync("A", At(-1), At(100), 0, Ct));
refused.Limit.ShouldBe(3);
await Should.ThrowAsync<HistoryReadLimitExceededException>(() => src.ReadRawAsync("A", At(-1), At(100), 4, Ct));
await Should.ThrowAsync<HistoryReadLimitExceededException>(() => src.ReadRawAsync("A", At(-1), At(100), uint.MaxValue, Ct));
Values(await src.ReadRawAsync("A", At(-1), At(100), 3, Ct)).ShouldBe([10d, 20d, 30d]);
Values(await src.ReadRawAsync("A", At(-1), At(100), 2, Ct)).ShouldBe([10d, 20d]);
// Three rows fit: everything / above the cap are served in full.
Values(await src.ReadRawAsync("A", At(10), At(40), 0, Ct)).ShouldBe([20d, 30d, 40d]);
Values(await src.ReadRawAsync("A", At(10), At(40), 4, Ct)).ShouldBe([20d, 30d, 40d]);
// Reverse reads are bounded the same way, keeping the LATEST rows.
await Should.ThrowAsync<HistoryReadLimitExceededException>(() => src.ReadRawAsync("A", At(100), At(-1), 0, Ct));
Values(await src.ReadRawAsync("A", At(100), At(-1), 3, Ct)).ShouldBe([50d, 40d, 30d]);
Values(await src.ReadRawAsync("A", At(40), At(10), 0, Ct)).ShouldBe([50d, 40d, 30d]);
// A point read and an empty window are unaffected.
Values(await src.ReadRawAsync("A", At(10), At(10), 0, Ct)).ShouldBe([20d]);
(await src.ReadRawAsync("A", At(50), At(60), 0, Ct)).Samples.ShouldBeEmpty();
src.GetHealthSnapshot().TotalFailures.ShouldBe(0, "a refused over-cap read is a client-request limit, not a backend fault");
}
[Fact]
public async Task Forward_read_honours_the_limit()
{
await ConformanceData.SeedAAsync(B);
Values(await B.CreateDataSource().ReadRawAsync("A", At(0), At(100), 2, Ct)).ShouldBe([10d, 20d]);
}
[Fact]
public async Task Zero_max_values_is_unbounded()
{
await ConformanceData.SeedAAsync(B);
Values(await B.CreateDataSource().ReadRawAsync("A", At(-1), At(100), 0, Ct)).ShouldBe([10d, 20d, 30d, 40d, 50d]);
Values(await B.CreateDataSource().ReadRawAsync("A", At(-1), At(100), uint.MaxValue, Ct)).Length.ShouldBe(5);
}
/// <summary>
/// #669: the node manager now pages an open-ended read (Part 11: only a StartTime / only an EndTime) with
/// <c>DateTime.MaxValue</c> / <c>DateTime.MinValue</c> as the missing bound, so every backend must bind both
/// extremes without overflow, ±infinity or a cast error.
/// </summary>
[Fact]
public async Task Extreme_bounds_MaxValue_and_MinValue_are_safe()
{
await ConformanceData.SeedAAsync(B);
var max = new DateTime(DateTime.MaxValue.Ticks, DateTimeKind.Utc);
var min = new DateTime(0, DateTimeKind.Utc);
Values(await B.CreateDataSource().ReadRawAsync("A", At(10), max, 2, Ct)).ShouldBe([20d, 30d]);
Values(await B.CreateDataSource().ReadRawAsync("A", At(100), min, 2, Ct)).ShouldBe([50d, 40d]);
Values(await B.CreateDataSource().ReadRawAsync("A", At(10), max, 0, Ct)).ShouldBe([20d, 30d, 40d, 50d]);
}
[Fact]
public void The_backend_reports_reverse_raw_support()
{
// #686: the server only sends start > end to a backend that declares it; these backends serve it natively.
using var src = B.CreateDataSource();
src.SupportsReverseRaw.ShouldBeTrue();
}
[Fact]
public async Task Reverse_read_is_descending_over_end_exclusive_start_inclusive()
{
await ConformanceData.SeedAAsync(B);
// start=+30, end=+10 ⇒ (+10, +30] descending.
Values(await B.CreateDataSource().ReadRawAsync("A", At(30), At(10), 0, Ct)).ShouldBe([40d, 30d]);
Values(await B.CreateDataSource().ReadRawAsync("A", At(100), At(-1), 2, Ct)).ShouldBe([50d, 40d]);
}
[Fact]
public async Task Equal_start_and_end_is_a_point_read()
{
await ConformanceData.SeedAAsync(B);
Values(await B.CreateDataSource().ReadRawAsync("A", At(10), At(10), 0, Ct)).ShouldBe([20d]);
// One resolution unit later: the stored sample is no longer at that instant.
var later = At(10).AddTicks(B.ResolutionTicks);
(await B.CreateDataSource().ReadRawAsync("A", later, later, 0, Ct)).Samples.ShouldBeEmpty();
}
[Fact]
public async Task A_resumed_page_from_the_last_timestamp_inclusive_continues_without_gaps()
{
// The node manager's paging: read cap N, resume at the last returned timestamp INCLUSIVE, trim it.
await ConformanceData.SeedAAsync(B);
var src = B.CreateDataSource();
var p1 = await src.ReadRawAsync("A", At(0), At(100), 2, Ct);
var p2 = await src.ReadRawAsync("A", p1.Samples[^1].SourceTimestampUtc!.Value, At(100), 2, Ct);
Values(p1).ShouldBe([10d, 20d]);
Values(p2).ShouldBe([20d, 30d]); // first is the boundary sample the node manager trims
}
[Fact]
public async Task Unknown_tag_is_an_empty_result_and_reads_never_create_tags()
{
var src = B.CreateDataSource();
(await src.ReadRawAsync("nope", At(0), At(10), 0, Ct)).Samples.ShouldBeEmpty();
await src.ReadAtTimeAsync("nope", [At(1)], Ct);
await src.ReadProcessedAsync("nope", At(0), At(10), TimeSpan.FromSeconds(5), HistoryAggregateType.Average, Ct);
(await B.TagExistsAsync("nope")).ShouldBeFalse();
src.GetHealthSnapshot().TotalFailures.ShouldBe(0);
}
[Fact]
public async Task Outage_markers_and_bad_samples_come_back_with_their_bad_status()
{
await B.SeedAsync("M",
(At(0), HistorianValue.FromDouble(10), Good),
(At(5), HistorianValue.Empty, BadNoCommunication),
(At(7), HistorianValue.FromDouble(99), Bad));
var r = await B.CreateDataSource().ReadRawAsync("M", At(0), At(10), 0, Ct);
r.Samples.Count.ShouldBe(3);
r.Samples[1].Value.ShouldBeNull();
r.Samples[1].StatusCode.ShouldBe(0x80310000u);
r.Samples[2].Value.ShouldBe(99d);
r.Samples[2].StatusCode.ShouldBe(0x80000000u);
}
[Fact]
public async Task Every_kind_round_trips_to_its_CLR_type()
{
var dt = new DateTime(2025, 1, 2, 3, 4, 5, DateTimeKind.Utc).AddTicks(7 * B.ResolutionTicks);
await B.SeedAsync("K",
(At(0), HistorianValue.FromInt64(-9_000_000_000L), Good),
(At(1), HistorianValue.FromBoolean(true), Good),
(At(2), HistorianValue.FromString("héllo"), Good),
(At(3), HistorianValue.FromDateTime(dt), Good),
(At(4), HistorianValue.FromDouble(1.5), Uncertain));
var s = (await B.CreateDataSource().ReadRawAsync("K", At(0), At(5), 0, Ct)).Samples;
s[0].Value.ShouldBe(-9_000_000_000L);
s[1].Value.ShouldBe(true);
s[2].Value.ShouldBe("héllo");
s[3].Value.ShouldBe(dt);
((DateTime)s[3].Value!).Kind.ShouldBe(DateTimeKind.Utc);
s[4].Value.ShouldBe(1.5);
s[4].StatusCode.ShouldBe(0x40000000u);
}
[Fact]
public async Task Successful_reads_are_counted_in_health()
{
await ConformanceData.SeedAAsync(B);
var src = B.CreateDataSource();
await src.ReadRawAsync("A", At(0), At(10), 0, Ct);
await src.ReadRawAsync("A", At(0), At(10), 0, Ct);
var h = src.GetHealthSnapshot();
h.TotalQueries.ShouldBe(2);
h.TotalSuccesses.ShouldBe(2);
h.ConsecutiveFailures.ShouldBe(0);
h.LastSuccessTime.ShouldNotBeNull();
h.ProcessConnectionOpen.ShouldBeTrue();
}
}