Files
lmxopcua/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer.IntegrationTests/ProcessedReadTests.cs
T
Joseph Doherty 8c6d6aff6a refactor(historian): extract Driver.Historian.Sql.Common from the SQL Server backend (no behaviour change) (1a)
Moves the database-independent logic of the SQL Server historian backend into a new
shared library for the upcoming QuestDB backend (#666): status codes and bits, value/column
mapping, StoredSample + quality clamp, the AtTime and aggregate evaluators, BucketPlan,
LiveWritePreparer and HistoryUpdatePlanner (both with a resolutionTicks parameter, default 1),
tag-name rules, the bootstrap failure cache, health counters, alarm event ids, the event
continuation and severity clamp. The SQL Server backend now calls it; its T-SQL, TVP
builders, connection, bootstrapper and catalog stay.

Tests: StatusCodeOracleTests, ValueWriterPrepareTests, the column half of SampleRowMapperTests
and ProcessedReadTests.Every_enum_member_is_either_served_or_refused move to the new
Sql.Common.Tests under the same names. SqlServer integration tests get mechanical renames
only. Test-name set across the three projects is identical to the baseline (160 tests).

Claude-Session: https://claude.ai/code/session_01Ef2ozxS7QPanGT8uoATbTe
2026-10-11 01:26:49 -04:00

377 lines
19 KiB
C#
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer.IntegrationTests;
/// <summary>
/// Push-down aggregates over tag A (+0=10, +10=20, +20=Bad(30), +30=40, +40=50), 20 s intervals from the
/// sub-millisecond <see cref="ReadTestDb.T0"/>, window [t0, t0+80s) ⇒ four buckets:
/// B0 [0,20) = {10, 20}; B1 [20,40) = {Bad 30, 40}; B2 [40,60) = {50}; B3 [60,80) = {} (empty).
/// Every expected value below is hand-computed.
/// </summary>
[Collection(SqlHistorianCollection.Name)]
public sealed class ProcessedReadTests(SqlHistorianFixture fx)
{
private static readonly CancellationToken Ct = CancellationToken.None;
private static readonly TimeSpan Twenty = TimeSpan.FromSeconds(20);
private static DateTime At(double s) => ReadTestDb.At(s);
// Calculated = InfoType DataValue 0x400 | Calculated 0x1, OR-ed onto the code.
private const uint GoodCalc = 0x00000401u;
private const uint UncertainCalc = 0x40A40401u; // UncertainDataSubNormal | 0x401
private const uint NoDataCalc = 0x809B0401u; // BadNoData | 0x401
private const uint InvalidInputsCalc = 0x80D60401u; // BadAggregateInvalidInputs | 0x401
private static async Task<HistoryReadResult> ReadA(ReadTestDb db, HistoryAggregateType agg) =>
await db.Source().ReadProcessedAsync("A", At(0), At(80), Twenty, agg, Ct);
[Fact]
public void The_status_literals_carry_the_SDK_Calculated_aggregate_bits()
{
new Opc.Ua.StatusCode(GoodCalc).AggregateBits.ShouldBe(Opc.Ua.AggregateBits.Calculated);
new Opc.Ua.StatusCode(UncertainCalc).AggregateBits.ShouldBe(Opc.Ua.AggregateBits.Calculated);
new Opc.Ua.StatusCode(UncertainCalc).CodeBits.ShouldBe(Opc.Ua.StatusCodes.UncertainDataSubNormal);
new Opc.Ua.StatusCode(NoDataCalc).CodeBits.ShouldBe(Opc.Ua.StatusCodes.BadNoData);
new Opc.Ua.StatusCode(NoDataCalc).AggregateBits.ShouldBe(Opc.Ua.AggregateBits.Calculated);
new Opc.Ua.StatusCode(InvalidInputsCalc).CodeBits.ShouldBe(Opc.Ua.StatusCodes.BadAggregateInvalidInputs);
}
[Theory]
[InlineData(HistoryAggregateType.Average, 15d, 40d, 50d)]
[InlineData(HistoryAggregateType.Minimum, 10d, 40d, 50d)]
[InlineData(HistoryAggregateType.Maximum, 20d, 40d, 50d)]
[InlineData(HistoryAggregateType.Total, 30d, 40d, 50d)]
[InlineData(HistoryAggregateType.Start, 10d, 40d, 50d)]
[InlineData(HistoryAggregateType.End, 20d, 40d, 50d)]
[InlineData(HistoryAggregateType.Delta, 10d, 0d, 0d)]
[InlineData(HistoryAggregateType.Range, 10d, 0d, 0d)]
public async Task Numeric_aggregates_over_the_fixture(HistoryAggregateType agg, double b0, double b1, double b2)
{
var db = await ReadTestDb.CreateAsync(fx);
await db.SeedAAsync();
var s = (await ReadA(db, agg)).Samples;
s.Count.ShouldBe(4);
s.Select(x => x.Value).ShouldBe([b0, b1, b2, null]);
// B1 holds a Bad sample alongside a usable one ⇒ Uncertain; the empty B3 ⇒ BadNoData.
s.Select(x => x.StatusCode).ShouldBe([GoodCalc, UncertainCalc, GoodCalc, NoDataCalc]);
s.Select(x => x.SourceTimestampUtc).ShouldBe([At(0), At(20), At(40), At(60)]); // bucket starts
}
[Fact]
public async Task Count_counts_usable_samples_as_int()
{
var db = await ReadTestDb.CreateAsync(fx);
await db.SeedAAsync();
var s = (await ReadA(db, HistoryAggregateType.Count)).Samples;
s.Select(x => x.Value).ShouldBe([(object?)2, 1, 1, null]);
s.Select(x => x.StatusCode).ShouldBe([GoodCalc, UncertainCalc, GoodCalc, NoDataCalc]);
}
[Theory]
[InlineData(HistoryAggregateType.PercentGood, 100d, 50d, 100d)]
[InlineData(HistoryAggregateType.PercentBad, 0d, 50d, 0d)]
public async Task Percent_aggregates_count_every_sample(HistoryAggregateType agg, double b0, double b1, double b2)
{
var db = await ReadTestDb.CreateAsync(fx);
await db.SeedAAsync();
var s = (await ReadA(db, agg)).Samples;
s.Select(x => x.Value).ShouldBe([b0, b1, b2, null]);
// A percentage is a statement about quality, so it is not itself degraded by the Bad input.
s.Select(x => x.StatusCode).ShouldBe([GoodCalc, GoodCalc, GoodCalc, NoDataCalc]);
}
[Fact]
public async Task Actual_time_aggregates_stamp_the_sample_time()
{
var db = await ReadTestDb.CreateAsync(fx);
await db.SeedAAsync();
var min = (await ReadA(db, HistoryAggregateType.MinimumActualTime)).Samples;
min.Select(x => x.Value).ShouldBe([10d, 40d, 50d, null]);
min.Select(x => x.SourceTimestampUtc).ShouldBe([At(0), At(30), At(40), At(60)]);
min.Select(x => x.StatusCode).ShouldBe([GoodCalc, UncertainCalc, GoodCalc, NoDataCalc]);
var max = (await ReadA(db, HistoryAggregateType.MaximumActualTime)).Samples;
max.Select(x => x.Value).ShouldBe([20d, 40d, 50d, null]);
max.Select(x => x.SourceTimestampUtc).ShouldBe([At(10), At(30), At(40), At(60)]);
}
[Fact]
public async Task Actual_time_ties_resolve_to_the_earliest_sample()
{
var db = await ReadTestDb.CreateAsync(fx);
await db.SeedAsync("T",
(At(0), HistorianValue.FromDouble(5), ReadTestDb.Good),
(At(2), HistorianValue.FromDouble(3), ReadTestDb.Good),
(At(4), HistorianValue.FromDouble(3), ReadTestDb.Good),
(At(6), HistorianValue.FromDouble(7), ReadTestDb.Good),
(At(8), HistorianValue.FromDouble(7), ReadTestDb.Good));
var src = db.Source();
var min = (await src.ReadProcessedAsync("T", At(0), At(10), TimeSpan.Zero, HistoryAggregateType.MinimumActualTime, Ct)).Samples.Single();
min.Value.ShouldBe(3d);
min.SourceTimestampUtc.ShouldBe(At(2));
var max = (await src.ReadProcessedAsync("T", At(0), At(10), TimeSpan.Zero, HistoryAggregateType.MaximumActualTime, Ct)).Samples.Single();
max.Value.ShouldBe(7d);
max.SourceTimestampUtc.ShouldBe(At(6));
}
[Fact]
public async Task Bucket_edges_are_start_inclusive_end_exclusive_to_the_tick()
{
// T0 is not millisecond-aligned: a DATEDIFF(millisecond) bucket formula would push the sample one tick
// before the +20 s edge into bucket 1 (it counts millisecond boundaries crossed, not elapsed time).
var db = await ReadTestDb.CreateAsync(fx);
var edge = At(20);
await db.SeedAsync("E",
(At(0).AddTicks(-1), HistorianValue.FromDouble(1000), ReadTestDb.Good), // before start: excluded
(At(0), HistorianValue.FromDouble(1), ReadTestDb.Good), // == start: B0
(edge.AddTicks(-1), HistorianValue.FromDouble(2), ReadTestDb.Good), // last tick of B0
(edge, HistorianValue.FromDouble(4), ReadTestDb.Good), // first tick of B1
(At(40).AddTicks(-1), HistorianValue.FromDouble(8), ReadTestDb.Good), // last tick of B1
(At(40), HistorianValue.FromDouble(2000), ReadTestDb.Good)); // == end: excluded
var s = (await db.Source().ReadProcessedAsync("E", At(0), At(40), Twenty, HistoryAggregateType.Total, Ct)).Samples;
s.Select(x => x.Value).ShouldBe([3d, 12d]);
s.ShouldAllBe(x => x.StatusCode == GoodCalc);
}
[Fact]
public async Task A_partial_last_interval_is_truncated_at_end()
{
var db = await ReadTestDb.CreateAsync(fx);
await db.SeedAAsync();
var s = (await db.Source().ReadProcessedAsync("A", At(0), At(50), Twenty, HistoryAggregateType.Average, Ct)).Samples;
s.Select(x => x.SourceTimestampUtc).ShouldBe([At(0), At(20), At(40)]);
s.Select(x => x.Value).ShouldBe([15d, 40d, 50d]);
}
[Fact]
public async Task A_non_positive_interval_is_one_interval_over_the_window()
{
var db = await ReadTestDb.CreateAsync(fx);
await db.SeedAAsync();
var src = db.Source();
var avg = (await src.ReadProcessedAsync("A", At(0), At(50), TimeSpan.Zero, HistoryAggregateType.Average, Ct)).Samples.Single();
avg.Value.ShouldBe(30d); // usable {10, 20, 40, 50}
avg.StatusCode.ShouldBe(UncertainCalc);
avg.SourceTimestampUtc.ShouldBe(At(0));
(await src.ReadProcessedAsync("A", At(0), At(50), TimeSpan.FromSeconds(-1), HistoryAggregateType.Count, Ct))
.Samples.Single().Value.ShouldBe(4);
}
[Fact]
public async Task A_whole_range_single_interval_from_1601_to_9999_does_not_overflow()
{
// 1601-01-01 is the OPC UA minimum DateTime. A nanosecond DATEDIFF_BIG overflows past ≈292 years (SQL 535),
// and a ns-scaled interval would wrap a long; both must be gone.
var db = await ReadTestDb.CreateAsync(fx);
await db.SeedAAsync();
var start = new DateTime(1601, 1, 1, 0, 0, 0, DateTimeKind.Utc);
var end = new DateTime(9999, 12, 31, 0, 0, 0, DateTimeKind.Utc);
var src = db.Source();
var avg = (await src.ReadProcessedAsync("A", start, end, TimeSpan.Zero, HistoryAggregateType.Average, Ct)).Samples.Single();
avg.Value.ShouldBe(30d); // usable {10, 20, 40, 50}
avg.StatusCode.ShouldBe(UncertainCalc);
avg.SourceTimestampUtc.ShouldBe(start);
(await src.ReadProcessedAsync("A", start, end, TimeSpan.Zero, HistoryAggregateType.Count, Ct))
.Samples.Single().Value.ShouldBe(4);
// Also a whole-DateTime-range window (0001 → 9999-12-31T23:59:59.9999999).
(await src.ReadProcessedAsync("A", DateTime.MinValue, DateTime.MaxValue, TimeSpan.Zero, HistoryAggregateType.Total, Ct))
.Samples.Single().Value.ShouldBe(120d);
// An interval wider than the window (here TimeSpan.MaxValue) is the same single interval, no overflow.
(await src.ReadProcessedAsync("A", start, end, TimeSpan.MaxValue, HistoryAggregateType.Total, Ct))
.Samples.Single().Value.ShouldBe(120d);
src.GetHealthSnapshot().TotalFailures.ShouldBe(0);
}
[Fact]
public async Task A_weekly_interval_over_four_centuries_lands_samples_in_the_right_week()
{
// Hand-computed (calendar arithmetic, independent of SQL): 1601-01-01 → 2027-01-01 is 155 593 days =
// 22 227.57 weeks ⇒ 22 228 intervals (under the 100k cap). 2026-10-09 08:00 is 155 509 days in ⇒ week
// index 22 215, which starts Monday 2026-10-05 00:00; every fixture sample (t0 … t0+40 s) is in it.
var db = await ReadTestDb.CreateAsync(fx);
await db.SeedAAsync();
var start = new DateTime(1601, 1, 1, 0, 0, 0, DateTimeKind.Utc);
var end = new DateTime(2027, 1, 1, 0, 0, 0, DateTimeKind.Utc);
var s = (await db.Source().ReadProcessedAsync("A", start, end, TimeSpan.FromDays(7), HistoryAggregateType.Average, Ct)).Samples;
s.Count.ShouldBe(22_228);
var hit = s[22_215];
hit.SourceTimestampUtc.ShouldBe(new DateTime(2026, 10, 5, 0, 0, 0, DateTimeKind.Utc));
hit.Value.ShouldBe(30d);
hit.StatusCode.ShouldBe(UncertainCalc);
s.Where((_, i) => i != 22_215).ShouldAllBe(x => x.StatusCode == NoDataCalc);
s[^1].SourceTimestampUtc.ShouldBe(new DateTime(2026, 12, 28, 0, 0, 0, DateTimeKind.Utc)); // partial last week
}
[Fact]
public async Task A_reverse_window_is_computed_forward_and_returned_descending()
{
var db = await ReadTestDb.CreateAsync(fx);
await db.SeedAAsync();
var s = (await db.Source().ReadProcessedAsync("A", At(80), At(0), Twenty, HistoryAggregateType.Average, Ct)).Samples;
s.Select(x => x.SourceTimestampUtc).ShouldBe([At(60), At(40), At(20), At(0)]);
s.Select(x => x.Value).ShouldBe([null, 50d, 40d, 15d]);
}
[Fact]
public async Task Equal_start_and_end_yields_no_intervals()
{
var db = await ReadTestDb.CreateAsync(fx);
await db.SeedAAsync();
(await db.Source().ReadProcessedAsync("A", At(10), At(10), Twenty, HistoryAggregateType.Average, Ct))
.Samples.ShouldBeEmpty();
}
[Fact]
public async Task Outage_markers_are_bad_samples_in_aggregates()
{
var db = await ReadTestDb.CreateAsync(fx);
await db.SeedAsync("M",
(At(0), HistorianValue.FromDouble(10), ReadTestDb.Good),
(At(5), HistorianValue.Empty, ReadTestDb.BadNoCommunication),
(At(10), HistorianValue.FromDouble(30), ReadTestDb.Good));
var src = db.Source();
async Task<DataValueSnapshot> One(HistoryAggregateType a) =>
(await src.ReadProcessedAsync("M", At(0), At(20), Twenty, a, Ct)).Samples.Single();
var avg = await One(HistoryAggregateType.Average);
avg.Value.ShouldBe(20d);
avg.StatusCode.ShouldBe(UncertainCalc);
(await One(HistoryAggregateType.Count)).Value.ShouldBe(2);
(await One(HistoryAggregateType.Start)).Value.ShouldBe(10d);
((double)(await One(HistoryAggregateType.PercentBad)).Value!).ShouldBe(33.333333333333336, 1e-9); // 100 × 1/3
}
[Fact]
public async Task An_interval_of_only_bad_samples_is_BadNoData_except_the_percentages()
{
var db = await ReadTestDb.CreateAsync(fx);
await db.SeedAsync("Z",
(At(1), HistorianValue.Empty, ReadTestDb.BadNoCommunication),
(At(2), HistorianValue.FromDouble(5), ReadTestDb.Bad));
var src = db.Source();
async Task<DataValueSnapshot> One(HistoryAggregateType a) =>
(await src.ReadProcessedAsync("Z", At(0), At(20), Twenty, a, Ct)).Samples.Single();
(await One(HistoryAggregateType.Average)).StatusCode.ShouldBe(NoDataCalc);
(await One(HistoryAggregateType.Count)).StatusCode.ShouldBe(NoDataCalc);
var pb = await One(HistoryAggregateType.PercentBad);
pb.Value.ShouldBe(100d);
pb.StatusCode.ShouldBe(GoodCalc);
}
[Fact]
public async Task Uncertain_samples_are_used_but_make_the_interval_uncertain()
{
var db = await ReadTestDb.CreateAsync(fx);
await db.SeedAsync("U",
(At(1), HistorianValue.FromDouble(4), ReadTestDb.Uncertain),
(At(2), HistorianValue.FromDouble(6), ReadTestDb.Good));
var s = (await db.Source().ReadProcessedAsync("U", At(0), At(20), Twenty, HistoryAggregateType.Average, Ct)).Samples.Single();
s.Value.ShouldBe(5d);
s.StatusCode.ShouldBe(UncertainCalc);
}
[Fact]
public async Task String_tags_serve_count_start_end_and_refuse_numeric_aggregates()
{
var db = await ReadTestDb.CreateAsync(fx);
await db.SeedAsync("S",
(At(0), HistorianValue.FromString("a"), ReadTestDb.Good),
(At(10), HistorianValue.FromString("b"), ReadTestDb.Good));
var src = db.Source();
async Task<DataValueSnapshot> One(HistoryAggregateType a) =>
(await src.ReadProcessedAsync("S", At(0), At(20), Twenty, a, Ct)).Samples.Single();
(await One(HistoryAggregateType.Count)).Value.ShouldBe(2);
(await One(HistoryAggregateType.Start)).Value.ShouldBe("a");
(await One(HistoryAggregateType.End)).Value.ShouldBe("b");
(await One(HistoryAggregateType.PercentGood)).Value.ShouldBe(100d);
foreach (var a in new[]
{
HistoryAggregateType.Average, HistoryAggregateType.Minimum, HistoryAggregateType.Maximum,
HistoryAggregateType.Total, HistoryAggregateType.Delta, HistoryAggregateType.Range,
HistoryAggregateType.MinimumActualTime, HistoryAggregateType.MaximumActualTime,
})
{
var r = await One(a);
r.StatusCode.ShouldBe(InvalidInputsCalc, a.ToString());
r.Value.ShouldBeNull();
}
}
[Fact]
public async Task DateTime_tags_refuse_numeric_aggregates()
{
var db = await ReadTestDb.CreateAsync(fx);
await db.SeedAsync("D", (At(0), HistorianValue.FromDateTime(At(-100)), ReadTestDb.Good));
(await db.Source().ReadProcessedAsync("D", At(0), At(20), Twenty, HistoryAggregateType.Average, Ct))
.Samples.Single().StatusCode.ShouldBe(InvalidInputsCalc);
}
[Fact]
public async Task Boolean_tags_aggregate_as_zero_or_one()
{
var db = await ReadTestDb.CreateAsync(fx);
await db.SeedAsync("B",
(At(0), HistorianValue.FromBoolean(true), ReadTestDb.Good),
(At(5), HistorianValue.FromBoolean(false), ReadTestDb.Good),
(At(10), HistorianValue.FromBoolean(true), ReadTestDb.Good),
(At(15), HistorianValue.FromBoolean(true), ReadTestDb.Good));
var src = db.Source();
async Task<DataValueSnapshot> One(HistoryAggregateType a) =>
(await src.ReadProcessedAsync("B", At(0), At(20), Twenty, a, Ct)).Samples.Single();
(await One(HistoryAggregateType.Average)).Value.ShouldBe(0.75d);
(await One(HistoryAggregateType.Total)).Value.ShouldBe(3d);
(await One(HistoryAggregateType.Start)).Value.ShouldBe(true);
(await One(HistoryAggregateType.Count)).Value.ShouldBe(4);
}
[Fact]
public async Task Unknown_tag_yields_every_interval_as_BadNoData()
{
var db = await ReadTestDb.CreateAsync(fx);
var s = (await db.Source().ReadProcessedAsync("nope", At(0), At(60), Twenty, HistoryAggregateType.Average, Ct)).Samples;
s.Count.ShouldBe(3);
s.ShouldAllBe(x => x.StatusCode == NoDataCalc && x.Value == null);
}
[Theory]
[InlineData(HistoryAggregateType.Interpolative)]
[InlineData(HistoryAggregateType.TimeAverage)]
[InlineData(HistoryAggregateType.StartBound)]
[InlineData(HistoryAggregateType.EndBound)]
[InlineData(HistoryAggregateType.DeltaBounds)]
[InlineData(HistoryAggregateType.DurationGood)]
[InlineData(HistoryAggregateType.DurationBad)]
[InlineData(HistoryAggregateType.WorstQuality)]
[InlineData(HistoryAggregateType.NumberOfTransitions)]
[InlineData(HistoryAggregateType.AnnotationCount)]
[InlineData(HistoryAggregateType.StandardDeviationPopulation)]
[InlineData(HistoryAggregateType.VariancePopulation)]
public async Task Unsupported_aggregates_throw_not_supported_and_are_not_health_failures(HistoryAggregateType agg)
{
var db = await ReadTestDb.CreateAsync(fx);
var src = db.Source();
var ex = await Should.ThrowAsync<HistoryAggregateNotSupportedException>(
() => src.ReadProcessedAsync("A", At(0), At(20), Twenty, agg, Ct));
ex.Aggregate.ShouldBe(agg);
src.GetHealthSnapshot().TotalQueries.ShouldBe(0);
}
[Fact]
public async Task The_bucket_count_is_capped_at_100000()
{
var db = await ReadTestDb.CreateAsync(fx);
var src = db.Source();
var oneTick = TimeSpan.FromTicks(1);
(await src.ReadProcessedAsync("nope", At(0), At(0).AddTicks(100_000), oneTick, HistoryAggregateType.Count, Ct))
.Samples.Count.ShouldBe(100_000);
await Should.ThrowAsync<ArgumentOutOfRangeException>(
() => src.ReadProcessedAsync("nope", At(0), At(0).AddTicks(100_001), oneTick, HistoryAggregateType.Count, Ct));
var h = src.GetHealthSnapshot();
h.TotalFailures.ShouldBe(0); // a client-request error, not a backend fault
}
}