Files
lmxopcua/tests/Drivers/Historian.Conformance/ProcessedConformance.cs
T
Joseph Doherty 0d206a0884 fix(historian): #680 review — recovery/only-sample TA2/Total2 value, outage-last + Boolean parity, docs
TimeAverage2/Total2: the interval the data ends inside with no non-Bad time
before its last sample (comms-loss recovery, a single sample) answers that
sample's value (Total2 0), Uncertain + Partial, instead of Bad/BadNoData —
as the SDK. Parity datasets RecoveryEnds, OutageLast, BoolSwitch with
documented masks; ProcessedConformance's reverse TA/Total mask narrowed to
the later-edge-sample interval. Docs: report-by-exception consequence,
outage-last and Boolean differences; CLAUDE.md historian bullet.
2026-10-11 16:02:22 -04:00

703 lines
40 KiB
C#
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common;
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Conformance;
/// <summary>
/// Push-down aggregates over tag A (+0=10, +10=20, +20=Bad(30), +30=40, +40=50), 20 s intervals from the
/// sub-millisecond <see cref="ConformanceData.T0"/>, window [t0, t0+80s) ⇒ four buckets:
/// B0 [0,20) = {10, 20}; B1 [20,40) = {Bad 30, 40}; B2 [40,60) = {50}; B3 [60,80) = {} (empty).
/// Every expected value below is hand-computed.
/// </summary>
public abstract class ProcessedConformance : ConformanceBase
{
private static readonly CancellationToken Ct = CancellationToken.None;
private static readonly TimeSpan Twenty = TimeSpan.FromSeconds(20);
// Calculated = InfoType DataValue 0x400 | Calculated 0x1, OR-ed onto the code.
private const uint GoodCalc = 0x00000401u;
private const uint UncertainCalc = 0x40A40401u; // UncertainDataSubNormal | 0x401
private const uint NoDataCalc = 0x809B0401u; // BadNoData | 0x401
private const uint InvalidInputsCalc = 0x80D60401u; // BadAggregateInvalidInputs | 0x401
private async Task<HistoryReadResult> ReadA(HistoryAggregateType agg) =>
await B.CreateDataSource().ReadProcessedAsync("A", At(0), At(80), Twenty, agg, Ct);
[Fact]
public void The_status_literals_carry_the_SDK_Calculated_aggregate_bits()
{
new Opc.Ua.StatusCode(GoodCalc).AggregateBits.ShouldBe(Opc.Ua.AggregateBits.Calculated);
new Opc.Ua.StatusCode(UncertainCalc).AggregateBits.ShouldBe(Opc.Ua.AggregateBits.Calculated);
new Opc.Ua.StatusCode(UncertainCalc).CodeBits.ShouldBe(Opc.Ua.StatusCodes.UncertainDataSubNormal);
new Opc.Ua.StatusCode(NoDataCalc).CodeBits.ShouldBe(Opc.Ua.StatusCodes.BadNoData);
new Opc.Ua.StatusCode(NoDataCalc).AggregateBits.ShouldBe(Opc.Ua.AggregateBits.Calculated);
new Opc.Ua.StatusCode(InvalidInputsCalc).CodeBits.ShouldBe(Opc.Ua.StatusCodes.BadAggregateInvalidInputs);
}
[Theory]
[InlineData(HistoryAggregateType.Average, 15d, 40d, 50d)]
[InlineData(HistoryAggregateType.Minimum, 10d, 40d, 50d)]
[InlineData(HistoryAggregateType.Maximum, 20d, 40d, 50d)]
[InlineData(HistoryAggregateType.Start, 10d, 40d, 50d)]
[InlineData(HistoryAggregateType.End, 20d, 40d, 50d)]
[InlineData(HistoryAggregateType.Delta, 10d, 0d, 0d)]
[InlineData(HistoryAggregateType.Range, 10d, 0d, 0d)]
public async Task Numeric_aggregates_over_the_fixture(HistoryAggregateType agg, double b0, double b1, double b2)
{
await ConformanceData.SeedAAsync(B);
var s = (await ReadA(agg)).Samples;
s.Count.ShouldBe(4);
s.Select(x => x.Value).ShouldBe([b0, b1, b2, null]);
// B1 holds a Bad sample alongside a usable one ⇒ Uncertain; the empty B3 ⇒ BadNoData.
s.Select(x => x.StatusCode).ShouldBe([GoodCalc, UncertainCalc, GoodCalc, NoDataCalc]);
s.Select(x => x.SourceTimestampUtc).ShouldBe([B.Round(At(0)), B.Round(At(20)), B.Round(At(40)), B.Round(At(60))]); // bucket starts
}
[Fact]
public async Task Count_counts_usable_samples_as_int()
{
await ConformanceData.SeedAAsync(B);
var s = (await ReadA(HistoryAggregateType.Count)).Samples;
s.Select(x => x.Value).ShouldBe([(object?)2, 1, 1, null]);
s.Select(x => x.StatusCode).ShouldBe([GoodCalc, UncertainCalc, GoodCalc, NoDataCalc]);
}
[Theory]
[InlineData(HistoryAggregateType.PercentGood, 100d, 50d, 100d)]
[InlineData(HistoryAggregateType.PercentBad, 0d, 50d, 0d)]
public async Task Percent_aggregates_are_shares_of_the_interval_duration(HistoryAggregateType agg, double b0, double b1,
double b2)
{
// #680 (Part 13 §5.4.3.31-34): stepped regions, each with the status of the sample starting it. B1: [20,30) Bad,
// [30,40) Good. B2 holds the end of the data (+40, nothing after) ⇒ its region runs to the interval end, Partial;
// B3 is after the end of the data ⇒ BadNoData.
await ConformanceData.SeedAAsync(B);
var s = (await ReadA(agg)).Samples;
s.Select(x => x.Value).ShouldBe([b0, b1, b2, null]);
// A percentage is a statement about quality, so it is not itself degraded by the Bad input.
s.Select(x => x.StatusCode).ShouldBe([GoodCalc, GoodCalc, GoodCalc | 0x4u, NoDataCalc]);
}
[Fact]
public async Task Actual_time_aggregates_stamp_the_sample_time()
{
await ConformanceData.SeedAAsync(B);
var min = (await ReadA(HistoryAggregateType.MinimumActualTime)).Samples;
min.Select(x => x.Value).ShouldBe([10d, 40d, 50d, null]);
min.Select(x => x.SourceTimestampUtc).ShouldBe([B.Round(At(0)), B.Round(At(30)), B.Round(At(40)), B.Round(At(60))]);
min.Select(x => x.StatusCode).ShouldBe([GoodCalc, UncertainCalc, GoodCalc, NoDataCalc]);
var max = (await ReadA(HistoryAggregateType.MaximumActualTime)).Samples;
max.Select(x => x.Value).ShouldBe([20d, 40d, 50d, null]);
max.Select(x => x.SourceTimestampUtc).ShouldBe([B.Round(At(10)), B.Round(At(30)), B.Round(At(40)), B.Round(At(60))]);
}
[Fact]
public async Task Actual_time_ties_resolve_to_the_earliest_sample()
{
await B.SeedAsync("T",
(At(0), HistorianValue.FromDouble(5), Good),
(At(2), HistorianValue.FromDouble(3), Good),
(At(4), HistorianValue.FromDouble(3), Good),
(At(6), HistorianValue.FromDouble(7), Good),
(At(8), HistorianValue.FromDouble(7), Good));
var src = B.CreateDataSource();
var min = (await src.ReadProcessedAsync("T", At(0), At(10), TimeSpan.Zero, HistoryAggregateType.MinimumActualTime, Ct)).Samples.Single();
min.Value.ShouldBe(3d);
min.SourceTimestampUtc.ShouldBe(B.Round(At(2)));
var max = (await src.ReadProcessedAsync("T", At(0), At(10), TimeSpan.Zero, HistoryAggregateType.MaximumActualTime, Ct)).Samples.Single();
max.Value.ShouldBe(7d);
max.SourceTimestampUtc.ShouldBe(B.Round(At(6)));
}
[Fact]
public async Task Bucket_edges_are_start_inclusive_end_exclusive_to_the_tick()
{
// T0 is not millisecond-aligned: a DATEDIFF(millisecond) bucket formula would push the sample one tick
// before the +20 s edge into bucket 1 (it counts millisecond boundaries crossed, not elapsed time).
var edge = At(20);
await B.SeedAsync("E",
(At(0).AddTicks(-B.ResolutionTicks), HistorianValue.FromDouble(1000), Good), // before start: excluded
(At(0), HistorianValue.FromDouble(1), Good), // == start: B0
(edge.AddTicks(-B.ResolutionTicks), HistorianValue.FromDouble(2), Good), // last tick of B0
(edge, HistorianValue.FromDouble(4), Good), // first tick of B1
(At(40).AddTicks(-B.ResolutionTicks), HistorianValue.FromDouble(8), Good), // last tick of B1
(At(40), HistorianValue.FromDouble(2000), Good)); // == end: excluded
var s = (await B.CreateDataSource().ReadProcessedAsync("E", At(0), At(40), Twenty, HistoryAggregateType.Average, Ct)).Samples;
s.Select(x => x.Value).ShouldBe([1.5d, 6d]);
s.ShouldAllBe(x => x.StatusCode == GoodCalc);
}
[Fact]
public async Task A_partial_last_interval_is_truncated_at_end()
{
await ConformanceData.SeedAAsync(B);
var s = (await B.CreateDataSource().ReadProcessedAsync("A", At(0), At(50), Twenty, HistoryAggregateType.Average, Ct)).Samples;
s.Select(x => x.SourceTimestampUtc).ShouldBe([B.Round(At(0)), B.Round(At(20)), B.Round(At(40))]);
s.Select(x => x.Value).ShouldBe([15d, 40d, 50d]);
}
[Fact]
public async Task A_non_positive_interval_is_one_interval_over_the_window()
{
await ConformanceData.SeedAAsync(B);
var src = B.CreateDataSource();
var avg = (await src.ReadProcessedAsync("A", At(0), At(50), TimeSpan.Zero, HistoryAggregateType.Average, Ct)).Samples.Single();
avg.Value.ShouldBe(30d); // usable {10, 20, 40, 50}
avg.StatusCode.ShouldBe(UncertainCalc);
avg.SourceTimestampUtc.ShouldBe(B.Round(At(0)));
(await src.ReadProcessedAsync("A", At(0), At(50), TimeSpan.FromSeconds(-1), HistoryAggregateType.Count, Ct))
.Samples.Single().Value.ShouldBe(4);
}
[Fact]
public async Task A_whole_range_single_interval_from_1601_to_9999_does_not_overflow()
{
// 1601-01-01 is the OPC UA minimum DateTime. A nanosecond DATEDIFF_BIG overflows past ≈292 years (SQL 535),
// and a ns-scaled interval would wrap a long; both must be gone.
await ConformanceData.SeedAAsync(B);
var start = new DateTime(1601, 1, 1, 0, 0, 0, DateTimeKind.Utc);
var end = new DateTime(9999, 12, 31, 0, 0, 0, DateTimeKind.Utc);
var src = B.CreateDataSource();
var avg = (await src.ReadProcessedAsync("A", start, end, TimeSpan.Zero, HistoryAggregateType.Average, Ct)).Samples.Single();
avg.Value.ShouldBe(30d); // usable {10, 20, 40, 50}
avg.StatusCode.ShouldBe(UncertainCalc);
avg.SourceTimestampUtc.ShouldBe(start);
(await src.ReadProcessedAsync("A", start, end, TimeSpan.Zero, HistoryAggregateType.Count, Ct))
.Samples.Single().Value.ShouldBe(4);
// Also a whole-DateTime-range window (0001 → 9999-12-31T23:59:59.9999999).
(await src.ReadProcessedAsync("A", DateTime.MinValue, DateTime.MaxValue, TimeSpan.Zero, HistoryAggregateType.Count, Ct))
.Samples.Single().Value.ShouldBe(4);
// An interval wider than the window (here TimeSpan.MaxValue) is the same single interval, no overflow.
(await src.ReadProcessedAsync("A", start, end, TimeSpan.MaxValue, HistoryAggregateType.Count, Ct))
.Samples.Single().Value.ShouldBe(4);
// The time-weighted path over the whole range. TimeAverage2 is not extrapolated past the end of the data (+40, #680):
// (150 + 200 + 450) value·s over the 30 non-Bad seconds = 80/3.
var tw = (await src.ReadProcessedAsync("A", start, end, TimeSpan.Zero, HistoryAggregateType.TimeAverage2, Ct)).Samples.Single();
((double)tw.Value!).ShouldBe(80d / 3d, 1e-6);
// The data starts (+0) strictly inside the one interval ⇒ Partial as well (#677, the SDK's m_startOfData rule).
tw.StatusCode.ShouldBe(UncertainCalc | 0x4u);
src.GetHealthSnapshot().TotalFailures.ShouldBe(0);
}
[Fact]
public async Task A_weekly_interval_over_four_centuries_lands_samples_in_the_right_week()
{
// Hand-computed (calendar arithmetic, independent of SQL): 1601-01-01 → 2027-01-01 is 155 593 days =
// 22 227.57 weeks ⇒ 22 228 intervals (under the 100k cap). 2026-10-09 08:00 is 155 509 days in ⇒ week
// index 22 215, which starts Monday 2026-10-05 00:00; every fixture sample (t0 … t0+40 s) is in it.
await ConformanceData.SeedAAsync(B);
var start = new DateTime(1601, 1, 1, 0, 0, 0, DateTimeKind.Utc);
var end = new DateTime(2027, 1, 1, 0, 0, 0, DateTimeKind.Utc);
var s = (await B.CreateDataSource().ReadProcessedAsync("A", start, end, TimeSpan.FromDays(7), HistoryAggregateType.Average, Ct)).Samples;
s.Count.ShouldBe(22_228);
var hit = s[22_215];
hit.SourceTimestampUtc.ShouldBe(new DateTime(2026, 10, 5, 0, 0, 0, DateTimeKind.Utc));
hit.Value.ShouldBe(30d);
hit.StatusCode.ShouldBe(UncertainCalc);
s.Where((_, i) => i != 22_215).ShouldAllBe(x => x.StatusCode == NoDataCalc);
s[^1].SourceTimestampUtc.ShouldBe(new DateTime(2026, 12, 28, 0, 0, 0, DateTimeKind.Utc)); // partial last week
}
[Fact]
public async Task A_reverse_window_is_anchored_at_its_later_time()
{
// #677 (Part 13 / SDK): At(80) → At(0) is the window (0,80] in intervals (60,80], (40,60], (20,40], (0,20], each
// stamped with its LATER edge, latest first. A sample on an edge belongs to the interval it ends: +40 is in
// (20,40], +20 (Bad) in (0,20], and +0 is outside the window. (The pre-#677 layout answered [60,80) … [0,20)
// stamped 60/40/20/0 with values null, 50, 40, 15.)
await ConformanceData.SeedAAsync(B);
var s = (await B.CreateDataSource().ReadProcessedAsync("A", At(80), At(0), Twenty, HistoryAggregateType.Average, Ct)).Samples;
s.Select(x => x.SourceTimestampUtc).ShouldBe([B.Round(At(80)), B.Round(At(60)), B.Round(At(40)), B.Round(At(20))]);
s.Select(x => x.Value).ShouldBe([null, null, 45d, 20d]);
s.Select(x => x.StatusCode).ShouldBe([NoDataCalc, NoDataCalc, GoodCalc, UncertainCalc]);
}
[Fact]
public async Task A_reverse_window_truncates_its_earliest_interval()
{
// At(50) → At(0), 20 s over tag A (+0=10, +10=20, +20=Bad, +30=40, +40=50): (30,50] = {+40} → 50 (+30 is its
// earlier edge, excluded); (10,30] = {+20 Bad, +30} → 40, Uncertain; (0,10] = {+10} → 20 — the truncated interval
// is the EARLIEST one. (Pre-#677: [0,20), [20,40), [40,50) stamped 40/20/0 with values 50, 40, 15.)
await ConformanceData.SeedAAsync(B);
var s = (await B.CreateDataSource().ReadProcessedAsync("A", At(50), At(0), Twenty, HistoryAggregateType.Average, Ct)).Samples;
s.Select(x => x.SourceTimestampUtc).ShouldBe([B.Round(At(50)), B.Round(At(30)), B.Round(At(10))]);
s.Select(x => x.Value).ShouldBe([50d, 40d, 20d]);
s.Select(x => x.StatusCode).ShouldBe([GoodCalc, UncertainCalc, GoodCalc]);
}
[Fact]
public async Task Reverse_bucket_edges_are_later_edge_inclusive_to_the_tick()
{
// The mirror of the forward edge test: At(40) → At(0), 20 s ⇒ (20,40], (0,20]. The bucket math is pushed down,
// so this pins the SQL, not just the C# plan.
var r = B.ResolutionTicks;
await B.SeedAsync("ER",
(At(0), HistorianValue.FromDouble(1000), Good), // == earlier edge: excluded
(At(0).AddTicks(r), HistorianValue.FromDouble(1), Good), // first tick of (0,20]
(At(20), HistorianValue.FromDouble(2), Good), // == edge: the LATER edge of (0,20]
(At(20).AddTicks(r), HistorianValue.FromDouble(4), Good), // first tick of (20,40]
(At(40), HistorianValue.FromDouble(8), Good), // == request start: included, in (20,40]
(At(40).AddTicks(r), HistorianValue.FromDouble(2000), Good)); // after the window: excluded
var s = (await B.CreateDataSource().ReadProcessedAsync("ER", At(40), At(0), Twenty, HistoryAggregateType.Average, Ct)).Samples;
s.Select(x => x.SourceTimestampUtc).ShouldBe([B.Round(At(40)), B.Round(At(20))]);
s.Select(x => x.Value).ShouldBe([6d, 1.5d]);
}
[Fact]
public async Task The_interval_the_data_starts_inside_carries_the_Partial_bit()
{
// #677 (SDK m_startOfData): the tag's first sample is +5, inside [0,20) ⇒ Partial there for every aggregate but
// Average; [20,40) is fully covered. With an earlier sample stored (even outside the window) nothing is Partial.
await B.SeedAsync("LS",
(At(5), HistorianValue.FromDouble(10), Good),
(At(15), HistorianValue.FromDouble(30), Good),
(At(25), HistorianValue.FromDouble(20), Good),
(At(60), HistorianValue.FromDouble(20), Good));
var src = B.CreateDataSource();
foreach (var agg in new[] { HistoryAggregateType.Count, HistoryAggregateType.Minimum, HistoryAggregateType.TimeAverage2 })
{
var s = (await src.ReadProcessedAsync("LS", At(0), At(40), Twenty, agg, Ct)).Samples;
(s[0].StatusCode & 0x4u).ShouldBe(0x4u, agg.ToString());
(s[1].StatusCode & 0x4u).ShouldBe(0u, agg.ToString());
}
(await src.ReadProcessedAsync("LS", At(0), At(40), Twenty, HistoryAggregateType.Average, Ct)).Samples
.ShouldAllBe(x => (x.StatusCode & 0x4u) == 0);
await B.SeedAsync("LS2",
(At(-100), HistorianValue.FromDouble(10), Good),
(At(5), HistorianValue.FromDouble(10), Good),
(At(15), HistorianValue.FromDouble(30), Good),
(At(25), HistorianValue.FromDouble(20), Good));
(await src.ReadProcessedAsync("LS2", At(0), At(40), Twenty, HistoryAggregateType.Count, Ct)).Samples
.ShouldAllBe(x => (x.StatusCode & 0x4u) == 0);
}
[Fact]
public async Task A_reverse_read_sets_no_start_of_data_Partial_and_matches_the_SDK()
{
// Review F1: the SDK applies the start-of-data rule to FORWARD reads only. Reverse 40 → 0 at 10 s over data that
// starts at +7: (0,10] holds the data's start but is not Partial — checked through the real SQL and the SDK oracle.
var rows = Enumerable.Range(0, 20)
.Select(i => (At(7 + i * 5), HistorianValue.FromDouble(i % 2 == 0 ? 20 : 40), Good)).ToArray();
await B.SeedAsync("RLS", rows);
var sdkRows = rows.Select(r => (B.Round(r.Item1), r.Item2, r.Item3)).ToArray();
var src = B.CreateDataSource();
var ten = TimeSpan.FromSeconds(10);
foreach (var agg in new[]
{
HistoryAggregateType.Count, HistoryAggregateType.Minimum, HistoryAggregateType.Range,
HistoryAggregateType.Delta, HistoryAggregateType.TimeAverage, HistoryAggregateType.TimeAverage2,
HistoryAggregateType.Total2,
})
{
var ours = (await src.ReadProcessedAsync("RLS", At(40), At(0), ten, agg, Ct)).Samples;
ours.ShouldAllBe(x => (x.StatusCode & 0x4u) == 0, agg.ToString());
var tw = AggregateFamilies.FamilyOf(agg) == AggregateFamily.TimeWeighted;
SdkAggregateOracle.ShouldMatch(ours, SdkAggregateOracle.Run(agg, B.Round(At(40)), B.Round(At(0)), ten, sdkRows),
$"{agg} reverse late start", compareCalculatedInterpolatedBits: tw,
// pre-existing, #680: the SDK's Delta is Good with a Bad sample present; ours Uncertain (none here).
compareCodeBits: agg != HistoryAggregateType.Delta);
}
}
/// <summary>Dense: every 2.5 s from −10 to +100, values cycling, one Bad sample at +22.5 (no empty interval).</summary>
private Task SeedDenseAsync() => B.SeedAsync("P", DenseRows());
private static (DateTime T, HistorianValue V, uint Status)[] DenseRows() => Enumerable.Range(0, 45)
.Select(i => (At(-10 + i * 2.5), HistorianValue.FromDouble(10 + (i * 7 % 13)), i == 13 ? Bad : Good))
.ToArray();
[Theory]
[InlineData(0, 80, 20)]
[InlineData(80, 0, 20)]
[InlineData(77, 3, 20)]
[InlineData(3, 77, 20)]
public async Task Processed_reads_match_the_SDK_aggregate_calculators(double from, double to, double intervalS)
{
// #677: the real backend (pushed-down SQL bucket math + the shared evaluators) against the pinned SDK's own Part 13
// calculators over the same stored rows — timestamps, values, status code bits and the Partial bit, forward and
// reverse. See SdkAggregateParityTests (Sql.Common.Tests) for the documented divergences this avoids/masks.
await SeedDenseAsync();
var rows = DenseRows().Select(r => (B.Round(r.T), r.V, r.Status)).ToArray();
var src = B.CreateDataSource();
var interval = TimeSpan.FromSeconds(intervalS);
// The SDK sees the window the backend actually serves (QuestDB rounds the bounds to µs).
var (start, end) = (B.Round(At(from)), B.Round(At(to)));
foreach (var agg in new[]
{
HistoryAggregateType.Average, HistoryAggregateType.Minimum, HistoryAggregateType.Maximum,
HistoryAggregateType.Count, HistoryAggregateType.TimeAverage, HistoryAggregateType.TimeAverage2,
HistoryAggregateType.Total, HistoryAggregateType.Total2, HistoryAggregateType.Interpolative,
HistoryAggregateType.Range, HistoryAggregateType.Start, HistoryAggregateType.End,
HistoryAggregateType.Delta, HistoryAggregateType.MinimumActualTime, HistoryAggregateType.MaximumActualTime,
HistoryAggregateType.PercentGood, HistoryAggregateType.PercentBad,
})
{
var ours = (await src.ReadProcessedAsync("P", At(from), At(to), interval, agg, Ct)).Samples.ToList();
var startOrEnd = agg is HistoryAggregateType.Start or HistoryAggregateType.End;
var tw = AggregateFamilies.FamilyOf(agg) == AggregateFamily.TimeWeighted;
if (tw && agg != HistoryAggregateType.Interpolative)
{
// Documented divergence: ours flags the truncated interval Partial, the SDK does not.
var plan = BucketPlan.Create(start, end, interval);
for (long i = 0; i < plan.BucketCount; i++)
if (plan.IsTruncated(i))
{
var k = (int)(plan.Reversed ? plan.BucketCount - 1 - i : i);
ours[k] = ours[k] with { StatusCode = ours[k].StatusCode & ~0x4u };
}
}
var reverseInterpolative = agg == HistoryAggregateType.Interpolative && from > to;
var sdk = SdkAggregateOracle.Run(agg, start, end, interval, rows);
if (from > to && agg is HistoryAggregateType.TimeAverage or HistoryAggregateType.Total)
{
// Documented SDK divergence (#680, see SdkAggregateParityTests.MaskDocumented): on a reverse read the SDK files
// a sample exactly on an interval's later edge as its early bound, so a TimeAverage/Total interval ending on a
// sample takes the status of the segment beyond it (Dense: +20 next to the Bad +22.5). Only that interval,
// only Good (ours) vs Uncertain (SDK), is aligned.
var plan = BucketPlan.Create(start, end, interval);
for (long i = 0; i < plan.BucketCount; i++)
{
var k = (int)(plan.BucketCount - 1 - i);
if (k < sdk.Count && rows.Any(r => r.Item1.Ticks == plan.UpperTicks(i))
&& (ours[k].StatusCode & 0xFFFF0000u) == 0
&& (sdk[k].StatusCode & 0xFFFF0000u) == 0x40A40000u)
ours[k] = ours[k] with { StatusCode = ours[k].StatusCode | 0x40A40000u };
}
}
SdkAggregateOracle.ShouldMatch(ours, sdk,
$"{agg} {from}→{to}/{intervalS}s",
compareCalculatedInterpolatedBits: tw && !reverseInterpolative,
// pre-existing, #680: Start/End raw-sample timestamp + status; Delta status (see SdkAggregateParityTests).
compareCodeBits: agg != HistoryAggregateType.Delta && !reverseInterpolative && !startOrEnd,
compareTimestamps: !startOrEnd);
}
}
[Fact]
public async Task Equal_start_and_end_yields_no_intervals()
{
await ConformanceData.SeedAAsync(B);
(await B.CreateDataSource().ReadProcessedAsync("A", At(10), At(10), Twenty, HistoryAggregateType.Average, Ct))
.Samples.ShouldBeEmpty();
}
[Fact]
public async Task Outage_markers_are_bad_samples_in_aggregates()
{
await B.SeedAsync("M",
(At(0), HistorianValue.FromDouble(10), Good),
(At(5), HistorianValue.Empty, BadNoCommunication),
(At(10), HistorianValue.FromDouble(30), Good));
var src = B.CreateDataSource();
async Task<DataValueSnapshot> One(HistoryAggregateType a) =>
(await src.ReadProcessedAsync("M", At(0), At(20), Twenty, a, Ct)).Samples.Single();
var avg = await One(HistoryAggregateType.Average);
avg.Value.ShouldBe(20d);
avg.StatusCode.ShouldBe(UncertainCalc);
(await One(HistoryAggregateType.Count)).Value.ShouldBe(2);
(await One(HistoryAggregateType.Start)).Value.ShouldBe(10d);
// #680: duration-based — [0,5) Good, [5,10) Bad, [10,20) Good (no end bound: the last region runs to the interval end).
((double)(await One(HistoryAggregateType.PercentBad)).Value!).ShouldBe(25d, 1e-9);
}
[Fact]
public async Task An_interval_of_only_bad_samples_is_BadNoData_except_the_percentages()
{
await B.SeedAsync("Z",
(At(1), HistorianValue.Empty, BadNoCommunication),
(At(2), HistorianValue.FromDouble(5), Bad));
var src = B.CreateDataSource();
async Task<DataValueSnapshot> One(HistoryAggregateType a) =>
(await src.ReadProcessedAsync("Z", At(0), At(20), Twenty, a, Ct)).Samples.Single();
(await One(HistoryAggregateType.Average)).StatusCode.ShouldBe(NoDataCalc);
(await One(HistoryAggregateType.Count)).StatusCode.ShouldBe(NoDataCalc);
var pb = await One(HistoryAggregateType.PercentBad);
pb.Value.ShouldBe(100d);
// The tag's data starts (+1) inside the interval ⇒ Partial on the non-Bad percentage (#677).
pb.StatusCode.ShouldBe(GoodCalc | 0x4u);
}
[Fact]
public async Task Uncertain_samples_are_used_but_make_the_interval_uncertain()
{
await B.SeedAsync("U",
(At(1), HistorianValue.FromDouble(4), Uncertain),
(At(2), HistorianValue.FromDouble(6), Good));
var s = (await B.CreateDataSource().ReadProcessedAsync("U", At(0), At(20), Twenty, HistoryAggregateType.Average, Ct)).Samples.Single();
s.Value.ShouldBe(5d);
s.StatusCode.ShouldBe(UncertainCalc);
}
[Fact]
public async Task String_tags_serve_count_start_end_and_refuse_numeric_aggregates()
{
await B.SeedAsync("S",
(At(0), HistorianValue.FromString("a"), Good),
(At(10), HistorianValue.FromString("b"), Good));
var src = B.CreateDataSource();
async Task<DataValueSnapshot> One(HistoryAggregateType a) =>
(await src.ReadProcessedAsync("S", At(0), At(20), Twenty, a, Ct)).Samples.Single();
(await One(HistoryAggregateType.Count)).Value.ShouldBe(2);
(await One(HistoryAggregateType.Start)).Value.ShouldBe("a");
(await One(HistoryAggregateType.End)).Value.ShouldBe("b");
(await One(HistoryAggregateType.PercentGood)).Value.ShouldBe(100d);
foreach (var a in new[]
{
HistoryAggregateType.Average, HistoryAggregateType.Minimum, HistoryAggregateType.Maximum,
HistoryAggregateType.Total, HistoryAggregateType.Delta, HistoryAggregateType.Range,
HistoryAggregateType.MinimumActualTime, HistoryAggregateType.MaximumActualTime,
})
{
var r = await One(a);
r.StatusCode.ShouldBe(InvalidInputsCalc, a.ToString());
r.Value.ShouldBeNull();
}
}
[Fact]
public async Task DateTime_tags_refuse_numeric_aggregates()
{
await B.SeedAsync("D", (At(0), HistorianValue.FromDateTime(At(-100)), Good));
(await B.CreateDataSource().ReadProcessedAsync("D", At(0), At(20), Twenty, HistoryAggregateType.Average, Ct))
.Samples.Single().StatusCode.ShouldBe(InvalidInputsCalc);
}
[Fact]
public async Task Boolean_tags_aggregate_as_zero_or_one()
{
await B.SeedAsync("B",
(At(0), HistorianValue.FromBoolean(true), Good),
(At(5), HistorianValue.FromBoolean(false), Good),
(At(10), HistorianValue.FromBoolean(true), Good),
(At(15), HistorianValue.FromBoolean(true), Good));
var src = B.CreateDataSource();
async Task<DataValueSnapshot> One(HistoryAggregateType a) =>
(await src.ReadProcessedAsync("B", At(0), At(20), Twenty, a, Ct)).Samples.Single();
(await One(HistoryAggregateType.Average)).Value.ShouldBe(0.75d);
// Total is time-weighted (#671): stepped 1 over [0,5), 0 over [5,10), 1 over [10,20) ⇒ 15 value·s.
(await One(HistoryAggregateType.Total)).Value.ShouldBe(15d);
(await One(HistoryAggregateType.Start)).Value.ShouldBe(true);
(await One(HistoryAggregateType.Count)).Value.ShouldBe(4);
}
[Fact]
public async Task Unknown_tag_yields_every_interval_as_BadNoData()
{
var s = (await B.CreateDataSource().ReadProcessedAsync("nope", At(0), At(60), Twenty, HistoryAggregateType.Average, Ct)).Samples;
s.Count.ShouldBe(3);
s.ShouldAllBe(x => x.StatusCode == NoDataCalc && x.Value == null);
}
[Theory]
[InlineData(HistoryAggregateType.StartBound)]
[InlineData(HistoryAggregateType.EndBound)]
[InlineData(HistoryAggregateType.DeltaBounds)]
[InlineData(HistoryAggregateType.DurationGood)]
[InlineData(HistoryAggregateType.DurationBad)]
[InlineData(HistoryAggregateType.WorstQuality)]
[InlineData(HistoryAggregateType.NumberOfTransitions)]
[InlineData(HistoryAggregateType.AnnotationCount)]
[InlineData(HistoryAggregateType.StandardDeviationPopulation)]
[InlineData(HistoryAggregateType.VariancePopulation)]
public async Task Unsupported_aggregates_throw_not_supported_and_are_not_health_failures(HistoryAggregateType agg)
{
var src = B.CreateDataSource();
var ex = await Should.ThrowAsync<HistoryAggregateNotSupportedException>(
() => src.ReadProcessedAsync("A", At(0), At(20), Twenty, agg, Ct));
ex.Aggregate.ShouldBe(agg);
src.GetHealthSnapshot().TotalQueries.ShouldBe(0);
}
// ------------------------------------------------------------------------------------------ Time-weighted (#671)
private const uint GoodRaw = 0x00000000u;
private const uint GoodInterp = 0x00000402u; // Good | InfoType 0x400 | Interpolated 0x2
private const uint UncertainInterp = 0x40A40402u; // UncertainDataSubNormal | 0x402
private const uint NoData = 0x809B0000u;
private const uint BadCalc = 0x80000401u; // generic Bad | Calculated (an all-outage interval)
/// <summary>
/// Sparse tag W: −30 = 0 G (bound before the window), +20 = 40 G, +30 = outage marker, +50 = 20 G, +100 = 20 G
/// (bound after the window). Window [t0, t0+80s). Simple-bounding signal: [−30,20) 0→40 sloped Good (24 at +0,
/// 32 at +10); [20,30) 40 held into the outage, Uncertain; [30,50) Bad; [50,100) 20 Good. Interpolated bounding:
/// the same, but [20,50) is sloped 40→20 straight across the outage (100/3 at +30, 80/3 at +40).
/// </summary>
private Task SeedWAsync() => B.SeedAsync("W",
(At(-30), HistorianValue.FromDouble(0), Good),
(At(20), HistorianValue.FromDouble(40), Good),
(At(30), HistorianValue.Empty, BadNoCommunication),
(At(50), HistorianValue.FromDouble(20), Good),
(At(100), HistorianValue.FromDouble(20), Good));
private async Task<IReadOnlyList<DataValueSnapshot>> ReadW(HistoryAggregateType agg, double intervalS) =>
(await B.CreateDataSource().ReadProcessedAsync("W", At(0), At(80), TimeSpan.FromSeconds(intervalS), agg, Ct)).Samples;
private static void Values(IReadOnlyList<DataValueSnapshot> s, params double?[] expected)
{
s.Count.ShouldBe(expected.Length);
for (var i = 0; i < expected.Length; i++)
{
if (expected[i] is { } e) ((double)s[i].Value!).ShouldBe(e, 1e-9, $"bucket {i}");
else s[i].Value.ShouldBeNull($"bucket {i}");
}
}
[Theory]
// 40 s intervals (wider than the sample gaps). B0 [0,40): raw [0,20) 24→40 = 640 Good + [20,30) 40 = 400 Uncertain,
// [30,40) Bad; bridged 640 + [20,40) 40→80/3 = 2000/3. B1 [40,80): raw [50,80) 20 = 600 Good, [40,50) Bad; bridged
// [40,50) 80/3→20 = 700/3 + 600.
[InlineData(HistoryAggregateType.TimeAverage, 98d / 3d, 125d / 6d)] // (3920/3)/40, (2500/3)/40
[InlineData(HistoryAggregateType.Total, 3920d / 3d, 2500d / 3d)] // TimeAverage × 40 s
[InlineData(HistoryAggregateType.TimeAverage2, 104d / 3d, 20d)] // 1040/30 s, 600/30 s
[InlineData(HistoryAggregateType.Total2, 1040d, 600d)]
public async Task Time_weighted_aggregates_over_a_sparse_tag_with_wide_intervals(HistoryAggregateType agg, double b0, double b1)
{
await SeedWAsync();
var s = await ReadW(agg, 40);
Values(s, b0, b1);
// Neither interval is Good throughout (an outage in each) ⇒ Uncertain.
s.Select(x => x.StatusCode).ShouldBe([UncertainCalc, UncertainCalc]);
s.Select(x => x.SourceTimestampUtc).ShouldBe([B.Round(At(0)), B.Round(At(40))]);
}
[Fact]
public async Task Interpolative_over_a_sparse_tag_with_wide_intervals()
{
await SeedWAsync();
var s = await ReadW(HistoryAggregateType.Interpolative, 40);
// +0: from the bound before the window, 0 + 40·(30/50) = 24, Good. +40: inside the outage, bridged 80/3, Uncertain.
Values(s, 24, 80d / 3d);
s.Select(x => x.StatusCode).ShouldBe([GoodInterp, UncertainInterp]);
}
[Fact]
public async Task TimeAverage2_over_a_sparse_tag_with_narrow_intervals()
{
await SeedWAsync();
// 10 s intervals (narrower than the gaps): [0,10) 24→32 = 28; [10,20) 32→40 = 36; [20,30) 40 held up to the
// outage (Uncertain time, which counts as Good: TreatUncertainAsBad = false); [30,40) and [40,50) inside the outage
// ⇒ generic Bad (data, but none non-Bad); [50,60) … [70,80) the flat 20 up to the bound after, Good.
var s = await ReadW(HistoryAggregateType.TimeAverage2, 10);
Values(s, 28, 36, 40, null, null, 20, 20, 20);
s.Select(x => x.StatusCode).ShouldBe(
[GoodCalc, GoodCalc, GoodCalc, BadCalc, BadCalc, GoodCalc, GoodCalc, GoodCalc]);
// Total2 of the first interval: 28 × 10 s.
((double)(await ReadW(HistoryAggregateType.Total2, 10))[0].Value!).ShouldBe(280d, 1e-9);
}
[Fact]
public async Task Interpolative_over_a_sparse_tag_with_narrow_intervals()
{
await SeedWAsync();
var s = await ReadW(HistoryAggregateType.Interpolative, 10);
// +0 24, +10 32 (sloped from the bound); +20 exact 40 (raw Good); +30 ON the outage marker ⇒ bridged 100/3;
// +40 bridged 80/3; +50 exact 20; +60, +70 on the Good 20→20 segment to the bound after.
Values(s, 24, 32, 40, 100d / 3d, 80d / 3d, 20, 20, 20);
s.Select(x => x.StatusCode).ShouldBe(
[GoodInterp, GoodInterp, GoodRaw, UncertainInterp, UncertainInterp, GoodRaw, GoodInterp, GoodInterp]);
}
[Fact]
public async Task A_reverse_Interpolative_read_is_anchored_at_its_later_time()
{
await SeedWAsync();
// At(80) → At(0), 40 s: points At(80) (on the Good 20→20 segment to the bound at +100 ⇒ 20) and At(40) (inside
// the outage ⇒ bridged 80/3, Uncertain), latest first. A forward read would answer At(0) and At(40) instead.
var s = (await B.CreateDataSource().ReadProcessedAsync("W", At(80), At(0), TimeSpan.FromSeconds(40),
HistoryAggregateType.Interpolative, Ct)).Samples;
s.Select(x => x.SourceTimestampUtc).ShouldBe([B.Round(At(80)), B.Round(At(40))]);
Values(s, 20, 80d / 3d);
s.Select(x => x.StatusCode).ShouldBe([GoodInterp, UncertainInterp]);
}
[Fact]
public async Task The_time_weighted_raw_row_cap_fails_the_read_past_the_cap_only()
{
// Tag A holds five samples in [0,80); W holds three in [0,80) plus two bound rows outside it.
await ConformanceData.SeedAAsync(B);
await SeedWAsync();
var atCap = B.CreateDataSource(5);
(await atCap.ReadProcessedAsync("A", At(0), At(80), Twenty, HistoryAggregateType.Total, Ct)).Samples.Count.ShouldBe(4);
var under = B.CreateDataSource(4);
var ex = await Should.ThrowAsync<HistoryReadLimitExceededException>(
() => under.ReadProcessedAsync("A", At(0), At(80), Twenty, HistoryAggregateType.Total, Ct));
ex.Limit.ShouldBe(4);
// #680: PercentGood/PercentBad are duration-based, served by the same window stream ⇒ capped too.
(await Should.ThrowAsync<HistoryReadLimitExceededException>(
() => under.ReadProcessedAsync("A", At(0), At(80), Twenty, HistoryAggregateType.PercentGood, Ct))).Limit.ShouldBe(4);
// Pushed-down (per-sample) aggregates are not capped; the bound rows outside the window do not count.
(await under.ReadProcessedAsync("A", At(0), At(80), Twenty, HistoryAggregateType.Average, Ct)).Samples.Count.ShouldBe(4);
(await B.CreateDataSource(3).ReadProcessedAsync("W", At(0), At(80), Twenty, HistoryAggregateType.TimeAverage, Ct))
.Samples.Count.ShouldBe(4);
// A client-request limit, not a backend fault.
under.GetHealthSnapshot().TotalFailures.ShouldBe(0);
// Non-positive = unbounded.
(await B.CreateDataSource(0).ReadProcessedAsync("A", At(0), At(80), Twenty, HistoryAggregateType.Total, Ct))
.Samples.Count.ShouldBe(4);
}
[Fact]
public async Task Outages_straddling_both_window_edges_use_the_four_bound_seeks()
{
// −10 = 0 G, −5 outage, +10 = 20 G, +20 outage, +30 = 40 G; window [0,20). The four seeks return −10 (latest
// usable before), −5 (latest before), +20 (first at/after end) and +30 (first usable at/after end).
await B.SeedAsync("O",
(At(-10), HistorianValue.FromDouble(0), Good),
(At(-5), HistorianValue.Empty, BadNoCommunication),
(At(10), HistorianValue.FromDouble(20), Good),
(At(20), HistorianValue.Empty, BadNoCommunication),
(At(30), HistorianValue.FromDouble(40), Good));
var src = B.CreateDataSource();
async Task<IReadOnlyList<DataValueSnapshot>> Read(HistoryAggregateType a, double intervalS) =>
(await src.ReadProcessedAsync("O", At(0), At(20), TimeSpan.FromSeconds(intervalS), a, Ct)).Samples;
// Bridged: −10..10 0→20 ⇒ [0,10) mean 15 (150); 10..30 20→40 ⇒ [10,20) mean 25 (250). 400 / 20 = 20.
// (Without the usable-before seek: 250/10 = 25; without the usable-after seek: [10,20) held 20 ⇒ 17.5.)
var ta = (await Read(HistoryAggregateType.TimeAverage, 0)).Single();
((double)ta.Value!).ShouldBe(20d, 1e-9);
ta.StatusCode.ShouldBe(UncertainCalc);
// Simple bounding: [0,10) inside the outage from −5; [10,20) 20 held into the outage at +20 ⇒ 200 / 10 s = 20.
// (Without the any-before seek: (150+200)/20 = 17.5; without the any-after seek: [10,20) 20→30 ⇒ 25.)
var ta2 = (await Read(HistoryAggregateType.TimeAverage2, 0)).Single();
((double)ta2.Value!).ShouldBe(20d, 1e-9);
ta2.StatusCode.ShouldBe(UncertainCalc);
// +0 inside the outage ⇒ bridged 10, Uncertain; +10 exact 20, raw Good.
var i = await Read(HistoryAggregateType.Interpolative, 10);
Values(i, 10, 20);
i.Select(x => x.StatusCode).ShouldBe([UncertainInterp, GoodRaw]);
}
[Fact]
public async Task Time_weighted_aggregates_on_an_unknown_tag_are_BadNoData()
{
var src = B.CreateDataSource();
(await src.ReadProcessedAsync("nope", At(0), At(40), Twenty, HistoryAggregateType.TimeAverage, Ct))
.Samples.ShouldAllBe(x => x.StatusCode == NoDataCalc);
(await src.ReadProcessedAsync("nope", At(0), At(40), Twenty, HistoryAggregateType.Interpolative, Ct))
.Samples.ShouldAllBe(x => x.StatusCode == NoData);
src.GetHealthSnapshot().TotalFailures.ShouldBe(0);
}
[Fact]
public async Task The_bucket_count_is_capped_at_100000()
{
var src = B.CreateDataSource();
var oneTick = TimeSpan.FromTicks(B.ResolutionTicks);
(await src.ReadProcessedAsync("nope", At(0), At(0).AddTicks(100_000 * B.ResolutionTicks), oneTick, HistoryAggregateType.Count, Ct))
.Samples.Count.ShouldBe(100_000);
await Should.ThrowAsync<ArgumentOutOfRangeException>(
() => src.ReadProcessedAsync("nope", At(0), At(0).AddTicks(100_001 * B.ResolutionTicks), oneTick, HistoryAggregateType.Count, Ct));
var h = src.GetHealthSnapshot();
h.TotalFailures.ShouldBe(0); // a client-request error, not a backend fault
}
}