dohertj2/lmxopcua
1
0
Fork 0
Code Issues 32 Pull Requests Actions Packages Projects Releases Wiki Activity

feat(historian): #677 Part 13/SDK reverse anchoring, start-of-data Partial, AggregateConfiguration, IntegralDivisor

Browse Source 
- BucketPlan: reverse processed reads anchored at the later time for every
  aggregate: window (end, start], partial interval earliest, results stamped
  with the later edge, latest first (SDK CreateSlice/GetTimestamp/backward
  GetValues). SqlServer adds @bucketAdjustTicks + flips the window predicate;
  QuestDB binds BucketBaseUnixTicks. IndexOf is the C# statement of the math.
- Partial bit on the interval the tag's data starts inside (SDK m_startOfData),
  every aggregate but Average; per-sample reads fetch the earliest sample.
- Node manager: non-default client AggregateConfiguration =>
  BadAggregateConfigurationRejected per node (ServerAggregateConfiguration,
  pinned to the SDK AggregateManager defaults).
- GatewayTagProvisioner sends IntegralDivisor = 1.
- SDK parity oracle (Opc.Ua.Server calculators) in unit + both backends'
  conformance suites, forward and reverse; remaining divergences documented.
- docs/Historian.md + CLAUDE.md release note (client-visible).
This commit is contained in:
Joseph Doherty
2026-10-11 11:26:00 -04:00
parent 825393299e
commit 27ca6cd791
24 changed files with 1134 additions and 135 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/Drivers/Historian.Conformance/ProcessedConformance.cs
+124 -5
View File
@@ -1,6 +1,7 @@
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common;
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Conformance;
@@ -171,7 +172,8 @@ public abstract class ProcessedConformance : ConformanceBase
// The time-weighted path over the whole range: ≈ the extrapolated 50 that dominates eight millennia.
var tw = (await src.ReadProcessedAsync("A", start, end, TimeSpan.Zero, HistoryAggregateType.TimeAverage2, Ct)).Samples.Single();
((double)tw.Value!).ShouldBe(50d, 1e-6);
tw.StatusCode.ShouldBe(UncertainCalc);
// The data starts (+0) strictly inside the one interval ⇒ Partial as well (#677, the SDK's m_startOfData rule).
tw.StatusCode.ShouldBe(UncertainCalc | 0x4u);
src.GetHealthSnapshot().TotalFailures.ShouldBe(0);
}
@@ -196,12 +198,128 @@ public abstract class ProcessedConformance : ConformanceBase
}
[Fact]
public async Task A_reverse_window_is_computed_forward_and_returned_descending()
public async Task A_reverse_window_is_anchored_at_its_later_time()
{
// #677 (Part 13 / SDK): At(80) → At(0) is the window (0,80] in intervals (60,80], (40,60], (20,40], (0,20], each
// stamped with its LATER edge, latest first. A sample on an edge belongs to the interval it ends: +40 is in
// (20,40], +20 (Bad) in (0,20], and +0 is outside the window. (The pre-#677 layout answered [60,80) … [0,20)
// stamped 60/40/20/0 with values null, 50, 40, 15.)
await ConformanceData.SeedAAsync(B);
var s = (await B.CreateDataSource().ReadProcessedAsync("A", At(80), At(0), Twenty, HistoryAggregateType.Average, Ct)).Samples;
s.Select(x => x.SourceTimestampUtc).ShouldBe([B.Round(At(60)), B.Round(At(40)), B.Round(At(20)), B.Round(At(0))]);
s.Select(x => x.Value).ShouldBe([null, 50d, 40d, 15d]);
s.Select(x => x.SourceTimestampUtc).ShouldBe([B.Round(At(80)), B.Round(At(60)), B.Round(At(40)), B.Round(At(20))]);
s.Select(x => x.Value).ShouldBe([null, null, 45d, 20d]);
s.Select(x => x.StatusCode).ShouldBe([NoDataCalc, NoDataCalc, GoodCalc, UncertainCalc]);
}
[Fact]
public async Task A_reverse_window_truncates_its_earliest_interval()
{
// At(50) → At(0), 20 s over tag A (+0=10, +10=20, +20=Bad, +30=40, +40=50): (30,50] = {+40} → 50 (+30 is its
// earlier edge, excluded); (10,30] = {+20 Bad, +30} → 40, Uncertain; (0,10] = {+10} → 20 — the truncated interval
// is the EARLIEST one. (Pre-#677: [0,20), [20,40), [40,50) stamped 40/20/0 with values 50, 40, 15.)
await ConformanceData.SeedAAsync(B);
var s = (await B.CreateDataSource().ReadProcessedAsync("A", At(50), At(0), Twenty, HistoryAggregateType.Average, Ct)).Samples;
s.Select(x => x.SourceTimestampUtc).ShouldBe([B.Round(At(50)), B.Round(At(30)), B.Round(At(10))]);
s.Select(x => x.Value).ShouldBe([50d, 40d, 20d]);
s.Select(x => x.StatusCode).ShouldBe([GoodCalc, UncertainCalc, GoodCalc]);
}
[Fact]
public async Task Reverse_bucket_edges_are_later_edge_inclusive_to_the_tick()
{
// The mirror of the forward edge test: At(40) → At(0), 20 s ⇒ (20,40], (0,20]. The bucket math is pushed down,
// so this pins the SQL, not just the C# plan.
var r = B.ResolutionTicks;
await B.SeedAsync("ER",
(At(0), HistorianValue.FromDouble(1000), Good), // == earlier edge: excluded
(At(0).AddTicks(r), HistorianValue.FromDouble(1), Good), // first tick of (0,20]
(At(20), HistorianValue.FromDouble(2), Good), // == edge: the LATER edge of (0,20]
(At(20).AddTicks(r), HistorianValue.FromDouble(4), Good), // first tick of (20,40]
(At(40), HistorianValue.FromDouble(8), Good), // == request start: included, in (20,40]
(At(40).AddTicks(r), HistorianValue.FromDouble(2000), Good)); // after the window: excluded
var s = (await B.CreateDataSource().ReadProcessedAsync("ER", At(40), At(0), Twenty, HistoryAggregateType.Average, Ct)).Samples;
s.Select(x => x.SourceTimestampUtc).ShouldBe([B.Round(At(40)), B.Round(At(20))]);
s.Select(x => x.Value).ShouldBe([6d, 1.5d]);
}
[Fact]
public async Task The_interval_the_data_starts_inside_carries_the_Partial_bit()
{
// #677 (SDK m_startOfData): the tag's first sample is +5, inside [0,20) ⇒ Partial there for every aggregate but
// Average; [20,40) is fully covered. With an earlier sample stored (even outside the window) nothing is Partial.
await B.SeedAsync("LS",
(At(5), HistorianValue.FromDouble(10), Good),
(At(15), HistorianValue.FromDouble(30), Good),
(At(25), HistorianValue.FromDouble(20), Good),
(At(60), HistorianValue.FromDouble(20), Good));
var src = B.CreateDataSource();
foreach (var agg in new[] { HistoryAggregateType.Count, HistoryAggregateType.Minimum, HistoryAggregateType.TimeAverage2 })
{
var s = (await src.ReadProcessedAsync("LS", At(0), At(40), Twenty, agg, Ct)).Samples;
(s[0].StatusCode & 0x4u).ShouldBe(0x4u, agg.ToString());
(s[1].StatusCode & 0x4u).ShouldBe(0u, agg.ToString());
}
(await src.ReadProcessedAsync("LS", At(0), At(40), Twenty, HistoryAggregateType.Average, Ct)).Samples
.ShouldAllBe(x => (x.StatusCode & 0x4u) == 0);
await B.SeedAsync("LS2",
(At(-100), HistorianValue.FromDouble(10), Good),
(At(5), HistorianValue.FromDouble(10), Good),
(At(15), HistorianValue.FromDouble(30), Good),
(At(25), HistorianValue.FromDouble(20), Good));
(await src.ReadProcessedAsync("LS2", At(0), At(40), Twenty, HistoryAggregateType.Count, Ct)).Samples
.ShouldAllBe(x => (x.StatusCode & 0x4u) == 0);
}
/// <summary>Dense: every 2.5 s from −10 to +100, values cycling, one Bad sample at +22.5 (no empty interval).</summary>
private Task SeedDenseAsync() => B.SeedAsync("P", DenseRows());
private static (DateTime T, HistorianValue V, uint Status)[] DenseRows() => Enumerable.Range(0, 45)
.Select(i => (At(-10 + i * 2.5), HistorianValue.FromDouble(10 + (i * 7 % 13)), i == 13 ? Bad : Good))
.ToArray();
[Theory]
[InlineData(0, 80, 20)]
[InlineData(80, 0, 20)]
[InlineData(77, 3, 20)]
[InlineData(3, 77, 20)]
public async Task Processed_reads_match_the_SDK_aggregate_calculators(double from, double to, double intervalS)
{
// #677: the real backend (pushed-down SQL bucket math + the shared evaluators) against the pinned SDK's own Part 13
// calculators over the same stored rows — timestamps, values, status code bits and the Partial bit, forward and
// reverse. See SdkAggregateParityTests (Sql.Common.Tests) for the documented divergences this avoids/masks.
await SeedDenseAsync();
var rows = DenseRows().Select(r => (B.Round(r.T), r.V, r.Status)).ToArray();
var src = B.CreateDataSource();
var interval = TimeSpan.FromSeconds(intervalS);
// The SDK sees the window the backend actually serves (QuestDB rounds the bounds to µs).
var (start, end) = (B.Round(At(from)), B.Round(At(to)));
foreach (var agg in new[]
{
HistoryAggregateType.Average, HistoryAggregateType.Minimum, HistoryAggregateType.Maximum,
HistoryAggregateType.Count, HistoryAggregateType.TimeAverage, HistoryAggregateType.TimeAverage2,
HistoryAggregateType.Total, HistoryAggregateType.Total2, HistoryAggregateType.Interpolative,
})
{
var ours = (await src.ReadProcessedAsync("P", At(from), At(to), interval, agg, Ct)).Samples.ToList();
var tw = AggregateFamilies.FamilyOf(agg) == AggregateFamily.TimeWeighted;
if (tw && agg != HistoryAggregateType.Interpolative)
{
// Documented divergence: ours flags the truncated interval Partial, the SDK does not.
var plan = BucketPlan.Create(start, end, interval);
for (long i = 0; i < plan.BucketCount; i++)
if (plan.IsTruncated(i))
{
var k = (int)(plan.Reversed ? plan.BucketCount - 1 - i : i);
ours[k] = ours[k] with { StatusCode = ours[k].StatusCode & ~0x4u };
}
}
var reverseInterpolative = agg == HistoryAggregateType.Interpolative && from > to;
SdkAggregateOracle.ShouldMatch(ours, SdkAggregateOracle.Run(agg, start, end, interval, rows),
$"{agg} {from}→{to}/{intervalS}s",
compareCalculatedInterpolatedBits: tw && !reverseInterpolative,
compareCodeBits: agg is not (HistoryAggregateType.TimeAverage or HistoryAggregateType.Total) && !reverseInterpolative);
}
}
[Fact]
@@ -245,7 +363,8 @@ public abstract class ProcessedConformance : ConformanceBase
(await One(HistoryAggregateType.Count)).StatusCode.ShouldBe(NoDataCalc);
var pb = await One(HistoryAggregateType.PercentBad);
pb.Value.ShouldBe(100d);
pb.StatusCode.ShouldBe(GoodCalc);
// The tag's data starts (+1) inside the interval ⇒ Partial on the non-Bad percentage (#677).
pb.StatusCode.ShouldBe(GoodCalc | 0x4u);
}
[Fact]