dohertj2/lmxopcua
1
0
Fork 0
Code Issues 32 Pull Requests Actions Packages Projects Releases Wiki Activity

fix(historian): #680 lane B — per-sample Count/Start/End/Delta per Part 13 + forward end-of-data Partial

Browse Source 
Count: Good samples only, value-based status (all Bad => generic Bad), 0/Good on an
empty interval inside the tag's data range [first, last] (both directions).
Start/End: the raw first/last sample of any quality (own value, status, timestamp).
Delta: Uncertain only when the interval's first or last sample is not usable.
Forward end-of-data Partial (SDK m_endOfData, earlier edge inclusive) for every
per-sample aggregate but Average (PercentGood/Bad left to lane A).
Backends push down first/last-any-quality columns and fetch the tag's data range
(earliest <= window end, latest >= window start) only when NeedsDataRange.
SDK parity masks for Start/End/Delta/Count removed; new parity datasets.
This commit is contained in:
Joseph Doherty
2026-10-11 15:52:49 -04:00
parent f6a2628656
commit dbfe2fc4de
13 changed files with 644 additions and 136 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common.Tests/SdkAggregateParityTests.cs
+125 -15
View File
@@ -16,9 +16,9 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common.Tests;
/// <remarks>
/// Deliberate, documented divergences the datasets avoid (they are not anchoring questions):
/// <list type="bullet">
/// <item>an <b>empty</b> interval inside the data range: the SDK's Count answers 0/Good, ours <c>BadNoData</c>;</item>
/// <item><b>Uncertain</b> samples in a per-sample aggregate: the SDK's Average/Count/Min/Max use Good samples only,
/// ours every usable (Good + Uncertain) sample (TreatUncertainAsBad = false applied consistently);</item>
/// <item><b>Uncertain</b> samples in a per-sample aggregate: the SDK's Average/Min/Max use Good samples only,
/// ours every usable (Good + Uncertain) sample (TreatUncertainAsBad = false applied consistently). Count matches
/// the SDK since #680 (Good samples only);</item>
/// <item>the per-sample <b>Calculated / MultipleValues</b> info bits (the SDK's MinMax sets Calculated only when the
/// extreme is not at the interval start, and flags ties) — those comparisons check code bits + Partial only;</item>
/// <item>a <b>truncated</b> interval: ours sets Partial on a non-Bad time-weighted result (Part 13 §5.3.3.2 Case 3
@@ -46,6 +46,8 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common.Tests;
/// <c>UpdateSlice</c> files it as the early bound (below), so the interval takes the status of the segment beyond
/// it (Uncertain when that segment skips a Bad sample); ours keeps the interval's own segments — masked by
/// <see cref="MaskDocumented"/>;</item>
/// <item>a <b>Bad</b> per-sample result never carries Partial (the SDK sets start/end-of-data Partial on Bad results
/// too) — the datasets keep the data's start and end out of all-Bad intervals;</item>
/// <item><b>Interpolative, reverse, a sample exactly at the point:</b> the SDK's backward <c>UpdateSlice</c> files
/// a sample AT the point as the early bound (<c>&gt;= 0</c>), so it interpolates from it to the next Good sample —
/// same value, but the Interpolated bit, and Uncertain when a Bad sample sits between. Ours returns the raw
@@ -164,7 +166,7 @@ public sealed class SdkAggregateParityTests
if (AggregateFamilies.FamilyOf(agg) == AggregateFamily.TimeWeighted)
return TimeWeightedAggregator.Evaluate(agg, plan, data, Now);
return AggregateEvaluator.EvaluateAll(agg, plan, InMemoryRows(plan, data, agg == HistoryAggregateType.MaximumActualTime),
data.Min(s => s.TimeUtc), Now);
data.Min(s => s.TimeUtc), data.Max(s => s.TimeUtc), Now);
}
private static IReadOnlyList<SdkAggregateOracle.Result> Sdk(HistoryAggregateType agg, double from, double to,
@@ -305,12 +307,7 @@ public sealed class SdkAggregateParityTests
{
SdkAggregateOracle.ShouldMatch(Ours(agg, from, to, intervalS, Dense), Sdk(agg, from, to, intervalS, Dense),
$"{agg} Dense {from}→{to}/{intervalS}s", compareCalculatedInterpolatedBits: false,
// pre-existing, #680: the SDK's Start/End return the raw first/last sample (any quality) with its own
// timestamp and stored status; ours return the first/last USABLE value stamped with the interval, under the
// per-sample status rule. The SDK's Delta is Good even when a Bad sample sits in the interval; ours is
// Uncertain. Values are compared wherever the SDK's answer is not Bad.
compareTimestamps: !StartOrEnd(agg),
compareCodeBits: !StartOrEnd(agg) && agg != HistoryAggregateType.Delta);
compareValues: true);
}
catch (Exception ex)
{
@@ -327,8 +324,6 @@ public sealed class SdkAggregateParityTests
HistoryAggregateType.Delta, HistoryAggregateType.MinimumActualTime, HistoryAggregateType.MaximumActualTime,
];
private static bool StartOrEnd(HistoryAggregateType agg) =>
agg is HistoryAggregateType.Start or HistoryAggregateType.End;
[Theory]
[InlineData(HistoryAggregateType.TimeAverage)]
@@ -351,9 +346,7 @@ public sealed class SdkAggregateParityTests
var ours = Ours(agg, 40, 0, 10, LateStart);
SdkAggregateOracle.ShouldMatch(ours, Sdk(agg, 40, 0, 10, LateStart), $"{agg} reverse LateStart",
compareCalculatedInterpolatedBits: tw,
// pre-existing, #680: Start's raw-sample timestamp/status (see Per_sample_aggregates_match_the_SDK).
compareTimestamps: !StartOrEnd(agg),
compareCodeBits: !StartOrEnd(agg) && agg != HistoryAggregateType.Delta);
compareCodeBits: true);
ours.ShouldAllBe(x => (x.StatusCode & HistorianInfoBits.HistorianPartial) == 0);
}
@@ -379,6 +372,121 @@ public sealed class SdkAggregateParityTests
ours.Skip(1).ShouldAllBe(x => (x.StatusCode & HistorianInfoBits.HistorianPartial) == 0);
}
/// <summary>
/// #680: sparse data with Good, Uncertain and Bad samples, empty intervals inside the data range, and intervals before
/// and after it: +1 G, +2 U, +25 Bad, +45 G, +70 G.
/// </summary>
private static readonly StoredSample[] Gappy =
[
S(1, HistorianValue.FromDouble(10), Good),
S(2, HistorianValue.FromDouble(11), Uncertain),
S(25, HistorianValue.FromDouble(12), Bad),
S(45, HistorianValue.FromDouble(13), Good),
S(70, HistorianValue.FromDouble(1), Good),
];
/// <summary>#680: Bad samples leading, trailing and mid-interval, plus an Uncertain one (Start / End / Delta).</summary>
private static readonly StoredSample[] Bracketed =
[
S(-5, HistorianValue.FromDouble(1), Good),
S(1, HistorianValue.FromDouble(10), Bad),
S(3, HistorianValue.FromDouble(11), Good),
S(5, HistorianValue.FromDouble(12), Bad),
S(7, HistorianValue.FromDouble(13), Uncertain),
S(9, HistorianValue.FromDouble(15), Good),
S(12, HistorianValue.FromDouble(20), Good),
S(14, HistorianValue.Empty, BadNoCommunication),
S(16, HistorianValue.FromDouble(21), Good),
S(19, HistorianValue.Empty, BadNoCommunication),
S(40, HistorianValue.FromDouble(1), Good),
];
public static TheoryData<double, double, double> GappyWindows => new()
{
{ 0, 60, 10 }, { 60, 0, 10 }, // empty intervals inside the data range; +70 is past the window
{ -10, 1, 5 }, { 1, -10, 5 }, // intervals before the first sample; one ending ON it (forward)
{ 70, 85, 5 }, { 85, 70, 5 }, // intervals after the last sample; one starting ON it (reverse)
{ 0, 1, 1 }, // one interval ending exactly on the first sample: empty but in range
};
[Theory]
[MemberData(nameof(GappyWindows))]
public void Count_Start_End_and_Delta_match_the_SDK_across_the_data_range(double from, double to, double intervalS)
{
// #680: Count over Good samples only with the value-based status, 0 (not BadNoData) for an empty interval inside
// [first, last], BadNoData outside it; Start/End the raw first/last sample; Delta's status from its bounding samples.
var failures = new List<string>();
foreach (var agg in new[]
{
HistoryAggregateType.Count, HistoryAggregateType.Start, HistoryAggregateType.End, HistoryAggregateType.Delta,
})
foreach (var (name, data) in new[] { ("Gappy", Gappy), ("Bracketed", Bracketed) })
{
try
{
SdkAggregateOracle.ShouldMatch(Ours(agg, from, to, intervalS, data), Sdk(agg, from, to, intervalS, data),
$"{agg} {name} {from}→{to}/{intervalS}s", compareCalculatedInterpolatedBits: false);
}
catch (Exception ex)
{
failures.Add(ex.Message);
}
}
failures.ShouldBeEmpty(string.Join("\n", failures));
}
[Theory]
[InlineData(0, 20, 10)]
[InlineData(20, 0, 10)]
[InlineData(0, 20, 0)]
[InlineData(20, 0, 0)]
public void Start_End_and_Delta_over_bad_bounding_samples_match_the_SDK_exactly(double from, double to, double intervalS)
{
// #680: with the raw sample's own status (incl. a Bad one and its null value) Start/End must match the SDK to the
// bit; Delta's Calculated bit as well.
foreach (var agg in new[] { HistoryAggregateType.Start, HistoryAggregateType.End, HistoryAggregateType.Delta })
SdkAggregateOracle.ShouldMatch(Ours(agg, from, to, intervalS, Bracketed), Sdk(agg, from, to, intervalS, Bracketed),
$"{agg} Bracketed {from}→{to}/{intervalS}s");
}
/// <summary>#680: data ending inside the window (−5, +3, +25) and ending exactly on an interval edge (+20).</summary>
private static readonly StoredSample[] EarlyEnd =
[S(-5, HistorianValue.FromDouble(1), Good), S(3, HistorianValue.FromDouble(2), Good), S(25, HistorianValue.FromDouble(3), Good)];
private static readonly StoredSample[] EndOnEdge =
[S(-5, HistorianValue.FromDouble(1), Good), S(3, HistorianValue.FromDouble(2), Good), S(20, HistorianValue.FromDouble(3), Good)];
[Theory]
[InlineData(0, 40, 10)]
[InlineData(0, 40, 25)]
[InlineData(0, 40, 0)]
[InlineData(40, 0, 10)]
public void The_interval_the_data_ends_inside_matches_the_SDK_Partial_bit(double from, double to, double intervalS)
{
// #680 (Part 13 §5.3.3.2 Case 2; SDK m_endOfData): forward, the interval holding the last sample (its earlier edge
// included) is Partial for every per-sample aggregate but Average. Reverse 40 → 0 over EndOnEdge: the SDK's
// reverse rule (strictly inside, see the remarks) does not fire on an edge, and ours sets nothing.
var failures = new List<string>();
var datasets = from > to ? new[] { ("EndOnEdge", EndOnEdge) } : new[] { ("EarlyEnd", EarlyEnd), ("EndOnEdge", EndOnEdge) };
foreach (var agg in PerSample)
foreach (var (name, data) in datasets)
{
try
{
SdkAggregateOracle.ShouldMatch(Ours(agg, from, to, intervalS, data), Sdk(agg, from, to, intervalS, data),
$"{agg} {name} {from}→{to}/{intervalS}s", compareCalculatedInterpolatedBits: false);
}
catch (Exception ex)
{
failures.Add(ex.Message);
}
}
failures.ShouldBeEmpty(string.Join("\n", failures));
// Not vacuous: the forward EarlyEnd read does flag [20,30) (and only it).
Ours(HistoryAggregateType.Minimum, 0, 40, 10, EarlyEnd).Select(x => x.StatusCode & HistorianInfoBits.HistorianPartial)
.ShouldBe([0u, 0u, HistorianInfoBits.HistorianPartial, 0u]);
}
/// <summary>The per-bucket statistics the backends' SQL computes, built in memory with <see cref="BucketPlan.IndexOf"/>.</summary>
internal static IEnumerable<BucketRow> InMemoryRows(BucketPlan plan, IEnumerable<StoredSample> samples, bool maximum = false)
{
@@ -408,6 +516,8 @@ public sealed class SdkAggregateParityTests
Last = usable.Count == 0 ? HistorianValue.Empty : usable[^1].Value,
Extreme = extreme?.Value.Number,
ExtremeTime = extreme?.TimeUtc,
FirstAny = all[0],
LastAny = all[^1],
};
}
}