dohertj2/lmxopcua
1
0
Fork 0
Code Issues 32 Pull Requests Actions Packages Projects Releases Wiki Activity

feat(historian): Part 13 time-weighted aggregates on SqlServer + QuestDb (#671)

Browse Source 
New AggregateFamily.TimeWeighted serving TimeAverage, TimeAverage2, Total,
Total2 and Interpolative, evaluated by the shared Sql.Common
TimeWeightedAggregator for both SQL backends (identical results by
construction). HistoryAggregateType gains TimeAverage2 + Total2 (appended);
OtOpcUaNodeManager.MapAggregate maps their NodeIds. Gateway's mapper already
refuses unknown defined members => BadAggregateNotSupported per node.

Semantics (server defaults, client AggregateConfiguration ignored):
TreatUncertainAsBad=false, PercentDataBad=PercentDataGood=100,
UseSlopedExtrapolation=false. Sloped for Double/Int64, stepped for Boolean;
a usable sample followed by a Bad one is held up to the outage (AtTime rule);
stepped Uncertain extrapolation past the last sample. TimeAverage/Total use
interpolated bounding (bridge outages); TimeAverage2/Total2 exclude Bad time.
Bounding values: latest usable + latest any before start, first any + first
usable at/after end (four TOP 1 / LIMIT 1 seeks).

CLIENT-VISIBLE: Total on SqlServer/QuestDb was SUM of samples; it is now the
time-weighted integral (TimeAverage x interval seconds), matching Gateway's
Integral. Release note in docs/Historian.md.

Trade-off (deviation from the preferred hybrid pushdown, allowed by the
design): the time-weighted family is NOT pushed down. Each read streams the
window's rows plus the four bounds (SQL Server: one five-result-set batch;
QuestDB: bounds UNION ALL + window select) into the C# evaluator. Network is
O(samples in window), memory O(buckets). Writing the per-segment
interpolation/edge splitting twice in two SQL dialects and keeping them
bit-identical was judged the bigger risk; this also makes the QuestDB
lag/lead spike moot.

Tests: 35 hand-computed TimeWeightedAggregator unit cases (each of 10
implementation mutations goes red); served-aggregate pin 13 -> 17;
conformance sparse fixture (intervals wider and narrower than sample gaps)
+ an outage straddling both window edges (each of the four SQL seeks
disabled in turn goes red); node-manager NodeId mapping test.
This commit is contained in:
Joseph Doherty
2026-10-11 08:44:26 -04:00
parent db6a53036b
commit 1f88669304
17 changed files with 1118 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/Drivers/Historian.Conformance/ProcessedConformance.cs
+141 -10
View File
@@ -39,7 +39,6 @@ public abstract class ProcessedConformance : ConformanceBase
[InlineData(HistoryAggregateType.Average, 15d, 40d, 50d)]
[InlineData(HistoryAggregateType.Minimum, 10d, 40d, 50d)]
[InlineData(HistoryAggregateType.Maximum, 20d, 40d, 50d)]
[InlineData(HistoryAggregateType.Total, 30d, 40d, 50d)]
[InlineData(HistoryAggregateType.Start, 10d, 40d, 50d)]
[InlineData(HistoryAggregateType.End, 20d, 40d, 50d)]
[InlineData(HistoryAggregateType.Delta, 10d, 0d, 0d)]
@@ -121,8 +120,8 @@ public abstract class ProcessedConformance : ConformanceBase
(edge, HistorianValue.FromDouble(4), Good), // first tick of B1
(At(40).AddTicks(-B.ResolutionTicks), HistorianValue.FromDouble(8), Good), // last tick of B1
(At(40), HistorianValue.FromDouble(2000), Good)); // == end: excluded
var s = (await B.CreateDataSource().ReadProcessedAsync("E", At(0), At(40), Twenty, HistoryAggregateType.Total, Ct)).Samples;
s.Select(x => x.Value).ShouldBe([3d, 12d]);
var s = (await B.CreateDataSource().ReadProcessedAsync("E", At(0), At(40), Twenty, HistoryAggregateType.Average, Ct)).Samples;
s.Select(x => x.Value).ShouldBe([1.5d, 6d]);
s.ShouldAllBe(x => x.StatusCode == GoodCalc);
}
@@ -164,11 +163,15 @@ public abstract class ProcessedConformance : ConformanceBase
(await src.ReadProcessedAsync("A", start, end, TimeSpan.Zero, HistoryAggregateType.Count, Ct))
.Samples.Single().Value.ShouldBe(4);
// Also a whole-DateTime-range window (0001 → 9999-12-31T23:59:59.9999999).
(await src.ReadProcessedAsync("A", DateTime.MinValue, DateTime.MaxValue, TimeSpan.Zero, HistoryAggregateType.Total, Ct))
.Samples.Single().Value.ShouldBe(120d);
(await src.ReadProcessedAsync("A", DateTime.MinValue, DateTime.MaxValue, TimeSpan.Zero, HistoryAggregateType.Count, Ct))
.Samples.Single().Value.ShouldBe(4);
// An interval wider than the window (here TimeSpan.MaxValue) is the same single interval, no overflow.
(await src.ReadProcessedAsync("A", start, end, TimeSpan.MaxValue, HistoryAggregateType.Total, Ct))
.Samples.Single().Value.ShouldBe(120d);
(await src.ReadProcessedAsync("A", start, end, TimeSpan.MaxValue, HistoryAggregateType.Count, Ct))
.Samples.Single().Value.ShouldBe(4);
// The time-weighted path over the whole range: ≈ the extrapolated 50 that dominates eight millennia.
var tw = (await src.ReadProcessedAsync("A", start, end, TimeSpan.Zero, HistoryAggregateType.TimeAverage2, Ct)).Samples.Single();
((double)tw.Value!).ShouldBe(50d, 1e-6);
tw.StatusCode.ShouldBe(UncertainCalc);
src.GetHealthSnapshot().TotalFailures.ShouldBe(0);
}
@@ -303,7 +306,8 @@ public abstract class ProcessedConformance : ConformanceBase
async Task<DataValueSnapshot> One(HistoryAggregateType a) =>
(await src.ReadProcessedAsync("B", At(0), At(20), Twenty, a, Ct)).Samples.Single();
(await One(HistoryAggregateType.Average)).Value.ShouldBe(0.75d);
(await One(HistoryAggregateType.Total)).Value.ShouldBe(3d);
// Total is time-weighted (#671): stepped 1 over [0,5), 0 over [5,10), 1 over [10,20) ⇒ 15 value·s.
(await One(HistoryAggregateType.Total)).Value.ShouldBe(15d);
(await One(HistoryAggregateType.Start)).Value.ShouldBe(true);
(await One(HistoryAggregateType.Count)).Value.ShouldBe(4);
}
@@ -317,8 +321,6 @@ public abstract class ProcessedConformance : ConformanceBase
}
[Theory]
[InlineData(HistoryAggregateType.Interpolative)]
[InlineData(HistoryAggregateType.TimeAverage)]
[InlineData(HistoryAggregateType.StartBound)]
[InlineData(HistoryAggregateType.EndBound)]
[InlineData(HistoryAggregateType.DeltaBounds)]
@@ -338,6 +340,135 @@ public abstract class ProcessedConformance : ConformanceBase
src.GetHealthSnapshot().TotalQueries.ShouldBe(0);
}
// ------------------------------------------------------------------------------------------ Time-weighted (#671)
private const uint GoodRaw = 0x00000000u;
private const uint GoodInterp = 0x00000402u; // Good | InfoType 0x400 | Interpolated 0x2
private const uint UncertainInterp = 0x40A40402u; // UncertainDataSubNormal | 0x402
private const uint NoData = 0x809B0000u;
/// <summary>
/// Sparse tag W: −30 = 0 G (bound before the window), +20 = 40 G, +30 = outage marker, +50 = 20 G, +100 = 20 G
/// (bound after the window). Window [t0, t0+80s). Simple-bounding signal: [−30,20) 0→40 sloped Good (24 at +0,
/// 32 at +10); [20,30) 40 held into the outage, Uncertain; [30,50) Bad; [50,100) 20 Good. Interpolated bounding:
/// the same, but [20,50) is sloped 40→20 straight across the outage (100/3 at +30, 80/3 at +40).
/// </summary>
private Task SeedWAsync() => B.SeedAsync("W",
(At(-30), HistorianValue.FromDouble(0), Good),
(At(20), HistorianValue.FromDouble(40), Good),
(At(30), HistorianValue.Empty, BadNoCommunication),
(At(50), HistorianValue.FromDouble(20), Good),
(At(100), HistorianValue.FromDouble(20), Good));
private async Task<IReadOnlyList<DataValueSnapshot>> ReadW(HistoryAggregateType agg, double intervalS) =>
(await B.CreateDataSource().ReadProcessedAsync("W", At(0), At(80), TimeSpan.FromSeconds(intervalS), agg, Ct)).Samples;
private static void Values(IReadOnlyList<DataValueSnapshot> s, params double?[] expected)
{
s.Count.ShouldBe(expected.Length);
for (var i = 0; i < expected.Length; i++)
{
if (expected[i] is { } e) ((double)s[i].Value!).ShouldBe(e, 1e-9, $"bucket {i}");
else s[i].Value.ShouldBeNull($"bucket {i}");
}
}
[Theory]
// 40 s intervals (wider than the sample gaps). B0 [0,40): raw [0,20) 24→40 = 640 Good + [20,30) 40 = 400 Uncertain,
// [30,40) Bad; bridged 640 + [20,40) 40→80/3 = 2000/3. B1 [40,80): raw [50,80) 20 = 600 Good, [40,50) Bad; bridged
// [40,50) 80/3→20 = 700/3 + 600.
[InlineData(HistoryAggregateType.TimeAverage, 98d / 3d, 125d / 6d)] // (3920/3)/40, (2500/3)/40
[InlineData(HistoryAggregateType.Total, 3920d / 3d, 2500d / 3d)] // TimeAverage × 40 s
[InlineData(HistoryAggregateType.TimeAverage2, 104d / 3d, 20d)] // 1040/30 s, 600/30 s
[InlineData(HistoryAggregateType.Total2, 1040d, 600d)]
public async Task Time_weighted_aggregates_over_a_sparse_tag_with_wide_intervals(HistoryAggregateType agg, double b0, double b1)
{
await SeedWAsync();
var s = await ReadW(agg, 40);
Values(s, b0, b1);
// Neither interval is Good throughout (an outage in each) ⇒ Uncertain.
s.Select(x => x.StatusCode).ShouldBe([UncertainCalc, UncertainCalc]);
s.Select(x => x.SourceTimestampUtc).ShouldBe([B.Round(At(0)), B.Round(At(40))]);
}
[Fact]
public async Task Interpolative_over_a_sparse_tag_with_wide_intervals()
{
await SeedWAsync();
var s = await ReadW(HistoryAggregateType.Interpolative, 40);
// +0: from the bound before the window, 0 + 40·(30/50) = 24, Good. +40: inside the outage, bridged 80/3, Uncertain.
Values(s, 24, 80d / 3d);
s.Select(x => x.StatusCode).ShouldBe([GoodInterp, UncertainInterp]);
}
[Fact]
public async Task TimeAverage2_over_a_sparse_tag_with_narrow_intervals()
{
await SeedWAsync();
// 10 s intervals (narrower than the gaps): [0,10) 24→32 = 28 G; [10,20) 32→40 = 36 G; [20,30) 40 held, Uncertain;
// [30,40) and [40,50) inside the outage ⇒ BadNoData; [50,60) … [70,80) the flat 20 up to the bound after, Good.
var s = await ReadW(HistoryAggregateType.TimeAverage2, 10);
Values(s, 28, 36, 40, null, null, 20, 20, 20);
s.Select(x => x.StatusCode).ShouldBe(
[GoodCalc, GoodCalc, UncertainCalc, NoDataCalc, NoDataCalc, GoodCalc, GoodCalc, GoodCalc]);
// Total2 of the first interval: 28 × 10 s.
((double)(await ReadW(HistoryAggregateType.Total2, 10))[0].Value!).ShouldBe(280d, 1e-9);
}
[Fact]
public async Task Interpolative_over_a_sparse_tag_with_narrow_intervals()
{
await SeedWAsync();
var s = await ReadW(HistoryAggregateType.Interpolative, 10);
// +0 24, +10 32 (sloped from the bound); +20 exact 40 (raw Good); +30 ON the outage marker ⇒ bridged 100/3;
// +40 bridged 80/3; +50 exact 20; +60, +70 on the Good 20→20 segment to the bound after.
Values(s, 24, 32, 40, 100d / 3d, 80d / 3d, 20, 20, 20);
s.Select(x => x.StatusCode).ShouldBe(
[GoodInterp, GoodInterp, GoodRaw, UncertainInterp, UncertainInterp, GoodRaw, GoodInterp, GoodInterp]);
}
[Fact]
public async Task Outages_straddling_both_window_edges_use_the_four_bound_seeks()
{
// −10 = 0 G, −5 outage, +10 = 20 G, +20 outage, +30 = 40 G; window [0,20). The four seeks return −10 (latest
// usable before), −5 (latest before), +20 (first at/after end) and +30 (first usable at/after end).
await B.SeedAsync("O",
(At(-10), HistorianValue.FromDouble(0), Good),
(At(-5), HistorianValue.Empty, BadNoCommunication),
(At(10), HistorianValue.FromDouble(20), Good),
(At(20), HistorianValue.Empty, BadNoCommunication),
(At(30), HistorianValue.FromDouble(40), Good));
var src = B.CreateDataSource();
async Task<IReadOnlyList<DataValueSnapshot>> Read(HistoryAggregateType a, double intervalS) =>
(await src.ReadProcessedAsync("O", At(0), At(20), TimeSpan.FromSeconds(intervalS), a, Ct)).Samples;
// Bridged: −10..10 0→20 ⇒ [0,10) mean 15 (150); 10..30 20→40 ⇒ [10,20) mean 25 (250). 400 / 20 = 20.
// (Without the usable-before seek: 250/10 = 25; without the usable-after seek: [10,20) held 20 ⇒ 17.5.)
var ta = (await Read(HistoryAggregateType.TimeAverage, 0)).Single();
((double)ta.Value!).ShouldBe(20d, 1e-9);
ta.StatusCode.ShouldBe(UncertainCalc);
// Simple bounding: [0,10) inside the outage from −5; [10,20) 20 held into the outage at +20 ⇒ 200 / 10 s = 20.
// (Without the any-before seek: (150+200)/20 = 17.5; without the any-after seek: [10,20) 20→30 ⇒ 25.)
var ta2 = (await Read(HistoryAggregateType.TimeAverage2, 0)).Single();
((double)ta2.Value!).ShouldBe(20d, 1e-9);
ta2.StatusCode.ShouldBe(UncertainCalc);
// +0 inside the outage ⇒ bridged 10, Uncertain; +10 exact 20, raw Good.
var i = await Read(HistoryAggregateType.Interpolative, 10);
Values(i, 10, 20);
i.Select(x => x.StatusCode).ShouldBe([UncertainInterp, GoodRaw]);
}
[Fact]
public async Task Time_weighted_aggregates_on_an_unknown_tag_are_BadNoData()
{
var src = B.CreateDataSource();
(await src.ReadProcessedAsync("nope", At(0), At(40), Twenty, HistoryAggregateType.TimeAverage, Ct))
.Samples.ShouldAllBe(x => x.StatusCode == NoDataCalc);
(await src.ReadProcessedAsync("nope", At(0), At(40), Twenty, HistoryAggregateType.Interpolative, Ct))
.Samples.ShouldAllBe(x => x.StatusCode == NoData);
src.GetHealthSnapshot().TotalFailures.ShouldBe(0);
}
[Fact]
public async Task The_bucket_count_is_capped_at_100000()
{