Joseph Doherty
1f88669304
New AggregateFamily.TimeWeighted serving TimeAverage, TimeAverage2, Total, Total2 and Interpolative, evaluated by the shared Sql.Common TimeWeightedAggregator for both SQL backends (identical results by construction). HistoryAggregateType gains TimeAverage2 + Total2 (appended); OtOpcUaNodeManager.MapAggregate maps their NodeIds. Gateway's mapper already refuses unknown defined members => BadAggregateNotSupported per node. Semantics (server defaults, client AggregateConfiguration ignored): TreatUncertainAsBad=false, PercentDataBad=PercentDataGood=100, UseSlopedExtrapolation=false. Sloped for Double/Int64, stepped for Boolean; a usable sample followed by a Bad one is held up to the outage (AtTime rule); stepped Uncertain extrapolation past the last sample. TimeAverage/Total use interpolated bounding (bridge outages); TimeAverage2/Total2 exclude Bad time. Bounding values: latest usable + latest any before start, first any + first usable at/after end (four TOP 1 / LIMIT 1 seeks). CLIENT-VISIBLE: Total on SqlServer/QuestDb was SUM of samples; it is now the time-weighted integral (TimeAverage x interval seconds), matching Gateway's Integral. Release note in docs/Historian.md. Trade-off (deviation from the preferred hybrid pushdown, allowed by the design): the time-weighted family is NOT pushed down. Each read streams the window's rows plus the four bounds (SQL Server: one five-result-set batch; QuestDB: bounds UNION ALL + window select) into the C# evaluator. Network is O(samples in window), memory O(buckets). Writing the per-segment interpolation/edge splitting twice in two SQL dialects and keeping them bit-identical was judged the bigger risk; this also makes the QuestDB lag/lead spike moot. Tests: 35 hand-computed TimeWeightedAggregator unit cases (each of 10 implementation mutations goes red); served-aggregate pin 13 -> 17; conformance sparse fixture (intervals wider and narrower than sample gaps) + an outage straddling both window edges (each of the four SQL seeks disabled in turn goes red); node-manager NodeId mapping test.
346 lines
16 KiB
C#
346 lines
16 KiB
C#
using Shouldly;
|
||
using Xunit;
|
||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||
using static ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common.Tests.Fixture;
|
||
|
||
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common.Tests;
|
||
|
||
/// <summary>
|
||
/// The Part 13 time-weighted aggregates (#671) over hand-built sample streams. Every literal is hand-computed in the
|
||
/// comment above it (areas in value·seconds; "raw" = the simple-bounding signal, "bridged" = interpolated bounding).
|
||
/// </summary>
|
||
public sealed class TimeWeightedAggregatorTests
|
||
{
|
||
private static readonly DateTime Now = At(3600);
|
||
private const uint GoodRaw = 0x00000000u;
|
||
private const uint NoData = 0x809B0000u;
|
||
private const uint InvalidInputs = 0x80D60000u;
|
||
|
||
private static IReadOnlyList<DataValueSnapshot> Run(HistoryAggregateType agg, double from, double to, double intervalS,
|
||
params StoredSample[] samples) =>
|
||
TimeWeightedAggregator.Evaluate(agg, BucketPlan.Create(At(from), At(to), TimeSpan.FromSeconds(intervalS)), samples, Now);
|
||
|
||
private static void Values(IReadOnlyList<DataValueSnapshot> s, params double?[] expected)
|
||
{
|
||
s.Count.ShouldBe(expected.Length);
|
||
for (var i = 0; i < expected.Length; i++)
|
||
{
|
||
if (expected[i] is { } e) ((double)s[i].Value!).ShouldBe(e, 1e-9, $"bucket {i}");
|
||
else s[i].Value.ShouldBeNull($"bucket {i}");
|
||
}
|
||
}
|
||
|
||
// ---------------------------------------------------------------- tag A: +0=10 G, +10=20 G, +20=Bad, +30=40 G, +40=50 G
|
||
// Window [0,80), 20 s buckets. Raw: [0,10) 10→20 Good (150); [10,20) 20 held into the outage, Uncertain (200);
|
||
// [20,30) Bad; [30,40) 40→50 Good (450); [40,80) 50 extrapolated, Uncertain (2000).
|
||
// Bridged: [0,10) 150; [10,30) 20→40 = 250 in B0 + 350 in B1; [30,40) 450; [40,80) 50 held.
|
||
|
||
[Fact]
|
||
public void TimeAverage_bridges_the_outage_on_tag_A()
|
||
{
|
||
// B0 (150+250)/20 = 20; B1 (350+450)/20 = 40; B2, B3 = 50. Only B1's [30,40) and B0's [0,10) are Good ⇒ all Uncertain.
|
||
var s = Run(HistoryAggregateType.TimeAverage, 0, 80, 20, A);
|
||
Values(s, 20, 40, 50, 50);
|
||
s.Select(x => x.StatusCode).ShouldBe([UncertainCalc, UncertainCalc, UncertainCalc, UncertainCalc]);
|
||
s.Select(x => x.SourceTimestampUtc).ShouldBe([At(0), At(20), At(40), At(60)]);
|
||
}
|
||
|
||
[Fact]
|
||
public void Total_is_TimeAverage_times_interval_seconds_on_tag_A()
|
||
{
|
||
// 20×20, 40×20, 50×20, 50×20.
|
||
Values(Run(HistoryAggregateType.Total, 0, 80, 20, A), 400, 800, 1000, 1000);
|
||
}
|
||
|
||
[Fact]
|
||
public void TimeAverage2_excludes_the_bad_interval_on_tag_A()
|
||
{
|
||
// B0 350/20 = 17.5; B1 450/10 = 45 (the Bad [20,30) excluded); B2, B3 = 50.
|
||
var s = Run(HistoryAggregateType.TimeAverage2, 0, 80, 20, A);
|
||
Values(s, 17.5, 45, 50, 50);
|
||
s.Select(x => x.StatusCode).ShouldBe([UncertainCalc, UncertainCalc, UncertainCalc, UncertainCalc]);
|
||
}
|
||
|
||
[Fact]
|
||
public void Total2_integrates_only_the_non_bad_data_on_tag_A()
|
||
{
|
||
// B0 150+200 = 350; B1 450; B2, B3 1000.
|
||
Values(Run(HistoryAggregateType.Total2, 0, 80, 20, A), 350, 450, 1000, 1000);
|
||
}
|
||
|
||
[Fact]
|
||
public void Interpolative_on_tag_A()
|
||
{
|
||
// +0 exact Good 10 (raw status); +20 sits on the Bad sample ⇒ bridged 20→40 at the midpoint = 30, Uncertain;
|
||
// +40 exact Good 50; +60 past the last sample ⇒ 50 extrapolated, Uncertain.
|
||
var s = Run(HistoryAggregateType.Interpolative, 0, 80, 20, A);
|
||
Values(s, 10, 30, 50, 50);
|
||
s.Select(x => x.StatusCode).ShouldBe([GoodRaw, UncertainInterpolated, GoodRaw, UncertainInterpolated]);
|
||
s.Select(x => x.SourceTimestampUtc).ShouldBe([At(0), At(20), At(40), At(60)]);
|
||
s.ShouldAllBe(x => x.ServerTimestampUtc == Now);
|
||
}
|
||
|
||
// ---------------------------------------------------------------- bounding values on both sides, crossing segments
|
||
|
||
private static readonly StoredSample[] Bounded =
|
||
[
|
||
S(-10, HistorianValue.FromDouble(0), Good), // bound before the window
|
||
S(10, HistorianValue.FromDouble(20), Good),
|
||
S(30, HistorianValue.FromDouble(20), Good), // bound after the window
|
||
];
|
||
|
||
[Theory]
|
||
// Window [0,20), 10 s. Segment -10..10 (0→20) crosses the window start: B0 [0,10) = 10→20, mean 15.
|
||
// B1 [10,20) = 20 flat (crossing the window end into the bound). All Good.
|
||
[InlineData(HistoryAggregateType.TimeAverage, 15d, 20d)]
|
||
[InlineData(HistoryAggregateType.TimeAverage2, 15d, 20d)]
|
||
[InlineData(HistoryAggregateType.Total, 150d, 200d)]
|
||
[InlineData(HistoryAggregateType.Total2, 150d, 200d)]
|
||
public void Bounding_values_before_and_after_the_window_are_used(HistoryAggregateType agg, double b0, double b1)
|
||
{
|
||
var s = Run(agg, 0, 20, 10, Bounded);
|
||
Values(s, b0, b1);
|
||
s.ShouldAllBe(x => x.StatusCode == GoodCalc);
|
||
}
|
||
|
||
[Fact]
|
||
public void Interpolative_interpolates_from_the_bound_before_the_window()
|
||
{
|
||
// +0: 0 + (20−0)·(10/20) = 10, Good both ends; +10 exact Good 20.
|
||
var s = Run(HistoryAggregateType.Interpolative, 0, 20, 10, Bounded);
|
||
Values(s, 10, 20);
|
||
s.Select(x => x.StatusCode).ShouldBe([GoodInterpolated, GoodRaw]);
|
||
}
|
||
|
||
[Fact]
|
||
public void A_segment_spanning_several_buckets_is_split_at_every_edge()
|
||
{
|
||
// One segment 0..40 (0→40), window [0,40), 10 s: bucket means 5, 15, 25, 35; Total2 50, 150, 250, 350.
|
||
StoredSample[] line = [S(0, HistorianValue.FromDouble(0), Good), S(40, HistorianValue.FromDouble(40), Good)];
|
||
Values(Run(HistoryAggregateType.TimeAverage2, 0, 40, 10, line), 5, 15, 25, 35);
|
||
Values(Run(HistoryAggregateType.Total2, 0, 40, 10, line), 50, 150, 250, 350);
|
||
Values(Run(HistoryAggregateType.Interpolative, 0, 40, 10, line), 0, 10, 20, 30);
|
||
}
|
||
|
||
// ---------------------------------------------------------------- empty buckets between bounds
|
||
|
||
[Fact]
|
||
public void An_empty_bucket_with_bounds_on_both_sides_is_interpolated_not_no_data()
|
||
{
|
||
// -5 = 0, +25 = 30 (slope 1/s); window [0,20), 10 s holds no sample at all.
|
||
StoredSample[] sparse = [S(-5, HistorianValue.FromDouble(0), Good), S(25, HistorianValue.FromDouble(30), Good)];
|
||
var interp = Run(HistoryAggregateType.Interpolative, 0, 20, 10, sparse);
|
||
Values(interp, 5, 15);
|
||
interp.ShouldAllBe(x => x.StatusCode == GoodInterpolated);
|
||
// B0 5→15 mean 10; B1 15→25 mean 20.
|
||
var avg = Run(HistoryAggregateType.TimeAverage, 0, 20, 10, sparse);
|
||
Values(avg, 10, 20);
|
||
avg.ShouldAllBe(x => x.StatusCode == GoodCalc);
|
||
Values(Run(HistoryAggregateType.Total, 0, 20, 10, sparse), 100, 200);
|
||
}
|
||
|
||
[Fact]
|
||
public void Past_the_last_sample_the_value_is_extrapolated_stepped_and_uncertain()
|
||
{
|
||
// -5 = 7 and nothing later: held at 7 over [0,10).
|
||
StoredSample[] one = [S(-5, HistorianValue.FromDouble(7), Good)];
|
||
var s = Run(HistoryAggregateType.TimeAverage2, 0, 10, 0, one).Single();
|
||
((double)s.Value!).ShouldBe(7d);
|
||
s.StatusCode.ShouldBe(UncertainCalc);
|
||
var i = Run(HistoryAggregateType.Interpolative, 0, 10, 0, one).Single();
|
||
((double)i.Value!).ShouldBe(7d);
|
||
i.StatusCode.ShouldBe(UncertainInterpolated);
|
||
}
|
||
|
||
[Theory]
|
||
[InlineData(HistoryAggregateType.TimeAverage)]
|
||
[InlineData(HistoryAggregateType.TimeAverage2)]
|
||
[InlineData(HistoryAggregateType.Total)]
|
||
[InlineData(HistoryAggregateType.Total2)]
|
||
public void No_data_at_all_is_BadNoData_with_the_calculated_bit(HistoryAggregateType agg)
|
||
{
|
||
var s = Run(agg, 0, 20, 10);
|
||
s.ShouldAllBe(x => x.StatusCode == NoDataCalc && x.Value == null);
|
||
}
|
||
|
||
[Fact]
|
||
public void Interpolative_with_no_data_is_plain_BadNoData()
|
||
{
|
||
// The only sample is at +15: nothing at or before either bucket start (+0, +10).
|
||
var s = Run(HistoryAggregateType.Interpolative, 0, 20, 10, S(15, HistorianValue.FromDouble(1), Good));
|
||
s.ShouldAllBe(x => x.StatusCode == NoData && x.Value == null);
|
||
}
|
||
|
||
// ---------------------------------------------------------------- Bad intervals: TimeAverage vs TimeAverage2/Total2
|
||
|
||
private static readonly StoredSample[] Outage =
|
||
[
|
||
S(0, HistorianValue.FromDouble(10), Good),
|
||
S(5, HistorianValue.Empty, BadNoCommunication),
|
||
S(15, HistorianValue.FromDouble(30), Good),
|
||
S(25, HistorianValue.FromDouble(30), Good), // bound after the window
|
||
];
|
||
|
||
[Fact]
|
||
public void TimeAverage2_and_Total2_exclude_the_bad_interval()
|
||
{
|
||
// Window [0,20) one bucket. Raw: [0,5) 10 held into the outage (50, Uncertain); [5,15) Bad; [15,20) 30 Good (150).
|
||
// TimeAverage2 = 200 / 10 s = 20; Total2 = 200.
|
||
var ta2 = Run(HistoryAggregateType.TimeAverage2, 0, 20, 0, Outage).Single();
|
||
((double)ta2.Value!).ShouldBe(20d);
|
||
ta2.StatusCode.ShouldBe(UncertainCalc);
|
||
((double)Run(HistoryAggregateType.Total2, 0, 20, 0, Outage).Single().Value!).ShouldBe(200d);
|
||
}
|
||
|
||
[Fact]
|
||
public void TimeAverage_and_Total_interpolate_across_the_bad_interval()
|
||
{
|
||
// Bridged: [0,15) 10→30, mean 20 ⇒ 300; [15,20) 30 ⇒ 150. TimeAverage = 450/20 = 22.5; Total = 22.5 × 20 = 450.
|
||
var ta = Run(HistoryAggregateType.TimeAverage, 0, 20, 0, Outage).Single();
|
||
((double)ta.Value!).ShouldBe(22.5d);
|
||
ta.StatusCode.ShouldBe(UncertainCalc);
|
||
((double)Run(HistoryAggregateType.Total, 0, 20, 0, Outage).Single().Value!).ShouldBe(450d);
|
||
}
|
||
|
||
[Fact]
|
||
public void An_interval_entirely_inside_an_outage_is_BadNoData_but_Interpolative_bridges_it()
|
||
{
|
||
// Window [10,15): raw is Bad throughout ⇒ no non-Bad time ⇒ BadNoData. Interpolative at +10 = 10 + 20·(10/15).
|
||
Run(HistoryAggregateType.TimeAverage, 10, 15, 0, Outage).Single().StatusCode.ShouldBe(NoDataCalc);
|
||
Run(HistoryAggregateType.TimeAverage2, 10, 15, 0, Outage).Single().StatusCode.ShouldBe(NoDataCalc);
|
||
var i = Run(HistoryAggregateType.Interpolative, 10, 15, 0, Outage).Single();
|
||
((double)i.Value!).ShouldBe(10d + 20d * 10d / 15d, 1e-9);
|
||
i.StatusCode.ShouldBe(UncertainInterpolated);
|
||
}
|
||
|
||
[Fact]
|
||
public void Interpolative_on_a_segment_that_runs_into_an_outage_is_uncertain()
|
||
{
|
||
// +2 lies between the Good +0 and the Bad +5: bridged 10→30 over [0,15) ⇒ 10 + 20·(2/15); the simple-bounding
|
||
// signal there is held into an outage, so the point is Uncertain even though both bridging ends are Good.
|
||
var i = Run(HistoryAggregateType.Interpolative, 2, 20, 0, Outage).Single();
|
||
((double)i.Value!).ShouldBe(10d + 20d * 2d / 15d, 1e-9);
|
||
i.StatusCode.ShouldBe(UncertainInterpolated);
|
||
}
|
||
|
||
[Fact]
|
||
public void The_latest_usable_bound_is_found_behind_a_bad_bound()
|
||
{
|
||
// -10 = 0 G, -5 Bad, +10 = 20 G, +30 = 20 G. Window [0,20) one bucket.
|
||
// Bridged: -10..10 0→20 ⇒ [0,10) 10→20 (150); [10,20) 20 (200) ⇒ TimeAverage = 17.5.
|
||
// Raw: [-5,10) Bad; [10,20) 20 Good (200) ⇒ TimeAverage2 = 20. Good 10 s of 20 ⇒ Uncertain.
|
||
StoredSample[] s =
|
||
[
|
||
S(-10, HistorianValue.FromDouble(0), Good), S(-5, HistorianValue.Empty, BadNoCommunication),
|
||
S(10, HistorianValue.FromDouble(20), Good), S(30, HistorianValue.FromDouble(20), Good),
|
||
];
|
||
((double)Run(HistoryAggregateType.TimeAverage, 0, 20, 0, s).Single().Value!).ShouldBe(17.5d);
|
||
var ta2 = Run(HistoryAggregateType.TimeAverage2, 0, 20, 0, s).Single();
|
||
((double)ta2.Value!).ShouldBe(20d);
|
||
ta2.StatusCode.ShouldBe(UncertainCalc);
|
||
var i = Run(HistoryAggregateType.Interpolative, 0, 20, 0, s).Single();
|
||
((double)i.Value!).ShouldBe(10d); // inside the outage, bridged -10..10 at 0
|
||
i.StatusCode.ShouldBe(UncertainInterpolated);
|
||
}
|
||
|
||
// ---------------------------------------------------------------- stepped Boolean, Uncertain, invalid inputs
|
||
|
||
private static readonly StoredSample[] Switch =
|
||
[
|
||
S(0, HistorianValue.FromBoolean(true), Good),
|
||
S(5, HistorianValue.FromBoolean(false), Good),
|
||
S(15, HistorianValue.FromBoolean(true), Good),
|
||
S(25, HistorianValue.FromBoolean(true), Good), // bound after the window
|
||
];
|
||
|
||
[Fact]
|
||
public void Booleans_are_stepped_not_sloped()
|
||
{
|
||
// Window [0,20), 10 s. B0: 1 over [0,5), 0 over [5,10) ⇒ 0.5 (sloped would give 0.375). B1: 0 over [10,15),
|
||
// 1 over [15,20) ⇒ 0.5. All Good. Total: 5 and 5.
|
||
var s = Run(HistoryAggregateType.TimeAverage, 0, 20, 10, Switch);
|
||
Values(s, 0.5, 0.5);
|
||
s.ShouldAllBe(x => x.StatusCode == GoodCalc);
|
||
Values(Run(HistoryAggregateType.Total, 0, 20, 10, Switch), 5, 5);
|
||
// Interpolative at +10: stepped from +5 (false) ⇒ 0, Good.
|
||
var i = Run(HistoryAggregateType.Interpolative, 0, 20, 10, Switch);
|
||
Values(i, 1, 0);
|
||
i.Select(x => x.StatusCode).ShouldBe([GoodRaw, GoodInterpolated]);
|
||
}
|
||
|
||
[Fact]
|
||
public void An_uncertain_sample_makes_its_segment_uncertain()
|
||
{
|
||
// 0 = 4 Uncertain, 10 = 6 Good, 20 = 6 Good: [0,10) sloped 4→6, quality Uncertain (worse end) ⇒ mean 5;
|
||
// [10,20) 6 Good. Window [0,20), 10 s.
|
||
StoredSample[] s =
|
||
[
|
||
S(0, HistorianValue.FromDouble(4), Uncertain), S(10, HistorianValue.FromDouble(6), Good),
|
||
S(20, HistorianValue.FromDouble(6), Good),
|
||
];
|
||
var r = Run(HistoryAggregateType.TimeAverage2, 0, 20, 10, s);
|
||
Values(r, 5, 6);
|
||
r.Select(x => x.StatusCode).ShouldBe([UncertainCalc, GoodCalc]);
|
||
}
|
||
|
||
[Fact]
|
||
public void Int64_samples_are_sloped_like_doubles()
|
||
{
|
||
StoredSample[] s = [S(0, HistorianValue.FromInt64(0), Good), S(10, HistorianValue.FromInt64(10), Good)];
|
||
Values(Run(HistoryAggregateType.TimeAverage2, 0, 10, 0, s), 5);
|
||
}
|
||
|
||
[Theory]
|
||
[InlineData(HistoryAggregateType.TimeAverage)]
|
||
[InlineData(HistoryAggregateType.TimeAverage2)]
|
||
[InlineData(HistoryAggregateType.Total)]
|
||
[InlineData(HistoryAggregateType.Total2)]
|
||
public void A_usable_string_sample_makes_the_interval_invalid_inputs(HistoryAggregateType agg)
|
||
{
|
||
StoredSample[] s = [S(0, HistorianValue.FromString("a"), Good), S(10, HistorianValue.FromString("b"), Good)];
|
||
var r = Run(agg, 0, 20, 0, s).Single();
|
||
r.StatusCode.ShouldBe(InvalidInputsCalc);
|
||
r.Value.ShouldBeNull();
|
||
}
|
||
|
||
[Fact]
|
||
public void Interpolative_on_a_string_is_invalid_inputs()
|
||
{
|
||
StoredSample[] s = [S(0, HistorianValue.FromString("a"), Good), S(10, HistorianValue.FromString("b"), Good)];
|
||
Run(HistoryAggregateType.Interpolative, 0, 20, 10, s).ShouldAllBe(x => x.StatusCode == InvalidInputs);
|
||
}
|
||
|
||
// ---------------------------------------------------------------- plan shape
|
||
|
||
[Fact]
|
||
public void A_reverse_window_is_computed_forward_and_returned_descending()
|
||
{
|
||
var s = TimeWeightedAggregator.Evaluate(HistoryAggregateType.TimeAverage2,
|
||
BucketPlan.Create(At(80), At(0), TimeSpan.FromSeconds(20)), A, Now);
|
||
s.Select(x => x.SourceTimestampUtc).ShouldBe([At(60), At(40), At(20), At(0)]);
|
||
Values(s, 50, 50, 45, 17.5);
|
||
}
|
||
|
||
[Fact]
|
||
public void A_truncated_last_bucket_uses_its_own_duration_for_Total()
|
||
{
|
||
// Window [0,50), 20 s: B2 = [40,50) is 10 s of the extrapolated 50 ⇒ TimeAverage 50, Total 500.
|
||
Values(Run(HistoryAggregateType.Total, 0, 50, 20, A), 400, 800, 500);
|
||
}
|
||
|
||
[Fact]
|
||
public void A_duplicate_bound_is_ignored()
|
||
{
|
||
// The same row fetched as both "latest usable" and "latest any" bound.
|
||
var dup = new[] { Bounded[0], Bounded[0], Bounded[1], Bounded[2], Bounded[2] };
|
||
Values(Run(HistoryAggregateType.TimeAverage2, 0, 20, 10, dup), 15, 20);
|
||
}
|
||
|
||
[Fact]
|
||
public void A_non_time_weighted_aggregate_is_refused()
|
||
{
|
||
Should.Throw<HistoryAggregateNotSupportedException>(() =>
|
||
new TimeWeightedAggregator(HistoryAggregateType.Average, BucketPlan.Create(At(0), At(10), TimeSpan.Zero)));
|
||
}
|
||
}
|