fix(historian): #671 review — raw-row cap, reverse Interpolative, SDK status rule, Partial bit
M1: the time-weighted family (incl. Total, formerly a pushed-down SUM) streams raw window rows into the C# evaluator. New shared knob ServerHistorian:MaxTimeWeightedRawRows (default 1,000,000; <=0 unbounded with a Validate warning) flows through HistorianBackendRegistry into both providers. The window query is capped at cap+1 (SQL Server TOP (@maxRows), QuestDB LIMIT $4); one row past the cap throws the new Core.Abstractions HistoryReadLimitExceededException, which the node manager maps to BadTooManyOperations for that node. Not counted as a backend health failure; bound rows are not counted; per-sample aggregates are not capped. M2: reverse Interpolative is anchored at the later time (Part 13): points at End - i*interval, emitted latest first. 10:00->09:00 @1h yields 10:00. L1: Partial bit (0x4, = Opc.Ua.AggregateBits.Partial) on non-Bad results of a truncated last interval (time-weighted family; per-sample family is a follow-up). L2/L3: aligned with the SDK AggregateCalculator.GetTimeBasedStatusCode (decompiled from Opc.Ua.Server 1.5.378.106): TreatUncertainAsBad=false counts Uncertain time (Uncertain samples, held-into-outage, extrapolation) as Good time; data but no non-Bad time => generic Bad (0x80000000); BadNoData only when there is no data at all. Consequence: a live tag's latest interval reads Good (documented). L5: release note no longer claims Total matches the Gateway Integral exactly (AVEVA's divisor is unverified); it notes Total is now evaluated in C# and subject to the cap. Tests: unit (reverse Interpolative x3, Partial, Bad vs BadNoData incl. an open-ended outage, cap helpers, SDK bit oracles); conformance (reverse Interpolative, raw-row cap at/over/0, bounds not counted, health unaffected); node manager BadTooManyOperations; options default + warning. Each new behaviour falsified by mutation (aggregator x9, data-source cap checks on both backends against the real DBs, node-manager catch).
This commit is contained in:
Joseph Doherty
+8
@@ -39,6 +39,14 @@ public sealed class StatusCodeOracleTests
|
||||
HistorianInfoBits.WithCalculated(interpolatedGood).ShouldBe(0x400u | HistorianInfoBits.HistorianCalculated);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Partial_bit_matches_the_SDK() =>
|
||||
((uint)Opc.Ua.AggregateBits.Partial).ShouldBe(HistorianInfoBits.HistorianPartial);
|
||||
|
||||
[Fact]
|
||||
public void Generic_bad_matches_the_SDK() =>
|
||||
HistorianStatusCodes.Bad.ShouldBe(Opc.Ua.StatusCodes.Bad);
|
||||
|
||||
[Fact]
|
||||
public void Info_type_is_data_value() =>
|
||||
HistorianInfoBits.InfoTypeDataValue.ShouldBe(0x400u);
|
||||
|
||||
+101
-16
@@ -15,6 +15,8 @@ public sealed class TimeWeightedAggregatorTests
|
||||
private const uint GoodRaw = 0x00000000u;
|
||||
private const uint NoData = 0x809B0000u;
|
||||
private const uint InvalidInputs = 0x80D60000u;
|
||||
private const uint BadCalc = 0x80000401u; // generic Bad | Calculated (an all-outage interval)
|
||||
private const uint GoodCalcPartial = 0x00000405u; // Good | Calculated | Partial (truncated last interval)
|
||||
|
||||
private static IReadOnlyList<DataValueSnapshot> Run(HistoryAggregateType agg, double from, double to, double intervalS,
|
||||
params StoredSample[] samples) =>
|
||||
@@ -38,10 +40,12 @@ public sealed class TimeWeightedAggregatorTests
|
||||
[Fact]
|
||||
public void TimeAverage_bridges_the_outage_on_tag_A()
|
||||
{
|
||||
// B0 (150+250)/20 = 20; B1 (350+450)/20 = 40; B2, B3 = 50. Only B1's [30,40) and B0's [0,10) are Good ⇒ all Uncertain.
|
||||
// B0 (150+250)/20 = 20; B1 (350+450)/20 = 40; B2, B3 = 50. Status (SDK rule, Uncertain time counts as Good):
|
||||
// B0 is non-Bad throughout (held-into-outage time is Uncertain) ⇒ Good; B1 has the Bad [20,30) ⇒ Uncertain;
|
||||
// B2, B3 are extrapolated (Uncertain) throughout ⇒ Good.
|
||||
var s = Run(HistoryAggregateType.TimeAverage, 0, 80, 20, A);
|
||||
Values(s, 20, 40, 50, 50);
|
||||
s.Select(x => x.StatusCode).ShouldBe([UncertainCalc, UncertainCalc, UncertainCalc, UncertainCalc]);
|
||||
s.Select(x => x.StatusCode).ShouldBe([GoodCalc, UncertainCalc, GoodCalc, GoodCalc]);
|
||||
s.Select(x => x.SourceTimestampUtc).ShouldBe([At(0), At(20), At(40), At(60)]);
|
||||
}
|
||||
|
||||
@@ -58,7 +62,7 @@ public sealed class TimeWeightedAggregatorTests
|
||||
// B0 350/20 = 17.5; B1 450/10 = 45 (the Bad [20,30) excluded); B2, B3 = 50.
|
||||
var s = Run(HistoryAggregateType.TimeAverage2, 0, 80, 20, A);
|
||||
Values(s, 17.5, 45, 50, 50);
|
||||
s.Select(x => x.StatusCode).ShouldBe([UncertainCalc, UncertainCalc, UncertainCalc, UncertainCalc]);
|
||||
s.Select(x => x.StatusCode).ShouldBe([GoodCalc, UncertainCalc, GoodCalc, GoodCalc]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -140,13 +144,14 @@ public sealed class TimeWeightedAggregatorTests
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Past_the_last_sample_the_value_is_extrapolated_stepped_and_uncertain()
|
||||
public void Past_the_last_sample_the_value_is_extrapolated_stepped()
|
||||
{
|
||||
// -5 = 7 and nothing later: held at 7 over [0,10).
|
||||
// -5 = 7 and nothing later: held at 7 over [0,10). The extrapolated time is Uncertain, which the SDK rule
|
||||
// (TreatUncertainAsBad = false) counts as Good time ⇒ the interval is Good; the Interpolative point is Uncertain.
|
||||
StoredSample[] one = [S(-5, HistorianValue.FromDouble(7), Good)];
|
||||
var s = Run(HistoryAggregateType.TimeAverage2, 0, 10, 0, one).Single();
|
||||
((double)s.Value!).ShouldBe(7d);
|
||||
s.StatusCode.ShouldBe(UncertainCalc);
|
||||
s.StatusCode.ShouldBe(GoodCalc);
|
||||
var i = Run(HistoryAggregateType.Interpolative, 0, 10, 0, one).Single();
|
||||
((double)i.Value!).ShouldBe(7d);
|
||||
i.StatusCode.ShouldBe(UncertainInterpolated);
|
||||
@@ -203,11 +208,15 @@ public sealed class TimeWeightedAggregatorTests
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void An_interval_entirely_inside_an_outage_is_BadNoData_but_Interpolative_bridges_it()
|
||||
public void An_interval_entirely_inside_an_outage_is_Bad_but_Interpolative_bridges_it()
|
||||
{
|
||||
// Window [10,15): raw is Bad throughout ⇒ no non-Bad time ⇒ BadNoData. Interpolative at +10 = 10 + 20·(10/15).
|
||||
Run(HistoryAggregateType.TimeAverage, 10, 15, 0, Outage).Single().StatusCode.ShouldBe(NoDataCalc);
|
||||
Run(HistoryAggregateType.TimeAverage2, 10, 15, 0, Outage).Single().StatusCode.ShouldBe(NoDataCalc);
|
||||
// Window [10,15): raw is Bad throughout ⇒ data but no non-Bad time ⇒ generic Bad (the SDK rule; BadNoData is
|
||||
// reserved for no data at all). Interpolative at +10 = 10 + 20·(10/15).
|
||||
var ta = Run(HistoryAggregateType.TimeAverage, 10, 15, 0, Outage).Single();
|
||||
ta.StatusCode.ShouldBe(BadCalc);
|
||||
ta.Value.ShouldBeNull();
|
||||
Run(HistoryAggregateType.TimeAverage2, 10, 15, 0, Outage).Single().StatusCode.ShouldBe(BadCalc);
|
||||
Run(HistoryAggregateType.Total2, 10, 15, 0, Outage).Single().StatusCode.ShouldBe(BadCalc);
|
||||
var i = Run(HistoryAggregateType.Interpolative, 10, 15, 0, Outage).Single();
|
||||
((double)i.Value!).ShouldBe(10d + 20d * 10d / 15d, 1e-9);
|
||||
i.StatusCode.ShouldBe(UncertainInterpolated);
|
||||
@@ -269,10 +278,10 @@ public sealed class TimeWeightedAggregatorTests
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void An_uncertain_sample_makes_its_segment_uncertain()
|
||||
public void Uncertain_time_counts_as_good_time()
|
||||
{
|
||||
// 0 = 4 Uncertain, 10 = 6 Good, 20 = 6 Good: [0,10) sloped 4→6, quality Uncertain (worse end) ⇒ mean 5;
|
||||
// [10,20) 6 Good. Window [0,20), 10 s.
|
||||
// TreatUncertainAsBad = false (the SDK's time-based status rule): Uncertain time counts toward PercentDataGood.
|
||||
// 0 = 4 Uncertain, 10 = 6 Good, 20 = 6 Good: [0,10) sloped 4→6 ⇒ mean 5; [10,20) 6. Window [0,20), 10 s.
|
||||
StoredSample[] s =
|
||||
[
|
||||
S(0, HistorianValue.FromDouble(4), Uncertain), S(10, HistorianValue.FromDouble(6), Good),
|
||||
@@ -280,7 +289,7 @@ public sealed class TimeWeightedAggregatorTests
|
||||
];
|
||||
var r = Run(HistoryAggregateType.TimeAverage2, 0, 20, 10, s);
|
||||
Values(r, 5, 6);
|
||||
r.Select(x => x.StatusCode).ShouldBe([UncertainCalc, GoodCalc]);
|
||||
r.Select(x => x.StatusCode).ShouldBe([GoodCalc, GoodCalc]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -322,10 +331,86 @@ public sealed class TimeWeightedAggregatorTests
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_truncated_last_bucket_uses_its_own_duration_for_Total()
|
||||
public void A_truncated_last_bucket_uses_its_own_duration_for_Total_and_is_Partial()
|
||||
{
|
||||
// Window [0,50), 20 s: B2 = [40,50) is 10 s of the extrapolated 50 ⇒ TimeAverage 50, Total 500.
|
||||
Values(Run(HistoryAggregateType.Total, 0, 50, 20, A), 400, 800, 500);
|
||||
var s = Run(HistoryAggregateType.Total, 0, 50, 20, A);
|
||||
Values(s, 400, 800, 500);
|
||||
// Only the truncated interval carries the Partial bit.
|
||||
s.Select(x => x.StatusCode).ShouldBe([GoodCalc, UncertainCalc, GoodCalcPartial]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void An_outage_with_no_later_sample_is_Bad_to_the_window_end_not_no_data()
|
||||
{
|
||||
// 0 = 10 G, then an outage marker at +5 and nothing after: [10,20) lies wholly inside the still-open outage.
|
||||
StoredSample[] open = [S(0, HistorianValue.FromDouble(10), Good), S(5, HistorianValue.Empty, BadNoCommunication)];
|
||||
Run(HistoryAggregateType.TimeAverage2, 10, 20, 0, open).Single().StatusCode.ShouldBe(BadCalc);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_bad_or_no_data_truncated_interval_carries_no_partial_bit()
|
||||
{
|
||||
// Window [0,15), 10 s: B1 = [10,15) is inside the outage ⇒ Bad without Partial.
|
||||
Run(HistoryAggregateType.TimeAverage2, 0, 15, 10, Outage)[1].StatusCode.ShouldBe(BadCalc);
|
||||
Run(HistoryAggregateType.TimeAverage2, 0, 15, 10)[1].StatusCode.ShouldBe(NoDataCalc);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- reverse Interpolative (Part 13: anchored at the later time)
|
||||
|
||||
[Fact]
|
||||
public void A_reverse_Interpolative_read_is_anchored_at_its_later_time()
|
||||
{
|
||||
// Request At(20) → At(0), 20 s: one point, at At(20) (not At(0)). Bounded: segment 10..30 is 20 flat, Good.
|
||||
// A forward read of the same window would give the value at At(0) = 10.
|
||||
var s = TimeWeightedAggregator.Evaluate(HistoryAggregateType.Interpolative,
|
||||
BucketPlan.Create(At(20), At(0), TimeSpan.FromSeconds(20)), Bounded, Now);
|
||||
s.Single().SourceTimestampUtc.ShouldBe(At(20));
|
||||
((double)s.Single().Value!).ShouldBe(20d);
|
||||
s.Single().StatusCode.ShouldBe(GoodInterpolated);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_reverse_Interpolative_read_steps_back_from_the_later_time_latest_first()
|
||||
{
|
||||
// Tag A, At(80) → At(0), 20 s: points 80, 60, 40, 20 (latest first). 80 and 60 are past the last sample ⇒ 50
|
||||
// extrapolated, Uncertain; 40 exact Good 50; 20 sits on the Bad sample ⇒ bridged 30, Uncertain.
|
||||
var s = TimeWeightedAggregator.Evaluate(HistoryAggregateType.Interpolative,
|
||||
BucketPlan.Create(At(80), At(0), TimeSpan.FromSeconds(20)), A, Now);
|
||||
s.Select(x => x.SourceTimestampUtc).ShouldBe([At(80), At(60), At(40), At(20)]);
|
||||
Values(s, 50, 50, 50, 30);
|
||||
s.Select(x => x.StatusCode).ShouldBe([UncertainInterpolated, UncertainInterpolated, GoodRaw, UncertainInterpolated]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_reverse_Interpolative_read_with_a_partial_interval_ends_inside_the_window()
|
||||
{
|
||||
// At(25) → At(0), 10 s: three intervals ⇒ points 25, 15, 5 (never before the window start).
|
||||
// Bounded: -10..10 slope 1/s from 0 ⇒ +5 = 15; 10..30 flat 20 ⇒ +15 = 20, +25 = 20.
|
||||
var s = TimeWeightedAggregator.Evaluate(HistoryAggregateType.Interpolative,
|
||||
BucketPlan.Create(At(25), At(0), TimeSpan.FromSeconds(10)), Bounded, Now);
|
||||
s.Select(x => x.SourceTimestampUtc).ShouldBe([At(25), At(15), At(5)]);
|
||||
Values(s, 20, 20, 15);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- raw-row cap helpers
|
||||
|
||||
[Theory]
|
||||
[InlineData(1_000_000, 1_000_001)]
|
||||
[InlineData(1, 2)]
|
||||
[InlineData(0, int.MaxValue)]
|
||||
[InlineData(-5, int.MaxValue)]
|
||||
[InlineData(int.MaxValue, int.MaxValue)]
|
||||
public void The_window_fetch_limit_is_one_past_the_cap(int cap, int expected) =>
|
||||
TimeWeightedAggregator.WindowFetchLimit(cap).ShouldBe(expected);
|
||||
|
||||
[Fact]
|
||||
public void The_window_row_check_fails_only_past_a_positive_cap()
|
||||
{
|
||||
TimeWeightedAggregator.CheckWindowRows(4, 4, "T");
|
||||
TimeWeightedAggregator.CheckWindowRows(5_000_000, 0, "T");
|
||||
var ex = Should.Throw<HistoryReadLimitExceededException>(() => TimeWeightedAggregator.CheckWindowRows(5, 4, "T"));
|
||||
ex.Limit.ShouldBe(4);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
|
||||
Reference in New Issue
Block a user