fix(historian): #680 review — recovery/only-sample TA2/Total2 value, outage-last + Boolean parity, docs
TimeAverage2/Total2: the interval the data ends inside with no non-Bad time before its last sample (comms-loss recovery, a single sample) answers that sample's value (Total2 0), Uncertain + Partial, instead of Bad/BadNoData — as the SDK. Parity datasets RecoveryEnds, OutageLast, BoolSwitch with documented masks; ProcessedConformance's reverse TA/Total mask narrowed to the later-edge-sample interval. Docs: report-by-exception consequence, outage-last and Boolean differences; CLAUDE.md historian bullet.
This commit is contained in:
Joseph Doherty
@@ -352,16 +352,28 @@ public abstract class ProcessedConformance : ConformanceBase
|
||||
}
|
||||
}
|
||||
var reverseInterpolative = agg == HistoryAggregateType.Interpolative && from > to;
|
||||
// Documented SDK divergence (#680, see SdkAggregateParityTests): on a reverse read the SDK files a sample exactly
|
||||
// on an interval's later edge as its early bound, so TimeAverage/Total there take the status of the segment
|
||||
// beyond it (Dense: +20 next to the Bad +22.5).
|
||||
var reverseEdgeStatus = from > to && agg is HistoryAggregateType.TimeAverage or HistoryAggregateType.Total;
|
||||
SdkAggregateOracle.ShouldMatch(ours, SdkAggregateOracle.Run(agg, start, end, interval, rows),
|
||||
var sdk = SdkAggregateOracle.Run(agg, start, end, interval, rows);
|
||||
if (from > to && agg is HistoryAggregateType.TimeAverage or HistoryAggregateType.Total)
|
||||
{
|
||||
// Documented SDK divergence (#680, see SdkAggregateParityTests.MaskDocumented): on a reverse read the SDK files
|
||||
// a sample exactly on an interval's later edge as its early bound, so a TimeAverage/Total interval ending on a
|
||||
// sample takes the status of the segment beyond it (Dense: +20 next to the Bad +22.5). Only that interval,
|
||||
// only Good (ours) vs Uncertain (SDK), is aligned.
|
||||
var plan = BucketPlan.Create(start, end, interval);
|
||||
for (long i = 0; i < plan.BucketCount; i++)
|
||||
{
|
||||
var k = (int)(plan.BucketCount - 1 - i);
|
||||
if (k < sdk.Count && rows.Any(r => r.Item1.Ticks == plan.UpperTicks(i))
|
||||
&& (ours[k].StatusCode & 0xFFFF0000u) == 0
|
||||
&& (sdk[k].StatusCode & 0xFFFF0000u) == 0x40A40000u)
|
||||
ours[k] = ours[k] with { StatusCode = ours[k].StatusCode | 0x40A40000u };
|
||||
}
|
||||
}
|
||||
SdkAggregateOracle.ShouldMatch(ours, sdk,
|
||||
$"{agg} {from}→{to}/{intervalS}s",
|
||||
compareCalculatedInterpolatedBits: tw && !reverseInterpolative,
|
||||
// pre-existing, #680: Start/End raw-sample timestamp + status; Delta status (see SdkAggregateParityTests).
|
||||
compareCodeBits: agg != HistoryAggregateType.Delta && !reverseInterpolative && !startOrEnd
|
||||
&& !reverseEdgeStatus,
|
||||
compareCodeBits: agg != HistoryAggregateType.Delta && !reverseInterpolative && !startOrEnd,
|
||||
compareTimestamps: !startOrEnd);
|
||||
}
|
||||
}
|
||||
|
||||
+88
-1
@@ -32,6 +32,16 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common.Tests;
|
||||
/// <c>GetRegionsInValueSet</c>); Part 13 ends TimeAverage2/Total2 at the last sample and runs the PercentGood/Bad
|
||||
/// region to the interval end, which ours does (#680; pinned by <c>TimeWeightedAggregatorTests</c>). That
|
||||
/// interval's value is not compared; its status, Partial bit and timestamp are.</item>
|
||||
/// <item><b>an outage marker as the last sample</b> (<see cref="OutageLast"/>): ours runs the outage to the window
|
||||
/// end — TimeAverage2/Total2 <c>Bad</c>, the percentages 100 % Bad, no end-of-data Partial on the interval holding
|
||||
/// it; the SDK treats the data as ended (<c>BadNoData</c> after it, Partial on that interval); and TimeAverage/Total
|
||||
/// over the span the last usable value is held up to the marker are Uncertain here (no Good value follows ⇒
|
||||
/// extrapolated, Part 13), Good in the SDK — masked by
|
||||
/// <see cref="MaskDocumented"/>;</item>
|
||||
/// <item><b>Booleans</b> (<see cref="BoolSwitch"/>): ours steps them (Part 13: a non-analog value is held, and the
|
||||
/// bound next to an Uncertain sample takes the earlier sample's status); the SDK interpolates them as numbers — its
|
||||
/// values (e.g. 1 vs our 0.5 over an interval switching half-way) and the Good/Uncertain choice are not compared,
|
||||
/// Bad vs non-Bad, timestamps and Partial are;</item>
|
||||
/// <item><b>TimeAverage / Total, reverse, a sample exactly on an interval's later edge:</b> the SDK's backward
|
||||
/// <c>UpdateSlice</c> files it as the early bound (below), so the interval takes the status of the segment beyond
|
||||
/// it (Uncertain when that segment skips a Bad sample); ours keeps the interval's own segments — masked by
|
||||
@@ -91,6 +101,48 @@ public sealed class SdkAggregateParityTests
|
||||
S(100, HistorianValue.FromDouble(40), Good),
|
||||
];
|
||||
|
||||
/// <summary>
|
||||
/// #680 review: the comms-loss recovery shape — a Good sample, an outage marker, then ONE recovery sample and nothing
|
||||
/// after (+27.5, inside an interval of every window, never on an edge): the interval holding the recovery sample has
|
||||
/// no non-Bad time before it, so TimeAverage2/Total2 answer the sample's value (Total2 0), Uncertain + Partial.
|
||||
/// </summary>
|
||||
private static readonly StoredSample[] RecoveryEnds =
|
||||
[
|
||||
S(-10, HistorianValue.FromDouble(10), Good),
|
||||
S(5, HistorianValue.Empty, BadNoCommunication),
|
||||
S(27.5, HistorianValue.FromDouble(20), Good),
|
||||
];
|
||||
|
||||
/// <summary>
|
||||
/// #680 review: an outage marker as the LAST sample (+31, nothing after) — ours: the outage runs to the window end
|
||||
/// (TimeAverage2/Total2 Bad, the percentages 100 % Bad); the SDK: the data ended (BadNoData, Partial on the interval
|
||||
/// holding the marker). Also TimeAverage/Total over the span the last usable value is held up to the marker: ours
|
||||
/// Uncertain (Part 13 — no Good value after it, so extrapolated), the SDK Good. Documented, deliberate differences.
|
||||
/// </summary>
|
||||
private static readonly StoredSample[] OutageLast =
|
||||
[
|
||||
S(-10, HistorianValue.FromDouble(10), Good),
|
||||
S(25, HistorianValue.FromDouble(20), Good),
|
||||
S(31, HistorianValue.Empty, BadNoCommunication),
|
||||
];
|
||||
|
||||
/// <summary>
|
||||
/// #680 review: Boolean (stepped) data with an Uncertain and a Bad sample. Ours steps Booleans (Part 13: a
|
||||
/// non-analog value is held); the SDK interpolates them as numbers and bounds them sloped — so only timestamps,
|
||||
/// status and Partial are compared for the values it computes that way (see <see cref="MaskDocumented"/>).
|
||||
/// </summary>
|
||||
private static readonly StoredSample[] BoolSwitch =
|
||||
[
|
||||
S(-10, HistorianValue.FromBoolean(true), Good),
|
||||
S(10, HistorianValue.FromBoolean(false), Good),
|
||||
S(27.5, HistorianValue.FromBoolean(true), Uncertain),
|
||||
S(32, HistorianValue.FromBoolean(false), Good),
|
||||
S(47.5, HistorianValue.FromBoolean(true), Good),
|
||||
S(62, HistorianValue.FromBoolean(true), Bad),
|
||||
S(71, HistorianValue.FromBoolean(false), Good),
|
||||
S(100, HistorianValue.FromBoolean(true), Good),
|
||||
];
|
||||
|
||||
/// <summary>Data that starts inside the window (no sample before +7) and runs past its end.</summary>
|
||||
private static readonly StoredSample[] LateStart = Enumerable.Range(0, 20)
|
||||
.Select(i => S(7 + i * 5, HistorianValue.FromDouble(i % 2 == 0 ? 20 : 40), Good))
|
||||
@@ -148,7 +200,11 @@ public sealed class SdkAggregateParityTests
|
||||
var plan = BucketPlan.Create(At(from), At(to), TimeSpan.FromSeconds(intervalS));
|
||||
var list = ours.ToList();
|
||||
var last = data.Max(x => x.TimeUtc).Ticks;
|
||||
var dataEnds = last < Math.Max(At(from).Ticks, At(to).Ticks) && data.Single(x => x.TimeUtc.Ticks == last).IsUsable;
|
||||
var endsInWindow = last < Math.Max(At(from).Ticks, At(to).Ticks);
|
||||
var dataEnds = endsInWindow && data.Single(x => x.TimeUtc.Ticks == last).IsUsable;
|
||||
var outageEnds = endsInWindow && !dataEnds;
|
||||
var boolean = data.Any(x => x.Value.Kind == HistorianValueKind.Boolean);
|
||||
var percent = agg is HistoryAggregateType.PercentGood or HistoryAggregateType.PercentBad;
|
||||
for (long i = 0; i < plan.BucketCount; i++)
|
||||
{
|
||||
var k = (int)(plan.Reversed ? plan.BucketCount - 1 - i : i);
|
||||
@@ -170,6 +226,36 @@ public sealed class SdkAggregateParityTests
|
||||
if (dataEnds && (twoVariant || agg is HistoryAggregateType.PercentGood or HistoryAggregateType.PercentBad)
|
||||
&& last >= plan.LowerTicks(i) && last < plan.UpperTicks(i) && sdk[k].Value is not null)
|
||||
list[k] = list[k] with { Value = sdk[k].Value };
|
||||
// An outage marker as the last sample: ours runs the outage to the window end (no end-of-data Partial on the
|
||||
// interval holding it; Bad / 100 % Bad after it); the SDK treats the data as ended there.
|
||||
if (outageEnds && (twoVariant || percent))
|
||||
{
|
||||
if (last >= plan.LowerTicks(i) && last < plan.UpperTicks(i))
|
||||
list[k] = list[k] with
|
||||
{
|
||||
StatusCode = (list[k].StatusCode & ~HistorianInfoBits.HistorianPartial)
|
||||
| (sdk[k].StatusCode & HistorianInfoBits.HistorianPartial),
|
||||
};
|
||||
else if (plan.LowerTicks(i) > last && (sdk[k].StatusCode & StatusBits.CodeBitsMask) == HistorianStatusCodes.BadNoData)
|
||||
list[k] = list[k] with { Value = null, StatusCode = HistorianStatusCodes.BadNoData };
|
||||
}
|
||||
// ...and TimeAverage/Total hold the last usable value up to that marker: Part 13 calls that extrapolation (no
|
||||
// Good value after it) ⇒ Uncertain; the SDK, which sees a later (Bad) sample, reports the held span Good.
|
||||
var lastUsable = data.Where(x => x.IsUsable).Max(x => x.TimeUtc).Ticks;
|
||||
if (outageEnds && agg is HistoryAggregateType.TimeAverage or HistoryAggregateType.Total
|
||||
&& plan.LowerTicks(i) < last && plan.UpperTicks(i) > lastUsable
|
||||
&& (list[k].StatusCode & StatusBits.CodeBitsMask) == HistorianStatusCodes.UncertainDataSubNormal
|
||||
&& (sdk[k].StatusCode & StatusBits.CodeBitsMask) == HistorianStatusCodes.Good)
|
||||
list[k] = list[k] with { StatusCode = list[k].StatusCode & ~StatusBits.CodeBitsMask };
|
||||
// Booleans: the SDK slopes them (values, and the status of a bound next to an Uncertain sample); ours steps.
|
||||
if (boolean && sdk[k].Value is not null && list[k].Value is not null)
|
||||
list[k] = list[k] with { Value = sdk[k].Value };
|
||||
if (boolean && (StatusBits.IsGood(list[k].StatusCode) || StatusBits.IsGood(sdk[k].StatusCode))
|
||||
&& !StatusBits.IsBad(list[k].StatusCode) && !StatusBits.IsBad(sdk[k].StatusCode))
|
||||
list[k] = list[k] with
|
||||
{
|
||||
StatusCode = (list[k].StatusCode & ~StatusBits.CodeBitsMask) | (sdk[k].StatusCode & StatusBits.CodeBitsMask),
|
||||
};
|
||||
}
|
||||
return list;
|
||||
}
|
||||
@@ -188,6 +274,7 @@ public sealed class SdkAggregateParityTests
|
||||
foreach (var (name, data) in new[]
|
||||
{
|
||||
("Bounded", Bounded), ("Dense", Dense), ("EndsInside", EndsInside), ("OutageInside", OutageInside),
|
||||
("RecoveryEnds", RecoveryEnds), ("OutageLast", OutageLast), ("BoolSwitch", BoolSwitch),
|
||||
})
|
||||
{
|
||||
try
|
||||
|
||||
+78
@@ -180,6 +180,84 @@ public sealed class TimeWeightedAggregatorTests
|
||||
((double)Run(HistoryAggregateType.TimeAverage, 0, 20, 0, late).Single().Value!).ShouldBe(10d);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_recovery_sample_after_an_outage_that_ends_the_data_is_its_own_value_Uncertain_and_Partial()
|
||||
{
|
||||
// #680 review (the comms-loss recovery shape): −10 = 10 G, +5 outage, +25 = 20 G (the recovery), nothing after.
|
||||
// [0,20): 10 held into the outage over [0,5) ⇒ TimeAverage2 10 (Total2 50), Uncertain. [20,40) holds the end of the
|
||||
// data but no non-Bad time before it ⇒ the sample's value 20 (Total2 0), Uncertain + Partial — never Bad/null (the
|
||||
// SDK answers the same). [40,60) is after the end of the data ⇒ BadNoData.
|
||||
StoredSample[] s =
|
||||
[
|
||||
S(-10, HistorianValue.FromDouble(10), Good), S(5, HistorianValue.Empty, BadNoCommunication),
|
||||
S(25, HistorianValue.FromDouble(20), Good),
|
||||
];
|
||||
var ta2 = Run(HistoryAggregateType.TimeAverage2, 0, 60, 20, s);
|
||||
Values(ta2, 10, 20, null);
|
||||
ta2.Select(x => x.StatusCode).ShouldBe([UncertainCalc, UncertainCalcPartial, NoDataCalc]);
|
||||
var t2 = Run(HistoryAggregateType.Total2, 0, 60, 20, s);
|
||||
Values(t2, 50, 0, null);
|
||||
t2.Select(x => x.StatusCode).ShouldBe([UncertainCalc, UncertainCalcPartial, NoDataCalc]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_single_sample_is_its_own_value_Uncertain_and_Partial()
|
||||
{
|
||||
// #680 review: the tag's only sample, +25 = 20. One interval [0,60) ⇒ 20 (Total2 0), Uncertain + Partial (time
|
||||
// before the data is Bad). At 20 s: [0,20) before the data ⇒ BadNoData; [20,40) the sample ⇒ 20, Uncertain +
|
||||
// Partial; [40,60) after the end of the data ⇒ BadNoData.
|
||||
StoredSample[] one = [S(25, HistorianValue.FromDouble(20), Good)];
|
||||
var whole = Run(HistoryAggregateType.TimeAverage2, 0, 60, 0, one).Single();
|
||||
((double)whole.Value!).ShouldBe(20d);
|
||||
whole.StatusCode.ShouldBe(UncertainCalcPartial);
|
||||
var total = Run(HistoryAggregateType.Total2, 0, 60, 0, one).Single();
|
||||
((double)total.Value!).ShouldBe(0d);
|
||||
total.StatusCode.ShouldBe(UncertainCalcPartial);
|
||||
var split = Run(HistoryAggregateType.TimeAverage2, 0, 60, 20, one);
|
||||
Values(split, null, 20, null);
|
||||
split.Select(x => x.StatusCode).ShouldBe([NoDataCalc, UncertainCalcPartial, NoDataCalc]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void An_outage_marker_as_the_last_sample_runs_Bad_to_the_window_end()
|
||||
{
|
||||
// #680 review, a deliberate difference from the SDK (which treats the data as ended at the marker): −10 = 10 G,
|
||||
// +25 = 20 G, +30 outage, nothing after. TimeAverage2 [20,40): [20,25) 18.5714…→20 (96.43) + [25,30) 20 held into
|
||||
// the outage (100) over 10 s = 19.642857…, Uncertain, NOT Partial (the data did not end — the tag is in an
|
||||
// outage); [40,60) inside the still-open outage ⇒ generic Bad. PercentBad [40,60) = 100, Good. TimeAverage holds
|
||||
// 20 to the window end — extrapolated (no Good value follows) ⇒ Uncertain.
|
||||
StoredSample[] s =
|
||||
[
|
||||
S(-10, HistorianValue.FromDouble(10), Good), S(25, HistorianValue.FromDouble(20), Good),
|
||||
S(30, HistorianValue.Empty, BadNoCommunication),
|
||||
];
|
||||
var ta2 = Run(HistoryAggregateType.TimeAverage2, 0, 60, 20, s);
|
||||
((double)ta2[1].Value!).ShouldBe(19.642857142857142, 1e-9);
|
||||
ta2.Select(x => x.StatusCode).Skip(1).ShouldBe([UncertainCalc, BadCalc]);
|
||||
var pb = Run(HistoryAggregateType.PercentBad, 0, 60, 20, s);
|
||||
Values(pb, 0, 50, 100);
|
||||
pb.ShouldAllBe(x => x.StatusCode == GoodCalc);
|
||||
Run(HistoryAggregateType.TimeAverage, 0, 60, 20, s).Select(x => x.StatusCode).Skip(1)
|
||||
.ShouldBe([UncertainCalc, UncertainCalc]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Boolean_percent_regions_take_the_stepped_bound_status()
|
||||
{
|
||||
// #680 review: −10 = true G, +15 = true U, +30 = false G, +50 = false G. [0,20): the bound at +0 is stepped from the
|
||||
// Good −10 (a Boolean is held, not sloped toward the Uncertain +15) ⇒ [0,15) Good, [15,20) Uncertain ⇒ PercentGood
|
||||
// 75. (The SDK slopes Booleans, so it calls the bound Uncertain and answers 0.) [20,40): [20,30) Uncertain (bound
|
||||
// stepped from the Uncertain +15), [30,40) Good ⇒ 50.
|
||||
StoredSample[] s =
|
||||
[
|
||||
S(-10, HistorianValue.FromBoolean(true), Good), S(15, HistorianValue.FromBoolean(true), Uncertain),
|
||||
S(30, HistorianValue.FromBoolean(false), Good), S(50, HistorianValue.FromBoolean(false), Good),
|
||||
];
|
||||
Values(Run(HistoryAggregateType.PercentGood, 0, 40, 20, s), 75, 50);
|
||||
// TimeAverage over the same Booleans is stepped: [20,40) = 1 over [20,30), 0 over [30,40) ⇒ 0.5 (the SDK: 1).
|
||||
Values(Run(HistoryAggregateType.TimeAverage, 0, 40, 20, s), 1, 0.5);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- #680: PercentGood / PercentBad are duration-based
|
||||
|
||||
[Fact]
|
||||
|
||||
Reference in New Issue
Block a user