Merge feat/677-aggregates: reverse processed reads SDK-anchored, start-of-data Partial (forward), AggregateConfiguration accept-defaults, IntegralDivisor=1, SDK parity tests (#677)

This commit is contained in:
Joseph Doherty
2026-10-11 11:47:52 -04:00
24 changed files with 1291 additions and 135 deletions
@@ -1,6 +1,7 @@
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common;
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Conformance;
@@ -171,7 +172,8 @@ public abstract class ProcessedConformance : ConformanceBase
// The time-weighted path over the whole range: ≈ the extrapolated 50 that dominates eight millennia.
var tw = (await src.ReadProcessedAsync("A", start, end, TimeSpan.Zero, HistoryAggregateType.TimeAverage2, Ct)).Samples.Single();
((double)tw.Value!).ShouldBe(50d, 1e-6);
tw.StatusCode.ShouldBe(UncertainCalc);
// The data starts (+0) strictly inside the one interval ⇒ Partial as well (#677, the SDK's m_startOfData rule).
tw.StatusCode.ShouldBe(UncertainCalc | 0x4u);
src.GetHealthSnapshot().TotalFailures.ShouldBe(0);
}
@@ -196,12 +198,165 @@ public abstract class ProcessedConformance : ConformanceBase
}
[Fact]
public async Task A_reverse_window_is_computed_forward_and_returned_descending()
public async Task A_reverse_window_is_anchored_at_its_later_time()
{
// #677 (Part 13 / SDK): At(80) → At(0) is the window (0,80] in intervals (60,80], (40,60], (20,40], (0,20], each
// stamped with its LATER edge, latest first. A sample on an edge belongs to the interval it ends: +40 is in
// (20,40], +20 (Bad) in (0,20], and +0 is outside the window. (The pre-#677 layout answered [60,80) … [0,20)
// stamped 60/40/20/0 with values null, 50, 40, 15.)
await ConformanceData.SeedAAsync(B);
var s = (await B.CreateDataSource().ReadProcessedAsync("A", At(80), At(0), Twenty, HistoryAggregateType.Average, Ct)).Samples;
s.Select(x => x.SourceTimestampUtc).ShouldBe([B.Round(At(60)), B.Round(At(40)), B.Round(At(20)), B.Round(At(0))]);
s.Select(x => x.Value).ShouldBe([null, 50d, 40d, 15d]);
s.Select(x => x.SourceTimestampUtc).ShouldBe([B.Round(At(80)), B.Round(At(60)), B.Round(At(40)), B.Round(At(20))]);
s.Select(x => x.Value).ShouldBe([null, null, 45d, 20d]);
s.Select(x => x.StatusCode).ShouldBe([NoDataCalc, NoDataCalc, GoodCalc, UncertainCalc]);
}
[Fact]
public async Task A_reverse_window_truncates_its_earliest_interval()
{
// At(50) → At(0), 20 s over tag A (+0=10, +10=20, +20=Bad, +30=40, +40=50): (30,50] = {+40} → 50 (+30 is its
// earlier edge, excluded); (10,30] = {+20 Bad, +30} → 40, Uncertain; (0,10] = {+10} → 20 — the truncated interval
// is the EARLIEST one. (Pre-#677: [0,20), [20,40), [40,50) stamped 40/20/0 with values 50, 40, 15.)
await ConformanceData.SeedAAsync(B);
var s = (await B.CreateDataSource().ReadProcessedAsync("A", At(50), At(0), Twenty, HistoryAggregateType.Average, Ct)).Samples;
s.Select(x => x.SourceTimestampUtc).ShouldBe([B.Round(At(50)), B.Round(At(30)), B.Round(At(10))]);
s.Select(x => x.Value).ShouldBe([50d, 40d, 20d]);
s.Select(x => x.StatusCode).ShouldBe([GoodCalc, UncertainCalc, GoodCalc]);
}
[Fact]
public async Task Reverse_bucket_edges_are_later_edge_inclusive_to_the_tick()
{
// The mirror of the forward edge test: At(40) → At(0), 20 s ⇒ (20,40], (0,20]. The bucket math is pushed down,
// so this pins the SQL, not just the C# plan.
var r = B.ResolutionTicks;
await B.SeedAsync("ER",
(At(0), HistorianValue.FromDouble(1000), Good), // == earlier edge: excluded
(At(0).AddTicks(r), HistorianValue.FromDouble(1), Good), // first tick of (0,20]
(At(20), HistorianValue.FromDouble(2), Good), // == edge: the LATER edge of (0,20]
(At(20).AddTicks(r), HistorianValue.FromDouble(4), Good), // first tick of (20,40]
(At(40), HistorianValue.FromDouble(8), Good), // == request start: included, in (20,40]
(At(40).AddTicks(r), HistorianValue.FromDouble(2000), Good)); // after the window: excluded
var s = (await B.CreateDataSource().ReadProcessedAsync("ER", At(40), At(0), Twenty, HistoryAggregateType.Average, Ct)).Samples;
s.Select(x => x.SourceTimestampUtc).ShouldBe([B.Round(At(40)), B.Round(At(20))]);
s.Select(x => x.Value).ShouldBe([6d, 1.5d]);
}
[Fact]
public async Task The_interval_the_data_starts_inside_carries_the_Partial_bit()
{
// #677 (SDK m_startOfData): the tag's first sample is +5, inside [0,20) ⇒ Partial there for every aggregate but
// Average; [20,40) is fully covered. With an earlier sample stored (even outside the window) nothing is Partial.
await B.SeedAsync("LS",
(At(5), HistorianValue.FromDouble(10), Good),
(At(15), HistorianValue.FromDouble(30), Good),
(At(25), HistorianValue.FromDouble(20), Good),
(At(60), HistorianValue.FromDouble(20), Good));
var src = B.CreateDataSource();
foreach (var agg in new[] { HistoryAggregateType.Count, HistoryAggregateType.Minimum, HistoryAggregateType.TimeAverage2 })
{
var s = (await src.ReadProcessedAsync("LS", At(0), At(40), Twenty, agg, Ct)).Samples;
(s[0].StatusCode & 0x4u).ShouldBe(0x4u, agg.ToString());
(s[1].StatusCode & 0x4u).ShouldBe(0u, agg.ToString());
}
(await src.ReadProcessedAsync("LS", At(0), At(40), Twenty, HistoryAggregateType.Average, Ct)).Samples
.ShouldAllBe(x => (x.StatusCode & 0x4u) == 0);
await B.SeedAsync("LS2",
(At(-100), HistorianValue.FromDouble(10), Good),
(At(5), HistorianValue.FromDouble(10), Good),
(At(15), HistorianValue.FromDouble(30), Good),
(At(25), HistorianValue.FromDouble(20), Good));
(await src.ReadProcessedAsync("LS2", At(0), At(40), Twenty, HistoryAggregateType.Count, Ct)).Samples
.ShouldAllBe(x => (x.StatusCode & 0x4u) == 0);
}
[Fact]
public async Task A_reverse_read_sets_no_start_of_data_Partial_and_matches_the_SDK()
{
// Review F1: the SDK applies the start-of-data rule to FORWARD reads only. Reverse 40 → 0 at 10 s over data that
// starts at +7: (0,10] holds the data's start but is not Partial — checked through the real SQL and the SDK oracle.
var rows = Enumerable.Range(0, 20)
.Select(i => (At(7 + i * 5), HistorianValue.FromDouble(i % 2 == 0 ? 20 : 40), Good)).ToArray();
await B.SeedAsync("RLS", rows);
var sdkRows = rows.Select(r => (B.Round(r.Item1), r.Item2, r.Item3)).ToArray();
var src = B.CreateDataSource();
var ten = TimeSpan.FromSeconds(10);
foreach (var agg in new[]
{
HistoryAggregateType.Count, HistoryAggregateType.Minimum, HistoryAggregateType.Range,
HistoryAggregateType.Delta, HistoryAggregateType.TimeAverage, HistoryAggregateType.TimeAverage2,
HistoryAggregateType.Total2,
})
{
var ours = (await src.ReadProcessedAsync("RLS", At(40), At(0), ten, agg, Ct)).Samples;
ours.ShouldAllBe(x => (x.StatusCode & 0x4u) == 0, agg.ToString());
var tw = AggregateFamilies.FamilyOf(agg) == AggregateFamily.TimeWeighted;
SdkAggregateOracle.ShouldMatch(ours, SdkAggregateOracle.Run(agg, B.Round(At(40)), B.Round(At(0)), ten, sdkRows),
$"{agg} reverse late start", compareCalculatedInterpolatedBits: tw,
// pre-existing, #680: the SDK's Delta is Good with a Bad sample present; ours Uncertain (none here).
compareCodeBits: agg != HistoryAggregateType.Delta);
}
}
/// <summary>Dense: every 2.5 s from −10 to +100, values cycling, one Bad sample at +22.5 (no empty interval).</summary>
private Task SeedDenseAsync() => B.SeedAsync("P", DenseRows());
private static (DateTime T, HistorianValue V, uint Status)[] DenseRows() => Enumerable.Range(0, 45)
.Select(i => (At(-10 + i * 2.5), HistorianValue.FromDouble(10 + (i * 7 % 13)), i == 13 ? Bad : Good))
.ToArray();
[Theory]
[InlineData(0, 80, 20)]
[InlineData(80, 0, 20)]
[InlineData(77, 3, 20)]
[InlineData(3, 77, 20)]
public async Task Processed_reads_match_the_SDK_aggregate_calculators(double from, double to, double intervalS)
{
// #677: the real backend (pushed-down SQL bucket math + the shared evaluators) against the pinned SDK's own Part 13
// calculators over the same stored rows — timestamps, values, status code bits and the Partial bit, forward and
// reverse. See SdkAggregateParityTests (Sql.Common.Tests) for the documented divergences this avoids/masks.
await SeedDenseAsync();
var rows = DenseRows().Select(r => (B.Round(r.T), r.V, r.Status)).ToArray();
var src = B.CreateDataSource();
var interval = TimeSpan.FromSeconds(intervalS);
// The SDK sees the window the backend actually serves (QuestDB rounds the bounds to µs).
var (start, end) = (B.Round(At(from)), B.Round(At(to)));
foreach (var agg in new[]
{
HistoryAggregateType.Average, HistoryAggregateType.Minimum, HistoryAggregateType.Maximum,
HistoryAggregateType.Count, HistoryAggregateType.TimeAverage, HistoryAggregateType.TimeAverage2,
HistoryAggregateType.Total, HistoryAggregateType.Total2, HistoryAggregateType.Interpolative,
HistoryAggregateType.Range, HistoryAggregateType.Start, HistoryAggregateType.End,
HistoryAggregateType.Delta, HistoryAggregateType.MinimumActualTime, HistoryAggregateType.MaximumActualTime,
HistoryAggregateType.PercentGood, HistoryAggregateType.PercentBad,
})
{
var ours = (await src.ReadProcessedAsync("P", At(from), At(to), interval, agg, Ct)).Samples.ToList();
var startOrEnd = agg is HistoryAggregateType.Start or HistoryAggregateType.End;
var tw = AggregateFamilies.FamilyOf(agg) == AggregateFamily.TimeWeighted;
if (tw && agg != HistoryAggregateType.Interpolative)
{
// Documented divergence: ours flags the truncated interval Partial, the SDK does not.
var plan = BucketPlan.Create(start, end, interval);
for (long i = 0; i < plan.BucketCount; i++)
if (plan.IsTruncated(i))
{
var k = (int)(plan.Reversed ? plan.BucketCount - 1 - i : i);
ours[k] = ours[k] with { StatusCode = ours[k].StatusCode & ~0x4u };
}
}
var reverseInterpolative = agg == HistoryAggregateType.Interpolative && from > to;
SdkAggregateOracle.ShouldMatch(ours, SdkAggregateOracle.Run(agg, start, end, interval, rows),
$"{agg} {from}→{to}/{intervalS}s",
compareCalculatedInterpolatedBits: tw && !reverseInterpolative,
// pre-existing, #680: TimeAverage/Total status rule; Start/End raw-sample timestamp + status; Delta status;
// PercentGood/Bad per-sample vs per-duration values (see SdkAggregateParityTests).
compareCodeBits: agg is not (HistoryAggregateType.TimeAverage or HistoryAggregateType.Total
or HistoryAggregateType.Delta) && !reverseInterpolative && !startOrEnd,
compareTimestamps: !startOrEnd,
compareValues: agg is not (HistoryAggregateType.PercentGood or HistoryAggregateType.PercentBad));
}
}
[Fact]
@@ -245,7 +400,8 @@ public abstract class ProcessedConformance : ConformanceBase
(await One(HistoryAggregateType.Count)).StatusCode.ShouldBe(NoDataCalc);
var pb = await One(HistoryAggregateType.PercentBad);
pb.Value.ShouldBe(100d);
pb.StatusCode.ShouldBe(GoodCalc);
// The tag's data starts (+1) inside the interval ⇒ Partial on the non-Bad percentage (#677).
pb.StatusCode.ShouldBe(GoodCalc | 0x4u);
}
[Fact]
@@ -0,0 +1,162 @@
using Opc.Ua;
using Opc.Ua.Server;
using Shouldly;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Conformance;
/// <summary>
/// Drives the pinned OPC UA SDK's own Part 13 aggregate calculators (<c>Opc.Ua.Server</c> — the classes the SDK's
/// <c>AggregateManager</c> registers) in-process over a sample set, so our processed reads can be compared with the
/// reference implementation bucket by bucket (#677). The calculators run under the server's default
/// <see cref="AggregateConfiguration"/> (TreatUncertainAsBad = false, PercentDataBad = PercentDataGood = 100,
/// UseSlopedExtrapolation = false), sloped interpolation, source timestamps.
/// </summary>
/// <remarks>
/// Shared (source-linked) by <c>Sql.Common.Tests</c> and both SQL backends' integration suites.
/// </remarks>
public static class SdkAggregateOracle
{
/// <summary>One processed value: value (double / int / null), full status code, timestamp.</summary>
public readonly record struct Result(object? Value, uint StatusCode, DateTime Timestamp);
/// <summary>The server-default aggregate configuration (what <c>AggregateManager.GetDefaultConfiguration</c> returns).</summary>
public static AggregateConfiguration DefaultConfiguration => new()
{
TreatUncertainAsBad = false,
PercentDataBad = 100,
PercentDataGood = 100,
UseSlopedExtrapolation = false,
UseServerCapabilitiesDefaults = false,
};
/// <summary>Runs the SDK calculator for <paramref name="aggregate"/> over <paramref name="samples"/>.</summary>
/// <param name="aggregate">The aggregate.</param>
/// <param name="start">Request start (later than <paramref name="end"/> for a reverse read).</param>
/// <param name="end">Request end.</param>
/// <param name="interval">Processing interval.</param>
/// <param name="samples">Every stored sample of the tag, any order (sorted here).</param>
/// <returns>The SDK's processed values, in the order it returns them.</returns>
public static IReadOnlyList<Result> Run(HistoryAggregateType aggregate, DateTime start, DateTime end, TimeSpan interval,
IEnumerable<(DateTime Time, HistorianValue Value, uint Status)> samples)
{
var id = AggregateId(aggregate);
var ms = interval.TotalMilliseconds;
var cfg = DefaultConfiguration;
IAggregateCalculator calc = aggregate switch
{
HistoryAggregateType.Average or HistoryAggregateType.TimeAverage or HistoryAggregateType.TimeAverage2
or HistoryAggregateType.Total or HistoryAggregateType.Total2 =>
new AverageAggregateCalculator(id, start, end, ms, false, cfg, null),
HistoryAggregateType.Minimum or HistoryAggregateType.Maximum or HistoryAggregateType.Range
or HistoryAggregateType.MinimumActualTime or HistoryAggregateType.MaximumActualTime =>
new MinMaxAggregateCalculator(id, start, end, ms, false, cfg, null),
HistoryAggregateType.Count => new CountAggregateCalculator(id, start, end, ms, false, cfg, null),
HistoryAggregateType.PercentGood or HistoryAggregateType.PercentBad =>
new StatusAggregateCalculator(id, start, end, ms, false, cfg, null),
HistoryAggregateType.Start or HistoryAggregateType.End or HistoryAggregateType.Delta =>
new StartEndAggregateCalculator(id, start, end, ms, false, cfg, null),
HistoryAggregateType.Interpolative => new AggregateCalculator(id, start, end, ms, false, cfg, null),
_ => throw new ArgumentOutOfRangeException(nameof(aggregate), aggregate, "no SDK calculator mapped"),
};
// The SDK expects values in the direction of the read: ascending forward, descending reverse.
var ordered = samples.OrderBy(s => s.Time).ToList();
if (end < start) ordered.Reverse();
foreach (var s in ordered) calc.QueueRawValue(ToDataValue(s.Time, s.Value, s.Status));
var results = new List<Result>();
for (var guard = 0; guard < 100_000; guard++)
{
var dv = calc.GetProcessedValue(true);
if (dv is null) break;
results.Add(new Result(dv.Value, dv.StatusCode.Code, DateTime.SpecifyKind(dv.SourceTimestamp, DateTimeKind.Utc)));
}
return results;
}
/// <summary>
/// Asserts our snapshots equal the SDK's: same count, same timestamps, same status code bits (severity + sub-code)
/// and Partial bit (and optionally the Calculated / Interpolated bits), same value (doubles to 1e-9; ints exactly;
/// null on a Bad status). A failure lists every differing interval side by side.
/// </summary>
/// <param name="ours">Our processed values.</param>
/// <param name="sdk">The SDK's.</param>
/// <param name="label">Context for failure messages.</param>
/// <param name="compareCalculatedInterpolatedBits">Also compare the Calculated / Interpolated aggregate bits.</param>
/// <param name="compareCodeBits">Compare the severity + sub-code (false only for a documented status divergence).</param>
/// <param name="compareValues">Compare the values (false only for a documented value divergence).</param>
/// <param name="compareTimestamps">Compare the timestamps (false only for a documented timestamp divergence).</param>
/// <param name="comparePartialBit">Compare the Partial bit (false only for a documented divergence).</param>
public static void ShouldMatch(IReadOnlyList<DataValueSnapshot> ours, IReadOnlyList<Result> sdk, string label,
bool compareCalculatedInterpolatedBits = true, bool compareCodeBits = true, bool compareValues = true,
bool compareTimestamps = true, bool comparePartialBit = true)
{
var problems = new List<string>();
if (ours.Count != sdk.Count) problems.Add($"interval count: ours {ours.Count}, sdk {sdk.Count}");
for (var i = 0; i < Math.Min(ours.Count, sdk.Count); i++)
{
var o = ours[i];
var s = sdk[i];
var bitMask = (comparePartialBit ? 0x4u : 0u) | (compareCodeBits ? 0xFFFF0000u : 0u)
| (compareCalculatedInterpolatedBits ? 0x3u : 0u);
var same = (!compareTimestamps || o.SourceTimestampUtc == s.Timestamp)
&& (o.StatusCode & bitMask) == (s.StatusCode & bitMask);
if (!compareValues || (!compareCodeBits && StatusCode.IsBad(s.StatusCode)))
{
// Values not compared (or the SDK answered Bad under a status this comparison does not check).
}
else if (same && !StatusCode.IsBad(s.StatusCode))
{
same = s.Value switch
{
null => o.Value is null,
int n => o.Value is not null && Convert.ToInt32(o.Value) == n,
_ => o.Value is not null && Math.Abs(Convert.ToDouble(o.Value) - Convert.ToDouble(s.Value)) <= 1e-9,
};
}
else if (same && compareCodeBits) same = o.Value is null;
if (!same)
problems.Add($"[{i}] ours {o.SourceTimestampUtc:HH:mm:ss.fffffff} {o.Value ?? "null"} 0x{o.StatusCode:X8}" +
$" | sdk {s.Timestamp:HH:mm:ss.fffffff} {s.Value ?? "null"} 0x{s.StatusCode:X8}");
}
problems.ShouldBeEmpty($"{label} differs from the SDK:\n" + string.Join("\n", problems));
}
private static DataValue ToDataValue(DateTime t, HistorianValue v, uint status) => new()
{
WrappedValue = v.Kind switch
{
HistorianValueKind.Empty => Variant.Null,
HistorianValueKind.Int64 => new Variant(v.Integer),
HistorianValueKind.Boolean => new Variant(v.Number != 0),
HistorianValueKind.String => new Variant(v.Text),
_ => new Variant(v.Number),
},
StatusCode = status,
SourceTimestamp = t,
ServerTimestamp = t,
};
private static NodeId AggregateId(HistoryAggregateType a) => a switch
{
HistoryAggregateType.Average => ObjectIds.AggregateFunction_Average,
HistoryAggregateType.Minimum => ObjectIds.AggregateFunction_Minimum,
HistoryAggregateType.Maximum => ObjectIds.AggregateFunction_Maximum,
HistoryAggregateType.Range => ObjectIds.AggregateFunction_Range,
HistoryAggregateType.MinimumActualTime => ObjectIds.AggregateFunction_MinimumActualTime,
HistoryAggregateType.MaximumActualTime => ObjectIds.AggregateFunction_MaximumActualTime,
HistoryAggregateType.Count => ObjectIds.AggregateFunction_Count,
HistoryAggregateType.PercentGood => ObjectIds.AggregateFunction_PercentGood,
HistoryAggregateType.PercentBad => ObjectIds.AggregateFunction_PercentBad,
HistoryAggregateType.Start => ObjectIds.AggregateFunction_Start,
HistoryAggregateType.End => ObjectIds.AggregateFunction_End,
HistoryAggregateType.Delta => ObjectIds.AggregateFunction_Delta,
HistoryAggregateType.TimeAverage => ObjectIds.AggregateFunction_TimeAverage,
HistoryAggregateType.TimeAverage2 => ObjectIds.AggregateFunction_TimeAverage2,
HistoryAggregateType.Total => ObjectIds.AggregateFunction_Total,
HistoryAggregateType.Total2 => ObjectIds.AggregateFunction_Total2,
HistoryAggregateType.Interpolative => ObjectIds.AggregateFunction_Interpolative,
_ => throw new ArgumentOutOfRangeException(nameof(a), a, "no SDK aggregate id mapped"),
};
}
@@ -78,6 +78,27 @@ public sealed class GatewayTagProvisionerTests
Assert.True(defs[0].StorageRateMs > 0, "provisioned tags must carry a positive storage rate (0 fails EnsureTags).");
}
[Fact]
public async Task Definition_carries_an_integral_divisor_of_one()
{
// #677: AVEVA's Total integral is value·seconds ÷ IntegralDivisor. The proto3 double default is 0, which the
// gateway passes through, so every provisioned tag must say 1 explicitly — the unit the SQL backends' Total uses.
var fake = new FakeHistorianGatewayClient { EnsureTagsResult = new TagOperationResults() };
var p = Provisioner(fake);
await p.EnsureTagsAsync(
new[]
{
new HistorianTagProvisionRequest("Pump1.Temp", DriverDataType.Float32, null, null),
new HistorianTagProvisionRequest("Pump1.Run", DriverDataType.Boolean, null, null),
},
TestContext.Current.CancellationToken);
var defs = fake.LastEnsureDefinitions!;
Assert.Equal(2, defs.Count);
Assert.All(defs, d => Assert.Equal(1.0, d.IntegralDivisor));
}
[Fact]
public async Task Maps_metadata_and_coalesces_null_metadata_to_empty()
{
@@ -82,7 +82,8 @@ public sealed class ReadTests(QuestDbFixture fx) : IAsyncLifetime
foreach (var agg in new[] { HistoryAggregateType.Average, HistoryAggregateType.Start, HistoryAggregateType.MaximumActualTime })
{
var s = (await src.ReadProcessedAsync("A", start, end, TimeSpan.Zero, agg, Ct)).Samples.Single();
s.StatusCode.ShouldBe(UncertainCalc, agg.ToString());
// The data starts (+0) strictly inside the one interval ⇒ Partial on every aggregate but Average (#677).
s.StatusCode.ShouldBe(agg == HistoryAggregateType.Average ? UncertainCalc : UncertainCalc | 0x4u, agg.ToString());
}
(await src.ReadProcessedAsync("A", start, end, TimeSpan.Zero, HistoryAggregateType.Average, Ct)).Samples.Single().Value.ShouldBe(30d);
(await src.ReadProcessedAsync("A", start, end, TimeSpan.Zero, HistoryAggregateType.Start, Ct)).Samples.Single().Value.ShouldBe(10d);
@@ -14,6 +14,8 @@
<PackageReference Include="xunit.v3"/>
<PackageReference Include="Shouldly"/>
<PackageReference Include="Microsoft.NET.Test.Sdk"/>
<!-- SDK Part 13 aggregate calculators: the conformance suite's processed-read parity oracle (#677). -->
<PackageReference Include="OPCFoundation.NetStandard.Opc.Ua.Server"/>
<PackageReference Include="xunit.runner.visualstudio">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
@@ -37,9 +37,7 @@ public sealed class AggregateEvaluatorTests
private static IReadOnlyList<DataValueSnapshot> ReadA(HistoryAggregateType agg, bool maximum = false)
{
var plan = BucketPlan.Create(At(0), At(80), TimeSpan.FromSeconds(20));
var evaluated = RowsA(maximum).ToDictionary(kv => kv.Key,
kv => AggregateEvaluator.Evaluate(agg, kv.Value, plan.BucketStart(kv.Key), Now));
return plan.Assemble(evaluated, t => AggregateEvaluator.NoData(t, Now));
return AggregateEvaluator.EvaluateAll(agg, plan, RowsA(maximum).Values, At(0), Now);
}
[Theory]
@@ -28,8 +28,10 @@ public sealed class BucketPlanTests
{
var p = BucketPlan.Create(At(0), At(50), TimeSpan.FromSeconds(20));
p.BucketCount.ShouldBe(3);
p.BucketStart(2).ShouldBe(At(40));
p.BucketStart(2).Kind.ShouldBe(DateTimeKind.Utc);
p.BucketTimestamp(2).ShouldBe(At(40));
p.BucketTimestamp(2).Kind.ShouldBe(DateTimeKind.Utc);
p.IsTruncated(2).ShouldBeTrue();
p.IsTruncated(1).ShouldBeFalse();
}
[Fact]
@@ -43,14 +45,104 @@ public sealed class BucketPlanTests
}
[Fact]
public void A_reverse_window_is_planned_forward_and_assembled_descending()
public void A_reverse_window_is_anchored_at_its_later_time_and_stamped_with_later_edges()
{
// #677 (Part 13 / SDK CreateSlice + GetTimestamp): At(80) → At(0), 20 s ⇒ (60,80], (40,60], (20,40], (0,20],
// each stamped with its later edge, latest first.
var p = BucketPlan.Create(At(80), At(0), TimeSpan.FromSeconds(20));
p.Reversed.ShouldBeTrue();
p.StartUtc.ShouldBe(At(0));
p.EndUtc.ShouldBe(At(80));
var s = p.Assemble(new Dictionary<long, DataValueSnapshot>(), t => new DataValueSnapshot(null, 0, t, t));
s.Select(x => x.SourceTimestampUtc).ShouldBe([At(60), At(40), At(20), At(0)]);
s.Select(x => x.SourceTimestampUtc).ShouldBe([At(80), At(60), At(40), At(20)]);
}
[Fact]
public void A_reverse_window_puts_the_partial_interval_at_its_earliest_end()
{
// At(50) → At(0), 20 s: (30,50], (10,30], (0,10] — the truncated bucket is the EARLIEST (the old layout truncated
// the latest: [0,20), [20,40), [40,50)).
var p = BucketPlan.Create(At(50), At(0), TimeSpan.FromSeconds(20));
p.BucketCount.ShouldBe(3);
p.LowerTicks(0).ShouldBe(At(0).Ticks);
p.UpperTicks(0).ShouldBe(At(10).Ticks);
p.IsTruncated(0).ShouldBeTrue();
p.LowerTicks(2).ShouldBe(At(30).Ticks);
p.UpperTicks(2).ShouldBe(At(50).Ticks);
p.IsTruncated(2).ShouldBeFalse();
var s = p.Assemble(new Dictionary<long, DataValueSnapshot>(), t => new DataValueSnapshot(null, 0, t, t));
s.Select(x => x.SourceTimestampUtc).ShouldBe([At(50), At(30), At(10)]);
}
[Fact]
public void Forward_buckets_are_start_inclusive_and_reverse_buckets_later_edge_inclusive()
{
var fwd = BucketPlan.Create(At(0), At(50), TimeSpan.FromSeconds(20));
fwd.IndexOf(At(0)).ShouldBe(0);
fwd.IndexOf(At(20)).ShouldBe(1);
fwd.IndexOf(At(20).AddTicks(-1)).ShouldBe(0);
fwd.IndexOf(At(49.9999999)).ShouldBe(2);
fwd.IndexOf(At(50)).ShouldBeNull();
fwd.IndexOf(At(0).AddTicks(-1)).ShouldBeNull();
// (0,10] (10,30] (30,50]: the later time is in the window, the earlier one is not (as a reverse raw read).
var rev = BucketPlan.Create(At(50), At(0), TimeSpan.FromSeconds(20));
rev.IndexOf(At(50)).ShouldBe(2);
rev.IndexOf(At(30)).ShouldBe(1);
rev.IndexOf(At(30).AddTicks(1)).ShouldBe(2);
rev.IndexOf(At(10)).ShouldBe(0);
rev.IndexOf(At(10).AddTicks(1)).ShouldBe(1);
rev.IndexOf(At(0).AddTicks(1)).ShouldBe(0);
rev.IndexOf(At(0)).ShouldBeNull();
rev.IndexOf(At(50).AddTicks(1)).ShouldBeNull();
}
[Fact]
public void The_backend_bucket_formula_matches_IndexOf_in_both_directions()
{
// b = (t − start + adjust) / interval and b = (unixTicks(t) − base) / interval — what the SQL computes.
foreach (var p in new[]
{
BucketPlan.Create(At(0), At(50), TimeSpan.FromSeconds(20)),
BucketPlan.Create(At(50), At(0), TimeSpan.FromSeconds(20)),
BucketPlan.Create(At(60), At(0), TimeSpan.FromSeconds(20)),
BucketPlan.Create(At(0).AddTicks(70), At(0), TimeSpan.FromTicks(3)),
})
{
for (var t = p.StartUtc.Ticks - 2; t <= p.EndUtc.Ticks + 2; t += Math.Max(1, (p.EndUtc.Ticks - p.StartUtc.Ticks) / 997))
{
var at = new DateTime(t, DateTimeKind.Utc);
if (p.IndexOf(at) is not { } idx) continue;
((t - p.StartUtc.Ticks + p.BucketIndexAdjustTicks) / p.IntervalTicks).ShouldBe(idx);
(((at - DateTime.UnixEpoch).Ticks - p.BucketBaseUnixTicks) / p.IntervalTicks).ShouldBe(idx);
p.LowerTicks(idx).ShouldBeLessThanOrEqualTo(t);
p.UpperTicks(idx).ShouldBeGreaterThanOrEqualTo(t);
}
}
}
[Fact]
public void Data_starts_inside_only_strictly_between_the_bucket_edges()
{
var p = BucketPlan.Create(At(0), At(40), TimeSpan.FromSeconds(20));
p.DataStartsInside(0, At(5)).ShouldBeTrue();
p.DataStartsInside(1, At(5)).ShouldBeFalse();
p.DataStartsInside(1, At(20)).ShouldBeFalse(); // on the edge: data covers the whole interval
p.DataStartsInside(0, null).ShouldBeFalse();
p.DataStartsInside(0, At(-5)).ShouldBeFalse();
}
[Fact]
public void The_earliest_sample_is_needed_only_for_forward_reads_of_aggregates_other_than_Average()
{
// Review F2: backends skip the earliest-sample query when its answer cannot change a result.
var fwd = BucketPlan.Create(At(0), At(40), TimeSpan.FromSeconds(10));
var rev = BucketPlan.Create(At(40), At(0), TimeSpan.FromSeconds(10));
AggregateEvaluator.NeedsEarliestSample(HistoryAggregateType.Count, fwd).ShouldBeTrue();
AggregateEvaluator.NeedsEarliestSample(HistoryAggregateType.Average, fwd).ShouldBeFalse();
AggregateEvaluator.NeedsEarliestSample(HistoryAggregateType.Count, rev).ShouldBeFalse();
AggregateEvaluator.NeedsEarliestSample(HistoryAggregateType.Count, BucketPlan.Create(At(0), At(0), TimeSpan.Zero))
.ShouldBeFalse();
}
[Fact]
@@ -64,6 +156,11 @@ public sealed class BucketPlanTests
s[2].SourceTimestampUtc.ShouldBe(At(40));
}
[Fact]
public void BucketBaseUnixTicks_is_the_start_forward()
=> BucketPlan.Create(DateTime.UnixEpoch.AddTicks(1234567), DateTime.UnixEpoch.AddSeconds(10), TimeSpan.FromSeconds(1))
.BucketBaseUnixTicks.ShouldBe(1234567L);
[Fact]
public void StartUnixTicks_is_ticks_since_the_unix_epoch()
=> BucketPlan.Create(DateTime.UnixEpoch.AddTicks(1234567), DateTime.UnixEpoch.AddSeconds(10), TimeSpan.Zero)
@@ -0,0 +1,259 @@
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Driver.Historian.Conformance;
using static ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common.Tests.Fixture;
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common.Tests;
/// <summary>
/// #677: our processed reads against the pinned SDK's own Part 13 aggregate calculators (<see cref="SdkAggregateOracle"/>),
/// forward AND reverse, bucket by bucket — timestamps, values and status. The per-sample aggregates are bucketed with
/// <see cref="BucketPlan.IndexOf"/> (the C# statement of the backends' SQL bucket math; the integration suites run the
/// same comparison through the real SQL) and evaluated by <see cref="AggregateEvaluator.EvaluateAll"/>; the
/// time-weighted ones by <see cref="TimeWeightedAggregator"/>.
/// </summary>
/// <remarks>
/// Deliberate, documented divergences the datasets avoid (they are not anchoring questions):
/// <list type="bullet">
/// <item>an <b>empty</b> interval inside the data range: the SDK's Count answers 0/Good, ours <c>BadNoData</c>;</item>
/// <item><b>Uncertain</b> samples in a per-sample aggregate: the SDK's Average/Count/Min/Max use Good samples only,
/// ours every usable (Good + Uncertain) sample (TreatUncertainAsBad = false applied consistently);</item>
/// <item>the per-sample <b>Calculated / MultipleValues</b> info bits (the SDK's MinMax sets Calculated only when the
/// extreme is not at the interval start, and flags ties) — those comparisons check code bits + Partial only;</item>
/// <item>a <b>truncated</b> interval: ours sets Partial on a non-Bad time-weighted result (the SDK does not), so the
/// non-multiple windows mask that one bucket's Partial bit;</item>
/// <item>the SDK's <b>end-of-data</b> Partial rule (forward, the interval holding the last sample when nothing follows
/// and nothing was extrapolated) is not implemented: every dataset carries data past the window end.</item>
/// </list>
/// Pre-existing forward divergences this parity test surfaced (#671 behaviour, unchanged here — forward results were
/// to stay as they are; recorded for a follow-up rather than fixed in place):
/// <list type="bullet">
/// <item><b>TimeAverage / Total status:</b> the SDK (interpolated bounds) reports <c>UncertainDataSubNormal</c> when
/// <i>any</i> region of the interval is non-Good (e.g. a segment bridged across a Bad sample); ours applies the
/// time-based rule (Uncertain time counts as Good) to these two as well, so a bridged-but-fully-covered interval
/// is Good. Values and timestamps agree; the status code bits are not compared for these two.</item>
/// <item><b>Total over an interval the data starts inside:</b> the SDK integrates only the defined part
/// (value·s over the covered seconds); ours is <c>TimeAverage × interval seconds</c>. The Partial bit and timestamp
/// agree; that value is not compared.</item>
/// <item><b>Interpolative, reverse, a sample exactly at the point:</b> the SDK's backward <c>UpdateSlice</c> files
/// a sample AT the point as the early bound (<c>&gt;= 0</c>), so it interpolates from it to the next Good sample —
/// same value, but the Interpolated bit, and Uncertain when a Bad sample sits between. Ours returns the raw
/// sample with its own status in both directions (as the SDK itself does forward). Reverse Interpolative
/// therefore compares timestamps and values (the anchoring question) but not status.</item>
/// </list>
/// </remarks>
public sealed class SdkAggregateParityTests
{
private static readonly DateTime Now = At(3600);
/// <summary>Dense: every 2.5 s from −10 to +100, values cycling, one Bad sample at +22.5 — no empty interval.</summary>
private static readonly StoredSample[] Dense = Enumerable.Range(0, 45)
.Select(i => (t: -10 + i * 2.5, i))
.Select(x => S(x.t, HistorianValue.FromDouble(10 + (x.i * 7 % 13)), x.t == 22.5 ? Bad : Good))
.ToArray();
/// <summary>Tag A with Good bounds on both sides, plus an Uncertain sample and an outage marker further on.</summary>
private static readonly StoredSample[] Bounded =
[
S(-10, HistorianValue.FromDouble(5), Good),
.. A,
S(55, HistorianValue.FromDouble(70), Uncertain),
S(62, HistorianValue.Empty, BadNoCommunication),
S(71, HistorianValue.FromDouble(30), Good),
S(100, HistorianValue.FromDouble(60), Good),
];
/// <summary>Data that starts inside the window (no sample before +7) and runs past its end.</summary>
private static readonly StoredSample[] LateStart = Enumerable.Range(0, 20)
.Select(i => S(7 + i * 5, HistorianValue.FromDouble(i % 2 == 0 ? 20 : 40), Good))
.ToArray();
public static TheoryData<double, double, double> Windows => new()
{
{ 0, 80, 20 }, { 80, 0, 20 }, // whole intervals (samples sit on the edges: the inclusivity question)
{ 0, 50, 20 }, { 50, 0, 20 }, // a truncated interval: latest forward, EARLIEST reverse
{ 5, 80, 15 }, { 80, 5, 15 }, // a misaligned window
{ 3, 77, 20 }, { 77, 3, 20 }, // misaligned and truncated
{ 0, 60, 0 }, { 60, 0, 0 }, // one interval
};
private static IReadOnlyList<DataValueSnapshot> Ours(HistoryAggregateType agg, double from, double to, double intervalS,
StoredSample[] data)
{
var plan = BucketPlan.Create(At(from), At(to), TimeSpan.FromSeconds(intervalS));
if (AggregateFamilies.FamilyOf(agg) == AggregateFamily.TimeWeighted)
return TimeWeightedAggregator.Evaluate(agg, plan, data, Now);
return AggregateEvaluator.EvaluateAll(agg, plan, InMemoryRows(plan, data, agg == HistoryAggregateType.MaximumActualTime),
data.Min(s => s.TimeUtc), Now);
}
private static IReadOnlyList<SdkAggregateOracle.Result> Sdk(HistoryAggregateType agg, double from, double to,
double intervalS, StoredSample[] data)
{
var interval = intervalS > 0 ? TimeSpan.FromSeconds(intervalS) : (At(to) - At(from)).Duration();
return SdkAggregateOracle.Run(agg, At(from), At(to), interval, data.Select(s => (s.TimeUtc, s.Value, s.Status)));
}
/// <summary>Masks the Partial bit of our one truncated interval (a documented divergence: ours flags it, the SDK does not).</summary>
private static IReadOnlyList<DataValueSnapshot> MaskTruncation(IReadOnlyList<DataValueSnapshot> ours, double from,
double to, double intervalS)
{
var plan = BucketPlan.Create(At(from), At(to), TimeSpan.FromSeconds(intervalS));
var list = ours.ToList();
for (long i = 0; i < plan.BucketCount; i++)
{
if (!plan.IsTruncated(i)) continue;
var k = (int)(plan.Reversed ? plan.BucketCount - 1 - i : i);
list[k] = list[k] with { StatusCode = list[k].StatusCode & ~HistorianInfoBits.HistorianPartial };
}
return list;
}
[Theory]
[MemberData(nameof(Windows))]
public void Time_weighted_aggregates_match_the_SDK(double from, double to, double intervalS)
{
foreach (var agg in new[]
{
HistoryAggregateType.TimeAverage, HistoryAggregateType.TimeAverage2, HistoryAggregateType.Total,
HistoryAggregateType.Total2, HistoryAggregateType.Interpolative,
})
foreach (var (name, data) in new[] { ("Bounded", Bounded), ("Dense", Dense) })
SdkAggregateOracle.ShouldMatch(MaskTruncation(Ours(agg, from, to, intervalS, data), from, to, intervalS),
Sdk(agg, from, to, intervalS, data), $"{agg} {name} {from}→{to}/{intervalS}s",
compareCalculatedInterpolatedBits: !(agg == HistoryAggregateType.Interpolative && from > to),
compareCodeBits: agg is not (HistoryAggregateType.TimeAverage or HistoryAggregateType.Total)
&& !(agg == HistoryAggregateType.Interpolative && from > to));
}
[Theory]
[MemberData(nameof(Windows))]
public void Per_sample_aggregates_match_the_SDK(double from, double to, double intervalS)
{
var failures = new List<string>();
foreach (var agg in PerSample)
{
try
{
SdkAggregateOracle.ShouldMatch(Ours(agg, from, to, intervalS, Dense), Sdk(agg, from, to, intervalS, Dense),
$"{agg} Dense {from}→{to}/{intervalS}s", compareCalculatedInterpolatedBits: false,
compareValues: !PercentAggregate(agg),
// pre-existing, #680: the SDK's Start/End return the raw first/last sample (any quality) with its own
// timestamp and stored status; ours return the first/last USABLE value stamped with the interval, under the
// per-sample status rule. The SDK's Delta is Good even when a Bad sample sits in the interval; ours is
// Uncertain. Values are compared wherever the SDK's answer is not Bad.
compareTimestamps: !StartOrEnd(agg),
compareCodeBits: !StartOrEnd(agg) && agg != HistoryAggregateType.Delta);
}
catch (Exception ex)
{
failures.Add(ex.Message);
}
}
failures.ShouldBeEmpty(string.Join("\n", failures));
}
private static readonly HistoryAggregateType[] PerSample =
[
HistoryAggregateType.Average, HistoryAggregateType.Minimum, HistoryAggregateType.Maximum,
HistoryAggregateType.Count, HistoryAggregateType.Range, HistoryAggregateType.Start, HistoryAggregateType.End,
HistoryAggregateType.Delta, HistoryAggregateType.MinimumActualTime, HistoryAggregateType.MaximumActualTime,
HistoryAggregateType.PercentGood, HistoryAggregateType.PercentBad,
];
/// <summary>
/// pre-existing, #680: ours is the per-SAMPLE percentage (100 × Good samples / all samples); the SDK's
/// <c>StatusAggregateCalculator</c> is the per-DURATION one (Good time / interval time). Different definitions, so the
/// values are not compared; timestamp, status and Partial still are.
/// </summary>
private static bool StartOrEnd(HistoryAggregateType agg) =>
agg is HistoryAggregateType.Start or HistoryAggregateType.End;
private static bool PercentAggregate(HistoryAggregateType agg) =>
agg is HistoryAggregateType.PercentGood or HistoryAggregateType.PercentBad;
[Theory]
[InlineData(HistoryAggregateType.TimeAverage)]
[InlineData(HistoryAggregateType.TimeAverage2)]
[InlineData(HistoryAggregateType.Total)]
[InlineData(HistoryAggregateType.Total2)]
[InlineData(HistoryAggregateType.Minimum)]
[InlineData(HistoryAggregateType.Maximum)]
[InlineData(HistoryAggregateType.Count)]
[InlineData(HistoryAggregateType.Range)]
[InlineData(HistoryAggregateType.Delta)]
[InlineData(HistoryAggregateType.Start)]
[InlineData(HistoryAggregateType.Average)]
public void A_reverse_read_over_data_starting_inside_the_window_sets_no_start_of_data_Partial(HistoryAggregateType agg)
{
// Review F1: the SDK applies its start-of-data rule only on a FORWARD read (a reverse read keys m_startOfData on the
// first value it is fed — the latest — which is past the window here). Reverse 40 → 0 at 10 s, first sample +7:
// (0,10] holds the start of the data but is NOT Partial.
var tw = AggregateFamilies.FamilyOf(agg) == AggregateFamily.TimeWeighted;
var ours = Ours(agg, 40, 0, 10, LateStart);
SdkAggregateOracle.ShouldMatch(ours, Sdk(agg, 40, 0, 10, LateStart), $"{agg} reverse LateStart",
compareCalculatedInterpolatedBits: tw,
// pre-existing, #680: Total over the interval the data starts inside (TimeAverage × interval vs the SDK's
// integral of the covered part); Start's raw-sample timestamp/status (see Per_sample_aggregates_match_the_SDK).
compareValues: agg != HistoryAggregateType.Total,
compareTimestamps: !StartOrEnd(agg),
compareCodeBits: !StartOrEnd(agg) && agg != HistoryAggregateType.Delta);
ours.ShouldAllBe(x => (x.StatusCode & HistorianInfoBits.HistorianPartial) == 0);
}
[Theory]
[InlineData(HistoryAggregateType.TimeAverage)]
[InlineData(HistoryAggregateType.TimeAverage2)]
[InlineData(HistoryAggregateType.Total)]
[InlineData(HistoryAggregateType.Total2)]
[InlineData(HistoryAggregateType.Minimum)]
[InlineData(HistoryAggregateType.Maximum)]
[InlineData(HistoryAggregateType.Count)]
[InlineData(HistoryAggregateType.Average)]
public void The_interval_the_data_starts_inside_matches_the_SDK_Partial_bit(HistoryAggregateType agg)
{
// Forward [0,40) at 10 s: the data starts at +7, inside [0,10) — the SDK sets Partial there (m_startOfData) for
// every calculator but plain Average; [10,20) … are fully covered.
var tw = AggregateFamilies.FamilyOf(agg) == AggregateFamily.TimeWeighted;
var ours = Ours(agg, 0, 40, 10, LateStart);
SdkAggregateOracle.ShouldMatch(ours, Sdk(agg, 0, 40, 10, LateStart), $"{agg} LateStart",
compareCalculatedInterpolatedBits: tw, compareValues: agg != HistoryAggregateType.Total);
(ours[0].StatusCode & HistorianInfoBits.HistorianPartial).ShouldBe(
agg == HistoryAggregateType.Average ? 0u : HistorianInfoBits.HistorianPartial);
ours.Skip(1).ShouldAllBe(x => (x.StatusCode & HistorianInfoBits.HistorianPartial) == 0);
}
/// <summary>The per-bucket statistics the backends' SQL computes, built in memory with <see cref="BucketPlan.IndexOf"/>.</summary>
internal static IEnumerable<BucketRow> InMemoryRows(BucketPlan plan, IEnumerable<StoredSample> samples, bool maximum = false)
{
foreach (var g in samples.Select(s => (s, b: plan.IndexOf(s.TimeUtc))).Where(x => x.b is not null)
.GroupBy(x => x.b!.Value).OrderBy(g => g.Key))
{
var all = g.Select(x => x.s).OrderBy(s => s.TimeUtc).ToList();
var usable = all.Where(s => s.IsUsable).ToList();
var numeric = usable.Where(s => s.Value.Kind is HistorianValueKind.Double or HistorianValueKind.Int64
or HistorianValueKind.Boolean).ToList();
// ActualTime: the extreme usable value, ties ⇒ earliest (the SQL's ROW_NUMBER order).
var extreme = numeric.OrderBy(s => maximum ? -s.Value.Number : s.Value.Number).ThenBy(s => s.TimeUtc)
.Cast<StoredSample?>().FirstOrDefault();
yield return new BucketRow
{
Bucket = g.Key,
Total = all.Count,
Usable = usable.Count,
Good = all.Count(s => s.IsGood),
Bad = all.Count - usable.Count,
NonNumeric = usable.Count - numeric.Count,
Avg = numeric.Count == 0 ? null : numeric.Average(s => s.Value.Number),
Min = numeric.Count == 0 ? null : numeric.Min(s => s.Value.Number),
Max = numeric.Count == 0 ? null : numeric.Max(s => s.Value.Number),
Sum = numeric.Count == 0 ? null : numeric.Sum(s => s.Value.Number),
First = usable.Count == 0 ? HistorianValue.Empty : usable[0].Value,
Last = usable.Count == 0 ? HistorianValue.Empty : usable[^1].Value,
Extreme = extreme?.Value.Number,
ExtremeTime = extreme?.TimeUtc,
};
}
}
}
@@ -322,14 +322,46 @@ public sealed class TimeWeightedAggregatorTests
// ---------------------------------------------------------------- plan shape
[Fact]
public void A_reverse_window_is_computed_forward_and_returned_descending()
public void A_reverse_window_is_returned_latest_first_stamped_with_later_edges()
{
// #677: the four whole intervals are the same spans as forward, but each is stamped with its later edge.
var s = TimeWeightedAggregator.Evaluate(HistoryAggregateType.TimeAverage2,
BucketPlan.Create(At(80), At(0), TimeSpan.FromSeconds(20)), A, Now);
s.Select(x => x.SourceTimestampUtc).ShouldBe([At(60), At(40), At(20), At(0)]);
s.Select(x => x.SourceTimestampUtc).ShouldBe([At(80), At(60), At(40), At(20)]);
Values(s, 50, 50, 45, 17.5);
}
[Fact]
public void A_reverse_window_anchors_at_its_later_time_and_truncates_the_earliest_interval()
{
// Tag A, At(50) → At(0), 20 s ⇒ (30,50], (10,30], (0,10]. Raw signal: [0,10) 10→20 Good (150); [10,20) 20 held
// into the outage, Uncertain (200); [20,30) Bad; [30,40) 40→50 Good (450); [40,50) 50 extrapolated (500).
// (30,50]: 950/20 = 47.5 Good; (10,30]: 200/10 = 20, the Bad 10 s ⇒ Uncertain; (0,10]: 150/10 = 15, truncated ⇒
// Partial. The old layout would have answered [0,20), [20,40), [40,50) stamped 0/20/40, partial at the LATEST end.
var s = TimeWeightedAggregator.Evaluate(HistoryAggregateType.TimeAverage2,
BucketPlan.Create(At(50), At(0), TimeSpan.FromSeconds(20)), A, Now);
s.Select(x => x.SourceTimestampUtc).ShouldBe([At(50), At(30), At(10)]);
Values(s, 47.5, 20, 15);
s.Select(x => x.StatusCode).ShouldBe([GoodCalc, UncertainCalc, GoodCalcPartial]);
}
[Fact]
public void The_interval_the_data_starts_inside_is_Partial()
{
// #677 (SDK m_startOfData): no sample before +5 at all ⇒ [0,10) holds the start of the data ⇒ Partial (its first
// 5 s are no-data time ⇒ Uncertain); [10,20) is fully covered ⇒ no Partial.
StoredSample[] late = [S(5, HistorianValue.FromDouble(10), Good), S(30, HistorianValue.FromDouble(10), Good)];
var s = Run(HistoryAggregateType.TimeAverage, 0, 20, 10, late);
Values(s, 10, 10);
s.Select(x => x.StatusCode).ShouldBe([UncertainCalc | 0x4u, GoodCalc]);
// A bound before the window (the data started earlier) ⇒ no Partial.
StoredSample[] early = [S(-5, HistorianValue.FromDouble(10), Good), S(30, HistorianValue.FromDouble(10), Good)];
Run(HistoryAggregateType.TimeAverage, 0, 20, 10, early).ShouldAllBe(x => x.StatusCode == GoodCalc);
// Data starting exactly on an interval edge covers that interval ⇒ no Partial.
StoredSample[] edge = [S(10, HistorianValue.FromDouble(10), Good), S(30, HistorianValue.FromDouble(10), Good)];
Run(HistoryAggregateType.TimeAverage, 0, 20, 10, edge)[1].StatusCode.ShouldBe(GoodCalc);
}
[Fact]
public void A_truncated_last_bucket_uses_its_own_duration_for_Total_and_is_Partial()
{
@@ -15,12 +15,18 @@
<PackageReference Include="Microsoft.NET.Test.Sdk"/>
<!-- SDK oracle for the status-code / aggregate-bit tests. -->
<PackageReference Include="OPCFoundation.NetStandard.Opc.Ua.Core"/>
<!-- SDK Part 13 aggregate calculators: the parity oracle for processed reads (#677). -->
<PackageReference Include="OPCFoundation.NetStandard.Opc.Ua.Server"/>
<PackageReference Include="xunit.runner.visualstudio">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
</ItemGroup>
<ItemGroup>
<Compile Include="..\Historian.Conformance\SdkAggregateOracle.cs" Link="SdkAggregateOracle.cs"/>
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\..\..\src\Drivers\ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common\ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common.csproj"/>
</ItemGroup>
@@ -13,6 +13,8 @@
<PackageReference Include="xunit.v3"/>
<PackageReference Include="Shouldly"/>
<PackageReference Include="Microsoft.NET.Test.Sdk"/>
<!-- SDK Part 13 aggregate calculators: the conformance suite's processed-read parity oracle (#677). -->
<PackageReference Include="OPCFoundation.NetStandard.Opc.Ua.Server"/>
<PackageReference Include="xunit.runner.visualstudio">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
@@ -278,6 +278,94 @@ public sealed class NodeManagerHistoryReadTests : IDisposable
await host.DisposeAsync();
}
/// <summary>
/// #677: a client <c>AggregateConfiguration</c> that differs from the server defaults is refused per node with
/// <c>BadAggregateConfigurationRejected</c> (it used to be silently ignored); the source is never invoked.
/// </summary>
[Theory]
[InlineData(true, (byte)100, (byte)100, false)] // TreatUncertainAsBad
[InlineData(false, (byte)50, (byte)100, false)] // PercentDataBad
[InlineData(false, (byte)100, (byte)80, false)] // PercentDataGood
[InlineData(false, (byte)100, (byte)100, true)] // UseSlopedExtrapolation
[InlineData(false, (byte)0, (byte)0, false)] // an all-zero structure without UseServerCapabilitiesDefaults
public async Task Processed_with_a_non_default_aggregate_configuration_is_rejected(
bool treatUncertainAsBad, byte percentBad, byte percentGood, bool sloped)
{
var (host, server) = await BootAsync();
var nm = server.NodeManager!;
var fake = new RecordingHistorianDataSource { ProcessedResult = Empty() };
nm.HistorianDataSource = fake;
nm.EnsureVariable("eq-1/cfg", parentFolderNodeId: null, displayName: "Cfg", dataType: "Float",
writable: false, historianTagname: "WW.Cfg", realm: AddressSpaceRealm.Uns);
var nodeId = nm.TryGetVariable("eq-1/cfg")!.NodeId;
var (_, errors) = InvokeHistoryRead(server, nm, new ReadProcessedDetails
{
StartTime = DateTime.UtcNow.AddHours(-1), EndTime = DateTime.UtcNow, ProcessingInterval = 10_000.0,
AggregateType = new NodeIdCollection { ObjectIds.AggregateFunction_TimeAverage },
AggregateConfiguration = new AggregateConfiguration
{
UseServerCapabilitiesDefaults = false, TreatUncertainAsBad = treatUncertainAsBad,
PercentDataBad = percentBad, PercentDataGood = percentGood, UseSlopedExtrapolation = sloped,
},
}, nodeId);
errors[0].StatusCode.Code.ShouldBe(StatusCodes.BadAggregateConfigurationRejected);
fake.LastCall.ShouldBeNull();
await host.DisposeAsync();
}
/// <summary>
/// #677: <c>UseServerCapabilitiesDefaults</c> (whatever the other fields say), or a configuration spelling out exactly
/// the server defaults, is served.
/// </summary>
[Theory]
[InlineData(true, true, (byte)0, (byte)0, true)]
[InlineData(false, false, (byte)100, (byte)100, false)]
public async Task Processed_with_the_server_default_aggregate_configuration_is_served(
bool useServerDefaults, bool treatUncertainAsBad, byte percentBad, byte percentGood, bool sloped)
{
var (host, server) = await BootAsync();
var nm = server.NodeManager!;
var fake = new RecordingHistorianDataSource { ProcessedResult = Empty() };
nm.HistorianDataSource = fake;
nm.EnsureVariable("eq-1/cfg-ok", parentFolderNodeId: null, displayName: "CfgOk", dataType: "Float",
writable: false, historianTagname: "WW.CfgOk", realm: AddressSpaceRealm.Uns);
var nodeId = nm.TryGetVariable("eq-1/cfg-ok")!.NodeId;
var (_, errors) = InvokeHistoryRead(server, nm, new ReadProcessedDetails
{
StartTime = DateTime.UtcNow.AddHours(-1), EndTime = DateTime.UtcNow, ProcessingInterval = 10_000.0,
AggregateType = new NodeIdCollection { ObjectIds.AggregateFunction_TimeAverage },
AggregateConfiguration = new AggregateConfiguration
{
UseServerCapabilitiesDefaults = useServerDefaults, TreatUncertainAsBad = treatUncertainAsBad,
PercentDataBad = percentBad, PercentDataGood = percentGood, UseSlopedExtrapolation = sloped,
},
}, nodeId);
StatusCode.IsGood(errors[0].StatusCode).ShouldBeTrue();
fake.LastCall.ShouldBe("Processed");
fake.LastAggregate.ShouldBe(HistoryAggregateType.TimeAverage);
await host.DisposeAsync();
}
/// <summary>#677: the server's declared aggregate defaults are the SDK's own server defaults.</summary>
[Fact]
public void The_server_aggregate_defaults_agree_with_the_SDK_AggregateManager()
{
var sdk = new Opc.Ua.Server.AggregateManager(null!).GetDefaultConfiguration(null);
var ours = ServerAggregateConfiguration.Defaults;
ours.TreatUncertainAsBad.ShouldBe(sdk.TreatUncertainAsBad);
ours.PercentDataBad.ShouldBe(sdk.PercentDataBad);
ours.PercentDataGood.ShouldBe(sdk.PercentDataGood);
ours.UseSlopedExtrapolation.ShouldBe(sdk.UseSlopedExtrapolation);
ServerAggregateConfiguration.IsServable(sdk).ShouldBeTrue();
ServerAggregateConfiguration.IsServable(null).ShouldBeTrue();
}
/// <summary>#285: a Part 13 aggregate beyond the original five (Interpolative) is forwarded to the
/// backend as its own <see cref="HistoryAggregateType"/> member.</summary>
[Fact]