From 5f225f0b4410df738d3d04192af673222381909b Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Mon, 12 Oct 2026 10:00:30 -0400 Subject: [PATCH] =?UTF-8?q?fix(historian):=20#681=20on=20#682/#684=20?= =?UTF-8?q?=E2=80=94=20Gateway=20served=20set=20is=20the=2012-member=20Ser?= =?UTF-8?q?vedAggregates;=20raw=20buckets=20use=20Good-only=20values?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - #682: ServedAggregates (ten raw-evaluated per-sample + TimeAverage/Total) replaces the placeholder AggregateModeMapper.Served; NativeAggregates is derived from the mapper's switch. The Gateway expectations in AggregateModeMapperTests, HistorianBackendRegistryTests and AdvertisedAggregateFunctionsWireTests move to the 12-member set (the wire tests' 'not served by Gateway' example is now TimeAverage2). - #684: RawBucketAccumulator applies the SQL backends' Good-only value columns (Avg/Min/Max/Sum and the ActualTime extreme over Good numeric samples), UncertainMin/UncertainMax over Uncertain numeric samples, and GoodNonNumeric. The parity oracle carries the #684 InMemoryRows and the DenseUncertain / UncertainBeyond / UncertainText datasets. --- docs/Historian.md | 4 ++ .../RawAggregateReader.cs | 28 +++++++++--- .../GatewayRawAggregateTests.cs | 44 +++++++++++++++---- .../AdvertisedAggregateFunctionsWireTests.cs | 10 ++--- 4 files changed, 66 insertions(+), 20 deletions(-) diff --git a/docs/Historian.md b/docs/Historian.md index 2677061e..0faa5f86 100644 --- a/docs/Historian.md +++ b/docs/Historian.md @@ -800,6 +800,10 @@ and evicts its shared historian session (see the HistorianGateway issue text in server ceiling" — `Historian:ReadLimits:MaxValuesCeiling`, default 1,000,000, with `AllowStreamingBeyondCeiling=false`; the default `1,000,000 + 18` exceeds it) is also `BadTooManyOperations`, not a backend fault — lower `MaxTimeWeightedRawRows` to `ceiling − 18` on such a gateway. A caller cancellation (client cancel, `HistoryReadDeadline`) is not counted as a backend fault. +- **Bucket statistics** are `RawBucketAccumulator`, the C# statement of the SQL backends' per-sample columns — incl. #684: + Average / Minimum / Maximum / Range / *ActualTime over **Good** samples only, the Uncertain extremes feeding only the Min/Max + status rule, and an Uncertain String/DateTime sample not refusing the Good-value aggregates. A change to those SQL columns + must be repeated there. - **Status** is the raw read's quality mapping. C-002: the gateway discards quality on write, so AVEVA history is normally all Good. ⚠️ **Known pre-existing limitation:** `SampleMapper` falls back to AVEVA's own `Quality` field when `OpcQuality` is 0 — but 0 is also the OPC DA Bad value, so a Bad row whose AVEVA `Quality` is ≥ 64 (e.g. 133) maps Uncertain/Good. Raw reads have always done diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway/RawAggregateReader.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway/RawAggregateReader.cs index 2f4a9a3a..f2e59d80 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway/RawAggregateReader.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway/RawAggregateReader.cs @@ -211,8 +211,10 @@ internal static class RawAggregateReader /// /// Builds the per-bucket statistics () the SQL backends' aggregate queries produce, from a stream /// of raw samples — the C# statement of SqlAggregateQueries' columns: counts by class (usable = non-Bad with a -/// value; good = usable with Good severity), Avg/Min/Max/Sum over the usable numeric values, first/last usable value by -/// time, the extreme usable numeric value with its time (ties ⇒ earliest), and the first/last sample of any quality. +/// value; good = usable with Good severity; NonNumeric = usable String/DateTime; GoodNonNumeric = good String/DateTime), +/// Avg/Min/Max/Sum over the GOOD numeric values and UncertainMin/UncertainMax over the usable-but-not-Good numeric values +/// (#684), first/last usable value by time, the extreme GOOD numeric value with its time (ties ⇒ earliest), and the +/// first/last sample of any quality. /// /// /// Keep in step with the SQL backends' per-sample value columns: a change to which samples feed a statistic there @@ -229,8 +231,9 @@ internal sealed class RawBucketAccumulator private sealed class State { - public int Total, Usable, Good, NonNumeric, Numeric; + public int Total, Usable, Good, NonNumeric, GoodNonNumeric, GoodNumeric, UncertainNumeric; public double Sum, Min = double.PositiveInfinity, Max = double.NegativeInfinity; + public double UncertainMin = double.PositiveInfinity, UncertainMax = double.NegativeInfinity; public StoredSample? First, Last, FirstAny, LastAny, Extreme; } @@ -251,9 +254,18 @@ internal sealed class RawBucketAccumulator if (!s.Value.TryGetDouble(out var v)) { st.NonNumeric++; + if (s.IsGood) st.GoodNonNumeric++; return; } - st.Numeric++; + if (!s.IsGood) + { + // #684: an Uncertain value is not part of the value set; it only feeds the Min/Max status rule. + st.UncertainNumeric++; + if (v < st.UncertainMin) st.UncertainMin = v; + if (v > st.UncertainMax) st.UncertainMax = v; + return; + } + st.GoodNumeric++; st.Sum += v; if (v < st.Min) st.Min = v; if (v > st.Max) st.Max = v; @@ -272,7 +284,8 @@ internal sealed class RawBucketAccumulator _buckets.OrderBy(kv => kv.Key).Select(kv => { var st = kv.Value; - var hasNumeric = st.Numeric > 0; + var hasNumeric = st.GoodNumeric > 0; + var hasUncertain = st.UncertainNumeric > 0; return new BucketRow { Bucket = kv.Key, @@ -281,10 +294,13 @@ internal sealed class RawBucketAccumulator Good = st.Good, Bad = st.Total - st.Usable, NonNumeric = st.NonNumeric, - Avg = hasNumeric ? st.Sum / st.Numeric : null, + GoodNonNumeric = st.GoodNonNumeric, + Avg = hasNumeric ? st.Sum / st.GoodNumeric : null, Min = hasNumeric ? st.Min : null, Max = hasNumeric ? st.Max : null, Sum = hasNumeric ? st.Sum : null, + UncertainMin = hasUncertain ? st.UncertainMin : null, + UncertainMax = hasUncertain ? st.UncertainMax : null, First = st.First?.Value ?? HistorianValue.Empty, Last = st.Last?.Value ?? HistorianValue.Empty, Extreme = st.Extreme?.Value.Number, diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests/GatewayRawAggregateTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests/GatewayRawAggregateTests.cs index b3a7a4f8..5ce178df 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests/GatewayRawAggregateTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests/GatewayRawAggregateTests.cs @@ -78,10 +78,29 @@ public sealed class GatewayRawAggregateTests /// Equal extremes in one bucket (ActualTime ties ⇒ earliest). private static readonly StoredSample[] Ties = [D(1, 5), D(3, 9), D(4, 1), D(6, 9), D(8, 1), D(25, 2)]; + /// #684 (Sql.Common's DenseUncertain): Dense with every 5th sample an in-range Uncertain 16. + private static readonly StoredSample[] DenseUncertain = Enumerable.Range(0, 45) + .Select(i => (t: -10 + i * 2.5, i)) + .Select(x => x.i % 5 == 0 && x.t != 22.5 + ? D(x.t, 16, Uncertain) + : D(x.t, 10 + (x.i * 7 % 13), x.t == 22.5 ? Bad : Good)) + .ToArray(); + + /// #684 (Sql.Common's UncertainBeyond): Uncertain values below and above the Good extremes in one interval. + private static readonly StoredSample[] UncertainBeyond = [D(-5, 20), D(2, 10), D(4, 5, Uncertain), D(6, 99, Uncertain), D(8, 30), D(25, 20)]; + + /// #684: an Uncertain string among Good numbers (does not refuse the Good-value aggregates) and a Good one. + private static readonly StoredSample[] UncertainText = + [ + D(-5, 1), D(2, 4), S(4, HistorianValue.FromString("u"), Uncertain), D(6, 8), S(14, HistorianValue.FromString("g"), Good), + D(16, 3), + ]; + private static readonly (string Name, StoredSample[] Data)[] Datasets = [ ("A", A), ("Dense", Dense), ("Gappy", Gappy), ("Bracketed", Bracketed), ("EarlyEnd", EarlyEnd), ("EndOnEdge", EndOnEdge), ("LateStart", LateStart), ("OutageEnds", OutageEnds), ("Mixed", Mixed), ("Ties", Ties), + ("DenseUncertain", DenseUncertain), ("UncertainBeyond", UncertainBeyond), ("UncertainText", UncertainText), ]; private static readonly (double From, double To, double Interval)[] Windows = @@ -174,7 +193,7 @@ public sealed class GatewayRawAggregateTests earliest, latest, Now); } - /// Independent LINQ statement of the per-bucket statistics (copied from Sql.Common.Tests). + /// Independent LINQ statement of the per-bucket statistics (copied from Sql.Common.Tests, #684 semantics). private static IEnumerable InMemoryRows(BucketPlan plan, IEnumerable samples, bool maximum) { foreach (var g in samples.Select(s => (s, b: plan.IndexOf(s.TimeUtc))).Where(x => x.b is not null) @@ -184,7 +203,9 @@ public sealed class GatewayRawAggregateTests var usable = all.Where(s => s.IsUsable).ToList(); var numeric = usable.Where(s => s.Value.Kind is HistorianValueKind.Double or HistorianValueKind.Int64 or HistorianValueKind.Boolean).ToList(); - var extreme = numeric.OrderBy(s => maximum ? -s.Value.Number : s.Value.Number).ThenBy(s => s.TimeUtc) + var good = numeric.Where(s => s.IsGood).ToList(); + var uncertain = numeric.Where(s => !s.IsGood).ToList(); + var extreme = good.OrderBy(s => maximum ? -s.Value.Number : s.Value.Number).ThenBy(s => s.TimeUtc) .Cast().FirstOrDefault(); yield return new BucketRow { @@ -194,10 +215,13 @@ public sealed class GatewayRawAggregateTests Good = all.Count(s => s.IsGood), Bad = all.Count - usable.Count, NonNumeric = usable.Count - numeric.Count, - Avg = numeric.Count == 0 ? null : numeric.Average(s => s.Value.Number), - Min = numeric.Count == 0 ? null : numeric.Min(s => s.Value.Number), - Max = numeric.Count == 0 ? null : numeric.Max(s => s.Value.Number), - Sum = numeric.Count == 0 ? null : numeric.Sum(s => s.Value.Number), + GoodNonNumeric = usable.Count(s => s.IsGood) - numeric.Count(s => s.IsGood), + Avg = good.Count == 0 ? null : good.Average(s => s.Value.Number), + Min = good.Count == 0 ? null : good.Min(s => s.Value.Number), + Max = good.Count == 0 ? null : good.Max(s => s.Value.Number), + Sum = good.Count == 0 ? null : good.Sum(s => s.Value.Number), + UncertainMin = uncertain.Count == 0 ? null : uncertain.Min(s => s.Value.Number), + UncertainMax = uncertain.Count == 0 ? null : uncertain.Max(s => s.Value.Number), First = usable.Count == 0 ? HistorianValue.Empty : usable[0].Value, Last = usable.Count == 0 ? HistorianValue.Empty : usable[^1].Value, Extreme = extreme?.Value.Number, @@ -525,16 +549,18 @@ public sealed class GatewayRawAggregateTests foreach (var maximum in new[] { false, true }) { var acc = new RawBucketAccumulator(maximum); - foreach (var s in Dense.Reverse()) + foreach (var s in DenseUncertain.Reverse()) if (plan.IndexOf(s.TimeUtc) is { } b) acc.Add(b, s); - var expected = InMemoryRows(plan, Dense, maximum).ToList(); + var expected = InMemoryRows(plan, DenseUncertain, maximum).ToList(); var actual = acc.Rows().ToList(); actual.Count.ShouldBe(expected.Count); foreach (var (a, e) in actual.Zip(expected)) { (a.Bucket, a.Total, a.Usable, a.Good, a.Bad, a.NonNumeric).ShouldBe((e.Bucket, e.Total, e.Usable, e.Good, e.Bad, e.NonNumeric)); (a.Min, a.Max, a.Extreme, a.ExtremeTime).ShouldBe((e.Min, e.Max, e.Extreme, e.ExtremeTime)); - a.Avg!.Value.ShouldBe(e.Avg!.Value, 1e-9); + (a.UncertainMin, a.UncertainMax, a.GoodNonNumeric).ShouldBe((e.UncertainMin, e.UncertainMax, e.GoodNonNumeric)); + (a.Avg is null).ShouldBe(e.Avg is null); + if (a.Avg is { } avg) avg.ShouldBe(e.Avg!.Value, 1e-9); (a.First, a.Last, a.FirstAny, a.LastAny).ShouldBe((e.First, e.Last, e.FirstAny, e.LastAny)); } } diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/AdvertisedAggregateFunctionsWireTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/AdvertisedAggregateFunctionsWireTests.cs index c97cfc0c..bcece177 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/AdvertisedAggregateFunctionsWireTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/AdvertisedAggregateFunctionsWireTests.cs @@ -109,16 +109,16 @@ public sealed class AdvertisedAggregateFunctionsWireTests rig.Nm.HistorianDataSource = Backend("SqlServer"); (await AdvertisedAsync(session, ObjectIds.HistoryServerCapabilities_AggregateFunctions, ct)).ShouldBe(SqlNames); - (await ParentsAsync(session, ObjectIds.AggregateFunction_Average, ct)).ShouldBe( + (await ParentsAsync(session, ObjectIds.AggregateFunction_TimeAverage2, ct)).ShouldBe( [ObjectIds.Server_ServerCapabilities_AggregateFunctions, ObjectIds.HistoryServerCapabilities_AggregateFunctions], ignoreOrder: true); - // Gateway has no arithmetic Average: dropped from the history folder (inverse reference too), kept in the - // server folder. + // Gateway has no TimeAverage2 (#681: no prior-sample read for Part 13 bounds): dropped from the history folder + // (inverse reference too), kept in the server folder. rig.Nm.HistorianDataSource = Backend("Gateway"); (await AdvertisedAsync(session, ObjectIds.HistoryServerCapabilities_AggregateFunctions, ct)).ShouldBe(GatewayNames); (await AdvertisedAsync(session, ObjectIds.Server_ServerCapabilities_AggregateFunctions, ct)).ShouldBe(AllSdkNames); - (await ParentsAsync(session, ObjectIds.AggregateFunction_Average, ct)).ShouldBe( + (await ParentsAsync(session, ObjectIds.AggregateFunction_TimeAverage2, ct)).ShouldBe( [ObjectIds.Server_ServerCapabilities_AggregateFunctions]); // And back: removed references are restored. @@ -130,7 +130,7 @@ public sealed class AdvertisedAggregateFunctionsWireTests [Theory] [InlineData("StandardDeviationSample")] // served by no historian backend, never in the history folder - [InlineData("Average")] // not served by the Gateway backend wired below + [InlineData("TimeAverage2")] // not served by the Gateway backend wired below (#681) public async Task A_live_AggregateFilter_subscription_for_an_unadvertised_aggregate_still_works(string aggregateName) { // The trim touches only the advertised folders, not the AggregateManager's calculator factories, which is what