fix(historian): #681 on #682/#684 — Gateway served set is the 12-member ServedAggregates; raw buckets use Good-only values

- #682: ServedAggregates (ten raw-evaluated per-sample + TimeAverage/Total) replaces the placeholder AggregateModeMapper.Served; NativeAggregates is derived from the mapper's switch. The Gateway expectations in AggregateModeMapperTests, HistorianBackendRegistryTests and AdvertisedAggregateFunctionsWireTests move to the 12-member set (the wire tests' 'not served by Gateway' example is now TimeAverage2). - #684: RawBucketAccumulator applies the SQL backends' Good-only value columns (Avg/Min/Max/Sum and the ActualTime extreme over Good numeric samples), UncertainMin/UncertainMax over Uncertain numeric samples, and GoodNonNumeric. The parity oracle carries the #684 InMemoryRows and the DenseUncertain / UncertainBeyond / UncertainText datasets.
2026-10-12 10:00:30 -04:00
parent 78f604e0cd
commit 5f225f0b44
4 changed files with 66 additions and 20 deletions
@@ -800,6 +800,10 @@ and evicts its shared historian session (see the HistorianGateway issue text in
  server ceiling" — `Historian:ReadLimits:MaxValuesCeiling`, default 1,000,000, with `AllowStreamingBeyondCeiling=false`; the
  default `1,000,000 + 18` exceeds it) is also `BadTooManyOperations`, not a backend fault — lower `MaxTimeWeightedRawRows` to
  `ceiling − 18` on such a gateway. A caller cancellation (client cancel, `HistoryReadDeadline`) is not counted as a backend fault.
+- **Bucket statistics** are `RawBucketAccumulator`, the C# statement of the SQL backends' per-sample columns — incl. #684:
+  Average / Minimum / Maximum / Range / *ActualTime over **Good** samples only, the Uncertain extremes feeding only the Min/Max
+  status rule, and an Uncertain String/DateTime sample not refusing the Good-value aggregates. A change to those SQL columns
+  must be repeated there.
 - **Status** is the raw read's quality mapping. C-002: the gateway discards quality on write, so AVEVA history is normally all Good.
  ⚠️ **Known pre-existing limitation:** `SampleMapper` falls back to AVEVA's own `Quality` field when `OpcQuality` is 0 — but 0 is
  also the OPC DA Bad value, so a Bad row whose AVEVA `Quality` is ≥ 64 (e.g. 133) maps Uncertain/Good. Raw reads have always done
@@ -211,8 +211,10 @@ internal static class RawAggregateReader
 /// <summary>
 ///     Builds the per-bucket statistics (<see cref="BucketRow"/>) the SQL backends' aggregate queries produce, from a stream
 ///     of raw samples — the C# statement of <c>SqlAggregateQueries</c>' columns: counts by class (usable = non-Bad with a
-///     value; good = usable with Good severity), Avg/Min/Max/Sum over the usable numeric values, first/last usable value by
-///     time, the extreme usable numeric value with its time (ties ⇒ earliest), and the first/last sample of any quality.
+///     value; good = usable with Good severity; NonNumeric = usable String/DateTime; GoodNonNumeric = good String/DateTime),
+///     Avg/Min/Max/Sum over the GOOD numeric values and UncertainMin/UncertainMax over the usable-but-not-Good numeric values
+///     (#684), first/last usable value by time, the extreme GOOD numeric value with its time (ties ⇒ earliest), and the
+///     first/last sample of any quality.
 /// </summary>
 /// <remarks>
 ///     Keep in step with the SQL backends' per-sample value columns: a change to which samples feed a statistic there
@@ -229,8 +231,9 @@ internal sealed class RawBucketAccumulator

    private sealed class State
    {
-        public int Total, Usable, Good, NonNumeric, Numeric;
+        public int Total, Usable, Good, NonNumeric, GoodNonNumeric, GoodNumeric, UncertainNumeric;
        public double Sum, Min = double.PositiveInfinity, Max = double.NegativeInfinity;
+        public double UncertainMin = double.PositiveInfinity, UncertainMax = double.NegativeInfinity;
        public StoredSample? First, Last, FirstAny, LastAny, Extreme;
    }

@@ -251,9 +254,18 @@ internal sealed class RawBucketAccumulator
        if (!s.Value.TryGetDouble(out var v))
        {
            st.NonNumeric++;
+            if (s.IsGood) st.GoodNonNumeric++;
            return;
        }
-        st.Numeric++;
+        if (!s.IsGood)
+        {
+            // #684: an Uncertain value is not part of the value set; it only feeds the Min/Max status rule.
+            st.UncertainNumeric++;
+            if (v < st.UncertainMin) st.UncertainMin = v;
+            if (v > st.UncertainMax) st.UncertainMax = v;
+            return;
+        }
+        st.GoodNumeric++;
        st.Sum += v;
        if (v < st.Min) st.Min = v;
        if (v > st.Max) st.Max = v;
@@ -272,7 +284,8 @@ internal sealed class RawBucketAccumulator
        _buckets.OrderBy(kv => kv.Key).Select(kv =>
        {
            var st = kv.Value;
-            var hasNumeric = st.Numeric > 0;
+            var hasNumeric = st.GoodNumeric > 0;
+            var hasUncertain = st.UncertainNumeric > 0;
            return new BucketRow
            {
                Bucket = kv.Key,
@@ -281,10 +294,13 @@ internal sealed class RawBucketAccumulator
                Good = st.Good,
                Bad = st.Total - st.Usable,
                NonNumeric = st.NonNumeric,
-                Avg = hasNumeric ? st.Sum / st.Numeric : null,
+                GoodNonNumeric = st.GoodNonNumeric,
+                Avg = hasNumeric ? st.Sum / st.GoodNumeric : null,
                Min = hasNumeric ? st.Min : null,
                Max = hasNumeric ? st.Max : null,
                Sum = hasNumeric ? st.Sum : null,
+                UncertainMin = hasUncertain ? st.UncertainMin : null,
+                UncertainMax = hasUncertain ? st.UncertainMax : null,
                First = st.First?.Value ?? HistorianValue.Empty,
                Last = st.Last?.Value ?? HistorianValue.Empty,
                Extreme = st.Extreme?.Value.Number,
@@ -78,10 +78,29 @@ public sealed class GatewayRawAggregateTests
    /// <summary>Equal extremes in one bucket (ActualTime ties ⇒ earliest).</summary>
    private static readonly StoredSample[] Ties = [D(1, 5), D(3, 9), D(4, 1), D(6, 9), D(8, 1), D(25, 2)];

+    /// <summary>#684 (Sql.Common's DenseUncertain): Dense with every 5th sample an in-range Uncertain 16.</summary>
+    private static readonly StoredSample[] DenseUncertain = Enumerable.Range(0, 45)
+        .Select(i => (t: -10 + i * 2.5, i))
+        .Select(x => x.i % 5 == 0 && x.t != 22.5
+            ? D(x.t, 16, Uncertain)
+            : D(x.t, 10 + (x.i * 7 % 13), x.t == 22.5 ? Bad : Good))
+        .ToArray();
+
+    /// <summary>#684 (Sql.Common's UncertainBeyond): Uncertain values below and above the Good extremes in one interval.</summary>
+    private static readonly StoredSample[] UncertainBeyond = [D(-5, 20), D(2, 10), D(4, 5, Uncertain), D(6, 99, Uncertain), D(8, 30), D(25, 20)];
+
+    /// <summary>#684: an Uncertain string among Good numbers (does not refuse the Good-value aggregates) and a Good one.</summary>
+    private static readonly StoredSample[] UncertainText =
+    [
+        D(-5, 1), D(2, 4), S(4, HistorianValue.FromString("u"), Uncertain), D(6, 8), S(14, HistorianValue.FromString("g"), Good),
+        D(16, 3),
+    ];
+
    private static readonly (string Name, StoredSample[] Data)[] Datasets =
    [
        ("A", A), ("Dense", Dense), ("Gappy", Gappy), ("Bracketed", Bracketed), ("EarlyEnd", EarlyEnd),
        ("EndOnEdge", EndOnEdge), ("LateStart", LateStart), ("OutageEnds", OutageEnds), ("Mixed", Mixed), ("Ties", Ties),
+        ("DenseUncertain", DenseUncertain), ("UncertainBeyond", UncertainBeyond), ("UncertainText", UncertainText),
    ];

    private static readonly (double From, double To, double Interval)[] Windows =
@@ -174,7 +193,7 @@ public sealed class GatewayRawAggregateTests
            earliest, latest, Now);
    }

-    /// <summary>Independent LINQ statement of the per-bucket statistics (copied from Sql.Common.Tests).</summary>
+    /// <summary>Independent LINQ statement of the per-bucket statistics (copied from Sql.Common.Tests, #684 semantics).</summary>
    private static IEnumerable<BucketRow> InMemoryRows(BucketPlan plan, IEnumerable<StoredSample> samples, bool maximum)
    {
        foreach (var g in samples.Select(s => (s, b: plan.IndexOf(s.TimeUtc))).Where(x => x.b is not null)
@@ -184,7 +203,9 @@ public sealed class GatewayRawAggregateTests
            var usable = all.Where(s => s.IsUsable).ToList();
            var numeric = usable.Where(s => s.Value.Kind is HistorianValueKind.Double or HistorianValueKind.Int64
                or HistorianValueKind.Boolean).ToList();
-            var extreme = numeric.OrderBy(s => maximum ? -s.Value.Number : s.Value.Number).ThenBy(s => s.TimeUtc)
+            var good = numeric.Where(s => s.IsGood).ToList();
+            var uncertain = numeric.Where(s => !s.IsGood).ToList();
+            var extreme = good.OrderBy(s => maximum ? -s.Value.Number : s.Value.Number).ThenBy(s => s.TimeUtc)
                .Cast<StoredSample?>().FirstOrDefault();
            yield return new BucketRow
            {
@@ -194,10 +215,13 @@ public sealed class GatewayRawAggregateTests
                Good = all.Count(s => s.IsGood),
                Bad = all.Count - usable.Count,
                NonNumeric = usable.Count - numeric.Count,
-                Avg = numeric.Count == 0 ? null : numeric.Average(s => s.Value.Number),
-                Min = numeric.Count == 0 ? null : numeric.Min(s => s.Value.Number),
-                Max = numeric.Count == 0 ? null : numeric.Max(s => s.Value.Number),
-                Sum = numeric.Count == 0 ? null : numeric.Sum(s => s.Value.Number),
+                GoodNonNumeric = usable.Count(s => s.IsGood) - numeric.Count(s => s.IsGood),
+                Avg = good.Count == 0 ? null : good.Average(s => s.Value.Number),
+                Min = good.Count == 0 ? null : good.Min(s => s.Value.Number),
+                Max = good.Count == 0 ? null : good.Max(s => s.Value.Number),
+                Sum = good.Count == 0 ? null : good.Sum(s => s.Value.Number),
+                UncertainMin = uncertain.Count == 0 ? null : uncertain.Min(s => s.Value.Number),
+                UncertainMax = uncertain.Count == 0 ? null : uncertain.Max(s => s.Value.Number),
                First = usable.Count == 0 ? HistorianValue.Empty : usable[0].Value,
                Last = usable.Count == 0 ? HistorianValue.Empty : usable[^1].Value,
                Extreme = extreme?.Value.Number,
@@ -525,16 +549,18 @@ public sealed class GatewayRawAggregateTests
        foreach (var maximum in new[] { false, true })
        {
            var acc = new RawBucketAccumulator(maximum);
-            foreach (var s in Dense.Reverse())
+            foreach (var s in DenseUncertain.Reverse())
                if (plan.IndexOf(s.TimeUtc) is { } b) acc.Add(b, s);
-            var expected = InMemoryRows(plan, Dense, maximum).ToList();
+            var expected = InMemoryRows(plan, DenseUncertain, maximum).ToList();
            var actual = acc.Rows().ToList();
            actual.Count.ShouldBe(expected.Count);
            foreach (var (a, e) in actual.Zip(expected))
            {
                (a.Bucket, a.Total, a.Usable, a.Good, a.Bad, a.NonNumeric).ShouldBe((e.Bucket, e.Total, e.Usable, e.Good, e.Bad, e.NonNumeric));
                (a.Min, a.Max, a.Extreme, a.ExtremeTime).ShouldBe((e.Min, e.Max, e.Extreme, e.ExtremeTime));
-                a.Avg!.Value.ShouldBe(e.Avg!.Value, 1e-9);
+                (a.UncertainMin, a.UncertainMax, a.GoodNonNumeric).ShouldBe((e.UncertainMin, e.UncertainMax, e.GoodNonNumeric));
+                (a.Avg is null).ShouldBe(e.Avg is null);
+                if (a.Avg is { } avg) avg.ShouldBe(e.Avg!.Value, 1e-9);
                (a.First, a.Last, a.FirstAny, a.LastAny).ShouldBe((e.First, e.Last, e.FirstAny, e.LastAny));
            }
        }
@@ -109,16 +109,16 @@ public sealed class AdvertisedAggregateFunctionsWireTests

            rig.Nm.HistorianDataSource = Backend("SqlServer");
            (await AdvertisedAsync(session, ObjectIds.HistoryServerCapabilities_AggregateFunctions, ct)).ShouldBe(SqlNames);
-            (await ParentsAsync(session, ObjectIds.AggregateFunction_Average, ct)).ShouldBe(
+            (await ParentsAsync(session, ObjectIds.AggregateFunction_TimeAverage2, ct)).ShouldBe(
                [ObjectIds.Server_ServerCapabilities_AggregateFunctions, ObjectIds.HistoryServerCapabilities_AggregateFunctions],
                ignoreOrder: true);

-            // Gateway has no arithmetic Average: dropped from the history folder (inverse reference too), kept in the
-            // server folder.
+            // Gateway has no TimeAverage2 (#681: no prior-sample read for Part 13 bounds): dropped from the history folder
+            // (inverse reference too), kept in the server folder.
            rig.Nm.HistorianDataSource = Backend("Gateway");
            (await AdvertisedAsync(session, ObjectIds.HistoryServerCapabilities_AggregateFunctions, ct)).ShouldBe(GatewayNames);
            (await AdvertisedAsync(session, ObjectIds.Server_ServerCapabilities_AggregateFunctions, ct)).ShouldBe(AllSdkNames);
-            (await ParentsAsync(session, ObjectIds.AggregateFunction_Average, ct)).ShouldBe(
+            (await ParentsAsync(session, ObjectIds.AggregateFunction_TimeAverage2, ct)).ShouldBe(
                [ObjectIds.Server_ServerCapabilities_AggregateFunctions]);

            // And back: removed references are restored.
@@ -130,7 +130,7 @@ public sealed class AdvertisedAggregateFunctionsWireTests

    [Theory]
    [InlineData("StandardDeviationSample")] // served by no historian backend, never in the history folder
-    [InlineData("Average")]                 // not served by the Gateway backend wired below
+    [InlineData("TimeAverage2")]            // not served by the Gateway backend wired below (#681)
    public async Task A_live_AggregateFilter_subscription_for_an_unadvertised_aggregate_still_works(string aggregateName)
    {
        // The trim touches only the advertised folders, not the AggregateManager's calculator factories, which is what