dohertj2/lmxopcua
1
0
Fork 0
Code Issues 32 Pull Requests Actions Packages Projects Releases Wiki Activity

fix(historian): per-sample Average/Min/Max/Range/ActualTime use Good samples only, Part 13 min/max status (#684)

Browse Source 
Part 13 defines Average/Minimum/Maximum/Range/*ActualTime over Good raw values
(independent of TreatUncertainAsBad) and the SDK's calculators skip Uncertain values
under the server's TreatUncertainAsBad=false. SqlServer/QuestDB value columns now use
the good flag (g) instead of usable (u); new UncMin/UncMax columns feed the Part 13
Minimum/Maximum/Range status rule (Uncertain on a Bad sample or an Uncertain value
beyond the Good extreme -- the SDK applies only the Bad half). An Uncertain-only
interval is BadNoData. Uncertain-bearing parity datasets (in-memory + real SQL) match
the SDK unmasked; the beyond-extreme divergence is pinned explicitly.

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Joseph Doherty
2026-10-12 09:03:47 -04:00
parent 113f1eaf7c
commit a11ab0dc08
14 changed files with 533 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb/QuestDbAggregateQueries.cs
+22 -11
View File
@@ -58,17 +58,20 @@ internal static class QuestDbAggregateQueries
/// <summary>
/// Average / Minimum / Maximum / Count / Range (PercentGood / PercentBad and Total are time-weighted, not here).
/// Family columns: <c>6 avg_value, 7 min_value, 8 max_value, 9 sum_value</c> (DOUBLE, NULL when no usable sample).
/// Family columns: <c>6 avg_value, 7 min_value, 8 max_value, 9 sum_value</c> over the <b>Good</b> numeric samples
/// (DOUBLE, NULL when none — #684, Part 13 / the SDK), then <c>10 unc_min, 11 unc_max</c> over the Uncertain ones (the
/// Part 13 Minimum/Maximum/Range status rule).
/// </summary>
/// <param name="sample">Prefixed sample table name.</param>
/// <returns>The SQL text.</returns>
/// <param name="reverse">True for a reverse read (window <c>($4, $5]</c>).</param>
public static string Simple(string sample, bool reverse = false) =>
"SELECT b, " + CommonCounts() + ", " +
"avg(CASE WHEN u = 1 THEN num_value END) AS avg_value, " +
"min(CASE WHEN u = 1 THEN num_value END) AS min_value, " +
"max(CASE WHEN u = 1 THEN num_value END) AS max_value, " +
"sum(CASE WHEN u = 1 THEN num_value END) AS sum_value " +
"avg(CASE WHEN g = 1 THEN num_value END) AS avg_value, " +
"min(CASE WHEN g = 1 THEN num_value END) AS min_value, " +
"max(CASE WHEN g = 1 THEN num_value END) AS max_value, " +
"sum(CASE WHEN g = 1 THEN num_value END) AS sum_value, " +
UncertainExtremes() + " " +
"FROM (" + Classified(sample, reverse) + ") GROUP BY b ORDER BY b";
/// <summary>Ordinal of the StartEnd family's first-sample-of-any-quality columns (<see cref="StartEnd"/>).</summary>
@@ -126,9 +129,9 @@ internal static class QuestDbAggregateQueries
}
/// <summary>
/// MinimumActualTime / MaximumActualTime: the usable sample with the smallest (largest) <c>num_value</c>, ties
/// broken by the earliest <c>ts</c>. Family columns: <c>6 num_value, 7 ts</c> (NULL when the bucket has no usable
/// numeric sample).
/// MinimumActualTime / MaximumActualTime: the <b>Good</b> sample (#684) with the smallest (largest) <c>num_value</c>,
/// ties broken by the earliest <c>ts</c>. Family columns: <c>6 num_value, 7 ts</c> (NULL when the bucket has no Good
/// numeric sample), then <c>8 unc_min, 9 unc_max</c> as in <see cref="Simple"/>.
/// </summary>
/// <remarks>
/// Deliberately still two references to the classified rows (counts + the ranked extreme), unlike
@@ -143,11 +146,12 @@ internal static class QuestDbAggregateQueries
/// <param name="reverse">True for a reverse read (window <c>($4, $5]</c>).</param>
public static string ActualTime(string sample, bool maximum, bool reverse = false) =>
"WITH s AS (" + Classified(sample, reverse) + "), " +
"st AS (SELECT b, " + CommonCounts() + " FROM s GROUP BY b), " +
"st AS (SELECT b, " + CommonCounts() + ", " + UncertainExtremes() + " FROM s GROUP BY b), " +
"r AS (SELECT b, num_value, ts, " +
"row_number() OVER (PARTITION BY b ORDER BY num_value " + (maximum ? "DESC" : "ASC") + ", ts ASC) AS rn " +
"FROM s WHERE u = 1 AND num_value IS NOT NULL) " +
"SELECT st.*, r.num_value, r.ts " +
"FROM s WHERE g = 1 AND num_value IS NOT NULL) " +
"SELECT st.b, st.total_count, st.usable_count, st.good_count, st.bad_count, st.non_numeric_count, " +
"r.num_value, r.ts, st.unc_min, st.unc_max " +
"FROM st LEFT JOIN (SELECT * FROM r WHERE rn = 1) r ON r.b = st.b ORDER BY st.b";
/// <summary>
@@ -217,6 +221,13 @@ internal static class QuestDbAggregateQueries
$"CASE WHEN status < {GoodLimit} AND kind <> {EmptyKind} THEN 1 ELSE 0 END AS g " +
$"FROM {sample} WHERE tag = $3 AND " + (reverse ? "ts > $4 AND ts <= $5" : "ts >= $4 AND ts < $5") + " AND NOT deleted";
/// <summary>
/// The minimum and maximum <b>Uncertain</b> (usable, not Good) numeric value of the bucket — <c>unc_min, unc_max</c>
/// (#684). Not result values: Part 13 makes Minimum/Maximum/Range Uncertain when one lies beyond the Good extreme.
/// </summary>
private static string UncertainExtremes() =>
"min(CASE WHEN u = 1 AND g = 0 THEN num_value END) AS unc_min, max(CASE WHEN u = 1 AND g = 0 THEN num_value END) AS unc_max";
/// <summary>The five common count columns (after <c>b</c>).</summary>
private static string CommonCounts() =>
"count() AS total_count, sum(u) AS usable_count, sum(g) AS good_count, count() - sum(u) AS bad_count, " +
src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb/QuestDbHistorianDataSource.cs
+4
View File
@@ -617,6 +617,8 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource
row.Min = D(reader, f + 1);
row.Max = D(reader, f + 2);
row.Sum = D(reader, f + 3);
row.UncertainMin = D(reader, f + 4);
row.UncertainMax = D(reader, f + 5);
break;
case AggregateFamily.StartEnd:
row.First = V(reader, f);
@@ -627,6 +629,8 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource
default:
row.Extreme = D(reader, f);
row.ExtremeTime = reader.IsDBNull(f + 1) ? null : QuestDbConnection.ReadUtc(reader, f + 1);
row.UncertainMin = D(reader, f + 2);
row.UncertainMax = D(reader, f + 3);
break;
}
return row;
src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common/AggregateEvaluator.cs
+40 -7
View File
@@ -9,7 +9,11 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common;
/// <para>
/// Per interval: no samples ⇒ <c>BadNoData</c> (Count: 0 inside the data range). Otherwise no usable sample ⇒
/// <c>BadNoData</c>; a numeric aggregate with a usable String/DateTime sample ⇒ <c>BadAggregateInvalidInputs</c>;
/// else the value, Good when every sample is Good, <c>UncertainDataSubNormal</c> when any is Bad or Uncertain.
/// else the value. #684: Average / Minimum / Maximum / Range / *ActualTime are computed over <b>Good</b> values only
/// (Part 13 and the SDK — an Uncertain sample is never part of the value; an interval with no Good value is
/// <c>BadNoData</c>). Status: Average value-based (Good only when every sample is Good, else
/// <c>UncertainDataSubNormal</c>); Minimum / Maximum / Range / *ActualTime per their Part 13 tables — Uncertain when a
/// Bad sample exists or an Uncertain value lies beyond the reported extreme (<see cref="MinMaxStatus"/>).
/// Every calculated status — including <c>BadNoData</c> — carries the Calculated info bits. Timestamps are the
/// bucket's <see cref="BucketPlan.BucketTimestamp"/> (its start forward, its later edge reverse), except
/// Minimum/MaximumActualTime (the sample's time; ties ⇒ earliest) and Start/End. Every aggregate but
@@ -72,13 +76,24 @@ public static class AggregateEvaluator
if (r.Usable == 0) return NoData(bucketStart, now);
if (r.NonNumeric > 0) return Calc(null, HistorianStatusCodes.BadAggregateInvalidInputs);
var status = r.Good == r.Total ? HistorianStatusCodes.Good : HistorianStatusCodes.UncertainDataSubNormal;
switch (aggregate)
{
case HistoryAggregateType.Average: return Calc(r.Avg, status);
case HistoryAggregateType.Minimum: return Calc(r.Min, status);
case HistoryAggregateType.Maximum: return Calc(r.Max, status);
case HistoryAggregateType.Range: return Calc(r.Max - r.Min, status);
// #684 (Part 13 §5.4.3.5; SDK ComputeAverage): the mean of the GOOD values — an interval with none (only
// Uncertain / Bad) is BadNoData; value-based status: Good only when every sample is Good, else Uncertain.
case HistoryAggregateType.Average:
if (r.Avg is null) return NoData(bucketStart, now);
return Calc(r.Avg, r.Good == r.Total ? HistorianStatusCodes.Good : HistorianStatusCodes.UncertainDataSubNormal);
// #684 (Part 13 §5.4.3.10/11/14; SDK ComputeMinMax): the extreme(s) of the GOOD values; Uncertain when a Bad
// sample exists or an Uncertain value lies beyond the reported extreme (see MinMaxStatus).
case HistoryAggregateType.Minimum:
if (r.Min is null) return NoData(bucketStart, now);
return Calc(r.Min, MinMaxStatus(r, r.Min, r.Max, low: true, high: false));
case HistoryAggregateType.Maximum:
if (r.Max is null) return NoData(bucketStart, now);
return Calc(r.Max, MinMaxStatus(r, r.Min, r.Max, low: false, high: true));
case HistoryAggregateType.Range:
if (r.Min is null || r.Max is null) return NoData(bucketStart, now);
return Calc(r.Max - r.Min, MinMaxStatus(r, r.Min, r.Max, low: true, high: true));
case HistoryAggregateType.Delta:
// #680 (SDK ComputeDelta): last usable − first usable; Uncertain only when the interval's first or last
// sample was skipped (not usable) — a Bad sample in the middle does not degrade it.
@@ -96,12 +111,30 @@ public static class AggregateEvaluator
// Live-gate 8 F1: usable numeric samples counted but no extreme picked (an inconsistent row) is no data,
// never a value-less Good.
if (r.Extreme is null || r.ExtremeTime is null) return NoData(bucketStart, now);
return Calc(r.Extreme, status, r.ExtremeTime);
var min = aggregate == HistoryAggregateType.MinimumActualTime;
return Calc(r.Extreme, MinMaxStatus(r, r.Extreme, r.Extreme, low: min, high: !min), r.ExtremeTime);
default:
throw new HistoryAggregateNotSupportedException(aggregate);
}
}
/// <summary>
/// #684 — the Part 13 status of Minimum / Maximum / Range / *ActualTime (Tables 57–62): <c>Uncertain_DataSubNormal</c>
/// when a Bad sample exists in the interval, or when an Uncertain value is less than the minimum Good value
/// (<paramref name="low"/>) / greater than the maximum Good value (<paramref name="high"/>); else Good. An Uncertain
/// value inside the Good range leaves it Good.
/// </summary>
/// <remarks>
/// The SDK's <c>ComputeMinMax</c> (1.5.378) applies only the Bad half: it collects the Uncertain extremes and never
/// reads them, so it answers Good where Part 13 says Uncertain. Ours follows the specification there — the more
/// conservative of the two, and never a status better than the inputs warrant. The comparison is strict, so an
/// Uncertain value EQUAL to the Good extreme does not degrade it.
/// </remarks>
private static uint MinMaxStatus(BucketRow r, double? goodMin, double? goodMax, bool low, bool high) =>
r.Bad > 0 || (low && r.UncertainMin < goodMin) || (high && r.UncertainMax > goodMax)
? HistorianStatusCodes.UncertainDataSubNormal
: HistorianStatusCodes.Good;
/// <summary>
/// True for the per-sample aggregates the SDK sets the start-of-data Partial bit on (every calculator but
/// <c>AverageAggregateCalculator</c>'s plain <c>Average</c>, which turns <c>SetPartialBit</c> off).
src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common/BucketRow.cs
+23 -6
View File
@@ -9,7 +9,11 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common;
/// <remarks>
/// Sample classes: <i>usable</i> = severity Good or Uncertain AND a stored value (an Empty sample is an outage marker
/// and counts as Bad whatever its code); <i>good</i> = usable with severity Good; <i>bad</i> = not usable.
/// Every value field is computed over usable samples only, except <see cref="FirstAny"/> / <see cref="LastAny"/>.
/// <see cref="Avg"/> / <see cref="Min"/> / <see cref="Max"/> / <see cref="Sum"/> / <see cref="Extreme"/> are computed over
/// <b>Good</b> numeric samples only (#684 — Part 13 defines Average / Minimum / Maximum / Range / *ActualTime over Good
/// raw values, as the SDK's calculators do); <see cref="UncertainMin"/> / <see cref="UncertainMax"/> over the Uncertain
/// ones (the Part 13 Minimum/Maximum/Range status rule); <see cref="First"/> / <see cref="Last"/> over usable samples;
/// <see cref="FirstAny"/> / <see cref="LastAny"/> over every sample.
/// </remarks>
public sealed class BucketRow
{
@@ -31,16 +35,16 @@ public sealed class BucketRow
/// <summary>Usable String / DateTime samples.</summary>
public int NonNumeric { get; set; }
/// <summary>Average of the usable numeric values (null when none).</summary>
/// <summary>Average of the Good numeric values (null when none — #684).</summary>
public double? Avg { get; set; }
/// <summary>Minimum of the usable numeric values (null when none).</summary>
/// <summary>Minimum of the Good numeric values (null when none — #684).</summary>
public double? Min { get; set; }
/// <summary>Maximum of the usable numeric values (null when none).</summary>
/// <summary>Maximum of the Good numeric values (null when none — #684).</summary>
public double? Max { get; set; }
/// <summary>Sum of the usable numeric values (null when none).</summary>
/// <summary>Sum of the Good numeric values (null when none — #684).</summary>
public double? Sum { get; set; }
/// <summary>First usable sample's value by time (Empty when none).</summary>
@@ -49,12 +53,25 @@ public sealed class BucketRow
/// <summary>Last usable sample's value by time (Empty when none).</summary>
public HistorianValue Last { get; set; }
/// <summary>Extreme (min or max) usable numeric value (null when none).</summary>
/// <summary>Extreme (min or max) Good numeric value (null when none — #684).</summary>
public double? Extreme { get; set; }
/// <summary>Time of <see cref="Extreme"/> (ties ⇒ earliest; null when none).</summary>
public DateTime? ExtremeTime { get; set; }
/// <summary>
/// Minimum of the bucket's <b>Uncertain</b> (usable, not Good) numeric values (null when none). Not a result value:
/// Part 13 makes Minimum / MinimumActualTime / Range <c>Uncertain_DataSubNormal</c> when an Uncertain value is less
/// than the minimum Good value (#684).
/// </summary>
public double? UncertainMin { get; set; }
/// <summary>
/// Maximum of the bucket's Uncertain numeric values (null when none): Maximum / MaximumActualTime / Range are
/// Uncertain when one is greater than the maximum Good value (#684).
/// </summary>
public double? UncertainMax { get; set; }
/// <summary>
/// The bucket's first sample by time of <b>any</b> quality (null when the bucket is empty): Start returns it raw, and
/// Delta is Uncertain when it is not usable (#680, Part 13 Tables 73/74 — the SDK's <c>StartEndAggregateCalculator</c>).
src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/SqlAggregateQueries.cs
+23 -12
View File
@@ -32,7 +32,8 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer;
/// and bit 30): <i>usable</i> = severity Good or Uncertain (<c>StatusCode &gt;= 0</c>) AND a stored value
/// (<c>ValueKind &lt;&gt; 0</c>) — an <c>Empty</c> sample is an outage marker and counts as Bad whatever its
/// code; <i>good</i> = usable with severity Good (<c>StatusCode &lt; 0x40000000</c>); <i>bad</i> = not
/// usable. Every value column below is computed over usable samples only.
/// usable. Average/Minimum/Maximum/Range/*ActualTime value columns are computed over <i>good</i> samples only
/// (#684); Start/End/Delta picks over usable samples (or every sample).
/// </para>
/// Common result columns (ordinals 0–5): <c>Bucket, TotalCount, UsableCount, GoodCount, BadCount,
/// NonNumericCount</c> (usable String/DateTime samples). Family columns follow.
@@ -47,8 +48,9 @@ internal static class SqlAggregateQueries
/// <summary>
/// Average / Minimum / Maximum / Count / Range (PercentGood / PercentBad and Total are time-weighted, not here).
/// Family columns: <c>6 AvgValue, 7 MinValue, 8 MaxValue, 9 SumValue</c> (float, NULL when no usable
/// numeric sample).
/// Family columns: <c>6 AvgValue, 7 MinValue, 8 MaxValue, 9 SumValue</c> over the <b>Good</b> numeric samples (float,
/// NULL when none — #684: Part 13 and the SDK compute these over Good values only), then <c>10 UncMin, 11 UncMax</c>
/// over the Uncertain ones (the Part 13 Minimum/Maximum/Range status rule, <see cref="AggregateEvaluator"/>).
/// </summary>
/// <param name="sample">Schema-qualified <c>Sample</c> table name.</param>
/// <returns>The SQL text.</returns>
@@ -56,10 +58,11 @@ internal static class SqlAggregateQueries
public static string Simple(string sample, bool reverse = false) =>
Classified(sample, reverse) +
"SELECT b, " + CommonCounts() + ", " +
"AVG(CASE WHEN u = 1 THEN NumValue END) AS AvgValue, " +
"MIN(CASE WHEN u = 1 THEN NumValue END) AS MinValue, " +
"MAX(CASE WHEN u = 1 THEN NumValue END) AS MaxValue, " +
"SUM(CASE WHEN u = 1 THEN NumValue END) AS SumValue " +
"AVG(CASE WHEN g = 1 THEN NumValue END) AS AvgValue, " +
"MIN(CASE WHEN g = 1 THEN NumValue END) AS MinValue, " +
"MAX(CASE WHEN g = 1 THEN NumValue END) AS MaxValue, " +
"SUM(CASE WHEN g = 1 THEN NumValue END) AS SumValue, " +
UncertainExtremes() + " " +
"FROM s GROUP BY b ORDER BY b;";
/// <summary>Ordinal of the StartEnd family's first-sample-of-any-quality columns (<see cref="StartEnd"/>).</summary>
@@ -107,9 +110,9 @@ internal static class SqlAggregateQueries
}
/// <summary>
/// MinimumActualTime / MaximumActualTime: the usable sample with the smallest (largest) <c>NumValue</c>,
/// MinimumActualTime / MaximumActualTime: the <b>Good</b> sample (#684) with the smallest (largest) <c>NumValue</c>,
/// ties broken by the earliest <c>SourceTimeUtc</c>. Family columns: <c>6 NumValue, 7 SourceTimeUtc</c>
/// (NULL when the bucket has no usable numeric sample).
/// (NULL when the bucket has no Good numeric sample), then <c>8 UncMin, 9 UncMax</c> as in <see cref="Simple"/>.
/// </summary>
/// <remarks>
/// Deliberately still two references to <c>s</c> (counts + the ranked extreme), unlike <see cref="StartEnd"/>:
@@ -124,11 +127,12 @@ internal static class SqlAggregateQueries
/// <param name="reverse">True for a reverse read (window <c>(@start, @end]</c>).</param>
public static string ActualTime(string sample, bool maximum, bool reverse = false) =>
Classified(sample, reverse) +
", st AS (SELECT b, " + CommonCounts() + " FROM s GROUP BY b)" +
", st AS (SELECT b, " + CommonCounts() + ", " + UncertainExtremes() + " FROM s GROUP BY b)" +
", r AS (SELECT b, NumValue, SourceTimeUtc, " +
"ROW_NUMBER() OVER (PARTITION BY b ORDER BY NumValue " + (maximum ? "DESC" : "ASC") + ", SourceTimeUtc ASC) AS rn " +
"FROM s WHERE u = 1 AND NumValue IS NOT NULL) " +
"SELECT st.*, r.NumValue, r.SourceTimeUtc " +
"FROM s WHERE g = 1 AND NumValue IS NOT NULL) " +
"SELECT st.b, st.TotalCount, st.UsableCount, st.GoodCount, st.BadCount, st.NonNumericCount, " +
"r.NumValue, r.SourceTimeUtc, st.UncMin, st.UncMax " +
"FROM st LEFT JOIN r ON r.b = st.b AND r.rn = 1 ORDER BY st.b;";
/// <summary>
@@ -187,6 +191,13 @@ internal static class SqlAggregateQueries
$"FROM {sample} WHERE TagId = @id AND " +
(reverse ? "SourceTimeUtc > @start AND SourceTimeUtc <= @end) " : "SourceTimeUtc >= @start AND SourceTimeUtc < @end) ");
/// <summary>
/// The minimum and maximum <b>Uncertain</b> (usable, not Good) numeric value of the bucket — <c>UncMin, UncMax</c>
/// (#684). Not result values: Part 13 makes Minimum/Maximum/Range Uncertain when one lies beyond the Good extreme.
/// </summary>
private static string UncertainExtremes() =>
"MIN(CASE WHEN u = 1 AND g = 0 THEN NumValue END) AS UncMin, MAX(CASE WHEN u = 1 AND g = 0 THEN NumValue END) AS UncMax";
/// <summary>The five common count columns (after <c>b</c>).</summary>
private static string CommonCounts() =>
"COUNT(*) AS TotalCount, SUM(u) AS UsableCount, SUM(g) AS GoodCount, COUNT(*) - SUM(u) AS BadCount, " +
src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/SqlServerHistorianDataSource.cs
+4
View File
@@ -564,6 +564,8 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
row.Min = D(reader, f + 1);
row.Max = D(reader, f + 2);
row.Sum = D(reader, f + 3);
row.UncertainMin = D(reader, f + 4);
row.UncertainMax = D(reader, f + 5);
break;
case AggregateFamily.StartEnd:
row.First = V(reader, f);
@@ -574,6 +576,8 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
default:
row.Extreme = D(reader, f);
row.ExtremeTime = reader.IsDBNull(f + 1) ? null : SqlHistorianConnection.GetUtc(reader, f + 1);
row.UncertainMin = D(reader, f + 2);
row.UncertainMax = D(reader, f + 3);
break;
}
return row;