dohertj2/lmxopcua
1
0
Fork 0
Code Issues 14 Pull Requests Actions Packages Projects Releases Wiki Activity

fix(historian): #680 lane B — per-sample Count/Start/End/Delta per Part 13 + forward end-of-data Partial

Browse Source 
Count: Good samples only, value-based status (all Bad => generic Bad), 0/Good on an
empty interval inside the tag's data range [first, last] (both directions).
Start/End: the raw first/last sample of any quality (own value, status, timestamp).
Delta: Uncertain only when the interval's first or last sample is not usable.
Forward end-of-data Partial (SDK m_endOfData, earlier edge inclusive) for every
per-sample aggregate but Average (PercentGood/Bad left to lane A).
Backends push down first/last-any-quality columns and fetch the tag's data range
(earliest <= window end, latest >= window start) only when NeedsDataRange.
SDK parity masks for Start/End/Delta/Count removed; new parity datasets.
This commit is contained in:
Joseph Doherty
2026-10-11 15:52:49 -04:00
parent f6a2628656
commit dbfe2fc4de
13 changed files with 644 additions and 136 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb/QuestDbAggregateQueries.cs
+36 -12
View File
@@ -71,11 +71,17 @@ internal static class QuestDbAggregateQueries
"sum(CASE WHEN u = 1 THEN num_value END) AS sum_value " +
"FROM (" + Classified(sample, reverse) + ") GROUP BY b ORDER BY b";
/// <summary>Ordinal of the StartEnd family's first-sample-of-any-quality columns (<see cref="StartEnd"/>).</summary>
public const int FirstAnyColumn = FirstFamilyColumn + 10;
/// <summary>
/// Start / End / Delta: the first and last usable sample of each bucket by <c>ts</c>.
/// Family columns: <c>6..10</c> first sample (<c>kind, num_value, int_value, str_value, ts</c>), <c>11..15</c> last
/// sample (same order). A bucket without a usable sample has NULL <c>ts</c> but <c>kind = 0</c> (BYTE has no NULL),
/// so the reader keys absence on the NULL time.
/// Start / End / Delta: the first and last usable sample of each bucket by <c>ts</c> (Delta's value), and the first
/// and last sample of <b>any</b> quality (Start / End, and Delta's status — #680).
/// Family columns: <c>6..10</c> first usable sample (<c>kind, num_value, int_value, str_value, ts</c>), <c>11..15</c>
/// last usable sample (same order) — a bucket without a usable sample has NULL <c>ts</c> but <c>kind = 0</c> (BYTE
/// has no NULL), so the reader keys absence on the NULL time; then <c>16..21</c> the first sample of any quality
/// (<c>ts, kind, num_value, int_value, str_value, status</c>) and <c>22..27</c> the last (same order), present for
/// every returned (non-empty) bucket.
/// </summary>
/// <param name="sample">Prefixed sample table name.</param>
/// <returns>The SQL text.</returns>
@@ -85,12 +91,19 @@ internal static class QuestDbAggregateQueries
"st AS (SELECT b, " + CommonCounts() + " FROM s GROUP BY b), " +
"f AS (SELECT b, kind, num_value, int_value, str_value, ts, " +
"row_number() OVER (PARTITION BY b ORDER BY ts ASC) AS rf, " +
"row_number() OVER (PARTITION BY b ORDER BY ts DESC) AS rl FROM s WHERE u = 1) " +
"row_number() OVER (PARTITION BY b ORDER BY ts DESC) AS rl FROM s WHERE u = 1), " +
"a AS (SELECT b, ts, kind, num_value, int_value, str_value, status, " +
"row_number() OVER (PARTITION BY b ORDER BY ts ASC) AS rf, " +
"row_number() OVER (PARTITION BY b ORDER BY ts DESC) AS rl FROM s) " +
"SELECT st.*, " +
"x.kind, x.num_value, x.int_value, x.str_value, x.ts, " +
"y.kind, y.num_value, y.int_value, y.str_value, y.ts " +
"y.kind, y.num_value, y.int_value, y.str_value, y.ts, " +
"p.ts, p.kind, p.num_value, p.int_value, p.str_value, p.status, " +
"q.ts, q.kind, q.num_value, q.int_value, q.str_value, q.status " +
"FROM st LEFT JOIN (SELECT * FROM f WHERE rf = 1) x ON x.b = st.b " +
"LEFT JOIN (SELECT * FROM f WHERE rl = 1) y ON y.b = st.b ORDER BY st.b";
"LEFT JOIN (SELECT * FROM f WHERE rl = 1) y ON y.b = st.b " +
"LEFT JOIN (SELECT * FROM a WHERE rf = 1) p ON p.b = st.b " +
"LEFT JOIN (SELECT * FROM a WHERE rl = 1) q ON q.b = st.b ORDER BY st.b";
/// <summary>
/// MinimumActualTime / MaximumActualTime: the usable sample with the smallest (largest) <c>num_value</c>, ties
@@ -150,18 +163,29 @@ internal static class QuestDbAggregateQueries
internal const string BucketExpression = "(cast(ts AS LONG) * 10 - $1) / $2";
/// <summary>
/// The tag's earliest live sample (any quality) before the window end: the start-of-data Partial rule (#677; forward
/// reads only, so the forward window's exclusive end bounds the scan). Parameters <c>$1</c> tag, <c>$2</c> the window
/// end. Column: <c>ts</c>; no row when the tag has none before the end.
/// The tag's earliest live sample (any quality) at or before the window's later edge — half of the tag's data range
/// (the start-of-data Partial rule, #677, and Count's in-range empty interval, #680; see
/// <see cref="AggregateEvaluator.NeedsDataRange"/>). Parameters <c>$1</c> tag, <c>$2</c> the later edge. Column:
/// <c>ts</c>; no row when the tag has none — the data range then does not reach the window.
/// </summary>
/// <param name="sample">Prefixed sample table name.</param>
/// <returns>The SQL text.</returns>
public static string EarliestSample(string sample) =>
$"SELECT ts FROM {sample} WHERE tag = $1 AND ts < $2 AND NOT deleted ORDER BY ts ASC LIMIT 1";
$"SELECT ts FROM {sample} WHERE tag = $1 AND ts <= $2 AND NOT deleted ORDER BY ts ASC LIMIT 1";
/// <summary>
/// The tag's latest live sample (any quality) at or after the window's earlier edge — the other half of the data
/// range (the end-of-data Partial rule and Count's in-range empty interval, #680). Parameters <c>$1</c> tag,
/// <c>$2</c> the earlier edge. Column: <c>ts</c>; no row when the tag has none.
/// </summary>
/// <param name="sample">Prefixed sample table name.</param>
/// <returns>The SQL text.</returns>
public static string LatestSample(string sample) =>
$"SELECT ts FROM {sample} WHERE tag = $1 AND ts >= $2 AND NOT deleted ORDER BY ts DESC LIMIT 1";
/// <summary>The classified subquery <c>s</c>: the window's live samples with bucket index and the usable/good flags.</summary>
private static string Classified(string sample, bool reverse) =>
"SELECT " + BucketExpression + " AS b, ts, kind, num_value, int_value, str_value, " +
"SELECT " + BucketExpression + " AS b, ts, kind, num_value, int_value, str_value, status, " +
$"CASE WHEN status < {NotBadLimit} AND kind <> {EmptyKind} THEN 1 ELSE 0 END AS u, " +
$"CASE WHEN status < {GoodLimit} AND kind <> {EmptyKind} THEN 1 ELSE 0 END AS g " +
$"FROM {sample} WHERE tag = $3 AND " + (reverse ? "ts > $4 AND ts <= $5" : "ts >= $4 AND ts < $5") + " AND NOT deleted";
src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb/QuestDbHistorianDataSource.cs
+22 -11
View File
@@ -472,7 +472,7 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource
return new HistoryReadResult(twResult, ContinuationPoint: null);
}
var rows = new List<BucketRow>();
DateTime? earliest = null;
DateTime? earliest = null, latest = null;
if (plan.BucketCount > 0 && await ResolveAsync(fullReference, cancellationToken).ConfigureAwait(false))
{
var sample = _conn.SampleTable;
@@ -494,20 +494,19 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource
while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false))
rows.Add(ReadBucket(reader, family));
}
// The tag's earliest live sample before the window end: the start-of-data Partial rule (#677) — forward
// reads of an aggregate other than Average only; skipped otherwise.
if (AggregateEvaluator.NeedsEarliestSample(aggregate, plan))
// The tag's data range (earliest + latest live sample): the start/end-of-data Partial rules (#677/#680 —
// forward reads of an aggregate other than Average) and Count's in-range empty interval (#680); skipped
// otherwise.
if (AggregateEvaluator.NeedsDataRange(aggregate, plan))
{
await using var cmd = _conn.CreateCommand(conn, QuestDbAggregateQueries.EarliestSample(sample));
QuestDbConnection.BindText(cmd, fullReference);
QuestDbConnection.BindTimestamp(cmd, plan.EndUtc);
await using var reader = await cmd.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false);
if (await reader.ReadAsync(cancellationToken).ConfigureAwait(false) && !reader.IsDBNull(0))
earliest = QuestDbConnection.ReadUtc(reader, 0);
earliest = await ReadBoundAsync(conn, QuestDbAggregateQueries.EarliestSample(sample), fullReference,
plan.EndUtc, cancellationToken).ConfigureAwait(false);
latest = await ReadBoundAsync(conn, QuestDbAggregateQueries.LatestSample(sample), fullReference,
plan.StartUtc, cancellationToken).ConfigureAwait(false);
}
}
var result = AggregateEvaluator.EvaluateAll(aggregate, plan, rows, earliest, now);
var result = AggregateEvaluator.EvaluateAll(aggregate, plan, rows, earliest, latest, now);
RecordSuccess();
return new HistoryReadResult(result, ContinuationPoint: null);
@@ -565,6 +564,16 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource
if (bounds[3] is { } usableAfter) tw.Add(usableAfter);
}
/// <summary>Runs a one-column <c>ts</c> seek (<see cref="QuestDbAggregateQueries.EarliestSample"/> / <c>LatestSample</c>).</summary>
private async Task<DateTime?> ReadBoundAsync(NpgsqlConnection conn, string sql, string tag, DateTime edge, CancellationToken ct)
{
await using var cmd = _conn.CreateCommand(conn, sql);
QuestDbConnection.BindText(cmd, tag);
QuestDbConnection.BindTimestamp(cmd, edge);
await using var reader = await cmd.ExecuteReaderAsync(ct).ConfigureAwait(false);
return await reader.ReadAsync(ct).ConfigureAwait(false) && !reader.IsDBNull(0) ? QuestDbConnection.ReadUtc(reader, 0) : null;
}
private static BucketRow ReadBucket(NpgsqlDataReader reader, AggregateFamily family)
{
static double? D(NpgsqlDataReader r, int i) => r.IsDBNull(i) ? null : r.GetDouble(i);
@@ -597,6 +606,8 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource
case AggregateFamily.StartEnd:
row.First = V(reader, f);
row.Last = V(reader, f + 5);
row.FirstAny = ReadStored(reader, QuestDbAggregateQueries.FirstAnyColumn);
row.LastAny = ReadStored(reader, QuestDbAggregateQueries.FirstAnyColumn + 6);
break;
default:
row.Extreme = D(reader, f);
src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common/AggregateEvaluator.cs
+80 -28
View File
@@ -10,11 +10,18 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common;
/// Per interval: no samples ⇒ <c>BadNoData</c>. PercentGood / PercentBad ⇒ <c>100 × good|bad / all</c> over
/// every sample, status Good. Otherwise no usable sample ⇒ <c>BadNoData</c>; a numeric aggregate with a usable
/// String/DateTime sample ⇒ <c>BadAggregateInvalidInputs</c>; else the value, Good when every sample is Good,
/// <c>UncertainDataSubNormal</c> when any is Bad or Uncertain. Every status — including <c>BadNoData</c> —
/// carries the Calculated info bits. Timestamps are the bucket's <see cref="BucketPlan.BucketTimestamp"/> (its
/// start forward, its later edge reverse), except Minimum/MaximumActualTime (the sample's time; ties ⇒ earliest).
/// Every aggregate but <c>Average</c> also carries the Partial bit on the bucket the data starts inside
/// (<see cref="EvaluateAll"/>).
/// <c>UncertainDataSubNormal</c> when any is Bad or Uncertain. Every calculated status — including
/// <c>BadNoData</c> — carries the Calculated info bits. Timestamps are the bucket's
/// <see cref="BucketPlan.BucketTimestamp"/> (its start forward, its later edge reverse), except
/// Minimum/MaximumActualTime (the sample's time; ties ⇒ earliest) and Start/End. Every aggregate but <c>Average</c>
/// also carries the Partial bit on a forward read's bucket the data starts or ends inside (<see cref="EvaluateAll"/>).
/// </para>
/// <para>
/// #680 (Part 13 / the SDK's calculators): <b>Count</b> counts Good samples only, value-based status (all Good ⇒
/// Good, all Bad ⇒ the generic Bad, else Uncertain), and is 0/Good on an empty bucket inside the tag's data range;
/// <b>Start / End</b> are the raw first / last sample of any quality (own value — null when Bad —, own status
/// without the Calculated bit, own timestamp); <b>Delta</b> is Uncertain only when the bucket's first or last
/// sample is not usable.
/// </para>
/// <para>
/// Values: Count is <see cref="int"/>; Start/End keep the sample's type; every other aggregate is
@@ -44,26 +51,35 @@ public static class AggregateEvaluator
case HistoryAggregateType.PercentGood: return Calc(100d * r.Good / r.Total, HistorianStatusCodes.Good);
case HistoryAggregateType.PercentBad: return Calc(100d * r.Bad / r.Total, HistorianStatusCodes.Good);
}
switch (aggregate)
{
// #680 (Part 13 Tables 73/74; SDK ComputeStartEnd): the raw first / last sample of ANY quality — its own value
// (null when Bad), its own status (Raw: no Calculated bit) and its own timestamp.
case HistoryAggregateType.Start: return Raw(r.FirstAny, bucketStart, now);
case HistoryAggregateType.End: return Raw(r.LastAny, bucketStart, now);
// #680 (Part 13 §5.4.3.21; SDK ComputeCount): Good samples only; value-based status — Good when every sample
// is Good, the generic Bad when every one is Bad (value null), else Uncertain.
case HistoryAggregateType.Count:
if (r.Bad == r.Total) return Calc(null, HistorianStatusCodes.Bad);
return Calc(r.Good, r.Good == r.Total ? HistorianStatusCodes.Good : HistorianStatusCodes.UncertainDataSubNormal);
}
if (r.Usable == 0) return NoData(bucketStart, now);
var anyText = aggregate is HistoryAggregateType.Count or HistoryAggregateType.Start or HistoryAggregateType.End;
if (!anyText && r.NonNumeric > 0)
return Calc(null, HistorianStatusCodes.BadAggregateInvalidInputs);
if (r.NonNumeric > 0) return Calc(null, HistorianStatusCodes.BadAggregateInvalidInputs);
var status = r.Good == r.Total ? HistorianStatusCodes.Good : HistorianStatusCodes.UncertainDataSubNormal;
switch (aggregate)
{
case HistoryAggregateType.Count: return Calc(r.Usable, status);
case HistoryAggregateType.Average: return Calc(r.Avg, status);
case HistoryAggregateType.Minimum: return Calc(r.Min, status);
case HistoryAggregateType.Maximum: return Calc(r.Max, status);
case HistoryAggregateType.Range: return Calc(r.Max - r.Min, status);
case HistoryAggregateType.Start: return Calc(r.First.ToObject(), status);
case HistoryAggregateType.End: return Calc(r.Last.ToObject(), status);
case HistoryAggregateType.Delta:
return r.First.TryGetDouble(out var a) && r.Last.TryGetDouble(out var b)
? Calc(b - a, status)
: Calc(null, HistorianStatusCodes.BadAggregateInvalidInputs);
// #680 (SDK ComputeDelta): last usable − first usable; Uncertain only when the interval's first or last
// sample was skipped (not usable) — a Bad sample in the middle does not degrade it.
if (!r.First.TryGetDouble(out var a) || !r.Last.TryGetDouble(out var b))
return Calc(null, HistorianStatusCodes.BadAggregateInvalidInputs);
var bounded = r.FirstAny is { IsUsable: true } && r.LastAny is { IsUsable: true };
return Calc(b - a, bounded ? HistorianStatusCodes.Good : HistorianStatusCodes.UncertainDataSubNormal);
case HistoryAggregateType.MinimumActualTime:
case HistoryAggregateType.MaximumActualTime:
return Calc(r.Extreme, status, r.ExtremeTime);
@@ -81,49 +97,85 @@ public static class AggregateEvaluator
public static bool SetsStartOfDataPartial(HistoryAggregateType aggregate) => aggregate != HistoryAggregateType.Average;
/// <summary>
/// True when a per-sample read must fetch the tag's earliest sample: the start-of-data Partial rule applies only to a
/// <b>forward</b> read (as in the SDK, whose reverse read keys <c>m_startOfData</c> on the first value it is fed — the
/// latest) of an aggregate other than <c>Average</c>. Otherwise the backend skips that query.
/// True for the per-sample aggregates the forward end-of-data Partial rule (#680) applies to: the same set as
/// <see cref="SetsStartOfDataPartial"/> less PercentGood / PercentBad, which #680 moves to the time-weighted family
/// (their per-duration definition) — not changed here.
/// </summary>
/// <param name="aggregate">Requested aggregate.</param>
/// <returns>Whether the end-of-data Partial rule applies.</returns>
public static bool SetsEndOfDataPartial(HistoryAggregateType aggregate) =>
SetsStartOfDataPartial(aggregate)
&& aggregate is not (HistoryAggregateType.PercentGood or HistoryAggregateType.PercentBad);
/// <summary>
/// True when a per-sample read must fetch the tag's data range — its earliest and latest stored samples of any
/// quality — because the answer can change a result: the start- and end-of-data Partial rules on a <b>forward</b>
/// read of an aggregate other than <c>Average</c> (as in the SDK, which applies both to forward reads only), and
/// Count in either direction (an empty interval inside the range is 0, outside it <c>BadNoData</c>, #680).
/// Otherwise the backend skips both queries.
/// </summary>
/// <param name="aggregate">Requested aggregate.</param>
/// <param name="plan">The bucket plan.</param>
/// <returns>Whether the earliest sample is needed.</returns>
public static bool NeedsEarliestSample(HistoryAggregateType aggregate, BucketPlan plan)
/// <returns>Whether the earliest and latest samples are needed.</returns>
public static bool NeedsDataRange(HistoryAggregateType aggregate, BucketPlan plan)
{
ArgumentNullException.ThrowIfNull(plan);
return !plan.Reversed && plan.BucketCount > 0 && SetsStartOfDataPartial(aggregate);
return plan.BucketCount > 0
&& (aggregate == HistoryAggregateType.Count || (!plan.Reversed && SetsStartOfDataPartial(aggregate)));
}
/// <summary>
/// Evaluates every non-empty bucket row and assembles one result per interval in the plan's order (descending for a
/// reverse read): each value stamped with <see cref="BucketPlan.BucketTimestamp"/> (the earlier edge forward, the
/// later edge reverse); empty buckets <c>BadNoData</c>; and — on a forward read, for every aggregate but
/// <c>Average</c> — the Partial bit on a non-Bad result of the bucket the tag's data starts inside (#677, the SDK's
/// <c>m_startOfData</c> rule; the SDK applies it to forward reads only, <see cref="NeedsEarliestSample"/>).
/// later edge reverse) — Start/End/ActualTime with their sample's own time; empty buckets <c>BadNoData</c>, except a
/// Count inside the tag's data range (0, Good — #680); and, on a forward read, for every aggregate but
/// <c>Average</c>, the Partial bit on a non-Bad result of the bucket the tag's data starts strictly inside (#677, the
/// SDK's <c>m_startOfData</c> rule) or ends inside (#680, <c>m_endOfData</c>, the bucket's earlier edge included).
/// A Bad result never carries Partial, and a reverse read none at all (<see cref="NeedsDataRange"/>).
/// </summary>
/// <param name="aggregate">A per-sample (Simple / StartEnd / ActualTime) aggregate.</param>
/// <param name="plan">The bucket plan.</param>
/// <param name="rows">The backend's non-empty bucket rows (ascending bucket index).</param>
/// <param name="earliestSampleUtc">The tag's earliest stored sample of any quality, or null when none.</param>
/// <param name="latestSampleUtc">The tag's latest stored sample of any quality, or null when none.</param>
/// <param name="now">Server timestamp.</param>
/// <returns><see cref="BucketPlan.BucketCount"/> results.</returns>
public static IReadOnlyList<DataValueSnapshot> EvaluateAll(HistoryAggregateType aggregate, BucketPlan plan,
IEnumerable<BucketRow> rows, DateTime? earliestSampleUtc, DateTime now)
IEnumerable<BucketRow> rows, DateTime? earliestSampleUtc, DateTime? latestSampleUtc, DateTime now)
{
ArgumentNullException.ThrowIfNull(plan);
ArgumentNullException.ThrowIfNull(rows);
var evaluated = new Dictionary<long, DataValueSnapshot>();
var partialRule = NeedsEarliestSample(aggregate, plan);
var startRule = !plan.Reversed && SetsStartOfDataPartial(aggregate);
var endRule = !plan.Reversed && SetsEndOfDataPartial(aggregate);
foreach (var row in rows)
{
var v = Evaluate(aggregate, row, plan.BucketTimestamp(row.Bucket), now);
if (partialRule && !StatusBits.IsBad(v.StatusCode) && plan.DataStartsInside(row.Bucket, earliestSampleUtc))
v = v with { StatusCode = v.StatusCode | HistorianInfoBits.HistorianPartial };
if (!StatusBits.IsBad(v.StatusCode)
&& ((startRule && plan.DataStartsInside(row.Bucket, earliestSampleUtc))
|| (endRule && plan.DataEndsInside(row.Bucket, latestSampleUtc))))
// A raw Start/End sample gains the DataValue InfoType with its first aggregate bit (as the SDK's
// SetAggregateBits does); a calculated status already carries it.
v = v with { StatusCode = v.StatusCode | HistorianInfoBits.InfoTypeDataValue | HistorianInfoBits.HistorianPartial };
evaluated[row.Bucket] = v;
}
if (aggregate == HistoryAggregateType.Count)
{
// #680 (Part 13 §5.4.3.21; SDK GetValues): an empty interval inside [earliest, latest] is a Count of 0, Good.
for (long i = 0; i < plan.BucketCount; i++)
if (!evaluated.ContainsKey(i) && plan.WithinDataRange(i, earliestSampleUtc, latestSampleUtc))
evaluated[i] = new DataValueSnapshot(0, HistorianInfoBits.WithCalculated(HistorianStatusCodes.Good),
plan.BucketTimestamp(i), now);
}
return plan.Assemble(evaluated, t => NoData(t, now));
}
/// <summary>A raw stored sample as an aggregate result (Start / End): value null when Bad, its own status and time.</summary>
private static DataValueSnapshot Raw(StoredSample? sample, DateTime bucketStart, DateTime now) =>
sample is not { } s
? NoData(bucketStart, now)
: new(StatusBits.IsBad(s.Status) ? null : s.Value.ToObject(), s.Status, s.TimeUtc, now);
/// <summary>The result for a bucket with no samples (or no usable sample): <c>BadNoData</c> + Calculated.</summary>
/// <param name="t">The bucket's start time.</param>
/// <param name="now">Server timestamp.</param>
src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common/BucketPlan.cs
+24
View File
@@ -155,6 +155,30 @@ public sealed class BucketPlan
public bool DataStartsInside(long index, DateTime? earliestSampleUtc) =>
earliestSampleUtc is { } e && e.Ticks > LowerTicks(index) && e.Ticks < UpperTicks(index);
/// <summary>
/// True when the latest stored sample of the tag lies in bucket <paramref name="index"/> (its earlier edge
/// included): the data ends part-way through the interval, so Part 13 §5.3.3.2 / the SDK set the Partial bit on a
/// forward read (#680, the SDK's <c>m_endOfData &gt;= startTime &amp;&amp; m_endOfData &lt; endTime</c>).
/// </summary>
/// <param name="index">Zero-based ascending bucket index.</param>
/// <param name="latestSampleUtc">The tag's latest stored sample (any quality), or null when it has none.</param>
/// <returns>Whether the data ends inside the bucket.</returns>
public bool DataEndsInside(long index, DateTime? latestSampleUtc) =>
latestSampleUtc is { } l && l.Ticks >= LowerTicks(index) && l.Ticks < UpperTicks(index);
/// <summary>
/// True when bucket <paramref name="index"/> overlaps the tag's data range <c>[earliest, latest]</c> — neither
/// wholly before the first sample nor wholly after the last (the SDK's <c>GetValues</c> null test:
/// <c>start &gt; last || end &lt; first</c> ⇒ no data, both edges inclusive in either direction). An empty bucket
/// inside the range is a Count of 0, not <c>BadNoData</c> (#680, Part 13 §5.4.3.21).
/// </summary>
/// <param name="index">Zero-based ascending bucket index.</param>
/// <param name="earliestSampleUtc">The tag's earliest stored sample (any quality), or null.</param>
/// <param name="latestSampleUtc">The tag's latest stored sample (any quality), or null.</param>
/// <returns>Whether the bucket is within the data range.</returns>
public bool WithinDataRange(long index, DateTime? earliestSampleUtc, DateTime? latestSampleUtc) =>
earliestSampleUtc is { } e && latestSampleUtc is { } l && LowerTicks(index) <= l.Ticks && UpperTicks(index) >= e.Ticks;
/// <summary>One snapshot per bucket in result order; missing buckets get <paramref name="empty"/>; descending if reversed.</summary>
/// <param name="byBucket">Evaluated snapshots of the non-empty buckets, by ascending bucket index.</param>
/// <param name="empty">Builds the snapshot of a bucket with no row, from its <see cref="BucketTimestamp"/>.</param>
src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common/BucketRow.cs
+10 -1
View File
@@ -9,7 +9,7 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common;
/// <remarks>
/// Sample classes: <i>usable</i> = severity Good or Uncertain AND a stored value (an Empty sample is an outage marker
/// and counts as Bad whatever its code); <i>good</i> = usable with severity Good; <i>bad</i> = not usable.
/// Every value field is computed over usable samples only.
/// Every value field is computed over usable samples only, except <see cref="FirstAny"/> / <see cref="LastAny"/>.
/// </remarks>
public sealed class BucketRow
{
@@ -54,4 +54,13 @@ public sealed class BucketRow
/// <summary>Time of <see cref="Extreme"/> (ties ⇒ earliest; null when none).</summary>
public DateTime? ExtremeTime { get; set; }
/// <summary>
/// The bucket's first sample by time of <b>any</b> quality (null when the bucket is empty): Start returns it raw, and
/// Delta is Uncertain when it is not usable (#680, Part 13 Tables 73/74 — the SDK's <c>StartEndAggregateCalculator</c>).
/// </summary>
public StoredSample? FirstAny { get; set; }
/// <summary>The bucket's last sample by time of any quality (null when the bucket is empty): End, Delta (#680).</summary>
public StoredSample? LastAny { get; set; }
}
src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/SqlAggregateQueries.cs
+29 -12
View File
@@ -62,10 +62,16 @@ internal static class SqlAggregateQueries
"SUM(CASE WHEN u = 1 THEN NumValue END) AS SumValue " +
"FROM s GROUP BY b ORDER BY b;";
/// <summary>Ordinal of the StartEnd family's first-sample-of-any-quality columns (<see cref="StartEnd"/>).</summary>
public const int FirstAnyColumn = FirstFamilyColumn + 10;
/// <summary>
/// Start / End / Delta: the first and last usable sample of each bucket by <c>SourceTimeUtc</c>.
/// Family columns: <c>6..10</c> first sample (<c>ValueKind, NumValue, IntValue, StrValue, SourceTimeUtc</c>),
/// <c>11..15</c> last sample (same order); all NULL when the bucket has no usable sample.
/// Start / End / Delta: the first and last usable sample of each bucket by <c>SourceTimeUtc</c> (Delta's value), and
/// the first and last sample of <b>any</b> quality (Start / End, and Delta's status — #680).
/// Family columns: <c>6..10</c> first usable sample (<c>ValueKind, NumValue, IntValue, StrValue, SourceTimeUtc</c>),
/// <c>11..15</c> last usable sample (same order) — all NULL when the bucket has no usable sample; then
/// <c>16..21</c> the first sample of any quality (<c>SourceTimeUtc, ValueKind, NumValue, IntValue, StrValue,
/// StatusCode</c>) and <c>22..27</c> the last (same order) — never NULL for a returned (non-empty) bucket.
/// </summary>
/// <param name="sample">Schema-qualified <c>Sample</c> table name.</param>
/// <returns>The SQL text.</returns>
@@ -75,11 +81,17 @@ internal static class SqlAggregateQueries
", st AS (SELECT b, " + CommonCounts() + " FROM s GROUP BY b)" +
", f AS (SELECT b, ValueKind, NumValue, IntValue, StrValue, SourceTimeUtc, " +
"ROW_NUMBER() OVER (PARTITION BY b ORDER BY SourceTimeUtc ASC) AS rf, " +
"ROW_NUMBER() OVER (PARTITION BY b ORDER BY SourceTimeUtc DESC) AS rl FROM s WHERE u = 1) " +
"ROW_NUMBER() OVER (PARTITION BY b ORDER BY SourceTimeUtc DESC) AS rl FROM s WHERE u = 1)" +
", a AS (SELECT b, SourceTimeUtc, ValueKind, NumValue, IntValue, StrValue, StatusCode, " +
"ROW_NUMBER() OVER (PARTITION BY b ORDER BY SourceTimeUtc ASC) AS rf, " +
"ROW_NUMBER() OVER (PARTITION BY b ORDER BY SourceTimeUtc DESC) AS rl FROM s) " +
"SELECT st.*, " +
"x.ValueKind, x.NumValue, x.IntValue, x.StrValue, x.SourceTimeUtc, " +
"y.ValueKind, y.NumValue, y.IntValue, y.StrValue, y.SourceTimeUtc " +
"FROM st LEFT JOIN f x ON x.b = st.b AND x.rf = 1 LEFT JOIN f y ON y.b = st.b AND y.rl = 1 ORDER BY st.b;";
"y.ValueKind, y.NumValue, y.IntValue, y.StrValue, y.SourceTimeUtc, " +
"p.SourceTimeUtc, p.ValueKind, p.NumValue, p.IntValue, p.StrValue, p.StatusCode, " +
"q.SourceTimeUtc, q.ValueKind, q.NumValue, q.IntValue, q.StrValue, q.StatusCode " +
"FROM st LEFT JOIN f x ON x.b = st.b AND x.rf = 1 LEFT JOIN f y ON y.b = st.b AND y.rl = 1 " +
"JOIN a p ON p.b = st.b AND p.rf = 1 JOIN a q ON q.b = st.b AND q.rl = 1 ORDER BY st.b;";
/// <summary>
/// MinimumActualTime / MaximumActualTime: the usable sample with the smallest (largest) <c>NumValue</c>,
@@ -130,21 +142,26 @@ internal static class SqlAggregateQueries
}
/// <summary>
/// The earliest stored sample of the tag (any quality), one row (<c>NULL</c> when none): the start-of-data Partial
/// rule (#677; forward reads only). A <c>MIN</c> over the <c>(TagId, SourceTimeUtc)</c> key — a single seek — bounded
/// by the window end (a sample at or past it cannot lie inside an interval).
/// The tag's data range as one row of two columns (each <c>NULL</c> when none): <c>0</c> the earliest stored sample
/// of any quality at or before the window's later edge, <c>1</c> the latest at or after its earlier edge. Used by the
/// start-of-data (#677) and end-of-data (#680) Partial rules and by Count's in-range empty interval (#680) — only when
/// <see cref="AggregateEvaluator.NeedsDataRange"/> says the answer can change a result. Each is a <c>MIN</c> /
/// <c>MAX</c> over the <c>(TagId, SourceTimeUtc)</c> key — a single seek — bounded by the window (a sample past the
/// later edge cannot start the data inside it, nor one before the earlier edge end it there; a NULL then means "the
/// data range does not reach the window").
/// </summary>
/// <param name="sample">Schema-qualified <c>Sample</c> table name.</param>
/// <returns>The SQL text.</returns>
public static string EarliestSample(string sample) =>
$"SELECT MIN(SourceTimeUtc) FROM {sample} WHERE TagId = @id AND SourceTimeUtc < @end;";
public static string DataRange(string sample) =>
$"SELECT (SELECT MIN(SourceTimeUtc) FROM {sample} WHERE TagId = @id AND SourceTimeUtc <= @end), " +
$"(SELECT MAX(SourceTimeUtc) FROM {sample} WHERE TagId = @id AND SourceTimeUtc >= @start);";
/// <summary>The <c>s</c> CTE: the window's samples with bucket index and the usable/good flags.</summary>
private static string Classified(string sample, bool reverse) =>
"WITH s AS (SELECT (DATEDIFF_BIG(microsecond, @start, SourceTimeUtc) * 10 " +
"+ (DATEPART(nanosecond, SourceTimeUtc) / 100) % 10 - (DATEPART(nanosecond, @start) / 100) % 10 " +
"+ @bucketAdjustTicks) / @intervalTicks AS b, " +
"SourceTimeUtc, ValueKind, NumValue, IntValue, StrValue, " +
"SourceTimeUtc, ValueKind, NumValue, IntValue, StrValue, StatusCode, " +
"CASE WHEN StatusCode >= 0 AND ValueKind <> 0 THEN 1 ELSE 0 END AS u, " +
"CASE WHEN StatusCode >= 0 AND StatusCode < 1073741824 AND ValueKind <> 0 THEN 1 ELSE 0 END AS g " +
$"FROM {sample} WHERE TagId = @id AND " +
src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/SqlServerHistorianDataSource.cs
+19 -11
View File
@@ -410,8 +410,11 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
/// String/DateTime sample ⇒ <c>BadAggregateInvalidInputs</c>; else the value, Good when every sample is Good,
/// <c>UncertainDataSubNormal</c> when any is Bad or Uncertain. Every status — including <c>BadNoData</c> —
/// carries the Calculated info bits. Timestamps are the bucket start (its later edge on a reverse read), except
/// Minimum/MaximumActualTime (the sample's time; ties ⇒ earliest). Every aggregate but Average sets the Partial
/// bit on the bucket the tag's earliest sample lies strictly inside (#677).
/// Minimum/MaximumActualTime (the sample's time; ties ⇒ earliest). On a forward read every aggregate but Average
/// sets the Partial bit on the bucket the tag's earliest sample lies strictly inside (#677) or its latest sample
/// lies in (#680). #680: Count counts Good samples (value-based status; 0/Good on an empty bucket inside the tag's
/// data range), Start/End are the raw first/last sample of any quality, Delta is Uncertain only when the bucket's
/// first or last sample is not usable (see <see cref="AggregateEvaluator"/>).
/// </para>
/// <para>
/// Values: Count is <see cref="int"/>; Start/End keep the sample's type; every other aggregate is
@@ -444,7 +447,7 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
return new HistoryReadResult(twResult, ContinuationPoint: null);
}
var rows = new List<BucketRow>();
DateTime? earliest = null;
DateTime? earliest = null, latest = null;
if (tagId is { } id)
{
var sample = _conn.Q("Sample");
@@ -454,10 +457,10 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
AggregateFamily.StartEnd => SqlAggregateQueries.StartEnd(sample, plan.Reversed),
_ => SqlAggregateQueries.ActualTime(sample, aggregate == HistoryAggregateType.MaximumActualTime, plan.Reversed),
};
// Second result set: the tag's earliest sample (the start-of-data Partial rule, #677) — forward reads of an
// aggregate other than Average only.
var needsEarliest = AggregateEvaluator.NeedsEarliestSample(aggregate, plan);
if (needsEarliest) sql += " " + SqlAggregateQueries.EarliestSample(sample);
// Second result set: the tag's data range (earliest + latest sample) — the start/end-of-data Partial rules
// (#677/#680, forward reads of an aggregate other than Average) and Count's in-range empty interval (#680).
var needsRange = AggregateEvaluator.NeedsDataRange(aggregate, plan);
if (needsRange) sql += " " + SqlAggregateQueries.DataRange(sample);
await using var conn = await _conn.OpenAsync(cancellationToken).ConfigureAwait(false);
await using var cmd = _conn.CreateCommand(conn, sql);
cmd.Parameters.Add(new SqlParameter("@id", SqlDbType.Int) { Value = id });
@@ -468,12 +471,15 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
await using var reader = await cmd.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false);
while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false))
rows.Add(ReadBucket(reader, family));
if (needsEarliest && await reader.NextResultAsync(cancellationToken).ConfigureAwait(false)
&& await reader.ReadAsync(cancellationToken).ConfigureAwait(false) && !reader.IsDBNull(0))
earliest = DateTime.SpecifyKind(reader.GetDateTime(0), DateTimeKind.Utc);
if (needsRange && await reader.NextResultAsync(cancellationToken).ConfigureAwait(false)
&& await reader.ReadAsync(cancellationToken).ConfigureAwait(false))
{
if (!reader.IsDBNull(0)) earliest = SqlHistorianConnection.GetUtc(reader, 0);
if (!reader.IsDBNull(1)) latest = SqlHistorianConnection.GetUtc(reader, 1);
}
}
var result = AggregateEvaluator.EvaluateAll(aggregate, plan, rows, earliest, now);
var result = AggregateEvaluator.EvaluateAll(aggregate, plan, rows, earliest, latest, now);
RecordSuccess();
return new HistoryReadResult(result, ContinuationPoint: null);
@@ -547,6 +553,8 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
case AggregateFamily.StartEnd:
row.First = V(reader, f);
row.Last = V(reader, f + 5);
row.FirstAny = ReadStored(reader, SqlAggregateQueries.FirstAnyColumn);
row.LastAny = ReadStored(reader, SqlAggregateQueries.FirstAnyColumn + 6);
break;
default:
row.Extreme = D(reader, f);