dohertj2/lmxopcua
1
0
Fork 0
Code Issues 32 Pull Requests Actions Packages Projects Releases Wiki Activity

Merge feat/677-aggregates: reverse processed reads SDK-anchored, start-of-data Partial (forward), AggregateConfiguration accept-defaults, IntegralDivisor=1, SDK parity tests (#677)

Browse Source
This commit is contained in:
Joseph Doherty
2026-10-11 11:47:52 -04:00
parent 50c09e21d5 7349260231
commit 945ec2c069
24 changed files with 1291 additions and 135 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/SqlAggregateQueries.cs
+33 -13
View File
@@ -5,13 +5,18 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer;
/// <summary>
/// Push-down T-SQL for processed (aggregate) reads (spec §4 "Processed"), one query per
/// <see cref="AggregateFamily"/> (<see cref="AggregateFamilies.FamilyOf"/> picks it). Every query takes
/// <c>@id</c> (int TagId), <c>@start</c>/<c>@end</c> (<c>datetime2(7)</c>, the forward window
/// <c>[start, end)</c>) and <c>@intervalTicks</c> (bigint bucket width in 100 ns ticks), and returns one row
/// per <b>non-empty</b> bucket. The caller generates every interval and fills the empty ones.
/// <c>@id</c> (int TagId), <c>@start</c>/<c>@end</c> (<c>datetime2(7)</c>, the earlier / later window edges —
/// <see cref="BucketPlan.StartUtc"/>/<see cref="BucketPlan.EndUtc"/>), <c>@intervalTicks</c> (bigint bucket width in
/// 100 ns ticks) and <c>@bucketAdjustTicks</c> (bigint, <see cref="BucketPlan.BucketIndexAdjustTicks"/>), and returns
/// one row per <b>non-empty</b> bucket (ascending bucket index). The caller generates every interval and fills the
/// empty ones.
/// </summary>
/// <remarks>
/// <para>
/// <b>Bucketing.</b> <c>b = elapsedTicks / @intervalTicks</c>, where
/// <b>Bucketing.</b> <c>b = (elapsedTicks + @bucketAdjustTicks) / @intervalTicks</c> (#677: the adjust is 0 for a
/// forward read and re-anchors a reverse read's grid at its later time — <see cref="BucketPlan.IndexOf"/> is the C#
/// statement of the same math). The window is <c>[@start, @end)</c> forward and <c>(@start, @end]</c> reverse (a
/// reverse read includes its later time), selected by the <c>reverse</c> argument. <c>elapsedTicks</c> is
/// <c>elapsedTicks = DATEDIFF_BIG(microsecond, @start, t) * 10 + tick-of-microsecond(t) − tick-of-microsecond(@start)</c>
/// and <c>tick-of-microsecond(x) = (DATEPART(nanosecond, x) / 100) % 10</c>. The spec sketched
/// <c>millisecond</c>, but <c>DATEDIFF</c> counts unit <i>boundaries crossed</i>, not elapsed time, so with a
@@ -47,8 +52,9 @@ internal static class SqlAggregateQueries
/// </summary>
/// <param name="sample">Schema-qualified <c>Sample</c> table name.</param>
/// <returns>The SQL text.</returns>
public static string Simple(string sample) =>
Classified(sample) +
/// <param name="reverse">True for a reverse read (window <c>(@start, @end]</c>).</param>
public static string Simple(string sample, bool reverse = false) =>
Classified(sample, reverse) +
"SELECT b, " + CommonCounts() + ", " +
"AVG(CASE WHEN u = 1 THEN NumValue END) AS AvgValue, " +
"MIN(CASE WHEN u = 1 THEN NumValue END) AS MinValue, " +
@@ -63,8 +69,9 @@ internal static class SqlAggregateQueries
/// </summary>
/// <param name="sample">Schema-qualified <c>Sample</c> table name.</param>
/// <returns>The SQL text.</returns>
public static string StartEnd(string sample) =>
Classified(sample) +
/// <param name="reverse">True for a reverse read (window <c>(@start, @end]</c>).</param>
public static string StartEnd(string sample, bool reverse = false) =>
Classified(sample, reverse) +
", st AS (SELECT b, " + CommonCounts() + " FROM s GROUP BY b)" +
", f AS (SELECT b, ValueKind, NumValue, IntValue, StrValue, SourceTimeUtc, " +
"ROW_NUMBER() OVER (PARTITION BY b ORDER BY SourceTimeUtc ASC) AS rf, " +
@@ -82,8 +89,9 @@ internal static class SqlAggregateQueries
/// <param name="sample">Schema-qualified <c>Sample</c> table name.</param>
/// <param name="maximum">True for MaximumActualTime.</param>
/// <returns>The SQL text.</returns>
public static string ActualTime(string sample, bool maximum) =>
Classified(sample) +
/// <param name="reverse">True for a reverse read (window <c>(@start, @end]</c>).</param>
public static string ActualTime(string sample, bool maximum, bool reverse = false) =>
Classified(sample, reverse) +
", st AS (SELECT b, " + CommonCounts() + " FROM s GROUP BY b)" +
", r AS (SELECT b, NumValue, SourceTimeUtc, " +
"ROW_NUMBER() OVER (PARTITION BY b ORDER BY NumValue " + (maximum ? "DESC" : "ASC") + ", SourceTimeUtc ASC) AS rn " +
@@ -121,14 +129,26 @@ internal static class SqlAggregateQueries
$"SELECT TOP 1 {cols} {key}SourceTimeUtc >= @end{usable} ORDER BY SourceTimeUtc ASC;";
}
/// <summary>
/// The earliest stored sample of the tag (any quality), one row (<c>NULL</c> when none): the start-of-data Partial
/// rule (#677; forward reads only). A <c>MIN</c> over the <c>(TagId, SourceTimeUtc)</c> key — a single seek — bounded
/// by the window end (a sample at or past it cannot lie inside an interval).
/// </summary>
/// <param name="sample">Schema-qualified <c>Sample</c> table name.</param>
/// <returns>The SQL text.</returns>
public static string EarliestSample(string sample) =>
$"SELECT MIN(SourceTimeUtc) FROM {sample} WHERE TagId = @id AND SourceTimeUtc < @end;";
/// <summary>The <c>s</c> CTE: the window's samples with bucket index and the usable/good flags.</summary>
private static string Classified(string sample) =>
private static string Classified(string sample, bool reverse) =>
"WITH s AS (SELECT (DATEDIFF_BIG(microsecond, @start, SourceTimeUtc) * 10 " +
"+ (DATEPART(nanosecond, SourceTimeUtc) / 100) % 10 - (DATEPART(nanosecond, @start) / 100) % 10) / @intervalTicks AS b, " +
"+ (DATEPART(nanosecond, SourceTimeUtc) / 100) % 10 - (DATEPART(nanosecond, @start) / 100) % 10 " +
"+ @bucketAdjustTicks) / @intervalTicks AS b, " +
"SourceTimeUtc, ValueKind, NumValue, IntValue, StrValue, " +
"CASE WHEN StatusCode >= 0 AND ValueKind <> 0 THEN 1 ELSE 0 END AS u, " +
"CASE WHEN StatusCode >= 0 AND StatusCode < 1073741824 AND ValueKind <> 0 THEN 1 ELSE 0 END AS g " +
$"FROM {sample} WHERE TagId = @id AND SourceTimeUtc >= @start AND SourceTimeUtc < @end) ";
$"FROM {sample} WHERE TagId = @id AND " +
(reverse ? "SourceTimeUtc > @start AND SourceTimeUtc <= @end) " : "SourceTimeUtc >= @start AND SourceTimeUtc < @end) ");
/// <summary>The five common count columns (after <c>b</c>).</summary>
private static string CommonCounts() =>
src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/SqlServerHistorianDataSource.cs
+21 -13
View File
@@ -400,16 +400,18 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
/// One result per interval of <c>[start, end)</c>, bucket <c>i</c> = <c>[start + i·interval,
/// start + (i+1)·interval)</c> (start-inclusive, end-exclusive to the tick; the last bucket is truncated at
/// <c>end</c>). <paramref name="interval"/> &lt;= 0 ⇒ one interval <c>[start, end)</c>. <c>start == end</c>
/// ⇒ no intervals. <c>end &lt; start</c> (reverse) ⇒ the window is swapped, computed forward, and returned
/// in descending bucket order (buckets stay anchored at the earlier time).
/// ⇒ no intervals. <c>end &lt; start</c> (reverse, #677) ⇒ Part 13 / SDK anchoring: window <c>(end, start]</c>,
/// buckets stepping back from <c>start</c> (the partial one is the earliest), each stamped with its later edge,
/// returned latest first (<see cref="BucketPlan"/>).
/// </para>
/// <para>
/// Per interval: no samples ⇒ <c>BadNoData</c>. PercentGood / PercentBad ⇒ <c>100 × good|bad / all</c> over
/// every sample, status Good. Otherwise no usable sample ⇒ <c>BadNoData</c>; a numeric aggregate with a usable
/// String/DateTime sample ⇒ <c>BadAggregateInvalidInputs</c>; else the value, Good when every sample is Good,
/// <c>UncertainDataSubNormal</c> when any is Bad or Uncertain. Every status — including <c>BadNoData</c> —
/// carries the Calculated info bits. Timestamps are the bucket start, except Minimum/MaximumActualTime
/// (the sample's time; ties ⇒ earliest).
/// carries the Calculated info bits. Timestamps are the bucket start (its later edge on a reverse read), except
/// Minimum/MaximumActualTime (the sample's time; ties ⇒ earliest). Every aggregate but Average sets the Partial
/// bit on the bucket the tag's earliest sample lies strictly inside (#677).
/// </para>
/// <para>
/// Values: Count is <see cref="int"/>; Start/End keep the sample's type; every other aggregate is
@@ -432,7 +434,6 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
try
{
var now = _time.GetUtcNow().UtcDateTime;
var evaluated = new Dictionary<long, DataValueSnapshot>();
var tagId = plan.BucketCount == 0 ? null : await ResolveAsync(fullReference, cancellationToken).ConfigureAwait(false);
if (family == AggregateFamily.TimeWeighted)
{
@@ -442,30 +443,37 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
RecordSuccess();
return new HistoryReadResult(twResult, ContinuationPoint: null);
}
var rows = new List<BucketRow>();
DateTime? earliest = null;
if (tagId is { } id)
{
var sample = _conn.Q("Sample");
var sql = family switch
{
AggregateFamily.Simple => SqlAggregateQueries.Simple(sample),
AggregateFamily.StartEnd => SqlAggregateQueries.StartEnd(sample),
_ => SqlAggregateQueries.ActualTime(sample, aggregate == HistoryAggregateType.MaximumActualTime),
AggregateFamily.Simple => SqlAggregateQueries.Simple(sample, plan.Reversed),
AggregateFamily.StartEnd => SqlAggregateQueries.StartEnd(sample, plan.Reversed),
_ => SqlAggregateQueries.ActualTime(sample, aggregate == HistoryAggregateType.MaximumActualTime, plan.Reversed),
};
// Second result set: the tag's earliest sample (the start-of-data Partial rule, #677) — forward reads of an
// aggregate other than Average only.
var needsEarliest = AggregateEvaluator.NeedsEarliestSample(aggregate, plan);
if (needsEarliest) sql += " " + SqlAggregateQueries.EarliestSample(sample);
await using var conn = await _conn.OpenAsync(cancellationToken).ConfigureAwait(false);
await using var cmd = _conn.CreateCommand(conn, sql);
cmd.Parameters.Add(new SqlParameter("@id", SqlDbType.Int) { Value = id });
SqlHistorianConnection.AddDateTime2(cmd, "@start", DateTime.SpecifyKind(plan.StartUtc, DateTimeKind.Utc));
SqlHistorianConnection.AddDateTime2(cmd, "@end", DateTime.SpecifyKind(plan.EndUtc, DateTimeKind.Utc));
cmd.Parameters.Add(new SqlParameter("@intervalTicks", SqlDbType.BigInt) { Value = plan.IntervalTicks });
cmd.Parameters.Add(new SqlParameter("@bucketAdjustTicks", SqlDbType.BigInt) { Value = plan.BucketIndexAdjustTicks });
await using var reader = await cmd.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false);
while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false))
{
var row = ReadBucket(reader, family);
evaluated[row.Bucket] = AggregateEvaluator.Evaluate(aggregate, row, plan.BucketStart(row.Bucket), now);
}
rows.Add(ReadBucket(reader, family));
if (needsEarliest && await reader.NextResultAsync(cancellationToken).ConfigureAwait(false)
&& await reader.ReadAsync(cancellationToken).ConfigureAwait(false) && !reader.IsDBNull(0))
earliest = DateTime.SpecifyKind(reader.GetDateTime(0), DateTimeKind.Utc);
}
var result = plan.Assemble(evaluated, bucketStart => AggregateEvaluator.NoData(bucketStart, now));
var result = AggregateEvaluator.EvaluateAll(aggregate, plan, rows, earliest, now);
RecordSuccess();
return new HistoryReadResult(result, ContinuationPoint: null);