Joseph Doherty
910befc84c
Start/End/Delta: one GROUP BY computes the counts and each bucket's four pick
times (MIN/MAX SourceTimeUtc, any quality and usable); each pick's columns are
then fetched by a seek on the clustered key (TagId, SourceTimeUtc). Replaces the
windowed ROW_NUMBER form (live-gate 8 F1), whose four window sorts spilled.
Output identical (EXCEPT both ways empty, 6M rows, forward + reverse incl. a
sub-microsecond reverse start). 6M rows / 30 days, avg of 3 x 2 rounds:
30d@1h 18.6-31.7 s -> 3.4 s (reverse 18.1-29.9 s -> 3.4-5.2 s)
7d@1d 5.2-6.1 s -> 0.8 s
1d@1min 0.31-0.40 s -> 0.16 s (1d@7min reverse 0.46 -> 0.12 s)
The seeks are separate reads within one statement: under READ COMMITTED a
HistoryUpdate delete between the aggregate and the seek nulls that pick; the
null-tolerant reader (2ceed561) makes that bucket's Start/End/Delta BadNoData,
transiently. Unit test pins it; documented in the query remarks + Historian.md.
QuestDB single-scan StartEnd unchanged.
215 lines
16 KiB
C#
215 lines
16 KiB
C#
using ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common;
|
||
|
||
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer;
|
||
|
||
/// <summary>
|
||
/// Push-down T-SQL for processed (aggregate) reads (spec §4 "Processed"), one query per
|
||
/// <see cref="AggregateFamily"/> (<see cref="AggregateFamilies.FamilyOf"/> picks it). Every query takes
|
||
/// <c>@id</c> (int TagId), <c>@start</c>/<c>@end</c> (<c>datetime2(7)</c>, the earlier / later window edges —
|
||
/// <see cref="BucketPlan.StartUtc"/>/<see cref="BucketPlan.EndUtc"/>), <c>@intervalTicks</c> (bigint bucket width in
|
||
/// 100 ns ticks) and <c>@bucketAdjustTicks</c> (bigint, <see cref="BucketPlan.BucketIndexAdjustTicks"/>), and returns
|
||
/// one row per <b>non-empty</b> bucket (ascending bucket index). The caller generates every interval and fills the
|
||
/// empty ones.
|
||
/// </summary>
|
||
/// <remarks>
|
||
/// <para>
|
||
/// <b>Bucketing.</b> <c>b = (elapsedTicks + @bucketAdjustTicks) / @intervalTicks</c> (#677: the adjust is 0 for a
|
||
/// forward read and re-anchors a reverse read's grid at its later time — <see cref="BucketPlan.IndexOf"/> is the C#
|
||
/// statement of the same math). The window is <c>[@start, @end)</c> forward and <c>(@start, @end]</c> reverse (a
|
||
/// reverse read includes its later time), selected by the <c>reverse</c> argument. <c>elapsedTicks</c> is
|
||
/// <c>elapsedTicks = DATEDIFF_BIG(microsecond, @start, t) * 10 + tick-of-microsecond(t) − tick-of-microsecond(@start)</c>
|
||
/// and <c>tick-of-microsecond(x) = (DATEPART(nanosecond, x) / 100) % 10</c>. The spec sketched
|
||
/// <c>millisecond</c>, but <c>DATEDIFF</c> counts unit <i>boundaries crossed</i>, not elapsed time, so with a
|
||
/// non-millisecond-aligned <c>@start</c> a sample one tick before a bucket edge would land in the next
|
||
/// bucket. Counting microsecond boundaries and correcting by the two sub-microsecond tick digits gives
|
||
/// the exact elapsed time in 100 ns ticks (10·(⌊t/10⌋ − ⌊s/10⌋) + t mod 10 − s mod 10 = t − s), so the
|
||
/// integer division is start-inclusive / end-exclusive to the tick. Unlike a nanosecond <c>DATEDIFF_BIG</c>
|
||
/// (which overflows a bigint past ≈292 years, SQL error 535), the microsecond count over the whole
|
||
/// <c>datetime2</c> range (0001–9999, ≈3.2e17 µs, ≈3.2e18 ticks) fits a bigint.
|
||
/// </para>
|
||
/// <para>
|
||
/// <b>Sample classes</b> (status stored bit-for-bit in an <c>int</c>, so the severity bits are the sign
|
||
/// and bit 30): <i>usable</i> = severity Good or Uncertain (<c>StatusCode >= 0</c>) AND a stored value
|
||
/// (<c>ValueKind <> 0</c>) — an <c>Empty</c> sample is an outage marker and counts as Bad whatever its
|
||
/// code; <i>good</i> = usable with severity Good (<c>StatusCode < 0x40000000</c>); <i>bad</i> = not
|
||
/// usable. Average/Minimum/Maximum/Range/*ActualTime value columns are computed over <i>good</i> samples only
|
||
/// (#684); Start/End/Delta picks over usable samples (or every sample).
|
||
/// </para>
|
||
/// Common result columns (ordinals 0–5): <c>Bucket, TotalCount, UsableCount, GoodCount, BadCount,
|
||
/// NonNumericCount</c> (usable String/DateTime samples). Family columns follow.
|
||
/// </remarks>
|
||
internal static class SqlAggregateQueries
|
||
{
|
||
/// <summary>Ordinal of the first family-specific column.</summary>
|
||
public const int FirstFamilyColumn = 6;
|
||
|
||
/// <summary><c>ValueKind</c> of String (4) and DateTime (5): usable but not numeric.</summary>
|
||
private const string NonNumericKinds = "4, 5";
|
||
|
||
/// <summary>
|
||
/// Average / Minimum / Maximum / Count / Range (PercentGood / PercentBad and Total are time-weighted, not here).
|
||
/// Family columns: <c>6 AvgValue, 7 MinValue, 8 MaxValue, 9 SumValue</c> over the <b>Good</b> numeric samples (float,
|
||
/// NULL when none — #684: Part 13 and the SDK compute these over Good values only), then <c>10 UncMin, 11 UncMax</c>
|
||
/// over the Uncertain ones (the Part 13 Minimum/Maximum/Range status rule, <see cref="AggregateEvaluator"/>), and
|
||
/// <c>12 GoodNonNumericCount</c>.
|
||
/// </summary>
|
||
/// <param name="sample">Schema-qualified <c>Sample</c> table name.</param>
|
||
/// <returns>The SQL text.</returns>
|
||
/// <param name="reverse">True for a reverse read (window <c>(@start, @end]</c>).</param>
|
||
public static string Simple(string sample, bool reverse = false) =>
|
||
Classified(sample, reverse) +
|
||
"SELECT b, " + CommonCounts() + ", " +
|
||
"AVG(CASE WHEN g = 1 THEN NumValue END) AS AvgValue, " +
|
||
"MIN(CASE WHEN g = 1 THEN NumValue END) AS MinValue, " +
|
||
"MAX(CASE WHEN g = 1 THEN NumValue END) AS MaxValue, " +
|
||
"SUM(CASE WHEN g = 1 THEN NumValue END) AS SumValue, " +
|
||
UncertainExtremes() + " " +
|
||
"FROM s GROUP BY b ORDER BY b;";
|
||
|
||
/// <summary>Ordinal of the StartEnd family's first-sample-of-any-quality columns (<see cref="StartEnd"/>).</summary>
|
||
public const int FirstAnyColumn = FirstFamilyColumn + 10;
|
||
|
||
/// <summary>
|
||
/// Start / End / Delta: the first and last usable sample of each bucket by <c>SourceTimeUtc</c> (Delta's value), and
|
||
/// the first and last sample of <b>any</b> quality (Start / End, and Delta's status — #680).
|
||
/// Family columns: <c>6..10</c> first usable sample (<c>ValueKind, NumValue, IntValue, StrValue, SourceTimeUtc</c>),
|
||
/// <c>11..15</c> last usable sample (same order) — all NULL when the bucket has no usable sample; then
|
||
/// <c>16..21</c> the first sample of any quality (<c>SourceTimeUtc, ValueKind, NumValue, IntValue, StrValue,
|
||
/// StatusCode</c>) and <c>22..27</c> the last (same order) — present for every returned (non-empty) bucket.
|
||
/// </summary>
|
||
/// <remarks>
|
||
/// <para>
|
||
/// <b>Aggregate, then join back on the key (#683).</b> One <c>GROUP BY</c> over the window computes the counts and
|
||
/// each bucket's four pick TIMES — <c>MIN</c>/<c>MAX(SourceTimeUtc)</c> over every sample (any quality) and over
|
||
/// the usable ones — and each pick's columns are then fetched by a seek on the clustered key
|
||
/// <c>(TagId, SourceTimeUtc)</c> (unique, so one row per pick; a NULL usable time joins nothing ⇒ NULL columns,
|
||
/// as before). Output is identical to the previous windowed form (one <c>ROW_NUMBER</c> pass + conditional
|
||
/// <c>MAX</c>, live-gate 8 F1) — <c>EXCEPT</c> both ways is empty on 6M rows, forward and reverse — without that
|
||
/// form's four window sorts, which spilled: on 6M rows / 30 days, 30 d @ 1 h ~3.4 s vs 18–32 s, 7 d @ 1 d
|
||
/// ~0.8 s vs 5.2–6.1 s, 1 d @ 1 min ~0.16 s vs 0.31–0.40 s (docs/Historian.md).
|
||
/// </para>
|
||
/// <para>
|
||
/// ⚠️ The key seeks are separate reads of the table within one statement, so under READ COMMITTED a sample
|
||
/// deleted (HistoryUpdate DeleteRaw/DeleteAtTime) between the aggregate and its seek leaves that pick NULL while
|
||
/// the counts still include it. The reader maps a NULL pick to "absent" (<c>ReadStoredOrNull</c>, live-gate 8 F1)
|
||
/// and the evaluator answers that bucket's Start/End/Delta <c>BadNoData</c> — transient, until the next read. A
|
||
/// sample replaced in between is picked with its new value (same time); a sample inserted in between is in
|
||
/// neither half (the pick times come from the aggregate).
|
||
/// </para>
|
||
/// </remarks>
|
||
/// <param name="sample">Schema-qualified <c>Sample</c> table name.</param>
|
||
/// <returns>The SQL text.</returns>
|
||
/// <param name="reverse">True for a reverse read (window <c>(@start, @end]</c>).</param>
|
||
public static string StartEnd(string sample, bool reverse = false)
|
||
{
|
||
// A pick's columns from its joined row, in the established order: usable picks (kind, num, int, str, time), any-
|
||
// quality picks (time, kind, num, int, str, status).
|
||
static string Usable(string a) => $"{a}.ValueKind, {a}.NumValue, {a}.IntValue, {a}.StrValue, {a}.SourceTimeUtc";
|
||
static string Any(string a) => $"{a}.SourceTimeUtc, {a}.ValueKind, {a}.NumValue, {a}.IntValue, {a}.StrValue, {a}.StatusCode";
|
||
string Join(string a, string time) => $"LEFT JOIN {sample} {a} ON {a}.TagId = @id AND {a}.SourceTimeUtc = st.{time} ";
|
||
return Classified(sample, reverse) +
|
||
", st AS (SELECT b, " + CommonCounts() + ", " +
|
||
"MIN(CASE WHEN u = 1 THEN SourceTimeUtc END) AS fu, MAX(CASE WHEN u = 1 THEN SourceTimeUtc END) AS lu, " +
|
||
"MIN(SourceTimeUtc) AS fa, MAX(SourceTimeUtc) AS la FROM s GROUP BY b) " +
|
||
"SELECT st.b, st.TotalCount, st.UsableCount, st.GoodCount, st.BadCount, st.NonNumericCount, " +
|
||
Usable("x") + ", " + Usable("y") + ", " + Any("p") + ", " + Any("q") + " FROM st " +
|
||
Join("x", "fu") + Join("y", "lu") + Join("p", "fa") + Join("q", "la") + "ORDER BY st.b;";
|
||
}
|
||
|
||
/// <summary>
|
||
/// MinimumActualTime / MaximumActualTime: the <b>Good</b> sample (#684) with the smallest (largest) <c>NumValue</c>,
|
||
/// ties broken by the earliest <c>SourceTimeUtc</c>. Family columns: <c>6 NumValue, 7 SourceTimeUtc</c>
|
||
/// (NULL when the bucket has no Good numeric sample), then <c>8 UncMin, 9 UncMax, 10 GoodNonNumericCount</c> as in <see cref="Simple"/>.
|
||
/// </summary>
|
||
/// <remarks>
|
||
/// Deliberately still two references to <c>s</c> (counts + the ranked extreme), unlike <see cref="StartEnd"/>:
|
||
/// every single-scan form measured (a <c>ROW_NUMBER</c> or windowed <c>MIN</c>/<c>MAX</c> over every row) was
|
||
/// 1.2–2× slower on day-plus windows than ranking only the filtered usable rows. Live-gate 8 F1's inconsistency —
|
||
/// a sample committed between the two scans, counted but not ranked — is instead absorbed by the reader: a
|
||
/// NULL extreme in a bucket with usable samples evaluates to <c>BadNoData</c> (never a value-less Good).
|
||
/// </remarks>
|
||
/// <param name="sample">Schema-qualified <c>Sample</c> table name.</param>
|
||
/// <param name="maximum">True for MaximumActualTime.</param>
|
||
/// <returns>The SQL text.</returns>
|
||
/// <param name="reverse">True for a reverse read (window <c>(@start, @end]</c>).</param>
|
||
public static string ActualTime(string sample, bool maximum, bool reverse = false) =>
|
||
Classified(sample, reverse) +
|
||
", st AS (SELECT b, " + CommonCounts() + ", " + UncertainExtremes() + " FROM s GROUP BY b)" +
|
||
", r AS (SELECT b, NumValue, SourceTimeUtc, " +
|
||
"ROW_NUMBER() OVER (PARTITION BY b ORDER BY NumValue " + (maximum ? "DESC" : "ASC") + ", SourceTimeUtc ASC) AS rn " +
|
||
"FROM s WHERE g = 1 AND NumValue IS NOT NULL) " +
|
||
"SELECT st.b, st.TotalCount, st.UsableCount, st.GoodCount, st.BadCount, st.NonNumericCount, " +
|
||
"r.NumValue, r.SourceTimeUtc, st.UncMin, st.UncMax, st.GoodNonNumericCount " +
|
||
"FROM st LEFT JOIN r ON r.b = st.b AND r.rn = 1 ORDER BY st.b;";
|
||
|
||
/// <summary>
|
||
/// Time-weighted family (#671: TimeAverage / TimeAverage2 / Total / Total2 / Interpolative) — <b>not</b> pushed down.
|
||
/// One batch of five result sets, each <c>SourceTimeUtc, ValueKind, NumValue, IntValue, StrValue, StatusCode</c>:
|
||
/// (1) the latest usable sample before <c>@start</c>, (2) the latest sample of any quality before <c>@start</c>,
|
||
/// (3) the samples in <c>[@start, @end)</c> ascending, at most <c>@maxRows</c> (the raw-row cap + 1), (4) the first sample of any quality at or after
|
||
/// <c>@end</c>, (5) the first usable one at or after <c>@end</c>. Each bound is a <c>TOP 1</c> seek on the
|
||
/// <c>(TagId, SourceTimeUtc)</c> key. The caller streams them, in that order, into
|
||
/// <see cref="TimeWeightedAggregator"/> (shared with QuestDB, so both backends agree exactly).
|
||
/// </summary>
|
||
/// <remarks>
|
||
/// Trade-off: the window's rows travel to the server instead of one row per bucket. Part 13's per-segment
|
||
/// interpolation, bucket-edge splitting and the two bounding-value definitions would otherwise have to be written
|
||
/// twice (T-SQL and QuestDB SQL) and kept bit-identical; evaluating once in C# keeps the backends equal by
|
||
/// construction. Memory stays O(buckets): the rows are streamed, never materialised.
|
||
/// </remarks>
|
||
/// <param name="sample">Schema-qualified <c>Sample</c> table name.</param>
|
||
/// <returns>The SQL batch.</returns>
|
||
public static string TimeWeighted(string sample)
|
||
{
|
||
const string cols = "SourceTimeUtc, ValueKind, NumValue, IntValue, StrValue, StatusCode";
|
||
const string usable = " AND StatusCode >= 0 AND ValueKind <> 0";
|
||
var key = $"FROM {sample} WHERE TagId = @id AND ";
|
||
return
|
||
$"SELECT TOP 1 {cols} {key}SourceTimeUtc < @start{usable} ORDER BY SourceTimeUtc DESC; " +
|
||
$"SELECT TOP 1 {cols} {key}SourceTimeUtc < @start ORDER BY SourceTimeUtc DESC; " +
|
||
$"SELECT TOP (@maxRows) {cols} {key}SourceTimeUtc >= @start AND SourceTimeUtc < @end ORDER BY SourceTimeUtc ASC; " +
|
||
$"SELECT TOP 1 {cols} {key}SourceTimeUtc >= @end ORDER BY SourceTimeUtc ASC; " +
|
||
$"SELECT TOP 1 {cols} {key}SourceTimeUtc >= @end{usable} ORDER BY SourceTimeUtc ASC;";
|
||
}
|
||
|
||
/// <summary>
|
||
/// The tag's data range as one row of two columns (each <c>NULL</c> when none): <c>0</c> the earliest stored sample
|
||
/// of any quality at or before the window's later edge, <c>1</c> the latest at or after its earlier edge. Used by the
|
||
/// start-of-data (#677) and end-of-data (#680) Partial rules and by Count's in-range empty interval (#680) — only when
|
||
/// <see cref="AggregateEvaluator.NeedsDataRange"/> says the answer can change a result. Each is a <c>MIN</c> /
|
||
/// <c>MAX</c> over the <c>(TagId, SourceTimeUtc)</c> key — a single seek — bounded by the window (a sample past the
|
||
/// later edge cannot start the data inside it, nor one before the earlier edge end it there; a NULL then means "the
|
||
/// data range does not reach the window").
|
||
/// </summary>
|
||
/// <param name="sample">Schema-qualified <c>Sample</c> table name.</param>
|
||
/// <returns>The SQL text.</returns>
|
||
public static string DataRange(string sample) =>
|
||
$"SELECT (SELECT MIN(SourceTimeUtc) FROM {sample} WHERE TagId = @id AND SourceTimeUtc <= @end), " +
|
||
$"(SELECT MAX(SourceTimeUtc) FROM {sample} WHERE TagId = @id AND SourceTimeUtc >= @start);";
|
||
|
||
/// <summary>The <c>s</c> CTE: the window's samples with bucket index and the usable/good flags.</summary>
|
||
private static string Classified(string sample, bool reverse) =>
|
||
"WITH s AS (SELECT (DATEDIFF_BIG(microsecond, @start, SourceTimeUtc) * 10 " +
|
||
"+ (DATEPART(nanosecond, SourceTimeUtc) / 100) % 10 - (DATEPART(nanosecond, @start) / 100) % 10 " +
|
||
"+ @bucketAdjustTicks) / @intervalTicks AS b, " +
|
||
"SourceTimeUtc, ValueKind, NumValue, IntValue, StrValue, StatusCode, " +
|
||
"CASE WHEN StatusCode >= 0 AND ValueKind <> 0 THEN 1 ELSE 0 END AS u, " +
|
||
"CASE WHEN StatusCode >= 0 AND StatusCode < 1073741824 AND ValueKind <> 0 THEN 1 ELSE 0 END AS g " +
|
||
$"FROM {sample} WHERE TagId = @id AND " +
|
||
(reverse ? "SourceTimeUtc > @start AND SourceTimeUtc <= @end) " : "SourceTimeUtc >= @start AND SourceTimeUtc < @end) ");
|
||
|
||
/// <summary>
|
||
/// The minimum and maximum <b>Uncertain</b> (usable, not Good) numeric value of the bucket — <c>UncMin, UncMax</c>
|
||
/// (#684). Not result values: Part 13 makes Minimum/Maximum/Range Uncertain when one lies beyond the Good extreme.
|
||
/// Then <c>GoodNonNumericCount</c>, the Good String/DateTime samples (the Good-value aggregates' refusal check).
|
||
/// </summary>
|
||
private static string UncertainExtremes() =>
|
||
"MIN(CASE WHEN u = 1 AND g = 0 THEN NumValue END) AS UncMin, MAX(CASE WHEN u = 1 AND g = 0 THEN NumValue END) AS UncMax, " +
|
||
$"SUM(CASE WHEN g = 1 AND ValueKind IN ({NonNumericKinds}) THEN 1 ELSE 0 END) AS GoodNonNumericCount";
|
||
|
||
/// <summary>The five common count columns (after <c>b</c>).</summary>
|
||
private static string CommonCounts() =>
|
||
"COUNT(*) AS TotalCount, SUM(u) AS UsableCount, SUM(g) AS GoodCount, COUNT(*) - SUM(u) AS BadCount, " +
|
||
$"SUM(CASE WHEN u = 1 AND ValueKind IN ({NonNumericKinds}) THEN 1 ELSE 0 END) AS NonNumericCount";
|
||
}
|