Joseph Doherty
225 lines
16 KiB
C#
225 lines
16 KiB
C#
using System.Globalization;
|
||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||
using ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common;
|
||
|
||
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb;
|
||
|
||
/// <summary>
|
||
/// Push-down QuestDB SQL for processed (aggregate) reads (spec §3 "Processed", T0 Q8-confirmed forms), one query
|
||
/// per <see cref="AggregateFamily"/>, mirroring the SQL Server backend's <c>SqlAggregateQueries</c> family split and
|
||
/// column order. Every query takes five positional parameters:
|
||
/// <list type="number">
|
||
/// <item><c>$1</c> — the bucket base in 100 ns ticks since the Unix epoch
|
||
/// (<see cref="BucketPlan.BucketBaseUnixTicks"/>, <c>Bigint</c>: the rounded window start forward; re-anchored at
|
||
/// the later time for a reverse read, #677);</item>
|
||
/// <item><c>$2</c> — the bucket width in ticks (<see cref="BucketPlan.IntervalTicks"/>, <c>Bigint</c>);</item>
|
||
/// <item><c>$3</c> — the tag (<c>Varchar</c>);</item>
|
||
/// <item><c>$4</c>/<c>$5</c> — the earlier / later window edges (<c>TimestampTz</c> via
|
||
/// <see cref="QuestDbConnection.BindTimestamp"/>): <c>[$4, $5)</c> forward, <c>($4, $5]</c> reverse.</item>
|
||
/// </list>
|
||
/// Each returns one row per <b>non-empty</b> bucket; the caller fills the empty ones.
|
||
/// </summary>
|
||
/// <remarks>
|
||
/// <para>
|
||
/// <b>Bucketing.</b> <c>b = (cast(ts AS LONG) * 10 − $1) / $2</c>. <c>cast(ts AS LONG)</c> is µs since 1970, so
|
||
/// <c>× 10</c> is ticks since 1970, so the integer division is exact to the tick, exactly like SQL Server's
|
||
/// (<see cref="BucketPlan.IndexOf"/> is the C# statement of the math). Every in-window row is at or after the base
|
||
/// (forward) or strictly after the window start with the base one tick past the grid origin (reverse), so the
|
||
/// dividend is never negative (no truncation-toward-zero surprise), and nothing overflows before 9999.
|
||
/// <c>SAMPLE BY</c> is deliberately not used: its alignment rules differ from ours.
|
||
/// </para>
|
||
/// <para>
|
||
/// <b>Sample classes (S6).</b> The status is stored as the <b>unsigned</b> 32-bit code in a <c>LONG</c>, so
|
||
/// "not Bad" is <c>status < 0x80000000</c> (not T-SQL's sign test). <i>usable</i> = not Bad AND
|
||
/// <c>kind <> 0</c> (an Empty sample is an outage marker and counts as Bad whatever its code);
|
||
/// <i>good</i> = usable AND <c>status < 0x40000000</c>; <i>bad</i> = not usable; <i>non-numeric</i> = usable
|
||
/// String/DateTime. The literals are generated from <see cref="StatusBits"/> and <see cref="HistorianValueKind"/>.
|
||
/// </para>
|
||
/// <para>Common result columns (ordinals 0–5): <c>b, total_count, usable_count, good_count, bad_count,
|
||
/// non_numeric_count</c>. Family columns follow from <see cref="FirstFamilyColumn"/>.</para>
|
||
/// </remarks>
|
||
internal static class QuestDbAggregateQueries
|
||
{
|
||
/// <summary>Ordinal of the first family-specific column.</summary>
|
||
public const int FirstFamilyColumn = 6;
|
||
|
||
/// <summary><c>status < SeverityBadBit</c>, as a decimal LONG literal.</summary>
|
||
internal static readonly string NotBadLimit = StatusBits.SeverityBadBit.ToString(CultureInfo.InvariantCulture);
|
||
|
||
/// <summary><c>status < SeverityUncertainBit</c>, as a decimal LONG literal.</summary>
|
||
internal static readonly string GoodLimit = StatusBits.SeverityUncertainBit.ToString(CultureInfo.InvariantCulture);
|
||
|
||
/// <summary>The <c>kind</c> values of String and DateTime: usable but not numeric.</summary>
|
||
internal static readonly string NonNumericKinds =
|
||
((byte)HistorianValueKind.String).ToString(CultureInfo.InvariantCulture) + ", " +
|
||
((byte)HistorianValueKind.DateTime).ToString(CultureInfo.InvariantCulture);
|
||
|
||
private static readonly string EmptyKind = ((byte)HistorianValueKind.Empty).ToString(CultureInfo.InvariantCulture);
|
||
|
||
/// <summary>
|
||
/// Average / Minimum / Maximum / Count / Range (PercentGood / PercentBad and Total are time-weighted, not here).
|
||
/// Family columns: <c>6 avg_value, 7 min_value, 8 max_value, 9 sum_value</c> (DOUBLE, NULL when no usable sample).
|
||
/// </summary>
|
||
/// <param name="sample">Prefixed sample table name.</param>
|
||
/// <returns>The SQL text.</returns>
|
||
/// <param name="reverse">True for a reverse read (window <c>($4, $5]</c>).</param>
|
||
public static string Simple(string sample, bool reverse = false) =>
|
||
"SELECT b, " + CommonCounts() + ", " +
|
||
"avg(CASE WHEN u = 1 THEN num_value END) AS avg_value, " +
|
||
"min(CASE WHEN u = 1 THEN num_value END) AS min_value, " +
|
||
"max(CASE WHEN u = 1 THEN num_value END) AS max_value, " +
|
||
"sum(CASE WHEN u = 1 THEN num_value END) AS sum_value " +
|
||
"FROM (" + Classified(sample, reverse) + ") GROUP BY b ORDER BY b";
|
||
|
||
/// <summary>Ordinal of the StartEnd family's first-sample-of-any-quality columns (<see cref="StartEnd"/>).</summary>
|
||
public const int FirstAnyColumn = FirstFamilyColumn + 10;
|
||
|
||
/// <summary>
|
||
/// Start / End / Delta: the first and last usable sample of each bucket by <c>ts</c> (Delta's value), and the first
|
||
/// and last sample of <b>any</b> quality (Start / End, and Delta's status — #680).
|
||
/// Family columns: <c>6..10</c> first usable sample (<c>kind, num_value, int_value, str_value, ts</c>), <c>11..15</c>
|
||
/// last usable sample (same order) — a bucket without a usable sample has NULL <c>ts</c> but <c>kind = 0</c> (BYTE
|
||
/// has no NULL), so the reader keys absence on the NULL time; then <c>16..21</c> the first sample of any quality
|
||
/// (<c>ts, kind, num_value, int_value, str_value, status</c>) and <c>22..27</c> the last (same order), present for
|
||
/// every returned (non-empty) bucket.
|
||
/// </summary>
|
||
/// <remarks>
|
||
/// <para>
|
||
/// <b>One scan, one GROUP BY (live-gate 8 F1).</b> The counts and all four picked samples come from a single read
|
||
/// of the sample table. The first/last sample of any quality are QuestDB's <c>first()</c>/<c>last()</c> (the
|
||
/// first/last row of the group in designated-timestamp order — null values included, so every column comes from
|
||
/// the same row); the first/last <i>usable</i> sample are <c>arg_min</c>/<c>arg_max</c> keyed on <c>uts</c>, the
|
||
/// row's time when usable and NULL otherwise (a NULL key is skipped, a NULL value is returned as is, and
|
||
/// <c>(ts, tag)</c> is unique — DEDUP — so there are no ties). <c>arg_min</c> has no VARCHAR form, so the usable
|
||
/// picks carry no <c>str_value</c>: only Delta reads them, and only when the bucket has no usable non-numeric
|
||
/// sample.
|
||
/// </para>
|
||
/// <para>
|
||
/// The previous form referenced its CTE four times (counts + three self-joins); QuestDB opens a separate table
|
||
/// reader per reference, so a row committed between two of them produced a bucket whose counts included a sample
|
||
/// its first/last join could not find (a NULL first-of-any-quality time that threw) — ~1 in 90k reads of a tag
|
||
/// being written. One reference means one reader at one table version. A windowed (<c>row_number()</c>)
|
||
/// single-scan form was also correct but ~2× slower; this one is not slower than the old query.
|
||
/// </para>
|
||
/// <para>
|
||
/// ⚠️ <c>first()</c>/<c>last()</c> following timestamp order is QuestDB 10.0.1 IMPLEMENTATION behaviour (rows reach
|
||
/// the GROUP BY in row-id order, and the parallel merge keeps row id), not a documented guarantee. It was verified
|
||
/// on 20M rows with O3 inserts, single-threaded and parallel. Adding a JOIN, <c>ORDER BY</c> or <c>UNION</c> inside
|
||
/// <c>Classified</c>, or upgrading QuestDB, can silently break Start/End — re-verify <c>first(ts) = min(ts)</c>.
|
||
/// </para>
|
||
/// </remarks>
|
||
/// <param name="sample">Prefixed sample table name.</param>
|
||
/// <returns>The SQL text.</returns>
|
||
/// <param name="reverse">True for a reverse read (window <c>($4, $5]</c>).</param>
|
||
public static string StartEnd(string sample, bool reverse = false)
|
||
{
|
||
// The usable picks: kind, num_value, int_value, str_value (not available — see remarks), ts.
|
||
static string Usable(string f, string g) =>
|
||
$"cast({f}(kind, uts) AS SHORT), {f}(num_value, uts), {f}(int_value, uts), cast(NULL AS VARCHAR), {g}(uts)";
|
||
// The any-quality picks: ts, kind, num_value, int_value, str_value, status — all from the group's first / last row.
|
||
static string Any(string f) =>
|
||
$"{f}(ts), cast({f}(kind) AS SHORT), {f}(num_value), {f}(int_value), {f}(str_value), {f}(status)";
|
||
return "SELECT b, " + CommonCounts() + ", " +
|
||
Usable("arg_min", "min") + ", " + Usable("arg_max", "max") + ", " + Any("first") + ", " + Any("last") + " " +
|
||
"FROM (SELECT b, ts, kind, num_value, int_value, str_value, status, u, g, CASE WHEN u = 1 THEN ts END AS uts " +
|
||
"FROM (" + Classified(sample, reverse) + ")) GROUP BY b ORDER BY b";
|
||
}
|
||
|
||
/// <summary>
|
||
/// MinimumActualTime / MaximumActualTime: the usable sample with the smallest (largest) <c>num_value</c>, ties
|
||
/// broken by the earliest <c>ts</c>. Family columns: <c>6 num_value, 7 ts</c> (NULL when the bucket has no usable
|
||
/// numeric sample).
|
||
/// </summary>
|
||
/// <remarks>
|
||
/// Deliberately still two references to the classified rows (counts + the ranked extreme), unlike
|
||
/// <see cref="StartEnd"/>: the single-scan <c>row_number()</c> form measured ~2× slower, and QuestDB's
|
||
/// <c>arg_max</c>/<c>arg_min</c> cannot express the earliest-time tie-break. Live-gate 8 F1's inconsistency (a row
|
||
/// committed between the two readers — counted but not ranked) is absorbed by the reader and evaluator instead: a
|
||
/// NULL extreme in a bucket with usable samples evaluates to <c>BadNoData</c>, never a value-less Good.
|
||
/// </remarks>
|
||
/// <param name="sample">Prefixed sample table name.</param>
|
||
/// <param name="maximum">True for MaximumActualTime.</param>
|
||
/// <returns>The SQL text.</returns>
|
||
/// <param name="reverse">True for a reverse read (window <c>($4, $5]</c>).</param>
|
||
public static string ActualTime(string sample, bool maximum, bool reverse = false) =>
|
||
"WITH s AS (" + Classified(sample, reverse) + "), " +
|
||
"st AS (SELECT b, " + CommonCounts() + " FROM s GROUP BY b), " +
|
||
"r AS (SELECT b, num_value, ts, " +
|
||
"row_number() OVER (PARTITION BY b ORDER BY num_value " + (maximum ? "DESC" : "ASC") + ", ts ASC) AS rn " +
|
||
"FROM s WHERE u = 1 AND num_value IS NOT NULL) " +
|
||
"SELECT st.*, r.num_value, r.ts " +
|
||
"FROM st LEFT JOIN (SELECT * FROM r WHERE rn = 1) r ON r.b = st.b ORDER BY st.b";
|
||
|
||
/// <summary>
|
||
/// Time-weighted family (#671), bounds statement — <b>not</b> pushed down (see <see cref="TimeWeightedWindow"/>).
|
||
/// Four <c>LIMIT 1</c> seeks <c>UNION ALL</c>-ed, each tagged with <c>k</c>: 0 = latest usable sample before the
|
||
/// start, 1 = latest sample of any quality before the start, 2 = first sample of any quality at or after the end,
|
||
/// 3 = first usable one at or after the end. Parameters: <c>$1</c> tag, <c>$2</c>/<c>$3</c> the forward window
|
||
/// start / end. Columns: <c>k</c>, then <c>ts, kind, num_value, int_value, str_value, status, server_ts</c>.
|
||
/// </summary>
|
||
/// <param name="sample">Prefixed sample table name.</param>
|
||
/// <returns>The SQL text.</returns>
|
||
public static string TimeWeightedBounds(string sample)
|
||
{
|
||
var usable = $" AND status < {NotBadLimit} AND kind <> {EmptyKind}";
|
||
string Seek(int k, string cmp, string filter, string order) =>
|
||
$"SELECT * FROM (SELECT {k.ToString(CultureInfo.InvariantCulture)} AS k, {SampleColumns} FROM {sample} " +
|
||
$"WHERE tag = $1 AND ts {cmp}{filter} AND NOT deleted ORDER BY ts {order} LIMIT 1)";
|
||
return Seek(0, "< $2", usable, "DESC") + " UNION ALL " + Seek(1, "< $2", "", "DESC") + " UNION ALL " +
|
||
Seek(2, ">= $3", "", "ASC") + " UNION ALL " + Seek(3, ">= $3", usable, "ASC");
|
||
}
|
||
|
||
/// <summary>
|
||
/// Time-weighted family (#671), window statement: the live samples in <c>[$2, $3)</c> for tag <c>$1</c>, ascending,
|
||
/// at most <c>$4</c> (the raw-row cap + 1).
|
||
/// Columns: <c>ts, kind, num_value, int_value, str_value, status, server_ts</c>. The caller feeds the before-bounds,
|
||
/// these rows and the after-bounds, in that order, into <see cref="TimeWeightedAggregator"/> — the same evaluator
|
||
/// the SQL Server backend uses, so the two agree exactly. Trade-off: the window's rows travel to the server rather
|
||
/// than one row per bucket; Part 13's segment interpolation and bucket-edge splitting are then written once
|
||
/// instead of twice in two SQL dialects that must stay bit-identical.
|
||
/// </summary>
|
||
/// <param name="sample">Prefixed sample table name.</param>
|
||
/// <returns>The SQL text.</returns>
|
||
public static string TimeWeightedWindow(string sample) =>
|
||
$"SELECT {SampleColumns} FROM {sample} WHERE tag = $1 AND ts >= $2 AND ts < $3 AND NOT deleted ORDER BY ts ASC LIMIT $4";
|
||
|
||
/// <summary>The row columns the time-weighted statements return (after <c>k</c> for the bounds).</summary>
|
||
private const string SampleColumns = "ts, kind, num_value, int_value, str_value, status, server_ts";
|
||
|
||
/// <summary>The bucket-index expression (exposed for tests).</summary>
|
||
internal const string BucketExpression = "(cast(ts AS LONG) * 10 - $1) / $2";
|
||
|
||
/// <summary>
|
||
/// The tag's earliest live sample (any quality) at or before the window's later edge — half of the tag's data range
|
||
/// (the start-of-data Partial rule, #677, and Count's in-range empty interval, #680; see
|
||
/// <see cref="AggregateEvaluator.NeedsDataRange"/>). Parameters <c>$1</c> tag, <c>$2</c> the later edge. Column:
|
||
/// <c>ts</c>; no row when the tag has none — the data range then does not reach the window.
|
||
/// </summary>
|
||
/// <param name="sample">Prefixed sample table name.</param>
|
||
/// <returns>The SQL text.</returns>
|
||
public static string EarliestSample(string sample) =>
|
||
$"SELECT ts FROM {sample} WHERE tag = $1 AND ts <= $2 AND NOT deleted ORDER BY ts ASC LIMIT 1";
|
||
|
||
/// <summary>
|
||
/// The tag's latest live sample (any quality) at or after the window's earlier edge — the other half of the data
|
||
/// range (the end-of-data Partial rule and Count's in-range empty interval, #680). Parameters <c>$1</c> tag,
|
||
/// <c>$2</c> the earlier edge. Column: <c>ts</c>; no row when the tag has none.
|
||
/// </summary>
|
||
/// <param name="sample">Prefixed sample table name.</param>
|
||
/// <returns>The SQL text.</returns>
|
||
public static string LatestSample(string sample) =>
|
||
$"SELECT ts FROM {sample} WHERE tag = $1 AND ts >= $2 AND NOT deleted ORDER BY ts DESC LIMIT 1";
|
||
|
||
/// <summary>The classified subquery <c>s</c>: the window's live samples with bucket index and the usable/good flags.</summary>
|
||
private static string Classified(string sample, bool reverse) =>
|
||
"SELECT " + BucketExpression + " AS b, ts, kind, num_value, int_value, str_value, status, " +
|
||
$"CASE WHEN status < {NotBadLimit} AND kind <> {EmptyKind} THEN 1 ELSE 0 END AS u, " +
|
||
$"CASE WHEN status < {GoodLimit} AND kind <> {EmptyKind} THEN 1 ELSE 0 END AS g " +
|
||
$"FROM {sample} WHERE tag = $3 AND " + (reverse ? "ts > $4 AND ts <= $5" : "ts >= $4 AND ts < $5") + " AND NOT deleted";
|
||
|
||
/// <summary>The five common count columns (after <c>b</c>).</summary>
|
||
private static string CommonCounts() =>
|
||
"count() AS total_count, sum(u) AS usable_count, sum(g) AS good_count, count() - sum(u) AS bad_count, " +
|
||
$"sum(CASE WHEN u = 1 AND kind IN ({NonNumericKinds}) THEN 1 ELSE 0 END) AS non_numeric_count";
|
||
}
|