fix(historian): live-gate 8 F1 — per-sample counts and picks from one table version (#680 follow-up)
QuestDB Start/End/Delta reads of a tag being written intermittently failed BadHistoryOperationUnsupported (InvalidCastException: Column 'ts_3' is null): the StartEnd query referenced its CTE four times and QuestDB opens a table reader per reference, so a row committed between two of them was counted but not picked. MinimumActualTime/MaximumActualTime had the same race and returned a value-less Good. (a) SQL - QuestDB StartEnd: one scan, one GROUP BY — first()/last() for the any-quality picks, arg_min/arg_max keyed on the usable rows' time for the usable picks (no str_value: arg_min has no VARCHAR form; only Delta reads them). Byte-identical output to the old query on a 1M-row tag, ~15% faster. - SqlServer StartEnd: one windowed pass + conditional MAX (was five CTE references); identical output, 1.4-3x faster. - ActualTime kept on both backends: every single-scan form measured 1.2-2.4x slower; its inconsistency is absorbed by (b). (b) Defence in depth on both backends - ReadBucket takes DbDataReader and reads absent picks as null (ReadStoredOrNull) instead of throwing. - AggregateEvaluator: Delta with a missing first/last usable pick and ActualTime with a missing extreme are BadNoData (were BadAggregateInvalidInputs / a value-less Good); Start/End with no pick already were. - Unit tests (Sql.Common evaluator, QuestDb + SqlServer mappers over a DataTableReader) fail with the guards removed. LiveWriteReadRaceStressTests (QuestDB integration, opt-in via HISTORIAN_QUESTDB_STRESS_SECONDS, Category=SlowUnit): single-row writer vs 4 tight-loop readers. Old code: 6 faults in ~105k reads; new: 0 in 106k. Docs: Historian.md QuestDB processed-read note; F2 (Min/Max on the interval start: SDK raw, ours Calculated) and F3 (integer tags: SDK casts interpolated values to the integer type, ours Double) added to the SDK-parity deviations.
This commit is contained in:
Joseph Doherty
@@ -71,33 +71,53 @@ internal static class SqlAggregateQueries
|
||||
/// Family columns: <c>6..10</c> first usable sample (<c>ValueKind, NumValue, IntValue, StrValue, SourceTimeUtc</c>),
|
||||
/// <c>11..15</c> last usable sample (same order) — all NULL when the bucket has no usable sample; then
|
||||
/// <c>16..21</c> the first sample of any quality (<c>SourceTimeUtc, ValueKind, NumValue, IntValue, StrValue,
|
||||
/// StatusCode</c>) and <c>22..27</c> the last (same order) — never NULL for a returned (non-empty) bucket.
|
||||
/// StatusCode</c>) and <c>22..27</c> the last (same order) — present for every returned (non-empty) bucket.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <b>One scan (live-gate 8 F1).</b> The counts and all four picked samples come from one reference to <c>s</c>:
|
||||
/// a windowed pass ranks each row within its bucket (<c>rfa</c>/<c>rla</c> over every sample, <c>rfu</c>/<c>rlu</c>
|
||||
/// within its usable / not-usable partition) and the outer <c>GROUP BY</c> both counts and lifts each picked row's
|
||||
/// columns with a conditional <c>MAX</c> (one row per bucket matches each pick). The previous form referenced the
|
||||
/// CTE five times; SQL Server expands each reference into its own scan, and under READ COMMITTED nothing makes those
|
||||
/// scans see the same rows, so a sample committed between two of them could be counted but not picked (QuestDB hit
|
||||
/// exactly that). It is also fewer scans of the key range.
|
||||
/// </remarks>
|
||||
/// <param name="sample">Schema-qualified <c>Sample</c> table name.</param>
|
||||
/// <returns>The SQL text.</returns>
|
||||
/// <param name="reverse">True for a reverse read (window <c>(@start, @end]</c>).</param>
|
||||
public static string StartEnd(string sample, bool reverse = false) =>
|
||||
Classified(sample, reverse) +
|
||||
", st AS (SELECT b, " + CommonCounts() + " FROM s GROUP BY b)" +
|
||||
", f AS (SELECT b, ValueKind, NumValue, IntValue, StrValue, SourceTimeUtc, " +
|
||||
"ROW_NUMBER() OVER (PARTITION BY b ORDER BY SourceTimeUtc ASC) AS rf, " +
|
||||
"ROW_NUMBER() OVER (PARTITION BY b ORDER BY SourceTimeUtc DESC) AS rl FROM s WHERE u = 1)" +
|
||||
", a AS (SELECT b, SourceTimeUtc, ValueKind, NumValue, IntValue, StrValue, StatusCode, " +
|
||||
"ROW_NUMBER() OVER (PARTITION BY b ORDER BY SourceTimeUtc ASC) AS rf, " +
|
||||
"ROW_NUMBER() OVER (PARTITION BY b ORDER BY SourceTimeUtc DESC) AS rl FROM s) " +
|
||||
"SELECT st.*, " +
|
||||
"x.ValueKind, x.NumValue, x.IntValue, x.StrValue, x.SourceTimeUtc, " +
|
||||
"y.ValueKind, y.NumValue, y.IntValue, y.StrValue, y.SourceTimeUtc, " +
|
||||
"p.SourceTimeUtc, p.ValueKind, p.NumValue, p.IntValue, p.StrValue, p.StatusCode, " +
|
||||
"q.SourceTimeUtc, q.ValueKind, q.NumValue, q.IntValue, q.StrValue, q.StatusCode " +
|
||||
"FROM st LEFT JOIN f x ON x.b = st.b AND x.rf = 1 LEFT JOIN f y ON y.b = st.b AND y.rl = 1 " +
|
||||
"JOIN a p ON p.b = st.b AND p.rf = 1 JOIN a q ON q.b = st.b AND q.rl = 1 ORDER BY st.b;";
|
||||
public static string StartEnd(string sample, bool reverse = false)
|
||||
{
|
||||
static string Pick(string when, bool any) =>
|
||||
(any ? $"MAX(CASE WHEN {when} THEN SourceTimeUtc END), " : "") +
|
||||
$"MAX(CASE WHEN {when} THEN ValueKind END), MAX(CASE WHEN {when} THEN NumValue END), " +
|
||||
$"MAX(CASE WHEN {when} THEN IntValue END), MAX(CASE WHEN {when} THEN StrValue END), " +
|
||||
(any ? $"MAX(CASE WHEN {when} THEN StatusCode END)" : $"MAX(CASE WHEN {when} THEN SourceTimeUtc END)");
|
||||
return Classified(sample, reverse) +
|
||||
", w AS (SELECT b, SourceTimeUtc, ValueKind, NumValue, IntValue, StrValue, StatusCode, u, g, " +
|
||||
"ROW_NUMBER() OVER (PARTITION BY b ORDER BY SourceTimeUtc ASC) AS rfa, " +
|
||||
"ROW_NUMBER() OVER (PARTITION BY b ORDER BY SourceTimeUtc DESC) AS rla, " +
|
||||
"ROW_NUMBER() OVER (PARTITION BY b, u ORDER BY SourceTimeUtc ASC) AS rfu, " +
|
||||
"ROW_NUMBER() OVER (PARTITION BY b, u ORDER BY SourceTimeUtc DESC) AS rlu FROM s) " +
|
||||
"SELECT b, " + CommonCounts() + ", " +
|
||||
Pick("u = 1 AND rfu = 1", any: false) + ", " +
|
||||
Pick("u = 1 AND rlu = 1", any: false) + ", " +
|
||||
Pick("rfa = 1", any: true) + ", " +
|
||||
Pick("rla = 1", any: true) + " " +
|
||||
"FROM w GROUP BY b ORDER BY b;";
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// MinimumActualTime / MaximumActualTime: the usable sample with the smallest (largest) <c>NumValue</c>,
|
||||
/// ties broken by the earliest <c>SourceTimeUtc</c>. Family columns: <c>6 NumValue, 7 SourceTimeUtc</c>
|
||||
/// (NULL when the bucket has no usable numeric sample).
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Deliberately still two references to <c>s</c> (counts + the ranked extreme), unlike <see cref="StartEnd"/>:
|
||||
/// every single-scan form measured (a <c>ROW_NUMBER</c> or windowed <c>MIN</c>/<c>MAX</c> over every row) was
|
||||
/// 1.2–2× slower on day-plus windows than ranking only the filtered usable rows. Live-gate 8 F1's inconsistency —
|
||||
/// a sample committed between the two scans, counted but not ranked — is instead absorbed by the reader: a
|
||||
/// NULL extreme in a bucket with usable samples evaluates to <c>BadNoData</c> (never a value-less Good).
|
||||
/// </remarks>
|
||||
/// <param name="sample">Schema-qualified <c>Sample</c> table name.</param>
|
||||
/// <param name="maximum">True for MaximumActualTime.</param>
|
||||
/// <returns>The SQL text.</returns>
|
||||
|
||||
@@ -106,7 +106,7 @@ internal sealed class SqlHistorianConnection
|
||||
/// <param name="reader">Reader positioned on a row.</param>
|
||||
/// <param name="ordinal">Column ordinal.</param>
|
||||
/// <returns>UTC timestamp.</returns>
|
||||
public static DateTime GetUtc(SqlDataReader reader, int ordinal) =>
|
||||
public static DateTime GetUtc(System.Data.Common.DbDataReader reader, int ordinal) =>
|
||||
DateTime.SpecifyKind(reader.GetDateTime(ordinal), DateTimeKind.Utc);
|
||||
|
||||
/// <summary>True for connection/timeout/deadlock/throttling errors worth retrying.</summary>
|
||||
|
||||
+25
-8
@@ -1,4 +1,5 @@
|
||||
using System.Data;
|
||||
using System.Data.Common;
|
||||
using System.Text;
|
||||
using Microsoft.Data.SqlClient;
|
||||
using Microsoft.Extensions.Logging;
|
||||
@@ -524,12 +525,22 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
|
||||
} while (await reader.NextResultAsync(ct).ConfigureAwait(false));
|
||||
}
|
||||
|
||||
private static BucketRow ReadBucket(SqlDataReader reader, AggregateFamily family)
|
||||
/// <summary>
|
||||
/// Maps one per-sample aggregate row (<see cref="SqlAggregateQueries"/> column layout) to a <see cref="BucketRow"/>.
|
||||
/// Every picked sample — first/last usable, first/last of any quality, the extreme — is optional: a NULL reads as
|
||||
/// "no such sample", never an exception (live-gate 8 F1: on QuestDB a row whose counts and picks disagreed threw
|
||||
/// out of <see cref="ReadStored"/> and failed the node's whole read; the evaluator answers such a bucket from what
|
||||
/// is present, or <c>BadNoData</c>).
|
||||
/// </summary>
|
||||
/// <param name="reader">Reader positioned on a bucket row.</param>
|
||||
/// <param name="family">The query's aggregate family.</param>
|
||||
/// <returns>The bucket statistics.</returns>
|
||||
internal static BucketRow ReadBucket(DbDataReader reader, AggregateFamily family)
|
||||
{
|
||||
static double? D(SqlDataReader r, int i) => r.IsDBNull(i) ? null : r.GetDouble(i);
|
||||
static long? L(SqlDataReader r, int i) => r.IsDBNull(i) ? null : r.GetInt64(i);
|
||||
static string? S(SqlDataReader r, int i) => r.IsDBNull(i) ? null : r.GetString(i);
|
||||
static HistorianValue V(SqlDataReader r, int i) =>
|
||||
static double? D(DbDataReader r, int i) => r.IsDBNull(i) ? null : r.GetDouble(i);
|
||||
static long? L(DbDataReader r, int i) => r.IsDBNull(i) ? null : r.GetInt64(i);
|
||||
static string? S(DbDataReader r, int i) => r.IsDBNull(i) ? null : r.GetString(i);
|
||||
static HistorianValue V(DbDataReader r, int i) =>
|
||||
r.IsDBNull(i) ? HistorianValue.Empty : SampleColumnMapper.FromColumns(r.GetByte(i), D(r, i + 1), L(r, i + 2), S(r, i + 3));
|
||||
|
||||
var row = new BucketRow
|
||||
@@ -553,8 +564,8 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
|
||||
case AggregateFamily.StartEnd:
|
||||
row.First = V(reader, f);
|
||||
row.Last = V(reader, f + 5);
|
||||
row.FirstAny = ReadStored(reader, SqlAggregateQueries.FirstAnyColumn);
|
||||
row.LastAny = ReadStored(reader, SqlAggregateQueries.FirstAnyColumn + 6);
|
||||
row.FirstAny = ReadStoredOrNull(reader, SqlAggregateQueries.FirstAnyColumn);
|
||||
row.LastAny = ReadStoredOrNull(reader, SqlAggregateQueries.FirstAnyColumn + 6);
|
||||
break;
|
||||
default:
|
||||
row.Extreme = D(reader, f);
|
||||
@@ -572,7 +583,7 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
|
||||
/// (<see cref="StoredSample.FromStored"/>) — defence in depth for rows written before the writer forced every
|
||||
/// Empty sample Bad.
|
||||
/// </summary>
|
||||
private static StoredSample ReadStored(SqlDataReader r, int o)
|
||||
private static StoredSample ReadStored(DbDataReader r, int o)
|
||||
{
|
||||
var value = SampleColumnMapper.FromColumns(
|
||||
r.GetByte(o + 1),
|
||||
@@ -583,6 +594,12 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
|
||||
return StoredSample.FromStored(SqlHistorianConnection.GetUtc(r, o), value, status);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// <see cref="ReadStored"/> for an optional picked sample: a NULL <c>SourceTimeUtc</c> means the row carries no such
|
||||
/// sample (live-gate 8 F1), so the result is null rather than an exception.
|
||||
/// </summary>
|
||||
private static StoredSample? ReadStoredOrNull(DbDataReader r, int o) => r.IsDBNull(o) ? null : ReadStored(r, o);
|
||||
|
||||
/// <summary>Read-only tag lookup (never creates a row); runs the schema bootstrap on a cache miss.</summary>
|
||||
private Task<int?> ResolveAsync(string name, CancellationToken ct) => _catalog.TryGetTagIdAsync(name, ct);
|
||||
|
||||
|
||||
Reference in New Issue
Block a user