fix(historian): live-gate 8 F1 — per-sample counts and picks from one table version (#680 follow-up)
QuestDB Start/End/Delta reads of a tag being written intermittently failed BadHistoryOperationUnsupported (InvalidCastException: Column 'ts_3' is null): the StartEnd query referenced its CTE four times and QuestDB opens a table reader per reference, so a row committed between two of them was counted but not picked. MinimumActualTime/MaximumActualTime had the same race and returned a value-less Good. (a) SQL - QuestDB StartEnd: one scan, one GROUP BY — first()/last() for the any-quality picks, arg_min/arg_max keyed on the usable rows' time for the usable picks (no str_value: arg_min has no VARCHAR form; only Delta reads them). Byte-identical output to the old query on a 1M-row tag, ~15% faster. - SqlServer StartEnd: one windowed pass + conditional MAX (was five CTE references); identical output, 1.4-3x faster. - ActualTime kept on both backends: every single-scan form measured 1.2-2.4x slower; its inconsistency is absorbed by (b). (b) Defence in depth on both backends - ReadBucket takes DbDataReader and reads absent picks as null (ReadStoredOrNull) instead of throwing. - AggregateEvaluator: Delta with a missing first/last usable pick and ActualTime with a missing extreme are BadNoData (were BadAggregateInvalidInputs / a value-less Good); Start/End with no pick already were. - Unit tests (Sql.Common evaluator, QuestDb + SqlServer mappers over a DataTableReader) fail with the guards removed. LiveWriteReadRaceStressTests (QuestDB integration, opt-in via HISTORIAN_QUESTDB_STRESS_SECONDS, Category=SlowUnit): single-row writer vs 4 tight-loop readers. Old code: 6 faults in ~105k reads; new: 0 in 106k. Docs: Historian.md QuestDB processed-read note; F2 (Min/Max on the interval start: SDK raw, ours Calculated) and F3 (integer tags: SDK casts interpolated values to the integer type, ours Double) added to the SDK-parity deviations.
This commit is contained in:
+25
-8
@@ -1,4 +1,5 @@
|
||||
using System.Data;
|
||||
using System.Data.Common;
|
||||
using System.Text;
|
||||
using Microsoft.Data.SqlClient;
|
||||
using Microsoft.Extensions.Logging;
|
||||
@@ -524,12 +525,22 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
|
||||
} while (await reader.NextResultAsync(ct).ConfigureAwait(false));
|
||||
}
|
||||
|
||||
private static BucketRow ReadBucket(SqlDataReader reader, AggregateFamily family)
|
||||
/// <summary>
|
||||
/// Maps one per-sample aggregate row (<see cref="SqlAggregateQueries"/> column layout) to a <see cref="BucketRow"/>.
|
||||
/// Every picked sample — first/last usable, first/last of any quality, the extreme — is optional: a NULL reads as
|
||||
/// "no such sample", never an exception (live-gate 8 F1: on QuestDB a row whose counts and picks disagreed threw
|
||||
/// out of <see cref="ReadStored"/> and failed the node's whole read; the evaluator answers such a bucket from what
|
||||
/// is present, or <c>BadNoData</c>).
|
||||
/// </summary>
|
||||
/// <param name="reader">Reader positioned on a bucket row.</param>
|
||||
/// <param name="family">The query's aggregate family.</param>
|
||||
/// <returns>The bucket statistics.</returns>
|
||||
internal static BucketRow ReadBucket(DbDataReader reader, AggregateFamily family)
|
||||
{
|
||||
static double? D(SqlDataReader r, int i) => r.IsDBNull(i) ? null : r.GetDouble(i);
|
||||
static long? L(SqlDataReader r, int i) => r.IsDBNull(i) ? null : r.GetInt64(i);
|
||||
static string? S(SqlDataReader r, int i) => r.IsDBNull(i) ? null : r.GetString(i);
|
||||
static HistorianValue V(SqlDataReader r, int i) =>
|
||||
static double? D(DbDataReader r, int i) => r.IsDBNull(i) ? null : r.GetDouble(i);
|
||||
static long? L(DbDataReader r, int i) => r.IsDBNull(i) ? null : r.GetInt64(i);
|
||||
static string? S(DbDataReader r, int i) => r.IsDBNull(i) ? null : r.GetString(i);
|
||||
static HistorianValue V(DbDataReader r, int i) =>
|
||||
r.IsDBNull(i) ? HistorianValue.Empty : SampleColumnMapper.FromColumns(r.GetByte(i), D(r, i + 1), L(r, i + 2), S(r, i + 3));
|
||||
|
||||
var row = new BucketRow
|
||||
@@ -553,8 +564,8 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
|
||||
case AggregateFamily.StartEnd:
|
||||
row.First = V(reader, f);
|
||||
row.Last = V(reader, f + 5);
|
||||
row.FirstAny = ReadStored(reader, SqlAggregateQueries.FirstAnyColumn);
|
||||
row.LastAny = ReadStored(reader, SqlAggregateQueries.FirstAnyColumn + 6);
|
||||
row.FirstAny = ReadStoredOrNull(reader, SqlAggregateQueries.FirstAnyColumn);
|
||||
row.LastAny = ReadStoredOrNull(reader, SqlAggregateQueries.FirstAnyColumn + 6);
|
||||
break;
|
||||
default:
|
||||
row.Extreme = D(reader, f);
|
||||
@@ -572,7 +583,7 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
|
||||
/// (<see cref="StoredSample.FromStored"/>) — defence in depth for rows written before the writer forced every
|
||||
/// Empty sample Bad.
|
||||
/// </summary>
|
||||
private static StoredSample ReadStored(SqlDataReader r, int o)
|
||||
private static StoredSample ReadStored(DbDataReader r, int o)
|
||||
{
|
||||
var value = SampleColumnMapper.FromColumns(
|
||||
r.GetByte(o + 1),
|
||||
@@ -583,6 +594,12 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
|
||||
return StoredSample.FromStored(SqlHistorianConnection.GetUtc(r, o), value, status);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// <see cref="ReadStored"/> for an optional picked sample: a NULL <c>SourceTimeUtc</c> means the row carries no such
|
||||
/// sample (live-gate 8 F1), so the result is null rather than an exception.
|
||||
/// </summary>
|
||||
private static StoredSample? ReadStoredOrNull(DbDataReader r, int o) => r.IsDBNull(o) ? null : ReadStored(r, o);
|
||||
|
||||
/// <summary>Read-only tag lookup (never creates a row); runs the schema bootstrap on a cache miss.</summary>
|
||||
private Task<int?> ResolveAsync(string name, CancellationToken ct) => _catalog.TryGetTagIdAsync(name, ct);
|
||||
|
||||
|
||||
Reference in New Issue
Block a user