2ceed561ad
QuestDB Start/End/Delta reads of a tag being written intermittently failed BadHistoryOperationUnsupported (InvalidCastException: Column 'ts_3' is null): the StartEnd query referenced its CTE four times and QuestDB opens a table reader per reference, so a row committed between two of them was counted but not picked. MinimumActualTime/MaximumActualTime had the same race and returned a value-less Good. (a) SQL - QuestDB StartEnd: one scan, one GROUP BY — first()/last() for the any-quality picks, arg_min/arg_max keyed on the usable rows' time for the usable picks (no str_value: arg_min has no VARCHAR form; only Delta reads them). Byte-identical output to the old query on a 1M-row tag, ~15% faster. - SqlServer StartEnd: one windowed pass + conditional MAX (was five CTE references); identical output, 1.4-3x faster. - ActualTime kept on both backends: every single-scan form measured 1.2-2.4x slower; its inconsistency is absorbed by (b). (b) Defence in depth on both backends - ReadBucket takes DbDataReader and reads absent picks as null (ReadStoredOrNull) instead of throwing. - AggregateEvaluator: Delta with a missing first/last usable pick and ActualTime with a missing extreme are BadNoData (were BadAggregateInvalidInputs / a value-less Good); Start/End with no pick already were. - Unit tests (Sql.Common evaluator, QuestDb + SqlServer mappers over a DataTableReader) fail with the guards removed. LiveWriteReadRaceStressTests (QuestDB integration, opt-in via HISTORIAN_QUESTDB_STRESS_SECONDS, Category=SlowUnit): single-row writer vs 4 tight-loop readers. Old code: 6 faults in ~105k reads; new: 0 in 106k. Docs: Historian.md QuestDB processed-read note; F2 (Min/Max on the interval start: SDK raw, ours Calculated) and F3 (integer tags: SDK casts interpolated values to the integer type, ours Double) added to the SDK-parity deviations.
113 lines
4.6 KiB
C#
113 lines
4.6 KiB
C#
using System.Data;
|
|
using Shouldly;
|
|
using Xunit;
|
|
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
|
using ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common;
|
|
|
|
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer.Tests;
|
|
|
|
/// <summary>
|
|
/// Live-gate 8 F1 (defence in depth on SQL Server, where it was not observed): the per-sample aggregate row mapper
|
|
/// (<see cref="SqlServerHistorianDataSource.ReadBucket"/>) over a hand-built row in the
|
|
/// <see cref="SqlAggregateQueries.StartEnd"/> column layout. A row whose counts include samples but whose
|
|
/// first/last-of-any-quality picks are NULL used to throw out of <c>ReadStored</c> and fail the node's whole read; it now
|
|
/// maps to absent picks the evaluator answers as <c>BadNoData</c>.
|
|
/// </summary>
|
|
public sealed class BucketRowMappingTests
|
|
{
|
|
private const uint NoDataCalc = 0x809B0401u;
|
|
private static readonly DateTime T = new(2026, 10, 11, 12, 0, 0, DateTimeKind.Utc);
|
|
|
|
/// <summary>b + 5 counts, two usable picks (kind, num, int, str, ts), two any picks (ts, kind, num, int, str, status).</summary>
|
|
private static DataTable StartEndTable()
|
|
{
|
|
var t = new DataTable();
|
|
t.Columns.Add("b", typeof(long));
|
|
foreach (var c in new[] { "total", "usable", "good", "bad", "nonnum" }) t.Columns.Add(c, typeof(int));
|
|
for (var p = 0; p < 2; p++)
|
|
{
|
|
t.Columns.Add($"k{p}", typeof(byte));
|
|
t.Columns.Add($"n{p}", typeof(double));
|
|
t.Columns.Add($"i{p}", typeof(long));
|
|
t.Columns.Add($"s{p}", typeof(string));
|
|
t.Columns.Add($"t{p}", typeof(DateTime));
|
|
}
|
|
for (var p = 0; p < 2; p++)
|
|
{
|
|
t.Columns.Add($"at{p}", typeof(DateTime));
|
|
t.Columns.Add($"ak{p}", typeof(byte));
|
|
t.Columns.Add($"an{p}", typeof(double));
|
|
t.Columns.Add($"ai{p}", typeof(long));
|
|
t.Columns.Add($"as{p}", typeof(string));
|
|
t.Columns.Add($"st{p}", typeof(int));
|
|
}
|
|
return t;
|
|
}
|
|
|
|
private static DataRow Counts(DataTable t, long bucket, int total)
|
|
{
|
|
var row = t.NewRow();
|
|
row[0] = bucket;
|
|
row[1] = total; row[2] = total; row[3] = total; row[4] = 0; row[5] = 0;
|
|
return row;
|
|
}
|
|
|
|
private static BucketRow Map(DataTable t, AggregateFamily family)
|
|
{
|
|
using var r = t.CreateDataReader();
|
|
r.Read().ShouldBeTrue();
|
|
return SqlServerHistorianDataSource.ReadBucket(r, family);
|
|
}
|
|
|
|
[Fact]
|
|
public void A_row_with_counts_but_null_first_and_last_of_any_quality_maps_to_absent_picks_without_throwing()
|
|
{
|
|
var t = StartEndTable();
|
|
t.Rows.Add(Counts(t, 3, 2));
|
|
|
|
var b = Map(t, AggregateFamily.StartEnd);
|
|
b.Bucket.ShouldBe(3);
|
|
b.Total.ShouldBe(2);
|
|
b.FirstAny.ShouldBeNull();
|
|
b.LastAny.ShouldBeNull();
|
|
b.First.Kind.ShouldBe(HistorianValueKind.Empty);
|
|
|
|
foreach (var agg in new[] { HistoryAggregateType.Start, HistoryAggregateType.End, HistoryAggregateType.Delta })
|
|
AggregateEvaluator.Evaluate(agg, b, T, T).StatusCode.ShouldBe(NoDataCalc);
|
|
}
|
|
|
|
[Fact]
|
|
public void A_consistent_row_still_maps_every_pick()
|
|
{
|
|
var t = StartEndTable();
|
|
var row = Counts(t, 0, 2);
|
|
row["k0"] = (byte)1; row["n0"] = 10d; row["t0"] = T;
|
|
row["k1"] = (byte)1; row["n1"] = 20d; row["t1"] = T.AddSeconds(1);
|
|
row["at0"] = T; row["ak0"] = (byte)1; row["an0"] = 10d; row["st0"] = 0;
|
|
row["at1"] = T.AddSeconds(1); row["ak1"] = (byte)1; row["an1"] = 20d; row["st1"] = 0;
|
|
t.Rows.Add(row);
|
|
|
|
var b = Map(t, AggregateFamily.StartEnd);
|
|
b.First.ShouldBe(HistorianValue.FromDouble(10));
|
|
b.Last.ShouldBe(HistorianValue.FromDouble(20));
|
|
b.FirstAny.ShouldNotBeNull().TimeUtc.ShouldBe(T);
|
|
b.LastAny.ShouldNotBeNull().Value.ShouldBe(HistorianValue.FromDouble(20));
|
|
AggregateEvaluator.Evaluate(HistoryAggregateType.Delta, b, T, T).Value.ShouldBe(10d);
|
|
}
|
|
|
|
[Fact]
|
|
public void An_actual_time_row_with_counts_but_no_extreme_is_no_data()
|
|
{
|
|
var t = new DataTable();
|
|
t.Columns.Add("b", typeof(long));
|
|
foreach (var c in new[] { "total", "usable", "good", "bad", "nonnum" }) t.Columns.Add(c, typeof(int));
|
|
t.Columns.Add("NumValue", typeof(double));
|
|
t.Columns.Add("SourceTimeUtc", typeof(DateTime));
|
|
t.Rows.Add(0L, 1, 1, 1, 0, 0, DBNull.Value, DBNull.Value);
|
|
|
|
var b = Map(t, AggregateFamily.ActualTime);
|
|
b.Extreme.ShouldBeNull();
|
|
AggregateEvaluator.Evaluate(HistoryAggregateType.MaximumActualTime, b, T, T).StatusCode.ShouldBe(NoDataCalc);
|
|
}
|
|
}
|