fix(historian): live-gate 8 F1 — per-sample counts and picks from one table version (#680 follow-up)

QuestDB Start/End/Delta reads of a tag being written intermittently failed BadHistoryOperationUnsupported (InvalidCastException: Column 'ts_3' is null): the StartEnd query referenced its CTE four times and QuestDB opens a table reader per reference, so a row committed between two of them was counted but not picked. MinimumActualTime/MaximumActualTime had the same race and returned a value-less Good. (a) SQL - QuestDB StartEnd: one scan, one GROUP BY — first()/last() for the any-quality picks, arg_min/arg_max keyed on the usable rows' time for the usable picks (no str_value: arg_min has no VARCHAR form; only Delta reads them). Byte-identical output to the old query on a 1M-row tag, ~15% faster. - SqlServer StartEnd: one windowed pass + conditional MAX (was five CTE references); identical output, 1.4-3x faster. - ActualTime kept on both backends: every single-scan form measured 1.2-2.4x slower; its inconsistency is absorbed by (b). (b) Defence in depth on both backends - ReadBucket takes DbDataReader and reads absent picks as null (ReadStoredOrNull) instead of throwing. - AggregateEvaluator: Delta with a missing first/last usable pick and ActualTime with a missing extreme are BadNoData (were BadAggregateInvalidInputs / a value-less Good); Start/End with no pick already were. - Unit tests (Sql.Common evaluator, QuestDb + SqlServer mappers over a DataTableReader) fail with the guards removed. LiveWriteReadRaceStressTests (QuestDB integration, opt-in via HISTORIAN_QUESTDB_STRESS_SECONDS, Category=SlowUnit): single-row writer vs 4 tight-loop readers. Old code: 6 faults in ~105k reads; new: 0 in 106k. Docs: Historian.md QuestDB processed-read note; F2 (Min/Max on the interval start: SDK raw, ours Calculated) and F3 (integer tags: SDK casts interpolated values to the integer type, ours Double) added to the SDK-parity deviations.
2026-10-11 18:28:29 -04:00
parent 11f792538e
commit 2ceed561ad
13 changed files with 549 additions and 54 deletions
@@ -0,0 +1,98 @@
+using System.Collections.Concurrent;
+using Shouldly;
+using Xunit;
+using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
+using ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common;
+
+namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb.IntegrationTests;
+
+/// <summary>
+///     Live-gate 8 F1 (#680 follow-up): per-sample processed reads of a tag that is being written. The StartEnd and
+///     ActualTime queries reference their classified rows more than once; QuestDB evaluates each reference with its own
+///     table reader, so a row committed between two of them produced a bucket whose counts included a sample its
+///     first/last (or extreme) join could not find — a NULL first-of-any-quality time threw <c>InvalidCastException</c>
+///     and failed the node's whole read (~1 in 90k reads on the rig), and ActualTime returned a value-less Good. StartEnd
+///     now reads the table once; ActualTime keeps its form and an inconsistent bucket evaluates to <c>BadNoData</c>. This
+///     test interleaves a continuous single-row writer with tight-loop reads and asserts that no read throws and no
+///     non-Bad result is value-less.
+/// </summary>
+/// <remarks>
+///     Opt-in: runs only when <c>HISTORIAN_QUESTDB_STRESS_SECONDS</c> is set (and the fixture's connection is), so it never
+///     slows the default suite. Category <c>SlowUnit</c> as well, for filter-based exclusion.
+/// </remarks>
+[Collection(QuestDbCollection.Name)]
+[Trait("Category", "SlowUnit")]
+public sealed class LiveWriteReadRaceStressTests(QuestDbFixture fx)
+{
+    /// <summary>Seconds to run the race loop for; absent ⇒ the test skips.</summary>
+    public const string SecondsEnvVar = "HISTORIAN_QUESTDB_STRESS_SECONDS";
+
+    private static readonly HistoryAggregateType[] Aggregates =
+    [
+        HistoryAggregateType.Start, HistoryAggregateType.End, HistoryAggregateType.Delta,
+        HistoryAggregateType.MinimumActualTime, HistoryAggregateType.MaximumActualTime,
+    ];
+
+    [Fact]
+    public async Task Per_sample_reads_of_a_tag_being_written_never_fault_and_never_return_a_valueless_non_bad_result()
+    {
+        if (!int.TryParse(Environment.GetEnvironmentVariable(SecondsEnvVar), out var seconds) || seconds <= 0)
+            Assert.Skip($"{SecondsEnvVar} is not set; the live write/read race stress test is opt-in.");
+        await using var b = await QuestDbConformanceBackend.CreateConcreteAsync(fx);
+        const string tag = "Race";
+        await b.NewCatalog().EnsureAsync(tag, 0, CancellationToken.None);
+        var src = b.CreateDataSource();
+
+        using var stop = new CancellationTokenSource(TimeSpan.FromSeconds(seconds));
+        long written = 0, reads = 0;
+        var faults = new ConcurrentBag<string>();
+
+        var writer = Task.Run(async () =>
+        {
+            var i = 0;
+            while (!stop.IsCancellationRequested)
+            {
+                // One committed row per statement (one WAL transaction each), every 7th Bad, never waiting for visibility.
+                var status = ++i % 7 == 0 ? HistorianStatusCodes.Bad : 0u;
+                await b.InsertRowsNoWaitAsync(tag, [(DateTime.UtcNow, HistorianValue.FromDouble(i), status, false)]);
+                Interlocked.Increment(ref written);
+            }
+        });
+
+        var readers = Enumerable.Range(0, 4).Select(_ => Task.Run(async () =>
+        {
+            while (!stop.IsCancellationRequested)
+            {
+                var now = DateTime.UtcNow;
+                var start = new DateTime(now.Ticks / TimeSpan.TicksPerSecond / 10 * 10 * TimeSpan.TicksPerSecond, DateTimeKind.Utc)
+                    .AddSeconds(-30);
+                var end = start.AddSeconds(70);
+                foreach (var agg in Aggregates)
+                foreach (var reverse in new[] { false, true })
+                {
+                    try
+                    {
+                        var r = await src.ReadProcessedAsync(tag, reverse ? end : start, reverse ? start : end,
+                            TimeSpan.FromSeconds(10), agg, CancellationToken.None);
+                        foreach (var v in r.Samples)
+                            if (!StatusBits.IsBad(v.StatusCode) && v.Value is null)
+                                faults.Add($"{agg} {(reverse ? "rev" : "fwd")}: non-Bad 0x{v.StatusCode:X8} with no value at {v.SourceTimestampUtc:O}");
+                    }
+                    catch (Exception ex)
+                    {
+                        faults.Add($"{agg} {(reverse ? "rev" : "fwd")}: {ex.GetType().Name}: {ex.Message}");
+                    }
+                    Interlocked.Increment(ref reads);
+                }
+            }
+        })).ToArray();
+
+        await Task.WhenAll(readers.Append(writer));
+        var summary = $"race stress: {seconds}s, {Interlocked.Read(ref written)} rows written, {Interlocked.Read(ref reads)} reads, {faults.Count} faults";
+        var output = TestContext.Current.TestOutputHelper;
+        output?.WriteLine(summary);
+        foreach (var f in faults.Take(20)) output?.WriteLine("fault: " + f);
+        written.ShouldBeGreaterThan(0);
+        faults.ShouldBeEmpty(summary);
+    }
+}
@@ -107,6 +107,19 @@ internal sealed class QuestDbConformanceBackend : IConformanceBackend
    /// <param name="tag">The tag.</param>
    /// <param name="rows">Rows: time (already rounded by the caller or not — it is bound through BindTimestamp), value, status, deleted.</param>
    public async Task InsertRowsAsync(string tag, (DateTime T, HistorianValue V, uint Status, bool Deleted)[] rows)
+    {
+        if (rows.Length == 0) return;
+        await InsertRowsNoWaitAsync(tag, rows);
+        await Wal.WaitAsync(Conn.SampleTable, CancellationToken.None);
+    }
+
+    /// <summary>
+    ///     <see cref="InsertRowsAsync"/> without the WAL visibility wait: the rows become visible whenever QuestDB applies
+    ///     them (the live write/read race stress test commits one row per call and never waits).
+    /// </summary>
+    /// <param name="tag">The tag.</param>
+    /// <param name="rows">Rows: time, value, status, deleted.</param>
+    public async Task InsertRowsNoWaitAsync(string tag, (DateTime T, HistorianValue V, uint Status, bool Deleted)[] rows)
    {
        if (rows.Length == 0) return;
        await using (var c = await Conn.OpenAsync(CancellationToken.None))
@@ -140,7 +153,6 @@ internal sealed class QuestDbConformanceBackend : IConformanceBackend
                await cmd.ExecuteNonQueryAsync();
            }
        }
-        await Wal.WaitAsync(Conn.SampleTable, CancellationToken.None);
    }

    /// <inheritdoc/>
@@ -0,0 +1,116 @@
+using System.Data;
+using Shouldly;
+using Xunit;
+using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
+using ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common;
+
+namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb.Tests;
+
+/// <summary>
+///     Live-gate 8 F1: the per-sample aggregate row mapper (<see cref="QuestDbHistorianDataSource.ReadBucket"/>) over a
+///     hand-built row in the <see cref="QuestDbAggregateQueries.StartEnd"/> column layout. A row whose counts include
+///     samples but whose first/last-of-any-quality picks are NULL used to throw <c>InvalidCastException: Column 'ts' is
+///     null</c> and fail the node's whole read; it now maps to absent picks the evaluator answers as <c>BadNoData</c>.
+/// </summary>
+public sealed class BucketRowMappingTests
+{
+    private const uint NoDataCalc = 0x809B0401u;
+    private static readonly DateTime T = new(2026, 10, 11, 12, 0, 0, DateTimeKind.Utc);
+
+    /// <summary>The StartEnd layout: b + 5 counts, two usable picks (kind, num, int, str, ts), two any picks (ts, kind, num, int, str, status).</summary>
+    private static DataTable StartEndTable()
+    {
+        var t = new DataTable();
+        t.Columns.Add("b", typeof(long));
+        foreach (var c in new[] { "total", "usable", "good", "bad", "nonnum" }) t.Columns.Add(c, typeof(long));
+        for (var p = 0; p < 2; p++)
+        {
+            t.Columns.Add($"k{p}", typeof(short));
+            t.Columns.Add($"n{p}", typeof(double));
+            t.Columns.Add($"i{p}", typeof(long));
+            t.Columns.Add($"s{p}", typeof(string));
+            t.Columns.Add($"t{p}", typeof(DateTime));
+        }
+        for (var p = 0; p < 2; p++)
+        {
+            t.Columns.Add($"at{p}", typeof(DateTime));
+            t.Columns.Add($"ak{p}", typeof(short));
+            t.Columns.Add($"an{p}", typeof(double));
+            t.Columns.Add($"ai{p}", typeof(long));
+            t.Columns.Add($"as{p}", typeof(string));
+            t.Columns.Add($"st{p}", typeof(long));
+        }
+        return t;
+    }
+
+    private static DataRow Counts(DataTable t, long bucket, long total)
+    {
+        var row = t.NewRow();
+        row[0] = bucket;
+        row[1] = total; row[2] = total; row[3] = total; row[4] = 0L; row[5] = 0L;
+        return row;
+    }
+
+    private static BucketRow Map(DataTable t, AggregateFamily family)
+    {
+        using var r = t.CreateDataReader();
+        r.Read().ShouldBeTrue();
+        return QuestDbHistorianDataSource.ReadBucket(r, family);
+    }
+
+    [Fact]
+    public void A_row_with_counts_but_null_first_and_last_of_any_quality_maps_to_absent_picks_without_throwing()
+    {
+        var t = StartEndTable();
+        var row = Counts(t, 3, 2);
+        // A non-matching BYTE pick reads back as kind 0 (BYTE has no NULL) with every other column NULL.
+        row["k0"] = (short)0;
+        row["k1"] = (short)0;
+        t.Rows.Add(row);
+
+        var b = Map(t, AggregateFamily.StartEnd);
+        b.Bucket.ShouldBe(3);
+        b.Total.ShouldBe(2);
+        b.FirstAny.ShouldBeNull();
+        b.LastAny.ShouldBeNull();
+        b.First.Kind.ShouldBe(HistorianValueKind.Empty);
+        b.Last.Kind.ShouldBe(HistorianValueKind.Empty);
+
+        foreach (var agg in new[] { HistoryAggregateType.Start, HistoryAggregateType.End, HistoryAggregateType.Delta })
+            AggregateEvaluator.Evaluate(agg, b, T, T).StatusCode.ShouldBe(NoDataCalc);
+    }
+
+    [Fact]
+    public void A_consistent_row_still_maps_every_pick()
+    {
+        var t = StartEndTable();
+        var row = Counts(t, 0, 2);
+        row["k0"] = (short)1; row["n0"] = 10d; row["t0"] = T;
+        row["k1"] = (short)1; row["n1"] = 20d; row["t1"] = T.AddSeconds(1);
+        row["at0"] = T; row["ak0"] = (short)1; row["an0"] = 10d; row["st0"] = 0L;
+        row["at1"] = T.AddSeconds(1); row["ak1"] = (short)1; row["an1"] = 20d; row["st1"] = 0L;
+        t.Rows.Add(row);
+
+        var b = Map(t, AggregateFamily.StartEnd);
+        b.First.ShouldBe(HistorianValue.FromDouble(10));
+        b.Last.ShouldBe(HistorianValue.FromDouble(20));
+        b.FirstAny.ShouldNotBeNull().TimeUtc.ShouldBe(T);
+        b.LastAny.ShouldNotBeNull().Value.ShouldBe(HistorianValue.FromDouble(20));
+        AggregateEvaluator.Evaluate(HistoryAggregateType.Delta, b, T, T).Value.ShouldBe(10d);
+    }
+
+    [Fact]
+    public void An_actual_time_row_with_counts_but_no_extreme_is_no_data()
+    {
+        var t = new DataTable();
+        t.Columns.Add("b", typeof(long));
+        foreach (var c in new[] { "total", "usable", "good", "bad", "nonnum" }) t.Columns.Add(c, typeof(long));
+        t.Columns.Add("num_value", typeof(double));
+        t.Columns.Add("ts", typeof(DateTime));
+        t.Rows.Add(0L, 1L, 1L, 1L, 0L, 0L, DBNull.Value, DBNull.Value);
+
+        var b = Map(t, AggregateFamily.ActualTime);
+        b.Extreme.ShouldBeNull();
+        AggregateEvaluator.Evaluate(HistoryAggregateType.MaximumActualTime, b, T, T).StatusCode.ShouldBe(NoDataCalc);
+    }
+}
@@ -257,4 +257,52 @@ public sealed class AggregateEvaluatorTests
        avg.Value.ShouldBe(15d);
        avg.StatusCode.ShouldBe(UncertainCalc);
    }
+
+    // ---- Live-gate 8 F1: a row whose counts and picks disagree (a backend that read them from two table versions).
+
+    /// <summary>Counts say B0 holds two Good samples, but none of the picked samples came back.</summary>
+    private static BucketRow InconsistentRow() => new()
+    {
+        Bucket = 0, Total = 2, Usable = 2, Good = 2, Bad = 0, NonNumeric = 0, Avg = 15, Min = 10, Max = 20, Sum = 30,
+        First = HistorianValue.Empty, Last = HistorianValue.Empty, Extreme = null, ExtremeTime = null,
+        FirstAny = null, LastAny = null,
+    };
+
+    [Theory]
+    [InlineData(HistoryAggregateType.Start)]
+    [InlineData(HistoryAggregateType.End)]
+    [InlineData(HistoryAggregateType.Delta)]
+    [InlineData(HistoryAggregateType.MinimumActualTime)]
+    [InlineData(HistoryAggregateType.MaximumActualTime)]
+    public void A_bucket_whose_picked_samples_are_missing_is_no_data_not_a_throw_or_a_valueless_good(HistoryAggregateType agg)
+    {
+        var v = AggregateEvaluator.Evaluate(agg, InconsistentRow(), At(0), Now);
+        v.Value.ShouldBeNull();
+        v.StatusCode.ShouldBe(NoDataCalc);
+    }
+
+    [Fact]
+    public void A_delta_whose_first_usable_pick_alone_is_missing_is_no_data_not_invalid_inputs()
+    {
+        var row = InconsistentRow();
+        row.Last = HistorianValue.FromDouble(20);
+        row.FirstAny = A[0];
+        row.LastAny = A[1];
+        AggregateEvaluator.Evaluate(HistoryAggregateType.Delta, row, At(0), Now).StatusCode.ShouldBe(NoDataCalc);
+    }
+
+    [Fact]
+    public void An_inconsistent_bucket_does_not_fail_the_whole_read_and_its_neighbours_keep_their_values()
+    {
+        var rows = RowsA();
+        rows[0] = InconsistentRow();
+        var plan = BucketPlan.Create(At(0), At(80), TimeSpan.FromSeconds(20));
+        foreach (var agg in new[] { HistoryAggregateType.Start, HistoryAggregateType.Delta, HistoryAggregateType.MaximumActualTime })
+        {
+            var s = AggregateEvaluator.EvaluateAll(agg, plan, rows.Values, At(0), At(40), Now);
+            s.Count.ShouldBe(4);
+            s[0].StatusCode.ShouldBe(NoDataCalc);
+            s[2].Value.ShouldNotBeNull();
+        }
+    }
 }
@@ -0,0 +1,112 @@
+using System.Data;
+using Shouldly;
+using Xunit;
+using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
+using ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common;
+
+namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer.Tests;
+
+/// <summary>
+///     Live-gate 8 F1 (defence in depth on SQL Server, where it was not observed): the per-sample aggregate row mapper
+///     (<see cref="SqlServerHistorianDataSource.ReadBucket"/>) over a hand-built row in the
+///     <see cref="SqlAggregateQueries.StartEnd"/> column layout. A row whose counts include samples but whose
+///     first/last-of-any-quality picks are NULL used to throw out of <c>ReadStored</c> and fail the node's whole read; it now
+///     maps to absent picks the evaluator answers as <c>BadNoData</c>.
+/// </summary>
+public sealed class BucketRowMappingTests
+{
+    private const uint NoDataCalc = 0x809B0401u;
+    private static readonly DateTime T = new(2026, 10, 11, 12, 0, 0, DateTimeKind.Utc);
+
+    /// <summary>b + 5 counts, two usable picks (kind, num, int, str, ts), two any picks (ts, kind, num, int, str, status).</summary>
+    private static DataTable StartEndTable()
+    {
+        var t = new DataTable();
+        t.Columns.Add("b", typeof(long));
+        foreach (var c in new[] { "total", "usable", "good", "bad", "nonnum" }) t.Columns.Add(c, typeof(int));
+        for (var p = 0; p < 2; p++)
+        {
+            t.Columns.Add($"k{p}", typeof(byte));
+            t.Columns.Add($"n{p}", typeof(double));
+            t.Columns.Add($"i{p}", typeof(long));
+            t.Columns.Add($"s{p}", typeof(string));
+            t.Columns.Add($"t{p}", typeof(DateTime));
+        }
+        for (var p = 0; p < 2; p++)
+        {
+            t.Columns.Add($"at{p}", typeof(DateTime));
+            t.Columns.Add($"ak{p}", typeof(byte));
+            t.Columns.Add($"an{p}", typeof(double));
+            t.Columns.Add($"ai{p}", typeof(long));
+            t.Columns.Add($"as{p}", typeof(string));
+            t.Columns.Add($"st{p}", typeof(int));
+        }
+        return t;
+    }
+
+    private static DataRow Counts(DataTable t, long bucket, int total)
+    {
+        var row = t.NewRow();
+        row[0] = bucket;
+        row[1] = total; row[2] = total; row[3] = total; row[4] = 0; row[5] = 0;
+        return row;
+    }
+
+    private static BucketRow Map(DataTable t, AggregateFamily family)
+    {
+        using var r = t.CreateDataReader();
+        r.Read().ShouldBeTrue();
+        return SqlServerHistorianDataSource.ReadBucket(r, family);
+    }
+
+    [Fact]
+    public void A_row_with_counts_but_null_first_and_last_of_any_quality_maps_to_absent_picks_without_throwing()
+    {
+        var t = StartEndTable();
+        t.Rows.Add(Counts(t, 3, 2));
+
+        var b = Map(t, AggregateFamily.StartEnd);
+        b.Bucket.ShouldBe(3);
+        b.Total.ShouldBe(2);
+        b.FirstAny.ShouldBeNull();
+        b.LastAny.ShouldBeNull();
+        b.First.Kind.ShouldBe(HistorianValueKind.Empty);
+
+        foreach (var agg in new[] { HistoryAggregateType.Start, HistoryAggregateType.End, HistoryAggregateType.Delta })
+            AggregateEvaluator.Evaluate(agg, b, T, T).StatusCode.ShouldBe(NoDataCalc);
+    }
+
+    [Fact]
+    public void A_consistent_row_still_maps_every_pick()
+    {
+        var t = StartEndTable();
+        var row = Counts(t, 0, 2);
+        row["k0"] = (byte)1; row["n0"] = 10d; row["t0"] = T;
+        row["k1"] = (byte)1; row["n1"] = 20d; row["t1"] = T.AddSeconds(1);
+        row["at0"] = T; row["ak0"] = (byte)1; row["an0"] = 10d; row["st0"] = 0;
+        row["at1"] = T.AddSeconds(1); row["ak1"] = (byte)1; row["an1"] = 20d; row["st1"] = 0;
+        t.Rows.Add(row);
+
+        var b = Map(t, AggregateFamily.StartEnd);
+        b.First.ShouldBe(HistorianValue.FromDouble(10));
+        b.Last.ShouldBe(HistorianValue.FromDouble(20));
+        b.FirstAny.ShouldNotBeNull().TimeUtc.ShouldBe(T);
+        b.LastAny.ShouldNotBeNull().Value.ShouldBe(HistorianValue.FromDouble(20));
+        AggregateEvaluator.Evaluate(HistoryAggregateType.Delta, b, T, T).Value.ShouldBe(10d);
+    }
+
+    [Fact]
+    public void An_actual_time_row_with_counts_but_no_extreme_is_no_data()
+    {
+        var t = new DataTable();
+        t.Columns.Add("b", typeof(long));
+        foreach (var c in new[] { "total", "usable", "good", "bad", "nonnum" }) t.Columns.Add(c, typeof(int));
+        t.Columns.Add("NumValue", typeof(double));
+        t.Columns.Add("SourceTimeUtc", typeof(DateTime));
+        t.Rows.Add(0L, 1, 1, 1, 0, 0, DBNull.Value, DBNull.Value);
+
+        var b = Map(t, AggregateFamily.ActualTime);
+        b.Extreme.ShouldBeNull();
+        AggregateEvaluator.Evaluate(HistoryAggregateType.MaximumActualTime, b, T, T).StatusCode.ShouldBe(NoDataCalc);
+    }
+}