fix(historian): page by the backend's timestamp resolution; QuestDB AtTime stamps the requested time

IHistorianDataSource.TimestampResolutionTicks (default 1; QuestDB 10). A drained tie cluster now steps off T by the resolution, so a forward one-value-per-page read on a microsecond backend no longer re-reads the same sample forever (C1). QuestDB AtTime results carry the client's original request time (M3); regression for the NEXT seek skipping a tombstone (M1). Conformance backend wires the real writer and updater. Claude-Session: https://claude.ai/code/session_01Ef2ozxS7QPanGT8uoATbTe
2026-10-11 02:30:25 -04:00
parent 13d628a07f
commit d2be213e86
9 changed files with 330 additions and 19 deletions
@@ -0,0 +1,85 @@
+using Shouldly;
+using Xunit;
+using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
+using ZB.MOM.WW.OtOpcUa.Driver.Historian.Conformance;
+using ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common;
+
+namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb.IntegrationTests;
+
+/// <summary>
+///     QuestDB AtTime regressions found in the T5 review (M1, M3), and the data source's reported timestamp
+///     resolution (the HistoryRead paging step, C1).
+/// </summary>
+[Collection(QuestDbCollection.Name)]
+public sealed class AtTimeRegressionTests(QuestDbFixture fx) : IAsyncLifetime
+{
+    private static readonly CancellationToken Ct = CancellationToken.None;
+    private const uint GoodInterpolated = 0x00000402u;
+
+    private QuestDbConformanceBackend _b = null!;
+
+    private static DateTime At(double s) => ConformanceData.At(s);
+
+    public async ValueTask InitializeAsync() => _b = await QuestDbConformanceBackend.CreateConcreteAsync(fx);
+
+    public async ValueTask DisposeAsync()
+    {
+        if (_b is not null) await _b.DisposeAsync();
+    }
+
+    /// <summary>
+    ///     M3: a request time with a sub-microsecond remainder is answered stamped with THAT time (as SQL Server does),
+    ///     not with the rounded time the seek used. The value is still the one at the rounded time.
+    /// </summary>
+    [Fact]
+    public async Task AtTime_result_is_stamped_with_the_original_sub_microsecond_request_time()
+    {
+        await ConformanceData.SeedAAsync(_b);                    // +0=10, +10=20, ...
+        var requested = _b.Round(At(5)).AddTicks(3);             // 0.3 µs past a microsecond (T0 itself is not µs-aligned)
+        var unspecified = DateTime.SpecifyKind(_b.Round(At(7)).AddTicks(7), DateTimeKind.Unspecified);
+
+        var s = (await _b.CreateDataSource().ReadAtTimeAsync("A", [requested, unspecified], Ct)).Samples;
+
+        s[0].SourceTimestampUtc.ShouldBe(requested);
+        s[0].SourceTimestampUtc!.Value.Kind.ShouldBe(DateTimeKind.Utc);
+        s[0].Value.ShouldBe(15d);                                // evaluated at the rounded time Round(At(5))
+        s[0].StatusCode.ShouldBe(GoodInterpolated);
+        s[1].SourceTimestampUtc.ShouldBe(DateTime.SpecifyKind(unspecified, DateTimeKind.Utc));
+        s[1].SourceTimestampUtc!.Value.Ticks.ShouldBe(unspecified.Ticks);
+    }
+
+    /// <summary>M3 on the unknown-tag path: every BadNoData entry carries its own requested time.</summary>
+    [Fact]
+    public async Task AtTime_unknown_tag_entries_are_stamped_with_the_original_request_time()
+    {
+        var requested = _b.Round(At(5)).AddTicks(3);
+        var s = (await _b.CreateDataSource().ReadAtTimeAsync("nope", [requested], Ct)).Samples;
+        s.Single().StatusCode.ShouldBe(HistorianStatusCodes.BadNoData);
+        s.Single().SourceTimestampUtc.ShouldBe(requested);
+    }
+
+    /// <summary>
+    ///     M1: the NEXT seek skips a tombstone and finds the next live row. With +10 deleted, At(5) lies between the
+    ///     live +0 (10) and the live +20 (30), so it interpolates to 15, Good. Seeing the tombstone as "next" would
+    ///     instead hold 10 as Uncertain (an Empty BadNoData neighbour is not usable).
+    /// </summary>
+    [Fact]
+    public async Task AtTime_next_seek_skips_a_tombstone_and_finds_the_next_live_row()
+    {
+        await _b.SeedAsync("M",
+            (At(0), HistorianValue.FromDouble(10), ConformanceData.Good),
+            (At(10), HistorianValue.FromDouble(20), ConformanceData.Good),
+            (At(20), HistorianValue.FromDouble(30), ConformanceData.Good));
+        await _b.InsertRowsAsync("M", [(At(10), HistorianValue.Empty, HistorianStatusCodes.BadNoData, true)]);
+
+        var s = (await _b.CreateDataSource().ReadAtTimeAsync("M", [At(5)], Ct)).Samples.Single();
+
+        s.Value.ShouldBe(15d);                                   // 10 + (30 - 10) * 5/20
+        s.StatusCode.ShouldBe(GoodInterpolated);
+    }
+
+    /// <summary>C1: the data source reports QuestDB's microsecond storage resolution to the paging code.</summary>
+    [Fact]
+    public void Data_source_reports_a_ten_tick_timestamp_resolution() =>
+        ((IHistorianDataSource)_b.CreateDataSource()).TimestampResolutionTicks.ShouldBe(10);
+}
@@ -0,0 +1,55 @@
+using ZB.MOM.WW.OtOpcUa.Driver.Historian.Conformance;
+using Xunit;
+
+namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb.IntegrationTests;
+
+// One concrete subclass per conformance class: the suite's tests run here against QuestDB.
+
+[Collection(QuestDbCollection.Name)]
+public sealed class QuestDbRawReadConformance(QuestDbFixture fx) : RawReadConformance
+{
+    /// <inheritdoc/>
+    protected override Task<IConformanceBackend> CreateBackendAsync() => QuestDbConformanceBackend.CreateAsync(fx);
+}
+
+[Collection(QuestDbCollection.Name)]
+public sealed class QuestDbAtTimeConformance(QuestDbFixture fx) : AtTimeConformance
+{
+    /// <inheritdoc/>
+    protected override Task<IConformanceBackend> CreateBackendAsync() => QuestDbConformanceBackend.CreateAsync(fx);
+}
+
+[Collection(QuestDbCollection.Name)]
+public sealed class QuestDbProcessedConformance(QuestDbFixture fx) : ProcessedConformance
+{
+    /// <inheritdoc/>
+    protected override Task<IConformanceBackend> CreateBackendAsync() => QuestDbConformanceBackend.CreateAsync(fx);
+}
+
+[Collection(QuestDbCollection.Name)]
+public sealed class QuestDbQualityContractConformance(QuestDbFixture fx) : QualityContractConformance
+{
+    /// <inheritdoc/>
+    protected override Task<IConformanceBackend> CreateBackendAsync() => QuestDbConformanceBackend.CreateAsync(fx);
+}
+
+[Collection(QuestDbCollection.Name)]
+public sealed class QuestDbLiveWriteConformance(QuestDbFixture fx) : LiveWriteConformance
+{
+    /// <inheritdoc/>
+    protected override Task<IConformanceBackend> CreateBackendAsync() => QuestDbConformanceBackend.CreateAsync(fx);
+}
+
+[Collection(QuestDbCollection.Name)]
+public sealed class QuestDbUpdateConformance(QuestDbFixture fx) : UpdateConformance
+{
+    /// <inheritdoc/>
+    protected override Task<IConformanceBackend> CreateBackendAsync() => QuestDbConformanceBackend.CreateAsync(fx);
+}
+
+[Collection(QuestDbCollection.Name)]
+public sealed class QuestDbTagNameConformance(QuestDbFixture fx) : TagNameConformance
+{
+    /// <inheritdoc/>
+    protected override Task<IConformanceBackend> CreateBackendAsync() => QuestDbConformanceBackend.CreateAsync(fx);
+}
@@ -76,14 +76,13 @@ internal sealed class QuestDbConformanceBackend : IConformanceBackend
        new QuestDbHistorianDataSource(Conn, Boot, NewCatalog(), timeProvider: clock);

    /// <inheritdoc/>
-    /// <remarks>TODO(T7): build the T4 <c>QuestDbHistorianValueWriter</c> here once T4 is merged.</remarks>
+    /// <remarks>The T4 live-value writer over a fresh catalog (fresh positive cache) and this instance's WAL wait.</remarks>
    public IHistorianValueWriter CreateValueWriter(Func<DateTime>? utcNow = null) =>
-        throw new NotSupportedException("TODO(T7): the QuestDB value writer (T4) is not merged into this branch yet.");
+        new QuestDbHistorianValueWriter(Conn, NewCatalog(), Wal, utcNow: utcNow);

    /// <inheritdoc/>
-    /// <remarks>TODO(T7): build the T6 <c>QuestDbHistorianDataUpdater</c> here once T6 is merged.</remarks>
-    public IHistorianDataUpdater CreateUpdater() =>
-        throw new NotSupportedException("TODO(T7): the QuestDB history updater (T6) is not merged into this branch yet.");
+    /// <remarks>The T6 soft-delete updater over a fresh catalog and this instance's WAL wait.</remarks>
+    public IHistorianDataUpdater CreateUpdater() => new QuestDbHistorianDataUpdater(Conn, NewCatalog(), Wal);

    /// <inheritdoc/>
    /// <remarks>
@@ -345,6 +345,51 @@ public sealed class HistoryPagingTests
        store.TryTake(null, cp3).ShouldNotBeNull(); // retained
    }

+    [Fact]
+    public void SliceTieCluster_forward_drain_steps_by_the_backend_resolution()
+    {
+        // A microsecond backend (QuestDB, 10 ticks) truncates T + 1 tick back to T, so the drained cluster must be
+        // stepped off by the resolution, not by one tick, or the same sample comes back forever.
+        var t = new DateTime(2026, 1, 1, 0, 0, 5, DateTimeKind.Utc);
+
+        HistoryPaging.SliceTieCluster(
+            clusterCount: 1, skip: 1, cap: 1, boundaryT: t, endUtc: t.AddHours(1),
+            out _, out var sliceCount, out var nextStartUtc, out var nextSkip, resolutionTicks: 10);
+
+        sliceCount.ShouldBe(0);
+        nextStartUtc.ShouldBe(t.AddTicks(10));
+        nextSkip.ShouldBe(0);
+    }
+
+    [Fact]
+    public void SliceTieCluster_reverse_drain_steps_down_by_the_backend_resolution()
+    {
+        var t = new DateTime(2026, 1, 1, 0, 0, 5, DateTimeKind.Utc);
+
+        HistoryPaging.SliceTieCluster(
+            clusterCount: 1, skip: 1, cap: 1, boundaryT: t, endUtc: t.AddHours(-1),
+            out _, out _, out var nextStartUtc, out _, reverse: true, resolutionTicks: 10);
+
+        nextStartUtc.ShouldBe(t.AddTicks(-10));
+    }
+
+    [Fact]
+    public void SliceTieCluster_forward_drain_terminates_when_one_resolution_step_passes_the_end()
+    {
+        var t = new DateTime(2026, 1, 1, 0, 0, 5, DateTimeKind.Utc);
+
+        HistoryPaging.SliceTieCluster(
+            clusterCount: 1, skip: 1, cap: 1, boundaryT: t, endUtc: t.AddTicks(9),
+            out _, out _, out var nextStartUtc, out _, resolutionTicks: 10);
+
+        nextStartUtc.ShouldBeNull();
+    }
+
+    [Fact]
+    public void SliceTieCluster_rejects_a_resolution_below_one() =>
+        Should.Throw<ArgumentOutOfRangeException>(() => HistoryPaging.SliceTieCluster(
+            1, 1, 1, DateTime.UtcNow, DateTime.UtcNow.AddHours(1), out _, out _, out _, out _, resolutionTicks: 0));
+
    private static HistoryContinuationState RawState() => new(
        "WW.Tag", DateTime.UtcNow, DateTime.UtcNow.AddHours(1),
        BoundarySkipCount: 1, NumValuesPerNode: 50);
@@ -357,6 +357,56 @@ public sealed class NodeManagerHistoryReadPagingTests : IDisposable
        await host.DisposeAsync();
    }

+    /// <summary>
+    ///     Forward paging at <c>NumValuesPerNode = 1</c> over a MICROSECOND backend (QuestDB: it truncates every inbound
+    ///     time to 10 ticks and reports <c>TimestampResolutionTicks = 10</c>) returns every sample exactly once and
+    ///     terminates. Before the fix the drained one-sample "cluster" resumed at <c>T + 1 tick</c>, which the backend
+    ///     truncated back to <c>T</c>: the same sample came back with no boundary skip, and the read looped forever.
+    /// </summary>
+    [Fact]
+    public async Task Raw_forward_read_with_one_value_per_page_on_a_microsecond_backend_returns_every_sample_once()
+    {
+        var (host, server) = await BootAsync();
+        var nm = server.NodeManager!;
+        nm.HistoryContinuationStore = new InMemoryHistoryContinuationStore();
+        // Microsecond-spaced samples, so a one-tick step would land strictly between two of them.
+        var series = Enumerable.Range(0, 6)
+            .Select(i => { var t = Epoch.AddTicks(10L * i); return new DataValueSnapshot((double)i, StatusCodes.Good, t, t); })
+            .ToArray();
+        nm.HistorianDataSource = new MicrosecondHistorianDataSource(series);
+
+        nm.EnsureVariable("eq-1/us", parentFolderNodeId: null, displayName: "Us", dataType: "Double",
+            writable: false, historianTagname: "WW.Us", realm: AddressSpaceRealm.Uns);
+        var nodeId = nm.TryGetVariable("eq-1/us")!.NodeId;
+
+        var collected = await DrainAsync(nm, nodeId, start: Epoch, end: Epoch.AddHours(1), max: 1);
+
+        collected.ShouldBe([0d, 1d, 2d, 3d, 4d, 5d]);
+        await host.DisposeAsync();
+    }
+
+    /// <summary>The reverse mirror on the same microsecond backend: newest first, each sample once.</summary>
+    [Fact]
+    public async Task Raw_reverse_read_with_one_value_per_page_on_a_microsecond_backend_returns_every_sample_once()
+    {
+        var (host, server) = await BootAsync();
+        var nm = server.NodeManager!;
+        nm.HistoryContinuationStore = new InMemoryHistoryContinuationStore();
+        var series = Enumerable.Range(0, 6)
+            .Select(i => { var t = Epoch.AddTicks(10L * (i + 1)); return new DataValueSnapshot((double)i, StatusCodes.Good, t, t); })
+            .ToArray();
+        nm.HistorianDataSource = new MicrosecondHistorianDataSource(series);
+
+        nm.EnsureVariable("eq-1/usrev", parentFolderNodeId: null, displayName: "UsRev", dataType: "Double",
+            writable: false, historianTagname: "WW.UsRev", realm: AddressSpaceRealm.Uns);
+        var nodeId = nm.TryGetVariable("eq-1/usrev")!.NodeId;
+
+        var collected = await DrainAsync(nm, nodeId, start: Epoch.AddHours(1), end: Epoch, max: 1);
+
+        collected.ShouldBe([5d, 4d, 3d, 2d, 1d, 0d]);
+        await host.DisposeAsync();
+    }
+
    /// <summary>Pages a Raw read to completion and returns the union of the pages in arrival order.</summary>
    private static Task<List<double>> DrainAsync(OtOpcUaNodeManager nm, NodeId nodeId, DateTime start, DateTime end, uint max)
    {
@@ -520,6 +570,43 @@ public sealed class NodeManagerHistoryReadPagingTests : IDisposable
        public void Dispose() { }
    }

+    /// <summary>
+    ///     A microsecond-resolution backend, shaped like QuestDB: every inbound window bound is TRUNCATED to 10 ticks
+    ///     (as <c>QuestDbConnection.Round</c> does) before the series is filtered, and the source reports
+    ///     <see cref="TimestampResolutionTicks"/> = 10.
+    /// </summary>
+    private sealed class MicrosecondHistorianDataSource(IReadOnlyList<DataValueSnapshot> series) : IHistorianDataSource
+    {
+        private readonly SeriesHistorianDataSource _inner = new(series);
+
+        public long TimestampResolutionTicks => 10;
+
+        private static DateTime Truncate(DateTime t) => new(t.Ticks - t.Ticks % 10, DateTimeKind.Utc);
+
+        public Task<HistorianRead> ReadRawAsync(
+            string fullReference, DateTime startUtc, DateTime endUtc, uint maxValuesPerNode,
+            CancellationToken cancellationToken) =>
+            _inner.ReadRawAsync(fullReference, Truncate(startUtc), Truncate(endUtc), maxValuesPerNode, cancellationToken);
+
+        public Task<HistorianRead> ReadProcessedAsync(
+            string fullReference, DateTime startUtc, DateTime endUtc, TimeSpan interval,
+            HistoryAggregateType aggregate, CancellationToken cancellationToken) =>
+            _inner.ReadProcessedAsync(fullReference, startUtc, endUtc, interval, aggregate, cancellationToken);
+
+        public Task<HistorianRead> ReadAtTimeAsync(
+            string fullReference, IReadOnlyList<DateTime> timestampsUtc, CancellationToken cancellationToken) =>
+            _inner.ReadAtTimeAsync(fullReference, timestampsUtc, cancellationToken);
+
+        public Task<HistoricalEventsResult> ReadEventsAsync(
+            string? sourceName, DateTime startUtc, DateTime endUtc, int maxEvents,
+            CancellationToken cancellationToken) =>
+            _inner.ReadEventsAsync(sourceName, startUtc, endUtc, maxEvents, cancellationToken);
+
+        public HistorianHealthSnapshot GetHealthSnapshot() => _inner.GetHealthSnapshot();
+
+        public void Dispose() { }
+    }
+
    private async Task<(OpcUaApplicationHost Host, OtOpcUaSdkServer Server)> BootAsync()
    {
        var applicationUri = $"urn:OtOpcUa.HistoryPagingTest:{Guid.NewGuid():N}";