diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Abstractions/Historian/IHistorianDataSource.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Abstractions/Historian/IHistorianDataSource.cs index 2cba63ed..926881c1 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Abstractions/Historian/IHistorianDataSource.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Abstractions/Historian/IHistorianDataSource.cs @@ -17,6 +17,12 @@ public interface IHistorianDataSource : IDisposable /// /// Read raw historical samples for a single tag over a time range. /// + /// + /// Forward (startUtc < endUtc) is [start, end) ascending; startUtc == endUtc is a point read. + /// startUtc > endUtc is the Part 11 reverse read (end, start] newest-first — but only a backend that + /// declares is ever sent one (Gitea #686); the server serves a reverse read on any + /// other backend from bounded forward reads. + /// /// The full reference of the tag to read. /// The start of the time range in UTC. /// The end of the time range in UTC. @@ -30,6 +36,16 @@ public interface IHistorianDataSource : IDisposable uint maxValuesPerNode, CancellationToken cancellationToken); + /// + /// Whether this backend can serve a reverse (newest-first) raw read itself — with + /// startUtc > endUtc (Gitea #686). The server checks it before every raw read. When false it never + /// sends start > end: it serves the reverse read — an explicit one, a resumed reverse page, and the + /// end-only capped read Part 11 reads backward from EndTime — from a series of bounded FORWARD reads walking back + /// from the later bound (ReverseRawReader in the OPC UA server). Default false, the safe direction: a + /// source that only reads forward is still answered correctly. A backend declares it itself (a guard test pins that). + /// + bool SupportsReverseRaw => false; + /// /// Read processed (interval-bucketed) samples — average / min / max / count / etc. /// A bucket with no source data returns a sample whose diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Abstractions/Historian/NullHistorianDataSource.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Abstractions/Historian/NullHistorianDataSource.cs index d0553988..769f1d4e 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Abstractions/Historian/NullHistorianDataSource.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Abstractions/Historian/NullHistorianDataSource.cs @@ -55,6 +55,11 @@ public sealed class NullHistorianDataSource : IHistorianDataSource int maxEvents, CancellationToken cancellationToken) => Task.FromResult(EmptyEvents); + /// + /// true (Gitea #686): there is no history, so the empty result is the correct answer in either + /// direction — no need for the server to walk back through forward reads to find nothing. + public bool SupportsReverseRaw => true; + /// /// false (Gitea #675): there is no history to read in either direction, and an explicit reverse read /// is answered BadHistoryOperationUnsupported like any backend that cannot read newest-first. diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway/GatewayHistorianDataSource.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway/GatewayHistorianDataSource.cs index aa87cbef..5fcdf0c5 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway/GatewayHistorianDataSource.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway/GatewayHistorianDataSource.cs @@ -76,10 +76,18 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis } /// + /// + /// A reverse window (start > end) is refused with before anything is sent + /// and is not recorded as a health failure (Gitea #686): the gateway rejects it InvalidArgument, and the server + /// never asks — is false, so it walks a reverse read back with forward reads. + /// public async Task ReadRawAsync( string fullReference, DateTime startUtc, DateTime endUtc, uint maxValuesPerNode, CancellationToken cancellationToken) { + if (startUtc > endUtc) + throw new NotSupportedException("The HistorianGateway reads raw history oldest-first only (start > end)."); + try { // The gateway seam caps with an int; OPC UA hands us a uint, so clamp to int range. @@ -235,6 +243,14 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis } } + /// + /// + /// false (Gitea #686): the gateway's ReadRaw streams ascending only and rejects end < start + /// (InvalidArgument). The server serves reverse and end-only capped raw reads from bounded forward reads + /// instead, and never sends this source a reverse window ( refuses one unsent). + /// + public bool SupportsReverseRaw => false; + /// /// /// false: the gateway's ReadEvents streams ascending only (Gitea #675). The server therefore refuses an diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb/QuestDbHistorianDataSource.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb/QuestDbHistorianDataSource.cs index 5bc333ea..07256c45 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb/QuestDbHistorianDataSource.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb/QuestDbHistorianDataSource.cs @@ -355,6 +355,10 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource // ------------------------------------------------------------------------------------------ Events + /// + /// Reverse raw reads are served natively ((end, start], ORDER BY … DESC; #669/#686). + public bool SupportsReverseRaw => true; + /// /// Reverse events reads are served natively (ORDER BY … DESC, Gitea #675). public bool SupportsReverseEvents => true; diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/SqlServerHistorianDataSource.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/SqlServerHistorianDataSource.cs index 4c7f1a44..27aeaf4c 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/SqlServerHistorianDataSource.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/SqlServerHistorianDataSource.cs @@ -312,6 +312,10 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource // ------------------------------------------------------------------------------------------ Events + /// + /// Reverse raw reads are served natively ((end, start], ORDER BY … DESC; #669/#686). + public bool SupportsReverseRaw => true; + /// /// Reverse events reads are served natively (ORDER BY … DESC, Gitea #675). public bool SupportsReverseEvents => true; diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/OtOpcUaNodeManager.cs b/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/OtOpcUaNodeManager.cs index 3ae1fe4e..77712df5 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/OtOpcUaNodeManager.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/OtOpcUaNodeManager.cs @@ -6219,6 +6219,8 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2 // Part 11: with NumValuesPerNode > 0 one bound may be left unset. Only a StartTime ⇒ page FORWARD from // it to an open upper bound; only an EndTime ⇒ page BACKWARD from it (a reverse read, start > end). // Without this an unset EndTime (MinValue) looked like a reverse read to a lower bound of "now". + // Every backend serves the backward read (#686): natively when it declares SupportsReverseRaw, otherwise + // walked back from bounded forward reads below — unlike the events path, which keeps it forward there. if (numValuesPerNode > 0) { if (endUtc == DateTime.MinValue && startUtc != DateTime.MinValue) @@ -6229,9 +6231,20 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2 } // HistoryRead is NOT under the node-manager Lock — awaiting the async source is safe. - var sourceResult = await HistorianDataSource - .ReadRawAsync(tagname, startUtc, endUtc, numValuesPerNode, ct) - .ConfigureAwait(false); + // A reverse window (explicit, resumed, or the end-only read above) goes to the backend only when it declares it + // can read newest-first; otherwise it is walked back from bounded forward reads (#686 — the Gateway rejects + // start > end, which failed the node and counted as a backend fault). + var source = HistorianDataSource; + var sourceResult = startUtc > endUtc && !source.SupportsReverseRaw + ? new HistorianRead( + await ReverseRawReader.ReadAsync( + source, tagname, startUtc, endUtc, numValuesPerNode, + (int)Math.Min(int.MaxValue, (long)Math.Max(1, MaxTieClusterOverfetch) + 1), ct) + .ConfigureAwait(false), + ContinuationPoint: null) + : await source + .ReadRawAsync(tagname, startUtc, endUtc, numValuesPerNode, ct) + .ConfigureAwait(false); var backendFull = HistoryPaging.IsFullPage(sourceResult.Samples.Count, numValuesPerNode); @@ -6348,6 +6361,14 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2 errors[handle.Index] = StatusCodes.BadTimeout; results[handle.Index] = new SdkHistoryReadResult { StatusCode = StatusCodes.BadTimeout }; } + catch (HistoryReadLimitExceededException ex) + { + // A request limit (#686: a reverse read walked back from forward reads met a tie cluster over the row + // budget) — BadTooManyOperations for THIS node, as on the processed path; not a backend fault. + _logger.LogWarning("OtOpcUaNodeManager: HistoryReadRaw limit exceeded for node {NodeId}: {Message}", handle.NodeId, ex.Message); + errors[handle.Index] = StatusCodes.BadTooManyOperations; + results[handle.Index] = new SdkHistoryReadResult { StatusCode = StatusCodes.BadTooManyOperations }; + } catch (Exception ex) { // One node's backend failure must not throw out of the batch — Bad for THIS node only. diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/ReverseRawReader.cs b/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/ReverseRawReader.cs new file mode 100644 index 00000000..feadf0e0 --- /dev/null +++ b/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/ReverseRawReader.cs @@ -0,0 +1,120 @@ +using ZB.MOM.WW.OtOpcUa.Core.Abstractions; + +namespace ZB.MOM.WW.OtOpcUa.OpcUaServer; + +/// +/// Gitea #686: serves a Part 11 reverse raw read — (lo, hi] newest-first, capped — on a historian backend that can +/// only read forward ( = false: the Gateway, whose +/// ReadRaw streams ascending and rejects start > end). +/// +/// +/// +/// Walk back in segments. Starting at hi, read one segment (segLo, segHi] FORWARD, keep it +/// newest-first, and step to the next older segment until the cap is met or lo is reached. A segment starts +/// at and grows ×4 while segments come back sparse, so "the latest N" of a live tag is +/// usually one read, and sparse history decades back is found in a few dozen tiny reads rather than one read of +/// all history. +/// +/// +/// Never answer from a truncated read. Each segment read asks for at most segmentCap rows (the server's +/// MaxTieClusterOverfetch + 1 over-fetch bound). A read that comes back FULL may have been cut short — its +/// newest rows are exactly the ones a reverse read needs — so it is discarded and the segment re-read at a quarter of +/// the width. A full read of a segment already one timestamp-resolution wide (a single-timestamp tie cluster larger +/// than the bound) cannot be served and throws +/// (⇒ BadTooManyOperations, a request limit, never a backend fault). +/// +/// +/// Edges, for any forward contract. A segment is requested as [segLo, segHi + resolution] and filtered to +/// (segLo, segHi] on the samples' own timestamps, so it is exact whether the backend's forward end is exclusive +/// (the interface contract) or inclusive (the gateway), and whether it truncates its bounds (QuestDB-style). The +/// filter also drops the rows an AVEVA Full read fabricates at the query start (the boundary / no-data row, +/// #687) and at the query end. Adjacent segments share no timestamp, so nothing is returned twice. +/// +/// +/// Ties come back in the backend's forward order reversed — the same order the node manager's tie-cluster +/// over-fetch uses for a reverse read — so a resumed page's boundary-tie skip lines up. +/// +/// +/// Floor. The walk stops at (the Unix epoch; its earliest request bound is one tick +/// before it): the AVEVA historian holds nothing earlier, and an end-only read's open lower bound (year 1) would +/// otherwise cost dozens of empty reads and send a bound outside the historian's time encoding. Samples before the +/// floor are not returned by a walked-back read; a window entirely before it is empty. +/// +/// +internal static class ReverseRawReader +{ + /// The first segment's width. + internal static readonly TimeSpan InitialSegment = TimeSpan.FromMinutes(1); + + /// The earliest instant a walk reads (see the remarks). + internal static readonly DateTime Floor = DateTime.UnixEpoch; + + /// Reads (lo, hi] newest-first from forward reads. + /// A backend that reads forward. + /// Historian tag. + /// The later (inclusive) bound — the request's StartTime or a reverse resume point. + /// The earlier (exclusive) bound. + /// Cap; 0 = every sample in the window. + /// Rows one segment read may return before it is treated as truncated (≥ 2). + /// Cancellation. + /// At most samples, newest first. + /// A single-timestamp tie cluster exceeds the segment cap. + public static async Task> ReadAsync( + IHistorianDataSource source, string tagname, DateTime hi, DateTime lo, uint maxValues, int segmentCap, + CancellationToken ct) + { + var resolution = Math.Max(1L, source.TimestampResolutionTicks); + var cap = (uint)Math.Max(2, segmentCap); + var result = new List(); + var width = Math.Max(resolution, InitialSegment.Ticks); + var segHiTicks = hi.Ticks; + + // The walk's exclusive lower edge. Clamped to one tick before the floor, so a sample exactly at the floor is still + // in the window (and the row a backend fabricates at that query start is still filtered out). + var stopTicks = lo.Ticks < Floor.Ticks ? Floor.Ticks - 1 : lo.Ticks; + + while (segHiTicks > stopTicks && (maxValues == 0 || result.Count < maxValues)) + { + ct.ThrowIfCancellationRequested(); + var segLoTicks = segHiTicks - stopTicks <= width ? stopTicks : segHiTicks - width; + var requestEndTicks = segHiTicks > DateTime.MaxValue.Ticks - resolution + ? DateTime.MaxValue.Ticks + : segHiTicks + resolution; + + var rows = (await source.ReadRawAsync( + tagname, + new DateTime(segLoTicks, DateTimeKind.Utc), + new DateTime(requestEndTicks, DateTimeKind.Utc), + cap, ct) + .ConfigureAwait(false)).Samples; + + if (rows.Count >= cap) + { + // Possibly truncated: its newest rows — the ones needed — may be missing. Narrow and re-read. + if (width <= resolution) + throw new HistoryReadLimitExceededException(cap - 1, + $"A reverse raw read of '{tagname}' at {new DateTime(segHiTicks, DateTimeKind.Utc):O} found more than " + + $"{cap - 1} samples at one timestamp; the backend reads forward only, so the latest of them cannot be " + + "selected. Increase ServerHistorian:MaxTieClusterOverfetch."); + width = Math.Max(resolution, width / 4); + continue; + } + + var inSegment = 0; + for (var i = rows.Count - 1; i >= 0; i--) + { + var ts = (rows[i].SourceTimestampUtc ?? DateTime.MinValue).Ticks; + if (ts <= segLoTicks || ts > segHiTicks) continue; + inSegment++; + if (maxValues == 0 || result.Count < maxValues) result.Add(rows[i]); + } + + // Sparse ⇒ widen; otherwise keep the width that fit. + if (inSegment <= cap / 8) + width = width > DateTime.MaxValue.Ticks / 4 ? DateTime.MaxValue.Ticks : width * 4; + segHiTicks = segLoTicks; + } + + return result; + } +} diff --git a/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Abstractions.Tests/Historian/HistorianDataSourceReverseEventsGuardTests.cs b/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Abstractions.Tests/Historian/HistorianDataSourceReverseEventsGuardTests.cs index fdc4d365..107f72ee 100644 --- a/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Abstractions.Tests/Historian/HistorianDataSourceReverseEventsGuardTests.cs +++ b/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Abstractions.Tests/Historian/HistorianDataSourceReverseEventsGuardTests.cs @@ -64,6 +64,22 @@ public sealed class HistorianDataSourceReverseEventsGuardTests } } + [Fact] + public void Every_implementation_declares_its_reverse_raw_capability_itself() + { + // Gitea #686: the default (false) is safe — the server walks back with forward reads — but a backend that serves + // reverse natively and inherited it would silently lose its native path, and a wrapper of the Gateway that claimed + // true would send start > end to a gateway that rejects it. Declare it, both ways. + foreach (var type in ProductionImplementations()) + { + var map = type.GetInterfaceMap(typeof(IHistorianDataSource)); + var getter = map.TargetMethods[Array.FindIndex(map.InterfaceMethods, + m => m.Name == "get_" + nameof(IHistorianDataSource.SupportsReverseRaw))]; + getter.DeclaringType.ShouldBe(type, + $"{type.FullName} must declare SupportsReverseRaw itself, not inherit the interface default"); + } + } + [Fact] public void Every_implementation_declares_its_supported_aggregates_itself() { diff --git a/tests/Drivers/Historian.Conformance/RawReadConformance.cs b/tests/Drivers/Historian.Conformance/RawReadConformance.cs index 41282291..8a04adc6 100644 --- a/tests/Drivers/Historian.Conformance/RawReadConformance.cs +++ b/tests/Drivers/Historian.Conformance/RawReadConformance.cs @@ -56,6 +56,14 @@ public abstract class RawReadConformance : ConformanceBase Values(await B.CreateDataSource().ReadRawAsync("A", At(10), max, 0, Ct)).ShouldBe([20d, 30d, 40d, 50d]); } + [Fact] + public void The_backend_reports_reverse_raw_support() + { + // #686: the server only sends start > end to a backend that declares it; these backends serve it natively. + using var src = B.CreateDataSource(); + src.SupportsReverseRaw.ShouldBeTrue(); + } + [Fact] public async Task Reverse_read_is_descending_over_end_exclusive_start_inclusive() { diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests/GatewayRawDirectionTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests/GatewayRawDirectionTests.cs new file mode 100644 index 00000000..515ec866 --- /dev/null +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests/GatewayRawDirectionTests.cs @@ -0,0 +1,45 @@ +using Microsoft.Extensions.Logging.Abstractions; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Core.Abstractions; + +namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests; + +/// +/// Gitea #686: the gateway's ReadRaw streams ascending only and rejects start > end with +/// InvalidArgument, which used to be recorded as a backend health failure. The source declares it cannot read +/// newest-first (so the server walks back with forward reads), and refuses a reverse window itself without sending +/// anything or counting a failure. +/// +public sealed class GatewayRawDirectionTests +{ + private static CancellationToken Ct => TestContext.Current.CancellationToken; + + [Fact] + public void The_gateway_source_declares_it_cannot_read_raw_newest_first() + { + IHistorianDataSource ds = new GatewayHistorianDataSource( + new FakeHistorianGatewayClient(), NullLogger.Instance); + ds.SupportsReverseRaw.ShouldBeFalse(); + } + + [Fact] + public async Task A_reverse_raw_window_is_refused_unsent_and_is_not_a_backend_failure() + { + var fake = new FakeHistorianGatewayClient(); + IHistorianDataSource ds = new GatewayHistorianDataSource(fake, NullLogger.Instance); + var end = new DateTime(2026, 10, 12, 18, 7, 45, DateTimeKind.Utc); + + await Should.ThrowAsync(() => + ds.ReadRawAsync("Gate681", end, new DateTime(0, DateTimeKind.Utc), 3, Ct)); + + fake.ReadRawCallCount.ShouldBe(0); + var health = ds.GetHealthSnapshot(); + health.TotalFailures.ShouldBe(0); + health.ConsecutiveFailures.ShouldBe(0); + + // Positive control: a forward read of the same window is sent. + await ds.ReadRawAsync("Gate681", new DateTime(2026, 10, 12, 18, 0, 0, DateTimeKind.Utc), end, 3, Ct); + fake.ReadRawCallCount.ShouldBe(1); + } +} diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.Tests/NodeManagerHistoryReadPagingTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.Tests/NodeManagerHistoryReadPagingTests.cs index 43ba469f..8ed393ff 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.Tests/NodeManagerHistoryReadPagingTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.Tests/NodeManagerHistoryReadPagingTests.cs @@ -466,6 +466,169 @@ public sealed class NodeManagerHistoryReadPagingTests : IDisposable await host.DisposeAsync(); } + // --- #686: reverse / end-only raw reads on a backend that only reads forward (the Gateway) ------------------------- + + /// Boots a node manager over a and one historized variable. + private async Task<(OpcUaApplicationHost Host, OtOpcUaNodeManager Nm, NodeId NodeId)> BootForwardOnlyAsync( + ForwardOnlyHistorianDataSource source, string name) + { + var (host, server) = await BootAsync(); + var nm = server.NodeManager!; + nm.HistoryContinuationStore = new InMemoryHistoryContinuationStore(); + nm.HistorianDataSource = source; + nm.EnsureVariable($"eq-1/{name}", parentFolderNodeId: null, displayName: name, dataType: "Double", + writable: false, historianTagname: $"WW.{name}", realm: AddressSpaceRealm.Uns); + return (host, nm, nm.TryGetVariable($"eq-1/{name}")!.NodeId); + } + + /// + /// #686: an end-only capped read (Part 11: page BACKWARD from EndTime) on a backend that cannot read newest-first + /// pages every sample newest-first — and never sends that backend start > end (the gateway rejects it + /// InvalidArgument, which used to fail the node and count as a backend fault). + /// + [Fact] + public async Task Raw_end_only_read_on_a_forward_only_backend_pages_descending_without_a_reverse_window() + { + var source = new ForwardOnlyHistorianDataSource(MakeSeries(count: 250, stepSeconds: 1)); + var (host, nm, nodeId) = await BootForwardOnlyAsync(source, "FwdEndOnly"); + + var collected = await DrainAsync(nm, nodeId, DateTime.MinValue, Epoch.AddHours(1), max: 100); + + collected.ShouldBe(Enumerable.Range(0, 250).Reverse().Select(i => (double)i)); + source.ReverseWindowCount.ShouldBe(0); + await host.DisposeAsync(); + } + + /// #686: the first page of an end-only capped read is the N values AT OR BEFORE EndTime, newest first — + /// not the oldest N in history (the pre-#669 forward meaning), and not the synthetic boundary row the AVEVA-shaped + /// fake emits at each query start. + [Fact] + public async Task Raw_end_only_capped_read_on_a_forward_only_backend_returns_the_latest_values_before_end_time() + { + var source = new ForwardOnlyHistorianDataSource(MakeSeries(count: 250, stepSeconds: 1)); + var (host, nm, nodeId) = await BootForwardOnlyAsync(source, "FwdLatest"); + + var (values, error, cp) = ReadRaw(nm, nodeId, DateTime.MinValue, Epoch.AddSeconds(100), max: 3, inboundCp: null); + + error.StatusCode.Code.ShouldBe(StatusCodes.Good); + values.ShouldBe([100d, 99d, 98d]); + cp.ShouldNotBeNull(); + source.ReverseWindowCount.ShouldBe(0); + await host.DisposeAsync(); + } + + /// #686: an explicit reverse read with no cap returns the whole window newest-first in one page. + [Fact] + public async Task Raw_unbounded_explicit_reverse_read_on_a_forward_only_backend_returns_the_window_descending() + { + var source = new ForwardOnlyHistorianDataSource(MakeSeries(count: 250, stepSeconds: 1)); + var (host, nm, nodeId) = await BootForwardOnlyAsync(source, "FwdRevAll"); + + // (end, start] = (Epoch+9s, Epoch+200s] — both edges exercised. + var (values, error, cp) = ReadRaw(nm, nodeId, Epoch.AddSeconds(200), Epoch.AddSeconds(9), max: 0, inboundCp: null); + + error.StatusCode.Code.ShouldBe(StatusCodes.Good); + values.ShouldBe(Enumerable.Range(10, 191).Reverse().Select(i => (double)i)); + cp.ShouldBeNull(); + source.ReverseWindowCount.ShouldBe(0); + await host.DisposeAsync(); + } + + /// #686: a reverse read paging through a tie cluster on a forward-only backend — every value once, newest first. + [Fact] + public async Task Raw_reverse_read_on_a_forward_only_backend_pages_through_a_tie_cluster() + { + // No AVEVA boundary row here: paging WITHIN the cluster re-reads it with the node manager's start == end point + // over-fetch, and a row fabricated at a point read's start is indistinguishable from a tie at that instant — + // dropping it is the Gateway source's job (#687), not the walk's. + var source = new ForwardOnlyHistorianDataSource( + MakeSeriesWithTieCluster(distinctBefore: 3, tieCount: 5, distinctAfter: 2), emitBoundaryRow: false); + var (host, nm, nodeId) = await BootForwardOnlyAsync(source, "FwdRevBurst"); + + var collected = await DrainAsync(nm, nodeId, start: Epoch.AddHours(1), end: Epoch.AddSeconds(-1), max: 2); + + collected.ShouldBe(Enumerable.Range(0, 10).Reverse().Select(i => (double)i)); + source.ReverseWindowCount.ShouldBe(0); + await host.DisposeAsync(); + } + + /// #686: a segment denser than the per-read row budget (MaxTieClusterOverfetch) is narrowed and re-read, + /// never answered from a truncated forward read. + [Fact] + public async Task Raw_reverse_read_on_a_forward_only_backend_narrows_a_segment_over_the_row_budget() + { + var source = new ForwardOnlyHistorianDataSource(MakeSeries(count: 250, stepSeconds: 1)); + var (host, nm, nodeId) = await BootForwardOnlyAsync(source, "FwdDense"); + nm.MaxTieClusterOverfetch = 16; // a 1-minute segment holds 60 samples + + var collected = await DrainAsync(nm, nodeId, DateTime.MinValue, Epoch.AddHours(1), max: 100); + + collected.ShouldBe(Enumerable.Range(0, 250).Reverse().Select(i => (double)i)); + source.MaxRowsReturned.ShouldBeLessThanOrEqualTo(17); + source.ReverseWindowCount.ShouldBe(0); + await host.DisposeAsync(); + } + + /// #686: sparse history far before EndTime is found, in a bounded number of forward reads. + [Fact] + public async Task Raw_end_only_read_on_a_forward_only_backend_finds_sparse_history_in_bounded_reads() + { + static DataValueSnapshot At(int year, double v) + { + var t = new DateTime(year, 1, 1, 0, 0, 0, DateTimeKind.Utc); + return new DataValueSnapshot(v, StatusCodes.Good, t, t); + } + + var source = new ForwardOnlyHistorianDataSource([At(1990, 1), At(2000, 2)]); + var (host, nm, nodeId) = await BootForwardOnlyAsync(source, "FwdSparse"); + + var (values, error, cp) = ReadRaw(nm, nodeId, DateTime.MinValue, Epoch, max: 5, inboundCp: null); + + error.StatusCode.Code.ShouldBe(StatusCodes.Good); + values.ShouldBe([2d, 1d]); + cp.ShouldBeNull(); + source.ReadCount.ShouldBeLessThan(40); + source.ReverseWindowCount.ShouldBe(0); + await host.DisposeAsync(); + } + + /// #686: a single-timestamp tie cluster larger than the row budget cannot be read newest-first from + /// forward reads — the node is refused BadTooManyOperations (a request limit, not a backend fault). + [Fact] + public async Task Raw_reverse_read_on_a_forward_only_backend_refuses_a_tie_cluster_over_the_row_budget() + { + var source = new ForwardOnlyHistorianDataSource(MakeSeriesWithTieCluster(distinctBefore: 1, tieCount: 10, distinctAfter: 0)); + var (host, nm, nodeId) = await BootForwardOnlyAsync(source, "FwdHugeTie"); + nm.MaxTieClusterOverfetch = 4; + + var (_, error, cp) = ReadRaw(nm, nodeId, Epoch.AddHours(1), Epoch.AddSeconds(-1), max: 2, inboundCp: null); + + error.StatusCode.Code.ShouldBe(StatusCodes.BadTooManyOperations); + cp.ShouldBeNull(); + source.ReverseWindowCount.ShouldBe(0); + await host.DisposeAsync(); + } + + /// A backend that DECLARES native reverse support is still sent the reverse window — one read. + [Fact] + public async Task Raw_end_only_read_on_a_reverse_capable_backend_is_one_native_reverse_read() + { + var (host, server) = await BootAsync(); + var nm = server.NodeManager!; + nm.HistoryContinuationStore = new InMemoryHistoryContinuationStore(); + var source = new SeriesHistorianDataSource(MakeSeries(count: 250, stepSeconds: 1)); + nm.HistorianDataSource = source; + nm.EnsureVariable("eq-1/native", parentFolderNodeId: null, displayName: "Native", dataType: "Double", + writable: false, historianTagname: "WW.Native", realm: AddressSpaceRealm.Uns); + var nodeId = nm.TryGetVariable("eq-1/native")!.NodeId; + + var (values, _, _) = ReadRaw(nm, nodeId, DateTime.MinValue, Epoch.AddSeconds(100), max: 3, inboundCp: null); + + values.ShouldBe([100d, 99d, 98d]); + source.ReadCount.ShouldBe(1); + await host.DisposeAsync(); + } + /// Pages a Raw read to completion and returns the union of the pages in arrival order. private static Task> DrainAsync(OtOpcUaNodeManager nm, NodeId nodeId, DateTime start, DateTime end, uint max) { @@ -584,6 +747,9 @@ public sealed class NodeManagerHistoryReadPagingTests : IDisposable private sealed class SeriesHistorianDataSource(IReadOnlyList series) : IHistorianDataSource { private int _readCount; + + /// Serves start > end natively — the SQL backends' contract (#686: declared, like they do). + public bool SupportsReverseRaw => true; public int ReadCount => _readCount; public void ResetReadCount() => _readCount = 0; @@ -629,6 +795,70 @@ public sealed class NodeManagerHistoryReadPagingTests : IDisposable public void Dispose() { } } + /// + /// #686: a Gateway-shaped backend that reads FORWARD only. is false; a reverse window + /// (start > end) is counted and rejected as the gateway does (InvalidArgument). A forward read is + /// inclusive [start, end] and, like an AVEVA Full read, first emits a synthetic boundary row stamped at + /// the query start carrying the value in effect there (Bad 0 when there is none) — counted against the cap. + /// + private sealed class ForwardOnlyHistorianDataSource( + IReadOnlyList series, bool emitBoundaryRow = true) : IHistorianDataSource + { + private int _readCount; + private int _reverseWindowCount; + private int _maxRowsReturned; + + public int ReadCount => _readCount; + public int ReverseWindowCount => _reverseWindowCount; + public int MaxRowsReturned => _maxRowsReturned; + + public bool SupportsReverseRaw => false; + + public Task ReadRawAsync( + string fullReference, DateTime startUtc, DateTime endUtc, uint maxValuesPerNode, + CancellationToken cancellationToken) + { + Interlocked.Increment(ref _readCount); + if (startUtc > endUtc) + { + Interlocked.Increment(ref _reverseWindowCount); + throw new InvalidOperationException("start_time must be less than or equal to end_time. (InvalidArgument)"); + } + + static DateTime Ts(DataValueSnapshot s) => s.SourceTimestampUtc ?? DateTime.MinValue; + var prior = series.LastOrDefault(s => Ts(s) <= startUtc); + var rows = new List(); + if (emitBoundaryRow) + rows.Add(prior is null + ? new DataValueSnapshot(0d, StatusCodes.BadNoData, startUtc, startUtc) + : new DataValueSnapshot(prior.Value, prior.StatusCode, startUtc, startUtc)); + rows.AddRange(series.Where(s => Ts(s) >= startUtc && Ts(s) <= endUtc)); + var page = maxValuesPerNode == 0 ? rows : rows.Take((int)maxValuesPerNode).ToList(); + int seen; + while (page.Count > (seen = Volatile.Read(ref _maxRowsReturned)) + && Interlocked.CompareExchange(ref _maxRowsReturned, page.Count, seen) != seen) { } + return Task.FromResult(new HistorianRead(page, null)); + } + + public Task ReadProcessedAsync( + string fullReference, DateTime startUtc, DateTime endUtc, TimeSpan interval, + HistoryAggregateType aggregate, CancellationToken cancellationToken) => + Task.FromResult(new HistorianRead(Array.Empty(), null)); + + public Task ReadAtTimeAsync( + string fullReference, IReadOnlyList timestampsUtc, CancellationToken cancellationToken) => + Task.FromResult(new HistorianRead(Array.Empty(), null)); + + public Task ReadEventsAsync( + string? sourceName, DateTime startUtc, DateTime endUtc, int maxEvents, + CancellationToken cancellationToken) => + Task.FromResult(new HistoricalEventsResult(Array.Empty(), null)); + + public HistorianHealthSnapshot GetHealthSnapshot() => NullHistorianDataSource.Instance.GetHealthSnapshot(); + + public void Dispose() { } + } + /// /// A microsecond-resolution backend, shaped like QuestDB: every inbound window bound is TRUNCATED to 10 ticks /// (as QuestDbConnection.Round does) before the series is filtered, and the source reports @@ -640,6 +870,8 @@ public sealed class NodeManagerHistoryReadPagingTests : IDisposable public long TimestampResolutionTicks => 10; + public bool SupportsReverseRaw => true; // QuestDB serves reverse natively + private static DateTime Truncate(DateTime t) => new(t.Ticks - t.Ticks % 10, DateTimeKind.Utc); public Task ReadRawAsync(