fix(historian): #686 serve reverse/end-only raw reads on forward-only backends (Gateway)

#669 turned an end-only capped HistoryRead-Raw into a reverse read (start > end)
for every backend; the gateway rejects start > end (InvalidArgument), so on the
default Gateway backend every end-only capped and every explicit reverse raw read
failed the node and was counted as a backend health failure.

- IHistorianDataSource.SupportsReverseRaw (default false; SqlServer/QuestDB/Null
  true, Gateway false; guard test pins that every production backend declares it).
- OtOpcUaNodeManager sends a reverse window only to a backend that declares it;
  otherwise ReverseRawReader walks (lo, hi] back in bounded forward segment reads
  (1 min, x4 while sparse; each <= MaxTieClusterOverfetch + 1 rows; a full read is
  re-read narrower, never answered truncated; filtered on sample timestamps, which
  also drops AVEVA's fabricated query-start rows; stops at the Unix epoch). A
  one-timestamp tie cluster over the bound => BadTooManyOperations.
- ServeRawPagedAsync maps HistoryReadLimitExceededException to BadTooManyOperations.
- GatewayHistorianDataSource.ReadRawAsync refuses start > end unsent, without
  recording a health failure.
This commit is contained in:
Joseph Doherty
2026-10-12 14:42:27 -04:00
parent 70fb1f1fda
commit 410c8d8976
11 changed files with 490 additions and 3 deletions
@@ -17,6 +17,12 @@ public interface IHistorianDataSource : IDisposable
/// <summary>
/// Read raw historical samples for a single tag over a time range.
/// </summary>
/// <remarks>
/// Forward (<c>startUtc &lt; endUtc</c>) is <c>[start, end)</c> ascending; <c>startUtc == endUtc</c> is a point read.
/// <c>startUtc &gt; endUtc</c> is the Part 11 reverse read <c>(end, start]</c> newest-first — but only a backend that
/// declares <see cref="SupportsReverseRaw"/> is ever sent one (Gitea #686); the server serves a reverse read on any
/// other backend from bounded forward reads.
/// </remarks>
/// <param name="fullReference">The full reference of the tag to read.</param>
/// <param name="startUtc">The start of the time range in UTC.</param>
/// <param name="endUtc">The end of the time range in UTC.</param>
@@ -30,6 +36,16 @@ public interface IHistorianDataSource : IDisposable
uint maxValuesPerNode,
CancellationToken cancellationToken);
/// <summary>
/// Whether this backend can serve a reverse (newest-first) raw read itself — <see cref="ReadRawAsync"/> with
/// <c>startUtc &gt; endUtc</c> (Gitea #686). The server checks it before every raw read. When <c>false</c> it never
/// sends <c>start &gt; end</c>: it serves the reverse read — an explicit one, a resumed reverse page, and the
/// end-only capped read Part 11 reads backward from EndTime — from a series of bounded FORWARD reads walking back
/// from the later bound (<c>ReverseRawReader</c> in the OPC UA server). Default <c>false</c>, the safe direction: a
/// source that only reads forward is still answered correctly. A backend declares it itself (a guard test pins that).
/// </summary>
bool SupportsReverseRaw => false;
/// <summary>
/// Read processed (interval-bucketed) samples — average / min / max / count / etc.
/// A bucket with no source data returns a sample whose
@@ -55,6 +55,11 @@ public sealed class NullHistorianDataSource : IHistorianDataSource
int maxEvents,
CancellationToken cancellationToken) => Task.FromResult(EmptyEvents);
/// <inheritdoc />
/// <remarks><c>true</c> (Gitea #686): there is no history, so the empty result is the correct answer in either
/// direction — no need for the server to walk back through forward reads to find nothing.</remarks>
public bool SupportsReverseRaw => true;
/// <inheritdoc />
/// <remarks><c>false</c> (Gitea #675): there is no history to read in either direction, and an explicit reverse read
/// is answered <c>BadHistoryOperationUnsupported</c> like any backend that cannot read newest-first.</remarks>
@@ -76,10 +76,18 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
}
/// <inheritdoc />
/// <remarks>
/// A reverse window (<c>start &gt; end</c>) is refused with <see cref="NotSupportedException"/> before anything is sent
/// and is not recorded as a health failure (Gitea #686): the gateway rejects it <c>InvalidArgument</c>, and the server
/// never asks — <see cref="SupportsReverseRaw"/> is <c>false</c>, so it walks a reverse read back with forward reads.
/// </remarks>
public async Task<HistoryReadResult> ReadRawAsync(
string fullReference, DateTime startUtc, DateTime endUtc, uint maxValuesPerNode,
CancellationToken cancellationToken)
{
if (startUtc > endUtc)
throw new NotSupportedException("The HistorianGateway reads raw history oldest-first only (start > end).");
try
{
// The gateway seam caps with an int; OPC UA hands us a uint, so clamp to int range.
@@ -235,6 +243,14 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
}
}
/// <inheritdoc />
/// <remarks>
/// <c>false</c> (Gitea #686): the gateway's <c>ReadRaw</c> streams ascending only and rejects <c>end &lt; start</c>
/// (<c>InvalidArgument</c>). The server serves reverse and end-only capped raw reads from bounded forward reads
/// instead, and never sends this source a reverse window (<see cref="ReadRawAsync"/> refuses one unsent).
/// </remarks>
public bool SupportsReverseRaw => false;
/// <inheritdoc />
/// <remarks>
/// <c>false</c>: the gateway's <c>ReadEvents</c> streams ascending only (Gitea #675). The server therefore refuses an
@@ -355,6 +355,10 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource
// ------------------------------------------------------------------------------------------ Events
/// <inheritdoc />
/// <remarks>Reverse raw reads are served natively (<c>(end, start]</c>, <c>ORDER BY … DESC</c>; #669/#686).</remarks>
public bool SupportsReverseRaw => true;
/// <inheritdoc />
/// <remarks>Reverse events reads are served natively (<c>ORDER BY … DESC</c>, Gitea #675).</remarks>
public bool SupportsReverseEvents => true;
@@ -312,6 +312,10 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
// ------------------------------------------------------------------------------------------ Events
/// <inheritdoc />
/// <remarks>Reverse raw reads are served natively (<c>(end, start]</c>, <c>ORDER BY … DESC</c>; #669/#686).</remarks>
public bool SupportsReverseRaw => true;
/// <inheritdoc />
/// <remarks>Reverse events reads are served natively (<c>ORDER BY … DESC</c>, Gitea #675).</remarks>
public bool SupportsReverseEvents => true;
@@ -6219,6 +6219,8 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2
// Part 11: with NumValuesPerNode > 0 one bound may be left unset. Only a StartTime ⇒ page FORWARD from
// it to an open upper bound; only an EndTime ⇒ page BACKWARD from it (a reverse read, start > end).
// Without this an unset EndTime (MinValue) looked like a reverse read to a lower bound of "now".
// Every backend serves the backward read (#686): natively when it declares SupportsReverseRaw, otherwise
// walked back from bounded forward reads below — unlike the events path, which keeps it forward there.
if (numValuesPerNode > 0)
{
if (endUtc == DateTime.MinValue && startUtc != DateTime.MinValue)
@@ -6229,9 +6231,20 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2
}
// HistoryRead is NOT under the node-manager Lock — awaiting the async source is safe.
var sourceResult = await HistorianDataSource
.ReadRawAsync(tagname, startUtc, endUtc, numValuesPerNode, ct)
.ConfigureAwait(false);
// A reverse window (explicit, resumed, or the end-only read above) goes to the backend only when it declares it
// can read newest-first; otherwise it is walked back from bounded forward reads (#686 — the Gateway rejects
// start > end, which failed the node and counted as a backend fault).
var source = HistorianDataSource;
var sourceResult = startUtc > endUtc && !source.SupportsReverseRaw
? new HistorianRead(
await ReverseRawReader.ReadAsync(
source, tagname, startUtc, endUtc, numValuesPerNode,
(int)Math.Min(int.MaxValue, (long)Math.Max(1, MaxTieClusterOverfetch) + 1), ct)
.ConfigureAwait(false),
ContinuationPoint: null)
: await source
.ReadRawAsync(tagname, startUtc, endUtc, numValuesPerNode, ct)
.ConfigureAwait(false);
var backendFull = HistoryPaging.IsFullPage(sourceResult.Samples.Count, numValuesPerNode);
@@ -6348,6 +6361,14 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2
errors[handle.Index] = StatusCodes.BadTimeout;
results[handle.Index] = new SdkHistoryReadResult { StatusCode = StatusCodes.BadTimeout };
}
catch (HistoryReadLimitExceededException ex)
{
// A request limit (#686: a reverse read walked back from forward reads met a tie cluster over the row
// budget) — BadTooManyOperations for THIS node, as on the processed path; not a backend fault.
_logger.LogWarning("OtOpcUaNodeManager: HistoryReadRaw limit exceeded for node {NodeId}: {Message}", handle.NodeId, ex.Message);
errors[handle.Index] = StatusCodes.BadTooManyOperations;
results[handle.Index] = new SdkHistoryReadResult { StatusCode = StatusCodes.BadTooManyOperations };
}
catch (Exception ex)
{
// One node's backend failure must not throw out of the batch — Bad for THIS node only.
@@ -0,0 +1,120 @@
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
namespace ZB.MOM.WW.OtOpcUa.OpcUaServer;
/// <summary>
/// Gitea #686: serves a Part 11 reverse raw read — <c>(lo, hi]</c> newest-first, capped — on a historian backend that can
/// only read forward (<see cref="IHistorianDataSource.SupportsReverseRaw"/> = <c>false</c>: the Gateway, whose
/// <c>ReadRaw</c> streams ascending and rejects <c>start &gt; end</c>).
/// </summary>
/// <remarks>
/// <para>
/// <b>Walk back in segments.</b> Starting at <c>hi</c>, read one segment <c>(segLo, segHi]</c> FORWARD, keep it
/// newest-first, and step to the next older segment until the cap is met or <c>lo</c> is reached. A segment starts
/// at <see cref="InitialSegment"/> and grows ×4 while segments come back sparse, so "the latest N" of a live tag is
/// usually one read, and sparse history decades back is found in a few dozen tiny reads rather than one read of
/// all history.
/// </para>
/// <para>
/// <b>Never answer from a truncated read.</b> Each segment read asks for at most <c>segmentCap</c> rows (the server's
/// <c>MaxTieClusterOverfetch + 1</c> over-fetch bound). A read that comes back FULL may have been cut short — its
/// newest rows are exactly the ones a reverse read needs — so it is discarded and the segment re-read at a quarter of
/// the width. A full read of a segment already one timestamp-resolution wide (a single-timestamp tie cluster larger
/// than the bound) cannot be served and throws <see cref="HistoryReadLimitExceededException"/>
/// (⇒ <c>BadTooManyOperations</c>, a request limit, never a backend fault).
/// </para>
/// <para>
/// <b>Edges, for any forward contract.</b> A segment is requested as <c>[segLo, segHi + resolution]</c> and filtered to
/// <c>(segLo, segHi]</c> on the samples' own timestamps, so it is exact whether the backend's forward end is exclusive
/// (the interface contract) or inclusive (the gateway), and whether it truncates its bounds (QuestDB-style). The
/// filter also drops the rows an AVEVA <c>Full</c> read fabricates at the query start (the boundary / no-data row,
/// #687) and at the query end. Adjacent segments share no timestamp, so nothing is returned twice.
/// </para>
/// <para>
/// <b>Ties</b> come back in the backend's forward order reversed — the same order the node manager's tie-cluster
/// over-fetch uses for a reverse read — so a resumed page's boundary-tie skip lines up.
/// </para>
/// <para>
/// <b>Floor.</b> The walk stops at <see cref="Floor"/> (the Unix epoch; its earliest request bound is one tick
/// before it): the AVEVA historian holds nothing earlier, and an end-only read's open lower bound (year 1) would
/// otherwise cost dozens of empty reads and send a bound outside the historian's time encoding. Samples before the
/// floor are not returned by a walked-back read; a window entirely before it is empty.
/// </para>
/// </remarks>
internal static class ReverseRawReader
{
/// <summary>The first segment's width.</summary>
internal static readonly TimeSpan InitialSegment = TimeSpan.FromMinutes(1);
/// <summary>The earliest instant a walk reads (see the remarks).</summary>
internal static readonly DateTime Floor = DateTime.UnixEpoch;
/// <summary>Reads <c>(lo, hi]</c> newest-first from forward reads.</summary>
/// <param name="source">A backend that reads forward.</param>
/// <param name="tagname">Historian tag.</param>
/// <param name="hi">The later (inclusive) bound — the request's StartTime or a reverse resume point.</param>
/// <param name="lo">The earlier (exclusive) bound.</param>
/// <param name="maxValues">Cap; <c>0</c> = every sample in the window.</param>
/// <param name="segmentCap">Rows one segment read may return before it is treated as truncated (≥ 2).</param>
/// <param name="ct">Cancellation.</param>
/// <returns>At most <paramref name="maxValues"/> samples, newest first.</returns>
/// <exception cref="HistoryReadLimitExceededException">A single-timestamp tie cluster exceeds the segment cap.</exception>
public static async Task<IReadOnlyList<DataValueSnapshot>> ReadAsync(
IHistorianDataSource source, string tagname, DateTime hi, DateTime lo, uint maxValues, int segmentCap,
CancellationToken ct)
{
var resolution = Math.Max(1L, source.TimestampResolutionTicks);
var cap = (uint)Math.Max(2, segmentCap);
var result = new List<DataValueSnapshot>();
var width = Math.Max(resolution, InitialSegment.Ticks);
var segHiTicks = hi.Ticks;
// The walk's exclusive lower edge. Clamped to one tick before the floor, so a sample exactly at the floor is still
// in the window (and the row a backend fabricates at that query start is still filtered out).
var stopTicks = lo.Ticks < Floor.Ticks ? Floor.Ticks - 1 : lo.Ticks;
while (segHiTicks > stopTicks && (maxValues == 0 || result.Count < maxValues))
{
ct.ThrowIfCancellationRequested();
var segLoTicks = segHiTicks - stopTicks <= width ? stopTicks : segHiTicks - width;
var requestEndTicks = segHiTicks > DateTime.MaxValue.Ticks - resolution
? DateTime.MaxValue.Ticks
: segHiTicks + resolution;
var rows = (await source.ReadRawAsync(
tagname,
new DateTime(segLoTicks, DateTimeKind.Utc),
new DateTime(requestEndTicks, DateTimeKind.Utc),
cap, ct)
.ConfigureAwait(false)).Samples;
if (rows.Count >= cap)
{
// Possibly truncated: its newest rows — the ones needed — may be missing. Narrow and re-read.
if (width <= resolution)
throw new HistoryReadLimitExceededException(cap - 1,
$"A reverse raw read of '{tagname}' at {new DateTime(segHiTicks, DateTimeKind.Utc):O} found more than " +
$"{cap - 1} samples at one timestamp; the backend reads forward only, so the latest of them cannot be " +
"selected. Increase ServerHistorian:MaxTieClusterOverfetch.");
width = Math.Max(resolution, width / 4);
continue;
}
var inSegment = 0;
for (var i = rows.Count - 1; i >= 0; i--)
{
var ts = (rows[i].SourceTimestampUtc ?? DateTime.MinValue).Ticks;
if (ts <= segLoTicks || ts > segHiTicks) continue;
inSegment++;
if (maxValues == 0 || result.Count < maxValues) result.Add(rows[i]);
}
// Sparse ⇒ widen; otherwise keep the width that fit.
if (inSegment <= cap / 8)
width = width > DateTime.MaxValue.Ticks / 4 ? DateTime.MaxValue.Ticks : width * 4;
segHiTicks = segLoTicks;
}
return result;
}
}
@@ -64,6 +64,22 @@ public sealed class HistorianDataSourceReverseEventsGuardTests
}
}
[Fact]
public void Every_implementation_declares_its_reverse_raw_capability_itself()
{
// Gitea #686: the default (false) is safe — the server walks back with forward reads — but a backend that serves
// reverse natively and inherited it would silently lose its native path, and a wrapper of the Gateway that claimed
// true would send start > end to a gateway that rejects it. Declare it, both ways.
foreach (var type in ProductionImplementations())
{
var map = type.GetInterfaceMap(typeof(IHistorianDataSource));
var getter = map.TargetMethods[Array.FindIndex(map.InterfaceMethods,
m => m.Name == "get_" + nameof(IHistorianDataSource.SupportsReverseRaw))];
getter.DeclaringType.ShouldBe(type,
$"{type.FullName} must declare SupportsReverseRaw itself, not inherit the interface default");
}
}
[Fact]
public void Every_implementation_declares_its_supported_aggregates_itself()
{
@@ -56,6 +56,14 @@ public abstract class RawReadConformance : ConformanceBase
Values(await B.CreateDataSource().ReadRawAsync("A", At(10), max, 0, Ct)).ShouldBe([20d, 30d, 40d, 50d]);
}
[Fact]
public void The_backend_reports_reverse_raw_support()
{
// #686: the server only sends start > end to a backend that declares it; these backends serve it natively.
using var src = B.CreateDataSource();
src.SupportsReverseRaw.ShouldBeTrue();
}
[Fact]
public async Task Reverse_read_is_descending_over_end_exclusive_start_inclusive()
{
@@ -0,0 +1,45 @@
using Microsoft.Extensions.Logging.Abstractions;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests;
/// <summary>
/// Gitea #686: the gateway's <c>ReadRaw</c> streams ascending only and rejects <c>start &gt; end</c> with
/// <c>InvalidArgument</c>, which used to be recorded as a backend health failure. The source declares it cannot read
/// newest-first (so the server walks back with forward reads), and refuses a reverse window itself without sending
/// anything or counting a failure.
/// </summary>
public sealed class GatewayRawDirectionTests
{
private static CancellationToken Ct => TestContext.Current.CancellationToken;
[Fact]
public void The_gateway_source_declares_it_cannot_read_raw_newest_first()
{
IHistorianDataSource ds = new GatewayHistorianDataSource(
new FakeHistorianGatewayClient(), NullLogger<GatewayHistorianDataSource>.Instance);
ds.SupportsReverseRaw.ShouldBeFalse();
}
[Fact]
public async Task A_reverse_raw_window_is_refused_unsent_and_is_not_a_backend_failure()
{
var fake = new FakeHistorianGatewayClient();
IHistorianDataSource ds = new GatewayHistorianDataSource(fake, NullLogger<GatewayHistorianDataSource>.Instance);
var end = new DateTime(2026, 10, 12, 18, 7, 45, DateTimeKind.Utc);
await Should.ThrowAsync<NotSupportedException>(() =>
ds.ReadRawAsync("Gate681", end, new DateTime(0, DateTimeKind.Utc), 3, Ct));
fake.ReadRawCallCount.ShouldBe(0);
var health = ds.GetHealthSnapshot();
health.TotalFailures.ShouldBe(0);
health.ConsecutiveFailures.ShouldBe(0);
// Positive control: a forward read of the same window is sent.
await ds.ReadRawAsync("Gate681", new DateTime(2026, 10, 12, 18, 0, 0, DateTimeKind.Utc), end, 3, Ct);
fake.ReadRawCallCount.ShouldBe(1);
}
}
@@ -466,6 +466,169 @@ public sealed class NodeManagerHistoryReadPagingTests : IDisposable
await host.DisposeAsync();
}
// --- #686: reverse / end-only raw reads on a backend that only reads forward (the Gateway) -------------------------
/// <summary>Boots a node manager over a <see cref="ForwardOnlyHistorianDataSource"/> and one historized variable.</summary>
private async Task<(OpcUaApplicationHost Host, OtOpcUaNodeManager Nm, NodeId NodeId)> BootForwardOnlyAsync(
ForwardOnlyHistorianDataSource source, string name)
{
var (host, server) = await BootAsync();
var nm = server.NodeManager!;
nm.HistoryContinuationStore = new InMemoryHistoryContinuationStore();
nm.HistorianDataSource = source;
nm.EnsureVariable($"eq-1/{name}", parentFolderNodeId: null, displayName: name, dataType: "Double",
writable: false, historianTagname: $"WW.{name}", realm: AddressSpaceRealm.Uns);
return (host, nm, nm.TryGetVariable($"eq-1/{name}")!.NodeId);
}
/// <summary>
/// #686: an end-only capped read (Part 11: page BACKWARD from EndTime) on a backend that cannot read newest-first
/// pages every sample newest-first — and never sends that backend <c>start &gt; end</c> (the gateway rejects it
/// <c>InvalidArgument</c>, which used to fail the node and count as a backend fault).
/// </summary>
[Fact]
public async Task Raw_end_only_read_on_a_forward_only_backend_pages_descending_without_a_reverse_window()
{
var source = new ForwardOnlyHistorianDataSource(MakeSeries(count: 250, stepSeconds: 1));
var (host, nm, nodeId) = await BootForwardOnlyAsync(source, "FwdEndOnly");
var collected = await DrainAsync(nm, nodeId, DateTime.MinValue, Epoch.AddHours(1), max: 100);
collected.ShouldBe(Enumerable.Range(0, 250).Reverse().Select(i => (double)i));
source.ReverseWindowCount.ShouldBe(0);
await host.DisposeAsync();
}
/// <summary>#686: the first page of an end-only capped read is the N values AT OR BEFORE EndTime, newest first —
/// not the oldest N in history (the pre-#669 forward meaning), and not the synthetic boundary row the AVEVA-shaped
/// fake emits at each query start.</summary>
[Fact]
public async Task Raw_end_only_capped_read_on_a_forward_only_backend_returns_the_latest_values_before_end_time()
{
var source = new ForwardOnlyHistorianDataSource(MakeSeries(count: 250, stepSeconds: 1));
var (host, nm, nodeId) = await BootForwardOnlyAsync(source, "FwdLatest");
var (values, error, cp) = ReadRaw(nm, nodeId, DateTime.MinValue, Epoch.AddSeconds(100), max: 3, inboundCp: null);
error.StatusCode.Code.ShouldBe(StatusCodes.Good);
values.ShouldBe([100d, 99d, 98d]);
cp.ShouldNotBeNull();
source.ReverseWindowCount.ShouldBe(0);
await host.DisposeAsync();
}
/// <summary>#686: an explicit reverse read with no cap returns the whole window newest-first in one page.</summary>
[Fact]
public async Task Raw_unbounded_explicit_reverse_read_on_a_forward_only_backend_returns_the_window_descending()
{
var source = new ForwardOnlyHistorianDataSource(MakeSeries(count: 250, stepSeconds: 1));
var (host, nm, nodeId) = await BootForwardOnlyAsync(source, "FwdRevAll");
// (end, start] = (Epoch+9s, Epoch+200s] — both edges exercised.
var (values, error, cp) = ReadRaw(nm, nodeId, Epoch.AddSeconds(200), Epoch.AddSeconds(9), max: 0, inboundCp: null);
error.StatusCode.Code.ShouldBe(StatusCodes.Good);
values.ShouldBe(Enumerable.Range(10, 191).Reverse().Select(i => (double)i));
cp.ShouldBeNull();
source.ReverseWindowCount.ShouldBe(0);
await host.DisposeAsync();
}
/// <summary>#686: a reverse read paging through a tie cluster on a forward-only backend — every value once, newest first.</summary>
[Fact]
public async Task Raw_reverse_read_on_a_forward_only_backend_pages_through_a_tie_cluster()
{
// No AVEVA boundary row here: paging WITHIN the cluster re-reads it with the node manager's start == end point
// over-fetch, and a row fabricated at a point read's start is indistinguishable from a tie at that instant —
// dropping it is the Gateway source's job (#687), not the walk's.
var source = new ForwardOnlyHistorianDataSource(
MakeSeriesWithTieCluster(distinctBefore: 3, tieCount: 5, distinctAfter: 2), emitBoundaryRow: false);
var (host, nm, nodeId) = await BootForwardOnlyAsync(source, "FwdRevBurst");
var collected = await DrainAsync(nm, nodeId, start: Epoch.AddHours(1), end: Epoch.AddSeconds(-1), max: 2);
collected.ShouldBe(Enumerable.Range(0, 10).Reverse().Select(i => (double)i));
source.ReverseWindowCount.ShouldBe(0);
await host.DisposeAsync();
}
/// <summary>#686: a segment denser than the per-read row budget (<c>MaxTieClusterOverfetch</c>) is narrowed and re-read,
/// never answered from a truncated forward read.</summary>
[Fact]
public async Task Raw_reverse_read_on_a_forward_only_backend_narrows_a_segment_over_the_row_budget()
{
var source = new ForwardOnlyHistorianDataSource(MakeSeries(count: 250, stepSeconds: 1));
var (host, nm, nodeId) = await BootForwardOnlyAsync(source, "FwdDense");
nm.MaxTieClusterOverfetch = 16; // a 1-minute segment holds 60 samples
var collected = await DrainAsync(nm, nodeId, DateTime.MinValue, Epoch.AddHours(1), max: 100);
collected.ShouldBe(Enumerable.Range(0, 250).Reverse().Select(i => (double)i));
source.MaxRowsReturned.ShouldBeLessThanOrEqualTo(17);
source.ReverseWindowCount.ShouldBe(0);
await host.DisposeAsync();
}
/// <summary>#686: sparse history far before EndTime is found, in a bounded number of forward reads.</summary>
[Fact]
public async Task Raw_end_only_read_on_a_forward_only_backend_finds_sparse_history_in_bounded_reads()
{
static DataValueSnapshot At(int year, double v)
{
var t = new DateTime(year, 1, 1, 0, 0, 0, DateTimeKind.Utc);
return new DataValueSnapshot(v, StatusCodes.Good, t, t);
}
var source = new ForwardOnlyHistorianDataSource([At(1990, 1), At(2000, 2)]);
var (host, nm, nodeId) = await BootForwardOnlyAsync(source, "FwdSparse");
var (values, error, cp) = ReadRaw(nm, nodeId, DateTime.MinValue, Epoch, max: 5, inboundCp: null);
error.StatusCode.Code.ShouldBe(StatusCodes.Good);
values.ShouldBe([2d, 1d]);
cp.ShouldBeNull();
source.ReadCount.ShouldBeLessThan(40);
source.ReverseWindowCount.ShouldBe(0);
await host.DisposeAsync();
}
/// <summary>#686: a single-timestamp tie cluster larger than the row budget cannot be read newest-first from
/// forward reads — the node is refused <c>BadTooManyOperations</c> (a request limit, not a backend fault).</summary>
[Fact]
public async Task Raw_reverse_read_on_a_forward_only_backend_refuses_a_tie_cluster_over_the_row_budget()
{
var source = new ForwardOnlyHistorianDataSource(MakeSeriesWithTieCluster(distinctBefore: 1, tieCount: 10, distinctAfter: 0));
var (host, nm, nodeId) = await BootForwardOnlyAsync(source, "FwdHugeTie");
nm.MaxTieClusterOverfetch = 4;
var (_, error, cp) = ReadRaw(nm, nodeId, Epoch.AddHours(1), Epoch.AddSeconds(-1), max: 2, inboundCp: null);
error.StatusCode.Code.ShouldBe(StatusCodes.BadTooManyOperations);
cp.ShouldBeNull();
source.ReverseWindowCount.ShouldBe(0);
await host.DisposeAsync();
}
/// <summary>A backend that DECLARES native reverse support is still sent the reverse window — one read.</summary>
[Fact]
public async Task Raw_end_only_read_on_a_reverse_capable_backend_is_one_native_reverse_read()
{
var (host, server) = await BootAsync();
var nm = server.NodeManager!;
nm.HistoryContinuationStore = new InMemoryHistoryContinuationStore();
var source = new SeriesHistorianDataSource(MakeSeries(count: 250, stepSeconds: 1));
nm.HistorianDataSource = source;
nm.EnsureVariable("eq-1/native", parentFolderNodeId: null, displayName: "Native", dataType: "Double",
writable: false, historianTagname: "WW.Native", realm: AddressSpaceRealm.Uns);
var nodeId = nm.TryGetVariable("eq-1/native")!.NodeId;
var (values, _, _) = ReadRaw(nm, nodeId, DateTime.MinValue, Epoch.AddSeconds(100), max: 3, inboundCp: null);
values.ShouldBe([100d, 99d, 98d]);
source.ReadCount.ShouldBe(1);
await host.DisposeAsync();
}
/// <summary>Pages a Raw read to completion and returns the union of the pages in arrival order.</summary>
private static Task<List<double>> DrainAsync(OtOpcUaNodeManager nm, NodeId nodeId, DateTime start, DateTime end, uint max)
{
@@ -584,6 +747,9 @@ public sealed class NodeManagerHistoryReadPagingTests : IDisposable
private sealed class SeriesHistorianDataSource(IReadOnlyList<DataValueSnapshot> series) : IHistorianDataSource
{
private int _readCount;
/// <summary>Serves start &gt; end natively — the SQL backends' contract (#686: declared, like they do).</summary>
public bool SupportsReverseRaw => true;
public int ReadCount => _readCount;
public void ResetReadCount() => _readCount = 0;
@@ -629,6 +795,70 @@ public sealed class NodeManagerHistoryReadPagingTests : IDisposable
public void Dispose() { }
}
/// <summary>
/// #686: a Gateway-shaped backend that reads FORWARD only. <see cref="SupportsReverseRaw"/> is false; a reverse window
/// (<c>start &gt; end</c>) is counted and rejected as the gateway does (<c>InvalidArgument</c>). A forward read is
/// inclusive <c>[start, end]</c> and, like an AVEVA <c>Full</c> read, first emits a synthetic boundary row stamped at
/// the query start carrying the value in effect there (Bad 0 when there is none) — counted against the cap.
/// </summary>
private sealed class ForwardOnlyHistorianDataSource(
IReadOnlyList<DataValueSnapshot> series, bool emitBoundaryRow = true) : IHistorianDataSource
{
private int _readCount;
private int _reverseWindowCount;
private int _maxRowsReturned;
public int ReadCount => _readCount;
public int ReverseWindowCount => _reverseWindowCount;
public int MaxRowsReturned => _maxRowsReturned;
public bool SupportsReverseRaw => false;
public Task<HistorianRead> ReadRawAsync(
string fullReference, DateTime startUtc, DateTime endUtc, uint maxValuesPerNode,
CancellationToken cancellationToken)
{
Interlocked.Increment(ref _readCount);
if (startUtc > endUtc)
{
Interlocked.Increment(ref _reverseWindowCount);
throw new InvalidOperationException("start_time must be less than or equal to end_time. (InvalidArgument)");
}
static DateTime Ts(DataValueSnapshot s) => s.SourceTimestampUtc ?? DateTime.MinValue;
var prior = series.LastOrDefault(s => Ts(s) <= startUtc);
var rows = new List<DataValueSnapshot>();
if (emitBoundaryRow)
rows.Add(prior is null
? new DataValueSnapshot(0d, StatusCodes.BadNoData, startUtc, startUtc)
: new DataValueSnapshot(prior.Value, prior.StatusCode, startUtc, startUtc));
rows.AddRange(series.Where(s => Ts(s) >= startUtc && Ts(s) <= endUtc));
var page = maxValuesPerNode == 0 ? rows : rows.Take((int)maxValuesPerNode).ToList();
int seen;
while (page.Count > (seen = Volatile.Read(ref _maxRowsReturned))
&& Interlocked.CompareExchange(ref _maxRowsReturned, page.Count, seen) != seen) { }
return Task.FromResult(new HistorianRead(page, null));
}
public Task<HistorianRead> ReadProcessedAsync(
string fullReference, DateTime startUtc, DateTime endUtc, TimeSpan interval,
HistoryAggregateType aggregate, CancellationToken cancellationToken) =>
Task.FromResult(new HistorianRead(Array.Empty<DataValueSnapshot>(), null));
public Task<HistorianRead> ReadAtTimeAsync(
string fullReference, IReadOnlyList<DateTime> timestampsUtc, CancellationToken cancellationToken) =>
Task.FromResult(new HistorianRead(Array.Empty<DataValueSnapshot>(), null));
public Task<HistoricalEventsResult> ReadEventsAsync(
string? sourceName, DateTime startUtc, DateTime endUtc, int maxEvents,
CancellationToken cancellationToken) =>
Task.FromResult(new HistoricalEventsResult(Array.Empty<HistoricalEvent>(), null));
public HistorianHealthSnapshot GetHealthSnapshot() => NullHistorianDataSource.Instance.GetHealthSnapshot();
public void Dispose() { }
}
/// <summary>
/// A microsecond-resolution backend, shaped like QuestDB: every inbound window bound is TRUNCATED to 10 ticks
/// (as <c>QuestDbConnection.Round</c> does) before the series is filtered, and the source reports
@@ -640,6 +870,8 @@ public sealed class NodeManagerHistoryReadPagingTests : IDisposable
public long TimestampResolutionTicks => 10;
public bool SupportsReverseRaw => true; // QuestDB serves reverse natively
private static DateTime Truncate(DateTime t) => new(t.Ticks - t.Ticks % 10, DateTimeKind.Utc);
public Task<HistorianRead> ReadRawAsync(