fix(historian): #686 serve reverse/end-only raw reads on forward-only backends (Gateway)
#669 turned an end-only capped HistoryRead-Raw into a reverse read (start > end) for every backend; the gateway rejects start > end (InvalidArgument), so on the default Gateway backend every end-only capped and every explicit reverse raw read failed the node and was counted as a backend health failure. - IHistorianDataSource.SupportsReverseRaw (default false; SqlServer/QuestDB/Null true, Gateway false; guard test pins that every production backend declares it). - OtOpcUaNodeManager sends a reverse window only to a backend that declares it; otherwise ReverseRawReader walks (lo, hi] back in bounded forward segment reads (1 min, x4 while sparse; each <= MaxTieClusterOverfetch + 1 rows; a full read is re-read narrower, never answered truncated; filtered on sample timestamps, which also drops AVEVA's fabricated query-start rows; stops at the Unix epoch). A one-timestamp tie cluster over the bound => BadTooManyOperations. - ServeRawPagedAsync maps HistoryReadLimitExceededException to BadTooManyOperations. - GatewayHistorianDataSource.ReadRawAsync refuses start > end unsent, without recording a health failure.
This commit is contained in:
@@ -17,6 +17,12 @@ public interface IHistorianDataSource : IDisposable
|
||||
/// <summary>
|
||||
/// Read raw historical samples for a single tag over a time range.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Forward (<c>startUtc < endUtc</c>) is <c>[start, end)</c> ascending; <c>startUtc == endUtc</c> is a point read.
|
||||
/// <c>startUtc > endUtc</c> is the Part 11 reverse read <c>(end, start]</c> newest-first — but only a backend that
|
||||
/// declares <see cref="SupportsReverseRaw"/> is ever sent one (Gitea #686); the server serves a reverse read on any
|
||||
/// other backend from bounded forward reads.
|
||||
/// </remarks>
|
||||
/// <param name="fullReference">The full reference of the tag to read.</param>
|
||||
/// <param name="startUtc">The start of the time range in UTC.</param>
|
||||
/// <param name="endUtc">The end of the time range in UTC.</param>
|
||||
@@ -30,6 +36,16 @@ public interface IHistorianDataSource : IDisposable
|
||||
uint maxValuesPerNode,
|
||||
CancellationToken cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// Whether this backend can serve a reverse (newest-first) raw read itself — <see cref="ReadRawAsync"/> with
|
||||
/// <c>startUtc > endUtc</c> (Gitea #686). The server checks it before every raw read. When <c>false</c> it never
|
||||
/// sends <c>start > end</c>: it serves the reverse read — an explicit one, a resumed reverse page, and the
|
||||
/// end-only capped read Part 11 reads backward from EndTime — from a series of bounded FORWARD reads walking back
|
||||
/// from the later bound (<c>ReverseRawReader</c> in the OPC UA server). Default <c>false</c>, the safe direction: a
|
||||
/// source that only reads forward is still answered correctly. A backend declares it itself (a guard test pins that).
|
||||
/// </summary>
|
||||
bool SupportsReverseRaw => false;
|
||||
|
||||
/// <summary>
|
||||
/// Read processed (interval-bucketed) samples — average / min / max / count / etc.
|
||||
/// A bucket with no source data returns a sample whose
|
||||
|
||||
@@ -55,6 +55,11 @@ public sealed class NullHistorianDataSource : IHistorianDataSource
|
||||
int maxEvents,
|
||||
CancellationToken cancellationToken) => Task.FromResult(EmptyEvents);
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks><c>true</c> (Gitea #686): there is no history, so the empty result is the correct answer in either
|
||||
/// direction — no need for the server to walk back through forward reads to find nothing.</remarks>
|
||||
public bool SupportsReverseRaw => true;
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks><c>false</c> (Gitea #675): there is no history to read in either direction, and an explicit reverse read
|
||||
/// is answered <c>BadHistoryOperationUnsupported</c> like any backend that cannot read newest-first.</remarks>
|
||||
|
||||
@@ -76,10 +76,18 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>
|
||||
/// A reverse window (<c>start > end</c>) is refused with <see cref="NotSupportedException"/> before anything is sent
|
||||
/// and is not recorded as a health failure (Gitea #686): the gateway rejects it <c>InvalidArgument</c>, and the server
|
||||
/// never asks — <see cref="SupportsReverseRaw"/> is <c>false</c>, so it walks a reverse read back with forward reads.
|
||||
/// </remarks>
|
||||
public async Task<HistoryReadResult> ReadRawAsync(
|
||||
string fullReference, DateTime startUtc, DateTime endUtc, uint maxValuesPerNode,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (startUtc > endUtc)
|
||||
throw new NotSupportedException("The HistorianGateway reads raw history oldest-first only (start > end).");
|
||||
|
||||
try
|
||||
{
|
||||
// The gateway seam caps with an int; OPC UA hands us a uint, so clamp to int range.
|
||||
@@ -235,6 +243,14 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>
|
||||
/// <c>false</c> (Gitea #686): the gateway's <c>ReadRaw</c> streams ascending only and rejects <c>end < start</c>
|
||||
/// (<c>InvalidArgument</c>). The server serves reverse and end-only capped raw reads from bounded forward reads
|
||||
/// instead, and never sends this source a reverse window (<see cref="ReadRawAsync"/> refuses one unsent).
|
||||
/// </remarks>
|
||||
public bool SupportsReverseRaw => false;
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>
|
||||
/// <c>false</c>: the gateway's <c>ReadEvents</c> streams ascending only (Gitea #675). The server therefore refuses an
|
||||
|
||||
@@ -355,6 +355,10 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource
|
||||
|
||||
// ------------------------------------------------------------------------------------------ Events
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>Reverse raw reads are served natively (<c>(end, start]</c>, <c>ORDER BY … DESC</c>; #669/#686).</remarks>
|
||||
public bool SupportsReverseRaw => true;
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>Reverse events reads are served natively (<c>ORDER BY … DESC</c>, Gitea #675).</remarks>
|
||||
public bool SupportsReverseEvents => true;
|
||||
|
||||
+4
@@ -312,6 +312,10 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
|
||||
|
||||
// ------------------------------------------------------------------------------------------ Events
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>Reverse raw reads are served natively (<c>(end, start]</c>, <c>ORDER BY … DESC</c>; #669/#686).</remarks>
|
||||
public bool SupportsReverseRaw => true;
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>Reverse events reads are served natively (<c>ORDER BY … DESC</c>, Gitea #675).</remarks>
|
||||
public bool SupportsReverseEvents => true;
|
||||
|
||||
@@ -6219,6 +6219,8 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2
|
||||
// Part 11: with NumValuesPerNode > 0 one bound may be left unset. Only a StartTime ⇒ page FORWARD from
|
||||
// it to an open upper bound; only an EndTime ⇒ page BACKWARD from it (a reverse read, start > end).
|
||||
// Without this an unset EndTime (MinValue) looked like a reverse read to a lower bound of "now".
|
||||
// Every backend serves the backward read (#686): natively when it declares SupportsReverseRaw, otherwise
|
||||
// walked back from bounded forward reads below — unlike the events path, which keeps it forward there.
|
||||
if (numValuesPerNode > 0)
|
||||
{
|
||||
if (endUtc == DateTime.MinValue && startUtc != DateTime.MinValue)
|
||||
@@ -6229,9 +6231,20 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2
|
||||
}
|
||||
|
||||
// HistoryRead is NOT under the node-manager Lock — awaiting the async source is safe.
|
||||
var sourceResult = await HistorianDataSource
|
||||
.ReadRawAsync(tagname, startUtc, endUtc, numValuesPerNode, ct)
|
||||
.ConfigureAwait(false);
|
||||
// A reverse window (explicit, resumed, or the end-only read above) goes to the backend only when it declares it
|
||||
// can read newest-first; otherwise it is walked back from bounded forward reads (#686 — the Gateway rejects
|
||||
// start > end, which failed the node and counted as a backend fault).
|
||||
var source = HistorianDataSource;
|
||||
var sourceResult = startUtc > endUtc && !source.SupportsReverseRaw
|
||||
? new HistorianRead(
|
||||
await ReverseRawReader.ReadAsync(
|
||||
source, tagname, startUtc, endUtc, numValuesPerNode,
|
||||
(int)Math.Min(int.MaxValue, (long)Math.Max(1, MaxTieClusterOverfetch) + 1), ct)
|
||||
.ConfigureAwait(false),
|
||||
ContinuationPoint: null)
|
||||
: await source
|
||||
.ReadRawAsync(tagname, startUtc, endUtc, numValuesPerNode, ct)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
var backendFull = HistoryPaging.IsFullPage(sourceResult.Samples.Count, numValuesPerNode);
|
||||
|
||||
@@ -6348,6 +6361,14 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2
|
||||
errors[handle.Index] = StatusCodes.BadTimeout;
|
||||
results[handle.Index] = new SdkHistoryReadResult { StatusCode = StatusCodes.BadTimeout };
|
||||
}
|
||||
catch (HistoryReadLimitExceededException ex)
|
||||
{
|
||||
// A request limit (#686: a reverse read walked back from forward reads met a tie cluster over the row
|
||||
// budget) — BadTooManyOperations for THIS node, as on the processed path; not a backend fault.
|
||||
_logger.LogWarning("OtOpcUaNodeManager: HistoryReadRaw limit exceeded for node {NodeId}: {Message}", handle.NodeId, ex.Message);
|
||||
errors[handle.Index] = StatusCodes.BadTooManyOperations;
|
||||
results[handle.Index] = new SdkHistoryReadResult { StatusCode = StatusCodes.BadTooManyOperations };
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
// One node's backend failure must not throw out of the batch — Bad for THIS node only.
|
||||
|
||||
@@ -0,0 +1,120 @@
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.OpcUaServer;
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #686: serves a Part 11 reverse raw read — <c>(lo, hi]</c> newest-first, capped — on a historian backend that can
|
||||
/// only read forward (<see cref="IHistorianDataSource.SupportsReverseRaw"/> = <c>false</c>: the Gateway, whose
|
||||
/// <c>ReadRaw</c> streams ascending and rejects <c>start > end</c>).
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>Walk back in segments.</b> Starting at <c>hi</c>, read one segment <c>(segLo, segHi]</c> FORWARD, keep it
|
||||
/// newest-first, and step to the next older segment until the cap is met or <c>lo</c> is reached. A segment starts
|
||||
/// at <see cref="InitialSegment"/> and grows ×4 while segments come back sparse, so "the latest N" of a live tag is
|
||||
/// usually one read, and sparse history decades back is found in a few dozen tiny reads rather than one read of
|
||||
/// all history.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Never answer from a truncated read.</b> Each segment read asks for at most <c>segmentCap</c> rows (the server's
|
||||
/// <c>MaxTieClusterOverfetch + 1</c> over-fetch bound). A read that comes back FULL may have been cut short — its
|
||||
/// newest rows are exactly the ones a reverse read needs — so it is discarded and the segment re-read at a quarter of
|
||||
/// the width. A full read of a segment already one timestamp-resolution wide (a single-timestamp tie cluster larger
|
||||
/// than the bound) cannot be served and throws <see cref="HistoryReadLimitExceededException"/>
|
||||
/// (⇒ <c>BadTooManyOperations</c>, a request limit, never a backend fault).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Edges, for any forward contract.</b> A segment is requested as <c>[segLo, segHi + resolution]</c> and filtered to
|
||||
/// <c>(segLo, segHi]</c> on the samples' own timestamps, so it is exact whether the backend's forward end is exclusive
|
||||
/// (the interface contract) or inclusive (the gateway), and whether it truncates its bounds (QuestDB-style). The
|
||||
/// filter also drops the rows an AVEVA <c>Full</c> read fabricates at the query start (the boundary / no-data row,
|
||||
/// #687) and at the query end. Adjacent segments share no timestamp, so nothing is returned twice.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Ties</b> come back in the backend's forward order reversed — the same order the node manager's tie-cluster
|
||||
/// over-fetch uses for a reverse read — so a resumed page's boundary-tie skip lines up.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Floor.</b> The walk stops at <see cref="Floor"/> (the Unix epoch; its earliest request bound is one tick
|
||||
/// before it): the AVEVA historian holds nothing earlier, and an end-only read's open lower bound (year 1) would
|
||||
/// otherwise cost dozens of empty reads and send a bound outside the historian's time encoding. Samples before the
|
||||
/// floor are not returned by a walked-back read; a window entirely before it is empty.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
internal static class ReverseRawReader
|
||||
{
|
||||
/// <summary>The first segment's width.</summary>
|
||||
internal static readonly TimeSpan InitialSegment = TimeSpan.FromMinutes(1);
|
||||
|
||||
/// <summary>The earliest instant a walk reads (see the remarks).</summary>
|
||||
internal static readonly DateTime Floor = DateTime.UnixEpoch;
|
||||
|
||||
/// <summary>Reads <c>(lo, hi]</c> newest-first from forward reads.</summary>
|
||||
/// <param name="source">A backend that reads forward.</param>
|
||||
/// <param name="tagname">Historian tag.</param>
|
||||
/// <param name="hi">The later (inclusive) bound — the request's StartTime or a reverse resume point.</param>
|
||||
/// <param name="lo">The earlier (exclusive) bound.</param>
|
||||
/// <param name="maxValues">Cap; <c>0</c> = every sample in the window.</param>
|
||||
/// <param name="segmentCap">Rows one segment read may return before it is treated as truncated (≥ 2).</param>
|
||||
/// <param name="ct">Cancellation.</param>
|
||||
/// <returns>At most <paramref name="maxValues"/> samples, newest first.</returns>
|
||||
/// <exception cref="HistoryReadLimitExceededException">A single-timestamp tie cluster exceeds the segment cap.</exception>
|
||||
public static async Task<IReadOnlyList<DataValueSnapshot>> ReadAsync(
|
||||
IHistorianDataSource source, string tagname, DateTime hi, DateTime lo, uint maxValues, int segmentCap,
|
||||
CancellationToken ct)
|
||||
{
|
||||
var resolution = Math.Max(1L, source.TimestampResolutionTicks);
|
||||
var cap = (uint)Math.Max(2, segmentCap);
|
||||
var result = new List<DataValueSnapshot>();
|
||||
var width = Math.Max(resolution, InitialSegment.Ticks);
|
||||
var segHiTicks = hi.Ticks;
|
||||
|
||||
// The walk's exclusive lower edge. Clamped to one tick before the floor, so a sample exactly at the floor is still
|
||||
// in the window (and the row a backend fabricates at that query start is still filtered out).
|
||||
var stopTicks = lo.Ticks < Floor.Ticks ? Floor.Ticks - 1 : lo.Ticks;
|
||||
|
||||
while (segHiTicks > stopTicks && (maxValues == 0 || result.Count < maxValues))
|
||||
{
|
||||
ct.ThrowIfCancellationRequested();
|
||||
var segLoTicks = segHiTicks - stopTicks <= width ? stopTicks : segHiTicks - width;
|
||||
var requestEndTicks = segHiTicks > DateTime.MaxValue.Ticks - resolution
|
||||
? DateTime.MaxValue.Ticks
|
||||
: segHiTicks + resolution;
|
||||
|
||||
var rows = (await source.ReadRawAsync(
|
||||
tagname,
|
||||
new DateTime(segLoTicks, DateTimeKind.Utc),
|
||||
new DateTime(requestEndTicks, DateTimeKind.Utc),
|
||||
cap, ct)
|
||||
.ConfigureAwait(false)).Samples;
|
||||
|
||||
if (rows.Count >= cap)
|
||||
{
|
||||
// Possibly truncated: its newest rows — the ones needed — may be missing. Narrow and re-read.
|
||||
if (width <= resolution)
|
||||
throw new HistoryReadLimitExceededException(cap - 1,
|
||||
$"A reverse raw read of '{tagname}' at {new DateTime(segHiTicks, DateTimeKind.Utc):O} found more than " +
|
||||
$"{cap - 1} samples at one timestamp; the backend reads forward only, so the latest of them cannot be " +
|
||||
"selected. Increase ServerHistorian:MaxTieClusterOverfetch.");
|
||||
width = Math.Max(resolution, width / 4);
|
||||
continue;
|
||||
}
|
||||
|
||||
var inSegment = 0;
|
||||
for (var i = rows.Count - 1; i >= 0; i--)
|
||||
{
|
||||
var ts = (rows[i].SourceTimestampUtc ?? DateTime.MinValue).Ticks;
|
||||
if (ts <= segLoTicks || ts > segHiTicks) continue;
|
||||
inSegment++;
|
||||
if (maxValues == 0 || result.Count < maxValues) result.Add(rows[i]);
|
||||
}
|
||||
|
||||
// Sparse ⇒ widen; otherwise keep the width that fit.
|
||||
if (inSegment <= cap / 8)
|
||||
width = width > DateTime.MaxValue.Ticks / 4 ? DateTime.MaxValue.Ticks : width * 4;
|
||||
segHiTicks = segLoTicks;
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
}
|
||||
+16
@@ -64,6 +64,22 @@ public sealed class HistorianDataSourceReverseEventsGuardTests
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Every_implementation_declares_its_reverse_raw_capability_itself()
|
||||
{
|
||||
// Gitea #686: the default (false) is safe — the server walks back with forward reads — but a backend that serves
|
||||
// reverse natively and inherited it would silently lose its native path, and a wrapper of the Gateway that claimed
|
||||
// true would send start > end to a gateway that rejects it. Declare it, both ways.
|
||||
foreach (var type in ProductionImplementations())
|
||||
{
|
||||
var map = type.GetInterfaceMap(typeof(IHistorianDataSource));
|
||||
var getter = map.TargetMethods[Array.FindIndex(map.InterfaceMethods,
|
||||
m => m.Name == "get_" + nameof(IHistorianDataSource.SupportsReverseRaw))];
|
||||
getter.DeclaringType.ShouldBe(type,
|
||||
$"{type.FullName} must declare SupportsReverseRaw itself, not inherit the interface default");
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Every_implementation_declares_its_supported_aggregates_itself()
|
||||
{
|
||||
|
||||
@@ -56,6 +56,14 @@ public abstract class RawReadConformance : ConformanceBase
|
||||
Values(await B.CreateDataSource().ReadRawAsync("A", At(10), max, 0, Ct)).ShouldBe([20d, 30d, 40d, 50d]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void The_backend_reports_reverse_raw_support()
|
||||
{
|
||||
// #686: the server only sends start > end to a backend that declares it; these backends serve it natively.
|
||||
using var src = B.CreateDataSource();
|
||||
src.SupportsReverseRaw.ShouldBeTrue();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Reverse_read_is_descending_over_end_exclusive_start_inclusive()
|
||||
{
|
||||
|
||||
+45
@@ -0,0 +1,45 @@
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #686: the gateway's <c>ReadRaw</c> streams ascending only and rejects <c>start > end</c> with
|
||||
/// <c>InvalidArgument</c>, which used to be recorded as a backend health failure. The source declares it cannot read
|
||||
/// newest-first (so the server walks back with forward reads), and refuses a reverse window itself without sending
|
||||
/// anything or counting a failure.
|
||||
/// </summary>
|
||||
public sealed class GatewayRawDirectionTests
|
||||
{
|
||||
private static CancellationToken Ct => TestContext.Current.CancellationToken;
|
||||
|
||||
[Fact]
|
||||
public void The_gateway_source_declares_it_cannot_read_raw_newest_first()
|
||||
{
|
||||
IHistorianDataSource ds = new GatewayHistorianDataSource(
|
||||
new FakeHistorianGatewayClient(), NullLogger<GatewayHistorianDataSource>.Instance);
|
||||
ds.SupportsReverseRaw.ShouldBeFalse();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_reverse_raw_window_is_refused_unsent_and_is_not_a_backend_failure()
|
||||
{
|
||||
var fake = new FakeHistorianGatewayClient();
|
||||
IHistorianDataSource ds = new GatewayHistorianDataSource(fake, NullLogger<GatewayHistorianDataSource>.Instance);
|
||||
var end = new DateTime(2026, 10, 12, 18, 7, 45, DateTimeKind.Utc);
|
||||
|
||||
await Should.ThrowAsync<NotSupportedException>(() =>
|
||||
ds.ReadRawAsync("Gate681", end, new DateTime(0, DateTimeKind.Utc), 3, Ct));
|
||||
|
||||
fake.ReadRawCallCount.ShouldBe(0);
|
||||
var health = ds.GetHealthSnapshot();
|
||||
health.TotalFailures.ShouldBe(0);
|
||||
health.ConsecutiveFailures.ShouldBe(0);
|
||||
|
||||
// Positive control: a forward read of the same window is sent.
|
||||
await ds.ReadRawAsync("Gate681", new DateTime(2026, 10, 12, 18, 0, 0, DateTimeKind.Utc), end, 3, Ct);
|
||||
fake.ReadRawCallCount.ShouldBe(1);
|
||||
}
|
||||
}
|
||||
+232
@@ -466,6 +466,169 @@ public sealed class NodeManagerHistoryReadPagingTests : IDisposable
|
||||
await host.DisposeAsync();
|
||||
}
|
||||
|
||||
// --- #686: reverse / end-only raw reads on a backend that only reads forward (the Gateway) -------------------------
|
||||
|
||||
/// <summary>Boots a node manager over a <see cref="ForwardOnlyHistorianDataSource"/> and one historized variable.</summary>
|
||||
private async Task<(OpcUaApplicationHost Host, OtOpcUaNodeManager Nm, NodeId NodeId)> BootForwardOnlyAsync(
|
||||
ForwardOnlyHistorianDataSource source, string name)
|
||||
{
|
||||
var (host, server) = await BootAsync();
|
||||
var nm = server.NodeManager!;
|
||||
nm.HistoryContinuationStore = new InMemoryHistoryContinuationStore();
|
||||
nm.HistorianDataSource = source;
|
||||
nm.EnsureVariable($"eq-1/{name}", parentFolderNodeId: null, displayName: name, dataType: "Double",
|
||||
writable: false, historianTagname: $"WW.{name}", realm: AddressSpaceRealm.Uns);
|
||||
return (host, nm, nm.TryGetVariable($"eq-1/{name}")!.NodeId);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// #686: an end-only capped read (Part 11: page BACKWARD from EndTime) on a backend that cannot read newest-first
|
||||
/// pages every sample newest-first — and never sends that backend <c>start > end</c> (the gateway rejects it
|
||||
/// <c>InvalidArgument</c>, which used to fail the node and count as a backend fault).
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task Raw_end_only_read_on_a_forward_only_backend_pages_descending_without_a_reverse_window()
|
||||
{
|
||||
var source = new ForwardOnlyHistorianDataSource(MakeSeries(count: 250, stepSeconds: 1));
|
||||
var (host, nm, nodeId) = await BootForwardOnlyAsync(source, "FwdEndOnly");
|
||||
|
||||
var collected = await DrainAsync(nm, nodeId, DateTime.MinValue, Epoch.AddHours(1), max: 100);
|
||||
|
||||
collected.ShouldBe(Enumerable.Range(0, 250).Reverse().Select(i => (double)i));
|
||||
source.ReverseWindowCount.ShouldBe(0);
|
||||
await host.DisposeAsync();
|
||||
}
|
||||
|
||||
/// <summary>#686: the first page of an end-only capped read is the N values AT OR BEFORE EndTime, newest first —
|
||||
/// not the oldest N in history (the pre-#669 forward meaning), and not the synthetic boundary row the AVEVA-shaped
|
||||
/// fake emits at each query start.</summary>
|
||||
[Fact]
|
||||
public async Task Raw_end_only_capped_read_on_a_forward_only_backend_returns_the_latest_values_before_end_time()
|
||||
{
|
||||
var source = new ForwardOnlyHistorianDataSource(MakeSeries(count: 250, stepSeconds: 1));
|
||||
var (host, nm, nodeId) = await BootForwardOnlyAsync(source, "FwdLatest");
|
||||
|
||||
var (values, error, cp) = ReadRaw(nm, nodeId, DateTime.MinValue, Epoch.AddSeconds(100), max: 3, inboundCp: null);
|
||||
|
||||
error.StatusCode.Code.ShouldBe(StatusCodes.Good);
|
||||
values.ShouldBe([100d, 99d, 98d]);
|
||||
cp.ShouldNotBeNull();
|
||||
source.ReverseWindowCount.ShouldBe(0);
|
||||
await host.DisposeAsync();
|
||||
}
|
||||
|
||||
/// <summary>#686: an explicit reverse read with no cap returns the whole window newest-first in one page.</summary>
|
||||
[Fact]
|
||||
public async Task Raw_unbounded_explicit_reverse_read_on_a_forward_only_backend_returns_the_window_descending()
|
||||
{
|
||||
var source = new ForwardOnlyHistorianDataSource(MakeSeries(count: 250, stepSeconds: 1));
|
||||
var (host, nm, nodeId) = await BootForwardOnlyAsync(source, "FwdRevAll");
|
||||
|
||||
// (end, start] = (Epoch+9s, Epoch+200s] — both edges exercised.
|
||||
var (values, error, cp) = ReadRaw(nm, nodeId, Epoch.AddSeconds(200), Epoch.AddSeconds(9), max: 0, inboundCp: null);
|
||||
|
||||
error.StatusCode.Code.ShouldBe(StatusCodes.Good);
|
||||
values.ShouldBe(Enumerable.Range(10, 191).Reverse().Select(i => (double)i));
|
||||
cp.ShouldBeNull();
|
||||
source.ReverseWindowCount.ShouldBe(0);
|
||||
await host.DisposeAsync();
|
||||
}
|
||||
|
||||
/// <summary>#686: a reverse read paging through a tie cluster on a forward-only backend — every value once, newest first.</summary>
|
||||
[Fact]
|
||||
public async Task Raw_reverse_read_on_a_forward_only_backend_pages_through_a_tie_cluster()
|
||||
{
|
||||
// No AVEVA boundary row here: paging WITHIN the cluster re-reads it with the node manager's start == end point
|
||||
// over-fetch, and a row fabricated at a point read's start is indistinguishable from a tie at that instant —
|
||||
// dropping it is the Gateway source's job (#687), not the walk's.
|
||||
var source = new ForwardOnlyHistorianDataSource(
|
||||
MakeSeriesWithTieCluster(distinctBefore: 3, tieCount: 5, distinctAfter: 2), emitBoundaryRow: false);
|
||||
var (host, nm, nodeId) = await BootForwardOnlyAsync(source, "FwdRevBurst");
|
||||
|
||||
var collected = await DrainAsync(nm, nodeId, start: Epoch.AddHours(1), end: Epoch.AddSeconds(-1), max: 2);
|
||||
|
||||
collected.ShouldBe(Enumerable.Range(0, 10).Reverse().Select(i => (double)i));
|
||||
source.ReverseWindowCount.ShouldBe(0);
|
||||
await host.DisposeAsync();
|
||||
}
|
||||
|
||||
/// <summary>#686: a segment denser than the per-read row budget (<c>MaxTieClusterOverfetch</c>) is narrowed and re-read,
|
||||
/// never answered from a truncated forward read.</summary>
|
||||
[Fact]
|
||||
public async Task Raw_reverse_read_on_a_forward_only_backend_narrows_a_segment_over_the_row_budget()
|
||||
{
|
||||
var source = new ForwardOnlyHistorianDataSource(MakeSeries(count: 250, stepSeconds: 1));
|
||||
var (host, nm, nodeId) = await BootForwardOnlyAsync(source, "FwdDense");
|
||||
nm.MaxTieClusterOverfetch = 16; // a 1-minute segment holds 60 samples
|
||||
|
||||
var collected = await DrainAsync(nm, nodeId, DateTime.MinValue, Epoch.AddHours(1), max: 100);
|
||||
|
||||
collected.ShouldBe(Enumerable.Range(0, 250).Reverse().Select(i => (double)i));
|
||||
source.MaxRowsReturned.ShouldBeLessThanOrEqualTo(17);
|
||||
source.ReverseWindowCount.ShouldBe(0);
|
||||
await host.DisposeAsync();
|
||||
}
|
||||
|
||||
/// <summary>#686: sparse history far before EndTime is found, in a bounded number of forward reads.</summary>
|
||||
[Fact]
|
||||
public async Task Raw_end_only_read_on_a_forward_only_backend_finds_sparse_history_in_bounded_reads()
|
||||
{
|
||||
static DataValueSnapshot At(int year, double v)
|
||||
{
|
||||
var t = new DateTime(year, 1, 1, 0, 0, 0, DateTimeKind.Utc);
|
||||
return new DataValueSnapshot(v, StatusCodes.Good, t, t);
|
||||
}
|
||||
|
||||
var source = new ForwardOnlyHistorianDataSource([At(1990, 1), At(2000, 2)]);
|
||||
var (host, nm, nodeId) = await BootForwardOnlyAsync(source, "FwdSparse");
|
||||
|
||||
var (values, error, cp) = ReadRaw(nm, nodeId, DateTime.MinValue, Epoch, max: 5, inboundCp: null);
|
||||
|
||||
error.StatusCode.Code.ShouldBe(StatusCodes.Good);
|
||||
values.ShouldBe([2d, 1d]);
|
||||
cp.ShouldBeNull();
|
||||
source.ReadCount.ShouldBeLessThan(40);
|
||||
source.ReverseWindowCount.ShouldBe(0);
|
||||
await host.DisposeAsync();
|
||||
}
|
||||
|
||||
/// <summary>#686: a single-timestamp tie cluster larger than the row budget cannot be read newest-first from
|
||||
/// forward reads — the node is refused <c>BadTooManyOperations</c> (a request limit, not a backend fault).</summary>
|
||||
[Fact]
|
||||
public async Task Raw_reverse_read_on_a_forward_only_backend_refuses_a_tie_cluster_over_the_row_budget()
|
||||
{
|
||||
var source = new ForwardOnlyHistorianDataSource(MakeSeriesWithTieCluster(distinctBefore: 1, tieCount: 10, distinctAfter: 0));
|
||||
var (host, nm, nodeId) = await BootForwardOnlyAsync(source, "FwdHugeTie");
|
||||
nm.MaxTieClusterOverfetch = 4;
|
||||
|
||||
var (_, error, cp) = ReadRaw(nm, nodeId, Epoch.AddHours(1), Epoch.AddSeconds(-1), max: 2, inboundCp: null);
|
||||
|
||||
error.StatusCode.Code.ShouldBe(StatusCodes.BadTooManyOperations);
|
||||
cp.ShouldBeNull();
|
||||
source.ReverseWindowCount.ShouldBe(0);
|
||||
await host.DisposeAsync();
|
||||
}
|
||||
|
||||
/// <summary>A backend that DECLARES native reverse support is still sent the reverse window — one read.</summary>
|
||||
[Fact]
|
||||
public async Task Raw_end_only_read_on_a_reverse_capable_backend_is_one_native_reverse_read()
|
||||
{
|
||||
var (host, server) = await BootAsync();
|
||||
var nm = server.NodeManager!;
|
||||
nm.HistoryContinuationStore = new InMemoryHistoryContinuationStore();
|
||||
var source = new SeriesHistorianDataSource(MakeSeries(count: 250, stepSeconds: 1));
|
||||
nm.HistorianDataSource = source;
|
||||
nm.EnsureVariable("eq-1/native", parentFolderNodeId: null, displayName: "Native", dataType: "Double",
|
||||
writable: false, historianTagname: "WW.Native", realm: AddressSpaceRealm.Uns);
|
||||
var nodeId = nm.TryGetVariable("eq-1/native")!.NodeId;
|
||||
|
||||
var (values, _, _) = ReadRaw(nm, nodeId, DateTime.MinValue, Epoch.AddSeconds(100), max: 3, inboundCp: null);
|
||||
|
||||
values.ShouldBe([100d, 99d, 98d]);
|
||||
source.ReadCount.ShouldBe(1);
|
||||
await host.DisposeAsync();
|
||||
}
|
||||
|
||||
/// <summary>Pages a Raw read to completion and returns the union of the pages in arrival order.</summary>
|
||||
private static Task<List<double>> DrainAsync(OtOpcUaNodeManager nm, NodeId nodeId, DateTime start, DateTime end, uint max)
|
||||
{
|
||||
@@ -584,6 +747,9 @@ public sealed class NodeManagerHistoryReadPagingTests : IDisposable
|
||||
private sealed class SeriesHistorianDataSource(IReadOnlyList<DataValueSnapshot> series) : IHistorianDataSource
|
||||
{
|
||||
private int _readCount;
|
||||
|
||||
/// <summary>Serves start > end natively — the SQL backends' contract (#686: declared, like they do).</summary>
|
||||
public bool SupportsReverseRaw => true;
|
||||
public int ReadCount => _readCount;
|
||||
|
||||
public void ResetReadCount() => _readCount = 0;
|
||||
@@ -629,6 +795,70 @@ public sealed class NodeManagerHistoryReadPagingTests : IDisposable
|
||||
public void Dispose() { }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// #686: a Gateway-shaped backend that reads FORWARD only. <see cref="SupportsReverseRaw"/> is false; a reverse window
|
||||
/// (<c>start > end</c>) is counted and rejected as the gateway does (<c>InvalidArgument</c>). A forward read is
|
||||
/// inclusive <c>[start, end]</c> and, like an AVEVA <c>Full</c> read, first emits a synthetic boundary row stamped at
|
||||
/// the query start carrying the value in effect there (Bad 0 when there is none) — counted against the cap.
|
||||
/// </summary>
|
||||
private sealed class ForwardOnlyHistorianDataSource(
|
||||
IReadOnlyList<DataValueSnapshot> series, bool emitBoundaryRow = true) : IHistorianDataSource
|
||||
{
|
||||
private int _readCount;
|
||||
private int _reverseWindowCount;
|
||||
private int _maxRowsReturned;
|
||||
|
||||
public int ReadCount => _readCount;
|
||||
public int ReverseWindowCount => _reverseWindowCount;
|
||||
public int MaxRowsReturned => _maxRowsReturned;
|
||||
|
||||
public bool SupportsReverseRaw => false;
|
||||
|
||||
public Task<HistorianRead> ReadRawAsync(
|
||||
string fullReference, DateTime startUtc, DateTime endUtc, uint maxValuesPerNode,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
Interlocked.Increment(ref _readCount);
|
||||
if (startUtc > endUtc)
|
||||
{
|
||||
Interlocked.Increment(ref _reverseWindowCount);
|
||||
throw new InvalidOperationException("start_time must be less than or equal to end_time. (InvalidArgument)");
|
||||
}
|
||||
|
||||
static DateTime Ts(DataValueSnapshot s) => s.SourceTimestampUtc ?? DateTime.MinValue;
|
||||
var prior = series.LastOrDefault(s => Ts(s) <= startUtc);
|
||||
var rows = new List<DataValueSnapshot>();
|
||||
if (emitBoundaryRow)
|
||||
rows.Add(prior is null
|
||||
? new DataValueSnapshot(0d, StatusCodes.BadNoData, startUtc, startUtc)
|
||||
: new DataValueSnapshot(prior.Value, prior.StatusCode, startUtc, startUtc));
|
||||
rows.AddRange(series.Where(s => Ts(s) >= startUtc && Ts(s) <= endUtc));
|
||||
var page = maxValuesPerNode == 0 ? rows : rows.Take((int)maxValuesPerNode).ToList();
|
||||
int seen;
|
||||
while (page.Count > (seen = Volatile.Read(ref _maxRowsReturned))
|
||||
&& Interlocked.CompareExchange(ref _maxRowsReturned, page.Count, seen) != seen) { }
|
||||
return Task.FromResult(new HistorianRead(page, null));
|
||||
}
|
||||
|
||||
public Task<HistorianRead> ReadProcessedAsync(
|
||||
string fullReference, DateTime startUtc, DateTime endUtc, TimeSpan interval,
|
||||
HistoryAggregateType aggregate, CancellationToken cancellationToken) =>
|
||||
Task.FromResult(new HistorianRead(Array.Empty<DataValueSnapshot>(), null));
|
||||
|
||||
public Task<HistorianRead> ReadAtTimeAsync(
|
||||
string fullReference, IReadOnlyList<DateTime> timestampsUtc, CancellationToken cancellationToken) =>
|
||||
Task.FromResult(new HistorianRead(Array.Empty<DataValueSnapshot>(), null));
|
||||
|
||||
public Task<HistoricalEventsResult> ReadEventsAsync(
|
||||
string? sourceName, DateTime startUtc, DateTime endUtc, int maxEvents,
|
||||
CancellationToken cancellationToken) =>
|
||||
Task.FromResult(new HistoricalEventsResult(Array.Empty<HistoricalEvent>(), null));
|
||||
|
||||
public HistorianHealthSnapshot GetHealthSnapshot() => NullHistorianDataSource.Instance.GetHealthSnapshot();
|
||||
|
||||
public void Dispose() { }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A microsecond-resolution backend, shaped like QuestDB: every inbound window bound is TRUNCATED to 10 ticks
|
||||
/// (as <c>QuestDbConnection.Round</c> does) before the series is filtered, and the source reports
|
||||
@@ -640,6 +870,8 @@ public sealed class NodeManagerHistoryReadPagingTests : IDisposable
|
||||
|
||||
public long TimestampResolutionTicks => 10;
|
||||
|
||||
public bool SupportsReverseRaw => true; // QuestDB serves reverse natively
|
||||
|
||||
private static DateTime Truncate(DateTime t) => new(t.Ticks - t.Ticks % 10, DateTimeKind.Utc);
|
||||
|
||||
public Task<HistorianRead> ReadRawAsync(
|
||||
|
||||
Reference in New Issue
Block a user