fix(historian): #686 review — bounded, density-aware reverse raw walk; uncapped Gateway raw reads

1. Uncapped reverse raw read on a forward-only backend = ONE forward read of the
   window, filtered and reversed in memory (was a segment-by-segment walk).
2. Segment width is aimed at ~half the row budget from the density just observed
   (overflow: from the truncated read's own sample span) and carried page to page
   in HistoryContinuationState.SegmentWidthTicks.
3. The walk starts no later than now + 1 day and never reads before the epoch.
4. Request bounds are rounded outward to the millisecond, so a row AVEVA
   fabricates at a rounded-up query start cannot land inside the segment.
5. Budget M = max(1, MaxTieClusterOverfetch); reads ask M + 2 rows, completeness is
   'short OR last row past the segment', and a read crowded by rows before the
   segment is re-read once (<= 2M + 4). A cluster of exactly M, neighbouring
   clusters each within M, and M <= 1 are all served; only one timestamp over M
   is refused. ServerHistorianOptions text updated.
6. The raw and events tie-path oversized-cluster refusal is BadTooManyOperations
   (was BadHistoryOperationUnsupported), consistent with the walk.
7. GatewayHistorianDataSource.ReadRawAsync: a caller cancellation is not a backend
   failure; NumValuesPerNode = 0 is sent as int.MaxValue (the client package and
   the gateway reject max_values <= 0, so every uncapped Gateway raw read failed);
   a MaxValuesCeiling refusal is a HistoryReadLimitExceededException.
8. Tests: upper-edge filter, epoch floor, look-ahead, ms alignment, exact/adjacent
   clusters, minimal budget, carried density, and the walk over the real Gateway
   source + FakeHistorianGatewayClient (OpcUaServer internals visible to it).
This commit is contained in:
Joseph Doherty
2026-10-12 15:11:21 -04:00
parent be2f0cc14d
commit cf1af55720
9 changed files with 554 additions and 109 deletions
@@ -90,8 +90,9 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
try
{
// The gateway seam caps with an int; OPC UA hands us a uint, so clamp to int range.
var maxValues = (int)Math.Min(maxValuesPerNode, int.MaxValue);
// The gateway seam caps with an int; OPC UA hands us a uint, so clamp to int range. 0 ("all values") is sent
// unbounded: the client package and the gateway reject max_values <= 0 (#686 review).
var maxValues = maxValuesPerNode == 0 ? int.MaxValue : (int)Math.Min(maxValuesPerNode, int.MaxValue);
var samples = new List<HistorianSample>();
await foreach (var sample in _client
.ReadRawAsync(fullReference, startUtc, endUtc, maxValues, cancellationToken)
@@ -104,6 +105,18 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
RecordOutcome(success: true, error: null);
return new HistoryReadResult(snapshots, ContinuationPoint: null);
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
throw; // the caller (client cancel / HistoryReadDeadline) asked for this — not a backend fault
}
catch (Exception ex) when (IsMaxValuesCeilingRejection(ex))
{
// An unbounded read over Historian:ReadLimits:MaxValuesCeiling with AllowStreamingBeyondCeiling=false: a
// client-request limit (BadTooManyOperations), not a backend fault.
throw new HistoryReadLimitExceededException(maxValuesPerNode,
$"The HistorianGateway refused the row budget of the raw read of '{fullReference}' (its " +
"Historian:ReadLimits:MaxValuesCeiling); set NumValuesPerNode or narrow the window.");
}
catch (Exception ex)
{
RecordReadFailure(ex);
@@ -51,6 +51,10 @@ namespace ZB.MOM.WW.OtOpcUa.OpcUaServer;
/// <c>(EndUtc, NextStartUtc]</c>. A resumed events read takes its direction from here, never from the resume
/// request's details. Raw keeps deriving it from the window (<c>NextStartUtc &gt; EndUtc</c>).
/// </param>
/// <param name="SegmentWidthTicks">
/// <see cref="HistoryCursorKind.Raw"/> reverse reads on a forward-only backend only (Gitea #686): the segment width the
/// walk-back ended on, so the next page starts at the observed density instead of re-discovering it. 0 = none.
/// </param>
internal sealed record HistoryContinuationState(
string Tagname,
DateTime NextStartUtc,
@@ -60,7 +64,8 @@ internal sealed record HistoryContinuationState(
HistoryCursorKind Kind = HistoryCursorKind.Raw,
long LastModifiedId = 0,
DateTime WindowStartUtc = default,
bool Reverse = false);
bool Reverse = false,
long SegmentWidthTicks = 0);
/// <summary>The HistoryRead kind that issued a <see cref="HistoryContinuationState"/>.</summary>
internal enum HistoryCursorKind
@@ -536,8 +536,9 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2
/// cap. When a resume read stalls on such a cluster (the boundary-tie trim empties the page), the
/// paging over-fetches up to <c>MaxTieClusterOverfetch + 1</c> ties at that single timestamp (a
/// <c>start == end</c> read) and slices through them via <see cref="HistoryPaging.SliceTieCluster"/>.
/// A cluster strictly larger than this still surfaces <c>BadHistoryOperationUnsupported</c> for that
/// node (the absurd-burst backstop). Mirrors the configured
/// A cluster strictly larger than this still fails that node <c>BadTooManyOperations</c> (the absurd-burst
/// backstop; <c>BadHistoryOperationUnsupported</c> before #686). Also the per-read row budget of a reverse raw read
/// walked back from forward reads (<see cref="ReverseRawReader"/>). Mirrors the configured
/// <c>ServerHistorianOptions.MaxTieClusterOverfetch</c>; the Host sets it at <c>StartAsync</c>. The
/// default (65536) survives when the historian section is absent.
/// </summary>
@@ -5359,8 +5360,8 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2
"OtOpcUaNodeManager: HistoryReadEvents tie cluster at {StartUtc:O} for source '{Source}' has {Count} events, " +
"exceeding MaxTieClusterOverfetch={Max}; cannot page within it. Increase MaxTieClusterOverfetch.",
startUtc, sourceName, cluster.Count, MaxTieClusterOverfetch);
errors[handle.Index] = StatusCodes.BadHistoryOperationUnsupported;
results[handle.Index] = new SdkHistoryReadResult { StatusCode = StatusCodes.BadHistoryOperationUnsupported };
errors[handle.Index] = StatusCodes.BadTooManyOperations;
results[handle.Index] = new SdkHistoryReadResult { StatusCode = StatusCodes.BadTooManyOperations };
return;
}
@@ -6183,6 +6184,7 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2
{
DateTime startUtc;
var boundarySkip = 0;
long segmentWidth = 0; // #686: the walk-back's width, carried page to page
string tagname;
if (inboundCp is { Length: > 0 })
@@ -6203,6 +6205,7 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2
boundarySkip = state.BoundarySkipCount;
endUtc = state.EndUtc;
numValuesPerNode = state.NumValuesPerNode;
segmentWidth = state.SegmentWidthTicks;
}
else
{
@@ -6235,16 +6238,22 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2
// can read newest-first; otherwise it is walked back from bounded forward reads (#686 — the Gateway rejects
// start > end, which failed the node and counted as a backend fault).
var source = HistorianDataSource;
var sourceResult = startUtc > endUtc && !source.SupportsReverseRaw
? new HistorianRead(
await ReverseRawReader.ReadAsync(
source, tagname, startUtc, endUtc, numValuesPerNode,
(int)Math.Min(int.MaxValue, (long)Math.Max(1, MaxTieClusterOverfetch) + 1), ct)
.ConfigureAwait(false),
ContinuationPoint: null)
: await source
HistorianRead sourceResult;
if (startUtc > endUtc && !source.SupportsReverseRaw)
{
var page = await ReverseRawReader.ReadAsync(
source, tagname, startUtc, endUtc, numValuesPerNode, MaxTieClusterOverfetch,
Clock.GetUtcNow().UtcDateTime, segmentWidth, ct)
.ConfigureAwait(false);
segmentWidth = page.WidthHintTicks;
sourceResult = new HistorianRead(page.Samples, ContinuationPoint: null);
}
else
{
sourceResult = await source
.ReadRawAsync(tagname, startUtc, endUtc, numValuesPerNode, ct)
.ConfigureAwait(false);
}
var backendFull = HistoryPaging.IsFullPage(sourceResult.Samples.Count, numValuesPerNode);
@@ -6293,8 +6302,8 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2
"OtOpcUaNodeManager: HistoryReadRaw tie cluster at {StartUtc:O} for tag '{Tagname}' has {Count} samples, " +
"exceeding MaxTieClusterOverfetch={Max}; cannot page within it. Increase MaxTieClusterOverfetch.",
startUtc, tagname, cluster.Count, MaxTieClusterOverfetch);
errors[handle.Index] = StatusCodes.BadHistoryOperationUnsupported;
results[handle.Index] = new SdkHistoryReadResult { StatusCode = StatusCodes.BadHistoryOperationUnsupported };
errors[handle.Index] = StatusCodes.BadTooManyOperations;
results[handle.Index] = new SdkHistoryReadResult { StatusCode = StatusCodes.BadTooManyOperations };
return;
}
@@ -6315,7 +6324,7 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2
if (nextStartUtc is { } resumeAt)
{
var clusterState = new HistoryContinuationState(
tagname, resumeAt, endUtc, nextSkip, numValuesPerNode);
tagname, resumeAt, endUtc, nextSkip, numValuesPerNode, SegmentWidthTicks: segmentWidth);
clusterCp = _historyContinuationStore.Save(session, clusterState);
}
@@ -6338,7 +6347,7 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2
{
HistoryPaging.ComputeResumeCursor(samples, out var nextStart, out var skip);
var nextState = new HistoryContinuationState(
tagname, nextStart, endUtc, skip, numValuesPerNode);
tagname, nextStart, endUtc, skip, numValuesPerNode, SegmentWidthTicks: segmentWidth);
// Save may return null (no session on this request) ⇒ degrade to single-shot for this node.
// Built AFTER historyData so a failure projecting samples can never orphan a stored cursor.
outboundCp = _historyContinuationStore.Save(session, nextState);
@@ -2,6 +2,11 @@ using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
namespace ZB.MOM.WW.OtOpcUa.OpcUaServer;
/// <summary>One walked-back reverse raw page (<see cref="ReverseRawReader"/>).</summary>
/// <param name="Samples">At most the requested cap, newest first.</param>
/// <param name="WidthHintTicks">The segment width the walk ended on — the next page's starting width (0 = none).</param>
internal readonly record struct ReverseRawPage(IReadOnlyList<DataValueSnapshot> Samples, long WidthHintTicks);
/// <summary>
/// Gitea #686: serves a Part 11 reverse raw read — <c>(lo, hi]</c> newest-first, capped — on a historian backend that can
/// only read forward (<see cref="IHistorianDataSource.SupportsReverseRaw"/> = <c>false</c>: the Gateway, whose
@@ -9,112 +14,191 @@ namespace ZB.MOM.WW.OtOpcUa.OpcUaServer;
/// </summary>
/// <remarks>
/// <para>
/// <b>Walk back in segments.</b> Starting at <c>hi</c>, read one segment <c>(segLo, segHi]</c> FORWARD, keep it
/// newest-first, and step to the next older segment until the cap is met or <c>lo</c> is reached. A segment starts
/// at <see cref="InitialSegment"/> and grows ×4 while segments come back sparse, so "the latest N" of a live tag is
/// usually one read, and sparse history decades back is found in a few dozen tiny reads rather than one read of
/// all history.
/// <b>Uncapped</b> (<c>maxValues = 0</c>): ONE forward read of the whole window, filtered and reversed in memory — the
/// same rows, round trips and memory as an uncapped forward read (and as a native backend's uncapped reverse read).
/// </para>
/// <para>
/// <b>Never answer from a truncated read.</b> Each segment read asks for at most <c>segmentCap</c> rows (the server's
/// <c>MaxTieClusterOverfetch + 1</c> over-fetch bound). A read that comes back FULL may have been cut short — its
/// newest rows are exactly the ones a reverse read needs — so it is discarded and the segment re-read at a quarter of
/// the width. A full read of a segment already one timestamp-resolution wide (a single-timestamp tie cluster larger
/// than the bound) cannot be served and throws <see cref="HistoryReadLimitExceededException"/>
/// <b>Capped: walk back in segments.</b> Starting at <c>hi</c>, read one segment <c>(segLo, segHi]</c> FORWARD, keep
/// it newest-first, and step to the next older segment until the cap is met or <c>lo</c> is reached. The first
/// segment is <see cref="InitialSegment"/> wide, or the width the previous page of the same read ended on (carried in
/// the continuation point). After each read the next width is re-aimed at about half the row budget from the density
/// just observed, growing at most ×4 per step — so a live tag's "latest N" is usually one read, a dense tag does not
/// throw away a full read per page, and sparse history is found in a few dozen tiny reads.
/// </para>
/// <para>
/// <b>Never answer from a truncated read.</b> The row budget is <c>M = max(1, MaxTieClusterOverfetch)</c>; a segment
/// read asks for <c>M + 2</c> rows (room for the row an AVEVA read fabricates at its query start, and one more to see
/// past the segment). Rows stream ascending, so a read is complete for its segment when it came back short OR its last
/// row is already past <c>segHi</c>. An incomplete read is discarded: when rows BEFORE the segment (a neighbouring
/// tie cluster inside the alignment slack, the fabricated row) crowded it out it is re-read once with room for them
/// (at most <c>2M + 4</c> rows); otherwise the segment is narrowed (to about half the budget at the density the
/// truncated read showed) and re-read. Only a single timestamp holding more than <c>M</c> samples — a segment already
/// one timestamp-resolution wide that still does not fit — is refused: <see cref="HistoryReadLimitExceededException"/>
/// (⇒ <c>BadTooManyOperations</c>, a request limit, never a backend fault).
/// </para>
/// <para>
/// <b>Edges, for any forward contract.</b> A segment is requested as <c>[segLo, segHi + resolution]</c> and filtered to
/// <c>(segLo, segHi]</c> on the samples' own timestamps, so it is exact whether the backend's forward end is exclusive
/// (the interface contract) or inclusive (the gateway), and whether it truncates its bounds (QuestDB-style). The
/// filter also drops the rows an AVEVA <c>Full</c> read fabricates at the query start (the boundary / no-data row,
/// #687) and at the query end. Adjacent segments share no timestamp, so nothing is returned twice.
/// <b>Edges.</b> A segment is requested as <c>[⌊segLo⌋, ⌈segHi + 1 tick⌉]</c>, both rounded outward to the millisecond
/// (the gateway's / AVEVA's resolution), and filtered to <c>(segLo, segHi]</c> on the samples' own timestamps. That is
/// exact whether the backend's forward end is exclusive (the interface contract) or inclusive (the gateway) and
/// whether it truncates its bounds (QuestDB-style); and because the request start is already on a millisecond, a row
/// an AVEVA read fabricates at its (possibly rounded-up) query start can never land inside the segment (#687).
/// Adjacent segments share no timestamp, so nothing is returned twice.
/// </para>
/// <para>
/// <b>Ties</b> come back in the backend's forward order reversed — the same order the node manager's tie-cluster
/// over-fetch uses for a reverse read — so a resumed page's boundary-tie skip lines up.
/// </para>
/// <para>
/// <b>Floor.</b> The walk stops at <see cref="Floor"/> (the Unix epoch; its earliest request bound is one tick
/// before it): the AVEVA historian holds nothing earlier, and an end-only read's open lower bound (year 1) would
/// otherwise cost dozens of empty reads and send a bound outside the historian's time encoding. Samples before the
/// floor are not returned by a walked-back read; a window entirely before it is empty.
/// <b>Bounds.</b> The walk never asks for data after now + <see cref="LookAhead"/> (one day — the historian stores
/// nothing further ahead; a <c>StartTime = DateTime.MaxValue</c> read would otherwise send centuries-ahead query
/// starts), nor before <see cref="Floor"/> (the Unix epoch — AVEVA holds nothing earlier, and an end-only read's year-1
/// lower bound is outside the historian's time encoding). A walked-back read therefore returns only samples strictly
/// after the epoch and no later than now + one day.
/// </para>
/// </remarks>
internal static class ReverseRawReader
{
/// <summary>The first segment's width.</summary>
/// <summary>The first segment's width when no hint is carried.</summary>
internal static readonly TimeSpan InitialSegment = TimeSpan.FromMinutes(1);
/// <summary>The earliest instant a walk reads (see the remarks).</summary>
/// <summary>The walk's exclusive lower limit (see the remarks).</summary>
internal static readonly DateTime Floor = DateTime.UnixEpoch;
/// <summary>How far past "now" the walk starts at most (the same horizon as the Gateway's raw-aggregate look-ahead).</summary>
internal static readonly TimeSpan LookAhead = TimeSpan.FromDays(1);
/// <summary>Request bounds are rounded outward to this (the gateway / AVEVA timestamp resolution).</summary>
private const long Alignment = TimeSpan.TicksPerMillisecond;
/// <summary>Reads <c>(lo, hi]</c> newest-first from forward reads.</summary>
/// <param name="source">A backend that reads forward.</param>
/// <param name="tagname">Historian tag.</param>
/// <param name="hi">The later (inclusive) bound — the request's StartTime or a reverse resume point.</param>
/// <param name="lo">The earlier (exclusive) bound.</param>
/// <param name="maxValues">Cap; <c>0</c> = every sample in the window.</param>
/// <param name="segmentCap">Rows one segment read may return before it is treated as truncated (≥ 2).</param>
/// <param name="maxValues">Cap; <c>0</c> = every sample in the window (one forward read).</param>
/// <param name="maxTieCluster">Row budget per segment read (<c>MaxTieClusterOverfetch</c>; values below 1 count as 1).</param>
/// <param name="nowUtc">Now, for the look-ahead clamp.</param>
/// <param name="widthHintTicks">Starting segment width (the previous page's <see cref="ReverseRawPage.WidthHintTicks"/>), 0 = none.</param>
/// <param name="ct">Cancellation.</param>
/// <returns>At most <paramref name="maxValues"/> samples, newest first.</returns>
/// <exception cref="HistoryReadLimitExceededException">A single-timestamp tie cluster exceeds the segment cap.</exception>
public static async Task<IReadOnlyList<DataValueSnapshot>> ReadAsync(
IHistorianDataSource source, string tagname, DateTime hi, DateTime lo, uint maxValues, int segmentCap,
CancellationToken ct)
/// <returns>The page, newest first, and the width to start the next page with.</returns>
/// <exception cref="HistoryReadLimitExceededException">One timestamp holds more than the row budget.</exception>
public static async Task<ReverseRawPage> ReadAsync(
IHistorianDataSource source, string tagname, DateTime hi, DateTime lo, uint maxValues, int maxTieCluster,
DateTime nowUtc, long widthHintTicks, CancellationToken ct)
{
var resolution = Math.Max(1L, source.TimestampResolutionTicks);
var cap = (uint)Math.Max(2, segmentCap);
var granularity = Math.Max(Alignment, resolution);
var budget = Math.Max(1, maxTieCluster);
var result = new List<DataValueSnapshot>();
var width = Math.Max(resolution, InitialSegment.Ticks);
var segHiTicks = hi.Ticks;
// The walk's exclusive lower edge. Clamped to one tick before the floor, so a sample exactly at the floor is still
// in the window (and the row a backend fabricates at that query start is still filtered out).
var stopTicks = lo.Ticks < Floor.Ticks ? Floor.Ticks - 1 : lo.Ticks;
var horizon = nowUtc.Ticks > DateTime.MaxValue.Ticks - LookAhead.Ticks ? DateTime.MaxValue.Ticks : nowUtc.Ticks + LookAhead.Ticks;
var hiTicks = Math.Min(hi.Ticks, horizon);
var stopTicks = Math.Max(lo.Ticks, Floor.Ticks);
if (hiTicks <= stopTicks) return new ReverseRawPage(result, widthHintTicks);
while (segHiTicks > stopTicks && (maxValues == 0 || result.Count < maxValues))
if (maxValues == 0)
{
ct.ThrowIfCancellationRequested();
var segLoTicks = segHiTicks - stopTicks <= width ? stopTicks : segHiTicks - width;
var requestEndTicks = segHiTicks > DateTime.MaxValue.Ticks - resolution
? DateTime.MaxValue.Ticks
: segHiTicks + resolution;
var rows = (await source.ReadRawAsync(
tagname,
new DateTime(segLoTicks, DateTimeKind.Utc),
new DateTime(requestEndTicks, DateTimeKind.Utc),
cap, ct)
.ConfigureAwait(false)).Samples;
if (rows.Count >= cap)
// One forward read of the whole window, reversed — the uncapped forward read's own cost and bound.
var all = (await source.ReadRawAsync(tagname, Utc(AlignDown(stopTicks, granularity)),
Utc(AlignUp(hiTicks + 1, granularity)), 0, ct).ConfigureAwait(false)).Samples;
for (var i = all.Count - 1; i >= 0; i--)
{
// Possibly truncated: its newest rows — the ones needed — may be missing. Narrow and re-read.
if (width <= resolution)
throw new HistoryReadLimitExceededException(cap - 1,
$"A reverse raw read of '{tagname}' at {new DateTime(segHiTicks, DateTimeKind.Utc):O} found more than " +
$"{cap - 1} samples at one timestamp; the backend reads forward only, so the latest of them cannot be " +
"selected. Increase ServerHistorian:MaxTieClusterOverfetch.");
width = Math.Max(resolution, width / 4);
continue;
var ts = Ticks(all[i]);
if (ts > stopTicks && ts <= hiTicks) result.Add(all[i]);
}
var inSegment = 0;
for (var i = rows.Count - 1; i >= 0; i--)
{
var ts = (rows[i].SourceTimestampUtc ?? DateTime.MinValue).Ticks;
if (ts <= segLoTicks || ts > segHiTicks) continue;
inSegment++;
if (maxValues == 0 || result.Count < maxValues) result.Add(rows[i]);
}
// Sparse ⇒ widen; otherwise keep the width that fit.
if (inSegment <= cap / 8)
width = width > DateTime.MaxValue.Ticks / 4 ? DateTime.MaxValue.Ticks : width * 4;
segHiTicks = segLoTicks;
return new ReverseRawPage(result, widthHintTicks);
}
return result;
var maxWidth = DateTime.MaxValue.Ticks;
var width = widthHintTicks > 0 ? Math.Clamp(widthHintTicks, resolution, maxWidth) : Math.Max(resolution, InitialSegment.Ticks);
var baseCap = budget + 2;
var maxCap = 2 * budget + 4;
var segHi = hiTicks;
while (segHi > stopTicks && result.Count < maxValues)
{
var segLo = segHi - stopTicks <= width ? stopTicks : segHi - width;
var start = Utc(AlignDown(segLo, granularity));
var end = Utc(AlignUp(segHi + 1, granularity));
var cap = baseCap;
IReadOnlyList<DataValueSnapshot> rows;
var narrowed = false;
while (true)
{
ct.ThrowIfCancellationRequested();
rows = (await source.ReadRawAsync(tagname, start, end, (uint)cap, ct).ConfigureAwait(false)).Samples;
if (rows.Count < cap || Ticks(rows[^1]) > segHi) break; // complete for (segLo, segHi]
var leading = 0;
var inner = 0;
var firstInner = 0L;
foreach (var r in rows)
{
var ts = Ticks(r);
if (ts <= segLo) leading++;
else if (ts <= segHi && inner++ == 0) firstInner = ts;
}
// Crowded out by rows before the segment, with the segment itself possibly within budget: re-read once
// with room for them.
var roomier = leading + budget + 2;
if (leading > 0 && inner <= budget && roomier > cap && roomier <= maxCap)
{
cap = roomier;
continue;
}
if (width <= resolution)
throw new HistoryReadLimitExceededException(budget,
$"A reverse raw read of '{tagname}' at {Utc(segHi):O} found more than {budget} samples at one " +
"timestamp; the backend reads forward only, so the latest of them cannot be selected. Increase " +
"ServerHistorian:MaxTieClusterOverfetch.");
// Narrow to about half the budget at the density the truncated read's own samples showed (their span, not
// the segment's — the segment may open on an empty stretch), at least halving.
var lastTs = Ticks(rows[^1]);
var aimed = inner > 1 && lastTs > firstInner
? (long)Math.Min((double)(lastTs - firstInner) * (budget / 2.0) / (inner - 1), width / 2.0)
: width / 4;
width = Math.Max(resolution, aimed);
narrowed = true;
break;
}
if (narrowed) continue; // re-read the same segHi, narrower
var count = 0;
for (var i = rows.Count - 1; i >= 0; i--)
{
var ts = Ticks(rows[i]);
if (ts <= segLo || ts > segHi) continue;
count++;
if (result.Count < maxValues) result.Add(rows[i]);
}
// Re-aim at about half the budget from the density just seen; grow at most x4.
var grown = width > maxWidth / 4 ? maxWidth : width * 4;
width = count == 0
? grown
: (long)Math.Clamp((double)width * Math.Max(1.0, budget / 2.0) / count, resolution, grown);
segHi = segLo;
}
return new ReverseRawPage(result, width);
}
private static long Ticks(DataValueSnapshot s) => (s.SourceTimestampUtc ?? DateTime.MinValue).Ticks;
private static DateTime Utc(long ticks) => new(ticks, DateTimeKind.Utc);
private static long AlignDown(long ticks, long step) => ticks - ticks % step;
private static long AlignUp(long ticks, long step)
{
var down = AlignDown(ticks, step);
if (down == ticks) return ticks;
return down > DateTime.MaxValue.Ticks - step ? DateTime.MaxValue.Ticks : down + step;
}
}
@@ -35,6 +35,8 @@
<!-- Gitea #562: the over-the-wire event-revalidation test asserts the server-side fan-out, because a
deleted item resurrected onto a notifier is invisible to the client that deleted it. -->
<InternalsVisibleTo Include="ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests"/>
<!-- Gitea #686: the reverse-raw walk-back is driven over the real Gateway source + its fake gateway client. -->
<InternalsVisibleTo Include="ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests"/>
</ItemGroup>
</Project>
@@ -81,8 +81,11 @@ public sealed class ServerHistorianOptions
/// cap). When a resume read stalls on such a cluster, the node manager over-fetches up to
/// <c>MaxTieClusterOverfetch + 1</c> ties at that single timestamp (a <c>start == end</c> read) and
/// pages through them; this bounds how large a single-timestamp burst the server will buffer in
/// memory before it gives up and surfaces <c>BadHistoryOperationUnsupported</c> for that node. The
/// default (65536) comfortably covers any realistic same-millisecond burst.
/// memory before it gives up and surfaces <c>BadTooManyOperations</c> for that node (#686; was
/// <c>BadHistoryOperationUnsupported</c>). It is also the per-read row budget of a reverse raw read walked
/// back from forward reads on a backend that cannot read newest-first (the Gateway): one timestamp holding
/// more samples than this is refused <c>BadTooManyOperations</c>. The default (65536) comfortably covers
/// any realistic same-millisecond burst; a value below 1 is treated as 1.
/// </summary>
public int MaxTieClusterOverfetch { get; init; } = 65536;
@@ -167,7 +170,7 @@ public sealed class ServerHistorianOptions
// the over-fetch code path only runs when a real IHistorianDataSource is wired in,
// so a zero/negative value is harmless (and noise-free) when the historian is disabled.
if (MaxTieClusterOverfetch <= 0)
warnings.Add($"ServerHistorian:MaxTieClusterOverfetch is {MaxTieClusterOverfetch} — must be > 0; HistoryRead-Raw cannot page within an oversized tie cluster and will surface BadHistoryOperationUnsupported for those reads.");
warnings.Add($"ServerHistorian:MaxTieClusterOverfetch is {MaxTieClusterOverfetch} — must be > 0 (treated as 1); HistoryRead-Raw and -Events cannot page within a tie cluster larger than that, and a Gateway reverse raw read cannot serve one timestamp holding more samples — those reads surface BadTooManyOperations.");
// S3 HistoryRead concurrency knobs — a non-positive value disables the corresponding bound, so warn
// (these gate the per-node fan-out / batch limiter that only run when a real data source is wired).
if (HistoryReadBatchParallelism <= 0)