feat(historian): ServerHistorian:MaxRawValuesPerRead caps raw pages and backend reads

Review R08:1.1: an uncapped HistoryRead-Raw (NumValuesPerNode = 0) read a
tag's whole window into memory on every backend (SQL TOP(int.MaxValue),
QuestDB LIMIT, Gateway max_values = int.MaxValue + a second snapshot copy).

New ServerHistorian:MaxRawValuesPerRead (default 10 000; <= 0 unbounded with
a Validate() warning; 1 raised to 2). Each backend data source takes it,
reports it as IHistorianDataSource.MaxRawValuesPerRead (the Gateway: min
with a strict gateway's learned ceiling), so the node manager's existing
RawPageLimit paging serves N = 0 and N > cap as pages of at most the cap
with a continuation point. Independently, a backend raw read asking for 0
or more than the cap reads cap + 1 rows and is REFUSED
(HistoryReadLimitExceededException => BadTooManyOperations) when the window
holds more - never answered truncated, since the server's paging reads a
short answer as the last page. Bound through the Host registry, the
SqlServer/QuestDb providers and GatewayHistorian.CreateDataSource.

Tests: options (default/binding/warnings), provider binding, Gateway unit +
through a real SDK node manager (uncapped -> 999/998/998/605 + CPs), and a
conformance case for the SQL backends.

Refs #736

Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT
This commit is contained in:
Joseph Doherty
2026-10-22 06:29:10 -04:00
parent ad2b5c4044
commit 6669fae7e5
17 changed files with 621 additions and 33 deletions
@@ -67,6 +67,12 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
/// <summary>The tie-cluster limit the crowding re-reads honour — exposed so the factory wiring is testable.</summary>
internal int MaxPreWindowRows => _maxPreWindowRows;
/// <summary>
/// The server raw cap (<c>ServerHistorian:MaxRawValuesPerRead</c>, R08:1.1): the most samples one raw read may
/// return; 0 = none. See <see cref="MaxRawValuesPerRead"/> and <see cref="ReadRawWindowAsync"/>.
/// </summary>
private readonly int _maxRawValuesPerRead;
/// <summary>A strict gateway's row ceiling, learned from its first refusal.</summary>
private readonly GatewayRowCeiling _ceiling;
private readonly GatewayTagPresenceCache _tags;
@@ -96,11 +102,19 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
/// The most rows before a raw window a crowding re-read makes room for (<c>ServerHistorian:MaxTieClusterOverfetch</c>,
/// values below 1 count as 1). More ⇒ <see cref="HistoryReadLimitExceededException"/> (<c>BadTooManyOperations</c>).
/// </param>
/// <param name="maxRawValuesPerRead">
/// The server raw cap (<c>ServerHistorian:MaxRawValuesPerRead</c>, R08:1.1): the most samples one
/// <see cref="ReadRawAsync"/> may return — reported as <see cref="MaxRawValuesPerRead"/> (bounded further by a strict
/// gateway's learned ceiling), so the server pages at it. A read asking for more (or for everything) asks the gateway
/// for cap + 1 rows and is refused when the window holds more, never truncated. &lt;= 0 = unbounded (the pre-#736
/// behaviour: an uncapped read is sent as <see cref="int.MaxValue"/>).
/// </param>
public GatewayHistorianDataSource(IHistorianGatewayClient client, ILogger<GatewayHistorianDataSource> logger,
int maxRawRows = TimeWeightedAggregator.DefaultMaxWindowRows, TimeProvider? timeProvider = null,
bool atTimeFromRaw = true, int maxPreWindowRows = DefaultMaxPreWindowRows)
bool atTimeFromRaw = true, int maxPreWindowRows = DefaultMaxPreWindowRows, int maxRawValuesPerRead = 0)
{
_maxPreWindowRows = Math.Max(1, maxPreWindowRows);
_maxRawValuesPerRead = Math.Max(0, maxRawValuesPerRead);
_atTimeFromRaw = atTimeFromRaw;
ArgumentNullException.ThrowIfNull(client);
ArgumentNullException.ThrowIfNull(logger);
@@ -203,6 +217,13 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
/// a cancelled gateway stream evicts its pooled historian session). <c>ReturnBounds</c> is not plumbed to the
/// data source on any backend, so bounding values are never synthesised here.
/// </para>
/// <para>
/// <b>The server raw cap</b> (R08:1.1, #736). With a cap, a read asking for everything (<c>0</c>) or for more than
/// the cap runs as a capped read of cap + 1 — one more than it may return — so an over-cap window is DETECTED and
/// refused (<see cref="HistoryReadLimitExceededException"/>) rather than answered truncated (the server's paging
/// reads a short answer as the last page) and never sent as <see cref="int.MaxValue"/>. The server itself never asks
/// for more than the cap (it pages at <see cref="MaxRawValuesPerRead"/>); this bounds a caller that bypasses it.
/// </para>
/// </remarks>
private async Task<List<HistorianSample>> ReadRawWindowAsync(
string fullReference, DateTime startUtc, DateTime endUtc, uint maxValuesPerNode,
@@ -210,6 +231,9 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
{
// The gateway seam caps with an int; OPC UA hands us a uint, so clamp to int range.
var cap = (int)Math.Min(maxValuesPerNode, int.MaxValue);
var serverCap = _maxRawValuesPerRead;
var probe = serverCap > 0 && (cap == 0 || cap > serverCap);
if (probe) cap = (int)Math.Min(int.MaxValue, (long)serverCap + 1);
if (cap == 0)
{
var all = await PassAsync(int.MaxValue).ConfigureAwait(false);
@@ -225,7 +249,7 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
var asked = (int)Math.Min(int.MaxValue, (long)cap + RawAggregateReader.PreWindowAllowance);
var pass = await PassAsync(asked).ConfigureAwait(false);
if (pass.Result.Complete) return pass.Result.Samples;
if (pass.Result.Complete) return Bounded(pass.Result.Samples);
// A strict gateway's ceiling already bounded that read: a larger budget would only be refused again. The
// refusal carries the pre-window rows it saw: the server re-reads the page that many values smaller.
@@ -256,7 +280,7 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
}
if (budget <= pass.Budget) break; // no more room, never retry the same full read
pass = await PassAsync(budget).ConfigureAwait(false);
if (pass.Result.Complete) return pass.Result.Samples;
if (pass.Result.Complete) return Bounded(pass.Result.Samples);
preWindow = Math.Max(preWindow, pass.Result.PreWindowRows);
if (pass.Budget < budget) break; // bounded by a ceiling learned meanwhile
if (room >= _maxPreWindowRows) break; // the limit's worth of room did not suffice
@@ -271,6 +295,16 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
// Pre-window rows the allowance did not cover are what crowded the read; fewer are not the cause.
static int Crowding(int preWindowRows) => preWindowRows > RawAggregateReader.PreWindowAllowance ? preWindowRows : 0;
// The cap + 1 probe came back with one sample too many: the window holds more than the server raw cap.
List<HistorianSample> Bounded(List<HistorianSample> samples)
{
if (probe && samples.Count > serverCap)
throw new HistoryReadLimitExceededException(serverCap,
$"The raw read of '{fullReference}' holds more than ServerHistorian:MaxRawValuesPerRead ({serverCap}) " +
"samples; set NumValuesPerNode at or below it to page, or narrow the window.");
return samples;
}
// One drained pass; a strict gateway's refusal of the budget is retried once at the ceiling it names.
Task<((List<HistorianSample> Samples, int PreWindowRows, bool Complete) Result, int Budget)> PassAsync(int budget) =>
_ceiling.ReadAsync(budget,
@@ -531,13 +565,20 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
public bool SupportsReverseRaw => false;
/// <inheritdoc />
// 0 until a strict gateway (AllowStreamingBeyondCeiling=false) has refused a budget and
// named its ceiling (GatewayRowCeiling); then that ceiling less the
// RawAggregateReader.PreWindowAllowance a capped read adds for AVEVA's boundary row — so a page of this
// size asks for exactly the ceiling. Back to 0 when a re-probe finds the gateway streaming again.
public int MaxRawValuesPerRead => _ceiling.Learned is var c and > 0
? Math.Max(1, c - RawAggregateReader.PreWindowAllowance)
: 0;
// The server raw cap (ServerHistorian:MaxRawValuesPerRead; 0 = none), bounded further by a strict gateway's
// (AllowStreamingBeyondCeiling=false) learned ceiling once a refusal has named it (GatewayRowCeiling) — that
// ceiling less the RawAggregateReader.PreWindowAllowance a capped read adds for AVEVA's boundary row, so a page of
// this size asks for exactly the ceiling. The learned part is forgotten when a re-probe finds the gateway streaming
// again; the cap stays. 0 only with no cap and no learned ceiling.
public int MaxRawValuesPerRead
{
get
{
var learned = _ceiling.Learned is var c and > 0 ? Math.Max(1, c - RawAggregateReader.PreWindowAllowance) : 0;
var cap = _maxRawValuesPerRead;
return cap > 0 && learned > 0 ? Math.Min(cap, learned) : cap > 0 ? cap : learned;
}
}
/// <inheritdoc />
// false: the gateway's ReadEvents streams ascending only. The server never sends it a