feat(historian): ServerHistorian:MaxRawValuesPerRead caps raw pages and backend reads

Review R08:1.1: an uncapped HistoryRead-Raw (NumValuesPerNode = 0) read a
tag's whole window into memory on every backend (SQL TOP(int.MaxValue),
QuestDB LIMIT, Gateway max_values = int.MaxValue + a second snapshot copy).

New ServerHistorian:MaxRawValuesPerRead (default 10 000; <= 0 unbounded with
a Validate() warning; 1 raised to 2). Each backend data source takes it,
reports it as IHistorianDataSource.MaxRawValuesPerRead (the Gateway: min
with a strict gateway's learned ceiling), so the node manager's existing
RawPageLimit paging serves N = 0 and N > cap as pages of at most the cap
with a continuation point. Independently, a backend raw read asking for 0
or more than the cap reads cap + 1 rows and is REFUSED
(HistoryReadLimitExceededException => BadTooManyOperations) when the window
holds more - never answered truncated, since the server's paging reads a
short answer as the last page. Bound through the Host registry, the
SqlServer/QuestDb providers and GatewayHistorian.CreateDataSource.

Tests: options (default/binding/warnings), provider binding, Gateway unit +
through a real SDK node manager (uncapped -> 999/998/998/605 + CPs), and a
conformance case for the SQL backends.

Refs #736

Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT
This commit is contained in:
Joseph Doherty
2026-10-22 06:29:10 -04:00
parent ad2b5c4044
commit 6669fae7e5
17 changed files with 621 additions and 33 deletions
@@ -66,6 +66,7 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource
private readonly TimeProvider _time;
private readonly HistorianHealthCounters _health = new();
private readonly int _maxTimeWeightedRawRows;
private readonly int _maxRawValuesPerRead;
/// <summary>Creates the data source over the provider's shared connection, bootstrapper and tag catalog.</summary>
/// <param name="conn">Connection helper.</param>
@@ -74,11 +75,19 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource
/// <param name="log">Optional logger.</param>
/// <param name="timeProvider">Clock for "now" (AtTime beyond the last sample, server timestamps); system by default.</param>
/// <param name="maxTimeWeightedRawRows">Raw in-window row cap of one time-weighted read (&lt;= 0 = unbounded).</param>
/// <param name="maxRawValuesPerRead">
/// The server raw cap (<c>ServerHistorian:MaxRawValuesPerRead</c>, R08:1.1): the most samples one
/// <see cref="ReadRawAsync"/> may return — reported as <see cref="MaxRawValuesPerRead"/>, so the server pages at it.
/// A read asking for more (or for everything) is refused when the window holds more, never truncated.
/// &lt;= 0 = unbounded (the pre-#736 behaviour).
/// </param>
internal QuestDbHistorianDataSource(
QuestDbConnection conn, QuestDbSchemaBootstrapper bootstrap, QuestDbTagCatalog catalog, ILogger? log = null,
TimeProvider? timeProvider = null, int maxTimeWeightedRawRows = TimeWeightedAggregator.DefaultMaxWindowRows)
TimeProvider? timeProvider = null, int maxTimeWeightedRawRows = TimeWeightedAggregator.DefaultMaxWindowRows,
int maxRawValuesPerRead = 0)
{
_maxTimeWeightedRawRows = maxTimeWeightedRawRows;
_maxRawValuesPerRead = Math.Max(0, maxRawValuesPerRead);
_conn = conn ?? throw new ArgumentNullException(nameof(conn));
_bootstrap = bootstrap ?? throw new ArgumentNullException(nameof(bootstrap));
_catalog = catalog ?? throw new ArgumentNullException(nameof(catalog));
@@ -93,12 +102,21 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource
// (int.MaxValue); larger values clamp. At most top samples, ContinuationPoint always null: the node
// manager pages by re-reading from the last returned timestamp inclusive, which the (ts, tag) dedup
// key makes exact.
// The server raw cap (R08:1.1, #736): with a cap, a read asking for everything or for more than the cap reads
// cap + 1 rows — one more than it may return — so an over-cap window is DETECTED and refused
// (HistoryReadLimitExceededException ⇒ BadTooManyOperations) rather than answered truncated: the server's paging
// reads a short answer as the last page, so a silent LIMIT would lose data. The server itself never asks for more
// than the cap (it pages at MaxRawValuesPerRead); this bounds a caller that bypasses it.
/// <inheritdoc />
public async Task<HistoryReadResult> ReadRawAsync(
string fullReference, DateTime startUtc, DateTime endUtc, uint maxValuesPerNode, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(fullReference);
var top = maxValuesPerNode == 0 ? int.MaxValue : (int)Math.Min(maxValuesPerNode, int.MaxValue);
var cap = _maxRawValuesPerRead;
var probe = cap > 0 && (maxValuesPerNode == 0 || maxValuesPerNode > (uint)cap);
var top = probe
? (int)Math.Min(int.MaxValue, (long)cap + 1)
: maxValuesPerNode == 0 ? int.MaxValue : (int)Math.Min(maxValuesPerNode, int.MaxValue);
try
{
var samples = new List<DataValueSnapshot>();
@@ -129,9 +147,19 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource
}
}
if (probe && samples.Count > cap)
throw new HistoryReadLimitExceededException(cap,
$"The raw read of '{fullReference}' holds more than ServerHistorian:MaxRawValuesPerRead ({cap}) samples; " +
"set NumValuesPerNode at or below it to page, or narrow the window.");
RecordSuccess();
return new HistoryReadResult(samples, ContinuationPoint: null);
}
catch (HistoryReadLimitExceededException)
{
// A client-request limit (the server raw cap), not a backend fault: not counted as a failure.
throw;
}
catch (Exception ex)
{
RecordFailure(ex);
@@ -360,9 +388,11 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource
/// <inheritdoc />
public bool SupportsReverseRaw => true;
// 0: a capped raw read is one bounded statement - no backend row ceiling.
// The server raw cap (ServerHistorian:MaxRawValuesPerRead; 0 = none): the server pages every raw read at it, so a
// client's NumValuesPerNode = 0 is served as pages of at most this many values. QuestDB itself has no row ceiling -
// a capped raw read is one bounded statement.
/// <inheritdoc />
public int MaxRawValuesPerRead => 0;
public int MaxRawValuesPerRead => _maxRawValuesPerRead;
// Reverse events reads are served natively (ORDER BY ... DESC).
/// <inheritdoc />