Merge lane H1: HistoryRead raw cap (#736)
Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT
This commit is contained in:
+68
-21
@@ -67,6 +67,12 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
/// <summary>The tie-cluster limit the crowding re-reads honour — exposed so the factory wiring is testable.</summary>
|
||||
internal int MaxPreWindowRows => _maxPreWindowRows;
|
||||
|
||||
/// <summary>
|
||||
/// The server raw cap (<c>ServerHistorian:MaxRawValuesPerRead</c>, R08:1.1): the most samples one raw read may
|
||||
/// return; 0 = none. See <see cref="MaxRawValuesPerRead"/> and <see cref="ReadRawWindowAsync"/>.
|
||||
/// </summary>
|
||||
private readonly int _maxRawValuesPerRead;
|
||||
|
||||
/// <summary>A strict gateway's row ceiling, learned from its first refusal.</summary>
|
||||
private readonly GatewayRowCeiling _ceiling;
|
||||
private readonly GatewayTagPresenceCache _tags;
|
||||
@@ -96,11 +102,19 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
/// The most rows before a raw window a crowding re-read makes room for (<c>ServerHistorian:MaxTieClusterOverfetch</c>,
|
||||
/// values below 1 count as 1). More ⇒ <see cref="HistoryReadLimitExceededException"/> (<c>BadTooManyOperations</c>).
|
||||
/// </param>
|
||||
/// <param name="maxRawValuesPerRead">
|
||||
/// The server raw cap (<c>ServerHistorian:MaxRawValuesPerRead</c>, R08:1.1): the most samples one
|
||||
/// <see cref="ReadRawAsync"/> may return — reported as <see cref="MaxRawValuesPerRead"/> (bounded further by a strict
|
||||
/// gateway's learned ceiling), so the server pages at it. A read asking for more (or for everything) asks the gateway
|
||||
/// for cap + 1 rows and is refused when the window holds more, never truncated. <= 0 = unbounded (the pre-#736
|
||||
/// behaviour: an uncapped read is sent as <see cref="int.MaxValue"/>).
|
||||
/// </param>
|
||||
public GatewayHistorianDataSource(IHistorianGatewayClient client, ILogger<GatewayHistorianDataSource> logger,
|
||||
int maxRawRows = TimeWeightedAggregator.DefaultMaxWindowRows, TimeProvider? timeProvider = null,
|
||||
bool atTimeFromRaw = true, int maxPreWindowRows = DefaultMaxPreWindowRows)
|
||||
bool atTimeFromRaw = true, int maxPreWindowRows = DefaultMaxPreWindowRows, int maxRawValuesPerRead = 0)
|
||||
{
|
||||
_maxPreWindowRows = Math.Max(1, maxPreWindowRows);
|
||||
_maxRawValuesPerRead = Math.Max(0, maxRawValuesPerRead);
|
||||
_atTimeFromRaw = atTimeFromRaw;
|
||||
ArgumentNullException.ThrowIfNull(client);
|
||||
ArgumentNullException.ThrowIfNull(logger);
|
||||
@@ -203,6 +217,13 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
/// a cancelled gateway stream evicts its pooled historian session). <c>ReturnBounds</c> is not plumbed to the
|
||||
/// data source on any backend, so bounding values are never synthesised here.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>The server raw cap</b> (R08:1.1, #736). With a cap, a read asking for everything (<c>0</c>) or for more than
|
||||
/// the cap runs as a capped read of cap + 1 — one more than it may return — so an over-cap window is DETECTED and
|
||||
/// refused (<see cref="HistoryReadLimitExceededException"/>) rather than answered truncated (the server's paging
|
||||
/// reads a short answer as the last page) and never sent as <see cref="int.MaxValue"/>. The server itself never asks
|
||||
/// for more than the cap (it pages at <see cref="MaxRawValuesPerRead"/>); this bounds a caller that bypasses it.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
private async Task<List<HistorianSample>> ReadRawWindowAsync(
|
||||
string fullReference, DateTime startUtc, DateTime endUtc, uint maxValuesPerNode,
|
||||
@@ -210,30 +231,39 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
{
|
||||
// The gateway seam caps with an int; OPC UA hands us a uint, so clamp to int range.
|
||||
var cap = (int)Math.Min(maxValuesPerNode, int.MaxValue);
|
||||
if (cap == 0)
|
||||
var serverCap = _maxRawValuesPerRead;
|
||||
// Under the server raw cap, a read asking for everything or for more than the cap runs as a capped read of
|
||||
// cap + 1 (budgetCap); `cap` stays what the client asked for, and a refusal names the cap that applied (limit).
|
||||
var probe = serverCap > 0 && (cap == 0 || cap > serverCap);
|
||||
var budgetCap = probe ? (int)Math.Min(int.MaxValue, (long)serverCap + 1) : cap;
|
||||
var limit = probe ? serverCap : cap;
|
||||
var advice = probe
|
||||
? $"set NumValuesPerNode at or below ServerHistorian:MaxRawValuesPerRead ({serverCap}) to page"
|
||||
: cap == 0 ? "set NumValuesPerNode to page" : "lower NumValuesPerNode";
|
||||
if (budgetCap == 0)
|
||||
{
|
||||
var all = await PassAsync(int.MaxValue).ConfigureAwait(false);
|
||||
if (all.Result.Complete) return all.Result.Samples;
|
||||
// Only reachable on a strict gateway: the ceiling re-read delivered the whole ceiling without reaching
|
||||
// the window end — the window may hold more, and "all values" is never answered from a truncated read.
|
||||
throw new HistoryReadLimitExceededException(cap,
|
||||
throw new HistoryReadLimitExceededException(limit,
|
||||
$"The raw read of '{fullReference}' holds more than the HistorianGateway's row ceiling ({all.Budget}, its " +
|
||||
"Historian:ReadLimits:MaxValuesCeiling with AllowStreamingBeyondCeiling=false); set NumValuesPerNode to page, " +
|
||||
"or narrow the window.")
|
||||
{ PreWindowRows = Crowding(all.Result.PreWindowRows) };
|
||||
}
|
||||
|
||||
var asked = (int)Math.Min(int.MaxValue, (long)cap + RawAggregateReader.PreWindowAllowance);
|
||||
var asked = (int)Math.Min(int.MaxValue, (long)budgetCap + RawAggregateReader.PreWindowAllowance);
|
||||
var pass = await PassAsync(asked).ConfigureAwait(false);
|
||||
if (pass.Result.Complete) return pass.Result.Samples;
|
||||
if (pass.Result.Complete) return Bounded(pass.Result.Samples);
|
||||
|
||||
// A strict gateway's ceiling already bounded that read: a larger budget would only be refused again. The
|
||||
// refusal carries the pre-window rows it saw: the server re-reads the page that many values smaller.
|
||||
if (pass.Budget < asked)
|
||||
throw new HistoryReadLimitExceededException(cap,
|
||||
throw new HistoryReadLimitExceededException(limit,
|
||||
$"The raw read of '{fullReference}' could not be read completely within the HistorianGateway's row ceiling " +
|
||||
$"({pass.Budget}, its Historian:ReadLimits:MaxValuesCeiling with AllowStreamingBeyondCeiling=false); lower " +
|
||||
"NumValuesPerNode or narrow the window.")
|
||||
$"({pass.Budget}, its Historian:ReadLimits:MaxValuesCeiling with AllowStreamingBeyondCeiling=false); {advice}, " +
|
||||
"or narrow the window.")
|
||||
{ PreWindowRows = Crowding(pass.Result.PreWindowRows) };
|
||||
|
||||
// The margin crowded the window out: re-read with room for the pre-window rows. A read that was itself cut short saw
|
||||
@@ -245,25 +275,25 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
var room = Math.Min(Math.Max(1, preWindow), _maxPreWindowRows);
|
||||
for (var attempt = 0; attempt < MaxCrowdedReReads && preWindow <= _maxPreWindowRows; attempt++)
|
||||
{
|
||||
var budget = (int)Math.Min(int.MaxValue, (long)cap + room + RawAggregateReader.PreWindowAllowance);
|
||||
var budget = (int)Math.Min(int.MaxValue, (long)budgetCap + room + RawAggregateReader.PreWindowAllowance);
|
||||
// doubling can leap over the ceiling while the cluster still fits. Spend the remaining room once.
|
||||
if (_ceiling.Learned is var known and > 0 && budget > known)
|
||||
{
|
||||
var available = known - cap - RawAggregateReader.PreWindowAllowance;
|
||||
var available = known - budgetCap - RawAggregateReader.PreWindowAllowance;
|
||||
if (available <= 0) break;
|
||||
room = Math.Min(room, available);
|
||||
budget = cap + room + RawAggregateReader.PreWindowAllowance;
|
||||
budget = budgetCap + room + RawAggregateReader.PreWindowAllowance;
|
||||
}
|
||||
if (budget <= pass.Budget) break; // no more room, never retry the same full read
|
||||
pass = await PassAsync(budget).ConfigureAwait(false);
|
||||
if (pass.Result.Complete) return pass.Result.Samples;
|
||||
if (pass.Result.Complete) return Bounded(pass.Result.Samples);
|
||||
preWindow = Math.Max(preWindow, pass.Result.PreWindowRows);
|
||||
if (pass.Budget < budget) break; // bounded by a ceiling learned meanwhile
|
||||
if (room >= _maxPreWindowRows) break; // the limit's worth of room did not suffice
|
||||
room = (int)Math.Min(_maxPreWindowRows, Math.Max(preWindow, 2L * room));
|
||||
}
|
||||
|
||||
throw new HistoryReadLimitExceededException(cap,
|
||||
throw new HistoryReadLimitExceededException(limit,
|
||||
$"The raw read of '{fullReference}' could not be read completely: more than {pass.Result.PreWindowRows} rows lie in " +
|
||||
$"the {RawAggregateReader.BoundaryMargin.TotalMilliseconds} ms before the window and exhausted the row budget.")
|
||||
{ PreWindowRows = preWindow };
|
||||
@@ -271,10 +301,20 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
// Pre-window rows the allowance did not cover are what crowded the read; fewer are not the cause.
|
||||
static int Crowding(int preWindowRows) => preWindowRows > RawAggregateReader.PreWindowAllowance ? preWindowRows : 0;
|
||||
|
||||
// The cap + 1 probe came back with one sample too many: the window holds more than the server raw cap.
|
||||
List<HistorianSample> Bounded(List<HistorianSample> samples)
|
||||
{
|
||||
if (probe && samples.Count > serverCap)
|
||||
throw new HistoryReadLimitExceededException(serverCap,
|
||||
$"The raw read of '{fullReference}' holds more than ServerHistorian:MaxRawValuesPerRead ({serverCap}) " +
|
||||
"samples; set NumValuesPerNode at or below it to page, or narrow the window.");
|
||||
return samples;
|
||||
}
|
||||
|
||||
// One drained pass; a strict gateway's refusal of the budget is retried once at the ceiling it names.
|
||||
Task<((List<HistorianSample> Samples, int PreWindowRows, bool Complete) Result, int Budget)> PassAsync(int budget) =>
|
||||
_ceiling.ReadAsync(budget,
|
||||
b => ReadRawPassAsync(fullReference, startUtc, endUtc, cap, b, cancellationToken),
|
||||
b => ReadRawPassAsync(fullReference, startUtc, endUtc, budgetCap, b, cancellationToken),
|
||||
(r, _) => !r.Complete);
|
||||
}
|
||||
|
||||
@@ -531,13 +571,20 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
public bool SupportsReverseRaw => false;
|
||||
|
||||
/// <inheritdoc />
|
||||
// 0 until a strict gateway (AllowStreamingBeyondCeiling=false) has refused a budget and
|
||||
// named its ceiling (GatewayRowCeiling); then that ceiling less the
|
||||
// RawAggregateReader.PreWindowAllowance a capped read adds for AVEVA's boundary row — so a page of this
|
||||
// size asks for exactly the ceiling. Back to 0 when a re-probe finds the gateway streaming again.
|
||||
public int MaxRawValuesPerRead => _ceiling.Learned is var c and > 0
|
||||
? Math.Max(1, c - RawAggregateReader.PreWindowAllowance)
|
||||
: 0;
|
||||
// The server raw cap (ServerHistorian:MaxRawValuesPerRead; 0 = none), bounded further by a strict gateway's
|
||||
// (AllowStreamingBeyondCeiling=false) learned ceiling once a refusal has named it (GatewayRowCeiling) — that
|
||||
// ceiling less the RawAggregateReader.PreWindowAllowance a capped read adds for AVEVA's boundary row, so a page of
|
||||
// this size asks for exactly the ceiling. The learned part is forgotten when a re-probe finds the gateway streaming
|
||||
// again; the cap stays. 0 only with no cap and no learned ceiling.
|
||||
public int MaxRawValuesPerRead
|
||||
{
|
||||
get
|
||||
{
|
||||
var learned = _ceiling.Learned is var c and > 0 ? Math.Max(1, c - RawAggregateReader.PreWindowAllowance) : 0;
|
||||
var cap = _maxRawValuesPerRead;
|
||||
return cap > 0 && learned > 0 ? Math.Min(cap, learned) : cap > 0 ? cap : learned;
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
// false: the gateway's ReadEvents streams ascending only. The server never sends it a
|
||||
|
||||
+3
-1
@@ -44,7 +44,9 @@ public static class GatewayHistorian
|
||||
// AtTime from raw reads + the shared evaluator; false = AVEVA's own interpolation (kill switch).
|
||||
atTimeFromRaw: options.GatewayAtTimeFromRaw,
|
||||
// A crowding re-read never makes room for more pre-window rows than the tie-cluster limit.
|
||||
maxPreWindowRows: options.MaxTieClusterOverfetch);
|
||||
maxPreWindowRows: options.MaxTieClusterOverfetch,
|
||||
// The server raw cap (R08:1.1): the page limit the server reads at, and the bound on every raw read.
|
||||
maxRawValuesPerRead: options.EffectiveMaxRawValuesPerRead);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
||||
+34
-4
@@ -66,6 +66,7 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource
|
||||
private readonly TimeProvider _time;
|
||||
private readonly HistorianHealthCounters _health = new();
|
||||
private readonly int _maxTimeWeightedRawRows;
|
||||
private readonly int _maxRawValuesPerRead;
|
||||
|
||||
/// <summary>Creates the data source over the provider's shared connection, bootstrapper and tag catalog.</summary>
|
||||
/// <param name="conn">Connection helper.</param>
|
||||
@@ -74,11 +75,19 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource
|
||||
/// <param name="log">Optional logger.</param>
|
||||
/// <param name="timeProvider">Clock for "now" (AtTime beyond the last sample, server timestamps); system by default.</param>
|
||||
/// <param name="maxTimeWeightedRawRows">Raw in-window row cap of one time-weighted read (<= 0 = unbounded).</param>
|
||||
/// <param name="maxRawValuesPerRead">
|
||||
/// The server raw cap (<c>ServerHistorian:MaxRawValuesPerRead</c>, R08:1.1): the most samples one
|
||||
/// <see cref="ReadRawAsync"/> may return — reported as <see cref="MaxRawValuesPerRead"/>, so the server pages at it.
|
||||
/// A read asking for more (or for everything) is refused when the window holds more, never truncated.
|
||||
/// <= 0 = unbounded (the pre-#736 behaviour).
|
||||
/// </param>
|
||||
internal QuestDbHistorianDataSource(
|
||||
QuestDbConnection conn, QuestDbSchemaBootstrapper bootstrap, QuestDbTagCatalog catalog, ILogger? log = null,
|
||||
TimeProvider? timeProvider = null, int maxTimeWeightedRawRows = TimeWeightedAggregator.DefaultMaxWindowRows)
|
||||
TimeProvider? timeProvider = null, int maxTimeWeightedRawRows = TimeWeightedAggregator.DefaultMaxWindowRows,
|
||||
int maxRawValuesPerRead = 0)
|
||||
{
|
||||
_maxTimeWeightedRawRows = maxTimeWeightedRawRows;
|
||||
_maxRawValuesPerRead = Math.Max(0, maxRawValuesPerRead);
|
||||
_conn = conn ?? throw new ArgumentNullException(nameof(conn));
|
||||
_bootstrap = bootstrap ?? throw new ArgumentNullException(nameof(bootstrap));
|
||||
_catalog = catalog ?? throw new ArgumentNullException(nameof(catalog));
|
||||
@@ -93,12 +102,21 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource
|
||||
// (int.MaxValue); larger values clamp. At most top samples, ContinuationPoint always null: the node
|
||||
// manager pages by re-reading from the last returned timestamp inclusive, which the (ts, tag) dedup
|
||||
// key makes exact.
|
||||
// The server raw cap (R08:1.1, #736): with a cap, a read asking for everything or for more than the cap reads
|
||||
// cap + 1 rows — one more than it may return — so an over-cap window is DETECTED and refused
|
||||
// (HistoryReadLimitExceededException ⇒ BadTooManyOperations) rather than answered truncated: the server's paging
|
||||
// reads a short answer as the last page, so a silent LIMIT would lose data. The server itself never asks for more
|
||||
// than the cap (it pages at MaxRawValuesPerRead); this bounds a caller that bypasses it.
|
||||
/// <inheritdoc />
|
||||
public async Task<HistoryReadResult> ReadRawAsync(
|
||||
string fullReference, DateTime startUtc, DateTime endUtc, uint maxValuesPerNode, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(fullReference);
|
||||
var top = maxValuesPerNode == 0 ? int.MaxValue : (int)Math.Min(maxValuesPerNode, int.MaxValue);
|
||||
var cap = _maxRawValuesPerRead;
|
||||
var probe = cap > 0 && (maxValuesPerNode == 0 || maxValuesPerNode > (uint)cap);
|
||||
var top = probe
|
||||
? (int)Math.Min(int.MaxValue, (long)cap + 1)
|
||||
: maxValuesPerNode == 0 ? int.MaxValue : (int)Math.Min(maxValuesPerNode, int.MaxValue);
|
||||
try
|
||||
{
|
||||
var samples = new List<DataValueSnapshot>();
|
||||
@@ -129,9 +147,19 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource
|
||||
}
|
||||
}
|
||||
|
||||
if (probe && samples.Count > cap)
|
||||
throw new HistoryReadLimitExceededException(cap,
|
||||
$"The raw read of '{fullReference}' holds more than ServerHistorian:MaxRawValuesPerRead ({cap}) samples; " +
|
||||
"set NumValuesPerNode at or below it to page, or narrow the window.");
|
||||
|
||||
RecordSuccess();
|
||||
return new HistoryReadResult(samples, ContinuationPoint: null);
|
||||
}
|
||||
catch (HistoryReadLimitExceededException)
|
||||
{
|
||||
// A client-request limit (the server raw cap), not a backend fault: not counted as a failure.
|
||||
throw;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
RecordFailure(ex);
|
||||
@@ -360,9 +388,11 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource
|
||||
/// <inheritdoc />
|
||||
public bool SupportsReverseRaw => true;
|
||||
|
||||
// 0: a capped raw read is one bounded statement - no backend row ceiling.
|
||||
// The server raw cap (ServerHistorian:MaxRawValuesPerRead; 0 = none): the server pages every raw read at it, so a
|
||||
// client's NumValuesPerNode = 0 is served as pages of at most this many values. QuestDB itself has no row ceiling -
|
||||
// a capped raw read is one bounded statement.
|
||||
/// <inheritdoc />
|
||||
public int MaxRawValuesPerRead => 0;
|
||||
public int MaxRawValuesPerRead => _maxRawValuesPerRead;
|
||||
|
||||
// Reverse events reads are served natively (ORDER BY ... DESC).
|
||||
/// <inheritdoc />
|
||||
|
||||
@@ -35,14 +35,17 @@ public sealed class QuestDbHistorianProvider : IHistorianBackendProvider, IAsync
|
||||
/// <summary>Creates the provider over the <c>ServerHistorian:QuestDb</c> options (validated elsewhere).</summary>
|
||||
/// <param name="options">The <c>ServerHistorian:QuestDb</c> options.</param>
|
||||
/// <param name="maxTimeWeightedRawRows">The shared <c>ServerHistorian:MaxTimeWeightedRawRows</c> cap (<= 0 = unbounded).</param>
|
||||
/// <param name="maxRawValuesPerRead">The shared <c>ServerHistorian:MaxRawValuesPerRead</c> raw cap (<= 0 = unbounded; R08:1.1).</param>
|
||||
public QuestDbHistorianProvider(QuestDbHistorianOptions options,
|
||||
int maxTimeWeightedRawRows = TimeWeightedAggregator.DefaultMaxWindowRows)
|
||||
int maxTimeWeightedRawRows = TimeWeightedAggregator.DefaultMaxWindowRows, int maxRawValuesPerRead = 0)
|
||||
{
|
||||
_options = options ?? throw new ArgumentNullException(nameof(options));
|
||||
_maxTimeWeightedRawRows = maxTimeWeightedRawRows;
|
||||
_maxRawValuesPerRead = maxRawValuesPerRead;
|
||||
}
|
||||
|
||||
private readonly int _maxTimeWeightedRawRows;
|
||||
private readonly int _maxRawValuesPerRead;
|
||||
|
||||
/// <inheritdoc />
|
||||
public string Name => "QuestDb";
|
||||
@@ -75,7 +78,8 @@ public sealed class QuestDbHistorianProvider : IHistorianBackendProvider, IAsync
|
||||
{
|
||||
var s = Get(sp);
|
||||
return new QuestDbHistorianDataSource(s.Conn, s.Bootstrap, s.Catalog,
|
||||
Loggers(sp).CreateLogger<QuestDbHistorianDataSource>(), maxTimeWeightedRawRows: _maxTimeWeightedRawRows);
|
||||
Loggers(sp).CreateLogger<QuestDbHistorianDataSource>(), maxTimeWeightedRawRows: _maxTimeWeightedRawRows,
|
||||
maxRawValuesPerRead: _maxRawValuesPerRead);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
|
||||
+36
-4
@@ -61,6 +61,7 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
|
||||
|
||||
private readonly HistorianHealthCounters _health = new();
|
||||
private readonly int _maxTimeWeightedRawRows;
|
||||
private readonly int _maxRawValuesPerRead;
|
||||
|
||||
/// <summary>Creates the data source over the provider's shared connection, bootstrapper and tag catalog.</summary>
|
||||
/// <param name="conn">Connection helper.</param>
|
||||
@@ -69,11 +70,19 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
|
||||
/// <param name="log">Optional logger.</param>
|
||||
/// <param name="timeProvider">Clock for "now" (AtTime beyond the last sample, server timestamps); system by default.</param>
|
||||
/// <param name="maxTimeWeightedRawRows">Raw in-window row cap of one time-weighted read (<= 0 = unbounded).</param>
|
||||
/// <param name="maxRawValuesPerRead">
|
||||
/// The server raw cap (<c>ServerHistorian:MaxRawValuesPerRead</c>, R08:1.1): the most samples one
|
||||
/// <see cref="ReadRawAsync"/> may return — reported as <see cref="MaxRawValuesPerRead"/>, so the server pages at it.
|
||||
/// A read asking for more (or for everything) is refused when the window holds more, never truncated.
|
||||
/// <= 0 = unbounded (the pre-#736 behaviour).
|
||||
/// </param>
|
||||
internal SqlServerHistorianDataSource(
|
||||
SqlHistorianConnection conn, SqlServerSchemaBootstrapper bootstrap, SqlServerTagCatalog catalog, ILogger? log = null,
|
||||
TimeProvider? timeProvider = null, int maxTimeWeightedRawRows = TimeWeightedAggregator.DefaultMaxWindowRows)
|
||||
TimeProvider? timeProvider = null, int maxTimeWeightedRawRows = TimeWeightedAggregator.DefaultMaxWindowRows,
|
||||
int maxRawValuesPerRead = 0)
|
||||
{
|
||||
_maxTimeWeightedRawRows = maxTimeWeightedRawRows;
|
||||
_maxRawValuesPerRead = Math.Max(0, maxRawValuesPerRead);
|
||||
_conn = conn ?? throw new ArgumentNullException(nameof(conn));
|
||||
_bootstrap = bootstrap ?? throw new ArgumentNullException(nameof(bootstrap));
|
||||
_catalog = catalog ?? throw new ArgumentNullException(nameof(catalog));
|
||||
@@ -90,13 +99,22 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
|
||||
// <c>0</c> is the node manager's "unbounded" sentinel; values above <see cref="int.MaxValue"/> clamp.
|
||||
// The node manager pages by re-reading from the last returned timestamp inclusive and trimming it; the
|
||||
// <c>(TagId, SourceTimeUtc)</c> primary key rules out tie clusters, so that is exact.
|
||||
// The server raw cap (R08:1.1, #736): with a cap, a read asking for everything or for more than the cap reads
|
||||
// cap + 1 rows — one more than it may return — so an over-cap window is DETECTED and refused
|
||||
// (HistoryReadLimitExceededException ⇒ BadTooManyOperations) rather than answered truncated: the server's paging
|
||||
// reads a short answer as the last page, so a silent TOP would lose data. The server itself never asks for more than
|
||||
// the cap (it pages at MaxRawValuesPerRead); this bounds a caller that bypasses it.
|
||||
// </remarks>
|
||||
/// <inheritdoc />
|
||||
public async Task<HistoryReadResult> ReadRawAsync(
|
||||
string fullReference, DateTime startUtc, DateTime endUtc, uint maxValuesPerNode, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(fullReference);
|
||||
var top = maxValuesPerNode == 0 ? int.MaxValue : (int)Math.Min(maxValuesPerNode, int.MaxValue);
|
||||
var cap = _maxRawValuesPerRead;
|
||||
var probe = cap > 0 && (maxValuesPerNode == 0 || maxValuesPerNode > (uint)cap);
|
||||
var top = probe
|
||||
? (int)Math.Min(int.MaxValue, (long)cap + 1)
|
||||
: maxValuesPerNode == 0 ? int.MaxValue : (int)Math.Min(maxValuesPerNode, int.MaxValue);
|
||||
try
|
||||
{
|
||||
var tagId = await ResolveAsync(fullReference, cancellationToken).ConfigureAwait(false);
|
||||
@@ -126,9 +144,19 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
|
||||
}
|
||||
}
|
||||
|
||||
if (probe && samples.Count > cap)
|
||||
throw new HistoryReadLimitExceededException(cap,
|
||||
$"The raw read of '{fullReference}' holds more than ServerHistorian:MaxRawValuesPerRead ({cap}) samples; " +
|
||||
"set NumValuesPerNode at or below it to page, or narrow the window.");
|
||||
|
||||
RecordSuccess();
|
||||
return new HistoryReadResult(samples, ContinuationPoint: null);
|
||||
}
|
||||
catch (HistoryReadLimitExceededException)
|
||||
{
|
||||
// A client-request limit (the server raw cap), not a backend fault: not counted as a failure.
|
||||
throw;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
RecordFailure(ex);
|
||||
@@ -319,9 +347,13 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
|
||||
/// <inheritdoc />
|
||||
public bool SupportsReverseRaw => true;
|
||||
|
||||
// <remarks><c>0</c>: a capped raw read is one bounded statement — no backend row ceiling.</remarks>
|
||||
// <remarks>
|
||||
// The server raw cap (ServerHistorian:MaxRawValuesPerRead; 0 = none): the server pages every raw read at it, so a
|
||||
// client's NumValuesPerNode = 0 is served as pages of at most this many values. SQL Server itself has no row
|
||||
// ceiling — a capped raw read is one bounded statement.
|
||||
// </remarks>
|
||||
/// <inheritdoc />
|
||||
public int MaxRawValuesPerRead => 0;
|
||||
public int MaxRawValuesPerRead => _maxRawValuesPerRead;
|
||||
|
||||
// <remarks>Reverse events reads are served natively (<c>ORDER BY … DESC</c>).</remarks>
|
||||
/// <inheritdoc />
|
||||
|
||||
+6
-2
@@ -31,14 +31,17 @@ public sealed class SqlServerHistorianProvider : IHistorianBackendProvider
|
||||
/// <summary>Creates the provider over the <c>ServerHistorian:SqlServer</c> options (validated elsewhere).</summary>
|
||||
/// <param name="options">The <c>ServerHistorian:SqlServer</c> options.</param>
|
||||
/// <param name="maxTimeWeightedRawRows">The shared <c>ServerHistorian:MaxTimeWeightedRawRows</c> cap (<= 0 = unbounded).</param>
|
||||
/// <param name="maxRawValuesPerRead">The shared <c>ServerHistorian:MaxRawValuesPerRead</c> raw cap (<= 0 = unbounded; R08:1.1).</param>
|
||||
public SqlServerHistorianProvider(SqlServerHistorianOptions options,
|
||||
int maxTimeWeightedRawRows = TimeWeightedAggregator.DefaultMaxWindowRows)
|
||||
int maxTimeWeightedRawRows = TimeWeightedAggregator.DefaultMaxWindowRows, int maxRawValuesPerRead = 0)
|
||||
{
|
||||
_options = options ?? throw new ArgumentNullException(nameof(options));
|
||||
_maxTimeWeightedRawRows = maxTimeWeightedRawRows;
|
||||
_maxRawValuesPerRead = maxRawValuesPerRead;
|
||||
}
|
||||
|
||||
private readonly int _maxTimeWeightedRawRows;
|
||||
private readonly int _maxRawValuesPerRead;
|
||||
|
||||
/// <inheritdoc />
|
||||
public string Name => "SqlServer";
|
||||
@@ -69,7 +72,8 @@ public sealed class SqlServerHistorianProvider : IHistorianBackendProvider
|
||||
{
|
||||
var s = Get(sp);
|
||||
return new SqlServerHistorianDataSource(s.Conn, s.Bootstrap, s.Catalog,
|
||||
Loggers(sp).CreateLogger<SqlServerHistorianDataSource>(), maxTimeWeightedRawRows: _maxTimeWeightedRawRows);
|
||||
Loggers(sp).CreateLogger<SqlServerHistorianDataSource>(), maxTimeWeightedRawRows: _maxTimeWeightedRawRows,
|
||||
maxRawValuesPerRead: _maxRawValuesPerRead);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
|
||||
@@ -45,8 +45,8 @@ public static class HistorianBackendRegistry
|
||||
new Dictionary<string, Func<ServerHistorianOptions, IConfiguration, IHistorianBackendProvider>>(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
[ServerHistorianOptions.GatewayBackendName] = (o, _) => new GatewayHistorianProvider(o),
|
||||
[SqlServerBackendName] = (o, c) => new SqlServerHistorianProvider(BindSqlServer(c), o.MaxTimeWeightedRawRows),
|
||||
[QuestDbBackendName] = (o, c) => new QuestDbHistorianProvider(BindQuestDb(c), o.MaxTimeWeightedRawRows),
|
||||
[SqlServerBackendName] = (o, c) => new SqlServerHistorianProvider(BindSqlServer(c), o.MaxTimeWeightedRawRows, o.EffectiveMaxRawValuesPerRead),
|
||||
[QuestDbBackendName] = (o, c) => new QuestDbHistorianProvider(BindQuestDb(c), o.MaxTimeWeightedRawRows, o.EffectiveMaxRawValuesPerRead),
|
||||
};
|
||||
|
||||
/// <summary>The registered backend names, for validation messages.</summary>
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
using Opc.Ua;
|
||||
using Opc.Ua.Server;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.OpcUaServer;
|
||||
@@ -110,24 +112,62 @@ internal enum HistoryCursorKind
|
||||
/// <c>NumValuesPerNode</c> samples with a null continuation point — so paging is synthesised here,
|
||||
/// time-based:
|
||||
/// <list type="bullet">
|
||||
/// <item>A page that returns EXACTLY the requested cap (<c>NumValuesPerNode > 0</c>) MAY have
|
||||
/// more behind it ⇒ emit a continuation point.</item>
|
||||
/// <item>A short page (fewer than the cap) is the last page ⇒ no continuation point.</item>
|
||||
/// <item><c>NumValuesPerNode == 0</c> means "all values, no limit" (OPC UA Part 11) ⇒ never page;
|
||||
/// return everything in one shot.</item>
|
||||
/// <item>A page whose read returned the lookahead row past it MAY have more behind it ⇒ emit a continuation
|
||||
/// point (<see cref="IsFullPage"/> judges a read against what was asked for).</item>
|
||||
/// <item>A short read (no lookahead row) is the last page ⇒ no continuation point.</item>
|
||||
/// <item><c>NumValuesPerNode == 0</c> means "all values" (OPC UA Part 11) — but not "in one shot": the page is
|
||||
/// sized to the backend's <see cref="IHistorianDataSource.MaxRawValuesPerRead"/> by <see cref="RawPageLimit"/>
|
||||
/// (the server raw cap <c>ServerHistorian:MaxRawValuesPerRead</c>, default 10 000, and/or a strict gateway's
|
||||
/// learned ceiling; review R08:1.1, #736), and a full page carries a continuation point like any other. Only
|
||||
/// with no limit at all (<c>0</c>, the opt-out) is everything returned in one shot.</item>
|
||||
/// </list>
|
||||
/// All methods are static + pure so they unit-test without a server, a session, or the SDK.
|
||||
/// All paging methods are static + pure so they unit-test without a server, a session, or the SDK;
|
||||
/// <see cref="AdvertiseMaxReturnDataValues"/> is the one SDK-facing member.
|
||||
/// </summary>
|
||||
internal static class HistoryPaging
|
||||
{
|
||||
/// <summary>
|
||||
/// Decide whether a just-returned page is "full" and therefore MAY be followed by more data —
|
||||
/// the signal to emit a continuation point. A page is full when the client asked for a finite
|
||||
/// cap (<paramref name="numValuesPerNode"/> > 0) and the backend returned exactly that many
|
||||
/// samples. A short page (or an unlimited <c>0</c> request) is terminal.
|
||||
/// Advertise the server raw cap as <c>HistoryServerCapabilities/MaxReturnDataValues</c> (i=11273; Part 11 §5.4.2:
|
||||
/// "the maximum number of values that can be returned by the Server for each Node", 0 = no limit). The cap is
|
||||
/// the most values one HistoryRead-Raw page returns (<c>ServerHistorian:MaxRawValuesPerRead</c>), so a client can
|
||||
/// size its <c>NumValuesPerNode</c> and expect a continuation point past it. Idempotent; a re-sync with a
|
||||
/// different cap overwrites. Never throws: a missing diagnostics manager or node, or any SDK failure writing the
|
||||
/// property, leaves the SDK default (0) and returns <c>false</c> for the caller to log — HistoryRead itself still
|
||||
/// enforces the cap.
|
||||
/// </summary>
|
||||
/// <param name="returnedCount">The number of samples the backend returned for this page.</param>
|
||||
/// <param name="numValuesPerNode">The client's per-page cap; <c>0</c> means "no limit".</param>
|
||||
/// <param name="server">The running server.</param>
|
||||
/// <param name="maxRawValuesPerRead">The cap (<= 0 advertises 0 = no limit).</param>
|
||||
/// <returns><c>true</c> when the property was written.</returns>
|
||||
public static bool AdvertiseMaxReturnDataValues(IServerInternal? server, int maxRawValuesPerRead)
|
||||
{
|
||||
if (server?.DiagnosticsNodeManager is not DiagnosticsNodeManager diagnostics) return false;
|
||||
try
|
||||
{
|
||||
lock (diagnostics.Lock)
|
||||
{
|
||||
var capabilities = diagnostics.GetDefaultHistoryCapabilities();
|
||||
if (capabilities?.MaxReturnDataValues is not { } property) return false;
|
||||
property.Value = (uint)Math.Max(0, maxRawValuesPerRead);
|
||||
property.ClearChangeMasks(diagnostics.SystemContext, false);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
catch (Exception)
|
||||
{
|
||||
return false; // advisory only: the caller logs; the cap is enforced by HistoryRead regardless
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Decide whether a just-returned read is "full" and therefore MAY be followed by more data —
|
||||
/// the signal to emit a continuation point. A read is full when it was issued with a finite request
|
||||
/// cap (<paramref name="numValuesPerNode"/> > 0 — the <c>RawPagePlan.RequestCap</c>: page + boundary ties +
|
||||
/// lookahead, which under a backend limit or the server raw cap is always finite, uncapped client reads included)
|
||||
/// and the backend returned at least that many samples. A short read is terminal; so is a read issued with a
|
||||
/// request cap of <c>0</c>, which only happens with no limit at all (one shot of the whole window).
|
||||
/// </summary>
|
||||
/// <param name="returnedCount">The number of samples the backend returned for this read.</param>
|
||||
/// <param name="numValuesPerNode">The request cap the read was issued with; <c>0</c> means "no limit".</param>
|
||||
/// <returns><c>true</c> when a continuation point should be emitted; otherwise <c>false</c>.</returns>
|
||||
public static bool IsFullPage(int returnedCount, uint numValuesPerNode) =>
|
||||
numValuesPerNode > 0 && returnedCount >= numValuesPerNode;
|
||||
|
||||
@@ -11,17 +11,22 @@ internal readonly record struct RawPagePlan(int Limit, uint PageCap, uint Reques
|
||||
|
||||
/// <summary>
|
||||
/// sizes a HistoryRead-Raw page to what the backend can return in one read
|
||||
/// (<see cref="IHistorianDataSource.MaxRawValuesPerRead"/> = L — a strict HistorianGateway's learned row ceiling, less
|
||||
/// its boundary-row allowance). With a limit the read — page P + boundary ties k + the lookahead row — is at most L,
|
||||
/// so P = min(want, L − k − 1) (all values: P = L − k − 1); a full page carries a continuation point like any other, as
|
||||
/// Part 11 allows fewer values than <c>NumValuesPerNode</c> with one. Previously a cap above a strict gateway's ceiling
|
||||
/// (or an uncapped read of a window holding more) failed <c>BadTooManyOperations</c>.
|
||||
/// (<see cref="IHistorianDataSource.MaxRawValuesPerRead"/> = L — the server raw cap
|
||||
/// <c>ServerHistorian:MaxRawValuesPerRead</c>, default 10 000, which every backend reports (review R08:1.1, #736),
|
||||
/// and/or a strict HistorianGateway's learned row ceiling less its boundary-row allowance, whichever is lower). With
|
||||
/// a limit the read — page P + boundary ties k + the lookahead row — is at most L, so P = min(want, L − k − 1) (all
|
||||
/// values: P = L − k − 1, i.e. a first page of 9 999 at the default cap); a full page carries a continuation point like
|
||||
/// any other, as Part 11 allows fewer values than <c>NumValuesPerNode</c> with one. Previously a cap above a strict
|
||||
/// gateway's ceiling (or an uncapped read of a window holding more) failed <c>BadTooManyOperations</c>, and before
|
||||
/// #736 an uncapped read on a backend without a ceiling read the whole window in one shot.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>Never a silent truncation.</b> The backend still answers a read in full or refuses it; only the server's page is
|
||||
/// smaller, and a page is judged full against what was asked for. A read the backend cannot serve at all (a tie
|
||||
/// cluster larger than one read) stays <c>BadTooManyOperations</c>.
|
||||
/// cluster larger than one read) stays <c>BadTooManyOperations</c>. The backends enforce the same cap themselves
|
||||
/// (a read asking for more than L, or for everything, is refused when the window holds more than L rows), so this
|
||||
/// planner is what keeps the server's own reads inside it: with L > 0 no read it plans ever asks for 0 or more than L.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>One re-read, planned again.</b> A refused read (<see cref="HistoryReadLimitExceededException"/>) is re-planned and
|
||||
|
||||
@@ -154,6 +154,42 @@ public sealed class ServerHistorianOptions
|
||||
/// </summary>
|
||||
public int MaxTimeWeightedRawRows { get; init; } = 1_000_000;
|
||||
|
||||
/// <summary>The default of <see cref="MaxRawValuesPerRead"/>: 10 000 values per raw page / backend read.</summary>
|
||||
public const int DefaultMaxRawValuesPerRead = 10_000;
|
||||
|
||||
/// <summary>
|
||||
/// The smallest usable <see cref="MaxRawValuesPerRead"/>: a resumed page's read must carry at least one value, the
|
||||
/// boundary tie it re-reads and trims, AND the lookahead row that tells the server more data follows. At 2 a resumed
|
||||
/// page has no room at all and every value costs an empty page with a continuation point (the #692 defect, re-created
|
||||
/// by configuration), so 1 and 2 are raised to 3 (<see cref="EffectiveMaxRawValuesPerRead"/>).
|
||||
/// </summary>
|
||||
public const int MinMaxRawValuesPerRead = 3;
|
||||
|
||||
/// <summary>
|
||||
/// Every backend: the most raw values the server returns on ONE HistoryRead-Raw page, and the most rows one
|
||||
/// backend raw read may materialise — the server's own row ceiling (review R08:1.1, #736). A client's
|
||||
/// <c>NumValuesPerNode = 0</c> ("all values") no longer reads a whole window into memory: it is served as pages of at
|
||||
/// most this many values, each with a continuation point (Part 11 allows fewer values than requested with one), through
|
||||
/// the same tie-safe paging a capped read uses (<c>RawPageLimit</c>: page + boundary ties + lookahead ≤ cap, so a first
|
||||
/// page holds cap − 1 values). A <c>NumValuesPerNode</c> above it is paged at it too. Advertised as
|
||||
/// <c>HistoryServerCapabilities/MaxReturnDataValues</c>. The backends' own <c>TOP</c> / <c>LIMIT</c> / <c>max_values</c>
|
||||
/// are bounded by it independently: a raw read asking for more than this (or for everything) reads at most cap + 1 rows
|
||||
/// and is REFUSED (<c>BadTooManyOperations</c>) when the window holds more — never answered truncated — so a caller that
|
||||
/// bypasses the node manager's paging cannot bypass the bound. On the Gateway backend a strict gateway's learned row
|
||||
/// ceiling (less its boundary-row allowance) still applies when lower. The tie-cluster bound of a resumed page is also
|
||||
/// at most cap − 1 (as under a gateway ceiling; <see cref="MaxTieClusterOverfetch"/> above that is inert), and a
|
||||
/// page whose boundary tie cluster is that large steps past it with one empty page. Default 10 000. Non-positive =
|
||||
/// unbounded, the pre-#736 behaviour (a <see cref="Validate"/> warning); below <see cref="MinMaxRawValuesPerRead"/>
|
||||
/// (3) is raised to it.
|
||||
/// </summary>
|
||||
public int MaxRawValuesPerRead { get; init; } = DefaultMaxRawValuesPerRead;
|
||||
|
||||
/// <summary>
|
||||
/// <see cref="MaxRawValuesPerRead"/> as the backends and the node manager apply it: <c>0</c> (unbounded) when
|
||||
/// non-positive, otherwise at least <see cref="MinMaxRawValuesPerRead"/>. The one place the rule lives.
|
||||
/// </summary>
|
||||
public int EffectiveMaxRawValuesPerRead => MaxRawValuesPerRead <= 0 ? 0 : Math.Max(MinMaxRawValuesPerRead, MaxRawValuesPerRead);
|
||||
|
||||
/// <summary>
|
||||
/// how far into the future an OPC UA HistoryUpdate (Insert / Replace / Update) item's
|
||||
/// <c>SourceTimestamp</c> may lie, measured from the server clock. An item beyond <c>now + skew</c> is refused
|
||||
@@ -208,6 +244,10 @@ public sealed class ServerHistorianOptions
|
||||
warnings.Add($"ServerHistorian:MaxConcurrentHistoryReadBatches is {MaxConcurrentHistoryReadBatches} — must be > 0; the process-wide HistoryRead batch limiter is disabled.");
|
||||
if (MaxTimeWeightedRawRows <= 0)
|
||||
warnings.Add($"ServerHistorian:MaxTimeWeightedRawRows is {MaxTimeWeightedRawRows} — must be > 0; time-weighted aggregate reads on the SqlServer/QuestDb backends stream an unbounded number of raw samples.");
|
||||
if (MaxRawValuesPerRead <= 0)
|
||||
warnings.Add($"ServerHistorian:MaxRawValuesPerRead is {MaxRawValuesPerRead} — must be > 0; HistoryRead-Raw pages and backend raw reads are unbounded, so a client asking for a whole window (NumValuesPerNode = 0) materialises every sample in memory.");
|
||||
else if (MaxRawValuesPerRead < MinMaxRawValuesPerRead)
|
||||
warnings.Add($"ServerHistorian:MaxRawValuesPerRead is {MaxRawValuesPerRead} — must be at least {MinMaxRawValuesPerRead} (a value, its boundary tie and the lookahead row); treated as {MinMaxRawValuesPerRead}.");
|
||||
if (EventReadHorizon < TimeSpan.Zero)
|
||||
warnings.Add($"ServerHistorian:EventReadHorizon is {EventReadHorizon} — negative is treated as no horizon (00:00:00); HistoryReadEvents reads every window in full.");
|
||||
if (HistoryReadDeadline <= TimeSpan.Zero)
|
||||
|
||||
Reference in New Issue
Block a user