diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway/GatewayHistorianDataSource.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway/GatewayHistorianDataSource.cs index 6f3dfd71..e5c19a87 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway/GatewayHistorianDataSource.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway/GatewayHistorianDataSource.cs @@ -67,6 +67,12 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis /// The tie-cluster limit the crowding re-reads honour — exposed so the factory wiring is testable. internal int MaxPreWindowRows => _maxPreWindowRows; + /// + /// The server raw cap (ServerHistorian:MaxRawValuesPerRead, R08:1.1): the most samples one raw read may + /// return; 0 = none. See and . + /// + private readonly int _maxRawValuesPerRead; + /// A strict gateway's row ceiling, learned from its first refusal. private readonly GatewayRowCeiling _ceiling; private readonly GatewayTagPresenceCache _tags; @@ -96,11 +102,19 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis /// The most rows before a raw window a crowding re-read makes room for (ServerHistorian:MaxTieClusterOverfetch, /// values below 1 count as 1). More ⇒ (BadTooManyOperations). /// + /// + /// The server raw cap (ServerHistorian:MaxRawValuesPerRead, R08:1.1): the most samples one + /// may return — reported as (bounded further by a strict + /// gateway's learned ceiling), so the server pages at it. A read asking for more (or for everything) asks the gateway + /// for cap + 1 rows and is refused when the window holds more, never truncated. <= 0 = unbounded (the pre-#736 + /// behaviour: an uncapped read is sent as ). + /// public GatewayHistorianDataSource(IHistorianGatewayClient client, ILogger logger, int maxRawRows = TimeWeightedAggregator.DefaultMaxWindowRows, TimeProvider? timeProvider = null, - bool atTimeFromRaw = true, int maxPreWindowRows = DefaultMaxPreWindowRows) + bool atTimeFromRaw = true, int maxPreWindowRows = DefaultMaxPreWindowRows, int maxRawValuesPerRead = 0) { _maxPreWindowRows = Math.Max(1, maxPreWindowRows); + _maxRawValuesPerRead = Math.Max(0, maxRawValuesPerRead); _atTimeFromRaw = atTimeFromRaw; ArgumentNullException.ThrowIfNull(client); ArgumentNullException.ThrowIfNull(logger); @@ -203,6 +217,13 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis /// a cancelled gateway stream evicts its pooled historian session). ReturnBounds is not plumbed to the /// data source on any backend, so bounding values are never synthesised here. /// + /// + /// The server raw cap (R08:1.1, #736). With a cap, a read asking for everything (0) or for more than + /// the cap runs as a capped read of cap + 1 — one more than it may return — so an over-cap window is DETECTED and + /// refused () rather than answered truncated (the server's paging + /// reads a short answer as the last page) and never sent as . The server itself never asks + /// for more than the cap (it pages at ); this bounds a caller that bypasses it. + /// /// private async Task> ReadRawWindowAsync( string fullReference, DateTime startUtc, DateTime endUtc, uint maxValuesPerNode, @@ -210,30 +231,39 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis { // The gateway seam caps with an int; OPC UA hands us a uint, so clamp to int range. var cap = (int)Math.Min(maxValuesPerNode, int.MaxValue); - if (cap == 0) + var serverCap = _maxRawValuesPerRead; + // Under the server raw cap, a read asking for everything or for more than the cap runs as a capped read of + // cap + 1 (budgetCap); `cap` stays what the client asked for, and a refusal names the cap that applied (limit). + var probe = serverCap > 0 && (cap == 0 || cap > serverCap); + var budgetCap = probe ? (int)Math.Min(int.MaxValue, (long)serverCap + 1) : cap; + var limit = probe ? serverCap : cap; + var advice = probe + ? $"set NumValuesPerNode at or below ServerHistorian:MaxRawValuesPerRead ({serverCap}) to page" + : cap == 0 ? "set NumValuesPerNode to page" : "lower NumValuesPerNode"; + if (budgetCap == 0) { var all = await PassAsync(int.MaxValue).ConfigureAwait(false); if (all.Result.Complete) return all.Result.Samples; // Only reachable on a strict gateway: the ceiling re-read delivered the whole ceiling without reaching // the window end — the window may hold more, and "all values" is never answered from a truncated read. - throw new HistoryReadLimitExceededException(cap, + throw new HistoryReadLimitExceededException(limit, $"The raw read of '{fullReference}' holds more than the HistorianGateway's row ceiling ({all.Budget}, its " + "Historian:ReadLimits:MaxValuesCeiling with AllowStreamingBeyondCeiling=false); set NumValuesPerNode to page, " + "or narrow the window.") { PreWindowRows = Crowding(all.Result.PreWindowRows) }; } - var asked = (int)Math.Min(int.MaxValue, (long)cap + RawAggregateReader.PreWindowAllowance); + var asked = (int)Math.Min(int.MaxValue, (long)budgetCap + RawAggregateReader.PreWindowAllowance); var pass = await PassAsync(asked).ConfigureAwait(false); - if (pass.Result.Complete) return pass.Result.Samples; + if (pass.Result.Complete) return Bounded(pass.Result.Samples); // A strict gateway's ceiling already bounded that read: a larger budget would only be refused again. The // refusal carries the pre-window rows it saw: the server re-reads the page that many values smaller. if (pass.Budget < asked) - throw new HistoryReadLimitExceededException(cap, + throw new HistoryReadLimitExceededException(limit, $"The raw read of '{fullReference}' could not be read completely within the HistorianGateway's row ceiling " + - $"({pass.Budget}, its Historian:ReadLimits:MaxValuesCeiling with AllowStreamingBeyondCeiling=false); lower " + - "NumValuesPerNode or narrow the window.") + $"({pass.Budget}, its Historian:ReadLimits:MaxValuesCeiling with AllowStreamingBeyondCeiling=false); {advice}, " + + "or narrow the window.") { PreWindowRows = Crowding(pass.Result.PreWindowRows) }; // The margin crowded the window out: re-read with room for the pre-window rows. A read that was itself cut short saw @@ -245,25 +275,25 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis var room = Math.Min(Math.Max(1, preWindow), _maxPreWindowRows); for (var attempt = 0; attempt < MaxCrowdedReReads && preWindow <= _maxPreWindowRows; attempt++) { - var budget = (int)Math.Min(int.MaxValue, (long)cap + room + RawAggregateReader.PreWindowAllowance); + var budget = (int)Math.Min(int.MaxValue, (long)budgetCap + room + RawAggregateReader.PreWindowAllowance); // doubling can leap over the ceiling while the cluster still fits. Spend the remaining room once. if (_ceiling.Learned is var known and > 0 && budget > known) { - var available = known - cap - RawAggregateReader.PreWindowAllowance; + var available = known - budgetCap - RawAggregateReader.PreWindowAllowance; if (available <= 0) break; room = Math.Min(room, available); - budget = cap + room + RawAggregateReader.PreWindowAllowance; + budget = budgetCap + room + RawAggregateReader.PreWindowAllowance; } if (budget <= pass.Budget) break; // no more room, never retry the same full read pass = await PassAsync(budget).ConfigureAwait(false); - if (pass.Result.Complete) return pass.Result.Samples; + if (pass.Result.Complete) return Bounded(pass.Result.Samples); preWindow = Math.Max(preWindow, pass.Result.PreWindowRows); if (pass.Budget < budget) break; // bounded by a ceiling learned meanwhile if (room >= _maxPreWindowRows) break; // the limit's worth of room did not suffice room = (int)Math.Min(_maxPreWindowRows, Math.Max(preWindow, 2L * room)); } - throw new HistoryReadLimitExceededException(cap, + throw new HistoryReadLimitExceededException(limit, $"The raw read of '{fullReference}' could not be read completely: more than {pass.Result.PreWindowRows} rows lie in " + $"the {RawAggregateReader.BoundaryMargin.TotalMilliseconds} ms before the window and exhausted the row budget.") { PreWindowRows = preWindow }; @@ -271,10 +301,20 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis // Pre-window rows the allowance did not cover are what crowded the read; fewer are not the cause. static int Crowding(int preWindowRows) => preWindowRows > RawAggregateReader.PreWindowAllowance ? preWindowRows : 0; + // The cap + 1 probe came back with one sample too many: the window holds more than the server raw cap. + List Bounded(List samples) + { + if (probe && samples.Count > serverCap) + throw new HistoryReadLimitExceededException(serverCap, + $"The raw read of '{fullReference}' holds more than ServerHistorian:MaxRawValuesPerRead ({serverCap}) " + + "samples; set NumValuesPerNode at or below it to page, or narrow the window."); + return samples; + } + // One drained pass; a strict gateway's refusal of the budget is retried once at the ceiling it names. Task<((List Samples, int PreWindowRows, bool Complete) Result, int Budget)> PassAsync(int budget) => _ceiling.ReadAsync(budget, - b => ReadRawPassAsync(fullReference, startUtc, endUtc, cap, b, cancellationToken), + b => ReadRawPassAsync(fullReference, startUtc, endUtc, budgetCap, b, cancellationToken), (r, _) => !r.Complete); } @@ -531,13 +571,20 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis public bool SupportsReverseRaw => false; /// - // 0 until a strict gateway (AllowStreamingBeyondCeiling=false) has refused a budget and - // named its ceiling (GatewayRowCeiling); then that ceiling less the - // RawAggregateReader.PreWindowAllowance a capped read adds for AVEVA's boundary row — so a page of this - // size asks for exactly the ceiling. Back to 0 when a re-probe finds the gateway streaming again. - public int MaxRawValuesPerRead => _ceiling.Learned is var c and > 0 - ? Math.Max(1, c - RawAggregateReader.PreWindowAllowance) - : 0; + // The server raw cap (ServerHistorian:MaxRawValuesPerRead; 0 = none), bounded further by a strict gateway's + // (AllowStreamingBeyondCeiling=false) learned ceiling once a refusal has named it (GatewayRowCeiling) — that + // ceiling less the RawAggregateReader.PreWindowAllowance a capped read adds for AVEVA's boundary row, so a page of + // this size asks for exactly the ceiling. The learned part is forgotten when a re-probe finds the gateway streaming + // again; the cap stays. 0 only with no cap and no learned ceiling. + public int MaxRawValuesPerRead + { + get + { + var learned = _ceiling.Learned is var c and > 0 ? Math.Max(1, c - RawAggregateReader.PreWindowAllowance) : 0; + var cap = _maxRawValuesPerRead; + return cap > 0 && learned > 0 ? Math.Min(cap, learned) : cap > 0 ? cap : learned; + } + } /// // false: the gateway's ReadEvents streams ascending only. The server never sends it a diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway/GatewayHistorianServiceCollectionExtensions.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway/GatewayHistorianServiceCollectionExtensions.cs index 496ba830..a2a9b0d2 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway/GatewayHistorianServiceCollectionExtensions.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway/GatewayHistorianServiceCollectionExtensions.cs @@ -44,7 +44,9 @@ public static class GatewayHistorian // AtTime from raw reads + the shared evaluator; false = AVEVA's own interpolation (kill switch). atTimeFromRaw: options.GatewayAtTimeFromRaw, // A crowding re-read never makes room for more pre-window rows than the tie-cluster limit. - maxPreWindowRows: options.MaxTieClusterOverfetch); + maxPreWindowRows: options.MaxTieClusterOverfetch, + // The server raw cap (R08:1.1): the page limit the server reads at, and the bound on every raw read. + maxRawValuesPerRead: options.EffectiveMaxRawValuesPerRead); } /// diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb/QuestDbHistorianDataSource.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb/QuestDbHistorianDataSource.cs index a0d9871a..49b814d6 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb/QuestDbHistorianDataSource.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb/QuestDbHistorianDataSource.cs @@ -66,6 +66,7 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource private readonly TimeProvider _time; private readonly HistorianHealthCounters _health = new(); private readonly int _maxTimeWeightedRawRows; + private readonly int _maxRawValuesPerRead; /// Creates the data source over the provider's shared connection, bootstrapper and tag catalog. /// Connection helper. @@ -74,11 +75,19 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource /// Optional logger. /// Clock for "now" (AtTime beyond the last sample, server timestamps); system by default. /// Raw in-window row cap of one time-weighted read (<= 0 = unbounded). + /// + /// The server raw cap (ServerHistorian:MaxRawValuesPerRead, R08:1.1): the most samples one + /// may return — reported as , so the server pages at it. + /// A read asking for more (or for everything) is refused when the window holds more, never truncated. + /// <= 0 = unbounded (the pre-#736 behaviour). + /// internal QuestDbHistorianDataSource( QuestDbConnection conn, QuestDbSchemaBootstrapper bootstrap, QuestDbTagCatalog catalog, ILogger? log = null, - TimeProvider? timeProvider = null, int maxTimeWeightedRawRows = TimeWeightedAggregator.DefaultMaxWindowRows) + TimeProvider? timeProvider = null, int maxTimeWeightedRawRows = TimeWeightedAggregator.DefaultMaxWindowRows, + int maxRawValuesPerRead = 0) { _maxTimeWeightedRawRows = maxTimeWeightedRawRows; + _maxRawValuesPerRead = Math.Max(0, maxRawValuesPerRead); _conn = conn ?? throw new ArgumentNullException(nameof(conn)); _bootstrap = bootstrap ?? throw new ArgumentNullException(nameof(bootstrap)); _catalog = catalog ?? throw new ArgumentNullException(nameof(catalog)); @@ -93,12 +102,21 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource // (int.MaxValue); larger values clamp. At most top samples, ContinuationPoint always null: the node // manager pages by re-reading from the last returned timestamp inclusive, which the (ts, tag) dedup // key makes exact. + // The server raw cap (R08:1.1, #736): with a cap, a read asking for everything or for more than the cap reads + // cap + 1 rows — one more than it may return — so an over-cap window is DETECTED and refused + // (HistoryReadLimitExceededException ⇒ BadTooManyOperations) rather than answered truncated: the server's paging + // reads a short answer as the last page, so a silent LIMIT would lose data. The server itself never asks for more + // than the cap (it pages at MaxRawValuesPerRead); this bounds a caller that bypasses it. /// public async Task ReadRawAsync( string fullReference, DateTime startUtc, DateTime endUtc, uint maxValuesPerNode, CancellationToken cancellationToken) { ArgumentNullException.ThrowIfNull(fullReference); - var top = maxValuesPerNode == 0 ? int.MaxValue : (int)Math.Min(maxValuesPerNode, int.MaxValue); + var cap = _maxRawValuesPerRead; + var probe = cap > 0 && (maxValuesPerNode == 0 || maxValuesPerNode > (uint)cap); + var top = probe + ? (int)Math.Min(int.MaxValue, (long)cap + 1) + : maxValuesPerNode == 0 ? int.MaxValue : (int)Math.Min(maxValuesPerNode, int.MaxValue); try { var samples = new List(); @@ -129,9 +147,19 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource } } + if (probe && samples.Count > cap) + throw new HistoryReadLimitExceededException(cap, + $"The raw read of '{fullReference}' holds more than ServerHistorian:MaxRawValuesPerRead ({cap}) samples; " + + "set NumValuesPerNode at or below it to page, or narrow the window."); + RecordSuccess(); return new HistoryReadResult(samples, ContinuationPoint: null); } + catch (HistoryReadLimitExceededException) + { + // A client-request limit (the server raw cap), not a backend fault: not counted as a failure. + throw; + } catch (Exception ex) { RecordFailure(ex); @@ -360,9 +388,11 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource /// public bool SupportsReverseRaw => true; - // 0: a capped raw read is one bounded statement - no backend row ceiling. + // The server raw cap (ServerHistorian:MaxRawValuesPerRead; 0 = none): the server pages every raw read at it, so a + // client's NumValuesPerNode = 0 is served as pages of at most this many values. QuestDB itself has no row ceiling - + // a capped raw read is one bounded statement. /// - public int MaxRawValuesPerRead => 0; + public int MaxRawValuesPerRead => _maxRawValuesPerRead; // Reverse events reads are served natively (ORDER BY ... DESC). /// diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb/QuestDbHistorianProvider.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb/QuestDbHistorianProvider.cs index 36579dfb..34ca413e 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb/QuestDbHistorianProvider.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb/QuestDbHistorianProvider.cs @@ -35,14 +35,17 @@ public sealed class QuestDbHistorianProvider : IHistorianBackendProvider, IAsync /// Creates the provider over the ServerHistorian:QuestDb options (validated elsewhere). /// The ServerHistorian:QuestDb options. /// The shared ServerHistorian:MaxTimeWeightedRawRows cap (<= 0 = unbounded). + /// The shared ServerHistorian:MaxRawValuesPerRead raw cap (<= 0 = unbounded; R08:1.1). public QuestDbHistorianProvider(QuestDbHistorianOptions options, - int maxTimeWeightedRawRows = TimeWeightedAggregator.DefaultMaxWindowRows) + int maxTimeWeightedRawRows = TimeWeightedAggregator.DefaultMaxWindowRows, int maxRawValuesPerRead = 0) { _options = options ?? throw new ArgumentNullException(nameof(options)); _maxTimeWeightedRawRows = maxTimeWeightedRawRows; + _maxRawValuesPerRead = maxRawValuesPerRead; } private readonly int _maxTimeWeightedRawRows; + private readonly int _maxRawValuesPerRead; /// public string Name => "QuestDb"; @@ -75,7 +78,8 @@ public sealed class QuestDbHistorianProvider : IHistorianBackendProvider, IAsync { var s = Get(sp); return new QuestDbHistorianDataSource(s.Conn, s.Bootstrap, s.Catalog, - Loggers(sp).CreateLogger(), maxTimeWeightedRawRows: _maxTimeWeightedRawRows); + Loggers(sp).CreateLogger(), maxTimeWeightedRawRows: _maxTimeWeightedRawRows, + maxRawValuesPerRead: _maxRawValuesPerRead); } /// diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/SqlServerHistorianDataSource.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/SqlServerHistorianDataSource.cs index 07d6887e..6525a173 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/SqlServerHistorianDataSource.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/SqlServerHistorianDataSource.cs @@ -61,6 +61,7 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource private readonly HistorianHealthCounters _health = new(); private readonly int _maxTimeWeightedRawRows; + private readonly int _maxRawValuesPerRead; /// Creates the data source over the provider's shared connection, bootstrapper and tag catalog. /// Connection helper. @@ -69,11 +70,19 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource /// Optional logger. /// Clock for "now" (AtTime beyond the last sample, server timestamps); system by default. /// Raw in-window row cap of one time-weighted read (<= 0 = unbounded). + /// + /// The server raw cap (ServerHistorian:MaxRawValuesPerRead, R08:1.1): the most samples one + /// may return — reported as , so the server pages at it. + /// A read asking for more (or for everything) is refused when the window holds more, never truncated. + /// <= 0 = unbounded (the pre-#736 behaviour). + /// internal SqlServerHistorianDataSource( SqlHistorianConnection conn, SqlServerSchemaBootstrapper bootstrap, SqlServerTagCatalog catalog, ILogger? log = null, - TimeProvider? timeProvider = null, int maxTimeWeightedRawRows = TimeWeightedAggregator.DefaultMaxWindowRows) + TimeProvider? timeProvider = null, int maxTimeWeightedRawRows = TimeWeightedAggregator.DefaultMaxWindowRows, + int maxRawValuesPerRead = 0) { _maxTimeWeightedRawRows = maxTimeWeightedRawRows; + _maxRawValuesPerRead = Math.Max(0, maxRawValuesPerRead); _conn = conn ?? throw new ArgumentNullException(nameof(conn)); _bootstrap = bootstrap ?? throw new ArgumentNullException(nameof(bootstrap)); _catalog = catalog ?? throw new ArgumentNullException(nameof(catalog)); @@ -90,13 +99,22 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource // 0 is the node manager's "unbounded" sentinel; values above clamp. // The node manager pages by re-reading from the last returned timestamp inclusive and trimming it; the // (TagId, SourceTimeUtc) primary key rules out tie clusters, so that is exact. + // The server raw cap (R08:1.1, #736): with a cap, a read asking for everything or for more than the cap reads + // cap + 1 rows — one more than it may return — so an over-cap window is DETECTED and refused + // (HistoryReadLimitExceededException ⇒ BadTooManyOperations) rather than answered truncated: the server's paging + // reads a short answer as the last page, so a silent TOP would lose data. The server itself never asks for more than + // the cap (it pages at MaxRawValuesPerRead); this bounds a caller that bypasses it. // /// public async Task ReadRawAsync( string fullReference, DateTime startUtc, DateTime endUtc, uint maxValuesPerNode, CancellationToken cancellationToken) { ArgumentNullException.ThrowIfNull(fullReference); - var top = maxValuesPerNode == 0 ? int.MaxValue : (int)Math.Min(maxValuesPerNode, int.MaxValue); + var cap = _maxRawValuesPerRead; + var probe = cap > 0 && (maxValuesPerNode == 0 || maxValuesPerNode > (uint)cap); + var top = probe + ? (int)Math.Min(int.MaxValue, (long)cap + 1) + : maxValuesPerNode == 0 ? int.MaxValue : (int)Math.Min(maxValuesPerNode, int.MaxValue); try { var tagId = await ResolveAsync(fullReference, cancellationToken).ConfigureAwait(false); @@ -126,9 +144,19 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource } } + if (probe && samples.Count > cap) + throw new HistoryReadLimitExceededException(cap, + $"The raw read of '{fullReference}' holds more than ServerHistorian:MaxRawValuesPerRead ({cap}) samples; " + + "set NumValuesPerNode at or below it to page, or narrow the window."); + RecordSuccess(); return new HistoryReadResult(samples, ContinuationPoint: null); } + catch (HistoryReadLimitExceededException) + { + // A client-request limit (the server raw cap), not a backend fault: not counted as a failure. + throw; + } catch (Exception ex) { RecordFailure(ex); @@ -319,9 +347,13 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource /// public bool SupportsReverseRaw => true; - // 0: a capped raw read is one bounded statement — no backend row ceiling. + // + // The server raw cap (ServerHistorian:MaxRawValuesPerRead; 0 = none): the server pages every raw read at it, so a + // client's NumValuesPerNode = 0 is served as pages of at most this many values. SQL Server itself has no row + // ceiling — a capped raw read is one bounded statement. + // /// - public int MaxRawValuesPerRead => 0; + public int MaxRawValuesPerRead => _maxRawValuesPerRead; // Reverse events reads are served natively (ORDER BY … DESC). /// diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/SqlServerHistorianProvider.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/SqlServerHistorianProvider.cs index 80cfe919..52884a95 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/SqlServerHistorianProvider.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/SqlServerHistorianProvider.cs @@ -31,14 +31,17 @@ public sealed class SqlServerHistorianProvider : IHistorianBackendProvider /// Creates the provider over the ServerHistorian:SqlServer options (validated elsewhere). /// The ServerHistorian:SqlServer options. /// The shared ServerHistorian:MaxTimeWeightedRawRows cap (<= 0 = unbounded). + /// The shared ServerHistorian:MaxRawValuesPerRead raw cap (<= 0 = unbounded; R08:1.1). public SqlServerHistorianProvider(SqlServerHistorianOptions options, - int maxTimeWeightedRawRows = TimeWeightedAggregator.DefaultMaxWindowRows) + int maxTimeWeightedRawRows = TimeWeightedAggregator.DefaultMaxWindowRows, int maxRawValuesPerRead = 0) { _options = options ?? throw new ArgumentNullException(nameof(options)); _maxTimeWeightedRawRows = maxTimeWeightedRawRows; + _maxRawValuesPerRead = maxRawValuesPerRead; } private readonly int _maxTimeWeightedRawRows; + private readonly int _maxRawValuesPerRead; /// public string Name => "SqlServer"; @@ -69,7 +72,8 @@ public sealed class SqlServerHistorianProvider : IHistorianBackendProvider { var s = Get(sp); return new SqlServerHistorianDataSource(s.Conn, s.Bootstrap, s.Catalog, - Loggers(sp).CreateLogger(), maxTimeWeightedRawRows: _maxTimeWeightedRawRows); + Loggers(sp).CreateLogger(), maxTimeWeightedRawRows: _maxTimeWeightedRawRows, + maxRawValuesPerRead: _maxRawValuesPerRead); } /// diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/HistorianBackendRegistry.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/HistorianBackendRegistry.cs index 221180d2..0fdba329 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/HistorianBackendRegistry.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/HistorianBackendRegistry.cs @@ -45,8 +45,8 @@ public static class HistorianBackendRegistry new Dictionary>(StringComparer.OrdinalIgnoreCase) { [ServerHistorianOptions.GatewayBackendName] = (o, _) => new GatewayHistorianProvider(o), - [SqlServerBackendName] = (o, c) => new SqlServerHistorianProvider(BindSqlServer(c), o.MaxTimeWeightedRawRows), - [QuestDbBackendName] = (o, c) => new QuestDbHistorianProvider(BindQuestDb(c), o.MaxTimeWeightedRawRows), + [SqlServerBackendName] = (o, c) => new SqlServerHistorianProvider(BindSqlServer(c), o.MaxTimeWeightedRawRows, o.EffectiveMaxRawValuesPerRead), + [QuestDbBackendName] = (o, c) => new QuestDbHistorianProvider(BindQuestDb(c), o.MaxTimeWeightedRawRows, o.EffectiveMaxRawValuesPerRead), }; /// The registered backend names, for validation messages. diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/HistoryPaging.cs b/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/HistoryPaging.cs index 36172326..bee16369 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/HistoryPaging.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/HistoryPaging.cs @@ -1,3 +1,5 @@ +using Opc.Ua; +using Opc.Ua.Server; using ZB.MOM.WW.OtOpcUa.Core.Abstractions; namespace ZB.MOM.WW.OtOpcUa.OpcUaServer; @@ -110,24 +112,62 @@ internal enum HistoryCursorKind /// NumValuesPerNode samples with a null continuation point — so paging is synthesised here, /// time-based: /// -/// A page that returns EXACTLY the requested cap (NumValuesPerNode > 0) MAY have -/// more behind it ⇒ emit a continuation point. -/// A short page (fewer than the cap) is the last page ⇒ no continuation point. -/// NumValuesPerNode == 0 means "all values, no limit" (OPC UA Part 11) ⇒ never page; -/// return everything in one shot. +/// A page whose read returned the lookahead row past it MAY have more behind it ⇒ emit a continuation +/// point ( judges a read against what was asked for). +/// A short read (no lookahead row) is the last page ⇒ no continuation point. +/// NumValuesPerNode == 0 means "all values" (OPC UA Part 11) — but not "in one shot": the page is +/// sized to the backend's by +/// (the server raw cap ServerHistorian:MaxRawValuesPerRead, default 10 000, and/or a strict gateway's +/// learned ceiling; review R08:1.1, #736), and a full page carries a continuation point like any other. Only +/// with no limit at all (0, the opt-out) is everything returned in one shot. /// -/// All methods are static + pure so they unit-test without a server, a session, or the SDK. +/// All paging methods are static + pure so they unit-test without a server, a session, or the SDK; +/// is the one SDK-facing member. /// internal static class HistoryPaging { /// - /// Decide whether a just-returned page is "full" and therefore MAY be followed by more data — - /// the signal to emit a continuation point. A page is full when the client asked for a finite - /// cap ( > 0) and the backend returned exactly that many - /// samples. A short page (or an unlimited 0 request) is terminal. + /// Advertise the server raw cap as HistoryServerCapabilities/MaxReturnDataValues (i=11273; Part 11 §5.4.2: + /// "the maximum number of values that can be returned by the Server for each Node", 0 = no limit). The cap is + /// the most values one HistoryRead-Raw page returns (ServerHistorian:MaxRawValuesPerRead), so a client can + /// size its NumValuesPerNode and expect a continuation point past it. Idempotent; a re-sync with a + /// different cap overwrites. Never throws: a missing diagnostics manager or node, or any SDK failure writing the + /// property, leaves the SDK default (0) and returns false for the caller to log — HistoryRead itself still + /// enforces the cap. /// - /// The number of samples the backend returned for this page. - /// The client's per-page cap; 0 means "no limit". + /// The running server. + /// The cap (<= 0 advertises 0 = no limit). + /// true when the property was written. + public static bool AdvertiseMaxReturnDataValues(IServerInternal? server, int maxRawValuesPerRead) + { + if (server?.DiagnosticsNodeManager is not DiagnosticsNodeManager diagnostics) return false; + try + { + lock (diagnostics.Lock) + { + var capabilities = diagnostics.GetDefaultHistoryCapabilities(); + if (capabilities?.MaxReturnDataValues is not { } property) return false; + property.Value = (uint)Math.Max(0, maxRawValuesPerRead); + property.ClearChangeMasks(diagnostics.SystemContext, false); + return true; + } + } + catch (Exception) + { + return false; // advisory only: the caller logs; the cap is enforced by HistoryRead regardless + } + } + + /// + /// Decide whether a just-returned read is "full" and therefore MAY be followed by more data — + /// the signal to emit a continuation point. A read is full when it was issued with a finite request + /// cap ( > 0 — the RawPagePlan.RequestCap: page + boundary ties + + /// lookahead, which under a backend limit or the server raw cap is always finite, uncapped client reads included) + /// and the backend returned at least that many samples. A short read is terminal; so is a read issued with a + /// request cap of 0, which only happens with no limit at all (one shot of the whole window). + /// + /// The number of samples the backend returned for this read. + /// The request cap the read was issued with; 0 means "no limit". /// true when a continuation point should be emitted; otherwise false. public static bool IsFullPage(int returnedCount, uint numValuesPerNode) => numValuesPerNode > 0 && returnedCount >= numValuesPerNode; diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/RawPageLimit.cs b/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/RawPageLimit.cs index 3adb40db..e4af5d11 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/RawPageLimit.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/RawPageLimit.cs @@ -11,17 +11,22 @@ internal readonly record struct RawPagePlan(int Limit, uint PageCap, uint Reques /// /// sizes a HistoryRead-Raw page to what the backend can return in one read -/// ( = L — a strict HistorianGateway's learned row ceiling, less -/// its boundary-row allowance). With a limit the read — page P + boundary ties k + the lookahead row — is at most L, -/// so P = min(want, L − k − 1) (all values: P = L − k − 1); a full page carries a continuation point like any other, as -/// Part 11 allows fewer values than NumValuesPerNode with one. Previously a cap above a strict gateway's ceiling -/// (or an uncapped read of a window holding more) failed BadTooManyOperations. +/// ( = L — the server raw cap +/// ServerHistorian:MaxRawValuesPerRead, default 10 000, which every backend reports (review R08:1.1, #736), +/// and/or a strict HistorianGateway's learned row ceiling less its boundary-row allowance, whichever is lower). With +/// a limit the read — page P + boundary ties k + the lookahead row — is at most L, so P = min(want, L − k − 1) (all +/// values: P = L − k − 1, i.e. a first page of 9 999 at the default cap); a full page carries a continuation point like +/// any other, as Part 11 allows fewer values than NumValuesPerNode with one. Previously a cap above a strict +/// gateway's ceiling (or an uncapped read of a window holding more) failed BadTooManyOperations, and before +/// #736 an uncapped read on a backend without a ceiling read the whole window in one shot. /// /// /// /// Never a silent truncation. The backend still answers a read in full or refuses it; only the server's page is /// smaller, and a page is judged full against what was asked for. A read the backend cannot serve at all (a tie -/// cluster larger than one read) stays BadTooManyOperations. +/// cluster larger than one read) stays BadTooManyOperations. The backends enforce the same cap themselves +/// (a read asking for more than L, or for everything, is refused when the window holds more than L rows), so this +/// planner is what keeps the server's own reads inside it: with L > 0 no read it plans ever asks for 0 or more than L. /// /// /// One re-read, planned again. A refused read () is re-planned and diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Historian/ServerHistorianOptions.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Historian/ServerHistorianOptions.cs index d19dbe38..ba2fe9c7 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Historian/ServerHistorianOptions.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Historian/ServerHistorianOptions.cs @@ -154,6 +154,42 @@ public sealed class ServerHistorianOptions /// public int MaxTimeWeightedRawRows { get; init; } = 1_000_000; + /// The default of : 10 000 values per raw page / backend read. + public const int DefaultMaxRawValuesPerRead = 10_000; + + /// + /// The smallest usable : a resumed page's read must carry at least one value, the + /// boundary tie it re-reads and trims, AND the lookahead row that tells the server more data follows. At 2 a resumed + /// page has no room at all and every value costs an empty page with a continuation point (the #692 defect, re-created + /// by configuration), so 1 and 2 are raised to 3 (). + /// + public const int MinMaxRawValuesPerRead = 3; + + /// + /// Every backend: the most raw values the server returns on ONE HistoryRead-Raw page, and the most rows one + /// backend raw read may materialise — the server's own row ceiling (review R08:1.1, #736). A client's + /// NumValuesPerNode = 0 ("all values") no longer reads a whole window into memory: it is served as pages of at + /// most this many values, each with a continuation point (Part 11 allows fewer values than requested with one), through + /// the same tie-safe paging a capped read uses (RawPageLimit: page + boundary ties + lookahead ≤ cap, so a first + /// page holds cap − 1 values). A NumValuesPerNode above it is paged at it too. Advertised as + /// HistoryServerCapabilities/MaxReturnDataValues. The backends' own TOP / LIMIT / max_values + /// are bounded by it independently: a raw read asking for more than this (or for everything) reads at most cap + 1 rows + /// and is REFUSED (BadTooManyOperations) when the window holds more — never answered truncated — so a caller that + /// bypasses the node manager's paging cannot bypass the bound. On the Gateway backend a strict gateway's learned row + /// ceiling (less its boundary-row allowance) still applies when lower. The tie-cluster bound of a resumed page is also + /// at most cap − 1 (as under a gateway ceiling; above that is inert), and a + /// page whose boundary tie cluster is that large steps past it with one empty page. Default 10 000. Non-positive = + /// unbounded, the pre-#736 behaviour (a warning); below + /// (3) is raised to it. + /// + public int MaxRawValuesPerRead { get; init; } = DefaultMaxRawValuesPerRead; + + /// + /// as the backends and the node manager apply it: 0 (unbounded) when + /// non-positive, otherwise at least . The one place the rule lives. + /// + public int EffectiveMaxRawValuesPerRead => MaxRawValuesPerRead <= 0 ? 0 : Math.Max(MinMaxRawValuesPerRead, MaxRawValuesPerRead); + /// /// how far into the future an OPC UA HistoryUpdate (Insert / Replace / Update) item's /// SourceTimestamp may lie, measured from the server clock. An item beyond now + skew is refused @@ -208,6 +244,10 @@ public sealed class ServerHistorianOptions warnings.Add($"ServerHistorian:MaxConcurrentHistoryReadBatches is {MaxConcurrentHistoryReadBatches} — must be > 0; the process-wide HistoryRead batch limiter is disabled."); if (MaxTimeWeightedRawRows <= 0) warnings.Add($"ServerHistorian:MaxTimeWeightedRawRows is {MaxTimeWeightedRawRows} — must be > 0; time-weighted aggregate reads on the SqlServer/QuestDb backends stream an unbounded number of raw samples."); + if (MaxRawValuesPerRead <= 0) + warnings.Add($"ServerHistorian:MaxRawValuesPerRead is {MaxRawValuesPerRead} — must be > 0; HistoryRead-Raw pages and backend raw reads are unbounded, so a client asking for a whole window (NumValuesPerNode = 0) materialises every sample in memory."); + else if (MaxRawValuesPerRead < MinMaxRawValuesPerRead) + warnings.Add($"ServerHistorian:MaxRawValuesPerRead is {MaxRawValuesPerRead} — must be at least {MinMaxRawValuesPerRead} (a value, its boundary tie and the lookahead row); treated as {MinMaxRawValuesPerRead}."); if (EventReadHorizon < TimeSpan.Zero) warnings.Add($"ServerHistorian:EventReadHorizon is {EventReadHorizon} — negative is treated as no horizon (00:00:00); HistoryReadEvents reads every window in full."); if (HistoryReadDeadline <= TimeSpan.Zero) diff --git a/tests/Drivers/Historian.Conformance/IConformanceBackend.cs b/tests/Drivers/Historian.Conformance/IConformanceBackend.cs index afc9f69f..970e7c77 100644 --- a/tests/Drivers/Historian.Conformance/IConformanceBackend.cs +++ b/tests/Drivers/Historian.Conformance/IConformanceBackend.cs @@ -22,9 +22,14 @@ public interface IConformanceBackend : IAsyncDisposable /// Its "now"; null means the system clock. IHistorianDataSource CreateDataSource(TimeProvider? clock = null); - /// A fresh data source whose time-weighted raw-row cap (ServerHistorian:MaxTimeWeightedRawRows) is . - /// The cap (<= 0 = unbounded). - IHistorianDataSource CreateDataSource(int maxTimeWeightedRawRows); + /// + /// A fresh data source whose time-weighted raw-row cap (ServerHistorian:MaxTimeWeightedRawRows) is + /// and whose server raw cap (ServerHistorian:MaxRawValuesPerRead, + /// R08:1.1 / #736) is . + /// + /// The time-weighted cap (<= 0 = unbounded). + /// The raw cap (<= 0 = unbounded, the default). + IHistorianDataSource CreateDataSource(int maxTimeWeightedRawRows, int maxRawValuesPerRead = 0); /// A fresh live-value writer over this backend's store. /// The clock used to stamp values that carry no timestamp; null means the system clock. diff --git a/tests/Drivers/Historian.Conformance/RawReadConformance.cs b/tests/Drivers/Historian.Conformance/RawReadConformance.cs index 8a04adc6..39c1d79f 100644 --- a/tests/Drivers/Historian.Conformance/RawReadConformance.cs +++ b/tests/Drivers/Historian.Conformance/RawReadConformance.cs @@ -25,6 +25,43 @@ public abstract class RawReadConformance : ConformanceBase r.ContinuationPoint.ShouldBeNull(); } + /// + /// R08:1.1 (#736): the server raw cap bounds the backend's own TOP / LIMIT. A read asking for everything + /// (or for more than the cap) reads at most cap + 1 rows and is refused when the window holds more — never answered + /// truncated (a short answer is what the server's paging reads as "the last page") — and is served in full when it fits. + /// A read at or under the cap is unchanged. The refusal is a client-request limit, not a backend fault. + /// + [Fact] + public async Task The_server_raw_cap_bounds_every_read_and_refuses_an_over_cap_window_rather_than_truncating() + { + await ConformanceData.SeedAAsync(B); // five samples at +0, +10, +20, +30, +40 + var src = B.CreateDataSource(maxTimeWeightedRawRows: 1_000_000, maxRawValuesPerRead: 3); + src.MaxRawValuesPerRead.ShouldBe(3); + + // Five rows under a cap of three: everything / above the cap ⇒ refused; at the cap ⇒ the first three; under ⇒ as asked. + var refused = await Should.ThrowAsync(() => src.ReadRawAsync("A", At(-1), At(100), 0, Ct)); + refused.Limit.ShouldBe(3); + await Should.ThrowAsync(() => src.ReadRawAsync("A", At(-1), At(100), 4, Ct)); + await Should.ThrowAsync(() => src.ReadRawAsync("A", At(-1), At(100), uint.MaxValue, Ct)); + Values(await src.ReadRawAsync("A", At(-1), At(100), 3, Ct)).ShouldBe([10d, 20d, 30d]); + Values(await src.ReadRawAsync("A", At(-1), At(100), 2, Ct)).ShouldBe([10d, 20d]); + + // Three rows fit: everything / above the cap are served in full. + Values(await src.ReadRawAsync("A", At(10), At(40), 0, Ct)).ShouldBe([20d, 30d, 40d]); + Values(await src.ReadRawAsync("A", At(10), At(40), 4, Ct)).ShouldBe([20d, 30d, 40d]); + + // Reverse reads are bounded the same way, keeping the LATEST rows. + await Should.ThrowAsync(() => src.ReadRawAsync("A", At(100), At(-1), 0, Ct)); + Values(await src.ReadRawAsync("A", At(100), At(-1), 3, Ct)).ShouldBe([50d, 40d, 30d]); + Values(await src.ReadRawAsync("A", At(40), At(10), 0, Ct)).ShouldBe([50d, 40d, 30d]); + + // A point read and an empty window are unaffected. + Values(await src.ReadRawAsync("A", At(10), At(10), 0, Ct)).ShouldBe([20d]); + (await src.ReadRawAsync("A", At(50), At(60), 0, Ct)).Samples.ShouldBeEmpty(); + + src.GetHealthSnapshot().TotalFailures.ShouldBe(0, "a refused over-cap read is a client-request limit, not a backend fault"); + } + [Fact] public async Task Forward_read_honours_the_limit() { diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests/GatewayRawCapTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests/GatewayRawCapTests.cs new file mode 100644 index 00000000..c1c5da7f --- /dev/null +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests/GatewayRawCapTests.cs @@ -0,0 +1,305 @@ +using Google.Protobuf.WellKnownTypes; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging.Abstractions; +using Opc.Ua; +using Opc.Ua.Server; +using Shouldly; +using Xunit; +using ZB.MOM.WW.HistorianGateway.Contracts.Grpc; +using ZB.MOM.WW.OtOpcUa.Commons.OpcUa; +using ZB.MOM.WW.OtOpcUa.Core.Abstractions; +using ZB.MOM.WW.OtOpcUa.OpcUaServer; +using ZB.MOM.WW.OtOpcUa.Runtime.Historian; +using SdkHistoryReadResult = Opc.Ua.HistoryReadResult; + +namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests; + +/// +/// Review R08:1.1 (#736): the server's raw cap (ServerHistorian:MaxRawValuesPerRead) bounds the Gateway +/// backend's max_values independently of the node manager. An uncapped raw read (NumValuesPerNode = 0) +/// used to be sent as and materialise the whole window twice (the protobuf list and the +/// snapshot list); a read above the cap now asks for cap + 1 rows and is REFUSED (, +/// BadTooManyOperations) when the window holds more — never answered truncated, because a short answer is what the +/// server's paging reads as "the last page". The data source reports the cap as its +/// (or a strict gateway's lower learned ceiling), so the node +/// manager's existing ceiling paging serves NumValuesPerNode = 0 as pages of at most the cap, each with a +/// continuation point. +/// +public sealed class GatewayRawCapTests : IDisposable +{ + private static CancellationToken Ct => TestContext.Current.CancellationToken; + + private static readonly DateTime T0 = new(2026, 10, 22, 10, 0, 0, DateTimeKind.Utc); + + private const int Cap = 1000; + + private readonly string _pkiRoot = Path.Combine(Path.GetTempPath(), $"otopcua-gwrawcap-{Guid.NewGuid():N}"); + + private static DateTime At(int seconds) => T0.AddSeconds(seconds); + + private static HistorianSample Row(DateTime t, double v, uint quality = 0) => + new() { Tag = "T", Timestamp = Timestamp.FromDateTime(t), NumericValue = v, OpcQuality = 192, Quality = quality }; + + /// AVEVA-shaped: a boundary row at the query start, [start, end] inclusive, capped at max. + private static FakeHistorianGatewayClient Store(int count, int? ceiling = null) + { + var stored = Enumerable.Range(0, count).Select(i => (T: At(i), V: (double)i)).ToList(); + return new FakeHistorianGatewayClient + { + MaxValuesCeiling = ceiling, + RawSource = (start, end, max) => + { + if (max <= 0) throw new ArgumentOutOfRangeException(nameof(max), "max_values must be positive"); // the client package + var rows = new List(); + if (stored.All(s => s.T != start)) + { + var prior = stored.LastOrDefault(s => s.T < start); + rows.Add(prior.T == default + ? new HistorianSample { Tag = "T", Timestamp = Timestamp.FromDateTime(start), NumericValue = 0, OpcQuality = 0, Quality = 1 } + : Row(start, prior.V, 133)); + } + rows.AddRange(stored.Where(s => s.T >= start && s.T <= end).Select(s => Row(s.T, s.V))); + return rows.Take(max); + }, + }; + } + + private static GatewayHistorianDataSource Source(FakeHistorianGatewayClient fake, int cap = Cap) => + new(fake, NullLogger.Instance, maxRawValuesPerRead: cap); + + private static int Probe => Cap + 1 + RawAggregateReader.PreWindowAllowance; // cap + 1 rows + the boundary-row allowance + + // ---- the data source alone ---------------------------------------------------------------------------------------- + + [Fact] + public void Without_a_cap_the_source_is_unchanged() + { + new GatewayHistorianDataSource(Store(10), NullLogger.Instance).MaxRawValuesPerRead.ShouldBe(0); + Source(Store(10), cap: 0).MaxRawValuesPerRead.ShouldBe(0); + } + + [Fact] + public async Task The_cap_is_the_page_limit_until_a_lower_gateway_ceiling_is_learned() + { + Source(Store(3600)).MaxRawValuesPerRead.ShouldBe(Cap); + + // A strict gateway whose ceiling is ABOVE the cap: the cap wins, nothing is refused by the gateway. + var higher = Store(3600, ceiling: 2000); + var ds = Source(higher); + await Should.ThrowAsync(() => ds.ReadRawAsync("T", At(0), At(3600), 0, Ct)); + higher.RawCalls.ShouldAllBe(c => c.MaxValues <= 2000); + ds.MaxRawValuesPerRead.ShouldBe(Cap); + + // A strict gateway whose ceiling is BELOW the cap: the learned ceiling (less the allowance) wins once known. + var lower = Store(3600, ceiling: 500); + var ds2 = Source(lower); + ds2.MaxRawValuesPerRead.ShouldBe(Cap); + await Should.ThrowAsync(() => ds2.ReadRawAsync("T", At(0), At(3600), 0, Ct)); + ds2.MaxRawValuesPerRead.ShouldBe(500 - RawAggregateReader.PreWindowAllowance); + } + + [Fact] + public async Task An_uncapped_read_over_the_cap_is_refused_never_truncated_and_never_asks_for_everything() + { + var fake = Store(3600); + var ds = Source(fake); + + var ex = await Should.ThrowAsync(() => ds.ReadRawAsync("T", At(0), At(3600), 0, Ct)); + + ex.Limit.ShouldBe(Cap); + ex.Message.ShouldContain("MaxRawValuesPerRead"); + fake.RawCalls.Select(c => c.MaxValues).ShouldBe([Probe]); // never int.MaxValue + fake.RawStreamsCompleted.ShouldBe(1, "the bounded stream is drained, never abandoned"); + ds.GetHealthSnapshot().TotalFailures.ShouldBe(0, "a client-request limit is not a backend fault"); + } + + [Theory] + [InlineData(10)] + [InlineData(Cap - 1)] + [InlineData(Cap)] + public async Task An_uncapped_read_that_fits_the_cap_is_served_in_full(int count) + { + var fake = Store(count); + var r = await Source(fake).ReadRawAsync("T", At(0), At(3600), 0, Ct); + + r.Samples.Select(s => s.Value).ShouldBe(Enumerable.Range(0, count).Select(i => (object?)(double)i)); + fake.RawCalls.Select(c => c.MaxValues).ShouldBe([Probe]); + } + + [Theory] + [InlineData(Cap + 1u)] + [InlineData(5000u)] + [InlineData(uint.MaxValue)] + public async Task A_client_cap_above_the_server_cap_is_bounded_by_it(uint n) + { + await Should.ThrowAsync(() => Source(Store(3600)).ReadRawAsync("T", At(0), At(3600), n, Ct)); + + var fake = Store(900); + var r = await Source(fake).ReadRawAsync("T", At(0), At(3600), n, Ct); + r.Samples.Count.ShouldBe(900); + fake.RawCalls.ShouldAllBe(c => c.MaxValues == Probe); + } + + [Theory] + [InlineData(Cap)] + [InlineData(Cap - 1)] + [InlineData(1)] + public async Task A_client_cap_at_or_under_the_server_cap_reads_as_before(int n) + { + var fake = Store(3600); + var r = await Source(fake).ReadRawAsync("T", At(0), At(3600), (uint)n, Ct); + + r.Samples.Count.ShouldBe(n); + fake.RawCalls.Select(c => c.MaxValues).ShouldBe([n + RawAggregateReader.PreWindowAllowance]); + } + + [Fact] + public async Task A_point_read_and_an_empty_window_are_unaffected() + { + var fake = Store(3600); + var ds = Source(fake); + (await ds.ReadRawAsync("T", At(5), At(5), 0, Ct)).Samples.Single().Value.ShouldBe(5d); + (await ds.ReadRawAsync("T", At(5000), At(6000), 0, Ct)).Samples.ShouldBeEmpty(); + fake.RawCalls.ShouldAllBe(c => c.MaxValues == Probe); + } + + // ---- the factory binding -------------------------------------------------------------------------------------------- + + [Theory] + [InlineData(777, 777)] + [InlineData(0, 0)] + [InlineData(-1, 0)] + [InlineData(1, ServerHistorianOptions.MinMaxRawValuesPerRead)] + public void The_factory_binds_the_effective_option(int configured, int expected) + { + var options = new ServerHistorianOptions + { + Endpoint = "http://localhost:5222", UseTls = false, ApiKey = "histgw_test_secret", MaxRawValuesPerRead = configured, + }; + using var ds = GatewayHistorian.CreateDataSource(options, new ServiceCollection().BuildServiceProvider()); + ds.ShouldBeOfType().MaxRawValuesPerRead.ShouldBe(expected); + } + + [Fact] + public void The_default_option_caps_the_source_at_ten_thousand() + { + var options = new ServerHistorianOptions { Endpoint = "http://localhost:5222", UseTls = false, ApiKey = "histgw_test_secret" }; + using var ds = new GatewayHistorianProvider(options).CreateDataSource(new ServiceCollection().BuildServiceProvider()); + ds.MaxRawValuesPerRead.ShouldBe(ServerHistorianOptions.DefaultMaxRawValuesPerRead); + } + + // ---- through the node manager --------------------------------------------------------------------------------------- + + [Theory] + [InlineData(0u, false)] + [InlineData(0u, true)] + [InlineData(5000u, false)] + [InlineData(uint.MaxValue, false)] + public async Task An_uncapped_HistoryRead_pages_at_the_cap_with_continuation_points(uint n, bool reverse) + { + var fake = Store(3600); + var (host, nm, nodeId) = await BootAsync(fake); + try + { + var (values, pages, status) = reverse + ? DrainFrom(nm, nodeId, At(3600), At(-1), n, null) + : DrainFrom(nm, nodeId, At(0), At(3600), n, null); + + status.Code.ShouldBe(StatusCodes.Good); + var expected = Enumerable.Range(0, 3600).Select(i => (double)i); + values.ShouldBe(reverse ? expected.Reverse() : expected); + pages.Count.ShouldBeGreaterThan(1, "a year-sized window at N = 0 is paged, not one shot"); + pages.ShouldAllBe(p => p > 0 && p <= Cap); + pages[0].ShouldBe(Cap - 1); // page + lookahead fills a read + fake.RawCalls.ShouldAllBe(c => c.MaxValues > 0 && c.MaxValues <= Cap + RawAggregateReader.PreWindowAllowance); + } + finally { await host.DisposeAsync(); } + } + + [Fact] + public async Task A_bounded_uncapped_HistoryRead_pages_within_the_cap() + { + var fake = Store(3600); + var (host, nm, nodeId) = await BootAsync(fake); + try + { + var (values, pages, status) = DrainFrom(nm, nodeId, At(10).AddMilliseconds(500), At(3000).AddMilliseconds(500), 0, null, returnBounds: true); + status.Code.ShouldBe(StatusCodes.Good); + values.ShouldBe(Enumerable.Range(10, 2992).Select(i => (double)i)); // start bound +10, data +11..+3000, end bound +3001 + pages.ShouldAllBe(p => p <= Cap); + fake.RawCalls.ShouldAllBe(c => c.MaxValues > 0 && c.MaxValues <= Cap + RawAggregateReader.PreWindowAllowance); + } + finally { await host.DisposeAsync(); } + } + + // ---- helpers --------------------------------------------------------------------------------------------------------- + + private async Task<(OpcUaApplicationHost Host, OtOpcUaNodeManager Nm, NodeId NodeId)> BootAsync(FakeHistorianGatewayClient fake) + { + var started = await OpcUaHostStart.StartAsync( + port => new OpcUaApplicationHost( + new OpcUaApplicationHostOptions + { + ApplicationName = "OtOpcUa.GatewayRawCapTest", + ApplicationUri = $"urn:OtOpcUa.GatewayRawCapTest:{Guid.NewGuid():N}", + OpcUaPort = port, + PublicHostname = "localhost", + PkiStoreRoot = _pkiRoot, + }, + NullLogger.Instance), + () => new OtOpcUaSdkServer(), Ct); + var nm = started.Server.NodeManager!; + nm.HistoryContinuationStore = new InMemoryHistoryContinuationStore(); + nm.HistorianDataSource = Source(fake); + nm.EnsureVariable("eq-1/pv", parentFolderNodeId: null, displayName: "PV", dataType: "Double", + writable: false, historianTagname: "T", realm: AddressSpaceRealm.Uns); + return (started.Host, nm, nm.TryGetVariable("eq-1/pv")!.NodeId); + } + + private static (List Values, List Pages, StatusCode Status) DrainFrom( + OtOpcUaNodeManager nm, NodeId nodeId, DateTime start, DateTime end, uint max, byte[]? cp, int maxPages = 20, + bool returnBounds = false) + { + var values = new List(); + var pages = new List(); + ServiceResult error; + do + { + (var page, error, cp) = ReadRaw(nm, nodeId, start, end, max, cp, returnBounds); + values.AddRange(page); + pages.Add(page.Count); + pages.Count.ShouldBeLessThanOrEqualTo(maxPages, "paging must terminate"); + } + while (cp is not null && StatusCode.IsGood(error.StatusCode)); + return (values, pages, error.StatusCode); + } + + private static (IReadOnlyList Values, ServiceResult Error, byte[]? Cp) ReadRaw( + OtOpcUaNodeManager nm, NodeId nodeId, DateTime start, DateTime end, uint max, byte[]? inboundCp, bool returnBounds = false) + { + var context = new OperationContext(new RequestHeader(), secureChannelContext: null, RequestType.HistoryRead, identity: null); + var details = new ReadRawModifiedDetails + { + ReturnBounds = returnBounds, IsReadModified = false, StartTime = start, EndTime = end, NumValuesPerNode = max, + }; + var nodesToRead = new List { new() { NodeId = nodeId, ContinuationPoint = inboundCp } }; + var results = new List { null! }; + var errors = new List { null! }; + nm.HistoryRead(context, details, TimestampsToReturn.Both, releaseContinuationPoints: false, nodesToRead, results, errors); + + var values = new List(); + if (results[0]?.HistoryData is { } ho && !ExtensionObject.IsNull(ho)) + foreach (var dv in ((HistoryData)ExtensionObject.ToEncodeable(ho)).DataValues) + values.Add(Convert.ToDouble(dv.Value)); + return (values, errors[0], results[0]?.ContinuationPoint); + } + + public void Dispose() + { + if (Directory.Exists(_pkiRoot)) + { + try { Directory.Delete(_pkiRoot, recursive: true); } + catch { /* best-effort */ } + } + } +} diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb.IntegrationTests/QuestDbConformanceBackend.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb.IntegrationTests/QuestDbConformanceBackend.cs index 5c90931d..7c8857f0 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb.IntegrationTests/QuestDbConformanceBackend.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb.IntegrationTests/QuestDbConformanceBackend.cs @@ -110,8 +110,9 @@ internal sealed class QuestDbConformanceBackend : IConformanceBackend new QuestDbHistorianDataSource(Conn, Boot, NewCatalog(), timeProvider: clock); /// - public IHistorianDataSource CreateDataSource(int maxTimeWeightedRawRows) => - new QuestDbHistorianDataSource(Conn, Boot, NewCatalog(), maxTimeWeightedRawRows: maxTimeWeightedRawRows); + public IHistorianDataSource CreateDataSource(int maxTimeWeightedRawRows, int maxRawValuesPerRead = 0) => + new QuestDbHistorianDataSource(Conn, Boot, NewCatalog(), maxTimeWeightedRawRows: maxTimeWeightedRawRows, + maxRawValuesPerRead: maxRawValuesPerRead); /// /// The T4 live-value writer over a fresh catalog (fresh positive cache) and this instance's WAL wait. diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb.Tests/QuestDbHistorianProviderTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb.Tests/QuestDbHistorianProviderTests.cs index c67a9b03..cf950e99 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb.Tests/QuestDbHistorianProviderTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb.Tests/QuestDbHistorianProviderTests.cs @@ -47,6 +47,21 @@ public sealed class QuestDbHistorianProviderTests p.SupportedKinds.Count.ShouldBe(6); } + /// R08:1.1 (#736): the server raw cap the Host binds reaches the data source; without one it is unbounded as before. + [Fact] + public void The_server_raw_cap_reaches_the_data_source() + { + var sp = new NullSp(); + using var plain = Make(); + plain.CreateDataSource(sp).MaxRawValuesPerRead.ShouldBe(0); + + using var capped = new QuestDbHistorianProvider(new QuestDbHistorianOptions + { + ConnectionString = "Host=localhost;Port=18812;Username=admin;Password=quest;Database=qdb;ServerCompatibilityMode=NoTypeLoading", + }, maxRawValuesPerRead: 2500); + capped.CreateDataSource(sp).MaxRawValuesPerRead.ShouldBe(2500); + } + [Fact] public void Creates_all_five_seams_without_touching_questdb() { diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer.IntegrationTests/ReadTestDb.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer.IntegrationTests/ReadTestDb.cs index 0fd95144..ce1cefd9 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer.IntegrationTests/ReadTestDb.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer.IntegrationTests/ReadTestDb.cs @@ -45,8 +45,9 @@ internal sealed class ReadTestDb /// A fresh data source (fresh catalog cache) over this schema; is its "now". public SqlServerHistorianDataSource Source(TimeProvider? clock = null, - int maxTimeWeightedRawRows = TimeWeightedAggregator.DefaultMaxWindowRows) => - new(Conn, Boot, new SqlServerTagCatalog(Conn, Boot), timeProvider: clock, maxTimeWeightedRawRows: maxTimeWeightedRawRows); + int maxTimeWeightedRawRows = TimeWeightedAggregator.DefaultMaxWindowRows, int maxRawValuesPerRead = 0) => + new(Conn, Boot, new SqlServerTagCatalog(Conn, Boot), timeProvider: clock, maxTimeWeightedRawRows: maxTimeWeightedRawRows, + maxRawValuesPerRead: maxRawValuesPerRead); /// Inserts a tag row and the given samples. ServerTimeUtc = source time + 1 s (so tests can tell them apart). public async Task SeedAsync(string tag, params (DateTime T, HistorianValue V, uint Status)[] samples) diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer.IntegrationTests/SqlServerConformanceBackend.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer.IntegrationTests/SqlServerConformanceBackend.cs index 69afa55b..8c5ecc94 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer.IntegrationTests/SqlServerConformanceBackend.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer.IntegrationTests/SqlServerConformanceBackend.cs @@ -26,7 +26,8 @@ internal sealed class SqlServerConformanceBackend : IConformanceBackend public IHistorianDataSource CreateDataSource(TimeProvider? clock = null) => _db.Source(clock); /// - public IHistorianDataSource CreateDataSource(int maxTimeWeightedRawRows) => _db.Source(null, maxTimeWeightedRawRows); + public IHistorianDataSource CreateDataSource(int maxTimeWeightedRawRows, int maxRawValuesPerRead = 0) => + _db.Source(null, maxTimeWeightedRawRows, maxRawValuesPerRead); /// public IHistorianValueWriter CreateValueWriter(Func? utcNow = null) => diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer.Tests/SqlServerHistorianProviderTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer.Tests/SqlServerHistorianProviderTests.cs index f3ab5176..668c76f7 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer.Tests/SqlServerHistorianProviderTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer.Tests/SqlServerHistorianProviderTests.cs @@ -47,6 +47,20 @@ public sealed class SqlServerHistorianProviderTests p.SupportedKinds.Count.ShouldBe(6); } + /// R08:1.1 (#736): the server raw cap the Host binds reaches the data source; without one it is unbounded as before. + [Fact] + public void The_server_raw_cap_reaches_the_data_source() + { + var sp = new NullSp(); + Make().CreateDataSource(sp).MaxRawValuesPerRead.ShouldBe(0); + + var capped = new SqlServerHistorianProvider(new SqlServerHistorianOptions + { + ConnectionString = "Server=localhost;User Id=sa;Password=x;TrustServerCertificate=True", + }, maxRawValuesPerRead: 2500); + capped.CreateDataSource(sp).MaxRawValuesPerRead.ShouldBe(2500); + } + [Fact] public void Creates_all_five_seams_without_touching_sql() { diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.Tests/HistoryServerCapabilitiesAdvertisementTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.Tests/HistoryServerCapabilitiesAdvertisementTests.cs new file mode 100644 index 00000000..0aaca5b1 --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.Tests/HistoryServerCapabilitiesAdvertisementTests.cs @@ -0,0 +1,58 @@ +using Opc.Ua; +using Opc.Ua.Server; +using Shouldly; +using Xunit; + +namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.Tests; + +/// +/// R08:1.1 (#736): the server raw cap (ServerHistorian:MaxRawValuesPerRead) is advertised as +/// HistoryServerCapabilities/MaxReturnDataValues (i=11273) through , +/// read back the way a client would — through the SDK master node manager — over a real server. +/// +public sealed class HistoryServerCapabilitiesAdvertisementTests(BoundsServerFixture fx) : IClassFixture +{ + private async Task ReadAdvertisedAsync() + { + var context = new OperationContext(new RequestHeader(), secureChannelContext: null, RequestType.Read, identity: null); + var nodesToRead = new ReadValueIdCollection + { + new ReadValueId { NodeId = VariableIds.HistoryServerCapabilities_MaxReturnDataValues, AttributeId = Attributes.Value }, + }; + var (values, _) = await fx.Nm.Server.NodeManager.ReadAsync(context, 0, TimestampsToReturn.Neither, nodesToRead, TestContext.Current.CancellationToken); + values.Count.ShouldBe(1); + StatusCode.IsGood(values[0].StatusCode).ShouldBeTrue(values[0].StatusCode.ToString()); + return (uint)values[0].Value; + } + + [Fact] + public async Task The_cap_is_advertised_as_MaxReturnDataValues_and_a_re_sync_overwrites_it() + { + await fx.Gate.WaitAsync(TestContext.Current.CancellationToken); + try + { + HistoryPaging.AdvertiseMaxReturnDataValues(fx.Nm.Server, 10_000).ShouldBeTrue(); + (await ReadAdvertisedAsync()).ShouldBe(10_000u); + + HistoryPaging.AdvertiseMaxReturnDataValues(fx.Nm.Server, 2_500).ShouldBeTrue(); + (await ReadAdvertisedAsync()).ShouldBe(2_500u); + + // Unbounded (the opt-out) advertises Part 11's "no limit". + HistoryPaging.AdvertiseMaxReturnDataValues(fx.Nm.Server, 0).ShouldBeTrue(); + (await ReadAdvertisedAsync()).ShouldBe(0u); + HistoryPaging.AdvertiseMaxReturnDataValues(fx.Nm.Server, -5).ShouldBeTrue(); + (await ReadAdvertisedAsync()).ShouldBe(0u); + } + finally + { + HistoryPaging.AdvertiseMaxReturnDataValues(fx.Nm.Server, 0); + fx.Gate.Release(); + } + } + + [Fact] + public void No_server_is_a_quiet_false() + { + HistoryPaging.AdvertiseMaxReturnDataValues(null, 10_000).ShouldBeFalse(); + } +} diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.Tests/NodeManagerHistoryReadRawCapFuzzTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.Tests/NodeManagerHistoryReadRawCapFuzzTests.cs new file mode 100644 index 00000000..6a9e78cd --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.Tests/NodeManagerHistoryReadRawCapFuzzTests.cs @@ -0,0 +1,436 @@ +using Opc.Ua; +using Opc.Ua.Server; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Core.Abstractions; +using HistorianRead = ZB.MOM.WW.OtOpcUa.Core.Abstractions.HistoryReadResult; +using SdkHistoryReadResult = Opc.Ua.HistoryReadResult; + +namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.Tests; + +/// +/// R08:1.1 (#736): the paged-vs-unpaged fuzz for the server raw cap. Every paging change in this repo has had boundary +/// bugs that the hand-written cases missed (silent truncation, an endless cursor, an empty page between every value), so +/// the cap is proved the only way that has worked: random series (bursts, gaps, sub-millisecond stamps, tie clusters), +/// random windows (empty, on a tie, one step past a tie, between samples, forward / reverse / end-only, with and without +/// bounds) and random caps — NumValuesPerNode 0 / 1 / 2 / small / huge, the server cap 3 … 1000 or off, the +/// tie-cluster budget 3 / 8 / 65536 — paged through a REAL SDK node manager against a wire-faithful fake of each backend +/// (SqlServer: tick precision, native reverse, TOP; QuestDB: µs truncation of bounds AND stamps, native reverse, +/// LIMIT; Gateway: forward only, [start, end) after its own boundary-row filtering, walked back by the +/// server) that enforces the backend half of the cap exactly as the real sources do (a read asking for 0 or more than +/// the cap reads cap + 1 rows and is refused when the window holds more). +/// +/// The oracle is the same node manager's one-shot answer with the cap OFF (NumValuesPerNode = 0, no limit), +/// which for a plain read is also checked against the series itself. The concatenation of the pages must equal it +/// exactly; the read must terminate within the number of values plus two pages; no page may exceed the cap; the fake +/// must never see a request for 0 or more than the cap (the "bounded by the same cap" invariant); and a +/// BadTooManyOperations is accepted only when a tie cluster (or, on the Gateway shape, one millisecond) is +/// larger than the budget that the cap and MaxTieClusterOverfetch leave — after an exact prefix. The run is +/// non-vacuous by construction: it asserts that empty windows, mid-tie windows, reverse reads, NumValuesPerNode +/// of 1 and 2, the smallest server cap (3) and every shape occurred, and that most iterations were exact. +/// +/// +/// Iterations: OTOPCUA_PAGING_FUZZ_ITERATIONS (default 10 000, the lane gate). Seeds are stable, so a failure +/// names a reproducible seed. +/// +/// +public sealed class NodeManagerHistoryReadRawCapFuzzTests(BoundsServerFixture fx) : IClassFixture +{ + private static readonly DateTime T0 = new(2026, 10, 22, 12, 0, 0, DateTimeKind.Utc); + + /// The backend the fake is shaped like. + public enum Shape { SqlServer, QuestDb, Gateway } + + private enum Direction { Forward, Reverse, EndOnly } + + private static int Iterations => + int.TryParse(Environment.GetEnvironmentVariable("OTOPCUA_PAGING_FUZZ_ITERATIONS"), out var n) && n > 0 ? n : 10_000; + + [Fact] + public async Task Paged_equals_unpaged_on_random_series_windows_and_caps_and_always_terminates() + { + var iterations = Iterations; + var counts = new Dictionary(); + void Count(string key) => counts[key] = counts.GetValueOrDefault(key) + 1; + + await fx.Gate.WaitAsync(TestContext.Current.CancellationToken); + try + { + for (var seed = 0; seed < iterations; seed++) + { + var rng = new Random(seed); + var shape = (Shape)(seed % 3); + var resolution = shape == Shape.QuestDb ? 10L : 1L; + var series = RandomSeries(rng, resolution); + var cap = rng.Next(10) switch { 0 => 0, 1 or 2 => 3, 3 => 4, 4 => 5, 5 => 7, 6 => 10, 7 => 16, 8 => 33, _ => 1000 }; + var n = rng.Next(9) switch { 0 or 1 => 0u, 2 or 3 => 1u, 4 or 5 => 2u, 6 => (uint)rng.Next(3, 20), 7 => (uint)rng.Next(20, 200), _ => 100_000u }; + var tieBudget = rng.Next(4) switch { 0 => 3, 1 => 8, _ => 65536 }; + var direction = rng.Next(5) switch { 0 => Direction.Reverse, 1 => Direction.EndOnly, _ => Direction.Forward }; + if (direction == Direction.EndOnly && n == 0) direction = Direction.Forward; // Part 11: one bound only with a cap + var bounds = direction != Direction.EndOnly && rng.Next(4) == 0; + + var a = PickBoundary(rng, series, resolution); + var b = PickBoundary(rng, series, resolution); + var (lo, hi) = a <= b ? (a, b) : (b, a); + var (start, end) = direction switch + { + Direction.Forward => (lo, hi), + Direction.Reverse => (hi, lo), + _ => (DateTime.MinValue, hi), + }; + var tag = $"seed {seed} {shape} cap {cap} N {n} tie {tieBudget} {direction} bounds {bounds} [{start:O} .. {end:O}]"; + + // The walk-back (Gateway shape only: reverse / end-only reads, and the bound lookups of any bounded read) refuses + // a millisecond holding more samples than its budget — computed up front, over the whole series, because the + // walk and the lookups read around the window. + var walks = shape == Shape.Gateway && (direction != Direction.Forward || bounds); + var msMax = walks ? MillisecondMax(series) : 0; + + // The oracle: the one-shot answer with the cap off (and, for a plain read, the series itself). + var truthSource = new CappedFake(series, shape, cap: 0); + fx.Nm.HistoricalSourceForTest(truthSource, tieBudget); + // An end-only read is paged backward from EndTime (the node manager flips it to a reverse read down to the + // beginning of time); its oracle is the explicit reverse read down to the Unix epoch — the walk-back's floor, + // and Part 11 (the SDK checks it) needs both bounds set when NumValuesPerNode = 0. + var (truth, truthError, truthCp) = direction == Direction.EndOnly + ? Read(hi, DateTime.UnixEpoch, 0, bounds: false, cp: null) + : Read(start, end, 0, bounds, cp: null); + if (truthError.StatusCode.Code == StatusCodes.BadTooManyOperations) + { + // Only the walk-back can refuse an uncapped one-shot read, and only past its budget (max(1, M) with no cap). + (walks && msMax > Math.Max(1, tieBudget)).ShouldBeTrue($"the oracle read was refused: {tag} (ms max {msMax})"); + Count("oracle refused"); + continue; + } + truthError.StatusCode.Code.ShouldBe(StatusCodes.Good, $"oracle: {tag}"); + truthCp.ShouldBeNull($"oracle (no cap, N = 0) is one page: {tag}"); + if (!bounds) + Render(truth).ShouldBe(Render(Oracle(series, start, end, direction, resolution)), $"oracle vs series: {tag}"); + truthSource.BadRequests.ShouldBe(0, $"a reverse window reached a forward-only backend on the oracle read: {tag}"); + + var windowTies = LargestTieCluster(truth); + if (truth.Count == 0) Count("empty"); + if (windowTies >= 2) Count("ties in window"); + if (EndsOnTie(series, start, end, direction, resolution)) Count("window ends on a tie"); + if (direction == Direction.Reverse) Count("reverse"); + if (direction == Direction.EndOnly) Count("end-only"); + if (n == 1) Count("N=1"); + if (n == 2) Count("N=2"); + if (n == 0) Count("N=0"); + if (cap == 3) Count("cap=3"); + if (bounds) Count("bounds"); + Count(shape.ToString()); + + // A boundary tie cluster as large as the cap leaves a resumed page no room: the server steps past it with one + // empty page carrying a continuation point (the documented #694 ceiling behaviour) — the one empty page allowed. + var oversizedClusters = cap > 0 + ? truth.Where(v => !(v.Value is null && v.StatusCode.Code == StatusCodes.BadBoundNotFound)) + .GroupBy(v => v.SourceTimestamp.Ticks).Count(g => g.Count() >= Math.Max(2, cap - 1)) + : 0; + + // The read under test. + var source = new CappedFake(series, shape, cap); + fx.Nm.HistoricalSourceForTest(source, tieBudget); + var pages = new List>(); + var collected = new List(); + byte[]? cp = null; + StatusCode status; + do + { + var (values, error, next) = Read(start, end, n, bounds, cp); + status = error.StatusCode; + if (!StatusCode.IsGood(status)) break; + pages.Add(values); + collected.AddRange(values); + cp = next; + pages.Count.ShouldBeLessThanOrEqualTo(truth.Count + 2 + oversizedClusters, $"paging must terminate: {tag} pages {Sizes(pages)}"); + } + while (cp is not null); + + source.OverCapRequests.ShouldBe(0, $"the server asked the backend for 0 or more than the cap: {tag}"); + source.BadRequests.ShouldBe(0, $"a reverse window reached a forward-only backend: {tag}"); + + if (status.Code == StatusCodes.BadTooManyOperations) + { + // Legitimate only past the budget the cap leaves: a tie cluster larger than min(M, cap − 1), or on the + // Gateway shape a millisecond holding more than min(M, cap − 2) samples (the walk-back's budget). + var overfetchBound = cap > 0 ? Math.Min(Math.Max(1, tieBudget), Math.Max(1, cap - 1)) : Math.Max(1, tieBudget); + var walkBudget = cap > 0 ? Math.Max(1, Math.Min(tieBudget, cap - 2)) : Math.Max(1, tieBudget); + var legit = windowTies > overfetchBound || (walks && msMax > walkBudget); + legit.ShouldBeTrue($"refused with no tie cluster over {overfetchBound} (max {windowTies}; ms max {msMax} vs {walkBudget}): {tag}"); + Render(collected).ShouldBe(Render(truth.Take(collected.Count)), $"the prefix before the refusal: {tag}"); + Count("refused"); + continue; + } + + status.Code.ShouldBe(StatusCodes.Good, $"{tag}"); + Render(collected).ShouldBe(Render(truth), $"paged ≠ unpaged: {tag} pages {Sizes(pages)}"); + if (cap > 0) + { + // A bounded read whose data fits one page carries its start bound, the data and its end bound: the node + // manager fits the end bound against the client's N but not against the server cap (follow-up F2), so that + // one page may hold cap + 1 values; every other page is at most the cap. + var bothBoundsOnOnePage = bounds && pages.Count == 1; + pages.ShouldAllBe(p => p.Count <= cap + (bothBoundsOnOnePage ? 1 : 0), $"a page over the cap: {tag} pages {Sizes(pages)}"); + if (bothBoundsOnOnePage && pages[0].Count == cap + 1) Count("bounded single page at cap+1 (F2)"); + } + if (n > 0) pages.ShouldAllBe(p => p.Count <= n, $"a page over N: {tag} pages {Sizes(pages)}"); + for (var i = 0; i < pages.Count - 1; i++) + { + if (pages[i].Count > 0) continue; + // Allowed only as the step past a boundary tie cluster the cap cannot page within. + var previousLast = i > 0 && pages[i - 1].Count > 0 ? pages[i - 1][^1].SourceTimestamp.Ticks : (long?)null; + var clusterAtBoundary = previousLast is { } t ? truth.Count(v => v.SourceTimestamp.Ticks == t) : 0; + (cap > 0 && clusterAtBoundary >= Math.Max(2, cap - 1)).ShouldBeTrue( + $"an empty page with a continuation point (page {i}, boundary cluster {clusterAtBoundary}): {tag} pages {Sizes(pages)}"); + Count("empty step over an oversized tie cluster"); + } + if (pages.Count > 1) Count("paged"); + if (pages.Count > 1 && n == 0) Count("paged at N=0"); + // A cost guard against a read explosion, not a budget. A native shape reads once per page plus the bound + // lookups. The Gateway shape's walk-back has its own cost profile — ~30 reads a page on a dense tag + // (#691/#693), far more on a sparse series under a tiny tie budget (seed 320: 82 reads with NO cap, seed 6428: + // 120+) — so it is judged RELATIVE to the same request with the cap off (the pre-#736 behaviour), per page: + // the cap must not make the walk explode compared to before. The baseline asks for the page size the cap + // imposes (N = cap − 1): an uncapped N = 0 reverse read used to be ONE int.MaxValue forward read — the very + // memory vector the cap removes — so "what N = 0 cost before" is not the comparison. + var allowedReads = 8 * pages.Count + 8; + if (shape == Shape.Gateway) + { + var baseline = new CappedFake(series, shape, cap: 0); + fx.Nm.HistoricalSourceForTest(baseline, tieBudget); + var baseN = cap > 0 && (n == 0 || n > (uint)cap) ? (uint)(cap - 1) : n; + var basePages = 0; + byte[]? baseCp = null; + do + { + var (_, baseError, next) = Read(start, end, baseN, bounds, baseCp); + if (!StatusCode.IsGood(baseError.StatusCode)) break; + basePages++; + baseCp = next; + } + while (baseCp is not null && basePages <= truth.Count + 2 + oversizedClusters); + allowedReads = 2 * baseline.Reads * Math.Max(1, pages.Count) / Math.Max(1, basePages) + 32 * pages.Count + 16; + } + source.Reads.ShouldBeLessThanOrEqualTo(allowedReads, + $"backend reads exploded: {source.Reads} for {pages.Count} pages (allowed {allowedReads}): {tag}"); + Count("exact"); + } + } + finally + { + fx.Nm.HistoricalSourceForTest(NullHistorianDataSource.Instance, 65536); + fx.Gate.Release(); + } + + // Non-vacuous: every category the brief names occurred, and most runs were exact. + var summary = string.Join(", ", counts.OrderBy(kv => kv.Key).Select(kv => $"{kv.Key}={kv.Value}")); + foreach (var required in new[] { "empty", "ties in window", "window ends on a tie", "reverse", "end-only", "N=1", "N=2", "N=0", "cap=3", "bounds", "paged", "paged at N=0", "SqlServer", "QuestDb", "Gateway" }) + counts.GetValueOrDefault(required).ShouldBeGreaterThan(iterations / 200, $"{required} under-represented: {summary}"); + counts.GetValueOrDefault("exact").ShouldBeGreaterThan(iterations * 6 / 10, summary); + TestContext.Current.TestOutputHelper?.WriteLine($"raw-cap fuzz: {iterations} iterations — {summary}"); + } + + // --- generators ----------------------------------------------------------------------------- + + private static DateTime Ts(DataValueSnapshot s) => s.SourceTimestampUtc ?? DateTime.MinValue; + + private static DateTime Trunc(DateTime t, long resolution) => new(t.Ticks - t.Ticks % resolution, DateTimeKind.Utc); + + /// Random ascending series: bursts, gaps, sub-millisecond stamps and tie clusters; stamps stored at the resolution. + private static List RandomSeries(Random rng, long resolution) + { + var list = new List(); + var t = T0.Ticks - rng.Next(0, 3) * TimeSpan.TicksPerHour; + var count = rng.Next(1, 120); + var v = 0; + while (list.Count < count) + { + t += rng.Next(6) switch + { + 0 => rng.Next(1, 10), // sub-millisecond (sub-µs on QuestDB: collapses onto a tie) + 1 => rng.Next(1, 5) * TimeSpan.TicksPerMillisecond, // dense + 2 => rng.Next(1, 3) * TimeSpan.TicksPerSecond, + 3 => rng.Next(1, 30) * TimeSpan.TicksPerMinute, // a gap + _ => rng.Next(1, 1000) * TimeSpan.TicksPerMillisecond, + }; + var ties = rng.Next(7) == 0 ? rng.Next(2, 9) : 1; + for (var k = 0; k < ties; k++) + { + var ts = Trunc(new DateTime(t, DateTimeKind.Utc), resolution); + var status = rng.Next(12) == 0 ? StatusCodes.Bad : StatusCodes.Good; + list.Add(new DataValueSnapshot(status == StatusCodes.Bad ? null : (double)v, status, ts, ts)); + v++; + } + } + + return list; + } + + /// A window bound: before / after the data, exactly on a sample (a tie, maybe), one step past one, a tick before one, or between two. + private static DateTime PickBoundary(Random rng, List series, long resolution) + { + var first = Ts(series[0]); + var last = Ts(series[^1]); + var i = rng.Next(series.Count); + var t = Ts(series[i]); + switch (rng.Next(9)) + { + case 0: return first.AddSeconds(-rng.Next(1, 100)); + case 1: return last.AddSeconds(rng.Next(1, 100)); + case 2: + case 3: return t; + case 4: return t.AddTicks(resolution); + case 5: return t.AddTicks(-1); // sub-resolution: a microsecond backend truncates it back ONTO the sample + case 6: return t.AddMilliseconds(1); + default: + var next = i + 1 < series.Count ? Ts(series[i + 1]) : t.AddSeconds(10); + return new DateTime(t.Ticks + (long)((next.Ticks - t.Ticks) * rng.NextDouble()), DateTimeKind.Utc); + } + } + + /// The series' own answer for a plain (no bounds) read: forward [start, end), point, reverse / end-only (end, start] newest first. + private static List Oracle(List series, DateTime start, DateTime end, Direction direction, long resolution) + { + var s = Trunc(start, resolution); + var e = Trunc(end, resolution); + IEnumerable window = direction switch + { + // start == end is Part 11's point read whichever way it was generated: the node manager reads it forward. + Direction.Forward or Direction.Reverse when s == e => series.Where(x => Ts(x) == s), + Direction.Forward => series.Where(x => Ts(x) >= s && Ts(x) < e), + Direction.Reverse => series.Where(x => Ts(x) > e && Ts(x) <= s).Reverse(), + _ => series.Where(x => Ts(x) <= e).Reverse(), + }; + return window.Select(x => new DataValue(new Variant(x.Value), new StatusCode(x.StatusCode), Ts(x), Ts(x))).ToList(); + } + + /// The largest tie cluster among the returned stored values (a bound placeholder is not a stored value). + private static int LargestTieCluster(IReadOnlyList truth) => + truth.Where(v => !(v.Value is null && v.StatusCode.Code == StatusCodes.BadBoundNotFound)) + .GroupBy(v => v.SourceTimestamp.Ticks).Select(g => g.Count()).DefaultIfEmpty(0).Max(); + + /// The most samples within one millisecond anywhere in the series (the walk-back's refusal rule). + private static int MillisecondMax(List series) + { + var msMax = 0; + for (int i = 0, j = 0; i < series.Count; i++) + { + while (Ts(series[i]).Ticks - Ts(series[j]).Ticks >= TimeSpan.TicksPerMillisecond) j++; + msMax = Math.Max(msMax, i - j + 1); + } + + return msMax; + } + + private static bool EndsOnTie(List series, DateTime start, DateTime end, Direction direction, long resolution) + { + var edge = Trunc(direction == Direction.Forward ? end : direction == Direction.Reverse ? start : end, resolution); + return series.Count(x => Ts(x) == edge) >= 2; + } + + // --- helpers ---------------------------------------------------------------------------------- + + private static string Sizes(IEnumerable> pages) => "[" + string.Join(",", pages.Select(p => p.Count)) + "]"; + + private static string Render(IEnumerable values) => + string.Join(" | ", values.Select(v => $"{v.SourceTimestamp.Ticks}={v.Value ?? "null"}/{v.StatusCode.Code:X8}")); + + private (IReadOnlyList Values, ServiceResult Error, byte[]? Cp) Read(DateTime start, DateTime end, uint max, bool bounds, byte[]? cp) + { + var context = new OperationContext(new RequestHeader(), secureChannelContext: null, RequestType.HistoryRead, identity: null); + var details = new ReadRawModifiedDetails + { + StartTime = start, EndTime = end, NumValuesPerNode = max, ReturnBounds = bounds, IsReadModified = false, + }; + var nodesToRead = new List { new() { NodeId = fx.NodeId, ContinuationPoint = cp } }; + var results = new List { null! }; + var errors = new List { null! }; + fx.Nm.HistoryRead(context, details, TimestampsToReturn.Both, releaseContinuationPoints: false, nodesToRead, results, errors); + + var values = new List(); + if (results[0]?.HistoryData is { } ho && !ExtensionObject.IsNull(ho)) + values.AddRange(((HistoryData)ExtensionObject.ToEncodeable(ho)).DataValues); + return (values, errors[0], results[0]?.ContinuationPoint); + } + + /// + /// A series-backed historian shaped like one backend, enforcing the backend half of the server raw cap exactly as + /// the real data sources do: it reports the cap as ; a read asking for 0 or more than + /// the cap reads cap + 1 rows and throws when the window holds more + /// (never a truncated answer). It also records every such request () — the server must + /// never issue one — and every reverse window sent to the forward-only shape (). + /// + private sealed class CappedFake(IReadOnlyList series, Shape shape, int cap) : IHistorianDataSource + { + private readonly Lock _gate = new(); + + public int Reads { get; private set; } + public int OverCapRequests { get; private set; } + public int BadRequests { get; private set; } + + public bool SupportsReverseRaw => shape != Shape.Gateway; + + public long TimestampResolutionTicks => shape == Shape.QuestDb ? 10 : 1; + + public int MaxRawValuesPerRead => cap; + + private DateTime Trunc(DateTime t) => NodeManagerHistoryReadRawCapFuzzTests.Trunc(t, TimestampResolutionTicks); + + public Task ReadRawAsync( + string fullReference, DateTime startUtc, DateTime endUtc, uint maxValuesPerNode, CancellationToken cancellationToken) + { + var probe = cap > 0 && (maxValuesPerNode == 0 || maxValuesPerNode > (uint)cap); + lock (_gate) + { + Reads++; + if (probe) OverCapRequests++; + if (shape == Shape.Gateway && startUtc > endUtc) BadRequests++; + } + + if (shape == Shape.Gateway && startUtc > endUtc) + throw new InvalidOperationException("start_time must be less than or equal to end_time. (InvalidArgument)"); + + startUtc = Trunc(startUtc); + endUtc = Trunc(endUtc); + List window; + if (startUtc > endUtc) window = series.Where(s => Ts(s) > endUtc && Ts(s) <= startUtc).Reverse().ToList(); + else if (startUtc == endUtc) window = series.Where(s => Ts(s) == startUtc).ToList(); + else window = series.Where(s => Ts(s) >= startUtc && Ts(s) < endUtc).ToList(); + + var top = probe ? cap + 1 : maxValuesPerNode == 0 ? int.MaxValue : (int)Math.Min(maxValuesPerNode, int.MaxValue); + var page = window.Take(top).ToList(); + if (probe && page.Count > cap) + throw new HistoryReadLimitExceededException(cap, $"more than MaxRawValuesPerRead ({cap}) samples"); + return Task.FromResult(new HistorianRead(page, null)); + } + + public Task ReadProcessedAsync( + string fullReference, DateTime startUtc, DateTime endUtc, TimeSpan interval, + HistoryAggregateType aggregate, CancellationToken cancellationToken) => + Task.FromResult(new HistorianRead(Array.Empty(), null)); + + public Task ReadAtTimeAsync( + string fullReference, IReadOnlyList timestampsUtc, CancellationToken cancellationToken) => + Task.FromResult(new HistorianRead(Array.Empty(), null)); + + public Task ReadEventsAsync( + string? sourceName, DateTime startUtc, DateTime endUtc, int maxEvents, CancellationToken cancellationToken) => + Task.FromResult(new HistoricalEventsResult(Array.Empty(), null)); + + public HistorianHealthSnapshot GetHealthSnapshot() => NullHistorianDataSource.Instance.GetHealthSnapshot(); + + public void Dispose() { } + } +} + +/// Test-only wiring shared by the paging fuzz: swap the source and the tie budget, with a fresh continuation store. +internal static class NodeManagerFuzzExtensions +{ + public static void HistoricalSourceForTest(this OtOpcUaNodeManager nm, IHistorianDataSource source, int tieBudget) + { + nm.HistoryContinuationStore = new InMemoryHistoryContinuationStore(); + nm.MaxTieClusterOverfetch = tieBudget; + nm.HistorianDataSource = source; + } +} diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.Tests/RawPageLimitTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.Tests/RawPageLimitTests.cs index fdd3e597..7818258d 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.Tests/RawPageLimitTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.Tests/RawPageLimitTests.cs @@ -24,6 +24,15 @@ public sealed class RawPageLimitTests [InlineData(5000u, 983, 984, 0u, 0u, true)] // the ties alone fill a read: never a 0 (uncapped) request [InlineData(5000u, 5000, 984, 0u, 0u, true)] // never an underflow [InlineData(5000u, 0, 1, 0u, 0u, true)] + // R08:1.1 (#736): the server raw cap (default 10 000) is the limit every backend reports — an uncapped read pages at it. + [InlineData(0u, 0, 10_000, 9_999u, 10_000u, false)] // NumValuesPerNode = 0: a first page of cap − 1 + the lookahead row + [InlineData(0u, 1, 10_000, 9_998u, 10_000u, false)] // a resumed page re-reads its one boundary tie + [InlineData(100_000u, 0, 10_000, 9_999u, 10_000u, false)] // a client cap above the server cap is bounded by it + [InlineData(10_000u, 0, 10_000, 9_999u, 10_000u, false)] // exactly the cap: still cap − 1 (the lookahead row needs room) + [InlineData(500u, 0, 10_000, 500u, 501u, false)] // under the cap: unchanged + [InlineData(0u, 0, 3, 2u, 3u, false)] // the smallest usable cap (MinMaxRawValuesPerRead = 3): a first page of two + [InlineData(0u, 1, 3, 1u, 3u, false)] // … and a resumed page still carries one value beside its boundary tie + [InlineData(0u, 1, 2, 0u, 0u, true)] // at 2 a resumed page has NO room — every value would cost an empty page (why 3 is the floor) public void Plan_fits_page_ties_and_lookahead_into_the_limit(uint want, int k, int limit, uint page, uint request, bool noRoom) { var plan = RawPageLimit.Plan(want, k, limit); diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Historian/ServerHistorianOptionsTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Historian/ServerHistorianOptionsTests.cs index 8e310f30..08d700b2 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Historian/ServerHistorianOptionsTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Historian/ServerHistorianOptionsTests.cs @@ -168,4 +168,68 @@ public sealed class ServerHistorianOptionsTests }.Validate(); Assert.Empty(w); } + + // ── MaxRawValuesPerRead (R08:1.1, #736) ───────────────────────────────────────────────────── + + [Fact] + public void MaxRawValuesPerRead_defaults_to_ten_thousand_and_binds() + { + var o = new ServerHistorianOptions(); + Assert.Equal(10_000, o.MaxRawValuesPerRead); + Assert.Equal(10_000, ServerHistorianOptions.DefaultMaxRawValuesPerRead); + Assert.Equal(10_000, o.EffectiveMaxRawValuesPerRead); + + var bound = new ConfigurationBuilder() + .AddInMemoryCollection(new Dictionary { ["ServerHistorian:MaxRawValuesPerRead"] = "2500" }) + .Build() + .GetSection(ServerHistorianOptions.SectionName) + .Get()!; + Assert.Equal(2500, bound.MaxRawValuesPerRead); + Assert.Equal(2500, bound.EffectiveMaxRawValuesPerRead); + Assert.Empty(new ServerHistorianOptions + { + Enabled = true, Endpoint = "https://h:5222", ApiKey = "histgw_x_y", MaxRawValuesPerRead = 2500, + }.Validate()); + } + + [Theory] + [InlineData(0)] + [InlineData(-5)] + public void Nonpositive_MaxRawValuesPerRead_is_unbounded_and_warns_when_enabled(int value) + { + var o = new ServerHistorianOptions + { + Enabled = true, Endpoint = "https://h:5222", ApiKey = "histgw_x_y", MaxRawValuesPerRead = value, + }; + Assert.Equal(0, o.EffectiveMaxRawValuesPerRead); + var w = o.Validate(); + var m = Assert.Single(w); + Assert.Contains("MaxRawValuesPerRead", m); + Assert.Contains("unbounded", m); + } + + [Theory] + [InlineData(1)] + [InlineData(2)] + public void A_MaxRawValuesPerRead_below_three_is_raised_to_three_and_warns_when_enabled(int value) + { + // A resumed page's read must carry a value, the boundary tie it re-reads and the lookahead row that says "more + // follows": at 2 it has no room and every value costs an empty page with a continuation point (the #692 defect). + var o = new ServerHistorianOptions + { + Enabled = true, Endpoint = "https://h:5222", ApiKey = "histgw_x_y", MaxRawValuesPerRead = value, + }; + Assert.Equal(3, o.EffectiveMaxRawValuesPerRead); + Assert.Equal(ServerHistorianOptions.MinMaxRawValuesPerRead, o.EffectiveMaxRawValuesPerRead); + var m = Assert.Single(o.Validate()); + Assert.Contains("MaxRawValuesPerRead", m); + Assert.Equal(3, new ServerHistorianOptions { MaxRawValuesPerRead = 3 }.EffectiveMaxRawValuesPerRead); + } + + [Fact] + public void Disabled_with_an_odd_MaxRawValuesPerRead_is_silent() + { + Assert.Empty(new ServerHistorianOptions { Enabled = false, MaxRawValuesPerRead = 0 }.Validate()); + Assert.Empty(new ServerHistorianOptions { Enabled = false, MaxRawValuesPerRead = 1 }.Validate()); + } }