From 8114f13a28cc41683350c96d0dfd11b446a667de Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Mon, 12 Oct 2026 15:19:34 -0400 Subject: [PATCH] =?UTF-8?q?fix(historian):=20#687=20review=20=E2=80=94=20o?= =?UTF-8?q?ne=20bounded=20re-read=20instead=20of=20failing,=20raw=20budget?= =?UTF-8?q?=20unclamped,=20exclusive=20forward=20end?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - M1/M2: when the 1 ms pre-window margin (boundary row + stored samples, e.g. a >16-sample tie cluster at start-1ms / lastTs-1ms) uses up the cap + 16 budget, re-read ONCE asking for cap + m + 16 (m = pre-window rows seen); fail BadTooManyOperations only if the re-read is still short because of them. The raw budget is no longer clamped at the gateway default ceiling (AVEVA always emits the boundary row, so page sizes 999,985-1,000,000 always came back short); the #688 clamp stays on the per-sample aggregate path only. - Merge trap: NumValuesPerNode 0 => int.MaxValue, never through the clamp. - L3: the forward end is exclusive ([start, end), the IHistorianDataSource contract); a point read (start == end, the tie-cluster over-fetch) inclusive. - RequestStartFor never goes before the Unix epoch (#686's walk-back floor). - L4 (docs): an Uncertain native row for an all-NULL interval is unverified live. --- CLAUDE.md | 2 +- docs/Historian.md | 37 ++++-- .../GatewayHistorianDataSource.cs | 79 ++++++++---- .../RawAggregateReader.cs | 25 ++-- .../GatewayRawBoundaryRowTests.cs | 122 +++++++++++++++--- 5 files changed, 202 insertions(+), 63 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index c9c902df..9672b8ce 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1364,7 +1364,7 @@ historian tagname, which is the tag's driver `FullName`. The server dispatches all OPC UA HistoryRead to the registered `IHistorianDataSource` — the `GatewayHistorianDataSource` read client when enabled, else the `NullHistorianDataSource` default -(historized nodes return `GoodNoData` for Raw/AtTime — Processed is `BadAggregateNotSupported` since #682). Supported variants: Raw (⚠️ #687, client-visible: Gateway raw reads no longer return AVEVA's synthetic start-boundary row — asked from 1 ms early with `NumValuesPerNode + 16` rows and filtered by timestamp, the #681 classification — so an empty window is `GoodNoData`, not a fabricated value at StartTime; `ReturnBounds` is ignored on every backend), Processed +(historized nodes return `GoodNoData` for Raw/AtTime — Processed is `BadAggregateNotSupported` since #682). Supported variants: Raw (⚠️ #687, client-visible: Gateway raw reads no longer return AVEVA's synthetic start-boundary row — asked from 1 ms early (epoch-floored) with `NumValuesPerNode + 16` rows (unclamped; 0 ⇒ `int.MaxValue`) and filtered by timestamp — so an empty window is `GoodNoData`, not a fabricated value at StartTime; the forward end is now exclusive (a point read inclusive); pre-window rows that crowd the budget trigger ONE re-read with room for them, else `BadTooManyOperations`; `ReturnBounds` is ignored on every backend), Processed (backend-dependent: SqlServer/QuestDb serve seventeen; Gateway twelve = `GatewayHistorianDataSource.ServedAggregates` — ⚠️ #681, client-visible: the ten per-sample ones (Average, Count, Min/Max, Range, Start/End/Delta, Min/MaxActualTime) are evaluated from RAW gateway reads by the shared `Sql.Common` evaluator — identical to the SQL backends, Calculated bits, interval-start stamps, #677 reverse; Min/Max are no longer AVEVA modes, the read moves every raw sample of the window (a month of 1 Hz ⇒ `BadTooManyOperations` at the default `MaxTimeWeightedRawRows` cap), and every gateway stream is consumed to completion (an abandoned one makes the gateway evict its shared historian session) — plus TimeAverage/Total via AVEVA modes (interval-start stamped + Calculated; ⚠️ #685: AVEVA's extra pre-window row is dropped ⇒ one row per interval in `[Start, End)`, and an Uncertain (opc 64) interval keeps its value as `UncertainDataSubNormal` — AVEVA's TWA excludes NULL-gap time where the SQL backends bridge it; per-interval values still need gateway fix historiangw#27); the other time-weighted five stay `BadAggregateNotSupported` because the gateway has no prior-sample read for Part 13 bounds; `docs/Historian.md` §Aggregate set), and AtTime over historized variable nodes; Events over alarm-owning equipment-folder event-notifier nodes. Reads are ungated (served from any redundancy node); authorization uses the standard `AccessLevels.HistoryRead` bit set at materialization. diff --git a/docs/Historian.md b/docs/Historian.md index cdbc450b..64363ba9 100644 --- a/docs/Historian.md +++ b/docs/Historian.md @@ -682,18 +682,28 @@ data. > `18:06:21 v=30 Good`), a window before the tag's history answered `v=0 Bad` instead of `GoodNoData`, and a start-only read led > with the fabricated row. It is now dropped; such windows read `GoodNoData`. -- **How.** The same classification the raw-evaluated aggregates use (#681, `RawAggregateReader.RequestStartFor` / - `IsWindowSample` — one authority): the gateway is asked from **1 ms before** the window, so the boundary row lands before it, and - only rows stamped at or after StartTime are returned. The decision is by **timestamp**, not by value or quality — a stored NULL - sample looks exactly like the no-data boundary row on the wire (value 0, Bad), and a stored NULL inside the window is still - returned (Bad), as it is a stored sample. -- **Paging (#669) is unaffected.** The gateway is asked for `NumValuesPerNode + 16` rows (`NumValuesPerNode = 0` stays unlimited), - so the boundary row cannot shorten a page: a full page still holds exactly `NumValuesPerNode` stored samples and carries a - continuation point, and a resumed page (which re-reads from the last returned timestamp inclusive, and the tie-cluster over-fetch - `start == end` read) no longer gets a fabricated row at its resume point. If more than 16 rows lie in the 1 ms before the window - and exhaust that allowance, the page cannot be read completely and the node fails (`HistoryReadLimitExceededException`, not - counted as a backend fault) rather than answering a short page that would end the client's paging early. Every gateway stream is - still drained to completion. +- **How.** The same request start the raw-evaluated aggregates use (#681, `RawAggregateReader.RequestStartFor` — one authority): + the gateway is asked from **1 ms before** the window (never before the Unix epoch, the start of the historian's time encoding), + so the boundary row lands before it, and only rows `RawAggregateReader.IsWindowSample` are returned. The decision is by + **timestamp**, not by value or quality — a stored NULL sample looks exactly like the no-data boundary row on the wire (value 0, + Bad), and a stored NULL inside the window is still returned (Bad), as it is a stored sample. +- **The forward end is exclusive** (`[StartTime, EndTime)`, the `IHistorianDataSource` contract and what the SQL backends return; + the gateway's own `ReadRaw` end is inclusive, so a sample stored exactly at EndTime used to be returned). A point read + (`StartTime == EndTime` — the server's tie-cluster over-fetch) still returns the samples at that instant. The #686 reverse + walk-back is exact either way. +- **Row budget.** `NumValuesPerNode = 0` ("all values") asks the gateway for an unbounded `max_values` (`int.MaxValue` — the client + package and the gateway reject 0, #686). A cap asks for `NumValuesPerNode + 16` rows, **not** clamped at the gateway's default + ceiling (#688 clamps only the per-sample aggregate budget): AVEVA always emits the boundary row, so a clamped budget would make + every page size from 999,985 to 1,000,000 come back short. A gateway with `AllowStreamingBeyondCeiling=false` refuses an + over-ceiling budget instead ⇒ `BadTooManyOperations` (not a backend fault); a default gateway serves it. +- **Paging (#669) is unaffected.** A full page still holds exactly `NumValuesPerNode` stored samples and carries a continuation + point, and a resumed page (which re-reads from the last returned timestamp inclusive, and the tie-cluster over-fetch + `start == end` read) no longer gets a fabricated row at its resume point. A read is complete when the stream came back short, + filled the page, or reached EndTime. If rows **before** the window — the boundary row plus stored samples inside the 1 ms margin, + e.g. a tie cluster at `StartTime − 1 ms` or at a resume point `− 1 ms` — used up the budget, the window is re-read **once** + asking for `NumValuesPerNode + m + 16` (m = the pre-window rows the first read saw, unclamped). Only if that re-read is still + short because of pre-window rows does the node fail `BadTooManyOperations` (not a backend fault) rather than answer a short page + that would end the client's paging early. Every gateway stream is still drained to completion. - **`ReturnBounds` is not honoured** — on any backend: the server does not pass it to the data source, so no bounding values are returned (see [Known limitations](#known-limitations)). The AVEVA boundary row is not used as a bound either: it is stamped at the query start rather than carrying the time of the real prior sample, so it could only ever be a Part 11 *interpolated* bound, which @@ -845,6 +855,9 @@ fixed: - ⚠️ **End-to-end correctness also needs the gateway fix historiangw#27:** the gateway sends the aggregate interval to AVEVA in ticks where AVEVA reads milliseconds (10,000× too large), so today a native read returns one row for the whole window (plus the pre-window row, now dropped). OtOpcUa does not work around it; per-interval values return once the gateway is fixed. +- **Not verified live:** what AVEVA answers for an interval holding **only** NULL samples (no good time at all). If it marks such a + row Uncertain (`opc_quality` 64–191) with a placeholder value, that value is now passed through as Uncertain; the gate only + observed Uncertain rows that also held good time. Re-check on the VPN once historiangw#27 is fixed. **Integral unit (#677).** AVEVA's integral is `∫ value dt` (seconds) ÷ the tag's `IntegralDivisor`. The proto3 `double` default for `HistorianTagDefinition.IntegralDivisor` is `0`, so `GatewayTagProvisioner` now sends **`IntegralDivisor = 1`** explicitly on every diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway/GatewayHistorianDataSource.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway/GatewayHistorianDataSource.cs index 828fc113..79986db2 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway/GatewayHistorianDataSource.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway/GatewayHistorianDataSource.cs @@ -123,8 +123,8 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis } /// - /// The stored samples of [startUtc, endUtc], oldest first, at most - /// (0 = no cap) — without AVEVA's synthetic rows (#687). + /// The stored samples of [startUtc, endUtc) (a point read start == end: of that instant), oldest first, + /// at most (0 = all) — without AVEVA's synthetic rows (#687). /// /// /// @@ -133,48 +133,81 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis /// a NULL row (value 0, Bad). Part 11 Raw with ReturnBounds = false wants stored values only, and the /// server's paging (#669) counts every returned row toward the page and resumes at the last returned timestamp, /// so a fabricated row would also be counted as a sample and re-fabricated at every resume point. The read - /// therefore uses the same classification as the raw-evaluated aggregates (#681): it is requested from + /// therefore uses the same request start as the raw-evaluated aggregates (#681): it is requested from /// (1 ms early) and only rows /// are kept — the decision is by TIMESTAMP, because a stored NULL - /// sample and the no-data boundary row look the same on the wire (both carry a value 0 with a Bad quality). + /// sample and the no-data boundary row look the same on the wire (both carry a value 0 with a Bad quality). The + /// forward end is exclusive (the interface contract; the gateway's own end is inclusive), a point read inclusive. /// /// - /// The gateway is asked for extra rows so the boundary row - /// (and any stored sample inside the margin) cannot shorten a page: a full page still holds exactly the cap of - /// stored samples, so the server's "full page ⇒ continuation point" rule is unchanged. If the stream delivers its - /// whole budget yet fewer than the cap belong to the window, the page cannot be read completely; answering a short - /// page would end the client's paging early, so the read fails with - /// instead. The stream is always consumed to completion (never - /// broken out of — a cancelled gateway stream evicts its pooled historian session). ReturnBounds is not - /// plumbed to the data source on any backend, so bounding values are never synthesised here. + /// Row budget. 0 ("all values") asks for (the client package and the + /// gateway reject max_values <= 0, #686). A cap asks for cap + + /// rows — NOT clamped at the gateway's default ceiling (#688 + /// clamps only the per-sample aggregate budget): AVEVA always emits the boundary row, so a clamped budget would make + /// every page size within the allowance of the ceiling come back short. A strict gateway refuses an over-ceiling + /// budget instead (⇒ , BadTooManyOperations). + /// + /// + /// Completeness. A full page must hold exactly the cap of stored samples — a short page ends the client's + /// paging. A read is complete when the stream came back short, filled the cap, or reached the window end. If rows + /// BEFORE the window (the boundary row and stored samples inside the 1 ms margin, e.g. a tie cluster at + /// start − 1 ms) used up the budget, the window is read ONCE more asking for cap + m + + /// (m = the pre-window rows the first read saw); only if that + /// re-read is still short because of pre-window rows does the read fail with + /// . Every stream is consumed to completion (never broken out of — + /// a cancelled gateway stream evicts its pooled historian session). ReturnBounds is not plumbed to the + /// data source on any backend, so bounding values are never synthesised here. /// /// private async Task> ReadRawWindowAsync( string fullReference, DateTime startUtc, DateTime endUtc, uint maxValuesPerNode, CancellationToken cancellationToken) { - // The gateway seam caps with an int; OPC UA hands us a uint, so clamp to int range. 0 ("all values") is sent as - // int.MaxValue: the client package and the gateway reject max_values <= 0 (#686 review). + // The gateway seam caps with an int; OPC UA hands us a uint, so clamp to int range. var cap = (int)Math.Min(maxValuesPerNode, int.MaxValue); - var budget = cap == 0 ? int.MaxValue : (int)Math.Min(int.MaxValue, (long)cap + RawAggregateReader.PreWindowAllowance); + if (cap == 0) + return (await ReadRawPassAsync(fullReference, startUtc, endUtc, cap, int.MaxValue, cancellationToken) + .ConfigureAwait(false)).Samples; + + var first = await ReadRawPassAsync(fullReference, startUtc, endUtc, cap, + (int)Math.Min(int.MaxValue, (long)cap + RawAggregateReader.PreWindowAllowance), cancellationToken).ConfigureAwait(false); + if (first.Complete) return first.Samples; + + // The margin crowded the window out: one bounded re-read with room for the pre-window rows already seen. + var second = await ReadRawPassAsync(fullReference, startUtc, endUtc, cap, + (int)Math.Min(int.MaxValue, (long)cap + first.PreWindowRows + RawAggregateReader.PreWindowAllowance), + cancellationToken).ConfigureAwait(false); + if (second.Complete) return second.Samples; + + throw new HistoryReadLimitExceededException(cap, + $"The raw read of '{fullReference}' could not be read completely: more than {second.PreWindowRows} rows lie in " + + $"the {RawAggregateReader.BoundaryMargin.TotalMilliseconds} ms before the window and exhausted the row budget."); + } + + /// One forward raw read of a window, drained, with its pre-window row count and whether it is complete. + private async Task<(List Samples, int PreWindowRows, bool Complete)> ReadRawPassAsync( + string fullReference, DateTime startUtc, DateTime endUtc, int cap, int budget, CancellationToken cancellationToken) + { var samples = new List(); long received = 0; + var preWindow = 0; + var reachedEnd = false; await foreach (var sample in _client .ReadRawAsync(fullReference, RawAggregateReader.RequestStartFor(startUtc), endUtc, budget, cancellationToken) .ConfigureAwait(false)) { received++; // Filter while draining — the rest of the (bounded) stream is consumed, never abandoned. - if (RawAggregateReader.IsWindowSample(sample, startUtc) && (cap == 0 || samples.Count < cap)) - samples.Add(sample); + if (RawAggregateReader.IsWindowSample(sample, startUtc, endUtc)) + { + if (cap == 0 || samples.Count < cap) samples.Add(sample); + } + else if (sample.Timestamp is { } ts && ts.ToDateTime() < startUtc) preWindow++; + else if (sample.Timestamp is not null) reachedEnd = true; } - if (cap > 0 && received >= budget && samples.Count < cap) - throw new HistoryReadLimitExceededException(cap, - $"The raw read of '{fullReference}' could not be read completely: more than " + - $"{RawAggregateReader.PreWindowAllowance} rows lie in the {RawAggregateReader.BoundaryMargin.TotalMilliseconds} ms " + - "before the window and exhausted the row budget."); - return samples; + var complete = received < budget || samples.Count >= cap || reachedEnd || cap == 0; + return (samples, preWindow, complete); } /// diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway/RawAggregateReader.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway/RawAggregateReader.cs index 484b56c4..1723cb1d 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway/RawAggregateReader.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway/RawAggregateReader.cs @@ -80,25 +80,32 @@ internal static class RawAggregateReader /// earlier, so the AVEVA start-boundary row — stamped exactly at the QUERY start, carrying the value in effect there /// (Quality = 133) or, with no earlier data, a NULL row (value 0, Bad OPC quality) — lands BEFORE the window /// (). The one place both the raw-evaluated aggregates (#681) and a raw HistoryRead - /// (#687) decide what the request start is. No margin when the window starts within one margin of - /// . + /// (#687) decide what the request start is. Never before (the Unix epoch — the start of + /// the historian's time encoding, the same floor the server's reverse walk-back uses, #686): a window starting less + /// than one margin after it is requested from the epoch itself. /// /// The window's (inclusive) start. /// The start to request. public static DateTime RequestStartFor(DateTime windowStartUtc) => - windowStartUtc.Ticks > BoundaryMargin.Ticks ? windowStartUtc - BoundaryMargin : windowStartUtc; + windowStartUtc - ProbeFloor > BoundaryMargin ? windowStartUtc - BoundaryMargin + : windowStartUtc > ProbeFloor ? ProbeFloor + : windowStartUtc; /// - /// True when a row of a stream requested from is a stored sample of the window that - /// begins at : it is stamped at or after the window start. A row before it is the - /// AVEVA boundary row (whatever its value or quality — a stored NULL looks the same on the wire, so the timestamp is - /// the evidence) or a stored sample inside the margin; a row without a timestamp is never a sample. + /// True when a row of a forward raw stream requested from is a stored sample of the raw + /// window [windowStartUtc, windowEndUtc) — the contract + /// (exclusive end, as on the SQL backends; the gateway's own ReadRaw end is inclusive) — or, for a point read + /// (start == end, the server's tie-cluster over-fetch), of that one instant. A row before the start is the AVEVA + /// boundary row (whatever its value or quality — a stored NULL looks the same on the wire, so the timestamp is the + /// evidence) or a stored sample inside the margin; a row without a timestamp is never a sample. /// /// The gateway row. /// The window's (inclusive) start. + /// The window's end: exclusive, or inclusive when it equals the start. /// Whether the row belongs to the window. - public static bool IsWindowSample(HistorianSample raw, DateTime windowStartUtc) => - raw.Timestamp is { } ts && ts.ToDateTime() >= windowStartUtc; + public static bool IsWindowSample(HistorianSample raw, DateTime windowStartUtc, DateTime windowEndUtc) => + raw.Timestamp is { } ts && ts.ToDateTime() is var t && t >= windowStartUtc + && (t < windowEndUtc || (t == windowEndUtc && windowStartUtc == windowEndUtc)); /// True for the aggregates this reader serves: the per-sample families of . /// Requested aggregate. diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests/GatewayRawBoundaryRowTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests/GatewayRawBoundaryRowTests.cs index 4a95d364..11d62de9 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests/GatewayRawBoundaryRowTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests/GatewayRawBoundaryRowTests.cs @@ -125,14 +125,6 @@ public sealed class GatewayRawBoundaryRowTests fake.RawCalls.Single().ShouldBe((At(30) - RawAggregateReader.BoundaryMargin, At(55), 5 + RawAggregateReader.PreWindowAllowance)); } - [Fact] - public async Task An_unlimited_read_stays_unlimited() - { - var fake = Gate(); - await Source(fake).ReadRawAsync("T", At(30), At(55), 0, Ct); - fake.RawCalls.Single().MaxValues.ShouldBe(0); - } - [Fact] public async Task A_full_page_holds_exactly_the_cap_and_the_stream_is_drained_not_abandoned() { @@ -148,13 +140,19 @@ public sealed class GatewayRawBoundaryRowTests [InlineData(3u)] public async Task Server_style_paging_returns_every_stored_sample_once_and_nothing_else(uint cap) { - // Ties at +60 (three samples) and a stored NULL at +40, over the AVEVA-shaped wire. - var data = Gate681.Select(x => (At(x.Sec), x.Value, x.Null)) - .Concat([(At(60), 2d, false), (At(60), 3d, false)]).ToList(); - var fake = Aveva(data); - var ds = Source(fake); + // Ties at +60 (three samples), a stored NULL at +40, a 20-sample tie cluster 1 ms before the +60 resume point + // (crowds that resume read's margin: more than the 16-row allowance) and one 1 ms before the window start (outside + // the window; crowds the first read's margin), over the AVEVA-shaped wire. var start = At(-10); var end = At(200); + var inside = Gate681.Select(x => (At(x.Sec), x.Value, x.Null)) + .Concat([(At(60), 2d, false), (At(60), 3d, false)]) + .Concat(Enumerable.Range(0, 20).Select(i => (At(60) - TimeSpan.FromMilliseconds(1), 100d + i, false))) + .ToList(); + var outside = Enumerable.Range(0, 20).Select(i => (start - TimeSpan.FromMilliseconds(1), 200d + i, false)); + var data = inside; + var fake = Aveva(inside.Concat(outside).ToList()); + var ds = Source(fake); var got = new List(); var skip = 0; var cursor = start; @@ -195,15 +193,103 @@ public sealed class GatewayRawBoundaryRowTests } [Fact] - public async Task A_margin_that_swallows_the_row_budget_fails_the_node_rather_than_answering_a_short_page() + public async Task A_margin_that_swallows_the_row_budget_is_re_read_once_with_room_for_it() { - // More stored rows inside the 1 ms margin than the allowance: the page could not be read, and a short page would - // end the client's paging early — refuse it (a request limit, not a backend fault). - var margin = Enumerable.Range(0, 40).Select(i => (At(30).AddTicks(-1 - i), (double)i, false)); + // 30 stored rows inside the 1 ms margin (plus AVEVA's boundary row) eat the whole first budget (5 + 16): the page is + // re-read ONCE asking for cap + m + 16 (m = the pre-window rows the first read saw), which fits, and the full page + // is answered — a short page would end the client's paging early. + var margin = Enumerable.Range(0, 30).Select(i => (At(30).AddTicks(-1 - i), (double)i, false)); + var window = Enumerable.Range(0, 10).Select(i => (At(31 + i), 1000d + i, false)); + var fake = Aveva(margin.Concat(window).ToList()); + var ds = Source(fake); + + var r = await ds.ReadRawAsync("T", At(30), At(50), 5, Ct); + + r.Samples.Select(s => (double)s.Value!).ShouldBe([1000d, 1001d, 1002d, 1003d, 1004d]); + fake.RawCalls.Select(c => c.MaxValues).ShouldBe([5 + 16, 5 + 21 + 16]); + fake.RawCalls.ShouldAllBe(c => c.Start == At(30) - RawAggregateReader.BoundaryMargin); + fake.RawStreamsCompleted.ShouldBe(2); + ds.GetHealthSnapshot().TotalFailures.ShouldBe(0); + } + + [Fact] + public async Task A_margin_still_crowding_the_re_read_fails_the_node_as_a_request_limit() + { + // 200 rows in the margin: the single bounded re-read (5 + 21 + 16) is still short of the cap because of them. + var margin = Enumerable.Range(0, 200).Select(i => (At(30).AddTicks(-1 - i), (double)i, false)); var fake = Aveva(margin.Append((At(35), 1d, false)).ToList()); var ds = Source(fake); + await Should.ThrowAsync(() => ds.ReadRawAsync("T", At(30), At(40), 5, Ct)); + + fake.ReadRawCallCount.ShouldBe(2); // one bounded re-read, never more + fake.RawStreamsCompleted.ShouldBe(2); ds.GetHealthSnapshot().TotalFailures.ShouldBe(0); - fake.RawStreamsCompleted.ShouldBe(1); + } + + [Fact] + public async Task A_short_read_is_complete_and_is_not_re_read() + { + var fake = Gate(); + await Source(fake).ReadRawAsync("T", At(30), At(55), 5, Ct); + fake.ReadRawCallCount.ShouldBe(1); + } + + [Fact] + public async Task A_read_that_reached_past_the_window_end_is_complete_and_is_not_re_read() + { + // Budget 3 + 16 = 19 = boundary + 16 margin rows + the one window row + the row stored at the (exclusive) end: + // the stream is used up and the page is short, but it reached the end — every window row was seen. + var margin = Enumerable.Range(0, 16).Select(i => (At(30).AddTicks(-1 - i), (double)i, false)); + var fake = Aveva(margin.Append((At(35), 1d, false)).Append((At(40), 2d, false)).ToList()); + var ds = Source(fake); + + var r = await ds.ReadRawAsync("T", At(30), At(40), 3, Ct); + + r.Samples.Select(s => (double)s.Value!).ShouldBe([1d]); + fake.RawYieldedCount.ShouldBe(19); + fake.ReadRawCallCount.ShouldBe(1); + } + + [Theory] // M1: the raw budget is NOT clamped at the gateway's default ceiling — AVEVA always emits a boundary row, so a + // page size within 16 of the ceiling would always come back short (a strict gateway refuses instead ⇒ + // BadTooManyOperations, a default gateway serves it). + [InlineData(999_984u, 1_000_000)] + [InlineData(999_990u, 1_000_006)] + [InlineData(1_000_000u, 1_000_016)] + public async Task The_raw_row_budget_is_not_clamped_at_the_gateways_default_ceiling(uint cap, int expected) + { + var fake = Gate(); + await Source(fake).ReadRawAsync("T", At(30), At(55), cap, Ct); + fake.RawCalls.Single().MaxValues.ShouldBe(expected); + } + + // ---- L3: the forward end bound is exclusive; a point read is inclusive ---------------------------------------- + + [Fact] + public async Task A_stored_sample_exactly_at_the_window_end_is_not_returned() + { + var r = await Source(Gate()).ReadRawAsync("T", At(0), At(20), 0, Ct); + r.Samples.Select(s => s.SourceTimestampUtc).ShouldBe(new DateTime?[] { At(0), At(5) }); + } + + [Fact] + public async Task A_point_read_returns_the_samples_stored_at_that_instant() + { + var data = new List<(DateTime, double, bool)> { (At(10), 1, false), (At(20), 2, false), (At(20), 3, false), (At(30), 4, false) }; + var r = await Source(Aveva(data)).ReadRawAsync("T", At(20), At(20), 1000, Ct); + r.Samples.Select(s => (double)s.Value!).ShouldBe([2d, 3d]); + } + + // ---- the Unix-epoch floor ---------------------------------------------------------------------------------------- + + [Theory] // the gateway's time encoding starts at the Unix epoch: the 1 ms margin never reaches before it + [InlineData(0L)] + [InlineData(5_000L)] + public void The_request_start_never_precedes_the_Unix_epoch(long ticksAfterEpoch) + { + var start = DateTime.UnixEpoch.AddTicks(ticksAfterEpoch); + RawAggregateReader.RequestStartFor(start).ShouldBe(DateTime.UnixEpoch); + RawAggregateReader.RequestStartFor(At(0)).ShouldBe(At(0) - RawAggregateReader.BoundaryMargin); } }