From 186b7d5ea5b1badb0b8d363f80351528c2d7f116 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Mon, 12 Oct 2026 16:32:00 -0400 Subject: [PATCH] fix(historian): #691.5 a reverse walk proves the rest empty once and carries it page to page The last pages of a deep reverse page-through on a forward-only backend each walked down to 1970 one x4 growth step at a time (~14-16 reads, ~2.5 s live). After two empty segments the walk now asks once per page for the whole remainder at the page budget (a truncated answer is discarded and the walk carries on), and the stretch the page's last complete read proved empty rides the continuation point (HistoryContinuationState.EmptyFromTicks/EmptyToTicks, session-held, so no versioning) so the next page steps over it. Fuzz now pages with the walk's own hints and with true random empty stretches. --- CLAUDE.md | 2 +- docs/Historian.md | 10 +- .../HistoryPaging.cs | 15 ++- .../OtOpcUaNodeManager.cs | 14 +-- .../ReverseWalk.cs | 81 ++++++++++++++-- .../NodeManagerHistoryReadPagingTests.cs | 32 ++++++ .../ReverseWalkTests.cs | 97 ++++++++++++++++--- 7 files changed, 222 insertions(+), 29 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index af88cdb5..ce8f2845 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1353,7 +1353,7 @@ historian tagname, which is the tag's driver `FullName`. - **SQL Server** (`Driver.Historian.SqlServer`): `hist.Sample` PK `(TagId, SourceTimeUtc)` — writes are insert-if-absent, so outbox redelivery is idempotent. Stores every value kind with its **real status**; per-sample aggregates (10, simple definitions) are pushed down to T-SQL with bucket math in **100 ns ticks**; the Part 13 time-weighted seven (TimeAverage/TimeAverage2/Total/Total2/Interpolative, #671 — ⚠️ `Total` is now the time-weighted integral, no longer `SUM` — plus the **duration-based** PercentGood/PercentBad, #680) stream window + 4 bound rows into the shared `TimeWeightedAggregator` (both SQL backends, identical results; reverse `Interpolative` anchored at the later time; window capped by `ServerHistorian:MaxTimeWeightedRawRows` ⇒ `BadTooManyOperations`, percentages included). ⚠️ **#680 status rules (client-visible):** TimeAverage/Total are Uncertain if any part of the interval uses an Uncertain value, is bridged across a Bad sample, is extrapolated past the last sample (⇒ a live tag's latest interval is Uncertain) or is not covered; TimeAverage2/Total2 keep the SDK time-based rule (Uncertain time counts as Good) but never extrapolate past the last sample — that interval is Partial, later ones `BadNoData` (a report-by-exception tag with nothing after its last sample reads `BadNoData` until its next sample arrives). `docs/Historian.md` §"Time-weighted aggregates". ⚠️ **#680 per-sample (client-visible):** `Count` counts **Good** samples only (value-based status; an all-Bad interval is generic Bad, value null) and an empty interval **inside the tag's data range** is `0`/Good (outside it still `BadNoData`); `Start`/`End` return the **raw** first/last sample of any quality — its own timestamp and status, value null when Bad; `Delta` is Uncertain only when the interval's first or last sample is unusable; ⚠️ #684: `Average`/`Minimum`/`Maximum`/`Range`/`*ActualTime` use **Good samples only** (was Good + Uncertain; an Uncertain-only interval is `BadNoData`), Min/Max-family status Uncertain on a Bad sample, an Uncertain value beyond the Good extreme (Part 13 — the SDK ignores that) or an Uncertain non-numeric sample, and only a **Good** String/DateTime sample refuses them (`BadAggregateInvalidInputs`; Delta still refuses any usable one); and every per-sample aggregate but `Average` sets **end-of-data Partial** on the interval holding the last sample (forward incl. its earlier edge, reverse strictly inside), never on a Bad result. The data-range lookup is a key seek on SQL Server and a `tag` symbol-index seek on QuestDB (#683 — `{p}sample.tag` is `SYMBOL INDEX`; the bootstrapper ADDs it to a pre-#683 table, idempotent), read BEFORE the bucket statement so a racing append can never yield a Good `Count` of 0 (a concurrent HistoryUpdate delete is the mirror case and still can, transiently). `docs/Historian.md` §"Aggregates". Docker-dev recipe, schema and aggregate semantics: `docs/Historian.md` §"SQL Server backend". Integration tests need `HISTORIAN_SQL_TEST_CONNECTION`. **Retention (#667):** opt-in `SqlServer:RetentionDays` (+ `PurgeInterval` 1h, `PurgeBatchSize` 4000) -> `HistorianRetentionService` purges old `hist.Sample` (per tag) / `hist.Event` in `DELETE TOP(n)` batches, Primary-gated on `IHistorizationRoleView`, one purger per DB via `sp_getapplock`; counter `otopcua.historian.retention.purged{table}`. Optional `IHistorianBackendProvider.CreateRetention` (default null). `docs/Historian.md` §Retention. - **QuestDB** (`Driver.Historian.QuestDb`, `ServerHistorian:QuestDb:*`, Npgsql over PG wire; docker-dev overlay `docker-compose.questdb.yml`): same kinds/status/aggregates/HistoryUpdate as SqlServer, soft delete by tombstone (INSERT only — never `UPDATE`/`DELETE`; every read filters `NOT deleted`). ⚠️ **Timestamps are µs** (`QuestDbConnection.Round`/`BindTimestamp`; `IHistorianDataSource.TimestampResolutionTicks` = 10 steps HistoryRead paging), **pre-1970 cannot be stored** (writer drops, HistoryUpdate `BadOutOfRange`), every write waits for WAL visibility, TTL is enforced on write commits only, set Npgsql `Maximum Pool Size` < QuestDB's 64. `docs/Historian.md` §"QuestDB backend". - ⚠️ **Reverse processed reads are Part 13/SDK-anchored (#677, client-visible):** every aggregate on SqlServer/QuestDB now anchors a reverse read at its later time — window `(end, start]`, partial interval earliest, each stamped with its later edge (was: earlier-anchored, stamped earlier edge); on forward reads Partial also marks the interval the data starts inside (not Average — the SDK's rule, forward-only); a non-default client `AggregateConfiguration` ⇒ `BadAggregateConfigurationRejected`; Gateway provisioning sends `IntegralDivisor = 1`. SDK-parity tests: `SdkAggregateParityTests`. `docs/Historian.md` §Aggregates. -- ⚠️ **Reverse / end-only RAW reads work on Gateway (#686, client-visible):** `IHistorianDataSource.SupportsReverseRaw` (SqlServer/QuestDB/Null `true`, Gateway `false`, interface default `false`, guard-tested) — the node manager sends `start > end` only to a backend that declares it; otherwise `ReverseRawReader` serves it from FORWARD reads: uncapped ⇒ ONE forward read reversed in memory; capped ⇒ a bounded walk back in segments (ms-aligned request bounds, width aimed from observed density at ≈ N + 8 rows and carried page to page in the continuation point — #689: it used to aim at half `MaxTieClusterOverfetch`, ~32k rows per page on a dense tag; a read asks for `2N + 18` rows (a discovery segment ≤ 1,026), `MaxTieClusterOverfetch` is only the tie-cluster ceiling — a full read is re-read, never answered truncated; only one timestamp holding more than max(1, `MaxTieClusterOverfetch`) samples ⇒ `BadTooManyOperations`; walks no later than now + 1 day and never before the Unix epoch). The raw/events tie-path oversized-cluster refusal is now `BadTooManyOperations` too (was `BadHistoryOperationUnsupported`). Uncapped Gateway raw reads (forward too) used to send `max_values = 0`, which the client/gateway reject — now unbounded (`int.MaxValue`; a ceiling refusal ⇒ `BadTooManyOperations`), and a cancelled read is no longer a backend failure. Since #669 such reads failed the node on Gateway and counted as a backend fault (the gateway rejects start > end — `GatewayHistorianDataSource.ReadRawAsync` now refuses one unsent); before #669 an end-only Gateway read returned the OLDEST N ascending — now the latest N at/before EndTime, newest first. Events still differ (end-only forward / explicit reverse refused on Gateway, #675). `docs/Historian.md` §"Open-ended and reverse raw reads". +- ⚠️ **Reverse / end-only RAW reads work on Gateway (#686, client-visible):** `IHistorianDataSource.SupportsReverseRaw` (SqlServer/QuestDB/Null `true`, Gateway `false`, interface default `false`, guard-tested) — the node manager sends `start > end` only to a backend that declares it; otherwise `ReverseRawReader` serves it from FORWARD reads: uncapped ⇒ ONE forward read reversed in memory; capped ⇒ a bounded walk back in segments (ms-aligned request bounds, width aimed from observed density at ≈ N + 8 rows and carried page to page in the continuation point — #689: it used to aim at half `MaxTieClusterOverfetch`, ~32k rows per page on a dense tag; a read asks for `2N + 18` rows (a discovery segment ≤ 1,026), `MaxTieClusterOverfetch` is only the tie-cluster ceiling — a full read is re-read, never answered truncated; only one timestamp holding more than max(1, `MaxTieClusterOverfetch`) samples ⇒ `BadTooManyOperations`; walks no later than now + 1 day and never before the Unix epoch; #691: after two empty segments it asks once for the whole remainder, and the stretch a page proved empty rides the continuation point — the last pages of a deep page-through no longer each walk ~14 reads to 1970). The raw/events tie-path oversized-cluster refusal is now `BadTooManyOperations` too (was `BadHistoryOperationUnsupported`). Uncapped Gateway raw reads (forward too) used to send `max_values = 0`, which the client/gateway reject — now unbounded (`int.MaxValue`; a ceiling refusal ⇒ `BadTooManyOperations`), and a cancelled read is no longer a backend failure. Since #669 such reads failed the node on Gateway and counted as a backend fault (the gateway rejects start > end — `GatewayHistorianDataSource.ReadRawAsync` now refuses one unsent); before #669 an end-only Gateway read returned the OLDEST N ascending — now the latest N at/before EndTime, newest first. Events still differ (end-only forward / explicit reverse refused on Gateway, #675). `docs/Historian.md` §"Open-ended and reverse raw reads". - **Advertised aggregates (#682):** `IHistorianDataSource.SupportedAggregates` (Gateway 4 via `GatewayHistorianDataSource.ServedAggregates`, SqlServer/QuestDB 17 via `AggregateFamilies.Served`, Null **none**; a null set ⇒ all) is the one authority for `HistoryServerCapabilities/AggregateFunctions` (i=11201 — only its folder references are trimmed) and the up-front `BadAggregateNotSupported` in HistoryRead-Processed. ⚠️ `ServerCapabilities/AggregateFunctions` (i=2997) is deliberately NOT trimmed: it keeps the SDK's 37 because live `AggregateFilter` subscriptions still compute them all. ⚠️ Historian disabled ⇒ Processed is now `BadAggregateNotSupported`, not `GoodNoData`. `docs/Historian.md` §Advertised aggregates. - **Real status fix:** the recorder used to stamp every sample Good (`192`); it now carries the driver's status. ⚠️ On the **Gateway** backend this changes nothing stored: the gateway discards quality (C-002), so AVEVA still stores Good. - **Comms loss is historized as an outage marker** (`Empty` + `BadNoCommunication`, one per tag per outage) on SqlServer; dropped (`DroppedUnsupportedKind`) on Gateway. Outbox records are **format v2** — drain or delete `ContinuousHistorization:OutboxPath` before downgrading a node. diff --git a/docs/Historian.md b/docs/Historian.md index cac9ea6a..82b68546 100644 --- a/docs/Historian.md +++ b/docs/Historian.md @@ -659,11 +659,19 @@ backend: rows of its burst over and over). Only **one timestamp holding more than `M` samples** is refused, **`BadTooManyOperations`** (a request limit, not a backend fault); a cluster of exactly `M`, or neighbouring clusters each within `M`, is served. + - **Proving the rest empty (#691).** Walking down through empty history to the floor used to cost one read per ×4 + growth step — ~14–16 reads (~2.5 s live) on **every** last page of a deep page-through. Now, after two consecutive + empty segments, the walk asks **once per page** for the whole remainder down to the floor at the page budget: a + complete answer ends the walk in one read; a truncated one (deep history) is discarded and the walk carries on. And + the stretch the page's last complete read proved empty below its oldest row (often all the way to the floor) rides + the continuation point, so the next page steps straight over it. Data backfilled into such a stretch between pages + is not seen by that read — paging was never a snapshot (the forward path misses data inserted behind its cursor too). - **Bounds.** The walk starts no later than **now + 1 day** (a `StartTime = DateTime.MaxValue` read does not send year-9999 query starts) and never reads before the **Unix epoch** (AVEVA holds nothing earlier): a walked-back read returns only samples strictly after 1970-01-01T00:00:00Z and no later than now + 1 day. - Paging is unchanged — a reverse cursor resumes as a fresh walk from its resume point. + Paging is otherwise unchanged — a reverse cursor resumes as a fresh walk from its resume point, carrying the width and + the proven-empty stretch (session-held cursor state, never serialized, so the new fields need no versioning). - **Null** reports `true` (empty in either direction). > ⚠️ **Client-visible (#686).** Since #669 an end-only capped raw read, and any explicit reverse raw read, on the Gateway diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/HistoryPaging.cs b/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/HistoryPaging.cs index fe825f0e..595affbb 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/HistoryPaging.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/HistoryPaging.cs @@ -55,6 +55,13 @@ namespace ZB.MOM.WW.OtOpcUa.OpcUaServer; /// reverse reads on a forward-only backend only (Gitea #686): the segment width the /// walk-back ended on, so the next page starts at the observed density instead of re-discovering it. 0 = none. /// +/// +/// Walked reverse reads only (Gitea #691): with , a stretch (from, to) the previous +/// page's walk proved holds no row, so the next page steps over it rather than re-walking it (to the 1970 floor, ~14 +/// reads, at the end of a deep page-through). 0/0 = none. The cursor lives in the session's in-memory store (never +/// serialized), so a new field with a "none" default is all the versioning it needs. +/// +/// The exclusive upper end of that stretch. internal sealed record HistoryContinuationState( string Tagname, DateTime NextStartUtc, @@ -65,7 +72,13 @@ internal sealed record HistoryContinuationState( long LastModifiedId = 0, DateTime WindowStartUtc = default, bool Reverse = false, - long SegmentWidthTicks = 0); + long SegmentWidthTicks = 0, + long EmptyFromTicks = 0, + long EmptyToTicks = 0) +{ + /// The walk-back hint this cursor carries (#686/#689/#691). + public ReverseWalkHint WalkHint => new(SegmentWidthTicks, EmptyFromTicks, EmptyToTicks); +} /// The HistoryRead kind that issued a . internal enum HistoryCursorKind diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/OtOpcUaNodeManager.cs b/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/OtOpcUaNodeManager.cs index 7cecc9b4..6b6c5dca 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/OtOpcUaNodeManager.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/OtOpcUaNodeManager.cs @@ -6184,7 +6184,7 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2 { DateTime startUtc; var boundarySkip = 0; - long segmentWidth = 0; // #686: the walk-back's width, carried page to page + var walkHint = default(ReverseWalkHint); // #686/#689/#691: the walk-back's width + proven-empty stretch, page to page string tagname; if (inboundCp is { Length: > 0 }) @@ -6205,7 +6205,7 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2 boundarySkip = state.BoundarySkipCount; endUtc = state.EndUtc; numValuesPerNode = state.NumValuesPerNode; - segmentWidth = state.SegmentWidthTicks; + walkHint = state.WalkHint; } else { @@ -6243,9 +6243,9 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2 { var page = await ReverseRawReader.ReadAsync( source, tagname, startUtc, endUtc, numValuesPerNode, MaxTieClusterOverfetch, - Clock.GetUtcNow().UtcDateTime, segmentWidth, ct) + Clock.GetUtcNow().UtcDateTime, walkHint, ct) .ConfigureAwait(false); - segmentWidth = page.WidthHintTicks; + walkHint = page.Hint; sourceResult = new HistorianRead(page.Samples, ContinuationPoint: null); } else @@ -6324,7 +6324,8 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2 if (nextStartUtc is { } resumeAt) { var clusterState = new HistoryContinuationState( - tagname, resumeAt, endUtc, nextSkip, numValuesPerNode, SegmentWidthTicks: segmentWidth); + tagname, resumeAt, endUtc, nextSkip, numValuesPerNode, SegmentWidthTicks: walkHint.WidthTicks, + EmptyFromTicks: walkHint.EmptyFromTicks, EmptyToTicks: walkHint.EmptyToTicks); clusterCp = _historyContinuationStore.Save(session, clusterState); } @@ -6347,7 +6348,8 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2 { HistoryPaging.ComputeResumeCursor(samples, out var nextStart, out var skip); var nextState = new HistoryContinuationState( - tagname, nextStart, endUtc, skip, numValuesPerNode, SegmentWidthTicks: segmentWidth); + tagname, nextStart, endUtc, skip, numValuesPerNode, SegmentWidthTicks: walkHint.WidthTicks, + EmptyFromTicks: walkHint.EmptyFromTicks, EmptyToTicks: walkHint.EmptyToTicks); // Save may return null (no session on this request) ⇒ degrade to single-shot for this node. // Built AFTER historyData so a failure projecting samples can never orphan a stored cursor. outboundCp = _historyContinuationStore.Save(session, nextState); diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/ReverseWalk.cs b/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/ReverseWalk.cs index 94d1ecfa..f68f167e 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/ReverseWalk.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/ReverseWalk.cs @@ -2,7 +2,17 @@ namespace ZB.MOM.WW.OtOpcUa.OpcUaServer; /// What one walked-back page tells the next page of the same read (carried in the continuation point). /// The segment width the walk ended on, aimed at the page's density — the next page's first width (0 = none). -internal readonly record struct ReverseWalkHint(long WidthTicks); +/// +/// With : a stretch (EmptyFrom, EmptyTo) (both exclusive) a complete segment read +/// proved holds no row (#691) — typically from the oldest row the page saw down to the walk's floor. A resumed page +/// steps straight over it instead of re-walking it. Both 0 = none. +/// +/// The exclusive upper end of the empty stretch. +internal readonly record struct ReverseWalkHint(long WidthTicks, long EmptyFromTicks = 0, long EmptyToTicks = 0) +{ + /// Whether an empty stretch is carried. + public bool HasEmptyStretch => EmptyToTicks > EmptyFromTicks && EmptyToTicks > 0; +} /// One forward read of a segment: the rows (ascending by time) and whether a cap cut the read short. /// Row type. @@ -46,6 +56,15 @@ internal readonly record struct ForwardSegmentRead(IReadOnlyList Rows, boo /// fit at the tie budget — is refused. /// /// +/// Proving the rest empty (#691). Walking down to the floor through empty history used to take one read per ×4 +/// growth step (~14 reads from a minute to the epoch) — on every page that ran out of rows. After two consecutive empty +/// segments the walk asks once per page for the WHOLE remainder (lo, segHi] at the page budget: a complete +/// answer ends the walk in one read; a truncated one (deep history) is discarded and the walk carries on as before. And +/// the stretch a page's last complete segment proved empty below its oldest row rides the continuation point +/// (), so the next page steps over it — the rows of a page are often the +/// last ones before a stretch the previous page already read. +/// +/// /// Edges. A segment is requested as [⌊segLo⌋, ⌈segHi + 1 tick⌉], both rounded outward to the millisecond /// (the gateway's / AVEVA's resolution), and filtered to (segLo, segHi] on the rows' own timestamps — exact /// whether the backend's forward end is exclusive or inclusive and whether it truncates its bounds; a row fabricated at @@ -141,14 +160,27 @@ internal static class ReverseWalk var knownWidth = 0L; var segHi = hiTicks; + // #691: the remainder probe (once per page, after two empty segments) and the empty stretches. + var emptyRun = 0; + var probed = false; + var carried = hint.HasEmptyStretch ? (hint.EmptyFromTicks, hint.EmptyToTicks) : (0L, 0L); + (long From, long To) proven = (0L, 0L); + while (segHi > stopTicks && result.Count < want) { - var w = width; + if (carried.Item2 > carried.Item1 && segHi > carried.Item1 && segHi < carried.Item2) + { + segHi = carried.Item1; // (from, segHi] lies inside a stretch a previous page proved empty + continue; + } + + var probe = !probed && known == 0 && emptyRun >= 2 && segHi - stopTicks > width; + var w = probe ? segHi - stopTicks : width; if (known > 0) w = Math.Min(w, Math.Max(1, (segHi - known) / 2)); var segLo = segHi - stopTicks <= w ? stopTicks : segHi - w; var start = Utc(AlignDown(segLo, granularity)); var end = Utc(AlignUp(segHi + 1, granularity)); - var segBudget = discovering ? discoveryBudget : pageBudget; + var segBudget = discovering && !probe ? discoveryBudget : pageBudget; var cap = segBudget + 2; IReadOnlyList rows; var narrowed = false; @@ -160,6 +192,14 @@ internal static class ReverseWalk rows = segment.Rows; if (!segment.Truncated || rows.Count == 0 || ticksOf(rows[^1]) > segHi) break; // complete for (segLo, segHi] + if (probe) + { + // Deep history below: the probe saw only its oldest rows. Carry on walking from the same segHi. + probed = true; + narrowed = true; + break; + } + var leading = 0; var inner = 0; var firstInner = 0L; @@ -226,8 +266,18 @@ internal static class ReverseWalk if (narrowed) continue; // re-read the same segHi, narrower - var count = AppendNewestFirst(rows, ticksOf, tieOrder, segLo, segHi, result, (int)Math.Min(int.MaxValue, want)); + var count = AppendNewestFirst(rows, ticksOf, tieOrder, segLo, segHi, result, (int)Math.Min(int.MaxValue, want), + out var oldest); + // This complete read proves (segLo, oldest) empty — or the whole segment when it held nothing. + proven = count > 0 ? (segLo, oldest) : (segLo, segHi + 1); + if (probe) + { + probed = true; + segHi = segLo; // the whole remainder is read + continue; + } + emptyRun = count == 0 ? emptyRun + 1 : 0; var grown = width > maxWidth / 4 ? maxWidth : width * 4; if (count == 0) { @@ -253,13 +303,31 @@ internal static class ReverseWalk } } - return (result, new ReverseWalkHint(width)); + // Carry the stretch the last complete read proved empty, joined to the one carried in when they touch; when they + // do not, keep the one reaching the floor (it saves a whole walk), else the newer. + var next = carried; + if (proven.To > proven.From) + { + if (carried.Item2 > carried.Item1 && proven.From < carried.Item2 && carried.Item1 < proven.To) + next = (Math.Min(proven.From, carried.Item1), Math.Max(proven.To, carried.Item2)); + else if (!(carried.Item2 > carried.Item1 && carried.Item1 <= stopTicks) || proven.From <= stopTicks) + next = proven; + } + + return (result, new ReverseWalkHint(width, next.Item1, next.Item2)); } /// Appends the rows in (segLo, segHi] newest-first (until ) and counts them. private static int AppendNewestFirst( - IReadOnlyList rows, Func ticksOf, IComparer? tieOrder, long segLo, long segHi, List result, int limit) + IReadOnlyList rows, Func ticksOf, IComparer? tieOrder, long segLo, long segHi, List result, int limit) => + AppendNewestFirst(rows, ticksOf, tieOrder, segLo, segHi, result, limit, out _); + + /// As above, also returning the oldest in-range timestamp (meaningless when none). + private static int AppendNewestFirst( + IReadOnlyList rows, Func ticksOf, IComparer? tieOrder, long segLo, long segHi, List result, int limit, + out long oldest) { + oldest = long.MaxValue; IReadOnlyList ordered = rows; if (tieOrder is not null) { @@ -278,6 +346,7 @@ internal static class ReverseWalk var ts = ticksOf(ordered[i]); if (ts <= segLo || ts > segHi) continue; count++; + if (ts < oldest) oldest = ts; if (result.Count < limit) result.Add(ordered[i]); } diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.Tests/NodeManagerHistoryReadPagingTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.Tests/NodeManagerHistoryReadPagingTests.cs index ae063d0d..b25afb52 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.Tests/NodeManagerHistoryReadPagingTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.Tests/NodeManagerHistoryReadPagingTests.cs @@ -739,6 +739,38 @@ public sealed class NodeManagerHistoryReadPagingTests : IDisposable await host.DisposeAsync(); } + /// + /// #691: the last pages of a deep reverse page-through no longer each walk down to 1970 (live: ~14–16 reads, ~2.5 s + /// per page). The stretch a page proved empty below its oldest row rides the continuation point, and a page that runs + /// out of rows asks once for the whole remainder — so every page here is a couple of reads. + /// + [Fact] + public async Task Raw_reverse_page_through_on_a_forward_only_backend_does_not_rewalk_to_the_epoch_on_each_last_page() + { + var source = new ForwardOnlyHistorianDataSource(MakeSeries(count: 25, stepSeconds: 1)); + var (host, nm, nodeId) = await BootForwardOnlyAsync(source, "FwdDeep691"); + nm.Clock = new FixedClock(Epoch.AddHours(1)); + + var collected = new List(); + byte[]? cp = null; + var page = 0; + do + { + source.ResetCounts(); + var (values, error, next) = ReadRaw(nm, nodeId, DateTime.MinValue, Epoch.AddHours(1), max: 10, inboundCp: cp); + error.StatusCode.Code.ShouldBe(StatusCodes.Good); + collected.AddRange(values); + cp = next; + page++; + source.ReadCount.ShouldBeLessThanOrEqualTo(page == 1 ? 4 : 1, $"page {page}"); + } + while (cp is not null && page < 10); + + collected.ShouldBe(Enumerable.Range(0, 25).Reverse().Select(i => (double)i)); + source.MinRequestedStart.ShouldBeGreaterThanOrEqualTo(DateTime.UnixEpoch); + await host.DisposeAsync(); + } + /// #686 review: a reverse read from StartTime = DateTime.MaxValue walks back from now + 1 day, not from /// year 9999. [Fact] diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.Tests/ReverseWalkTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.Tests/ReverseWalkTests.cs index 3cb0370d..d44c20c2 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.Tests/ReverseWalkTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.Tests/ReverseWalkTests.cs @@ -46,23 +46,44 @@ public sealed class ReverseWalkTests 1 => new ReverseWalkHint(rng.Next(1, 10)), _ => new ReverseWalkHint((long)(rng.NextDouble() * TimeSpan.TicksPerHour)), }; - - var expected = series.Where(s => Ts(s) > Math.Max(lo, DateTime.UnixEpoch.Ticks) && Ts(s) <= hi) - .Select((s, i) => (s, i)).OrderByDescending(x => Ts(x.s)).ThenByDescending(x => x.i) - .Take((int)want).Select(x => (double)x.s.Value!).ToList(); - - try + if (rng.Next(2) == 0) { - var page = await ReverseRawReader.ReadAsync(source, "T", new DateTime(hi, DateTimeKind.Utc), - new DateTime(lo, DateTimeKind.Utc), want, tieBudget, T0.AddDays(1), hint, Ct); - var got = page.Samples.Select(s => (double)s.Value!).ToList(); - if (!got.SequenceEqual(expected)) - failures.Add($"seed {seed}: got [{string.Join(",", got)}] expected [{string.Join(",", expected)}]"); - if (page.Hint.WidthTicks <= 0) failures.Add($"seed {seed}: no width carried"); + // A TRUE empty stretch (#691): between two adjacent distinct timestamps, or from the floor to the first row. + var k = rng.Next(series.Count); + var from = k == 0 ? (rng.Next(2) == 0 ? 0 : DateTime.UnixEpoch.Ticks) : Ts(series[k - 1]); + var to = Ts(series[k]); + if (to > from) hint = hint with { EmptyFromTicks = from, EmptyToTicks = to }; } - catch (HistoryReadLimitExceededException) + + // Page on: each next page resumes just below the previous page's oldest timestamp with the hint it returned — + // so the hints the walk itself produces (width, proven-empty stretch) are exercised too. + for (var pageNo = 0; pageNo < 6 && hi > Math.Max(lo, DateTime.UnixEpoch.Ticks); pageNo++) { - if (maxCluster <= tieBudget) failures.Add($"seed {seed}: refused with no cluster over {tieBudget} (max {maxCluster})"); + var expected = series.Where(s => Ts(s) > Math.Max(lo, DateTime.UnixEpoch.Ticks) && Ts(s) <= hi) + .Select((s, i) => (s, i)).OrderByDescending(x => Ts(x.s)).ThenByDescending(x => x.i) + .Take((int)want).Select(x => (double)x.s.Value!).ToList(); + + try + { + var page = await ReverseRawReader.ReadAsync(source, "T", new DateTime(hi, DateTimeKind.Utc), + new DateTime(lo, DateTimeKind.Utc), want, tieBudget, T0.AddDays(1), hint, Ct); + var got = page.Samples.Select(s => (double)s.Value!).ToList(); + if (!got.SequenceEqual(expected)) + { + failures.Add($"seed {seed} page {pageNo}: got [{string.Join(",", got)}] expected [{string.Join(",", expected)}]"); + break; + } + + if (page.Hint.WidthTicks <= 0) failures.Add($"seed {seed}: no width carried"); + if (got.Count < want) break; + hint = page.Hint; + hi = Ts(page.Samples[^1]) - 1; + } + catch (HistoryReadLimitExceededException) + { + if (maxCluster <= tieBudget) failures.Add($"seed {seed}: refused with no cluster over {tieBudget} (max {maxCluster})"); + break; + } } if (source.MaxCap > 2L * Math.Max(Math.Max(tieBudget, ReverseWalk.DiscoveryRows), 2 * want + 16) + 4) @@ -116,6 +137,54 @@ public sealed class ReverseWalkTests source.RowsReturned.ShouldBeLessThanOrEqualTo(25 * (ReverseWalk.DiscoveryRows + 2)); } + /// + /// #691: the end of a walk proves the rest of the window empty in about one read — after two empty segments the walk + /// asks once for the whole remainder down to the floor — instead of one read per ×4 growth step to 1970 (~14 reads). + /// + [Fact] + public async Task Running_out_of_rows_proves_the_rest_of_the_window_empty_in_a_few_reads() + { + var series = Enumerable.Range(0, 5).Select(i => At(T0.Ticks + i * TimeSpan.TicksPerSecond, i)).ToList(); + var source = new AvevaShapedSource(series); + + var page = await ReverseRawReader.ReadAsync(source, "T", T0.AddSeconds(4), DateTime.MinValue, 10, 65536, T0, + default(ReverseWalkHint), Ct); + + page.Samples.Select(s => (double)s.Value!).ShouldBe([4d, 3d, 2d, 1d, 0d]); + source.Reads.ShouldBeLessThanOrEqualTo(4, "one segment holding the rows, two empty ones, then the remainder at once"); + } + + /// + /// #691: a page's last complete read proves the stretch below its oldest row empty, and the continuation carries it: + /// the next page steps straight over it to the floor instead of walking it again. + /// + [Fact] + public async Task A_resumed_page_steps_over_the_stretch_the_previous_page_proved_empty() + { + var series = Enumerable.Range(0, 25).Select(i => At(T0.Ticks + i * TimeSpan.TicksPerSecond, i)).ToList(); + var source = new AvevaShapedSource(series); + var now = T0.AddHours(1); + + // Page 1: an hour of nothing above the rows; the remainder probe reads all 25 and proves (1970, row 0) empty. + var p1 = await ReverseRawReader.ReadAsync(source, "T", now, DateTime.MinValue, 10, 65536, now, default(ReverseWalkHint), Ct); + p1.Samples.Select(s => (double)s.Value!).ShouldBe(Enumerable.Range(15, 10).Reverse().Select(i => (double)i)); + p1.Hint.HasEmptyStretch.ShouldBeTrue(); + p1.Hint.EmptyFromTicks.ShouldBeLessThanOrEqualTo(DateTime.UnixEpoch.Ticks); + p1.Hint.EmptyToTicks.ShouldBe(Ts(series[0])); + + var hint = p1.Hint; + var hi = p1.Samples[^1].SourceTimestampUtc!.Value.AddTicks(-1); + for (var page = 2; page <= 3; page++) + { + source.Reset(); + var p = await ReverseRawReader.ReadAsync(source, "T", hi, DateTime.MinValue, 10, 65536, now, hint, Ct); + p.Samples.Count.ShouldBe(page == 2 ? 10 : 5); + source.Reads.ShouldBe(1, $"page {page} reads its rows and steps over the proven-empty stretch to the floor"); + hint = p.Hint; + hi = p.Samples[^1].SourceTimestampUtc!.Value.AddTicks(-1); + } + } + private static long Ts(DataValueSnapshot s) => s.SourceTimestampUtc!.Value.Ticks; /// Random ascending series: bursts, gaps, sub-millisecond stamps and tie clusters.