Compare commits
14 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 5897764024 | |||
| 97668d26d3 | |||
| 66a2b0c3e9 | |||
| de8bc7a12c | |||
| 3c3513ea3e | |||
| a5a0496176 | |||
| 2609c5462e | |||
| 68a1f31397 | |||
| aa923cea83 | |||
| e04e1bf127 | |||
| 0d249f3ee9 | |||
| 3d7eea6280 | |||
| 8dfa165110 | |||
| 37390139fe |
+187
-19
@@ -112,6 +112,8 @@ Processed read is `BadAggregateNotSupported` and no history aggregate functions
|
||||
"CallTimeout": "00:00:30",
|
||||
"MaxTieClusterOverfetch": 65536,
|
||||
"HonorReturnBounds": true,
|
||||
"GatewayAtTimeFromRaw": true,
|
||||
"EventReadHorizon": "00:00:00",
|
||||
"HistoryReadBatchParallelism": 4,
|
||||
"MaxConcurrentHistoryReadBatches": 16,
|
||||
"HistoryReadDeadline": "00:01:00",
|
||||
@@ -144,8 +146,10 @@ Processed read is `BadAggregateNotSupported` and no history aggregate functions
|
||||
| `AllowUntrustedServerCertificate` | bool | `false` | **Gateway backend only.** Accept a self-signed / untrusted server certificate (dev / on-prem only). |
|
||||
| `CaCertificatePath` | string\|null | `null` | **Gateway backend only.** PEM CA file pinning the gateway's TLS chain. Null/empty uses the OS trust store. |
|
||||
| `CallTimeout` | TimeSpan | `00:00:30` | **Gateway backend only.** Per-call deadline applied to each unary gateway read. A read for a tag the historian does not yet know about — e.g. a freshly-historized tag that has **not** been provisioned (see [Tag auto-provisioning](#tag-auto-provisioning-ensuretags)) — can run to this **full** deadline before the gateway errors, so an unprovisioned-tag HistoryRead can block for ~30 s. Lower this to fail faster, at the cost of truncating legitimately long reads. |
|
||||
| `MaxTieClusterOverfetch` | int | `65536` | (Node-manager paging; all backends — a `SqlServer` PK rules out tie clusters, so it is moot there.) Maximum samples the server will fetch in one shot to page through a tie cluster (multiple samples sharing one `SourceTimestamp`). A cluster larger than this ceiling fails `BadTooManyOperations` (#686; was `BadHistoryOperationUnsupported`). Also the per-read row budget of a Gateway reverse raw read (walked back from forward reads, #686): one timestamp holding more samples is refused `BadTooManyOperations`. Values below 1 count as 1. Raise to handle abnormally large tie clusters; the default covers all normal-data cases. |
|
||||
| `MaxTieClusterOverfetch` | int | `65536` | (Node-manager paging; all backends — a `SqlServer` PK rules out tie clusters, so it is moot there.) Maximum samples the server will fetch in one shot to page through a tie cluster (multiple samples sharing one `SourceTimestamp`). A cluster larger than this ceiling fails `BadTooManyOperations` (#686; was `BadHistoryOperationUnsupported`). Also the per-read row budget of a Gateway reverse raw read (walked back from forward reads, #686): one timestamp holding more samples is refused `BadTooManyOperations`. Also (Gateway, #694 R2) the most rows inside the 1 ms look-back before a raw window that a crowding re-read makes room for. Values below 1 count as 1. Raise to handle abnormally large tie clusters; the default covers all normal-data cases. |
|
||||
| `HonorReturnBounds` | bool | `true` | (#691, all backends.) Whether HistoryRead-Raw honours `ReturnBounds` (Part 11 bounding values, [Bounding values](#bounding-values-returnbounds-691)). `false` is the kill switch back to the pre-#691 behaviour: the flag is ignored (stored samples only) and a modified read carrying it is not refused `Bad_InvalidArgument`. Note the .NET SDK defaults both `ReturnBounds` and `IsReadModified` to `true`. |
|
||||
| `GatewayAtTimeFromRaw` | bool | `true` | (#693, **Gateway backend only.**) Whether HistoryRead-AtTime is served from raw gateway reads evaluated with the SQL backends' AtTime rules ([Gateway AtTime](#gateway-attime-is-evaluated-from-raw-reads-693)). `false` is the kill switch back to the pre-#693 answer: AVEVA's own interpolation (the gateway's `ReadAtTime`) passed through unchanged. |
|
||||
| `EventReadHorizon` | TimeSpan | `00:00:00` | (#694, all backends, **opt-in, lossy**.) `00:00:00` = no horizon: every HistoryReadEvents read is exact to its own window. A positive span raises the earlier bound of every fresh events window to `now − EventReadHorizon` (the floor instant itself included), in **both** directions — a window wholly older is `GoodNoData` and is not sent; a page-through keeps the first page's floor. It exists for the Gateway, where a walked reverse / end-only read proves "nothing older" down to the Unix epoch (~3.8 s live; an unfiltered one can hit the gateway's 30 s SQL timeout) and a forward read from a far-past `StartTime` scans the same range. ⚠️ While set, events older than the horizon are **invisible** to HistoryReadEvents. Raw/Processed/AtTime are unaffected. Negative ⇒ treated as none (a `Validate()` warning). See [Continuation-point paging (Events)](#continuation-point-paging-events-669). |
|
||||
| `HistoryReadBatchParallelism` | int | `4` | Max historized nodes served **concurrently** within one HistoryRead batch (arch-review 03/S3). The SDK's `HistoryRead*` override surface is synchronous, so each arm bridges once at its boundary; this turns per-node serving from sequential (`N × CallTimeout` worst case) into `⌈N / P⌉ × CallTimeout`. Also caps the gateway-load multiplication a single client can command. `≤ 1` serves sequentially. |
|
||||
| `MaxConcurrentHistoryReadBatches` | int | `16` | Process-wide limit on HistoryRead (and HistoryUpdate) batches served **concurrently** (arch-review 03/S3). A flood of history clients degrades to `BadTooManyOperations` rather than exhausting the SDK request-thread pool. With `HistoryReadBatchParallelism` this caps total in-flight gateway reads at `MaxConcurrentHistoryReadBatches × HistoryReadBatchParallelism` (default 16 × 4 = 64). A batch that cannot get a slot within `min(HistoryReadDeadline, 5 s)` fails every handle `BadTooManyOperations`. |
|
||||
| `MaxTimeWeightedRawRows` | int | `1000000` | `SqlServer` / `QuestDb` (#671), and the `Gateway` per-sample aggregates (#681 — every per-sample read there streams raw samples, so the cap applies to the window's raw rows; see [Aggregate set](#aggregate-set-and-badaggregatenotsupported-285)). The most raw in-window samples one [time-weighted aggregate](#time-weighted-aggregates-part-13-671) read (TimeAverage, TimeAverage2, **Total**, Total2, Interpolative, and — since #680 — **PercentGood/PercentBad**) may stream into the server-side evaluator — that family is **not** pushed down to SQL. The window query is capped at this + 1 rows; one row past the cap fails **that node** with `BadTooManyOperations` (not counted as a backend fault) instead of holding a pooled connection until `HistoryReadDeadline` and returning `BadTimeout`. Bound rows outside the window do not count; per-sample aggregates are not capped on the SQL backends (pushed down). 1 000 000 ≈ 11.5 days of a 1 Hz tag. Non-positive = unbounded (a `Validate()` warning). `Gateway` (#688): the row budget it asks the gateway for (`cap + 18`) is clamped at the gateway's default `MaxValuesCeiling` (1,000,000) while the cap is at or under it, so the default works against a gateway with `AllowStreamingBeyondCeiling=false`; against a gateway with a *lower* ceiling the refused budget is retried once at the ceiling the gateway names (#691.3), so no change is needed (`ceiling − 18` only saves that round-trip). |
|
||||
@@ -569,7 +573,8 @@ node without `isHistorized` has no HistoryRead access bit and answers
|
||||
| Historized node, historian configured, time range empty | `GoodNoData` |
|
||||
| Historized node, historian NOT configured (`Enabled=false` / Null source) | `GoodNoData` (empty) for Raw / AtTime; Processed ⇒ `BadAggregateNotSupported` (#682) |
|
||||
| Non-historized node | `BadHistoryOperationUnsupported` |
|
||||
| Backend timeout or exception | `BadHistoryOperationUnsupported` per node; other nodes in the same batch are unaffected |
|
||||
| Backend transport outage (#697) | `BadNoCommunication` per node across Raw, Modified, Processed, AtTime and Events; cancelled/deadline-limited server reads retain `BadTimeout` |
|
||||
| Other backend exception / unsupported operation | `BadHistoryOperationUnsupported` per node; other nodes in the same batch are unaffected |
|
||||
| `Backend=SqlServer`, `AutoCreateSchema=false`, schema missing/behind | every read/write fails with a clear error naming the missing schema version (health `LastError`); reads answer `BadHistoryOperationUnsupported` per node; no host crash |
|
||||
|
||||
A historized node with no historian configured never returns an error status — it returns
|
||||
@@ -582,23 +587,44 @@ gateway is provisioned, without producing error spikes in connected clients.
|
||||
`NumValuesPerNode` samples with no continuation point of its own), so the server synthesises
|
||||
paging time-based:
|
||||
|
||||
- A page that returns **exactly** `NumValuesPerNode` samples (with `NumValuesPerNode > 0`) MAY
|
||||
have more behind it, so the server stores a resume cursor and returns an opaque continuation
|
||||
- Every page's backend read asks for **one lookahead row** past the page (`NumValuesPerNode + 1`, plus the boundary
|
||||
ties a resumed page is about to trim — see below). When the lookahead row comes back there is more data: the page
|
||||
returns exactly `NumValuesPerNode` values, and the server stores a resume cursor and returns an opaque continuation
|
||||
point (16 bytes). The client hands it back to fetch the next page.
|
||||
- A short page (fewer than the cap) is the last page — no continuation point.
|
||||
- When it does not, the page is the **last** — no continuation point, even when it is exactly full. (#692: a total that
|
||||
is an exact multiple of the cap used to end with an extra empty `GoodNoData` page.) The exception is an oversized tie
|
||||
cluster (below): its pages are sliced from the over-fetched cluster, its last slice is `min(N, remaining)` values and
|
||||
still carries a continuation point when the window goes on — so stepping past a cluster at the very end of the data can
|
||||
yield one empty final page.
|
||||
- `NumValuesPerNode == 0` ("all values, no limit") is never paged; the whole window returns in one
|
||||
shot.
|
||||
- The resume cursor is **tie-safe**: the next page resumes from the last returned sample's
|
||||
SourceTimestamp *inclusive* and drops the boundary samples already emitted, so samples sharing
|
||||
the boundary timestamp are neither duplicated nor skipped.
|
||||
the boundary timestamp are neither duplicated nor skipped. Because it re-reads them, a resumed page asks for
|
||||
`NumValuesPerNode + skip + 1` rows, so it still carries a full page. On a strict Gateway
|
||||
(`AllowStreamingBeyondCeiling=false`) that moves the row-ceiling edge by `skip + 1`: a `NumValuesPerNode` just under the
|
||||
learned ceiling may now cost one learning refusal (#691.3 re-asks at the ceiling). A page clamp to the learned ceiling is
|
||||
being added separately (#694).
|
||||
- ⚠️ **#692 (client-visible): every page is full** (outside an oversized tie cluster — see above). A resumed page used to ask for only `NumValuesPerNode` rows, so after
|
||||
the trim it netted `N − skip` values — and at `NumValuesPerNode = 1` **nothing**: the lone re-read boundary sample was
|
||||
trimmed, the empty "full" page was taken for a stalled tie cluster, and a `MaxTieClusterOverfetch + 1` (65,537-row)
|
||||
over-fetch stepped one tick to return an **empty page with a continuation point between every value** (live, AVEVA:
|
||||
8 values = 17 pages / 25 backend reads; on every backend, forward and reverse, with or without `ReturnBounds`). Now 8
|
||||
values at N = 1 is 8 pages and 8 reads. Clients that drained to the end were unaffected in content, only in cost; a
|
||||
client that stopped at the first empty page lost data.
|
||||
|
||||
> **Oversized tie clusters — within-timestamp paging.** When more samples share one
|
||||
> `SourceTimestamp` than the current page cap, the server detects that the cursor has stalled on
|
||||
> a tie cluster (the last returned timestamp equals the resume timestamp). It then **over-fetches
|
||||
> a tie cluster (every row of the resumed `N + skip + 1` read past the trimmed ties — the lookahead row included — still
|
||||
> sits on the resume timestamp; since #692 nothing else trips it). It then **over-fetches
|
||||
> the entire cluster** at that single timestamp up to a bounded ceiling controlled by
|
||||
> `ServerHistorian:MaxTieClusterOverfetch` (default **65 536**), then serves the cluster
|
||||
> `NumValuesPerNode` samples at a time across successive pages, advancing the cursor one tick past
|
||||
> the timestamp once the cluster is fully drained.
|
||||
> the timestamp once the cluster is fully drained. A cursor left inside the cluster is marked so (`WithinTieCluster`), and
|
||||
> its next page over-fetches the cluster directly instead of first re-reading rows it knows are ties. A drained cluster
|
||||
> resumes only while the next step is strictly before `EndTime` (truncated to the backend's resolution) — the forward
|
||||
> window is end-exclusive; resuming AT `EndTime` used to make the next read a point read that returned the samples at
|
||||
> `EndTime` as data (#692 review).
|
||||
>
|
||||
> **Reverse reads** (`start > end`, newest first) page the same way in the opposite direction: a drained
|
||||
> cluster steps the cursor one tick **down**, the read ends when that would reach the (excluded) `end`, and
|
||||
@@ -693,7 +719,8 @@ backend:
|
||||
> and counted as a backend failure. It is now sent unbounded (`int.MaxValue`); a gateway configured with
|
||||
> `AllowStreamingBeyondCeiling=false` refuses that over its `MaxValuesCeiling`; since #691.3 the read is retried once at the
|
||||
> ceiling the refusal names and answered when provably complete, else **`BadTooManyOperations`** (not a backend fault) — see
|
||||
> [#687](#gateway-raw-reads-return-stored-samples-only-687) "Strict gateway". A cancelled read (client cancel / `HistoryReadDeadline`) is no longer counted as a backend failure.
|
||||
> [#687](#gateway-raw-reads-return-stored-samples-only-687) "Strict gateway". Since #694 the HistoryRead pages such a read at the
|
||||
> learned ceiling with continuation points instead (same section, "Pages are sized to a strict gateway's ceiling"). A cancelled read (client cancel / `HistoryReadDeadline`) is no longer counted as a backend failure.
|
||||
|
||||
Continuation points are bound to the OPC UA session (the SDK's
|
||||
`ServerConfiguration.MaxHistoryContinuationPoints` cap, default 100, with oldest-eviction; points
|
||||
@@ -738,27 +765,153 @@ data.
|
||||
refusal of the remembered one. A **raised** ceiling (or a switch to streaming) is noticed by the only reads it matters to: when a
|
||||
read bounded by the remembered ceiling comes back full without proving completeness, it is re-probed **once** at its full budget
|
||||
— served ⇒ the ceiling is forgotten; refused naming a larger ceiling ⇒ that one is learned and the read re-issued at it; refused
|
||||
naming the same ceiling ⇒ the bounded result stands. No timer: a TTL would only add refusals for reads that already fit; the cost
|
||||
of the rule is one refused re-probe per genuinely over-ceiling read (which fails either way). So an uncapped read (`NumValuesPerNode = 0`, which used to be refused outright on a strict gateway, even for a 10-row
|
||||
naming the same ceiling ⇒ the bounded result stands. Raw and AtTime re-probe immediately. Per-sample aggregates (#696) reuse
|
||||
a refusal for **30 seconds** before re-probing a full stream; within that period an oversized window still fails honestly with
|
||||
`BadTooManyOperations`. A lowered ceiling is still re-learned immediately; a raise becomes visible on the next incomplete
|
||||
aggregate read after the cooldown. So an uncapped read (`NumValuesPerNode = 0`, which used to be refused outright on a strict
|
||||
gateway, even for a 10-row
|
||||
window) and a page size within 16 of the ceiling are now served when the window fits. A ceiling-bounded read is answered only
|
||||
when it is **provably complete** — the stream came back short of the ceiling, or reached `EndTime` (or, capped, filled the page);
|
||||
a stream that delivered the whole ceiling short of `EndTime` may be truncated and fails **`BadTooManyOperations`** (not a backend
|
||||
fault) — set `NumValuesPerNode` to page, or narrow the window. A refusal whose text carries no number falls back to the gateway's
|
||||
default ceiling (1,000,000). Every stream is still drained. The #686 reverse walk-back (segment budget `MaxTieClusterOverfetch + 2`)
|
||||
inherits this: against a ceiling below that budget a dense segment fails `BadTooManyOperations` instead of being narrowed.
|
||||
- **Paging (#669) is unaffected.** A full page still holds exactly `NumValuesPerNode` stored samples and carries a continuation
|
||||
default ceiling (1,000,000). Every stream is still drained.
|
||||
- ⚠️ **Pages are sized to a strict gateway's ceiling (#694, client-visible).** Until #694 a page size above a strict gateway's
|
||||
ceiling failed: with `MaxValuesCeiling = 1000`, `NumValuesPerNode = 5000` — or `0` over 3,600 rows — was `BadTooManyOperations`.
|
||||
The data source now reports the learned ceiling to the server as `IHistorianDataSource.MaxRawValuesPerRead` = **L** = ceiling − 16
|
||||
(`0` until a refusal has named a ceiling, and again after a re-probe finds the gateway streaming; `0` on every SQL backend). The
|
||||
node manager plans each page with it (`RawPageLimit`): a page's read is the page **P** + the boundary ties **k** it re-reads and
|
||||
trims + the #692 lookahead row, and with a limit that read is at most L — so **P = min(N, L − k − 1)** (uncapped: P = L − k − 1;
|
||||
with the ceiling at 1000 a first page holds 983 values, a resumed one 982), and a full page carries a continuation point like any
|
||||
other (Part 11 allows fewer than `NumValuesPerNode` with one). **This includes uncapped reads** (`NumValuesPerNode = 0`), which
|
||||
page instead of being refused; a window that fits is still one page without a continuation point. Without a limit the #692 plan
|
||||
is unchanged. Boundary ties that alone fill a read (k ≥ L − 1) go straight to the tie-cluster over-fetch — never an uncapped or
|
||||
underflowing request.
|
||||
- **First read after start-up:** the ceiling is unknown until refused once; the read that learned it is re-planned and re-issued
|
||||
ONCE (cost on that read: the refused attempt, the ceiling re-read that could not hold the window, the page-sized read; later
|
||||
pages are one read each). Reverse / end-only reads (the #686 walk-back) size their segments to L (tie budget
|
||||
`min(MaxTieClusterOverfetch, L − 2)`); on the page path the refusal reaches the page planner, which shrinks the **page** — the
|
||||
walk never re-runs itself at the ceiling with the page still at N (review F2: a cold end-only read with a ceiling of 20 and
|
||||
N = 3000 cost 3,023 gateway reads in one HistoryRead; now ≤ 40). Bound lookups (a page of one) keep the walk's own re-run.
|
||||
- **Crowded look-back (review F3).** A page asking for exactly the ceiling leaves no room for the #687 1 ms look-back, so more
|
||||
than 16 rows just before a page start (e.g. 20 ties 1 ms before it) used to fail `BadTooManyOperations`. The data source now
|
||||
reports the crowding rows on its refusal (`HistoryReadLimitExceededException.PreWindowRows`, only when they exceed the 16-row
|
||||
allowance), and the planner re-reads the page once that many values smaller — the source's own crowding re-read then fits under
|
||||
the ceiling. In the walk-back, a crowded segment is re-read that much smaller and counts as truncated.
|
||||
- **Tie clusters.** The over-fetch that pages within a cluster reads at most `min(MaxTieClusterOverfetch, L − 1)` + 1 rows (so it
|
||||
never trips a refusal); a larger cluster is **`BadTooManyOperations`** — the gateway cannot offset into it. A crowding refusal that
|
||||
leaves no room for a page is that answer too (it is never mistaken for "the cursor sits in a cluster", which would step the
|
||||
cursor a tick per page forever). Every sample before such a failure is served exactly once; nothing is truncated silently.
|
||||
- **Never a truncation:** the gateway read is still answered in full or refused; only the server's page is smaller.
|
||||
- A raised ceiling is not noticed by raw paging (its reads ask for at most the learned ceiling, so nothing re-probes); pages stay
|
||||
at the old size until a read asking for more (a per-sample aggregate) re-probes it, or the server restarts — correct results
|
||||
either way. A **lowered** ceiling is re-learned from the next page's refusal and that page re-planned.
|
||||
- ⚠️ **A window starting at or before the Unix epoch (review F1, fixed with #694).** Such a window gets no 1 ms look-back, so
|
||||
AVEVA's synthetic row lands exactly at its start; it is no sample, and it used to count as "past the window end" — a
|
||||
ceiling-truncated read was then reported complete (an uncapped forward read from the epoch answered its first ceiling − 1 rows,
|
||||
Good, no continuation point). It now counts as a pre-window row.
|
||||
- **Paging (#669) is unaffected.** A full page still holds exactly the page size of stored samples and carries a continuation
|
||||
point, and a resumed page (which re-reads from the last returned timestamp inclusive, and the tie-cluster over-fetch
|
||||
`start == end` read) no longer gets a fabricated row at its resume point. A read is complete when the stream came back short,
|
||||
filled the page, or reached EndTime. If rows **before** the window — the boundary row plus stored samples inside the 1 ms margin,
|
||||
e.g. a tie cluster at `StartTime − 1 ms` or at a resume point `− 1 ms` — used up the budget, the window is re-read **once**
|
||||
asking for `NumValuesPerNode + m + 16` (m = the pre-window rows the first read saw, unclamped). Only if that re-read is still
|
||||
short because of pre-window rows does the node fail `BadTooManyOperations` (not a backend fault) rather than answer a short page
|
||||
e.g. a tie cluster at `StartTime − 1 ms` or at a resume point `− 1 ms` — used up the budget, the window is re-read with room for
|
||||
them: `cap + room + 16`, room = the pre-window rows the first read saw, at least **doubling** per re-read while a read that was
|
||||
itself cut short keeps finding more (up to 8 re-reads, never past a known strict ceiling — review F4: a single re-read sized from
|
||||
a truncated read's count failed a 300–1,700-row cluster just before a resume point even on a default gateway). The room is
|
||||
**capped at `MaxTieClusterOverfetch`** (#694 R2, the limit every other tie path uses): more rows than that inside the look-back ⇒
|
||||
`BadTooManyOperations` after a bounded number of bounded reads — uncapped, a pathological million-row single-millisecond cluster
|
||||
on a streaming gateway would have been streamed several times over for one page. Only if the
|
||||
re-reads are still crowded does the node fail `BadTooManyOperations` (not a backend fault) rather than answer a short page
|
||||
that would end the client's paging early. Every gateway stream is still drained to completion.
|
||||
- **`ReturnBounds` is served by the server, not by this row** (#691, [Bounding values](#bounding-values-returnbounds-691)). The AVEVA
|
||||
boundary row is never used as a bound: it is stamped at the query start rather than carrying the time of the real prior sample, so
|
||||
it could only ever be a Part 11 *interpolated* bound. The start bound on the Gateway is the real prior sample, found by the
|
||||
#686 walk-back.
|
||||
|
||||
**Warm crowding ceiling (#695).** A raw crowding re-read spends the remaining room below a learned gateway ceiling before
|
||||
refusing; doubling the look-back allowance does not skip that final usable budget. Reverse paging retries a crowded segment
|
||||
with a smaller in-window budget derived from the learned ceiling and the observed pre-window rows.
|
||||
|
||||
**Missing tags (#653).** Gateway Raw, Processed and AtTime first check the tag catalog. An empty exact-name catalog result or an explicit
|
||||
`NotFound` returns no data without a history RPC: Raw is empty; AtTime and Processed return null/`BadNoData` at the
|
||||
requested times or bucket timestamps. Unsupported aggregate validation still runs first. A bounded per-data-source cache
|
||||
holds at most 4,096 tag names, with **5 seconds** for absence and **60 seconds** for presence, so newly provisioned tags become
|
||||
visible after the short negative expiry. Catalog faults and cancellation never become missing-tag cache entries. The adapter drains the filtered catalog stream and checks exact names, avoiding older gateways that misparse a missing `GetTagMetadata` response as `Internal`. Empty
|
||||
AtTime/Processed requests need no check. Events and modified-history capability checks are unaffected.
|
||||
|
||||
**Stopped-tag reverse discovery (#696).** On a strict Gateway, a truncated remainder probe retains its oldest-row evidence
|
||||
and density. Gap discovery uses the learned ceiling's available budget; ordinary page reads keep their page-sized budget.
|
||||
Only complete segments contribute to a page, preserving newest-first order, bounds, ties and continuation hints.
|
||||
|
||||
### Gateway AtTime is evaluated from raw reads (#693)
|
||||
|
||||
> ⚠️ **Client-visible (#693).** Gateway AtTime used to pass AVEVA's own interpolated answer (the gateway's `ReadAtTime`) through,
|
||||
> which broke the [AtTime rules](#attime-reads) every other backend follows (Part 11 §6.5.4). Live (`HistGW.LiveTest.Gate685`:
|
||||
> 100 @ 19:00:00, 107 @ 19:00:02, **NULL** @ 19:00:03, 108 @ 19:00:05) it returned **107 Good at the NULL's own time**, **null + Bad
|
||||
> at 19:00:05** (a stored Good 108 right after the NULL), **99.99999999944 for the stored 100**, and no Interpolated bit on any
|
||||
> interpolated value. It is now answered from **raw** reads with the SQL backends' shared `AtTimeEvaluator` (as #681 did for the
|
||||
> per-sample aggregates), so the same stored samples give the same answer on every backend: the exact stored value (no
|
||||
> Interpolated bit) at a stored time; null + the outage's Bad code at or after a stored NULL; the previous value held,
|
||||
> `UncertainDataSubNormal` + Interpolated, before a Bad neighbour; linear + Interpolated between usable neighbours; `BadNoData`
|
||||
> before the first sample; stepped after the last (`UncertainDataSubNormal` once later than now). Kill switch:
|
||||
> `ServerHistorian:GatewayAtTimeFromRaw = false`.
|
||||
|
||||
- **Read plan (`GatewayAtTimeReader`).** Each requested time needs its stored sample at-or-before (`prev`) and after (`next`; read only
|
||||
when the evaluator uses it — a usable `prev` that is not an exact hit). The distinct times are served by **one** forward window read
|
||||
`[first − 1 ms, last + 1 ms]` of at most 1,024 rows, plus a **walk-back** for the sample before the first time and one small forward
|
||||
**probe** (18 rows) for the sample after the last. Measured on the fake: ten times one second apart on a 1 Hz tag cost **1 read**
|
||||
(all exact hits) or **3** (interpolated); ten times an hour apart on a 1 Hz tag **12** (≈ 9.5k rows, never the 32k between); one
|
||||
time 400 days after the only samples **5**; a tag with no history **4**. A window read that fills its budget resolves the times before
|
||||
its last row; the next time either continues from there (if the rows in between should fit one read) or **jumps** to it with a
|
||||
small read (64 rows) that starts a density-sized margin earlier (so its `prev` is normally in the same read) — 1,000 times
|
||||
spread over ten years of a 1 Hz tag cost ≈ 1,000 reads of ≤ 64 rows (≈ 66k rows; it used to be 1,024-row reads).
|
||||
- **Walk-back** (the gateway reads forward only), in two phases. **Gallop:** segments stepping back from the time, the first sized
|
||||
from the window read's density (else 1 minute), growing ×4 when empty; after two empty segments the whole remainder down to the
|
||||
epoch is asked once — a complete answer ends the walk, a **full** one seeds the bisection with its last row. **Bisect** (once a
|
||||
stored row is known below — a full segment's, or an earlier window read's last row): read `(mid, hi]`; empty ⇒ `hi = mid`; full ⇒
|
||||
the known row moves up to its last row; complete with rows ⇒ done. Right after a full read one segment is aimed at about a quarter
|
||||
of a read at the density it showed (until an aimed segment comes back empty — a gap — then plain halving); below 2 ms the segment
|
||||
is the whole stretch from the known row (never a 100 ns step). Measured (review harness, 5 years of history, time = now − 1 min):
|
||||
a tag stopped 1 h / 1 d / 7 d / 30 d / 365 d ago costs 6–18 reads at 1/min, 21–25 at 1 Hz and 25–31 at 10 Hz; a 1 kHz tag behind
|
||||
a 10-minute gap 6 reads (< 10k rows). (Before the review fix a full segment reset the width and a full whole-remainder read was
|
||||
discarded, so a dense tag that stopped weeks ago gave up after 65 reads.) More than 1,024 stored samples within one millisecond
|
||||
fail the node `BadTooManyOperations` (not counted as a backend failure).
|
||||
- **Bounded.** One request takes at most **64 + 16 × distinct times** raw reads; past it the node fails `BadTooManyOperations`
|
||||
("needed more than N raw reads", naming a small strict-gateway ceiling when one bounded the reads). It is a runaway guard, not a
|
||||
budget (`HistoryReadDeadline` bounds the wall clock): a time inside a gap of a bursty tag legitimately costs a jump read plus a
|
||||
bisection back across the gap, ≈ 6 reads — a 1 h/day 1 Hz tag asked at noon on 30 / 100 / 300 consecutive days takes 186 / 606 /
|
||||
1,806 reads, which the first cut's 64 + 4 per time refused although AVEVA's own AtTime served it. A strict gateway whose ceiling is
|
||||
below 4 rows is refused up front with a message naming the ceiling; a small one (e.g. 10) is served by paging within the cap.
|
||||
- **Wire shape.** Every request bound is a whole millisecond and a row counts only **strictly inside** its request, so AVEVA's synthetic
|
||||
boundary row at the query start (#687) — and any row at the query end — is never taken for a stored sample. A Bad row is value-less
|
||||
(#690). Nothing is asked past now + 1 day (the gateway's AVEVA may refuse a far-future end; a time beyond it is stepped from the last
|
||||
sample). Every stream is drained. A strict gateway's ceiling (#691.3) is learned on its first refusal and honoured; a full read is
|
||||
ordinary paging here, so it is not re-probed.
|
||||
- **Values follow the node's type (type hint).** The gateway wire carries no data type — every historizable type is stored as an
|
||||
AVEVA analog (a Boolean as a 0/1 `Int2`) — so the node manager passes each node's scalar DataType with the AtTime call
|
||||
(`IHistorianDataSource.ReadAtTimeAsync(…, TypeCode? valueType, …)`, a default-interface overload: SqlServer / QuestDB / Null
|
||||
ignore it; Raw and UNS NodeIds give the same hint, arrays and other types none). The Gateway converts each raw row to that type
|
||||
**before** the shared evaluator runs, so its own type rules apply exactly as on the SQL backends: a **Boolean** node
|
||||
(value ≠ 0) is **stepped** — never 0.5; an **integer** node is interpolated, rounded half away from zero and returned as the
|
||||
node's own integer type (`Int32` ⇒ `int`); Float / Double interpolate as Double (as the SQL backends return them). With no
|
||||
hint (an unknown node) every numeric tag interpolates as Double. A **DateTime** node is stepped, but its value stays the
|
||||
gateway's Double: the Gateway never historizes a DateTime node (`HistorianTypeMapper` skips it at provisioning and the recorder drops
|
||||
non-numeric values), so such a node only has history if something else wrote that AVEVA tag, and the numeric encoding is not known
|
||||
here — the SQL backends store DateTime as its own kind and return a `DateTime`. Under an integer hint a NaN, ±∞ or
|
||||
out-of-Int64-range stored value is no integer: it is treated as unusable (as `BadNoData`), so a neighbour like that is never
|
||||
interpolated or saturated into a Good value.
|
||||
- **Client value types (#698, all backends).** Before returning Raw, Modified or AtTime history, the node manager converts non-null
|
||||
scalar values to the node's declared DataType (including Boolean, integer widths and Float). Arrays retain their stored values.
|
||||
Conversion failures produce a null value with `BadTypeMismatch` for that sample, preserving timestamps and modification metadata.
|
||||
DateTime requires an actual DateTime value: the gateway's numeric DateTime encoding is unknown and cannot be inferred.
|
||||
Processed results retain each aggregate's result type, so Count and averages can remain Double on a Boolean or integer node.
|
||||
- **Future timestamps (#696).** A bounded raw probe checks for stored samples between now and the requested future time. An empty
|
||||
future interval starts the prior-sample search near now; a full probe seeds the existing walk-back with a stored future sample.
|
||||
The result still carries the requested timestamp and the evaluator's Uncertain status when no later sample exists.
|
||||
- ⚠️ **Shared-evaluator fix (all backends that use it):** `AtTimeEvaluator` returned every interpolated `Int64` boxed as a
|
||||
**Double** (a C# `cond ? (long)x : double` conditional widens the long); its "keeping the Int64 type" tests passed because
|
||||
Shouldly's `ShouldBe` compares numbers across types. SqlServer / QuestDB interpolated `Int64` AtTime values are now `Int64`,
|
||||
as documented in [AtTime reads](#attime-reads).
|
||||
|
||||
### Bounding values (`ReturnBounds`, #691)
|
||||
|
||||
⚠️ **Client-visible — a breaking change for .NET clients that rely on SDK defaults.** Until #691
|
||||
@@ -792,7 +945,13 @@ switch `ServerHistorian:HonorReturnBounds = false` restores the pre-#691 behavio
|
||||
- **Counting and paging.** Bounds **count toward `NumValuesPerNode`** (§6.5.3.2: "any Bounding Values returned are included in the
|
||||
numValuesPerNode count"; `NumValuesPerNode = 1` returns the start bound alone). The start bound leads the **first** page and the
|
||||
end bound ends the **last**: the pending end bound rides the continuation point, and when a page is exactly full the end bound is
|
||||
the whole next page. Paging a bounded read at any page size concatenates to the unpaged answer.
|
||||
the whole next page. Paging a bounded read at any page size concatenates to the unpaged answer. At `NumValuesPerNode = 1`
|
||||
with no start bound found, the server first checks the read holds anything (else `GoodNoData`); since #692 what that check
|
||||
learns rides the continuation point — the walk-back hint and, when there is no data, the end bound it found — so the next
|
||||
page reads nothing it already read.
|
||||
- **Placeholders are marked, not recognised by shape (#692).** Only a `Bad_BoundNotFound` value the server synthesised is a
|
||||
placeholder; a stored row that carries `Bad_BoundNotFound` with a null value (writable through HistoryUpdate) is data, and
|
||||
never turns a read into the "no data and no bound" `GoodNoData` answer.
|
||||
- **How it is served — one authority, `HistoryBounds` in the OPC UA server.** "At or before T" is a newest-first read of `(0, T]`
|
||||
capped at one — native on SqlServer/QuestDB, the #686 walk-back of bounded forward reads on the Gateway (a tag with no sample
|
||||
before StartTime is proven empty down to 1970 by the #689 walk — after two empty segments it asks once for the whole remainder —
|
||||
@@ -857,6 +1016,15 @@ says is complete carries no continuation point. `NumValuesPerNode == 0` stays un
|
||||
the walk's biggest read (`2N + 18` rows, a discovery read 1,026, a tie-cluster read up to `2·MaxTieClusterOverfetch + 4`).
|
||||
That cap truncates **silently** (the stream just ends), so a segment it cut would look complete and drop its newest events —
|
||||
the same blindness a forward read has to it, but on the end a reverse read is asking for.
|
||||
- ⚠️ **Cost of "nothing older" (#694 item 2).** A walked read that runs out of events proves the rest of its window empty with
|
||||
one forward read down to its earlier bound — for an end-only read, the Unix epoch. On `Runtime.dbo.Events` that read is
|
||||
measured at **~3.8 s** for a source with few events (vs ~0.7 s for a bounded window), and an **unfiltered** deep read hit the
|
||||
gateway's 30 s SQL timeout; a forward read from a far-past `StartTime` scans the same range. The cost is the gateway's query
|
||||
over `Runtime.dbo.Events` (its index use is a historiangw matter, not fixable here). Two ways to keep it down: give the read an
|
||||
explicit `EndTime`/`StartTime` that bounds the window, or set the opt-in **`ServerHistorian:EventReadHorizon`** (below). The
|
||||
default stays exact — there is no cache of "earliest event per source", deliberately: events reach `Runtime.dbo.Events` from
|
||||
the alarm store-and-forward drain (which can deliver hours- or days-old events after an outage) and from AVEVA itself, so a
|
||||
remembered "nothing before X" can go stale silently.
|
||||
|
||||
⚠️ **Client-visible (#691.4).** On Gateway (the default backend) and Null an explicit reverse events read used to be refused
|
||||
`BadHistoryOperationUnsupported` (#675), and an end-only capped events read returned the **oldest** N events ascending. Both
|
||||
@@ -1620,7 +1788,7 @@ this family and **are** capped (they used to be pushed down). Raise the cap, or
|
||||
### Health
|
||||
|
||||
The read client reports `GetHealthSnapshot()` (query/failure counters, consecutive failures, `LastError` — e.g. a missing schema under
|
||||
`AutoCreateSchema=false`). A SQL error on a read surfaces as `BadHistoryOperationUnsupported` for **that node only**; a cancelled read
|
||||
`AutoCreateSchema=false`). A SQL connection outage on a read surfaces as `BadNoCommunication`; other SQL errors retain `BadHistoryOperationUnsupported` for **that node only**; a cancelled read
|
||||
(deadline, client abort) is not counted as a backend fault.
|
||||
|
||||
### Retention (`RetentionDays`)
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
# Missing-tag and Gateway performance fixes — 2026-10-13
|
||||
|
||||
Follow-up to pushed `97668d26` / PR #699. Implements #653 and the remaining #696 items 2/3 under the approved bounded design.
|
||||
|
||||
## Changes
|
||||
|
||||
- **#653:** Raw, Processed and AtTime check tag presence before history RPCs. The bounded 4,096-name cache uses a 5-second negative and 60-second positive lifetime. Only explicit absence or NotFound is cached as missing; failures and cancellation propagate. Raw is empty; AtTime and Processed retain requested/bucket timestamps with null/BadNoData. Unsupported aggregate validation runs first.
|
||||
- **Live compatibility adjustment:** the deployed gateway misparses missing `GetTagMetadata` replies as `Internal` (`InvalidDataException: GetTagInfo response ended unexpectedly`). The adapter therefore checks the filtered tag catalog and compares exact names, fully draining the stream before declaring presence/absence. It never turns the parser error into a missing tag. Wildcard matches alone do not prove presence, and a trailing stream fault cannot enter the cache.
|
||||
- **#696 item 2:** a truncated remainder probe seeds strict-ceiling reverse discovery with its observed rows/density; gap searches use the available discovery budget. Complete segments alone contribute to the newest-first page. Regression: a 7,200-row stopped 1 Hz tag, ceiling 1,000, requested latest three one day later, previously 17 raw calls; now meets a maximum of 12 with exactly 7199/7198/7197 returned. Ordinary page sizing and event discovery behavior are unchanged.
|
||||
- **#696 item 3:** per-sample aggregates reuse a ceiling refusal for 30 seconds instead of immediately re-probing on every full stream. The next incomplete aggregate read after expiry notices a raised ceiling; a lowered ceiling is re-learned immediately. Raw and AtTime re-probes remain immediate. During the cooldown an oversized aggregate remains BadTooManyOperations, never a truncated result.
|
||||
|
||||
## Verification evidence
|
||||
|
||||
New regressions failed before fixes: all initial 12 cases failed (missing preflight/caching, 17 discovery reads, duplicate aggregate refusal). The four catalog wire cases failed with the metadata adapter and pass with the catalog adapter. Final Gateway suite: 434 passed, six environment-gated skips.
|
||||
|
||||
Independent read-only review found no important defects. Concurrent cold requests can duplicate a metadata/catalog check; cache access and capacity mutations are locked. No single-flight guarantee was introduced. Catalog RPC contract streams all matches; the upstream SDK pages the native tag-query results, with no configured browse result ceiling.
|
||||
|
||||
## Live #653 confirmation
|
||||
|
||||
Measured the updated GatewayHistorianDataSource through the real adapter and local gateway against the existing remote AVEVA connection. Temporary historian:read key had a two-hour expiry and was revoked after the run. No historian writes/provisioning and no gateway deployment changes.
|
||||
|
||||
Nonexistent tag: `OtOpcUa.Issue653.Nonexistent.20261013`.
|
||||
|
||||
| Operation, five consecutive runs | Elapsed milliseconds | Result |
|
||||
|---|---|---|
|
||||
| Raw | 228, 0, 0, 0, 0 | empty |
|
||||
| Count processed | 2, 0, 0, 0, 0 | six null/BadNoData buckets |
|
||||
| AtTime | 1, 0, 0, 0, 0 | null/BadNoData at requested time |
|
||||
|
||||
These used one data source/cache: the first Raw call is cold, subsequent calls are cached. The prior direct-client measurement took ~30.6 seconds for missing-tag Raw/Aggregate and ~61.3 seconds for AtTime. Direct client RPCs are not modified; the fast-fail protection is in OtOpcUa.
|
||||
|
||||
**Control limitation:** SysTimeSec catalog presence succeeded, but its Raw RPC twice timed out upstream with DeadlineExceeded at ~30 seconds. The data source propagated IOException/communication failure rather than returning no data. A fresh control using the original package client (no data-source presence checks) also timed out with DeadlineExceeded after 30,656 ms, confirming this timeout is independent of the new checks. The earlier direct-client control had succeeded. This run therefore verifies missing-tag fast-fail and failure propagation, not healthy-tag live history behavior. No end-to-end deployed OPC UA claim is made.
|
||||
|
||||
#694 remains open for native aggregate rerun after historiangw#27. No sibling gateway code changed.
|
||||
|
||||
Full final-tree solution: 14,600 passed, 609 skipped, 0 failed across 62 suites. `dotnet test ZB.MOM.WW.OtOpcUa.slnx --verbosity quiet`, exit 0. Includes 122 passing OPC UA integration tests and 268 passing host integration tests (12 host skips). Environment-gated suites are included in the skip count.
|
||||
@@ -0,0 +1,28 @@
|
||||
# Historian follow-up results — 2026-10-13
|
||||
|
||||
Scope: approved bounded fixes for #695–#698, plus live measurement of #653. Base: `66a2b0c3`.
|
||||
|
||||
- #695: spend the remaining learned-ceiling room on crowding reads; reduce crowded reverse-read windows using the observed pre-window count. Regression cases cover ceilings 1,709 and 1,369, and reverse paging through a crowded millisecond.
|
||||
- #696 item 1: probe the future interval before starting the prior-sample search near now. Preserve stored future samples, requested timestamps and Uncertain status. Dense future probes still use the existing bounded walk. Items 2 and 3 remain open (stopped-tag strict-ceiling cost and aggregate ceiling re-probes).
|
||||
- #697: actual Gateway, SQL Server and QuestDB transport failures become `BadNoCommunication` on every HistoryRead arm. Cancellation remains `BadTimeout`; unsupported operations retain `BadHistoryOperationUnsupported`. Tests include real refused database connections, repeated reads and nested gRPC failures.
|
||||
- #698: normalize Raw, Modified and AtTime scalar values to the node DataType. Conversion failure gives per-sample null/`BadTypeMismatch`; arrays, audit metadata and timestamps survive. Processed results retain aggregate-specific types.
|
||||
- #694 item 3 remains blocked on historiangw#27. No upstream code was changed.
|
||||
|
||||
## Live #653 measurement
|
||||
|
||||
Five sequential calls per operation and tag, using client 0.3.0 against the running local gateway (`localhost:5221`) with its existing remote AVEVA connection. Commissioned a two-hour `historian:read` key, then revoked it after measurement. No historian writes or provisioning performed.
|
||||
|
||||
| Operation | SysTimeSec, elapsed ms | Nonexistent tag, elapsed ms |
|
||||
|---|---|---|
|
||||
| Metadata | 299, 85, 81, 82, 48 (success) | 89, 53, 41, 77, 76 (gateway exception) |
|
||||
| Raw | 143, 99, 145, 133, 159 (success) | 30260, 30634, 30675, 30754, 30628 (gateway exception) |
|
||||
| Aggregate | 271, 275, 309, 249, 304 (success) | 30564, 30605, 30583, 30575, 30806 (gateway exception) |
|
||||
| AtTime | 140, 88, 89, 127, 92 (success) | 61074, 61330, 61601, 61249, 61271 (gateway exception) |
|
||||
|
||||
Nonexistent tag: `OtOpcUa.Issue653.Nonexistent.20261013`. The delay is still present. Recommend a metadata pre-check with a short negative cache, caching only explicit missing-tag outcomes and never authentication, communication or arbitrary errors. This is follow-up implementation; #653 remains open. Direct gateway AtTime above differs from OtOpcUa's raw-based AtTime evaluator.
|
||||
|
||||
## Consumer and validation notes
|
||||
|
||||
ScadaBridge source and tests contain no HistoryRead consumer; no consumer change required. Independent review found a future-stored-sample edge case, which was fixed and covered before final verification. No deployment or live end-to-end OPC UA verification of these new fixes is claimed; the live run measures #653 through the gateway client.
|
||||
|
||||
Full solution: 14,581 passed, 609 skipped, 0 failed across 62 test suites. Command: `dotnet test ZB.MOM.WW.OtOpcUa.slnx --verbosity quiet`, exit 0. Skips include environment-gated integration suites.
|
||||
@@ -93,6 +93,14 @@ public sealed class HistoryReadLimitExceededException : InvalidOperationExceptio
|
||||
/// <summary>The configured limit that was exceeded.</summary>
|
||||
public long Limit { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #694: a raw read refused because rows BEFORE its window (inside the backend's look-back margin — a tie
|
||||
/// cluster just before a page start, the HistorianGateway's boundary row) crowded a ceiling-bounded row budget: how
|
||||
/// many such rows the read saw (a lower bound when the read was itself truncated). <c>0</c> = not a crowding refusal.
|
||||
/// The server re-reads such a page once, smaller by this many values (<c>RawPageLimit</c>).
|
||||
/// </summary>
|
||||
public int PreWindowRows { get; init; }
|
||||
|
||||
/// <summary>Creates the exception.</summary>
|
||||
/// <param name="limit">The configured limit.</param>
|
||||
/// <param name="message">Operator-facing description.</param>
|
||||
|
||||
@@ -46,6 +46,18 @@ public interface IHistorianDataSource : IDisposable
|
||||
/// </summary>
|
||||
bool SupportsReverseRaw => false;
|
||||
|
||||
/// <summary>
|
||||
/// The most values one capped <see cref="ReadRawAsync"/> can currently return without failing
|
||||
/// <see cref="HistoryReadLimitExceededException"/> on a BACKEND row ceiling (Gitea #694); <c>0</c> = no known limit.
|
||||
/// The server sizes its raw pages to it — a page is at most this many values and carries a continuation point when
|
||||
/// full (Part 11 allows fewer than <c>NumValuesPerNode</c> with a continuation point), instead of failing a page size
|
||||
/// the backend cannot read in one go. It may change at runtime (a strict HistorianGateway's ceiling is learned from
|
||||
/// its first refusal, #691.3), so it is read per read. A backend whose reads are never ceiling-bound returns <c>0</c>.
|
||||
/// The source still never truncates: a read asking for more than this is answered in full or refused. Default
|
||||
/// <c>0</c>; a backend declares it itself (a guard test pins that).
|
||||
/// </summary>
|
||||
int MaxRawValuesPerRead => 0;
|
||||
|
||||
/// <summary>
|
||||
/// Read processed (interval-bucketed) samples — average / min / max / count / etc.
|
||||
/// A bucket with no source data returns a sample whose
|
||||
@@ -81,6 +93,31 @@ public interface IHistorianDataSource : IDisposable
|
||||
IReadOnlyList<DateTime> timestampsUtc,
|
||||
CancellationToken cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #693: <see cref="ReadAtTimeAsync(string, IReadOnlyList{DateTime}, CancellationToken)"/> with the value type of
|
||||
/// the node being read, for a backend whose stored values do not carry their type (the Gateway: AVEVA stores every
|
||||
/// historized type as a numeric analog, so a Boolean comes back as a 0/1 Double). Such a backend uses the hint to apply
|
||||
/// the AtTime rules exactly as the typed backends do — a Boolean (or String / DateTime) node is stepped, an integer node
|
||||
/// is interpolated, rounded half away from zero and returned as the node's integer type, a floating-point node is
|
||||
/// interpolated. A backend that stores typed values ignores it: the default forwards to the hint-less overload, so
|
||||
/// SqlServer, QuestDB and the Null source need no change.
|
||||
/// </summary>
|
||||
/// <param name="fullReference">The full reference of the tag to read.</param>
|
||||
/// <param name="timestampsUtc">The list of timestamps to read values at.</param>
|
||||
/// <param name="valueType">
|
||||
/// The scalar value type of the node (<see cref="TypeCode.Boolean"/>, the integer codes, <see cref="TypeCode.Single"/>,
|
||||
/// <see cref="TypeCode.Double"/>, <see cref="TypeCode.String"/>, <see cref="TypeCode.DateTime"/>), or <c>null</c> when
|
||||
/// unknown, not scalar, or not one of those — the backend's own policy then applies.
|
||||
/// </param>
|
||||
/// <param name="cancellationToken">A cancellation token that can be used to cancel the operation.</param>
|
||||
/// <returns>One sample per requested timestamp, in the same order as <paramref name="timestampsUtc"/>.</returns>
|
||||
Task<HistoryReadResult> ReadAtTimeAsync(
|
||||
string fullReference,
|
||||
IReadOnlyList<DateTime> timestampsUtc,
|
||||
TypeCode? valueType,
|
||||
CancellationToken cancellationToken) =>
|
||||
ReadAtTimeAsync(fullReference, timestampsUtc, cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// Read historical alarm / event records — OPC UA HistoryReadEvents service.
|
||||
/// Distinct from any live event stream; sources here come from the historian's
|
||||
@@ -95,7 +132,7 @@ public interface IHistorianDataSource : IDisposable
|
||||
/// <c>uint maxValuesPerNode</c> because their OPC UA HistoryRead surface has no
|
||||
/// equivalent "use default" sentinel.
|
||||
///
|
||||
/// This surface declares <see cref="ReadAtTimeAsync"/> and <see cref="ReadEventsAsync(string?, DateTime, DateTime, int, CancellationToken)"/>
|
||||
/// This surface declares <see cref="ReadAtTimeAsync(string, IReadOnlyList{DateTime}, CancellationToken)"/> and <see cref="ReadEventsAsync(string?, DateTime, DateTime, int, CancellationToken)"/>
|
||||
/// as required members — a server-side historian owns the full read surface.
|
||||
/// </remarks>
|
||||
/// <param name="sourceName">The source name to filter events, or null to return events from all sources.</param>
|
||||
|
||||
@@ -60,6 +60,10 @@ public sealed class NullHistorianDataSource : IHistorianDataSource
|
||||
/// direction — no need for the server to walk back through forward reads to find nothing.</remarks>
|
||||
public bool SupportsReverseRaw => true;
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks><c>0</c>: no history, no ceiling (Gitea #694).</remarks>
|
||||
public int MaxRawValuesPerRead => 0;
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks><c>true</c>, like <see cref="SupportsReverseRaw"/> (Gitea #691.4 review; <c>false</c> under #675): there is no
|
||||
/// history to read in either direction — no need for the server to walk back through empty forward reads.</remarks>
|
||||
|
||||
@@ -0,0 +1,514 @@
|
||||
using ZB.MOM.WW.HistorianGateway.Contracts.Grpc;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Mapping;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway;
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #693: HistoryRead AtTime on the Gateway backend, served from RAW gateway reads and evaluated with the SQL backends'
|
||||
/// shared <see cref="AtTimeEvaluator"/> (Part 11 §6.5.4) — as #681 did for the per-sample aggregates — so the Gateway
|
||||
/// answers exactly what SqlServer and QuestDB answer for the same stored samples. AVEVA's own interpolated answer (the
|
||||
/// gateway's <c>ReadAtTime</c>) is not used: live, it returned the previous value as Good at a stored NULL's own time,
|
||||
/// null + Bad at a Good sample right after a NULL, a drifted value (99.99999999944) at an exact stored 100, and no
|
||||
/// Interpolated bit.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>What each requested time needs.</b> <c>prev</c> = the latest stored sample at or before it, <c>next</c> = the
|
||||
/// first stored sample after it. <c>next</c> is only read when the evaluator uses it (a usable <c>prev</c> that is not
|
||||
/// an exact hit).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Read plan — O(1) reads for a dense batch, never two round-trips per timestamp by default.</b> The distinct
|
||||
/// requested times are walked in ascending order with ONE forward window read <c>[t − 1 ms, max + 1 ms]</c> of at most
|
||||
/// <see cref="ReadRows"/> rows, which yields every in-window neighbour. The sample before the first time comes from a
|
||||
/// walk-back (below), the sample after the last time from one small forward probe. A window read that fills its budget
|
||||
/// (dense data, widely spread times) resolves the times before its last row; the next unresolved time either
|
||||
/// continues from that row (when the rows in between are expected to fit one read) or jumps there with a read that
|
||||
/// starts a density-sized margin before it (so its <c>prev</c> is normally in the same read). Ten times one second
|
||||
/// apart on a 1 Hz tag cost 1 read (all exact hits) or 3 (interpolated); ten times an hour apart about one read each.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Walk-back</b> (the gateway reads forward only), in two phases. <i>Gallop:</i> segments <c>(lo, hi]</c> stepping
|
||||
/// back from <c>hi</c>, each one forward read, the first sized from the density the window read showed (else
|
||||
/// <see cref="InitialWalkWidth"/>), an empty one growing the next ×4; after two empty segments the whole remainder down
|
||||
/// to the epoch is asked once — a complete answer ends the walk, a FULL one seeds the bisection with its last row.
|
||||
/// <i>Bisect</i> (once a stored row is known below <c>hi</c> — a full segment's last row, or the last row of an earlier
|
||||
/// window read): read <c>(mid, hi]</c>; empty ⇒ <c>hi = mid</c>; full ⇒ the known row moves up to its last row;
|
||||
/// complete with rows ⇒ its last row is the answer. Right after a full read the next segment is aimed at about a
|
||||
/// quarter of a read at the density it showed (one try — an empty aimed segment falls back to halving), and below
|
||||
/// 2 ms the segment is the whole stretch from the known row (never a 100 ns step).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Bounded.</b> One request may take at most <c>64 + 16 × distinct times</c> raw reads (<see cref="BaseReads"/>,
|
||||
/// <see cref="ReadsPerTime"/>); past it — or with a strict-gateway ceiling below <see cref="MinBudget"/> rows — the
|
||||
/// node fails <c>BadTooManyOperations</c> with a message naming the limit. A jump read (to a time far past the previous
|
||||
/// read) asks for only <see cref="JumpRows"/> rows, so a thousand times spread over ten years cost about a thousand
|
||||
/// small reads, not a thousand 1,024-row reads.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Wire shape.</b> Every request bound is a whole millisecond (AVEVA stores milliseconds and truncates request
|
||||
/// times); a row counts only strictly INSIDE its request — AVEVA's synthetic boundary row at the query start (the value
|
||||
/// in effect there, <c>Quality = 133</c>, or a NULL row) can never be taken for a stored sample, nor a possible row at
|
||||
/// the query end; the epoch and before hold nothing. A Bad row (a stored NULL arrives as value 0, <c>q=1 opc=0</c>,
|
||||
/// #690) is value-less. Every stream is drained, never abandoned (a cancelled gateway stream evicts its pooled
|
||||
/// historian session). Nothing is asked past <c>now + </c><see cref="RawAggregateReader.LookAheadHorizon"/> (the
|
||||
/// historian stores nothing further ahead, and the gateway's AVEVA may refuse a far-future end); a strict gateway's
|
||||
/// row ceiling (#691.3) is learned and honoured — a short read is ordinary paging here, never re-probed.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Values.</b> A gateway value is a Double or a String whatever the node's data type (AVEVA stores every historized
|
||||
/// type as a numeric analog — a Boolean as a 0/1 <c>Int2</c>). The node's type hint converts each row BEFORE the
|
||||
/// shared evaluator runs, so its own type rules apply exactly as on the SQL backends: Boolean (value ≠ 0) and String
|
||||
/// are stepped; an integer type is an Int64 — interpolated, rounded half away from zero — and the result is returned
|
||||
/// as the node's integer type; Float / Double (and no hint) interpolate linearly as Double.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
internal static class GatewayAtTimeReader
|
||||
{
|
||||
/// <summary>The row budget of a window read (the first, or one continuing from a full one) and a walk-back segment.</summary>
|
||||
internal const int ReadRows = 1024;
|
||||
|
||||
/// <summary>The row budget of a jump read (a window read restarting at a time far past the previous read's last row).</summary>
|
||||
internal const int JumpRows = 64;
|
||||
|
||||
/// <summary>The row budget of the forward probe for the sample after the last time (boundary row + ties + one).</summary>
|
||||
internal const int ProbeRows = 2 + RawAggregateReader.PreWindowAllowance;
|
||||
|
||||
/// <summary>The smallest row budget a read can work with (the boundary row, a tie, one sample beyond).</summary>
|
||||
internal const int MinBudget = 4;
|
||||
|
||||
/// <summary>Raw reads one request may always take …</summary>
|
||||
internal const int BaseReads = 64;
|
||||
|
||||
/// <summary>
|
||||
/// … plus this many per distinct requested time. A runaway guard, not a budget (<c>HistoryReadDeadline</c> bounds the
|
||||
/// wall clock): a time inside a gap of a bursty tag legitimately costs a jump read plus a bisection back across the gap
|
||||
/// (≈ 6 reads — a 1 h/day tag asked at noon on consecutive days), so 4 refused requests AVEVA's own AtTime served.
|
||||
/// </summary>
|
||||
internal const int ReadsPerTime = 16;
|
||||
|
||||
/// <summary>The first walk-back segment's width when the window read showed no density.</summary>
|
||||
internal static readonly TimeSpan InitialWalkWidth = TimeSpan.FromMinutes(1);
|
||||
|
||||
/// <summary>The widest walk-back segment sized from density, before ×4 growth.</summary>
|
||||
private static readonly TimeSpan MaxSeededWidth = TimeSpan.FromDays(1);
|
||||
|
||||
private const long Ms = TimeSpan.TicksPerMillisecond;
|
||||
|
||||
private static readonly TimeSpan OneMs = TimeSpan.FromMilliseconds(1);
|
||||
|
||||
private static readonly DateTime Floor = RawAggregateReader.ProbeFloor;
|
||||
|
||||
/// <summary>One drained read: the rows strictly inside the request (ascending) and whether the stream ended short.</summary>
|
||||
private readonly record struct Read(List<StoredSample> Rows, bool Complete);
|
||||
|
||||
/// <summary>Evaluates AtTime for <paramref name="timestampsUtc"/> (same length and order, duplicates included).</summary>
|
||||
/// <param name="client">The gateway client.</param>
|
||||
/// <param name="ceiling">The data source's learned gateway row ceiling (#691.3).</param>
|
||||
/// <param name="tag">Historian tag name.</param>
|
||||
/// <param name="timestampsUtc">Requested times (any kind; treated as UTC, as on the SQL backends).</param>
|
||||
/// <param name="now">Server timestamp, the beyond-now rule and the look-ahead horizon.</param>
|
||||
/// <param name="ct">Cancellation.</param>
|
||||
/// <param name="valueType">The node's scalar value type (the hinted <c>IHistorianDataSource.ReadAtTimeAsync</c>), or null.</param>
|
||||
/// <param name="maxReads">The request's raw-read cap; null = <see cref="BaseReads"/> + <see cref="ReadsPerTime"/> per distinct time.</param>
|
||||
/// <returns>One value per requested time, stamped with it.</returns>
|
||||
/// <exception cref="HistoryReadLimitExceededException">More than <see cref="ReadRows"/> rows within one millisecond, the
|
||||
/// read cap, or a strict gateway ceiling below <see cref="MinBudget"/>.</exception>
|
||||
public static async Task<IReadOnlyList<DataValueSnapshot>> ReadAsync(IHistorianGatewayClient client, GatewayRowCeiling ceiling,
|
||||
string tag, IReadOnlyList<DateTime> timestampsUtc, DateTime now, CancellationToken ct, TypeCode? valueType = null,
|
||||
int? maxReads = null)
|
||||
{
|
||||
var requested = timestampsUtc.Select(t => DateTime.SpecifyKind(t, DateTimeKind.Utc)).ToArray();
|
||||
if (requested.Length == 0) return [];
|
||||
var times = requested.Distinct().Order().ToArray();
|
||||
var ctx = new Ctx(client, ceiling, tag, ct, valueType, maxReads ?? BaseReads + ReadsPerTime * times.Length);
|
||||
var neighbours = await ResolveAsync(ctx, times, now).ConfigureAwait(false);
|
||||
var result = new DataValueSnapshot[requested.Length];
|
||||
for (var i = 0; i < requested.Length; i++)
|
||||
{
|
||||
var t = requested[i];
|
||||
var (prev, next) = neighbours[t.Ticks];
|
||||
// A DateTime node is stepped (as on the SQL backends): a usable non-numeric next sends the evaluator down its
|
||||
// stepped branch, with the next's own status.
|
||||
if (valueType == TypeCode.DateTime && next is { IsUsable: true } n)
|
||||
next = n with { Value = HistorianValue.FromString(string.Empty) };
|
||||
result[i] = ToNodeType(AtTimeEvaluator.Evaluate(t, prev, next, now, now), valueType);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/// <summary>One request's read context: the client, the type hint, and the read count against its cap.</summary>
|
||||
private sealed class Ctx(IHistorianGatewayClient client, GatewayRowCeiling ceiling, string tag, CancellationToken ct,
|
||||
TypeCode? valueType, int maxReads)
|
||||
{
|
||||
public IHistorianGatewayClient Client { get; } = client;
|
||||
public GatewayRowCeiling Ceiling { get; } = ceiling;
|
||||
public string Tag { get; } = tag;
|
||||
public CancellationToken Ct { get; } = ct;
|
||||
public TypeCode? ValueType { get; } = valueType;
|
||||
public int MaxReads { get; } = maxReads;
|
||||
public int Reads { get; set; }
|
||||
|
||||
/// <summary>The smallest budget a read was served with (a strict gateway's ceiling shows here).</summary>
|
||||
public int SmallestBudget { get; set; } = int.MaxValue;
|
||||
}
|
||||
|
||||
/// <summary>True for the integer type codes (evaluated as Int64, returned as the node's own type).</summary>
|
||||
private static bool IsInteger(TypeCode? t) =>
|
||||
t is TypeCode.SByte or TypeCode.Byte or TypeCode.Int16 or TypeCode.UInt16 or TypeCode.Int32 or TypeCode.UInt32
|
||||
or TypeCode.Int64 or TypeCode.UInt64;
|
||||
|
||||
/// <summary>
|
||||
/// An evaluated Int64 (integer hint) returned as the node's integer type. A value the type cannot hold (only possible
|
||||
/// for a stored value already outside the node's range) stays the evaluator's Int64.
|
||||
/// </summary>
|
||||
private static DataValueSnapshot ToNodeType(DataValueSnapshot v, TypeCode? valueType)
|
||||
{
|
||||
if (v.Value is not long l || !IsInteger(valueType) || valueType == TypeCode.Int64) return v;
|
||||
try
|
||||
{
|
||||
return v with { Value = Convert.ChangeType(l, valueType!.Value, System.Globalization.CultureInfo.InvariantCulture) };
|
||||
}
|
||||
catch (OverflowException)
|
||||
{
|
||||
return v;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>prev / next for every time of <paramref name="times"/> (distinct, ascending).</summary>
|
||||
private static async Task<Dictionary<long, (StoredSample? Prev, StoredSample? Next)>> ResolveAsync(
|
||||
Ctx ctx, DateTime[] times, DateTime now)
|
||||
{
|
||||
var result = new Dictionary<long, (StoredSample?, StoredSample?)>(times.Length);
|
||||
var horizon = MsFloor(DateTime.MaxValue - now > RawAggregateReader.LookAheadHorizon
|
||||
? DateTime.SpecifyKind(now + RawAggregateReader.LookAheadHorizon, DateTimeKind.Utc)
|
||||
: DateTime.MaxValue.AddTicks(-Ms));
|
||||
|
||||
// Nothing is stored at or before the epoch.
|
||||
var i = 0;
|
||||
for (; i < times.Length && times[i] <= Floor; i++) result[times[i].Ticks] = (null, null);
|
||||
if (i == times.Length) return result;
|
||||
|
||||
// Reads are planned on times clamped to the horizon (nothing is stored beyond it, and no read ends past it).
|
||||
var qe = Min(MsFloor(Min(times[^1], horizon)) + OneMs, horizon);
|
||||
var qs = RequestStart(Min(times[i], horizon));
|
||||
var budget = ReadRows;
|
||||
StoredSample? before = null; // the latest stored sample at or before qs …
|
||||
var beforeKnown = false; // … once known
|
||||
DateTime? walkKnown = null; // a stored sample exists here (at or before qs): the walk-back bisects above it
|
||||
var walkWidth = InitialWalkWidth;
|
||||
var probeFrom = -1; // first index whose next lies at or after qe (complete last read)
|
||||
|
||||
while (true)
|
||||
{
|
||||
var read = await ReadTrustedAsync(ctx, qs, qe, budget).ConfigureAwait(false);
|
||||
var rows = read.Rows;
|
||||
if (!read.Complete && rows.Count == 0)
|
||||
throw TooDense(ctx, qs);
|
||||
if (rows.Count >= 2 && !beforeKnown && walkKnown is null)
|
||||
walkWidth = Clamp(TimeSpan.FromTicks(4 * (rows[^1].TimeUtc - rows[0].TimeUtc).Ticks / (rows.Count - 1)));
|
||||
|
||||
// A short stream saw every row up to qe; a full one every row before its last row's time x.
|
||||
var x = read.Complete ? qe : rows[^1].TimeUtc;
|
||||
var j = -1; // index of the last row at or before the current time
|
||||
var k = i;
|
||||
for (; k < times.Length; k++)
|
||||
{
|
||||
var t = times[k];
|
||||
var tr = Min(t, horizon);
|
||||
if (!read.Complete && tr >= x) break;
|
||||
while (j + 1 < rows.Count && rows[j + 1].TimeUtc <= tr) j++;
|
||||
|
||||
StoredSample? prev;
|
||||
if (j >= 0) prev = rows[j];
|
||||
else
|
||||
{
|
||||
if (!beforeKnown)
|
||||
{
|
||||
// #696: prove the future stretch empty before jumping to now. Historian clocks or imports
|
||||
// can place real rows there; a full bounded probe seeds the original backward search.
|
||||
var searchEnd = qs;
|
||||
if (qs > now && walkKnown is null)
|
||||
{
|
||||
var future = await ReadTrustedAsync(ctx, MsFloor(now),
|
||||
Min(MsFloor(qs) + OneMs, horizon), ReadRows).ConfigureAwait(false);
|
||||
StoredSample? latest = null;
|
||||
foreach (var row in future.Rows)
|
||||
if (row.TimeUtc <= qs) latest = row;
|
||||
if (future.Complete)
|
||||
{
|
||||
before = latest;
|
||||
beforeKnown = latest is not null;
|
||||
searchEnd = now;
|
||||
}
|
||||
else if (latest is { } found)
|
||||
walkKnown = found.TimeUtc;
|
||||
}
|
||||
if (!beforeKnown)
|
||||
before = await WalkBackAsync(ctx, searchEnd, walkKnown, walkWidth).ConfigureAwait(false);
|
||||
beforeKnown = true;
|
||||
}
|
||||
prev = before;
|
||||
}
|
||||
|
||||
StoredSample? next = null;
|
||||
if (NeedsNext(t, prev) && t < horizon)
|
||||
{
|
||||
if (j + 1 < rows.Count) next = rows[j + 1];
|
||||
else if (probeFrom < 0) probeFrom = k; // after every row of a complete read: the probe answers it
|
||||
}
|
||||
result[t.Ticks] = (prev, next);
|
||||
}
|
||||
|
||||
if (k == times.Length) break;
|
||||
|
||||
// Truncated before times[k]: continue from the last row, or jump to times[k] with a density-sized margin and a
|
||||
// small budget (it needs that time's neighbours, and whatever later times follow closely).
|
||||
var tn = Min(times[k], horizon);
|
||||
var spanTicks = Math.Max(1, (x - rows[0].TimeUtc).Ticks);
|
||||
var density = (double)rows.Count / spanTicks; // rows per tick
|
||||
var gapRows = density * (tn - x).Ticks;
|
||||
var margin = Clamp(TimeSpan.FromTicks((long)Math.Min(TimeSpan.MaxValue.Ticks / 2.0, JumpRows / 4.0 / Math.Max(density, 1e-18))));
|
||||
var resume = MsFloor(x) - OneMs;
|
||||
var jump = gapRows >= ReadRows / 2.0;
|
||||
var nextQs = jump ? Max(RequestStart(tn - Min(margin, tn - Floor)), resume) : resume;
|
||||
if (nextQs <= qs) throw TooDense(ctx, x);
|
||||
|
||||
if (nextQs < x)
|
||||
{
|
||||
// Every row before x was seen: the latest at or before nextQs is known.
|
||||
var lastIndex = rows.FindLastIndex(r => r.TimeUtc <= nextQs);
|
||||
if (lastIndex >= 0)
|
||||
{
|
||||
before = rows[lastIndex];
|
||||
beforeKnown = true;
|
||||
}
|
||||
walkKnown = null;
|
||||
}
|
||||
else
|
||||
{
|
||||
beforeKnown = false;
|
||||
before = null;
|
||||
walkKnown = x;
|
||||
walkWidth = margin;
|
||||
}
|
||||
budget = jump ? JumpRows : ReadRows;
|
||||
qs = nextQs;
|
||||
i = k;
|
||||
}
|
||||
|
||||
if (probeFrom >= 0 && qe < horizon)
|
||||
{
|
||||
var after = await FirstAfterAsync(ctx, qe - OneMs, horizon, Min(times[^1], horizon)).ConfigureAwait(false);
|
||||
if (after is not null)
|
||||
for (var k = probeFrom; k < times.Length; k++)
|
||||
{
|
||||
var t = times[k];
|
||||
var (prev, next) = result[t.Ticks];
|
||||
if (next is null && t < horizon && NeedsNext(t, prev) && after.Value.TimeUtc > t)
|
||||
result[t.Ticks] = (prev, after);
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/// <summary>The evaluator reads <c>next</c> only for a usable <c>prev</c> that is not an exact hit.</summary>
|
||||
private static bool NeedsNext(DateTime t, StoredSample? prev) => prev is { IsUsable: true } p && p.TimeUtc != t;
|
||||
|
||||
/// <summary>One walk-back segment <c>(lo, hi]</c>: its oldest and newest rows, and whether the read was complete.</summary>
|
||||
private static async Task<(StoredSample? First, StoredSample? Last, bool Complete)> SegmentAsync(
|
||||
Ctx ctx, DateTime lo, DateTime hi)
|
||||
{
|
||||
var read = await ReadTrustedAsync(ctx, MsFloor(lo), MsFloor(hi) + OneMs, ReadRows).ConfigureAwait(false);
|
||||
StoredSample? first = null, last = null;
|
||||
foreach (var r in read.Rows)
|
||||
if (r.TimeUtc > lo && r.TimeUtc <= hi)
|
||||
{
|
||||
first ??= r;
|
||||
last = r;
|
||||
}
|
||||
return (first, last, read.Complete);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The latest stored sample with time in <c>(Floor, hi]</c> (see the type remarks: gallop, then bisect).
|
||||
/// <paramref name="known"/>: a stored sample exists at that time (≤ <paramref name="hi"/>) — the walk then bisects
|
||||
/// above it straight away, its first segment aimed at <paramref name="width"/>.
|
||||
/// </summary>
|
||||
private static async Task<StoredSample?> WalkBackAsync(Ctx ctx, DateTime hi, DateTime? known, TimeSpan width)
|
||||
{
|
||||
TimeSpan? aim = known is null ? null : width;
|
||||
|
||||
// Gallop: no stored row known below hi yet.
|
||||
var empties = 0;
|
||||
while (known is null)
|
||||
{
|
||||
if (hi <= Floor) return null;
|
||||
var rest = empties >= 2;
|
||||
var lo = rest || (hi - Floor).Ticks <= width.Ticks ? Floor : hi - width;
|
||||
var seg = await SegmentAsync(ctx, lo, hi).ConfigureAwait(false);
|
||||
if (seg.Complete)
|
||||
{
|
||||
if (seg.Last is { } found) return found;
|
||||
if (lo <= Floor) return null;
|
||||
hi = lo;
|
||||
empties++;
|
||||
width = Grow(width);
|
||||
continue;
|
||||
}
|
||||
// Full — a whole-remainder read of deep history too: its last row is a stored row below hi. Bisect above it.
|
||||
if (seg.Last is not { } seen) throw TooDense(ctx, lo);
|
||||
known = seen.TimeUtc;
|
||||
aim = AimFrom(seg.First!.Value.TimeUtc, seen.TimeUtc);
|
||||
}
|
||||
|
||||
// Bisect: a stored row exists at `known`, nothing newer than `hi` matters; the answer is the newest row in [known, hi].
|
||||
var aimMissed = false;
|
||||
while (true)
|
||||
{
|
||||
var k = known.Value;
|
||||
DateTime lo;
|
||||
var aimed = false;
|
||||
if (hi - k < TimeSpan.FromMilliseconds(2))
|
||||
lo = k.AddTicks(-1); // the whole stretch from the known row: never a sub-millisecond step
|
||||
else
|
||||
{
|
||||
lo = k + TimeSpan.FromTicks((hi - k).Ticks / 2);
|
||||
if (aim is { } a && hi - a > lo)
|
||||
{
|
||||
lo = hi - a; // an aimed try right after a full read — until one comes back empty (a gap: halve from then on)
|
||||
aimed = true;
|
||||
}
|
||||
}
|
||||
aim = null;
|
||||
|
||||
var seg = await SegmentAsync(ctx, lo, hi).ConfigureAwait(false);
|
||||
if (seg.Complete)
|
||||
{
|
||||
if (seg.Last is { } found) return found;
|
||||
if (lo < k) return null; // the known row is gone (deleted meanwhile): nothing is left to find
|
||||
hi = lo;
|
||||
aimMissed |= aimed;
|
||||
continue;
|
||||
}
|
||||
if (seg.Last is not { } seen || seen.TimeUtc <= k) throw TooDense(ctx, k);
|
||||
known = seen.TimeUtc;
|
||||
if (!aimMissed) aim = AimFrom(seg.First!.Value.TimeUtc, seen.TimeUtc);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>About a quarter of a read at the density a full read showed (its rows from <paramref name="first"/> to <paramref name="last"/>).</summary>
|
||||
private static TimeSpan AimFrom(DateTime first, DateTime last) =>
|
||||
TimeSpan.FromTicks(Math.Max(Ms, (last - first).Ticks / 4));
|
||||
|
||||
/// <summary>The first stored sample after <paramref name="after"/> in <c>(qs, end)</c>, from a small forward read.</summary>
|
||||
private static async Task<StoredSample?> FirstAfterAsync(Ctx ctx, DateTime qs, DateTime end, DateTime after)
|
||||
{
|
||||
foreach (var budget in new[] { ProbeRows, ReadRows })
|
||||
{
|
||||
var read = await ReadTrustedAsync(ctx, qs, end, budget).ConfigureAwait(false);
|
||||
foreach (var r in read.Rows)
|
||||
if (r.TimeUtc > after) return r;
|
||||
if (read.Complete) return null;
|
||||
}
|
||||
throw TooDense(ctx, qs);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// One forward read <c>[qs, qe]</c> (whole milliseconds), drained, keeping only rows strictly inside it and after the
|
||||
/// epoch — counted against the request's read cap. A strict gateway's ceiling is learned on refusal and then asked for
|
||||
/// directly; a full read is not re-probed (the caller pages on, it does not need the rest of this read).
|
||||
/// </summary>
|
||||
private static async Task<Read> ReadTrustedAsync(Ctx ctx, DateTime qs, DateTime qe, int budget)
|
||||
{
|
||||
if (++ctx.Reads > ctx.MaxReads)
|
||||
throw new HistoryReadLimitExceededException(ctx.MaxReads,
|
||||
$"The AtTime read of '{ctx.Tag}' needed more than {ctx.MaxReads} raw reads ({BaseReads} + {ReadsPerTime} per " +
|
||||
"requested time)" + CeilingNote(ctx) + "; the Gateway backend serves AtTime from raw reads — request fewer times.");
|
||||
var (read, used) = await ctx.Ceiling.ReadAsync(budget, b => PassAsync(ctx, qs, qe, b), (_, _) => false)
|
||||
.ConfigureAwait(false);
|
||||
if (used < ctx.SmallestBudget) ctx.SmallestBudget = used;
|
||||
if (used < MinBudget)
|
||||
throw new HistoryReadLimitExceededException(used,
|
||||
$"The HistorianGateway's row ceiling ({used}, its Historian:ReadLimits:MaxValuesCeiling with " +
|
||||
$"AllowStreamingBeyondCeiling=false) is below the {MinBudget} rows an AtTime raw read of '{ctx.Tag}' needs.");
|
||||
return read;
|
||||
}
|
||||
|
||||
private static string CeilingNote(Ctx ctx) =>
|
||||
ctx.SmallestBudget < JumpRows
|
||||
? $", with reads bounded by the HistorianGateway's row ceiling of {ctx.SmallestBudget} (Historian:ReadLimits:MaxValuesCeiling)"
|
||||
: string.Empty;
|
||||
|
||||
private static async Task<Read> PassAsync(Ctx ctx, DateTime qs, DateTime qe, int budget)
|
||||
{
|
||||
var rows = new List<StoredSample>();
|
||||
var received = 0;
|
||||
await foreach (var raw in ctx.Client.ReadRawAsync(ctx.Tag, qs, qe, budget, ctx.Ct).ConfigureAwait(false))
|
||||
{
|
||||
received++; // drained to completion, never broken out of
|
||||
if (raw.Timestamp is null) continue;
|
||||
var t = raw.Timestamp.ToDateTime();
|
||||
if (t > qs && t < qe && t > Floor) rows.Add(ToStored(raw, t, ctx.ValueType));
|
||||
}
|
||||
return new Read(rows, received < budget);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A gateway row as a stored sample of the node's type (see the type remarks); a Bad row carries no value (#690: a
|
||||
/// stored NULL arrives as value 0).
|
||||
/// </summary>
|
||||
private static StoredSample ToStored(HistorianSample raw, DateTime t, TypeCode? valueType)
|
||||
{
|
||||
var status = SampleMapper.ToStatus(raw);
|
||||
var value = StatusBits.IsBad(status) ? HistorianValue.Empty
|
||||
: raw.HasNumericValue ? Typed(raw.NumericValue, valueType)
|
||||
: raw.HasStringValue ? HistorianValue.FromString(raw.StringValue)
|
||||
: HistorianValue.Empty;
|
||||
return StoredSample.FromStored(t, value, status);
|
||||
}
|
||||
|
||||
/// <summary>A gateway number as the hinted kind, so the shared evaluator's own type branch decides stepped vs linear.</summary>
|
||||
private static HistorianValue Typed(double v, TypeCode? valueType) => valueType switch
|
||||
{
|
||||
TypeCode.Boolean => HistorianValue.FromBoolean(v != 0),
|
||||
TypeCode.String => HistorianValue.FromString(v.ToString("R", System.Globalization.CultureInfo.InvariantCulture)),
|
||||
// In Int64 range only (2^63 is exactly representable); NaN, ±∞ or out of range is no integer at all — Empty, which
|
||||
// StoredSample.FromStored makes BadNoData (unusable), never a saturated Good value.
|
||||
_ when IsInteger(valueType) => v >= -9.2233720368547758E18 && v < 9.2233720368547758E18
|
||||
? HistorianValue.FromInt64((long)Math.Round(v, MidpointRounding.AwayFromZero))
|
||||
: HistorianValue.Empty,
|
||||
_ => HistorianValue.FromDouble(v),
|
||||
};
|
||||
|
||||
/// <summary>The window read's start for a time: a millisecond before its millisecond (never before the epoch).</summary>
|
||||
private static DateTime RequestStart(DateTime t)
|
||||
{
|
||||
var s = MsFloor(t) - TimeSpan.FromMilliseconds(1);
|
||||
return s < Floor ? Floor : s;
|
||||
}
|
||||
|
||||
private static DateTime MsFloor(DateTime t) => new(t.Ticks - t.Ticks % Ms, DateTimeKind.Utc);
|
||||
|
||||
private static DateTime Min(DateTime a, DateTime b) => a <= b ? a : b;
|
||||
|
||||
private static DateTime Max(DateTime a, DateTime b) => a >= b ? a : b;
|
||||
|
||||
private static TimeSpan Min(TimeSpan a, TimeSpan b) => a <= b ? a : b;
|
||||
|
||||
private static TimeSpan Clamp(TimeSpan w) =>
|
||||
w < TimeSpan.FromMilliseconds(1) ? TimeSpan.FromMilliseconds(1) : w > MaxSeededWidth ? MaxSeededWidth : w;
|
||||
|
||||
private static TimeSpan Grow(TimeSpan w) => w.Ticks > TimeSpan.MaxValue.Ticks / 4 ? TimeSpan.MaxValue : w * 4;
|
||||
|
||||
private static HistoryReadLimitExceededException TooDense(Ctx ctx, DateTime at)
|
||||
{
|
||||
var rows = Math.Min(ReadRows, ctx.SmallestBudget);
|
||||
return new(rows,
|
||||
$"The AtTime read of '{ctx.Tag}' met more than {rows} stored samples within about one millisecond of {at:O}" +
|
||||
(rows < ReadRows ? $" (reads are bounded by the HistorianGateway's row ceiling of {rows})" : string.Empty) +
|
||||
"; the Gateway backend serves AtTime from raw reads.");
|
||||
}
|
||||
}
|
||||
+192
-18
@@ -46,9 +46,30 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
private readonly ILogger<GatewayHistorianDataSource> _logger;
|
||||
private readonly int _maxRawRows;
|
||||
private readonly TimeProvider _time;
|
||||
private readonly bool _atTimeFromRaw;
|
||||
|
||||
/// <summary>#693: AtTime from raw reads (true) or AVEVA's own interpolation (the kill switch).</summary>
|
||||
internal bool AtTimeFromRaw => _atTimeFromRaw;
|
||||
|
||||
/// <summary>The most #687 crowding re-reads one raw read makes (#694 F4): each at least doubles the pre-window room.</summary>
|
||||
private const int MaxCrowdedReReads = 8;
|
||||
|
||||
/// <summary>The default of <c>maxPreWindowRows</c>: <c>ServerHistorian:MaxTieClusterOverfetch</c>'s default.</summary>
|
||||
public const int DefaultMaxPreWindowRows = 65536;
|
||||
|
||||
/// <summary>
|
||||
/// The most rows before a raw window (inside the 1 ms look-back) a crowding re-read makes room for (#694 R2) — the
|
||||
/// server's tie-cluster limit (<c>ServerHistorian:MaxTieClusterOverfetch</c>), which every other tie path treats as
|
||||
/// the limit. More ⇒ <see cref="HistoryReadLimitExceededException"/> with <c>PreWindowRows</c> set.
|
||||
/// </summary>
|
||||
private readonly int _maxPreWindowRows;
|
||||
|
||||
/// <summary>The tie-cluster limit the crowding re-reads honour (#694 R2) — exposed so the factory wiring is testable.</summary>
|
||||
internal int MaxPreWindowRows => _maxPreWindowRows;
|
||||
|
||||
/// <summary>A strict gateway's row ceiling, learned from its first refusal (#691.3).</summary>
|
||||
private readonly GatewayRowCeiling _ceiling = new();
|
||||
private readonly GatewayRowCeiling _ceiling;
|
||||
private readonly GatewayTagPresenceCache _tags;
|
||||
|
||||
private readonly object _healthLock = new();
|
||||
private DateTime? _lastSuccessUtc;
|
||||
@@ -67,15 +88,28 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
/// <= 0 = unbounded). Past it the node fails <c>BadTooManyOperations</c>.
|
||||
/// </param>
|
||||
/// <param name="timeProvider">Clock for result server timestamps and the raw look-ahead horizon; null = system.</param>
|
||||
/// <param name="atTimeFromRaw">
|
||||
/// #693: serve AtTime from raw reads with the shared AtTime rules (<see cref="GatewayAtTimeReader"/>; default);
|
||||
/// <c>false</c> = AVEVA's own interpolation passed through (<c>ServerHistorian:GatewayAtTimeFromRaw</c> kill switch).
|
||||
/// </param>
|
||||
/// <param name="maxPreWindowRows">
|
||||
/// The most rows before a raw window a crowding re-read makes room for (#694 R2; <c>ServerHistorian:MaxTieClusterOverfetch</c>,
|
||||
/// values below 1 count as 1). More ⇒ <see cref="HistoryReadLimitExceededException"/> (<c>BadTooManyOperations</c>).
|
||||
/// </param>
|
||||
public GatewayHistorianDataSource(IHistorianGatewayClient client, ILogger<GatewayHistorianDataSource> logger,
|
||||
int maxRawRows = TimeWeightedAggregator.DefaultMaxWindowRows, TimeProvider? timeProvider = null)
|
||||
int maxRawRows = TimeWeightedAggregator.DefaultMaxWindowRows, TimeProvider? timeProvider = null,
|
||||
bool atTimeFromRaw = true, int maxPreWindowRows = DefaultMaxPreWindowRows)
|
||||
{
|
||||
_maxPreWindowRows = Math.Max(1, maxPreWindowRows);
|
||||
_atTimeFromRaw = atTimeFromRaw;
|
||||
ArgumentNullException.ThrowIfNull(client);
|
||||
ArgumentNullException.ThrowIfNull(logger);
|
||||
_client = client;
|
||||
_logger = logger;
|
||||
_maxRawRows = maxRawRows;
|
||||
_time = timeProvider ?? TimeProvider.System;
|
||||
_ceiling = new GatewayRowCeiling(_time);
|
||||
_tags = new GatewayTagPresenceCache(_client, _time);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
@@ -95,6 +129,11 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
|
||||
try
|
||||
{
|
||||
if (!await _tags.ExistsAsync(fullReference, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
RecordOutcome(success: true, error: null);
|
||||
return new HistoryReadResult([], ContinuationPoint: null);
|
||||
}
|
||||
var samples = await ReadRawWindowAsync(fullReference, startUtc, endUtc, maxValuesPerNode, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
var snapshots = SampleMapper.ToSnapshots(samples);
|
||||
@@ -121,6 +160,7 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
catch (Exception ex)
|
||||
{
|
||||
RecordReadFailure(ex);
|
||||
if (IsTransportFailure(ex)) throw new System.IO.IOException("HistorianGateway read transport is unavailable.", ex);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
@@ -181,29 +221,57 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
throw new HistoryReadLimitExceededException(cap,
|
||||
$"The raw read of '{fullReference}' holds more than the HistorianGateway's row ceiling ({all.Budget}, its " +
|
||||
"Historian:ReadLimits:MaxValuesCeiling with AllowStreamingBeyondCeiling=false); set NumValuesPerNode to page, " +
|
||||
"or narrow the window.");
|
||||
"or narrow the window.")
|
||||
{ PreWindowRows = Crowding(all.Result.PreWindowRows) };
|
||||
}
|
||||
|
||||
var asked = (int)Math.Min(int.MaxValue, (long)cap + RawAggregateReader.PreWindowAllowance);
|
||||
var first = await PassAsync(asked).ConfigureAwait(false);
|
||||
if (first.Result.Complete) return first.Result.Samples;
|
||||
var pass = await PassAsync(asked).ConfigureAwait(false);
|
||||
if (pass.Result.Complete) return pass.Result.Samples;
|
||||
|
||||
// A strict gateway's ceiling already bounded that read (#691.3): a larger budget would only be refused again.
|
||||
if (first.Budget < asked)
|
||||
// A strict gateway's ceiling already bounded that read (#691.3): a larger budget would only be refused again. The
|
||||
// refusal carries the pre-window rows it saw (#694): the server re-reads the page that many values smaller.
|
||||
if (pass.Budget < asked)
|
||||
throw new HistoryReadLimitExceededException(cap,
|
||||
$"The raw read of '{fullReference}' could not be read completely within the HistorianGateway's row ceiling " +
|
||||
$"({first.Budget}, its Historian:ReadLimits:MaxValuesCeiling with AllowStreamingBeyondCeiling=false); lower " +
|
||||
"NumValuesPerNode or narrow the window.");
|
||||
$"({pass.Budget}, its Historian:ReadLimits:MaxValuesCeiling with AllowStreamingBeyondCeiling=false); lower " +
|
||||
"NumValuesPerNode or narrow the window.")
|
||||
{ PreWindowRows = Crowding(pass.Result.PreWindowRows) };
|
||||
|
||||
// The margin crowded the window out: one bounded re-read with room for the pre-window rows already seen.
|
||||
var second = await PassAsync(
|
||||
(int)Math.Min(int.MaxValue, (long)cap + first.Result.PreWindowRows + RawAggregateReader.PreWindowAllowance))
|
||||
.ConfigureAwait(false);
|
||||
if (second.Result.Complete) return second.Result.Samples;
|
||||
// The margin crowded the window out: re-read with room for the pre-window rows. A read that was itself cut short saw
|
||||
// only some of them (#694 F4: a tie cluster of hundreds just before a resume point used to fail even with no
|
||||
// ceiling), so the room at least doubles per re-read — bounded by MaxCrowdedReReads, and never past a known ceiling.
|
||||
// #694 R2: the room is capped at the tie-cluster limit (MaxTieClusterOverfetch) — a pathological single-millisecond
|
||||
// cluster on a streaming gateway would otherwise be streamed several times over for one page.
|
||||
var preWindow = pass.Result.PreWindowRows;
|
||||
var room = Math.Min(Math.Max(1, preWindow), _maxPreWindowRows);
|
||||
for (var attempt = 0; attempt < MaxCrowdedReReads && preWindow <= _maxPreWindowRows; attempt++)
|
||||
{
|
||||
var budget = (int)Math.Min(int.MaxValue, (long)cap + room + RawAggregateReader.PreWindowAllowance);
|
||||
// #695: doubling can leap over the ceiling while the cluster still fits. Spend the remaining room once.
|
||||
if (_ceiling.Learned is var known and > 0 && budget > known)
|
||||
{
|
||||
var available = known - cap - RawAggregateReader.PreWindowAllowance;
|
||||
if (available <= 0) break;
|
||||
room = Math.Min(room, available);
|
||||
budget = cap + room + RawAggregateReader.PreWindowAllowance;
|
||||
}
|
||||
if (budget <= pass.Budget) break; // no more room, never retry the same full read
|
||||
pass = await PassAsync(budget).ConfigureAwait(false);
|
||||
if (pass.Result.Complete) return pass.Result.Samples;
|
||||
preWindow = Math.Max(preWindow, pass.Result.PreWindowRows);
|
||||
if (pass.Budget < budget) break; // bounded by a ceiling learned meanwhile
|
||||
if (room >= _maxPreWindowRows) break; // the limit's worth of room did not suffice
|
||||
room = (int)Math.Min(_maxPreWindowRows, Math.Max(preWindow, 2L * room));
|
||||
}
|
||||
|
||||
throw new HistoryReadLimitExceededException(cap,
|
||||
$"The raw read of '{fullReference}' could not be read completely: more than {second.Result.PreWindowRows} rows lie in " +
|
||||
$"the {RawAggregateReader.BoundaryMargin.TotalMilliseconds} ms before the window and exhausted the row budget.");
|
||||
$"The raw read of '{fullReference}' could not be read completely: more than {pass.Result.PreWindowRows} rows lie in " +
|
||||
$"the {RawAggregateReader.BoundaryMargin.TotalMilliseconds} ms before the window and exhausted the row budget.")
|
||||
{ PreWindowRows = preWindow };
|
||||
|
||||
// Pre-window rows the allowance did not cover are what crowded the read; fewer are not the cause (#694).
|
||||
static int Crowding(int preWindowRows) => preWindowRows > RawAggregateReader.PreWindowAllowance ? preWindowRows : 0;
|
||||
|
||||
// One drained pass; a strict gateway's refusal of the budget is retried once at the ceiling it names (#691.3).
|
||||
Task<((List<HistorianSample> Samples, int PreWindowRows, bool Complete) Result, int Budget)> PassAsync(int budget) =>
|
||||
@@ -230,7 +298,11 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
{
|
||||
if (cap == 0 || samples.Count < cap) samples.Add(sample);
|
||||
}
|
||||
else if (sample.Timestamp is { } ts && ts.ToDateTime() < startUtc) preWindow++;
|
||||
// A row before the window — or, for a window starting at/before the epoch (no look-back), AVEVA's synthetic row at
|
||||
// the query start, which is no sample either (#694 F1: it used to set reachedEnd, so a ceiling-truncated read
|
||||
// passed as complete) — is a pre-window row.
|
||||
else if (sample.Timestamp is { } ts
|
||||
&& (ts.ToDateTime() < startUtc || !RawAggregateReader.IsAfterFloor(ts.ToDateTime()))) preWindow++;
|
||||
else if (sample.Timestamp is not null) reachedEnd = true;
|
||||
}
|
||||
|
||||
@@ -287,6 +359,11 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
if (plan.BucketCount == 0) return new HistoryReadResult([], ContinuationPoint: null);
|
||||
try
|
||||
{
|
||||
if (!await _tags.ExistsAsync(fullReference, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
RecordOutcome(success: true, error: null);
|
||||
return new HistoryReadResult(plan.Assemble(new Dictionary<long, DataValueSnapshot>(), t => AggregateEvaluator.NoData(t, _time.GetUtcNow().UtcDateTime)), ContinuationPoint: null);
|
||||
}
|
||||
var byBucket = new Dictionary<long, DataValueSnapshot>();
|
||||
foreach (var request in NativeAggregatePlan.Requests(plan))
|
||||
{
|
||||
@@ -310,6 +387,7 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
catch (Exception ex)
|
||||
{
|
||||
RecordReadFailure(ex);
|
||||
if (IsTransportFailure(ex)) throw new System.IO.IOException("HistorianGateway read transport is unavailable.", ex);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
@@ -325,6 +403,11 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
try
|
||||
{
|
||||
var now = _time.GetUtcNow().UtcDateTime;
|
||||
if (plan.BucketCount > 0 && !await _tags.ExistsAsync(fullReference, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
RecordOutcome(success: true, error: null);
|
||||
return new HistoryReadResult(plan.Assemble(new Dictionary<long, DataValueSnapshot>(), t => AggregateEvaluator.NoData(t, now)), ContinuationPoint: null);
|
||||
}
|
||||
var values = await RawAggregateReader
|
||||
.ReadAsync(_client, _ceiling, fullReference, aggregate, plan, _maxRawRows, now, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
@@ -353,6 +436,7 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
catch (Exception ex)
|
||||
{
|
||||
RecordReadFailure(ex);
|
||||
if (IsTransportFailure(ex)) throw new System.IO.IOException("HistorianGateway read transport is unavailable.", ex);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
@@ -364,11 +448,75 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
internal static bool IsMaxValuesCeilingRejection(Exception ex) => GatewayReadLimits.IsCeilingRejection(ex);
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>
|
||||
/// #693: served from RAW gateway reads — each requested time's stored sample at-or-before and after it — evaluated with
|
||||
/// the SQL backends' shared <see cref="AtTimeEvaluator"/> (Part 11 §6.5.4), so the Gateway answers what SqlServer and
|
||||
/// QuestDB answer for the same stored samples; see <see cref="GatewayAtTimeReader"/> for the read plan and its cost.
|
||||
/// AVEVA's own interpolated answer (the gateway's <c>ReadAtTime</c>) returned the previous value as Good at a stored
|
||||
/// NULL's time, null + Bad at a Good sample right after a NULL, a drifted value at an exact stored sample, and no
|
||||
/// Interpolated bit; it is only used when <c>ServerHistorian:GatewayAtTimeFromRaw = false</c> (kill switch).
|
||||
/// A gateway value is a Double (or a String) whatever the node's type (AVEVA stores a Boolean as a 0/1 <c>Int2</c>), so
|
||||
/// without a type hint every numeric tag is interpolated linearly; the node manager passes the node's type through
|
||||
/// <see cref="ReadAtTimeAsync(string, IReadOnlyList{DateTime}, TypeCode?, CancellationToken)"/>.
|
||||
/// </remarks>
|
||||
public Task<HistoryReadResult> ReadAtTimeAsync(
|
||||
string fullReference, IReadOnlyList<DateTime> timestampsUtc, CancellationToken cancellationToken) =>
|
||||
ReadAtTimeAsync(fullReference, timestampsUtc, valueType: null, cancellationToken);
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>
|
||||
/// #693: the hint makes the AtTime rules type-exact, as on the SQL backends — the gateway's Double rows are converted to
|
||||
/// the node's type BEFORE the shared evaluator runs (Boolean = value ≠ 0 ⇒ stepped; an integer type ⇒ Int64 ⇒
|
||||
/// interpolated and rounded half away from zero, then returned as the node's integer type; String ⇒ stepped; Float /
|
||||
/// Double / no hint ⇒ linear Double). Ignored with the kill switch (AVEVA's own answer).
|
||||
/// </remarks>
|
||||
public async Task<HistoryReadResult> ReadAtTimeAsync(
|
||||
string fullReference, IReadOnlyList<DateTime> timestampsUtc, TypeCode? valueType, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(timestampsUtc);
|
||||
if (!_atTimeFromRaw) return await ReadAtTimeNativeAsync(fullReference, timestampsUtc, cancellationToken).ConfigureAwait(false);
|
||||
try
|
||||
{
|
||||
var now = _time.GetUtcNow().UtcDateTime;
|
||||
if (timestampsUtc.Count > 0 && !await _tags.ExistsAsync(fullReference, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
RecordOutcome(success: true, error: null);
|
||||
return MissingAtTime(timestampsUtc, now);
|
||||
}
|
||||
var values = await GatewayAtTimeReader
|
||||
.ReadAsync(_client, _ceiling, fullReference, timestampsUtc, now, cancellationToken, valueType)
|
||||
.ConfigureAwait(false);
|
||||
RecordOutcome(success: true, error: null);
|
||||
return new HistoryReadResult(values, ContinuationPoint: null);
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
throw; // caller-requested, not a backend fault
|
||||
}
|
||||
catch (HistoryReadLimitExceededException)
|
||||
{
|
||||
throw; // a client-request limit (a pathological tie cluster), not a backend fault
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
RecordReadFailure(ex);
|
||||
if (IsTransportFailure(ex)) throw new System.IO.IOException("HistorianGateway read transport is unavailable.", ex);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>The pre-#693 AtTime: AVEVA's own interpolation, aligned to the request (kill switch only).</summary>
|
||||
private async Task<HistoryReadResult> ReadAtTimeNativeAsync(
|
||||
string fullReference, IReadOnlyList<DateTime> timestampsUtc, CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (timestampsUtc.Count > 0 && !await _tags.ExistsAsync(fullReference, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
RecordOutcome(success: true, error: null);
|
||||
return MissingAtTime(timestampsUtc, _time.GetUtcNow().UtcDateTime);
|
||||
}
|
||||
|
||||
var samples = await _client
|
||||
.ReadAtTimeAsync(fullReference, timestampsUtc, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
@@ -379,6 +527,7 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
catch (Exception ex)
|
||||
{
|
||||
RecordReadFailure(ex);
|
||||
if (IsTransportFailure(ex)) throw new System.IO.IOException("HistorianGateway read transport is unavailable.", ex);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
@@ -391,6 +540,17 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
/// </remarks>
|
||||
public bool SupportsReverseRaw => false;
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>
|
||||
/// Gitea #694: <c>0</c> until a strict gateway (<c>AllowStreamingBeyondCeiling=false</c>) has refused a budget and
|
||||
/// named its ceiling (#691.3, <see cref="GatewayRowCeiling"/>); then that ceiling less the
|
||||
/// <see cref="RawAggregateReader.PreWindowAllowance"/> a capped read adds for AVEVA's boundary row — so a page of this
|
||||
/// size asks for exactly the ceiling. Back to <c>0</c> when a re-probe finds the gateway streaming again.
|
||||
/// </remarks>
|
||||
public int MaxRawValuesPerRead => _ceiling.Learned is var c and > 0
|
||||
? Math.Max(1, c - RawAggregateReader.PreWindowAllowance)
|
||||
: 0;
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>
|
||||
/// <c>false</c>: the gateway's <c>ReadEvents</c> streams ascending only (Gitea #675). The server never sends it a
|
||||
@@ -463,6 +623,7 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
catch (Exception ex)
|
||||
{
|
||||
RecordReadFailure(ex);
|
||||
if (IsTransportFailure(ex)) throw new System.IO.IOException("HistorianGateway read transport is unavailable.", ex);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
@@ -494,7 +655,7 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
|
||||
/// <summary>
|
||||
/// Reconciles a gateway at-time reply against the requested timestamps to honour the
|
||||
/// <see cref="IHistorianDataSource.ReadAtTimeAsync"/> contract: exactly one snapshot per
|
||||
/// <see cref="IHistorianDataSource.ReadAtTimeAsync(string, IReadOnlyList{DateTime}, CancellationToken)"/> contract: exactly one snapshot per
|
||||
/// requested timestamp, in request order. Returned samples are indexed by timestamp ticks;
|
||||
/// any requested timestamp the gateway did not return is filled with a Bad-quality
|
||||
/// (<c>0x80000000</c>) snapshot stamped at the requested time rather than positionally
|
||||
@@ -537,6 +698,19 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
return result;
|
||||
}
|
||||
|
||||
/// <summary>No-data values for a missing tag, stamped with the client's requested times.</summary>
|
||||
private static HistoryReadResult MissingAtTime(IReadOnlyList<DateTime> times, DateTime now) =>
|
||||
new(times.Select(t => AtTimeEvaluator.Evaluate(t, null, null, now, now)).ToArray(), ContinuationPoint: null);
|
||||
|
||||
/// <summary>Classifies actual gRPC/package transport failures without treating refusals as outages (#697).</summary>
|
||||
private static bool IsTransportFailure(Exception ex) => ex switch
|
||||
{
|
||||
ZB.MOM.WW.HistorianGateway.Client.HistorianGatewayUnavailableException => true,
|
||||
RpcException rpc => rpc.StatusCode is Grpc.Core.StatusCode.Unavailable or Grpc.Core.StatusCode.DeadlineExceeded,
|
||||
TimeoutException => true,
|
||||
_ => ex.InnerException is { } inner && IsTransportFailure(inner),
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// Records a failed read: bumps the health counters and logs a generic, redaction-safe
|
||||
/// debug line (no tag, host, or value). The exception itself is rethrown by the caller.
|
||||
|
||||
+5
-1
@@ -40,7 +40,11 @@ public static class GatewayHistorian
|
||||
HistorianGatewayClientAdapter.Create(options, loggerFactory),
|
||||
logger,
|
||||
// #681: the raw-evaluated per-sample aggregates share the SQL backends' raw-row cap.
|
||||
maxRawRows: options.MaxTimeWeightedRawRows);
|
||||
maxRawRows: options.MaxTimeWeightedRawRows,
|
||||
// #693: AtTime from raw reads + the shared evaluator; false = AVEVA's own interpolation (kill switch).
|
||||
atTimeFromRaw: options.GatewayAtTimeFromRaw,
|
||||
// #694 R2: a crowding re-read never makes room for more pre-window rows than the tie-cluster limit.
|
||||
maxPreWindowRows: options.MaxTieClusterOverfetch);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
||||
@@ -76,13 +76,15 @@ internal static class GatewayReadLimits
|
||||
/// the read is re-issued at <c>N</c>; refused naming <c>C</c> again ⇒ the <c>C</c>-bounded result stands (no repeat).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Why no TTL.</b> A raise only matters to a read that needs more than <c>C</c>, and such a read re-probes anyway
|
||||
/// (rule 3), so a timer would only add refusals for reads that fit. The cost of the rule is one refused re-probe per
|
||||
/// genuinely over-ceiling read — a read that fails <c>BadTooManyOperations</c> whatever is done.
|
||||
/// <b>Re-probe timing (#696).</b> Raw and AtTime reads re-probe incomplete streams immediately. Per-sample
|
||||
/// aggregate reads opt into a 30-second cooldown after a refusal; a full stream during it remains a request
|
||||
/// limit, never a truncated answer. After the cooldown the next incomplete read detects a raised ceiling.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
internal sealed class GatewayRowCeiling
|
||||
internal sealed class GatewayRowCeiling(TimeProvider? timeProvider = null)
|
||||
{
|
||||
private readonly TimeProvider _time = timeProvider ?? TimeProvider.System;
|
||||
private long _lastRefusalTicks;
|
||||
private int _learned; // 0 = unknown / not strict
|
||||
|
||||
/// <summary>The learned ceiling (0 = none).</summary>
|
||||
@@ -93,8 +95,9 @@ internal sealed class GatewayRowCeiling
|
||||
/// <param name="budget">The <c>max_values</c> the read wants.</param>
|
||||
/// <param name="pass">One drained read with the given budget.</param>
|
||||
/// <param name="needsMore">True when a pass with the given budget delivered it all without proving completeness.</param>
|
||||
/// <param name="deferRecentReprobe">Aggregate-only opt-in: reuse a refusal for 30 seconds before re-probing an incomplete read.</param>
|
||||
/// <returns>The result and the budget it was read with.</returns>
|
||||
public async Task<(T Result, int Budget)> ReadAsync<T>(int budget, Func<int, Task<T>> pass, Func<T, int, bool> needsMore)
|
||||
public async Task<(T Result, int Budget)> ReadAsync<T>(int budget, Func<int, Task<T>> pass, Func<T, int, bool> needsMore, bool deferRecentReprobe = false)
|
||||
{
|
||||
var learned = Learned;
|
||||
if (learned <= 0 || learned >= budget) return await AttemptAsync(budget, pass).ConfigureAwait(false);
|
||||
@@ -102,6 +105,10 @@ internal sealed class GatewayRowCeiling
|
||||
var bounded = await AttemptAsync(learned, pass).ConfigureAwait(false);
|
||||
if (bounded.Budget < learned || !needsMore(bounded.Result, bounded.Budget)) return bounded;
|
||||
|
||||
// #696: aggregate reads may reuse a recent refusal for 30 seconds. Raw paging keeps immediate re-probes.
|
||||
if (deferRecentReprobe && _time.GetUtcNow().UtcTicks - Volatile.Read(ref _lastRefusalTicks) < TimeSpan.FromSeconds(30).Ticks)
|
||||
return bounded;
|
||||
|
||||
// Rule 3: full at a possibly stale ceiling — re-probe once at the full budget.
|
||||
int fresh;
|
||||
try
|
||||
@@ -114,10 +121,16 @@ internal sealed class GatewayRowCeiling
|
||||
{
|
||||
// fall through (outside the catch)
|
||||
}
|
||||
Volatile.Write(ref _learned, fresh);
|
||||
Learn(fresh);
|
||||
return fresh > learned ? (await pass(fresh).ConfigureAwait(false), fresh) : bounded;
|
||||
}
|
||||
|
||||
private void Learn(int ceiling)
|
||||
{
|
||||
Volatile.Write(ref _lastRefusalTicks, _time.GetUtcNow().UtcTicks);
|
||||
Volatile.Write(ref _learned, ceiling);
|
||||
}
|
||||
|
||||
private async Task<(T Result, int Budget)> AttemptAsync<T>(int budget, Func<int, Task<T>> pass)
|
||||
{
|
||||
int ceiling;
|
||||
@@ -129,7 +142,7 @@ internal sealed class GatewayRowCeiling
|
||||
{
|
||||
// fall through to the retry (outside the catch, so a second refusal is not nested in this one)
|
||||
}
|
||||
Volatile.Write(ref _learned, ceiling);
|
||||
Learn(ceiling);
|
||||
return (await pass(ceiling).ConfigureAwait(false), ceiling);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
using Grpc.Core;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway;
|
||||
|
||||
/// <summary>Bounded metadata cache (#653). Missing tags expire quickly; failures never become missing tags.</summary>
|
||||
internal sealed class GatewayTagPresenceCache(IHistorianGatewayClient client, TimeProvider time)
|
||||
{
|
||||
internal const int Capacity = 4096;
|
||||
private readonly object _sync = new();
|
||||
private readonly Dictionary<string, (bool Exists, DateTimeOffset Expires)> _entries = new(StringComparer.Ordinal);
|
||||
|
||||
public async Task<bool> ExistsAsync(string tag, CancellationToken ct)
|
||||
{
|
||||
ct.ThrowIfCancellationRequested();
|
||||
lock (_sync)
|
||||
if (_entries.TryGetValue(tag, out var entry) && time.GetUtcNow() < entry.Expires) return entry.Exists;
|
||||
|
||||
bool exists;
|
||||
try { exists = await client.TagExistsAsync(tag, ct).ConfigureAwait(false); }
|
||||
catch (Exception ex) when (IsNotFound(ex)) { exists = false; }
|
||||
ct.ThrowIfCancellationRequested();
|
||||
lock (_sync)
|
||||
{
|
||||
if (!_entries.ContainsKey(tag) && _entries.Count >= Capacity) _entries.Remove(_entries.Keys.First());
|
||||
_entries[tag] = (exists, time.GetUtcNow().AddSeconds(exists ? 60 : 5));
|
||||
}
|
||||
return exists;
|
||||
}
|
||||
|
||||
private static bool IsNotFound(Exception ex)
|
||||
{
|
||||
for (Exception? e = ex; e is not null; e = e.InnerException)
|
||||
if (e is RpcException { StatusCode: StatusCode.NotFound }) return true;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
+11
@@ -81,6 +81,17 @@ public sealed class HistorianGatewayClientAdapter : IHistorianGatewayClient, IDi
|
||||
return new HistorianGatewayClientAdapter(HistorianGatewayClient.Create(clientOptions));
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<bool> TagExistsAsync(string tag, CancellationToken ct)
|
||||
{
|
||||
// #653: older gateway SDKs misparse a missing GetTagInfo response as Internal. A catalog query proves
|
||||
// absence without guessing from an error. Even a glob-like tag is checked by exact name, never by row count.
|
||||
var found = false;
|
||||
await foreach (var name in _inner.BrowseTagNamesAsync(tag, ct).ConfigureAwait(false))
|
||||
found |= string.Equals(name, tag, StringComparison.OrdinalIgnoreCase);
|
||||
return found; // drain first: a trailing stream fault must propagate and must not become a cache entry
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public IAsyncEnumerable<HistorianSample> ReadRawAsync(
|
||||
string tag, DateTime startUtc, DateTime endUtc, int maxValues, CancellationToken ct) =>
|
||||
|
||||
@@ -11,6 +11,12 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway;
|
||||
/// </summary>
|
||||
public interface IHistorianGatewayClient : IAsyncDisposable
|
||||
{
|
||||
/// <summary>Checks the tag catalog; false means explicitly absent, while failures propagate.</summary>
|
||||
/// <param name="tag">Historian tag name.</param>
|
||||
/// <param name="ct">Cancellation token.</param>
|
||||
/// <returns>Whether the historian contains the tag.</returns>
|
||||
Task<bool> TagExistsAsync(string tag, CancellationToken ct);
|
||||
|
||||
/// <summary>Streams raw historian samples for a tag over a time window.</summary>
|
||||
/// <param name="tag">The historian tag name to read.</param>
|
||||
/// <param name="startUtc">Inclusive start of the requested time window (UTC).</param>
|
||||
|
||||
@@ -188,7 +188,7 @@ internal static class RawAggregateReader
|
||||
var requested = MaxValuesFor(maxWindowRows);
|
||||
var (pass, maxValues) = await ceiling.ReadAsync(requested,
|
||||
budget => ReadWindowAsync(client, tag, aggregate, plan, requestStart, budget, maxWindowRows, ct),
|
||||
(r, budget) => !r.OverCap && r.Received >= budget).ConfigureAwait(false);
|
||||
(r, budget) => !r.OverCap && r.Received >= budget, deferRecentReprobe: true).ConfigureAwait(false);
|
||||
var (buckets, earliest, latest, received, overCap) = pass;
|
||||
if (overCap) throw LimitExceeded(tag, maxWindowRows);
|
||||
if (received >= maxValues) throw BudgetExhausted(tag, maxWindowRows, requested, maxValues);
|
||||
|
||||
@@ -137,6 +137,7 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource
|
||||
catch (Exception ex)
|
||||
{
|
||||
RecordFailure(ex);
|
||||
if (IsTransportFailure(ex)) throw new System.IO.IOException("Historian read transport is unavailable.", ex);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
@@ -232,6 +233,7 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource
|
||||
catch (Exception ex)
|
||||
{
|
||||
RecordFailure(ex);
|
||||
if (IsTransportFailure(ex)) throw new System.IO.IOException("Historian read transport is unavailable.", ex);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
@@ -316,6 +318,7 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource
|
||||
catch (Exception ex)
|
||||
{
|
||||
RecordFailure(ex);
|
||||
if (IsTransportFailure(ex)) throw new System.IO.IOException("Historian read transport is unavailable.", ex);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
@@ -359,6 +362,10 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource
|
||||
/// <remarks>Reverse raw reads are served natively (<c>(end, start]</c>, <c>ORDER BY … DESC</c>; #669/#686).</remarks>
|
||||
public bool SupportsReverseRaw => true;
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks><c>0</c>: a capped raw read is one bounded statement — no backend row ceiling (Gitea #694).</remarks>
|
||||
public int MaxRawValuesPerRead => 0;
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>Reverse events reads are served natively (<c>ORDER BY … DESC</c>, Gitea #675).</remarks>
|
||||
public bool SupportsReverseEvents => true;
|
||||
@@ -439,6 +446,7 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource
|
||||
catch (Exception ex)
|
||||
{
|
||||
RecordFailure(ex);
|
||||
if (IsTransportFailure(ex)) throw new System.IO.IOException("Historian read transport is unavailable.", ex);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
@@ -534,6 +542,7 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource
|
||||
catch (Exception ex)
|
||||
{
|
||||
RecordFailure(ex);
|
||||
if (IsTransportFailure(ex)) throw new System.IO.IOException("Historian read transport is unavailable.", ex);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
@@ -697,6 +706,11 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource
|
||||
|
||||
private void RecordSuccess() => _health.RecordSuccess();
|
||||
|
||||
/// <summary>Classifies provider transport/timeouts, keeping SQL/query errors distinct (#697).</summary>
|
||||
private static bool IsTransportFailure(Exception ex) => ex is NpgsqlException { IsTransient: true } and not PostgresException
|
||||
|| ex is PostgresException pg && (pg.SqlState.StartsWith("08", StringComparison.Ordinal)
|
||||
|| pg.SqlState is "57P01" or "57P02" or "57P03");
|
||||
|
||||
private void RecordFailure(Exception ex)
|
||||
{
|
||||
// A cancelled read (deadline / client abort) is not a backend fault (the counters ignore it too).
|
||||
|
||||
@@ -49,8 +49,10 @@ public static class AtTimeEvaluator
|
||||
{
|
||||
var fraction = (double)(t.Ticks - p.TimeUtc.Ticks) / (n.TimeUtc.Ticks - p.TimeUtc.Ticks);
|
||||
var v = p.Value.Number + (nv - p.Value.Number) * fraction;
|
||||
// Each arm boxed on its own: `cond ? (long)x : v` is a DOUBLE expression in C# (the long widens), which returned
|
||||
// every interpolated Int64 as a Double until #693.
|
||||
object value = p.Value.Kind == HistorianValueKind.Int64
|
||||
? (long)Math.Round(v, MidpointRounding.AwayFromZero)
|
||||
? (object)(long)Math.Round(v, MidpointRounding.AwayFromZero)
|
||||
: v;
|
||||
var status = p.IsGood && n.IsGood ? HistorianStatusCodes.Good : HistorianStatusCodes.UncertainDataSubNormal;
|
||||
return new DataValueSnapshot(value, HistorianInfoBits.WithInterpolated(status), t, serverTime);
|
||||
|
||||
+12
@@ -132,6 +132,7 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
|
||||
catch (Exception ex)
|
||||
{
|
||||
RecordFailure(ex);
|
||||
if (IsTransportFailure(ex)) throw new System.IO.IOException("Historian read transport is unavailable.", ex);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
@@ -229,6 +230,7 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
|
||||
catch (Exception ex) when (ex is not NotSupportedException)
|
||||
{
|
||||
RecordFailure(ex);
|
||||
if (IsTransportFailure(ex)) throw new System.IO.IOException("Historian read transport is unavailable.", ex);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
@@ -306,6 +308,7 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
|
||||
catch (Exception ex)
|
||||
{
|
||||
RecordFailure(ex);
|
||||
if (IsTransportFailure(ex)) throw new System.IO.IOException("Historian read transport is unavailable.", ex);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
@@ -316,6 +319,10 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
|
||||
/// <remarks>Reverse raw reads are served natively (<c>(end, start]</c>, <c>ORDER BY … DESC</c>; #669/#686).</remarks>
|
||||
public bool SupportsReverseRaw => true;
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks><c>0</c>: a capped raw read is one bounded statement — no backend row ceiling (Gitea #694).</remarks>
|
||||
public int MaxRawValuesPerRead => 0;
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>Reverse events reads are served natively (<c>ORDER BY … DESC</c>, Gitea #675).</remarks>
|
||||
public bool SupportsReverseEvents => true;
|
||||
@@ -397,6 +404,7 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
|
||||
catch (Exception ex)
|
||||
{
|
||||
RecordFailure(ex);
|
||||
if (IsTransportFailure(ex)) throw new System.IO.IOException("Historian read transport is unavailable.", ex);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
@@ -508,6 +516,7 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
|
||||
catch (Exception ex)
|
||||
{
|
||||
RecordFailure(ex);
|
||||
if (IsTransportFailure(ex)) throw new System.IO.IOException("Historian read transport is unavailable.", ex);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
@@ -631,6 +640,9 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
|
||||
|
||||
private void RecordSuccess() => _health.RecordSuccess();
|
||||
|
||||
/// <summary>Classifies provider transport/timeouts, keeping SQL/query errors distinct (#697).</summary>
|
||||
private static bool IsTransportFailure(Exception ex) => ex is SqlException sql && sql.Number is -2 or 0 or 53 or 64 or 233 or 258 or 10053 or 10054 or 10060 or 10061 or 10065 or 11001 or 40197 or 40613;
|
||||
|
||||
private void RecordFailure(Exception ex)
|
||||
{
|
||||
// A cancelled read (deadline / client abort) is not a backend fault (the counters ignore it too).
|
||||
|
||||
@@ -320,6 +320,9 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
|
||||
// Gitea #691: the ReturnBounds kill switch (default on).
|
||||
_server.NodeManager.HonorReturnBounds = HonorReturnBounds;
|
||||
|
||||
// Gitea #694: the opt-in HistoryReadEvents horizon (default off = exact to each read's own window).
|
||||
_server.NodeManager.EventReadHorizon = EventReadHorizon;
|
||||
|
||||
// Propagate the S3 HistoryRead concurrency bounds (arch-review 03/S3): bounded per-node fan-out
|
||||
// within a batch, the process-wide concurrent-batch limiter, and the per-request deadline. Defaults
|
||||
// (4 / 16 / 60 s) survive when the ServerHistorian section is absent.
|
||||
@@ -356,6 +359,12 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
|
||||
/// </summary>
|
||||
internal bool HonorReturnBounds => _serverHistorianOptions.HonorReturnBounds;
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #694 — the <c>ServerHistorian:EventReadHorizon</c> <c>StartAsync</c> applies to the node manager (default
|
||||
/// <see cref="TimeSpan.Zero"/> = none). Exposed so the wiring is testable without booting the SDK.
|
||||
/// </summary>
|
||||
internal TimeSpan EventReadHorizon => _serverHistorianOptions.EventReadHorizon;
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #674 — what <c>StartAsync</c> wires onto the node manager for HistoryUpdate: the Primary gate (read from
|
||||
/// the <see cref="IHistoryUpdateRoleView"/> on every call; null when none is registered, i.e. allow), the
|
||||
|
||||
@@ -35,7 +35,9 @@
|
||||
"AllowUntrustedServerCertificate": false,
|
||||
"CaCertificatePath": null,
|
||||
"CallTimeout": "00:00:30",
|
||||
"MaxTieClusterOverfetch": 65536
|
||||
"MaxTieClusterOverfetch": 65536,
|
||||
"EventReadHorizon": "00:00:00",
|
||||
"_EventReadHorizonComment": "Gitea #694, opt-in. 00:00:00 = no horizon: HistoryReadEvents is exact to each read's window (on the Gateway a reverse/end-only events read proves 'nothing older' with one read down to 1970 - seconds on a large Runtime.dbo.Events). A positive span (e.g. 365.00:00:00) never reads events older than now minus it, in either direction - LOSSY: older events become invisible to HistoryReadEvents."
|
||||
},
|
||||
"ContinuousHistorization": {
|
||||
"_comment": "Continuous historization of driver (non-Galaxy) tag values: a crash-safe FasterLog outbox + recorder draining to the ServerHistorian gateway's WriteLiveValues. Disabled => no recorder is spawned. Requires ServerHistorian to be configured; the gateway connection (endpoint/key/TLS) is sourced from the ServerHistorian section, not here.",
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
namespace ZB.MOM.WW.OtOpcUa.OpcUaServer;
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #694 item 2: the opt-in <c>ServerHistorian:EventReadHorizon</c> — HistoryReadEvents never reads events older than
|
||||
/// <c>now − horizon</c>. Off (<see cref="TimeSpan.Zero"/>, the default) a read is exact down to whatever its window says,
|
||||
/// which on the Gateway means a reverse / end-only read walked back with forward reads proves "nothing older" with one
|
||||
/// read down to the Unix epoch (#691) — ~3.8 s live on <c>Runtime.dbo.Events</c>, and an unfiltered one can hit the
|
||||
/// gateway's 30 s SQL timeout. A forward read whose StartTime is far in the past costs the same on that table. The
|
||||
/// horizon bounds both, at a stated price: events older than it are not returned, in either direction, on any backend.
|
||||
/// </summary>
|
||||
internal static class EventHorizon
|
||||
{
|
||||
/// <summary>
|
||||
/// Raises the EARLIER bound of a fresh events window to the horizon's floor (<c>now − horizon</c>, inclusive): the
|
||||
/// start of a forward <c>[start, end)</c> window, the exclusive lower end of a reverse <c>(end, start]</c> one.
|
||||
/// </summary>
|
||||
/// <param name="startUtc">The window start (the later bound of a reverse read).</param>
|
||||
/// <param name="endUtc">The window end (the exclusive earlier bound of a reverse read).</param>
|
||||
/// <param name="reverse">Whether the read is newest-first (<c>start > end</c>).</param>
|
||||
/// <param name="horizon">The horizon; non-positive = none (nothing changes).</param>
|
||||
/// <param name="nowUtc">Now.</param>
|
||||
/// <returns><c>false</c> when the whole window lies before the floor — the read is empty and must not be sent.</returns>
|
||||
public static bool TryClamp(ref DateTime startUtc, ref DateTime endUtc, bool reverse, TimeSpan horizon, DateTime nowUtc)
|
||||
{
|
||||
if (horizon <= TimeSpan.Zero) return true;
|
||||
var floor = nowUtc.Ticks - DateTime.MinValue.Ticks > horizon.Ticks
|
||||
? new DateTime(nowUtc.Ticks - horizon.Ticks, DateTimeKind.Utc)
|
||||
: new DateTime(0, DateTimeKind.Utc);
|
||||
if (floor.Ticks == 0) return true;
|
||||
|
||||
if (reverse)
|
||||
{
|
||||
if (startUtc < floor) return false; // (end, start] lies wholly before the floor
|
||||
var below = floor.AddTicks(-1); // exclusive lower end ⇒ the floor instant itself is read
|
||||
if (endUtc < below) endUtc = below;
|
||||
return true;
|
||||
}
|
||||
|
||||
if (endUtc <= floor) return false; // [start, end) lies wholly before the floor
|
||||
if (startUtc < floor) startUtc = floor;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
@@ -17,8 +17,13 @@ namespace ZB.MOM.WW.OtOpcUa.OpcUaServer;
|
||||
/// <param name="Open">The end time was not given (a one-sided read's sentinel): there is no end bound to look up.</param>
|
||||
/// <param name="DataExhausted">The data is drained and the end bound did not fit on the previous page.</param>
|
||||
/// <param name="LastReturnedUtc">The last value's time returned on an earlier page (stamps an open bound's placeholder).</param>
|
||||
/// <param name="Resolved">
|
||||
/// Gitea #692: the end bound an earlier page already looked up (a stored sample — never a placeholder), so the page that
|
||||
/// returns it does not read the backend again. <c>null</c> = not looked up yet.
|
||||
/// </param>
|
||||
internal sealed record HistoryEndBound(
|
||||
DateTime AtUtc, bool Reverse, bool Inclusive, bool Open, bool DataExhausted = false, DateTime? LastReturnedUtc = null);
|
||||
DateTime AtUtc, bool Reverse, bool Inclusive, bool Open, bool DataExhausted = false, DateTime? LastReturnedUtc = null,
|
||||
DataValueSnapshot? Resolved = null);
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #691: OPC UA Part 11 bounding values for HistoryRead-Raw (<c>ReadRawModifiedDetails.ReturnBounds</c>), composed
|
||||
@@ -59,6 +64,15 @@ internal static class HistoryBounds
|
||||
{
|
||||
private static readonly TimeSpan OpenBoundStep = TimeSpan.FromSeconds(1);
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #692: the placeholders this server synthesised (<see cref="NotFound"/>), by reference. A placeholder is known by
|
||||
/// this marker, never by its shape: a stored row can legitimately carry <c>Bad_BoundNotFound</c> with a null value (a
|
||||
/// client may write one through HistoryUpdate), and that row is data.
|
||||
/// </summary>
|
||||
private static readonly System.Runtime.CompilerServices.ConditionalWeakTable<DataValueSnapshot, object> Placeholders = new();
|
||||
|
||||
private static readonly object PlaceholderMark = new();
|
||||
|
||||
/// <summary>The end bound owed by a fresh read over the normalised window.</summary>
|
||||
/// <param name="startUtc">The (normalised) start: the later time of a reverse read.</param>
|
||||
/// <param name="endUtc">The (normalised) end; a one-sided read's sentinel (tick 0 / MaxValue) is an open bound.</param>
|
||||
@@ -66,11 +80,13 @@ internal static class HistoryBounds
|
||||
public static HistoryEndBound For(DateTime startUtc, DateTime endUtc) =>
|
||||
new(endUtc, Reverse: startUtc > endUtc, Inclusive: startUtc != endUtc, Open: IsOpen(endUtc));
|
||||
|
||||
/// <summary>Whether a returned value is a <see cref="NotFound"/> placeholder rather than a stored sample.</summary>
|
||||
/// <summary>
|
||||
/// Whether a returned value is a <see cref="NotFound"/> placeholder this server synthesised rather than a stored sample
|
||||
/// — by an explicit marker (#692), so a stored row whose status happens to be <c>Bad_BoundNotFound</c> is not taken for one.
|
||||
/// </summary>
|
||||
/// <param name="value">The value.</param>
|
||||
/// <returns><c>true</c> for a <c>Bad_BoundNotFound</c> placeholder.</returns>
|
||||
public static bool IsPlaceholder(DataValueSnapshot value) =>
|
||||
value.StatusCode == StatusCodes.BadBoundNotFound && value.Value is null;
|
||||
/// <returns><c>true</c> only for an instance <see cref="NotFound"/> produced.</returns>
|
||||
public static bool IsPlaceholder(DataValueSnapshot value) => Placeholders.TryGetValue(value, out _);
|
||||
|
||||
/// <summary>The pending end bound to store behind a continuation point, remembering the page's last returned time.</summary>
|
||||
/// <param name="bound">The pending end bound (null = bounds not requested).</param>
|
||||
@@ -84,8 +100,12 @@ internal static class HistoryBounds
|
||||
/// <summary>The <c>Bad_BoundNotFound</c> placeholder (null value) stamped at <paramref name="atUtc"/>.</summary>
|
||||
/// <param name="atUtc">The bound's time.</param>
|
||||
/// <returns>The placeholder.</returns>
|
||||
public static DataValueSnapshot NotFound(DateTime atUtc) =>
|
||||
new(null, StatusCodes.BadBoundNotFound, atUtc, atUtc);
|
||||
public static DataValueSnapshot NotFound(DateTime atUtc)
|
||||
{
|
||||
var placeholder = new DataValueSnapshot(null, StatusCodes.BadBoundNotFound, atUtc, atUtc);
|
||||
Placeholders.AddOrUpdate(placeholder, PlaceholderMark);
|
||||
return placeholder;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The start bound to return ahead of the data, or <c>null</c> when a stored sample sits on StartTime (that sample is
|
||||
@@ -123,6 +143,8 @@ internal static class HistoryBounds
|
||||
return NotFound(new DateTime(ticks, DateTimeKind.Utc));
|
||||
}
|
||||
|
||||
if (bound.Resolved is { } resolved) return resolved; // #692: an earlier page already looked it up
|
||||
|
||||
DataValueSnapshot? found;
|
||||
if (bound.Reverse)
|
||||
found = await LatestAtOrBeforeAsync(source, tagname, bound.AtUtc, maxTieCluster, nowUtc, ct).ConfigureAwait(false);
|
||||
|
||||
@@ -66,6 +66,11 @@ namespace ZB.MOM.WW.OtOpcUa.OpcUaServer;
|
||||
/// <see cref="HistoryCursorKind.Raw"/> with <c>ReturnBounds = true</c> only (Gitea #691): the end bound still owed —
|
||||
/// Part 11 returns it with the last page. <c>null</c> = bounds not requested.
|
||||
/// </param>
|
||||
/// <param name="WithinTieCluster">
|
||||
/// <see cref="HistoryCursorKind.Raw"/> only (Gitea #692): the previous page was a slice of an oversized tie cluster at
|
||||
/// <paramref name="NextStartUtc"/> that still has un-emitted ties — the next page over-fetches the cluster directly
|
||||
/// instead of first re-reading <c>N + skip + 1</c> rows that are known to be ties.
|
||||
/// </param>
|
||||
internal sealed record HistoryContinuationState(
|
||||
string Tagname,
|
||||
DateTime NextStartUtc,
|
||||
@@ -79,7 +84,8 @@ internal sealed record HistoryContinuationState(
|
||||
long SegmentWidthTicks = 0,
|
||||
long EmptyFromTicks = 0,
|
||||
long EmptyToTicks = 0,
|
||||
HistoryEndBound? PendingEndBound = null)
|
||||
HistoryEndBound? PendingEndBound = null,
|
||||
bool WithinTieCluster = false)
|
||||
{
|
||||
/// <summary>The walk-back hint this cursor carries (#686/#689/#691).</summary>
|
||||
public ReverseWalkHint WalkHint => new(SegmentWidthTicks, EmptyFromTicks, EmptyToTicks);
|
||||
@@ -252,12 +258,11 @@ internal static class HistoryPaging
|
||||
/// <param name="cap">The client's per-page cap (<c>NumValuesPerNode</c>); must be > 0 to make progress.</param>
|
||||
/// <param name="boundaryT">The single SourceTimestamp every tie in the cluster shares.</param>
|
||||
/// <param name="endUtc">The (inclusive) upper bound of the read window; unchanged across pages.
|
||||
/// OpcUaServer-003, accepted by design: the backend contract
|
||||
/// (<c>IHistorianDataSource.ReadRawAsync</c>) treats <c>endUtc</c> as EXCLUSIVE, so the
|
||||
/// <c>next <= endUtc</c> test here is deliberately the conservative side — at worst one extra
|
||||
/// empty resume page (<c>[endUtc, endUtc)</c> → GoodNoData, no continuation point), never a
|
||||
/// dropped or duplicated sample. Tightening to <c><</c> would be correct only if the live
|
||||
/// backend's boundary is confirmed exclusive; it is not worth the opposite off-by-one risk.</param>
|
||||
/// The forward window is end-EXCLUSIVE (Part 11; the <c>IHistorianDataSource.ReadRawAsync</c> contract), so a drained
|
||||
/// cluster continues only while <c>next</c> is strictly before <paramref name="endUtc"/> truncated to
|
||||
/// <paramref name="resolutionTicks"/>. (OpcUaServer-003 had kept <c>next <= endUtc</c> as "the conservative side",
|
||||
/// but it was not: resuming AT EndTime makes the next read a <c>start == end</c> point read, which returns the samples
|
||||
/// at EndTime as data — #692 review F1; on a microsecond backend EndTime is truncated first.)</param>
|
||||
/// <param name="sliceStart">The index into the cluster the emitted slice starts at (clamped to the count).</param>
|
||||
/// <param name="sliceCount">How many ties the emitted slice contains (may be <c>0</c> on a stale-skip self-heal).</param>
|
||||
/// <param name="nextStartUtc">The next page's inclusive start: <paramref name="boundaryT"/> while the
|
||||
@@ -300,7 +305,9 @@ internal static class HistoryPaging
|
||||
else
|
||||
{
|
||||
var next = boundaryT.AddTicks(resolutionTicks);
|
||||
if (next <= endUtc) { nextStartUtc = next; nextSkip = 0; }
|
||||
// The forward window is end-EXCLUSIVE, at the backend's resolution: resuming AT EndTime would turn the next read
|
||||
// into a start == end point read, which returns the samples at EndTime as data (#692 review F1).
|
||||
if (next.Ticks < endUtc.Ticks - endUtc.Ticks % resolutionTicks) { nextStartUtc = next; nextSkip = 0; }
|
||||
else { nextStartUtc = null; nextSkip = 0; }
|
||||
}
|
||||
}
|
||||
|
||||
@@ -533,7 +533,8 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2
|
||||
/// <summary>
|
||||
/// The upper bound on the bounded over-fetch <see cref="ServeRawPaged"/> uses to page WITHIN an
|
||||
/// oversized "tie cluster" — more raw samples sharing one SourceTimestamp than the client's per-page
|
||||
/// cap. When a resume read stalls on such a cluster (the boundary-tie trim empties the page), the
|
||||
/// cap. When a resume read stalls on such a cluster (every row of its <c>N + skip + 1</c> read, past the
|
||||
/// trimmed ties, still sits on the boundary timestamp — #692: only a genuine cluster larger than the page), the
|
||||
/// paging over-fetches up to <c>MaxTieClusterOverfetch + 1</c> ties at that single timestamp (a
|
||||
/// <c>start == end</c> read) and slices through them via <see cref="HistoryPaging.SliceTieCluster"/>.
|
||||
/// A cluster strictly larger than this still fails that node <c>BadTooManyOperations</c> (the absurd-burst
|
||||
@@ -551,6 +552,13 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2
|
||||
/// </summary>
|
||||
public bool HonorReturnBounds { get; set; } = true;
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #694: <c>ServerHistorian:EventReadHorizon</c> — HistoryReadEvents never reads events older than
|
||||
/// <c>now − EventReadHorizon</c> (<see cref="EventHorizon"/>). <see cref="TimeSpan.Zero"/> (the default) = no horizon:
|
||||
/// every read is exact down to its own window (a walked Gateway reverse read proves "nothing older" down to 1970).
|
||||
/// </summary>
|
||||
public TimeSpan EventReadHorizon { get; set; } = TimeSpan.Zero;
|
||||
|
||||
/// <summary>
|
||||
/// Maximum historized nodes served concurrently within one HistoryRead batch (arch-review 03/S3).
|
||||
/// Mirrors <c>ServerHistorianOptions.HistoryReadBatchParallelism</c>; the Host sets it at
|
||||
@@ -3398,6 +3406,36 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2
|
||||
return NodeId.Null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #693: the AtTime value-type hint of a historized variable — its DataType's built-in type as a
|
||||
/// <see cref="TypeCode"/> when the node is a scalar of a type the historian evaluates (Boolean, the integers, Float,
|
||||
/// Double, String, DateTime); <c>null</c> for an array, an unknown node, or any other type. A UNS mirror carries its
|
||||
/// backing raw tag's DataType, so either NodeId gives the same hint.
|
||||
/// </summary>
|
||||
/// <param name="nodeId">The HistoryRead request's node id.</param>
|
||||
/// <returns>The hint, or null.</returns>
|
||||
internal TypeCode? AtTimeValueType(NodeId nodeId)
|
||||
{
|
||||
if (!_variables.TryGetValue(MapKey(nodeId), out var variable) || variable.ValueRank != ValueRanks.Scalar) return null;
|
||||
return TypeInfo.GetBuiltInType(variable.DataType) switch
|
||||
{
|
||||
BuiltInType.Boolean => TypeCode.Boolean,
|
||||
BuiltInType.SByte => TypeCode.SByte,
|
||||
BuiltInType.Byte => TypeCode.Byte,
|
||||
BuiltInType.Int16 => TypeCode.Int16,
|
||||
BuiltInType.UInt16 => TypeCode.UInt16,
|
||||
BuiltInType.Int32 => TypeCode.Int32,
|
||||
BuiltInType.UInt32 => TypeCode.UInt32,
|
||||
BuiltInType.Int64 => TypeCode.Int64,
|
||||
BuiltInType.UInt64 => TypeCode.UInt64,
|
||||
BuiltInType.Float => TypeCode.Single,
|
||||
BuiltInType.Double => TypeCode.Double,
|
||||
BuiltInType.String => TypeCode.String,
|
||||
BuiltInType.DateTime => TypeCode.DateTime,
|
||||
_ => null,
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>Map a Tag.DataType string ("Boolean", "Int32", "Float", "Double", "String",
|
||||
/// "DateTime") to the OPC UA built-in NodeId. Unknown names fall back to BaseDataType
|
||||
/// (matches CreateVariable's default for lazy-created nodes).</summary>
|
||||
@@ -5091,6 +5129,7 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2
|
||||
|
||||
RunBounded(work, HistoryReadBatchParallelism);
|
||||
});
|
||||
NormalizeHistoryValues(nodesToProcess, results);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
@@ -5176,14 +5215,18 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2
|
||||
var work = new List<Func<Task>>(nodesToProcess.Count);
|
||||
foreach (var handle in nodesToProcess)
|
||||
{
|
||||
// #693: the node's scalar value type, for a backend whose stored values carry none (the Gateway).
|
||||
var valueType = AtTimeValueType(handle.NodeId);
|
||||
work.Add(() => ServeNodeAsync(handle, results, errors, (source, tagname, token) => source.ReadAtTimeAsync(
|
||||
tagname,
|
||||
timestamps,
|
||||
valueType,
|
||||
token), ct));
|
||||
}
|
||||
|
||||
RunBounded(work, HistoryReadBatchParallelism);
|
||||
});
|
||||
NormalizeHistoryValues(nodesToProcess, results);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
@@ -5305,6 +5348,19 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2
|
||||
|
||||
// Part 11: StartTime later than EndTime is a reverse read — events newest-first over (end, start].
|
||||
reverse = startUtc > endUtc;
|
||||
|
||||
// #694: the opt-in events horizon raises the window's earlier bound (the cursor then carries it). A window
|
||||
// wholly older than it is empty and is not sent (a forward [start, end) clamped past its end would be refused).
|
||||
if (!EventHorizon.TryClamp(ref startUtc, ref endUtc, reverse, EventReadHorizon, Clock.GetUtcNow().UtcDateTime))
|
||||
{
|
||||
results[handle.Index] = new SdkHistoryReadResult
|
||||
{
|
||||
StatusCode = StatusCodes.GoodNoData,
|
||||
HistoryData = new ExtensionObject(ProjectEvents([], selectClauses)),
|
||||
};
|
||||
errors[handle.Index] = ServiceResult.Good;
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// A reverse read on a backend that cannot read newest-first (the Gateway) is never sent there (#675 refused it
|
||||
@@ -5462,8 +5518,8 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2
|
||||
// One node's backend failure must not throw out of the batch — surface Bad for THIS node
|
||||
// only. Logged via the injected ILogger.
|
||||
_logger.LogError(ex, "OtOpcUaNodeManager: HistoryReadEvents failed for node {NodeId}", handle.NodeId);
|
||||
errors[handle.Index] = StatusCodes.BadHistoryOperationUnsupported;
|
||||
results[handle.Index] = new SdkHistoryReadResult { StatusCode = StatusCodes.BadHistoryOperationUnsupported };
|
||||
errors[handle.Index] = HistoryReadFailureStatus(ex);
|
||||
results[handle.Index] = new SdkHistoryReadResult { StatusCode = HistoryReadFailureStatus(ex) };
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6179,7 +6235,8 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2
|
||||
// Bad status for THIS node only. Logged through the injected ILogger
|
||||
// rather than swallowed silently.
|
||||
_logger.LogError(ex, "OtOpcUaNodeManager: HistoryRead failed for node {NodeId}", handle.NodeId);
|
||||
errors[handle.Index] = StatusCodes.BadHistoryOperationUnsupported;
|
||||
errors[handle.Index] = HistoryReadFailureStatus(ex);
|
||||
results[handle.Index] = new SdkHistoryReadResult { StatusCode = HistoryReadFailureStatus(ex) };
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6190,11 +6247,16 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2
|
||||
/// <list type="bullet">
|
||||
/// <item><b>Fresh read</b> (no inbound continuation point): read the window from
|
||||
/// <c>details.StartTime</c> to <paramref name="endUtc"/> capped at
|
||||
/// <paramref name="numValuesPerNode"/>. If the page comes back FULL (exactly the cap, and the
|
||||
/// cap is > 0), store a resume cursor and emit a continuation point.</item>
|
||||
/// <paramref name="numValuesPerNode"/> <b>+ 1</b> — one lookahead row (#692). If the lookahead row came
|
||||
/// back, return the first N, store a resume cursor and emit a continuation point; otherwise the page is
|
||||
/// the last and carries none (so a total that is an exact multiple of N ends on a full page, not on an
|
||||
/// empty trailing one).</item>
|
||||
/// <item><b>Resume read</b> (inbound continuation point present): take the stored cursor, read
|
||||
/// the next page from the boundary forward, trim already-emitted boundary ties, and emit a
|
||||
/// FRESH continuation point only if THIS page is also full — else null (done).</item>
|
||||
/// <c>N + boundarySkip + 1</c> rows from the boundary (the ties about to be trimmed, the page, and the
|
||||
/// lookahead row), trim the already-emitted boundary ties, and continue exactly as a fresh read — so every
|
||||
/// page carries N values. (Asking for only N used to net N − skip per resumed page, and at N = 1 an EMPTY
|
||||
/// page between every value: the trimmed page looked like a stalled tie cluster and paid a
|
||||
/// <see cref="MaxTieClusterOverfetch"/> + 1 over-fetch to step one tick — Gitea #692.)</item>
|
||||
/// </list>
|
||||
/// The resume cursor is tie-safe (see <see cref="HistoryPaging.ComputeResumeCursor"/> /
|
||||
/// <see cref="HistoryPaging.TrimBoundaryDuplicates"/>): the next page resumes from the boundary
|
||||
@@ -6238,6 +6300,7 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2
|
||||
var boundarySkip = 0;
|
||||
var walkHint = default(ReverseWalkHint); // #686/#689/#691: the walk-back's width + proven-empty stretch, page to page
|
||||
HistoryEndBound? endBound = null; // #691: the end bound still owed (null = bounds not requested)
|
||||
var withinTieCluster = false; // #692: the previous page sliced an oversized tie cluster that still has ties
|
||||
|
||||
string tagname;
|
||||
if (inboundCp is { Length: > 0 })
|
||||
@@ -6260,6 +6323,7 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2
|
||||
numValuesPerNode = state.NumValuesPerNode;
|
||||
walkHint = state.WalkHint;
|
||||
endBound = state.PendingEndBound;
|
||||
withinTieCluster = state.WithinTieCluster;
|
||||
}
|
||||
else
|
||||
{
|
||||
@@ -6314,18 +6378,29 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2
|
||||
}
|
||||
|
||||
// #691 review: whether a bounded read whose start bound was NOT found has any data or a found end bound — only
|
||||
// asked when a one-value page would otherwise be the start-bound placeholder alone.
|
||||
async Task<bool> BoundedReadHasAnythingAsync()
|
||||
// asked when a one-value page would otherwise be the start-bound placeholder alone. #692: what it learns rides the
|
||||
// cursor — the walk hint its walk-back computed, and, when there is no data, the end bound it found (so the next
|
||||
// page is that bound with no backend read at all).
|
||||
async Task<(bool Any, HistoryEndBound Pending)> BoundedReadHasAnythingAsync()
|
||||
{
|
||||
var src = HistorianDataSource;
|
||||
IReadOnlyList<DataValueSnapshot> first = startUtc > endUtc && !src.SupportsReverseRaw
|
||||
? (await ReverseRawReader.ReadAsync(src, tagname, startUtc, endUtc, 1, MaxTieClusterOverfetch, nowUtc, 0, ct)
|
||||
.ConfigureAwait(false)).Samples
|
||||
: (await src.ReadRawAsync(tagname, startUtc, endUtc, 1, ct).ConfigureAwait(false)).Samples;
|
||||
if (first.Any(x => HistoryBounds.InDataWindow(x, endBound!, resolution))) return true;
|
||||
IReadOnlyList<DataValueSnapshot> first;
|
||||
if (startUtc > endUtc && !src.SupportsReverseRaw)
|
||||
{
|
||||
var walked = await ReverseRawReader.ReadAsync(
|
||||
src, tagname, startUtc, endUtc, 1, MaxTieClusterOverfetch, nowUtc, walkHint, ct)
|
||||
.ConfigureAwait(false);
|
||||
walkHint = walked.Hint;
|
||||
first = walked.Samples;
|
||||
}
|
||||
else first = (await src.ReadRawAsync(tagname, startUtc, endUtc, 1, ct).ConfigureAwait(false)).Samples;
|
||||
|
||||
if (first.Any(x => HistoryBounds.InDataWindow(x, endBound!, resolution))) return (true, endBound!);
|
||||
var end = await HistoryBounds.EndBoundAsync(
|
||||
src, tagname, endBound!, startUtc, MaxTieClusterOverfetch, nowUtc, ct).ConfigureAwait(false);
|
||||
return !HistoryBounds.IsPlaceholder(end);
|
||||
return HistoryBounds.IsPlaceholder(end)
|
||||
? (false, endBound!)
|
||||
: (true, endBound! with { DataExhausted = true, Resolved = end });
|
||||
}
|
||||
|
||||
// #691: a resumed bounded read whose data is drained (the end bound did not fit on the previous page, or the
|
||||
@@ -6354,14 +6429,18 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2
|
||||
startBound = await HistoryBounds.StartBoundAsync(
|
||||
HistorianDataSource, tagname, startUtc, endBound.Reverse, MaxTieClusterOverfetch, nowUtc, ct)
|
||||
.ConfigureAwait(false);
|
||||
if (startBound is not null && numValuesPerNode == 1
|
||||
&& (!HistoryBounds.IsPlaceholder(startBound) || await BoundedReadHasAnythingAsync().ConfigureAwait(false)))
|
||||
var (anything, pending) = startBound is not null && numValuesPerNode == 1
|
||||
? HistoryBounds.IsPlaceholder(startBound)
|
||||
? await BoundedReadHasAnythingAsync().ConfigureAwait(false)
|
||||
: (true, endBound)
|
||||
: (false, endBound);
|
||||
if (startBound is not null && numValuesPerNode == 1 && anything)
|
||||
{
|
||||
// The page is the start bound alone; the data starts at StartTime on the next page.
|
||||
var boundOnlyCp = _historyContinuationStore.Save(session, new HistoryContinuationState(
|
||||
tagname, startUtc, endUtc, 0, numValuesPerNode, SegmentWidthTicks: walkHint.WidthTicks,
|
||||
EmptyFromTicks: walkHint.EmptyFromTicks, EmptyToTicks: walkHint.EmptyToTicks,
|
||||
PendingEndBound: HistoryBounds.Carry(endBound, [startBound])));
|
||||
PendingEndBound: HistoryBounds.Carry(pending, [startBound])));
|
||||
results[handle.Index] = new SdkHistoryReadResult
|
||||
{
|
||||
StatusCode = StatusCodes.Good,
|
||||
@@ -6388,60 +6467,97 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2
|
||||
if (startBound is not null && numValuesPerNode > 0) readCap = numValuesPerNode - 1;
|
||||
}
|
||||
|
||||
// #692: the page's read asks for the boundary ties it is about to trim (a resumed page re-reads them, because the
|
||||
// cursor resumes AT the boundary timestamp, inclusive) plus ONE lookahead row past the page. Without them a resumed
|
||||
// page netted N − skip values, and at N = 1 nothing at all: the lone boundary sample came back, was trimmed, the
|
||||
// "full" empty page was taken for a stalled tie cluster, and a MaxTieClusterOverfetch + 1 over-fetch stepped one
|
||||
// tick to return an EMPTY page with a continuation point — between every value. The lookahead row is what says
|
||||
// "more data follows": a page whose read found no row past it is the last, so the read no longer ends with an
|
||||
// empty trailing page either. (RawPageLimit.Plan computes the request: page + boundary ties + 1.)
|
||||
var resumed = inboundCp is { Length: > 0 };
|
||||
var boundaryTies = resumed ? Math.Max(0, boundarySkip) : 0;
|
||||
var want = readCap; // N, less the start bound on a bounded first page; 0 = all values
|
||||
|
||||
// HistoryRead is NOT under the node-manager Lock — awaiting the async source is safe.
|
||||
// A reverse window (explicit, resumed, or the end-only read above) goes to the backend only when it declares it
|
||||
// can read newest-first; otherwise it is walked back from bounded forward reads (#686 — the Gateway rejects
|
||||
// start > end, which failed the node and counted as a backend fault).
|
||||
// A cursor left inside an oversized tie cluster (#692) skips this read: every row it could return is a tie.
|
||||
// #694: a backend with a row ceiling (MaxRawValuesPerRead = L > 0, a strict gateway) sizes the page to it — the
|
||||
// read asks for page + boundary ties + lookahead ≤ L, and a full page carries a continuation point as always
|
||||
// (uncapped reads included). No room left after the boundary ties ⇒ the cursor sits in a tie cluster as large as
|
||||
// a read: straight to the over-fetch. See RawPageLimit.
|
||||
var source = HistorianDataSource;
|
||||
HistorianRead sourceResult;
|
||||
if (startUtc > endUtc && !source.SupportsReverseRaw)
|
||||
IReadOnlyList<DataValueSnapshot> samples = [];
|
||||
var backendFull = false;
|
||||
int pageLimit;
|
||||
if (!withinTieCluster)
|
||||
{
|
||||
var page = await ReverseRawReader.ReadAsync(
|
||||
source, tagname, startUtc, endUtc, readCap, MaxTieClusterOverfetch,
|
||||
nowUtc, walkHint, ct)
|
||||
.ConfigureAwait(false);
|
||||
walkHint = page.Hint;
|
||||
sourceResult = new HistorianRead(page.Samples, ContinuationPoint: null);
|
||||
var read = await RawPageLimit.ReadAsync(source, want, boundaryTies, async plan =>
|
||||
{
|
||||
if (startUtc > endUtc && !source.SupportsReverseRaw)
|
||||
{
|
||||
var page = await ReverseRawReader.ReadAsync(
|
||||
source, tagname, startUtc, endUtc, plan.RequestCap, MaxTieClusterOverfetch,
|
||||
nowUtc, walkHint, ct, retryOnLearnedLimit: false)
|
||||
.ConfigureAwait(false);
|
||||
walkHint = page.Hint;
|
||||
return new HistorianRead(page.Samples, ContinuationPoint: null);
|
||||
}
|
||||
|
||||
return await source.ReadRawAsync(tagname, startUtc, endUtc, plan.RequestCap, ct).ConfigureAwait(false);
|
||||
}).ConfigureAwait(false);
|
||||
pageLimit = read.Plan.Limit;
|
||||
|
||||
if (read.Plan.NoRoom)
|
||||
{
|
||||
if (!resumed)
|
||||
throw new HistoryReadLimitExceededException(pageLimit,
|
||||
$"The backend's row ceiling ({pageLimit} values per read) leaves no room for a raw page of '{tagname}'.");
|
||||
withinTieCluster = true; // every row a read could return is a boundary tie already emitted
|
||||
}
|
||||
else
|
||||
{
|
||||
readCap = read.Plan.PageCap;
|
||||
var sourceResult = read.Page!;
|
||||
backendFull = HistoryPaging.IsFullPage(sourceResult.Samples.Count, read.Plan.RequestCap);
|
||||
|
||||
// On a resume read, drop the boundary ties already returned on the prior page.
|
||||
samples = resumed
|
||||
? HistoryPaging.TrimBoundaryDuplicates(sourceResult.Samples, startUtc, boundarySkip)
|
||||
: sourceResult.Samples;
|
||||
|
||||
// #691: a bounded read's data window excludes the end time (a sample there is the end bound). A backend whose
|
||||
// forward end is inclusive (the gateway's own) returns it as data; dropping it also means the data is drained.
|
||||
if (endBound is not null)
|
||||
{
|
||||
var inside = samples.Where(x => HistoryBounds.InDataWindow(x, endBound, resolution)).ToList();
|
||||
if (inside.Count != samples.Count)
|
||||
{
|
||||
samples = inside;
|
||||
backendFull = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
sourceResult = await source
|
||||
.ReadRawAsync(tagname, startUtc, endUtc, readCap, ct)
|
||||
.ConfigureAwait(false);
|
||||
pageLimit = source.MaxRawValuesPerRead;
|
||||
}
|
||||
|
||||
var backendFull = HistoryPaging.IsFullPage(sourceResult.Samples.Count, readCap);
|
||||
|
||||
// On a resume read, drop the boundary ties already returned on the prior page.
|
||||
var samples = inboundCp is { Length: > 0 }
|
||||
? HistoryPaging.TrimBoundaryDuplicates(sourceResult.Samples, startUtc, boundarySkip)
|
||||
: sourceResult.Samples;
|
||||
|
||||
// #691: a bounded read's data window excludes the end time (a sample there is the end bound). A backend whose
|
||||
// forward end is inclusive (the gateway's own) returns it as data; dropping it also means the data is drained.
|
||||
if (endBound is not null)
|
||||
{
|
||||
var inside = samples.Where(x => HistoryBounds.InDataWindow(x, endBound, resolution)).ToList();
|
||||
if (inside.Count != samples.Count)
|
||||
{
|
||||
samples = inside;
|
||||
backendFull = false;
|
||||
}
|
||||
}
|
||||
|
||||
// Oversized tie cluster: a resume read whose FULL backend page does not advance past the resume
|
||||
// boundary timestamp `startUtc`. That happens when more samples share `startUtc` than the page
|
||||
// cap — a tie cluster larger than NumValuesPerNode. The fixed-(start,end,cap) backend can only
|
||||
// ever return the first `cap` of those ties, so the (timestamp, skip) cursor alone can never page
|
||||
// past (or even reliably within) the cluster: either the boundary-tie trim empties the page, or
|
||||
// it re-emits the same head ties forever. Detect BOTH stall shapes — an empty trimmed page, or a
|
||||
// non-empty page whose LAST sample is still at `startUtc` (no forward progress) — and, rather
|
||||
// than fail the read, over-fetch the WHOLE cluster with an explicit, BOUNDED cap (a start==end
|
||||
// read at the boundary timestamp) and page WITHIN the timestamp via SliceTieCluster.
|
||||
var stalledOnTieCluster = inboundCp is { Length: > 0 }
|
||||
&& backendFull
|
||||
&& (samples.Count == 0
|
||||
|| (samples[^1].SourceTimestampUtc ?? DateTime.MinValue) == startUtc);
|
||||
// Oversized tie cluster: a resume read whose FULL backend read does not advance past the resume boundary
|
||||
// timestamp `startUtc` — every row it returned past the trimmed ties, the lookahead row included, still sits on
|
||||
// `startUtc`. That happens only when more samples share `startUtc` than the page plus the already-emitted ties
|
||||
// plus one: a genuine tie cluster larger than the page. The fixed-(start,end,cap) backend can only ever return
|
||||
// the first `cap` of those ties, so growing the read page by page would re-read the cluster quadratically.
|
||||
// Instead over-fetch the WHOLE cluster with an explicit, BOUNDED cap (a start==end read at the boundary
|
||||
// timestamp) and page WITHIN the timestamp via SliceTieCluster. (#692: before the read asked for the trimmed ties
|
||||
// and a lookahead row, a resumed N = 1 page tripped this on every single sample.)
|
||||
var stalledOnTieCluster = resumed
|
||||
&& (withinTieCluster
|
||||
|| (backendFull
|
||||
&& (samples.Count == 0
|
||||
|| (samples[^1].SourceTimestampUtc ?? DateTime.MinValue) == startUtc)));
|
||||
if (stalledOnTieCluster)
|
||||
{
|
||||
// The over-fetch cap MUST be explicit and non-zero: a cap of 0 falls back to the backend's
|
||||
@@ -6449,7 +6565,12 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2
|
||||
// lets us DETECT a cluster strictly larger than the bound (the absurd-burst backstop below).
|
||||
// Compute in uint so +1 can never wrap: Math.Max(1,…) also guards against a zero/negative
|
||||
// config value slipping through (Validate() warns but does not block startup).
|
||||
var overfetchCap = (uint)Math.Max(1, MaxTieClusterOverfetch) + 1u;
|
||||
// #694: on a backend with a row ceiling the bound is also at most L − 1, so the over-fetch (bound + 1 rows)
|
||||
// fits one read and an oversized cluster is refused here rather than by a gateway refusal.
|
||||
var overfetchBound = pageLimit > 0
|
||||
? Math.Min(Math.Max(1, MaxTieClusterOverfetch), Math.Max(1, pageLimit - 1))
|
||||
: Math.Max(1, MaxTieClusterOverfetch);
|
||||
var overfetchCap = (uint)overfetchBound + 1u;
|
||||
IReadOnlyList<DataValueSnapshot> cluster = (await HistorianDataSource
|
||||
.ReadRawAsync(tagname, startUtc, startUtc, overfetchCap, ct)
|
||||
.ConfigureAwait(false)).Samples;
|
||||
@@ -6463,12 +6584,13 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2
|
||||
// Absurd burst: more ties than we're willing to buffer in memory. Preserve today's loud-fail
|
||||
// for that node rather than over-fetch an unbounded cluster; the operator's remedy is a
|
||||
// larger ServerHistorian:MaxTieClusterOverfetch (or NumValuesPerNode).
|
||||
if (cluster.Count > MaxTieClusterOverfetch)
|
||||
if (cluster.Count > overfetchBound)
|
||||
{
|
||||
_logger.LogError(
|
||||
"OtOpcUaNodeManager: HistoryReadRaw tie cluster at {StartUtc:O} for tag '{Tagname}' has {Count} samples, " +
|
||||
"exceeding MaxTieClusterOverfetch={Max}; cannot page within it. Increase MaxTieClusterOverfetch.",
|
||||
startUtc, tagname, cluster.Count, MaxTieClusterOverfetch);
|
||||
"exceeding {Max} (MaxTieClusterOverfetch, or the backend's row ceiling less one); cannot page within it. " +
|
||||
"Increase MaxTieClusterOverfetch (or the HistorianGateway MaxValuesCeiling).",
|
||||
startUtc, tagname, cluster.Count, overfetchBound);
|
||||
errors[handle.Index] = StatusCodes.BadTooManyOperations;
|
||||
results[handle.Index] = new SdkHistoryReadResult { StatusCode = StatusCodes.BadTooManyOperations };
|
||||
return;
|
||||
@@ -6476,8 +6598,15 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2
|
||||
|
||||
// Direction comes from the window itself (Part 11: start > end is a reverse read). A resumed reverse
|
||||
// read keeps start > end, because a reverse resume point is always strictly after endUtc.
|
||||
// #694: an uncapped read pages only at the backend's limit, so it slices at that page size — never 0 (a zero
|
||||
// slice hands back the same cursor forever; a limit that reverted to 0 under an uncapped cursor in a cluster
|
||||
// falls back to the cluster's own size).
|
||||
var sliceSize = numValuesPerNode > 0 ? numValuesPerNode
|
||||
: readCap > 0 ? readCap
|
||||
: pageLimit > 0 ? (uint)pageLimit
|
||||
: (uint)Math.Max(1, cluster.Count);
|
||||
HistoryPaging.SliceTieCluster(
|
||||
cluster.Count, boundarySkip, numValuesPerNode, startUtc, endUtc,
|
||||
cluster.Count, boundarySkip, sliceSize, startUtc, endUtc,
|
||||
out var sliceStart, out var sliceCount, out var nextStartUtc, out var nextSkip,
|
||||
reverse: reverse, resolutionTicks: Math.Max(1L, HistorianDataSource.TimestampResolutionTicks));
|
||||
|
||||
@@ -6493,7 +6622,8 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2
|
||||
var clusterState = new HistoryContinuationState(
|
||||
tagname, resumeAt, endUtc, nextSkip, numValuesPerNode, SegmentWidthTicks: walkHint.WidthTicks,
|
||||
EmptyFromTicks: walkHint.EmptyFromTicks, EmptyToTicks: walkHint.EmptyToTicks,
|
||||
PendingEndBound: HistoryBounds.Carry(endBound, slice));
|
||||
PendingEndBound: HistoryBounds.Carry(endBound, slice),
|
||||
WithinTieCluster: resumeAt == startUtc);
|
||||
clusterCp = _historyContinuationStore.Save(session, clusterState);
|
||||
}
|
||||
else if (endBound is not null)
|
||||
@@ -6512,6 +6642,24 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2
|
||||
return;
|
||||
}
|
||||
|
||||
// #692: more than the page came back (the lookahead row, after trimming) ⇒ there is more data: return the page and
|
||||
// resume after it. Otherwise the data is drained and this is the last data page.
|
||||
var more = readCap > 0 && samples.Count > readCap;
|
||||
if (more) samples = samples.Take((int)readCap).ToList();
|
||||
|
||||
HistoryContinuationState? nextState = null;
|
||||
if (more)
|
||||
{
|
||||
// A page that never left the boundary timestamp (ties continuing a resumed page) adds to the ties already
|
||||
// emitted there, or the next page would re-emit the earlier ones.
|
||||
HistoryPaging.ComputeResumeCursor(samples, out var nextStart, out var skip);
|
||||
if (resumed && nextStart == startUtc) skip += boundarySkip;
|
||||
nextState = new HistoryContinuationState(
|
||||
tagname, nextStart, endUtc, skip, numValuesPerNode, SegmentWidthTicks: walkHint.WidthTicks,
|
||||
EmptyFromTicks: walkHint.EmptyFromTicks, EmptyToTicks: walkHint.EmptyToTicks,
|
||||
PendingEndBound: HistoryBounds.Carry(endBound, samples));
|
||||
}
|
||||
|
||||
// #691: the start bound (first page only) leads the data.
|
||||
if (startBound is not null)
|
||||
{
|
||||
@@ -6520,18 +6668,11 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2
|
||||
samples = led;
|
||||
}
|
||||
|
||||
// The "full page" test is against the RAW backend count (before trimming): the backend honoured
|
||||
// the cap, so a full backend page ⇒ there may be more even if we trimmed some boundary ties.
|
||||
var historyData = ToHistoryDataFromSamples(samples);
|
||||
|
||||
byte[]? outboundCp = null;
|
||||
if (backendFull && samples.Count > 0)
|
||||
if (nextState is not null)
|
||||
{
|
||||
HistoryPaging.ComputeResumeCursor(samples, out var nextStart, out var skip);
|
||||
var nextState = new HistoryContinuationState(
|
||||
tagname, nextStart, endUtc, skip, numValuesPerNode, SegmentWidthTicks: walkHint.WidthTicks,
|
||||
EmptyFromTicks: walkHint.EmptyFromTicks, EmptyToTicks: walkHint.EmptyToTicks,
|
||||
PendingEndBound: HistoryBounds.Carry(endBound, samples));
|
||||
// Save may return null (no session on this request) ⇒ degrade to single-shot for this node.
|
||||
// Built AFTER historyData so a failure projecting samples can never orphan a stored cursor.
|
||||
outboundCp = _historyContinuationStore.Save(session, nextState);
|
||||
@@ -6584,7 +6725,8 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2
|
||||
{
|
||||
// One node's backend failure must not throw out of the batch — Bad for THIS node only.
|
||||
_logger.LogError(ex, "OtOpcUaNodeManager: HistoryReadRaw (paged) failed for node {NodeId}", handle.NodeId);
|
||||
errors[handle.Index] = StatusCodes.BadHistoryOperationUnsupported;
|
||||
errors[handle.Index] = HistoryReadFailureStatus(ex);
|
||||
results[handle.Index] = new SdkHistoryReadResult { StatusCode = HistoryReadFailureStatus(ex) };
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6691,8 +6833,8 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "OtOpcUaNodeManager: HistoryRead (modified) failed for node {NodeId}", handle.NodeId);
|
||||
errors[handle.Index] = StatusCodes.BadHistoryOperationUnsupported;
|
||||
results[handle.Index] = new SdkHistoryReadResult { StatusCode = StatusCodes.BadHistoryOperationUnsupported };
|
||||
errors[handle.Index] = HistoryReadFailureStatus(ex);
|
||||
results[handle.Index] = new SdkHistoryReadResult { StatusCode = HistoryReadFailureStatus(ex) };
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6741,6 +6883,49 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Transport failures are retryable; unsupported operations retain their distinct status (#697).</summary>
|
||||
private static uint HistoryReadFailureStatus(Exception error)
|
||||
{
|
||||
for (Exception? ex = error; ex is not null; ex = ex.InnerException)
|
||||
if (ex is System.IO.IOException or System.Net.Sockets.SocketException or System.Net.Http.HttpRequestException or TimeoutException)
|
||||
return StatusCodes.BadNoCommunication;
|
||||
return StatusCodes.BadHistoryOperationUnsupported;
|
||||
}
|
||||
|
||||
/// <summary>Normalize stored and AtTime history to each scalar node's declared DataType (#698).</summary>
|
||||
private void NormalizeHistoryValues(List<NodeHandle> handles, IList<SdkHistoryReadResult> results)
|
||||
{
|
||||
foreach (var handle in handles)
|
||||
{
|
||||
if (AtTimeValueType(handle.NodeId) is not { } type || results[handle.Index]?.HistoryData is not { } body) continue;
|
||||
var values = ExtensionObject.ToEncodeable(body) switch
|
||||
{
|
||||
HistoryData data => data.DataValues,
|
||||
_ => null,
|
||||
};
|
||||
if (values is null) continue;
|
||||
foreach (var value in values)
|
||||
{
|
||||
if (value.Value is null) continue;
|
||||
try
|
||||
{
|
||||
// A gateway number cannot reconstruct a DateTime or a String which was never stored.
|
||||
if ((type == TypeCode.DateTime && value.Value is not DateTime)
|
||||
|| (type == TypeCode.String && value.Value is not string)) throw new InvalidCastException();
|
||||
var converted = Convert.ChangeType(value.Value, type, System.Globalization.CultureInfo.InvariantCulture);
|
||||
if (converted is float f && float.IsInfinity(f) && value.Value is double d && double.IsFinite(d))
|
||||
throw new OverflowException();
|
||||
value.WrappedValue = new Variant(converted);
|
||||
}
|
||||
catch (Exception ex) when (ex is InvalidCastException or OverflowException or FormatException)
|
||||
{
|
||||
value.WrappedValue = Variant.Null;
|
||||
value.StatusCode = StatusCodes.BadTypeMismatch;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Project a plain sample list into an SDK <see cref="HistoryData"/> (the paged Raw path
|
||||
/// works on a trimmed <see cref="IReadOnlyList{T}"/> rather than a whole <see cref="HistorianRead"/>).</summary>
|
||||
/// <param name="samples">The samples to project (already trimmed of boundary duplicates).</param>
|
||||
|
||||
@@ -0,0 +1,97 @@
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.OpcUaServer;
|
||||
|
||||
/// <summary>How one HistoryRead-Raw page reads (<see cref="RawPageLimit.Plan"/>).</summary>
|
||||
/// <param name="Limit">The backend's <see cref="IHistorianDataSource.MaxRawValuesPerRead"/> the plan was made under (0 = none).</param>
|
||||
/// <param name="PageCap">The page: at most this many values are returned (0 = all values — only without a limit).</param>
|
||||
/// <param name="RequestCap">What the read asks for: page + the boundary ties it will trim + one lookahead row (0 = all).</param>
|
||||
/// <param name="NoRoom">The boundary ties alone fill a read — nothing can be read past them; serve the tie cluster.</param>
|
||||
internal readonly record struct RawPagePlan(int Limit, uint PageCap, uint RequestCap, bool NoRoom);
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #694: sizes a HistoryRead-Raw page to what the backend can return in one read
|
||||
/// (<see cref="IHistorianDataSource.MaxRawValuesPerRead"/> = L — a strict HistorianGateway's learned row ceiling, less
|
||||
/// its boundary-row allowance). With a limit the read — page P + boundary ties k + the #692 lookahead row — is at most L,
|
||||
/// so P = min(want, L − k − 1) (all values: P = L − k − 1); a full page carries a continuation point like any other, as
|
||||
/// Part 11 allows fewer values than <c>NumValuesPerNode</c> with one. Before #694 a cap above a strict gateway's ceiling
|
||||
/// (or an uncapped read of a window holding more) failed <c>BadTooManyOperations</c>.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>Never a silent truncation.</b> The backend still answers a read in full or refuses it; only the server's page is
|
||||
/// smaller, and a page is judged full against what was asked for. A read the backend cannot serve at all (a tie
|
||||
/// cluster larger than one read) stays <c>BadTooManyOperations</c>.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>One re-read, planned again.</b> A refused read (<see cref="HistoryReadLimitExceededException"/>) is re-planned and
|
||||
/// re-issued ONCE when (a) the backend learned a different limit while it ran — the first read after start-up teaches
|
||||
/// the ceiling — or (b) it was refused because rows before the window crowded the budget
|
||||
/// (<see cref="HistoryReadLimitExceededException.PreWindowRows"/> m): the page is then shrunk by m so that the
|
||||
/// backend's look-back fits beside it. Anything else is the answer.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
internal static class RawPageLimit
|
||||
{
|
||||
/// <summary>Plans one page.</summary>
|
||||
/// <param name="want">The page the client asked for (N, less a start bound; 0 = all values).</param>
|
||||
/// <param name="boundaryTies">Ties at the resume timestamp already returned, which the read re-reads and trims (k).</param>
|
||||
/// <param name="limit">The backend's limit L (<= 0 = none).</param>
|
||||
/// <param name="sizeLimit">The limit to size the page by (L, or L − m after a crowded refusal); defaults to <paramref name="limit"/>.</param>
|
||||
/// <returns>The plan.</returns>
|
||||
public static RawPagePlan Plan(uint want, int boundaryTies, int limit, int? sizeLimit = null)
|
||||
{
|
||||
var k = Math.Max(0, boundaryTies);
|
||||
if (limit <= 0)
|
||||
return new RawPagePlan(0, want, want == 0 ? 0u : (uint)Math.Min(uint.MaxValue, (long)want + k + 1L), false);
|
||||
|
||||
var room = (long)(sizeLimit ?? limit) - k - 1;
|
||||
if (room < 1) return new RawPagePlan(limit, 0, 0, true);
|
||||
var page = want == 0 ? room : Math.Min(want, room);
|
||||
return new RawPagePlan(limit, (uint)page, (uint)(page + k + 1), false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reads one page under the backend's current limit, re-planning and re-reading ONCE as described in the remarks.
|
||||
/// A <see cref="RawPagePlan.NoRoom"/> plan is returned without a read.
|
||||
/// </summary>
|
||||
/// <typeparam name="T">The page.</typeparam>
|
||||
/// <param name="source">The backend.</param>
|
||||
/// <param name="want">The page the client asked for (0 = all values).</param>
|
||||
/// <param name="boundaryTies">The boundary ties the read re-reads and trims.</param>
|
||||
/// <param name="read">One read with the given plan (it asks for <see cref="RawPagePlan.RequestCap"/>).</param>
|
||||
/// <returns>The page (null when <see cref="RawPagePlan.NoRoom"/>) and the plan it was read with.</returns>
|
||||
public static async Task<(T? Page, RawPagePlan Plan)> ReadAsync<T>(
|
||||
IHistorianDataSource source, uint want, int boundaryTies, Func<RawPagePlan, Task<T>> read)
|
||||
{
|
||||
var before = source.MaxRawValuesPerRead;
|
||||
var plan = Plan(want, boundaryTies, before);
|
||||
if (plan.NoRoom) return (default, plan);
|
||||
|
||||
HistoryReadLimitExceededException refused;
|
||||
try
|
||||
{
|
||||
return (await read(plan).ConfigureAwait(false), plan);
|
||||
}
|
||||
catch (HistoryReadLimitExceededException ex) when (source.MaxRawValuesPerRead is var now and > 0
|
||||
&& (now != before || ex.PreWindowRows > 0))
|
||||
{
|
||||
refused = ex; // re-read outside the catch: the second refusal is the answer
|
||||
}
|
||||
|
||||
var limit = source.MaxRawValuesPerRead;
|
||||
var shrink = Math.Max(0, refused.PreWindowRows);
|
||||
plan = Plan(want, boundaryTies, limit, (int)Math.Max(0L, (long)limit - shrink));
|
||||
// No room left once the crowding rows are allowed for: those rows are not (necessarily) the boundary ties, so this is
|
||||
// NOT "the cursor sits in a tie cluster" — the refusal is the answer (stepping the cursor through it a tick per page
|
||||
// would never end).
|
||||
// (A limit that changed under an un-crowded read leaves the boundary ties alone filling a read: that IS the tie
|
||||
// cluster at the cursor, served by the over-fetch.)
|
||||
if (plan.NoRoom)
|
||||
{
|
||||
if (shrink > 0) throw refused;
|
||||
return (default, plan);
|
||||
}
|
||||
return (await read(plan).ConfigureAwait(false), plan);
|
||||
}
|
||||
}
|
||||
@@ -58,25 +58,78 @@ internal static class ReverseRawReader
|
||||
/// <param name="ct">Cancellation.</param>
|
||||
/// <returns>The page, newest first, and the hint for the next page.</returns>
|
||||
/// <exception cref="HistoryReadLimitExceededException">One timestamp holds more than the tie budget.</exception>
|
||||
/// <remarks>
|
||||
/// Gitea #694: every segment read fits the backend's <see cref="IHistorianDataSource.MaxRawValuesPerRead"/> (a strict
|
||||
/// gateway's learned ceiling) — the walk's tie budget is bounded by it so its segments are sized for it, and a
|
||||
/// crowded re-read asking more is cut to it (and counts as truncated, never as complete). A walk that TEACHES the
|
||||
/// backend its limit (the first read after start-up) and is refused for it is re-run once with segments sized for the
|
||||
/// new limit — unless <paramref name="retryOnLearnedLimit"/> is false: the HistoryRead page path passes false, so the
|
||||
/// refusal reaches <see cref="RawPageLimit"/>, which shrinks the PAGE to the limit as well (re-running the walk for a
|
||||
/// page of N far above the limit cost thousands of segment reads). The bound lookups (a page of one) keep the re-run.
|
||||
/// The page size itself is never cut here: a short answer would read as the last page. A segment refused because rows
|
||||
/// just before it crowded the backend's budget (<see cref="HistoryReadLimitExceededException.PreWindowRows"/>) is
|
||||
/// re-read once that many rows smaller and counts as truncated when full. An uncapped walk (one forward read of the
|
||||
/// whole window) is never cut: it is answered in full or refused.
|
||||
/// </remarks>
|
||||
/// <param name="retryOnLearnedLimit">Re-run the walk once when it taught the backend its limit (see the remarks).</param>
|
||||
public static async Task<ReverseRawPage> ReadAsync(
|
||||
IHistorianDataSource source, string tagname, DateTime hi, DateTime lo, uint maxValues, int maxTieCluster,
|
||||
DateTime nowUtc, ReverseWalkHint hint, CancellationToken ct)
|
||||
DateTime nowUtc, ReverseWalkHint hint, CancellationToken ct, bool retryOnLearnedLimit = true)
|
||||
{
|
||||
var budget = Math.Max(1, maxTieCluster);
|
||||
var (rows, next) = await ReverseWalk.WalkAsync(
|
||||
async (start, end, cap, token) =>
|
||||
{
|
||||
var samples = (await source.ReadRawAsync(tagname, start, end, (uint)cap, token).ConfigureAwait(false)).Samples;
|
||||
return new ForwardSegmentRead<DataValueSnapshot>(samples, cap > 0 && samples.Count >= cap);
|
||||
},
|
||||
Ticks, tieOrder: null, hi, lo, maxValues, budget, source.TimestampResolutionTicks, nowUtc, hint,
|
||||
at => new HistoryReadLimitExceededException(budget,
|
||||
$"A reverse raw read of '{tagname}' at {at:O} found more than {budget} samples within one millisecond; the " +
|
||||
"backend reads forward only, so the latest of them cannot be selected. Increase " +
|
||||
"ServerHistorian:MaxTieClusterOverfetch."),
|
||||
ct)
|
||||
.ConfigureAwait(false);
|
||||
return new ReverseRawPage(rows, next.WidthTicks) { Hint = next };
|
||||
var before = source.MaxRawValuesPerRead;
|
||||
try
|
||||
{
|
||||
return await WalkAsync(before).ConfigureAwait(false);
|
||||
}
|
||||
catch (HistoryReadLimitExceededException) when (retryOnLearnedLimit && maxValues > 0
|
||||
&& source.MaxRawValuesPerRead is var now and > 0 && now != before)
|
||||
{
|
||||
// fall through (outside the catch): the walk taught the backend its limit — re-run it sized for that limit once.
|
||||
// Not an uncapped walk: it is one read of the whole window, which the limit does not change.
|
||||
}
|
||||
|
||||
return await WalkAsync(source.MaxRawValuesPerRead).ConfigureAwait(false);
|
||||
|
||||
async Task<ReverseRawPage> WalkAsync(int limit)
|
||||
{
|
||||
// A segment asks for at most its budget + 2 rows (ReverseWalk), so the budget is bounded 2 under the limit.
|
||||
var budget = Math.Max(1, maxTieCluster);
|
||||
if (limit > 0) budget = Math.Min(budget, Math.Max(1, limit - 2));
|
||||
var (rows, next) = await ReverseWalk.WalkAsync(
|
||||
async (start, end, cap, token) =>
|
||||
{
|
||||
var asked = limit > 0 && cap > limit ? limit : cap; // cap 0 (the whole window) is never cut
|
||||
IReadOnlyList<DataValueSnapshot> samples;
|
||||
for (var attempt = 0; ; attempt++)
|
||||
{
|
||||
try
|
||||
{
|
||||
samples = (await source.ReadRawAsync(tagname, start, end, (uint)asked, token).ConfigureAwait(false)).Samples;
|
||||
break;
|
||||
}
|
||||
catch (HistoryReadLimitExceededException ex) when (asked > 0 && ex.PreWindowRows > 0 && attempt < 2)
|
||||
{
|
||||
// #695: the discovery request may be smaller than the pre-window cluster. Size against
|
||||
// the ceiling, not that request; a second pass can discover more crowding rows.
|
||||
var ceilingRoom = source.MaxRawValuesPerRead is var learned and > 0 ? learned : asked;
|
||||
var smaller = Math.Min(asked - 1, ceilingRoom - ex.PreWindowRows - 1);
|
||||
if (smaller < 1) throw;
|
||||
asked = smaller;
|
||||
}
|
||||
}
|
||||
return new ForwardSegmentRead<DataValueSnapshot>(samples, asked > 0 && samples.Count >= asked);
|
||||
},
|
||||
Ticks, tieOrder: null, hi, lo, maxValues, budget, source.TimestampResolutionTicks, nowUtc, hint,
|
||||
at => new HistoryReadLimitExceededException(budget,
|
||||
$"A reverse raw read of '{tagname}' at {at:O} found more than {budget} samples within one millisecond; the " +
|
||||
"backend reads forward only, so the latest of them cannot be selected. Increase " +
|
||||
(budget < Math.Max(1, maxTieCluster)
|
||||
? "the backend's row ceiling (HistorianGateway Historian:ReadLimits:MaxValuesCeiling)."
|
||||
: "ServerHistorian:MaxTieClusterOverfetch.")),
|
||||
ct, useDiscoveryBudgetForGaps: limit > 0)
|
||||
.ConfigureAwait(false);
|
||||
return new ReverseRawPage(rows, next.WidthTicks) { Hint = next };
|
||||
}
|
||||
}
|
||||
|
||||
private static long Ticks(DataValueSnapshot s) => (s.SourceTimestampUtc ?? DateTime.MinValue).Ticks;
|
||||
|
||||
@@ -123,13 +123,14 @@ internal static class ReverseWalk
|
||||
/// <param name="hint">What the previous page of the same read carried (default = none).</param>
|
||||
/// <param name="tooManyAtOneTimestamp">Builds the exception for a timestamp holding more than M rows.</param>
|
||||
/// <param name="ct">Cancellation.</param>
|
||||
/// <param name="useDiscoveryBudgetForGaps">Use ceiling-sized discovery and retain truncated remainder rows (#696, strict raw backends only).</param>
|
||||
/// <returns>At most <paramref name="want"/> rows, newest first, and the hint for the next page.</returns>
|
||||
public static async Task<(List<T> Rows, ReverseWalkHint Next)> WalkAsync<T>(
|
||||
Func<DateTime, DateTime, int, CancellationToken, Task<ForwardSegmentRead<T>>> read,
|
||||
Func<T, long> ticksOf,
|
||||
IComparer<T>? tieOrder,
|
||||
DateTime hi, DateTime lo, uint want, int tieBudget, long resolutionTicks, DateTime nowUtc,
|
||||
ReverseWalkHint hint, Func<DateTime, Exception> tooManyAtOneTimestamp, CancellationToken ct)
|
||||
ReverseWalkHint hint, Func<DateTime, Exception> tooManyAtOneTimestamp, CancellationToken ct, bool useDiscoveryBudgetForGaps = false)
|
||||
{
|
||||
var resolution = Math.Max(1L, resolutionTicks);
|
||||
var granularity = Math.Max(Alignment, resolution);
|
||||
@@ -206,7 +207,8 @@ internal static class ReverseWalk
|
||||
var segLo = segHi - stopTicks <= w ? stopTicks : segHi - w;
|
||||
var start = Utc(AlignDown(segLo, granularity));
|
||||
var end = Utc(AlignUp(segHi + 1, granularity));
|
||||
var segBudget = discovering && !probe ? discoveryBudget : pageBudget;
|
||||
var segBudget = (discovering && !probe) || (useDiscoveryBudgetForGaps && (probe || known > 0))
|
||||
? discoveryBudget : pageBudget;
|
||||
var cap = segBudget + 2;
|
||||
IReadOnlyList<T> rows;
|
||||
var narrowed = false;
|
||||
@@ -220,10 +222,14 @@ internal static class ReverseWalk
|
||||
|
||||
if (probe)
|
||||
{
|
||||
// Deep history below: the probe saw only its oldest rows. Carry on walking from the same segHi.
|
||||
// #696: strict-ceiling discovery retains the rows/density instead of discarding the remainder probe.
|
||||
probed = true;
|
||||
narrowed = true;
|
||||
break;
|
||||
if (!useDiscoveryBudgetForGaps)
|
||||
{
|
||||
narrowed = true;
|
||||
break;
|
||||
}
|
||||
probe = false;
|
||||
}
|
||||
|
||||
var leading = 0;
|
||||
@@ -284,7 +290,7 @@ internal static class ReverseWalk
|
||||
// Rows exist at lastInner; what lies above it is unknown. Aim at about half the page budget at the
|
||||
// density the truncated rows showed (their span, not the segment's — it may open on an empty stretch).
|
||||
var aimed = inner > 1 && lastInner > firstInner
|
||||
? (long)Math.Min((double)(lastInner - firstInner) * (pageBudget / 2.0) / (inner - 1), maxWidth)
|
||||
? (long)Math.Min((double)(lastInner - firstInner) * ((useDiscoveryBudgetForGaps ? segBudget : pageBudget) / 2.0) / (inner - 1), maxWidth)
|
||||
: resolution;
|
||||
known = lastInner;
|
||||
knownWidth = Math.Max(resolution, aimed);
|
||||
|
||||
@@ -97,6 +97,27 @@ public sealed class ServerHistorianOptions
|
||||
/// </summary>
|
||||
public bool HonorReturnBounds { get; init; } = true;
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #693, <c>Gateway</c> backend only: whether HistoryRead-AtTime is served from RAW gateway reads evaluated with
|
||||
/// the SQL backends' shared AtTime rules (exact stored value at a stored time, linear between usable neighbours with
|
||||
/// the Interpolated bit, held before a Bad neighbour as Uncertain, null + the outage's Bad code inside an outage,
|
||||
/// <c>BadNoData</c> before the first sample). Default <c>true</c>. <c>false</c> is the kill switch back to the
|
||||
/// pre-#693 answer: AVEVA's own interpolation (the gateway's <c>ReadAtTime</c>), passed through unchanged.
|
||||
/// </summary>
|
||||
public bool GatewayAtTimeFromRaw { get; init; } = true;
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #694 (opt-in): HistoryReadEvents never reads events older than <c>now − EventReadHorizon</c> — the earlier
|
||||
/// bound of every fresh events window is raised to it (both directions, every backend; a window wholly older is
|
||||
/// <c>GoodNoData</c>). Default <see cref="TimeSpan.Zero"/> = no horizon: every read is exact to its own window. It
|
||||
/// exists for the Gateway, where proving "no older events" costs a read down to the Unix epoch on
|
||||
/// <c>Runtime.dbo.Events</c> (~3.8 s live per walked reverse / end-only read; an unfiltered one can hit the
|
||||
/// gateway's 30 s SQL timeout) and a forward read from a far-past StartTime costs the same. ⚠️ Lossy by design:
|
||||
/// events older than the horizon are invisible to HistoryReadEvents while it is set. Negative = none (a
|
||||
/// <see cref="Validate"/> warning).
|
||||
/// </summary>
|
||||
public TimeSpan EventReadHorizon { get; init; } = TimeSpan.Zero;
|
||||
|
||||
/// <summary>
|
||||
/// Maximum number of historized nodes served CONCURRENTLY within one HistoryRead batch (arch-review
|
||||
/// 03/S3). The SDK's <c>HistoryRead*</c> override surface is synchronous, so each arm block-bridges
|
||||
@@ -187,6 +208,8 @@ public sealed class ServerHistorianOptions
|
||||
warnings.Add($"ServerHistorian:MaxConcurrentHistoryReadBatches is {MaxConcurrentHistoryReadBatches} — must be > 0; the process-wide HistoryRead batch limiter is disabled.");
|
||||
if (MaxTimeWeightedRawRows <= 0)
|
||||
warnings.Add($"ServerHistorian:MaxTimeWeightedRawRows is {MaxTimeWeightedRawRows} — must be > 0; time-weighted aggregate reads on the SqlServer/QuestDb backends stream an unbounded number of raw samples.");
|
||||
if (EventReadHorizon < TimeSpan.Zero)
|
||||
warnings.Add($"ServerHistorian:EventReadHorizon is {EventReadHorizon} — negative is treated as no horizon (00:00:00); HistoryReadEvents reads every window in full.");
|
||||
if (HistoryReadDeadline <= TimeSpan.Zero)
|
||||
warnings.Add($"ServerHistorian:HistoryReadDeadline is {HistoryReadDeadline} — must be > 0; HistoryRead requests run unbounded (no per-request deadline).");
|
||||
return warnings;
|
||||
|
||||
+15
@@ -94,4 +94,19 @@ public sealed class HistorianDataSourceReverseEventsGuardTests
|
||||
$"{type.FullName} must declare SupportedAggregates itself, not inherit the interface default");
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Every_implementation_declares_its_raw_page_limit_itself()
|
||||
{
|
||||
// Gitea #694: the default (0 = no limit) is what a strict-ceiling Gateway must NOT report — a wrapper that inherited
|
||||
// it would bring back BadTooManyOperations for every page larger than the gateway's ceiling.
|
||||
foreach (var type in ProductionImplementations())
|
||||
{
|
||||
var map = type.GetInterfaceMap(typeof(IHistorianDataSource));
|
||||
var getter = map.TargetMethods[Array.FindIndex(map.InterfaceMethods,
|
||||
m => m.Name == "get_" + nameof(IHistorianDataSource.MaxRawValuesPerRead))];
|
||||
getter.DeclaringType.ShouldBe(type,
|
||||
$"{type.FullName} must declare MaxRawValuesPerRead itself, not inherit the interface default");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -139,6 +139,7 @@ public abstract class AtTimeConformance : ConformanceBase
|
||||
(At(10), HistorianValue.FromInt64(15), Good));
|
||||
var s = (await B.CreateDataSource().ReadAtTimeAsync("I", [At(5), At(2)], Ct)).Samples;
|
||||
s[0].Value.ShouldBe(13L); // 12.5 ⇒ 13 (away from zero)
|
||||
s[0].Value.ShouldBeOfType<long>(); // ShouldBe compares numbers across types — the type needs its own check (#693)
|
||||
s[1].Value.ShouldBe(11L); // 11.0
|
||||
s[0].StatusCode.ShouldBe(GoodInterpolated);
|
||||
}
|
||||
|
||||
+11
@@ -12,6 +12,17 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests;
|
||||
/// </summary>
|
||||
public sealed class FakeHistorianGatewayClient : IHistorianGatewayClient
|
||||
{
|
||||
public bool TagExistsResult = true;
|
||||
public int TagExistsCallCount;
|
||||
public Exception? TagExistsThrows;
|
||||
public Task<bool> TagExistsAsync(string tag, CancellationToken ct)
|
||||
{
|
||||
ct.ThrowIfCancellationRequested();
|
||||
TagExistsCallCount++;
|
||||
if (TagExistsThrows is { } error) throw error;
|
||||
return Task.FromResult(TagExistsResult);
|
||||
}
|
||||
|
||||
// ---- ReadRaw -------------------------------------------------------------------------------
|
||||
public IReadOnlyList<HistorianSample> RawSamples = Array.Empty<HistorianSample>();
|
||||
public string? LastReadRawTag;
|
||||
|
||||
@@ -0,0 +1,746 @@
|
||||
using Google.Protobuf.WellKnownTypes;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.HistorianGateway.Contracts.Grpc;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #693: Gateway HistoryRead AtTime is served from RAW reads and evaluated with the SQL backends' shared
|
||||
/// <c>AtTimeEvaluator</c> (Part 11 §6.5.4) — never AVEVA's own interpolated answer, which the live gate caught returning
|
||||
/// the previous value as Good at a stored NULL's own time, null + Bad at a Good sample right after a NULL, a drifted value
|
||||
/// at an exact stored sample, and no Interpolated bit. The fake is AVEVA-shaped (the real wire shape): ReadRaw over
|
||||
/// <c>[start, end]</c> ascending, capped, led by the synthetic boundary row at the query start (the value in effect there,
|
||||
/// <c>Quality = 133</c>, or a NULL row), NULL rows as <c>NumericValue = 0</c> + <c>q=1 opc=0</c>, millisecond timestamps;
|
||||
/// and it REJECTS what the real gateway rejects (<c>max_values <= 0</c>, <c>end < start</c>).
|
||||
/// </summary>
|
||||
public sealed class GatewayAtTimeTests
|
||||
{
|
||||
private static CancellationToken Ct => TestContext.Current.CancellationToken;
|
||||
|
||||
private const uint Good = 0u;
|
||||
private const uint Bad = 0x80000000u;
|
||||
private const uint BadNoData = 0x809B0000u;
|
||||
private const uint GoodInterpolated = 0x00000402u;
|
||||
private const uint UncertainInterpolated = 0x40A40402u; // UncertainDataSubNormal | InfoType DataValue | Interpolated
|
||||
|
||||
/// <summary>The live gate's tag <c>HistGW.LiveTest.Gate685</c>: 19:00:03 is a stored NULL.</summary>
|
||||
private static readonly DateTime T0 = new(2026, 10, 12, 19, 0, 0, DateTimeKind.Utc);
|
||||
|
||||
private static readonly DateTime Now = T0.AddHours(1);
|
||||
|
||||
private static DateTime At(double seconds) => T0.AddTicks((long)(seconds * TimeSpan.TicksPerSecond));
|
||||
|
||||
private static List<(DateTime T, double V, bool Null)> Gate685() =>
|
||||
[(At(0), 100, false), (At(2), 107, false), (At(3), 0, true), (At(5), 108, false)];
|
||||
|
||||
/// <summary>An AVEVA-shaped store over sorted rows, counting reads and rejecting what the real gateway rejects.</summary>
|
||||
private sealed class AvevaStore
|
||||
{
|
||||
private readonly (DateTime T, double V, bool Null)[] _rows;
|
||||
|
||||
public AvevaStore(IEnumerable<(DateTime T, double V, bool Null)> rows) => _rows = rows.OrderBy(r => r.T).ToArray();
|
||||
|
||||
/// <summary>Also emit a row at the exact query END carrying the value in effect there (unverified AVEVA behaviour).</summary>
|
||||
public bool TerminalRow { get; init; }
|
||||
|
||||
/// <summary>Total rows streamed (cost).</summary>
|
||||
public long RowsStreamed;
|
||||
|
||||
public FakeHistorianGatewayClient Client(int? ceiling = null) => new()
|
||||
{
|
||||
RawSource = Serve,
|
||||
MaxValuesCeiling = ceiling,
|
||||
};
|
||||
|
||||
private static HistorianSample Row(DateTime t, double v, bool isNull, uint quality = 0) => new()
|
||||
{
|
||||
Tag = "T",
|
||||
Timestamp = Timestamp.FromDateTime(t),
|
||||
NumericValue = isNull ? 0 : v,
|
||||
OpcQuality = isNull ? 0u : 192u,
|
||||
Quality = isNull ? 1u : quality,
|
||||
};
|
||||
|
||||
private int FirstAtOrAfter(DateTime t)
|
||||
{
|
||||
int lo = 0, hi = _rows.Length;
|
||||
while (lo < hi)
|
||||
{
|
||||
var mid = (lo + hi) / 2;
|
||||
if (_rows[mid].T < t) lo = mid + 1; else hi = mid;
|
||||
}
|
||||
return lo;
|
||||
}
|
||||
|
||||
private HistorianSample InEffect(DateTime t)
|
||||
{
|
||||
var i = FirstAtOrAfter(t) - 1;
|
||||
if (i < 0) return new HistorianSample { Tag = "T", Timestamp = Timestamp.FromDateTime(t), NumericValue = 0, OpcQuality = 0, Quality = 1 };
|
||||
var p = _rows[i];
|
||||
return Row(t, p.V, p.Null, p.Null ? 1u : 133u);
|
||||
}
|
||||
|
||||
private IEnumerable<HistorianSample> Serve(DateTime start, DateTime end, int max)
|
||||
{
|
||||
if (max <= 0)
|
||||
throw new Grpc.Core.RpcException(new Grpc.Core.Status(Grpc.Core.StatusCode.InvalidArgument, "max_values must be greater than zero."));
|
||||
if (end < start)
|
||||
throw new Grpc.Core.RpcException(new Grpc.Core.Status(Grpc.Core.StatusCode.InvalidArgument,
|
||||
"start_time must be less than or equal to end_time."));
|
||||
if (end > Now.AddDays(2))
|
||||
throw new Grpc.Core.RpcException(new Grpc.Core.Status(Grpc.Core.StatusCode.InvalidArgument,
|
||||
"end_time is outside the historian's time range"));
|
||||
if (start.Ticks % TimeSpan.TicksPerMillisecond != 0 || end.Ticks % TimeSpan.TicksPerMillisecond != 0)
|
||||
throw new InvalidOperationException("AVEVA truncates request times to the millisecond; send millisecond bounds.");
|
||||
|
||||
var rows = new List<HistorianSample>();
|
||||
var i = FirstAtOrAfter(start);
|
||||
if (i >= _rows.Length || _rows[i].T != start) rows.Add(InEffect(start));
|
||||
for (; i < _rows.Length && _rows[i].T <= end && rows.Count < max; i++)
|
||||
rows.Add(Row(_rows[i].T, _rows[i].V, _rows[i].Null));
|
||||
if (TerminalRow && rows.Count < max && (i >= _rows.Length || _rows[i].T > end) && _rows.All(r => r.T != end))
|
||||
rows.Add(InEffect(end));
|
||||
var served = rows.Take(max).ToList();
|
||||
RowsStreamed += served.Count;
|
||||
return served;
|
||||
}
|
||||
}
|
||||
|
||||
private static GatewayHistorianDataSource Source(FakeHistorianGatewayClient fake) =>
|
||||
new(fake, NullLogger<GatewayHistorianDataSource>.Instance, timeProvider: new FixedTime(Now));
|
||||
|
||||
private sealed class FixedTime(DateTime now) : TimeProvider
|
||||
{
|
||||
public override DateTimeOffset GetUtcNow() => new(now);
|
||||
}
|
||||
|
||||
private static async Task<IReadOnlyList<DataValueSnapshot>> AtTime(FakeHistorianGatewayClient fake, params DateTime[] times)
|
||||
{
|
||||
var r = await Source(fake).ReadAtTimeAsync("T", times, Ct);
|
||||
r.Samples.Count.ShouldBe(times.Length);
|
||||
for (var i = 0; i < times.Length; i++) r.Samples[i].SourceTimestampUtc.ShouldBe(times[i]);
|
||||
var refused = fake.MaxValuesCeiling is { } c ? fake.RawCalls.Count(x => x.MaxValues > c) : 0;
|
||||
fake.RawStreamsCompleted.ShouldBe(fake.ReadRawCallCount - refused); // never abandons a gateway stream
|
||||
fake.ReadAtTimeCallCount.ShouldBe(0); // never AVEVA's own interpolation
|
||||
return r.Samples;
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_future_time_on_a_live_tag_finds_the_prior_sample_without_walking_the_empty_future()
|
||||
{
|
||||
var store = new AvevaStore(Enumerable.Range(0, 7201).Select(i => (Now.AddSeconds(i - 7200), (double)i, false)));
|
||||
var fake = store.Client();
|
||||
var result = (await AtTime(fake, Now.AddHours(1))).Single();
|
||||
result.Value.ShouldBe(7200d);
|
||||
result.StatusCode.ShouldBe(UncertainInterpolated);
|
||||
fake.ReadRawCallCount.ShouldBeLessThanOrEqualTo(4);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_future_time_still_uses_a_stored_future_prior_sample()
|
||||
{
|
||||
var fake = new AvevaStore([(Now.AddSeconds(-1), 1d, false), (Now.AddMinutes(30), 2d, false)]).Client();
|
||||
var result = (await AtTime(fake, Now.AddHours(1))).Single();
|
||||
result.Value.ShouldBe(2d);
|
||||
result.StatusCode.ShouldBe(UncertainInterpolated);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_full_future_probe_walks_to_the_latest_stored_future_sample()
|
||||
{
|
||||
var fake = new AvevaStore(Enumerable.Range(0, 7201).Select(i => (Now.AddSeconds(i), (double)i, false))).Client();
|
||||
var result = (await AtTime(fake, Now.AddHours(3))).Single();
|
||||
result.Value.ShouldBe(7200d);
|
||||
result.StatusCode.ShouldBe(UncertainInterpolated);
|
||||
}
|
||||
|
||||
// ---- the four live findings ------------------------------------------------------------------------------------
|
||||
|
||||
[Fact] // live: 19:00:03 (a stored NULL) ⇒ 107 Good
|
||||
public async Task At_a_stored_NULLs_own_time_is_null_and_Bad()
|
||||
{
|
||||
var s = (await AtTime(new AvevaStore(Gate685()).Client(), At(3))).Single();
|
||||
s.Value.ShouldBeNull();
|
||||
s.StatusCode.ShouldBe(Bad);
|
||||
}
|
||||
|
||||
[Fact] // live: 19:00:05 (stored 108, right after the NULL) ⇒ null + Bad
|
||||
public async Task At_a_Good_sample_right_after_a_NULL_is_the_stored_value()
|
||||
{
|
||||
var s = (await AtTime(new AvevaStore(Gate685()).Client(), At(5))).Single();
|
||||
s.Value.ShouldBe(108d);
|
||||
s.StatusCode.ShouldBe(Good);
|
||||
}
|
||||
|
||||
[Fact] // live: 99.99999999944 for a stored 100
|
||||
public async Task At_an_exact_stored_sample_is_the_exact_stored_value_without_the_Interpolated_bit()
|
||||
{
|
||||
var s = await AtTime(new AvevaStore(Gate685()).Client(), At(0), At(2));
|
||||
s[0].Value.ShouldBe(100d);
|
||||
s[0].StatusCode.ShouldBe(Good);
|
||||
s[1].Value.ShouldBe(107d);
|
||||
s[1].StatusCode.ShouldBe(Good);
|
||||
}
|
||||
|
||||
[Fact] // live: no Interpolated bit
|
||||
public async Task Between_two_Good_samples_is_linear_with_the_Interpolated_bit()
|
||||
{
|
||||
var s = (await AtTime(new AvevaStore(Gate685()).Client(), At(1))).Single();
|
||||
s.Value.ShouldBe(103.5d);
|
||||
s.StatusCode.ShouldBe(GoodInterpolated);
|
||||
new Opc.Ua.StatusCode(s.StatusCode).AggregateBits.ShouldBe(Opc.Ua.AggregateBits.Interpolated);
|
||||
}
|
||||
|
||||
// ---- the rest of the AtTime rules --------------------------------------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public async Task Before_a_Bad_neighbour_the_previous_value_is_held_Uncertain_and_Interpolated()
|
||||
{
|
||||
var s = (await AtTime(new AvevaStore(Gate685()).Client(), At(2.5))).Single();
|
||||
s.Value.ShouldBe(107d);
|
||||
s.StatusCode.ShouldBe(UncertainInterpolated);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Inside_an_outage_is_null_with_the_outages_Bad_code()
|
||||
{
|
||||
var s = (await AtTime(new AvevaStore(Gate685()).Client(), At(4))).Single();
|
||||
s.Value.ShouldBeNull();
|
||||
s.StatusCode.ShouldBe(Bad);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Before_the_first_sample_is_BadNoData_not_the_boundary_rows_zero()
|
||||
{
|
||||
var s = await AtTime(new AvevaStore(Gate685()).Client(), At(-1), At(-0.001), At(-3600));
|
||||
s.ShouldAllBe(x => x.StatusCode == BadNoData && x.Value == null);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task After_the_last_sample_is_stepped_Good_until_now_then_Uncertain()
|
||||
{
|
||||
var s = await AtTime(new AvevaStore(Gate685()).Client(), At(10), Now.AddMinutes(5), Now.AddDays(3));
|
||||
s[0].Value.ShouldBe(108d);
|
||||
s[0].StatusCode.ShouldBe(GoodInterpolated);
|
||||
s[1].Value.ShouldBe(108d);
|
||||
s[1].StatusCode.ShouldBe(UncertainInterpolated); // later than now: not known to still hold
|
||||
s[2].Value.ShouldBe(108d);
|
||||
s[2].StatusCode.ShouldBe(UncertainInterpolated); // beyond the horizon: never sent to the gateway (it refuses)
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_mixed_request_keeps_order_and_duplicates()
|
||||
{
|
||||
var s = await AtTime(new AvevaStore(Gate685()).Client(), At(5), At(1), At(-1), At(3), At(1), At(2.5));
|
||||
s.Select(x => x.Value).ShouldBe(new object?[] { 108d, 103.5d, null, null, 103.5d, 107d });
|
||||
s.Select(x => x.StatusCode).ShouldBe(new[] { Good, GoodInterpolated, BadNoData, Bad, GoodInterpolated, UncertainInterpolated });
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task An_empty_request_reads_nothing()
|
||||
{
|
||||
var fake = new AvevaStore(Gate685()).Client();
|
||||
(await Source(fake).ReadAtTimeAsync("T", [], Ct)).Samples.ShouldBeEmpty();
|
||||
fake.ReadRawCallCount.ShouldBe(0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_sub_millisecond_request_time_interpolates_between_the_millisecond_samples()
|
||||
{
|
||||
var s = (await AtTime(new AvevaStore(Gate685()).Client(), At(0.0005))).Single();
|
||||
((double)s.Value!).ShouldBe(100.00175d, 1e-9);
|
||||
s.StatusCode.ShouldBe(GoodInterpolated);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_row_at_the_query_end_never_counts_as_a_stored_sample()
|
||||
{
|
||||
var store = new AvevaStore(Gate685()) { TerminalRow = true };
|
||||
var s = await AtTime(store.Client(), At(1), At(2.5), At(4), At(10));
|
||||
s.Select(x => x.Value).ShouldBe(new object?[] { 103.5d, 107d, null, 108d });
|
||||
s.Select(x => x.StatusCode).ShouldBe(new[] { GoodInterpolated, UncertainInterpolated, Bad, GoodInterpolated });
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_string_tag_is_stepped()
|
||||
{
|
||||
var fake = new FakeHistorianGatewayClient
|
||||
{
|
||||
RawSource = (start, end, max) =>
|
||||
{
|
||||
var rows = new List<HistorianSample>
|
||||
{
|
||||
new() { Tag = "T", Timestamp = Timestamp.FromDateTime(start), StringValue = "", OpcQuality = 0, Quality = 1 },
|
||||
};
|
||||
foreach (var (t, v) in new[] { (At(0), "a"), (At(10), "b") })
|
||||
if (t >= start && t <= end)
|
||||
rows.Add(new HistorianSample { Tag = "T", Timestamp = Timestamp.FromDateTime(t), StringValue = v, OpcQuality = 192 });
|
||||
return rows.Take(max);
|
||||
},
|
||||
};
|
||||
var s = (await AtTime(fake, At(5))).Single();
|
||||
s.Value.ShouldBe("a");
|
||||
s.StatusCode.ShouldBe(GoodInterpolated);
|
||||
}
|
||||
|
||||
// ---- the node's value type (the node manager's hint; AVEVA stores a Boolean as a 0/1 Int2) ------------------------
|
||||
|
||||
/// <summary>Reads through the INTERFACE's hinted overload, as the node manager does.</summary>
|
||||
private static async Task<IReadOnlyList<DataValueSnapshot>> AtTimeTyped(
|
||||
FakeHistorianGatewayClient fake, TypeCode? valueType, params DateTime[] times)
|
||||
{
|
||||
IHistorianDataSource source = Source(fake);
|
||||
var r = await source.ReadAtTimeAsync("T", times, valueType, Ct);
|
||||
r.Samples.Count.ShouldBe(times.Length);
|
||||
fake.ReadAtTimeCallCount.ShouldBe(0);
|
||||
return r.Samples;
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_Boolean_node_is_stepped_never_a_fraction()
|
||||
{
|
||||
var fake = new AvevaStore([(At(0), 0, false), (At(10), 1, false), (At(20), 0, false)]).Client();
|
||||
var s = await AtTimeTyped(fake, TypeCode.Boolean, At(5), At(10), At(15), At(25));
|
||||
s.Select(x => x.Value).ShouldBe(new object?[] { false, true, true, false });
|
||||
s.ShouldAllBe(x => x.Value is bool);
|
||||
s.Select(x => x.StatusCode).ShouldBe(new[] { GoodInterpolated, Good, GoodInterpolated, GoodInterpolated });
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Without_a_hint_a_Boolean_tag_interpolates_as_before()
|
||||
{
|
||||
var fake = new AvevaStore([(At(0), 0, false), (At(10), 1, false)]).Client();
|
||||
(await AtTimeTyped(fake, null, At(5))).Single().Value.ShouldBeOfType<double>().ShouldBe(0.5d);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task An_Int32_node_interpolates_rounds_half_away_from_zero_and_keeps_its_type()
|
||||
{
|
||||
var s = await AtTimeTyped(new AvevaStore(Gate685()).Client(), TypeCode.Int32, At(0), At(0.5), At(1), At(2.5), At(3));
|
||||
s[0].Value.ShouldBe(100); // exact hit, as stored, Int32
|
||||
s[0].StatusCode.ShouldBe(Good);
|
||||
s[1].Value.ShouldBe(102); // 101.75 ⇒ 102
|
||||
s[2].Value.ShouldBe(104); // 103.5 ⇒ 104 (half away from zero)
|
||||
s[1].StatusCode.ShouldBe(GoodInterpolated);
|
||||
s[3].Value.ShouldBe(107); // held before the NULL, Uncertain
|
||||
s[3].StatusCode.ShouldBe(UncertainInterpolated);
|
||||
s[4].Value.ShouldBeNull(); // the NULL itself
|
||||
s[4].StatusCode.ShouldBe(Bad);
|
||||
s.Take(4).Select(x => x.Value + ":" + x.Value!.GetType().Name).ShouldBe(["100:Int32", "102:Int32", "104:Int32", "107:Int32"]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_negative_midpoint_rounds_away_from_zero()
|
||||
{
|
||||
var fake = new AvevaStore([(At(0), -100, false), (At(2), -107, false)]).Client();
|
||||
(await AtTimeTyped(fake, TypeCode.Int16, At(1))).Single().Value.ShouldBeOfType<short>().ShouldBe((short)-104);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Other_integer_nodes_keep_their_own_type()
|
||||
{
|
||||
var fake = new AvevaStore(Gate685()).Client();
|
||||
(await AtTimeTyped(fake, TypeCode.UInt16, At(1))).Single().Value.ShouldBeOfType<ushort>().ShouldBe((ushort)104);
|
||||
(await AtTimeTyped(fake, TypeCode.Int64, At(1))).Single().Value.ShouldBeOfType<long>().ShouldBe(104L);
|
||||
(await AtTimeTyped(fake, TypeCode.Byte, At(1))).Single().Value.ShouldBeOfType<byte>().ShouldBe((byte)104);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Float_and_Double_nodes_interpolate_unchanged()
|
||||
{
|
||||
var fake = new AvevaStore(Gate685()).Client();
|
||||
(await AtTimeTyped(fake, TypeCode.Double, At(1))).Single().Value.ShouldBeOfType<double>().ShouldBe(103.5d);
|
||||
(await AtTimeTyped(fake, TypeCode.Single, At(1))).Single().Value.ShouldBeOfType<double>().ShouldBe(103.5d); // as the SQL backends (Double kind)
|
||||
}
|
||||
|
||||
// ---- cost ------------------------------------------------------------------------------------------------------
|
||||
|
||||
private static IEnumerable<(DateTime T, double V, bool Null)> OneHertz(DateTime from, int seconds) =>
|
||||
Enumerable.Range(0, seconds).Select(i => (from.AddSeconds(i), (double)i, false));
|
||||
|
||||
[Fact]
|
||||
public async Task Ten_exact_hits_on_dense_data_are_one_read()
|
||||
{
|
||||
var fake = new AvevaStore(OneHertz(T0.AddHours(-1), 3600)).Client();
|
||||
var times = Enumerable.Range(0, 10).Select(i => T0.AddMinutes(-30).AddSeconds(i)).ToArray();
|
||||
var s = await AtTime(fake, times);
|
||||
s.Select(x => x.Value).ShouldBe(Enumerable.Range(1800, 10).Select(i => (object?)(double)i));
|
||||
s.ShouldAllBe(x => x.StatusCode == Good);
|
||||
fake.ReadRawCallCount.ShouldBe(1);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Ten_interpolated_times_on_dense_data_are_three_reads()
|
||||
{
|
||||
var fake = new AvevaStore(OneHertz(T0.AddHours(-1), 3600)).Client();
|
||||
var times = Enumerable.Range(0, 10).Select(i => T0.AddMinutes(-30).AddSeconds(i + 0.5)).ToArray();
|
||||
var s = await AtTime(fake, times);
|
||||
s.Select(x => x.Value).ShouldBe(Enumerable.Range(1800, 10).Select(i => (object?)(i + 0.5)));
|
||||
s.ShouldAllBe(x => x.StatusCode == GoodInterpolated);
|
||||
fake.ReadRawCallCount.ShouldBe(3); // the window, the sample before the first time, the sample after the last
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Ten_times_an_hour_apart_on_dense_data_cost_about_one_read_each_not_the_rows_between()
|
||||
{
|
||||
var store = new AvevaStore(OneHertz(T0.AddHours(-12), 12 * 3600));
|
||||
var fake = store.Client();
|
||||
var times = Enumerable.Range(0, 10).Select(i => T0.AddHours(-11).AddHours(i).AddSeconds(0.5)).ToArray();
|
||||
var s = await AtTime(fake, times);
|
||||
s.Select(x => x.Value).ShouldBe(Enumerable.Range(0, 10).Select(i => (object?)(3600.0 * (i + 1) + 0.5)));
|
||||
s.ShouldAllBe(x => x.StatusCode == GoodInterpolated);
|
||||
fake.ReadRawCallCount.ShouldBe(12); // 10 window reads (each finds its own prev) + 1 walk-back + 1 probe
|
||||
store.RowsStreamed.ShouldBeLessThan(10 * 1100); // never the ~32,400 rows between the first and last time
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Sparse_history_far_back_is_found_in_a_bounded_number_of_reads()
|
||||
{
|
||||
var fake = new AvevaStore([(T0.AddDays(-400), 42, false), (T0.AddDays(-400).AddSeconds(10), 43, false)]).Client();
|
||||
var s = (await AtTime(fake, T0)).Single();
|
||||
s.Value.ShouldBe(43d);
|
||||
s.StatusCode.ShouldBe(GoodInterpolated);
|
||||
fake.ReadRawCallCount.ShouldBe(5); // window, 1-min + 4-min segments, the whole remainder, the probe after
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_tag_with_no_history_is_BadNoData_in_a_bounded_number_of_reads()
|
||||
{
|
||||
var fake = new AvevaStore([]).Client();
|
||||
var s = await AtTime(fake, At(1), At(2));
|
||||
s.ShouldAllBe(x => x.StatusCode == BadNoData);
|
||||
fake.ReadRawCallCount.ShouldBe(4); // window, two segments, the whole remainder (no probe: nothing to interpolate)
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Dense_data_far_denser_than_one_read_still_finds_the_neighbours()
|
||||
{
|
||||
// 1 kHz for 10 minutes: a 1-minute walk-back segment holds 60,000 rows (truncated) — it must narrow, not fail.
|
||||
var rows = Enumerable.Range(0, 600_000).Select(i => (T0.AddMinutes(-20).AddMilliseconds(i), (double)i, false));
|
||||
var fake = new AvevaStore(rows).Client();
|
||||
var t = T0.AddMinutes(-15).AddTicks(5000); // 0.5 ms past sample 300,000
|
||||
var s = (await AtTime(fake, t)).Single();
|
||||
s.Value.ShouldBe(300_000.5d);
|
||||
s.StatusCode.ShouldBe(GoodInterpolated);
|
||||
fake.ReadRawCallCount.ShouldBe(2); // the window (holds the prev) + the probe
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_walk_back_into_data_far_denser_than_one_read_narrows_instead_of_paging()
|
||||
{
|
||||
// 1 kHz for 10 minutes, then a 10-minute gap: the walk-back crosses the gap and lands in 600,000 rows.
|
||||
var rows = Enumerable.Range(0, 600_000).Select(i => (T0.AddMinutes(-20).AddMilliseconds(i), (double)i, false));
|
||||
var store = new AvevaStore(rows);
|
||||
var fake = store.Client();
|
||||
var s = (await AtTime(fake, T0.AddMinutes(-5))).Single();
|
||||
s.Value.ShouldBe(599_999d);
|
||||
s.StatusCode.ShouldBe(GoodInterpolated); // stepped from the last sample (before now)
|
||||
fake.ReadRawCallCount.ShouldBe(6); // window, 1m, 4m, remainder (full: seeds the bisection), one aimed segment, probe
|
||||
store.RowsStreamed.ShouldBeLessThan(10_000);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_strict_gateway_with_a_small_ceiling_is_still_served()
|
||||
{
|
||||
var store = new AvevaStore(OneHertz(T0.AddHours(-1), 3600));
|
||||
var fake = store.Client(ceiling: 100);
|
||||
var times = Enumerable.Range(0, 10).Select(i => T0.AddMinutes(-30).AddSeconds(i * 30 + 0.5)).ToArray();
|
||||
var s = await AtTime(fake, times);
|
||||
s.Select(x => x.Value).ShouldBe(Enumerable.Range(0, 10).Select(i => (object?)(1800 + i * 30 + 0.5)));
|
||||
s.ShouldAllBe(x => x.StatusCode == GoodInterpolated);
|
||||
fake.RawCalls.Count(c => c.MaxValues > 100).ShouldBe(1); // learned once, then remembered — a short page is not an error
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Success_is_recorded_once()
|
||||
{
|
||||
var fake = new AvevaStore(Gate685()).Client();
|
||||
var ds = Source(fake);
|
||||
await ds.ReadAtTimeAsync("T", [At(1), At(4)], Ct);
|
||||
var h = ds.GetHealthSnapshot();
|
||||
h.TotalQueries.ShouldBe(1);
|
||||
h.TotalSuccesses.ShouldBe(1);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_gateway_fault_is_a_recorded_failure()
|
||||
{
|
||||
var fake = new AvevaStore(Gate685()).Client();
|
||||
fake.ThrowOnRead = new Grpc.Core.RpcException(new Grpc.Core.Status(Grpc.Core.StatusCode.Unavailable, "down"));
|
||||
var ds = Source(fake);
|
||||
await Should.ThrowAsync<Exception>(() => ds.ReadAtTimeAsync("T", [At(1)], Ct));
|
||||
ds.GetHealthSnapshot().TotalFailures.ShouldBe(1);
|
||||
}
|
||||
|
||||
// ---- long histories (review H1/M1/L1): a generated store, too large to materialise -----------------------------
|
||||
|
||||
/// <summary>
|
||||
/// A regular series (value = sample index) over <c>[from, to]</c> every <paramref name="step"/>, optionally with a
|
||||
/// hole <c>(gapFrom, gapTo)</c>, served in the AVEVA wire shape (boundary row at the query start, ms bounds only,
|
||||
/// <c>max_values > 0</c>, <c>start <= end</c>, nothing past now + 2 days).
|
||||
/// </summary>
|
||||
private sealed class SeriesStore(DateTime from, DateTime to, TimeSpan step, DateTime? gapFrom = null, DateTime? gapTo = null)
|
||||
{
|
||||
public long RowsStreamed;
|
||||
|
||||
private bool InGap(DateTime t) => gapFrom is { } a && gapTo is { } b && t > a && t < b;
|
||||
|
||||
private DateTime? FirstAtOrAfter(DateTime t)
|
||||
{
|
||||
if (t > to) return null;
|
||||
var k = t <= from ? 0 : ((t - from).Ticks + step.Ticks - 1) / step.Ticks;
|
||||
var r = from.AddTicks(k * step.Ticks);
|
||||
if (InGap(r)) return FirstAtOrAfter(gapTo!.Value);
|
||||
return r <= to ? r : null;
|
||||
}
|
||||
|
||||
private DateTime? LastBefore(DateTime t)
|
||||
{
|
||||
if (t <= from) return null;
|
||||
var k = ((Min(t, to.AddTicks(1)) - from).Ticks - 1) / step.Ticks;
|
||||
var r = from.AddTicks(k * step.Ticks);
|
||||
return InGap(r) ? LastBefore(gapFrom!.Value.AddTicks(1)) : r;
|
||||
}
|
||||
|
||||
private static DateTime Min(DateTime a, DateTime b) => a <= b ? a : b;
|
||||
|
||||
/// <summary>The stored row at <paramref name="t"/>, stamped <paramref name="at"/> (the boundary row) when given.</summary>
|
||||
private HistorianSample Row(DateTime t, uint quality = 0, DateTime? at = null) => new()
|
||||
{
|
||||
Tag = "T", Timestamp = Timestamp.FromDateTime(at ?? t), NumericValue = (t - from).Ticks / step.Ticks, OpcQuality = 192,
|
||||
Quality = quality,
|
||||
};
|
||||
|
||||
public FakeHistorianGatewayClient Client(int? ceiling = null) => new() { RawSource = Serve, MaxValuesCeiling = ceiling };
|
||||
|
||||
private IEnumerable<HistorianSample> Serve(DateTime start, DateTime end, int max)
|
||||
{
|
||||
if (max <= 0 || end < start || end > Now.AddDays(2)
|
||||
|| start.Ticks % TimeSpan.TicksPerMillisecond != 0 || end.Ticks % TimeSpan.TicksPerMillisecond != 0)
|
||||
throw new Grpc.Core.RpcException(new Grpc.Core.Status(Grpc.Core.StatusCode.InvalidArgument, "refused"));
|
||||
var rows = new List<HistorianSample>();
|
||||
if (FirstAtOrAfter(start) != start)
|
||||
rows.Add(LastBefore(start) is { } p
|
||||
? Row(p, 133, start)
|
||||
: new HistorianSample { Tag = "T", Timestamp = Timestamp.FromDateTime(start), OpcQuality = 0, Quality = 1, NumericValue = 0 });
|
||||
for (var t = FirstAtOrAfter(start); t is { } u && u <= end && rows.Count < max; t = FirstAtOrAfter(u.AddTicks(1)))
|
||||
rows.Add(Row(u));
|
||||
RowsStreamed += rows.Count;
|
||||
return rows;
|
||||
}
|
||||
}
|
||||
|
||||
[Theory] // H1: a dense tag that stopped reporting — the walk crosses the silence by bisection, not by regrowing segments
|
||||
[InlineData(1.0, 30)]
|
||||
[InlineData(1.0, 365)]
|
||||
[InlineData(10.0, 30)]
|
||||
[InlineData(10.0, 365)]
|
||||
public async Task A_dense_tag_that_stopped_long_ago_is_found_in_a_bounded_number_of_reads(double hz, int stoppedDaysAgo)
|
||||
{
|
||||
var step = TimeSpan.FromTicks((long)(TimeSpan.TicksPerSecond / hz));
|
||||
var end = Now.AddDays(-stoppedDaysAgo);
|
||||
var fake = new SeriesStore(end.AddYears(-5), end, step).Client();
|
||||
var s = (await AtTime(fake, Now.AddMinutes(-1))).Single();
|
||||
s.Value.ShouldBe((double)((end - end.AddYears(-5)).Ticks / step.Ticks)); // the last sample, held
|
||||
s.StatusCode.ShouldBe(GoodInterpolated);
|
||||
fake.ReadRawCallCount.ShouldBeLessThan(40);
|
||||
}
|
||||
|
||||
[Fact] // H1: a time inside a month-long hole in the middle of a dense history
|
||||
public async Task A_time_inside_a_mid_history_gap_interpolates_across_it()
|
||||
{
|
||||
var from = Now.AddYears(-2);
|
||||
var gapFrom = Now.AddDays(-200);
|
||||
var gapTo = gapFrom.AddDays(30);
|
||||
var fake = new SeriesStore(from, Now, TimeSpan.FromSeconds(1), gapFrom, gapTo).Client();
|
||||
var t = gapFrom.AddDays(15);
|
||||
var s = (await AtTime(fake, t)).Single();
|
||||
var before = (gapFrom - from).Ticks / TimeSpan.TicksPerSecond;
|
||||
var after = (gapTo - from).Ticks / TimeSpan.TicksPerSecond;
|
||||
((double)s.Value!).ShouldBe(before + (after - before) * 0.5, 1e-6);
|
||||
s.StatusCode.ShouldBe(GoodInterpolated);
|
||||
fake.ReadRawCallCount.ShouldBeLessThan(40);
|
||||
}
|
||||
|
||||
[Fact] // H1: a jump to a time in a gap — the walk starts from the earlier read's last row, far below
|
||||
public async Task A_jump_into_a_gap_bisects_up_from_the_earlier_reads_last_row()
|
||||
{
|
||||
var from = Now.AddDays(-30);
|
||||
var gapFrom = from.AddDays(2);
|
||||
var gapTo = gapFrom.AddDays(10);
|
||||
var fake = new SeriesStore(from, Now, TimeSpan.FromSeconds(1), gapFrom, gapTo).Client();
|
||||
var t0 = from.AddSeconds(10.5);
|
||||
var t1 = gapFrom.AddDays(5);
|
||||
var s = await AtTime(fake, t0, t1);
|
||||
s[0].Value.ShouldBe(10.5d);
|
||||
var before = (gapFrom - from).Ticks / TimeSpan.TicksPerSecond;
|
||||
var after = (gapTo - from).Ticks / TimeSpan.TicksPerSecond;
|
||||
((double)s[1].Value!).ShouldBe(before + (after - before) * 0.5, 1e-6);
|
||||
fake.ReadRawCallCount.ShouldBeLessThan(40);
|
||||
}
|
||||
|
||||
[Fact] // M1: widely spread times cost one SMALL read each, not one 1,024-row read each
|
||||
public async Task A_thousand_times_over_ten_years_cost_one_small_read_each()
|
||||
{
|
||||
var store = new SeriesStore(Now.AddYears(-10), Now, TimeSpan.FromSeconds(1));
|
||||
var fake = store.Client();
|
||||
var rnd = new Random(1);
|
||||
var times = Enumerable.Range(0, 1000).Select(_ => Now.AddYears(-9).AddTicks((long)(rnd.NextDouble() * TimeSpan.TicksPerDay * 3000))).ToArray();
|
||||
var s = await AtTime(fake, times);
|
||||
s.ShouldAllBe(x => x.StatusCode == GoodInterpolated);
|
||||
fake.ReadRawCallCount.ShouldBeLessThanOrEqualTo(1010);
|
||||
store.RowsStreamed.ShouldBeLessThan(80_000); // ~64 rows a read (was ~1,000,000)
|
||||
}
|
||||
|
||||
[Fact] // L1: a strict-gateway ceiling too small to work with is refused with a message naming it
|
||||
public async Task A_ceiling_below_a_usable_read_is_refused_naming_the_ceiling()
|
||||
{
|
||||
var fake = new SeriesStore(Now.AddDays(-1), Now, TimeSpan.FromSeconds(1)).Client(ceiling: 2);
|
||||
var ex = await Should.ThrowAsync<HistoryReadLimitExceededException>(
|
||||
() => Source(fake).ReadAtTimeAsync("T", [Now.AddHours(-1)], Ct));
|
||||
ex.Message.ShouldContain("row ceiling (2");
|
||||
}
|
||||
|
||||
/// <summary>A bursty tag: 1 Hz for the first hour of every day (value = seconds since <paramref name="from"/>).</summary>
|
||||
private sealed class BurstStore(DateTime from, int days)
|
||||
{
|
||||
public long RowsStreamed;
|
||||
private const long Burst = 3600 * TimeSpan.TicksPerSecond;
|
||||
|
||||
private DateTime? FirstAtOrAfter(DateTime t)
|
||||
{
|
||||
if (t <= from) return from;
|
||||
var d = (t - from).Ticks / TimeSpan.TicksPerDay;
|
||||
var inDay = (t - from).Ticks % TimeSpan.TicksPerDay;
|
||||
var r = inDay >= Burst
|
||||
? from.AddDays(d + 1)
|
||||
: from.AddTicks(d * TimeSpan.TicksPerDay + (inDay + TimeSpan.TicksPerSecond - 1) / TimeSpan.TicksPerSecond * TimeSpan.TicksPerSecond);
|
||||
if ((r - from).Ticks % TimeSpan.TicksPerDay >= Burst) r = from.AddDays(d + 1);
|
||||
return r < from.AddDays(days) ? r : null;
|
||||
}
|
||||
|
||||
private DateTime? LastBefore(DateTime t)
|
||||
{
|
||||
if (t <= from) return null;
|
||||
var x = (t - from).Ticks - 1;
|
||||
var d = x / TimeSpan.TicksPerDay;
|
||||
var inDay = x % TimeSpan.TicksPerDay;
|
||||
var r = inDay >= Burst - TimeSpan.TicksPerSecond
|
||||
? from.AddTicks(d * TimeSpan.TicksPerDay + Burst - TimeSpan.TicksPerSecond)
|
||||
: from.AddTicks(d * TimeSpan.TicksPerDay + inDay / TimeSpan.TicksPerSecond * TimeSpan.TicksPerSecond);
|
||||
var last = from.AddDays(days - 1).AddTicks(Burst - TimeSpan.TicksPerSecond);
|
||||
return r > last ? last : r;
|
||||
}
|
||||
|
||||
private HistorianSample Row(DateTime t, uint quality = 0, DateTime? at = null) => new()
|
||||
{
|
||||
Tag = "T", Timestamp = Timestamp.FromDateTime(at ?? t), NumericValue = (t - from).Ticks / TimeSpan.TicksPerSecond,
|
||||
OpcQuality = 192, Quality = quality,
|
||||
};
|
||||
|
||||
public FakeHistorianGatewayClient Client() => new() { RawSource = Serve };
|
||||
|
||||
private IEnumerable<HistorianSample> Serve(DateTime start, DateTime end, int max)
|
||||
{
|
||||
if (max <= 0 || end < start || end > Now.AddDays(2)
|
||||
|| start.Ticks % TimeSpan.TicksPerMillisecond != 0 || end.Ticks % TimeSpan.TicksPerMillisecond != 0)
|
||||
throw new Grpc.Core.RpcException(new Grpc.Core.Status(Grpc.Core.StatusCode.InvalidArgument, "refused"));
|
||||
var rows = new List<HistorianSample>();
|
||||
if (FirstAtOrAfter(start) != start)
|
||||
rows.Add(LastBefore(start) is { } p
|
||||
? Row(p, 133, start)
|
||||
: new HistorianSample { Tag = "T", Timestamp = Timestamp.FromDateTime(start), OpcQuality = 0, Quality = 1 });
|
||||
for (var t = FirstAtOrAfter(start); t is { } u && u <= end && rows.Count < max; t = FirstAtOrAfter(u.AddTicks(1)))
|
||||
rows.Add(Row(u));
|
||||
RowsStreamed += rows.Count;
|
||||
return rows;
|
||||
}
|
||||
}
|
||||
|
||||
[Theory] // re-review: every time in a gap of a bursty tag costs a jump read + a bisection — the cap must allow that
|
||||
[InlineData(30)]
|
||||
[InlineData(100)]
|
||||
[InlineData(300)]
|
||||
public async Task A_bursty_tag_asked_at_noon_on_consecutive_days_is_served(int days)
|
||||
{
|
||||
var from = Now.Date.AddDays(-days - 1);
|
||||
var fake = new BurstStore(from, days + 1).Client();
|
||||
var times = Enumerable.Range(0, days).Select(d => from.AddDays(d).AddHours(12)).ToArray();
|
||||
var s = await AtTime(fake, times);
|
||||
for (var d = 0; d < days; d++)
|
||||
{
|
||||
double prev = d * 86400 + 3599, next = (d + 1) * 86400;
|
||||
((double)s[d].Value!).ShouldBe(prev + (next - prev) * (12 * 3600 - 3599.0) / (86400 - 3599.0), 1e-6);
|
||||
s[d].StatusCode.ShouldBe(GoodInterpolated);
|
||||
}
|
||||
fake.ReadRawCallCount.ShouldBeLessThanOrEqualTo(7 * days); // ≈ 6 a time; the cap is 64 + 16 a time
|
||||
}
|
||||
|
||||
[Fact] // L1: a small ceiling is served (paging), within the read cap
|
||||
public async Task A_small_ceiling_is_served_within_the_read_cap()
|
||||
{
|
||||
var fake = new SeriesStore(Now.AddYears(-1), Now, TimeSpan.FromSeconds(1)).Client(ceiling: 10);
|
||||
var times = Enumerable.Range(0, 50).Select(i => Now.AddDays(-300 + i * 5).AddSeconds(0.5)).ToArray();
|
||||
var s = await AtTime(fake, times);
|
||||
s.ShouldAllBe(x => x.StatusCode == GoodInterpolated);
|
||||
fake.ReadRawCallCount.ShouldBeLessThanOrEqualTo(GatewayAtTimeReader.BaseReads + GatewayAtTimeReader.ReadsPerTime * 50);
|
||||
}
|
||||
|
||||
[Fact] // L1: the request's read cap stops a walk with a clear error, never a hang (here a cap of 3 vs. a 5-read walk)
|
||||
public async Task The_read_cap_stops_a_request_with_a_clear_error()
|
||||
{
|
||||
var fake = new AvevaStore([(T0.AddDays(-400), 42, false), (T0.AddDays(-400).AddSeconds(10), 43, false)]).Client();
|
||||
var ex = await Should.ThrowAsync<HistoryReadLimitExceededException>(() => GatewayAtTimeReader.ReadAsync(
|
||||
fake, new GatewayRowCeiling(), "T", [T0], Now, Ct, maxReads: 3));
|
||||
ex.Message.ShouldContain("needed more than 3 raw reads");
|
||||
fake.ReadRawCallCount.ShouldBe(3);
|
||||
fake.RawStreamsCompleted.ShouldBe(3);
|
||||
}
|
||||
|
||||
[Fact] // the default cap scales with the request: 64 + 16 per distinct time (a runaway guard; a bursty tag needs ≈ 6)
|
||||
public async Task The_default_read_cap_is_64_plus_4_per_distinct_time()
|
||||
{
|
||||
(GatewayAtTimeReader.BaseReads, GatewayAtTimeReader.ReadsPerTime).ShouldBe((64, 16));
|
||||
await Task.CompletedTask;
|
||||
}
|
||||
|
||||
// ---- type hint edge cases (review L2/L3) ------------------------------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public async Task A_DateTime_node_is_stepped()
|
||||
{
|
||||
var fake = new AvevaStore([(At(0), 0, false), (At(10), 10, false)]).Client();
|
||||
var s = (await AtTimeTyped(fake, TypeCode.DateTime, At(5))).Single();
|
||||
s.Value.ShouldBe(0d);
|
||||
s.StatusCode.ShouldBe(GoodInterpolated);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(double.PositiveInfinity)]
|
||||
[InlineData(double.NaN)]
|
||||
[InlineData(1e30)]
|
||||
public async Task Under_an_integer_hint_a_non_integer_next_value_is_unusable_not_saturated(double next)
|
||||
{
|
||||
var fake = new AvevaStore([(At(0), 100, false), (At(10), next, false)]).Client();
|
||||
var s = (await AtTimeTyped(fake, TypeCode.Int32, At(5))).Single();
|
||||
s.Value.ShouldBeOfType<int>().ShouldBe(100); // held before an unusable neighbour
|
||||
s.StatusCode.ShouldBe(UncertainInterpolated);
|
||||
}
|
||||
|
||||
// ---- kill switch -----------------------------------------------------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public async Task The_kill_switch_restores_AVEVAs_own_answer()
|
||||
{
|
||||
var fake = new AvevaStore(Gate685()).Client();
|
||||
fake.AtTimeSamples = [new HistorianSample { NumericValue = 107, OpcQuality = 192, Timestamp = Timestamp.FromDateTime(At(3)) }];
|
||||
var ds = new GatewayHistorianDataSource(fake, NullLogger<GatewayHistorianDataSource>.Instance, atTimeFromRaw: false);
|
||||
var s = (await ds.ReadAtTimeAsync("T", [At(3)], Ct)).Samples.Single();
|
||||
s.Value.ShouldBe(107d);
|
||||
fake.ReadAtTimeCallCount.ShouldBe(1);
|
||||
fake.ReadRawCallCount.ShouldBe(0);
|
||||
}
|
||||
}
|
||||
+562
@@ -0,0 +1,562 @@
|
||||
using Google.Protobuf.WellKnownTypes;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Opc.Ua;
|
||||
using Opc.Ua.Server;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.HistorianGateway.Contracts.Grpc;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.OpcUaServer;
|
||||
using SdkHistoryReadResult = Opc.Ua.HistoryReadResult;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #694 item 1 (live gate F3): a strict gateway (<c>AllowStreamingBeyondCeiling=false</c>) with a LOWERED
|
||||
/// <c>MaxValuesCeiling</c> (1000) refused ordinary page sizes — <c>NumValuesPerNode = 5000</c>, or an uncapped read over
|
||||
/// 3,600 rows, came back <c>BadTooManyOperations</c>. The server now sizes the page to the learned ceiling
|
||||
/// (<see cref="IHistorianDataSource.MaxRawValuesPerRead"/> L = ceiling − 16): the read — page P + boundary ties k + the
|
||||
/// #692 lookahead row — is at most L, so a first page holds L − 1 values and a resumed one L − k − 1, each with a
|
||||
/// continuation point. Driven end to end: a REAL SDK server's node manager → the REAL
|
||||
/// <see cref="GatewayHistorianDataSource"/> → an AVEVA-shaped fake gateway client that refuses an over-ceiling
|
||||
/// <c>max_values</c> exactly as the real gateway does (<c>InvalidArgument</c> "max_values exceeds the server ceiling of N",
|
||||
/// wrapped by the client package) and rejects <c>max_values <= 0</c> as the client package does.
|
||||
/// </summary>
|
||||
public sealed class GatewayCeilingPagingTests : IDisposable
|
||||
{
|
||||
private static CancellationToken Ct => TestContext.Current.CancellationToken;
|
||||
|
||||
private static readonly DateTime T0 = new(2026, 10, 12, 10, 0, 0, DateTimeKind.Utc);
|
||||
|
||||
private const int Ceiling = 1000;
|
||||
private const int L = Ceiling - RawAggregateReader.PreWindowAllowance; // 984
|
||||
private const int FirstPage = L - 1; // 983: page + lookahead fills a read
|
||||
private const int NextPage = L - 2; // 982: + the one boundary tie a resumed page re-reads (distinct timestamps)
|
||||
|
||||
private readonly string _pkiRoot = Path.Combine(Path.GetTempPath(), $"otopcua-gwceiling-{Guid.NewGuid():N}");
|
||||
|
||||
private static DateTime At(int seconds) => T0.AddSeconds(seconds);
|
||||
|
||||
private static HistorianSample Row(DateTime t, double v, uint quality = 0) =>
|
||||
new() { Tag = "T", Timestamp = Timestamp.FromDateTime(t), NumericValue = v, OpcQuality = 192, Quality = quality };
|
||||
|
||||
/// <summary>
|
||||
/// AVEVA-shaped ReadRaw over <paramref name="stored"/>: a boundary row at the query start (the value in effect, or a
|
||||
/// NULL row before the first sample), then <c>[start, end]</c> inclusive ascending, capped at <c>max_values</c>.
|
||||
/// </summary>
|
||||
private static FakeHistorianGatewayClient Gateway(IReadOnlyList<(DateTime T, double V)> stored, int? ceiling) => new()
|
||||
{
|
||||
MaxValuesCeiling = ceiling,
|
||||
RawSource = (start, end, max) =>
|
||||
{
|
||||
if (max <= 0) throw new ArgumentOutOfRangeException(nameof(max), "max_values must be positive"); // the client package
|
||||
var rows = new List<HistorianSample>();
|
||||
if (stored.All(s => s.T != start))
|
||||
{
|
||||
var prior = stored.LastOrDefault(s => s.T < start);
|
||||
rows.Add(prior.T == default
|
||||
? new HistorianSample { Tag = "T", Timestamp = Timestamp.FromDateTime(start), NumericValue = 0, OpcQuality = 0, Quality = 1 }
|
||||
: Row(start, prior.V, 133));
|
||||
}
|
||||
rows.AddRange(stored.Where(s => s.T >= start && s.T <= end).Select(s => Row(s.T, s.V)));
|
||||
return rows.Take(max);
|
||||
},
|
||||
};
|
||||
|
||||
/// <summary>One sample per second, value = seconds, at +0 … +(count − 1).</summary>
|
||||
private static List<(DateTime T, double V)> PerSecond(int count) =>
|
||||
Enumerable.Range(0, count).Select(i => (At(i), (double)i)).ToList();
|
||||
|
||||
private static int Refusals(FakeHistorianGatewayClient fake) =>
|
||||
fake.RawCalls.Count(c => fake.MaxValuesCeiling is { } n && c.MaxValues > n);
|
||||
|
||||
private static void EveryRequestFits(FakeHistorianGatewayClient fake) =>
|
||||
fake.RawCalls.ShouldAllBe(c => c.MaxValues > 0 && c.MaxValues <= Ceiling); // never uncapped, never over the ceiling
|
||||
|
||||
// ---- the data source's page limit ------------------------------------------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public async Task The_page_limit_is_unknown_until_a_refusal_names_the_ceiling_and_forgotten_when_the_gateway_streams()
|
||||
{
|
||||
var fake = Gateway(PerSecond(3600), Ceiling);
|
||||
var ds = new GatewayHistorianDataSource(fake, NullLogger<GatewayHistorianDataSource>.Instance);
|
||||
ds.MaxRawValuesPerRead.ShouldBe(0);
|
||||
|
||||
await ds.ReadRawAsync("T", At(0), At(10), 0, Ct);
|
||||
ds.MaxRawValuesPerRead.ShouldBe(L); // a read of this size asks for exactly the ceiling
|
||||
|
||||
(await ds.ReadRawAsync("T", At(0), At(3600), (uint)L, Ct)).Samples.Count.ShouldBe(L);
|
||||
fake.RawCalls[^1].MaxValues.ShouldBe(Ceiling);
|
||||
Refusals(fake).ShouldBe(1);
|
||||
|
||||
// The operator enabled AllowStreamingBeyondCeiling: a ceiling-full uncapped read re-probes (#691.3) and forgets it.
|
||||
fake.MaxValuesCeiling = null;
|
||||
(await ds.ReadRawAsync("T", At(0), At(3600), 0, Ct)).Samples.Count.ShouldBe(3600);
|
||||
ds.MaxRawValuesPerRead.ShouldBe(0);
|
||||
}
|
||||
|
||||
// ---- through the node manager ----------------------------------------------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public async Task The_cold_first_read_costs_one_refusal_and_its_page_fits_the_learned_ceiling()
|
||||
{
|
||||
var fake = Gateway(PerSecond(3600), Ceiling);
|
||||
var (host, nm, nodeId) = await BootAsync(fake);
|
||||
try
|
||||
{
|
||||
var first = ReadRaw(nm, nodeId, At(0), At(3600), 5000, null);
|
||||
first.Error.StatusCode.Code.ShouldBe(StatusCodes.Good);
|
||||
first.Values.Count.ShouldBe(FirstPage);
|
||||
first.Cp.ShouldNotBeNull();
|
||||
// The budget the client asked for (+ lookahead + allowance, refused naming 1000), the ceiling re-read that cannot
|
||||
// hold the window, then the page sized to the learned ceiling.
|
||||
fake.RawCalls.Select(c => c.MaxValues).ShouldBe([5000 + 1 + RawAggregateReader.PreWindowAllowance, Ceiling, Ceiling]);
|
||||
|
||||
var (rest, _, status) = DrainFrom(nm, nodeId, At(0), At(3600), 5000, first.Cp);
|
||||
status.Code.ShouldBe(StatusCodes.Good);
|
||||
first.Values.Concat(rest).ShouldBe(PerSecond(3600).Select(s => s.Item2));
|
||||
Refusals(fake).ShouldBe(1);
|
||||
}
|
||||
finally { await host.DisposeAsync(); }
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(5000u, false)]
|
||||
[InlineData(0u, false)]
|
||||
[InlineData(5000u, true)]
|
||||
[InlineData(0u, true)]
|
||||
public async Task With_the_ceiling_known_every_page_is_exactly_the_ceiling_page_and_nothing_is_refused(uint n, bool reverse)
|
||||
{
|
||||
var stored = PerSecond(3600);
|
||||
var fake = Gateway(stored, Ceiling);
|
||||
var (host, nm, nodeId) = await BootAsync(fake);
|
||||
try
|
||||
{
|
||||
ReadRaw(nm, nodeId, At(0), At(3600), 5000, null); // learns the ceiling
|
||||
fake.RawCalls.Clear();
|
||||
|
||||
var (values, pages, status) = reverse
|
||||
? DrainFrom(nm, nodeId, At(3600), At(-1), n, null)
|
||||
: DrainFrom(nm, nodeId, At(0), At(3600), n, null);
|
||||
|
||||
status.Code.ShouldBe(StatusCodes.Good);
|
||||
var expected = stored.Select(s => s.Item2);
|
||||
values.ShouldBe(reverse ? expected.Reverse() : expected);
|
||||
pages[0].ShouldBe(FirstPage);
|
||||
pages.Skip(1).SkipLast(1).ShouldAllBe(p => p == NextPage);
|
||||
pages[^1].ShouldBeGreaterThan(0); // no empty trailing page
|
||||
Refusals(fake).ShouldBe(0);
|
||||
EveryRequestFits(fake);
|
||||
if (!reverse) fake.RawCalls.Count.ShouldBe(pages.Count); // one read per page
|
||||
}
|
||||
finally { await host.DisposeAsync(); }
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task An_uncapped_read_that_fits_the_ceiling_is_still_one_page_without_a_continuation_point()
|
||||
{
|
||||
var fake = Gateway(PerSecond(3600), Ceiling);
|
||||
var (host, nm, nodeId) = await BootAsync(fake);
|
||||
try
|
||||
{
|
||||
var (values, pages, _) = DrainFrom(nm, nodeId, At(0), At(10), 0, null);
|
||||
values.Count.ShouldBe(10);
|
||||
pages.Count.ShouldBe(1);
|
||||
fake.RawCalls.Select(c => c.MaxValues).ShouldBe([int.MaxValue, Ceiling]); // #691.3 unchanged
|
||||
|
||||
var again = DrainFrom(nm, nodeId, At(0), At(10), 0, null);
|
||||
again.Pages.Count.ShouldBe(1);
|
||||
fake.RawCalls[^1].MaxValues.ShouldBe(Ceiling); // the learned page size, one read
|
||||
}
|
||||
finally { await host.DisposeAsync(); }
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_default_gateway_is_unchanged_one_read_and_no_continuation_point()
|
||||
{
|
||||
var fake = Gateway(PerSecond(3600), ceiling: null);
|
||||
var (host, nm, nodeId) = await BootAsync(fake);
|
||||
try
|
||||
{
|
||||
var (values, pages, _) = DrainFrom(nm, nodeId, At(0), At(3600), 5000, null);
|
||||
values.Count.ShouldBe(3600);
|
||||
pages.Count.ShouldBe(1);
|
||||
fake.RawCalls.Select(c => c.MaxValues).ShouldBe([5000 + 1 + RawAggregateReader.PreWindowAllowance]);
|
||||
}
|
||||
finally { await host.DisposeAsync(); }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A tie cluster straddling a page boundary, at the sizes where the ceiling arithmetic turns: a read can carry L rows;
|
||||
/// the over-fetch that pages within a cluster reads at most L − 1 (+1 to detect more). Every outcome is exact, or
|
||||
/// <c>BadTooManyOperations</c> after an exact prefix — never an uncapped or over-ceiling request, never an endless read.
|
||||
/// </summary>
|
||||
[Theory]
|
||||
[InlineData(L - 2, 5000u)]
|
||||
[InlineData(L - 1, 5000u)]
|
||||
[InlineData(L, 5000u)]
|
||||
[InlineData(L + 1, 5000u)]
|
||||
[InlineData(Ceiling - 17, 5000u)]
|
||||
[InlineData(Ceiling - 16, 5000u)]
|
||||
[InlineData(Ceiling + 1, 5000u)]
|
||||
[InlineData(L - 2, 0u)]
|
||||
[InlineData(L - 1, 0u)]
|
||||
[InlineData(L + 1, 0u)]
|
||||
[InlineData(Ceiling + 1, 0u)]
|
||||
public async Task A_tie_cluster_at_a_page_boundary_is_exact_or_refused_never_truncated_or_endless(int size, uint n)
|
||||
{
|
||||
var stored = PerSecond(500)
|
||||
.Concat(Enumerable.Range(0, size).Select(i => (At(600), 10_000.0 + i)))
|
||||
.Concat(Enumerable.Range(0, 1500).Select(i => (At(700 + i), 20_000.0 + i)))
|
||||
.ToList();
|
||||
var fake = Gateway(stored, Ceiling);
|
||||
var (host, nm, nodeId) = await BootAsync(fake);
|
||||
try
|
||||
{
|
||||
ReadRaw(nm, nodeId, At(0), At(5), 5000, null); // learns the ceiling
|
||||
fake.RawCalls.Clear();
|
||||
|
||||
var (values, pages, status) = DrainFrom(nm, nodeId, At(0), At(5000), n, null, maxPages: 60);
|
||||
var expected = stored.Select(s => s.Item2).ToList();
|
||||
if (status.Code == StatusCodes.Good) values.ShouldBe(expected);
|
||||
else
|
||||
{
|
||||
status.Code.ShouldBe(StatusCodes.BadTooManyOperations);
|
||||
values.ShouldBe(expected.Take(values.Count)); // an exact prefix
|
||||
size.ShouldBeGreaterThan(L - 2, "a cluster that fits the over-fetch is always served");
|
||||
}
|
||||
pages.Count.ShouldBeLessThan(60);
|
||||
EveryRequestFits(fake);
|
||||
}
|
||||
finally { await host.DisposeAsync(); }
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task An_uncapped_cursor_inside_a_tie_cluster_finishes_exactly_when_the_gateway_stops_being_strict()
|
||||
{
|
||||
// 100 samples, then 983 ties: the first page ends inside the cluster. Then the operator enables streaming and the
|
||||
// data source forgets the ceiling (L back to 0) — the uncapped cursor must still finish exactly, not loop.
|
||||
var stored = PerSecond(100)
|
||||
.Concat(Enumerable.Range(0, L - 1).Select(i => (At(200), 10_000.0 + i)))
|
||||
.Concat(Enumerable.Range(0, 300).Select(i => (At(300 + i), 20_000.0 + i)))
|
||||
.ToList();
|
||||
var fake = Gateway(stored, Ceiling);
|
||||
var ds = new GatewayHistorianDataSource(fake, NullLogger<GatewayHistorianDataSource>.Instance);
|
||||
var (host, nm, nodeId) = await BootAsync(fake, ds);
|
||||
try
|
||||
{
|
||||
ReadRaw(nm, nodeId, At(0), At(5), 5000, null); // learns the ceiling
|
||||
var first = ReadRaw(nm, nodeId, At(0), At(5000), 0, null);
|
||||
first.Values.Count.ShouldBe(FirstPage);
|
||||
first.Cp.ShouldNotBeNull();
|
||||
|
||||
fake.MaxValuesCeiling = null;
|
||||
await ds.ReadRawAsync("T", At(0), At(5000), 0, Ct); // a ceiling-full read re-probes and forgets
|
||||
ds.MaxRawValuesPerRead.ShouldBe(0);
|
||||
|
||||
var (rest, pages, status) = DrainFrom(nm, nodeId, At(0), At(5000), 0, first.Cp, maxPages: 20);
|
||||
status.Code.ShouldBe(StatusCodes.Good);
|
||||
first.Values.Concat(rest).ShouldBe(stored.Select(s => s.Item2));
|
||||
pages.Count.ShouldBeLessThan(20);
|
||||
}
|
||||
finally { await host.DisposeAsync(); }
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_ceiling_lowered_mid_page_through_is_re_learned_and_the_read_stays_exact()
|
||||
{
|
||||
var stored = PerSecond(3600);
|
||||
var fake = Gateway(stored, Ceiling);
|
||||
var (host, nm, nodeId) = await BootAsync(fake);
|
||||
try
|
||||
{
|
||||
var got = new List<double>();
|
||||
var p1 = ReadRaw(nm, nodeId, At(0), At(3600), 5000, null);
|
||||
got.AddRange(p1.Values);
|
||||
var p2 = ReadRaw(nm, nodeId, At(0), At(3600), 5000, p1.Cp);
|
||||
got.AddRange(p2.Values);
|
||||
|
||||
fake.MaxValuesCeiling = 500; // the operator lowered it
|
||||
fake.RawCalls.Clear();
|
||||
var (rest, pages, status) = DrainFrom(nm, nodeId, At(0), At(3600), 5000, p2.Cp, maxPages: 20);
|
||||
status.Code.ShouldBe(StatusCodes.Good);
|
||||
got.AddRange(rest);
|
||||
got.ShouldBe(stored.Select(s => s.Item2));
|
||||
pages.ShouldAllBe(p => p <= 500 - RawAggregateReader.PreWindowAllowance);
|
||||
Refusals(fake).ShouldBe(1); // the one that names the new ceiling
|
||||
}
|
||||
finally { await host.DisposeAsync(); }
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Bounding_values_page_within_the_ceiling()
|
||||
{
|
||||
var stored = PerSecond(3600);
|
||||
var fake = Gateway(stored, Ceiling);
|
||||
var (host, nm, nodeId) = await BootAsync(fake);
|
||||
try
|
||||
{
|
||||
ReadRaw(nm, nodeId, At(0), At(5), 5000, null); // learns the ceiling
|
||||
fake.RawCalls.Clear();
|
||||
|
||||
// Start bound = the sample at +10 (before StartTime), end bound = the sample at +3001 (after EndTime).
|
||||
var start = At(10).AddMilliseconds(500);
|
||||
var end = At(3000).AddMilliseconds(500);
|
||||
var (values, pages, status) = DrainFrom(nm, nodeId, start, end, 5000, null, returnBounds: true);
|
||||
status.Code.ShouldBe(StatusCodes.Good);
|
||||
values.ShouldBe(Enumerable.Range(10, 2992).Select(i => (double)i));
|
||||
pages.ShouldAllBe(p => p <= L);
|
||||
Refusals(fake).ShouldBe(0);
|
||||
}
|
||||
finally { await host.DisposeAsync(); }
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(1000, 100_000u, 10)]
|
||||
[InlineData(1000, 5000u, 10)]
|
||||
[InlineData(20, 3000u, 40)]
|
||||
public async Task A_cold_end_only_read_with_a_huge_page_shrinks_the_page_not_the_walk(int ceiling, uint n, int maxReads)
|
||||
{
|
||||
// Review F2: the walk used to re-run itself at the learned ceiling while keeping the page at N — a cold end-only read
|
||||
// with ceiling 20 and N = 3000 cost 3,023 gateway reads in ONE HistoryRead (ceiling 1000, N = 100,000: 207). The
|
||||
// refusal now reaches the page planner, which shrinks the page to the ceiling.
|
||||
var dense = Enumerable.Range(0, 200_000).Select(i => (T0.AddMilliseconds(i), (double)i)).ToList();
|
||||
var fake = Gateway(dense, ceiling);
|
||||
var (host, nm, nodeId) = await BootAsync(fake);
|
||||
try
|
||||
{
|
||||
var r = ReadRaw(nm, nodeId, DateTime.MinValue, T0.AddMilliseconds(200_000), n, null);
|
||||
r.Error.StatusCode.Code.ShouldBe(StatusCodes.Good);
|
||||
r.Cp.ShouldNotBeNull();
|
||||
var page = ceiling - RawAggregateReader.PreWindowAllowance - 1;
|
||||
r.Values.ShouldBe(Enumerable.Range(200_000 - r.Values.Count, r.Values.Count).Reverse().Select(i => (double)i));
|
||||
r.Values.Count.ShouldBeLessThanOrEqualTo(page);
|
||||
fake.RawCalls.Count.ShouldBeLessThanOrEqualTo(maxReads);
|
||||
}
|
||||
finally { await host.DisposeAsync(); }
|
||||
}
|
||||
|
||||
// ---- F1: a window starting at/before the epoch gets no look-back (review of #694) -------------------------------------
|
||||
|
||||
private static List<(DateTime T, double V)> FromEpoch(int count) =>
|
||||
Enumerable.Range(1, count).Select(i => (DateTime.UnixEpoch.AddSeconds(i), (double)i)).ToList();
|
||||
|
||||
[Fact]
|
||||
public async Task A_ceiling_truncated_read_from_the_epoch_is_never_reported_complete()
|
||||
{
|
||||
// AVEVA's synthetic row lands AT the epoch start (no 1 ms look-back there). It used to count as "past the window
|
||||
// end", so a ceiling-truncated uncapped read answered its first 999 rows, Good, as if that were everything.
|
||||
var fake = Gateway(FromEpoch(3000), Ceiling);
|
||||
var ds = new GatewayHistorianDataSource(fake, NullLogger<GatewayHistorianDataSource>.Instance);
|
||||
await Should.ThrowAsync<HistoryReadLimitExceededException>(() =>
|
||||
ds.ReadRawAsync("T", DateTime.UnixEpoch, DateTime.UnixEpoch.AddDays(1), 0, Ct));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task An_uncapped_read_from_the_epoch_pages_every_sample()
|
||||
{
|
||||
var stored = FromEpoch(3000);
|
||||
var fake = Gateway(stored, Ceiling);
|
||||
var (host, nm, nodeId) = await BootAsync(fake);
|
||||
try
|
||||
{
|
||||
var (values, pages, status) = DrainFrom(nm, nodeId, DateTime.UnixEpoch, DateTime.UnixEpoch.AddDays(1), 0, null);
|
||||
status.Code.ShouldBe(StatusCodes.Good);
|
||||
values.ShouldBe(stored.Select(s => s.Item2));
|
||||
pages.Count.ShouldBeGreaterThan(1);
|
||||
}
|
||||
finally { await host.DisposeAsync(); }
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task An_end_only_read_far_past_epoch_data_returns_the_latest_values()
|
||||
{
|
||||
var stored = FromEpoch(3000);
|
||||
var fake = Gateway(stored, Ceiling);
|
||||
var (host, nm, nodeId) = await BootAsync(fake);
|
||||
try
|
||||
{
|
||||
ReadRaw(nm, nodeId, DateTime.UnixEpoch, DateTime.UnixEpoch.AddSeconds(5), 5000, null); // learns the ceiling
|
||||
var r = ReadRaw(nm, nodeId, DateTime.MinValue, DateTime.UnixEpoch.AddDays(30), 10, null);
|
||||
r.Error.StatusCode.Code.ShouldBe(StatusCodes.Good);
|
||||
r.Values.ShouldBe(Enumerable.Range(2991, 10).Reverse().Select(i => (double)i)); // the LATEST ten, newest first
|
||||
}
|
||||
finally { await host.DisposeAsync(); }
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_stopped_dense_tag_under_a_strict_ceiling_is_found_with_bounded_discovery_cost()
|
||||
{
|
||||
var stored = PerSecond(7200);
|
||||
var fake = Gateway(stored, Ceiling);
|
||||
var ds = new GatewayHistorianDataSource(fake, NullLogger<GatewayHistorianDataSource>.Instance);
|
||||
await ds.ReadRawAsync("T", At(0), At(1), 5000, Ct);
|
||||
fake.RawCalls.Clear();
|
||||
var page = await ReverseRawReader.ReadAsync(ds, "T", At(7200).AddDays(1), DateTime.UnixEpoch,
|
||||
3, 65536, At(7200).AddDays(1), 0, Ct);
|
||||
page.Samples.Select(s => s.Value).ShouldBe(new object?[] { 7199d, 7198d, 7197d });
|
||||
fake.RawCalls.Count.ShouldBeLessThanOrEqualTo(12);
|
||||
EveryRequestFits(fake);
|
||||
}
|
||||
|
||||
// ---- F3: rows crowding the look-back before a page start ---------------------------------------------------------------
|
||||
|
||||
[Theory]
|
||||
[InlineData(5000u)]
|
||||
[InlineData(0u)]
|
||||
[InlineData((uint)L)]
|
||||
[InlineData(900u)]
|
||||
public async Task Ties_just_before_a_page_start_are_read_around_not_refused(uint n)
|
||||
{
|
||||
// 962 samples, 20 ties at X, one sample at X + 1 ms: the first page (983) ends exactly at X + 1 ms. The next page's
|
||||
// 1 ms look-back holds the 20 ties — more than the 16-row allowance, and a ceiling-full page leaves no room for the
|
||||
// #687 re-read, so it used to fail BadTooManyOperations. The source now names the crowding rows and the page is
|
||||
// re-read that much smaller.
|
||||
var x = At(1000);
|
||||
var stored = Enumerable.Range(0, 962).Select(i => (At(i), (double)i))
|
||||
.Concat(Enumerable.Range(0, 20).Select(i => (x, 962.0 + i)))
|
||||
.Append((x.AddMilliseconds(1), 982.0))
|
||||
.Concat(Enumerable.Range(0, 3000).Select(i => (x.AddSeconds(1 + i), 983.0 + i)))
|
||||
.ToList();
|
||||
var fake = Gateway(stored, Ceiling);
|
||||
var (host, nm, nodeId) = await BootAsync(fake);
|
||||
try
|
||||
{
|
||||
ReadRaw(nm, nodeId, At(0), At(5), 5000, null); // learns the ceiling
|
||||
fake.RawCalls.Clear();
|
||||
|
||||
var (values, _, status) = DrainFrom(nm, nodeId, At(0), x.AddDays(1), n, null);
|
||||
status.Code.ShouldBe(StatusCodes.Good);
|
||||
values.ShouldBe(stored.Select(s => s.Item2));
|
||||
Refusals(fake).ShouldBe(0);
|
||||
EveryRequestFits(fake);
|
||||
}
|
||||
finally { await host.DisposeAsync(); }
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(1709, 1366)] // #695 reverse-walk trace
|
||||
[InlineData(1369, 1292)] // #695 bounds trace
|
||||
public async Task A_warm_read_uses_the_last_room_below_the_ceiling_for_pre_window_ties(int ceiling, int cluster)
|
||||
{
|
||||
var start = At(1000);
|
||||
var stored = Enumerable.Range(0, cluster).Select(i => (start.AddMilliseconds(-0.5), (double)i))
|
||||
.Append((start, 9000d)).Append((start.AddSeconds(1), 9001d)).ToList();
|
||||
var fake = Gateway(stored, ceiling);
|
||||
using var ds = new GatewayHistorianDataSource(fake, NullLogger<GatewayHistorianDataSource>.Instance);
|
||||
await ds.ReadRawAsync("T", At(0), At(1), 5000, Ct); // learn without crowding
|
||||
fake.RawCalls.Clear();
|
||||
var result = await ds.ReadRawAsync("T", start, start.AddSeconds(2), 1, Ct);
|
||||
result.Samples.Select(s => s.Value).ShouldBe([9000d]);
|
||||
fake.RawCalls.ShouldAllBe(c => c.MaxValues <= ceiling);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_warm_reverse_segment_can_shrink_again_when_the_pre_window_cluster_grows()
|
||||
{
|
||||
var start = At(1000);
|
||||
var stored = Enumerable.Range(0, 1366).Select(i => (start.AddMilliseconds(-0.5), (double)i))
|
||||
.Concat(Enumerable.Range(0, 500).Select(i => (start.AddMilliseconds(i), 9000d + i))).ToList();
|
||||
var fake = Gateway(stored, 1709);
|
||||
using var ds = new GatewayHistorianDataSource(fake, NullLogger<GatewayHistorianDataSource>.Instance);
|
||||
await ds.ReadRawAsync("T", At(0), At(1), 5000, Ct);
|
||||
var page = await ReverseRawReader.ReadAsync(ds, "T", start.AddMilliseconds(499), start.AddTicks(-1),
|
||||
249, 65536, At(2000), new ReverseWalkHint(499 * TimeSpan.TicksPerMillisecond), Ct);
|
||||
page.Samples.Select(s => (double)s.Value!).ShouldBe(Enumerable.Range(9251, 249).Reverse().Select(i => (double)i));
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(false)]
|
||||
[InlineData(true)]
|
||||
public async Task Gateway_transport_faults_surface_as_BadNoCommunication(bool deadline)
|
||||
{
|
||||
var fake = new FakeHistorianGatewayClient { ThrowOnRead = deadline
|
||||
? new ZB.MOM.WW.HistorianGateway.Client.HistorianGatewayException("deadline", new Grpc.Core.RpcException(new Grpc.Core.Status(Grpc.Core.StatusCode.DeadlineExceeded, "deadline")))
|
||||
: new ZB.MOM.WW.HistorianGateway.Client.HistorianGatewayUnavailableException("unavailable") };
|
||||
var (host, nm, nodeId) = await BootAsync(fake);
|
||||
try
|
||||
{
|
||||
var r = ReadRaw(nm, nodeId, At(0), At(10), 10, null);
|
||||
r.Error.StatusCode.Code.ShouldBe(StatusCodes.BadNoCommunication);
|
||||
}
|
||||
finally { await host.DisposeAsync(); }
|
||||
}
|
||||
|
||||
// ---- helpers ---------------------------------------------------------------------------------------------------------
|
||||
|
||||
private async Task<(OpcUaApplicationHost Host, OtOpcUaNodeManager Nm, NodeId NodeId)> BootAsync(
|
||||
FakeHistorianGatewayClient fake, GatewayHistorianDataSource? ds = null)
|
||||
{
|
||||
var started = await OpcUaHostStart.StartAsync(
|
||||
port => new OpcUaApplicationHost(
|
||||
new OpcUaApplicationHostOptions
|
||||
{
|
||||
ApplicationName = "OtOpcUa.GatewayCeilingPagingTest",
|
||||
ApplicationUri = $"urn:OtOpcUa.GatewayCeilingPagingTest:{Guid.NewGuid():N}",
|
||||
OpcUaPort = port,
|
||||
PublicHostname = "localhost",
|
||||
PkiStoreRoot = _pkiRoot,
|
||||
},
|
||||
NullLogger<OpcUaApplicationHost>.Instance),
|
||||
() => new OtOpcUaSdkServer(), Ct);
|
||||
var nm = started.Server.NodeManager!;
|
||||
nm.HistoryContinuationStore = new InMemoryHistoryContinuationStore();
|
||||
nm.HistorianDataSource = ds ?? new GatewayHistorianDataSource(fake, NullLogger<GatewayHistorianDataSource>.Instance);
|
||||
nm.EnsureVariable("eq-1/pv", parentFolderNodeId: null, displayName: "PV", dataType: "Double",
|
||||
writable: false, historianTagname: "T", realm: AddressSpaceRealm.Uns);
|
||||
return (started.Host, nm, nm.TryGetVariable("eq-1/pv")!.NodeId);
|
||||
}
|
||||
|
||||
/// <summary>Pages until no continuation point or a Bad status; the page sizes and the final status.</summary>
|
||||
private static (List<double> Values, List<int> Pages, StatusCode Status) DrainFrom(
|
||||
OtOpcUaNodeManager nm, NodeId nodeId, DateTime start, DateTime end, uint max, byte[]? cp, int maxPages = 20,
|
||||
bool returnBounds = false)
|
||||
{
|
||||
var values = new List<double>();
|
||||
var pages = new List<int>();
|
||||
ServiceResult error;
|
||||
do
|
||||
{
|
||||
(var page, error, cp) = ReadRaw(nm, nodeId, start, end, max, cp, returnBounds);
|
||||
values.AddRange(page);
|
||||
pages.Add(page.Count);
|
||||
pages.Count.ShouldBeLessThanOrEqualTo(maxPages, "paging must terminate");
|
||||
}
|
||||
while (cp is not null && StatusCode.IsGood(error.StatusCode));
|
||||
return (values, pages, error.StatusCode);
|
||||
}
|
||||
|
||||
private static (IReadOnlyList<double> Values, ServiceResult Error, byte[]? Cp) ReadRaw(
|
||||
OtOpcUaNodeManager nm, NodeId nodeId, DateTime start, DateTime end, uint max, byte[]? inboundCp, bool returnBounds = false)
|
||||
{
|
||||
var context = new OperationContext(new RequestHeader(), secureChannelContext: null, RequestType.HistoryRead, identity: null);
|
||||
var details = new ReadRawModifiedDetails
|
||||
{
|
||||
ReturnBounds = returnBounds,
|
||||
IsReadModified = false,
|
||||
StartTime = start,
|
||||
EndTime = end,
|
||||
NumValuesPerNode = max,
|
||||
};
|
||||
var nodesToRead = new List<HistoryReadValueId> { new() { NodeId = nodeId, ContinuationPoint = inboundCp } };
|
||||
var results = new List<SdkHistoryReadResult> { null! };
|
||||
var errors = new List<ServiceResult> { null! };
|
||||
nm.HistoryRead(context, details, TimestampsToReturn.Both, releaseContinuationPoints: false, nodesToRead, results, errors);
|
||||
|
||||
var values = new List<double>();
|
||||
if (results[0]?.HistoryData is { } ho && !ExtensionObject.IsNull(ho))
|
||||
foreach (var dv in ((HistoryData)ExtensionObject.ToEncodeable(ho)).DataValues)
|
||||
values.Add(Convert.ToDouble(dv.Value));
|
||||
return (values, errors[0], results[0]?.ContinuationPoint);
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
if (Directory.Exists(_pkiRoot))
|
||||
{
|
||||
try { Directory.Delete(_pkiRoot, recursive: true); }
|
||||
catch { /* best-effort */ }
|
||||
}
|
||||
}
|
||||
}
|
||||
+2
-2
@@ -106,7 +106,7 @@ public sealed class GatewayHistorianDataSourceTests
|
||||
var t0 = new DateTime(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc);
|
||||
var t1 = t0.AddSeconds(1);
|
||||
fake.AtTimeSamples = new[] { new HistorianSample { NumericValue = 5.0, OpcQuality = 192, Timestamp = Timestamp.FromDateTime(t0) } };
|
||||
var ds = new GatewayHistorianDataSource(fake, NullLogger<GatewayHistorianDataSource>.Instance);
|
||||
var ds = new GatewayHistorianDataSource(fake, NullLogger<GatewayHistorianDataSource>.Instance, atTimeFromRaw: false); // #693 kill switch
|
||||
var r = await ds.ReadAtTimeAsync("T", new[] { t0, t1 }, TestContext.Current.CancellationToken);
|
||||
Assert.Equal(2, r.Samples.Count); // exactly one per requested ts, in order
|
||||
Assert.Equal(5.0, r.Samples[0].Value);
|
||||
@@ -122,7 +122,7 @@ public sealed class GatewayHistorianDataSourceTests
|
||||
{
|
||||
new HistorianSample { NumericValue = 0, OpcQuality = 0, Quality = 1, Timestamp = Timestamp.FromDateTime(t0) },
|
||||
};
|
||||
var ds = new GatewayHistorianDataSource(fake, NullLogger<GatewayHistorianDataSource>.Instance);
|
||||
var ds = new GatewayHistorianDataSource(fake, NullLogger<GatewayHistorianDataSource>.Instance, atTimeFromRaw: false); // #693 kill switch
|
||||
var r = await ds.ReadAtTimeAsync("T", new[] { t0 }, TestContext.Current.CancellationToken);
|
||||
Assert.Null(r.Samples[0].Value);
|
||||
Assert.Equal(0x80000000u, r.Samples[0].StatusCode);
|
||||
|
||||
+126
@@ -0,0 +1,126 @@
|
||||
using Google.Protobuf.WellKnownTypes;
|
||||
using Grpc.Core;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.HistorianGateway.Contracts.Grpc;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests;
|
||||
|
||||
public sealed class GatewayMissingTagTests
|
||||
{
|
||||
private static CancellationToken Ct => TestContext.Current.CancellationToken;
|
||||
private static readonly DateTime Now = new(2026, 10, 13, 8, 0, 0, DateTimeKind.Utc);
|
||||
private sealed class Clock : TimeProvider
|
||||
{
|
||||
public DateTime Current = Now;
|
||||
public override DateTimeOffset GetUtcNow() => new(Current);
|
||||
}
|
||||
private static GatewayHistorianDataSource Source(FakeHistorianGatewayClient fake, Clock? clock = null, bool native = false) =>
|
||||
new(fake, NullLogger<GatewayHistorianDataSource>.Instance, timeProvider: clock, atTimeFromRaw: !native);
|
||||
|
||||
[Theory]
|
||||
[InlineData("Raw")]
|
||||
[InlineData("AtTime")]
|
||||
[InlineData("NativeAtTime")]
|
||||
[InlineData("Count")]
|
||||
[InlineData("TimeAverage")]
|
||||
public async Task A_missing_tag_returns_no_data_without_a_history_RPC(string kind)
|
||||
{
|
||||
var fake = new FakeHistorianGatewayClient { TagExistsResult = false };
|
||||
var ds = Source(fake, native: kind == "NativeAtTime");
|
||||
var result = kind switch
|
||||
{
|
||||
"Raw" => await ds.ReadRawAsync("Missing", Now.AddMinutes(-1), Now, 10, Ct),
|
||||
"AtTime" or "NativeAtTime" => await ds.ReadAtTimeAsync("Missing", [Now], Ct),
|
||||
_ => await ds.ReadProcessedAsync("Missing", Now.AddMinutes(-1), Now, TimeSpan.FromMinutes(1),
|
||||
kind == "Count" ? HistoryAggregateType.Count : HistoryAggregateType.TimeAverage, Ct),
|
||||
};
|
||||
if (kind == "Raw") result.Samples.ShouldBeEmpty();
|
||||
else
|
||||
{
|
||||
result.Samples.Count.ShouldBe(1);
|
||||
result.Samples[0].Value.ShouldBeNull();
|
||||
(result.Samples[0].StatusCode & 0xFFFF0000u).ShouldBe(0x809B0000u);
|
||||
result.Samples[0].SourceTimestampUtc.ShouldBe(kind.Contains("AtTime") ? Now : Now.AddMinutes(-1));
|
||||
}
|
||||
fake.TagExistsCallCount.ShouldBe(1);
|
||||
(fake.ReadRawCallCount + fake.ReadAggregateCallCount + fake.ReadAtTimeCallCount).ShouldBe(0);
|
||||
ds.GetHealthSnapshot().TotalFailures.ShouldBe(0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Negative_cache_expires_so_a_newly_provisioned_tag_can_be_read()
|
||||
{
|
||||
var fake = new FakeHistorianGatewayClient { TagExistsResult = false };
|
||||
var clock = new Clock(); var ds = Source(fake, clock);
|
||||
await ds.ReadRawAsync("T", Now.AddMinutes(-1), Now, 10, Ct);
|
||||
fake.TagExistsResult = true;
|
||||
fake.RawSamples = [new HistorianSample { Timestamp = Timestamp.FromDateTime(Now.AddSeconds(-1)), NumericValue = 42, OpcQuality = 192 }];
|
||||
(await ds.ReadRawAsync("T", Now.AddMinutes(-1), Now, 10, Ct)).Samples.ShouldBeEmpty();
|
||||
fake.TagExistsCallCount.ShouldBe(1);
|
||||
clock.Current = Now.AddSeconds(5);
|
||||
(await ds.ReadRawAsync("T", Now.AddMinutes(-1), Now, 10, Ct)).Samples.Single().Value.ShouldBe(42d);
|
||||
fake.TagExistsCallCount.ShouldBe(2);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Positive_cache_avoids_checks_per_page_and_expires_after_a_minute()
|
||||
{
|
||||
var fake = new FakeHistorianGatewayClient(); var clock = new Clock(); var ds = Source(fake, clock);
|
||||
await ds.ReadRawAsync("T", Now.AddMinutes(-1), Now, 10, Ct);
|
||||
await ds.ReadRawAsync("T", Now.AddMinutes(-1), Now, 10, Ct);
|
||||
fake.TagExistsCallCount.ShouldBe(1);
|
||||
clock.Current = Now.AddSeconds(60);
|
||||
await ds.ReadRawAsync("T", Now.AddMinutes(-1), Now, 10, Ct);
|
||||
fake.TagExistsCallCount.ShouldBe(2);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task An_explicit_nested_NotFound_is_cached_as_missing()
|
||||
{
|
||||
var fake = new FakeHistorianGatewayClient { TagExistsThrows = new InvalidOperationException("wrapper",
|
||||
new RpcException(new Status(StatusCode.NotFound, "absent"))) };
|
||||
var ds = Source(fake);
|
||||
(await ds.ReadRawAsync("T", Now.AddMinutes(-1), Now, 10, Ct)).Samples.ShouldBeEmpty();
|
||||
(await ds.ReadRawAsync("T", Now.AddMinutes(-1), Now, 10, Ct)).Samples.ShouldBeEmpty();
|
||||
fake.TagExistsCallCount.ShouldBe(1);
|
||||
fake.ReadRawCallCount.ShouldBe(0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task The_presence_cache_is_bounded_and_a_displaced_entry_is_checked_again()
|
||||
{
|
||||
var fake = new FakeHistorianGatewayClient { TagExistsResult = false }; var ds = Source(fake);
|
||||
for (var i = 0; i <= GatewayTagPresenceCache.Capacity; i++)
|
||||
await ds.ReadRawAsync($"T{i}", Now.AddMinutes(-1), Now, 10, Ct);
|
||||
await ds.ReadRawAsync("T0", Now.AddMinutes(-1), Now, 10, Ct);
|
||||
fake.TagExistsCallCount.ShouldBe(GatewayTagPresenceCache.Capacity + 2);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Caller_cancellation_is_not_cached_as_missing()
|
||||
{
|
||||
var fake = new FakeHistorianGatewayClient { TagExistsThrows = new OperationCanceledException() }; var ds = Source(fake);
|
||||
await Should.ThrowAsync<OperationCanceledException>(() => ds.ReadRawAsync("T", Now.AddMinutes(-1), Now, 10, Ct));
|
||||
fake.TagExistsThrows = null;
|
||||
await ds.ReadRawAsync("T", Now.AddMinutes(-1), Now, 10, Ct);
|
||||
fake.TagExistsCallCount.ShouldBe(2);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(StatusCode.Unavailable)]
|
||||
[InlineData(StatusCode.PermissionDenied)]
|
||||
[InlineData(StatusCode.Internal)]
|
||||
public async Task A_metadata_fault_is_not_cached_as_a_missing_tag(StatusCode code)
|
||||
{
|
||||
var fake = new FakeHistorianGatewayClient { TagExistsThrows = new RpcException(new Status(code, "failure")) };
|
||||
var ds = Source(fake);
|
||||
await Should.ThrowAsync<Exception>(() => ds.ReadRawAsync("T", Now.AddMinutes(-1), Now, 10, Ct));
|
||||
fake.TagExistsThrows = null;
|
||||
await ds.ReadRawAsync("T", Now.AddMinutes(-1), Now, 10, Ct);
|
||||
fake.TagExistsCallCount.ShouldBe(2);
|
||||
fake.ReadRawCallCount.ShouldBe(1);
|
||||
}
|
||||
}
|
||||
+58
-5
@@ -222,20 +222,73 @@ public sealed class GatewayRawBoundaryRowTests
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_margin_still_crowding_the_re_read_fails_the_node_as_a_request_limit()
|
||||
public async Task A_margin_crowding_a_truncated_read_is_re_read_with_growing_room()
|
||||
{
|
||||
// 200 rows in the margin: the single bounded re-read (5 + 21 + 16) is still short of the cap because of them.
|
||||
// #694 F4: 200 rows in the margin. The first read (5 + 16) saw only 21 of them, so a single re-read sized from that
|
||||
// count (5 + 21 + 16) was still short and the node failed — with no ceiling at all. The room now at least doubles per
|
||||
// re-read (21, 42, 84, 168, 336), so the window is served in a bounded number of reads.
|
||||
var margin = Enumerable.Range(0, 200).Select(i => (At(30).AddTicks(-1 - i), (double)i, false));
|
||||
var fake = Aveva(margin.Append((At(35), 1d, false)).ToList());
|
||||
var ds = Source(fake);
|
||||
|
||||
await Should.ThrowAsync<HistoryReadLimitExceededException>(() => ds.ReadRawAsync("T", At(30), At(40), 5, Ct));
|
||||
var r = await ds.ReadRawAsync("T", At(30), At(40), 5, Ct);
|
||||
|
||||
fake.ReadRawCallCount.ShouldBe(2); // one bounded re-read, never more
|
||||
fake.RawStreamsCompleted.ShouldBe(2);
|
||||
r.Samples.Select(s => s.Value).ShouldBe([1d]);
|
||||
fake.ReadRawCallCount.ShouldBe(6);
|
||||
fake.RawStreamsCompleted.ShouldBe(6);
|
||||
ds.GetHealthSnapshot().TotalFailures.ShouldBe(0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_margin_still_crowding_the_bounded_re_reads_fails_the_node_as_a_request_limit()
|
||||
{
|
||||
// 20,000 rows in the margin: past the bounded re-reads (21 · 2^8 ≈ 5,400 rows of room).
|
||||
var margin = Enumerable.Range(0, 20_000).Select(i => (At(30).AddTicks(-1 - i), (double)i, false));
|
||||
var fake = Aveva(margin.Append((At(35), 1d, false)).ToList());
|
||||
var ds = Source(fake);
|
||||
|
||||
var ex = await Should.ThrowAsync<HistoryReadLimitExceededException>(() => ds.ReadRawAsync("T", At(30), At(40), 5, Ct));
|
||||
|
||||
ex.PreWindowRows.ShouldBeGreaterThan(0);
|
||||
fake.ReadRawCallCount.ShouldBe(9); // the read + MaxCrowdedReReads, never more
|
||||
fake.RawStreamsCompleted.ShouldBe(9);
|
||||
ds.GetHealthSnapshot().TotalFailures.ShouldBe(0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task The_crowding_re_reads_never_make_room_for_more_than_the_tie_cluster_limit()
|
||||
{
|
||||
// #694 R2: the room used to double up to 8 re-reads with no cap — on a streaming gateway a pathological 1M-row
|
||||
// single-millisecond cluster streamed ~4M rows for one page where it used to fail at once. The room is capped at
|
||||
// MaxTieClusterOverfetch (here 100); past it the read is a request limit, after a bounded number of bounded reads.
|
||||
var margin = Enumerable.Range(0, 150).Select(i => (At(30).AddTicks(-1 - i), (double)i, false));
|
||||
var fake = Aveva(margin.Append((At(35), 1d, false)).ToList());
|
||||
var ds = new GatewayHistorianDataSource(fake, NullLogger<GatewayHistorianDataSource>.Instance, maxPreWindowRows: 100);
|
||||
|
||||
var ex = await Should.ThrowAsync<HistoryReadLimitExceededException>(() => ds.ReadRawAsync("T", At(30), At(40), 5, Ct));
|
||||
|
||||
ex.PreWindowRows.ShouldBeGreaterThan(100);
|
||||
// Room 21, 42, 84: that read already saw more than 100 rows before the window, so it stops — never asking past the cap.
|
||||
fake.ReadRawCallCount.ShouldBe(4);
|
||||
fake.RawCalls.Max(c => c.MaxValues).ShouldBeLessThanOrEqualTo(5 + 100 + RawAggregateReader.PreWindowAllowance);
|
||||
fake.RawYieldedCount.ShouldBeLessThanOrEqualTo(21 + 42 + 63 + 105);
|
||||
fake.RawStreamsCompleted.ShouldBe(4);
|
||||
ds.GetHealthSnapshot().TotalFailures.ShouldBe(0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Crowding_rows_just_under_the_tie_cluster_limit_are_read_around()
|
||||
{
|
||||
var margin = Enumerable.Range(0, 90).Select(i => (At(30).AddTicks(-1 - i), (double)i, false));
|
||||
var fake = Aveva(margin.Append((At(35), 1d, false)).ToList());
|
||||
var ds = new GatewayHistorianDataSource(fake, NullLogger<GatewayHistorianDataSource>.Instance, maxPreWindowRows: 100);
|
||||
|
||||
var r = await ds.ReadRawAsync("T", At(30), At(40), 5, Ct);
|
||||
|
||||
r.Samples.Select(s => s.Value).ShouldBe([1d]);
|
||||
fake.RawCalls.Max(c => c.MaxValues).ShouldBeLessThanOrEqualTo(5 + 100 + RawAggregateReader.PreWindowAllowance);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_short_read_is_complete_and_is_not_re_read()
|
||||
{
|
||||
|
||||
+24
@@ -57,6 +57,30 @@ public sealed class GatewayStrictCeilingTests
|
||||
public override DateTimeOffset GetUtcNow() => new(now);
|
||||
}
|
||||
|
||||
private sealed class MovingClock : TimeProvider
|
||||
{
|
||||
public DateTime Current = Now;
|
||||
public override DateTimeOffset GetUtcNow() => new(Current);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Aggregate_ceiling_re_probes_wait_thirty_seconds_then_notice_a_raised_limit()
|
||||
{
|
||||
var fake = Store(100, ceiling: 50);
|
||||
var clock = new MovingClock();
|
||||
var ds = new GatewayHistorianDataSource(fake, NullLogger<GatewayHistorianDataSource>.Instance, timeProvider: clock);
|
||||
Task<HistoryReadResult> Read() => ds.ReadProcessedAsync("T", At(0), At(100), TimeSpan.FromSeconds(100), HistoryAggregateType.Count, Ct);
|
||||
await Should.ThrowAsync<HistoryReadLimitExceededException>(Read);
|
||||
await Should.ThrowAsync<HistoryReadLimitExceededException>(Read);
|
||||
Refusals(fake).ShouldBe(1);
|
||||
fake.MaxValuesCeiling = 200;
|
||||
await Should.ThrowAsync<HistoryReadLimitExceededException>(Read);
|
||||
Refusals(fake).ShouldBe(1);
|
||||
clock.Current = Now.AddSeconds(30);
|
||||
(await Read()).Samples.Single().Value.ShouldBe(100);
|
||||
ds.MaxRawValuesPerRead.ShouldBe(200 - RawAggregateReader.PreWindowAllowance);
|
||||
}
|
||||
|
||||
// ---- raw, uncapped ---------------------------------------------------------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
|
||||
+90
@@ -0,0 +1,90 @@
|
||||
using System.Net;
|
||||
using Grpc.Core;
|
||||
using Microsoft.AspNetCore.Builder;
|
||||
using Microsoft.AspNetCore.Hosting;
|
||||
using Microsoft.AspNetCore.Server.Kestrel.Core;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.HistorianGateway.Contracts.Grpc;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Historian;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests;
|
||||
|
||||
public sealed class GatewayTagCatalogWireTests
|
||||
{
|
||||
private static CancellationToken Ct => TestContext.Current.CancellationToken;
|
||||
|
||||
[Theory]
|
||||
[InlineData("Pump1", true)]
|
||||
[InlineData("Missing", false)]
|
||||
[InlineData("Pump*", false)]
|
||||
public async Task Presence_is_proved_by_exact_names_without_using_the_broken_metadata_parser(string tag, bool expected)
|
||||
{
|
||||
var service = new TagsService();
|
||||
await using var server = await StartAsync(service);
|
||||
using var adapter = Create(server.Port);
|
||||
(await adapter.TagExistsAsync(tag, Ct)).ShouldBe(expected);
|
||||
service.Filter.ShouldBe(tag);
|
||||
service.MetadataCalls.ShouldBe(0);
|
||||
service.Completed.ShouldBe(1);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_catalog_stream_fault_after_a_matching_name_is_not_cached_as_presence()
|
||||
{
|
||||
var service = new TagsService { FailAfterRow = true };
|
||||
await using var server = await StartAsync(service);
|
||||
using var adapter = Create(server.Port);
|
||||
var source = new GatewayHistorianDataSource(adapter, NullLogger<GatewayHistorianDataSource>.Instance);
|
||||
var now = DateTime.UtcNow;
|
||||
await Should.ThrowAsync<Exception>(() => source.ReadRawAsync("Pump1", now.AddMinutes(-1), now, 10, Ct));
|
||||
service.FailAfterRow = false; service.Names = [];
|
||||
(await source.ReadRawAsync("Pump1", now.AddMinutes(-1), now, 10, Ct)).Samples.ShouldBeEmpty();
|
||||
service.BrowseCalls.ShouldBe(2);
|
||||
}
|
||||
|
||||
private static HistorianGatewayClientAdapter Create(int port) => HistorianGatewayClientAdapter.Create(
|
||||
new ServerHistorianOptions { Endpoint = $"http://127.0.0.1:{port}", ApiKey = "histgw_test_key", UseTls = false },
|
||||
NullLoggerFactory.Instance);
|
||||
|
||||
private sealed class TagsService : HistorianTags.HistorianTagsBase
|
||||
{
|
||||
public int MetadataCalls, BrowseCalls, Completed;
|
||||
public string? Filter;
|
||||
public bool FailAfterRow;
|
||||
public string[] Names = ["Pump1", "Pump2"];
|
||||
public override Task<GetTagMetadataReply> GetTagMetadata(GetTagMetadataRequest request, ServerCallContext context)
|
||||
{
|
||||
MetadataCalls++;
|
||||
throw new RpcException(new Status(StatusCode.Internal, "GetTagInfo response ended unexpectedly."));
|
||||
}
|
||||
public override async Task BrowseTagNames(BrowseTagNamesRequest request, IServerStreamWriter<TagName> stream, ServerCallContext context)
|
||||
{
|
||||
BrowseCalls++; Filter = request.Filter;
|
||||
foreach (var name in Names)
|
||||
{
|
||||
await stream.WriteAsync(new TagName { Name = name }, context.CancellationToken);
|
||||
if (FailAfterRow) throw new RpcException(new Status(StatusCode.Internal, "catalog failure"));
|
||||
}
|
||||
Completed++;
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class Server(WebApplication app, int port) : IAsyncDisposable
|
||||
{
|
||||
public int Port => port;
|
||||
public async ValueTask DisposeAsync() { await app.StopAsync(); await app.DisposeAsync(); }
|
||||
}
|
||||
private static async Task<Server> StartAsync(TagsService service)
|
||||
{
|
||||
var builder = WebApplication.CreateSlimBuilder();
|
||||
builder.Logging.ClearProviders();
|
||||
builder.WebHost.ConfigureKestrel(k => k.Listen(IPAddress.Loopback, 0, o => o.Protocols = HttpProtocols.Http2));
|
||||
builder.Services.AddGrpc(); builder.Services.AddSingleton(service);
|
||||
var app = builder.Build(); app.MapGrpcService<TagsService>(); await app.StartAsync();
|
||||
return new Server(app, new Uri(app.Urls.First()).Port);
|
||||
}
|
||||
}
|
||||
+27
@@ -117,4 +117,31 @@ public sealed class HistorianGatewayClientAdapterTests
|
||||
var dataSource = GatewayHistorian.CreateDataSource(opts, services);
|
||||
Assert.IsType<GatewayHistorianDataSource>(dataSource);
|
||||
}
|
||||
|
||||
[Theory] // #693: the ServerHistorian:GatewayAtTimeFromRaw kill switch reaches the data source (default on)
|
||||
[InlineData(null, true)]
|
||||
[InlineData(true, true)]
|
||||
[InlineData(false, false)]
|
||||
public void Factory_passes_the_AtTime_kill_switch(bool? configured, bool expected)
|
||||
{
|
||||
var opts = configured is { } c
|
||||
? new ServerHistorianOptions { Enabled = true, Endpoint = "https://localhost:5222", ApiKey = "histgw_x_y", GatewayAtTimeFromRaw = c }
|
||||
: new ServerHistorianOptions { Enabled = true, Endpoint = "https://localhost:5222", ApiKey = "histgw_x_y" };
|
||||
using var services = new ServiceCollection().BuildServiceProvider();
|
||||
var dataSource = Assert.IsType<GatewayHistorianDataSource>(GatewayHistorian.CreateDataSource(opts, services));
|
||||
dataSource.AtTimeFromRaw.ShouldBe(expected);
|
||||
}
|
||||
|
||||
[Theory] // #694 R2: ServerHistorian:MaxTieClusterOverfetch bounds the crowding re-reads (default 65536)
|
||||
[InlineData(null, 65536)]
|
||||
[InlineData(1000, 1000)]
|
||||
public void Factory_passes_the_tie_cluster_limit_to_the_crowding_re_reads(int? configured, int expected)
|
||||
{
|
||||
var opts = configured is { } c
|
||||
? new ServerHistorianOptions { Enabled = true, Endpoint = "https://localhost:5222", ApiKey = "histgw_x_y", MaxTieClusterOverfetch = c }
|
||||
: new ServerHistorianOptions { Enabled = true, Endpoint = "https://localhost:5222", ApiKey = "histgw_x_y" };
|
||||
using var services = new ServiceCollection().BuildServiceProvider();
|
||||
var dataSource = Assert.IsType<GatewayHistorianDataSource>(GatewayHistorian.CreateDataSource(opts, services));
|
||||
dataSource.MaxPreWindowRows.ShouldBe(expected);
|
||||
}
|
||||
}
|
||||
|
||||
+5
@@ -31,4 +31,9 @@
|
||||
<ProjectReference Include="..\..\..\src\Drivers\ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway\ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.csproj"/>
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<!-- #694: the node manager's HistoryRead paging over the REAL gateway data source boots a real SDK server. -->
|
||||
<Compile Include="..\..\Server\ZB.MOM.WW.OtOpcUa.OpcUaServer.Tests\OpcUaHostStart.cs" Link="OpcUaHostStart.cs"/>
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
|
||||
+27
@@ -11,6 +11,33 @@ public sealed class QuestDbHistorianProviderTests
|
||||
ConnectionString = "Host=localhost;Port=18812;Username=admin;Password=quest;Database=qdb;ServerCompatibilityMode=NoTypeLoading",
|
||||
});
|
||||
|
||||
[Theory]
|
||||
[InlineData("Raw")]
|
||||
[InlineData("AtTime")]
|
||||
[InlineData("Processed")]
|
||||
[InlineData("Events")]
|
||||
[InlineData("Modified")]
|
||||
public async Task A_connection_failure_is_exposed_as_a_transport_failure(string kind)
|
||||
{
|
||||
using var provider = new QuestDbHistorianProvider(new QuestDbHistorianOptions {
|
||||
ConnectionString = "Host=127.0.0.1;Port=1;Username=admin;Password=test;Database=qdb;Timeout=1;Pooling=False;ServerCompatibilityMode=NoTypeLoading", AutoCreateSchema = false
|
||||
});
|
||||
using var source = provider.CreateDataSource(new NullSp());
|
||||
var t = new DateTime(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc);
|
||||
Func<Task> read = kind switch
|
||||
{
|
||||
"Raw" => async () => await source.ReadRawAsync("T", t, t.AddHours(1), 10, TestContext.Current.CancellationToken),
|
||||
"AtTime" => async () => await source.ReadAtTimeAsync("T", [t], TestContext.Current.CancellationToken),
|
||||
"Processed" => async () => await source.ReadProcessedAsync("T", t, t.AddHours(1), TimeSpan.FromSeconds(10), HistoryAggregateType.Average, TestContext.Current.CancellationToken),
|
||||
"Events" => async () => await source.ReadEventsAsync("T", t, t.AddHours(1), 10, TestContext.Current.CancellationToken),
|
||||
_ => async () => await source.ReadModifiedAsync("T", t, t.AddHours(1), 10, null, TestContext.Current.CancellationToken)
|
||||
};
|
||||
var error = await Should.ThrowAsync<System.IO.IOException>(read);
|
||||
error.InnerException.ShouldNotBeNull();
|
||||
// A cached bootstrap outage must remain retryable on subsequent reads as well.
|
||||
(await Should.ThrowAsync<System.IO.IOException>(read)).InnerException.ShouldNotBeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Name_and_all_six_kinds()
|
||||
{
|
||||
|
||||
+5
@@ -83,6 +83,11 @@ public sealed class AtTimeEvaluatorTests
|
||||
var n = S(10, HistorianValue.FromInt64(15), Good);
|
||||
AtTimeEvaluator.Evaluate(At(5), p, n, Now, Now).Value.ShouldBe(13L); // 12.5 ⇒ 13
|
||||
AtTimeEvaluator.Evaluate(At(2), p, n, Now, Now).Value.ShouldBe(11L);
|
||||
// #693: ShouldBe compares numbers across types, so the type needs its own assertion — a C# conditional
|
||||
// `long : double` boxed a Double here, and the line above still passed.
|
||||
AtTimeEvaluator.Evaluate(At(5), p, n, Now, Now).Value.ShouldBeOfType<long>();
|
||||
AtTimeEvaluator.Evaluate(At(5), S(0, HistorianValue.FromDouble(10), Good), S(10, HistorianValue.FromDouble(15), Good), Now, Now)
|
||||
.Value.ShouldBeOfType<double>();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
|
||||
+27
@@ -11,6 +11,33 @@ public sealed class SqlServerHistorianProviderTests
|
||||
ConnectionString = "Server=localhost;User Id=sa;Password=x;TrustServerCertificate=True",
|
||||
});
|
||||
|
||||
[Theory]
|
||||
[InlineData("Raw")]
|
||||
[InlineData("AtTime")]
|
||||
[InlineData("Processed")]
|
||||
[InlineData("Events")]
|
||||
[InlineData("Modified")]
|
||||
public async Task A_connection_failure_is_exposed_as_a_transport_failure(string kind)
|
||||
{
|
||||
var provider = new SqlServerHistorianProvider(new SqlServerHistorianOptions {
|
||||
ConnectionString = "Server=127.0.0.1,1;User Id=sa;Password=test;TrustServerCertificate=True;Connect Timeout=1;ConnectRetryCount=0;Pooling=False", AutoCreateSchema = false
|
||||
});
|
||||
using var source = provider.CreateDataSource(new NullSp());
|
||||
var t = new DateTime(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc);
|
||||
Func<Task> read = kind switch
|
||||
{
|
||||
"Raw" => async () => await source.ReadRawAsync("T", t, t.AddHours(1), 10, TestContext.Current.CancellationToken),
|
||||
"AtTime" => async () => await source.ReadAtTimeAsync("T", [t], TestContext.Current.CancellationToken),
|
||||
"Processed" => async () => await source.ReadProcessedAsync("T", t, t.AddHours(1), TimeSpan.FromSeconds(10), HistoryAggregateType.Average, TestContext.Current.CancellationToken),
|
||||
"Events" => async () => await source.ReadEventsAsync("T", t, t.AddHours(1), 10, TestContext.Current.CancellationToken),
|
||||
_ => async () => await source.ReadModifiedAsync("T", t, t.AddHours(1), 10, null, TestContext.Current.CancellationToken)
|
||||
};
|
||||
var error = await Should.ThrowAsync<System.IO.IOException>(read);
|
||||
error.InnerException.ShouldNotBeNull();
|
||||
// A cached bootstrap outage must remain retryable on subsequent reads as well.
|
||||
(await Should.ThrowAsync<System.IO.IOException>(read)).InnerException.ShouldNotBeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Name_and_all_six_kinds()
|
||||
{
|
||||
|
||||
+12
@@ -84,6 +84,18 @@ public sealed class OtOpcUaServerHostedServiceHistoryUpdateTests
|
||||
Service(view: null, config: config).HonorReturnBounds.ShouldBeFalse();
|
||||
}
|
||||
|
||||
/// <summary>#694: the events horizon is bound from <c>ServerHistorian:EventReadHorizon</c>, default none.</summary>
|
||||
[Fact]
|
||||
public void The_event_read_horizon_comes_from_options_and_defaults_off()
|
||||
{
|
||||
Service(view: null).EventReadHorizon.ShouldBe(TimeSpan.Zero);
|
||||
var config = new ConfigurationBuilder().AddInMemoryCollection(new Dictionary<string, string?>
|
||||
{
|
||||
["ServerHistorian:EventReadHorizon"] = "365.00:00:00",
|
||||
}).Build();
|
||||
Service(view: null, config: config).EventReadHorizon.ShouldBe(TimeSpan.FromDays(365));
|
||||
}
|
||||
|
||||
private static OtOpcUaServerHostedService Service(
|
||||
IHistoryUpdateRoleView? view, IDriverPrimaryRoleSource? primary = null, IConfiguration? config = null,
|
||||
TimeProvider? clock = null) => new(
|
||||
|
||||
@@ -0,0 +1,63 @@
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.Tests;
|
||||
|
||||
/// <summary>Gitea #694: <see cref="EventHorizon.TryClamp"/> — the pure window arithmetic of <c>ServerHistorian:EventReadHorizon</c>.</summary>
|
||||
public sealed class EventHorizonTests
|
||||
{
|
||||
private static readonly DateTime Now = new(2026, 10, 13, 12, 0, 0, DateTimeKind.Utc);
|
||||
private static readonly TimeSpan H = TimeSpan.FromDays(30);
|
||||
private static readonly DateTime Floor = Now - H;
|
||||
|
||||
[Fact]
|
||||
public void No_horizon_changes_nothing()
|
||||
{
|
||||
var (s, e) = (DateTime.MinValue, Now);
|
||||
EventHorizon.TryClamp(ref s, ref e, reverse: false, TimeSpan.Zero, Now).ShouldBeTrue();
|
||||
(s, e).ShouldBe((DateTime.MinValue, Now));
|
||||
EventHorizon.TryClamp(ref s, ref e, reverse: false, TimeSpan.FromDays(-1), Now).ShouldBeTrue();
|
||||
(s, e).ShouldBe((DateTime.MinValue, Now));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_forward_window_starts_at_the_floor()
|
||||
{
|
||||
var (s, e) = (DateTime.UnixEpoch, Now);
|
||||
EventHorizon.TryClamp(ref s, ref e, reverse: false, H, Now).ShouldBeTrue();
|
||||
(s, e).ShouldBe((Floor, Now));
|
||||
|
||||
var (s2, e2) = (Floor.AddDays(1), Now);
|
||||
EventHorizon.TryClamp(ref s2, ref e2, reverse: false, H, Now).ShouldBeTrue();
|
||||
s2.ShouldBe(Floor.AddDays(1)); // already inside: unchanged
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_reverse_window_keeps_the_floor_instant()
|
||||
{
|
||||
var (s, e) = (Now, new DateTime(0, DateTimeKind.Utc));
|
||||
EventHorizon.TryClamp(ref s, ref e, reverse: true, H, Now).ShouldBeTrue();
|
||||
(s, e).ShouldBe((Now, Floor.AddTicks(-1))); // (end, start] — the floor itself is read
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_window_wholly_before_the_floor_is_empty()
|
||||
{
|
||||
var (s, e) = (Floor.AddDays(-2), Floor);
|
||||
EventHorizon.TryClamp(ref s, ref e, reverse: false, H, Now).ShouldBeFalse(); // [start, end) ends at the floor
|
||||
|
||||
var (rs, re) = (Floor.AddTicks(-1), Floor.AddDays(-2));
|
||||
EventHorizon.TryClamp(ref rs, ref re, reverse: true, H, Now).ShouldBeFalse();
|
||||
|
||||
var (ok, oe) = (Floor, Floor.AddDays(-2));
|
||||
EventHorizon.TryClamp(ref ok, ref oe, reverse: true, H, Now).ShouldBeTrue(); // (…, floor] holds the floor
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_horizon_longer_than_the_calendar_is_no_horizon()
|
||||
{
|
||||
var (s, e) = (DateTime.MinValue, Now);
|
||||
EventHorizon.TryClamp(ref s, ref e, reverse: false, TimeSpan.MaxValue, Now).ShouldBeTrue();
|
||||
s.ShouldBe(DateTime.MinValue);
|
||||
}
|
||||
}
|
||||
+231
@@ -0,0 +1,231 @@
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Opc.Ua;
|
||||
using Opc.Ua.Server;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using HistorianRead = ZB.MOM.WW.OtOpcUa.Core.Abstractions.HistoryReadResult;
|
||||
using SdkHistoryReadResult = Opc.Ua.HistoryReadResult;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #694 item 2 (live gate F4): a walked-back events read on the Gateway proves "nothing older" with one forward read
|
||||
/// down to the Unix epoch (#691) — ~3.8 s live on <c>Runtime.dbo.Events</c>, and an unfiltered deep read hit the gateway's
|
||||
/// 30 s SQL timeout. The opt-in <c>ServerHistorian:EventReadHorizon</c> (<see cref="OtOpcUaNodeManager.EventReadHorizon"/>)
|
||||
/// bounds every events read at <c>now − horizon</c>, in both directions; off (the default) nothing changes. The backend is
|
||||
/// shaped like the Gateway's SQL event read: forward only, <c>[start, end]</c> inclusive, refuses <c>start > end</c>
|
||||
/// (counted), records every requested window.
|
||||
/// </summary>
|
||||
public sealed class NodeManagerHistoryReadEventsHorizonTests : IDisposable
|
||||
{
|
||||
private static CancellationToken Ct => TestContext.Current.CancellationToken;
|
||||
private static readonly DateTime Now = new(2026, 10, 13, 12, 0, 0, DateTimeKind.Utc);
|
||||
private static readonly TimeSpan Horizon = TimeSpan.FromDays(30);
|
||||
private static readonly DateTime Floor = Now - Horizon;
|
||||
private readonly string _pkiRoot = Path.Combine(Path.GetTempPath(), $"otopcua-eventhorizon-{Guid.NewGuid():N}");
|
||||
|
||||
/// <summary>Five recent events (the last five minutes) — a source with little history, as in the live gate.</summary>
|
||||
private static List<HistoricalEvent> Recent() =>
|
||||
Enumerable.Range(0, 5).Select(i => Ev($"r{i}", Now.AddMinutes(-5 + i))).ToList();
|
||||
|
||||
[Fact]
|
||||
public async Task Without_a_horizon_a_walked_reverse_read_proves_the_rest_empty_down_to_the_epoch()
|
||||
{
|
||||
// The positive control: today's exact behaviour, and the cost the horizon removes.
|
||||
var (host, nm, node, gw) = await SetupAsync(Recent());
|
||||
try
|
||||
{
|
||||
var r = ReadEvents(nm, node, max: 10, start: DateTime.MinValue, end: Now);
|
||||
r.Error.StatusCode.Code.ShouldBe(StatusCodes.Good);
|
||||
r.Ids.ShouldBe(["r4", "r3", "r2", "r1", "r0"]);
|
||||
gw.Requests.Min(q => q.Start).ShouldBe(DateTime.UnixEpoch);
|
||||
gw.BadRequests.ShouldBe(0);
|
||||
}
|
||||
finally { await host.DisposeAsync(); }
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task With_a_horizon_the_walk_never_reads_before_it_and_answers_the_same()
|
||||
{
|
||||
var (host, nm, node, gw) = await SetupAsync(Recent());
|
||||
try
|
||||
{
|
||||
nm.EventReadHorizon = Horizon;
|
||||
var r = ReadEvents(nm, node, max: 10, start: DateTime.MinValue, end: Now);
|
||||
r.Error.StatusCode.Code.ShouldBe(StatusCodes.Good);
|
||||
r.Ids.ShouldBe(["r4", "r3", "r2", "r1", "r0"]);
|
||||
r.Cp.ShouldBeNull();
|
||||
// Requests are rounded outward to the millisecond, so the earliest one may start up to 1 ms before the floor.
|
||||
gw.Requests.Min(q => q.Start).ShouldBeGreaterThanOrEqualTo(Floor.AddMilliseconds(-1));
|
||||
gw.BadRequests.ShouldBe(0);
|
||||
}
|
||||
finally { await host.DisposeAsync(); }
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Events_older_than_the_horizon_are_not_returned_in_either_direction()
|
||||
{
|
||||
var events = new List<HistoricalEvent> { Ev("old", Floor.AddDays(-10)), Ev("edge", Floor), Ev("new", Now.AddDays(-1)) };
|
||||
var (host, nm, node, gw) = await SetupAsync(events);
|
||||
try
|
||||
{
|
||||
nm.EventReadHorizon = Horizon;
|
||||
|
||||
var reverse = ReadEvents(nm, node, max: 10, start: Now, end: DateTime.UnixEpoch);
|
||||
reverse.Ids.ShouldBe(["new", "edge"]); // the floor instant itself is inside
|
||||
|
||||
var forward = ReadEvents(nm, node, max: 10, start: DateTime.UnixEpoch, end: Now);
|
||||
forward.Ids.ShouldBe(["edge", "new"]);
|
||||
|
||||
gw.Requests.ShouldAllBe(q => q.Start >= Floor.AddMilliseconds(-1));
|
||||
|
||||
nm.EventReadHorizon = TimeSpan.Zero; // off again: exact to the window
|
||||
ReadEvents(nm, node, max: 10, start: DateTime.UnixEpoch, end: Now).Ids.ShouldBe(["old", "edge", "new"]);
|
||||
}
|
||||
finally { await host.DisposeAsync(); }
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_window_wholly_older_than_the_horizon_is_GoodNoData_and_never_sent()
|
||||
{
|
||||
var (host, nm, node, gw) = await SetupAsync([Ev("old", Floor.AddDays(-10))]);
|
||||
try
|
||||
{
|
||||
nm.EventReadHorizon = Horizon;
|
||||
var forward = ReadEvents(nm, node, max: 10, start: Floor.AddDays(-20), end: Floor.AddDays(-1));
|
||||
forward.Error.StatusCode.Code.ShouldBe(StatusCodes.Good);
|
||||
forward.Status.Code.ShouldBe(StatusCodes.GoodNoData);
|
||||
forward.Ids.ShouldBeEmpty();
|
||||
|
||||
var reverse = ReadEvents(nm, node, max: 10, start: Floor.AddDays(-1), end: Floor.AddDays(-20));
|
||||
reverse.Status.Code.ShouldBe(StatusCodes.GoodNoData);
|
||||
|
||||
gw.Requests.ShouldBeEmpty();
|
||||
}
|
||||
finally { await host.DisposeAsync(); }
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_page_through_under_a_horizon_keeps_its_floor_across_continuation_points()
|
||||
{
|
||||
var events = Enumerable.Range(0, 25).Select(i => Ev($"e{i:00}", Now.AddMinutes(-i))).ToList();
|
||||
events.Add(Ev("old", Floor.AddDays(-1)));
|
||||
var (host, nm, node, gw) = await SetupAsync(events);
|
||||
try
|
||||
{
|
||||
nm.EventReadHorizon = Horizon;
|
||||
var all = new List<string>();
|
||||
byte[]? cp = null;
|
||||
var pages = 0;
|
||||
do
|
||||
{
|
||||
var r = ReadEvents(nm, node, max: 10, start: DateTime.MinValue, end: Now, inboundCp: cp);
|
||||
r.Error.StatusCode.Code.ShouldBe(StatusCodes.Good);
|
||||
all.AddRange(r.Ids);
|
||||
cp = r.Cp;
|
||||
(++pages).ShouldBeLessThan(10);
|
||||
}
|
||||
while (cp is not null);
|
||||
|
||||
all.ShouldBe(Enumerable.Range(0, 25).Select(i => $"e{i:00}"));
|
||||
gw.Requests.ShouldAllBe(q => q.Start >= Floor.AddMilliseconds(-1));
|
||||
}
|
||||
finally { await host.DisposeAsync(); }
|
||||
}
|
||||
|
||||
// ---- helpers ---------------------------------------------------------------------------------------------------------
|
||||
|
||||
private static HistoricalEvent Ev(string id, DateTime at) => new(id, "src", at, at, "m", 500);
|
||||
|
||||
private async Task<(OpcUaApplicationHost Host, OtOpcUaNodeManager Nm, NodeId Node, GatewayShapedEventSource Gw)> SetupAsync(
|
||||
List<HistoricalEvent> events)
|
||||
{
|
||||
var started = await OpcUaHostStart.StartAsync(
|
||||
port => new OpcUaApplicationHost(
|
||||
new OpcUaApplicationHostOptions
|
||||
{
|
||||
ApplicationName = "OtOpcUa.EventHorizonTest",
|
||||
ApplicationUri = $"urn:OtOpcUa.EventHorizonTest:{Guid.NewGuid():N}",
|
||||
OpcUaPort = port,
|
||||
PublicHostname = "localhost",
|
||||
PkiStoreRoot = _pkiRoot,
|
||||
},
|
||||
NullLogger<OpcUaApplicationHost>.Instance),
|
||||
() => new OtOpcUaSdkServer(), Ct);
|
||||
var nm = started.Server.NodeManager!;
|
||||
nm.HistoryContinuationStore = new InMemoryHistoryContinuationStore();
|
||||
nm.Clock = new FixedClock(Now);
|
||||
var gw = new GatewayShapedEventSource(events);
|
||||
nm.HistorianDataSource = gw;
|
||||
nm.EnsureFolder("eq-evt", parentNodeId: null, displayName: "Equipment", realm: AddressSpaceRealm.Uns);
|
||||
nm.MaterialiseAlarmCondition("alarm-1", "eq-evt", "HighTemp", "OffNormalAlarm", severity: 700, realm: AddressSpaceRealm.Uns);
|
||||
return (started.Host, nm, nm.TryGetFolder("eq-evt")!.NodeId, gw);
|
||||
}
|
||||
|
||||
private static (List<string> Ids, ServiceResult Error, StatusCode Status, byte[]? Cp) ReadEvents(
|
||||
OtOpcUaNodeManager nm, NodeId node, uint max, DateTime start, DateTime end, byte[]? inboundCp = null)
|
||||
{
|
||||
var context = new OperationContext(new RequestHeader(), secureChannelContext: null, RequestType.HistoryRead, identity: null);
|
||||
var filter = new EventFilter();
|
||||
filter.SelectClauses.Add(new SimpleAttributeOperand(ObjectTypeIds.BaseEventType, new QualifiedName("EventId"))
|
||||
{
|
||||
AttributeId = Attributes.Value,
|
||||
});
|
||||
var details = new ReadEventDetails { StartTime = start, EndTime = end, NumValuesPerNode = max, Filter = filter };
|
||||
var nodesToRead = new List<HistoryReadValueId> { new() { NodeId = node, ContinuationPoint = inboundCp } };
|
||||
var results = new List<SdkHistoryReadResult> { null! };
|
||||
var errors = new List<ServiceResult> { null! };
|
||||
nm.HistoryRead(context, details, TimestampsToReturn.Both, false, nodesToRead, results, errors);
|
||||
|
||||
var ids = new List<string>();
|
||||
if (results[0]?.HistoryData is { } ho && !ExtensionObject.IsNull(ho))
|
||||
foreach (var e in ((HistoryEvent)ExtensionObject.ToEncodeable(ho)).Events)
|
||||
ids.Add(System.Text.Encoding.UTF8.GetString((byte[])e.EventFields[0].Value));
|
||||
return (ids, errors[0], results[0]?.StatusCode ?? StatusCodes.Good, results[0]?.ContinuationPoint);
|
||||
}
|
||||
|
||||
/// <summary>The Gateway's SQL event read: forward only, <c>[start, end]</c> inclusive, time-ordered, capped + flag.</summary>
|
||||
private sealed class GatewayShapedEventSource(List<HistoricalEvent> events) : IHistorianDataSource
|
||||
{
|
||||
public List<(DateTime Start, DateTime End, int Max)> Requests { get; } = [];
|
||||
public int BadRequests { get; private set; }
|
||||
public bool SupportsReverseEvents => false;
|
||||
|
||||
public Task<HistoricalEventsResult> ReadEventsAsync(string? n, DateTime s, DateTime e, int m, CancellationToken ct)
|
||||
{
|
||||
Requests.Add((s, e, m));
|
||||
if (s > e) BadRequests++; // the gateway's ValidateRange
|
||||
var window = events.Where(x => x.EventTimeUtc >= s && x.EventTimeUtc <= e).OrderBy(x => x.EventTimeUtc).ToList();
|
||||
var cap = m <= 0 ? int.MaxValue : m;
|
||||
return Task.FromResult(new HistoricalEventsResult(window.Take(cap).ToList(), window.Count > cap ? new byte[] { 1 } : null));
|
||||
}
|
||||
|
||||
public Task<HistoricalEventsResult> ReadEventsAsync(string? n, DateTime s, DateTime e, int m, bool reverse, CancellationToken ct) =>
|
||||
reverse ? throw new NotSupportedException("forward only") : ReadEventsAsync(n, s, e, m, ct);
|
||||
|
||||
public Task<HistorianRead> ReadRawAsync(string r, DateTime s, DateTime e, uint m, CancellationToken ct) =>
|
||||
NullHistorianDataSource.Instance.ReadRawAsync(r, s, e, m, ct);
|
||||
public Task<HistorianRead> ReadProcessedAsync(string r, DateTime s, DateTime e, TimeSpan i, HistoryAggregateType a, CancellationToken ct) =>
|
||||
NullHistorianDataSource.Instance.ReadProcessedAsync(r, s, e, i, a, ct);
|
||||
public Task<HistorianRead> ReadAtTimeAsync(string r, IReadOnlyList<DateTime> t, CancellationToken ct) =>
|
||||
NullHistorianDataSource.Instance.ReadAtTimeAsync(r, t, ct);
|
||||
public HistorianHealthSnapshot GetHealthSnapshot() => NullHistorianDataSource.Instance.GetHealthSnapshot();
|
||||
public void Dispose() { }
|
||||
}
|
||||
|
||||
private sealed class FixedClock(DateTime now) : TimeProvider
|
||||
{
|
||||
public override DateTimeOffset GetUtcNow() => new(now);
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
if (Directory.Exists(_pkiRoot))
|
||||
{
|
||||
try { Directory.Delete(_pkiRoot, recursive: true); }
|
||||
catch { /* best-effort */ }
|
||||
}
|
||||
}
|
||||
}
|
||||
+9
-5
@@ -295,12 +295,14 @@ public sealed class NodeManagerHistoryReadEventsTests : IDisposable
|
||||
/// HistoryRead call. The fake source MUST be invoked (proving we reached the bridge) and threw;
|
||||
/// the production catch now sets BOTH errors and results explicitly, so both are asserted here
|
||||
/// rather than relying on the SDK base pre-seeding results[i].</summary>
|
||||
[Fact]
|
||||
public async Task Events_backend_throw_yields_bad_status_and_does_not_escape()
|
||||
[Theory]
|
||||
[InlineData(false)]
|
||||
[InlineData(true)]
|
||||
public async Task Events_backend_throw_yields_bad_status_and_does_not_escape(bool communication)
|
||||
{
|
||||
var (host, server) = await BootAsync();
|
||||
var nm = server.NodeManager!;
|
||||
var fake = new RecordingHistorianDataSource { ThrowOnRead = true };
|
||||
var fake = new RecordingHistorianDataSource { ThrowOnRead = !communication, ReadException = communication ? new System.IO.IOException("lost connection") : null };
|
||||
nm.HistorianDataSource = fake;
|
||||
|
||||
const string equipmentId = "eq-boom";
|
||||
@@ -321,10 +323,10 @@ public sealed class NodeManagerHistoryReadEventsTests : IDisposable
|
||||
|
||||
// Authoritative per-node signal: errors[0] must be Bad.
|
||||
ServiceResult.IsBad(errors[0]).ShouldBeTrue();
|
||||
errors[0].StatusCode.Code.ShouldBe(StatusCodes.BadHistoryOperationUnsupported);
|
||||
errors[0].StatusCode.Code.ShouldBe(communication ? StatusCodes.BadNoCommunication : StatusCodes.BadHistoryOperationUnsupported);
|
||||
// The production catch now sets results[0] explicitly — assert it directly (not relying on base seeding).
|
||||
results[0].ShouldNotBeNull();
|
||||
results[0].StatusCode.Code.ShouldBe(StatusCodes.BadHistoryOperationUnsupported);
|
||||
results[0].StatusCode.Code.ShouldBe(communication ? StatusCodes.BadNoCommunication : StatusCodes.BadHistoryOperationUnsupported);
|
||||
// The source WAS entered (proving the override reached the bridge and the throw was swallowed).
|
||||
fake.EventsEntered.ShouldBeTrue();
|
||||
// ThrowOnRead fires before EventsCalled/LastSourceName are set — that's the throw path.
|
||||
@@ -380,6 +382,7 @@ public sealed class NodeManagerHistoryReadEventsTests : IDisposable
|
||||
private sealed class RecordingHistorianDataSource : IHistorianDataSource
|
||||
{
|
||||
public bool ThrowOnRead { get; init; }
|
||||
public Exception? ReadException { get; init; }
|
||||
public HistoricalEventsResult EventsResult { get; set; } =
|
||||
new(Array.Empty<HistoricalEvent>(), null);
|
||||
|
||||
@@ -411,6 +414,7 @@ public sealed class NodeManagerHistoryReadEventsTests : IDisposable
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
EventsEntered = true;
|
||||
if (ReadException is not null) throw ReadException;
|
||||
if (ThrowOnRead) throw new InvalidOperationException("backend boom for events");
|
||||
EventsCalled = true;
|
||||
LastSourceName = sourceName;
|
||||
|
||||
+30
-2
@@ -25,6 +25,34 @@ public sealed class NodeManagerHistoryReadModifiedTests : IDisposable
|
||||
private static readonly DateTime Epoch = new(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc);
|
||||
private readonly string _pkiRoot = Path.Combine(Path.GetTempPath(), $"otopcua-modified-{Guid.NewGuid():N}");
|
||||
|
||||
[Fact]
|
||||
public async Task Modified_values_use_the_scalar_node_type_and_preserve_the_audit_information()
|
||||
{
|
||||
var (host, nm, node, _) = await SetupAsync([Mod(1, 1, 2d, HistoryModificationType.Replace, "alice", Epoch)], "Float");
|
||||
try
|
||||
{
|
||||
var r = Read(nm, node, Epoch, Epoch.AddHours(1), 10, null);
|
||||
r.Data!.DataValues.Single().Value.ShouldBeOfType<float>().ShouldBe(2f);
|
||||
r.Data.ModificationInfos.Single().UserName.ShouldBe("alice");
|
||||
r.Data.ModificationInfos.Single().UpdateType.ShouldBe(HistoryUpdateType.Replace);
|
||||
}
|
||||
finally { await host.DisposeAsync(); }
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_modified_read_of_an_unreachable_backend_answers_BadNoCommunication()
|
||||
{
|
||||
var (host, nm, node, src) = await SetupAsync([]);
|
||||
try
|
||||
{
|
||||
src.Throw = new System.IO.IOException("connection lost");
|
||||
var r = Read(nm, node, Epoch, Epoch.AddHours(1), 10, null);
|
||||
r.Error.StatusCode.Code.ShouldBe(StatusCodes.BadNoCommunication);
|
||||
r.Result.StatusCode.Code.ShouldBe(StatusCodes.BadNoCommunication);
|
||||
}
|
||||
finally { await host.DisposeAsync(); }
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_modified_read_dispatches_to_the_trail_and_projects_modification_infos()
|
||||
{
|
||||
@@ -289,7 +317,7 @@ public sealed class NodeManagerHistoryReadModifiedTests : IDisposable
|
||||
}
|
||||
|
||||
private async Task<(OpcUaApplicationHost Host, OtOpcUaNodeManager Nm, NodeId Node, SqlLikeTrail Src)> SetupAsync(
|
||||
List<HistoryModifiedValue> trail)
|
||||
List<HistoryModifiedValue> trail, string dataType = "Double")
|
||||
{
|
||||
var started = await OpcUaHostStart.StartAsync(
|
||||
port => new OpcUaApplicationHost(
|
||||
@@ -307,7 +335,7 @@ public sealed class NodeManagerHistoryReadModifiedTests : IDisposable
|
||||
nm.HistoryContinuationStore = new InMemoryHistoryContinuationStore();
|
||||
var src = new SqlLikeTrail(trail);
|
||||
nm.HistorianDataSource = src;
|
||||
nm.EnsureVariable("eq-1/mod", null, "Mod", "Double", false, historianTagname: Tag, realm: AddressSpaceRealm.Uns);
|
||||
nm.EnsureVariable("eq-1/mod", null, "Mod", dataType, false, historianTagname: Tag, realm: AddressSpaceRealm.Uns);
|
||||
return (started.Host, nm, nm.TryGetVariable("eq-1/mod")!.NodeId, src);
|
||||
}
|
||||
|
||||
|
||||
+438
@@ -0,0 +1,438 @@
|
||||
using Opc.Ua;
|
||||
using Opc.Ua.Server;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using HistorianRead = ZB.MOM.WW.OtOpcUa.Core.Abstractions.HistoryReadResult;
|
||||
using SdkHistoryReadResult = Opc.Ua.HistoryReadResult;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #692: a paged HistoryRead-Raw fills every page. With <c>NumValuesPerNode = 1</c> a resumed page used to re-read
|
||||
/// only the boundary sample, trim it, take the empty page for a stalled tie cluster and over-fetch
|
||||
/// <c>MaxTieClusterOverfetch + 1</c> rows to step one tick — an EMPTY page (GoodNoData + continuation point) between every
|
||||
/// value (8 values = 17 pages / 25 backend reads), on every backend, forward and reverse, with or without ReturnBounds.
|
||||
/// Larger pages lost one value per resumed page the same way. Every page must now carry exactly N values (bounds count),
|
||||
/// the over-fetch must fire only for a genuine tie cluster, and the read must not end with an empty trailing page when the
|
||||
/// last data page already knew it was the last.
|
||||
/// </summary>
|
||||
public sealed class NodeManagerHistoryReadPageFillTests(BoundsServerFixture fx) : IClassFixture<BoundsServerFixture>
|
||||
{
|
||||
private static readonly DateTime T0 = new(2026, 10, 12, 19, 0, 0, DateTimeKind.Utc);
|
||||
|
||||
public enum Shape { Native, ForwardOnly, Microsecond }
|
||||
|
||||
public enum Direction { Forward, Reverse, EndOnly }
|
||||
|
||||
/// <summary>The live-gate repro (#692): 8 values one second apart, read over [19:00:00.1, 19:00:10.5).</summary>
|
||||
private static DataValueSnapshot[] EightValues() =>
|
||||
Enumerable.Range(1, 8).Select(i => Sample(i, T0.AddSeconds(i))).ToArray();
|
||||
|
||||
private static DataValueSnapshot Sample(double v, DateTime t) => new(v, StatusCodes.Good, t, t);
|
||||
|
||||
private static (DateTime Start, DateTime End) Window(Direction direction) => direction switch
|
||||
{
|
||||
Direction.Forward => (T0.AddMilliseconds(100), T0.AddMilliseconds(10_500)),
|
||||
Direction.Reverse => (T0.AddMilliseconds(10_500), T0.AddMilliseconds(100)),
|
||||
_ => (DateTime.MinValue, T0.AddMilliseconds(10_500)),
|
||||
};
|
||||
|
||||
public static TheoryData<Shape, Direction, bool> AllShapes()
|
||||
{
|
||||
var data = new TheoryData<Shape, Direction, bool>();
|
||||
foreach (var s in Enum.GetValues<Shape>())
|
||||
foreach (var d in Enum.GetValues<Direction>())
|
||||
foreach (var b in new[] { false, true })
|
||||
data.Add(s, d, b);
|
||||
return data;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// N = 1 paged to completion: every page carries exactly one value, the pages concatenate to the one-page answer, the
|
||||
/// read never asks the backend for a tie-cluster over-fetch, and the backend read count stays proportional to the
|
||||
/// pages (native backends: one data read per page plus the bound lookups).
|
||||
/// </summary>
|
||||
[Theory]
|
||||
[MemberData(nameof(AllShapes))]
|
||||
public async Task One_value_per_page_returns_one_value_on_every_page(Shape shape, Direction direction, bool bounds)
|
||||
{
|
||||
var source = Source(shape, EightValues());
|
||||
var (start, end) = Window(direction);
|
||||
|
||||
var expected = await SinglePageAsync(source, start, end, bounds);
|
||||
expected.Count.ShouldBeGreaterThanOrEqualTo(8);
|
||||
source.ResetCounts();
|
||||
|
||||
var pages = await DrainPagesAsync(source, start, end, 1, bounds);
|
||||
|
||||
Render(pages.SelectMany(p => p)).ShouldBe(Render(expected));
|
||||
pages.Select(p => p.Count).ShouldAllBe(c => c == 1, $"pages: {string.Join(",", pages.Select(p => p.Count))}");
|
||||
pages.Count.ShouldBe(expected.Count);
|
||||
source.MaxCapRequested.ShouldBeLessThanOrEqualTo((uint)(ReverseWalk.DiscoveryRows + 2),
|
||||
"no tie-cluster over-fetch for a series without ties");
|
||||
if (shape != Shape.ForwardOnly)
|
||||
source.Reads.ShouldBeLessThanOrEqualTo(pages.Count + 4, $"{source.Reads} backend reads for {pages.Count} pages");
|
||||
}
|
||||
|
||||
/// <summary>The live-gate numbers, pinned: 8 values at N = 1 on a native backend is 8 pages and 8 backend reads (was 17 / 25).</summary>
|
||||
[Fact]
|
||||
public async Task Eight_values_at_one_per_page_cost_eight_pages_and_eight_reads()
|
||||
{
|
||||
var source = Source(Shape.Native, EightValues());
|
||||
var (start, end) = Window(Direction.Forward);
|
||||
|
||||
var pages = await DrainPagesAsync(source, start, end, 1, bounds: false);
|
||||
|
||||
pages.Count.ShouldBe(8, $"{source.Reads} reads, {source.RequestedRows} rows requested");
|
||||
source.Reads.ShouldBe(8);
|
||||
source.RequestedRows.ShouldBeLessThanOrEqualTo(8 * 3);
|
||||
}
|
||||
|
||||
/// <summary>A series whose tie clusters straddle the page boundaries at every page size.</summary>
|
||||
private static DataValueSnapshot[] TiedValues()
|
||||
{
|
||||
var seconds = new[] { 1, 2, 2, 3, 3, 3, 4, 5, 5, 6, 7, 7, 7, 8 };
|
||||
return seconds.Select((s, i) => Sample(i, T0.AddSeconds(s))).ToArray();
|
||||
}
|
||||
|
||||
public static TheoryData<Shape, Direction, bool, uint> TiedShapes()
|
||||
{
|
||||
var data = new TheoryData<Shape, Direction, bool, uint>();
|
||||
foreach (var s in Enum.GetValues<Shape>())
|
||||
foreach (var d in Enum.GetValues<Direction>())
|
||||
foreach (var b in new[] { false, true })
|
||||
foreach (var n in new uint[] { 1, 2, 3 })
|
||||
data.Add(s, d, b, n);
|
||||
return data;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// #669 tie semantics at the page boundary are kept: ties that straddle a page boundary come back once each, in the
|
||||
/// read's order, at N = 1/2/3 — and every page that has a continuation point is full, no page is empty.
|
||||
/// </summary>
|
||||
[Theory]
|
||||
[MemberData(nameof(TiedShapes))]
|
||||
public async Task Tie_clusters_straddling_page_boundaries_page_without_loss_duplication_or_short_pages(
|
||||
Shape shape, Direction direction, bool bounds, uint n)
|
||||
{
|
||||
var source = Source(shape, TiedValues());
|
||||
var (start, end) = Window(direction);
|
||||
var expected = await SinglePageAsync(source, start, end, bounds);
|
||||
|
||||
var pages = await DrainPagesAsync(source, start, end, n, bounds);
|
||||
|
||||
Render(pages.SelectMany(p => p)).ShouldBe(Render(expected));
|
||||
pages.ShouldAllBe(p => p.Count > 0, "no empty page");
|
||||
for (var i = 0; i < pages.Count - 1; i++)
|
||||
pages[i].Count.ShouldBe((int)n, $"page {i} of {pages.Count}: {string.Join(",", pages.Select(p => p.Count))}");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A read whose total is an exact multiple of the page size ends on its last full page: the lookahead row tells the
|
||||
/// server there is nothing more, so there is no trailing 0-value page (the #692 gate's reverse N = 2 observation).
|
||||
/// </summary>
|
||||
[Theory]
|
||||
[InlineData(Shape.Native, Direction.Forward)]
|
||||
[InlineData(Shape.Native, Direction.Reverse)]
|
||||
[InlineData(Shape.ForwardOnly, Direction.Forward)]
|
||||
[InlineData(Shape.ForwardOnly, Direction.Reverse)]
|
||||
[InlineData(Shape.Microsecond, Direction.Reverse)]
|
||||
public async Task An_exact_multiple_ends_on_its_last_full_page(Shape shape, Direction direction)
|
||||
{
|
||||
var (start, end) = Window(direction);
|
||||
var pages = await DrainPagesAsync(Source(shape, EightValues()), start, end, 2, bounds: false);
|
||||
|
||||
pages.Select(p => p.Count).ShouldBe([2, 2, 2, 2]);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// #692 review F1: an oversized tie cluster one step below EndTime used to resume AT EndTime (the drain check was
|
||||
/// <c>next <= end</c>, but the forward window is end-exclusive), so the next read was a <c>start == end</c> point
|
||||
/// read that returned the samples AT EndTime as data. Paged must equal unpaged — neither holds the EndTime samples.
|
||||
/// The microsecond shape truncates EndTime (8.1 µs ⇒ 8.0 µs) before the same thing happens.
|
||||
/// </summary>
|
||||
[Theory]
|
||||
[InlineData(Shape.Native, 1L, 8L)]
|
||||
[InlineData(Shape.Microsecond, 10L, 81L)]
|
||||
public async Task A_tie_cluster_just_below_the_end_time_never_resumes_onto_the_end_time(Shape shape, long unit, long endTicks)
|
||||
{
|
||||
var series = new List<DataValueSnapshot>();
|
||||
var v = 0;
|
||||
foreach (var (at, count) in new[] { (5L, 5), (6L, 2), (7L, 3), (8L, 3) })
|
||||
for (var i = 0; i < count; i++)
|
||||
series.Add(Sample(v++, T0.AddTicks(at * unit)));
|
||||
var source = Source(shape, series);
|
||||
var (start, end) = (T0.AddSeconds(-3), T0.AddTicks(endTicks));
|
||||
|
||||
var unpaged = await SinglePageAsync(source, start, end, bounds: false);
|
||||
unpaged.Count.ShouldBe(10, "the samples at the (exclusive) end time are not data");
|
||||
|
||||
var pages = await DrainPagesAsync(source, start, end, 1, bounds: false);
|
||||
Render(pages.SelectMany(p => p)).ShouldBe(Render(unpaged));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// #692: a cursor left inside an oversized tie cluster is flagged, and its next page goes straight to the cluster
|
||||
/// over-fetch — one backend read — instead of first re-reading <c>N + skip + 1</c> rows it knows are ties.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task A_cursor_inside_a_tie_cluster_over_fetches_without_a_normal_read_first()
|
||||
{
|
||||
// a @1s, five ties @2s, b @3s; N = 1.
|
||||
var series = new List<DataValueSnapshot> { Sample(0, T0.AddSeconds(1)) };
|
||||
for (var i = 1; i <= 5; i++) series.Add(Sample(i, T0.AddSeconds(2)));
|
||||
series.Add(Sample(6, T0.AddSeconds(3)));
|
||||
var source = Source(Shape.Native, series);
|
||||
var (start, end) = (T0, T0.AddSeconds(10));
|
||||
|
||||
var readsPerPage = new List<int>();
|
||||
var all = new List<DataValue>();
|
||||
byte[]? cp = null;
|
||||
do
|
||||
{
|
||||
source.ResetCounts();
|
||||
var (values, error, next) = await ReadAsync(source, start, end, 1, bounds: false, cp);
|
||||
error.StatusCode.Code.ShouldBe(StatusCodes.Good);
|
||||
values.Count.ShouldBe(1);
|
||||
all.AddRange(values);
|
||||
readsPerPage.Add(source.Reads);
|
||||
cp = next;
|
||||
readsPerPage.Count.ShouldBeLessThan(20);
|
||||
}
|
||||
while (cp is not null);
|
||||
|
||||
all.Select(x => (double)x.Value).ShouldBe([0d, 1d, 2d, 3d, 4d, 5d, 6d]);
|
||||
// Page 1 [a], page 2 [t1]: one read each. Page 3 [t2]: the read stalls on the cluster, then the over-fetch (2).
|
||||
// Pages 4 [t3], 5 [t4], 6 [t5]: the flagged cursor over-fetches directly (1 each). Page 7 [b]: one read.
|
||||
readsPerPage.ShouldBe([1, 1, 2, 1, 1, 1, 1]);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// #692 (low): a one-value bounded read whose start bound is not found asks whether the read holds anything. When the
|
||||
/// answer is "only the end bound", the bound it just found rides the continuation point — the next page is served
|
||||
/// without reading the backend again.
|
||||
/// </summary>
|
||||
[Theory]
|
||||
[InlineData(Shape.Native, Direction.Forward)]
|
||||
[InlineData(Shape.Native, Direction.Reverse)]
|
||||
[InlineData(Shape.ForwardOnly, Direction.Forward)]
|
||||
[InlineData(Shape.ForwardOnly, Direction.Reverse)]
|
||||
[InlineData(Shape.Microsecond, Direction.Forward)]
|
||||
public async Task A_start_bound_only_page_carries_the_end_bound_it_found(Shape shape, Direction direction)
|
||||
{
|
||||
// Forward [5s, 7s): nothing at/before 5s, nothing inside, the end bound at 9s. Reverse (5s ← 7s]: mirrored.
|
||||
var series = direction == Direction.Forward ? new[] { Sample(9, T0.AddSeconds(9)) } : [Sample(3, T0.AddSeconds(3))];
|
||||
var source = Source(shape, series);
|
||||
var (start, end) = direction == Direction.Forward ? (T0.AddSeconds(5), T0.AddSeconds(7)) : (T0.AddSeconds(7), T0.AddSeconds(5));
|
||||
|
||||
var (first, _, cp) = await ReadAsync(source, start, end, 1, bounds: true, cp: null);
|
||||
first.Single().StatusCode.Code.ShouldBe(StatusCodes.BadBoundNotFound);
|
||||
cp.ShouldNotBeNull();
|
||||
|
||||
source.ResetCounts();
|
||||
var (second, _, cp2) = await ReadAsync(source, start, end, 1, bounds: true, cp: cp);
|
||||
second.Single().Value.ShouldBe(direction == Direction.Forward ? 9d : 3d);
|
||||
cp2.ShouldBeNull();
|
||||
source.Reads.ShouldBe(0, "the end bound was found by the first page and carried");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// #692 (low): a one-value bounded reverse read on a forward-only backend walks back to answer "is there any data";
|
||||
/// the walk hint that walk computed rides the continuation point (it used to be dropped).
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task A_start_bound_only_page_carries_the_walk_hint_it_computed()
|
||||
{
|
||||
var source = Source(Shape.ForwardOnly, EightValues());
|
||||
var store = new RecordingStore();
|
||||
await fx.Gate.WaitAsync(TestContext.Current.CancellationToken);
|
||||
try
|
||||
{
|
||||
fx.Nm.HistoryContinuationStore = store;
|
||||
fx.Nm.HistorianDataSource = source;
|
||||
// Reverse (0.1s ← 30s]: no sample at or after 30s ⇒ the start bound is a placeholder; the data is below.
|
||||
var (values, _, cp) = Read(T0.AddSeconds(30), T0.AddMilliseconds(100), 1, bounds: true, cp: null);
|
||||
values.Single().StatusCode.Code.ShouldBe(StatusCodes.BadBoundNotFound);
|
||||
cp.ShouldNotBeNull();
|
||||
store.Saved.Last().WalkHint.WidthTicks.ShouldBeGreaterThan(0);
|
||||
}
|
||||
finally
|
||||
{
|
||||
fx.Nm.HistoryContinuationStore = new InMemoryHistoryContinuationStore();
|
||||
fx.Gate.Release();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// #692 (low): only a SERVER-synthesised bound is a placeholder. A stored row that happens to carry
|
||||
/// <c>Bad_BoundNotFound</c> with a null value (a client can write one through HistoryUpdate) is data — it used to make
|
||||
/// the read look "all placeholders" and come back GoodNoData with no values.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task A_stored_row_with_the_bound_not_found_status_is_data_not_a_placeholder()
|
||||
{
|
||||
var stored = new DataValueSnapshot(null, StatusCodes.BadBoundNotFound, T0.AddSeconds(6), T0.AddSeconds(6));
|
||||
HistoryBounds.IsPlaceholder(stored).ShouldBeFalse();
|
||||
HistoryBounds.IsPlaceholder(HistoryBounds.NotFound(T0)).ShouldBeTrue();
|
||||
|
||||
var (values, _, cp) = await ReadAsync(Source(Shape.Native, [stored]), T0.AddSeconds(5), T0.AddSeconds(7), 0, bounds: true, cp: null);
|
||||
cp.ShouldBeNull();
|
||||
values.Count.ShouldBe(3, "start placeholder, the stored row, end placeholder");
|
||||
values[1].SourceTimestamp.ShouldBe(T0.AddSeconds(6));
|
||||
}
|
||||
|
||||
// --- helpers --------------------------------------------------------------------------------
|
||||
|
||||
private static string Render(IEnumerable<DataValue> values) =>
|
||||
string.Join(" | ", values.Select(v => $"{v.SourceTimestamp:HH:mm:ss.fffffff}={v.Value ?? "null"}/{v.StatusCode.Code:X8}"));
|
||||
|
||||
private async Task<List<DataValue>> SinglePageAsync(CountingSource source, DateTime start, DateTime end, bool bounds)
|
||||
{
|
||||
var (values, error, cp) = await ReadAsync(source, start, end, 10_000, bounds, cp: null);
|
||||
error.StatusCode.Code.ShouldBe(StatusCodes.Good);
|
||||
cp.ShouldBeNull();
|
||||
return values.ToList();
|
||||
}
|
||||
|
||||
private async Task<List<IReadOnlyList<DataValue>>> DrainPagesAsync(
|
||||
CountingSource source, DateTime start, DateTime end, uint max, bool bounds)
|
||||
{
|
||||
var pages = new List<IReadOnlyList<DataValue>>();
|
||||
byte[]? cp = null;
|
||||
do
|
||||
{
|
||||
var (values, error, next) = await ReadAsync(source, start, end, max, bounds, cp);
|
||||
error.StatusCode.Code.ShouldBe(StatusCodes.Good);
|
||||
pages.Add(values);
|
||||
cp = next;
|
||||
pages.Count.ShouldBeLessThan(80, "paging must terminate");
|
||||
}
|
||||
while (cp is not null);
|
||||
|
||||
return pages;
|
||||
}
|
||||
|
||||
private async Task<(IReadOnlyList<DataValue> Values, ServiceResult Error, byte[]? Cp)> ReadAsync(
|
||||
IHistorianDataSource source, DateTime start, DateTime end, uint max, bool bounds, byte[]? cp)
|
||||
{
|
||||
await fx.Gate.WaitAsync(TestContext.Current.CancellationToken);
|
||||
try
|
||||
{
|
||||
fx.Nm.HistorianDataSource = source;
|
||||
return Read(start, end, max, bounds, cp);
|
||||
}
|
||||
finally { fx.Gate.Release(); }
|
||||
}
|
||||
|
||||
private (IReadOnlyList<DataValue> Values, ServiceResult Error, byte[]? Cp) Read(
|
||||
DateTime start, DateTime end, uint max, bool bounds, byte[]? cp)
|
||||
{
|
||||
var context = new OperationContext(new RequestHeader(), secureChannelContext: null, RequestType.HistoryRead, identity: null);
|
||||
var details = new ReadRawModifiedDetails
|
||||
{
|
||||
StartTime = start, EndTime = end, NumValuesPerNode = max, ReturnBounds = bounds, IsReadModified = false,
|
||||
};
|
||||
var nodesToRead = new List<HistoryReadValueId> { new() { NodeId = fx.NodeId, ContinuationPoint = cp } };
|
||||
var results = new List<SdkHistoryReadResult> { null! };
|
||||
var errors = new List<ServiceResult> { null! };
|
||||
fx.Nm.HistoryRead(context, details, TimestampsToReturn.Both, releaseContinuationPoints: false, nodesToRead, results, errors);
|
||||
|
||||
var values = new List<DataValue>();
|
||||
if (results[0]?.HistoryData is { } ho && !ExtensionObject.IsNull(ho))
|
||||
values.AddRange(((HistoryData)ExtensionObject.ToEncodeable(ho)).DataValues);
|
||||
return (values, errors[0], results[0]?.ContinuationPoint);
|
||||
}
|
||||
|
||||
private static CountingSource Source(Shape shape, IReadOnlyList<DataValueSnapshot> series) => shape switch
|
||||
{
|
||||
Shape.Native => new CountingSource(series, reverse: true, resolution: 1),
|
||||
Shape.ForwardOnly => new CountingSource(series, reverse: false, resolution: 1),
|
||||
_ => new CountingSource(series, reverse: true, resolution: 10),
|
||||
};
|
||||
|
||||
/// <summary>Records every saved cursor (to inspect what rides a continuation point).</summary>
|
||||
private sealed class RecordingStore : IHistoryContinuationStore
|
||||
{
|
||||
private readonly InMemoryHistoryContinuationStore _inner = new();
|
||||
public List<HistoryContinuationState> Saved { get; } = [];
|
||||
|
||||
public byte[]? Save(ISession? session, HistoryContinuationState state)
|
||||
{
|
||||
Saved.Add(state);
|
||||
return _inner.Save(session, state);
|
||||
}
|
||||
|
||||
public HistoryContinuationState? TryTake(ISession? session, byte[] continuationPoint) => _inner.TryTake(session, continuationPoint);
|
||||
|
||||
public void Release(ISession? session, byte[] continuationPoint) => _inner.Release(session, continuationPoint);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A series-backed historian that counts what it is asked for. <c>reverse</c>: serves <c>start > end</c> newest-first
|
||||
/// (SqlServer/QuestDB); otherwise refuses it (the Gateway — the server walks back with forward reads). Forward reads
|
||||
/// are <c>[start, end)</c>, a point read the instant; <c>resolution</c> > 1 truncates the window bounds (QuestDB µs).
|
||||
/// </summary>
|
||||
private sealed class CountingSource(IReadOnlyList<DataValueSnapshot> series, bool reverse, long resolution) : IHistorianDataSource
|
||||
{
|
||||
private readonly Lock _gate = new();
|
||||
|
||||
public int Reads { get; private set; }
|
||||
public long RequestedRows { get; private set; }
|
||||
public uint MaxCapRequested { get; private set; }
|
||||
|
||||
public void ResetCounts()
|
||||
{
|
||||
lock (_gate) { Reads = 0; RequestedRows = 0; MaxCapRequested = 0; }
|
||||
}
|
||||
|
||||
public bool SupportsReverseRaw => reverse;
|
||||
|
||||
public long TimestampResolutionTicks => resolution;
|
||||
|
||||
private DateTime Trunc(DateTime t) => new(t.Ticks - t.Ticks % resolution, DateTimeKind.Utc);
|
||||
|
||||
public Task<HistorianRead> ReadRawAsync(
|
||||
string fullReference, DateTime startUtc, DateTime endUtc, uint maxValuesPerNode, CancellationToken cancellationToken)
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
Reads++;
|
||||
RequestedRows += maxValuesPerNode;
|
||||
if (maxValuesPerNode > MaxCapRequested) MaxCapRequested = maxValuesPerNode;
|
||||
}
|
||||
|
||||
static DateTime Ts(DataValueSnapshot s) => s.SourceTimestampUtc ?? DateTime.MinValue;
|
||||
startUtc = Trunc(startUtc);
|
||||
endUtc = Trunc(endUtc);
|
||||
List<DataValueSnapshot> window;
|
||||
if (startUtc > endUtc)
|
||||
{
|
||||
if (!reverse) throw new InvalidOperationException("start_time must be less than or equal to end_time. (InvalidArgument)");
|
||||
window = series.Where(s => Ts(s) > endUtc && Ts(s) <= startUtc).Reverse().ToList();
|
||||
}
|
||||
else if (startUtc == endUtc) window = series.Where(s => Ts(s) == startUtc).ToList();
|
||||
else window = series.Where(s => Ts(s) >= startUtc && Ts(s) < endUtc).ToList();
|
||||
|
||||
var page = maxValuesPerNode == 0 ? window : window.Take((int)maxValuesPerNode).ToList();
|
||||
return Task.FromResult(new HistorianRead(page, null));
|
||||
}
|
||||
|
||||
public Task<HistorianRead> ReadProcessedAsync(
|
||||
string fullReference, DateTime startUtc, DateTime endUtc, TimeSpan interval,
|
||||
HistoryAggregateType aggregate, CancellationToken cancellationToken) =>
|
||||
Task.FromResult(new HistorianRead(Array.Empty<DataValueSnapshot>(), null));
|
||||
|
||||
public Task<HistorianRead> ReadAtTimeAsync(
|
||||
string fullReference, IReadOnlyList<DateTime> timestampsUtc, CancellationToken cancellationToken) =>
|
||||
Task.FromResult(new HistorianRead(Array.Empty<DataValueSnapshot>(), null));
|
||||
|
||||
public Task<HistoricalEventsResult> ReadEventsAsync(
|
||||
string? sourceName, DateTime startUtc, DateTime endUtc, int maxEvents, CancellationToken cancellationToken) =>
|
||||
Task.FromResult(new HistoricalEventsResult(Array.Empty<HistoricalEvent>(), null));
|
||||
|
||||
public HistorianHealthSnapshot GetHealthSnapshot() => NullHistorianDataSource.Instance.GetHealthSnapshot();
|
||||
|
||||
public void Dispose() { }
|
||||
}
|
||||
}
|
||||
+19
-13
@@ -29,8 +29,8 @@ public sealed class NodeManagerHistoryReadPagingTests : IDisposable
|
||||
|
||||
/// <summary>A full first page emits a continuation point; resuming returns subsequent pages; the
|
||||
/// final short page clears the continuation point — and the union of pages is the complete series with
|
||||
/// no duplicate or skipped sample. (Resumed pages net slightly fewer than the cap because each resume
|
||||
/// re-reads + trims the boundary sample to stay tie-safe — see the design; the union is what matters.)</summary>
|
||||
/// no duplicate or skipped sample. (#692: every page is full — a resume re-reads the boundary ties it trims plus one
|
||||
/// lookahead row, so 250 at 100 is exactly three pages of 100, 100 and 50.)</summary>
|
||||
[Fact]
|
||||
public async Task Raw_pages_full_series_across_continuation_points()
|
||||
{
|
||||
@@ -64,7 +64,7 @@ public sealed class NodeManagerHistoryReadPagingTests : IDisposable
|
||||
while (cp is not null);
|
||||
|
||||
sawCp.ShouldBeTrue("a 250-sample series capped at 100 must page");
|
||||
pageCount.ShouldBeGreaterThan(1);
|
||||
pageCount.ShouldBe(3, "100 + 100 + 50 — every page full (#692)");
|
||||
|
||||
// The union is the exact series — no duplicates, no skips, in order.
|
||||
collected.Count.ShouldBe(250);
|
||||
@@ -132,11 +132,11 @@ public sealed class NodeManagerHistoryReadPagingTests : IDisposable
|
||||
await host.DisposeAsync();
|
||||
}
|
||||
|
||||
/// <summary>An exactly-full series whose total is a multiple of the cap pages to a trailing
|
||||
/// EMPTY page (GoodNoData, no CP) — the server can't know the prior page was the last until it reads
|
||||
/// past the end.</summary>
|
||||
/// <summary>An exactly-full series whose total is a multiple of the cap ends on its last full page: each read asks for
|
||||
/// one lookahead row past the page (#692), so the server knows the page is the last and emits no continuation point —
|
||||
/// there is no trailing EMPTY page any more. A stale point resumed anyway still terminates with GoodNoData.</summary>
|
||||
[Fact]
|
||||
public async Task Raw_exact_multiple_terminates_with_empty_final_page()
|
||||
public async Task Raw_exact_multiple_ends_on_its_last_full_page()
|
||||
{
|
||||
var (host, server) = await BootAsync();
|
||||
var nm = server.NodeManager!;
|
||||
@@ -149,12 +149,17 @@ public sealed class NodeManagerHistoryReadPagingTests : IDisposable
|
||||
|
||||
var (r1, _, cp1) = ReadRaw(nm, nodeId, Epoch, Epoch.AddHours(1), max: 100, inboundCp: null);
|
||||
r1.Count.ShouldBe(100);
|
||||
cp1.ShouldNotBeNull("a full page emits a CP even when it happens to be the last");
|
||||
cp1.ShouldBeNull("the lookahead row found nothing past the page — it is the last");
|
||||
|
||||
var (r2, e2, cp2) = ReadRaw(nm, nodeId, Epoch, Epoch.AddHours(1), max: 100, inboundCp: cp1);
|
||||
r2.Count.ShouldBe(0);
|
||||
// Positive control: one more sample past the page ⇒ the same full page DOES carry a continuation point.
|
||||
nm.HistorianDataSource = new SeriesHistorianDataSource(MakeSeries(count: 101, stepSeconds: 1));
|
||||
var (r2, e2, cp2) = ReadRaw(nm, nodeId, Epoch, Epoch.AddHours(1), max: 100, inboundCp: null);
|
||||
r2.Count.ShouldBe(100);
|
||||
e2.StatusCode.Code.ShouldBe(StatusCodes.Good);
|
||||
cp2.ShouldBeNull("the empty trailing page terminates paging");
|
||||
cp2.ShouldNotBeNull();
|
||||
var (r3, _, cp3) = ReadRaw(nm, nodeId, Epoch, Epoch.AddHours(1), max: 100, inboundCp: cp2);
|
||||
r3.ShouldBe([100d]);
|
||||
cp3.ShouldBeNull();
|
||||
|
||||
await host.DisposeAsync();
|
||||
}
|
||||
@@ -675,8 +680,9 @@ public sealed class NodeManagerHistoryReadPagingTests : IDisposable
|
||||
var (values, error, next) = ReadRaw(nm, nodeId, DateTime.MinValue, end, max: 3, inboundCp: cp);
|
||||
error.StatusCode.Code.ShouldBe(StatusCodes.Good);
|
||||
values.ShouldNotBeEmpty();
|
||||
values[^1].ShouldBe((double)(expected - 2), $"page {page} continues exactly");
|
||||
expected = (int)values[^1];
|
||||
// #692: every page is full (N = 3) — a resumed page used to net N − 1 after trimming its boundary sample.
|
||||
values.ShouldBe([expected, expected - 1d, expected - 2d], $"page {page} continues exactly");
|
||||
expected -= 3;
|
||||
next.ShouldNotBeNull();
|
||||
cp = next;
|
||||
|
||||
|
||||
@@ -66,7 +66,7 @@ public sealed class NodeManagerHistoryReadTests : IDisposable
|
||||
fake.LastTagname.ShouldBe("WW.Temp");
|
||||
fake.LastStart.ShouldBe(start);
|
||||
fake.LastEnd.ShouldBe(end);
|
||||
fake.LastMaxValues.ShouldBe(100u);
|
||||
fake.LastMaxValues.ShouldBe(101u, "the page plus one lookahead row (#692) — it tells the server whether more follows");
|
||||
|
||||
errors[0].StatusCode.Code.ShouldBe(StatusCodes.Good);
|
||||
results[0].StatusCode.Code.ShouldBe(StatusCodes.Good);
|
||||
@@ -586,6 +586,78 @@ public sealed class NodeManagerHistoryReadTests : IDisposable
|
||||
await host.DisposeAsync();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// #693: AtTime passes each node's scalar value type to the source (the Gateway needs it — its stored values carry no
|
||||
/// type), through a Raw NodeId and a UNS NodeId alike; an array node passes none. (The fake above implements only the
|
||||
/// hint-less overload, and <see cref="AtTime_dispatches_requested_timestamps"/> proves the interface default forwards.)
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task AtTime_passes_the_nodes_value_type_for_Raw_and_UNS_nodes()
|
||||
{
|
||||
var (host, server) = await BootAsync();
|
||||
var nm = server.NodeManager!;
|
||||
var fake = new HintRecordingSource();
|
||||
nm.HistorianDataSource = fake;
|
||||
|
||||
nm.EnsureVariable("Plant/Modbus/dev1/Flag", parentFolderNodeId: null, displayName: "Flag", dataType: "Boolean",
|
||||
writable: false, historianTagname: "R.Flag", realm: AddressSpaceRealm.Raw);
|
||||
nm.EnsureVariable("Area/Line/Eq/Flag", parentFolderNodeId: null, displayName: "Flag", dataType: "Boolean",
|
||||
writable: false, historianTagname: "U.Flag", realm: AddressSpaceRealm.Uns);
|
||||
nm.EnsureVariable("Plant/Modbus/dev1/Count", parentFolderNodeId: null, displayName: "Count", dataType: "Int32",
|
||||
writable: false, historianTagname: "R.Count", realm: AddressSpaceRealm.Raw);
|
||||
nm.EnsureVariable("Area/Line/Eq/Flow", parentFolderNodeId: null, displayName: "Flow", dataType: "Double",
|
||||
writable: false, historianTagname: "U.Flow", realm: AddressSpaceRealm.Uns);
|
||||
nm.EnsureVariable("Plant/Modbus/dev1/Arr", parentFolderNodeId: null, displayName: "Arr", dataType: "Int16",
|
||||
writable: false, historianTagname: "R.Arr", realm: AddressSpaceRealm.Raw, isArray: true, arrayLength: 4);
|
||||
|
||||
string[] paths = ["Plant/Modbus/dev1/Flag", "Area/Line/Eq/Flag", "Plant/Modbus/dev1/Count", "Area/Line/Eq/Flow", "Plant/Modbus/dev1/Arr"];
|
||||
AddressSpaceRealm[] realms = [AddressSpaceRealm.Raw, AddressSpaceRealm.Uns, AddressSpaceRealm.Raw, AddressSpaceRealm.Uns, AddressSpaceRealm.Raw];
|
||||
var nodeIds = paths.Select((p, i) => nm.TryGetVariable(p, realms[i])!.NodeId).ToArray();
|
||||
|
||||
var (_, errors) = InvokeHistoryRead(server, nm,
|
||||
new ReadAtTimeDetails { ReqTimes = new DateTimeCollection { DateTime.UtcNow.AddMinutes(-1) } }, nodeIds);
|
||||
|
||||
errors.ShouldAllBe(e => StatusCode.IsGood(e.StatusCode));
|
||||
fake.Hints["R.Flag"].ShouldBe(TypeCode.Boolean);
|
||||
fake.Hints["U.Flag"].ShouldBe(TypeCode.Boolean);
|
||||
fake.Hints["R.Count"].ShouldBe(TypeCode.Int32);
|
||||
fake.Hints["U.Flow"].ShouldBe(TypeCode.Double);
|
||||
fake.Hints["R.Arr"].ShouldBeNull();
|
||||
|
||||
await host.DisposeAsync();
|
||||
}
|
||||
|
||||
/// <summary>A source recording the AtTime value-type hint per tagname (the hinted overload only).</summary>
|
||||
private sealed class HintRecordingSource : IHistorianDataSource
|
||||
{
|
||||
public System.Collections.Concurrent.ConcurrentDictionary<string, TypeCode?> Hints { get; } = new();
|
||||
|
||||
public Task<HistorianRead> ReadAtTimeAsync(
|
||||
string fullReference, IReadOnlyList<DateTime> timestampsUtc, TypeCode? valueType, CancellationToken cancellationToken)
|
||||
{
|
||||
Hints[fullReference] = valueType;
|
||||
return Task.FromResult(Empty());
|
||||
}
|
||||
|
||||
public Task<HistorianRead> ReadAtTimeAsync(
|
||||
string fullReference, IReadOnlyList<DateTime> timestampsUtc, CancellationToken cancellationToken) =>
|
||||
throw new InvalidOperationException("the node manager must call the hinted overload");
|
||||
|
||||
public Task<HistorianRead> ReadRawAsync(string fullReference, DateTime startUtc, DateTime endUtc, uint maxValuesPerNode,
|
||||
CancellationToken cancellationToken) => Task.FromResult(Empty());
|
||||
|
||||
public Task<HistorianRead> ReadProcessedAsync(string fullReference, DateTime startUtc, DateTime endUtc, TimeSpan interval,
|
||||
HistoryAggregateType aggregate, CancellationToken cancellationToken) => Task.FromResult(Empty());
|
||||
|
||||
public Task<HistoricalEventsResult> ReadEventsAsync(string? sourceName, DateTime startUtc, DateTime endUtc, int maxEvents,
|
||||
CancellationToken cancellationToken) =>
|
||||
Task.FromResult(new HistoricalEventsResult(Array.Empty<HistoricalEvent>(), null));
|
||||
|
||||
public HistorianHealthSnapshot GetHealthSnapshot() => NullHistorianDataSource.Instance.GetHealthSnapshot();
|
||||
|
||||
public void Dispose() { }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Two nodes in a single Raw HistoryRead batch — one backend succeeds, one throws. Proves that
|
||||
/// per-node error isolation is enforced across handle indices: the good node's slot is
|
||||
@@ -714,6 +786,132 @@ public sealed class NodeManagerHistoryReadTests : IDisposable
|
||||
return (results, errors);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("Raw")]
|
||||
[InlineData("AtTime")]
|
||||
[InlineData("Processed")]
|
||||
public async Task An_unreachable_backend_answers_a_communication_status(string kind)
|
||||
{
|
||||
var (host, server) = await BootAsync();
|
||||
try
|
||||
{
|
||||
var nm = server.NodeManager!;
|
||||
nm.HistorianDataSource = new RecordingHistorianDataSource { ReadException = new System.IO.IOException("connection lost") };
|
||||
nm.EnsureVariable("eq/comm", null, "Comm", "Double", false, historianTagname: "T", realm: AddressSpaceRealm.Uns);
|
||||
var t = DateTime.UtcNow.AddMinutes(-1);
|
||||
HistoryReadDetails details = kind switch
|
||||
{
|
||||
"Raw" => new ReadRawModifiedDetails { IsReadModified = false, ReturnBounds = false, StartTime = t, EndTime = t.AddSeconds(10) },
|
||||
"AtTime" => new ReadAtTimeDetails { ReqTimes = new DateTimeCollection { t } },
|
||||
_ => new ReadProcessedDetails { StartTime = t, EndTime = t.AddSeconds(10), ProcessingInterval = 1000,
|
||||
AggregateType = new NodeIdCollection { ObjectIds.AggregateFunction_Average } }
|
||||
};
|
||||
var (results, errors) = InvokeHistoryRead(server, nm, details, nm.TryGetVariable("eq/comm")!.NodeId);
|
||||
errors[0].StatusCode.Code.ShouldBe(StatusCodes.BadNoCommunication);
|
||||
results[0].StatusCode.Code.ShouldBe(StatusCodes.BadNoCommunication);
|
||||
}
|
||||
finally { await host.DisposeAsync(); }
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("Int32", "Raw")]
|
||||
[InlineData("Boolean", "Raw")]
|
||||
[InlineData("Float", "Raw")]
|
||||
[InlineData("Int32", "AtTime")]
|
||||
[InlineData("Boolean", "AtTime")]
|
||||
[InlineData("Float", "AtTime")]
|
||||
public async Task History_values_use_the_nodes_scalar_type(string dataType, string kind)
|
||||
{
|
||||
var (host, server) = await BootAsync();
|
||||
try
|
||||
{
|
||||
var nm = server.NodeManager!;
|
||||
var t = DateTime.UtcNow.AddMinutes(-1);
|
||||
var value = new HistorianRead([new DataValueSnapshot(1d, StatusCodes.Good, t, t)], null);
|
||||
nm.HistorianDataSource = new RecordingHistorianDataSource { RawResult = value, AtTimeResult = value };
|
||||
nm.EnsureVariable("eq/typed", null, "Typed", dataType, false, historianTagname: "T", realm: AddressSpaceRealm.Uns);
|
||||
HistoryReadDetails details = kind == "Raw"
|
||||
? new ReadRawModifiedDetails { IsReadModified = false, ReturnBounds = false, StartTime = t, EndTime = t.AddSeconds(10) }
|
||||
: new ReadAtTimeDetails { ReqTimes = new DateTimeCollection { t } };
|
||||
var (results, errors) = InvokeHistoryRead(server, nm, details, nm.TryGetVariable("eq/typed")!.NodeId);
|
||||
errors[0].StatusCode.Code.ShouldBe(StatusCodes.Good);
|
||||
var actual = ((HistoryData)ExtensionObject.ToEncodeable(results[0].HistoryData)).DataValues.Single();
|
||||
if (dataType == "Int32") actual.Value.ShouldBeOfType<int>().ShouldBe(1);
|
||||
if (dataType == "Boolean") actual.Value.ShouldBeOfType<bool>().ShouldBeTrue();
|
||||
if (dataType == "Float") actual.Value.ShouldBeOfType<float>().ShouldBe(1f);
|
||||
actual.SourceTimestamp.ShouldBe(t);
|
||||
actual.StatusCode.Code.ShouldBe(StatusCodes.Good);
|
||||
}
|
||||
finally { await host.DisposeAsync(); }
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("Int16", 40000d)]
|
||||
[InlineData("DateTime", 1d)]
|
||||
public async Task An_unrepresentable_historical_value_is_null_with_BadTypeMismatch(string type, double input)
|
||||
{
|
||||
var (host, server) = await BootAsync();
|
||||
try
|
||||
{
|
||||
var nm = server.NodeManager!;
|
||||
var t = DateTime.UtcNow.AddMinutes(-1);
|
||||
nm.EnsureVariable("eq/mismatch", null, "Mismatch", type, false, historianTagname: "T", realm: AddressSpaceRealm.Uns);
|
||||
nm.HistorianDataSource = new RecordingHistorianDataSource { RawResult = new HistorianRead([new(input, StatusCodes.Good, t, t)], null) };
|
||||
var (results, _) = InvokeHistoryRead(server, nm, new ReadRawModifiedDetails { IsReadModified = false, ReturnBounds = false,
|
||||
StartTime = t, EndTime = t.AddSeconds(10) }, nm.TryGetVariable("eq/mismatch")!.NodeId);
|
||||
var value = ((HistoryData)ExtensionObject.ToEncodeable(results[0].HistoryData)).DataValues.Single();
|
||||
value.Value.ShouldBeNull();
|
||||
value.StatusCode.Code.ShouldBe(StatusCodes.BadTypeMismatch);
|
||||
value.SourceTimestamp.ShouldBe(t);
|
||||
value.ServerTimestamp.ShouldBe(t);
|
||||
}
|
||||
finally { await host.DisposeAsync(); }
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Array_and_null_history_values_preserve_quality_and_timestamps()
|
||||
{
|
||||
var (host, server) = await BootAsync();
|
||||
try
|
||||
{
|
||||
var nm = server.NodeManager!;
|
||||
var t = DateTime.UtcNow.AddMinutes(-1);
|
||||
var serverTime = t.AddSeconds(1);
|
||||
nm.EnsureVariable("eq/array", null, "Array", "Int32", false, historianTagname: "T", realm: AddressSpaceRealm.Raw, isArray: true, arrayLength: 2);
|
||||
nm.HistorianDataSource = new RecordingHistorianDataSource { RawResult = new HistorianRead([
|
||||
new(new int[] { 1, 2 }, StatusCodes.Uncertain, t, serverTime),
|
||||
new(null, StatusCodes.BadNoCommunication, t.AddSeconds(1), serverTime)], null) };
|
||||
var (results, _) = InvokeHistoryRead(server, nm, new ReadRawModifiedDetails { IsReadModified = false, ReturnBounds = false,
|
||||
StartTime = t, EndTime = t.AddSeconds(10) }, nm.TryGetVariable("eq/array", AddressSpaceRealm.Raw)!.NodeId);
|
||||
var values = ((HistoryData)ExtensionObject.ToEncodeable(results[0].HistoryData)).DataValues;
|
||||
values[0].Value.ShouldBeOfType<int[]>().ShouldBe([1, 2]);
|
||||
values[0].StatusCode.Code.ShouldBe(StatusCodes.Uncertain);
|
||||
values[0].SourceTimestamp.ShouldBe(t);
|
||||
values[0].ServerTimestamp.ShouldBe(serverTime);
|
||||
values[1].Value.ShouldBeNull();
|
||||
values[1].StatusCode.Code.ShouldBe(StatusCodes.BadNoCommunication);
|
||||
}
|
||||
finally { await host.DisposeAsync(); }
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Processed_Count_keeps_its_aggregate_type_on_a_Boolean_node()
|
||||
{
|
||||
var (host, server) = await BootAsync();
|
||||
try
|
||||
{
|
||||
var nm = server.NodeManager!;
|
||||
var t = DateTime.UtcNow.AddMinutes(-1);
|
||||
nm.EnsureVariable("eq/count", null, "Count", "Boolean", false, historianTagname: "T", realm: AddressSpaceRealm.Uns);
|
||||
nm.HistorianDataSource = new RecordingHistorianDataSource { ProcessedResult = new HistorianRead([new(3d, StatusCodes.Good, t, t)], null) };
|
||||
var (results, _) = InvokeHistoryRead(server, nm, new ReadProcessedDetails { StartTime = t, EndTime = t.AddSeconds(10),
|
||||
ProcessingInterval = 1000, AggregateType = new NodeIdCollection { ObjectIds.AggregateFunction_Count } }, nm.TryGetVariable("eq/count")!.NodeId);
|
||||
var value = ((HistoryData)ExtensionObject.ToEncodeable(results[0].HistoryData)).DataValues.Single();
|
||||
value.Value.ShouldBeOfType<double>().ShouldBe(3d);
|
||||
}
|
||||
finally { await host.DisposeAsync(); }
|
||||
}
|
||||
|
||||
private static HistorianRead Empty() => new(Array.Empty<DataValueSnapshot>(), null);
|
||||
|
||||
/// <summary>A recording fake historian source — captures the last call's kind + arguments and returns
|
||||
@@ -721,6 +919,7 @@ public sealed class NodeManagerHistoryReadTests : IDisposable
|
||||
private sealed class RecordingHistorianDataSource : IHistorianDataSource
|
||||
{
|
||||
public bool ThrowOnRead { get; init; }
|
||||
public Exception? ReadException { get; init; }
|
||||
public bool AggregateNotSupported { get; init; }
|
||||
public bool LimitExceeded { get; init; }
|
||||
public IReadOnlySet<HistoryAggregateType>? Supported { get; init; } = HistoryAggregateTypes.All;
|
||||
@@ -742,6 +941,7 @@ public sealed class NodeManagerHistoryReadTests : IDisposable
|
||||
string fullReference, DateTime startUtc, DateTime endUtc, uint maxValuesPerNode,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (ReadException is not null) throw ReadException;
|
||||
if (ThrowOnRead) throw new InvalidOperationException("backend boom");
|
||||
LastCall = "Raw";
|
||||
LastTagname = fullReference;
|
||||
@@ -755,6 +955,7 @@ public sealed class NodeManagerHistoryReadTests : IDisposable
|
||||
string fullReference, DateTime startUtc, DateTime endUtc, TimeSpan interval,
|
||||
HistoryAggregateType aggregate, CancellationToken cancellationToken)
|
||||
{
|
||||
if (ReadException is not null) throw ReadException;
|
||||
if (ThrowOnRead) throw new InvalidOperationException("backend boom");
|
||||
if (AggregateNotSupported) throw new HistoryAggregateNotSupportedException(aggregate);
|
||||
if (LimitExceeded) throw new HistoryReadLimitExceededException(4, "too many raw rows");
|
||||
@@ -770,6 +971,7 @@ public sealed class NodeManagerHistoryReadTests : IDisposable
|
||||
public Task<HistorianRead> ReadAtTimeAsync(
|
||||
string fullReference, IReadOnlyList<DateTime> timestampsUtc, CancellationToken cancellationToken)
|
||||
{
|
||||
if (ReadException is not null) throw ReadException;
|
||||
if (ThrowOnRead) throw new InvalidOperationException("backend boom");
|
||||
LastCall = "AtTime";
|
||||
LastTagname = fullReference;
|
||||
|
||||
@@ -0,0 +1,142 @@
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #694: <see cref="RawPageLimit"/> — the page plan (P = min(want, L − k − 1), request = P + k + 1 ≤ L) and the one
|
||||
/// re-plan after a refusal. The end-to-end behaviour over the real Gateway data source is pinned in
|
||||
/// <c>GatewayCeilingPagingTests</c>.
|
||||
/// </summary>
|
||||
public sealed class RawPageLimitTests
|
||||
{
|
||||
[Theory]
|
||||
// want, k, L ⇒ page, request, noRoom
|
||||
[InlineData(5000u, 0, 0, 5000u, 5001u, false)] // no limit: #692 unchanged (page + ties + lookahead)
|
||||
[InlineData(5000u, 3, 0, 5000u, 5004u, false)]
|
||||
[InlineData(0u, 3, 0, 0u, 0u, false)] // no limit, uncapped: one uncapped read
|
||||
[InlineData(5000u, 0, 984, 983u, 984u, false)] // over the limit: the request fills the limit exactly
|
||||
[InlineData(5000u, 10, 984, 973u, 984u, false)] // boundary ties take room from the page
|
||||
[InlineData(10u, 10, 984, 10u, 21u, false)] // under the limit: unchanged
|
||||
[InlineData(0u, 0, 984, 983u, 984u, false)] // uncapped: a limit-sized page
|
||||
[InlineData(0u, 982, 984, 1u, 984u, false)]
|
||||
[InlineData(5000u, 983, 984, 0u, 0u, true)] // the ties alone fill a read: never a 0 (uncapped) request
|
||||
[InlineData(5000u, 5000, 984, 0u, 0u, true)] // never an underflow
|
||||
[InlineData(5000u, 0, 1, 0u, 0u, true)]
|
||||
public void Plan_fits_page_ties_and_lookahead_into_the_limit(uint want, int k, int limit, uint page, uint request, bool noRoom)
|
||||
{
|
||||
var plan = RawPageLimit.Plan(want, k, limit);
|
||||
plan.PageCap.ShouldBe(page);
|
||||
plan.RequestCap.ShouldBe(request);
|
||||
plan.NoRoom.ShouldBe(noRoom);
|
||||
if (limit > 0 && !noRoom) plan.RequestCap.ShouldBeLessThanOrEqualTo((uint)limit);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_read_that_teaches_the_limit_is_re_planned_once_at_it()
|
||||
{
|
||||
var source = new LimitSource();
|
||||
var asked = new List<uint>();
|
||||
var (page, plan) = await RawPageLimit.ReadAsync(source, 5000u, 0, p =>
|
||||
{
|
||||
asked.Add(p.RequestCap);
|
||||
if (p.RequestCap > 984) { source.Limit = 984; throw new HistoryReadLimitExceededException(1000, "ceiling"); }
|
||||
return Task.FromResult("ok");
|
||||
});
|
||||
page.ShouldBe("ok");
|
||||
plan.PageCap.ShouldBe(983u);
|
||||
asked.ShouldBe([5001u, 984u]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_crowded_refusal_shrinks_the_page_by_the_pre_window_rows()
|
||||
{
|
||||
var source = new LimitSource { Limit = 984 };
|
||||
var asked = new List<uint>();
|
||||
var (_, plan) = await RawPageLimit.ReadAsync(source, 0u, 2, p =>
|
||||
{
|
||||
asked.Add(p.RequestCap);
|
||||
if (asked.Count == 1) throw new HistoryReadLimitExceededException(1000, "crowded") { PreWindowRows = 21 };
|
||||
return Task.FromResult("ok");
|
||||
});
|
||||
asked.ShouldBe([984u, 963u]);
|
||||
plan.PageCap.ShouldBe(960u);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_refusal_at_an_unchanged_limit_without_crowding_is_the_answer()
|
||||
{
|
||||
var source = new LimitSource { Limit = 984 };
|
||||
var calls = 0;
|
||||
await Should.ThrowAsync<HistoryReadLimitExceededException>(() => RawPageLimit.ReadAsync<string>(source, 5000u, 0, _ =>
|
||||
{
|
||||
calls++;
|
||||
throw new HistoryReadLimitExceededException(1000, "tie cluster");
|
||||
}));
|
||||
calls.ShouldBe(1);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_second_refusal_is_not_retried_again()
|
||||
{
|
||||
var source = new LimitSource();
|
||||
var calls = 0;
|
||||
await Should.ThrowAsync<HistoryReadLimitExceededException>(() => RawPageLimit.ReadAsync<string>(source, 0u, 0, _ =>
|
||||
{
|
||||
calls++;
|
||||
source.Limit = 984 - calls; // even if the limit keeps moving
|
||||
throw new HistoryReadLimitExceededException(1000, "refused") { PreWindowRows = 5 };
|
||||
}));
|
||||
calls.ShouldBe(2);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task No_room_is_returned_without_a_read()
|
||||
{
|
||||
var source = new LimitSource { Limit = 10 };
|
||||
var calls = 0;
|
||||
var (page, plan) = await RawPageLimit.ReadAsync(source, 5u, 9, _ => { calls++; return Task.FromResult("x"); });
|
||||
plan.NoRoom.ShouldBeTrue();
|
||||
page.ShouldBeNull();
|
||||
calls.ShouldBe(0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Other_failures_are_not_retried()
|
||||
{
|
||||
var source = new LimitSource();
|
||||
var calls = 0;
|
||||
await Should.ThrowAsync<InvalidOperationException>(() => RawPageLimit.ReadAsync<string>(source, 5000u, 0, _ =>
|
||||
{
|
||||
calls++;
|
||||
source.Limit = 984;
|
||||
throw new InvalidOperationException("backend fault");
|
||||
}));
|
||||
calls.ShouldBe(1);
|
||||
}
|
||||
|
||||
/// <summary>Only <see cref="IHistorianDataSource.MaxRawValuesPerRead"/> matters here.</summary>
|
||||
private sealed class LimitSource : IHistorianDataSource
|
||||
{
|
||||
public int Limit;
|
||||
|
||||
public int MaxRawValuesPerRead => Limit;
|
||||
|
||||
public Task<HistoryReadResult> ReadRawAsync(string fullReference, DateTime startUtc, DateTime endUtc, uint maxValuesPerNode,
|
||||
CancellationToken cancellationToken) => throw new NotSupportedException();
|
||||
|
||||
public Task<HistoryReadResult> ReadProcessedAsync(string fullReference, DateTime startUtc, DateTime endUtc, TimeSpan interval,
|
||||
HistoryAggregateType aggregate, CancellationToken cancellationToken) => throw new NotSupportedException();
|
||||
|
||||
public Task<HistoryReadResult> ReadAtTimeAsync(string fullReference, IReadOnlyList<DateTime> timestampsUtc,
|
||||
CancellationToken cancellationToken) => throw new NotSupportedException();
|
||||
|
||||
public Task<HistoricalEventsResult> ReadEventsAsync(string? sourceName, DateTime startUtc, DateTime endUtc, int maxEvents,
|
||||
CancellationToken cancellationToken) => throw new NotSupportedException();
|
||||
|
||||
public HistorianHealthSnapshot GetHealthSnapshot() => NullHistorianDataSource.Instance.GetHealthSnapshot();
|
||||
|
||||
public void Dispose() { }
|
||||
}
|
||||
}
|
||||
@@ -119,6 +119,24 @@ public sealed class ServerHistorianOptionsTests
|
||||
{ Enabled = true, Endpoint = "https://h:5222", ApiKey = "histgw_x_y", HonorReturnBounds = false }.Validate());
|
||||
}
|
||||
|
||||
/// <summary>#694: the events horizon defaults off, binds as a TimeSpan, and only a negative value warns.</summary>
|
||||
[Fact]
|
||||
public void EventReadHorizon_defaults_off_binds_and_only_a_negative_value_warns()
|
||||
{
|
||||
Assert.Equal(TimeSpan.Zero, new ServerHistorianOptions().EventReadHorizon);
|
||||
var bound = new ConfigurationBuilder()
|
||||
.AddInMemoryCollection(new Dictionary<string, string?> { ["ServerHistorian:EventReadHorizon"] = "3650.00:00:00" })
|
||||
.Build()
|
||||
.GetSection(ServerHistorianOptions.SectionName)
|
||||
.Get<ServerHistorianOptions>();
|
||||
Assert.Equal(TimeSpan.FromDays(3650), bound!.EventReadHorizon);
|
||||
Assert.Empty(new ServerHistorianOptions
|
||||
{ Enabled = true, Endpoint = "https://h:5222", ApiKey = "histgw_x_y", EventReadHorizon = TimeSpan.FromDays(30) }.Validate());
|
||||
var w = new ServerHistorianOptions
|
||||
{ Enabled = true, Endpoint = "https://h:5222", ApiKey = "histgw_x_y", EventReadHorizon = TimeSpan.FromDays(-1) }.Validate();
|
||||
Assert.Contains(w, m => m.Contains("EventReadHorizon"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void HistoryUpdateMaxFutureSkew_defaults_to_one_day_binds_and_zero_is_silent()
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user