Merge fix/690-691-gateway-mapping: Bad Gateway rows carry no value, learned gateway ceiling, native TimeAverage/Total grid both directions (#690 #691.2 #691.3)
This commit is contained in:
Joseph Doherty
@@ -1364,8 +1364,8 @@ historian tagname, which is the tag's driver `FullName`.
|
||||
|
||||
The server dispatches all OPC UA HistoryRead to the registered `IHistorianDataSource` — the
|
||||
`GatewayHistorianDataSource` read client when enabled, else the `NullHistorianDataSource` default
|
||||
(historized nodes return `GoodNoData` for Raw/AtTime — Processed is `BadAggregateNotSupported` since #682). Supported variants: Raw (⚠️ #687, client-visible: Gateway raw reads no longer return AVEVA's synthetic start-boundary row — asked from 1 ms early (epoch-floored) with `NumValuesPerNode + 16` rows (unclamped; 0 ⇒ `int.MaxValue`) and filtered by timestamp — so an empty window is `GoodNoData`, not a fabricated value at StartTime; the forward end is now exclusive (a point read inclusive); pre-window rows that crowd the budget trigger ONE re-read with room for them, else `BadTooManyOperations`; `ReturnBounds` is ignored on every backend), Processed
|
||||
(backend-dependent: SqlServer/QuestDb serve seventeen; Gateway twelve = `GatewayHistorianDataSource.ServedAggregates` — ⚠️ #681, client-visible: the ten per-sample ones (Average, Count, Min/Max, Range, Start/End/Delta, Min/MaxActualTime) are evaluated from RAW gateway reads by the shared `Sql.Common` evaluator — identical to the SQL backends, Calculated bits, interval-start stamps, #677 reverse; Min/Max are no longer AVEVA modes, the read moves every raw sample of the window (a month of 1 Hz ⇒ `BadTooManyOperations` at the default `MaxTimeWeightedRawRows` cap), and every gateway stream is consumed to completion (an abandoned one makes the gateway evict its shared historian session) — plus TimeAverage/Total via AVEVA modes (interval-start stamped + Calculated; ⚠️ #685: AVEVA's extra pre-window row is dropped ⇒ one row per interval in `[Start, End)`, and an Uncertain (opc 64) interval keeps its value as `UncertainDataSubNormal` — AVEVA's TWA excludes NULL-gap time where the SQL backends bridge it; per-interval values still need gateway fix historiangw#27); the other time-weighted five stay `BadAggregateNotSupported` because the gateway has no prior-sample read for Part 13 bounds; `docs/Historian.md` §Aggregate set), and AtTime over historized variable nodes; Events over
|
||||
(historized nodes return `GoodNoData` for Raw/AtTime — Processed is `BadAggregateNotSupported` since #682). Supported variants: Raw (⚠️ #687, client-visible: Gateway raw reads no longer return AVEVA's synthetic start-boundary row — asked from 1 ms early (epoch-floored) with `NumValuesPerNode + 16` rows (unclamped; 0 ⇒ `int.MaxValue`) and filtered by timestamp — so an empty window is `GoodNoData`, not a fabricated value at StartTime; the forward end is now exclusive (a point read inclusive); pre-window rows that crowd the budget trigger ONE re-read with room for them, else `BadTooManyOperations`; `ReturnBounds` is ignored on every backend; ⚠️ #690: a Gateway Bad row — a stored NULL, which histsdk sends as value 0 — now reads value **null** + Bad on Raw and AtTime, as on the SQL backends; ⚠️ #691.3: a strict gateway (`AllowStreamingBeyondCeiling=false`) that refuses a raw budget is re-asked ONCE at the ceiling its refusal names — raw and per-sample reads alike; the ceiling is then remembered per data source (`GatewayRowCeiling`: refusals count as gateway errors), re-learned from a refusal when lowered, and re-probed once at the full budget when a ceiling-bounded read comes back full (raised / streaming) — so an uncapped (`NumValuesPerNode = 0`) read of a small window is served; a ceiling-bounded stream that is not provably complete is `BadTooManyOperations`, never a truncated answer), Processed
|
||||
(backend-dependent: SqlServer/QuestDb serve seventeen; Gateway twelve = `GatewayHistorianDataSource.ServedAggregates` — ⚠️ #681, client-visible: the ten per-sample ones (Average, Count, Min/Max, Range, Start/End/Delta, Min/MaxActualTime) are evaluated from RAW gateway reads by the shared `Sql.Common` evaluator — identical to the SQL backends, Calculated bits, interval-start stamps, #677 reverse; Min/Max are no longer AVEVA modes, the read moves every raw sample of the window (a month of 1 Hz ⇒ `BadTooManyOperations` at the default `MaxTimeWeightedRawRows` cap), and every gateway stream is consumed to completion (an abandoned one makes the gateway evict its shared historian session) — plus TimeAverage/Total via AVEVA modes (interval-start stamped + Calculated; ⚠️ #685: AVEVA's extra pre-window row is dropped ⇒ one row per interval in `[Start, End)`, and an Uncertain (opc 64) interval keeps its value as `UncertainDataSubNormal` — AVEVA's TWA excludes NULL-gap time where the SQL backends bridge it; per-interval values still need gateway fix historiangw#27; ⚠️ #691.2: native reads use the SQL backends' grid in both directions via forward requests only — a REVERSE read (refused by the gateway before) is anchored at its later time (#677), and the clipped interval (earliest reverse, LATEST forward — which used to be averaged past EndTime) is its own exact-span read marked Partial unless Bad; an interval wider than the window is sent as the window, interval 0 as one interval; until historiangw#27 lands both directions return Good-status wrong values); the other time-weighted five stay `BadAggregateNotSupported` because the gateway has no prior-sample read for Part 13 bounds; `docs/Historian.md` §Aggregate set), and AtTime over historized variable nodes; Events over
|
||||
alarm-owning equipment-folder event-notifier nodes. Reads are ungated (served from any redundancy node);
|
||||
authorization uses the standard `AccessLevels.HistoryRead` bit set at materialization.
|
||||
|
||||
|
||||
+52
-11
@@ -146,7 +146,7 @@ Processed read is `BadAggregateNotSupported` and no history aggregate functions
|
||||
| `MaxTieClusterOverfetch` | int | `65536` | (Node-manager paging; all backends — a `SqlServer` PK rules out tie clusters, so it is moot there.) Maximum samples the server will fetch in one shot to page through a tie cluster (multiple samples sharing one `SourceTimestamp`). A cluster larger than this ceiling fails `BadTooManyOperations` (#686; was `BadHistoryOperationUnsupported`). Also the per-read row budget of a Gateway reverse raw read (walked back from forward reads, #686): one timestamp holding more samples is refused `BadTooManyOperations`. Values below 1 count as 1. Raise to handle abnormally large tie clusters; the default covers all normal-data cases. |
|
||||
| `HistoryReadBatchParallelism` | int | `4` | Max historized nodes served **concurrently** within one HistoryRead batch (arch-review 03/S3). The SDK's `HistoryRead*` override surface is synchronous, so each arm bridges once at its boundary; this turns per-node serving from sequential (`N × CallTimeout` worst case) into `⌈N / P⌉ × CallTimeout`. Also caps the gateway-load multiplication a single client can command. `≤ 1` serves sequentially. |
|
||||
| `MaxConcurrentHistoryReadBatches` | int | `16` | Process-wide limit on HistoryRead (and HistoryUpdate) batches served **concurrently** (arch-review 03/S3). A flood of history clients degrades to `BadTooManyOperations` rather than exhausting the SDK request-thread pool. With `HistoryReadBatchParallelism` this caps total in-flight gateway reads at `MaxConcurrentHistoryReadBatches × HistoryReadBatchParallelism` (default 16 × 4 = 64). A batch that cannot get a slot within `min(HistoryReadDeadline, 5 s)` fails every handle `BadTooManyOperations`. |
|
||||
| `MaxTimeWeightedRawRows` | int | `1000000` | `SqlServer` / `QuestDb` (#671), and the `Gateway` per-sample aggregates (#681 — every per-sample read there streams raw samples, so the cap applies to the window's raw rows; see [Aggregate set](#aggregate-set-and-badaggregatenotsupported-285)). The most raw in-window samples one [time-weighted aggregate](#time-weighted-aggregates-part-13-671) read (TimeAverage, TimeAverage2, **Total**, Total2, Interpolative, and — since #680 — **PercentGood/PercentBad**) may stream into the server-side evaluator — that family is **not** pushed down to SQL. The window query is capped at this + 1 rows; one row past the cap fails **that node** with `BadTooManyOperations` (not counted as a backend fault) instead of holding a pooled connection until `HistoryReadDeadline` and returning `BadTimeout`. Bound rows outside the window do not count; per-sample aggregates are not capped on the SQL backends (pushed down). 1 000 000 ≈ 11.5 days of a 1 Hz tag. Non-positive = unbounded (a `Validate()` warning). `Gateway` (#688): the row budget it asks the gateway for (`cap + 18`) is clamped at the gateway's default `MaxValuesCeiling` (1,000,000) while the cap is at or under it, so the default works against a gateway with `AllowStreamingBeyondCeiling=false`; against a gateway with a *lower* ceiling, set this to `ceiling − 18`. |
|
||||
| `MaxTimeWeightedRawRows` | int | `1000000` | `SqlServer` / `QuestDb` (#671), and the `Gateway` per-sample aggregates (#681 — every per-sample read there streams raw samples, so the cap applies to the window's raw rows; see [Aggregate set](#aggregate-set-and-badaggregatenotsupported-285)). The most raw in-window samples one [time-weighted aggregate](#time-weighted-aggregates-part-13-671) read (TimeAverage, TimeAverage2, **Total**, Total2, Interpolative, and — since #680 — **PercentGood/PercentBad**) may stream into the server-side evaluator — that family is **not** pushed down to SQL. The window query is capped at this + 1 rows; one row past the cap fails **that node** with `BadTooManyOperations` (not counted as a backend fault) instead of holding a pooled connection until `HistoryReadDeadline` and returning `BadTimeout`. Bound rows outside the window do not count; per-sample aggregates are not capped on the SQL backends (pushed down). 1 000 000 ≈ 11.5 days of a 1 Hz tag. Non-positive = unbounded (a `Validate()` warning). `Gateway` (#688): the row budget it asks the gateway for (`cap + 18`) is clamped at the gateway's default `MaxValuesCeiling` (1,000,000) while the cap is at or under it, so the default works against a gateway with `AllowStreamingBeyondCeiling=false`; against a gateway with a *lower* ceiling the refused budget is retried once at the ceiling the gateway names (#691.3), so no change is needed (`ceiling − 18` only saves that round-trip). |
|
||||
| `HistoryReadDeadline` | TimeSpan | `00:01:00` | Per-request wall-clock deadline for a HistoryRead (and HistoryUpdate) batch (arch-review 03/S3). A node still in flight at the deadline surfaces `BadTimeout`, so one request can never hold an SDK request thread longer than this regardless of node count. Non-positive = unbounded (a `Validate()` warning). |
|
||||
| `HistoryUpdateMaxFutureSkew` | TimeSpan | `1.00:00:00` | #674. How far past the server clock a HistoryUpdate (Insert / Replace / Update) item's `SourceTimestamp` may lie; an item beyond `now + skew` is refused **`BadOutOfRange`** by the node manager before any backend sees it (every backend). Guards against a far-future typo (a 2062 timestamp). The live recorder is not bounded. Non-positive disables the bound (no warning — a deliberate choice). See [HistoryUpdate](#historyupdate). |
|
||||
|
||||
@@ -664,8 +664,9 @@ backend:
|
||||
> ⚠️ **Also fixed with #686: uncapped raw reads on Gateway.** `NumValuesPerNode = 0` ("all values") used to be forwarded as
|
||||
> `max_values = 0`, which the client package and the gateway reject — every uncapped raw read (forward too) failed the node
|
||||
> and counted as a backend failure. It is now sent unbounded (`int.MaxValue`); a gateway configured with
|
||||
> `AllowStreamingBeyondCeiling=false` refuses that over its `MaxValuesCeiling`, which is answered **`BadTooManyOperations`**
|
||||
> (not a backend fault). A cancelled read (client cancel / `HistoryReadDeadline`) is no longer counted as a backend failure.
|
||||
> `AllowStreamingBeyondCeiling=false` refuses that over its `MaxValuesCeiling`; since #691.3 the read is retried once at the
|
||||
> ceiling the refusal names and answered when provably complete, else **`BadTooManyOperations`** (not a backend fault) — see
|
||||
> [#687](#gateway-raw-reads-return-stored-samples-only-687) "Strict gateway". A cancelled read (client cancel / `HistoryReadDeadline`) is no longer counted as a backend failure.
|
||||
|
||||
Continuation points are bound to the OPC UA session (the SDK's
|
||||
`ServerConfiguration.MaxHistoryContinuationPoints` cap, default 100, with oldest-eviction; points
|
||||
@@ -686,7 +687,12 @@ data.
|
||||
the gateway is asked from **1 ms before** the window (never before the Unix epoch, the start of the historian's time encoding),
|
||||
so the boundary row lands before it, and only rows `RawAggregateReader.IsWindowSample` are returned. The decision is by
|
||||
**timestamp**, not by value or quality — a stored NULL sample looks exactly like the no-data boundary row on the wire (value 0,
|
||||
Bad), and a stored NULL inside the window is still returned (Bad), as it is a stored sample. A row stamped **at or before the
|
||||
Bad), and a stored NULL inside the window is still returned (Bad), as it is a stored sample. ⚠️ **Since #690 it is returned
|
||||
with a null value** (client-visible): histsdk puts a value on every row, so the NULL used to come back as value `0` + Bad (`""`
|
||||
on a string tag), a fabricated 0 for any client that reads the value without the status. `SampleMapper.ToSnapshot` drops the
|
||||
value of every row whose mapped status is **Bad** (AVEVA stores a Bad-quality sample as NULL, so a Bad row never carries a real
|
||||
value) — raw reads and AtTime alike, matching SqlServer/QuestDB (an Empty sample reads null + Bad). Uncertain rows keep their
|
||||
value; the raw-evaluated aggregates were already unaffected (a Bad sample is never usable, and Start/End null a Bad value). A row stamped **at or before the
|
||||
epoch** is never a stored sample (AVEVA stores nothing there — the #686 walk-back's floor too): a window starting at or before
|
||||
the epoch gets no margin, so AVEVA's no-data row lands exactly at its start; raw reads and the per-sample aggregates both drop it.
|
||||
- **The forward end is exclusive** (`[StartTime, EndTime)`, the `IHistorianDataSource` contract and what the SQL backends return;
|
||||
@@ -696,8 +702,23 @@ data.
|
||||
- **Row budget.** `NumValuesPerNode = 0` ("all values") asks the gateway for an unbounded `max_values` (`int.MaxValue` — the client
|
||||
package and the gateway reject 0, #686). A cap asks for `NumValuesPerNode + 16` rows, **not** clamped at the gateway's default
|
||||
ceiling (#688 clamps only the per-sample aggregate budget): AVEVA always emits the boundary row, so a clamped budget would make
|
||||
every page size from 999,985 to 1,000,000 come back short. A gateway with `AllowStreamingBeyondCeiling=false` refuses an
|
||||
over-ceiling budget instead ⇒ `BadTooManyOperations` (not a backend fault); a default gateway serves it.
|
||||
every page size from 999,985 to 1,000,000 come back short. A default gateway serves any budget.
|
||||
- ⚠️ **Strict gateway (#691.3, client-visible).** A gateway with `AllowStreamingBeyondCeiling=false` refuses a budget over its
|
||||
`MaxValuesCeiling` (before streaming) and names the ceiling in the refusal; the gateway has no RPC that reports it. The read is
|
||||
then **re-issued once with the ceiling as the budget**, and the ceiling is **remembered per data source** (`GatewayRowCeiling`,
|
||||
shared by raw and per-sample reads): every refused attempt counts as an error in the gateway's own outcome metrics, so after the
|
||||
first refusal later reads ask for at most the learned ceiling and are not refused. A **lowered** ceiling is re-learned from the
|
||||
refusal of the remembered one. A **raised** ceiling (or a switch to streaming) is noticed by the only reads it matters to: when a
|
||||
read bounded by the remembered ceiling comes back full without proving completeness, it is re-probed **once** at its full budget
|
||||
— served ⇒ the ceiling is forgotten; refused naming a larger ceiling ⇒ that one is learned and the read re-issued at it; refused
|
||||
naming the same ceiling ⇒ the bounded result stands. No timer: a TTL would only add refusals for reads that already fit; the cost
|
||||
of the rule is one refused re-probe per genuinely over-ceiling read (which fails either way). So an uncapped read (`NumValuesPerNode = 0`, which used to be refused outright on a strict gateway, even for a 10-row
|
||||
window) and a page size within 16 of the ceiling are now served when the window fits. A ceiling-bounded read is answered only
|
||||
when it is **provably complete** — the stream came back short of the ceiling, or reached `EndTime` (or, capped, filled the page);
|
||||
a stream that delivered the whole ceiling short of `EndTime` may be truncated and fails **`BadTooManyOperations`** (not a backend
|
||||
fault) — set `NumValuesPerNode` to page, or narrow the window. A refusal whose text carries no number falls back to the gateway's
|
||||
default ceiling (1,000,000). Every stream is still drained. The #686 reverse walk-back (segment budget `MaxTieClusterOverfetch + 2`)
|
||||
inherits this: against a ceiling below that budget a dense segment fails `BadTooManyOperations` instead of being narrowed.
|
||||
- **Paging (#669) is unaffected.** A full page still holds exactly `NumValuesPerNode` stored samples and carries a continuation
|
||||
point, and a resumed page (which re-reads from the last returned timestamp inclusive, and the tie-cluster over-fetch
|
||||
`start == end` read) no longer gets a fabricated row at its resume point. A read is complete when the stream came back short,
|
||||
@@ -865,8 +886,26 @@ fixed:
|
||||
`HistorianTagDefinition.IntegralDivisor` is `0`, so `GatewayTagProvisioner` now sends **`IntegralDivisor = 1`** explicitly on every
|
||||
`EnsureTags` — the value·seconds unit the SQL backends' `Total` uses. A tag provisioned earlier keeps its divisor until its next
|
||||
`EnsureTags` (an upsert). The live magnitude check against AVEVA (a constant 1 integrated over 1 s reads 1) passed on 2026-10-12 at the AVEVA
|
||||
level; end to end it waits on historiangw#27 (above). Reverse processed reads of the AVEVA-native `TimeAverage`/`Total` are not re-anchored by #677 (the gateway refuses `end < start`);
|
||||
the raw-evaluated per-sample aggregates (#681) do follow #677. Since #681 every native bucket is stamped with its interval **start**
|
||||
level; end to end it waits on historiangw#27 (above). ⚠️ **Native reads on the SQL backends' interval grid (#691.2, client-visible).** Both directions now use `BucketPlan`, served by
|
||||
**forward** native requests only (`NativeAggregatePlan` — AVEVA anchors a native read's interval grid at the request start, and the
|
||||
gateway refuses `start > end` and `interval <= 0`):
|
||||
- **Reverse** (`StartTime > EndTime`) used to reach the gateway as `start > end` and fail the node (`InvalidArgument`, counted as a backend
|
||||
failure). It now follows #677 like the SQL backends and the raw-evaluated per-sample aggregates (#681): window `(EndTime, StartTime]`,
|
||||
intervals stepping back from `StartTime`, the clipped interval the **earliest**, each stamped with its **later** edge, newest first —
|
||||
one read `[O + I·f, StartTime)` with the interval `I` for the whole intervals (`O = StartTime − n·I`, `f` = 1 when clipped) and a
|
||||
second read over exactly the clipped span `[EndTime, O + I)` with its own width as the interval. A time-weighted value over `[a, b)` and
|
||||
over `(a, b]` is the same integral (two instants apart), so each forward row is the reverse interval's value.
|
||||
- **Forward** windows that are not a whole number of intervals used to send one request whose last AVEVA interval ran **past EndTime**;
|
||||
now the whole intervals `[StartTime, StartTime + (n−1)·I)` are one read and the clipped **latest** interval `[StartTime + (n−1)·I,
|
||||
EndTime)` is its own read over exactly that span. An interval wider than the window is sent as the window, and `interval = 0` as one
|
||||
interval over the window (it used to reach the gateway as 0 and be refused). An empty window (`StartTime == EndTime`) sends nothing.
|
||||
- Stamps match SqlServer/QuestDB (forward: earlier edge, ascending; reverse: later edge, descending), and the clipped interval carries
|
||||
the **Partial** bit unless Bad (Part 13 §5.3.3.2 Case 3, as the SQL backends' time-weighted aggregates — which also mark the intervals
|
||||
the tag's data starts/ends inside; AVEVA's rows carry no data range, so that is not reproduced). Rows are otherwise mapped as before
|
||||
(#685). An interval AVEVA answers no row for is absent (the SQL backends answer it `BadNoData`).
|
||||
- ⚠️ **Until historiangw#27 is fixed, native values are wrong in BOTH directions with a Good status:** AVEVA answers one row per request
|
||||
(the whole span's value), which lands on that request's first interval as a Good, Calculated value while the other intervals are
|
||||
absent. Pinned against a native-mode fake only — re-run the forward and reverse native legs live once the gateway is fixed. Since #681 every native bucket is stamped with its interval **start**
|
||||
(Part 13; it was the interval end) and carries the Calculated info bits (`Good` ⇒ `0x00000401`, no data ⇒ `BadNoData | 0x401`).
|
||||
The live AVEVA gate (2026-10-12) confirmed `start_time` is the interval start (and the `value·s` unit). Previously noted as
|
||||
pending: it reads the aggregate row's `start_time`, which histsdk decodes from a row
|
||||
@@ -912,9 +951,11 @@ and evicts its shared historian session (see the HistorianGateway issue text in
|
||||
streaming gateway and keeps its `+ 18`. Past the cap the rest of the (bounded) stream is drained, then the node fails
|
||||
`BadTooManyOperations`; a stream that delivers the whole budget cannot be proven complete and fails the same way (so with the
|
||||
default cap a window of more than ~999,982 rows fails, as it would one row later). The gateway's own refusal of the budget
|
||||
(`InvalidArgument` "max_values exceeds the server ceiling" — `Historian:ReadLimits:MaxValuesCeiling` lowered below the default,
|
||||
with `AllowStreamingBeyondCeiling=false`) is also `BadTooManyOperations`, not a backend fault — lower `MaxTimeWeightedRawRows` to
|
||||
`ceiling − 18` on such a gateway. A raw read's budget is **not** clamped (see [#687](#gateway-raw-reads-return-stored-samples-only-687)). A caller cancellation (client cancel, `HistoryReadDeadline`) is not counted as a backend fault.
|
||||
(`InvalidArgument` "max_values exceeds the server ceiling of N" — `Historian:ReadLimits:MaxValuesCeiling` lowered below the
|
||||
default, or an unbounded `MaxTimeWeightedRawRows ≤ 0`, with `AllowStreamingBeyondCeiling=false`) is **retried once with N as the
|
||||
budget** (#691.3): a window that fits is served, one that delivers all N rows fails `BadTooManyOperations` as above (not a backend
|
||||
fault). Lowering `MaxTimeWeightedRawRows` to `ceiling − 18` on such a gateway is no longer needed — it only saves the refused
|
||||
round-trip. A raw read's budget is **not** clamped (see [#687](#gateway-raw-reads-return-stored-samples-only-687)). A caller cancellation (client cancel, `HistoryReadDeadline`) is not counted as a backend fault.
|
||||
- **Bucket statistics** are `RawBucketAccumulator`, the C# statement of the SQL backends' per-sample columns — incl. #684:
|
||||
Average / Minimum / Maximum / Range / *ActualTime over **Good** samples only, the Uncertain extremes feeding only the Min/Max
|
||||
status rule, and an Uncertain String/DateTime sample not refusing the Good-value aggregates. A change to those SQL columns
|
||||
|
||||
+72
-33
@@ -47,6 +47,9 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
private readonly int _maxRawRows;
|
||||
private readonly TimeProvider _time;
|
||||
|
||||
/// <summary>A strict gateway's row ceiling, learned from its first refusal (#691.3).</summary>
|
||||
private readonly GatewayRowCeiling _ceiling = new();
|
||||
|
||||
private readonly object _healthLock = new();
|
||||
private DateTime? _lastSuccessUtc;
|
||||
private DateTime? _lastFailureUtc;
|
||||
@@ -144,8 +147,12 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
/// gateway reject <c>max_values <= 0</c>, #686). A cap asks for <c>cap + </c>
|
||||
/// <see cref="RawAggregateReader.PreWindowAllowance"/> rows — NOT clamped at the gateway's default ceiling (#688
|
||||
/// clamps only the per-sample aggregate budget): AVEVA always emits the boundary row, so a clamped budget would make
|
||||
/// every page size within the allowance of the ceiling come back short. A strict gateway refuses an over-ceiling
|
||||
/// budget instead (⇒ <see cref="HistoryReadLimitExceededException"/>, <c>BadTooManyOperations</c>).
|
||||
/// every page size within the allowance of the ceiling come back short. A strict gateway
|
||||
/// (<c>AllowStreamingBeyondCeiling=false</c>) refuses an over-ceiling budget and names its ceiling; the read is
|
||||
/// then re-issued ONCE with the ceiling as budget (#691.3, <see cref="GatewayReadLimits"/>) — so an uncapped read
|
||||
/// of a small window is served — and a ceiling-bounded read that is not provably complete (it delivered the
|
||||
/// whole ceiling without reaching the window end, or a page short of its cap) fails
|
||||
/// <see cref="HistoryReadLimitExceededException"/> (<c>BadTooManyOperations</c>), never a truncated answer.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Completeness.</b> A full page must hold exactly the cap of stored samples — a short page ends the client's
|
||||
@@ -166,22 +173,43 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
// The gateway seam caps with an int; OPC UA hands us a uint, so clamp to int range.
|
||||
var cap = (int)Math.Min(maxValuesPerNode, int.MaxValue);
|
||||
if (cap == 0)
|
||||
return (await ReadRawPassAsync(fullReference, startUtc, endUtc, cap, int.MaxValue, cancellationToken)
|
||||
.ConfigureAwait(false)).Samples;
|
||||
{
|
||||
var all = await PassAsync(int.MaxValue).ConfigureAwait(false);
|
||||
if (all.Result.Complete) return all.Result.Samples;
|
||||
// Only reachable on a strict gateway (#691.3): the ceiling re-read delivered the whole ceiling without reaching
|
||||
// the window end — the window may hold more, and "all values" is never answered from a truncated read.
|
||||
throw new HistoryReadLimitExceededException(cap,
|
||||
$"The raw read of '{fullReference}' holds more than the HistorianGateway's row ceiling ({all.Budget}, its " +
|
||||
"Historian:ReadLimits:MaxValuesCeiling with AllowStreamingBeyondCeiling=false); set NumValuesPerNode to page, " +
|
||||
"or narrow the window.");
|
||||
}
|
||||
|
||||
var first = await ReadRawPassAsync(fullReference, startUtc, endUtc, cap,
|
||||
(int)Math.Min(int.MaxValue, (long)cap + RawAggregateReader.PreWindowAllowance), cancellationToken).ConfigureAwait(false);
|
||||
if (first.Complete) return first.Samples;
|
||||
var asked = (int)Math.Min(int.MaxValue, (long)cap + RawAggregateReader.PreWindowAllowance);
|
||||
var first = await PassAsync(asked).ConfigureAwait(false);
|
||||
if (first.Result.Complete) return first.Result.Samples;
|
||||
|
||||
// A strict gateway's ceiling already bounded that read (#691.3): a larger budget would only be refused again.
|
||||
if (first.Budget < asked)
|
||||
throw new HistoryReadLimitExceededException(cap,
|
||||
$"The raw read of '{fullReference}' could not be read completely within the HistorianGateway's row ceiling " +
|
||||
$"({first.Budget}, its Historian:ReadLimits:MaxValuesCeiling with AllowStreamingBeyondCeiling=false); lower " +
|
||||
"NumValuesPerNode or narrow the window.");
|
||||
|
||||
// The margin crowded the window out: one bounded re-read with room for the pre-window rows already seen.
|
||||
var second = await ReadRawPassAsync(fullReference, startUtc, endUtc, cap,
|
||||
(int)Math.Min(int.MaxValue, (long)cap + first.PreWindowRows + RawAggregateReader.PreWindowAllowance),
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
if (second.Complete) return second.Samples;
|
||||
var second = await PassAsync(
|
||||
(int)Math.Min(int.MaxValue, (long)cap + first.Result.PreWindowRows + RawAggregateReader.PreWindowAllowance))
|
||||
.ConfigureAwait(false);
|
||||
if (second.Result.Complete) return second.Result.Samples;
|
||||
|
||||
throw new HistoryReadLimitExceededException(cap,
|
||||
$"The raw read of '{fullReference}' could not be read completely: more than {second.PreWindowRows} rows lie in " +
|
||||
$"The raw read of '{fullReference}' could not be read completely: more than {second.Result.PreWindowRows} rows lie in " +
|
||||
$"the {RawAggregateReader.BoundaryMargin.TotalMilliseconds} ms before the window and exhausted the row budget.");
|
||||
|
||||
// One drained pass; a strict gateway's refusal of the budget is retried once at the ceiling it names (#691.3).
|
||||
Task<((List<HistorianSample> Samples, int PreWindowRows, bool Complete) Result, int Budget)> PassAsync(int budget) =>
|
||||
_ceiling.ReadAsync(budget,
|
||||
b => ReadRawPassAsync(fullReference, startUtc, endUtc, cap, b, cancellationToken),
|
||||
(r, _) => !r.Complete);
|
||||
}
|
||||
|
||||
/// <summary>One forward raw read of a window, drained, with its pre-window row count and whether it is complete.</summary>
|
||||
@@ -206,7 +234,9 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
else if (sample.Timestamp is not null) reachedEnd = true;
|
||||
}
|
||||
|
||||
var complete = received < budget || samples.Count >= cap || reachedEnd || cap == 0;
|
||||
// cap == 0 ("all values") is complete only when the stream proves it: short of the budget (always, unless a strict
|
||||
// gateway's ceiling became the budget — #691.3) or past the window end.
|
||||
var complete = received < budget || reachedEnd || (cap > 0 && samples.Count >= cap);
|
||||
return (samples, preWindow, complete);
|
||||
}
|
||||
|
||||
@@ -223,7 +253,9 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
/// <para>
|
||||
/// <b>TimeAverage / Total</b> stay native AVEVA retrieval modes (<see cref="AggregateModeMapper"/>), stamped with
|
||||
/// the interval start and the Calculated bit; AVEVA's extra row for the interval before the window is dropped and
|
||||
/// an Uncertain interval keeps its value (#685, <see cref="SampleMapper.ToAggregateSnapshots"/>). Everything else is <see cref="HistoryAggregateNotSupportedException"/>
|
||||
/// an Uncertain interval keeps its value (#685). Both directions use the SQL backends' interval grid with forward
|
||||
/// requests only — a reverse read (which the gateway refuses) anchored at its later time (#677), the clipped interval
|
||||
/// read over exactly its span and marked Partial (#691.2, <see cref="NativeAggregatePlan"/>). Everything else is <see cref="HistoryAggregateNotSupportedException"/>
|
||||
/// (see <see cref="ServedAggregates"/>).
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
@@ -240,22 +272,36 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
// Mapped OUTSIDE the failure-recording try: an aggregate the gateway has no mode for is a client
|
||||
// request the server answers BadAggregateNotSupported, not a backend fault to count (#285).
|
||||
var mode = AggregateModeMapper.ToRetrievalMode(aggregate);
|
||||
// Planned OUTSIDE the failure-recording try: too many intervals is a client request limit, not a backend fault.
|
||||
return await ReadNativeAsync(fullReference, BucketPlan.Create(startUtc, endUtc, interval), mode, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A native TimeAverage / Total read on the SQL backends' interval grid, either direction (#691.2): forward native
|
||||
/// requests only, the clipped interval read over exactly its span. See <see cref="NativeAggregatePlan"/>.
|
||||
/// </summary>
|
||||
private async Task<HistoryReadResult> ReadNativeAsync(string fullReference, BucketPlan plan, RetrievalMode mode,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (plan.BucketCount == 0) return new HistoryReadResult([], ContinuationPoint: null);
|
||||
try
|
||||
{
|
||||
// TimeAverage / Total are native gateway retrieval modes (TimeWeightedAverage / Integral) — no
|
||||
// client-side scaling.
|
||||
var buckets = new List<HistorianAggregateSample>();
|
||||
await foreach (var bucket in _client
|
||||
.ReadAggregateAsync(fullReference, startUtc, endUtc, mode, interval, cancellationToken)
|
||||
.ConfigureAwait(false))
|
||||
var byBucket = new Dictionary<long, DataValueSnapshot>();
|
||||
foreach (var request in NativeAggregatePlan.Requests(plan))
|
||||
{
|
||||
buckets.Add(bucket);
|
||||
var buckets = new List<HistorianAggregateSample>();
|
||||
await foreach (var bucket in _client
|
||||
.ReadAggregateAsync(fullReference, request.StartUtc, request.EndUtc, mode, request.Interval, cancellationToken)
|
||||
.ConfigureAwait(false))
|
||||
{
|
||||
buckets.Add(bucket); // drained to completion, mapped afterwards
|
||||
}
|
||||
NativeAggregatePlan.Place(plan, request, buckets, byBucket);
|
||||
}
|
||||
|
||||
// Filtered while mapping, after the stream was drained (#685: AVEVA's extra pre-window row).
|
||||
var snapshots = SampleMapper.ToAggregateSnapshots(buckets, startUtc, endUtc);
|
||||
RecordOutcome(success: true, error: null);
|
||||
return new HistoryReadResult(snapshots, ContinuationPoint: null);
|
||||
return new HistoryReadResult(NativeAggregatePlan.Assemble(plan, byBucket), ContinuationPoint: null);
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
@@ -280,7 +326,7 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
{
|
||||
var now = _time.GetUtcNow().UtcDateTime;
|
||||
var values = await RawAggregateReader
|
||||
.ReadAsync(_client, fullReference, aggregate, plan, _maxRawRows, now, cancellationToken)
|
||||
.ReadAsync(_client, _ceiling, fullReference, aggregate, plan, _maxRawRows, now, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
RecordOutcome(success: true, error: null);
|
||||
return new HistoryReadResult(values, ContinuationPoint: null);
|
||||
@@ -315,14 +361,7 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
/// True when <paramref name="ex"/> (or an inner exception — the client package wraps the <see cref="RpcException"/>)
|
||||
/// is the gateway's <c>InvalidArgument</c> "max_values exceeds the server ceiling" rejection.
|
||||
/// </summary>
|
||||
internal static bool IsMaxValuesCeilingRejection(Exception ex)
|
||||
{
|
||||
for (var e = ex; e is not null; e = e.InnerException)
|
||||
if (e is RpcException { StatusCode: StatusCode.InvalidArgument } rpc
|
||||
&& rpc.Status.Detail.Contains("max_values exceeds the server ceiling", StringComparison.Ordinal))
|
||||
return true;
|
||||
return false;
|
||||
}
|
||||
internal static bool IsMaxValuesCeilingRejection(Exception ex) => GatewayReadLimits.IsCeilingRejection(ex);
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<HistoryReadResult> ReadAtTimeAsync(
|
||||
|
||||
@@ -0,0 +1,135 @@
|
||||
using System.Globalization;
|
||||
using System.Text.RegularExpressions;
|
||||
using Grpc.Core;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway;
|
||||
|
||||
/// <summary>
|
||||
/// The HistorianGateway's raw-read row ceiling (<c>Historian:ReadLimits:MaxValuesCeiling</c>, default 1,000,000) as seen
|
||||
/// from this side of the wire. With <c>AllowStreamingBeyondCeiling=false</c> (a "strict" gateway) the gateway refuses
|
||||
/// any <c>ReadRaw</c> whose <c>max_values</c> exceeds the ceiling with <c>InvalidArgument</c> — BEFORE it streams a row
|
||||
/// — and names the ceiling in the refusal (<c>"max_values exceeds the server ceiling of {n}; …"</c>). The gateway has
|
||||
/// no RPC that reports the ceiling, so the refusal is how it is learned (#691.3).
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// A read whose budget the gateway refuses is re-issued ONCE with the ceiling as its budget, and the ceiling is
|
||||
/// remembered per data source (<see cref="GatewayRowCeiling"/>); the caller treats a stream that delivered the whole
|
||||
/// ceiling as possibly truncated (never answered from).
|
||||
/// </remarks>
|
||||
internal static class GatewayReadLimits
|
||||
{
|
||||
private const string CeilingRejectionText = "max_values exceeds the server ceiling";
|
||||
|
||||
private static readonly Regex CeilingNumber = new(@"max_values exceeds the server ceiling of (\d+)",
|
||||
RegexOptions.CultureInvariant | RegexOptions.Compiled);
|
||||
|
||||
/// <summary>
|
||||
/// True when <paramref name="ex"/> (or an inner exception — the client package wraps the <see cref="RpcException"/>)
|
||||
/// is the gateway's <c>InvalidArgument</c> "max_values exceeds the server ceiling" rejection.
|
||||
/// </summary>
|
||||
/// <param name="ex">The exception.</param>
|
||||
/// <returns>Whether it is the ceiling rejection.</returns>
|
||||
public static bool IsCeilingRejection(Exception ex) => Rejection(ex) is not null;
|
||||
|
||||
/// <summary>
|
||||
/// The ceiling a ceiling rejection names, or — when <paramref name="ex"/> is a ceiling rejection that does not
|
||||
/// carry a parseable number — the gateway's default (<see cref="RawAggregateReader.GatewayDefaultMaxValuesCeiling"/>).
|
||||
/// </summary>
|
||||
/// <param name="ex">The exception.</param>
|
||||
/// <param name="ceiling">The ceiling (> 0) when this returns true.</param>
|
||||
/// <returns>Whether <paramref name="ex"/> is a ceiling rejection.</returns>
|
||||
public static bool TryGetCeiling(Exception ex, out int ceiling)
|
||||
{
|
||||
ceiling = 0;
|
||||
if (Rejection(ex) is not { } rpc) return false;
|
||||
var m = CeilingNumber.Match(rpc.Status.Detail);
|
||||
ceiling = m.Success && int.TryParse(m.Groups[1].Value, NumberStyles.None, CultureInfo.InvariantCulture, out var n) && n > 0
|
||||
? n
|
||||
: RawAggregateReader.GatewayDefaultMaxValuesCeiling;
|
||||
return true;
|
||||
}
|
||||
|
||||
private static RpcException? Rejection(Exception ex)
|
||||
{
|
||||
for (var e = ex; e is not null; e = e.InnerException)
|
||||
if (e is RpcException { StatusCode: StatusCode.InvalidArgument } rpc
|
||||
&& rpc.Status.Detail.Contains(CeilingRejectionText, StringComparison.Ordinal))
|
||||
return rpc;
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A strict gateway's row ceiling as learned by ONE data source (#691.3 review): every refused attempt counts as an
|
||||
/// error in the gateway's own outcome metrics, so the ceiling a refusal names is remembered and later reads ask for at
|
||||
/// most it — no refusal per read in the steady state. Thread-safe (one <see cref="int"/>, read/written atomically; a
|
||||
/// race costs at most one extra refusal or re-probe, never a wrong answer).
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>Rules.</b> (1) Unknown ceiling: ask for the full budget; a refusal naming <c>N</c> < budget learns <c>N</c> and
|
||||
/// the read is re-issued once at <c>N</c>. (2) Learned ceiling <c>C</c> below the budget: ask for <c>C</c> — a refusal
|
||||
/// there (the gateway was LOWERED) re-learns its <c>N</c> as in (1). (3) If that <c>C</c>-bounded read came back
|
||||
/// full but not provably complete, the gateway may have been RAISED (or switched to streaming) since <c>C</c> was
|
||||
/// learned, and answering <c>BadTooManyOperations</c> from a stale ceiling would be wrong: the read is re-probed ONCE
|
||||
/// at the full budget — served ⇒ the ceiling is forgotten; refused naming a larger <c>N</c> ⇒ <c>N</c> is learned and
|
||||
/// the read is re-issued at <c>N</c>; refused naming <c>C</c> again ⇒ the <c>C</c>-bounded result stands (no repeat).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Why no TTL.</b> A raise only matters to a read that needs more than <c>C</c>, and such a read re-probes anyway
|
||||
/// (rule 3), so a timer would only add refusals for reads that fit. The cost of the rule is one refused re-probe per
|
||||
/// genuinely over-ceiling read — a read that fails <c>BadTooManyOperations</c> whatever is done.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
internal sealed class GatewayRowCeiling
|
||||
{
|
||||
private int _learned; // 0 = unknown / not strict
|
||||
|
||||
/// <summary>The learned ceiling (0 = none).</summary>
|
||||
public int Learned => Volatile.Read(ref _learned);
|
||||
|
||||
/// <summary>Runs a drained read under the rules above.</summary>
|
||||
/// <typeparam name="T">The pass result.</typeparam>
|
||||
/// <param name="budget">The <c>max_values</c> the read wants.</param>
|
||||
/// <param name="pass">One drained read with the given budget.</param>
|
||||
/// <param name="needsMore">True when a pass with the given budget delivered it all without proving completeness.</param>
|
||||
/// <returns>The result and the budget it was read with.</returns>
|
||||
public async Task<(T Result, int Budget)> ReadAsync<T>(int budget, Func<int, Task<T>> pass, Func<T, int, bool> needsMore)
|
||||
{
|
||||
var learned = Learned;
|
||||
if (learned <= 0 || learned >= budget) return await AttemptAsync(budget, pass).ConfigureAwait(false);
|
||||
|
||||
var bounded = await AttemptAsync(learned, pass).ConfigureAwait(false);
|
||||
if (bounded.Budget < learned || !needsMore(bounded.Result, bounded.Budget)) return bounded;
|
||||
|
||||
// Rule 3: full at a possibly stale ceiling — re-probe once at the full budget.
|
||||
int fresh;
|
||||
try
|
||||
{
|
||||
var full = await pass(budget).ConfigureAwait(false);
|
||||
Interlocked.CompareExchange(ref _learned, 0, learned);
|
||||
return (full, budget);
|
||||
}
|
||||
catch (Exception ex) when (GatewayReadLimits.TryGetCeiling(ex, out fresh) && fresh < budget)
|
||||
{
|
||||
// fall through (outside the catch)
|
||||
}
|
||||
Volatile.Write(ref _learned, fresh);
|
||||
return fresh > learned ? (await pass(fresh).ConfigureAwait(false), fresh) : bounded;
|
||||
}
|
||||
|
||||
private async Task<(T Result, int Budget)> AttemptAsync<T>(int budget, Func<int, Task<T>> pass)
|
||||
{
|
||||
int ceiling;
|
||||
try
|
||||
{
|
||||
return (await pass(budget).ConfigureAwait(false), budget);
|
||||
}
|
||||
catch (Exception ex) when (GatewayReadLimits.TryGetCeiling(ex, out ceiling) && ceiling < budget)
|
||||
{
|
||||
// fall through to the retry (outside the catch, so a second refusal is not nested in this one)
|
||||
}
|
||||
Volatile.Write(ref _learned, ceiling);
|
||||
return (await pass(ceiling).ConfigureAwait(false), ceiling);
|
||||
}
|
||||
}
|
||||
@@ -23,11 +23,21 @@ internal static class SampleMapper
|
||||
/// <summary>Maps a single raw sample to a value snapshot.</summary>
|
||||
/// <param name="sample">The gateway raw sample.</param>
|
||||
/// <returns>The driver-agnostic snapshot.</returns>
|
||||
/// <remarks>
|
||||
/// #690: a Bad row carries no value. histsdk puts a value on EVERY row (<c>NumericValue = 0</c> on a numeric row,
|
||||
/// <c>StringValue = ""</c> on a string one), so a stored NULL arrives as <c>0</c> / <c>""</c> with a Bad OPC quality
|
||||
/// (live: <c>q=1 opc=0</c>) — that 0 is not a value, and AVEVA stores a Bad-quality sample as NULL, so a Bad row never
|
||||
/// carries a real one. The value is therefore dropped whenever the mapped status is Bad, matching the SqlServer/QuestDB
|
||||
/// backends (an Empty sample is null + Bad) and the aggregate mapper's no-data rows. Uncertain rows keep their value.
|
||||
/// </remarks>
|
||||
public static DataValueSnapshot ToSnapshot(HistorianSample sample)
|
||||
{
|
||||
var status = ToStatus(sample);
|
||||
// proto3 explicit presence: prefer the numeric value, else the string value, else null.
|
||||
object? value;
|
||||
if (sample.HasNumericValue)
|
||||
if (StatusBits.IsBad(status))
|
||||
value = null;
|
||||
else if (sample.HasNumericValue)
|
||||
value = sample.NumericValue; // boxes as System.Double
|
||||
else if (sample.HasStringValue)
|
||||
value = sample.StringValue;
|
||||
@@ -36,7 +46,7 @@ internal static class SampleMapper
|
||||
|
||||
return new DataValueSnapshot(
|
||||
Value: value,
|
||||
StatusCode: ToStatus(sample),
|
||||
StatusCode: status,
|
||||
SourceTimestampUtc: sample.Timestamp?.ToDateTime(), // Utc kind
|
||||
ServerTimestampUtc: DateTime.UtcNow);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,112 @@
|
||||
using ZB.MOM.WW.HistorianGateway.Contracts.Grpc;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Mapping;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway;
|
||||
|
||||
/// <summary>
|
||||
/// Serves a processed read of the AVEVA-native <c>TimeAverage</c> / <c>Total</c> on the SQL backends' interval grid
|
||||
/// (<see cref="BucketPlan"/>), in either direction: forward — window <c>[start, end)</c>, intervals stepping forward
|
||||
/// from <c>start</c>, the clipped interval the LATEST, each stamped with its earlier edge, oldest first; reverse (#677,
|
||||
/// #691.2) — window <c>(end, start]</c>, intervals stepping back from <c>start</c>, the clipped interval the EARLIEST,
|
||||
/// each stamped with its later edge, newest first. The gateway refuses <c>start > end</c> and <c>interval <= 0</c>,
|
||||
/// so every request it sees is forward with a positive interval.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>Forward requests on the plan's grid.</b> AVEVA anchors a native read's interval grid at the request start (live,
|
||||
/// 2026-10-12: rows at <c>start + j·interval</c>, plus one extra row for the interval before it, #685). So the plan's
|
||||
/// whole intervals are requested as ONE forward read spanning exactly them with the interval <c>I</c>, and the
|
||||
/// clipped interval (when the window is not a whole number of intervals) as a second forward read over exactly its
|
||||
/// span with its own width as the interval — AVEVA never averages time outside the window (before it on a reverse
|
||||
/// read, past <c>EndTime</c> on a forward one). An interval wider than the window is sent as the window, and
|
||||
/// <c>interval <= 0</c> as one interval over the whole window (it used to reach the gateway as 0 and be refused).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Soundness of reverse.</b> A time-weighted average / integral over <c>[a, b)</c> and over <c>(a, b]</c> integrate
|
||||
/// the same function over sets differing by two instants, so a forward native row IS the value of the reverse
|
||||
/// interval with the same edges. The one place the half-open side could matter is AVEVA's status for an interval
|
||||
/// whose only non-good sample lies exactly on an edge — accepted, as on every native row.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Mapping.</b> Each row is mapped as before (<see cref="SampleMapper.ToAggregateSnapshot"/>: Uncertain keeps its
|
||||
/// value, no data ⇒ <c>BadNoData</c>, Calculated bits), placed by its interval start (else end), restamped with
|
||||
/// <see cref="BucketPlan.BucketTimestamp"/>, and the clipped interval carries the Partial bit unless Bad (Part 13
|
||||
/// §5.3.3.2 Case 3, as the SQL backends' time-weighted aggregates — which additionally mark the intervals the data
|
||||
/// starts/ends inside; AVEVA's rows carry no data range, so those are not reproduced). An interval AVEVA answers no
|
||||
/// row for is absent (not a fabricated <c>BadNoData</c>, unlike the SQL backends).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// ⚠️ Today historiangw#27 (the interval reaches AVEVA in ticks where it reads milliseconds) makes AVEVA answer ONE
|
||||
/// Good row per request — the whole-intervals read lands on its earliest interval with a whole-span value and the rest
|
||||
/// are absent — in both directions. The mapping needs no change once the gateway is fixed.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
internal static class NativeAggregatePlan
|
||||
{
|
||||
/// <summary>One forward native request and the ascending bucket index of its first interval.</summary>
|
||||
/// <param name="StartUtc">Forward request start (inclusive).</param>
|
||||
/// <param name="EndUtc">Forward request end.</param>
|
||||
/// <param name="Interval">The request's interval (> 0).</param>
|
||||
/// <param name="FirstBucket">Ascending plan bucket of the request's first interval.</param>
|
||||
/// <param name="BucketCount">Number of plan buckets the request covers.</param>
|
||||
internal readonly record struct Request(DateTime StartUtc, DateTime EndUtc, TimeSpan Interval, long FirstBucket, long BucketCount);
|
||||
|
||||
/// <summary>The forward requests serving a plan: the whole intervals first, then the clipped one (if any).</summary>
|
||||
/// <param name="plan">A plan with at least one bucket.</param>
|
||||
/// <returns>One or two forward requests.</returns>
|
||||
public static IReadOnlyList<Request> Requests(BucketPlan plan)
|
||||
{
|
||||
var interval = TimeSpan.FromTicks(plan.IntervalTicks);
|
||||
var n = plan.BucketCount;
|
||||
var clipped = plan.IsTruncated(0) ? 0L : plan.IsTruncated(n - 1) ? n - 1 : -1L;
|
||||
if (clipped < 0) return [new Request(plan.StartUtc, plan.EndUtc, interval, 0, n)];
|
||||
|
||||
var clipLo = new DateTime(plan.LowerTicks(clipped), DateTimeKind.Utc);
|
||||
var clipHi = new DateTime(plan.UpperTicks(clipped), DateTimeKind.Utc);
|
||||
var requests = new List<Request>(2);
|
||||
if (n > 1)
|
||||
requests.Add(clipped == 0
|
||||
? new Request(clipHi, plan.EndUtc, interval, 1, n - 1) // reverse: the clipped one is the earliest
|
||||
: new Request(plan.StartUtc, clipLo, interval, 0, n - 1)); // forward: the clipped one is the latest
|
||||
requests.Add(new Request(clipLo, clipHi, clipHi - clipLo, clipped, 1));
|
||||
return requests;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Maps a request's rows onto the plan's buckets: rows outside the request window (AVEVA's pre-window row, #685)
|
||||
/// are dropped; each in-window row is placed by its interval start (or, without one, its end).
|
||||
/// </summary>
|
||||
/// <param name="plan">The plan.</param>
|
||||
/// <param name="request">The forward request the rows answer.</param>
|
||||
/// <param name="rows">The drained rows.</param>
|
||||
/// <param name="byBucket">Results by ascending bucket index (first row per bucket wins).</param>
|
||||
public static void Place(BucketPlan plan, Request request, IEnumerable<HistorianAggregateSample> rows,
|
||||
Dictionary<long, DataValueSnapshot> byBucket)
|
||||
{
|
||||
var interval = request.Interval.Ticks;
|
||||
foreach (var row in rows)
|
||||
{
|
||||
if (!SampleMapper.IsInWindow(row, request.StartUtc, request.EndUtc)) continue;
|
||||
long offset = row.StartTime is { } s
|
||||
? (s.ToDateTime().Ticks - request.StartUtc.Ticks) / interval
|
||||
: (row.EndTime.ToDateTime().Ticks - request.StartUtc.Ticks - 1) / interval; // IsInWindow ⇒ end in (lo, hi]
|
||||
if (offset < 0 || offset >= request.BucketCount) continue;
|
||||
var index = request.FirstBucket + offset;
|
||||
if (byBucket.ContainsKey(index)) continue;
|
||||
|
||||
var snapshot = SampleMapper.ToAggregateSnapshot(row);
|
||||
var status = snapshot.StatusCode;
|
||||
if (plan.IsTruncated(index) && !StatusBits.IsBad(status)) status |= HistorianInfoBits.HistorianPartial;
|
||||
byBucket[index] = snapshot with { StatusCode = status, SourceTimestampUtc = plan.BucketTimestamp(index) };
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>The placed results in the read's order (ascending forward, descending reverse).</summary>
|
||||
/// <param name="plan">The plan.</param>
|
||||
/// <param name="byBucket">Results by ascending bucket index.</param>
|
||||
/// <returns>The results.</returns>
|
||||
public static IReadOnlyList<DataValueSnapshot> Assemble(BucketPlan plan, IReadOnlyDictionary<long, DataValueSnapshot> byBucket) =>
|
||||
(plan.Reversed ? byBucket.OrderByDescending(kv => kv.Key) : byBucket.OrderBy(kv => kv.Key)).Select(kv => kv.Value).ToList();
|
||||
}
|
||||
@@ -49,7 +49,10 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway;
|
||||
/// requested row budget — has been drained. The gateway is asked for at most <c>cap + 2 + </c>
|
||||
/// <see cref="PreWindowAllowance"/> rows, clamped at <see cref="GatewayDefaultMaxValuesCeiling"/> while the cap is at
|
||||
/// or under it (#688, <see cref="ClampToGatewayCeiling"/>); a stream that delivers all of them cannot be proven
|
||||
/// complete and fails the same way rather than answering from a truncated window.
|
||||
/// complete and fails the same way rather than answering from a truncated window. A strict gateway whose ceiling is
|
||||
/// below that budget (a ceiling under 1,000,000, or an unbounded cap) refuses it naming its ceiling; the window is
|
||||
/// then re-read once with the ceiling as budget (#691.3, <see cref="GatewayReadLimits"/>), and the same
|
||||
/// "delivered the whole budget ⇒ cannot be proven complete" rule applies.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Values and status.</b> A sample's value is its numeric value (Double), else its string value, else Empty; its
|
||||
@@ -163,6 +166,7 @@ internal static class RawAggregateReader
|
||||
|
||||
/// <summary>Reads and evaluates one per-sample processed read.</summary>
|
||||
/// <param name="client">The gateway client.</param>
|
||||
/// <param name="ceiling">The data source's learned gateway row ceiling (#691.3).</param>
|
||||
/// <param name="tag">Historian tag name.</param>
|
||||
/// <param name="aggregate">A per-sample aggregate (<see cref="Serves"/>).</param>
|
||||
/// <param name="plan">The bucket plan.</param>
|
||||
@@ -172,14 +176,51 @@ internal static class RawAggregateReader
|
||||
/// <returns>One value per interval, in the plan's order.</returns>
|
||||
/// <exception cref="HistoryReadLimitExceededException">The window holds more than <paramref name="maxWindowRows"/> rows,
|
||||
/// or the stream could not be read to completion within the requested row budget.</exception>
|
||||
public static async Task<IReadOnlyList<DataValueSnapshot>> ReadAsync(IHistorianGatewayClient client, string tag,
|
||||
public static async Task<IReadOnlyList<DataValueSnapshot>> ReadAsync(IHistorianGatewayClient client, GatewayRowCeiling ceiling, string tag,
|
||||
HistoryAggregateType aggregate, BucketPlan plan, int maxWindowRows, DateTime now, CancellationToken ct)
|
||||
{
|
||||
if (plan.BucketCount == 0) return [];
|
||||
var needsRange = AggregateEvaluator.NeedsDataRange(aggregate, plan);
|
||||
var requestStart = RequestStartFor(plan.StartUtc);
|
||||
var maxValues = MaxValuesFor(maxWindowRows);
|
||||
|
||||
// A strict gateway that refuses the budget is re-asked once at the ceiling it names (#691.3; e.g. a ceiling below
|
||||
// the 1,000,000 #688 clamps at, or an unbounded cap) — and a stream that fills that ceiling is never evaluated.
|
||||
var requested = MaxValuesFor(maxWindowRows);
|
||||
var (pass, maxValues) = await ceiling.ReadAsync(requested,
|
||||
budget => ReadWindowAsync(client, tag, aggregate, plan, requestStart, budget, maxWindowRows, ct),
|
||||
(r, budget) => !r.OverCap && r.Received >= budget).ConfigureAwait(false);
|
||||
var (buckets, earliest, latest, received, overCap) = pass;
|
||||
if (overCap) throw LimitExceeded(tag, maxWindowRows);
|
||||
if (received >= maxValues) throw BudgetExhausted(tag, maxWindowRows, requested, maxValues);
|
||||
|
||||
if (needsRange)
|
||||
{
|
||||
// Earlier data: no non-Bad pre-window row ⇒ ask explicitly (normally only when the tag's history really starts
|
||||
// inside the window).
|
||||
var earlierSeen = earliest is { } e && (e < plan.StartUtc || (plan.Reversed && e == plan.StartUtc));
|
||||
if (!earlierSeen && plan.StartUtc.AddTicks(-1) > ProbeFloor)
|
||||
if (await FirstSampleAsync(client, tag, ProbeFloor, plan.StartUtc.AddTicks(-1), ct).ConfigureAwait(false) is { } before)
|
||||
earliest = before;
|
||||
|
||||
// Later data: the main read covers a sample exactly at the window end (forward: that is already later data);
|
||||
// otherwise a small separate read from the end to now + horizon.
|
||||
var laterSeen = latest is { } l && (plan.Reversed ? l > plan.EndUtc : l >= plan.EndUtc);
|
||||
var horizon = DateTime.MaxValue - now > LookAheadHorizon
|
||||
? DateTime.SpecifyKind(now + LookAheadHorizon, DateTimeKind.Utc)
|
||||
: DateTime.MaxValue;
|
||||
if (!laterSeen && horizon > plan.EndUtc)
|
||||
if (await FirstSampleAsync(client, tag, plan.EndUtc, horizon, ct).ConfigureAwait(false) is { } after)
|
||||
latest = after;
|
||||
}
|
||||
|
||||
return AggregateEvaluator.EvaluateAll(aggregate, plan, buckets.Rows(), earliest, latest, now);
|
||||
}
|
||||
|
||||
/// <summary>The window read of <see cref="ReadAsync"/> with one row budget, drained.</summary>
|
||||
private static async Task<(RawBucketAccumulator Buckets, DateTime? Earliest, DateTime? Latest, long Received, bool OverCap)>
|
||||
ReadWindowAsync(IHistorianGatewayClient client, string tag, HistoryAggregateType aggregate, BucketPlan plan,
|
||||
DateTime requestStart, int maxValues, int maxWindowRows, CancellationToken ct)
|
||||
{
|
||||
var buckets = new RawBucketAccumulator(aggregate == HistoryAggregateType.MaximumActualTime);
|
||||
DateTime? earliest = null, latest = null;
|
||||
long windowRows = 0, received = 0;
|
||||
@@ -214,30 +255,7 @@ internal static class RawAggregateReader
|
||||
if (t >= plan.StartUtc && (latest is null || t > latest)) latest = t;
|
||||
}
|
||||
}
|
||||
if (overCap || (maxWindowRows > 0 && received >= maxValues))
|
||||
throw LimitExceeded(tag, maxWindowRows);
|
||||
|
||||
if (needsRange)
|
||||
{
|
||||
// Earlier data: no non-Bad pre-window row ⇒ ask explicitly (normally only when the tag's history really starts
|
||||
// inside the window).
|
||||
var earlierSeen = earliest is { } e && (e < plan.StartUtc || (plan.Reversed && e == plan.StartUtc));
|
||||
if (!earlierSeen && plan.StartUtc.AddTicks(-1) > ProbeFloor)
|
||||
if (await FirstSampleAsync(client, tag, ProbeFloor, plan.StartUtc.AddTicks(-1), ct).ConfigureAwait(false) is { } before)
|
||||
earliest = before;
|
||||
|
||||
// Later data: the main read covers a sample exactly at the window end (forward: that is already later data);
|
||||
// otherwise a small separate read from the end to now + horizon.
|
||||
var laterSeen = latest is { } l && (plan.Reversed ? l > plan.EndUtc : l >= plan.EndUtc);
|
||||
var horizon = DateTime.MaxValue - now > LookAheadHorizon
|
||||
? DateTime.SpecifyKind(now + LookAheadHorizon, DateTimeKind.Utc)
|
||||
: DateTime.MaxValue;
|
||||
if (!laterSeen && horizon > plan.EndUtc)
|
||||
if (await FirstSampleAsync(client, tag, plan.EndUtc, horizon, ct).ConfigureAwait(false) is { } after)
|
||||
latest = after;
|
||||
}
|
||||
|
||||
return AggregateEvaluator.EvaluateAll(aggregate, plan, buckets.Rows(), earliest, latest, now);
|
||||
return (buckets, earliest, latest, received, overCap);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -266,6 +284,26 @@ internal static class RawAggregateReader
|
||||
/// </summary>
|
||||
private static bool IsNotBadOnTheWire(HistorianSample raw) => (byte)raw.OpcQuality >= 64; // low byte, as SampleMapper.ToStatus reads it
|
||||
|
||||
/// <summary>
|
||||
/// The stream delivered its whole row budget, so the window cannot be proven complete. Names the limit that actually
|
||||
/// bound the budget: the gateway's ceiling (a strict gateway refused the requested budget and named it, #691.3), the
|
||||
/// #688 clamp at the gateway's default ceiling, or the configured cap.
|
||||
/// </summary>
|
||||
private static HistoryReadLimitExceededException BudgetExhausted(string tag, int maxWindowRows, int requested, int budget)
|
||||
{
|
||||
if (budget < requested)
|
||||
return new HistoryReadLimitExceededException(budget,
|
||||
$"The processed read of '{tag}' covers more raw samples than the HistorianGateway's row ceiling ({budget}, its " +
|
||||
"Historian:ReadLimits:MaxValuesCeiling with AllowStreamingBeyondCeiling=false); the Gateway backend evaluates " +
|
||||
"per-sample aggregates from raw reads — narrow the window.");
|
||||
if (maxWindowRows > 0 && budget < (long)maxWindowRows + 2 + PreWindowAllowance)
|
||||
return new HistoryReadLimitExceededException(budget,
|
||||
$"The processed read of '{tag}' covers more raw samples than the HistorianGateway's default row ceiling " +
|
||||
$"({budget}) the request is clamped at; the Gateway backend evaluates per-sample aggregates from raw reads — " +
|
||||
"narrow the window.");
|
||||
return LimitExceeded(tag, maxWindowRows);
|
||||
}
|
||||
|
||||
private static HistoryReadLimitExceededException LimitExceeded(string tag, int maxWindowRows) =>
|
||||
new(maxWindowRows,
|
||||
$"The processed read of '{tag}' covers more than {maxWindowRows} raw samples " +
|
||||
|
||||
+42
-3
@@ -100,12 +100,35 @@ public sealed class FakeHistorianGatewayClient : IHistorianGatewayClient
|
||||
LastReadRawStartUtc = startUtc;
|
||||
LastReadRawEndUtc = endUtc;
|
||||
LastReadRawMaxValues = maxValues;
|
||||
ReadRawCallCount++;
|
||||
RawCalls.Add((startUtc, endUtc, maxValues));
|
||||
lock (RawCalls)
|
||||
{
|
||||
ReadRawCallCount++;
|
||||
RawCalls.Add((startUtc, endUtc, maxValues));
|
||||
}
|
||||
if (MaxValuesCeiling is { } ceiling && maxValues > ceiling)
|
||||
{
|
||||
// The real gateway's refusal (HistorianReadService.ReadRaw), raised on the first MoveNext before any row, as
|
||||
// the client package surfaces it: a HistorianGatewayException wrapping the RpcException.
|
||||
var rpc = new Grpc.Core.RpcException(new Grpc.Core.Status(Grpc.Core.StatusCode.InvalidArgument,
|
||||
CeilingRejectionDetail ?? $"max_values exceeds the server ceiling of {ceiling}; read large ranges by time-slicing " +
|
||||
"(split the range and read the slices sequentially), or enable " +
|
||||
"Historian:ReadLimits:AllowStreamingBeyondCeiling to page through server-side."));
|
||||
return ToAsyncStream(Array.Empty<HistorianSample>(),
|
||||
new ZB.MOM.WW.HistorianGateway.Client.HistorianGatewayException(rpc.Status.Detail, rpc), ct);
|
||||
}
|
||||
var items = RawSource is null ? RawSamples : RawSource(startUtc, endUtc, maxValues).ToList();
|
||||
return CountingStream(ToAsyncStream(items, ThrowOnRead, ct));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A strict gateway (<c>Historian:ReadLimits:AllowStreamingBeyondCeiling=false</c>) with this
|
||||
/// <c>MaxValuesCeiling</c>: a ReadRaw asking for more is refused <c>InvalidArgument</c> (#691.3).
|
||||
/// </summary>
|
||||
public int? MaxValuesCeiling;
|
||||
|
||||
/// <summary>Overrides the refusal's detail text (an older/other gateway wording).</summary>
|
||||
public string? CeilingRejectionDetail;
|
||||
|
||||
public IAsyncEnumerable<HistorianAggregateSample> ReadAggregateAsync(
|
||||
string tag,
|
||||
DateTime startUtc,
|
||||
@@ -120,9 +143,25 @@ public sealed class FakeHistorianGatewayClient : IHistorianGatewayClient
|
||||
LastAggregateMode = mode;
|
||||
LastAggregateInterval = interval;
|
||||
ReadAggregateCallCount++;
|
||||
return ToAsyncStream(AggregateSamples, ThrowOnRead, ct);
|
||||
AggregateCalls.Add((startUtc, endUtc, mode, interval));
|
||||
if (AggregateSource is null) return ToAsyncStream(AggregateSamples, ThrowOnRead, ct);
|
||||
if (startUtc > endUtc || interval <= TimeSpan.Zero)
|
||||
{
|
||||
// The real gateway's request validation (ValidateRange; interval must be > 0), raised before any row.
|
||||
var rpc = new Grpc.Core.RpcException(new Grpc.Core.Status(Grpc.Core.StatusCode.InvalidArgument,
|
||||
startUtc > endUtc ? "start_time must be less than or equal to end_time." : "interval must be greater than zero."));
|
||||
return ToAsyncStream(Array.Empty<HistorianAggregateSample>(),
|
||||
new ZB.MOM.WW.HistorianGateway.Client.HistorianGatewayException(rpc.Status.Detail, rpc), ct);
|
||||
}
|
||||
return ToAsyncStream(AggregateSource(startUtc, endUtc, mode, interval).ToList(), ThrowOnRead, ct);
|
||||
}
|
||||
|
||||
/// <summary>When set, ReadAggregate serves <c>AggregateSource(start, end, mode, interval)</c> (and refuses start > end).</summary>
|
||||
public Func<DateTime, DateTime, RetrievalMode, TimeSpan, IEnumerable<HistorianAggregateSample>>? AggregateSource;
|
||||
|
||||
/// <summary>Every ReadAggregate call's (start, end, mode, interval), in order.</summary>
|
||||
public readonly List<(DateTime Start, DateTime End, RetrievalMode Mode, TimeSpan Interval)> AggregateCalls = new();
|
||||
|
||||
public Task<IReadOnlyList<HistorianSample>> ReadAtTimeAsync(
|
||||
string tag,
|
||||
IReadOnlyList<DateTime> timestampsUtc,
|
||||
|
||||
+19
-2
@@ -29,12 +29,14 @@ public sealed class GatewayHistorianDataSourceTests
|
||||
Assert.Equal(100 + RawAggregateReader.PreWindowAllowance, fake.LastReadRawMaxValues);
|
||||
}
|
||||
|
||||
private static readonly DateTime Hour0 = new(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc);
|
||||
|
||||
[Fact]
|
||||
public async Task ReadProcessed_uses_aggregate_mode_mapping()
|
||||
{
|
||||
var fake = new FakeHistorianGatewayClient();
|
||||
var ds = new GatewayHistorianDataSource(fake, NullLogger<GatewayHistorianDataSource>.Instance);
|
||||
await ds.ReadProcessedAsync("T", default, default, TimeSpan.FromSeconds(60), HistoryAggregateType.Total, TestContext.Current.CancellationToken);
|
||||
await ds.ReadProcessedAsync("T", Hour0, Hour0.AddHours(1), TimeSpan.FromSeconds(60), HistoryAggregateType.Total, TestContext.Current.CancellationToken); // an empty window sends nothing
|
||||
Assert.Equal(RetrievalMode.Integral, fake.LastAggregateMode);
|
||||
Assert.Equal(TimeSpan.FromSeconds(60), fake.LastAggregateInterval);
|
||||
}
|
||||
@@ -62,7 +64,7 @@ public sealed class GatewayHistorianDataSourceTests
|
||||
{
|
||||
var fake = new FakeHistorianGatewayClient();
|
||||
var ds = new GatewayHistorianDataSource(fake, NullLogger<GatewayHistorianDataSource>.Instance);
|
||||
await ds.ReadProcessedAsync("T", default, default, TimeSpan.FromSeconds(60), HistoryAggregateType.TimeAverage, TestContext.Current.CancellationToken);
|
||||
await ds.ReadProcessedAsync("T", Hour0, Hour0.AddHours(1), TimeSpan.FromSeconds(60), HistoryAggregateType.TimeAverage, TestContext.Current.CancellationToken); // an empty window sends nothing
|
||||
Assert.Equal(RetrievalMode.TimeWeightedAverage, fake.LastAggregateMode);
|
||||
}
|
||||
|
||||
@@ -111,6 +113,21 @@ public sealed class GatewayHistorianDataSourceTests
|
||||
Assert.Equal(0x80000000u, r.Samples[1].StatusCode); // gap → Bad at requested ts
|
||||
}
|
||||
|
||||
[Fact] // #690: an at-time NULL (histsdk: value 0 + Bad) is null + Bad, like the SQL backends' AtTime of a Bad sample
|
||||
public async Task ReadAtTime_of_a_NULL_row_carries_no_value()
|
||||
{
|
||||
var fake = new FakeHistorianGatewayClient();
|
||||
var t0 = new DateTime(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc);
|
||||
fake.AtTimeSamples = new[]
|
||||
{
|
||||
new HistorianSample { NumericValue = 0, OpcQuality = 0, Quality = 1, Timestamp = Timestamp.FromDateTime(t0) },
|
||||
};
|
||||
var ds = new GatewayHistorianDataSource(fake, NullLogger<GatewayHistorianDataSource>.Instance);
|
||||
var r = await ds.ReadAtTimeAsync("T", new[] { t0 }, TestContext.Current.CancellationToken);
|
||||
Assert.Null(r.Samples[0].Value);
|
||||
Assert.Equal(0x80000000u, r.Samples[0].StatusCode);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Empty_window_is_not_a_fault()
|
||||
{
|
||||
|
||||
+216
@@ -0,0 +1,216 @@
|
||||
using Google.Protobuf.WellKnownTypes;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.HistorianGateway.Contracts.Grpc;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// #691.2: a forward or reverse processed read of the AVEVA-native <c>TimeAverage</c> / <c>Total</c> on the SQL
|
||||
/// backends' interval grid. The gateway refuses <c>start > end</c>, so a reverse read is served by FORWARD native
|
||||
/// requests whose bucket grid is anchored at the request's later time (#677: window <c>(end, start]</c>, intervals
|
||||
/// stepping back from <c>start</c>, the clipped interval the earliest, each stamped with its later edge, results newest
|
||||
/// first); a forward read's clipped LATEST interval is read over exactly its span (review). The fake is a native-mode AVEVA
|
||||
/// model: a step-held stored series; for a forward request <c>[s, e)</c> with interval <c>I</c> it answers one row per
|
||||
/// interval of the grid anchored at <c>s</c> (plus AVEVA's extra row for the interval before <c>s</c>, #685), each the
|
||||
/// time-weighted average / integral of exactly that interval — the per-interval behaviour the gateway will have once
|
||||
/// historiangw#27 is fixed.
|
||||
/// </summary>
|
||||
public sealed class GatewayNativeAggregateGridTests
|
||||
{
|
||||
private static readonly CancellationToken Ct = CancellationToken.None;
|
||||
private static readonly DateTime T0 = new(2026, 10, 12, 18, 0, 0, DateTimeKind.Utc);
|
||||
|
||||
private const uint GoodCalc = 0x00000401u;
|
||||
private const uint Partial = 0x4u;
|
||||
private const uint NoDataCalc = 0x809B0401u;
|
||||
|
||||
private static DateTime At(double seconds) => T0.AddTicks((long)(seconds * TimeSpan.TicksPerSecond));
|
||||
|
||||
/// <summary>Step-held series: value changes at these offsets (s); data starts at −100 s.</summary>
|
||||
private static readonly (double T, double V)[] Series = [(-100, 1), (7, 5), (23, 2), (41, 10), (55, 4), (80, 6)];
|
||||
|
||||
/// <summary>∫ v dt (value·s) of the step-held series over [a, b); null when no data covers any of it.</summary>
|
||||
private static double? Integral(DateTime a, DateTime b)
|
||||
{
|
||||
double sum = 0;
|
||||
var covered = false;
|
||||
for (var i = 0; i < Series.Length; i++)
|
||||
{
|
||||
var from = At(Series[i].T);
|
||||
var to = i + 1 < Series.Length ? At(Series[i + 1].T) : DateTime.MaxValue;
|
||||
var lo = from > a ? from : a;
|
||||
var hi = to < b ? to : b;
|
||||
if (hi <= lo) continue;
|
||||
covered = true;
|
||||
sum += Series[i].V * (hi - lo).TotalSeconds;
|
||||
}
|
||||
return covered ? sum : null;
|
||||
}
|
||||
|
||||
private static double? Expected(RetrievalMode mode, DateTime a, DateTime b) =>
|
||||
Integral(a, b) is { } i ? mode == RetrievalMode.Integral ? i : i / (b - a).TotalSeconds : null;
|
||||
|
||||
private static FakeHistorianGatewayClient Aveva() => new()
|
||||
{
|
||||
AggregateSource = (s, e, mode, interval) =>
|
||||
{
|
||||
var rows = new List<HistorianAggregateSample>();
|
||||
for (var lo = s - interval; lo < e; lo += interval) // the pre-window interval first (#685)
|
||||
{
|
||||
var hi = lo + interval;
|
||||
var v = Expected(mode, lo, hi);
|
||||
rows.Add(new HistorianAggregateSample
|
||||
{
|
||||
Tag = "T",
|
||||
StartTime = Timestamp.FromDateTime(lo),
|
||||
EndTime = Timestamp.FromDateTime(hi),
|
||||
Value = v ?? 0,
|
||||
OpcQuality = v is null ? 0u : 192u,
|
||||
RetrievalMode = mode,
|
||||
});
|
||||
}
|
||||
return rows;
|
||||
},
|
||||
};
|
||||
|
||||
private static GatewayHistorianDataSource Source(FakeHistorianGatewayClient fake) =>
|
||||
new(fake, NullLogger<GatewayHistorianDataSource>.Instance);
|
||||
|
||||
[Theory]
|
||||
[InlineData(HistoryAggregateType.TimeAverage, RetrievalMode.TimeWeightedAverage)]
|
||||
[InlineData(HistoryAggregateType.Total, RetrievalMode.Integral)]
|
||||
public async Task A_reverse_read_over_whole_intervals_is_anchored_at_its_later_time(HistoryAggregateType agg, RetrievalMode mode)
|
||||
{
|
||||
var fake = Aveva();
|
||||
var ds = Source(fake);
|
||||
var s = (await ds.ReadProcessedAsync("T", At(60), At(0), TimeSpan.FromSeconds(20), agg, Ct)).Samples;
|
||||
|
||||
// (0, 60] in three intervals, newest first, each stamped with its LATER edge (#677).
|
||||
s.Select(x => x.SourceTimestampUtc).ShouldBe(new DateTime?[] { At(60), At(40), At(20) });
|
||||
s.Select(x => (double)x.Value!).ShouldBe(
|
||||
[Expected(mode, At(40), At(60))!.Value, Expected(mode, At(20), At(40))!.Value, Expected(mode, At(0), At(20))!.Value],
|
||||
tolerance: 1e-9);
|
||||
s.ShouldAllBe(x => x.StatusCode == GoodCalc);
|
||||
fake.AggregateCalls.ShouldBe([(At(0), At(60), mode, TimeSpan.FromSeconds(20))]);
|
||||
ds.GetHealthSnapshot().TotalFailures.ShouldBe(0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_reverse_read_with_a_partial_interval_clips_the_EARLIEST_one_and_marks_it_Partial()
|
||||
{
|
||||
var fake = Aveva();
|
||||
var s = (await Source(fake).ReadProcessedAsync("T", At(70), At(5), TimeSpan.FromSeconds(20),
|
||||
HistoryAggregateType.TimeAverage, Ct)).Samples;
|
||||
|
||||
// (5, 70]: (50,70], (30,50], (10,30], and the clipped (5,10] — the grid steps back from 70, not forward from 5.
|
||||
s.Select(x => x.SourceTimestampUtc).ShouldBe(new DateTime?[] { At(70), At(50), At(30), At(10) });
|
||||
var m = RetrievalMode.TimeWeightedAverage;
|
||||
s.Select(x => (double)x.Value!).ShouldBe(
|
||||
[Expected(m, At(50), At(70))!.Value, Expected(m, At(30), At(50))!.Value, Expected(m, At(10), At(30))!.Value,
|
||||
Expected(m, At(5), At(10))!.Value],
|
||||
tolerance: 1e-9);
|
||||
s.Select(x => x.StatusCode).ShouldBe([GoodCalc, GoodCalc, GoodCalc, GoodCalc | Partial]);
|
||||
|
||||
// Two forward requests, neither reversed: the whole intervals on the later-anchored grid, and the clipped one alone
|
||||
// (its own width as the interval, so AVEVA computes it over exactly (5, 10]).
|
||||
fake.AggregateCalls.ShouldBe([
|
||||
(At(10), At(70), m, TimeSpan.FromSeconds(20)),
|
||||
(At(5), At(10), m, TimeSpan.FromSeconds(5)),
|
||||
]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_reverse_read_narrower_than_one_interval_is_one_interval_stamped_at_its_later_time()
|
||||
{
|
||||
var fake = Aveva();
|
||||
var s = (await Source(fake).ReadProcessedAsync("T", At(30), At(12), TimeSpan.FromSeconds(60),
|
||||
HistoryAggregateType.Total, Ct)).Samples;
|
||||
s.Single().SourceTimestampUtc.ShouldBe(At(30));
|
||||
((double)s.Single().Value!).ShouldBe(Expected(RetrievalMode.Integral, At(12), At(30))!.Value, 1e-9);
|
||||
s.Single().StatusCode.ShouldBe(GoodCalc); // a whole (clamped) interval, not a truncated one
|
||||
fake.AggregateCalls.ShouldBe([(At(12), At(30), RetrievalMode.Integral, TimeSpan.FromSeconds(18))]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_zero_interval_reverse_read_is_one_interval_over_the_window()
|
||||
{
|
||||
var fake = Aveva();
|
||||
var s = (await Source(fake).ReadProcessedAsync("T", At(30), At(0), TimeSpan.Zero,
|
||||
HistoryAggregateType.TimeAverage, Ct)).Samples;
|
||||
s.Single().SourceTimestampUtc.ShouldBe(At(30));
|
||||
fake.AggregateCalls.Single().Interval.ShouldBe(TimeSpan.FromSeconds(30));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_no_data_interval_is_BadNoData_without_the_Partial_bit()
|
||||
{
|
||||
// Data starts at −100 s: (−110, −90] holds some; (−130, −110] and the clipped earliest (−135, −130] hold none.
|
||||
var s = (await Source(Aveva()).ReadProcessedAsync("T", At(-90), At(-135), TimeSpan.FromSeconds(20),
|
||||
HistoryAggregateType.TimeAverage, Ct)).Samples;
|
||||
s.Select(x => x.SourceTimestampUtc).ShouldBe(new DateTime?[] { At(-90), At(-110), At(-130) });
|
||||
s[0].StatusCode.ShouldBe(GoodCalc);
|
||||
s[1].StatusCode.ShouldBe(NoDataCalc);
|
||||
s[2].StatusCode.ShouldBe(NoDataCalc); // the clipped interval, but Bad ⇒ no Partial bit
|
||||
s[2].Value.ShouldBeNull();
|
||||
}
|
||||
|
||||
// ---- forward reads on the same grid (#691.2 review): the clipped LATEST interval is its own exact-span read ----------
|
||||
|
||||
[Theory]
|
||||
[InlineData(HistoryAggregateType.TimeAverage, RetrievalMode.TimeWeightedAverage)]
|
||||
[InlineData(HistoryAggregateType.Total, RetrievalMode.Integral)]
|
||||
public async Task A_forward_read_with_a_partial_interval_clips_the_LATEST_one_and_marks_it_Partial(HistoryAggregateType agg,
|
||||
RetrievalMode mode)
|
||||
{
|
||||
var fake = Aveva();
|
||||
var s = (await Source(fake).ReadProcessedAsync("T", At(5), At(70), TimeSpan.FromSeconds(20), agg, Ct)).Samples;
|
||||
|
||||
// [5, 70): [5,25), [25,45), [45,65) and the clipped [65,70) — stamped with each EARLIER edge, oldest first, as on
|
||||
// SqlServer/QuestDB (BucketPlan forward; the truncated interval Partial unless Bad).
|
||||
s.Select(x => x.SourceTimestampUtc).ShouldBe(new DateTime?[] { At(5), At(25), At(45), At(65) });
|
||||
s.Select(x => (double)x.Value!).ShouldBe(
|
||||
[Expected(mode, At(5), At(25))!.Value, Expected(mode, At(25), At(45))!.Value, Expected(mode, At(45), At(65))!.Value,
|
||||
Expected(mode, At(65), At(70))!.Value],
|
||||
tolerance: 1e-9);
|
||||
s.Select(x => x.StatusCode).ShouldBe([GoodCalc, GoodCalc, GoodCalc, GoodCalc | Partial]);
|
||||
// AVEVA never averages past EndTime: the whole intervals, then the clipped one over exactly its span.
|
||||
fake.AggregateCalls.ShouldBe([
|
||||
(At(5), At(65), mode, TimeSpan.FromSeconds(20)),
|
||||
(At(65), At(70), mode, TimeSpan.FromSeconds(5)),
|
||||
]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_forward_interval_wider_than_the_window_is_sent_as_the_window()
|
||||
{
|
||||
var fake = Aveva();
|
||||
var s = (await Source(fake).ReadProcessedAsync("T", At(12), At(30), TimeSpan.FromSeconds(60),
|
||||
HistoryAggregateType.Total, Ct)).Samples;
|
||||
s.Single().SourceTimestampUtc.ShouldBe(At(12));
|
||||
s.Single().StatusCode.ShouldBe(GoodCalc);
|
||||
fake.AggregateCalls.ShouldBe([(At(12), At(30), RetrievalMode.Integral, TimeSpan.FromSeconds(18))]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_zero_interval_forward_read_is_one_interval_over_the_window_not_a_zero_interval_request()
|
||||
{
|
||||
var fake = Aveva();
|
||||
var s = (await Source(fake).ReadProcessedAsync("T", At(0), At(30), TimeSpan.Zero,
|
||||
HistoryAggregateType.TimeAverage, Ct)).Samples;
|
||||
s.Single().SourceTimestampUtc.ShouldBe(At(0));
|
||||
fake.AggregateCalls.Single().Interval.ShouldBe(TimeSpan.FromSeconds(30));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task The_forward_read_is_unchanged()
|
||||
{
|
||||
var fake = Aveva();
|
||||
var s = (await Source(fake).ReadProcessedAsync("T", At(0), At(60), TimeSpan.FromSeconds(20),
|
||||
HistoryAggregateType.TimeAverage, Ct)).Samples;
|
||||
s.Select(x => x.SourceTimestampUtc).ShouldBe(new DateTime?[] { At(0), At(20), At(40) });
|
||||
fake.AggregateCalls.ShouldBe([(At(0), At(60), RetrievalMode.TimeWeightedAverage, TimeSpan.FromSeconds(20))]);
|
||||
}
|
||||
}
|
||||
+3
-1
@@ -106,9 +106,11 @@ public sealed class GatewayNativeAggregateRowsTests
|
||||
public async Task The_window_filter_is_direction_agnostic()
|
||||
{
|
||||
// A reverse request (start > end) covers the same span (end, start]; a row for the interval before the span is out.
|
||||
// #691.2: served by a forward request on the later-anchored grid, newest first, stamped with each later edge.
|
||||
var wire = new[] { Row(T0 - I20, T0, 9, 192), Row(T0, T0 + I20, 1, 192), Row(T0 + I20, T0 + 2 * I20, 2, 192) };
|
||||
var s = await Read(wire, T0 + 2 * I20, T0);
|
||||
s.Select(x => x.Value).ShouldBe(new object?[] { 1d, 2d });
|
||||
s.Select(x => x.Value).ShouldBe(new object?[] { 2d, 1d });
|
||||
s.Select(x => x.SourceTimestampUtc).ShouldBe(new DateTime?[] { T0 + 2 * I20, T0 + I20 });
|
||||
}
|
||||
|
||||
// ---- D2: an Uncertain row keeps its value ------------------------------------------------------------------------
|
||||
|
||||
+9
@@ -109,6 +109,15 @@ public sealed class GatewayRawBoundaryRowTests
|
||||
r.Samples[0].StatusCode.ShouldBe(0x80000000u);
|
||||
}
|
||||
|
||||
[Fact] // #690: the wire's value 0 on a NULL row is not a value — SqlServer/QuestDB return null + Bad for the same sample
|
||||
public async Task A_stored_NULL_sample_carries_no_value_not_the_wires_zero()
|
||||
{
|
||||
var r = await Source(Gate()).ReadRawAsync("T", At(30), At(55), 0, Ct);
|
||||
r.Samples[0].SourceTimestampUtc.ShouldBe(At(40));
|
||||
r.Samples[0].Value.ShouldBeNull();
|
||||
r.Samples[1].Value.ShouldBe(5d); // a Good neighbour keeps its value
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_stored_sample_exactly_at_the_window_start_is_kept()
|
||||
{
|
||||
|
||||
+290
@@ -0,0 +1,290 @@
|
||||
using Google.Protobuf.WellKnownTypes;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.HistorianGateway.Contracts.Grpc;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// #691.3: a gateway with <c>Historian:ReadLimits:AllowStreamingBeyondCeiling=false</c> refuses any <c>ReadRaw</c> whose
|
||||
/// <c>max_values</c> exceeds its <c>MaxValuesCeiling</c> (before streaming, naming the ceiling). An uncapped raw read
|
||||
/// (<c>NumValuesPerNode = 0</c>, sent as <see cref="int.MaxValue"/>) used to be refused outright — even a 10-row window.
|
||||
/// The read is now re-issued once with the ceiling the refusal named, and a stream that delivered the whole ceiling
|
||||
/// without reaching the window end is never answered from (<c>BadTooManyOperations</c>). The fake is AVEVA-shaped:
|
||||
/// a boundary row at the query start, <c>[start, end]</c> inclusive, capped at <c>max</c>.
|
||||
/// </summary>
|
||||
public sealed class GatewayStrictCeilingTests
|
||||
{
|
||||
private static readonly CancellationToken Ct = CancellationToken.None;
|
||||
private static readonly DateTime T0 = new(2026, 10, 12, 19, 0, 0, DateTimeKind.Utc);
|
||||
private static readonly DateTime Now = T0.AddDays(1);
|
||||
|
||||
private static DateTime At(int seconds) => T0.AddSeconds(seconds);
|
||||
|
||||
private static HistorianSample Row(DateTime t, double v, uint quality = 0) =>
|
||||
new() { Tag = "T", Timestamp = Timestamp.FromDateTime(t), NumericValue = v, OpcQuality = 192, Quality = quality };
|
||||
|
||||
/// <summary>One stored sample per second, value = seconds, at +0 … +(count − 1); strict ceiling when given.</summary>
|
||||
private static FakeHistorianGatewayClient Store(int count, int? ceiling)
|
||||
{
|
||||
var stored = Enumerable.Range(0, count).Select(i => (T: At(i), V: (double)i)).ToList();
|
||||
return new FakeHistorianGatewayClient
|
||||
{
|
||||
MaxValuesCeiling = ceiling,
|
||||
RawSource = (start, end, max) =>
|
||||
{
|
||||
var rows = new List<HistorianSample>();
|
||||
if (stored.All(s => s.T != start))
|
||||
{
|
||||
var prior = stored.LastOrDefault(s => s.T < start);
|
||||
rows.Add(prior.T == default
|
||||
? new HistorianSample { Tag = "T", Timestamp = Timestamp.FromDateTime(start), NumericValue = 0, OpcQuality = 0, Quality = 1 }
|
||||
: Row(start, prior.V, 133));
|
||||
}
|
||||
rows.AddRange(stored.Where(s => s.T >= start && s.T <= end).Select(s => Row(s.T, s.V)));
|
||||
return rows.Take(max);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
private static GatewayHistorianDataSource Source(FakeHistorianGatewayClient fake, int maxRawRows = 1_000_000) =>
|
||||
new(fake, NullLogger<GatewayHistorianDataSource>.Instance, maxRawRows, new FixedClock(Now));
|
||||
|
||||
private sealed class FixedClock(DateTime now) : TimeProvider
|
||||
{
|
||||
public override DateTimeOffset GetUtcNow() => new(now);
|
||||
}
|
||||
|
||||
// ---- raw, uncapped ---------------------------------------------------------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public async Task An_uncapped_read_of_a_small_window_on_a_strict_gateway_is_served_with_the_ceiling_as_budget()
|
||||
{
|
||||
var fake = Store(10, ceiling: 1000);
|
||||
var ds = Source(fake);
|
||||
var r = await ds.ReadRawAsync("T", At(0), At(60), 0, Ct);
|
||||
|
||||
r.Samples.Select(s => s.Value).ShouldBe(Enumerable.Range(0, 10).Select(i => (object?)(double)i));
|
||||
fake.RawCalls.Select(c => c.MaxValues).ShouldBe([int.MaxValue, 1000]); // refused, then the ceiling it named
|
||||
fake.RawStreamsCompleted.ShouldBe(1);
|
||||
ds.GetHealthSnapshot().TotalFailures.ShouldBe(0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task An_uncapped_read_on_a_default_gateway_still_sends_one_unbounded_request()
|
||||
{
|
||||
var fake = Store(10, ceiling: null);
|
||||
await Source(fake).ReadRawAsync("T", At(0), At(60), 0, Ct);
|
||||
fake.RawCalls.Select(c => c.MaxValues).ShouldBe([int.MaxValue]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task An_uncapped_window_holding_more_than_the_ceiling_is_a_request_limit_never_a_truncated_answer()
|
||||
{
|
||||
var fake = Store(100, ceiling: 50);
|
||||
var ds = Source(fake);
|
||||
await Should.ThrowAsync<HistoryReadLimitExceededException>(() => ds.ReadRawAsync("T", At(0), At(200), 0, Ct));
|
||||
fake.RawStreamsCompleted.ShouldBe(1); // drained, never abandoned
|
||||
ds.GetHealthSnapshot().TotalFailures.ShouldBe(0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_ceiling_full_stream_that_reached_the_window_end_is_complete()
|
||||
{
|
||||
// 50 rows: the boundary row + +0 … +48; the window end is +48 (the gateway's end is inclusive), so the stream
|
||||
// provably ended at the window end — complete although it delivered the whole ceiling.
|
||||
var fake = Store(100, ceiling: 50);
|
||||
var r = await Source(fake).ReadRawAsync("T", At(0), At(48), 0, Ct);
|
||||
r.Samples.Count.ShouldBe(48); // [+0, +48): the end is exclusive
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_refusal_without_a_parseable_ceiling_retries_at_the_gateways_default_ceiling()
|
||||
{
|
||||
var fake = Store(10, ceiling: 1_000_000);
|
||||
fake.CeilingRejectionDetail = "max_values exceeds the server ceiling; read large ranges by time-slicing.";
|
||||
var r = await Source(fake).ReadRawAsync("T", At(0), At(60), 0, Ct);
|
||||
r.Samples.Count.ShouldBe(10);
|
||||
fake.RawCalls.Select(c => c.MaxValues).ShouldBe([int.MaxValue, RawAggregateReader.GatewayDefaultMaxValuesCeiling]);
|
||||
}
|
||||
|
||||
// ---- the learned ceiling (review: a refused attempt counts as an error in the gateway's own metrics) -----------------
|
||||
|
||||
private static int Refusals(FakeHistorianGatewayClient fake) =>
|
||||
fake.RawCalls.Count(c => fake.MaxValuesCeiling is { } n && c.MaxValues > n);
|
||||
|
||||
[Fact]
|
||||
public async Task The_ceiling_is_learned_once_per_data_source_so_later_reads_are_not_refused()
|
||||
{
|
||||
var fake = Store(10, ceiling: 1000);
|
||||
var ds = Source(fake);
|
||||
await ds.ReadRawAsync("T", At(0), At(60), 0, Ct);
|
||||
await ds.ReadRawAsync("T", At(0), At(60), 0, Ct);
|
||||
await ds.ReadRawAsync("T", At(0), At(60), 995, Ct);
|
||||
await ds.ReadProcessedAsync("T", At(0), At(10), TimeSpan.FromSeconds(10), HistoryAggregateType.Count, Ct);
|
||||
|
||||
fake.RawCalls.Select(c => c.MaxValues).Take(4).ShouldBe([int.MaxValue, 1000, 1000, 1000]);
|
||||
fake.RawCalls[4].MaxValues.ShouldBe(1000); // the per-sample read shares it (its own budget 1,000,000)
|
||||
Refusals(fake).ShouldBe(1);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_gateway_that_now_streams_is_re_probed_when_a_learned_ceiling_read_comes_back_full()
|
||||
{
|
||||
var fake = Store(100, ceiling: 50);
|
||||
var ds = Source(fake);
|
||||
await ds.ReadRawAsync("T", At(0), At(5), 0, Ct); // learns 50
|
||||
fake.MaxValuesCeiling = null; // the operator enabled AllowStreamingBeyondCeiling
|
||||
|
||||
var r = await ds.ReadRawAsync("T", At(0), At(200), 0, Ct);
|
||||
r.Samples.Count.ShouldBe(100); // not BadTooManyOperations from a stale ceiling
|
||||
fake.RawCalls.Skip(2).Select(c => c.MaxValues).ShouldBe([50, int.MaxValue]);
|
||||
|
||||
await ds.ReadRawAsync("T", At(0), At(5), 0, Ct);
|
||||
fake.RawCalls[^1].MaxValues.ShouldBe(int.MaxValue); // forgotten
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_raised_but_still_strict_ceiling_is_re_learned_from_the_re_probe()
|
||||
{
|
||||
var fake = Store(100, ceiling: 50);
|
||||
var ds = Source(fake);
|
||||
await ds.ReadRawAsync("T", At(0), At(5), 0, Ct);
|
||||
fake.MaxValuesCeiling = 500;
|
||||
|
||||
(await ds.ReadRawAsync("T", At(0), At(200), 0, Ct)).Samples.Count.ShouldBe(100);
|
||||
fake.RawCalls.Skip(2).Select(c => c.MaxValues).ShouldBe([50, int.MaxValue, 500]);
|
||||
|
||||
await ds.ReadRawAsync("T", At(0), At(5), 0, Ct);
|
||||
fake.RawCalls[^1].MaxValues.ShouldBe(500);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_genuinely_over_ceiling_read_costs_one_re_probe_and_no_repeat_at_the_same_ceiling()
|
||||
{
|
||||
var fake = Store(100, ceiling: 50);
|
||||
var ds = Source(fake);
|
||||
await ds.ReadRawAsync("T", At(0), At(5), 0, Ct);
|
||||
await Should.ThrowAsync<HistoryReadLimitExceededException>(() => ds.ReadRawAsync("T", At(0), At(200), 0, Ct));
|
||||
fake.RawCalls.Skip(2).Select(c => c.MaxValues).ShouldBe([50, int.MaxValue]);
|
||||
ds.GetHealthSnapshot().TotalFailures.ShouldBe(0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_lowered_ceiling_is_re_learned_from_the_refusal_of_the_learned_one()
|
||||
{
|
||||
var fake = Store(10, ceiling: 1000);
|
||||
var ds = Source(fake);
|
||||
await ds.ReadRawAsync("T", At(0), At(60), 0, Ct);
|
||||
fake.MaxValuesCeiling = 50;
|
||||
|
||||
(await ds.ReadRawAsync("T", At(0), At(60), 0, Ct)).Samples.Count.ShouldBe(10);
|
||||
await ds.ReadRawAsync("T", At(0), At(60), 0, Ct);
|
||||
fake.RawCalls.Skip(2).Select(c => c.MaxValues).ShouldBe([1000, 50, 50]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Concurrent_reads_share_the_learned_ceiling_safely()
|
||||
{
|
||||
var fake = Store(10, ceiling: 1000);
|
||||
var ds = Source(fake);
|
||||
await ds.ReadRawAsync("T", At(0), At(60), 0, Ct);
|
||||
var results = await Task.WhenAll(Enumerable.Range(0, 32).Select(_ => Task.Run(() => ds.ReadRawAsync("T", At(0), At(60), 0, Ct))));
|
||||
results.ShouldAllBe(r => r.Samples.Count == 10);
|
||||
Refusals(fake).ShouldBe(1);
|
||||
}
|
||||
|
||||
// ---- raw, capped -----------------------------------------------------------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public async Task A_cap_whose_allowance_exceeds_the_ceiling_is_served_with_the_ceiling_as_budget()
|
||||
{
|
||||
// cap 995 + 16 = 1011 > 1000: refused, re-asked with 1000 — a small window is served in full.
|
||||
var fake = Store(10, ceiling: 1000);
|
||||
var r = await Source(fake).ReadRawAsync("T", At(0), At(60), 995, Ct);
|
||||
r.Samples.Count.ShouldBe(10);
|
||||
fake.RawCalls.Select(c => c.MaxValues).ShouldBe([995 + RawAggregateReader.PreWindowAllowance, 1000]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_cap_above_the_ceiling_over_a_dense_window_is_a_request_limit_not_a_short_page()
|
||||
{
|
||||
// A short page would end the client's paging early (silent truncation), so it must fail instead.
|
||||
var fake = Store(100, ceiling: 50);
|
||||
var ds = Source(fake);
|
||||
await Should.ThrowAsync<HistoryReadLimitExceededException>(() => ds.ReadRawAsync("T", At(0), At(200), 80, Ct));
|
||||
fake.RawCalls.Count.ShouldBe(2); // the ceiling re-read is final: no third read at the same ceiling
|
||||
ds.GetHealthSnapshot().TotalFailures.ShouldBe(0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_cap_that_fits_the_ceiling_is_unaffected()
|
||||
{
|
||||
var fake = Store(100, ceiling: 50);
|
||||
var r = await Source(fake).ReadRawAsync("T", At(0), At(200), 10, Ct);
|
||||
r.Samples.Count.ShouldBe(10);
|
||||
fake.RawCalls.Select(c => c.MaxValues).ShouldBe([10 + RawAggregateReader.PreWindowAllowance]);
|
||||
}
|
||||
|
||||
// ---- per-sample aggregates (#688 clamp vs a ceiling below 1,000,000) ---------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public async Task A_per_sample_aggregate_on_a_gateway_with_a_lower_ceiling_is_served_when_the_window_fits()
|
||||
{
|
||||
var fake = Store(30, ceiling: 1000); // default cap 1,000,000 ⇒ budget 1,000,000 > 1000
|
||||
var s = (await Source(fake).ReadProcessedAsync("T", At(0), At(30), TimeSpan.FromSeconds(10),
|
||||
HistoryAggregateType.Count, Ct)).Samples;
|
||||
s.Select(x => x.Value).ShouldBe([10, 10, 10]);
|
||||
fake.RawCalls[0].MaxValues.ShouldBe(RawAggregateReader.GatewayDefaultMaxValuesCeiling);
|
||||
fake.RawCalls[1].MaxValues.ShouldBe(1000);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_per_sample_aggregate_whose_window_fills_the_lower_ceiling_is_a_request_limit()
|
||||
{
|
||||
var fake = Store(100, ceiling: 50);
|
||||
var ds = Source(fake);
|
||||
await Should.ThrowAsync<HistoryReadLimitExceededException>(() => ds.ReadProcessedAsync("T", At(0), At(100),
|
||||
TimeSpan.FromSeconds(10), HistoryAggregateType.Count, Ct));
|
||||
ds.GetHealthSnapshot().TotalFailures.ShouldBe(0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task An_unbounded_per_sample_aggregate_on_a_strict_gateway_is_served_with_the_ceiling()
|
||||
{
|
||||
var fake = Store(30, ceiling: 1000);
|
||||
var s = (await Source(fake, maxRawRows: 0).ReadProcessedAsync("T", At(0), At(30), TimeSpan.FromSeconds(10),
|
||||
HistoryAggregateType.Count, Ct)).Samples;
|
||||
s.Select(x => x.Value).ShouldBe([10, 10, 10]);
|
||||
fake.RawCalls.Take(2).Select(c => c.MaxValues).ShouldBe([int.MaxValue, 1000]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task An_unbounded_per_sample_aggregate_filling_the_ceiling_is_a_request_limit()
|
||||
{
|
||||
var fake = Store(100, ceiling: 50);
|
||||
await Should.ThrowAsync<HistoryReadLimitExceededException>(() => Source(fake, maxRawRows: 0).ReadProcessedAsync("T",
|
||||
At(0), At(100), TimeSpan.FromSeconds(10), HistoryAggregateType.Count, Ct));
|
||||
}
|
||||
|
||||
[Theory] // review: the message names the limit that actually bound the read — the gateway's ceiling, not the cap
|
||||
[InlineData(0)] // unbounded cap: used to say "more than 0 raw samples (MaxTimeWeightedRawRows)"
|
||||
[InlineData(1_000_000)] // default cap: used to cite 1,000,000
|
||||
public async Task A_ceiling_bound_per_sample_read_names_the_gateway_ceiling(int maxRawRows)
|
||||
{
|
||||
var ex = await Should.ThrowAsync<HistoryReadLimitExceededException>(() => Source(Store(100, ceiling: 50), maxRawRows)
|
||||
.ReadProcessedAsync("T", At(0), At(100), TimeSpan.FromSeconds(10), HistoryAggregateType.Count, Ct));
|
||||
ex.Message.ShouldContain("row ceiling (50,");
|
||||
ex.Message.ShouldNotContain("MaxTimeWeightedRawRows); the Gateway");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_cap_bound_per_sample_read_still_names_the_cap()
|
||||
{
|
||||
var ex = await Should.ThrowAsync<HistoryReadLimitExceededException>(() => Source(Store(100, ceiling: null), 20)
|
||||
.ReadProcessedAsync("T", At(0), At(100), TimeSpan.FromSeconds(10), HistoryAggregateType.Count, Ct));
|
||||
ex.Message.ShouldContain("more than 20 raw samples");
|
||||
}
|
||||
}
|
||||
+25
@@ -32,6 +32,31 @@ public sealed class SampleMapperTests
|
||||
Assert.Equal(0x80000000u, SampleMapper.ToSnapshot(s).StatusCode);
|
||||
}
|
||||
|
||||
[Theory] // #690: histsdk puts a value on every row, so a NULL row arrives as 0 / "" with a Bad quality
|
||||
[InlineData(0u, 1u)] // the live NULL row: opc_quality 0, AVEVA quality 1
|
||||
[InlineData(24u, 0u)] // any Bad OPC quality (out of service)
|
||||
public void A_Bad_numeric_row_carries_no_value(uint opc, uint quality)
|
||||
{
|
||||
var s = new HistorianSample { Tag = "T", NumericValue = 0, OpcQuality = opc, Quality = quality, Timestamp = Ts(2026, 1, 1, 0, 0, 0) };
|
||||
var snap = SampleMapper.ToSnapshot(s);
|
||||
Assert.True(snap.StatusCode >= 0x80000000u);
|
||||
Assert.Null(snap.Value);
|
||||
}
|
||||
|
||||
[Fact] // #690: a string tag's NULL row arrives as "" + Bad
|
||||
public void A_Bad_string_row_carries_no_value()
|
||||
{
|
||||
var s = new HistorianSample { Tag = "T", StringValue = "", OpcQuality = 0, Quality = 1, Timestamp = Ts(2026, 1, 1, 0, 0, 0) };
|
||||
Assert.Null(SampleMapper.ToSnapshot(s).Value);
|
||||
}
|
||||
|
||||
[Fact] // #690: an Uncertain row keeps its value (only Bad means "no value")
|
||||
public void An_Uncertain_row_keeps_its_value()
|
||||
{
|
||||
var s = new HistorianSample { Tag = "T", NumericValue = 3, OpcQuality = 64, Timestamp = Ts(2026, 1, 1, 0, 0, 0) };
|
||||
Assert.Equal(3d, SampleMapper.ToSnapshot(s).Value);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Aggregate_null_value_is_BadNoData()
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user