Compare commits
60 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 8828e999c4 | |||
| 4f7a07fbd3 | |||
| da2e441bb9 | |||
| c8e3e279b4 | |||
| 468f13cb56 | |||
| f71fe13826 | |||
| 6f0674296b | |||
| 19bc13bf6a | |||
| cd50db136e | |||
| eecf301200 | |||
| 76bdca90e9 | |||
| 8ff9f38bbd | |||
| 495606b9fb | |||
| f794dc50b6 | |||
| 7b23875070 | |||
| 109163621c | |||
| 0ccfbbee5c | |||
| dccab57d39 | |||
| a935e21bf8 | |||
| 3ec6006f57 | |||
| 618d98e822 | |||
| 5897764024 | |||
| 97668d26d3 | |||
| 66a2b0c3e9 | |||
| de8bc7a12c | |||
| 3c3513ea3e | |||
| a5a0496176 | |||
| 2609c5462e | |||
| 68a1f31397 | |||
| aa923cea83 | |||
| e04e1bf127 | |||
| 0d249f3ee9 | |||
| 3d7eea6280 | |||
| 8dfa165110 | |||
| 37390139fe | |||
| 15ccd9b722 | |||
| 942ca7bc14 | |||
| df0c1f3513 | |||
| ba46f4f47a | |||
| 934909b430 | |||
| ebc6b8d4d9 | |||
| f6681544e7 | |||
| 92aab7282c | |||
| 60febbf2dc | |||
| 88141d6ed5 | |||
| 9bb1363cd6 | |||
| 574c3aa1b0 | |||
| d28e3e64f1 | |||
| ccda27d220 | |||
| d277f34a96 | |||
| 186b7d5ea5 | |||
| d33992034d | |||
| 71d031fcd0 | |||
| f0d064a68c | |||
| 830b47f624 | |||
| 7b86059daf | |||
| ab5030174a | |||
| 0e47e887de | |||
| dbffb6bdb4 | |||
| 39552e15b9 |
+19
-22
@@ -1,7 +1,6 @@
|
||||
# CI for the v2 branch — runs on every push + PR to the v2-akka-fuse / master
|
||||
# branches. Layered into two jobs:
|
||||
# build dotnet restore + build (fast feedback on compile errors)
|
||||
# unit-tests the WHOLE solution minus the env-gated tiers (self-maintaining)
|
||||
# branches. One job restores and builds once, then tests the WHOLE solution
|
||||
# minus the env-gated tiers (self-maintaining).
|
||||
#
|
||||
# Skips E2E (Category=E2E) and LiveIntegration (needs a real gateway/PLC/VPN) — both
|
||||
# are env-gated tiers with no hosted-runner fixtures. (The skip-gated *.IntegrationTests
|
||||
@@ -20,38 +19,35 @@ on:
|
||||
branches: [v2-akka-fuse, master]
|
||||
workflow_dispatch: {}
|
||||
|
||||
# #707: one active run per repository across push/PR triggers. Gitea 1.26 supports
|
||||
# workflow concurrency; newer pending runs can supersede an older pending run.
|
||||
concurrency:
|
||||
group: lmxopcua-v2-ci
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
DOTNET_NOLOGO: "1"
|
||||
DOTNET_CLI_TELEMETRY_OPTOUT: "1"
|
||||
|
||||
jobs:
|
||||
|
||||
build:
|
||||
unit-tests:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-dotnet@v4
|
||||
with:
|
||||
dotnet-version: 10.0.x
|
||||
# #606: code-reviews/README.md must be regenerated from the per-module findings.md
|
||||
# files; --check exits non-zero on drift without writing (see REVIEW-PROCESS.md section 5).
|
||||
# #606: regenerated review summary must remain in sync with module findings.
|
||||
- name: code-reviews README drift check
|
||||
run: python3 code-reviews/regen-readme.py --check
|
||||
- name: TRX skip guard fixtures
|
||||
run: python3 scripts/ci/test-assert-not-all-skipped.py
|
||||
# #707: bound project workers and avoid concurrent NuGet restore work.
|
||||
- name: dotnet restore
|
||||
run: dotnet restore ZB.MOM.WW.OtOpcUa.slnx
|
||||
run: dotnet restore ZB.MOM.WW.OtOpcUa.slnx --disable-parallel -m:2
|
||||
- name: dotnet build
|
||||
run: dotnet build ZB.MOM.WW.OtOpcUa.slnx --no-restore --configuration Release
|
||||
|
||||
unit-tests:
|
||||
needs: build
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-dotnet@v4
|
||||
with:
|
||||
dotnet-version: 10.0.x
|
||||
- name: dotnet restore
|
||||
run: dotnet restore ZB.MOM.WW.OtOpcUa.slnx
|
||||
run: dotnet build ZB.MOM.WW.OtOpcUa.slnx --no-restore --configuration Release -m:2
|
||||
# #606: the runner's size is the context every timing-sensitive failure needs and the log never
|
||||
# recorded it (load average included: the two Gitea runners may be running jobs side by side).
|
||||
- name: runner capacity
|
||||
@@ -69,14 +65,15 @@ jobs:
|
||||
# are unreachable, so they run harmlessly here (and are counted by the skip-gate below).
|
||||
- name: dotnet test (whole solution, minus E2E + LiveIntegration)
|
||||
# #606: Akka TestKit time dilation for the Runtime + ControlPlane actor suites (read by their
|
||||
# harnesses, tests/Server/*/Harness/TestTimeFactor.cs). This leg runs every test assembly at once
|
||||
# on the runner, and their 2–8 s presence budgets — sized on an idle dev box — failed in a long
|
||||
# harnesses, tests/Server/*/Harness/TestTimeFactor.cs). Their 2–8 s presence budgets
|
||||
# — sized on an idle dev box — failed in a long
|
||||
# tail, a different test per run. Dilation only stretches waits: a presence wait still fails if
|
||||
# the message never comes, and absence windows get longer (stricter). Local runs stay at 1.
|
||||
env:
|
||||
AKKA_TEST_TIMEFACTOR: "3"
|
||||
run: >
|
||||
dotnet test ZB.MOM.WW.OtOpcUa.slnx --configuration Release --no-restore
|
||||
dotnet test ZB.MOM.WW.OtOpcUa.slnx --configuration Release --no-build
|
||||
-m:1 -p:TestTfmsInParallel=false --settings scripts/ci/ci.runsettings
|
||||
--filter "Category!=E2E&Category!=LiveIntegration"
|
||||
--logger "trx" --results-directory artifacts/trx
|
||||
# 07/S-2 fail-on-skip: a whole-solution run that executed ZERO tests is a masked outage
|
||||
|
||||
@@ -1353,8 +1353,9 @@ historian tagname, which is the tag's driver `FullName`.
|
||||
- **SQL Server** (`Driver.Historian.SqlServer`): `hist.Sample` PK `(TagId, SourceTimeUtc)` — writes are insert-if-absent, so outbox redelivery is idempotent. Stores every value kind with its **real status**; per-sample aggregates (10, simple definitions) are pushed down to T-SQL with bucket math in **100 ns ticks**; the Part 13 time-weighted seven (TimeAverage/TimeAverage2/Total/Total2/Interpolative, #671 — ⚠️ `Total` is now the time-weighted integral, no longer `SUM` — plus the **duration-based** PercentGood/PercentBad, #680) stream window + 4 bound rows into the shared `TimeWeightedAggregator` (both SQL backends, identical results; reverse `Interpolative` anchored at the later time; window capped by `ServerHistorian:MaxTimeWeightedRawRows` ⇒ `BadTooManyOperations`, percentages included). ⚠️ **#680 status rules (client-visible):** TimeAverage/Total are Uncertain if any part of the interval uses an Uncertain value, is bridged across a Bad sample, is extrapolated past the last sample (⇒ a live tag's latest interval is Uncertain) or is not covered; TimeAverage2/Total2 keep the SDK time-based rule (Uncertain time counts as Good) but never extrapolate past the last sample — that interval is Partial, later ones `BadNoData` (a report-by-exception tag with nothing after its last sample reads `BadNoData` until its next sample arrives). `docs/Historian.md` §"Time-weighted aggregates". ⚠️ **#680 per-sample (client-visible):** `Count` counts **Good** samples only (value-based status; an all-Bad interval is generic Bad, value null) and an empty interval **inside the tag's data range** is `0`/Good (outside it still `BadNoData`); `Start`/`End` return the **raw** first/last sample of any quality — its own timestamp and status, value null when Bad; `Delta` is Uncertain only when the interval's first or last sample is unusable; ⚠️ #684: `Average`/`Minimum`/`Maximum`/`Range`/`*ActualTime` use **Good samples only** (was Good + Uncertain; an Uncertain-only interval is `BadNoData`), Min/Max-family status Uncertain on a Bad sample, an Uncertain value beyond the Good extreme (Part 13 — the SDK ignores that) or an Uncertain non-numeric sample, and only a **Good** String/DateTime sample refuses them (`BadAggregateInvalidInputs`; Delta still refuses any usable one); and every per-sample aggregate but `Average` sets **end-of-data Partial** on the interval holding the last sample (forward incl. its earlier edge, reverse strictly inside), never on a Bad result. The data-range lookup is a key seek on SQL Server and a `tag` symbol-index seek on QuestDB (#683 — `{p}sample.tag` is `SYMBOL INDEX`; the bootstrapper ADDs it to a pre-#683 table, idempotent), read BEFORE the bucket statement so a racing append can never yield a Good `Count` of 0 (a concurrent HistoryUpdate delete is the mirror case and still can, transiently). `docs/Historian.md` §"Aggregates". Docker-dev recipe, schema and aggregate semantics: `docs/Historian.md` §"SQL Server backend". Integration tests need `HISTORIAN_SQL_TEST_CONNECTION`. **Retention (#667):** opt-in `SqlServer:RetentionDays` (+ `PurgeInterval` 1h, `PurgeBatchSize` 4000) -> `HistorianRetentionService` purges old `hist.Sample` (per tag) / `hist.Event` in `DELETE TOP(n)` batches, Primary-gated on `IHistorizationRoleView`, one purger per DB via `sp_getapplock`; counter `otopcua.historian.retention.purged{table}`. Optional `IHistorianBackendProvider.CreateRetention` (default null). `docs/Historian.md` §Retention.
|
||||
- **QuestDB** (`Driver.Historian.QuestDb`, `ServerHistorian:QuestDb:*`, Npgsql over PG wire; docker-dev overlay `docker-compose.questdb.yml`): same kinds/status/aggregates/HistoryUpdate as SqlServer, soft delete by tombstone (INSERT only — never `UPDATE`/`DELETE`; every read filters `NOT deleted`). ⚠️ **Timestamps are µs** (`QuestDbConnection.Round`/`BindTimestamp`; `IHistorianDataSource.TimestampResolutionTicks` = 10 steps HistoryRead paging), **pre-1970 cannot be stored** (writer drops, HistoryUpdate `BadOutOfRange`), every write waits for WAL visibility, TTL is enforced on write commits only, set Npgsql `Maximum Pool Size` < QuestDB's 64. `docs/Historian.md` §"QuestDB backend".
|
||||
- ⚠️ **Reverse processed reads are Part 13/SDK-anchored (#677, client-visible):** every aggregate on SqlServer/QuestDB now anchors a reverse read at its later time — window `(end, start]`, partial interval earliest, each stamped with its later edge (was: earlier-anchored, stamped earlier edge); on forward reads Partial also marks the interval the data starts inside (not Average — the SDK's rule, forward-only); a non-default client `AggregateConfiguration` ⇒ `BadAggregateConfigurationRejected`; Gateway provisioning sends `IntegralDivisor = 1`. SDK-parity tests: `SdkAggregateParityTests`. `docs/Historian.md` §Aggregates.
|
||||
- ⚠️ **Reverse / end-only RAW reads work on Gateway (#686, client-visible):** `IHistorianDataSource.SupportsReverseRaw` (SqlServer/QuestDB/Null `true`, Gateway `false`, interface default `false`, guard-tested) — the node manager sends `start > end` only to a backend that declares it; otherwise `ReverseRawReader` serves it from FORWARD reads: uncapped ⇒ ONE forward read reversed in memory; capped ⇒ a bounded walk back in segments (ms-aligned request bounds, width aimed from observed density and carried page to page in the continuation point, ≤ `MaxTieClusterOverfetch + 2` rows per read — a full read is re-read, never answered truncated; only one timestamp holding more than max(1, `MaxTieClusterOverfetch`) samples ⇒ `BadTooManyOperations`; walks no later than now + 1 day and never before the Unix epoch). The raw/events tie-path oversized-cluster refusal is now `BadTooManyOperations` too (was `BadHistoryOperationUnsupported`). Uncapped Gateway raw reads (forward too) used to send `max_values = 0`, which the client/gateway reject — now unbounded (`int.MaxValue`; a ceiling refusal ⇒ `BadTooManyOperations`), and a cancelled read is no longer a backend failure. Since #669 such reads failed the node on Gateway and counted as a backend fault (the gateway rejects start > end — `GatewayHistorianDataSource.ReadRawAsync` now refuses one unsent); before #669 an end-only Gateway read returned the OLDEST N ascending — now the latest N at/before EndTime, newest first. Events still differ (end-only forward / explicit reverse refused on Gateway, #675). `docs/Historian.md` §"Open-ended and reverse raw reads".
|
||||
- ⚠️ **Reverse / end-only RAW reads work on Gateway (#686, client-visible):** `IHistorianDataSource.SupportsReverseRaw` (SqlServer/QuestDB/Null `true`, Gateway `false`, interface default `false`, guard-tested) — the node manager sends `start > end` only to a backend that declares it; otherwise `ReverseRawReader` serves it from FORWARD reads: uncapped ⇒ ONE forward read reversed in memory; capped ⇒ a bounded walk back in segments (ms-aligned request bounds, width aimed from observed density at ≈ N + 8 rows and carried page to page in the continuation point — #689: it used to aim at half `MaxTieClusterOverfetch`, ~32k rows per page on a dense tag; a read asks for `2N + 18` rows (a discovery segment ≤ 1,026), `MaxTieClusterOverfetch` is only the tie-cluster ceiling — a full read is re-read, never answered truncated; only more than max(1, `MaxTieClusterOverfetch`) samples within ONE MILLISECOND ⇒ `BadTooManyOperations` (per ms, not per timestamp — requests are ms-rounded, and letting a segment narrow below 1 ms stepped a page through one ms a tick per read, ~10k reads); walks no later than now + 1 day and never before the Unix epoch; #691: after two empty segments it asks once for the whole remainder, and the stretch a page proved empty rides the continuation point — the last pages of a deep page-through no longer each walk ~14 reads to 1970). The raw/events tie-path oversized-cluster refusal is now `BadTooManyOperations` too (was `BadHistoryOperationUnsupported`). Uncapped Gateway raw reads (forward too) used to send `max_values = 0`, which the client/gateway reject — now unbounded (`int.MaxValue`; a ceiling refusal ⇒ `BadTooManyOperations`), and a cancelled read is no longer a backend failure. Since #669 such reads failed the node on Gateway and counted as a backend fault (the gateway rejects start > end — `GatewayHistorianDataSource.ReadRawAsync` now refuses one unsent); before #669 an end-only Gateway read returned the OLDEST N ascending — now the latest N at/before EndTime, newest first. Events follow since #691.4: a reverse/end-only capped events read on Gateway (`SupportsReverseEvents` false; Null now reports `true` and answers empty) is walked back the same way (ties sorted by EventId — the gateway orders by time only; one extra event found ⇒ continuation point) — it used to be refused `BadHistoryOperationUnsupported` (explicit reverse) or answered with the OLDEST N ascending (end-only); needs the gateway's `RuntimeDb:EventReadMaxRows` 0/unlimited (it truncates silently). Uncapped Gateway events reads (forward too) used to throw "Array dimensions exceeded" (result list pre-sized to `int.MaxValue`) — fixed. Above a truncated read the walk gallops up from the newest row seen, then bisects (never below 1 ms): a burst that ended 100 days ago ≈ 18 reads, a dense tag that stopped days ago ≈ 26. `docs/Historian.md` §"Open-ended and reverse raw reads".
|
||||
- **Advertised aggregates (#682):** `IHistorianDataSource.SupportedAggregates` (Gateway 4 via `GatewayHistorianDataSource.ServedAggregates`, SqlServer/QuestDB 17 via `AggregateFamilies.Served`, Null **none**; a null set ⇒ all) is the one authority for `HistoryServerCapabilities/AggregateFunctions` (i=11201 — only its folder references are trimmed) and the up-front `BadAggregateNotSupported` in HistoryRead-Processed. ⚠️ `ServerCapabilities/AggregateFunctions` (i=2997) is deliberately NOT trimmed: it keeps the SDK's 37 because live `AggregateFilter` subscriptions still compute them all. ⚠️ Historian disabled ⇒ Processed is now `BadAggregateNotSupported`, not `GoodNoData`. `docs/Historian.md` §Advertised aggregates.
|
||||
- **Raw `ReturnBounds` is honoured on every backend (#691, client-visible):** Part 11 bounding values (§4.6 Table 1 is the test oracle) are composed in the server by `HistoryBounds` from each backend's raw reads — "at or before" = capped reverse read (native on SQL backends, the #686 `ReverseRawReader` walk-back on Gateway, never AVEVA's synthetic boundary row), "at or after" = capped forward read. Bounds are stored samples of any quality, count toward `NumValuesPerNode`, start bound on the first page / end bound on the last (pending end bound rides the continuation point); missing ⇒ `Bad_BoundNotFound` (null, at the bound time; a one-sided read's open side ±1 s from the previous value). ⚠️ **Breaking for SDK-default clients:** the .NET SDK's `new ReadRawModifiedDetails()` defaults BOTH `ReturnBounds = true` AND `IsReadModified = true` — a client that cleared only `IsReadModified` now gets bounds, and one that set neither is a modified read with bounds ⇒ `Bad_InvalidArgument` (§6.5.3.3); set both explicitly (Client.Shared does; tests building details must too). No data AND no bound found ⇒ `GoodNoData`, no values (the pre-bounds answer). On Gateway the at-or-after lookup ends at now + 1 day (never a year-9999 end to AVEVA). Kill switch `ServerHistorian:HonorReturnBounds` (default `true`; `false` = pre-#691). `docs/Historian.md` §Bounding values.
|
||||
- **Real status fix:** the recorder used to stamp every sample Good (`192`); it now carries the driver's status. ⚠️ On the **Gateway** backend this changes nothing stored: the gateway discards quality (C-002), so AVEVA still stores Good.
|
||||
- **Comms loss is historized as an outage marker** (`Empty` + `BadNoCommunication`, one per tag per outage) on SqlServer; dropped (`DroppedUnsupportedKind`) on Gateway. Outbox records are **format v2** — drain or delete `ContinuousHistorization:OutboxPath` before downgrading a node.
|
||||
- **HistoryUpdate** (SqlServer + QuestDb; Gateway ⇒ `BadHistoryOperationUnsupported`): Insert/Replace/Update/DeleteRaw/DeleteAtTime, **Primary-only** since #674 (off-Primary ⇒ `BadInvalidState`, after authorization AND after planning — an unservable request (Remove / `IsDeleteModified` / not historized) stays `BadHistoryOperationUnsupported` off-Primary; the verdict is `IHistoryUpdateRoleView` = the data-plane verdict but **closed until this node's first redundancy snapshot**, so a pair cannot both accept in the boot window; admin-only nodes register no view ⇒ no gate; reads ungated), items with a `SourceTimestamp` past now + `ServerHistorian:HistoryUpdateMaxFutureSkew` (default 1 day, ≤0 disables) ⇒ `BadOutOfRange` in the node manager (every backend), gated in both ACL modes — flag off needs the `HistoryUpdate` role from a **Cluster-scoped** grant (+ key re-check), flag on uses `DataPlaneAuthorizer`'s HistoryUpdate operation; `WriteOperate` never implies it. The `HistoryWrite` AccessLevel bit is set only when an updater is wired at materialise time. `docs/Historian.md` §HistoryUpdate, `docs/security.md`. ⚠️ **SqlServer schema v1 is accepted (#674)** with `AutoCreateSchema=false`: writes/reads/events/retention work; modified reads + HistoryUpdate ⇒ node-level `BadHistoryOperationUnsupported` on every arm (the updater throws `NotSupportedException`, nothing sent; the `HistoryWrite` bit is still advertised — not knowable at materialise time), retention skips `SampleModified`, one Warning per boot (`SqlServerSchemaBootstrapper.SupportsModifiedTrail`; `RequiredVersion` = 1, `ModifiedTrailVersion` = 2). QuestDB: keep `cairo.ttl.use.wall.clock=true`; a QuestDB trail INSERT retried after a client timeout duplicates (no DEDUP key).
|
||||
@@ -1364,8 +1365,8 @@ historian tagname, which is the tag's driver `FullName`.
|
||||
|
||||
The server dispatches all OPC UA HistoryRead to the registered `IHistorianDataSource` — the
|
||||
`GatewayHistorianDataSource` read client when enabled, else the `NullHistorianDataSource` default
|
||||
(historized nodes return `GoodNoData` for Raw/AtTime — Processed is `BadAggregateNotSupported` since #682). Supported variants: Raw (⚠️ #687, client-visible: Gateway raw reads no longer return AVEVA's synthetic start-boundary row — asked from 1 ms early (epoch-floored) with `NumValuesPerNode + 16` rows (unclamped; 0 ⇒ `int.MaxValue`) and filtered by timestamp — so an empty window is `GoodNoData`, not a fabricated value at StartTime; the forward end is now exclusive (a point read inclusive); pre-window rows that crowd the budget trigger ONE re-read with room for them, else `BadTooManyOperations`; `ReturnBounds` is ignored on every backend), Processed
|
||||
(backend-dependent: SqlServer/QuestDb serve seventeen; Gateway twelve = `GatewayHistorianDataSource.ServedAggregates` — ⚠️ #681, client-visible: the ten per-sample ones (Average, Count, Min/Max, Range, Start/End/Delta, Min/MaxActualTime) are evaluated from RAW gateway reads by the shared `Sql.Common` evaluator — identical to the SQL backends, Calculated bits, interval-start stamps, #677 reverse; Min/Max are no longer AVEVA modes, the read moves every raw sample of the window (a month of 1 Hz ⇒ `BadTooManyOperations` at the default `MaxTimeWeightedRawRows` cap), and every gateway stream is consumed to completion (an abandoned one makes the gateway evict its shared historian session) — plus TimeAverage/Total via AVEVA modes (interval-start stamped + Calculated; ⚠️ #685: AVEVA's extra pre-window row is dropped ⇒ one row per interval in `[Start, End)`, and an Uncertain (opc 64) interval keeps its value as `UncertainDataSubNormal` — AVEVA's TWA excludes NULL-gap time where the SQL backends bridge it; per-interval values still need gateway fix historiangw#27); the other time-weighted five stay `BadAggregateNotSupported` because the gateway has no prior-sample read for Part 13 bounds; `docs/Historian.md` §Aggregate set), and AtTime over historized variable nodes; Events over
|
||||
(historized nodes return `GoodNoData` for Raw/AtTime — Processed is `BadAggregateNotSupported` since #682). Supported variants: Raw (⚠️ #687, client-visible: Gateway raw reads no longer return AVEVA's synthetic start-boundary row — asked from 1 ms early (epoch-floored) with `NumValuesPerNode + 16` rows (unclamped; 0 ⇒ `int.MaxValue`) and filtered by timestamp — so an empty window is `GoodNoData`, not a fabricated value at StartTime; the forward end is now exclusive (a point read inclusive); pre-window rows that crowd the budget trigger ONE re-read with room for them, else `BadTooManyOperations`; `ReturnBounds` is ignored on every backend; ⚠️ #690: a Gateway Bad row — a stored NULL, which histsdk sends as value 0 — now reads value **null** + Bad on Raw and AtTime, as on the SQL backends; ⚠️ #691.3: a strict gateway (`AllowStreamingBeyondCeiling=false`) that refuses a raw budget is re-asked ONCE at the ceiling its refusal names — raw and per-sample reads alike; the ceiling is then remembered per data source (`GatewayRowCeiling`: refusals count as gateway errors), re-learned from a refusal when lowered, and re-probed once at the full budget when a ceiling-bounded read comes back full (raised / streaming) — so an uncapped (`NumValuesPerNode = 0`) read of a small window is served; a ceiling-bounded stream that is not provably complete is `BadTooManyOperations`, never a truncated answer), Processed
|
||||
(backend-dependent: SqlServer/QuestDb serve seventeen; Gateway twelve = `GatewayHistorianDataSource.ServedAggregates` — ⚠️ #681, client-visible: the ten per-sample ones (Average, Count, Min/Max, Range, Start/End/Delta, Min/MaxActualTime) are evaluated from RAW gateway reads by the shared `Sql.Common` evaluator — identical to the SQL backends, Calculated bits, interval-start stamps, #677 reverse; Min/Max are no longer AVEVA modes, the read moves every raw sample of the window (a month of 1 Hz ⇒ `BadTooManyOperations` at the default `MaxTimeWeightedRawRows` cap), and every gateway stream is consumed to completion (an abandoned one makes the gateway evict its shared historian session) — plus TimeAverage/Total via AVEVA modes (interval-start stamped + Calculated; ⚠️ #685: AVEVA's extra pre-window row is dropped ⇒ one row per interval in `[Start, End)`, and an Uncertain (opc 64) interval keeps its value as `UncertainDataSubNormal` — AVEVA's TWA excludes NULL-gap time where the SQL backends bridge it; per-interval values still need gateway fix historiangw#27; ⚠️ #691.2: native reads use the SQL backends' grid in both directions via forward requests only — a REVERSE read (refused by the gateway before) is anchored at its later time (#677), and the clipped interval (earliest reverse, LATEST forward — which used to be averaged past EndTime) is its own exact-span read marked Partial unless Bad; an interval wider than the window is sent as the window, interval 0 as one interval; more than 100,000 intervals is refused up front (`BadHistoryOperationUnsupported`, as on the SQL backends — forward used to be served up to the gateway's 1,000,000); until historiangw#27 lands both directions return Good-status wrong values); the other time-weighted five stay `BadAggregateNotSupported` because the gateway has no prior-sample read for Part 13 bounds; `docs/Historian.md` §Aggregate set), and AtTime over historized variable nodes; Events over
|
||||
alarm-owning equipment-folder event-notifier nodes. Reads are ungated (served from any redundancy node);
|
||||
authorization uses the standard `AccessLevels.HistoryRead` bit set at materialization.
|
||||
|
||||
@@ -1381,6 +1382,7 @@ authorization uses the standard `AccessLevels.HistoryRead` bit set at materializ
|
||||
| `CaCertificatePath` | `null` | PEM CA file pinning the gateway TLS chain; null/empty uses the OS trust store |
|
||||
| `CallTimeout` | `00:00:30` | Per-call deadline for each unary gateway read |
|
||||
| `MaxTieClusterOverfetch` | `65536` | Bounded over-fetch the HistoryRead-Raw paging uses to page within an oversized same-timestamp tie cluster (retained from the prior backend) |
|
||||
| `HonorReturnBounds` | `true` | #691 kill switch: `false` ignores HistoryRead-Raw `ReturnBounds` as before (no bounding values, no modified-read `Bad_InvalidArgument`) |
|
||||
| `HistoryReadBatchParallelism` | `4` | Max nodes served concurrently within one HistoryRead batch (03/S3); `⌈N/P⌉ × CallTimeout` worst case instead of sequential |
|
||||
| `MaxConcurrentHistoryReadBatches` | `16` | Process-wide concurrent HistoryRead batch cap (03/S3); flood degrades to `BadTooManyOperations`. Total in-flight reads capped at 16 × 4 = 64 |
|
||||
| `MaxTimeWeightedRawRows` | `1000000` | `SqlServer`/`QuestDb` (#671): raw in-window rows one time-weighted aggregate read (incl. `Total` and, since #680, PercentGood/PercentBad) may stream into the C# evaluator — and, since #681, every `Gateway` per-sample read (raw-evaluated); window query capped at this + 1, past it ⇒ `BadTooManyOperations` for that node. ≤ 0 = unbounded. Gateway (#688): the `cap + 18` budget is clamped at the gateway's default `MaxValuesCeiling` (1,000,000) while the cap is ≤ it, so the default no longer trips a strict (`AllowStreamingBeyondCeiling=false`) gateway |
|
||||
@@ -1455,6 +1457,9 @@ The target HistorianGateway OtOpcUa points at **must** run with:
|
||||
|
||||
- `RuntimeDb:Enabled=true` — enables the `WriteLiveValues` SQL live path (continuous historization).
|
||||
- `RuntimeDb:EventReadsEnabled=true` — enables `ReadEvents` from `Runtime.dbo.Events` (alarm-history reads).
|
||||
- `RuntimeDb:EventReadMaxRows=0` (its default, unlimited) — or larger than the reverse-events walk's biggest read
|
||||
(`2·MaxTieClusterOverfetch + 4`). The cap truncates **silently**, so a walked-back reverse events read (#691.4) would
|
||||
lose the newest events of a segment it cut.
|
||||
- An API key carrying scopes **`historian:read`**, **`historian:write`**, **`historian:tags:write`**.
|
||||
|
||||
### Migration note (deployments upgrading from the Wonderware backend)
|
||||
|
||||
@@ -153,6 +153,19 @@ Ordering follows from that: the drain reads in `enqueued_at_utc` order (with `id
|
||||
as a tiebreak so the ordering is total), because a hashed key carries no
|
||||
insertion sequence.
|
||||
|
||||
Gateway delivery carries a separate nonzero `DeliveryId` GUID. Direct event instances
|
||||
create it once; retries and record copies preserve it. It is excluded from queue JSON
|
||||
and business equality, so the existing payload fingerprint and pair deduplication stay
|
||||
unchanged. The drain derives its delivery GUID solely from the persisted row `id`,
|
||||
including migrated legacy rows. It never hashes a reconstructed payload or mints an ID
|
||||
per send. The versioned encoding in `AlarmDeliveryIds` therefore survives retry,
|
||||
restart, payload edits and replicated failover without a schema migration.
|
||||
|
||||
Different business payloads at the same timestamp remain distinct; completely identical
|
||||
payloads deliberately converge. An explicitly empty direct delivery ID is rejected.
|
||||
The Gateway mapper sends this GUID as the wire event ID; the live alarm readback test
|
||||
requires that exact ID. SQL Server and QuestDB retain their existing event-ID schemes.
|
||||
|
||||
`EnqueueAsync` does a single `INSERT … ON CONFLICT DO NOTHING` on the hot path.
|
||||
To avoid a `SELECT COUNT(*)` on every enqueue, the sink keeps an in-memory
|
||||
non-dead-lettered row counter (seeded at startup, kept current by every mutation,
|
||||
|
||||
@@ -456,9 +456,15 @@ from the scripted `AlarmCommandRouter`:
|
||||
carrying the authored `FullName` as the `ConditionId` and the
|
||||
authenticated `OperatorUser`. The driver forwards this to the Galaxy
|
||||
gateway → AVEVA via `GatewayGalaxyAlarmAcknowledger`.
|
||||
**Fire-and-forget** — a failed upstream ack is not surfaced back to the
|
||||
OPC UA client (mirrors the Galaxy write-outcome limitation; the local
|
||||
`AlarmConditionState` SDK update already committed at step 1).
|
||||
The Galaxy adapter awaits the correlated `AcknowledgeAlarmAsync` RPC:
|
||||
gateway protocol failures throw, while a non-zero native return code is
|
||||
warning-logged. **The actor route back to OPC UA is fire-and-forget** —
|
||||
`DriverInstanceActor` logs and drops failures without replying to the
|
||||
OPC UA client, whose local `AlarmConditionState` SDK update already
|
||||
committed at step 1. This is a native-ack routing limitation; Galaxy
|
||||
value writes have a separate correlated write-outcome path. An awaited
|
||||
ack RPC does not prove the ack reached AVEVA event history: the H6 live
|
||||
verification remains [operator-gated](plans/2026-06-15-stillpending-phase-3-opcua-standards-design.md).
|
||||
|
||||
Only the **Acknowledge** is routed to the driver. **A native condition
|
||||
supports Acknowledge and AddComment only** (Gitea #587 review): `Confirm`,
|
||||
|
||||
+368
-59
@@ -111,6 +111,9 @@ Processed read is `BadAggregateNotSupported` and no history aggregate functions
|
||||
"CaCertificatePath": null,
|
||||
"CallTimeout": "00:00:30",
|
||||
"MaxTieClusterOverfetch": 65536,
|
||||
"HonorReturnBounds": true,
|
||||
"GatewayAtTimeFromRaw": true,
|
||||
"EventReadHorizon": "00:00:00",
|
||||
"HistoryReadBatchParallelism": 4,
|
||||
"MaxConcurrentHistoryReadBatches": 16,
|
||||
"HistoryReadDeadline": "00:01:00",
|
||||
@@ -143,10 +146,13 @@ Processed read is `BadAggregateNotSupported` and no history aggregate functions
|
||||
| `AllowUntrustedServerCertificate` | bool | `false` | **Gateway backend only.** Accept a self-signed / untrusted server certificate (dev / on-prem only). |
|
||||
| `CaCertificatePath` | string\|null | `null` | **Gateway backend only.** PEM CA file pinning the gateway's TLS chain. Null/empty uses the OS trust store. |
|
||||
| `CallTimeout` | TimeSpan | `00:00:30` | **Gateway backend only.** Per-call deadline applied to each unary gateway read. A read for a tag the historian does not yet know about — e.g. a freshly-historized tag that has **not** been provisioned (see [Tag auto-provisioning](#tag-auto-provisioning-ensuretags)) — can run to this **full** deadline before the gateway errors, so an unprovisioned-tag HistoryRead can block for ~30 s. Lower this to fail faster, at the cost of truncating legitimately long reads. |
|
||||
| `MaxTieClusterOverfetch` | int | `65536` | (Node-manager paging; all backends — a `SqlServer` PK rules out tie clusters, so it is moot there.) Maximum samples the server will fetch in one shot to page through a tie cluster (multiple samples sharing one `SourceTimestamp`). A cluster larger than this ceiling fails `BadTooManyOperations` (#686; was `BadHistoryOperationUnsupported`). Also the per-read row budget of a Gateway reverse raw read (walked back from forward reads, #686): one timestamp holding more samples is refused `BadTooManyOperations`. Values below 1 count as 1. Raise to handle abnormally large tie clusters; the default covers all normal-data cases. |
|
||||
| `MaxTieClusterOverfetch` | int | `65536` | (Node-manager paging; all backends — a `SqlServer` PK rules out tie clusters, so it is moot there.) Maximum samples the server will fetch in one shot to page through a tie cluster (multiple samples sharing one `SourceTimestamp`). A cluster larger than this ceiling fails `BadTooManyOperations` (#686; was `BadHistoryOperationUnsupported`). Also the per-read row budget of a Gateway reverse raw read (walked back from forward reads, #686): one timestamp holding more samples is refused `BadTooManyOperations`. Also (Gateway, #694 R2) the most rows inside the 1 ms look-back before a raw window that a crowding re-read makes room for. Values below 1 count as 1. Raise to handle abnormally large tie clusters; the default covers all normal-data cases. |
|
||||
| `HonorReturnBounds` | bool | `true` | (#691, all backends.) Whether HistoryRead-Raw honours `ReturnBounds` (Part 11 bounding values, [Bounding values](#bounding-values-returnbounds-691)). `false` is the kill switch back to the pre-#691 behaviour: the flag is ignored (stored samples only) and a modified read carrying it is not refused `Bad_InvalidArgument`. Note the .NET SDK defaults both `ReturnBounds` and `IsReadModified` to `true`. |
|
||||
| `GatewayAtTimeFromRaw` | bool | `true` | (#693, **Gateway backend only.**) Whether HistoryRead-AtTime is served from raw gateway reads evaluated with the SQL backends' AtTime rules ([Gateway AtTime](#gateway-attime-is-evaluated-from-raw-reads-693)). `false` is the kill switch back to the pre-#693 answer: AVEVA's own interpolation (the gateway's `ReadAtTime`) passed through unchanged. |
|
||||
| `EventReadHorizon` | TimeSpan | `00:00:00` | (#694, all backends, **opt-in, lossy**.) `00:00:00` = no horizon: every HistoryReadEvents read is exact to its own window. A positive span raises the earlier bound of every fresh events window to `now − EventReadHorizon` (the floor instant itself included), in **both** directions — a window wholly older is `GoodNoData` and is not sent; a page-through keeps the first page's floor. It exists for the Gateway, where a walked reverse / end-only read proves "nothing older" down to the Unix epoch (~3.8 s live; an unfiltered one can hit the gateway's 30 s SQL timeout) and a forward read from a far-past `StartTime` scans the same range. ⚠️ While set, events older than the horizon are **invisible** to HistoryReadEvents. Raw/Processed/AtTime are unaffected. Negative ⇒ treated as none (a `Validate()` warning). See [Continuation-point paging (Events)](#continuation-point-paging-events-669). |
|
||||
| `HistoryReadBatchParallelism` | int | `4` | Max historized nodes served **concurrently** within one HistoryRead batch (arch-review 03/S3). The SDK's `HistoryRead*` override surface is synchronous, so each arm bridges once at its boundary; this turns per-node serving from sequential (`N × CallTimeout` worst case) into `⌈N / P⌉ × CallTimeout`. Also caps the gateway-load multiplication a single client can command. `≤ 1` serves sequentially. |
|
||||
| `MaxConcurrentHistoryReadBatches` | int | `16` | Process-wide limit on HistoryRead (and HistoryUpdate) batches served **concurrently** (arch-review 03/S3). A flood of history clients degrades to `BadTooManyOperations` rather than exhausting the SDK request-thread pool. With `HistoryReadBatchParallelism` this caps total in-flight gateway reads at `MaxConcurrentHistoryReadBatches × HistoryReadBatchParallelism` (default 16 × 4 = 64). A batch that cannot get a slot within `min(HistoryReadDeadline, 5 s)` fails every handle `BadTooManyOperations`. |
|
||||
| `MaxTimeWeightedRawRows` | int | `1000000` | `SqlServer` / `QuestDb` (#671), and the `Gateway` per-sample aggregates (#681 — every per-sample read there streams raw samples, so the cap applies to the window's raw rows; see [Aggregate set](#aggregate-set-and-badaggregatenotsupported-285)). The most raw in-window samples one [time-weighted aggregate](#time-weighted-aggregates-part-13-671) read (TimeAverage, TimeAverage2, **Total**, Total2, Interpolative, and — since #680 — **PercentGood/PercentBad**) may stream into the server-side evaluator — that family is **not** pushed down to SQL. The window query is capped at this + 1 rows; one row past the cap fails **that node** with `BadTooManyOperations` (not counted as a backend fault) instead of holding a pooled connection until `HistoryReadDeadline` and returning `BadTimeout`. Bound rows outside the window do not count; per-sample aggregates are not capped on the SQL backends (pushed down). 1 000 000 ≈ 11.5 days of a 1 Hz tag. Non-positive = unbounded (a `Validate()` warning). `Gateway` (#688): the row budget it asks the gateway for (`cap + 18`) is clamped at the gateway's default `MaxValuesCeiling` (1,000,000) while the cap is at or under it, so the default works against a gateway with `AllowStreamingBeyondCeiling=false`; against a gateway with a *lower* ceiling, set this to `ceiling − 18`. |
|
||||
| `MaxTimeWeightedRawRows` | int | `1000000` | `SqlServer` / `QuestDb` (#671), and the `Gateway` per-sample aggregates (#681 — every per-sample read there streams raw samples, so the cap applies to the window's raw rows; see [Aggregate set](#aggregate-set-and-badaggregatenotsupported-285)). The most raw in-window samples one [time-weighted aggregate](#time-weighted-aggregates-part-13-671) read (TimeAverage, TimeAverage2, **Total**, Total2, Interpolative, and — since #680 — **PercentGood/PercentBad**) may stream into the server-side evaluator — that family is **not** pushed down to SQL. The window query is capped at this + 1 rows; one row past the cap fails **that node** with `BadTooManyOperations` (not counted as a backend fault) instead of holding a pooled connection until `HistoryReadDeadline` and returning `BadTimeout`. Bound rows outside the window do not count; per-sample aggregates are not capped on the SQL backends (pushed down). 1 000 000 ≈ 11.5 days of a 1 Hz tag. Non-positive = unbounded (a `Validate()` warning). `Gateway` (#688): the row budget it asks the gateway for (`cap + 18`) is clamped at the gateway's default `MaxValuesCeiling` (1,000,000) while the cap is at or under it, so the default works against a gateway with `AllowStreamingBeyondCeiling=false`; against a gateway with a *lower* ceiling the refused budget is retried once at the ceiling the gateway names (#691.3), so no change is needed (`ceiling − 18` only saves that round-trip). |
|
||||
| `HistoryReadDeadline` | TimeSpan | `00:01:00` | Per-request wall-clock deadline for a HistoryRead (and HistoryUpdate) batch (arch-review 03/S3). A node still in flight at the deadline surfaces `BadTimeout`, so one request can never hold an SDK request thread longer than this regardless of node count. Non-positive = unbounded (a `Validate()` warning). |
|
||||
| `HistoryUpdateMaxFutureSkew` | TimeSpan | `1.00:00:00` | #674. How far past the server clock a HistoryUpdate (Insert / Replace / Update) item's `SourceTimestamp` may lie; an item beyond `now + skew` is refused **`BadOutOfRange`** by the node manager before any backend sees it (every backend). Guards against a far-future typo (a 2062 timestamp). The live recorder is not bounded. Non-positive disables the bound (no warning — a deliberate choice). See [HistoryUpdate](#historyupdate). |
|
||||
|
||||
@@ -567,7 +573,8 @@ node without `isHistorized` has no HistoryRead access bit and answers
|
||||
| Historized node, historian configured, time range empty | `GoodNoData` |
|
||||
| Historized node, historian NOT configured (`Enabled=false` / Null source) | `GoodNoData` (empty) for Raw / AtTime; Processed ⇒ `BadAggregateNotSupported` (#682) |
|
||||
| Non-historized node | `BadHistoryOperationUnsupported` |
|
||||
| Backend timeout or exception | `BadHistoryOperationUnsupported` per node; other nodes in the same batch are unaffected |
|
||||
| Backend transport outage (#697) | `BadNoCommunication` per node across Raw, Modified, Processed, AtTime and Events; cancelled/deadline-limited server reads retain `BadTimeout` |
|
||||
| Other backend exception / unsupported operation | `BadHistoryOperationUnsupported` per node; other nodes in the same batch are unaffected |
|
||||
| `Backend=SqlServer`, `AutoCreateSchema=false`, schema missing/behind | every read/write fails with a clear error naming the missing schema version (health `LastError`); reads answer `BadHistoryOperationUnsupported` per node; no host crash |
|
||||
|
||||
A historized node with no historian configured never returns an error status — it returns
|
||||
@@ -580,23 +587,44 @@ gateway is provisioned, without producing error spikes in connected clients.
|
||||
`NumValuesPerNode` samples with no continuation point of its own), so the server synthesises
|
||||
paging time-based:
|
||||
|
||||
- A page that returns **exactly** `NumValuesPerNode` samples (with `NumValuesPerNode > 0`) MAY
|
||||
have more behind it, so the server stores a resume cursor and returns an opaque continuation
|
||||
- Every page's backend read asks for **one lookahead row** past the page (`NumValuesPerNode + 1`, plus the boundary
|
||||
ties a resumed page is about to trim — see below). When the lookahead row comes back there is more data: the page
|
||||
returns exactly `NumValuesPerNode` values, and the server stores a resume cursor and returns an opaque continuation
|
||||
point (16 bytes). The client hands it back to fetch the next page.
|
||||
- A short page (fewer than the cap) is the last page — no continuation point.
|
||||
- When it does not, the page is the **last** — no continuation point, even when it is exactly full. (#692: a total that
|
||||
is an exact multiple of the cap used to end with an extra empty `GoodNoData` page.) The exception is an oversized tie
|
||||
cluster (below): its pages are sliced from the over-fetched cluster, its last slice is `min(N, remaining)` values and
|
||||
still carries a continuation point when the window goes on — so stepping past a cluster at the very end of the data can
|
||||
yield one empty final page.
|
||||
- `NumValuesPerNode == 0` ("all values, no limit") is never paged; the whole window returns in one
|
||||
shot.
|
||||
- The resume cursor is **tie-safe**: the next page resumes from the last returned sample's
|
||||
SourceTimestamp *inclusive* and drops the boundary samples already emitted, so samples sharing
|
||||
the boundary timestamp are neither duplicated nor skipped.
|
||||
the boundary timestamp are neither duplicated nor skipped. Because it re-reads them, a resumed page asks for
|
||||
`NumValuesPerNode + skip + 1` rows, so it still carries a full page. On a strict Gateway
|
||||
(`AllowStreamingBeyondCeiling=false`) that moves the row-ceiling edge by `skip + 1`: a `NumValuesPerNode` just under the
|
||||
learned ceiling may now cost one learning refusal (#691.3 re-asks at the ceiling). A page clamp to the learned ceiling is
|
||||
being added separately (#694).
|
||||
- ⚠️ **#692 (client-visible): every page is full** (outside an oversized tie cluster — see above). A resumed page used to ask for only `NumValuesPerNode` rows, so after
|
||||
the trim it netted `N − skip` values — and at `NumValuesPerNode = 1` **nothing**: the lone re-read boundary sample was
|
||||
trimmed, the empty "full" page was taken for a stalled tie cluster, and a `MaxTieClusterOverfetch + 1` (65,537-row)
|
||||
over-fetch stepped one tick to return an **empty page with a continuation point between every value** (live, AVEVA:
|
||||
8 values = 17 pages / 25 backend reads; on every backend, forward and reverse, with or without `ReturnBounds`). Now 8
|
||||
values at N = 1 is 8 pages and 8 reads. Clients that drained to the end were unaffected in content, only in cost; a
|
||||
client that stopped at the first empty page lost data.
|
||||
|
||||
> **Oversized tie clusters — within-timestamp paging.** When more samples share one
|
||||
> `SourceTimestamp` than the current page cap, the server detects that the cursor has stalled on
|
||||
> a tie cluster (the last returned timestamp equals the resume timestamp). It then **over-fetches
|
||||
> a tie cluster (every row of the resumed `N + skip + 1` read past the trimmed ties — the lookahead row included — still
|
||||
> sits on the resume timestamp; since #692 nothing else trips it). It then **over-fetches
|
||||
> the entire cluster** at that single timestamp up to a bounded ceiling controlled by
|
||||
> `ServerHistorian:MaxTieClusterOverfetch` (default **65 536**), then serves the cluster
|
||||
> `NumValuesPerNode` samples at a time across successive pages, advancing the cursor one tick past
|
||||
> the timestamp once the cluster is fully drained.
|
||||
> the timestamp once the cluster is fully drained. A cursor left inside the cluster is marked so (`WithinTieCluster`), and
|
||||
> its next page over-fetches the cluster directly instead of first re-reading rows it knows are ties. A drained cluster
|
||||
> resumes only while the next step is strictly before `EndTime` (truncated to the backend's resolution) — the forward
|
||||
> window is end-exclusive; resuming AT `EndTime` used to make the next read a point read that returned the samples at
|
||||
> `EndTime` as data (#692 review).
|
||||
>
|
||||
> **Reverse reads** (`start > end`, newest first) page the same way in the opposite direction: a drained
|
||||
> cluster steps the cursor one tick **down**, the read ends when that would reach the (excluded) `end`, and
|
||||
@@ -639,33 +667,60 @@ backend:
|
||||
`[⌊segLo⌋, ⌈segHi + 1 tick⌉]`, both rounded **outward to the millisecond**, and filtered on the samples' own
|
||||
timestamps — exact for an inclusive or exclusive forward end, and the row AVEVA fabricates at a query's (possibly
|
||||
rounded-up) start can never land inside the segment. The first segment is **1 minute** (or the width the previous
|
||||
page ended on — it rides the continuation point); each next width is aimed at about half the row budget from the
|
||||
density just observed, growing at most ×4. So "the last N values" of a live tag is normally one read, a dense tag
|
||||
costs about one read per page, and sparse history is found in a few dozen tiny ones.
|
||||
- **Never answered from a truncated read.** The row budget is `M = max(1, MaxTieClusterOverfetch)`; a segment read asks
|
||||
for `M + 2` rows (room for the fabricated row and one to see past the segment). Rows stream ascending, so a read is
|
||||
complete when it came back short or its last row is past the segment; otherwise it is discarded — re-read once with
|
||||
room for the rows before the segment that crowded it (a neighbouring tie cluster, at most `2M + 4` rows), or narrowed
|
||||
to the density it showed. Only **one timestamp holding more than `M` samples** is refused, **`BadTooManyOperations`**
|
||||
(a request limit, not a backend fault); a cluster of exactly `M`, or neighbouring clusters each within `M`, is served.
|
||||
page ended on — it rides the continuation point); each next width is aimed from the density just observed at about
|
||||
half the **page budget** `2N + 16` (≈ N + 8 rows), growing at most ×4. So "the last N values" of a live tag is
|
||||
normally one read of about N rows, a dense tag costs about one N-sized read per page, and sparse history is found in
|
||||
a few dozen tiny ones.
|
||||
- **Read sizes (#689).** A segment read asks for its budget + 2 rows (room for the fabricated row and one to see past
|
||||
the segment). The budget is the page budget `2N + 16`, except a **discovery** segment — the first of a page with no
|
||||
carried width, or one after an empty segment, whose ×4 growth may overshoot into dense data — which may read up to
|
||||
**1,024** rows. The tie budget `M = max(1, MaxTieClusterOverfetch)` is only a **ceiling**: a single timestamp holding
|
||||
more rows than the segment budget escalates to `M`.
|
||||
⚠️ Before #689 every segment was aimed at half of `M` (≈ 32k rows by default) and that width was carried page to
|
||||
page, so on a 1 Hz tag from about page 5 every page of a "latest N" read fetched ~32,770 rows and took 2–5 s to
|
||||
return N.
|
||||
- **Never answered from a truncated read.** Rows stream ascending, so a read is complete when it came back short or its
|
||||
last row is past the segment; otherwise it is discarded — re-read once with room for the rows before the segment
|
||||
that crowded it (a neighbouring tie cluster, at most twice the budget + 4 rows), or narrowed: to the density it
|
||||
showed, and never again below the newest row it saw until what lies above that row is read. The gap above it is
|
||||
searched: a segment at that density ending on the top (data running right up to it); if empty, a **gallop** up from
|
||||
that row (`(row + d, top]`, `d` ×4 per step, following each truncated read) until a read comes back empty, then a
|
||||
**bisection** of the bracket — never narrower than the 1 ms request granularity, which only re-sent the same request.
|
||||
Measured: a dense tag that stopped two days ago ≈ **26** reads, a burst of just over 1,024 rows that ended 100 days ago
|
||||
≈ **18** (was 46 bisecting the whole gap) — rather than re-reading the oldest rows of a burst over and over. Only **more than `M` samples within one millisecond** are refused,
|
||||
**`BadTooManyOperations`** (a request limit, not a backend fault); a cluster of exactly `M`, or neighbouring
|
||||
clusters each within `M` and a millisecond or more apart, is served. ⚠️ The unit is the **millisecond**, not the
|
||||
timestamp (#691.4 re-review): requests are rounded to the millisecond, so a narrower segment is the same request —
|
||||
letting the width fall below it once stepped a page through one millisecond a tick at a time (~10,000 reads). Exact
|
||||
for the Gateway, which stores milliseconds; a sub-millisecond backend walked this way would refuse more than `M`
|
||||
samples inside one millisecond even when no single timestamp holds that many.
|
||||
- **Proving the rest empty (#691).** Walking down through empty history to the floor used to cost one read per ×4
|
||||
growth step — ~14–16 reads (~2.5 s live) on **every** last page of a deep page-through. Now, after two consecutive
|
||||
empty segments, the walk asks **once per page** for the whole remainder down to the floor at the page budget: a
|
||||
complete answer ends the walk in one read; a truncated one (deep history) is discarded and the walk carries on. And
|
||||
the stretch the page's last complete read proved empty below its oldest row (often all the way to the floor) rides
|
||||
the continuation point, so the next page steps straight over it. Data backfilled into such a stretch between pages
|
||||
is not seen by that read — paging was never a snapshot (the forward path misses data inserted behind its cursor too).
|
||||
- **Bounds.** The walk starts no later than **now + 1 day** (a `StartTime = DateTime.MaxValue` read does not send
|
||||
year-9999 query starts) and never reads before the **Unix epoch** (AVEVA holds nothing earlier): a walked-back read
|
||||
returns only samples strictly after 1970-01-01T00:00:00Z and no later than now + 1 day.
|
||||
|
||||
Paging is unchanged — a reverse cursor resumes as a fresh walk from its resume point.
|
||||
Paging is otherwise unchanged — a reverse cursor resumes as a fresh walk from its resume point, carrying the width and
|
||||
the proven-empty stretch (session-held cursor state, never serialized, so the new fields need no versioning).
|
||||
- **Null** reports `true` (empty in either direction).
|
||||
|
||||
> ⚠️ **Client-visible (#686).** Since #669 an end-only capped raw read, and any explicit reverse raw read, on the Gateway
|
||||
> backend failed the node (`BadHistoryOperationUnsupported`) and was counted as a historian backend failure (the gateway
|
||||
> refused `start > end`). Before #669 an end-only read on Gateway returned the **oldest** N values ascending. Both now return
|
||||
> the Part 11 answer — the latest values at or before `EndTime`, newest first. **Events differ:** an end-only events read on
|
||||
> Gateway still keeps its forward meaning and an explicit reverse events read is refused (see below).
|
||||
> the Part 11 answer — the latest values at or before `EndTime`, newest first. **Events** follow since #691.4 (see below).
|
||||
>
|
||||
> ⚠️ **Also fixed with #686: uncapped raw reads on Gateway.** `NumValuesPerNode = 0` ("all values") used to be forwarded as
|
||||
> `max_values = 0`, which the client package and the gateway reject — every uncapped raw read (forward too) failed the node
|
||||
> and counted as a backend failure. It is now sent unbounded (`int.MaxValue`); a gateway configured with
|
||||
> `AllowStreamingBeyondCeiling=false` refuses that over its `MaxValuesCeiling`, which is answered **`BadTooManyOperations`**
|
||||
> (not a backend fault). A cancelled read (client cancel / `HistoryReadDeadline`) is no longer counted as a backend failure.
|
||||
> `AllowStreamingBeyondCeiling=false` refuses that over its `MaxValuesCeiling`; since #691.3 the read is retried once at the
|
||||
> ceiling the refusal names and answered when provably complete, else **`BadTooManyOperations`** (not a backend fault) — see
|
||||
> [#687](#gateway-raw-reads-return-stored-samples-only-687) "Strict gateway". Since #694 the HistoryRead pages such a read at the
|
||||
> learned ceiling with continuation points instead (same section, "Pages are sized to a strict gateway's ceiling"). A cancelled read (client cancel / `HistoryReadDeadline`) is no longer counted as a backend failure.
|
||||
|
||||
Continuation points are bound to the OPC UA session (the SDK's
|
||||
`ServerConfiguration.MaxHistoryContinuationPoints` cap, default 100, with oldest-eviction; points
|
||||
@@ -686,7 +741,12 @@ data.
|
||||
the gateway is asked from **1 ms before** the window (never before the Unix epoch, the start of the historian's time encoding),
|
||||
so the boundary row lands before it, and only rows `RawAggregateReader.IsWindowSample` are returned. The decision is by
|
||||
**timestamp**, not by value or quality — a stored NULL sample looks exactly like the no-data boundary row on the wire (value 0,
|
||||
Bad), and a stored NULL inside the window is still returned (Bad), as it is a stored sample. A row stamped **at or before the
|
||||
Bad), and a stored NULL inside the window is still returned (Bad), as it is a stored sample. ⚠️ **Since #690 it is returned
|
||||
with a null value** (client-visible): histsdk puts a value on every row, so the NULL used to come back as value `0` + Bad (`""`
|
||||
on a string tag), a fabricated 0 for any client that reads the value without the status. `SampleMapper.ToSnapshot` drops the
|
||||
value of every row whose mapped status is **Bad** (AVEVA stores a Bad-quality sample as NULL, so a Bad row never carries a real
|
||||
value) — raw reads and AtTime alike, matching SqlServer/QuestDB (an Empty sample reads null + Bad). Uncertain rows keep their
|
||||
value; the raw-evaluated aggregates were already unaffected (a Bad sample is never usable, and Start/End null a Bad value). A row stamped **at or before the
|
||||
epoch** is never a stored sample (AVEVA stores nothing there — the #686 walk-back's floor too): a window starting at or before
|
||||
the epoch gets no margin, so AVEVA's no-data row lands exactly at its start; raw reads and the per-sample aggregates both drop it.
|
||||
- **The forward end is exclusive** (`[StartTime, EndTime)`, the `IHistorianDataSource` contract and what the SQL backends return;
|
||||
@@ -696,20 +756,220 @@ data.
|
||||
- **Row budget.** `NumValuesPerNode = 0` ("all values") asks the gateway for an unbounded `max_values` (`int.MaxValue` — the client
|
||||
package and the gateway reject 0, #686). A cap asks for `NumValuesPerNode + 16` rows, **not** clamped at the gateway's default
|
||||
ceiling (#688 clamps only the per-sample aggregate budget): AVEVA always emits the boundary row, so a clamped budget would make
|
||||
every page size from 999,985 to 1,000,000 come back short. A gateway with `AllowStreamingBeyondCeiling=false` refuses an
|
||||
over-ceiling budget instead ⇒ `BadTooManyOperations` (not a backend fault); a default gateway serves it.
|
||||
- **Paging (#669) is unaffected.** A full page still holds exactly `NumValuesPerNode` stored samples and carries a continuation
|
||||
every page size from 999,985 to 1,000,000 come back short. A default gateway serves any budget.
|
||||
- ⚠️ **Strict gateway (#691.3, client-visible).** A gateway with `AllowStreamingBeyondCeiling=false` refuses a budget over its
|
||||
`MaxValuesCeiling` (before streaming) and names the ceiling in the refusal; the gateway has no RPC that reports it. The read is
|
||||
then **re-issued once with the ceiling as the budget**, and the ceiling is **remembered per data source** (`GatewayRowCeiling`,
|
||||
shared by raw and per-sample reads): every refused attempt counts as an error in the gateway's own outcome metrics, so after the
|
||||
first refusal later reads ask for at most the learned ceiling and are not refused. A **lowered** ceiling is re-learned from the
|
||||
refusal of the remembered one. A **raised** ceiling (or a switch to streaming) is noticed by the only reads it matters to: when a
|
||||
read bounded by the remembered ceiling comes back full without proving completeness, it is re-probed **once** at its full budget
|
||||
— served ⇒ the ceiling is forgotten; refused naming a larger ceiling ⇒ that one is learned and the read re-issued at it; refused
|
||||
naming the same ceiling ⇒ the bounded result stands. Raw and AtTime re-probe immediately. Per-sample aggregates (#696) reuse
|
||||
a refusal for **30 seconds** before re-probing a full stream; within that period an oversized window still fails honestly with
|
||||
`BadTooManyOperations`. A lowered ceiling is still re-learned immediately; a raise becomes visible on the next incomplete
|
||||
aggregate read after the cooldown. So an uncapped read (`NumValuesPerNode = 0`, which used to be refused outright on a strict
|
||||
gateway, even for a 10-row
|
||||
window) and a page size within 16 of the ceiling are now served when the window fits. A ceiling-bounded read is answered only
|
||||
when it is **provably complete** — the stream came back short of the ceiling, or reached `EndTime` (or, capped, filled the page);
|
||||
a stream that delivered the whole ceiling short of `EndTime` may be truncated and fails **`BadTooManyOperations`** (not a backend
|
||||
fault) — set `NumValuesPerNode` to page, or narrow the window. A refusal whose text carries no number falls back to the gateway's
|
||||
default ceiling (1,000,000). Every stream is still drained.
|
||||
- ⚠️ **Pages are sized to a strict gateway's ceiling (#694, client-visible).** Until #694 a page size above a strict gateway's
|
||||
ceiling failed: with `MaxValuesCeiling = 1000`, `NumValuesPerNode = 5000` — or `0` over 3,600 rows — was `BadTooManyOperations`.
|
||||
The data source now reports the learned ceiling to the server as `IHistorianDataSource.MaxRawValuesPerRead` = **L** = ceiling − 16
|
||||
(`0` until a refusal has named a ceiling, and again after a re-probe finds the gateway streaming; `0` on every SQL backend). The
|
||||
node manager plans each page with it (`RawPageLimit`): a page's read is the page **P** + the boundary ties **k** it re-reads and
|
||||
trims + the #692 lookahead row, and with a limit that read is at most L — so **P = min(N, L − k − 1)** (uncapped: P = L − k − 1;
|
||||
with the ceiling at 1000 a first page holds 983 values, a resumed one 982), and a full page carries a continuation point like any
|
||||
other (Part 11 allows fewer than `NumValuesPerNode` with one). **This includes uncapped reads** (`NumValuesPerNode = 0`), which
|
||||
page instead of being refused; a window that fits is still one page without a continuation point. Without a limit the #692 plan
|
||||
is unchanged. Boundary ties that alone fill a read (k ≥ L − 1) go straight to the tie-cluster over-fetch — never an uncapped or
|
||||
underflowing request.
|
||||
- **First read after start-up:** the ceiling is unknown until refused once; the read that learned it is re-planned and re-issued
|
||||
ONCE (cost on that read: the refused attempt, the ceiling re-read that could not hold the window, the page-sized read; later
|
||||
pages are one read each). Reverse / end-only reads (the #686 walk-back) size their segments to L (tie budget
|
||||
`min(MaxTieClusterOverfetch, L − 2)`); on the page path the refusal reaches the page planner, which shrinks the **page** — the
|
||||
walk never re-runs itself at the ceiling with the page still at N (review F2: a cold end-only read with a ceiling of 20 and
|
||||
N = 3000 cost 3,023 gateway reads in one HistoryRead; now ≤ 40). Bound lookups (a page of one) keep the walk's own re-run.
|
||||
- **Crowded look-back (review F3).** A page asking for exactly the ceiling leaves no room for the #687 1 ms look-back, so more
|
||||
than 16 rows just before a page start (e.g. 20 ties 1 ms before it) used to fail `BadTooManyOperations`. The data source now
|
||||
reports the crowding rows on its refusal (`HistoryReadLimitExceededException.PreWindowRows`, only when they exceed the 16-row
|
||||
allowance), and the planner re-reads the page once that many values smaller — the source's own crowding re-read then fits under
|
||||
the ceiling. In the walk-back, a crowded segment is re-read that much smaller and counts as truncated.
|
||||
- **Tie clusters.** The over-fetch that pages within a cluster reads at most `min(MaxTieClusterOverfetch, L − 1)` + 1 rows (so it
|
||||
never trips a refusal); a larger cluster is **`BadTooManyOperations`** — the gateway cannot offset into it. A crowding refusal that
|
||||
leaves no room for a page is that answer too (it is never mistaken for "the cursor sits in a cluster", which would step the
|
||||
cursor a tick per page forever). Every sample before such a failure is served exactly once; nothing is truncated silently.
|
||||
- **Never a truncation:** the gateway read is still answered in full or refused; only the server's page is smaller.
|
||||
- A raised ceiling is not noticed by raw paging (its reads ask for at most the learned ceiling, so nothing re-probes); pages stay
|
||||
at the old size until a read asking for more (a per-sample aggregate) re-probes it, or the server restarts — correct results
|
||||
either way. A **lowered** ceiling is re-learned from the next page's refusal and that page re-planned.
|
||||
- ⚠️ **A window starting at or before the Unix epoch (review F1, fixed with #694).** Such a window gets no 1 ms look-back, so
|
||||
AVEVA's synthetic row lands exactly at its start; it is no sample, and it used to count as "past the window end" — a
|
||||
ceiling-truncated read was then reported complete (an uncapped forward read from the epoch answered its first ceiling − 1 rows,
|
||||
Good, no continuation point). It now counts as a pre-window row.
|
||||
- **Paging (#669) is unaffected.** A full page still holds exactly the page size of stored samples and carries a continuation
|
||||
point, and a resumed page (which re-reads from the last returned timestamp inclusive, and the tie-cluster over-fetch
|
||||
`start == end` read) no longer gets a fabricated row at its resume point. A read is complete when the stream came back short,
|
||||
filled the page, or reached EndTime. If rows **before** the window — the boundary row plus stored samples inside the 1 ms margin,
|
||||
e.g. a tie cluster at `StartTime − 1 ms` or at a resume point `− 1 ms` — used up the budget, the window is re-read **once**
|
||||
asking for `NumValuesPerNode + m + 16` (m = the pre-window rows the first read saw, unclamped). Only if that re-read is still
|
||||
short because of pre-window rows does the node fail `BadTooManyOperations` (not a backend fault) rather than answer a short page
|
||||
e.g. a tie cluster at `StartTime − 1 ms` or at a resume point `− 1 ms` — used up the budget, the window is re-read with room for
|
||||
them: `cap + room + 16`, room = the pre-window rows the first read saw, at least **doubling** per re-read while a read that was
|
||||
itself cut short keeps finding more (up to 8 re-reads, never past a known strict ceiling — review F4: a single re-read sized from
|
||||
a truncated read's count failed a 300–1,700-row cluster just before a resume point even on a default gateway). The room is
|
||||
**capped at `MaxTieClusterOverfetch`** (#694 R2, the limit every other tie path uses): more rows than that inside the look-back ⇒
|
||||
`BadTooManyOperations` after a bounded number of bounded reads — uncapped, a pathological million-row single-millisecond cluster
|
||||
on a streaming gateway would have been streamed several times over for one page. Only if the
|
||||
re-reads are still crowded does the node fail `BadTooManyOperations` (not a backend fault) rather than answer a short page
|
||||
that would end the client's paging early. Every gateway stream is still drained to completion.
|
||||
- **`ReturnBounds` is not honoured** — on any backend: the server does not pass it to the data source, so no bounding values are
|
||||
returned (see [Known limitations](#known-limitations)). The AVEVA boundary row is not used as a bound either: it is stamped at the
|
||||
query start rather than carrying the time of the real prior sample, so it could only ever be a Part 11 *interpolated* bound, which
|
||||
this backend does not compute.
|
||||
- **`ReturnBounds` is served by the server, not by this row** (#691, [Bounding values](#bounding-values-returnbounds-691)). The AVEVA
|
||||
boundary row is never used as a bound: it is stamped at the query start rather than carrying the time of the real prior sample, so
|
||||
it could only ever be a Part 11 *interpolated* bound. The start bound on the Gateway is the real prior sample, found by the
|
||||
#686 walk-back.
|
||||
|
||||
**Warm crowding ceiling (#695).** A raw crowding re-read spends the remaining room below a learned gateway ceiling before
|
||||
refusing; doubling the look-back allowance does not skip that final usable budget. Reverse paging retries a crowded segment
|
||||
with a smaller in-window budget derived from the learned ceiling and the observed pre-window rows.
|
||||
|
||||
**Missing tags (#653).** Gateway Raw, Processed and AtTime first check the tag catalog. An empty exact-name catalog result or an explicit
|
||||
`NotFound` returns no data without a history RPC: Raw is empty; AtTime and Processed return null/`BadNoData` at the
|
||||
requested times or bucket timestamps. Unsupported aggregate validation still runs first. A bounded per-data-source cache
|
||||
holds at most 4,096 tag names, with **5 seconds** for absence and **60 seconds** for presence, so newly provisioned tags become
|
||||
visible after the short negative expiry. Catalog faults and cancellation never become missing-tag cache entries. The adapter drains the filtered catalog stream and checks exact names, avoiding older gateways that misparse a missing `GetTagMetadata` response as `Internal`. Empty
|
||||
AtTime/Processed requests need no check. Events and modified-history capability checks are unaffected.
|
||||
|
||||
**Stopped-tag reverse discovery (#696).** On a strict Gateway, a truncated remainder probe retains its oldest-row evidence
|
||||
and density. Gap discovery uses the learned ceiling's available budget; ordinary page reads keep their page-sized budget.
|
||||
Only complete segments contribute to a page, preserving newest-first order, bounds, ties and continuation hints.
|
||||
|
||||
### Gateway AtTime is evaluated from raw reads (#693)
|
||||
|
||||
> ⚠️ **Client-visible (#693).** Gateway AtTime used to pass AVEVA's own interpolated answer (the gateway's `ReadAtTime`) through,
|
||||
> which broke the [AtTime rules](#attime-reads) every other backend follows (Part 11 §6.5.4). Live (`HistGW.LiveTest.Gate685`:
|
||||
> 100 @ 19:00:00, 107 @ 19:00:02, **NULL** @ 19:00:03, 108 @ 19:00:05) it returned **107 Good at the NULL's own time**, **null + Bad
|
||||
> at 19:00:05** (a stored Good 108 right after the NULL), **99.99999999944 for the stored 100**, and no Interpolated bit on any
|
||||
> interpolated value. It is now answered from **raw** reads with the SQL backends' shared `AtTimeEvaluator` (as #681 did for the
|
||||
> per-sample aggregates), so the same stored samples give the same answer on every backend: the exact stored value (no
|
||||
> Interpolated bit) at a stored time; null + the outage's Bad code at or after a stored NULL; the previous value held,
|
||||
> `UncertainDataSubNormal` + Interpolated, before a Bad neighbour; linear + Interpolated between usable neighbours; `BadNoData`
|
||||
> before the first sample; stepped after the last (`UncertainDataSubNormal` once later than now). Kill switch:
|
||||
> `ServerHistorian:GatewayAtTimeFromRaw = false`.
|
||||
|
||||
- **Read plan (`GatewayAtTimeReader`).** Each requested time needs its stored sample at-or-before (`prev`) and after (`next`; read only
|
||||
when the evaluator uses it — a usable `prev` that is not an exact hit). The distinct times are served by **one** forward window read
|
||||
`[first − 1 ms, last + 1 ms]` of at most 1,024 rows, plus a **walk-back** for the sample before the first time and one small forward
|
||||
**probe** (18 rows) for the sample after the last. Measured on the fake: ten times one second apart on a 1 Hz tag cost **1 read**
|
||||
(all exact hits) or **3** (interpolated); ten times an hour apart on a 1 Hz tag **12** (≈ 9.5k rows, never the 32k between); one
|
||||
time 400 days after the only samples **5**; a tag with no history **4**. A window read that fills its budget resolves the times before
|
||||
its last row; the next time either continues from there (if the rows in between should fit one read) or **jumps** to it with a
|
||||
small read (64 rows) that starts a density-sized margin earlier (so its `prev` is normally in the same read) — 1,000 times
|
||||
spread over ten years of a 1 Hz tag cost ≈ 1,000 reads of ≤ 64 rows (≈ 66k rows; it used to be 1,024-row reads).
|
||||
- **Walk-back** (the gateway reads forward only), in two phases. **Gallop:** segments stepping back from the time, the first sized
|
||||
from the window read's density (else 1 minute), growing ×4 when empty; after two empty segments the whole remainder down to the
|
||||
epoch is asked once — a complete answer ends the walk, a **full** one seeds the bisection with its last row. **Bisect** (once a
|
||||
stored row is known below — a full segment's, or an earlier window read's last row): read `(mid, hi]`; empty ⇒ `hi = mid`; full ⇒
|
||||
the known row moves up to its last row; complete with rows ⇒ done. Right after a full read one segment is aimed at about a quarter
|
||||
of a read at the density it showed (until an aimed segment comes back empty — a gap — then plain halving); below 2 ms the segment
|
||||
is the whole stretch from the known row (never a 100 ns step). Measured (review harness, 5 years of history, time = now − 1 min):
|
||||
a tag stopped 1 h / 1 d / 7 d / 30 d / 365 d ago costs 6–18 reads at 1/min, 21–25 at 1 Hz and 25–31 at 10 Hz; a 1 kHz tag behind
|
||||
a 10-minute gap 6 reads (< 10k rows). (Before the review fix a full segment reset the width and a full whole-remainder read was
|
||||
discarded, so a dense tag that stopped weeks ago gave up after 65 reads.) More than 1,024 stored samples within one millisecond
|
||||
fail the node `BadTooManyOperations` (not counted as a backend failure).
|
||||
- **Bounded.** One request takes at most **64 + 16 × distinct times** raw reads; past it the node fails `BadTooManyOperations`
|
||||
("needed more than N raw reads", naming a small strict-gateway ceiling when one bounded the reads). It is a runaway guard, not a
|
||||
budget (`HistoryReadDeadline` bounds the wall clock): a time inside a gap of a bursty tag legitimately costs a jump read plus a
|
||||
bisection back across the gap, ≈ 6 reads — a 1 h/day 1 Hz tag asked at noon on 30 / 100 / 300 consecutive days takes 186 / 606 /
|
||||
1,806 reads, which the first cut's 64 + 4 per time refused although AVEVA's own AtTime served it. A strict gateway whose ceiling is
|
||||
below 4 rows is refused up front with a message naming the ceiling; a small one (e.g. 10) is served by paging within the cap.
|
||||
- **Wire shape.** Every request bound is a whole millisecond and a row counts only **strictly inside** its request, so AVEVA's synthetic
|
||||
boundary row at the query start (#687) — and any row at the query end — is never taken for a stored sample. A Bad row is value-less
|
||||
(#690). Nothing is asked past now + 1 day (the gateway's AVEVA may refuse a far-future end; a time beyond it is stepped from the last
|
||||
sample). Every stream is drained. A strict gateway's ceiling (#691.3) is learned on its first refusal and honoured; a full read is
|
||||
ordinary paging here, so it is not re-probed.
|
||||
- **Values follow the node's type (type hint).** The gateway wire carries no data type — every historizable type is stored as an
|
||||
AVEVA analog (a Boolean as a 0/1 `Int2`) — so the node manager passes each node's scalar DataType with the AtTime call
|
||||
(`IHistorianDataSource.ReadAtTimeAsync(…, TypeCode? valueType, …)`, a default-interface overload: SqlServer / QuestDB / Null
|
||||
ignore it; Raw and UNS NodeIds give the same hint, arrays and other types none). The Gateway converts each raw row to that type
|
||||
**before** the shared evaluator runs, so its own type rules apply exactly as on the SQL backends: a **Boolean** node
|
||||
(value ≠ 0) is **stepped** — never 0.5; an **integer** node is interpolated, rounded half away from zero and returned as the
|
||||
node's own integer type (`Int32` ⇒ `int`); Float / Double interpolate as Double (as the SQL backends return them). With no
|
||||
hint (an unknown node) every numeric tag interpolates as Double. A **DateTime** node is stepped, but its value stays the
|
||||
gateway's Double: the Gateway never historizes a DateTime node (`HistorianTypeMapper` skips it at provisioning and the recorder drops
|
||||
non-numeric values), so such a node only has history if something else wrote that AVEVA tag, and the numeric encoding is not known
|
||||
here — the SQL backends store DateTime as its own kind and return a `DateTime`. Under an integer hint a NaN, ±∞ or
|
||||
out-of-Int64-range stored value is no integer: it is treated as unusable (as `BadNoData`), so a neighbour like that is never
|
||||
interpolated or saturated into a Good value.
|
||||
- **Client value types (#698, all backends).** Before returning Raw, Modified or AtTime history, the node manager converts non-null
|
||||
scalar values to the node's declared DataType (including Boolean, integer widths and Float). Arrays retain their stored values.
|
||||
Conversion failures produce a null value with `BadTypeMismatch` for that sample, preserving timestamps and modification metadata.
|
||||
DateTime requires an actual DateTime value: the gateway's numeric DateTime encoding is unknown and cannot be inferred.
|
||||
Processed results retain each aggregate's result type, so Count and averages can remain Double on a Boolean or integer node.
|
||||
- **Future timestamps (#696).** A bounded raw probe checks for stored samples between now and the requested future time. An empty
|
||||
future interval starts the prior-sample search near now; a full probe seeds the existing walk-back with a stored future sample.
|
||||
The result still carries the requested timestamp and the evaluator's Uncertain status when no later sample exists.
|
||||
- ⚠️ **Shared-evaluator fix (all backends that use it):** `AtTimeEvaluator` returned every interpolated `Int64` boxed as a
|
||||
**Double** (a C# `cond ? (long)x : double` conditional widens the long); its "keeping the Int64 type" tests passed because
|
||||
Shouldly's `ShouldBe` compares numbers across types. SqlServer / QuestDB interpolated `Int64` AtTime values are now `Int64`,
|
||||
as documented in [AtTime reads](#attime-reads).
|
||||
|
||||
### Bounding values (`ReturnBounds`, #691)
|
||||
|
||||
⚠️ **Client-visible — a breaking change for .NET clients that rely on SDK defaults.** Until #691
|
||||
`ReadRawModifiedDetails.ReturnBounds` was ignored on every backend. It is now honoured on **every** backend (SqlServer, QuestDB,
|
||||
Gateway, and none), following OPC UA Part 11 §4.6 / §6.5.3.2 and checked against every row of Part 11 Table 1. **The OPC Foundation
|
||||
.NET SDK's `new ReadRawModifiedDetails()` defaults BOTH `ReturnBounds = true` AND `IsReadModified = true`** (verified on
|
||||
1.5.378.106), so:
|
||||
|
||||
- a client that clears `IsReadModified` but never sets `ReturnBounds` used to get stored values only and **now gets bounding values**;
|
||||
- a client that sets neither (a bare `new ReadRawModifiedDetails { StartTime, EndTime }`) is a *modified* read with bounds, which
|
||||
used to be served (or `BadHistoryOperationUnsupported` on a backend with no trail) and is **now `Bad_InvalidArgument`** (§6.5.3.3).
|
||||
|
||||
Set both explicitly. `Client.Shared` (Client.CLI / Client.UI) already does, and ScadaBridge does not issue HistoryRead. The kill
|
||||
switch `ServerHistorian:HonorReturnBounds = false` restores the pre-#691 behaviour (both changes) fleet-wide.
|
||||
|
||||
- **What a bound is.** Forward (`StartTime < EndTime`): the start bound is the stored sample **at or immediately before**
|
||||
StartTime, the end bound the one **at or immediately after** EndTime. Reverse mirrors it (start bound at or after StartTime, end
|
||||
bound at or before EndTime), newest first. A sample exactly **on** StartTime is the bound and is already the data's first value
|
||||
(nothing extra is added); a sample exactly on EndTime is added as the end bound (the data window excludes EndTime).
|
||||
`StartTime == EndTime` reads forward: the data is the samples at that instant, the end bound the first sample strictly after it.
|
||||
- **A bound is a stored sample of any quality** — a Bad sample or a comms-loss outage marker comes back as stored. Inside a tie
|
||||
cluster the bound is the cluster's first sample in the read's direction.
|
||||
- **Nothing at all ⇒ `GoodNoData`.** A read with no data in the window **and** neither bound found returns no values and
|
||||
`GoodNoData` — the answer it gave before bounds (§6.5.3.2: bounds make the result a success only "if one or both exist"). This is
|
||||
judged over the whole read (it only arises on a single, final first page); once any bound or value exists, a missing bound is
|
||||
returned as its placeholder.
|
||||
- **Missing bound:** `Bad_BoundNotFound`, null value, stamped with StartTime / EndTime. A one-sided read's open side (only a
|
||||
StartTime, or only an EndTime) is never looked up; it ends the read as `Bad_BoundNotFound` stamped one second after (forward) or
|
||||
before (reverse) the previous returned value (Table 1 footnotes a/b). `Bad_BoundNotSupported` is never returned — every backend
|
||||
can find a bound.
|
||||
- **Counting and paging.** Bounds **count toward `NumValuesPerNode`** (§6.5.3.2: "any Bounding Values returned are included in the
|
||||
numValuesPerNode count"; `NumValuesPerNode = 1` returns the start bound alone). The start bound leads the **first** page and the
|
||||
end bound ends the **last**: the pending end bound rides the continuation point, and when a page is exactly full the end bound is
|
||||
the whole next page. Paging a bounded read at any page size concatenates to the unpaged answer. At `NumValuesPerNode = 1`
|
||||
with no start bound found, the server first checks the read holds anything (else `GoodNoData`); since #692 what that check
|
||||
learns rides the continuation point — the walk-back hint and, when there is no data, the end bound it found — so the next
|
||||
page reads nothing it already read.
|
||||
- **Placeholders are marked, not recognised by shape (#692).** Only a `Bad_BoundNotFound` value the server synthesised is a
|
||||
placeholder; a stored row that carries `Bad_BoundNotFound` with a null value (writable through HistoryUpdate) is data, and
|
||||
never turns a read into the "no data and no bound" `GoodNoData` answer.
|
||||
- **How it is served — one authority, `HistoryBounds` in the OPC UA server.** "At or before T" is a newest-first read of `(0, T]`
|
||||
capped at one — native on SqlServer/QuestDB, the #686 walk-back of bounded forward reads on the Gateway (a tag with no sample
|
||||
before StartTime is proven empty down to 1970 by the #689 walk — after two empty segments it asks once for the whole remainder —
|
||||
so a missing start bound costs a few small reads, not one per ×4 step). "At or after T" is a forward read
|
||||
of `[T, MaxValue)` capped at one — on the **Gateway** it ends at **now + 1 day** instead (the walk-back's horizon; the
|
||||
HistorianGateway only checks `start ≤ end`, so a year-9999 end would reach AVEVA), and an end bound beyond that horizon is
|
||||
`Bad_BoundNotFound` without a read. SqlServer/QuestDB keep `MaxValue` (they bind it safely and may hold samples further ahead).
|
||||
"Strictly after" starts one storage resolution later. On QuestDB, whose window bounds truncate to
|
||||
the microsecond, "on the bound" compares at that resolution, so a sample at ⌊StartTime⌋ is never returned twice. A first page
|
||||
therefore costs up to three backend reads (start bound, data, end bound). The `BoundingValueConformance` suite pins those exact
|
||||
lookup reads on SqlServer and QuestDB.
|
||||
⚠️ On the **Gateway** the start-bound walk-back is subject to the #689 per-millisecond tie limit: if the millisecond holding the
|
||||
bound has more than `MaxTieClusterOverfetch` samples, the lookup cannot pick the cluster's last sample without reading it all, so
|
||||
the **whole node** read fails `BadTooManyOperations` (even though the data read alone would succeed) rather than return a wrong bound.
|
||||
- **Modified reads:** `IsReadModified = true` with `ReturnBounds = true` is `Bad_InvalidArgument` per node (Part 11 §6.5.3.3).
|
||||
- **Kill switch:** `ServerHistorian:HonorReturnBounds` (default `true`). `false` ignores `ReturnBounds` exactly as before #691 —
|
||||
stored samples only, and no modified-read rejection.
|
||||
- **Not affected:** Processed, AtTime and Events reads (they have no `ReturnBounds`).
|
||||
|
||||
### Continuation-point paging (Events) (#669)
|
||||
|
||||
@@ -723,30 +983,55 @@ says is complete carries no continuation point. `NumValuesPerNode == 0` stays un
|
||||
that many more rows than the cap, so every page short of the last is a FULL `NumValuesPerNode` page (#675; resumed pages used
|
||||
to come back short by the skip).
|
||||
- A tie cluster larger than the cap is paged within by over-fetching the half-open window `[T, T + resolution)` up to
|
||||
`MaxTieClusterOverfetch`; beyond that bound the node answers `BadHistoryOperationUnsupported` (raise the bound).
|
||||
`MaxTieClusterOverfetch`; beyond that bound the node answers `BadTooManyOperations` (raise the bound).
|
||||
- **Tie exactness depends on the backend.** SqlServer and QuestDB order by `(EventTimeUtc, EventId)`, so the skip count is exact.
|
||||
The Gateway backend has no stable intra-timestamp order (and only a truncation flag), so ties at a page boundary are
|
||||
best-effort there.
|
||||
The Gateway backend has no stable intra-timestamp order (and only a truncation flag), so ties at a page boundary of a
|
||||
**forward** read are best-effort there; a walked-back **reverse** read (below) sorts them by `EventId` itself and is exact.
|
||||
- **Reverse event reads page newest-first (#675).** `StartTime > EndTime` reads the Part 11 reverse window `(EndTime, StartTime]`
|
||||
with events in DESCENDING `(EventTimeUtc, EventId)` order, so a cap keeps the **latest** N and a continuation point is issued
|
||||
when the source flags more. The direction is stored **in the cursor**: a reverse point resumes reverse (and a forward one
|
||||
forward) whatever the resume request's details say. Reverse tie clusters are over-fetched forward and sliced from the newest
|
||||
end, exactly as Raw does. Before #675 a capped reverse read silently returned the EARLIEST N, ascending, Good, with no
|
||||
continuation point.
|
||||
- **End-only events reads: backward on SqlServer/QuestDB, forward on Gateway (#675).** `StartTime` unset
|
||||
(`DateTime.MinValue`) with an `EndTime` and `NumValuesPerNode > 0` pages backward from `EndTime` over `(MinValue, EndTime]` —
|
||||
the same rule as Raw — **only on a backend that reports `SupportsReverseEvents`** (SqlServer, QuestDB). On Gateway (the
|
||||
default backend) and Null it keeps its pre-#675 forward `[MinValue, EndTime)` meaning, paged oldest-first, rather than
|
||||
becoming a refused reverse read. Start-only reads page forward to an open upper bound (`DateTime.MaxValue`, UTC) everywhere.
|
||||
Without a cap an end-only read is forward `[MinValue, EndTime)` everywhere.
|
||||
- **Backends.** The server checks `IHistorianDataSource.SupportsReverseEvents` and then calls
|
||||
`ReadEventsAsync(…, bool reverse, …)`. SqlServer and QuestDB serve reverse natively (`… > lo AND … <= hi ORDER BY time DESC,
|
||||
id DESC`) and report `true`. **Gateway and Null report `false`**: an explicit reverse read (`StartTime > EndTime`) is answered
|
||||
`BadHistoryOperationUnsupported` **without calling the backend** — the gateway streams ascending only and rejects `end < start`,
|
||||
and draining the whole window to keep its tail would be exactly the unbounded read the cap prevents. Because the refusal is a
|
||||
capability check, not a caught exception, a `NotSupportedException` thrown from inside a backend (a driver library) is logged
|
||||
as a backend fault like any other. Every implementation (and any future wrapper) must declare both members itself — the
|
||||
interface defaults (`false` / throw) would silently disable reverse reads; `HistorianDataSourceReverseEventsGuardTests` pins it.
|
||||
- **End-only events reads page backward on every backend (#675, #691.4).** `StartTime` unset (`DateTime.MinValue`) with an
|
||||
`EndTime` and `NumValuesPerNode > 0` pages backward from `EndTime` over `(MinValue, EndTime]` — the same rule as Raw.
|
||||
Start-only reads page forward to an open upper bound (`DateTime.MaxValue`, UTC) everywhere. Without a cap an end-only read
|
||||
is forward `[MinValue, EndTime)` everywhere.
|
||||
- **Backends.** The server checks `IHistorianDataSource.SupportsReverseEvents`. SqlServer and QuestDB serve reverse natively
|
||||
(`… > lo AND … <= hi ORDER BY time DESC, id DESC`, the `ReadEventsAsync(…, bool reverse, …)` overload) and report `true`.
|
||||
**Gateway reports `false`** — the gateway streams ascending only and rejects `end < start` — and is never sent a reverse
|
||||
read (Null reports `true` and answers empty either way, as for raw): since **#691.4** the server walks it back from bounded **forward** reads with the raw walk (#686/#689/#691:
|
||||
page-sized ms-aligned segments filtered to `(segLo, segHi]`, a read the source flagged truncated is never answered from,
|
||||
the now + 1 day / 1970 bounds, the remainder probe and the proven-empty stretch carried in the cursor). Event specifics:
|
||||
- **Ties** are sorted by `EventId` (ordinal) on every read — the gateway orders by `EventTimeUtc` only, in any order within a
|
||||
timestamp — so the page is DESCENDING `(EventTimeUtc, EventId)` and a resumed page's boundary skip lines up; the
|
||||
oversized-cluster over-fetch keeps exactly the events at `T` (the gateway's end is inclusive) in the same order.
|
||||
- **"More exists"** is found by walking for one event past the page (the walk has no source truncation flag to hand on).
|
||||
- **Uncapped** reverse reads are one forward read of the window, sorted and reversed.
|
||||
- ⚠️ **Also fixed (#691.4 review): uncapped events reads on Gateway.** `NumValuesPerNode = 0` reaches the source as
|
||||
`int.MaxValue`, and `GatewayHistorianDataSource` pre-sized its result list to it ("Array dimensions exceeded") — every
|
||||
uncapped Gateway events read, forward too, failed the node and counted as a backend failure. The capacity is now bounded.
|
||||
- A timestamp holding more than `MaxTieClusterOverfetch` events ⇒ **`BadTooManyOperations`** for that node.
|
||||
- ⚠️ **Gateway prerequisite:** the gateway's `RuntimeDb:EventReadMaxRows` must be `0` (its default, unlimited) or larger than
|
||||
the walk's biggest read (`2N + 18` rows, a discovery read 1,026, a tie-cluster read up to `2·MaxTieClusterOverfetch + 4`).
|
||||
That cap truncates **silently** (the stream just ends), so a segment it cut would look complete and drop its newest events —
|
||||
the same blindness a forward read has to it, but on the end a reverse read is asking for.
|
||||
- ⚠️ **Cost of "nothing older" (#694 item 2).** A walked read that runs out of events proves the rest of its window empty with
|
||||
one forward read down to its earlier bound — for an end-only read, the Unix epoch. On `Runtime.dbo.Events` that read is
|
||||
measured at **~3.8 s** for a source with few events (vs ~0.7 s for a bounded window), and an **unfiltered** deep read hit the
|
||||
gateway's 30 s SQL timeout; a forward read from a far-past `StartTime` scans the same range. The cost is the gateway's query
|
||||
over `Runtime.dbo.Events` (its index use is a historiangw matter, not fixable here). Two ways to keep it down: give the read an
|
||||
explicit `EndTime`/`StartTime` that bounds the window, or set the opt-in **`ServerHistorian:EventReadHorizon`** (below). The
|
||||
default stays exact — there is no cache of "earliest event per source", deliberately: events reach `Runtime.dbo.Events` from
|
||||
the alarm store-and-forward drain (which can deliver hours- or days-old events after an outage) and from AVEVA itself, so a
|
||||
remembered "nothing before X" can go stale silently.
|
||||
|
||||
⚠️ **Client-visible (#691.4).** On Gateway (the default backend) and Null an explicit reverse events read used to be refused
|
||||
`BadHistoryOperationUnsupported` (#675), and an end-only capped events read returned the **oldest** N events ascending. Both
|
||||
now return the latest events newest-first. Because the capability is checked, not a caught exception, a
|
||||
`NotSupportedException` thrown from inside a backend (a driver library) is still logged as a backend fault. Every
|
||||
implementation (and any future wrapper) must declare both members itself — the interface defaults (`false` / throw) would
|
||||
silently send a native-reverse backend through the walk; `HistorianDataSourceReverseEventsGuardTests` pins it.
|
||||
- **Gateway open upper bound.** The `DateTime.MaxValue` (UTC) bound of a start-only forward read is pinned offline through the
|
||||
published client's proto `Timestamp` conversion (captured by an in-process gRPC server) and the FILETIME encoding
|
||||
(`GatewayEventBoundsTests`). What a live AVEVA Historian does with a year-9999 bound is still unverified (needs the VPN).
|
||||
@@ -865,8 +1150,30 @@ fixed:
|
||||
`HistorianTagDefinition.IntegralDivisor` is `0`, so `GatewayTagProvisioner` now sends **`IntegralDivisor = 1`** explicitly on every
|
||||
`EnsureTags` — the value·seconds unit the SQL backends' `Total` uses. A tag provisioned earlier keeps its divisor until its next
|
||||
`EnsureTags` (an upsert). The live magnitude check against AVEVA (a constant 1 integrated over 1 s reads 1) passed on 2026-10-12 at the AVEVA
|
||||
level; end to end it waits on historiangw#27 (above). Reverse processed reads of the AVEVA-native `TimeAverage`/`Total` are not re-anchored by #677 (the gateway refuses `end < start`);
|
||||
the raw-evaluated per-sample aggregates (#681) do follow #677. Since #681 every native bucket is stamped with its interval **start**
|
||||
level; end to end it waits on historiangw#27 (above). ⚠️ **Native reads on the SQL backends' interval grid (#691.2, client-visible).** Both directions now use `BucketPlan`, served by
|
||||
**forward** native requests only (`NativeAggregatePlan` — AVEVA anchors a native read's interval grid at the request start, and the
|
||||
gateway refuses `start > end` and `interval <= 0`):
|
||||
- **Reverse** (`StartTime > EndTime`) used to reach the gateway as `start > end` and fail the node (`InvalidArgument`, counted as a backend
|
||||
failure). It now follows #677 like the SQL backends and the raw-evaluated per-sample aggregates (#681): window `(EndTime, StartTime]`,
|
||||
intervals stepping back from `StartTime`, the clipped interval the **earliest**, each stamped with its **later** edge, newest first —
|
||||
one read `[O + I·f, StartTime)` with the interval `I` for the whole intervals (`O = StartTime − n·I`, `f` = 1 when clipped) and a
|
||||
second read over exactly the clipped span `[EndTime, O + I)` with its own width as the interval. A time-weighted value over `[a, b)` and
|
||||
over `(a, b]` is the same integral (two instants apart), so each forward row is the reverse interval's value.
|
||||
- **Forward** windows that are not a whole number of intervals used to send one request whose last AVEVA interval ran **past EndTime**;
|
||||
now the whole intervals `[StartTime, StartTime + (n−1)·I)` are one read and the clipped **latest** interval `[StartTime + (n−1)·I,
|
||||
EndTime)` is its own read over exactly that span. An interval wider than the window is sent as the window, and `interval = 0` as one
|
||||
interval over the window (it used to reach the gateway as 0 and be refused). An empty window (`StartTime == EndTime`) sends nothing.
|
||||
- Stamps match SqlServer/QuestDB (forward: earlier edge, ascending; reverse: later edge, descending), and the clipped interval carries
|
||||
the **Partial** bit unless Bad (Part 13 §5.3.3.2 Case 3, as the SQL backends' time-weighted aggregates — which also mark the intervals
|
||||
the tag's data starts/ends inside; AVEVA's rows carry no data range, so that is not reproduced). Rows are otherwise mapped as before
|
||||
(#685). An interval AVEVA answers no row for is absent (the SQL backends answer it `BadNoData`).
|
||||
- ⚠️ **Interval cap (client-visible):** forward native reads now go through `BucketPlan` too, so a window of more than 100,000
|
||||
intervals (e.g. 1 s over 2 days) is refused up front with `BadHistoryOperationUnsupported`, as on the SQL backends — it used to be
|
||||
served up to the gateway's own `MaxAggregateIntervals` (1,000,000). Not counted as a backend failure. A clipped interval shorter than
|
||||
1 ms (a sub-ms window end) is sent as a sub-ms interval request; re-check it in the post-historiangw#27 live leg.
|
||||
- ⚠️ **Until historiangw#27 is fixed, native values are wrong in BOTH directions with a Good status:** AVEVA answers one row per request
|
||||
(the whole span's value), which lands on that request's first interval as a Good, Calculated value while the other intervals are
|
||||
absent. Pinned against a native-mode fake only — re-run the forward and reverse native legs live once the gateway is fixed. Since #681 every native bucket is stamped with its interval **start**
|
||||
(Part 13; it was the interval end) and carries the Calculated info bits (`Good` ⇒ `0x00000401`, no data ⇒ `BadNoData | 0x401`).
|
||||
The live AVEVA gate (2026-10-12) confirmed `start_time` is the interval start (and the `value·s` unit). Previously noted as
|
||||
pending: it reads the aggregate row's `start_time`, which histsdk decodes from a row
|
||||
@@ -912,9 +1219,11 @@ and evicts its shared historian session (see the HistorianGateway issue text in
|
||||
streaming gateway and keeps its `+ 18`. Past the cap the rest of the (bounded) stream is drained, then the node fails
|
||||
`BadTooManyOperations`; a stream that delivers the whole budget cannot be proven complete and fails the same way (so with the
|
||||
default cap a window of more than ~999,982 rows fails, as it would one row later). The gateway's own refusal of the budget
|
||||
(`InvalidArgument` "max_values exceeds the server ceiling" — `Historian:ReadLimits:MaxValuesCeiling` lowered below the default,
|
||||
with `AllowStreamingBeyondCeiling=false`) is also `BadTooManyOperations`, not a backend fault — lower `MaxTimeWeightedRawRows` to
|
||||
`ceiling − 18` on such a gateway. A raw read's budget is **not** clamped (see [#687](#gateway-raw-reads-return-stored-samples-only-687)). A caller cancellation (client cancel, `HistoryReadDeadline`) is not counted as a backend fault.
|
||||
(`InvalidArgument` "max_values exceeds the server ceiling of N" — `Historian:ReadLimits:MaxValuesCeiling` lowered below the
|
||||
default, or an unbounded `MaxTimeWeightedRawRows ≤ 0`, with `AllowStreamingBeyondCeiling=false`) is **retried once with N as the
|
||||
budget** (#691.3): a window that fits is served, one that delivers all N rows fails `BadTooManyOperations` as above (not a backend
|
||||
fault). Lowering `MaxTimeWeightedRawRows` to `ceiling − 18` on such a gateway is no longer needed — it only saves the refused
|
||||
round-trip. A raw read's budget is **not** clamped (see [#687](#gateway-raw-reads-return-stored-samples-only-687)). A caller cancellation (client cancel, `HistoryReadDeadline`) is not counted as a backend fault.
|
||||
- **Bucket statistics** are `RawBucketAccumulator`, the C# statement of the SQL backends' per-sample columns — incl. #684:
|
||||
Average / Minimum / Maximum / Range / *ActualTime over **Good** samples only, the Uncertain extremes feeding only the Min/Max
|
||||
status rule, and an Uncertain String/DateTime sample not refusing the Good-value aggregates. A change to those SQL columns
|
||||
@@ -1009,8 +1318,8 @@ whatever the SDK sets — #682 touches only the i=11201 folder (follow-up candid
|
||||
sample per requested timestamp, so the single-shot backend returns the complete result in one
|
||||
read and there is no "full page ⇒ maybe more" signal to page on. Returning the full result with
|
||||
no continuation point is spec-conformant.
|
||||
- **`ReturnBounds` is ignored** (every backend): Raw reads return stored values only; no bounding values are added at the window
|
||||
edges. The node manager does not pass the flag to `IHistorianDataSource`.
|
||||
- **Bounds are raw only:** `ReturnBounds` (#691, [Bounding values](#bounding-values-returnbounds-691)) never interpolates a bound —
|
||||
it is always a stored sample or `Bad_BoundNotFound`.
|
||||
- **No modified-value history on the `Gateway` backend.** Modified reads (`IsReadModified = true`) are served by
|
||||
`SqlServer` and `QuestDb` ([Modified-value history](#modified-value-history-part-11), #668); the HistorianGateway exposes no
|
||||
modified/edited-history surface, so on `Gateway` (and with no historian) they return `BadHistoryOperationUnsupported`.
|
||||
@@ -1479,7 +1788,7 @@ this family and **are** capped (they used to be pushed down). Raise the cap, or
|
||||
### Health
|
||||
|
||||
The read client reports `GetHealthSnapshot()` (query/failure counters, consecutive failures, `LastError` — e.g. a missing schema under
|
||||
`AutoCreateSchema=false`). A SQL error on a read surfaces as `BadHistoryOperationUnsupported` for **that node only**; a cancelled read
|
||||
`AutoCreateSchema=false`). A SQL connection outage on a read surfaces as `BadNoCommunication`; other SQL errors retain `BadHistoryOperationUnsupported` for **that node only**; a cancelled read
|
||||
(deadline, client abort) is not counted as a backend fault.
|
||||
|
||||
### Retention (`RetentionDays`)
|
||||
|
||||
@@ -325,8 +325,10 @@ routes through a separate `NativeAlarmAckRouter` seam (instead of the scripted
|
||||
FullName)` inverse map, Primary-gated) → the owning driver's
|
||||
`IAlarmSource.AcknowledgeAsync` → Galaxy gateway → AVEVA. The call carries the
|
||||
authenticated operator's display name via the `AlarmAcknowledgeRequest.OperatorUser`
|
||||
field. This is fire-and-forget — a failed upstream ack is not surfaced back to the
|
||||
OPC UA client (mirrors the Galaxy write-outcome limitation). See [AlarmTracking.md
|
||||
field. The actor route back to OPC UA is fire-and-forget — a failed upstream ack
|
||||
is logged rather than surfaced back to the OPC UA client. The Galaxy adapter
|
||||
awaits its correlated RPC; this does not establish delivery to AVEVA event history.
|
||||
Galaxy value writes use a separate correlated write-outcome path. See [AlarmTracking.md
|
||||
§Native alarm acknowledge → AVEVA](AlarmTracking.md#native-alarm-acknowledge--aveva)
|
||||
for the full routing diagram.
|
||||
|
||||
|
||||
+45
-7
@@ -198,15 +198,53 @@ Timers are **read-only** this phase; a write attempt returns `BadNotWritable`.
|
||||
### Counter read
|
||||
|
||||
**Address**: `C{n}` (e.g. `C0`, `C10`). **DataType must be `Int32`**. The OPC UA node is
|
||||
a scalar `Int32` whose value is the current **count** (raw word from the PLC).
|
||||
a scalar `Int32` containing the unsigned **raw big-endian word**, widened to `Int32`:
|
||||
`(bytes[0] << 8) | bytes[1]`. `S7.Net`'s `Counter.FromByteArray` performs no BCD
|
||||
reinterpretation; for example, bytes `01 23` surface as `291`. This describes the
|
||||
current decoder, not verified agreement with a PLC's displayed count.
|
||||
Counters are **read-only** this phase; a write attempt returns `BadNotWritable`.
|
||||
|
||||
> **Known limitation — Counter BCD encoding on S7-300/400**: `S7.Net`'s
|
||||
> `Counter.FromByteArray` returns the raw big-endian word without BCD decode.
|
||||
> On **classic S7-300/400** the C-area word is BCD-encoded (0–999), so the
|
||||
> surfaced value can differ from the actual count on that hardware. S7-1200/1500
|
||||
> use IEC/DB counters (plain integers) where the raw word is correct. BCD
|
||||
> reinterpretation for legacy C-area is a live-hardware-gated follow-up.
|
||||
The S7comm ReadVar **C-area response encoding remains unverified on hardware**.
|
||||
PLC instruction representations do not by themselves establish network encoding:
|
||||
Siemens documents a binary current value for [L: Load counter](https://docs.tia.siemens.cloud/r/en-us/v21/stl-s7-300-s7-400/stl-mnemonic-s7-300-s7-400/counter-operations-s7-300-s7-400/l-load-counter-s7-300-s7-400)
|
||||
and separate `CV` and `CV_BCD` outputs for [S_CD](https://docs.tia.siemens.cloud/r/en-us/v21/lad-s7-300-s7-400/counter-operations-s7-300-s7-400/simatic-counters-s7-300-s7-400/s_cd-assign-parameters-and-count-down-s7-300-s7-400?contentId=reR_xapFX_Ike2opHIvfzg).
|
||||
|
||||
IEC counter values stored in DB memory use the **DB address and declared integer
|
||||
type**, a different read path from `C{n}`. CPU family alone does not select the
|
||||
counter kind or prove its wire representation. In particular, Siemens also lists
|
||||
legacy SIMATIC counters for S7-1500 in its [instruction comparison (pages 23–24)](https://cache.industry.siemens.com/dl/files/777/109986777/att_1322640/v1/s7_1500_compare_table_en.pdf?download=true).
|
||||
The driver has no counter encoding selector; hardware-faithful counter support
|
||||
requires independent raw-response evidence before any interpretation change.
|
||||
|
||||
### Counter evidence follow-up (#616)
|
||||
|
||||
The existing `S7_1500SpanReadTests` exercises the python-snap7 profile's synthetic
|
||||
raw words (`C0 = 4711`, `C7 = 31337`). These are outside the classic SIMATIC count
|
||||
range `0..999` documented by S_CD. These expectations describe raw-word handling;
|
||||
they do not establish classic counter count semantics. The recorded device-0 run cannot prove nonzero
|
||||
numbering. The nonzero test is environment-gated, but its options remain
|
||||
`CpuType = S71500`, rack `0`, slot `0`: setting `S7_TIMER_NUMBER_ADDRESSING=1`
|
||||
does not configure a classic CPU or make the profile a classic counter gate.
|
||||
|
||||
Before claiming hardware validation or closing #616:
|
||||
|
||||
- Prepare a separate target-specific semantic gate with the actual CPU type,
|
||||
rack, slot and TSAP/connection settings. Keep the synthetic raw-word profile
|
||||
as its own evidence of the existing decoder contract.
|
||||
- Record CPU model/order number, firmware, endpoint, connection settings and
|
||||
driver/S7netplus plus simulator versions where applicable.
|
||||
- Observe independent PLC-side current counts and binary/BCD reference outputs
|
||||
for valid classic counts such as `123`, a second distinguishing count (`42`
|
||||
or `999`), and zero. Record the instruction/counter kind and DB reference
|
||||
addresses; do not derive reference values from the driver's output.
|
||||
- Capture the actual two-byte S7comm C-area responses alongside those references.
|
||||
Do not infer response bytes from a simulator seed or expected BCD value.
|
||||
- Verify several distinct nonzero timer/counter numbers against their independent
|
||||
PLC values and captured requests/responses, and verify the read-only contract.
|
||||
|
||||
The raw-byte/count and nonzero-numbering gates remain open. Password/protection
|
||||
support inherited from #303/#632 is a separate follow-up, not evidence supplied
|
||||
by these counter checks.
|
||||
|
||||
### Deferrals (not yet implemented)
|
||||
|
||||
|
||||
@@ -120,7 +120,9 @@ OPC UA client Acknowledge on a NATIVE condition
|
||||
under Lock, like `AlarmCommandRouter`).
|
||||
- **Driver faulted / reconnecting** → `AcknowledgeAsync` fails or no-ops per the driver; the local condition
|
||||
state still reflects the SDK-applied ack (the OPC UA client sees Good). Surfacing a *failed* upstream ack
|
||||
back to the client is out of scope (mirrors the Galaxy write-outcome fire-and-forget limitation).
|
||||
back to the client is out of scope for this native-ack actor route. **Current clarification
|
||||
(2026-10-13):** the Galaxy adapter awaits a correlated ack RPC, but the actor route does not
|
||||
return its outcome to OPC UA; Galaxy value writes have a separate correlated write-outcome path.
|
||||
- **H4 native Enable/Disable** → `BadNotSupported` (decision #2).
|
||||
- **H2-bit:** no runtime behavior change for existing grants except closing the HistoryRead⇒HistoryUpdate
|
||||
authorization hole; the service itself stays rejected.
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
# Alarm delivery identity results — lmxopcua #705
|
||||
|
||||
Alarm events now carry a nonzero delivery GUID created once for direct delivery and preserved across retries and record copies. It is excluded from queue JSON and business-field equality/hash. Queued delivery overrides it with a frozen versioned UUID encoding derived only from the persisted queue row key, preserving existing payload JSON, fingerprints, schema, deduplication and AlarmId. The Gateway mapper sends that GUID and rejects explicitly empty identities. SQL Server and QuestDB identity logic is unchanged.
|
||||
|
||||
Coverage includes literal historical JSON/fingerprint/GUID vectors; ten-field equality and hash behavior; direct copies/retries; queue retry/reopen/payload edits/dead-letter revival; identical-input convergence; distinct same-time business events; real replicated queue convergence; and legacy migration. Reconstructed direct objects receive new identities; durable reconstruction uses the queue. Completely identical business payloads remain intentionally deduplicated.
|
||||
|
||||
TDD: direct wire regression failed on an empty ID; queued retry/reopen regression failed on changing IDs. Final affected runs: Core51/51, Host migration/convergence16/16, Gateway453passed/6live skipped/0failed. Full offline solution:14,635passed/611skipped/0failed across62suites, exit0. Independent final review and diffcheck clear. Fixture configuration variables were removed before all runs.
|
||||
|
||||
No live event writes, deployed sender rollout or historical row rewrite occurred. The live round-trip test now requires exact delivery GUID readback; actual persistence remains the open Gateway #28 gate, alongside its separate #39 timestamp-offset provider failure. Upstream History controls #29 and lmxopcua #694/#700 remain open.
|
||||
@@ -0,0 +1,9 @@
|
||||
# Gateway tag-presence concurrency coverage — 2026-10-13
|
||||
|
||||
Approved tests-only fix for #701. Added 15 deterministic cases for overlapping positive/negative cold lookups, expiry refreshes, capacity crossing, successful-result isolation from faults/cancellation, and cancellation after asynchronous completion. Fake presence calls support an async hook and atomic counter. Production cache is unchanged; no single-flight promise.
|
||||
|
||||
The synthetic 16-caller cold/expiry bursts produced 16 catalog RPCs. Assertions permit one through16; this records amplification without making it a requirement or a live-cost claim. Newclass15/15 and Gateway449/449 passed. Independent review found no actionable defects; diff check clean.
|
||||
|
||||
First full solution run:14,614passed/609skipped/1failed across62suites. Existing restart/reconnect wire test failed, then isolated retry passed1/1. Full repeat with detailed console logging passed14,615/609skip/0fail across62suites, exit0, including OPC UA integration122/122 and host integration268passed/12skip. #702 tracks the unresolved intermittent failure; it is not closed merely because the repeat passed.
|
||||
|
||||
Both full runs excluded configured live fixture variables from the environment; environment-gated tests skipped. Existing NU1510/xUnit duplicate-InlineData warnings remain pre-existing. No production behavior or deployment changed by this patch.
|
||||
@@ -0,0 +1,36 @@
|
||||
# Missing-tag and Gateway performance fixes — 2026-10-13
|
||||
|
||||
Follow-up to pushed `97668d26` / PR #699. Implements #653 and the remaining #696 items 2/3 under the approved bounded design.
|
||||
|
||||
## Changes
|
||||
|
||||
- **#653:** Raw, Processed and AtTime check tag presence before history RPCs. The bounded 4,096-name cache uses a 5-second negative and 60-second positive lifetime. Only explicit absence or NotFound is cached as missing; failures and cancellation propagate. Raw is empty; AtTime and Processed retain requested/bucket timestamps with null/BadNoData. Unsupported aggregate validation runs first.
|
||||
- **Live compatibility adjustment:** the deployed gateway misparses missing `GetTagMetadata` replies as `Internal` (`InvalidDataException: GetTagInfo response ended unexpectedly`). The adapter therefore checks the filtered tag catalog and compares exact names, fully draining the stream before declaring presence/absence. It never turns the parser error into a missing tag. Wildcard matches alone do not prove presence, and a trailing stream fault cannot enter the cache.
|
||||
- **#696 item 2:** a truncated remainder probe seeds strict-ceiling reverse discovery with its observed rows/density; gap searches use the available discovery budget. Complete segments alone contribute to the newest-first page. Regression: a 7,200-row stopped 1 Hz tag, ceiling 1,000, requested latest three one day later, previously 17 raw calls; now meets a maximum of 12 with exactly 7199/7198/7197 returned. Ordinary page sizing and event discovery behavior are unchanged.
|
||||
- **#696 item 3:** per-sample aggregates reuse a ceiling refusal for 30 seconds instead of immediately re-probing on every full stream. The next incomplete aggregate read after expiry notices a raised ceiling; a lowered ceiling is re-learned immediately. Raw and AtTime re-probes remain immediate. During the cooldown an oversized aggregate remains BadTooManyOperations, never a truncated result.
|
||||
|
||||
## Verification evidence
|
||||
|
||||
New regressions failed before fixes: all initial 12 cases failed (missing preflight/caching, 17 discovery reads, duplicate aggregate refusal). The four catalog wire cases failed with the metadata adapter and pass with the catalog adapter. Final Gateway suite: 434 passed, six environment-gated skips.
|
||||
|
||||
Independent read-only review found no important defects. Concurrent cold requests can duplicate a metadata/catalog check; cache access and capacity mutations are locked. No single-flight guarantee was introduced. Catalog RPC contract streams all matches; the upstream SDK pages the native tag-query results, with no configured browse result ceiling.
|
||||
|
||||
## Live #653 confirmation
|
||||
|
||||
Measured the updated GatewayHistorianDataSource through the real adapter and local gateway against the existing remote AVEVA connection. Temporary historian:read key had a two-hour expiry and was revoked after the run. No historian writes/provisioning and no gateway deployment changes.
|
||||
|
||||
Nonexistent tag: `OtOpcUa.Issue653.Nonexistent.20261013`.
|
||||
|
||||
| Operation, five consecutive runs | Elapsed milliseconds | Result |
|
||||
|---|---|---|
|
||||
| Raw | 228, 0, 0, 0, 0 | empty |
|
||||
| Count processed | 2, 0, 0, 0, 0 | six null/BadNoData buckets |
|
||||
| AtTime | 1, 0, 0, 0, 0 | null/BadNoData at requested time |
|
||||
|
||||
These used one data source/cache: the first Raw call is cold, subsequent calls are cached. The prior direct-client measurement took ~30.6 seconds for missing-tag Raw/Aggregate and ~61.3 seconds for AtTime. Direct client RPCs are not modified; the fast-fail protection is in OtOpcUa.
|
||||
|
||||
**Control limitation:** SysTimeSec catalog presence succeeded, but its Raw RPC twice timed out upstream with DeadlineExceeded at ~30 seconds. The data source propagated IOException/communication failure rather than returning no data. A fresh control using the original package client (no data-source presence checks) also timed out with DeadlineExceeded after 30,656 ms, confirming this timeout is independent of the new checks. The earlier direct-client control had succeeded. This run therefore verifies missing-tag fast-fail and failure propagation, not healthy-tag live history behavior. No end-to-end deployed OPC UA claim is made.
|
||||
|
||||
#694 remains open for native aggregate rerun after historiangw#27. No sibling gateway code changed.
|
||||
|
||||
Full final-tree solution: 14,600 passed, 609 skipped, 0 failed across 62 suites. `dotnet test ZB.MOM.WW.OtOpcUa.slnx --verbosity quiet`, exit 0. Includes 122 passing OPC UA integration tests and 268 passing host integration tests (12 host skips). Environment-gated suites are included in the skip count.
|
||||
@@ -0,0 +1,28 @@
|
||||
# Historian follow-up results — 2026-10-13
|
||||
|
||||
Scope: approved bounded fixes for #695–#698, plus live measurement of #653. Base: `66a2b0c3`.
|
||||
|
||||
- #695: spend the remaining learned-ceiling room on crowding reads; reduce crowded reverse-read windows using the observed pre-window count. Regression cases cover ceilings 1,709 and 1,369, and reverse paging through a crowded millisecond.
|
||||
- #696 item 1: probe the future interval before starting the prior-sample search near now. Preserve stored future samples, requested timestamps and Uncertain status. Dense future probes still use the existing bounded walk. Items 2 and 3 remain open (stopped-tag strict-ceiling cost and aggregate ceiling re-probes).
|
||||
- #697: actual Gateway, SQL Server and QuestDB transport failures become `BadNoCommunication` on every HistoryRead arm. Cancellation remains `BadTimeout`; unsupported operations retain `BadHistoryOperationUnsupported`. Tests include real refused database connections, repeated reads and nested gRPC failures.
|
||||
- #698: normalize Raw, Modified and AtTime scalar values to the node DataType. Conversion failure gives per-sample null/`BadTypeMismatch`; arrays, audit metadata and timestamps survive. Processed results retain aggregate-specific types.
|
||||
- #694 item 3 remains blocked on historiangw#27. No upstream code was changed.
|
||||
|
||||
## Live #653 measurement
|
||||
|
||||
Five sequential calls per operation and tag, using client 0.3.0 against the running local gateway (`localhost:5221`) with its existing remote AVEVA connection. Commissioned a two-hour `historian:read` key, then revoked it after measurement. No historian writes or provisioning performed.
|
||||
|
||||
| Operation | SysTimeSec, elapsed ms | Nonexistent tag, elapsed ms |
|
||||
|---|---|---|
|
||||
| Metadata | 299, 85, 81, 82, 48 (success) | 89, 53, 41, 77, 76 (gateway exception) |
|
||||
| Raw | 143, 99, 145, 133, 159 (success) | 30260, 30634, 30675, 30754, 30628 (gateway exception) |
|
||||
| Aggregate | 271, 275, 309, 249, 304 (success) | 30564, 30605, 30583, 30575, 30806 (gateway exception) |
|
||||
| AtTime | 140, 88, 89, 127, 92 (success) | 61074, 61330, 61601, 61249, 61271 (gateway exception) |
|
||||
|
||||
Nonexistent tag: `OtOpcUa.Issue653.Nonexistent.20261013`. The delay is still present. Recommend a metadata pre-check with a short negative cache, caching only explicit missing-tag outcomes and never authentication, communication or arbitrary errors. This is follow-up implementation; #653 remains open. Direct gateway AtTime above differs from OtOpcUa's raw-based AtTime evaluator.
|
||||
|
||||
## Consumer and validation notes
|
||||
|
||||
ScadaBridge source and tests contain no HistoryRead consumer; no consumer change required. Independent review found a future-stored-sample edge case, which was fixed and covered before final verification. No deployment or live end-to-end OPC UA verification of these new fixes is claimed; the live run measures #653 through the gateway client.
|
||||
|
||||
Full solution: 14,581 passed, 609 skipped, 0 failed across 62 test suites. Command: `dotnet test ZB.MOM.WW.OtOpcUa.slnx --verbosity quiet`, exit 0. Skips include environment-gated integration suites.
|
||||
@@ -0,0 +1,9 @@
|
||||
# Restart loss reason test correction — #702
|
||||
|
||||
The full integration reproduction failed at the restart wire test reason assertion: it required BadServerHalted but correctly received BadSessionIdInvalid. Production already supports that definitive loss reason.
|
||||
|
||||
The test now accepts an explicit semantic set: four direct definitive session statuses and two sustained closed-channel statuses. All BadNoCommunication, retained-value, unhealthy-state, frozen-read-stamp, one-notification and explicit reinitialization/recovery assertions remain unchanged. Only test expectation/comments changed; production unchanged. Existing deterministic driver tests independently enforce direct-status ordering/count and the sustained-channel threshold.
|
||||
|
||||
Verification: isolated restart1/1, deterministic status routes30/30, full OPC UA integration122/122. Full offline solution: 14,613 passed / 611 skipped / 0 failed, exit0, 61 reported suite summaries. Host integration266 passed /14 skipped; unavailable environment/fixture tests remain skips. Independent review found no important defects; git diff --check passed. Actual prior red run121pass/1fail, detailed console/TRX confirms the exact reason mismatch.
|
||||
|
||||
Logs/TRX retained in the Codex thread workspace under work/issue702-*. No Gateway or deployment behavior changed by this test correction. Separate live gates remain open.
|
||||
@@ -0,0 +1,53 @@
|
||||
# Whole-solution CI
|
||||
|
||||
The `v2-ci` workflow restores, builds and tests in one job so the test step uses
|
||||
that job's Release binaries. Restore disables parallel package work; restore and
|
||||
build use at most two MSBuild workers. Testing uses one MSBuild worker, disables
|
||||
parallel target-framework runs and loads `ci.runsettings`: one VSTest process and
|
||||
two xUnit collection workers with conservative scheduling. Assemblies that disable
|
||||
parallelization keep that setting.
|
||||
|
||||
The workflow retains the E2E/LiveIntegration exclusions, includes SlowUnit, keeps
|
||||
Akka's test time factor at three, and requires at least 12,000 executed tests.
|
||||
Runner capacity and TRX files remain available for diagnosing failures. These
|
||||
limits reduce contention; they do not prove every timing failure has one cause.
|
||||
There are no automatic test retries or changed assertion deadlines.
|
||||
|
||||
Gitea 1.26 supports the repository-wide `lmxopcua-v2-ci` concurrency group. Only
|
||||
one run is active; an active run is not cancelled, but a newer pending run may
|
||||
supersede an older pending run. The group does not serialize other repositories
|
||||
using the same host.
|
||||
|
||||
To reproduce the bounded test leg after restore and Release build:
|
||||
|
||||
```bash
|
||||
AKKA_TEST_TIMEFACTOR=3 dotnet test ZB.MOM.WW.OtOpcUa.slnx \
|
||||
--configuration Release --no-build -m:1 -p:TestTfmsInParallel=false \
|
||||
--settings scripts/ci/ci.runsettings \
|
||||
--filter "Category!=E2E&Category!=LiveIntegration" \
|
||||
--logger trx --results-directory artifacts/trx
|
||||
MIN_EXECUTED=12000 bash scripts/ci/assert-not-all-skipped.sh artifacts/trx/*.trx
|
||||
```
|
||||
|
||||
The guard uses the existing TRX `Counters/@executed` value for the aggregate
|
||||
floor. Its `skipped` diagnostic counts individual `Results/UnitTestResult`
|
||||
records whose outcome is `NotExecuted`, using the document's XML namespace.
|
||||
Some VSTest reports set summary `notExecuted` to zero even when these records
|
||||
are present. Skip records are counted per file, including repeated runs; they
|
||||
are not a count of unique test methods and do not affect the executed floor.
|
||||
For malformed XML, the diagnostic retains the old summary-counter fallback;
|
||||
this guard does not add XML validation to its existing gate policy.
|
||||
|
||||
Run the deterministic script fixtures without building or running the solution:
|
||||
|
||||
```bash
|
||||
python3 scripts/ci/test-assert-not-all-skipped.py
|
||||
```
|
||||
|
||||
Verified hosted baseline: [Gitea run 975](https://gitea.dohertylan.com/dohertj2/lmxopcua/actions/runs/975)
|
||||
completed successfully on 2026-10-13 for commit
|
||||
`76bdca90e94e814e06ef10b78271e2b492b2ba32` (merged by PR #711).
|
||||
Restore, Release build, README drift check, the serialized whole-solution test
|
||||
step, the 12,000-test floor and TRX upload all passed. The floor counted
|
||||
14,630 executed tests across 62 TRX files. This baseline predates the individual
|
||||
skip-reporting correction; its printed zero skip diagnostic is not skip evidence.
|
||||
@@ -33,12 +33,31 @@ for f in "$@"; do
|
||||
exit 2
|
||||
fi
|
||||
seen=$((seen + 1))
|
||||
# The trx <Counters .../> element in <ResultSummary> carries executed + notExecuted counts.
|
||||
# Keep the executed-counter floor unchanged; notExecuted is only a fallback
|
||||
# for malformed XML that this guard historically accepted.
|
||||
counters="$(grep -oE '<Counters[^>]*/?>' "$f" | head -1)"
|
||||
ex="$(printf '%s' "$counters" | grep -oE 'executed="[0-9]+"' | grep -oE '[0-9]+' || true)"
|
||||
ne="$(printf '%s' "$counters" | grep -oE 'notExecuted="[0-9]+"' | grep -oE '[0-9]+' || true)"
|
||||
ex="${ex:-0}"
|
||||
ne="${ne:-0}"
|
||||
# VSTest can emit notExecuted="0" despite individual NotExecuted results.
|
||||
# Count direct Results/UnitTestResult records in the document's namespace,
|
||||
# excluding foreign elements, nested records and escaped diagnostic text.
|
||||
ne="$(python3 - "$f" "$ne" <<'PY'
|
||||
import sys
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
try:
|
||||
root = ET.parse(sys.argv[1]).getroot()
|
||||
except ET.ParseError:
|
||||
# Skip reporting must not introduce a new malformed-XML gate failure.
|
||||
print(sys.argv[2])
|
||||
else:
|
||||
namespace = root.tag.split("}", 1)[0] + "}" if root.tag.startswith("{") else ""
|
||||
results = root.findall(f"{namespace}Results/{namespace}UnitTestResult")
|
||||
print(sum(result.get("outcome") == "NotExecuted" for result in results))
|
||||
PY
|
||||
)"
|
||||
total_executed=$((total_executed + ex))
|
||||
total_skipped=$((total_skipped + ne))
|
||||
echo " $(basename "$f"): executed=${ex} skipped=${ne}"
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<RunSettings>
|
||||
<!-- #707: project scheduling is also bounded by dotnet test -m:1. -->
|
||||
<RunConfiguration>
|
||||
<MaxCpuCount>1</MaxCpuCount>
|
||||
</RunConfiguration>
|
||||
<xUnit>
|
||||
<!-- Preserve each assembly's parallelization opt-out; bound enabled collections. -->
|
||||
<MaxParallelThreads>2</MaxParallelThreads>
|
||||
<ParallelAlgorithm>conservative</ParallelAlgorithm>
|
||||
</xUnit>
|
||||
</RunSettings>
|
||||
@@ -0,0 +1,108 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Deterministic subprocess checks for the TRX executed-floor guard."""
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
|
||||
SCRIPT = Path(__file__).with_name("assert-not-all-skipped.sh")
|
||||
NS = "http://microsoft.com/schemas/VisualStudio/TeamTest/2010"
|
||||
|
||||
|
||||
def trx(executed, outcomes, summary_skipped=0, namespace=NS, extra=""):
|
||||
declaration = f' xmlns="{namespace}"' if namespace else ""
|
||||
results = "".join(f'<UnitTestResult outcome="{outcome}"/>' for outcome in outcomes)
|
||||
return (f'<TestRun{declaration}><Results>{results}{extra}</Results>'
|
||||
f'<ResultSummary><Counters total="{len(outcomes)}" executed="{executed}" '
|
||||
f'notExecuted="{summary_skipped}"/></ResultSummary></TestRun>')
|
||||
|
||||
|
||||
class SkipGuardTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.directory = tempfile.TemporaryDirectory(prefix="trx-skip-fixtures-")
|
||||
self.addCleanup(self.directory.cleanup)
|
||||
self.root = Path(self.directory.name)
|
||||
|
||||
def write(self, name, content):
|
||||
path = self.root / name
|
||||
path.write_text(content, encoding="utf-8")
|
||||
return path
|
||||
|
||||
def run_guard(self, *paths, minimum=None):
|
||||
env = os.environ.copy()
|
||||
env.pop("MIN_EXECUTED", None)
|
||||
if minimum is not None:
|
||||
env["MIN_EXECUTED"] = str(minimum)
|
||||
return subprocess.run(["bash", str(SCRIPT), *map(str, paths)], env=env,
|
||||
text=True, capture_output=True, check=False)
|
||||
|
||||
def assert_report(self, result, status, executed, skipped, files=1):
|
||||
self.assertEqual(status, result.returncode, result.stdout + result.stderr)
|
||||
self.assertIn(f"files={files} executed={executed} skipped={skipped} ", result.stdout)
|
||||
|
||||
def test_mixed_results_ignore_zero_summary_skip_counter(self):
|
||||
# A summary-only skip count misses the three actual NotExecuted records.
|
||||
path = self.write("mixed.trx", trx(827, ["Passed"] * 827 + ["NotExecuted"] * 3))
|
||||
result = self.run_guard(path)
|
||||
self.assert_report(result, 0, 827, 3)
|
||||
self.assertIn("mixed.trx: executed=827 skipped=3", result.stdout)
|
||||
|
||||
def test_all_skipped_fails_default_floor_but_report_only_passes(self):
|
||||
path = self.write("all-skipped.trx", trx(0, ["NotExecuted"] * 188))
|
||||
self.assert_report(self.run_guard(path), 1, 0, 188)
|
||||
self.assert_report(self.run_guard(path, minimum=0), 0, 0, 188)
|
||||
|
||||
def test_zero_tests_fails_default_floor_but_report_only_passes(self):
|
||||
path = self.write("zero.trx", trx(0, []))
|
||||
self.assert_report(self.run_guard(path), 1, 0, 0)
|
||||
self.assert_report(self.run_guard(path, minimum=0), 0, 0, 0)
|
||||
|
||||
def test_agreeing_summary_and_outcomes_are_not_double_counted(self):
|
||||
path = self.write("agreeing.trx", trx(1, ["Passed", "NotExecuted"], 1))
|
||||
self.assert_report(self.run_guard(path), 0, 1, 1)
|
||||
|
||||
def test_outcomes_are_authoritative_when_summary_overstates_skips(self):
|
||||
path = self.write("overstated.trx", trx(1, ["Passed", "NotExecuted"], 9))
|
||||
self.assert_report(self.run_guard(path), 0, 1, 1)
|
||||
|
||||
def test_namespace_path_and_exact_outcome_exclude_decoys(self):
|
||||
extra = ('<Other><UnitTestResult outcome="NotExecuted"/></Other>'
|
||||
'<foreign:UnitTestResult xmlns:foreign="urn:other" outcome="NotExecuted"/>'
|
||||
'<Output><Message><UnitTestResult outcome="NotExecuted"/></Message></Output>')
|
||||
content = trx(2, ["Passed", "Failed", "NotExecuted", "notexecuted"], extra=extra)
|
||||
# Prefixes on Results/UnitTestResult must work just like the default namespace.
|
||||
content = content.replace('<TestRun ', f'<TestRun xmlns:t="{NS}" ')
|
||||
content = content.replace('<Results>', '<t:Results>').replace('</Results>', '</t:Results>')
|
||||
content = content.replace('<UnitTestResult ', '<t:UnitTestResult ')
|
||||
path = self.write("namespaced.trx", content)
|
||||
self.assert_report(self.run_guard(path), 0, 2, 1)
|
||||
|
||||
def test_unnamespaced_results_remain_supported(self):
|
||||
path = self.write("legacy.trx", trx(1, ["Passed", "NotExecuted"], namespace=""))
|
||||
self.assert_report(self.run_guard(path), 0, 1, 1)
|
||||
|
||||
def test_executed_counter_controls_aggregate_floor_at_boundary(self):
|
||||
# Counters retain authority even when result records disagree with executed.
|
||||
mixed = self.write("floor.trx", trx(12000, ["Passed", "NotExecuted"]))
|
||||
skipped = self.write("skipped.trx", trx(0, ["NotExecuted"]))
|
||||
self.assert_report(self.run_guard(mixed, skipped, minimum=12000), 0, 12000, 2, 2)
|
||||
self.assert_report(self.run_guard(mixed, skipped, minimum=12001), 1, 12000, 2, 2)
|
||||
|
||||
def test_missing_arguments_and_file_keep_exit_two(self):
|
||||
self.assertEqual(2, self.run_guard().returncode)
|
||||
self.assertEqual(2, self.run_guard(self.root / "missing.trx").returncode)
|
||||
|
||||
def test_missing_counters_keeps_existing_exit_one(self):
|
||||
path = self.write("no-counters.trx", '<TestRun><Results/></TestRun>')
|
||||
self.assertEqual(1, self.run_guard(path, minimum=0).returncode)
|
||||
|
||||
def test_malformed_xml_keeps_summary_fallback_and_gate_status(self):
|
||||
path = self.write("malformed.trx", '<TestRun><Counters executed="2" notExecuted="3"/>')
|
||||
self.assert_report(self.run_guard(path, minimum=2), 0, 2, 3)
|
||||
self.assert_report(self.run_guard(path, minimum=3), 1, 2, 3)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main(verbosity=2)
|
||||
@@ -1,21 +1,38 @@
|
||||
namespace ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
|
||||
|
||||
/// <summary>
|
||||
/// Late-binding adapter that holds an inner <see cref="IServiceLevelPublisher"/> reference
|
||||
/// swappable at runtime. Mirrors <see cref="DeferredAddressSpaceSink"/>: Akka actors resolve
|
||||
/// the publisher at DI time, but the production <c>SdkServiceLevelPublisher</c> only exists
|
||||
/// after <c>StandardServer.Start</c>. The Host's hosted service swaps the inner once the SDK
|
||||
/// is up; until then writes route through <see cref="NullServiceLevelPublisher"/>.
|
||||
/// Late-binding adapter that accepts and retains the latest requested ServiceLevel until
|
||||
/// the SDK publisher is available. Binding synchronously replays that request, including zero.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Publishing and binding are serialized through the synchronous inner call. Inner publishers
|
||||
/// must not call back into this adapter. Acceptance while unbound is not an SDK write confirmation.
|
||||
/// A failed call retains both the requested level and the installed publisher for later recovery.
|
||||
/// </remarks>
|
||||
public sealed class DeferredServiceLevelPublisher : IServiceLevelPublisher
|
||||
{
|
||||
private volatile IServiceLevelPublisher _inner = NullServiceLevelPublisher.Instance;
|
||||
private readonly object _gate = new();
|
||||
private IServiceLevelPublisher? _inner;
|
||||
private byte? _latest;
|
||||
|
||||
/// <summary>Swap the underlying publisher. Pass null to revert to the Null no-op.</summary>
|
||||
/// <param name="inner">The publisher implementation to use, or null to use the null publisher.</param>
|
||||
public void SetInner(IServiceLevelPublisher? inner) =>
|
||||
_inner = inner ?? NullServiceLevelPublisher.Instance;
|
||||
/// <summary>Bind and replay the latest request, or unbind while retaining that request.</summary>
|
||||
/// <param name="inner">The publisher implementation to use, or null to unbind.</param>
|
||||
public void SetInner(IServiceLevelPublisher? inner)
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
_inner = inner;
|
||||
if (_latest is { } latest) _inner?.Publish(latest);
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public void Publish(byte serviceLevel) => _inner.Publish(serviceLevel);
|
||||
public void Publish(byte serviceLevel)
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
_latest = serviceLevel;
|
||||
_inner?.Publish(serviceLevel);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -93,6 +93,14 @@ public sealed class HistoryReadLimitExceededException : InvalidOperationExceptio
|
||||
/// <summary>The configured limit that was exceeded.</summary>
|
||||
public long Limit { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #694: a raw read refused because rows BEFORE its window (inside the backend's look-back margin — a tie
|
||||
/// cluster just before a page start, the HistorianGateway's boundary row) crowded a ceiling-bounded row budget: how
|
||||
/// many such rows the read saw (a lower bound when the read was itself truncated). <c>0</c> = not a crowding refusal.
|
||||
/// The server re-reads such a page once, smaller by this many values (<c>RawPageLimit</c>).
|
||||
/// </summary>
|
||||
public int PreWindowRows { get; init; }
|
||||
|
||||
/// <summary>Creates the exception.</summary>
|
||||
/// <param name="limit">The configured limit.</param>
|
||||
/// <param name="message">Operator-facing description.</param>
|
||||
|
||||
@@ -46,6 +46,18 @@ public interface IHistorianDataSource : IDisposable
|
||||
/// </summary>
|
||||
bool SupportsReverseRaw => false;
|
||||
|
||||
/// <summary>
|
||||
/// The most values one capped <see cref="ReadRawAsync"/> can currently return without failing
|
||||
/// <see cref="HistoryReadLimitExceededException"/> on a BACKEND row ceiling (Gitea #694); <c>0</c> = no known limit.
|
||||
/// The server sizes its raw pages to it — a page is at most this many values and carries a continuation point when
|
||||
/// full (Part 11 allows fewer than <c>NumValuesPerNode</c> with a continuation point), instead of failing a page size
|
||||
/// the backend cannot read in one go. It may change at runtime (a strict HistorianGateway's ceiling is learned from
|
||||
/// its first refusal, #691.3), so it is read per read. A backend whose reads are never ceiling-bound returns <c>0</c>.
|
||||
/// The source still never truncates: a read asking for more than this is answered in full or refused. Default
|
||||
/// <c>0</c>; a backend declares it itself (a guard test pins that).
|
||||
/// </summary>
|
||||
int MaxRawValuesPerRead => 0;
|
||||
|
||||
/// <summary>
|
||||
/// Read processed (interval-bucketed) samples — average / min / max / count / etc.
|
||||
/// A bucket with no source data returns a sample whose
|
||||
@@ -81,6 +93,31 @@ public interface IHistorianDataSource : IDisposable
|
||||
IReadOnlyList<DateTime> timestampsUtc,
|
||||
CancellationToken cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #693: <see cref="ReadAtTimeAsync(string, IReadOnlyList{DateTime}, CancellationToken)"/> with the value type of
|
||||
/// the node being read, for a backend whose stored values do not carry their type (the Gateway: AVEVA stores every
|
||||
/// historized type as a numeric analog, so a Boolean comes back as a 0/1 Double). Such a backend uses the hint to apply
|
||||
/// the AtTime rules exactly as the typed backends do — a Boolean (or String / DateTime) node is stepped, an integer node
|
||||
/// is interpolated, rounded half away from zero and returned as the node's integer type, a floating-point node is
|
||||
/// interpolated. A backend that stores typed values ignores it: the default forwards to the hint-less overload, so
|
||||
/// SqlServer, QuestDB and the Null source need no change.
|
||||
/// </summary>
|
||||
/// <param name="fullReference">The full reference of the tag to read.</param>
|
||||
/// <param name="timestampsUtc">The list of timestamps to read values at.</param>
|
||||
/// <param name="valueType">
|
||||
/// The scalar value type of the node (<see cref="TypeCode.Boolean"/>, the integer codes, <see cref="TypeCode.Single"/>,
|
||||
/// <see cref="TypeCode.Double"/>, <see cref="TypeCode.String"/>, <see cref="TypeCode.DateTime"/>), or <c>null</c> when
|
||||
/// unknown, not scalar, or not one of those — the backend's own policy then applies.
|
||||
/// </param>
|
||||
/// <param name="cancellationToken">A cancellation token that can be used to cancel the operation.</param>
|
||||
/// <returns>One sample per requested timestamp, in the same order as <paramref name="timestampsUtc"/>.</returns>
|
||||
Task<HistoryReadResult> ReadAtTimeAsync(
|
||||
string fullReference,
|
||||
IReadOnlyList<DateTime> timestampsUtc,
|
||||
TypeCode? valueType,
|
||||
CancellationToken cancellationToken) =>
|
||||
ReadAtTimeAsync(fullReference, timestampsUtc, cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// Read historical alarm / event records — OPC UA HistoryReadEvents service.
|
||||
/// Distinct from any live event stream; sources here come from the historian's
|
||||
@@ -95,7 +132,7 @@ public interface IHistorianDataSource : IDisposable
|
||||
/// <c>uint maxValuesPerNode</c> because their OPC UA HistoryRead surface has no
|
||||
/// equivalent "use default" sentinel.
|
||||
///
|
||||
/// This surface declares <see cref="ReadAtTimeAsync"/> and <see cref="ReadEventsAsync(string?, DateTime, DateTime, int, CancellationToken)"/>
|
||||
/// This surface declares <see cref="ReadAtTimeAsync(string, IReadOnlyList{DateTime}, CancellationToken)"/> and <see cref="ReadEventsAsync(string?, DateTime, DateTime, int, CancellationToken)"/>
|
||||
/// as required members — a server-side historian owns the full read surface.
|
||||
/// </remarks>
|
||||
/// <param name="sourceName">The source name to filter events, or null to return events from all sources.</param>
|
||||
@@ -120,9 +157,9 @@ public interface IHistorianDataSource : IDisposable
|
||||
/// <summary>
|
||||
/// Whether this backend can serve a reverse (newest-first) events read — the reverse overload of
|
||||
/// <see cref="ReadEventsAsync(string?, DateTime, DateTime, int, bool, CancellationToken)"/> (Gitea #675). The server
|
||||
/// checks it BEFORE calling the source: an explicit reverse read (<c>StartTime > EndTime</c>) on a backend that
|
||||
/// cannot is answered <c>BadHistoryOperationUnsupported</c> without a call, and an end-only capped read, which Part 11
|
||||
/// reads backward, keeps its forward <c>[MinValue, EndTime)</c> meaning there. Default <c>false</c>; an
|
||||
/// checks it BEFORE calling the source: a reverse read (explicit <c>StartTime > EndTime</c>, or an end-only capped
|
||||
/// read, which Part 11 reads backward) on a backend that cannot is never sent the reverse overload — since Gitea
|
||||
/// #691.4 the server walks it back from bounded FORWARD reads instead (#675 refused it). Default <c>false</c>; an
|
||||
/// implementation (or a wrapper) declares it itself — a guard test pins that.
|
||||
/// </summary>
|
||||
bool SupportsReverseEvents => false;
|
||||
@@ -143,10 +180,10 @@ public interface IHistorianDataSource : IDisposable
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// The default forwards a forward read to the five-argument overload and throws
|
||||
/// <see cref="NotSupportedException"/> for a reverse one — a backend that cannot read newest-first (Gateway,
|
||||
/// Null). The server never relies on that throw: it consults <see cref="SupportsReverseEvents"/> first and
|
||||
/// answers <c>BadHistoryOperationUnsupported</c> without calling, rather than returning the earliest events in
|
||||
/// ascending order as if they were the answer. A backend that serves reverse must override BOTH members.
|
||||
/// <see cref="NotSupportedException"/> for a reverse one — a backend that cannot read newest-first (the
|
||||
/// Gateway). The server never relies on that throw: it consults <see cref="SupportsReverseEvents"/> first and walks
|
||||
/// such a read back from forward reads (#691.4), never returning the earliest events in ascending order as if
|
||||
/// they were the answer. A backend that serves reverse natively must override BOTH members.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
/// <param name="sourceName">The source name to filter events, or null to return events from all sources.</param>
|
||||
|
||||
@@ -61,13 +61,16 @@ public sealed class NullHistorianDataSource : IHistorianDataSource
|
||||
public bool SupportsReverseRaw => true;
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks><c>false</c> (Gitea #675): there is no history to read in either direction, and an explicit reverse read
|
||||
/// is answered <c>BadHistoryOperationUnsupported</c> like any backend that cannot read newest-first.</remarks>
|
||||
public bool SupportsReverseEvents => false;
|
||||
/// <remarks><c>0</c>: no history, no ceiling (Gitea #694).</remarks>
|
||||
public int MaxRawValuesPerRead => 0;
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>Forward ⇒ the empty result; reverse ⇒ <see cref="NotSupportedException"/>, the interface contract for a
|
||||
/// backend without <see cref="SupportsReverseEvents"/>.</remarks>
|
||||
/// <remarks><c>true</c>, like <see cref="SupportsReverseRaw"/> (Gitea #691.4 review; <c>false</c> under #675): there is no
|
||||
/// history to read in either direction — no need for the server to walk back through empty forward reads.</remarks>
|
||||
public bool SupportsReverseEvents => true;
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>The empty result in either direction.</remarks>
|
||||
public Task<HistoricalEventsResult> ReadEventsAsync(
|
||||
string? sourceName,
|
||||
DateTime startUtc,
|
||||
@@ -75,9 +78,7 @@ public sealed class NullHistorianDataSource : IHistorianDataSource
|
||||
int maxEvents,
|
||||
bool reverse,
|
||||
CancellationToken cancellationToken) =>
|
||||
reverse
|
||||
? throw new NotSupportedException("The null historian cannot read events newest-first (reverse).")
|
||||
: Task.FromResult(EmptyEvents);
|
||||
Task.FromResult(EmptyEvents);
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian;
|
||||
|
||||
/// <summary>Stable delivery GUIDs for persisted alarm queue rows.</summary>
|
||||
public static class AlarmDeliveryIds
|
||||
{
|
||||
/// <summary>
|
||||
/// Converts an existing durable row key into a nonzero version-8 GUID. Version 1 of this encoding hashes
|
||||
/// UTF-8 "OtOpcUa.AlarmDelivery/v1\0" followed by the exact persisted key with SHA-256, takes the first
|
||||
/// 16 bytes, sets the UUID version/variant bits, and interprets the bytes in network order.
|
||||
/// This encoding must remain stable across retries, upgrades and replicated failover.
|
||||
/// </summary>
|
||||
/// <param name="queueId">The persisted row key, never a hash recomputed from the current payload.</param>
|
||||
/// <returns>The row's stable delivery identity.</returns>
|
||||
public static Guid FromQueueId(string queueId)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(queueId);
|
||||
var digest = SHA256.HashData(Encoding.UTF8.GetBytes("OtOpcUa.AlarmDelivery/v1\0" + queueId));
|
||||
digest[6] = (byte)((digest[6] & 0x0F) | 0x80);
|
||||
digest[8] = (byte)((digest[8] & 0x3F) | 0x80);
|
||||
return new Guid(digest.AsSpan(0, 16), bigEndian: true);
|
||||
}
|
||||
}
|
||||
@@ -1,3 +1,4 @@
|
||||
using System.Text.Json.Serialization;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian;
|
||||
@@ -34,4 +35,31 @@ public sealed record AlarmHistorianEvent(
|
||||
string Message,
|
||||
string User,
|
||||
string? Comment,
|
||||
DateTime TimestampUtc);
|
||||
DateTime TimestampUtc)
|
||||
{
|
||||
/// <summary>
|
||||
/// Delivery identity, minted once for direct sends and replaced by the durable queue's identity when drained.
|
||||
/// Excluded from payload serialization and business equality so redundant enqueues still converge.
|
||||
/// A record copy preserves this identity; an explicitly empty identity is invalid for delivery.
|
||||
/// </summary>
|
||||
[JsonIgnore]
|
||||
public Guid DeliveryId { get; init; } = Guid.NewGuid();
|
||||
|
||||
/// <summary>Compares the ten business fields, excluding transport delivery identity.</summary>
|
||||
/// <param name="other">The event to compare.</param>
|
||||
/// <returns>Whether the business fields are equal.</returns>
|
||||
public bool Equals(AlarmHistorianEvent? other) => ReferenceEquals(this, other) || other is not null
|
||||
&& AlarmId == other.AlarmId && EquipmentPath == other.EquipmentPath && AlarmName == other.AlarmName
|
||||
&& AlarmTypeName == other.AlarmTypeName && Severity == other.Severity && EventKind == other.EventKind
|
||||
&& Message == other.Message && User == other.User && Comment == other.Comment && TimestampUtc == other.TimestampUtc;
|
||||
|
||||
/// <summary>Hashes the same business fields used by equality, excluding delivery identity.</summary>
|
||||
/// <returns>The business-field hash code.</returns>
|
||||
public override int GetHashCode()
|
||||
{
|
||||
var hash = new HashCode();
|
||||
hash.Add(AlarmId); hash.Add(EquipmentPath); hash.Add(AlarmName); hash.Add(AlarmTypeName); hash.Add(Severity);
|
||||
hash.Add(EventKind); hash.Add(Message); hash.Add(User); hash.Add(Comment); hash.Add(TimestampUtc);
|
||||
return hash.ToHashCode();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -603,6 +603,9 @@ public sealed class LocalDbStoreAndForwardSink : IAlarmHistorianSink, IDisposabl
|
||||
try
|
||||
{
|
||||
evt = JsonSerializer.Deserialize<AlarmHistorianEvent>(payload);
|
||||
// The durable key survives retry, payload edits, legacy migration and replicated failover.
|
||||
// Never derive delivery identity from a reserialized payload or an individual RPC attempt.
|
||||
if (evt is not null) evt = evt with { DeliveryId = AlarmDeliveryIds.FromQueueId(id) };
|
||||
}
|
||||
catch (JsonException)
|
||||
{
|
||||
|
||||
@@ -0,0 +1,514 @@
|
||||
using ZB.MOM.WW.HistorianGateway.Contracts.Grpc;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Mapping;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway;
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #693: HistoryRead AtTime on the Gateway backend, served from RAW gateway reads and evaluated with the SQL backends'
|
||||
/// shared <see cref="AtTimeEvaluator"/> (Part 11 §6.5.4) — as #681 did for the per-sample aggregates — so the Gateway
|
||||
/// answers exactly what SqlServer and QuestDB answer for the same stored samples. AVEVA's own interpolated answer (the
|
||||
/// gateway's <c>ReadAtTime</c>) is not used: live, it returned the previous value as Good at a stored NULL's own time,
|
||||
/// null + Bad at a Good sample right after a NULL, a drifted value (99.99999999944) at an exact stored 100, and no
|
||||
/// Interpolated bit.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>What each requested time needs.</b> <c>prev</c> = the latest stored sample at or before it, <c>next</c> = the
|
||||
/// first stored sample after it. <c>next</c> is only read when the evaluator uses it (a usable <c>prev</c> that is not
|
||||
/// an exact hit).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Read plan — O(1) reads for a dense batch, never two round-trips per timestamp by default.</b> The distinct
|
||||
/// requested times are walked in ascending order with ONE forward window read <c>[t − 1 ms, max + 1 ms]</c> of at most
|
||||
/// <see cref="ReadRows"/> rows, which yields every in-window neighbour. The sample before the first time comes from a
|
||||
/// walk-back (below), the sample after the last time from one small forward probe. A window read that fills its budget
|
||||
/// (dense data, widely spread times) resolves the times before its last row; the next unresolved time either
|
||||
/// continues from that row (when the rows in between are expected to fit one read) or jumps there with a read that
|
||||
/// starts a density-sized margin before it (so its <c>prev</c> is normally in the same read). Ten times one second
|
||||
/// apart on a 1 Hz tag cost 1 read (all exact hits) or 3 (interpolated); ten times an hour apart about one read each.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Walk-back</b> (the gateway reads forward only), in two phases. <i>Gallop:</i> segments <c>(lo, hi]</c> stepping
|
||||
/// back from <c>hi</c>, each one forward read, the first sized from the density the window read showed (else
|
||||
/// <see cref="InitialWalkWidth"/>), an empty one growing the next ×4; after two empty segments the whole remainder down
|
||||
/// to the epoch is asked once — a complete answer ends the walk, a FULL one seeds the bisection with its last row.
|
||||
/// <i>Bisect</i> (once a stored row is known below <c>hi</c> — a full segment's last row, or the last row of an earlier
|
||||
/// window read): read <c>(mid, hi]</c>; empty ⇒ <c>hi = mid</c>; full ⇒ the known row moves up to its last row;
|
||||
/// complete with rows ⇒ its last row is the answer. Right after a full read the next segment is aimed at about a
|
||||
/// quarter of a read at the density it showed (one try — an empty aimed segment falls back to halving), and below
|
||||
/// 2 ms the segment is the whole stretch from the known row (never a 100 ns step).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Bounded.</b> One request may take at most <c>64 + 16 × distinct times</c> raw reads (<see cref="BaseReads"/>,
|
||||
/// <see cref="ReadsPerTime"/>); past it — or with a strict-gateway ceiling below <see cref="MinBudget"/> rows — the
|
||||
/// node fails <c>BadTooManyOperations</c> with a message naming the limit. A jump read (to a time far past the previous
|
||||
/// read) asks for only <see cref="JumpRows"/> rows, so a thousand times spread over ten years cost about a thousand
|
||||
/// small reads, not a thousand 1,024-row reads.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Wire shape.</b> Every request bound is a whole millisecond (AVEVA stores milliseconds and truncates request
|
||||
/// times); a row counts only strictly INSIDE its request — AVEVA's synthetic boundary row at the query start (the value
|
||||
/// in effect there, <c>Quality = 133</c>, or a NULL row) can never be taken for a stored sample, nor a possible row at
|
||||
/// the query end; the epoch and before hold nothing. A Bad row (a stored NULL arrives as value 0, <c>q=1 opc=0</c>,
|
||||
/// #690) is value-less. Every stream is drained, never abandoned (a cancelled gateway stream evicts its pooled
|
||||
/// historian session). Nothing is asked past <c>now + </c><see cref="RawAggregateReader.LookAheadHorizon"/> (the
|
||||
/// historian stores nothing further ahead, and the gateway's AVEVA may refuse a far-future end); a strict gateway's
|
||||
/// row ceiling (#691.3) is learned and honoured — a short read is ordinary paging here, never re-probed.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Values.</b> A gateway value is a Double or a String whatever the node's data type (AVEVA stores every historized
|
||||
/// type as a numeric analog — a Boolean as a 0/1 <c>Int2</c>). The node's type hint converts each row BEFORE the
|
||||
/// shared evaluator runs, so its own type rules apply exactly as on the SQL backends: Boolean (value ≠ 0) and String
|
||||
/// are stepped; an integer type is an Int64 — interpolated, rounded half away from zero — and the result is returned
|
||||
/// as the node's integer type; Float / Double (and no hint) interpolate linearly as Double.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
internal static class GatewayAtTimeReader
|
||||
{
|
||||
/// <summary>The row budget of a window read (the first, or one continuing from a full one) and a walk-back segment.</summary>
|
||||
internal const int ReadRows = 1024;
|
||||
|
||||
/// <summary>The row budget of a jump read (a window read restarting at a time far past the previous read's last row).</summary>
|
||||
internal const int JumpRows = 64;
|
||||
|
||||
/// <summary>The row budget of the forward probe for the sample after the last time (boundary row + ties + one).</summary>
|
||||
internal const int ProbeRows = 2 + RawAggregateReader.PreWindowAllowance;
|
||||
|
||||
/// <summary>The smallest row budget a read can work with (the boundary row, a tie, one sample beyond).</summary>
|
||||
internal const int MinBudget = 4;
|
||||
|
||||
/// <summary>Raw reads one request may always take …</summary>
|
||||
internal const int BaseReads = 64;
|
||||
|
||||
/// <summary>
|
||||
/// … plus this many per distinct requested time. A runaway guard, not a budget (<c>HistoryReadDeadline</c> bounds the
|
||||
/// wall clock): a time inside a gap of a bursty tag legitimately costs a jump read plus a bisection back across the gap
|
||||
/// (≈ 6 reads — a 1 h/day tag asked at noon on consecutive days), so 4 refused requests AVEVA's own AtTime served.
|
||||
/// </summary>
|
||||
internal const int ReadsPerTime = 16;
|
||||
|
||||
/// <summary>The first walk-back segment's width when the window read showed no density.</summary>
|
||||
internal static readonly TimeSpan InitialWalkWidth = TimeSpan.FromMinutes(1);
|
||||
|
||||
/// <summary>The widest walk-back segment sized from density, before ×4 growth.</summary>
|
||||
private static readonly TimeSpan MaxSeededWidth = TimeSpan.FromDays(1);
|
||||
|
||||
private const long Ms = TimeSpan.TicksPerMillisecond;
|
||||
|
||||
private static readonly TimeSpan OneMs = TimeSpan.FromMilliseconds(1);
|
||||
|
||||
private static readonly DateTime Floor = RawAggregateReader.ProbeFloor;
|
||||
|
||||
/// <summary>One drained read: the rows strictly inside the request (ascending) and whether the stream ended short.</summary>
|
||||
private readonly record struct Read(List<StoredSample> Rows, bool Complete);
|
||||
|
||||
/// <summary>Evaluates AtTime for <paramref name="timestampsUtc"/> (same length and order, duplicates included).</summary>
|
||||
/// <param name="client">The gateway client.</param>
|
||||
/// <param name="ceiling">The data source's learned gateway row ceiling (#691.3).</param>
|
||||
/// <param name="tag">Historian tag name.</param>
|
||||
/// <param name="timestampsUtc">Requested times (any kind; treated as UTC, as on the SQL backends).</param>
|
||||
/// <param name="now">Server timestamp, the beyond-now rule and the look-ahead horizon.</param>
|
||||
/// <param name="ct">Cancellation.</param>
|
||||
/// <param name="valueType">The node's scalar value type (the hinted <c>IHistorianDataSource.ReadAtTimeAsync</c>), or null.</param>
|
||||
/// <param name="maxReads">The request's raw-read cap; null = <see cref="BaseReads"/> + <see cref="ReadsPerTime"/> per distinct time.</param>
|
||||
/// <returns>One value per requested time, stamped with it.</returns>
|
||||
/// <exception cref="HistoryReadLimitExceededException">More than <see cref="ReadRows"/> rows within one millisecond, the
|
||||
/// read cap, or a strict gateway ceiling below <see cref="MinBudget"/>.</exception>
|
||||
public static async Task<IReadOnlyList<DataValueSnapshot>> ReadAsync(IHistorianGatewayClient client, GatewayRowCeiling ceiling,
|
||||
string tag, IReadOnlyList<DateTime> timestampsUtc, DateTime now, CancellationToken ct, TypeCode? valueType = null,
|
||||
int? maxReads = null)
|
||||
{
|
||||
var requested = timestampsUtc.Select(t => DateTime.SpecifyKind(t, DateTimeKind.Utc)).ToArray();
|
||||
if (requested.Length == 0) return [];
|
||||
var times = requested.Distinct().Order().ToArray();
|
||||
var ctx = new Ctx(client, ceiling, tag, ct, valueType, maxReads ?? BaseReads + ReadsPerTime * times.Length);
|
||||
var neighbours = await ResolveAsync(ctx, times, now).ConfigureAwait(false);
|
||||
var result = new DataValueSnapshot[requested.Length];
|
||||
for (var i = 0; i < requested.Length; i++)
|
||||
{
|
||||
var t = requested[i];
|
||||
var (prev, next) = neighbours[t.Ticks];
|
||||
// A DateTime node is stepped (as on the SQL backends): a usable non-numeric next sends the evaluator down its
|
||||
// stepped branch, with the next's own status.
|
||||
if (valueType == TypeCode.DateTime && next is { IsUsable: true } n)
|
||||
next = n with { Value = HistorianValue.FromString(string.Empty) };
|
||||
result[i] = ToNodeType(AtTimeEvaluator.Evaluate(t, prev, next, now, now), valueType);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/// <summary>One request's read context: the client, the type hint, and the read count against its cap.</summary>
|
||||
private sealed class Ctx(IHistorianGatewayClient client, GatewayRowCeiling ceiling, string tag, CancellationToken ct,
|
||||
TypeCode? valueType, int maxReads)
|
||||
{
|
||||
public IHistorianGatewayClient Client { get; } = client;
|
||||
public GatewayRowCeiling Ceiling { get; } = ceiling;
|
||||
public string Tag { get; } = tag;
|
||||
public CancellationToken Ct { get; } = ct;
|
||||
public TypeCode? ValueType { get; } = valueType;
|
||||
public int MaxReads { get; } = maxReads;
|
||||
public int Reads { get; set; }
|
||||
|
||||
/// <summary>The smallest budget a read was served with (a strict gateway's ceiling shows here).</summary>
|
||||
public int SmallestBudget { get; set; } = int.MaxValue;
|
||||
}
|
||||
|
||||
/// <summary>True for the integer type codes (evaluated as Int64, returned as the node's own type).</summary>
|
||||
private static bool IsInteger(TypeCode? t) =>
|
||||
t is TypeCode.SByte or TypeCode.Byte or TypeCode.Int16 or TypeCode.UInt16 or TypeCode.Int32 or TypeCode.UInt32
|
||||
or TypeCode.Int64 or TypeCode.UInt64;
|
||||
|
||||
/// <summary>
|
||||
/// An evaluated Int64 (integer hint) returned as the node's integer type. A value the type cannot hold (only possible
|
||||
/// for a stored value already outside the node's range) stays the evaluator's Int64.
|
||||
/// </summary>
|
||||
private static DataValueSnapshot ToNodeType(DataValueSnapshot v, TypeCode? valueType)
|
||||
{
|
||||
if (v.Value is not long l || !IsInteger(valueType) || valueType == TypeCode.Int64) return v;
|
||||
try
|
||||
{
|
||||
return v with { Value = Convert.ChangeType(l, valueType!.Value, System.Globalization.CultureInfo.InvariantCulture) };
|
||||
}
|
||||
catch (OverflowException)
|
||||
{
|
||||
return v;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>prev / next for every time of <paramref name="times"/> (distinct, ascending).</summary>
|
||||
private static async Task<Dictionary<long, (StoredSample? Prev, StoredSample? Next)>> ResolveAsync(
|
||||
Ctx ctx, DateTime[] times, DateTime now)
|
||||
{
|
||||
var result = new Dictionary<long, (StoredSample?, StoredSample?)>(times.Length);
|
||||
var horizon = MsFloor(DateTime.MaxValue - now > RawAggregateReader.LookAheadHorizon
|
||||
? DateTime.SpecifyKind(now + RawAggregateReader.LookAheadHorizon, DateTimeKind.Utc)
|
||||
: DateTime.MaxValue.AddTicks(-Ms));
|
||||
|
||||
// Nothing is stored at or before the epoch.
|
||||
var i = 0;
|
||||
for (; i < times.Length && times[i] <= Floor; i++) result[times[i].Ticks] = (null, null);
|
||||
if (i == times.Length) return result;
|
||||
|
||||
// Reads are planned on times clamped to the horizon (nothing is stored beyond it, and no read ends past it).
|
||||
var qe = Min(MsFloor(Min(times[^1], horizon)) + OneMs, horizon);
|
||||
var qs = RequestStart(Min(times[i], horizon));
|
||||
var budget = ReadRows;
|
||||
StoredSample? before = null; // the latest stored sample at or before qs …
|
||||
var beforeKnown = false; // … once known
|
||||
DateTime? walkKnown = null; // a stored sample exists here (at or before qs): the walk-back bisects above it
|
||||
var walkWidth = InitialWalkWidth;
|
||||
var probeFrom = -1; // first index whose next lies at or after qe (complete last read)
|
||||
|
||||
while (true)
|
||||
{
|
||||
var read = await ReadTrustedAsync(ctx, qs, qe, budget).ConfigureAwait(false);
|
||||
var rows = read.Rows;
|
||||
if (!read.Complete && rows.Count == 0)
|
||||
throw TooDense(ctx, qs);
|
||||
if (rows.Count >= 2 && !beforeKnown && walkKnown is null)
|
||||
walkWidth = Clamp(TimeSpan.FromTicks(4 * (rows[^1].TimeUtc - rows[0].TimeUtc).Ticks / (rows.Count - 1)));
|
||||
|
||||
// A short stream saw every row up to qe; a full one every row before its last row's time x.
|
||||
var x = read.Complete ? qe : rows[^1].TimeUtc;
|
||||
var j = -1; // index of the last row at or before the current time
|
||||
var k = i;
|
||||
for (; k < times.Length; k++)
|
||||
{
|
||||
var t = times[k];
|
||||
var tr = Min(t, horizon);
|
||||
if (!read.Complete && tr >= x) break;
|
||||
while (j + 1 < rows.Count && rows[j + 1].TimeUtc <= tr) j++;
|
||||
|
||||
StoredSample? prev;
|
||||
if (j >= 0) prev = rows[j];
|
||||
else
|
||||
{
|
||||
if (!beforeKnown)
|
||||
{
|
||||
// #696: prove the future stretch empty before jumping to now. Historian clocks or imports
|
||||
// can place real rows there; a full bounded probe seeds the original backward search.
|
||||
var searchEnd = qs;
|
||||
if (qs > now && walkKnown is null)
|
||||
{
|
||||
var future = await ReadTrustedAsync(ctx, MsFloor(now),
|
||||
Min(MsFloor(qs) + OneMs, horizon), ReadRows).ConfigureAwait(false);
|
||||
StoredSample? latest = null;
|
||||
foreach (var row in future.Rows)
|
||||
if (row.TimeUtc <= qs) latest = row;
|
||||
if (future.Complete)
|
||||
{
|
||||
before = latest;
|
||||
beforeKnown = latest is not null;
|
||||
searchEnd = now;
|
||||
}
|
||||
else if (latest is { } found)
|
||||
walkKnown = found.TimeUtc;
|
||||
}
|
||||
if (!beforeKnown)
|
||||
before = await WalkBackAsync(ctx, searchEnd, walkKnown, walkWidth).ConfigureAwait(false);
|
||||
beforeKnown = true;
|
||||
}
|
||||
prev = before;
|
||||
}
|
||||
|
||||
StoredSample? next = null;
|
||||
if (NeedsNext(t, prev) && t < horizon)
|
||||
{
|
||||
if (j + 1 < rows.Count) next = rows[j + 1];
|
||||
else if (probeFrom < 0) probeFrom = k; // after every row of a complete read: the probe answers it
|
||||
}
|
||||
result[t.Ticks] = (prev, next);
|
||||
}
|
||||
|
||||
if (k == times.Length) break;
|
||||
|
||||
// Truncated before times[k]: continue from the last row, or jump to times[k] with a density-sized margin and a
|
||||
// small budget (it needs that time's neighbours, and whatever later times follow closely).
|
||||
var tn = Min(times[k], horizon);
|
||||
var spanTicks = Math.Max(1, (x - rows[0].TimeUtc).Ticks);
|
||||
var density = (double)rows.Count / spanTicks; // rows per tick
|
||||
var gapRows = density * (tn - x).Ticks;
|
||||
var margin = Clamp(TimeSpan.FromTicks((long)Math.Min(TimeSpan.MaxValue.Ticks / 2.0, JumpRows / 4.0 / Math.Max(density, 1e-18))));
|
||||
var resume = MsFloor(x) - OneMs;
|
||||
var jump = gapRows >= ReadRows / 2.0;
|
||||
var nextQs = jump ? Max(RequestStart(tn - Min(margin, tn - Floor)), resume) : resume;
|
||||
if (nextQs <= qs) throw TooDense(ctx, x);
|
||||
|
||||
if (nextQs < x)
|
||||
{
|
||||
// Every row before x was seen: the latest at or before nextQs is known.
|
||||
var lastIndex = rows.FindLastIndex(r => r.TimeUtc <= nextQs);
|
||||
if (lastIndex >= 0)
|
||||
{
|
||||
before = rows[lastIndex];
|
||||
beforeKnown = true;
|
||||
}
|
||||
walkKnown = null;
|
||||
}
|
||||
else
|
||||
{
|
||||
beforeKnown = false;
|
||||
before = null;
|
||||
walkKnown = x;
|
||||
walkWidth = margin;
|
||||
}
|
||||
budget = jump ? JumpRows : ReadRows;
|
||||
qs = nextQs;
|
||||
i = k;
|
||||
}
|
||||
|
||||
if (probeFrom >= 0 && qe < horizon)
|
||||
{
|
||||
var after = await FirstAfterAsync(ctx, qe - OneMs, horizon, Min(times[^1], horizon)).ConfigureAwait(false);
|
||||
if (after is not null)
|
||||
for (var k = probeFrom; k < times.Length; k++)
|
||||
{
|
||||
var t = times[k];
|
||||
var (prev, next) = result[t.Ticks];
|
||||
if (next is null && t < horizon && NeedsNext(t, prev) && after.Value.TimeUtc > t)
|
||||
result[t.Ticks] = (prev, after);
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/// <summary>The evaluator reads <c>next</c> only for a usable <c>prev</c> that is not an exact hit.</summary>
|
||||
private static bool NeedsNext(DateTime t, StoredSample? prev) => prev is { IsUsable: true } p && p.TimeUtc != t;
|
||||
|
||||
/// <summary>One walk-back segment <c>(lo, hi]</c>: its oldest and newest rows, and whether the read was complete.</summary>
|
||||
private static async Task<(StoredSample? First, StoredSample? Last, bool Complete)> SegmentAsync(
|
||||
Ctx ctx, DateTime lo, DateTime hi)
|
||||
{
|
||||
var read = await ReadTrustedAsync(ctx, MsFloor(lo), MsFloor(hi) + OneMs, ReadRows).ConfigureAwait(false);
|
||||
StoredSample? first = null, last = null;
|
||||
foreach (var r in read.Rows)
|
||||
if (r.TimeUtc > lo && r.TimeUtc <= hi)
|
||||
{
|
||||
first ??= r;
|
||||
last = r;
|
||||
}
|
||||
return (first, last, read.Complete);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The latest stored sample with time in <c>(Floor, hi]</c> (see the type remarks: gallop, then bisect).
|
||||
/// <paramref name="known"/>: a stored sample exists at that time (≤ <paramref name="hi"/>) — the walk then bisects
|
||||
/// above it straight away, its first segment aimed at <paramref name="width"/>.
|
||||
/// </summary>
|
||||
private static async Task<StoredSample?> WalkBackAsync(Ctx ctx, DateTime hi, DateTime? known, TimeSpan width)
|
||||
{
|
||||
TimeSpan? aim = known is null ? null : width;
|
||||
|
||||
// Gallop: no stored row known below hi yet.
|
||||
var empties = 0;
|
||||
while (known is null)
|
||||
{
|
||||
if (hi <= Floor) return null;
|
||||
var rest = empties >= 2;
|
||||
var lo = rest || (hi - Floor).Ticks <= width.Ticks ? Floor : hi - width;
|
||||
var seg = await SegmentAsync(ctx, lo, hi).ConfigureAwait(false);
|
||||
if (seg.Complete)
|
||||
{
|
||||
if (seg.Last is { } found) return found;
|
||||
if (lo <= Floor) return null;
|
||||
hi = lo;
|
||||
empties++;
|
||||
width = Grow(width);
|
||||
continue;
|
||||
}
|
||||
// Full — a whole-remainder read of deep history too: its last row is a stored row below hi. Bisect above it.
|
||||
if (seg.Last is not { } seen) throw TooDense(ctx, lo);
|
||||
known = seen.TimeUtc;
|
||||
aim = AimFrom(seg.First!.Value.TimeUtc, seen.TimeUtc);
|
||||
}
|
||||
|
||||
// Bisect: a stored row exists at `known`, nothing newer than `hi` matters; the answer is the newest row in [known, hi].
|
||||
var aimMissed = false;
|
||||
while (true)
|
||||
{
|
||||
var k = known.Value;
|
||||
DateTime lo;
|
||||
var aimed = false;
|
||||
if (hi - k < TimeSpan.FromMilliseconds(2))
|
||||
lo = k.AddTicks(-1); // the whole stretch from the known row: never a sub-millisecond step
|
||||
else
|
||||
{
|
||||
lo = k + TimeSpan.FromTicks((hi - k).Ticks / 2);
|
||||
if (aim is { } a && hi - a > lo)
|
||||
{
|
||||
lo = hi - a; // an aimed try right after a full read — until one comes back empty (a gap: halve from then on)
|
||||
aimed = true;
|
||||
}
|
||||
}
|
||||
aim = null;
|
||||
|
||||
var seg = await SegmentAsync(ctx, lo, hi).ConfigureAwait(false);
|
||||
if (seg.Complete)
|
||||
{
|
||||
if (seg.Last is { } found) return found;
|
||||
if (lo < k) return null; // the known row is gone (deleted meanwhile): nothing is left to find
|
||||
hi = lo;
|
||||
aimMissed |= aimed;
|
||||
continue;
|
||||
}
|
||||
if (seg.Last is not { } seen || seen.TimeUtc <= k) throw TooDense(ctx, k);
|
||||
known = seen.TimeUtc;
|
||||
if (!aimMissed) aim = AimFrom(seg.First!.Value.TimeUtc, seen.TimeUtc);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>About a quarter of a read at the density a full read showed (its rows from <paramref name="first"/> to <paramref name="last"/>).</summary>
|
||||
private static TimeSpan AimFrom(DateTime first, DateTime last) =>
|
||||
TimeSpan.FromTicks(Math.Max(Ms, (last - first).Ticks / 4));
|
||||
|
||||
/// <summary>The first stored sample after <paramref name="after"/> in <c>(qs, end)</c>, from a small forward read.</summary>
|
||||
private static async Task<StoredSample?> FirstAfterAsync(Ctx ctx, DateTime qs, DateTime end, DateTime after)
|
||||
{
|
||||
foreach (var budget in new[] { ProbeRows, ReadRows })
|
||||
{
|
||||
var read = await ReadTrustedAsync(ctx, qs, end, budget).ConfigureAwait(false);
|
||||
foreach (var r in read.Rows)
|
||||
if (r.TimeUtc > after) return r;
|
||||
if (read.Complete) return null;
|
||||
}
|
||||
throw TooDense(ctx, qs);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// One forward read <c>[qs, qe]</c> (whole milliseconds), drained, keeping only rows strictly inside it and after the
|
||||
/// epoch — counted against the request's read cap. A strict gateway's ceiling is learned on refusal and then asked for
|
||||
/// directly; a full read is not re-probed (the caller pages on, it does not need the rest of this read).
|
||||
/// </summary>
|
||||
private static async Task<Read> ReadTrustedAsync(Ctx ctx, DateTime qs, DateTime qe, int budget)
|
||||
{
|
||||
if (++ctx.Reads > ctx.MaxReads)
|
||||
throw new HistoryReadLimitExceededException(ctx.MaxReads,
|
||||
$"The AtTime read of '{ctx.Tag}' needed more than {ctx.MaxReads} raw reads ({BaseReads} + {ReadsPerTime} per " +
|
||||
"requested time)" + CeilingNote(ctx) + "; the Gateway backend serves AtTime from raw reads — request fewer times.");
|
||||
var (read, used) = await ctx.Ceiling.ReadAsync(budget, b => PassAsync(ctx, qs, qe, b), (_, _) => false)
|
||||
.ConfigureAwait(false);
|
||||
if (used < ctx.SmallestBudget) ctx.SmallestBudget = used;
|
||||
if (used < MinBudget)
|
||||
throw new HistoryReadLimitExceededException(used,
|
||||
$"The HistorianGateway's row ceiling ({used}, its Historian:ReadLimits:MaxValuesCeiling with " +
|
||||
$"AllowStreamingBeyondCeiling=false) is below the {MinBudget} rows an AtTime raw read of '{ctx.Tag}' needs.");
|
||||
return read;
|
||||
}
|
||||
|
||||
private static string CeilingNote(Ctx ctx) =>
|
||||
ctx.SmallestBudget < JumpRows
|
||||
? $", with reads bounded by the HistorianGateway's row ceiling of {ctx.SmallestBudget} (Historian:ReadLimits:MaxValuesCeiling)"
|
||||
: string.Empty;
|
||||
|
||||
private static async Task<Read> PassAsync(Ctx ctx, DateTime qs, DateTime qe, int budget)
|
||||
{
|
||||
var rows = new List<StoredSample>();
|
||||
var received = 0;
|
||||
await foreach (var raw in ctx.Client.ReadRawAsync(ctx.Tag, qs, qe, budget, ctx.Ct).ConfigureAwait(false))
|
||||
{
|
||||
received++; // drained to completion, never broken out of
|
||||
if (raw.Timestamp is null) continue;
|
||||
var t = raw.Timestamp.ToDateTime();
|
||||
if (t > qs && t < qe && t > Floor) rows.Add(ToStored(raw, t, ctx.ValueType));
|
||||
}
|
||||
return new Read(rows, received < budget);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A gateway row as a stored sample of the node's type (see the type remarks); a Bad row carries no value (#690: a
|
||||
/// stored NULL arrives as value 0).
|
||||
/// </summary>
|
||||
private static StoredSample ToStored(HistorianSample raw, DateTime t, TypeCode? valueType)
|
||||
{
|
||||
var status = SampleMapper.ToStatus(raw);
|
||||
var value = StatusBits.IsBad(status) ? HistorianValue.Empty
|
||||
: raw.HasNumericValue ? Typed(raw.NumericValue, valueType)
|
||||
: raw.HasStringValue ? HistorianValue.FromString(raw.StringValue)
|
||||
: HistorianValue.Empty;
|
||||
return StoredSample.FromStored(t, value, status);
|
||||
}
|
||||
|
||||
/// <summary>A gateway number as the hinted kind, so the shared evaluator's own type branch decides stepped vs linear.</summary>
|
||||
private static HistorianValue Typed(double v, TypeCode? valueType) => valueType switch
|
||||
{
|
||||
TypeCode.Boolean => HistorianValue.FromBoolean(v != 0),
|
||||
TypeCode.String => HistorianValue.FromString(v.ToString("R", System.Globalization.CultureInfo.InvariantCulture)),
|
||||
// In Int64 range only (2^63 is exactly representable); NaN, ±∞ or out of range is no integer at all — Empty, which
|
||||
// StoredSample.FromStored makes BadNoData (unusable), never a saturated Good value.
|
||||
_ when IsInteger(valueType) => v >= -9.2233720368547758E18 && v < 9.2233720368547758E18
|
||||
? HistorianValue.FromInt64((long)Math.Round(v, MidpointRounding.AwayFromZero))
|
||||
: HistorianValue.Empty,
|
||||
_ => HistorianValue.FromDouble(v),
|
||||
};
|
||||
|
||||
/// <summary>The window read's start for a time: a millisecond before its millisecond (never before the epoch).</summary>
|
||||
private static DateTime RequestStart(DateTime t)
|
||||
{
|
||||
var s = MsFloor(t) - TimeSpan.FromMilliseconds(1);
|
||||
return s < Floor ? Floor : s;
|
||||
}
|
||||
|
||||
private static DateTime MsFloor(DateTime t) => new(t.Ticks - t.Ticks % Ms, DateTimeKind.Utc);
|
||||
|
||||
private static DateTime Min(DateTime a, DateTime b) => a <= b ? a : b;
|
||||
|
||||
private static DateTime Max(DateTime a, DateTime b) => a >= b ? a : b;
|
||||
|
||||
private static TimeSpan Min(TimeSpan a, TimeSpan b) => a <= b ? a : b;
|
||||
|
||||
private static TimeSpan Clamp(TimeSpan w) =>
|
||||
w < TimeSpan.FromMilliseconds(1) ? TimeSpan.FromMilliseconds(1) : w > MaxSeededWidth ? MaxSeededWidth : w;
|
||||
|
||||
private static TimeSpan Grow(TimeSpan w) => w.Ticks > TimeSpan.MaxValue.Ticks / 4 ? TimeSpan.MaxValue : w * 4;
|
||||
|
||||
private static HistoryReadLimitExceededException TooDense(Ctx ctx, DateTime at)
|
||||
{
|
||||
var rows = Math.Min(ReadRows, ctx.SmallestBudget);
|
||||
return new(rows,
|
||||
$"The AtTime read of '{ctx.Tag}' met more than {rows} stored samples within about one millisecond of {at:O}" +
|
||||
(rows < ReadRows ? $" (reads are bounded by the HistorianGateway's row ceiling of {rows})" : string.Empty) +
|
||||
"; the Gateway backend serves AtTime from raw reads.");
|
||||
}
|
||||
}
|
||||
+259
-45
@@ -46,6 +46,30 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
private readonly ILogger<GatewayHistorianDataSource> _logger;
|
||||
private readonly int _maxRawRows;
|
||||
private readonly TimeProvider _time;
|
||||
private readonly bool _atTimeFromRaw;
|
||||
|
||||
/// <summary>#693: AtTime from raw reads (true) or AVEVA's own interpolation (the kill switch).</summary>
|
||||
internal bool AtTimeFromRaw => _atTimeFromRaw;
|
||||
|
||||
/// <summary>The most #687 crowding re-reads one raw read makes (#694 F4): each at least doubles the pre-window room.</summary>
|
||||
private const int MaxCrowdedReReads = 8;
|
||||
|
||||
/// <summary>The default of <c>maxPreWindowRows</c>: <c>ServerHistorian:MaxTieClusterOverfetch</c>'s default.</summary>
|
||||
public const int DefaultMaxPreWindowRows = 65536;
|
||||
|
||||
/// <summary>
|
||||
/// The most rows before a raw window (inside the 1 ms look-back) a crowding re-read makes room for (#694 R2) — the
|
||||
/// server's tie-cluster limit (<c>ServerHistorian:MaxTieClusterOverfetch</c>), which every other tie path treats as
|
||||
/// the limit. More ⇒ <see cref="HistoryReadLimitExceededException"/> with <c>PreWindowRows</c> set.
|
||||
/// </summary>
|
||||
private readonly int _maxPreWindowRows;
|
||||
|
||||
/// <summary>The tie-cluster limit the crowding re-reads honour (#694 R2) — exposed so the factory wiring is testable.</summary>
|
||||
internal int MaxPreWindowRows => _maxPreWindowRows;
|
||||
|
||||
/// <summary>A strict gateway's row ceiling, learned from its first refusal (#691.3).</summary>
|
||||
private readonly GatewayRowCeiling _ceiling;
|
||||
private readonly GatewayTagPresenceCache _tags;
|
||||
|
||||
private readonly object _healthLock = new();
|
||||
private DateTime? _lastSuccessUtc;
|
||||
@@ -64,15 +88,28 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
/// <= 0 = unbounded). Past it the node fails <c>BadTooManyOperations</c>.
|
||||
/// </param>
|
||||
/// <param name="timeProvider">Clock for result server timestamps and the raw look-ahead horizon; null = system.</param>
|
||||
/// <param name="atTimeFromRaw">
|
||||
/// #693: serve AtTime from raw reads with the shared AtTime rules (<see cref="GatewayAtTimeReader"/>; default);
|
||||
/// <c>false</c> = AVEVA's own interpolation passed through (<c>ServerHistorian:GatewayAtTimeFromRaw</c> kill switch).
|
||||
/// </param>
|
||||
/// <param name="maxPreWindowRows">
|
||||
/// The most rows before a raw window a crowding re-read makes room for (#694 R2; <c>ServerHistorian:MaxTieClusterOverfetch</c>,
|
||||
/// values below 1 count as 1). More ⇒ <see cref="HistoryReadLimitExceededException"/> (<c>BadTooManyOperations</c>).
|
||||
/// </param>
|
||||
public GatewayHistorianDataSource(IHistorianGatewayClient client, ILogger<GatewayHistorianDataSource> logger,
|
||||
int maxRawRows = TimeWeightedAggregator.DefaultMaxWindowRows, TimeProvider? timeProvider = null)
|
||||
int maxRawRows = TimeWeightedAggregator.DefaultMaxWindowRows, TimeProvider? timeProvider = null,
|
||||
bool atTimeFromRaw = true, int maxPreWindowRows = DefaultMaxPreWindowRows)
|
||||
{
|
||||
_maxPreWindowRows = Math.Max(1, maxPreWindowRows);
|
||||
_atTimeFromRaw = atTimeFromRaw;
|
||||
ArgumentNullException.ThrowIfNull(client);
|
||||
ArgumentNullException.ThrowIfNull(logger);
|
||||
_client = client;
|
||||
_logger = logger;
|
||||
_maxRawRows = maxRawRows;
|
||||
_time = timeProvider ?? TimeProvider.System;
|
||||
_ceiling = new GatewayRowCeiling(_time);
|
||||
_tags = new GatewayTagPresenceCache(_client, _time);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
@@ -92,6 +129,11 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
|
||||
try
|
||||
{
|
||||
if (!await _tags.ExistsAsync(fullReference, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
RecordOutcome(success: true, error: null);
|
||||
return new HistoryReadResult([], ContinuationPoint: null);
|
||||
}
|
||||
var samples = await ReadRawWindowAsync(fullReference, startUtc, endUtc, maxValuesPerNode, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
var snapshots = SampleMapper.ToSnapshots(samples);
|
||||
@@ -118,6 +160,7 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
catch (Exception ex)
|
||||
{
|
||||
RecordReadFailure(ex);
|
||||
if (IsTransportFailure(ex)) throw new System.IO.IOException("HistorianGateway read transport is unavailable.", ex);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
@@ -144,8 +187,12 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
/// gateway reject <c>max_values <= 0</c>, #686). A cap asks for <c>cap + </c>
|
||||
/// <see cref="RawAggregateReader.PreWindowAllowance"/> rows — NOT clamped at the gateway's default ceiling (#688
|
||||
/// clamps only the per-sample aggregate budget): AVEVA always emits the boundary row, so a clamped budget would make
|
||||
/// every page size within the allowance of the ceiling come back short. A strict gateway refuses an over-ceiling
|
||||
/// budget instead (⇒ <see cref="HistoryReadLimitExceededException"/>, <c>BadTooManyOperations</c>).
|
||||
/// every page size within the allowance of the ceiling come back short. A strict gateway
|
||||
/// (<c>AllowStreamingBeyondCeiling=false</c>) refuses an over-ceiling budget and names its ceiling; the read is
|
||||
/// then re-issued ONCE with the ceiling as budget (#691.3, <see cref="GatewayReadLimits"/>) — so an uncapped read
|
||||
/// of a small window is served — and a ceiling-bounded read that is not provably complete (it delivered the
|
||||
/// whole ceiling without reaching the window end, or a page short of its cap) fails
|
||||
/// <see cref="HistoryReadLimitExceededException"/> (<c>BadTooManyOperations</c>), never a truncated answer.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Completeness.</b> A full page must hold exactly the cap of stored samples — a short page ends the client's
|
||||
@@ -166,22 +213,71 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
// The gateway seam caps with an int; OPC UA hands us a uint, so clamp to int range.
|
||||
var cap = (int)Math.Min(maxValuesPerNode, int.MaxValue);
|
||||
if (cap == 0)
|
||||
return (await ReadRawPassAsync(fullReference, startUtc, endUtc, cap, int.MaxValue, cancellationToken)
|
||||
.ConfigureAwait(false)).Samples;
|
||||
{
|
||||
var all = await PassAsync(int.MaxValue).ConfigureAwait(false);
|
||||
if (all.Result.Complete) return all.Result.Samples;
|
||||
// Only reachable on a strict gateway (#691.3): the ceiling re-read delivered the whole ceiling without reaching
|
||||
// the window end — the window may hold more, and "all values" is never answered from a truncated read.
|
||||
throw new HistoryReadLimitExceededException(cap,
|
||||
$"The raw read of '{fullReference}' holds more than the HistorianGateway's row ceiling ({all.Budget}, its " +
|
||||
"Historian:ReadLimits:MaxValuesCeiling with AllowStreamingBeyondCeiling=false); set NumValuesPerNode to page, " +
|
||||
"or narrow the window.")
|
||||
{ PreWindowRows = Crowding(all.Result.PreWindowRows) };
|
||||
}
|
||||
|
||||
var first = await ReadRawPassAsync(fullReference, startUtc, endUtc, cap,
|
||||
(int)Math.Min(int.MaxValue, (long)cap + RawAggregateReader.PreWindowAllowance), cancellationToken).ConfigureAwait(false);
|
||||
if (first.Complete) return first.Samples;
|
||||
var asked = (int)Math.Min(int.MaxValue, (long)cap + RawAggregateReader.PreWindowAllowance);
|
||||
var pass = await PassAsync(asked).ConfigureAwait(false);
|
||||
if (pass.Result.Complete) return pass.Result.Samples;
|
||||
|
||||
// The margin crowded the window out: one bounded re-read with room for the pre-window rows already seen.
|
||||
var second = await ReadRawPassAsync(fullReference, startUtc, endUtc, cap,
|
||||
(int)Math.Min(int.MaxValue, (long)cap + first.PreWindowRows + RawAggregateReader.PreWindowAllowance),
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
if (second.Complete) return second.Samples;
|
||||
// A strict gateway's ceiling already bounded that read (#691.3): a larger budget would only be refused again. The
|
||||
// refusal carries the pre-window rows it saw (#694): the server re-reads the page that many values smaller.
|
||||
if (pass.Budget < asked)
|
||||
throw new HistoryReadLimitExceededException(cap,
|
||||
$"The raw read of '{fullReference}' could not be read completely within the HistorianGateway's row ceiling " +
|
||||
$"({pass.Budget}, its Historian:ReadLimits:MaxValuesCeiling with AllowStreamingBeyondCeiling=false); lower " +
|
||||
"NumValuesPerNode or narrow the window.")
|
||||
{ PreWindowRows = Crowding(pass.Result.PreWindowRows) };
|
||||
|
||||
// The margin crowded the window out: re-read with room for the pre-window rows. A read that was itself cut short saw
|
||||
// only some of them (#694 F4: a tie cluster of hundreds just before a resume point used to fail even with no
|
||||
// ceiling), so the room at least doubles per re-read — bounded by MaxCrowdedReReads, and never past a known ceiling.
|
||||
// #694 R2: the room is capped at the tie-cluster limit (MaxTieClusterOverfetch) — a pathological single-millisecond
|
||||
// cluster on a streaming gateway would otherwise be streamed several times over for one page.
|
||||
var preWindow = pass.Result.PreWindowRows;
|
||||
var room = Math.Min(Math.Max(1, preWindow), _maxPreWindowRows);
|
||||
for (var attempt = 0; attempt < MaxCrowdedReReads && preWindow <= _maxPreWindowRows; attempt++)
|
||||
{
|
||||
var budget = (int)Math.Min(int.MaxValue, (long)cap + room + RawAggregateReader.PreWindowAllowance);
|
||||
// #695: doubling can leap over the ceiling while the cluster still fits. Spend the remaining room once.
|
||||
if (_ceiling.Learned is var known and > 0 && budget > known)
|
||||
{
|
||||
var available = known - cap - RawAggregateReader.PreWindowAllowance;
|
||||
if (available <= 0) break;
|
||||
room = Math.Min(room, available);
|
||||
budget = cap + room + RawAggregateReader.PreWindowAllowance;
|
||||
}
|
||||
if (budget <= pass.Budget) break; // no more room, never retry the same full read
|
||||
pass = await PassAsync(budget).ConfigureAwait(false);
|
||||
if (pass.Result.Complete) return pass.Result.Samples;
|
||||
preWindow = Math.Max(preWindow, pass.Result.PreWindowRows);
|
||||
if (pass.Budget < budget) break; // bounded by a ceiling learned meanwhile
|
||||
if (room >= _maxPreWindowRows) break; // the limit's worth of room did not suffice
|
||||
room = (int)Math.Min(_maxPreWindowRows, Math.Max(preWindow, 2L * room));
|
||||
}
|
||||
|
||||
throw new HistoryReadLimitExceededException(cap,
|
||||
$"The raw read of '{fullReference}' could not be read completely: more than {second.PreWindowRows} rows lie in " +
|
||||
$"the {RawAggregateReader.BoundaryMargin.TotalMilliseconds} ms before the window and exhausted the row budget.");
|
||||
$"The raw read of '{fullReference}' could not be read completely: more than {pass.Result.PreWindowRows} rows lie in " +
|
||||
$"the {RawAggregateReader.BoundaryMargin.TotalMilliseconds} ms before the window and exhausted the row budget.")
|
||||
{ PreWindowRows = preWindow };
|
||||
|
||||
// Pre-window rows the allowance did not cover are what crowded the read; fewer are not the cause (#694).
|
||||
static int Crowding(int preWindowRows) => preWindowRows > RawAggregateReader.PreWindowAllowance ? preWindowRows : 0;
|
||||
|
||||
// One drained pass; a strict gateway's refusal of the budget is retried once at the ceiling it names (#691.3).
|
||||
Task<((List<HistorianSample> Samples, int PreWindowRows, bool Complete) Result, int Budget)> PassAsync(int budget) =>
|
||||
_ceiling.ReadAsync(budget,
|
||||
b => ReadRawPassAsync(fullReference, startUtc, endUtc, cap, b, cancellationToken),
|
||||
(r, _) => !r.Complete);
|
||||
}
|
||||
|
||||
/// <summary>One forward raw read of a window, drained, with its pre-window row count and whether it is complete.</summary>
|
||||
@@ -202,11 +298,17 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
{
|
||||
if (cap == 0 || samples.Count < cap) samples.Add(sample);
|
||||
}
|
||||
else if (sample.Timestamp is { } ts && ts.ToDateTime() < startUtc) preWindow++;
|
||||
// A row before the window — or, for a window starting at/before the epoch (no look-back), AVEVA's synthetic row at
|
||||
// the query start, which is no sample either (#694 F1: it used to set reachedEnd, so a ceiling-truncated read
|
||||
// passed as complete) — is a pre-window row.
|
||||
else if (sample.Timestamp is { } ts
|
||||
&& (ts.ToDateTime() < startUtc || !RawAggregateReader.IsAfterFloor(ts.ToDateTime()))) preWindow++;
|
||||
else if (sample.Timestamp is not null) reachedEnd = true;
|
||||
}
|
||||
|
||||
var complete = received < budget || samples.Count >= cap || reachedEnd || cap == 0;
|
||||
// cap == 0 ("all values") is complete only when the stream proves it: short of the budget (always, unless a strict
|
||||
// gateway's ceiling became the budget — #691.3) or past the window end.
|
||||
var complete = received < budget || reachedEnd || (cap > 0 && samples.Count >= cap);
|
||||
return (samples, preWindow, complete);
|
||||
}
|
||||
|
||||
@@ -223,7 +325,9 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
/// <para>
|
||||
/// <b>TimeAverage / Total</b> stay native AVEVA retrieval modes (<see cref="AggregateModeMapper"/>), stamped with
|
||||
/// the interval start and the Calculated bit; AVEVA's extra row for the interval before the window is dropped and
|
||||
/// an Uncertain interval keeps its value (#685, <see cref="SampleMapper.ToAggregateSnapshots"/>). Everything else is <see cref="HistoryAggregateNotSupportedException"/>
|
||||
/// an Uncertain interval keeps its value (#685). Both directions use the SQL backends' interval grid with forward
|
||||
/// requests only — a reverse read (which the gateway refuses) anchored at its later time (#677), the clipped interval
|
||||
/// read over exactly its span and marked Partial (#691.2, <see cref="NativeAggregatePlan"/>). Everything else is <see cref="HistoryAggregateNotSupportedException"/>
|
||||
/// (see <see cref="ServedAggregates"/>).
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
@@ -240,22 +344,41 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
// Mapped OUTSIDE the failure-recording try: an aggregate the gateway has no mode for is a client
|
||||
// request the server answers BadAggregateNotSupported, not a backend fault to count (#285).
|
||||
var mode = AggregateModeMapper.ToRetrievalMode(aggregate);
|
||||
// Planned OUTSIDE the failure-recording try: too many intervals is a client request limit, not a backend fault.
|
||||
return await ReadNativeAsync(fullReference, BucketPlan.Create(startUtc, endUtc, interval), mode, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A native TimeAverage / Total read on the SQL backends' interval grid, either direction (#691.2): forward native
|
||||
/// requests only, the clipped interval read over exactly its span. See <see cref="NativeAggregatePlan"/>.
|
||||
/// </summary>
|
||||
private async Task<HistoryReadResult> ReadNativeAsync(string fullReference, BucketPlan plan, RetrievalMode mode,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (plan.BucketCount == 0) return new HistoryReadResult([], ContinuationPoint: null);
|
||||
try
|
||||
{
|
||||
// TimeAverage / Total are native gateway retrieval modes (TimeWeightedAverage / Integral) — no
|
||||
// client-side scaling.
|
||||
var buckets = new List<HistorianAggregateSample>();
|
||||
await foreach (var bucket in _client
|
||||
.ReadAggregateAsync(fullReference, startUtc, endUtc, mode, interval, cancellationToken)
|
||||
.ConfigureAwait(false))
|
||||
if (!await _tags.ExistsAsync(fullReference, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
buckets.Add(bucket);
|
||||
RecordOutcome(success: true, error: null);
|
||||
return new HistoryReadResult(plan.Assemble(new Dictionary<long, DataValueSnapshot>(), t => AggregateEvaluator.NoData(t, _time.GetUtcNow().UtcDateTime)), ContinuationPoint: null);
|
||||
}
|
||||
var byBucket = new Dictionary<long, DataValueSnapshot>();
|
||||
foreach (var request in NativeAggregatePlan.Requests(plan))
|
||||
{
|
||||
var buckets = new List<HistorianAggregateSample>();
|
||||
await foreach (var bucket in _client
|
||||
.ReadAggregateAsync(fullReference, request.StartUtc, request.EndUtc, mode, request.Interval, cancellationToken)
|
||||
.ConfigureAwait(false))
|
||||
{
|
||||
buckets.Add(bucket); // drained to completion, mapped afterwards
|
||||
}
|
||||
NativeAggregatePlan.Place(plan, request, buckets, byBucket);
|
||||
}
|
||||
|
||||
// Filtered while mapping, after the stream was drained (#685: AVEVA's extra pre-window row).
|
||||
var snapshots = SampleMapper.ToAggregateSnapshots(buckets, startUtc, endUtc);
|
||||
RecordOutcome(success: true, error: null);
|
||||
return new HistoryReadResult(snapshots, ContinuationPoint: null);
|
||||
return new HistoryReadResult(NativeAggregatePlan.Assemble(plan, byBucket), ContinuationPoint: null);
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
@@ -264,6 +387,7 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
catch (Exception ex)
|
||||
{
|
||||
RecordReadFailure(ex);
|
||||
if (IsTransportFailure(ex)) throw new System.IO.IOException("HistorianGateway read transport is unavailable.", ex);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
@@ -279,8 +403,13 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
try
|
||||
{
|
||||
var now = _time.GetUtcNow().UtcDateTime;
|
||||
if (plan.BucketCount > 0 && !await _tags.ExistsAsync(fullReference, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
RecordOutcome(success: true, error: null);
|
||||
return new HistoryReadResult(plan.Assemble(new Dictionary<long, DataValueSnapshot>(), t => AggregateEvaluator.NoData(t, now)), ContinuationPoint: null);
|
||||
}
|
||||
var values = await RawAggregateReader
|
||||
.ReadAsync(_client, fullReference, aggregate, plan, _maxRawRows, now, cancellationToken)
|
||||
.ReadAsync(_client, _ceiling, fullReference, aggregate, plan, _maxRawRows, now, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
RecordOutcome(success: true, error: null);
|
||||
return new HistoryReadResult(values, ContinuationPoint: null);
|
||||
@@ -307,6 +436,7 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
catch (Exception ex)
|
||||
{
|
||||
RecordReadFailure(ex);
|
||||
if (IsTransportFailure(ex)) throw new System.IO.IOException("HistorianGateway read transport is unavailable.", ex);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
@@ -315,21 +445,78 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
/// True when <paramref name="ex"/> (or an inner exception — the client package wraps the <see cref="RpcException"/>)
|
||||
/// is the gateway's <c>InvalidArgument</c> "max_values exceeds the server ceiling" rejection.
|
||||
/// </summary>
|
||||
internal static bool IsMaxValuesCeilingRejection(Exception ex)
|
||||
{
|
||||
for (var e = ex; e is not null; e = e.InnerException)
|
||||
if (e is RpcException { StatusCode: StatusCode.InvalidArgument } rpc
|
||||
&& rpc.Status.Detail.Contains("max_values exceeds the server ceiling", StringComparison.Ordinal))
|
||||
return true;
|
||||
return false;
|
||||
}
|
||||
internal static bool IsMaxValuesCeilingRejection(Exception ex) => GatewayReadLimits.IsCeilingRejection(ex);
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>
|
||||
/// #693: served from RAW gateway reads — each requested time's stored sample at-or-before and after it — evaluated with
|
||||
/// the SQL backends' shared <see cref="AtTimeEvaluator"/> (Part 11 §6.5.4), so the Gateway answers what SqlServer and
|
||||
/// QuestDB answer for the same stored samples; see <see cref="GatewayAtTimeReader"/> for the read plan and its cost.
|
||||
/// AVEVA's own interpolated answer (the gateway's <c>ReadAtTime</c>) returned the previous value as Good at a stored
|
||||
/// NULL's time, null + Bad at a Good sample right after a NULL, a drifted value at an exact stored sample, and no
|
||||
/// Interpolated bit; it is only used when <c>ServerHistorian:GatewayAtTimeFromRaw = false</c> (kill switch).
|
||||
/// A gateway value is a Double (or a String) whatever the node's type (AVEVA stores a Boolean as a 0/1 <c>Int2</c>), so
|
||||
/// without a type hint every numeric tag is interpolated linearly; the node manager passes the node's type through
|
||||
/// <see cref="ReadAtTimeAsync(string, IReadOnlyList{DateTime}, TypeCode?, CancellationToken)"/>.
|
||||
/// </remarks>
|
||||
public Task<HistoryReadResult> ReadAtTimeAsync(
|
||||
string fullReference, IReadOnlyList<DateTime> timestampsUtc, CancellationToken cancellationToken) =>
|
||||
ReadAtTimeAsync(fullReference, timestampsUtc, valueType: null, cancellationToken);
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>
|
||||
/// #693: the hint makes the AtTime rules type-exact, as on the SQL backends — the gateway's Double rows are converted to
|
||||
/// the node's type BEFORE the shared evaluator runs (Boolean = value ≠ 0 ⇒ stepped; an integer type ⇒ Int64 ⇒
|
||||
/// interpolated and rounded half away from zero, then returned as the node's integer type; String ⇒ stepped; Float /
|
||||
/// Double / no hint ⇒ linear Double). Ignored with the kill switch (AVEVA's own answer).
|
||||
/// </remarks>
|
||||
public async Task<HistoryReadResult> ReadAtTimeAsync(
|
||||
string fullReference, IReadOnlyList<DateTime> timestampsUtc, TypeCode? valueType, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(timestampsUtc);
|
||||
if (!_atTimeFromRaw) return await ReadAtTimeNativeAsync(fullReference, timestampsUtc, cancellationToken).ConfigureAwait(false);
|
||||
try
|
||||
{
|
||||
var now = _time.GetUtcNow().UtcDateTime;
|
||||
if (timestampsUtc.Count > 0 && !await _tags.ExistsAsync(fullReference, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
RecordOutcome(success: true, error: null);
|
||||
return MissingAtTime(timestampsUtc, now);
|
||||
}
|
||||
var values = await GatewayAtTimeReader
|
||||
.ReadAsync(_client, _ceiling, fullReference, timestampsUtc, now, cancellationToken, valueType)
|
||||
.ConfigureAwait(false);
|
||||
RecordOutcome(success: true, error: null);
|
||||
return new HistoryReadResult(values, ContinuationPoint: null);
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
throw; // caller-requested, not a backend fault
|
||||
}
|
||||
catch (HistoryReadLimitExceededException)
|
||||
{
|
||||
throw; // a client-request limit (a pathological tie cluster), not a backend fault
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
RecordReadFailure(ex);
|
||||
if (IsTransportFailure(ex)) throw new System.IO.IOException("HistorianGateway read transport is unavailable.", ex);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>The pre-#693 AtTime: AVEVA's own interpolation, aligned to the request (kill switch only).</summary>
|
||||
private async Task<HistoryReadResult> ReadAtTimeNativeAsync(
|
||||
string fullReference, IReadOnlyList<DateTime> timestampsUtc, CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (timestampsUtc.Count > 0 && !await _tags.ExistsAsync(fullReference, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
RecordOutcome(success: true, error: null);
|
||||
return MissingAtTime(timestampsUtc, _time.GetUtcNow().UtcDateTime);
|
||||
}
|
||||
|
||||
var samples = await _client
|
||||
.ReadAtTimeAsync(fullReference, timestampsUtc, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
@@ -340,6 +527,7 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
catch (Exception ex)
|
||||
{
|
||||
RecordReadFailure(ex);
|
||||
if (IsTransportFailure(ex)) throw new System.IO.IOException("HistorianGateway read transport is unavailable.", ex);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
@@ -354,8 +542,20 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>
|
||||
/// <c>false</c>: the gateway's <c>ReadEvents</c> streams ascending only (Gitea #675). The server therefore refuses an
|
||||
/// explicit reverse events read without calling, and keeps an end-only capped read forward.
|
||||
/// Gitea #694: <c>0</c> until a strict gateway (<c>AllowStreamingBeyondCeiling=false</c>) has refused a budget and
|
||||
/// named its ceiling (#691.3, <see cref="GatewayRowCeiling"/>); then that ceiling less the
|
||||
/// <see cref="RawAggregateReader.PreWindowAllowance"/> a capped read adds for AVEVA's boundary row — so a page of this
|
||||
/// size asks for exactly the ceiling. Back to <c>0</c> when a re-probe finds the gateway streaming again.
|
||||
/// </remarks>
|
||||
public int MaxRawValuesPerRead => _ceiling.Learned is var c and > 0
|
||||
? Math.Max(1, c - RawAggregateReader.PreWindowAllowance)
|
||||
: 0;
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>
|
||||
/// <c>false</c>: the gateway's <c>ReadEvents</c> streams ascending only (Gitea #675). The server never sends it a
|
||||
/// reverse read; since #691.4 it walks one back from bounded forward reads (ties sorted by EventId — the gateway
|
||||
/// orders by event time only).
|
||||
/// </remarks>
|
||||
public bool SupportsReverseEvents => false;
|
||||
|
||||
@@ -365,10 +565,9 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>
|
||||
/// A reverse (newest-first) read is refused with <see cref="NotSupportedException"/> — the server reports
|
||||
/// <c>BadHistoryOperationUnsupported</c> (Gitea #675). The gateway's <c>ReadEvents</c> streams ascending only and
|
||||
/// rejects <c>end < start</c>, so the only way to serve "the latest N" would be to drain the whole window and
|
||||
/// keep its tail, an unbounded read the cap exists to prevent. Nothing is sent.
|
||||
/// A reverse (newest-first) read is refused with <see cref="NotSupportedException"/> (Gitea #675) — the gateway's
|
||||
/// <c>ReadEvents</c> streams ascending only and rejects <c>end < start</c>. The server never calls this with
|
||||
/// <c>reverse</c>: it walks a reverse read back from bounded forward reads (#691.4). Nothing is sent.
|
||||
/// </remarks>
|
||||
public Task<HistoricalEventsResult> ReadEventsAsync(
|
||||
string? sourceName, DateTime startUtc, DateTime endUtc, int maxEvents, bool reverse,
|
||||
@@ -386,7 +585,8 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
try
|
||||
{
|
||||
var hasCap = maxEvents > 0;
|
||||
var collected = new List<HistoricalEvent>(hasCap ? maxEvents : 0);
|
||||
// Bounded initial capacity: an uncapped read arrives as int.MaxValue ("Array dimensions exceeded", #691.4 review).
|
||||
var collected = new List<HistoricalEvent>(hasCap ? Math.Min(maxEvents, 1024) : 0);
|
||||
var truncated = false;
|
||||
|
||||
await foreach (var wireEvent in _client
|
||||
@@ -423,6 +623,7 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
catch (Exception ex)
|
||||
{
|
||||
RecordReadFailure(ex);
|
||||
if (IsTransportFailure(ex)) throw new System.IO.IOException("HistorianGateway read transport is unavailable.", ex);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
@@ -454,7 +655,7 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
|
||||
/// <summary>
|
||||
/// Reconciles a gateway at-time reply against the requested timestamps to honour the
|
||||
/// <see cref="IHistorianDataSource.ReadAtTimeAsync"/> contract: exactly one snapshot per
|
||||
/// <see cref="IHistorianDataSource.ReadAtTimeAsync(string, IReadOnlyList{DateTime}, CancellationToken)"/> contract: exactly one snapshot per
|
||||
/// requested timestamp, in request order. Returned samples are indexed by timestamp ticks;
|
||||
/// any requested timestamp the gateway did not return is filled with a Bad-quality
|
||||
/// (<c>0x80000000</c>) snapshot stamped at the requested time rather than positionally
|
||||
@@ -497,6 +698,19 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
return result;
|
||||
}
|
||||
|
||||
/// <summary>No-data values for a missing tag, stamped with the client's requested times.</summary>
|
||||
private static HistoryReadResult MissingAtTime(IReadOnlyList<DateTime> times, DateTime now) =>
|
||||
new(times.Select(t => AtTimeEvaluator.Evaluate(t, null, null, now, now)).ToArray(), ContinuationPoint: null);
|
||||
|
||||
/// <summary>Classifies actual gRPC/package transport failures without treating refusals as outages (#697).</summary>
|
||||
private static bool IsTransportFailure(Exception ex) => ex switch
|
||||
{
|
||||
ZB.MOM.WW.HistorianGateway.Client.HistorianGatewayUnavailableException => true,
|
||||
RpcException rpc => rpc.StatusCode is Grpc.Core.StatusCode.Unavailable or Grpc.Core.StatusCode.DeadlineExceeded,
|
||||
TimeoutException => true,
|
||||
_ => ex.InnerException is { } inner && IsTransportFailure(inner),
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// Records a failed read: bumps the health counters and logs a generic, redaction-safe
|
||||
/// debug line (no tag, host, or value). The exception itself is rethrown by the caller.
|
||||
|
||||
+5
-1
@@ -40,7 +40,11 @@ public static class GatewayHistorian
|
||||
HistorianGatewayClientAdapter.Create(options, loggerFactory),
|
||||
logger,
|
||||
// #681: the raw-evaluated per-sample aggregates share the SQL backends' raw-row cap.
|
||||
maxRawRows: options.MaxTimeWeightedRawRows);
|
||||
maxRawRows: options.MaxTimeWeightedRawRows,
|
||||
// #693: AtTime from raw reads + the shared evaluator; false = AVEVA's own interpolation (kill switch).
|
||||
atTimeFromRaw: options.GatewayAtTimeFromRaw,
|
||||
// #694 R2: a crowding re-read never makes room for more pre-window rows than the tie-cluster limit.
|
||||
maxPreWindowRows: options.MaxTieClusterOverfetch);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
||||
@@ -0,0 +1,148 @@
|
||||
using System.Globalization;
|
||||
using System.Text.RegularExpressions;
|
||||
using Grpc.Core;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway;
|
||||
|
||||
/// <summary>
|
||||
/// The HistorianGateway's raw-read row ceiling (<c>Historian:ReadLimits:MaxValuesCeiling</c>, default 1,000,000) as seen
|
||||
/// from this side of the wire. With <c>AllowStreamingBeyondCeiling=false</c> (a "strict" gateway) the gateway refuses
|
||||
/// any <c>ReadRaw</c> whose <c>max_values</c> exceeds the ceiling with <c>InvalidArgument</c> — BEFORE it streams a row
|
||||
/// — and names the ceiling in the refusal (<c>"max_values exceeds the server ceiling of {n}; …"</c>). The gateway has
|
||||
/// no RPC that reports the ceiling, so the refusal is how it is learned (#691.3).
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// A read whose budget the gateway refuses is re-issued ONCE with the ceiling as its budget, and the ceiling is
|
||||
/// remembered per data source (<see cref="GatewayRowCeiling"/>); the caller treats a stream that delivered the whole
|
||||
/// ceiling as possibly truncated (never answered from).
|
||||
/// </remarks>
|
||||
internal static class GatewayReadLimits
|
||||
{
|
||||
private const string CeilingRejectionText = "max_values exceeds the server ceiling";
|
||||
|
||||
private static readonly Regex CeilingNumber = new(@"max_values exceeds the server ceiling of (\d+)",
|
||||
RegexOptions.CultureInvariant | RegexOptions.Compiled);
|
||||
|
||||
/// <summary>
|
||||
/// True when <paramref name="ex"/> (or an inner exception — the client package wraps the <see cref="RpcException"/>)
|
||||
/// is the gateway's <c>InvalidArgument</c> "max_values exceeds the server ceiling" rejection.
|
||||
/// </summary>
|
||||
/// <param name="ex">The exception.</param>
|
||||
/// <returns>Whether it is the ceiling rejection.</returns>
|
||||
public static bool IsCeilingRejection(Exception ex) => Rejection(ex) is not null;
|
||||
|
||||
/// <summary>
|
||||
/// The ceiling a ceiling rejection names, or — when <paramref name="ex"/> is a ceiling rejection that does not
|
||||
/// carry a parseable number — the gateway's default (<see cref="RawAggregateReader.GatewayDefaultMaxValuesCeiling"/>).
|
||||
/// </summary>
|
||||
/// <param name="ex">The exception.</param>
|
||||
/// <param name="ceiling">The ceiling (> 0) when this returns true.</param>
|
||||
/// <returns>Whether <paramref name="ex"/> is a ceiling rejection.</returns>
|
||||
public static bool TryGetCeiling(Exception ex, out int ceiling)
|
||||
{
|
||||
ceiling = 0;
|
||||
if (Rejection(ex) is not { } rpc) return false;
|
||||
var m = CeilingNumber.Match(rpc.Status.Detail);
|
||||
ceiling = m.Success && int.TryParse(m.Groups[1].Value, NumberStyles.None, CultureInfo.InvariantCulture, out var n) && n > 0
|
||||
? n
|
||||
: RawAggregateReader.GatewayDefaultMaxValuesCeiling;
|
||||
return true;
|
||||
}
|
||||
|
||||
private static RpcException? Rejection(Exception ex)
|
||||
{
|
||||
for (var e = ex; e is not null; e = e.InnerException)
|
||||
if (e is RpcException { StatusCode: StatusCode.InvalidArgument } rpc
|
||||
&& rpc.Status.Detail.Contains(CeilingRejectionText, StringComparison.Ordinal))
|
||||
return rpc;
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A strict gateway's row ceiling as learned by ONE data source (#691.3 review): every refused attempt counts as an
|
||||
/// error in the gateway's own outcome metrics, so the ceiling a refusal names is remembered and later reads ask for at
|
||||
/// most it — no refusal per read in the steady state. Thread-safe (one <see cref="int"/>, read/written atomically; a
|
||||
/// race costs at most one extra refusal or re-probe, never a wrong answer).
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>Rules.</b> (1) Unknown ceiling: ask for the full budget; a refusal naming <c>N</c> < budget learns <c>N</c> and
|
||||
/// the read is re-issued once at <c>N</c>. (2) Learned ceiling <c>C</c> below the budget: ask for <c>C</c> — a refusal
|
||||
/// there (the gateway was LOWERED) re-learns its <c>N</c> as in (1). (3) If that <c>C</c>-bounded read came back
|
||||
/// full but not provably complete, the gateway may have been RAISED (or switched to streaming) since <c>C</c> was
|
||||
/// learned, and answering <c>BadTooManyOperations</c> from a stale ceiling would be wrong: the read is re-probed ONCE
|
||||
/// at the full budget — served ⇒ the ceiling is forgotten; refused naming a larger <c>N</c> ⇒ <c>N</c> is learned and
|
||||
/// the read is re-issued at <c>N</c>; refused naming <c>C</c> again ⇒ the <c>C</c>-bounded result stands (no repeat).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Re-probe timing (#696).</b> Raw and AtTime reads re-probe incomplete streams immediately. Per-sample
|
||||
/// aggregate reads opt into a 30-second cooldown after a refusal; a full stream during it remains a request
|
||||
/// limit, never a truncated answer. After the cooldown the next incomplete read detects a raised ceiling.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
internal sealed class GatewayRowCeiling(TimeProvider? timeProvider = null)
|
||||
{
|
||||
private readonly TimeProvider _time = timeProvider ?? TimeProvider.System;
|
||||
private long _lastRefusalTicks;
|
||||
private int _learned; // 0 = unknown / not strict
|
||||
|
||||
/// <summary>The learned ceiling (0 = none).</summary>
|
||||
public int Learned => Volatile.Read(ref _learned);
|
||||
|
||||
/// <summary>Runs a drained read under the rules above.</summary>
|
||||
/// <typeparam name="T">The pass result.</typeparam>
|
||||
/// <param name="budget">The <c>max_values</c> the read wants.</param>
|
||||
/// <param name="pass">One drained read with the given budget.</param>
|
||||
/// <param name="needsMore">True when a pass with the given budget delivered it all without proving completeness.</param>
|
||||
/// <param name="deferRecentReprobe">Aggregate-only opt-in: reuse a refusal for 30 seconds before re-probing an incomplete read.</param>
|
||||
/// <returns>The result and the budget it was read with.</returns>
|
||||
public async Task<(T Result, int Budget)> ReadAsync<T>(int budget, Func<int, Task<T>> pass, Func<T, int, bool> needsMore, bool deferRecentReprobe = false)
|
||||
{
|
||||
var learned = Learned;
|
||||
if (learned <= 0 || learned >= budget) return await AttemptAsync(budget, pass).ConfigureAwait(false);
|
||||
|
||||
var bounded = await AttemptAsync(learned, pass).ConfigureAwait(false);
|
||||
if (bounded.Budget < learned || !needsMore(bounded.Result, bounded.Budget)) return bounded;
|
||||
|
||||
// #696: aggregate reads may reuse a recent refusal for 30 seconds. Raw paging keeps immediate re-probes.
|
||||
if (deferRecentReprobe && _time.GetUtcNow().UtcTicks - Volatile.Read(ref _lastRefusalTicks) < TimeSpan.FromSeconds(30).Ticks)
|
||||
return bounded;
|
||||
|
||||
// Rule 3: full at a possibly stale ceiling — re-probe once at the full budget.
|
||||
int fresh;
|
||||
try
|
||||
{
|
||||
var full = await pass(budget).ConfigureAwait(false);
|
||||
Interlocked.CompareExchange(ref _learned, 0, learned);
|
||||
return (full, budget);
|
||||
}
|
||||
catch (Exception ex) when (GatewayReadLimits.TryGetCeiling(ex, out fresh) && fresh < budget)
|
||||
{
|
||||
// fall through (outside the catch)
|
||||
}
|
||||
Learn(fresh);
|
||||
return fresh > learned ? (await pass(fresh).ConfigureAwait(false), fresh) : bounded;
|
||||
}
|
||||
|
||||
private void Learn(int ceiling)
|
||||
{
|
||||
Volatile.Write(ref _lastRefusalTicks, _time.GetUtcNow().UtcTicks);
|
||||
Volatile.Write(ref _learned, ceiling);
|
||||
}
|
||||
|
||||
private async Task<(T Result, int Budget)> AttemptAsync<T>(int budget, Func<int, Task<T>> pass)
|
||||
{
|
||||
int ceiling;
|
||||
try
|
||||
{
|
||||
return (await pass(budget).ConfigureAwait(false), budget);
|
||||
}
|
||||
catch (Exception ex) when (GatewayReadLimits.TryGetCeiling(ex, out ceiling) && ceiling < budget)
|
||||
{
|
||||
// fall through to the retry (outside the catch, so a second refusal is not nested in this one)
|
||||
}
|
||||
Learn(ceiling);
|
||||
return (await pass(ceiling).ConfigureAwait(false), ceiling);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
using Grpc.Core;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway;
|
||||
|
||||
/// <summary>Bounded metadata cache (#653). Missing tags expire quickly; failures never become missing tags.</summary>
|
||||
internal sealed class GatewayTagPresenceCache(IHistorianGatewayClient client, TimeProvider time)
|
||||
{
|
||||
internal const int Capacity = 4096;
|
||||
private readonly object _sync = new();
|
||||
private readonly Dictionary<string, (bool Exists, DateTimeOffset Expires)> _entries = new(StringComparer.Ordinal);
|
||||
|
||||
public async Task<bool> ExistsAsync(string tag, CancellationToken ct)
|
||||
{
|
||||
ct.ThrowIfCancellationRequested();
|
||||
lock (_sync)
|
||||
if (_entries.TryGetValue(tag, out var entry) && time.GetUtcNow() < entry.Expires) return entry.Exists;
|
||||
|
||||
bool exists;
|
||||
try { exists = await client.TagExistsAsync(tag, ct).ConfigureAwait(false); }
|
||||
catch (Exception ex) when (IsNotFound(ex)) { exists = false; }
|
||||
ct.ThrowIfCancellationRequested();
|
||||
lock (_sync)
|
||||
{
|
||||
if (!_entries.ContainsKey(tag) && _entries.Count >= Capacity) _entries.Remove(_entries.Keys.First());
|
||||
_entries[tag] = (exists, time.GetUtcNow().AddSeconds(exists ? 60 : 5));
|
||||
}
|
||||
return exists;
|
||||
}
|
||||
|
||||
private static bool IsNotFound(Exception ex)
|
||||
{
|
||||
for (Exception? e = ex; e is not null; e = e.InnerException)
|
||||
if (e is RpcException { StatusCode: StatusCode.NotFound }) return true;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
+11
@@ -81,6 +81,17 @@ public sealed class HistorianGatewayClientAdapter : IHistorianGatewayClient, IDi
|
||||
return new HistorianGatewayClientAdapter(HistorianGatewayClient.Create(clientOptions));
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<bool> TagExistsAsync(string tag, CancellationToken ct)
|
||||
{
|
||||
// #653: older gateway SDKs misparse a missing GetTagInfo response as Internal. A catalog query proves
|
||||
// absence without guessing from an error. Even a glob-like tag is checked by exact name, never by row count.
|
||||
var found = false;
|
||||
await foreach (var name in _inner.BrowseTagNamesAsync(tag, ct).ConfigureAwait(false))
|
||||
found |= string.Equals(name, tag, StringComparison.OrdinalIgnoreCase);
|
||||
return found; // drain first: a trailing stream fault must propagate and must not become a cache entry
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public IAsyncEnumerable<HistorianSample> ReadRawAsync(
|
||||
string tag, DateTime startUtc, DateTime endUtc, int maxValues, CancellationToken ct) =>
|
||||
|
||||
@@ -11,6 +11,12 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway;
|
||||
/// </summary>
|
||||
public interface IHistorianGatewayClient : IAsyncDisposable
|
||||
{
|
||||
/// <summary>Checks the tag catalog; false means explicitly absent, while failures propagate.</summary>
|
||||
/// <param name="tag">Historian tag name.</param>
|
||||
/// <param name="ct">Cancellation token.</param>
|
||||
/// <returns>Whether the historian contains the tag.</returns>
|
||||
Task<bool> TagExistsAsync(string tag, CancellationToken ct);
|
||||
|
||||
/// <summary>Streams raw historian samples for a tag over a time window.</summary>
|
||||
/// <param name="tag">The historian tag name to read.</param>
|
||||
/// <param name="startUtc">Inclusive start of the requested time window (UTC).</param>
|
||||
|
||||
@@ -35,16 +35,16 @@ internal static class AlarmEventMapper
|
||||
/// <returns>The gateway wire event ready for <c>SendEvent</c>.</returns>
|
||||
public static HistorianEvent ToHistorianEvent(AlarmHistorianEvent alarm)
|
||||
{
|
||||
if (alarm.DeliveryId == Guid.Empty)
|
||||
throw new ArgumentException("Alarm delivery identity must be nonzero.", nameof(alarm));
|
||||
// Timestamp.FromDateTime requires a Utc-kind DateTime; coerce defensively (TimestampUtc is
|
||||
// already Utc by contract, but a caller could pass Unspecified).
|
||||
var eventTime = Timestamp.FromDateTime(DateTime.SpecifyKind(alarm.TimestampUtc, DateTimeKind.Utc));
|
||||
|
||||
var historianEvent = new HistorianEvent
|
||||
{
|
||||
// Deliberately DO NOT set HistorianEvent.Id: the gateway's SendEvent path rejects a
|
||||
// client-supplied event id (the server handler throws and the call fails permanently —
|
||||
// confirmed live). The historian assigns event identity server-side; the alarm's own id
|
||||
// is preserved below as a property for read-back correlation/traceability.
|
||||
// Identity belongs to the event/durable queue, never to an individual RPC attempt.
|
||||
Id = alarm.DeliveryId.ToString("D"),
|
||||
SourceName = alarm.EquipmentPath,
|
||||
Type = alarm.AlarmTypeName,
|
||||
EventTime = eventTime,
|
||||
|
||||
@@ -23,11 +23,21 @@ internal static class SampleMapper
|
||||
/// <summary>Maps a single raw sample to a value snapshot.</summary>
|
||||
/// <param name="sample">The gateway raw sample.</param>
|
||||
/// <returns>The driver-agnostic snapshot.</returns>
|
||||
/// <remarks>
|
||||
/// #690: a Bad row carries no value. histsdk puts a value on EVERY row (<c>NumericValue = 0</c> on a numeric row,
|
||||
/// <c>StringValue = ""</c> on a string one), so a stored NULL arrives as <c>0</c> / <c>""</c> with a Bad OPC quality
|
||||
/// (live: <c>q=1 opc=0</c>) — that 0 is not a value, and AVEVA stores a Bad-quality sample as NULL, so a Bad row never
|
||||
/// carries a real one. The value is therefore dropped whenever the mapped status is Bad, matching the SqlServer/QuestDB
|
||||
/// backends (an Empty sample is null + Bad) and the aggregate mapper's no-data rows. Uncertain rows keep their value.
|
||||
/// </remarks>
|
||||
public static DataValueSnapshot ToSnapshot(HistorianSample sample)
|
||||
{
|
||||
var status = ToStatus(sample);
|
||||
// proto3 explicit presence: prefer the numeric value, else the string value, else null.
|
||||
object? value;
|
||||
if (sample.HasNumericValue)
|
||||
if (StatusBits.IsBad(status))
|
||||
value = null;
|
||||
else if (sample.HasNumericValue)
|
||||
value = sample.NumericValue; // boxes as System.Double
|
||||
else if (sample.HasStringValue)
|
||||
value = sample.StringValue;
|
||||
@@ -36,7 +46,7 @@ internal static class SampleMapper
|
||||
|
||||
return new DataValueSnapshot(
|
||||
Value: value,
|
||||
StatusCode: ToStatus(sample),
|
||||
StatusCode: status,
|
||||
SourceTimestampUtc: sample.Timestamp?.ToDateTime(), // Utc kind
|
||||
ServerTimestampUtc: DateTime.UtcNow);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,112 @@
|
||||
using ZB.MOM.WW.HistorianGateway.Contracts.Grpc;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Mapping;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway;
|
||||
|
||||
/// <summary>
|
||||
/// Serves a processed read of the AVEVA-native <c>TimeAverage</c> / <c>Total</c> on the SQL backends' interval grid
|
||||
/// (<see cref="BucketPlan"/>), in either direction: forward — window <c>[start, end)</c>, intervals stepping forward
|
||||
/// from <c>start</c>, the clipped interval the LATEST, each stamped with its earlier edge, oldest first; reverse (#677,
|
||||
/// #691.2) — window <c>(end, start]</c>, intervals stepping back from <c>start</c>, the clipped interval the EARLIEST,
|
||||
/// each stamped with its later edge, newest first. The gateway refuses <c>start > end</c> and <c>interval <= 0</c>,
|
||||
/// so every request it sees is forward with a positive interval.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>Forward requests on the plan's grid.</b> AVEVA anchors a native read's interval grid at the request start (live,
|
||||
/// 2026-10-12: rows at <c>start + j·interval</c>, plus one extra row for the interval before it, #685). So the plan's
|
||||
/// whole intervals are requested as ONE forward read spanning exactly them with the interval <c>I</c>, and the
|
||||
/// clipped interval (when the window is not a whole number of intervals) as a second forward read over exactly its
|
||||
/// span with its own width as the interval — AVEVA never averages time outside the window (before it on a reverse
|
||||
/// read, past <c>EndTime</c> on a forward one). An interval wider than the window is sent as the window, and
|
||||
/// <c>interval <= 0</c> as one interval over the whole window (it used to reach the gateway as 0 and be refused).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Soundness of reverse.</b> A time-weighted average / integral over <c>[a, b)</c> and over <c>(a, b]</c> integrate
|
||||
/// the same function over sets differing by two instants, so a forward native row IS the value of the reverse
|
||||
/// interval with the same edges. The one place the half-open side could matter is AVEVA's status for an interval
|
||||
/// whose only non-good sample lies exactly on an edge — accepted, as on every native row.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Mapping.</b> Each row is mapped as before (<see cref="SampleMapper.ToAggregateSnapshot"/>: Uncertain keeps its
|
||||
/// value, no data ⇒ <c>BadNoData</c>, Calculated bits), placed by its interval start (else end), restamped with
|
||||
/// <see cref="BucketPlan.BucketTimestamp"/>, and the clipped interval carries the Partial bit unless Bad (Part 13
|
||||
/// §5.3.3.2 Case 3, as the SQL backends' time-weighted aggregates — which additionally mark the intervals the data
|
||||
/// starts/ends inside; AVEVA's rows carry no data range, so those are not reproduced). An interval AVEVA answers no
|
||||
/// row for is absent (not a fabricated <c>BadNoData</c>, unlike the SQL backends).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// ⚠️ Today historiangw#27 (the interval reaches AVEVA in ticks where it reads milliseconds) makes AVEVA answer ONE
|
||||
/// Good row per request — the whole-intervals read lands on its earliest interval with a whole-span value and the rest
|
||||
/// are absent — in both directions. The mapping needs no change once the gateway is fixed.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
internal static class NativeAggregatePlan
|
||||
{
|
||||
/// <summary>One forward native request and the ascending bucket index of its first interval.</summary>
|
||||
/// <param name="StartUtc">Forward request start (inclusive).</param>
|
||||
/// <param name="EndUtc">Forward request end.</param>
|
||||
/// <param name="Interval">The request's interval (> 0).</param>
|
||||
/// <param name="FirstBucket">Ascending plan bucket of the request's first interval.</param>
|
||||
/// <param name="BucketCount">Number of plan buckets the request covers.</param>
|
||||
internal readonly record struct Request(DateTime StartUtc, DateTime EndUtc, TimeSpan Interval, long FirstBucket, long BucketCount);
|
||||
|
||||
/// <summary>The forward requests serving a plan: the whole intervals first, then the clipped one (if any).</summary>
|
||||
/// <param name="plan">A plan with at least one bucket.</param>
|
||||
/// <returns>One or two forward requests.</returns>
|
||||
public static IReadOnlyList<Request> Requests(BucketPlan plan)
|
||||
{
|
||||
var interval = TimeSpan.FromTicks(plan.IntervalTicks);
|
||||
var n = plan.BucketCount;
|
||||
var clipped = plan.IsTruncated(0) ? 0L : plan.IsTruncated(n - 1) ? n - 1 : -1L;
|
||||
if (clipped < 0) return [new Request(plan.StartUtc, plan.EndUtc, interval, 0, n)];
|
||||
|
||||
var clipLo = new DateTime(plan.LowerTicks(clipped), DateTimeKind.Utc);
|
||||
var clipHi = new DateTime(plan.UpperTicks(clipped), DateTimeKind.Utc);
|
||||
var requests = new List<Request>(2);
|
||||
if (n > 1)
|
||||
requests.Add(clipped == 0
|
||||
? new Request(clipHi, plan.EndUtc, interval, 1, n - 1) // reverse: the clipped one is the earliest
|
||||
: new Request(plan.StartUtc, clipLo, interval, 0, n - 1)); // forward: the clipped one is the latest
|
||||
requests.Add(new Request(clipLo, clipHi, clipHi - clipLo, clipped, 1));
|
||||
return requests;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Maps a request's rows onto the plan's buckets: rows outside the request window (AVEVA's pre-window row, #685)
|
||||
/// are dropped; each in-window row is placed by its interval start (or, without one, its end).
|
||||
/// </summary>
|
||||
/// <param name="plan">The plan.</param>
|
||||
/// <param name="request">The forward request the rows answer.</param>
|
||||
/// <param name="rows">The drained rows.</param>
|
||||
/// <param name="byBucket">Results by ascending bucket index (first row per bucket wins).</param>
|
||||
public static void Place(BucketPlan plan, Request request, IEnumerable<HistorianAggregateSample> rows,
|
||||
Dictionary<long, DataValueSnapshot> byBucket)
|
||||
{
|
||||
var interval = request.Interval.Ticks;
|
||||
foreach (var row in rows)
|
||||
{
|
||||
if (!SampleMapper.IsInWindow(row, request.StartUtc, request.EndUtc)) continue;
|
||||
long offset = row.StartTime is { } s
|
||||
? (s.ToDateTime().Ticks - request.StartUtc.Ticks) / interval
|
||||
: (row.EndTime.ToDateTime().Ticks - request.StartUtc.Ticks - 1) / interval; // IsInWindow ⇒ end in (lo, hi]
|
||||
if (offset < 0 || offset >= request.BucketCount) continue;
|
||||
var index = request.FirstBucket + offset;
|
||||
if (byBucket.ContainsKey(index)) continue;
|
||||
|
||||
var snapshot = SampleMapper.ToAggregateSnapshot(row);
|
||||
var status = snapshot.StatusCode;
|
||||
if (plan.IsTruncated(index) && !StatusBits.IsBad(status)) status |= HistorianInfoBits.HistorianPartial;
|
||||
byBucket[index] = snapshot with { StatusCode = status, SourceTimestampUtc = plan.BucketTimestamp(index) };
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>The placed results in the read's order (ascending forward, descending reverse).</summary>
|
||||
/// <param name="plan">The plan.</param>
|
||||
/// <param name="byBucket">Results by ascending bucket index.</param>
|
||||
/// <returns>The results.</returns>
|
||||
public static IReadOnlyList<DataValueSnapshot> Assemble(BucketPlan plan, IReadOnlyDictionary<long, DataValueSnapshot> byBucket) =>
|
||||
(plan.Reversed ? byBucket.OrderByDescending(kv => kv.Key) : byBucket.OrderBy(kv => kv.Key)).Select(kv => kv.Value).ToList();
|
||||
}
|
||||
@@ -49,7 +49,10 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway;
|
||||
/// requested row budget — has been drained. The gateway is asked for at most <c>cap + 2 + </c>
|
||||
/// <see cref="PreWindowAllowance"/> rows, clamped at <see cref="GatewayDefaultMaxValuesCeiling"/> while the cap is at
|
||||
/// or under it (#688, <see cref="ClampToGatewayCeiling"/>); a stream that delivers all of them cannot be proven
|
||||
/// complete and fails the same way rather than answering from a truncated window.
|
||||
/// complete and fails the same way rather than answering from a truncated window. A strict gateway whose ceiling is
|
||||
/// below that budget (a ceiling under 1,000,000, or an unbounded cap) refuses it naming its ceiling; the window is
|
||||
/// then re-read once with the ceiling as budget (#691.3, <see cref="GatewayReadLimits"/>), and the same
|
||||
/// "delivered the whole budget ⇒ cannot be proven complete" rule applies.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Values and status.</b> A sample's value is its numeric value (Double), else its string value, else Empty; its
|
||||
@@ -163,6 +166,7 @@ internal static class RawAggregateReader
|
||||
|
||||
/// <summary>Reads and evaluates one per-sample processed read.</summary>
|
||||
/// <param name="client">The gateway client.</param>
|
||||
/// <param name="ceiling">The data source's learned gateway row ceiling (#691.3).</param>
|
||||
/// <param name="tag">Historian tag name.</param>
|
||||
/// <param name="aggregate">A per-sample aggregate (<see cref="Serves"/>).</param>
|
||||
/// <param name="plan">The bucket plan.</param>
|
||||
@@ -172,14 +176,51 @@ internal static class RawAggregateReader
|
||||
/// <returns>One value per interval, in the plan's order.</returns>
|
||||
/// <exception cref="HistoryReadLimitExceededException">The window holds more than <paramref name="maxWindowRows"/> rows,
|
||||
/// or the stream could not be read to completion within the requested row budget.</exception>
|
||||
public static async Task<IReadOnlyList<DataValueSnapshot>> ReadAsync(IHistorianGatewayClient client, string tag,
|
||||
public static async Task<IReadOnlyList<DataValueSnapshot>> ReadAsync(IHistorianGatewayClient client, GatewayRowCeiling ceiling, string tag,
|
||||
HistoryAggregateType aggregate, BucketPlan plan, int maxWindowRows, DateTime now, CancellationToken ct)
|
||||
{
|
||||
if (plan.BucketCount == 0) return [];
|
||||
var needsRange = AggregateEvaluator.NeedsDataRange(aggregate, plan);
|
||||
var requestStart = RequestStartFor(plan.StartUtc);
|
||||
var maxValues = MaxValuesFor(maxWindowRows);
|
||||
|
||||
// A strict gateway that refuses the budget is re-asked once at the ceiling it names (#691.3; e.g. a ceiling below
|
||||
// the 1,000,000 #688 clamps at, or an unbounded cap) — and a stream that fills that ceiling is never evaluated.
|
||||
var requested = MaxValuesFor(maxWindowRows);
|
||||
var (pass, maxValues) = await ceiling.ReadAsync(requested,
|
||||
budget => ReadWindowAsync(client, tag, aggregate, plan, requestStart, budget, maxWindowRows, ct),
|
||||
(r, budget) => !r.OverCap && r.Received >= budget, deferRecentReprobe: true).ConfigureAwait(false);
|
||||
var (buckets, earliest, latest, received, overCap) = pass;
|
||||
if (overCap) throw LimitExceeded(tag, maxWindowRows);
|
||||
if (received >= maxValues) throw BudgetExhausted(tag, maxWindowRows, requested, maxValues);
|
||||
|
||||
if (needsRange)
|
||||
{
|
||||
// Earlier data: no non-Bad pre-window row ⇒ ask explicitly (normally only when the tag's history really starts
|
||||
// inside the window).
|
||||
var earlierSeen = earliest is { } e && (e < plan.StartUtc || (plan.Reversed && e == plan.StartUtc));
|
||||
if (!earlierSeen && plan.StartUtc.AddTicks(-1) > ProbeFloor)
|
||||
if (await FirstSampleAsync(client, tag, ProbeFloor, plan.StartUtc.AddTicks(-1), ct).ConfigureAwait(false) is { } before)
|
||||
earliest = before;
|
||||
|
||||
// Later data: the main read covers a sample exactly at the window end (forward: that is already later data);
|
||||
// otherwise a small separate read from the end to now + horizon.
|
||||
var laterSeen = latest is { } l && (plan.Reversed ? l > plan.EndUtc : l >= plan.EndUtc);
|
||||
var horizon = DateTime.MaxValue - now > LookAheadHorizon
|
||||
? DateTime.SpecifyKind(now + LookAheadHorizon, DateTimeKind.Utc)
|
||||
: DateTime.MaxValue;
|
||||
if (!laterSeen && horizon > plan.EndUtc)
|
||||
if (await FirstSampleAsync(client, tag, plan.EndUtc, horizon, ct).ConfigureAwait(false) is { } after)
|
||||
latest = after;
|
||||
}
|
||||
|
||||
return AggregateEvaluator.EvaluateAll(aggregate, plan, buckets.Rows(), earliest, latest, now);
|
||||
}
|
||||
|
||||
/// <summary>The window read of <see cref="ReadAsync"/> with one row budget, drained.</summary>
|
||||
private static async Task<(RawBucketAccumulator Buckets, DateTime? Earliest, DateTime? Latest, long Received, bool OverCap)>
|
||||
ReadWindowAsync(IHistorianGatewayClient client, string tag, HistoryAggregateType aggregate, BucketPlan plan,
|
||||
DateTime requestStart, int maxValues, int maxWindowRows, CancellationToken ct)
|
||||
{
|
||||
var buckets = new RawBucketAccumulator(aggregate == HistoryAggregateType.MaximumActualTime);
|
||||
DateTime? earliest = null, latest = null;
|
||||
long windowRows = 0, received = 0;
|
||||
@@ -214,30 +255,7 @@ internal static class RawAggregateReader
|
||||
if (t >= plan.StartUtc && (latest is null || t > latest)) latest = t;
|
||||
}
|
||||
}
|
||||
if (overCap || (maxWindowRows > 0 && received >= maxValues))
|
||||
throw LimitExceeded(tag, maxWindowRows);
|
||||
|
||||
if (needsRange)
|
||||
{
|
||||
// Earlier data: no non-Bad pre-window row ⇒ ask explicitly (normally only when the tag's history really starts
|
||||
// inside the window).
|
||||
var earlierSeen = earliest is { } e && (e < plan.StartUtc || (plan.Reversed && e == plan.StartUtc));
|
||||
if (!earlierSeen && plan.StartUtc.AddTicks(-1) > ProbeFloor)
|
||||
if (await FirstSampleAsync(client, tag, ProbeFloor, plan.StartUtc.AddTicks(-1), ct).ConfigureAwait(false) is { } before)
|
||||
earliest = before;
|
||||
|
||||
// Later data: the main read covers a sample exactly at the window end (forward: that is already later data);
|
||||
// otherwise a small separate read from the end to now + horizon.
|
||||
var laterSeen = latest is { } l && (plan.Reversed ? l > plan.EndUtc : l >= plan.EndUtc);
|
||||
var horizon = DateTime.MaxValue - now > LookAheadHorizon
|
||||
? DateTime.SpecifyKind(now + LookAheadHorizon, DateTimeKind.Utc)
|
||||
: DateTime.MaxValue;
|
||||
if (!laterSeen && horizon > plan.EndUtc)
|
||||
if (await FirstSampleAsync(client, tag, plan.EndUtc, horizon, ct).ConfigureAwait(false) is { } after)
|
||||
latest = after;
|
||||
}
|
||||
|
||||
return AggregateEvaluator.EvaluateAll(aggregate, plan, buckets.Rows(), earliest, latest, now);
|
||||
return (buckets, earliest, latest, received, overCap);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -266,6 +284,26 @@ internal static class RawAggregateReader
|
||||
/// </summary>
|
||||
private static bool IsNotBadOnTheWire(HistorianSample raw) => (byte)raw.OpcQuality >= 64; // low byte, as SampleMapper.ToStatus reads it
|
||||
|
||||
/// <summary>
|
||||
/// The stream delivered its whole row budget, so the window cannot be proven complete. Names the limit that actually
|
||||
/// bound the budget: the gateway's ceiling (a strict gateway refused the requested budget and named it, #691.3), the
|
||||
/// #688 clamp at the gateway's default ceiling, or the configured cap.
|
||||
/// </summary>
|
||||
private static HistoryReadLimitExceededException BudgetExhausted(string tag, int maxWindowRows, int requested, int budget)
|
||||
{
|
||||
if (budget < requested)
|
||||
return new HistoryReadLimitExceededException(budget,
|
||||
$"The processed read of '{tag}' covers more raw samples than the HistorianGateway's row ceiling ({budget}, its " +
|
||||
"Historian:ReadLimits:MaxValuesCeiling with AllowStreamingBeyondCeiling=false); the Gateway backend evaluates " +
|
||||
"per-sample aggregates from raw reads — narrow the window.");
|
||||
if (maxWindowRows > 0 && budget < (long)maxWindowRows + 2 + PreWindowAllowance)
|
||||
return new HistoryReadLimitExceededException(budget,
|
||||
$"The processed read of '{tag}' covers more raw samples than the HistorianGateway's default row ceiling " +
|
||||
$"({budget}) the request is clamped at; the Gateway backend evaluates per-sample aggregates from raw reads — " +
|
||||
"narrow the window.");
|
||||
return LimitExceeded(tag, maxWindowRows);
|
||||
}
|
||||
|
||||
private static HistoryReadLimitExceededException LimitExceeded(string tag, int maxWindowRows) =>
|
||||
new(maxWindowRows,
|
||||
$"The processed read of '{tag}' covers more than {maxWindowRows} raw samples " +
|
||||
|
||||
@@ -137,6 +137,7 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource
|
||||
catch (Exception ex)
|
||||
{
|
||||
RecordFailure(ex);
|
||||
if (IsTransportFailure(ex)) throw new System.IO.IOException("Historian read transport is unavailable.", ex);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
@@ -232,6 +233,7 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource
|
||||
catch (Exception ex)
|
||||
{
|
||||
RecordFailure(ex);
|
||||
if (IsTransportFailure(ex)) throw new System.IO.IOException("Historian read transport is unavailable.", ex);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
@@ -316,6 +318,7 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource
|
||||
catch (Exception ex)
|
||||
{
|
||||
RecordFailure(ex);
|
||||
if (IsTransportFailure(ex)) throw new System.IO.IOException("Historian read transport is unavailable.", ex);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
@@ -359,6 +362,10 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource
|
||||
/// <remarks>Reverse raw reads are served natively (<c>(end, start]</c>, <c>ORDER BY … DESC</c>; #669/#686).</remarks>
|
||||
public bool SupportsReverseRaw => true;
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks><c>0</c>: a capped raw read is one bounded statement — no backend row ceiling (Gitea #694).</remarks>
|
||||
public int MaxRawValuesPerRead => 0;
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>Reverse events reads are served natively (<c>ORDER BY … DESC</c>, Gitea #675).</remarks>
|
||||
public bool SupportsReverseEvents => true;
|
||||
@@ -439,6 +446,7 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource
|
||||
catch (Exception ex)
|
||||
{
|
||||
RecordFailure(ex);
|
||||
if (IsTransportFailure(ex)) throw new System.IO.IOException("Historian read transport is unavailable.", ex);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
@@ -534,6 +542,7 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource
|
||||
catch (Exception ex)
|
||||
{
|
||||
RecordFailure(ex);
|
||||
if (IsTransportFailure(ex)) throw new System.IO.IOException("Historian read transport is unavailable.", ex);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
@@ -697,6 +706,11 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource
|
||||
|
||||
private void RecordSuccess() => _health.RecordSuccess();
|
||||
|
||||
/// <summary>Classifies provider transport/timeouts, keeping SQL/query errors distinct (#697).</summary>
|
||||
private static bool IsTransportFailure(Exception ex) => ex is NpgsqlException { IsTransient: true } and not PostgresException
|
||||
|| ex is PostgresException pg && (pg.SqlState.StartsWith("08", StringComparison.Ordinal)
|
||||
|| pg.SqlState is "57P01" or "57P02" or "57P03");
|
||||
|
||||
private void RecordFailure(Exception ex)
|
||||
{
|
||||
// A cancelled read (deadline / client abort) is not a backend fault (the counters ignore it too).
|
||||
|
||||
@@ -49,8 +49,10 @@ public static class AtTimeEvaluator
|
||||
{
|
||||
var fraction = (double)(t.Ticks - p.TimeUtc.Ticks) / (n.TimeUtc.Ticks - p.TimeUtc.Ticks);
|
||||
var v = p.Value.Number + (nv - p.Value.Number) * fraction;
|
||||
// Each arm boxed on its own: `cond ? (long)x : v` is a DOUBLE expression in C# (the long widens), which returned
|
||||
// every interpolated Int64 as a Double until #693.
|
||||
object value = p.Value.Kind == HistorianValueKind.Int64
|
||||
? (long)Math.Round(v, MidpointRounding.AwayFromZero)
|
||||
? (object)(long)Math.Round(v, MidpointRounding.AwayFromZero)
|
||||
: v;
|
||||
var status = p.IsGood && n.IsGood ? HistorianStatusCodes.Good : HistorianStatusCodes.UncertainDataSubNormal;
|
||||
return new DataValueSnapshot(value, HistorianInfoBits.WithInterpolated(status), t, serverTime);
|
||||
|
||||
+12
@@ -132,6 +132,7 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
|
||||
catch (Exception ex)
|
||||
{
|
||||
RecordFailure(ex);
|
||||
if (IsTransportFailure(ex)) throw new System.IO.IOException("Historian read transport is unavailable.", ex);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
@@ -229,6 +230,7 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
|
||||
catch (Exception ex) when (ex is not NotSupportedException)
|
||||
{
|
||||
RecordFailure(ex);
|
||||
if (IsTransportFailure(ex)) throw new System.IO.IOException("Historian read transport is unavailable.", ex);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
@@ -306,6 +308,7 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
|
||||
catch (Exception ex)
|
||||
{
|
||||
RecordFailure(ex);
|
||||
if (IsTransportFailure(ex)) throw new System.IO.IOException("Historian read transport is unavailable.", ex);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
@@ -316,6 +319,10 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
|
||||
/// <remarks>Reverse raw reads are served natively (<c>(end, start]</c>, <c>ORDER BY … DESC</c>; #669/#686).</remarks>
|
||||
public bool SupportsReverseRaw => true;
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks><c>0</c>: a capped raw read is one bounded statement — no backend row ceiling (Gitea #694).</remarks>
|
||||
public int MaxRawValuesPerRead => 0;
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>Reverse events reads are served natively (<c>ORDER BY … DESC</c>, Gitea #675).</remarks>
|
||||
public bool SupportsReverseEvents => true;
|
||||
@@ -397,6 +404,7 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
|
||||
catch (Exception ex)
|
||||
{
|
||||
RecordFailure(ex);
|
||||
if (IsTransportFailure(ex)) throw new System.IO.IOException("Historian read transport is unavailable.", ex);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
@@ -508,6 +516,7 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
|
||||
catch (Exception ex)
|
||||
{
|
||||
RecordFailure(ex);
|
||||
if (IsTransportFailure(ex)) throw new System.IO.IOException("Historian read transport is unavailable.", ex);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
@@ -631,6 +640,9 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
|
||||
|
||||
private void RecordSuccess() => _health.RecordSuccess();
|
||||
|
||||
/// <summary>Classifies provider transport/timeouts, keeping SQL/query errors distinct (#697).</summary>
|
||||
private static bool IsTransportFailure(Exception ex) => ex is SqlException sql && sql.Number is -2 or 0 or 53 or 64 or 233 or 258 or 10053 or 10054 or 10060 or 10061 or 10065 or 11001 or 40197 or 40613;
|
||||
|
||||
private void RecordFailure(Exception ex)
|
||||
{
|
||||
// A cancelled read (deadline / client abort) is not a backend fault (the counters ignore it too).
|
||||
|
||||
@@ -1465,7 +1465,7 @@ public sealed class S7Driver
|
||||
{
|
||||
// AREA precedence first (before the DataType switch): a Timer (T{n}) decodes its 2-byte
|
||||
// S5TIME word to a duration in SECONDS via S7.Net's Timer.FromByteArray (returns double);
|
||||
// a Counter (C{n}) decodes its 2-byte word to a COUNT via Counter.FromByteArray (returns
|
||||
// a Counter (C{n}) exposes its 2-byte RAW WORD via Counter.FromByteArray (returns
|
||||
// ushort, surfaced as a non-negative int to match the tag's Int32 type). Routing on Area
|
||||
// here is essential: a Timer tag is typed Float64, so letting it fall to the Float64 arm
|
||||
// below would ReadDoubleBigEndian over a 2-byte block = garbage/throw. Boxed explicitly so
|
||||
@@ -1474,11 +1474,12 @@ public sealed class S7Driver
|
||||
return (object)global::S7.Net.Types.Timer.FromByteArray(block);
|
||||
if (addr.Area is S7Area.Counter)
|
||||
{
|
||||
// NOTE: S7.Net Counter.FromByteArray returns the RAW big-endian word ((bytes[0]<<8)|bytes[1]),
|
||||
// NOT a BCD decode. On classic S7-300/400 the C-area word is BCD (0-999), so on that hardware
|
||||
// this raw value can differ from the displayed count; S7-1200/1500 use IEC/DB counters (plain
|
||||
// ints), where it is correct. Surfacing S7.Net's value verbatim is the faithful choice; BCD
|
||||
// reinterpretation for legacy C-area counters is a live-hardware-gated follow-up.
|
||||
// S7.Net Counter.FromByteArray returns the raw big-endian word ((bytes[0]<<8)|bytes[1]),
|
||||
// without BCD reinterpretation. This preserves the raw value, not a verified PLC count:
|
||||
// S7comm C-area response encoding still needs independent raw bytes and PLC-side
|
||||
// count/binary/BCD references (#616). CPU family alone does not prove that encoding.
|
||||
// IEC counter integers in DB memory take the DB type codec, not this C-area path;
|
||||
// S7-1500 also has legacy SIMATIC counters. See docs/drivers/S7.md for the evidence gate.
|
||||
return (object)(int)global::S7.Net.Types.Counter.FromByteArray(block);
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Host.OpcUa;
|
||||
|
||||
/// <summary>Coordinates endpoint cleanup after ServiceLevel binding fails during startup.</summary>
|
||||
internal static class FailedServiceLevelBindingCleanup
|
||||
{
|
||||
/// <summary>Stops and disposes the endpoint, returning whether ownership can be released.</summary>
|
||||
/// <param name="stop">The SDK stop operation.</param>
|
||||
/// <param name="dispose">The application-host disposal operation.</param>
|
||||
/// <param name="logger">The cleanup logger.</param>
|
||||
/// <returns>Whether cleanup completed successfully.</returns>
|
||||
internal static async Task<bool> StopAndDisposeAsync(
|
||||
Func<Task> stop, Func<ValueTask> dispose, ILogger logger)
|
||||
{
|
||||
var completed = true;
|
||||
try
|
||||
{
|
||||
await stop().ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
completed = false;
|
||||
logger.LogError(ex, "OtOpcUaServerHostedService: SDK stop failed after binding failure; retaining endpoint ownership for shutdown retry");
|
||||
}
|
||||
// Attempt application cleanup even when stopping faults; keep ownership on either failure.
|
||||
try
|
||||
{
|
||||
await dispose().ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
completed = false;
|
||||
logger.LogError(ex, "OtOpcUaServerHostedService: endpoint disposal failed after binding failure; retaining endpoint ownership for shutdown retry");
|
||||
}
|
||||
return completed;
|
||||
}
|
||||
}
|
||||
@@ -317,6 +317,12 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
|
||||
// section is absent; NodeManager is non-null here (guarded above).
|
||||
_server.NodeManager.MaxTieClusterOverfetch = _serverHistorianOptions.MaxTieClusterOverfetch;
|
||||
|
||||
// Gitea #691: the ReturnBounds kill switch (default on).
|
||||
_server.NodeManager.HonorReturnBounds = HonorReturnBounds;
|
||||
|
||||
// Gitea #694: the opt-in HistoryReadEvents horizon (default off = exact to each read's own window).
|
||||
_server.NodeManager.EventReadHorizon = EventReadHorizon;
|
||||
|
||||
// Propagate the S3 HistoryRead concurrency bounds (arch-review 03/S3): bounded per-node fan-out
|
||||
// within a batch, the process-wide concurrent-batch limiter, and the per-request deadline. Defaults
|
||||
// (4 / 16 / 60 s) survive when the ServerHistorian section is absent.
|
||||
@@ -333,11 +339,36 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
|
||||
_server.NodeManager.Clock = historyUpdate.Clock;
|
||||
|
||||
// ServiceLevel publisher needs IServerInternal — only available after Start.
|
||||
if (_server.CurrentInstance is { } serverInternal)
|
||||
try
|
||||
{
|
||||
_deferredServiceLevel.SetInner(new SdkServiceLevelPublisher(
|
||||
serverInternal,
|
||||
_loggerFactory.CreateLogger<SdkServiceLevelPublisher>()));
|
||||
if (_server.CurrentInstance is { } serverInternal)
|
||||
{
|
||||
_deferredServiceLevel.SetInner(new SdkServiceLevelPublisher(
|
||||
serverInternal,
|
||||
_loggerFactory.CreateLogger<SdkServiceLevelPublisher>()));
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex,
|
||||
"OtOpcUaServerHostedService: ServiceLevel binding failed; stopping the OPC UA endpoint while the admin host continues");
|
||||
try
|
||||
{
|
||||
await StopAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception cleanupError)
|
||||
{
|
||||
_logger.LogError(cleanupError, "OtOpcUaServerHostedService: failed to detach OPC UA bindings after startup failure");
|
||||
}
|
||||
var released = await FailedServiceLevelBindingCleanup.StopAndDisposeAsync(
|
||||
async () => await _server.StopAsync(CancellationToken.None).ConfigureAwait(false),
|
||||
() => _appHost.DisposeAsync(), _logger).ConfigureAwait(false);
|
||||
if (released)
|
||||
{
|
||||
_appHost = null;
|
||||
_server = null;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// arch-review 03/S19 — LAST, after the node manager is known non-null and after the ACL gate is
|
||||
@@ -347,6 +378,18 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
|
||||
_logger.LogInformation("OtOpcUaServerHostedService: SDK started, address-space + ServiceLevel sinks bound");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #691 — the <c>ServerHistorian:HonorReturnBounds</c> kill switch <c>StartAsync</c> applies to the node manager
|
||||
/// (default <c>true</c> when the section or key is absent). Exposed so the wiring is testable without booting the SDK.
|
||||
/// </summary>
|
||||
internal bool HonorReturnBounds => _serverHistorianOptions.HonorReturnBounds;
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #694 — the <c>ServerHistorian:EventReadHorizon</c> <c>StartAsync</c> applies to the node manager (default
|
||||
/// <see cref="TimeSpan.Zero"/> = none). Exposed so the wiring is testable without booting the SDK.
|
||||
/// </summary>
|
||||
internal TimeSpan EventReadHorizon => _serverHistorianOptions.EventReadHorizon;
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #674 — what <c>StartAsync</c> wires onto the node manager for HistoryUpdate: the Primary gate (read from
|
||||
/// the <see cref="IHistoryUpdateRoleView"/> on every call; null when none is registered, i.e. allow), the
|
||||
|
||||
@@ -35,7 +35,9 @@
|
||||
"AllowUntrustedServerCertificate": false,
|
||||
"CaCertificatePath": null,
|
||||
"CallTimeout": "00:00:30",
|
||||
"MaxTieClusterOverfetch": 65536
|
||||
"MaxTieClusterOverfetch": 65536,
|
||||
"EventReadHorizon": "00:00:00",
|
||||
"_EventReadHorizonComment": "Gitea #694, opt-in. 00:00:00 = no horizon: HistoryReadEvents is exact to each read's window (on the Gateway a reverse/end-only events read proves 'nothing older' with one read down to 1970 - seconds on a large Runtime.dbo.Events). A positive span (e.g. 365.00:00:00) never reads events older than now minus it, in either direction - LOSSY: older events become invisible to HistoryReadEvents."
|
||||
},
|
||||
"ContinuousHistorization": {
|
||||
"_comment": "Continuous historization of driver (non-Galaxy) tag values: a crash-safe FasterLog outbox + recorder draining to the ServerHistorian gateway's WriteLiveValues. Disabled => no recorder is spawned. Requires ServerHistorian to be configured; the gateway connection (endpoint/key/TLS) is sourced from the ServerHistorian section, not here.",
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
namespace ZB.MOM.WW.OtOpcUa.OpcUaServer;
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #694 item 2: the opt-in <c>ServerHistorian:EventReadHorizon</c> — HistoryReadEvents never reads events older than
|
||||
/// <c>now − horizon</c>. Off (<see cref="TimeSpan.Zero"/>, the default) a read is exact down to whatever its window says,
|
||||
/// which on the Gateway means a reverse / end-only read walked back with forward reads proves "nothing older" with one
|
||||
/// read down to the Unix epoch (#691) — ~3.8 s live on <c>Runtime.dbo.Events</c>, and an unfiltered one can hit the
|
||||
/// gateway's 30 s SQL timeout. A forward read whose StartTime is far in the past costs the same on that table. The
|
||||
/// horizon bounds both, at a stated price: events older than it are not returned, in either direction, on any backend.
|
||||
/// </summary>
|
||||
internal static class EventHorizon
|
||||
{
|
||||
/// <summary>
|
||||
/// Raises the EARLIER bound of a fresh events window to the horizon's floor (<c>now − horizon</c>, inclusive): the
|
||||
/// start of a forward <c>[start, end)</c> window, the exclusive lower end of a reverse <c>(end, start]</c> one.
|
||||
/// </summary>
|
||||
/// <param name="startUtc">The window start (the later bound of a reverse read).</param>
|
||||
/// <param name="endUtc">The window end (the exclusive earlier bound of a reverse read).</param>
|
||||
/// <param name="reverse">Whether the read is newest-first (<c>start > end</c>).</param>
|
||||
/// <param name="horizon">The horizon; non-positive = none (nothing changes).</param>
|
||||
/// <param name="nowUtc">Now.</param>
|
||||
/// <returns><c>false</c> when the whole window lies before the floor — the read is empty and must not be sent.</returns>
|
||||
public static bool TryClamp(ref DateTime startUtc, ref DateTime endUtc, bool reverse, TimeSpan horizon, DateTime nowUtc)
|
||||
{
|
||||
if (horizon <= TimeSpan.Zero) return true;
|
||||
var floor = nowUtc.Ticks - DateTime.MinValue.Ticks > horizon.Ticks
|
||||
? new DateTime(nowUtc.Ticks - horizon.Ticks, DateTimeKind.Utc)
|
||||
: new DateTime(0, DateTimeKind.Utc);
|
||||
if (floor.Ticks == 0) return true;
|
||||
|
||||
if (reverse)
|
||||
{
|
||||
if (startUtc < floor) return false; // (end, start] lies wholly before the floor
|
||||
var below = floor.AddTicks(-1); // exclusive lower end ⇒ the floor instant itself is read
|
||||
if (endUtc < below) endUtc = below;
|
||||
return true;
|
||||
}
|
||||
|
||||
if (endUtc <= floor) return false; // [start, end) lies wholly before the floor
|
||||
if (startUtc < floor) startUtc = floor;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,241 @@
|
||||
using Opc.Ua;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.OpcUaServer;
|
||||
|
||||
/// <summary>
|
||||
/// The end bound still owed to a HistoryRead-Raw read with <c>ReturnBounds = true</c> (Gitea #691). It rides the
|
||||
/// continuation point, because Part 11 returns the end bound with the LAST page only.
|
||||
/// </summary>
|
||||
/// <param name="AtUtc">The request's end time (the later bound of a forward read, the earlier of a reverse one).</param>
|
||||
/// <param name="Reverse">The read is newest-first (<c>StartTime > EndTime</c>, including an end-only read).</param>
|
||||
/// <param name="Inclusive">
|
||||
/// <c>false</c> only for <c>StartTime == EndTime</c>: the samples AT that instant are the data, so the end bound is the
|
||||
/// first sample strictly after it (Part 11 Table 1 footnote c). Otherwise the end bound is the sample at or beyond the
|
||||
/// end time, and the data window excludes the end time.
|
||||
/// </param>
|
||||
/// <param name="Open">The end time was not given (a one-sided read's sentinel): there is no end bound to look up.</param>
|
||||
/// <param name="DataExhausted">The data is drained and the end bound did not fit on the previous page.</param>
|
||||
/// <param name="LastReturnedUtc">The last value's time returned on an earlier page (stamps an open bound's placeholder).</param>
|
||||
/// <param name="Resolved">
|
||||
/// Gitea #692: the end bound an earlier page already looked up (a stored sample — never a placeholder), so the page that
|
||||
/// returns it does not read the backend again. <c>null</c> = not looked up yet.
|
||||
/// </param>
|
||||
internal sealed record HistoryEndBound(
|
||||
DateTime AtUtc, bool Reverse, bool Inclusive, bool Open, bool DataExhausted = false, DateTime? LastReturnedUtc = null,
|
||||
DataValueSnapshot? Resolved = null);
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #691: OPC UA Part 11 bounding values for HistoryRead-Raw (<c>ReadRawModifiedDetails.ReturnBounds</c>), composed
|
||||
/// in the server from every backend's existing raw read — the ONE authority, so SqlServer, QuestDB, Gateway and Null
|
||||
/// answer identically.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>What a bound is</b> (Part 11 §4.6, Table 1). Forward: the start bound is the stored sample AT or immediately
|
||||
/// BEFORE StartTime, the end bound the one AT or immediately AFTER EndTime. Reverse mirrors it: the start bound is at
|
||||
/// or after StartTime, the end bound at or before EndTime. A sample exactly on a bound IS the bound: on the start it
|
||||
/// is already the first sample of the data (the window includes StartTime), so nothing extra is added; on the end it is
|
||||
/// added (the window excludes EndTime). <c>StartTime == EndTime</c> reads forward, and its end bound is the first
|
||||
/// sample strictly after the instant. A bound is a stored sample of any quality — a Bad sample is returned as stored.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Missing bound.</b> <c>Bad_BoundNotFound</c>, a null value, stamped with the bound's time. A one-sided read's
|
||||
/// missing side (no EndTime / no StartTime) is never looked up; it is <c>Bad_BoundNotFound</c> stamped one second after
|
||||
/// (forward) or before (reverse) the previous returned value (Table 1 footnotes a and b).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Lookups.</b> "At or before T" is a newest-first read of <c>(0, T]</c> capped at one — native on a backend that
|
||||
/// declares <see cref="IHistorianDataSource.SupportsReverseRaw"/>, otherwise <see cref="ReverseRawReader"/>'s walk-back
|
||||
/// from bounded forward reads (the Gateway: NOT AVEVA's synthetic boundary row, which is stamped at the query start
|
||||
/// rather than at the prior sample's own time). "At or after T" is a forward read of <c>[T, MaxValue)</c> capped at one (on a
|
||||
/// forward-only backend ending at now + <see cref="ReverseRawReader.LookAhead"/> instead — #691 review);
|
||||
/// "strictly after T" starts one storage resolution later (lossless: nothing is stored in between). Inside a tie
|
||||
/// cluster the bound is the cluster's first sample in the read's direction — the forward read's first, the reverse
|
||||
/// read's first (the forward order's last).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Resolution.</b> "On the bound" compares at the backend's storage resolution, because a microsecond backend
|
||||
/// (QuestDB) truncates the window bounds it is sent: a sample at <c>⌊T⌋</c> is inside a window starting at <c>T</c>
|
||||
/// there, so it must not also be returned as the start bound.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
internal static class HistoryBounds
|
||||
{
|
||||
private static readonly TimeSpan OpenBoundStep = TimeSpan.FromSeconds(1);
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #692: the placeholders this server synthesised (<see cref="NotFound"/>), by reference. A placeholder is known by
|
||||
/// this marker, never by its shape: a stored row can legitimately carry <c>Bad_BoundNotFound</c> with a null value (a
|
||||
/// client may write one through HistoryUpdate), and that row is data.
|
||||
/// </summary>
|
||||
private static readonly System.Runtime.CompilerServices.ConditionalWeakTable<DataValueSnapshot, object> Placeholders = new();
|
||||
|
||||
private static readonly object PlaceholderMark = new();
|
||||
|
||||
/// <summary>The end bound owed by a fresh read over the normalised window.</summary>
|
||||
/// <param name="startUtc">The (normalised) start: the later time of a reverse read.</param>
|
||||
/// <param name="endUtc">The (normalised) end; a one-sided read's sentinel (tick 0 / MaxValue) is an open bound.</param>
|
||||
/// <returns>The pending end bound.</returns>
|
||||
public static HistoryEndBound For(DateTime startUtc, DateTime endUtc) =>
|
||||
new(endUtc, Reverse: startUtc > endUtc, Inclusive: startUtc != endUtc, Open: IsOpen(endUtc));
|
||||
|
||||
/// <summary>
|
||||
/// Whether a returned value is a <see cref="NotFound"/> placeholder this server synthesised rather than a stored sample
|
||||
/// — by an explicit marker (#692), so a stored row whose status happens to be <c>Bad_BoundNotFound</c> is not taken for one.
|
||||
/// </summary>
|
||||
/// <param name="value">The value.</param>
|
||||
/// <returns><c>true</c> only for an instance <see cref="NotFound"/> produced.</returns>
|
||||
public static bool IsPlaceholder(DataValueSnapshot value) => Placeholders.TryGetValue(value, out _);
|
||||
|
||||
/// <summary>The pending end bound to store behind a continuation point, remembering the page's last returned time.</summary>
|
||||
/// <param name="bound">The pending end bound (null = bounds not requested).</param>
|
||||
/// <param name="page">The page being returned.</param>
|
||||
/// <returns>The bound to carry, or null.</returns>
|
||||
public static HistoryEndBound? Carry(HistoryEndBound? bound, IReadOnlyList<DataValueSnapshot> page) =>
|
||||
bound is null || page.Count == 0 || page[^1].SourceTimestampUtc is not { } last
|
||||
? bound
|
||||
: bound with { LastReturnedUtc = last };
|
||||
|
||||
/// <summary>The <c>Bad_BoundNotFound</c> placeholder (null value) stamped at <paramref name="atUtc"/>.</summary>
|
||||
/// <param name="atUtc">The bound's time.</param>
|
||||
/// <returns>The placeholder.</returns>
|
||||
public static DataValueSnapshot NotFound(DateTime atUtc)
|
||||
{
|
||||
var placeholder = new DataValueSnapshot(null, StatusCodes.BadBoundNotFound, atUtc, atUtc);
|
||||
Placeholders.AddOrUpdate(placeholder, PlaceholderMark);
|
||||
return placeholder;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The start bound to return ahead of the data, or <c>null</c> when a stored sample sits on StartTime (that sample is
|
||||
/// the bound, and it is already the data's first sample).
|
||||
/// </summary>
|
||||
public static async Task<DataValueSnapshot?> StartBoundAsync(
|
||||
IHistorianDataSource source, string tagname, DateTime startUtc, bool reverse, int maxTieCluster, DateTime nowUtc,
|
||||
CancellationToken ct)
|
||||
{
|
||||
if (IsOpen(startUtc)) return NotFound(startUtc);
|
||||
var found = reverse
|
||||
? await EarliestAtOrAfterAsync(source, tagname, startUtc, nowUtc, ct).ConfigureAwait(false)
|
||||
: await LatestAtOrBeforeAsync(source, tagname, startUtc, maxTieCluster, nowUtc, ct).ConfigureAwait(false);
|
||||
if (found is null) return NotFound(startUtc);
|
||||
return SameInstant(Ticks(found), startUtc.Ticks, Resolution(source)) ? null : found;
|
||||
}
|
||||
|
||||
/// <summary>The end bound: the stored sample on or beyond the end time, else the placeholder.</summary>
|
||||
/// <param name="source">The backend.</param>
|
||||
/// <param name="tagname">Historian tag.</param>
|
||||
/// <param name="bound">The pending end bound.</param>
|
||||
/// <param name="previousReturnedUtc">The last value's time already returned (stamps an open bound's placeholder).</param>
|
||||
/// <param name="maxTieCluster">The walk-back's row budget (forward-only backends).</param>
|
||||
/// <param name="nowUtc">Now (the walk-back's look-ahead clamp).</param>
|
||||
/// <param name="ct">Cancellation.</param>
|
||||
public static async Task<DataValueSnapshot> EndBoundAsync(
|
||||
IHistorianDataSource source, string tagname, HistoryEndBound bound, DateTime previousReturnedUtc,
|
||||
int maxTieCluster, DateTime nowUtc, CancellationToken ct)
|
||||
{
|
||||
if (bound.Open)
|
||||
{
|
||||
var ticks = bound.Reverse
|
||||
? Math.Max(0L, previousReturnedUtc.Ticks - OpenBoundStep.Ticks)
|
||||
: Math.Min(DateTime.MaxValue.Ticks, previousReturnedUtc.Ticks + OpenBoundStep.Ticks);
|
||||
return NotFound(new DateTime(ticks, DateTimeKind.Utc));
|
||||
}
|
||||
|
||||
if (bound.Resolved is { } resolved) return resolved; // #692: an earlier page already looked it up
|
||||
|
||||
DataValueSnapshot? found;
|
||||
if (bound.Reverse)
|
||||
found = await LatestAtOrBeforeAsync(source, tagname, bound.AtUtc, maxTieCluster, nowUtc, ct).ConfigureAwait(false);
|
||||
else if (bound.Inclusive)
|
||||
found = await EarliestAtOrAfterAsync(source, tagname, bound.AtUtc, nowUtc, ct).ConfigureAwait(false);
|
||||
else
|
||||
{
|
||||
var resolution = Resolution(source);
|
||||
found = bound.AtUtc.Ticks > DateTime.MaxValue.Ticks - resolution
|
||||
? null
|
||||
: await EarliestAtOrAfterAsync(source, tagname, bound.AtUtc.AddTicks(resolution), nowUtc, ct).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
return found ?? NotFound(bound.AtUtc);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Whether a data sample lies inside the data window of a bounded read — a forward window excludes the end time (a
|
||||
/// sample there is the end bound), compared at the storage resolution. A backend whose forward end is inclusive, or a
|
||||
/// resumed read that steps onto the end time, would otherwise return the end bound twice.
|
||||
/// </summary>
|
||||
public static bool InDataWindow(DataValueSnapshot sample, HistoryEndBound bound, long resolution)
|
||||
{
|
||||
if (!bound.Inclusive || bound.Open) return true;
|
||||
var ts = Align(Ticks(sample), resolution);
|
||||
var end = Align(bound.AtUtc.Ticks, resolution);
|
||||
return bound.Reverse ? ts > end : ts < end;
|
||||
}
|
||||
|
||||
/// <summary>Whether a resumed read's next start is already past the data window (only the end bound remains).</summary>
|
||||
public static bool DataWindowDrained(DateTime nextStartUtc, HistoryEndBound bound, long resolution)
|
||||
{
|
||||
if (!bound.Inclusive || bound.Open) return false;
|
||||
var next = Align(nextStartUtc.Ticks, resolution);
|
||||
var end = Align(bound.AtUtc.Ticks, resolution);
|
||||
return bound.Reverse ? next <= end : next >= end;
|
||||
}
|
||||
|
||||
/// <summary>The latest stored sample at or before <paramref name="atUtc"/>, or null.</summary>
|
||||
internal static async Task<DataValueSnapshot?> LatestAtOrBeforeAsync(
|
||||
IHistorianDataSource source, string tagname, DateTime atUtc, int maxTieCluster, DateTime nowUtc, CancellationToken ct)
|
||||
{
|
||||
if (atUtc.Ticks == 0) return null;
|
||||
var floor = new DateTime(0, DateTimeKind.Utc);
|
||||
IReadOnlyList<DataValueSnapshot> rows = source.SupportsReverseRaw
|
||||
? (await source.ReadRawAsync(tagname, atUtc, floor, 1, ct).ConfigureAwait(false)).Samples
|
||||
: (await ReverseRawReader.ReadAsync(source, tagname, atUtc, floor, 1, maxTieCluster, nowUtc, 0, ct)
|
||||
.ConfigureAwait(false)).Samples;
|
||||
var limit = Align(atUtc.Ticks, Resolution(source));
|
||||
foreach (var row in rows)
|
||||
if (Align(Ticks(row), Resolution(source)) <= limit)
|
||||
return row;
|
||||
return null;
|
||||
}
|
||||
|
||||
/// <summary>The earliest stored sample at or after <paramref name="atUtc"/>, or null.</summary>
|
||||
/// <remarks>
|
||||
/// #691 review: on a backend that reads forward only (the Gateway) the read's end is clamped to now +
|
||||
/// <see cref="ReverseRawReader.LookAhead"/> — the same horizon the walk-back uses — and a time beyond it is "not found"
|
||||
/// without a read. The HistorianGateway only checks <c>start <= end</c>, so a <c>DateTime.MaxValue</c> end would
|
||||
/// reach AVEVA as a year-9999 query end (unverified: a rejection, or a fabricated row). The clamp is NOT applied to a
|
||||
/// backend that reads newest-first natively (SqlServer / QuestDB): they bind <c>MaxValue</c> safely (pinned by the
|
||||
/// <c>Extreme_bounds</c> conformance test) and can legitimately hold samples further ahead (HistoryUpdate's
|
||||
/// <c>HistoryUpdateMaxFutureSkew</c> is configurable, and driver-stamped live values carry device time) — clamping
|
||||
/// there would turn a real end bound into <c>Bad_BoundNotFound</c>.
|
||||
/// </remarks>
|
||||
internal static async Task<DataValueSnapshot?> EarliestAtOrAfterAsync(
|
||||
IHistorianDataSource source, string tagname, DateTime atUtc, DateTime nowUtc, CancellationToken ct)
|
||||
{
|
||||
var endTicks = DateTime.MaxValue.Ticks;
|
||||
if (!source.SupportsReverseRaw)
|
||||
endTicks = nowUtc.Ticks > DateTime.MaxValue.Ticks - ReverseRawReader.LookAhead.Ticks
|
||||
? DateTime.MaxValue.Ticks
|
||||
: nowUtc.Ticks + ReverseRawReader.LookAhead.Ticks;
|
||||
if (atUtc.Ticks >= endTicks) return null;
|
||||
var rows = (await source.ReadRawAsync(tagname, atUtc, new DateTime(endTicks, DateTimeKind.Utc), 1, ct)
|
||||
.ConfigureAwait(false)).Samples;
|
||||
var limit = Align(atUtc.Ticks, Resolution(source));
|
||||
foreach (var row in rows)
|
||||
if (Align(Ticks(row), Resolution(source)) >= limit)
|
||||
return row;
|
||||
return null;
|
||||
}
|
||||
|
||||
private static bool IsOpen(DateTime t) => t.Ticks == 0 || t.Ticks == DateTime.MaxValue.Ticks;
|
||||
|
||||
private static bool SameInstant(long a, long b, long resolution) => Align(a, resolution) == Align(b, resolution);
|
||||
|
||||
private static long Resolution(IHistorianDataSource source) => Math.Max(1L, source.TimestampResolutionTicks);
|
||||
|
||||
private static long Ticks(DataValueSnapshot s) => (s.SourceTimestampUtc ?? DateTime.MinValue).Ticks;
|
||||
|
||||
private static long Align(long ticks, long resolution) => ticks - ticks % resolution;
|
||||
}
|
||||
@@ -55,6 +55,22 @@ namespace ZB.MOM.WW.OtOpcUa.OpcUaServer;
|
||||
/// <see cref="HistoryCursorKind.Raw"/> reverse reads on a forward-only backend only (Gitea #686): the segment width the
|
||||
/// walk-back ended on, so the next page starts at the observed density instead of re-discovering it. 0 = none.
|
||||
/// </param>
|
||||
/// <param name="EmptyFromTicks">
|
||||
/// Walked reverse reads only (Gitea #691): with <paramref name="EmptyToTicks"/>, a stretch <c>(from, to)</c> the previous
|
||||
/// page's walk proved holds no row, so the next page steps over it rather than re-walking it (to the 1970 floor, ~14
|
||||
/// reads, at the end of a deep page-through). 0/0 = none. The cursor lives in the session's in-memory store (never
|
||||
/// serialized), so a new field with a "none" default is all the versioning it needs.
|
||||
/// </param>
|
||||
/// <param name="EmptyToTicks">The exclusive upper end of that stretch.</param>
|
||||
/// <param name="PendingEndBound">
|
||||
/// <see cref="HistoryCursorKind.Raw"/> with <c>ReturnBounds = true</c> only (Gitea #691): the end bound still owed —
|
||||
/// Part 11 returns it with the last page. <c>null</c> = bounds not requested.
|
||||
/// </param>
|
||||
/// <param name="WithinTieCluster">
|
||||
/// <see cref="HistoryCursorKind.Raw"/> only (Gitea #692): the previous page was a slice of an oversized tie cluster at
|
||||
/// <paramref name="NextStartUtc"/> that still has un-emitted ties — the next page over-fetches the cluster directly
|
||||
/// instead of first re-reading <c>N + skip + 1</c> rows that are known to be ties.
|
||||
/// </param>
|
||||
internal sealed record HistoryContinuationState(
|
||||
string Tagname,
|
||||
DateTime NextStartUtc,
|
||||
@@ -65,7 +81,15 @@ internal sealed record HistoryContinuationState(
|
||||
long LastModifiedId = 0,
|
||||
DateTime WindowStartUtc = default,
|
||||
bool Reverse = false,
|
||||
long SegmentWidthTicks = 0);
|
||||
long SegmentWidthTicks = 0,
|
||||
long EmptyFromTicks = 0,
|
||||
long EmptyToTicks = 0,
|
||||
HistoryEndBound? PendingEndBound = null,
|
||||
bool WithinTieCluster = false)
|
||||
{
|
||||
/// <summary>The walk-back hint this cursor carries (#686/#689/#691).</summary>
|
||||
public ReverseWalkHint WalkHint => new(SegmentWidthTicks, EmptyFromTicks, EmptyToTicks);
|
||||
}
|
||||
|
||||
/// <summary>The HistoryRead kind that issued a <see cref="HistoryContinuationState"/>.</summary>
|
||||
internal enum HistoryCursorKind
|
||||
@@ -234,12 +258,11 @@ internal static class HistoryPaging
|
||||
/// <param name="cap">The client's per-page cap (<c>NumValuesPerNode</c>); must be > 0 to make progress.</param>
|
||||
/// <param name="boundaryT">The single SourceTimestamp every tie in the cluster shares.</param>
|
||||
/// <param name="endUtc">The (inclusive) upper bound of the read window; unchanged across pages.
|
||||
/// OpcUaServer-003, accepted by design: the backend contract
|
||||
/// (<c>IHistorianDataSource.ReadRawAsync</c>) treats <c>endUtc</c> as EXCLUSIVE, so the
|
||||
/// <c>next <= endUtc</c> test here is deliberately the conservative side — at worst one extra
|
||||
/// empty resume page (<c>[endUtc, endUtc)</c> → GoodNoData, no continuation point), never a
|
||||
/// dropped or duplicated sample. Tightening to <c><</c> would be correct only if the live
|
||||
/// backend's boundary is confirmed exclusive; it is not worth the opposite off-by-one risk.</param>
|
||||
/// The forward window is end-EXCLUSIVE (Part 11; the <c>IHistorianDataSource.ReadRawAsync</c> contract), so a drained
|
||||
/// cluster continues only while <c>next</c> is strictly before <paramref name="endUtc"/> truncated to
|
||||
/// <paramref name="resolutionTicks"/>. (OpcUaServer-003 had kept <c>next <= endUtc</c> as "the conservative side",
|
||||
/// but it was not: resuming AT EndTime makes the next read a <c>start == end</c> point read, which returns the samples
|
||||
/// at EndTime as data — #692 review F1; on a microsecond backend EndTime is truncated first.)</param>
|
||||
/// <param name="sliceStart">The index into the cluster the emitted slice starts at (clamped to the count).</param>
|
||||
/// <param name="sliceCount">How many ties the emitted slice contains (may be <c>0</c> on a stale-skip self-heal).</param>
|
||||
/// <param name="nextStartUtc">The next page's inclusive start: <paramref name="boundaryT"/> while the
|
||||
@@ -282,7 +305,9 @@ internal static class HistoryPaging
|
||||
else
|
||||
{
|
||||
var next = boundaryT.AddTicks(resolutionTicks);
|
||||
if (next <= endUtc) { nextStartUtc = next; nextSkip = 0; }
|
||||
// The forward window is end-EXCLUSIVE, at the backend's resolution: resuming AT EndTime would turn the next read
|
||||
// into a start == end point read, which returns the samples at EndTime as data (#692 review F1).
|
||||
if (next.Ticks < endUtc.Ticks - endUtc.Ticks % resolutionTicks) { nextStartUtc = next; nextSkip = 0; }
|
||||
else { nextStartUtc = null; nextSkip = 0; }
|
||||
}
|
||||
}
|
||||
|
||||
@@ -558,10 +558,15 @@ public sealed class OpcUaApplicationHost : IAsyncDisposable
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
if (_impersonateHandler is not null && _server?.CurrentInstance?.SessionManager is { } sessionManager)
|
||||
try
|
||||
{
|
||||
try { sessionManager.ImpersonateUser -= _impersonateHandler; }
|
||||
catch (Exception ex) { _logger.LogWarning(ex, "OpcUaApplicationHost: detaching ImpersonateUser threw"); }
|
||||
// An explicit SDK stop may already have halted CurrentInstance; still continue disposal.
|
||||
if (_impersonateHandler is not null && _server?.CurrentInstance?.SessionManager is { } sessionManager)
|
||||
sessionManager.ImpersonateUser -= _impersonateHandler;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "OpcUaApplicationHost: detaching ImpersonateUser threw");
|
||||
}
|
||||
_impersonateHandler = null;
|
||||
|
||||
|
||||
@@ -533,17 +533,32 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2
|
||||
/// <summary>
|
||||
/// The upper bound on the bounded over-fetch <see cref="ServeRawPaged"/> uses to page WITHIN an
|
||||
/// oversized "tie cluster" — more raw samples sharing one SourceTimestamp than the client's per-page
|
||||
/// cap. When a resume read stalls on such a cluster (the boundary-tie trim empties the page), the
|
||||
/// cap. When a resume read stalls on such a cluster (every row of its <c>N + skip + 1</c> read, past the
|
||||
/// trimmed ties, still sits on the boundary timestamp — #692: only a genuine cluster larger than the page), the
|
||||
/// paging over-fetches up to <c>MaxTieClusterOverfetch + 1</c> ties at that single timestamp (a
|
||||
/// <c>start == end</c> read) and slices through them via <see cref="HistoryPaging.SliceTieCluster"/>.
|
||||
/// A cluster strictly larger than this still fails that node <c>BadTooManyOperations</c> (the absurd-burst
|
||||
/// backstop; <c>BadHistoryOperationUnsupported</c> before #686). Also the per-read row budget of a reverse raw read
|
||||
/// backstop; <c>BadHistoryOperationUnsupported</c> before #686). Also the tie-cluster ceiling (not the segment size, #689) of a reverse read
|
||||
/// walked back from forward reads (<see cref="ReverseRawReader"/>). Mirrors the configured
|
||||
/// <c>ServerHistorianOptions.MaxTieClusterOverfetch</c>; the Host sets it at <c>StartAsync</c>. The
|
||||
/// default (65536) survives when the historian section is absent.
|
||||
/// </summary>
|
||||
public int MaxTieClusterOverfetch { get; set; } = 65536;
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #691: whether HistoryRead-Raw honours <c>ReturnBounds</c> (Part 11 bounding values, <see cref="HistoryBounds"/>)
|
||||
/// — <c>ServerHistorian:HonorReturnBounds</c>, default <c>true</c>. <c>false</c> ignores the flag, as before #691,
|
||||
/// including the modified-read <c>Bad_InvalidArgument</c>.
|
||||
/// </summary>
|
||||
public bool HonorReturnBounds { get; set; } = true;
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #694: <c>ServerHistorian:EventReadHorizon</c> — HistoryReadEvents never reads events older than
|
||||
/// <c>now − EventReadHorizon</c> (<see cref="EventHorizon"/>). <see cref="TimeSpan.Zero"/> (the default) = no horizon:
|
||||
/// every read is exact down to its own window (a walked Gateway reverse read proves "nothing older" down to 1970).
|
||||
/// </summary>
|
||||
public TimeSpan EventReadHorizon { get; set; } = TimeSpan.Zero;
|
||||
|
||||
/// <summary>
|
||||
/// Maximum historized nodes served concurrently within one HistoryRead batch (arch-review 03/S3).
|
||||
/// Mirrors <c>ServerHistorianOptions.HistoryReadBatchParallelism</c>; the Host sets it at
|
||||
@@ -3391,6 +3406,36 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2
|
||||
return NodeId.Null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #693: the AtTime value-type hint of a historized variable — its DataType's built-in type as a
|
||||
/// <see cref="TypeCode"/> when the node is a scalar of a type the historian evaluates (Boolean, the integers, Float,
|
||||
/// Double, String, DateTime); <c>null</c> for an array, an unknown node, or any other type. A UNS mirror carries its
|
||||
/// backing raw tag's DataType, so either NodeId gives the same hint.
|
||||
/// </summary>
|
||||
/// <param name="nodeId">The HistoryRead request's node id.</param>
|
||||
/// <returns>The hint, or null.</returns>
|
||||
internal TypeCode? AtTimeValueType(NodeId nodeId)
|
||||
{
|
||||
if (!_variables.TryGetValue(MapKey(nodeId), out var variable) || variable.ValueRank != ValueRanks.Scalar) return null;
|
||||
return TypeInfo.GetBuiltInType(variable.DataType) switch
|
||||
{
|
||||
BuiltInType.Boolean => TypeCode.Boolean,
|
||||
BuiltInType.SByte => TypeCode.SByte,
|
||||
BuiltInType.Byte => TypeCode.Byte,
|
||||
BuiltInType.Int16 => TypeCode.Int16,
|
||||
BuiltInType.UInt16 => TypeCode.UInt16,
|
||||
BuiltInType.Int32 => TypeCode.Int32,
|
||||
BuiltInType.UInt32 => TypeCode.UInt32,
|
||||
BuiltInType.Int64 => TypeCode.Int64,
|
||||
BuiltInType.UInt64 => TypeCode.UInt64,
|
||||
BuiltInType.Float => TypeCode.Single,
|
||||
BuiltInType.Double => TypeCode.Double,
|
||||
BuiltInType.String => TypeCode.String,
|
||||
BuiltInType.DateTime => TypeCode.DateTime,
|
||||
_ => null,
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>Map a Tag.DataType string ("Boolean", "Int32", "Float", "Double", "String",
|
||||
/// "DateTime") to the OPC UA built-in NodeId. Unknown names fall back to BaseDataType
|
||||
/// (matches CreateVariable's default for lazy-created nodes).</summary>
|
||||
@@ -5056,6 +5101,15 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2
|
||||
var work = new List<Func<Task>>(nodesToProcess.Count);
|
||||
foreach (var handle in nodesToProcess)
|
||||
{
|
||||
if (details.IsReadModified && details.ReturnBounds && HonorReturnBounds)
|
||||
{
|
||||
// Part 11 §6.5.3.3: returnBounds "shall be set to FALSE" for a modified read, "otherwise the Server returns
|
||||
// a Bad_InvalidArgument" (#691). Off with the ServerHistorian:HonorReturnBounds kill switch.
|
||||
errors[handle.Index] = StatusCodes.BadInvalidArgument;
|
||||
results[handle.Index] = new SdkHistoryReadResult { StatusCode = StatusCodes.BadInvalidArgument };
|
||||
continue;
|
||||
}
|
||||
|
||||
if (details.IsReadModified)
|
||||
{
|
||||
// Part 11 modified-value history (#668): the prior values every HistoryUpdate recorded. Gated by the
|
||||
@@ -5070,11 +5124,12 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2
|
||||
work.Add(() => ServeRawPagedAsync(
|
||||
handle, session, nodesToRead, results, errors,
|
||||
details.StartTime, details.EndTime, details.NumValuesPerNode,
|
||||
ct));
|
||||
ct, details.ReturnBounds && HonorReturnBounds));
|
||||
}
|
||||
|
||||
RunBounded(work, HistoryReadBatchParallelism);
|
||||
});
|
||||
NormalizeHistoryValues(nodesToProcess, results);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
@@ -5160,14 +5215,18 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2
|
||||
var work = new List<Func<Task>>(nodesToProcess.Count);
|
||||
foreach (var handle in nodesToProcess)
|
||||
{
|
||||
// #693: the node's scalar value type, for a backend whose stored values carry none (the Gateway).
|
||||
var valueType = AtTimeValueType(handle.NodeId);
|
||||
work.Add(() => ServeNodeAsync(handle, results, errors, (source, tagname, token) => source.ReadAtTimeAsync(
|
||||
tagname,
|
||||
timestamps,
|
||||
valueType,
|
||||
token), ct));
|
||||
}
|
||||
|
||||
RunBounded(work, HistoryReadBatchParallelism);
|
||||
});
|
||||
NormalizeHistoryValues(nodesToProcess, results);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
@@ -5248,6 +5307,7 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2
|
||||
var endUtc = details.EndTime;
|
||||
var numValuesPerNode = details.NumValuesPerNode;
|
||||
var boundarySkip = 0;
|
||||
var walkHint = default(ReverseWalkHint); // #691.4: a walked-back read's width + proven-empty stretch
|
||||
bool reverse;
|
||||
|
||||
if (resumed)
|
||||
@@ -5271,35 +5331,43 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2
|
||||
// The direction is the CURSOR's (Gitea #675), never re-derived from the resume request's details: a
|
||||
// reverse read resumes reverse, a forward one forward.
|
||||
reverse = state.Reverse;
|
||||
walkHint = state.WalkHint;
|
||||
}
|
||||
else
|
||||
{
|
||||
// Part 11: with one bound unset, an unset EndTime (MinValue) with a StartTime means "page FORWARD from
|
||||
// StartTime" — normalise it to an open upper bound, or the cursor would store MinValue and the read would
|
||||
// look reversed. An unset StartTime with an EndTime and a count cap means "page BACKWARD from EndTime"
|
||||
// (#675, mirroring raw): the reverse window (MinValue, EndTime] — but only on a backend that can read
|
||||
// newest-first. Elsewhere (Gateway, the default backend) and without a count cap an end-only read keeps its
|
||||
// historical forward [MinValue, EndTime) meaning rather than becoming a refused reverse read.
|
||||
// (#675, mirroring raw): the reverse window (MinValue, EndTime] — on every backend since #691.4 (a backend
|
||||
// that cannot read newest-first is walked back below). Without a count cap an end-only read keeps its
|
||||
// forward [MinValue, EndTime) meaning.
|
||||
if (endUtc == DateTime.MinValue && startUtc != DateTime.MinValue)
|
||||
endUtc = new DateTime(DateTime.MaxValue.Ticks, DateTimeKind.Utc); // Utc kind: a proto Timestamp conversion rejects Unspecified
|
||||
else if (numValuesPerNode > 0 && startUtc == DateTime.MinValue && endUtc != DateTime.MinValue
|
||||
&& source.SupportsReverseEvents)
|
||||
else if (numValuesPerNode > 0 && startUtc == DateTime.MinValue && endUtc != DateTime.MinValue)
|
||||
(startUtc, endUtc) = (endUtc, new DateTime(0, DateTimeKind.Utc));
|
||||
|
||||
// Part 11: StartTime later than EndTime is a reverse read — events newest-first over (end, start].
|
||||
reverse = startUtc > endUtc;
|
||||
|
||||
// #694: the opt-in events horizon raises the window's earlier bound (the cursor then carries it). A window
|
||||
// wholly older than it is empty and is not sent (a forward [start, end) clamped past its end would be refused).
|
||||
if (!EventHorizon.TryClamp(ref startUtc, ref endUtc, reverse, EventReadHorizon, Clock.GetUtcNow().UtcDateTime))
|
||||
{
|
||||
results[handle.Index] = new SdkHistoryReadResult
|
||||
{
|
||||
StatusCode = StatusCodes.GoodNoData,
|
||||
HistoryData = new ExtensionObject(ProjectEvents([], selectClauses)),
|
||||
};
|
||||
errors[handle.Index] = ServiceResult.Good;
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// An explicit reverse read (or a reverse cursor resumed after the source was swapped) on a backend that cannot
|
||||
// read newest-first: refuse it BEFORE calling the source — never the earliest events in ascending order as if
|
||||
// they were the answer. Checked here rather than by catching an exception, so a NotSupportedException thrown
|
||||
// from inside a backend (a driver library, the projection) stays a backend fault.
|
||||
if (reverse && !source.SupportsReverseEvents)
|
||||
{
|
||||
errors[handle.Index] = StatusCodes.BadHistoryOperationUnsupported;
|
||||
results[handle.Index] = new SdkHistoryReadResult { StatusCode = StatusCodes.BadHistoryOperationUnsupported };
|
||||
return;
|
||||
}
|
||||
// A reverse read on a backend that cannot read newest-first (the Gateway) is never sent there (#675 refused it
|
||||
// here); since #691.4 it is walked back from bounded forward reads — the raw walk (#686), with ties ordered by
|
||||
// EventId because the gateway orders by event time only. Decided by the capability, not by catching an
|
||||
// exception, so a NotSupportedException thrown from inside a backend stays a backend fault.
|
||||
var walked = reverse && !source.SupportsReverseEvents;
|
||||
|
||||
// NOT under the node-manager Lock — awaiting the async source is safe here.
|
||||
// NumValuesPerNode==0 means "no limit — return ALL values" per OPC UA Part 4/11, so it is translated
|
||||
@@ -5309,8 +5377,26 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2
|
||||
var fetch = EventMaxEvents(numValuesPerNode);
|
||||
if (numValuesPerNode > 0 && boundarySkip > 0)
|
||||
fetch = (int)Math.Min(int.MaxValue - 1L, (long)fetch + boundarySkip);
|
||||
var sourceResult = await source.ReadEventsAsync(
|
||||
sourceName, startUtc, endUtc, fetch, reverse, ct).ConfigureAwait(false);
|
||||
HistoricalEventsResult sourceResult;
|
||||
if (walked)
|
||||
{
|
||||
// Walk for one more than the fetch: finding it is the "more events exist" signal a source gives with its
|
||||
// continuation point (the walk itself cannot flag truncation).
|
||||
var want = numValuesPerNode == 0 ? 0u : (uint)Math.Min(uint.MaxValue - 1L, (long)fetch + 1);
|
||||
var (rows, next) = await ReverseEventsReader.ReadAsync(
|
||||
source, sourceName, startUtc, endUtc, want, MaxTieClusterOverfetch, Clock.GetUtcNow().UtcDateTime,
|
||||
walkHint, ct)
|
||||
.ConfigureAwait(false);
|
||||
walkHint = next;
|
||||
var more = want > 0 && rows.Count > fetch;
|
||||
if (more) rows.RemoveRange(fetch, rows.Count - fetch);
|
||||
sourceResult = new HistoricalEventsResult(rows, more ? [0x01] : null);
|
||||
}
|
||||
else
|
||||
{
|
||||
sourceResult = await source.ReadEventsAsync(
|
||||
sourceName, startUtc, endUtc, fetch, reverse, ct).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
// Truncation is the SOURCE's word (non-null continuation point) — never inferred from the count, so a
|
||||
// page that happens to be exactly cap-sized but complete never emits a point.
|
||||
@@ -5350,6 +5436,12 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2
|
||||
sourceName, startUtc, startUtc.AddTicks(resolution), overfetchCap, reverse: false, ct)
|
||||
.ConfigureAwait(false)).Events;
|
||||
|
||||
// A walked read (#691.4) emitted its ties sorted by EventId; the gateway returns them in any order and its
|
||||
// end is inclusive — keep exactly the cluster, in the walk's order, so the skip indexes the same events.
|
||||
if (walked)
|
||||
cluster = cluster.Where(e => e.EventTimeUtc == startUtc)
|
||||
.OrderBy(e => e, EventIdTieOrder.Instance).ToList();
|
||||
|
||||
// The over-fetch reads the cluster FORWARD (ascending EventId), but a reverse read emitted its ties — and
|
||||
// counted boundarySkip — in descending order. Slice in the read's own direction, as raw does.
|
||||
if (reverse) cluster = cluster.Reverse().ToList();
|
||||
@@ -5378,7 +5470,8 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2
|
||||
{
|
||||
pendingCursor = new HistoryContinuationState(
|
||||
sourceName ?? string.Empty, resumeAt, endUtc, nextSkip, numValuesPerNode, HistoryCursorKind.Events,
|
||||
Reverse: reverse);
|
||||
Reverse: reverse, SegmentWidthTicks: walkHint.WidthTicks, EmptyFromTicks: walkHint.EmptyFromTicks,
|
||||
EmptyToTicks: walkHint.EmptyToTicks);
|
||||
}
|
||||
}
|
||||
else if (truncated && events.Count > 0)
|
||||
@@ -5390,7 +5483,8 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2
|
||||
for (var i = events.Count - 1; i >= 0 && events[i].EventTimeUtc == boundary; i--) skip++;
|
||||
pendingCursor = new HistoryContinuationState(
|
||||
sourceName ?? string.Empty, boundary, endUtc, skip, numValuesPerNode, HistoryCursorKind.Events,
|
||||
Reverse: reverse);
|
||||
Reverse: reverse, SegmentWidthTicks: walkHint.WidthTicks, EmptyFromTicks: walkHint.EmptyFromTicks,
|
||||
EmptyToTicks: walkHint.EmptyToTicks);
|
||||
}
|
||||
|
||||
var historyEvent = ProjectEvents(events, selectClauses);
|
||||
@@ -5411,13 +5505,21 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2
|
||||
errors[handle.Index] = StatusCodes.BadTimeout;
|
||||
results[handle.Index] = new SdkHistoryReadResult { StatusCode = StatusCodes.BadTimeout };
|
||||
}
|
||||
catch (HistoryReadLimitExceededException ex)
|
||||
{
|
||||
// A request limit (#691.4: a walked-back reverse read met a tie cluster over the budget) — BadTooManyOperations
|
||||
// for THIS node, as on the raw path; not a backend fault.
|
||||
_logger.LogWarning("OtOpcUaNodeManager: HistoryReadEvents limit exceeded for node {NodeId}: {Message}", handle.NodeId, ex.Message);
|
||||
errors[handle.Index] = StatusCodes.BadTooManyOperations;
|
||||
results[handle.Index] = new SdkHistoryReadResult { StatusCode = StatusCodes.BadTooManyOperations };
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
// One node's backend failure must not throw out of the batch — surface Bad for THIS node
|
||||
// only. Logged via the injected ILogger.
|
||||
_logger.LogError(ex, "OtOpcUaNodeManager: HistoryReadEvents failed for node {NodeId}", handle.NodeId);
|
||||
errors[handle.Index] = StatusCodes.BadHistoryOperationUnsupported;
|
||||
results[handle.Index] = new SdkHistoryReadResult { StatusCode = StatusCodes.BadHistoryOperationUnsupported };
|
||||
errors[handle.Index] = HistoryReadFailureStatus(ex);
|
||||
results[handle.Index] = new SdkHistoryReadResult { StatusCode = HistoryReadFailureStatus(ex) };
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6133,7 +6235,8 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2
|
||||
// Bad status for THIS node only. Logged through the injected ILogger
|
||||
// rather than swallowed silently.
|
||||
_logger.LogError(ex, "OtOpcUaNodeManager: HistoryRead failed for node {NodeId}", handle.NodeId);
|
||||
errors[handle.Index] = StatusCodes.BadHistoryOperationUnsupported;
|
||||
errors[handle.Index] = HistoryReadFailureStatus(ex);
|
||||
results[handle.Index] = new SdkHistoryReadResult { StatusCode = HistoryReadFailureStatus(ex) };
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6144,11 +6247,16 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2
|
||||
/// <list type="bullet">
|
||||
/// <item><b>Fresh read</b> (no inbound continuation point): read the window from
|
||||
/// <c>details.StartTime</c> to <paramref name="endUtc"/> capped at
|
||||
/// <paramref name="numValuesPerNode"/>. If the page comes back FULL (exactly the cap, and the
|
||||
/// cap is > 0), store a resume cursor and emit a continuation point.</item>
|
||||
/// <paramref name="numValuesPerNode"/> <b>+ 1</b> — one lookahead row (#692). If the lookahead row came
|
||||
/// back, return the first N, store a resume cursor and emit a continuation point; otherwise the page is
|
||||
/// the last and carries none (so a total that is an exact multiple of N ends on a full page, not on an
|
||||
/// empty trailing one).</item>
|
||||
/// <item><b>Resume read</b> (inbound continuation point present): take the stored cursor, read
|
||||
/// the next page from the boundary forward, trim already-emitted boundary ties, and emit a
|
||||
/// FRESH continuation point only if THIS page is also full — else null (done).</item>
|
||||
/// <c>N + boundarySkip + 1</c> rows from the boundary (the ties about to be trimmed, the page, and the
|
||||
/// lookahead row), trim the already-emitted boundary ties, and continue exactly as a fresh read — so every
|
||||
/// page carries N values. (Asking for only N used to net N − skip per resumed page, and at N = 1 an EMPTY
|
||||
/// page between every value: the trimmed page looked like a stalled tie cluster and paid a
|
||||
/// <see cref="MaxTieClusterOverfetch"/> + 1 over-fetch to step one tick — Gitea #692.)</item>
|
||||
/// </list>
|
||||
/// The resume cursor is tie-safe (see <see cref="HistoryPaging.ComputeResumeCursor"/> /
|
||||
/// <see cref="HistoryPaging.TrimBoundaryDuplicates"/>): the next page resumes from the boundary
|
||||
@@ -6167,6 +6275,11 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2
|
||||
/// <param name="startTimeUtc">The request window's (inclusive) lower bound, used for a fresh read.</param>
|
||||
/// <param name="endUtc">The (inclusive) upper bound of the read window; unchanged across pages.</param>
|
||||
/// <param name="numValuesPerNode">The client's per-page cap; <c>0</c> means "all values, no paging".</param>
|
||||
/// <param name="ct">Cancellation (the per-request deadline).</param>
|
||||
/// <param name="returnBounds">
|
||||
/// Part 11 bounding values (#691, <see cref="HistoryBounds"/>): the start bound leads the first page, the end bound
|
||||
/// ends the last, and both count toward <paramref name="numValuesPerNode"/>. A resumed read takes it from the cursor.
|
||||
/// </param>
|
||||
private async Task ServeRawPagedAsync(
|
||||
NodeHandle handle,
|
||||
ISession? session,
|
||||
@@ -6176,7 +6289,8 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2
|
||||
DateTime startTimeUtc,
|
||||
DateTime endUtc,
|
||||
uint numValuesPerNode,
|
||||
CancellationToken ct)
|
||||
CancellationToken ct,
|
||||
bool returnBounds = false)
|
||||
{
|
||||
var inboundCp = nodesToRead[handle.Index].ContinuationPoint;
|
||||
|
||||
@@ -6184,7 +6298,9 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2
|
||||
{
|
||||
DateTime startUtc;
|
||||
var boundarySkip = 0;
|
||||
long segmentWidth = 0; // #686: the walk-back's width, carried page to page
|
||||
var walkHint = default(ReverseWalkHint); // #686/#689/#691: the walk-back's width + proven-empty stretch, page to page
|
||||
HistoryEndBound? endBound = null; // #691: the end bound still owed (null = bounds not requested)
|
||||
var withinTieCluster = false; // #692: the previous page sliced an oversized tie cluster that still has ties
|
||||
|
||||
string tagname;
|
||||
if (inboundCp is { Length: > 0 })
|
||||
@@ -6205,7 +6321,9 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2
|
||||
boundarySkip = state.BoundarySkipCount;
|
||||
endUtc = state.EndUtc;
|
||||
numValuesPerNode = state.NumValuesPerNode;
|
||||
segmentWidth = state.SegmentWidthTicks;
|
||||
walkHint = state.WalkHint;
|
||||
endBound = state.PendingEndBound;
|
||||
withinTieCluster = state.WithinTieCluster;
|
||||
}
|
||||
else
|
||||
{
|
||||
@@ -6231,50 +6349,215 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2
|
||||
else if (startUtc == DateTime.MinValue && endUtc != DateTime.MinValue)
|
||||
(startUtc, endUtc) = (endUtc, new DateTime(0, DateTimeKind.Utc));
|
||||
}
|
||||
|
||||
if (returnBounds) endBound = HistoryBounds.For(startUtc, endUtc);
|
||||
}
|
||||
|
||||
var nowUtc = Clock.GetUtcNow().UtcDateTime;
|
||||
var resolution = Math.Max(1L, HistorianDataSource.TimestampResolutionTicks);
|
||||
|
||||
// #691: the data is drained — the end bound ends this page when it has room (Part 11: bounds count toward
|
||||
// NumValuesPerNode), else it is the whole next page. Returns the continuation point to emit, if any.
|
||||
async Task<byte[]?> AppendEndBoundAsync(List<DataValueSnapshot> page, DateTime resumedAtUtc)
|
||||
{
|
||||
// The previous returned value stamps an open bound's placeholder (Part 11 Table 1 footnotes a/b): this
|
||||
// page's last, else the last of an earlier page (carried in the cursor), else where the read resumed.
|
||||
var previous = (page.Count > 0 ? page[^1].SourceTimestampUtc : null)
|
||||
?? endBound!.LastReturnedUtc ?? resumedAtUtc;
|
||||
if (numValuesPerNode == 0 || page.Count < numValuesPerNode)
|
||||
{
|
||||
page.Add(await HistoryBounds.EndBoundAsync(
|
||||
HistorianDataSource, tagname, endBound!, previous, MaxTieClusterOverfetch, nowUtc, ct).ConfigureAwait(false));
|
||||
return null;
|
||||
}
|
||||
|
||||
return _historyContinuationStore.Save(session, new HistoryContinuationState(
|
||||
tagname, previous, endUtc, 0, numValuesPerNode, SegmentWidthTicks: walkHint.WidthTicks,
|
||||
EmptyFromTicks: walkHint.EmptyFromTicks, EmptyToTicks: walkHint.EmptyToTicks,
|
||||
PendingEndBound: HistoryBounds.Carry(endBound!, page)! with { DataExhausted = true }));
|
||||
}
|
||||
|
||||
// #691 review: whether a bounded read whose start bound was NOT found has any data or a found end bound — only
|
||||
// asked when a one-value page would otherwise be the start-bound placeholder alone. #692: what it learns rides the
|
||||
// cursor — the walk hint its walk-back computed, and, when there is no data, the end bound it found (so the next
|
||||
// page is that bound with no backend read at all).
|
||||
async Task<(bool Any, HistoryEndBound Pending)> BoundedReadHasAnythingAsync()
|
||||
{
|
||||
var src = HistorianDataSource;
|
||||
IReadOnlyList<DataValueSnapshot> first;
|
||||
if (startUtc > endUtc && !src.SupportsReverseRaw)
|
||||
{
|
||||
var walked = await ReverseRawReader.ReadAsync(
|
||||
src, tagname, startUtc, endUtc, 1, MaxTieClusterOverfetch, nowUtc, walkHint, ct)
|
||||
.ConfigureAwait(false);
|
||||
walkHint = walked.Hint;
|
||||
first = walked.Samples;
|
||||
}
|
||||
else first = (await src.ReadRawAsync(tagname, startUtc, endUtc, 1, ct).ConfigureAwait(false)).Samples;
|
||||
|
||||
if (first.Any(x => HistoryBounds.InDataWindow(x, endBound!, resolution))) return (true, endBound!);
|
||||
var end = await HistoryBounds.EndBoundAsync(
|
||||
src, tagname, endBound!, startUtc, MaxTieClusterOverfetch, nowUtc, ct).ConfigureAwait(false);
|
||||
return HistoryBounds.IsPlaceholder(end)
|
||||
? (false, endBound!)
|
||||
: (true, endBound! with { DataExhausted = true, Resolved = end });
|
||||
}
|
||||
|
||||
// #691: a resumed bounded read whose data is drained (the end bound did not fit on the previous page, or the
|
||||
// resume point already sits on the end time) owes only the end bound.
|
||||
if (endBound is not null && inboundCp is { Length: > 0 }
|
||||
&& (endBound.DataExhausted || HistoryBounds.DataWindowDrained(startUtc, endBound, resolution)))
|
||||
{
|
||||
var onlyBound = new List<DataValueSnapshot>(1);
|
||||
var afterCp = await AppendEndBoundAsync(onlyBound, startUtc).ConfigureAwait(false);
|
||||
results[handle.Index] = new SdkHistoryReadResult
|
||||
{
|
||||
StatusCode = StatusCodes.Good,
|
||||
HistoryData = new ExtensionObject(ToHistoryDataFromSamples(onlyBound)),
|
||||
ContinuationPoint = afterCp,
|
||||
};
|
||||
errors[handle.Index] = ServiceResult.Good;
|
||||
return;
|
||||
}
|
||||
|
||||
// #691: the start bound leads the first page and counts toward the cap; null when a sample sits on StartTime
|
||||
// (that sample is the bound, and it is the data's first).
|
||||
DataValueSnapshot? startBound = null;
|
||||
var readCap = numValuesPerNode;
|
||||
if (endBound is not null && inboundCp is not { Length: > 0 })
|
||||
{
|
||||
startBound = await HistoryBounds.StartBoundAsync(
|
||||
HistorianDataSource, tagname, startUtc, endBound.Reverse, MaxTieClusterOverfetch, nowUtc, ct)
|
||||
.ConfigureAwait(false);
|
||||
var (anything, pending) = startBound is not null && numValuesPerNode == 1
|
||||
? HistoryBounds.IsPlaceholder(startBound)
|
||||
? await BoundedReadHasAnythingAsync().ConfigureAwait(false)
|
||||
: (true, endBound)
|
||||
: (false, endBound);
|
||||
if (startBound is not null && numValuesPerNode == 1 && anything)
|
||||
{
|
||||
// The page is the start bound alone; the data starts at StartTime on the next page.
|
||||
var boundOnlyCp = _historyContinuationStore.Save(session, new HistoryContinuationState(
|
||||
tagname, startUtc, endUtc, 0, numValuesPerNode, SegmentWidthTicks: walkHint.WidthTicks,
|
||||
EmptyFromTicks: walkHint.EmptyFromTicks, EmptyToTicks: walkHint.EmptyToTicks,
|
||||
PendingEndBound: HistoryBounds.Carry(pending, [startBound])));
|
||||
results[handle.Index] = new SdkHistoryReadResult
|
||||
{
|
||||
StatusCode = StatusCodes.Good,
|
||||
HistoryData = new ExtensionObject(ToHistoryDataFromSamples([startBound])),
|
||||
ContinuationPoint = boundOnlyCp,
|
||||
};
|
||||
errors[handle.Index] = ServiceResult.Good;
|
||||
return;
|
||||
}
|
||||
|
||||
if (startBound is not null && numValuesPerNode == 1)
|
||||
{
|
||||
// Nothing at all — no data and no bound found (review of #691): the pre-bounds GoodNoData answer.
|
||||
results[handle.Index] = new SdkHistoryReadResult
|
||||
{
|
||||
StatusCode = StatusCodes.GoodNoData,
|
||||
HistoryData = new ExtensionObject(ToHistoryDataFromSamples([])),
|
||||
ContinuationPoint = null,
|
||||
};
|
||||
errors[handle.Index] = ServiceResult.Good;
|
||||
return;
|
||||
}
|
||||
|
||||
if (startBound is not null && numValuesPerNode > 0) readCap = numValuesPerNode - 1;
|
||||
}
|
||||
|
||||
// #692: the page's read asks for the boundary ties it is about to trim (a resumed page re-reads them, because the
|
||||
// cursor resumes AT the boundary timestamp, inclusive) plus ONE lookahead row past the page. Without them a resumed
|
||||
// page netted N − skip values, and at N = 1 nothing at all: the lone boundary sample came back, was trimmed, the
|
||||
// "full" empty page was taken for a stalled tie cluster, and a MaxTieClusterOverfetch + 1 over-fetch stepped one
|
||||
// tick to return an EMPTY page with a continuation point — between every value. The lookahead row is what says
|
||||
// "more data follows": a page whose read found no row past it is the last, so the read no longer ends with an
|
||||
// empty trailing page either. (RawPageLimit.Plan computes the request: page + boundary ties + 1.)
|
||||
var resumed = inboundCp is { Length: > 0 };
|
||||
var boundaryTies = resumed ? Math.Max(0, boundarySkip) : 0;
|
||||
var want = readCap; // N, less the start bound on a bounded first page; 0 = all values
|
||||
|
||||
// HistoryRead is NOT under the node-manager Lock — awaiting the async source is safe.
|
||||
// A reverse window (explicit, resumed, or the end-only read above) goes to the backend only when it declares it
|
||||
// can read newest-first; otherwise it is walked back from bounded forward reads (#686 — the Gateway rejects
|
||||
// start > end, which failed the node and counted as a backend fault).
|
||||
// A cursor left inside an oversized tie cluster (#692) skips this read: every row it could return is a tie.
|
||||
// #694: a backend with a row ceiling (MaxRawValuesPerRead = L > 0, a strict gateway) sizes the page to it — the
|
||||
// read asks for page + boundary ties + lookahead ≤ L, and a full page carries a continuation point as always
|
||||
// (uncapped reads included). No room left after the boundary ties ⇒ the cursor sits in a tie cluster as large as
|
||||
// a read: straight to the over-fetch. See RawPageLimit.
|
||||
var source = HistorianDataSource;
|
||||
HistorianRead sourceResult;
|
||||
if (startUtc > endUtc && !source.SupportsReverseRaw)
|
||||
IReadOnlyList<DataValueSnapshot> samples = [];
|
||||
var backendFull = false;
|
||||
int pageLimit;
|
||||
if (!withinTieCluster)
|
||||
{
|
||||
var page = await ReverseRawReader.ReadAsync(
|
||||
source, tagname, startUtc, endUtc, numValuesPerNode, MaxTieClusterOverfetch,
|
||||
Clock.GetUtcNow().UtcDateTime, segmentWidth, ct)
|
||||
.ConfigureAwait(false);
|
||||
segmentWidth = page.WidthHintTicks;
|
||||
sourceResult = new HistorianRead(page.Samples, ContinuationPoint: null);
|
||||
var read = await RawPageLimit.ReadAsync(source, want, boundaryTies, async plan =>
|
||||
{
|
||||
if (startUtc > endUtc && !source.SupportsReverseRaw)
|
||||
{
|
||||
var page = await ReverseRawReader.ReadAsync(
|
||||
source, tagname, startUtc, endUtc, plan.RequestCap, MaxTieClusterOverfetch,
|
||||
nowUtc, walkHint, ct, retryOnLearnedLimit: false)
|
||||
.ConfigureAwait(false);
|
||||
walkHint = page.Hint;
|
||||
return new HistorianRead(page.Samples, ContinuationPoint: null);
|
||||
}
|
||||
|
||||
return await source.ReadRawAsync(tagname, startUtc, endUtc, plan.RequestCap, ct).ConfigureAwait(false);
|
||||
}).ConfigureAwait(false);
|
||||
pageLimit = read.Plan.Limit;
|
||||
|
||||
if (read.Plan.NoRoom)
|
||||
{
|
||||
if (!resumed)
|
||||
throw new HistoryReadLimitExceededException(pageLimit,
|
||||
$"The backend's row ceiling ({pageLimit} values per read) leaves no room for a raw page of '{tagname}'.");
|
||||
withinTieCluster = true; // every row a read could return is a boundary tie already emitted
|
||||
}
|
||||
else
|
||||
{
|
||||
readCap = read.Plan.PageCap;
|
||||
var sourceResult = read.Page!;
|
||||
backendFull = HistoryPaging.IsFullPage(sourceResult.Samples.Count, read.Plan.RequestCap);
|
||||
|
||||
// On a resume read, drop the boundary ties already returned on the prior page.
|
||||
samples = resumed
|
||||
? HistoryPaging.TrimBoundaryDuplicates(sourceResult.Samples, startUtc, boundarySkip)
|
||||
: sourceResult.Samples;
|
||||
|
||||
// #691: a bounded read's data window excludes the end time (a sample there is the end bound). A backend whose
|
||||
// forward end is inclusive (the gateway's own) returns it as data; dropping it also means the data is drained.
|
||||
if (endBound is not null)
|
||||
{
|
||||
var inside = samples.Where(x => HistoryBounds.InDataWindow(x, endBound, resolution)).ToList();
|
||||
if (inside.Count != samples.Count)
|
||||
{
|
||||
samples = inside;
|
||||
backendFull = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
sourceResult = await source
|
||||
.ReadRawAsync(tagname, startUtc, endUtc, numValuesPerNode, ct)
|
||||
.ConfigureAwait(false);
|
||||
pageLimit = source.MaxRawValuesPerRead;
|
||||
}
|
||||
|
||||
var backendFull = HistoryPaging.IsFullPage(sourceResult.Samples.Count, numValuesPerNode);
|
||||
|
||||
// On a resume read, drop the boundary ties already returned on the prior page.
|
||||
var samples = inboundCp is { Length: > 0 }
|
||||
? HistoryPaging.TrimBoundaryDuplicates(sourceResult.Samples, startUtc, boundarySkip)
|
||||
: sourceResult.Samples;
|
||||
|
||||
// Oversized tie cluster: a resume read whose FULL backend page does not advance past the resume
|
||||
// boundary timestamp `startUtc`. That happens when more samples share `startUtc` than the page
|
||||
// cap — a tie cluster larger than NumValuesPerNode. The fixed-(start,end,cap) backend can only
|
||||
// ever return the first `cap` of those ties, so the (timestamp, skip) cursor alone can never page
|
||||
// past (or even reliably within) the cluster: either the boundary-tie trim empties the page, or
|
||||
// it re-emits the same head ties forever. Detect BOTH stall shapes — an empty trimmed page, or a
|
||||
// non-empty page whose LAST sample is still at `startUtc` (no forward progress) — and, rather
|
||||
// than fail the read, over-fetch the WHOLE cluster with an explicit, BOUNDED cap (a start==end
|
||||
// read at the boundary timestamp) and page WITHIN the timestamp via SliceTieCluster.
|
||||
var stalledOnTieCluster = inboundCp is { Length: > 0 }
|
||||
&& backendFull
|
||||
&& (samples.Count == 0
|
||||
|| (samples[^1].SourceTimestampUtc ?? DateTime.MinValue) == startUtc);
|
||||
// Oversized tie cluster: a resume read whose FULL backend read does not advance past the resume boundary
|
||||
// timestamp `startUtc` — every row it returned past the trimmed ties, the lookahead row included, still sits on
|
||||
// `startUtc`. That happens only when more samples share `startUtc` than the page plus the already-emitted ties
|
||||
// plus one: a genuine tie cluster larger than the page. The fixed-(start,end,cap) backend can only ever return
|
||||
// the first `cap` of those ties, so growing the read page by page would re-read the cluster quadratically.
|
||||
// Instead over-fetch the WHOLE cluster with an explicit, BOUNDED cap (a start==end read at the boundary
|
||||
// timestamp) and page WITHIN the timestamp via SliceTieCluster. (#692: before the read asked for the trimmed ties
|
||||
// and a lookahead row, a resumed N = 1 page tripped this on every single sample.)
|
||||
var stalledOnTieCluster = resumed
|
||||
&& (withinTieCluster
|
||||
|| (backendFull
|
||||
&& (samples.Count == 0
|
||||
|| (samples[^1].SourceTimestampUtc ?? DateTime.MinValue) == startUtc)));
|
||||
if (stalledOnTieCluster)
|
||||
{
|
||||
// The over-fetch cap MUST be explicit and non-zero: a cap of 0 falls back to the backend's
|
||||
@@ -6282,7 +6565,12 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2
|
||||
// lets us DETECT a cluster strictly larger than the bound (the absurd-burst backstop below).
|
||||
// Compute in uint so +1 can never wrap: Math.Max(1,…) also guards against a zero/negative
|
||||
// config value slipping through (Validate() warns but does not block startup).
|
||||
var overfetchCap = (uint)Math.Max(1, MaxTieClusterOverfetch) + 1u;
|
||||
// #694: on a backend with a row ceiling the bound is also at most L − 1, so the over-fetch (bound + 1 rows)
|
||||
// fits one read and an oversized cluster is refused here rather than by a gateway refusal.
|
||||
var overfetchBound = pageLimit > 0
|
||||
? Math.Min(Math.Max(1, MaxTieClusterOverfetch), Math.Max(1, pageLimit - 1))
|
||||
: Math.Max(1, MaxTieClusterOverfetch);
|
||||
var overfetchCap = (uint)overfetchBound + 1u;
|
||||
IReadOnlyList<DataValueSnapshot> cluster = (await HistorianDataSource
|
||||
.ReadRawAsync(tagname, startUtc, startUtc, overfetchCap, ct)
|
||||
.ConfigureAwait(false)).Samples;
|
||||
@@ -6296,12 +6584,13 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2
|
||||
// Absurd burst: more ties than we're willing to buffer in memory. Preserve today's loud-fail
|
||||
// for that node rather than over-fetch an unbounded cluster; the operator's remedy is a
|
||||
// larger ServerHistorian:MaxTieClusterOverfetch (or NumValuesPerNode).
|
||||
if (cluster.Count > MaxTieClusterOverfetch)
|
||||
if (cluster.Count > overfetchBound)
|
||||
{
|
||||
_logger.LogError(
|
||||
"OtOpcUaNodeManager: HistoryReadRaw tie cluster at {StartUtc:O} for tag '{Tagname}' has {Count} samples, " +
|
||||
"exceeding MaxTieClusterOverfetch={Max}; cannot page within it. Increase MaxTieClusterOverfetch.",
|
||||
startUtc, tagname, cluster.Count, MaxTieClusterOverfetch);
|
||||
"exceeding {Max} (MaxTieClusterOverfetch, or the backend's row ceiling less one); cannot page within it. " +
|
||||
"Increase MaxTieClusterOverfetch (or the HistorianGateway MaxValuesCeiling).",
|
||||
startUtc, tagname, cluster.Count, overfetchBound);
|
||||
errors[handle.Index] = StatusCodes.BadTooManyOperations;
|
||||
results[handle.Index] = new SdkHistoryReadResult { StatusCode = StatusCodes.BadTooManyOperations };
|
||||
return;
|
||||
@@ -6309,8 +6598,15 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2
|
||||
|
||||
// Direction comes from the window itself (Part 11: start > end is a reverse read). A resumed reverse
|
||||
// read keeps start > end, because a reverse resume point is always strictly after endUtc.
|
||||
// #694: an uncapped read pages only at the backend's limit, so it slices at that page size — never 0 (a zero
|
||||
// slice hands back the same cursor forever; a limit that reverted to 0 under an uncapped cursor in a cluster
|
||||
// falls back to the cluster's own size).
|
||||
var sliceSize = numValuesPerNode > 0 ? numValuesPerNode
|
||||
: readCap > 0 ? readCap
|
||||
: pageLimit > 0 ? (uint)pageLimit
|
||||
: (uint)Math.Max(1, cluster.Count);
|
||||
HistoryPaging.SliceTieCluster(
|
||||
cluster.Count, boundarySkip, numValuesPerNode, startUtc, endUtc,
|
||||
cluster.Count, boundarySkip, sliceSize, startUtc, endUtc,
|
||||
out var sliceStart, out var sliceCount, out var nextStartUtc, out var nextSkip,
|
||||
reverse: reverse, resolutionTicks: Math.Max(1L, HistorianDataSource.TimestampResolutionTicks));
|
||||
|
||||
@@ -6324,9 +6620,17 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2
|
||||
if (nextStartUtc is { } resumeAt)
|
||||
{
|
||||
var clusterState = new HistoryContinuationState(
|
||||
tagname, resumeAt, endUtc, nextSkip, numValuesPerNode, SegmentWidthTicks: segmentWidth);
|
||||
tagname, resumeAt, endUtc, nextSkip, numValuesPerNode, SegmentWidthTicks: walkHint.WidthTicks,
|
||||
EmptyFromTicks: walkHint.EmptyFromTicks, EmptyToTicks: walkHint.EmptyToTicks,
|
||||
PendingEndBound: HistoryBounds.Carry(endBound, slice),
|
||||
WithinTieCluster: resumeAt == startUtc);
|
||||
clusterCp = _historyContinuationStore.Save(session, clusterState);
|
||||
}
|
||||
else if (endBound is not null)
|
||||
{
|
||||
// #691: the data is drained — the end bound ends this page, or is the next one when this one is full.
|
||||
clusterCp = await AppendEndBoundAsync(slice, startUtc).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
results[handle.Index] = new SdkHistoryReadResult
|
||||
{
|
||||
@@ -6338,20 +6642,59 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2
|
||||
return;
|
||||
}
|
||||
|
||||
// The "full page" test is against the RAW backend count (before trimming): the backend honoured
|
||||
// the cap, so a full backend page ⇒ there may be more even if we trimmed some boundary ties.
|
||||
// #692: more than the page came back (the lookahead row, after trimming) ⇒ there is more data: return the page and
|
||||
// resume after it. Otherwise the data is drained and this is the last data page.
|
||||
var more = readCap > 0 && samples.Count > readCap;
|
||||
if (more) samples = samples.Take((int)readCap).ToList();
|
||||
|
||||
HistoryContinuationState? nextState = null;
|
||||
if (more)
|
||||
{
|
||||
// A page that never left the boundary timestamp (ties continuing a resumed page) adds to the ties already
|
||||
// emitted there, or the next page would re-emit the earlier ones.
|
||||
HistoryPaging.ComputeResumeCursor(samples, out var nextStart, out var skip);
|
||||
if (resumed && nextStart == startUtc) skip += boundarySkip;
|
||||
nextState = new HistoryContinuationState(
|
||||
tagname, nextStart, endUtc, skip, numValuesPerNode, SegmentWidthTicks: walkHint.WidthTicks,
|
||||
EmptyFromTicks: walkHint.EmptyFromTicks, EmptyToTicks: walkHint.EmptyToTicks,
|
||||
PendingEndBound: HistoryBounds.Carry(endBound, samples));
|
||||
}
|
||||
|
||||
// #691: the start bound (first page only) leads the data.
|
||||
if (startBound is not null)
|
||||
{
|
||||
var led = new List<DataValueSnapshot>(samples.Count + 2) { startBound };
|
||||
led.AddRange(samples);
|
||||
samples = led;
|
||||
}
|
||||
|
||||
var historyData = ToHistoryDataFromSamples(samples);
|
||||
|
||||
byte[]? outboundCp = null;
|
||||
if (backendFull && samples.Count > 0)
|
||||
if (nextState is not null)
|
||||
{
|
||||
HistoryPaging.ComputeResumeCursor(samples, out var nextStart, out var skip);
|
||||
var nextState = new HistoryContinuationState(
|
||||
tagname, nextStart, endUtc, skip, numValuesPerNode, SegmentWidthTicks: segmentWidth);
|
||||
// Save may return null (no session on this request) ⇒ degrade to single-shot for this node.
|
||||
// Built AFTER historyData so a failure projecting samples can never orphan a stored cursor.
|
||||
outboundCp = _historyContinuationStore.Save(session, nextState);
|
||||
}
|
||||
else if (endBound is not null)
|
||||
{
|
||||
// #691: the data is drained — the end bound ends this page, or is the next one when this one is full.
|
||||
var page = new List<DataValueSnapshot>(samples.Count + 1);
|
||||
page.AddRange(samples);
|
||||
outboundCp = await AppendEndBoundAsync(page, startUtc).ConfigureAwait(false);
|
||||
samples = page;
|
||||
historyData = ToHistoryDataFromSamples(samples);
|
||||
}
|
||||
|
||||
// #691 review: a whole bounded read with no data and no bound found (one page, every value a placeholder) is the
|
||||
// pre-bounds answer — GoodNoData, no values (§6.5.3.2: bounds make it a success only "if one or both exist").
|
||||
if (endBound is not null && inboundCp is not { Length: > 0 } && outboundCp is null
|
||||
&& samples.All(HistoryBounds.IsPlaceholder))
|
||||
{
|
||||
samples = [];
|
||||
historyData = ToHistoryDataFromSamples(samples);
|
||||
}
|
||||
|
||||
results[handle.Index] = new SdkHistoryReadResult
|
||||
{
|
||||
@@ -6382,7 +6725,8 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2
|
||||
{
|
||||
// One node's backend failure must not throw out of the batch — Bad for THIS node only.
|
||||
_logger.LogError(ex, "OtOpcUaNodeManager: HistoryReadRaw (paged) failed for node {NodeId}", handle.NodeId);
|
||||
errors[handle.Index] = StatusCodes.BadHistoryOperationUnsupported;
|
||||
errors[handle.Index] = HistoryReadFailureStatus(ex);
|
||||
results[handle.Index] = new SdkHistoryReadResult { StatusCode = HistoryReadFailureStatus(ex) };
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6489,8 +6833,8 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "OtOpcUaNodeManager: HistoryRead (modified) failed for node {NodeId}", handle.NodeId);
|
||||
errors[handle.Index] = StatusCodes.BadHistoryOperationUnsupported;
|
||||
results[handle.Index] = new SdkHistoryReadResult { StatusCode = StatusCodes.BadHistoryOperationUnsupported };
|
||||
errors[handle.Index] = HistoryReadFailureStatus(ex);
|
||||
results[handle.Index] = new SdkHistoryReadResult { StatusCode = HistoryReadFailureStatus(ex) };
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6539,6 +6883,49 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Transport failures are retryable; unsupported operations retain their distinct status (#697).</summary>
|
||||
private static uint HistoryReadFailureStatus(Exception error)
|
||||
{
|
||||
for (Exception? ex = error; ex is not null; ex = ex.InnerException)
|
||||
if (ex is System.IO.IOException or System.Net.Sockets.SocketException or System.Net.Http.HttpRequestException or TimeoutException)
|
||||
return StatusCodes.BadNoCommunication;
|
||||
return StatusCodes.BadHistoryOperationUnsupported;
|
||||
}
|
||||
|
||||
/// <summary>Normalize stored and AtTime history to each scalar node's declared DataType (#698).</summary>
|
||||
private void NormalizeHistoryValues(List<NodeHandle> handles, IList<SdkHistoryReadResult> results)
|
||||
{
|
||||
foreach (var handle in handles)
|
||||
{
|
||||
if (AtTimeValueType(handle.NodeId) is not { } type || results[handle.Index]?.HistoryData is not { } body) continue;
|
||||
var values = ExtensionObject.ToEncodeable(body) switch
|
||||
{
|
||||
HistoryData data => data.DataValues,
|
||||
_ => null,
|
||||
};
|
||||
if (values is null) continue;
|
||||
foreach (var value in values)
|
||||
{
|
||||
if (value.Value is null) continue;
|
||||
try
|
||||
{
|
||||
// A gateway number cannot reconstruct a DateTime or a String which was never stored.
|
||||
if ((type == TypeCode.DateTime && value.Value is not DateTime)
|
||||
|| (type == TypeCode.String && value.Value is not string)) throw new InvalidCastException();
|
||||
var converted = Convert.ChangeType(value.Value, type, System.Globalization.CultureInfo.InvariantCulture);
|
||||
if (converted is float f && float.IsInfinity(f) && value.Value is double d && double.IsFinite(d))
|
||||
throw new OverflowException();
|
||||
value.WrappedValue = new Variant(converted);
|
||||
}
|
||||
catch (Exception ex) when (ex is InvalidCastException or OverflowException or FormatException)
|
||||
{
|
||||
value.WrappedValue = Variant.Null;
|
||||
value.StatusCode = StatusCodes.BadTypeMismatch;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Project a plain sample list into an SDK <see cref="HistoryData"/> (the paged Raw path
|
||||
/// works on a trimmed <see cref="IReadOnlyList{T}"/> rather than a whole <see cref="HistorianRead"/>).</summary>
|
||||
/// <param name="samples">The samples to project (already trimmed of boundary duplicates).</param>
|
||||
|
||||
@@ -0,0 +1,97 @@
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.OpcUaServer;
|
||||
|
||||
/// <summary>How one HistoryRead-Raw page reads (<see cref="RawPageLimit.Plan"/>).</summary>
|
||||
/// <param name="Limit">The backend's <see cref="IHistorianDataSource.MaxRawValuesPerRead"/> the plan was made under (0 = none).</param>
|
||||
/// <param name="PageCap">The page: at most this many values are returned (0 = all values — only without a limit).</param>
|
||||
/// <param name="RequestCap">What the read asks for: page + the boundary ties it will trim + one lookahead row (0 = all).</param>
|
||||
/// <param name="NoRoom">The boundary ties alone fill a read — nothing can be read past them; serve the tie cluster.</param>
|
||||
internal readonly record struct RawPagePlan(int Limit, uint PageCap, uint RequestCap, bool NoRoom);
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #694: sizes a HistoryRead-Raw page to what the backend can return in one read
|
||||
/// (<see cref="IHistorianDataSource.MaxRawValuesPerRead"/> = L — a strict HistorianGateway's learned row ceiling, less
|
||||
/// its boundary-row allowance). With a limit the read — page P + boundary ties k + the #692 lookahead row — is at most L,
|
||||
/// so P = min(want, L − k − 1) (all values: P = L − k − 1); a full page carries a continuation point like any other, as
|
||||
/// Part 11 allows fewer values than <c>NumValuesPerNode</c> with one. Before #694 a cap above a strict gateway's ceiling
|
||||
/// (or an uncapped read of a window holding more) failed <c>BadTooManyOperations</c>.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>Never a silent truncation.</b> The backend still answers a read in full or refuses it; only the server's page is
|
||||
/// smaller, and a page is judged full against what was asked for. A read the backend cannot serve at all (a tie
|
||||
/// cluster larger than one read) stays <c>BadTooManyOperations</c>.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>One re-read, planned again.</b> A refused read (<see cref="HistoryReadLimitExceededException"/>) is re-planned and
|
||||
/// re-issued ONCE when (a) the backend learned a different limit while it ran — the first read after start-up teaches
|
||||
/// the ceiling — or (b) it was refused because rows before the window crowded the budget
|
||||
/// (<see cref="HistoryReadLimitExceededException.PreWindowRows"/> m): the page is then shrunk by m so that the
|
||||
/// backend's look-back fits beside it. Anything else is the answer.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
internal static class RawPageLimit
|
||||
{
|
||||
/// <summary>Plans one page.</summary>
|
||||
/// <param name="want">The page the client asked for (N, less a start bound; 0 = all values).</param>
|
||||
/// <param name="boundaryTies">Ties at the resume timestamp already returned, which the read re-reads and trims (k).</param>
|
||||
/// <param name="limit">The backend's limit L (<= 0 = none).</param>
|
||||
/// <param name="sizeLimit">The limit to size the page by (L, or L − m after a crowded refusal); defaults to <paramref name="limit"/>.</param>
|
||||
/// <returns>The plan.</returns>
|
||||
public static RawPagePlan Plan(uint want, int boundaryTies, int limit, int? sizeLimit = null)
|
||||
{
|
||||
var k = Math.Max(0, boundaryTies);
|
||||
if (limit <= 0)
|
||||
return new RawPagePlan(0, want, want == 0 ? 0u : (uint)Math.Min(uint.MaxValue, (long)want + k + 1L), false);
|
||||
|
||||
var room = (long)(sizeLimit ?? limit) - k - 1;
|
||||
if (room < 1) return new RawPagePlan(limit, 0, 0, true);
|
||||
var page = want == 0 ? room : Math.Min(want, room);
|
||||
return new RawPagePlan(limit, (uint)page, (uint)(page + k + 1), false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reads one page under the backend's current limit, re-planning and re-reading ONCE as described in the remarks.
|
||||
/// A <see cref="RawPagePlan.NoRoom"/> plan is returned without a read.
|
||||
/// </summary>
|
||||
/// <typeparam name="T">The page.</typeparam>
|
||||
/// <param name="source">The backend.</param>
|
||||
/// <param name="want">The page the client asked for (0 = all values).</param>
|
||||
/// <param name="boundaryTies">The boundary ties the read re-reads and trims.</param>
|
||||
/// <param name="read">One read with the given plan (it asks for <see cref="RawPagePlan.RequestCap"/>).</param>
|
||||
/// <returns>The page (null when <see cref="RawPagePlan.NoRoom"/>) and the plan it was read with.</returns>
|
||||
public static async Task<(T? Page, RawPagePlan Plan)> ReadAsync<T>(
|
||||
IHistorianDataSource source, uint want, int boundaryTies, Func<RawPagePlan, Task<T>> read)
|
||||
{
|
||||
var before = source.MaxRawValuesPerRead;
|
||||
var plan = Plan(want, boundaryTies, before);
|
||||
if (plan.NoRoom) return (default, plan);
|
||||
|
||||
HistoryReadLimitExceededException refused;
|
||||
try
|
||||
{
|
||||
return (await read(plan).ConfigureAwait(false), plan);
|
||||
}
|
||||
catch (HistoryReadLimitExceededException ex) when (source.MaxRawValuesPerRead is var now and > 0
|
||||
&& (now != before || ex.PreWindowRows > 0))
|
||||
{
|
||||
refused = ex; // re-read outside the catch: the second refusal is the answer
|
||||
}
|
||||
|
||||
var limit = source.MaxRawValuesPerRead;
|
||||
var shrink = Math.Max(0, refused.PreWindowRows);
|
||||
plan = Plan(want, boundaryTies, limit, (int)Math.Max(0L, (long)limit - shrink));
|
||||
// No room left once the crowding rows are allowed for: those rows are not (necessarily) the boundary ties, so this is
|
||||
// NOT "the cursor sits in a tie cluster" — the refusal is the answer (stepping the cursor through it a tick per page
|
||||
// would never end).
|
||||
// (A limit that changed under an un-crowded read leaves the boundary ties alone filling a read: that IS the tie
|
||||
// cluster at the cursor, served by the over-fetch.)
|
||||
if (plan.NoRoom)
|
||||
{
|
||||
if (shrink > 0) throw refused;
|
||||
return (default, plan);
|
||||
}
|
||||
return (await read(plan).ConfigureAwait(false), plan);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.OpcUaServer;
|
||||
|
||||
/// <summary>One walked-back reverse events page (<see cref="ReverseEventsReader"/>).</summary>
|
||||
/// <param name="Events">At most the requested count, newest first by <c>(EventTimeUtc, EventId)</c>.</param>
|
||||
/// <param name="Hint">What the next page of the same read resumes with.</param>
|
||||
internal readonly record struct ReverseEventsPage(List<HistoricalEvent> Events, ReverseWalkHint Hint);
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #691.4: serves a Part 11 reverse events read — <c>(lo, hi]</c> newest-first — on a backend that reads events
|
||||
/// forward only (<see cref="IHistorianDataSource.SupportsReverseEvents"/> = <c>false</c>: the Gateway) with the
|
||||
/// <see cref="ReverseWalk"/>. A segment read is cut short when the source flags truncation (its continuation point); ties
|
||||
/// are ordered by <see cref="EventIdTieOrder"/> because the gateway orders by event time only.
|
||||
/// </summary>
|
||||
internal static class ReverseEventsReader
|
||||
{
|
||||
/// <summary>Reads <c>(lo, hi]</c> newest-first from forward event reads.</summary>
|
||||
/// <param name="source">A backend that reads events forward.</param>
|
||||
/// <param name="sourceName">Event-source filter (null = all).</param>
|
||||
/// <param name="hi">The later (inclusive) bound.</param>
|
||||
/// <param name="lo">The earlier (exclusive) bound.</param>
|
||||
/// <param name="want">How many events; <c>0</c> = all in the window (one forward read).</param>
|
||||
/// <param name="maxTieCluster">The tie budget (<c>MaxTieClusterOverfetch</c>) — a ceiling only.</param>
|
||||
/// <param name="nowUtc">Now, for the look-ahead clamp.</param>
|
||||
/// <param name="hint">What the previous page carried (default = none).</param>
|
||||
/// <param name="ct">Cancellation.</param>
|
||||
/// <returns>The events, newest first, and the hint for the next page.</returns>
|
||||
/// <exception cref="HistoryReadLimitExceededException">One timestamp holds more than the tie budget.</exception>
|
||||
public static async Task<ReverseEventsPage> ReadAsync(
|
||||
IHistorianDataSource source, string? sourceName, DateTime hi, DateTime lo, uint want, int maxTieCluster,
|
||||
DateTime nowUtc, ReverseWalkHint hint, CancellationToken ct)
|
||||
{
|
||||
var budget = Math.Max(1, maxTieCluster);
|
||||
var (rows, next) = await ReverseWalk.WalkAsync<HistoricalEvent>(
|
||||
async (start, end, cap, token) =>
|
||||
{
|
||||
// cap 0 = the whole window: int.MaxValue is the events contract's "no cap" (a non-positive value would
|
||||
// mean the backend's DEFAULT cap).
|
||||
var r = await source.ReadEventsAsync(sourceName, start, end, cap == 0 ? int.MaxValue : cap, reverse: false, token)
|
||||
.ConfigureAwait(false);
|
||||
return new ForwardSegmentRead<HistoricalEvent>(r.Events, r.ContinuationPoint is not null);
|
||||
},
|
||||
ev => ev.EventTimeUtc.Ticks, EventIdTieOrder.Instance, hi, lo, want, budget, source.TimestampResolutionTicks,
|
||||
nowUtc, hint,
|
||||
at => new HistoryReadLimitExceededException(budget,
|
||||
$"A reverse events read of '{sourceName}' at {at:O} found more than {budget} events within one millisecond; the " +
|
||||
"backend reads forward only, so the latest of them cannot be selected. Increase " +
|
||||
"ServerHistorian:MaxTieClusterOverfetch."),
|
||||
ct)
|
||||
.ConfigureAwait(false);
|
||||
return new ReverseEventsPage(rows, next);
|
||||
}
|
||||
}
|
||||
@@ -5,70 +5,29 @@ namespace ZB.MOM.WW.OtOpcUa.OpcUaServer;
|
||||
/// <summary>One walked-back reverse raw page (<see cref="ReverseRawReader"/>).</summary>
|
||||
/// <param name="Samples">At most the requested cap, newest first.</param>
|
||||
/// <param name="WidthHintTicks">The segment width the walk ended on — the next page's starting width (0 = none).</param>
|
||||
internal readonly record struct ReverseRawPage(IReadOnlyList<DataValueSnapshot> Samples, long WidthHintTicks);
|
||||
internal readonly record struct ReverseRawPage(IReadOnlyList<DataValueSnapshot> Samples, long WidthHintTicks)
|
||||
{
|
||||
/// <summary>Everything the next page of the same read should resume with (its width is <see cref="WidthHintTicks"/>).</summary>
|
||||
public ReverseWalkHint Hint { get; init; } = new(WidthHintTicks);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #686: serves a Part 11 reverse raw read — <c>(lo, hi]</c> newest-first, capped — on a historian backend that can
|
||||
/// only read forward (<see cref="IHistorianDataSource.SupportsReverseRaw"/> = <c>false</c>: the Gateway, whose
|
||||
/// <c>ReadRaw</c> streams ascending and rejects <c>start > end</c>).
|
||||
/// <c>ReadRaw</c> streams ascending and rejects <c>start > end</c>). The walk itself — segment sizing (#689), the
|
||||
/// truncation rules, edges, ties and bounds — is <see cref="ReverseWalk"/>; this adapter supplies the raw read (a read is
|
||||
/// cut short when it returned its full cap) and keeps the backend's tie order.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>Uncapped</b> (<c>maxValues = 0</c>): ONE forward read of the whole window, filtered and reversed in memory — the
|
||||
/// same rows, round trips and memory as an uncapped forward read (and as a native backend's uncapped reverse read).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Capped: walk back in segments.</b> Starting at <c>hi</c>, read one segment <c>(segLo, segHi]</c> FORWARD, keep
|
||||
/// it newest-first, and step to the next older segment until the cap is met or <c>lo</c> is reached. The first
|
||||
/// segment is <see cref="InitialSegment"/> wide, or the width the previous page of the same read ended on (carried in
|
||||
/// the continuation point). After each read the next width is re-aimed at about half the row budget from the density
|
||||
/// just observed, growing at most ×4 per step — so a live tag's "latest N" is usually one read, a dense tag does not
|
||||
/// throw away a full read per page, and sparse history is found in a few dozen tiny reads.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Never answer from a truncated read.</b> The row budget is <c>M = max(1, MaxTieClusterOverfetch)</c>; a segment
|
||||
/// read asks for <c>M + 2</c> rows (room for the row an AVEVA read fabricates at its query start, and one more to see
|
||||
/// past the segment). Rows stream ascending, so a read is complete for its segment when it came back short OR its last
|
||||
/// row is already past <c>segHi</c>. An incomplete read is discarded: when rows BEFORE the segment (a neighbouring
|
||||
/// tie cluster inside the alignment slack, the fabricated row) crowded it out it is re-read once with room for them
|
||||
/// (at most <c>2M + 4</c> rows); otherwise the segment is narrowed (to about half the budget at the density the
|
||||
/// truncated read showed) and re-read. Only a single timestamp holding more than <c>M</c> samples — a segment already
|
||||
/// one timestamp-resolution wide that still does not fit — is refused: <see cref="HistoryReadLimitExceededException"/>
|
||||
/// (⇒ <c>BadTooManyOperations</c>, a request limit, never a backend fault).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Edges.</b> A segment is requested as <c>[⌊segLo⌋, ⌈segHi + 1 tick⌉]</c>, both rounded outward to the millisecond
|
||||
/// (the gateway's / AVEVA's resolution), and filtered to <c>(segLo, segHi]</c> on the samples' own timestamps. That is
|
||||
/// exact whether the backend's forward end is exclusive (the interface contract) or inclusive (the gateway) and
|
||||
/// whether it truncates its bounds (QuestDB-style); and because the request start is already on a millisecond, a row
|
||||
/// an AVEVA read fabricates at its (possibly rounded-up) query start can never land inside the segment (#687).
|
||||
/// Adjacent segments share no timestamp, so nothing is returned twice.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Ties</b> come back in the backend's forward order reversed — the same order the node manager's tie-cluster
|
||||
/// over-fetch uses for a reverse read — so a resumed page's boundary-tie skip lines up.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Bounds.</b> The walk never asks for data after now + <see cref="LookAhead"/> (one day — the historian stores
|
||||
/// nothing further ahead; a <c>StartTime = DateTime.MaxValue</c> read would otherwise send centuries-ahead query
|
||||
/// starts), nor before <see cref="Floor"/> (the Unix epoch — AVEVA holds nothing earlier, and an end-only read's year-1
|
||||
/// lower bound is outside the historian's time encoding). A walked-back read therefore returns only samples strictly
|
||||
/// after the epoch and no later than now + one day.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
internal static class ReverseRawReader
|
||||
{
|
||||
/// <summary>The first segment's width when no hint is carried.</summary>
|
||||
internal static readonly TimeSpan InitialSegment = TimeSpan.FromMinutes(1);
|
||||
internal static readonly TimeSpan InitialSegment = ReverseWalk.InitialSegment;
|
||||
|
||||
/// <summary>The walk's exclusive lower limit (see the remarks).</summary>
|
||||
internal static readonly DateTime Floor = DateTime.UnixEpoch;
|
||||
/// <summary>The walk's exclusive lower limit.</summary>
|
||||
internal static readonly DateTime Floor = ReverseWalk.Floor;
|
||||
|
||||
/// <summary>How far past "now" the walk starts at most (the same horizon as the Gateway's raw-aggregate look-ahead).</summary>
|
||||
internal static readonly TimeSpan LookAhead = TimeSpan.FromDays(1);
|
||||
|
||||
/// <summary>Request bounds are rounded outward to this (the gateway / AVEVA timestamp resolution).</summary>
|
||||
private const long Alignment = TimeSpan.TicksPerMillisecond;
|
||||
/// <summary>How far past "now" the walk starts at most.</summary>
|
||||
internal static readonly TimeSpan LookAhead = ReverseWalk.LookAhead;
|
||||
|
||||
/// <summary>Reads <c>(lo, hi]</c> newest-first from forward reads.</summary>
|
||||
/// <param name="source">A backend that reads forward.</param>
|
||||
@@ -76,129 +35,102 @@ internal static class ReverseRawReader
|
||||
/// <param name="hi">The later (inclusive) bound — the request's StartTime or a reverse resume point.</param>
|
||||
/// <param name="lo">The earlier (exclusive) bound.</param>
|
||||
/// <param name="maxValues">Cap; <c>0</c> = every sample in the window (one forward read).</param>
|
||||
/// <param name="maxTieCluster">Row budget per segment read (<c>MaxTieClusterOverfetch</c>; values below 1 count as 1).</param>
|
||||
/// <param name="maxTieCluster">The tie budget (<c>MaxTieClusterOverfetch</c>; values below 1 count as 1) — a ceiling only.</param>
|
||||
/// <param name="nowUtc">Now, for the look-ahead clamp.</param>
|
||||
/// <param name="widthHintTicks">Starting segment width (the previous page's <see cref="ReverseRawPage.WidthHintTicks"/>), 0 = none.</param>
|
||||
/// <param name="ct">Cancellation.</param>
|
||||
/// <returns>The page, newest first, and the width to start the next page with.</returns>
|
||||
/// <exception cref="HistoryReadLimitExceededException">One timestamp holds more than the row budget.</exception>
|
||||
/// <exception cref="HistoryReadLimitExceededException">One timestamp holds more than the tie budget.</exception>
|
||||
public static Task<ReverseRawPage> ReadAsync(
|
||||
IHistorianDataSource source, string tagname, DateTime hi, DateTime lo, uint maxValues, int maxTieCluster,
|
||||
DateTime nowUtc, long widthHintTicks, CancellationToken ct) =>
|
||||
ReadAsync(source, tagname, hi, lo, maxValues, maxTieCluster, nowUtc, new ReverseWalkHint(widthHintTicks), ct);
|
||||
|
||||
/// <summary>Reads <c>(lo, hi]</c> newest-first from forward reads, resuming with the previous page's full hint.</summary>
|
||||
/// <param name="source">A backend that reads forward.</param>
|
||||
/// <param name="tagname">Historian tag.</param>
|
||||
/// <param name="hi">The later (inclusive) bound.</param>
|
||||
/// <param name="lo">The earlier (exclusive) bound.</param>
|
||||
/// <param name="maxValues">Cap; <c>0</c> = every sample in the window (one forward read).</param>
|
||||
/// <param name="maxTieCluster">The tie budget — a ceiling only.</param>
|
||||
/// <param name="nowUtc">Now, for the look-ahead clamp.</param>
|
||||
/// <param name="hint">What the previous page carried (<see cref="ReverseRawPage.Hint"/>), default = none.</param>
|
||||
/// <param name="ct">Cancellation.</param>
|
||||
/// <returns>The page, newest first, and the hint for the next page.</returns>
|
||||
/// <exception cref="HistoryReadLimitExceededException">One timestamp holds more than the tie budget.</exception>
|
||||
/// <remarks>
|
||||
/// Gitea #694: every segment read fits the backend's <see cref="IHistorianDataSource.MaxRawValuesPerRead"/> (a strict
|
||||
/// gateway's learned ceiling) — the walk's tie budget is bounded by it so its segments are sized for it, and a
|
||||
/// crowded re-read asking more is cut to it (and counts as truncated, never as complete). A walk that TEACHES the
|
||||
/// backend its limit (the first read after start-up) and is refused for it is re-run once with segments sized for the
|
||||
/// new limit — unless <paramref name="retryOnLearnedLimit"/> is false: the HistoryRead page path passes false, so the
|
||||
/// refusal reaches <see cref="RawPageLimit"/>, which shrinks the PAGE to the limit as well (re-running the walk for a
|
||||
/// page of N far above the limit cost thousands of segment reads). The bound lookups (a page of one) keep the re-run.
|
||||
/// The page size itself is never cut here: a short answer would read as the last page. A segment refused because rows
|
||||
/// just before it crowded the backend's budget (<see cref="HistoryReadLimitExceededException.PreWindowRows"/>) is
|
||||
/// re-read once that many rows smaller and counts as truncated when full. An uncapped walk (one forward read of the
|
||||
/// whole window) is never cut: it is answered in full or refused.
|
||||
/// </remarks>
|
||||
/// <param name="retryOnLearnedLimit">Re-run the walk once when it taught the backend its limit (see the remarks).</param>
|
||||
public static async Task<ReverseRawPage> ReadAsync(
|
||||
IHistorianDataSource source, string tagname, DateTime hi, DateTime lo, uint maxValues, int maxTieCluster,
|
||||
DateTime nowUtc, long widthHintTicks, CancellationToken ct)
|
||||
DateTime nowUtc, ReverseWalkHint hint, CancellationToken ct, bool retryOnLearnedLimit = true)
|
||||
{
|
||||
var resolution = Math.Max(1L, source.TimestampResolutionTicks);
|
||||
var granularity = Math.Max(Alignment, resolution);
|
||||
var budget = Math.Max(1, maxTieCluster);
|
||||
var result = new List<DataValueSnapshot>();
|
||||
|
||||
var horizon = nowUtc.Ticks > DateTime.MaxValue.Ticks - LookAhead.Ticks ? DateTime.MaxValue.Ticks : nowUtc.Ticks + LookAhead.Ticks;
|
||||
var hiTicks = Math.Min(hi.Ticks, horizon);
|
||||
var stopTicks = Math.Max(lo.Ticks, Floor.Ticks);
|
||||
if (hiTicks <= stopTicks) return new ReverseRawPage(result, widthHintTicks);
|
||||
|
||||
if (maxValues == 0)
|
||||
var before = source.MaxRawValuesPerRead;
|
||||
try
|
||||
{
|
||||
// One forward read of the whole window, reversed — the uncapped forward read's own cost and bound.
|
||||
var all = (await source.ReadRawAsync(tagname, Utc(AlignDown(stopTicks, granularity)),
|
||||
Utc(AlignUp(hiTicks + 1, granularity)), 0, ct).ConfigureAwait(false)).Samples;
|
||||
for (var i = all.Count - 1; i >= 0; i--)
|
||||
{
|
||||
var ts = Ticks(all[i]);
|
||||
if (ts > stopTicks && ts <= hiTicks) result.Add(all[i]);
|
||||
}
|
||||
|
||||
return new ReverseRawPage(result, widthHintTicks);
|
||||
return await WalkAsync(before).ConfigureAwait(false);
|
||||
}
|
||||
catch (HistoryReadLimitExceededException) when (retryOnLearnedLimit && maxValues > 0
|
||||
&& source.MaxRawValuesPerRead is var now and > 0 && now != before)
|
||||
{
|
||||
// fall through (outside the catch): the walk taught the backend its limit — re-run it sized for that limit once.
|
||||
// Not an uncapped walk: it is one read of the whole window, which the limit does not change.
|
||||
}
|
||||
|
||||
var maxWidth = DateTime.MaxValue.Ticks;
|
||||
var width = widthHintTicks > 0 ? Math.Clamp(widthHintTicks, resolution, maxWidth) : Math.Max(resolution, InitialSegment.Ticks);
|
||||
var baseCap = budget + 2;
|
||||
var maxCap = 2 * budget + 4;
|
||||
var segHi = hiTicks;
|
||||
return await WalkAsync(source.MaxRawValuesPerRead).ConfigureAwait(false);
|
||||
|
||||
while (segHi > stopTicks && result.Count < maxValues)
|
||||
async Task<ReverseRawPage> WalkAsync(int limit)
|
||||
{
|
||||
var segLo = segHi - stopTicks <= width ? stopTicks : segHi - width;
|
||||
var start = Utc(AlignDown(segLo, granularity));
|
||||
var end = Utc(AlignUp(segHi + 1, granularity));
|
||||
var cap = baseCap;
|
||||
IReadOnlyList<DataValueSnapshot> rows;
|
||||
var narrowed = false;
|
||||
|
||||
while (true)
|
||||
{
|
||||
ct.ThrowIfCancellationRequested();
|
||||
rows = (await source.ReadRawAsync(tagname, start, end, (uint)cap, ct).ConfigureAwait(false)).Samples;
|
||||
if (rows.Count < cap || Ticks(rows[^1]) > segHi) break; // complete for (segLo, segHi]
|
||||
|
||||
var leading = 0;
|
||||
var inner = 0;
|
||||
var firstInner = 0L;
|
||||
foreach (var r in rows)
|
||||
{
|
||||
var ts = Ticks(r);
|
||||
if (ts <= segLo) leading++;
|
||||
else if (ts <= segHi && inner++ == 0) firstInner = ts;
|
||||
}
|
||||
|
||||
// Crowded out by rows before the segment, with the segment itself possibly within budget: re-read once
|
||||
// with room for them.
|
||||
var roomier = leading + budget + 2;
|
||||
if (leading > 0 && inner <= budget && roomier > cap && roomier <= maxCap)
|
||||
{
|
||||
cap = roomier;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (width <= resolution)
|
||||
throw new HistoryReadLimitExceededException(budget,
|
||||
$"A reverse raw read of '{tagname}' at {Utc(segHi):O} found more than {budget} samples at one " +
|
||||
"timestamp; the backend reads forward only, so the latest of them cannot be selected. Increase " +
|
||||
"ServerHistorian:MaxTieClusterOverfetch.");
|
||||
|
||||
// Narrow to about half the budget at the density the truncated read's own samples showed (their span, not
|
||||
// the segment's — the segment may open on an empty stretch), at least halving.
|
||||
var lastTs = Ticks(rows[^1]);
|
||||
var aimed = inner > 1 && lastTs > firstInner
|
||||
? (long)Math.Min((double)(lastTs - firstInner) * (budget / 2.0) / (inner - 1), width / 2.0)
|
||||
: width / 4;
|
||||
width = Math.Max(resolution, aimed);
|
||||
narrowed = true;
|
||||
break;
|
||||
}
|
||||
|
||||
if (narrowed) continue; // re-read the same segHi, narrower
|
||||
|
||||
var count = 0;
|
||||
for (var i = rows.Count - 1; i >= 0; i--)
|
||||
{
|
||||
var ts = Ticks(rows[i]);
|
||||
if (ts <= segLo || ts > segHi) continue;
|
||||
count++;
|
||||
if (result.Count < maxValues) result.Add(rows[i]);
|
||||
}
|
||||
|
||||
// Re-aim at about half the budget from the density just seen; grow at most x4.
|
||||
var grown = width > maxWidth / 4 ? maxWidth : width * 4;
|
||||
width = count == 0
|
||||
? grown
|
||||
: (long)Math.Clamp((double)width * Math.Max(1.0, budget / 2.0) / count, resolution, grown);
|
||||
segHi = segLo;
|
||||
// A segment asks for at most its budget + 2 rows (ReverseWalk), so the budget is bounded 2 under the limit.
|
||||
var budget = Math.Max(1, maxTieCluster);
|
||||
if (limit > 0) budget = Math.Min(budget, Math.Max(1, limit - 2));
|
||||
var (rows, next) = await ReverseWalk.WalkAsync(
|
||||
async (start, end, cap, token) =>
|
||||
{
|
||||
var asked = limit > 0 && cap > limit ? limit : cap; // cap 0 (the whole window) is never cut
|
||||
IReadOnlyList<DataValueSnapshot> samples;
|
||||
for (var attempt = 0; ; attempt++)
|
||||
{
|
||||
try
|
||||
{
|
||||
samples = (await source.ReadRawAsync(tagname, start, end, (uint)asked, token).ConfigureAwait(false)).Samples;
|
||||
break;
|
||||
}
|
||||
catch (HistoryReadLimitExceededException ex) when (asked > 0 && ex.PreWindowRows > 0 && attempt < 2)
|
||||
{
|
||||
// #695: the discovery request may be smaller than the pre-window cluster. Size against
|
||||
// the ceiling, not that request; a second pass can discover more crowding rows.
|
||||
var ceilingRoom = source.MaxRawValuesPerRead is var learned and > 0 ? learned : asked;
|
||||
var smaller = Math.Min(asked - 1, ceilingRoom - ex.PreWindowRows - 1);
|
||||
if (smaller < 1) throw;
|
||||
asked = smaller;
|
||||
}
|
||||
}
|
||||
return new ForwardSegmentRead<DataValueSnapshot>(samples, asked > 0 && samples.Count >= asked);
|
||||
},
|
||||
Ticks, tieOrder: null, hi, lo, maxValues, budget, source.TimestampResolutionTicks, nowUtc, hint,
|
||||
at => new HistoryReadLimitExceededException(budget,
|
||||
$"A reverse raw read of '{tagname}' at {at:O} found more than {budget} samples within one millisecond; the " +
|
||||
"backend reads forward only, so the latest of them cannot be selected. Increase " +
|
||||
(budget < Math.Max(1, maxTieCluster)
|
||||
? "the backend's row ceiling (HistorianGateway Historian:ReadLimits:MaxValuesCeiling)."
|
||||
: "ServerHistorian:MaxTieClusterOverfetch.")),
|
||||
ct, useDiscoveryBudgetForGaps: limit > 0)
|
||||
.ConfigureAwait(false);
|
||||
return new ReverseRawPage(rows, next.WidthTicks) { Hint = next };
|
||||
}
|
||||
|
||||
return new ReverseRawPage(result, width);
|
||||
}
|
||||
|
||||
private static long Ticks(DataValueSnapshot s) => (s.SourceTimestampUtc ?? DateTime.MinValue).Ticks;
|
||||
|
||||
private static DateTime Utc(long ticks) => new(ticks, DateTimeKind.Utc);
|
||||
|
||||
private static long AlignDown(long ticks, long step) => ticks - ticks % step;
|
||||
|
||||
private static long AlignUp(long ticks, long step)
|
||||
{
|
||||
var down = AlignDown(ticks, step);
|
||||
if (down == ticks) return ticks;
|
||||
return down > DateTime.MaxValue.Ticks - step ? DateTime.MaxValue.Ticks : down + step;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,454 @@
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.OpcUaServer;
|
||||
|
||||
/// <summary>What one walked-back page tells the next page of the same read (carried in the continuation point).</summary>
|
||||
/// <param name="WidthTicks">The segment width the walk ended on, aimed at the page's density — the next page's first width (0 = none).</param>
|
||||
/// <param name="EmptyFromTicks">
|
||||
/// With <paramref name="EmptyToTicks"/>: a stretch <c>(EmptyFrom, EmptyTo)</c> (both exclusive) a complete segment read
|
||||
/// proved holds no row (#691) — typically from the oldest row the page saw down to the walk's floor. A resumed page
|
||||
/// steps straight over it instead of re-walking it. Both 0 = none.
|
||||
/// </param>
|
||||
/// <param name="EmptyToTicks">The exclusive upper end of the empty stretch.</param>
|
||||
internal readonly record struct ReverseWalkHint(long WidthTicks, long EmptyFromTicks = 0, long EmptyToTicks = 0)
|
||||
{
|
||||
/// <summary>Whether an empty stretch is carried.</summary>
|
||||
public bool HasEmptyStretch => EmptyToTicks > EmptyFromTicks && EmptyToTicks > 0;
|
||||
}
|
||||
|
||||
/// <summary>One forward read of a segment: the rows (ascending by time) and whether a cap cut the read short.</summary>
|
||||
/// <typeparam name="T">Row type.</typeparam>
|
||||
/// <param name="Rows">Rows in ascending time order (ties in any order).</param>
|
||||
/// <param name="Truncated">A cap stopped the read — more rows may follow the last one.</param>
|
||||
internal readonly record struct ForwardSegmentRead<T>(IReadOnlyList<T> Rows, bool Truncated);
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #686/#689: the walk-back that serves a Part 11 reverse read — <c>(lo, hi]</c> newest-first, capped — from a
|
||||
/// backend that can only read FORWARD. Shared by raw values (<see cref="ReverseRawReader"/>) and events; the row type
|
||||
/// only has to expose a timestamp.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>Uncapped</b> (<c>want = 0</c>): ONE forward read of the whole window, filtered and reversed in memory.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Capped: walk back in segments.</b> Starting at <c>hi</c>, read one segment <c>(segLo, segHi]</c> FORWARD, keep
|
||||
/// it newest-first, and step to the next older segment until the cap is met or <c>lo</c> is reached. The first
|
||||
/// segment is <see cref="InitialSegment"/> wide, or the width the previous page of the same read ended on (carried in
|
||||
/// the continuation point). After each read the next width is re-aimed from the density just observed at about half
|
||||
/// the PAGE budget <c>2N + 16</c> (≈ N + 8 rows), growing at most ×4 per step (#689 — it used to aim at half the tie
|
||||
/// budget, ~32k rows, whatever N was) — so a live tag's "latest N" is usually one read of about N rows, and sparse
|
||||
/// history is found in a few dozen tiny reads.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Read sizes.</b> A segment read asks for its budget + 2 rows (room for a row an AVEVA read fabricates at its query
|
||||
/// start, and one to see past the segment). The budget is the page budget, except a <b>discovery</b> segment — the
|
||||
/// first of a page with no carried width, or one after an empty segment (whose ×4 growth may overshoot into dense
|
||||
/// data) — which may read up to <see cref="DiscoveryRows"/>. The tie budget <c>M = max(1, MaxTieClusterOverfetch)</c>
|
||||
/// is only a CEILING: a single timestamp holding more rows than the segment budget escalates to <c>M</c>.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Never answer from a truncated read.</b> Rows stream ascending, so a read is complete for its segment when it was
|
||||
/// not cut short OR its last row is already past <c>segHi</c>. An incomplete read is discarded: when rows BEFORE the
|
||||
/// segment (a neighbouring tie cluster inside the alignment slack, a fabricated row) crowded it out it is re-read once
|
||||
/// with room for them (at most twice the budget + 4); otherwise the segment is narrowed — to the density the truncated
|
||||
/// read showed, and never again below the newest row it saw (<c>known</c>: rows exist there, so walking past it before
|
||||
/// finding what lies above it would re-read the same dense stretch over and over; the gap above it is bisected
|
||||
/// instead). Only more than <c>M</c> rows within one millisecond — a segment no wider than the 1 ms request
|
||||
/// granularity that still does not fit at the tie budget — is refused (per millisecond, not per timestamp: narrower
|
||||
/// requests are the same request, and the Gateway stores milliseconds).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Proving the rest empty (#691).</b> Walking down to the floor through empty history used to take one read per ×4
|
||||
/// growth step (~14 reads from a minute to the epoch) — on every page that ran out of rows. After two consecutive empty
|
||||
/// segments the walk asks once per page for the WHOLE remainder <c>(lo, segHi]</c> at the page budget: a complete
|
||||
/// answer ends the walk in one read; a truncated one (deep history) is discarded and the walk carries on as before. And
|
||||
/// the stretch a page's last complete segment proved empty below its oldest row rides the continuation point
|
||||
/// (<see cref="ReverseWalkHint.EmptyFromTicks"/>), so the next page steps over it — the rows of a page are often the
|
||||
/// last ones before a stretch the previous page already read.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Edges.</b> A segment is requested as <c>[⌊segLo⌋, ⌈segHi + 1 tick⌉]</c>, both rounded outward to the millisecond
|
||||
/// (the gateway's / AVEVA's resolution), and filtered to <c>(segLo, segHi]</c> on the rows' own timestamps — exact
|
||||
/// whether the backend's forward end is exclusive or inclusive and whether it truncates its bounds; a row fabricated at
|
||||
/// the (millisecond) query start can never land inside the segment (#687). Adjacent segments share no timestamp.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Ties</b> come back in the backend's forward order reversed, or — with a tie order given (events, whose gateway
|
||||
/// order within a timestamp is arbitrary) — sorted by it and reversed, so a resumed page's boundary-tie skip lines up.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Bounds.</b> The walk never asks for data after now + <see cref="LookAhead"/> nor before <see cref="Floor"/> (the
|
||||
/// Unix epoch — AVEVA holds nothing earlier). A walked-back read returns only rows strictly after the epoch and no
|
||||
/// later than now + one day.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
internal static class ReverseWalk
|
||||
{
|
||||
/// <summary>The first segment's width when no hint is carried.</summary>
|
||||
internal static readonly TimeSpan InitialSegment = TimeSpan.FromMinutes(1);
|
||||
|
||||
/// <summary>The walk's exclusive lower limit.</summary>
|
||||
internal static readonly DateTime Floor = DateTime.UnixEpoch;
|
||||
|
||||
/// <summary>How far past "now" the walk starts at most.</summary>
|
||||
internal static readonly TimeSpan LookAhead = TimeSpan.FromDays(1);
|
||||
|
||||
/// <summary>Slack added to a page's row budget (#689): room for density variation between segments.</summary>
|
||||
internal const int PageSlack = 16;
|
||||
|
||||
/// <summary>The most rows a discovery segment (see the remarks) may read before it counts as truncated (#689).</summary>
|
||||
internal const int DiscoveryRows = 1024;
|
||||
|
||||
/// <summary>Request bounds are rounded outward to this (the gateway / AVEVA timestamp resolution).</summary>
|
||||
private const long Alignment = TimeSpan.TicksPerMillisecond;
|
||||
|
||||
/// <summary>The rows a segment is sized for: <c>2N + </c><see cref="PageSlack"/>, never above the tie budget.</summary>
|
||||
/// <param name="want">The page cap N (> 0).</param>
|
||||
/// <param name="tieBudget">The tie budget M (>= 1).</param>
|
||||
/// <returns>The page budget.</returns>
|
||||
internal static int PageBudget(uint want, int tieBudget) => (int)Math.Clamp(2L * want + PageSlack, 1L, Math.Max(1, tieBudget));
|
||||
|
||||
/// <summary>Reads <c>(lo, hi]</c> newest-first from forward reads.</summary>
|
||||
/// <typeparam name="T">Row type.</typeparam>
|
||||
/// <param name="read">One forward read <c>[start, end]</c> asking for at most <c>cap</c> rows (<c>0</c> = all).</param>
|
||||
/// <param name="ticksOf">A row's timestamp, in ticks.</param>
|
||||
/// <param name="tieOrder">Order within one timestamp (null = the backend's forward order).</param>
|
||||
/// <param name="hi">The later (inclusive) bound.</param>
|
||||
/// <param name="lo">The earlier (exclusive) bound.</param>
|
||||
/// <param name="want">Cap; <c>0</c> = every row in the window (one forward read).</param>
|
||||
/// <param name="tieBudget">The tie budget M (values below 1 count as 1).</param>
|
||||
/// <param name="resolutionTicks">The backend's timestamp resolution (>= 1).</param>
|
||||
/// <param name="nowUtc">Now, for the look-ahead clamp.</param>
|
||||
/// <param name="hint">What the previous page of the same read carried (default = none).</param>
|
||||
/// <param name="tooManyAtOneTimestamp">Builds the exception for a timestamp holding more than M rows.</param>
|
||||
/// <param name="ct">Cancellation.</param>
|
||||
/// <param name="useDiscoveryBudgetForGaps">Use ceiling-sized discovery and retain truncated remainder rows (#696, strict raw backends only).</param>
|
||||
/// <returns>At most <paramref name="want"/> rows, newest first, and the hint for the next page.</returns>
|
||||
public static async Task<(List<T> Rows, ReverseWalkHint Next)> WalkAsync<T>(
|
||||
Func<DateTime, DateTime, int, CancellationToken, Task<ForwardSegmentRead<T>>> read,
|
||||
Func<T, long> ticksOf,
|
||||
IComparer<T>? tieOrder,
|
||||
DateTime hi, DateTime lo, uint want, int tieBudget, long resolutionTicks, DateTime nowUtc,
|
||||
ReverseWalkHint hint, Func<DateTime, Exception> tooManyAtOneTimestamp, CancellationToken ct, bool useDiscoveryBudgetForGaps = false)
|
||||
{
|
||||
var resolution = Math.Max(1L, resolutionTicks);
|
||||
var granularity = Math.Max(Alignment, resolution);
|
||||
var budget = Math.Max(1, tieBudget);
|
||||
var result = new List<T>();
|
||||
|
||||
var horizon = nowUtc.Ticks > DateTime.MaxValue.Ticks - LookAhead.Ticks ? DateTime.MaxValue.Ticks : nowUtc.Ticks + LookAhead.Ticks;
|
||||
var hiTicks = Math.Min(hi.Ticks, horizon);
|
||||
var stopTicks = Math.Max(lo.Ticks, Floor.Ticks);
|
||||
if (hiTicks <= stopTicks) return (result, hint);
|
||||
|
||||
if (want == 0)
|
||||
{
|
||||
// One forward read of the whole window, reversed — the uncapped forward read's own cost and bound.
|
||||
var all = (await read(Utc(AlignDown(stopTicks, granularity)), Utc(AlignUp(hiTicks + 1, granularity)), 0, ct)
|
||||
.ConfigureAwait(false)).Rows;
|
||||
AppendNewestFirst(all, ticksOf, tieOrder, stopTicks, hiTicks, result, int.MaxValue);
|
||||
return (result, hint);
|
||||
}
|
||||
|
||||
var maxWidth = DateTime.MaxValue.Ticks;
|
||||
var discovering = hint.WidthTicks <= 0;
|
||||
var width = discovering ? Math.Max(resolution, InitialSegment.Ticks) : Math.Clamp(hint.WidthTicks, resolution, maxWidth);
|
||||
var pageBudget = PageBudget(want, budget);
|
||||
var discoveryBudget = Math.Min(budget, Math.Max(pageBudget, DiscoveryRows));
|
||||
|
||||
// `known` > 0: a truncated read saw rows up to this timestamp (below segHi), and nothing is known of (known, segHi].
|
||||
// Segments stay above it until that stretch is read; then `knownWidth` (the density the truncated read showed)
|
||||
// sizes the segment that ends on it. Searching the gap: first a top-anchored segment at that density (data running
|
||||
// right up to segHi); if it is empty, the data stops somewhere above `known`, so GALLOP up from it — read
|
||||
// (known + lowerBy, segHi] with lowerBy growing ×4, `known` following what each truncated read saw — until a read
|
||||
// comes back empty (the top is now bracketed below its segLo), then BISECT that bracket. Any complete read holding
|
||||
// rows ends the search. A burst that ended long before segHi is found in a few reads instead of bisecting the whole
|
||||
// gap down to the millisecond, and a long dense stretch is crossed geometrically, not a budget at a time (#691.4
|
||||
// review). `noGallop` falls back to plain top-anchored halving if a galloping read is crowded by rows before it.
|
||||
var known = 0L;
|
||||
var knownWidth = 0L;
|
||||
var lowerBy = 0L;
|
||||
var bisecting = false;
|
||||
var noGallop = false;
|
||||
var segHi = hiTicks;
|
||||
|
||||
// #691: the remainder probe (once per page, after two empty segments) and the empty stretches.
|
||||
var emptyRun = 0;
|
||||
var probed = false;
|
||||
var carried = hint.HasEmptyStretch ? (hint.EmptyFromTicks, hint.EmptyToTicks) : (0L, 0L);
|
||||
(long From, long To) proven = (0L, 0L);
|
||||
|
||||
while (segHi > stopTicks && result.Count < want)
|
||||
{
|
||||
if (carried.Item2 > carried.Item1 && segHi > carried.Item1 && segHi < carried.Item2)
|
||||
{
|
||||
segHi = carried.Item1; // (from, segHi] lies inside a stretch a previous page proved empty
|
||||
continue;
|
||||
}
|
||||
|
||||
var probe = !probed && known == 0 && emptyRun >= 2 && segHi - stopTicks > width;
|
||||
var w = probe ? segHi - stopTicks : width;
|
||||
var lowered = false;
|
||||
var bisected = false;
|
||||
if (known > 0)
|
||||
{
|
||||
var gap = segHi - known;
|
||||
lowered = lowerBy > 0 && lowerBy < gap;
|
||||
bisected = !lowered && (bisecting || lowerBy > 0) && gap > 2 * Math.Max(granularity, knownWidth);
|
||||
// Never narrower than the request granularity: requests are rounded outward to it, so a narrower segment
|
||||
// re-sends the same request — and a width left at one tick (a tie cluster's) stepped segHi down a tick per
|
||||
// read, ~10,000 identical reads (#691.4 re-review). Keep the Min: a truncated read crowded by rows before
|
||||
// the segment must still shrink the segment.
|
||||
w = lowered ? gap - lowerBy
|
||||
: bisected ? gap / 2
|
||||
: Math.Min(Math.Max(w, granularity), gap <= 2 * granularity ? gap : gap / 2);
|
||||
}
|
||||
var segLo = segHi - stopTicks <= w ? stopTicks : segHi - w;
|
||||
var start = Utc(AlignDown(segLo, granularity));
|
||||
var end = Utc(AlignUp(segHi + 1, granularity));
|
||||
var segBudget = (discovering && !probe) || (useDiscoveryBudgetForGaps && (probe || known > 0))
|
||||
? discoveryBudget : pageBudget;
|
||||
var cap = segBudget + 2;
|
||||
IReadOnlyList<T> rows;
|
||||
var narrowed = false;
|
||||
|
||||
while (true)
|
||||
{
|
||||
ct.ThrowIfCancellationRequested();
|
||||
var segment = await read(start, end, cap, ct).ConfigureAwait(false);
|
||||
rows = segment.Rows;
|
||||
if (!segment.Truncated || rows.Count == 0 || ticksOf(rows[^1]) > segHi) break; // complete for (segLo, segHi]
|
||||
|
||||
if (probe)
|
||||
{
|
||||
// #696: strict-ceiling discovery retains the rows/density instead of discarding the remainder probe.
|
||||
probed = true;
|
||||
if (!useDiscoveryBudgetForGaps)
|
||||
{
|
||||
narrowed = true;
|
||||
break;
|
||||
}
|
||||
probe = false;
|
||||
}
|
||||
|
||||
var leading = 0;
|
||||
var inner = 0;
|
||||
var firstInner = 0L;
|
||||
var lastInner = 0L;
|
||||
foreach (var r in rows)
|
||||
{
|
||||
var ts = ticksOf(r);
|
||||
if (ts <= segLo) leading++;
|
||||
else if (ts <= segHi)
|
||||
{
|
||||
if (inner++ == 0) firstInner = ts;
|
||||
lastInner = ts;
|
||||
}
|
||||
}
|
||||
|
||||
// Crowded out by rows before the segment, with the segment itself possibly within budget: re-read once
|
||||
// with room for them.
|
||||
var roomier = leading + segBudget + 2;
|
||||
if (leading > 0 && inner <= segBudget && roomier > cap && roomier <= 2 * segBudget + 4)
|
||||
{
|
||||
cap = roomier;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (segHi - segLo <= granularity)
|
||||
{
|
||||
// A segment no wider than the request granularity (1 ms — narrower requests are the same request) still
|
||||
// over its budget: a tie cluster, or a burst inside one millisecond. Escalate to the tie budget M before
|
||||
// refusing — up to M rows within one millisecond are served (#686); more are refused, per millisecond
|
||||
// (#691.4 re-review — exact for the millisecond-resolution Gateway).
|
||||
if (segBudget < budget)
|
||||
{
|
||||
segBudget = budget;
|
||||
cap = segBudget + 2;
|
||||
continue;
|
||||
}
|
||||
|
||||
throw tooManyAtOneTimestamp(Utc(segHi));
|
||||
}
|
||||
|
||||
// Narrow — the segment shrinks every time, so this terminates.
|
||||
if (inner > 0 && lastInner == segHi)
|
||||
{
|
||||
width = resolution; // a tie cluster at the top of the segment: read that one timestamp
|
||||
lowerBy = 0;
|
||||
bisecting = false;
|
||||
}
|
||||
else if (inner > 0 && (lowered || bisected))
|
||||
{
|
||||
// Data reaches past this segment's bottom: move `known` up to what this read saw; a gallop goes further.
|
||||
known = lastInner;
|
||||
if (lowered) lowerBy = lowerBy > maxWidth / 4 ? maxWidth : lowerBy * 4;
|
||||
}
|
||||
else if (inner > 0)
|
||||
{
|
||||
// Rows exist at lastInner; what lies above it is unknown. Aim at about half the page budget at the
|
||||
// density the truncated rows showed (their span, not the segment's — it may open on an empty stretch).
|
||||
var aimed = inner > 1 && lastInner > firstInner
|
||||
? (long)Math.Min((double)(lastInner - firstInner) * ((useDiscoveryBudgetForGaps ? segBudget : pageBudget) / 2.0) / (inner - 1), maxWidth)
|
||||
: resolution;
|
||||
known = lastInner;
|
||||
knownWidth = Math.Max(resolution, aimed);
|
||||
width = Math.Max(resolution, Math.Min(aimed, segHi - segLo - 1));
|
||||
lowerBy = 0;
|
||||
bisecting = false;
|
||||
}
|
||||
else
|
||||
{
|
||||
// Only rows before the segment: shrink it — top-anchored from here on (a gallop would re-send the same
|
||||
// crowded read).
|
||||
width = Math.Max(resolution, Math.Min(width, segHi - segLo) / 4);
|
||||
if (lowered || bisected) noGallop = true;
|
||||
lowerBy = 0;
|
||||
bisecting = false;
|
||||
}
|
||||
|
||||
discovering = false;
|
||||
narrowed = true;
|
||||
break;
|
||||
}
|
||||
|
||||
if (narrowed) continue; // re-read the same segHi, narrower
|
||||
|
||||
var count = AppendNewestFirst(rows, ticksOf, tieOrder, segLo, segHi, result, (int)Math.Min(int.MaxValue, want),
|
||||
out var oldest);
|
||||
// This complete read proves (segLo, oldest) empty — or the whole segment when it held nothing.
|
||||
proven = count > 0 ? (segLo, oldest) : (segLo, segHi + 1);
|
||||
if (probe)
|
||||
{
|
||||
probed = true;
|
||||
segHi = segLo; // the whole remainder is read
|
||||
continue;
|
||||
}
|
||||
|
||||
emptyRun = count == 0 ? emptyRun + 1 : 0;
|
||||
var grown = width > maxWidth / 4 ? maxWidth : width * 4;
|
||||
if (count == 0)
|
||||
{
|
||||
discovering = true; // ×4 growth may overshoot into dense data
|
||||
if (known > 0)
|
||||
{
|
||||
// An empty top-anchored segment: gallop up from `known`. An empty galloping read: the top is bracketed
|
||||
// in (known, segLo] — bisect it (an empty bisecting read keeps bisecting).
|
||||
if (lowered || bisected)
|
||||
{
|
||||
lowerBy = 0;
|
||||
bisecting = true;
|
||||
}
|
||||
else if (noGallop)
|
||||
{
|
||||
width = maxWidth; // halve the gap, top-anchored
|
||||
}
|
||||
else
|
||||
{
|
||||
lowerBy = Math.Max(knownWidth, granularity);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
width = grown;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
// Re-aim at about half the page budget from the density just seen.
|
||||
width = (long)Math.Clamp((double)(segHi - segLo) * Math.Max(1.0, pageBudget / 2.0) / count, resolution, grown);
|
||||
discovering = false;
|
||||
known = 0;
|
||||
lowerBy = 0;
|
||||
bisecting = false;
|
||||
noGallop = false;
|
||||
}
|
||||
|
||||
segHi = segLo;
|
||||
if (known > 0 && segHi <= known)
|
||||
{
|
||||
// Everything above the newest row the truncated read saw is read: the next segment ends on it, at its density.
|
||||
known = 0;
|
||||
lowerBy = 0;
|
||||
bisecting = false;
|
||||
noGallop = false;
|
||||
width = knownWidth;
|
||||
discovering = false;
|
||||
}
|
||||
}
|
||||
|
||||
// Carry the stretch the last complete read proved empty, joined to the one carried in when they touch; when they
|
||||
// do not, keep the one reaching the floor (it saves a whole walk), else the newer.
|
||||
var next = carried;
|
||||
if (proven.To > proven.From)
|
||||
{
|
||||
if (carried.Item2 > carried.Item1 && proven.From < carried.Item2 && carried.Item1 < proven.To)
|
||||
next = (Math.Min(proven.From, carried.Item1), Math.Max(proven.To, carried.Item2));
|
||||
else if (!(carried.Item2 > carried.Item1 && carried.Item1 <= stopTicks) || proven.From <= stopTicks)
|
||||
next = proven;
|
||||
}
|
||||
|
||||
return (result, new ReverseWalkHint(width, next.Item1, next.Item2));
|
||||
}
|
||||
|
||||
/// <summary>Appends the rows in <c>(segLo, segHi]</c> newest-first (until <paramref name="limit"/>) and counts them.</summary>
|
||||
private static int AppendNewestFirst<T>(
|
||||
IReadOnlyList<T> rows, Func<T, long> ticksOf, IComparer<T>? tieOrder, long segLo, long segHi, List<T> result, int limit) =>
|
||||
AppendNewestFirst(rows, ticksOf, tieOrder, segLo, segHi, result, limit, out _);
|
||||
|
||||
/// <summary>As above, also returning the oldest in-range timestamp (meaningless when none).</summary>
|
||||
private static int AppendNewestFirst<T>(
|
||||
IReadOnlyList<T> rows, Func<T, long> ticksOf, IComparer<T>? tieOrder, long segLo, long segHi, List<T> result, int limit,
|
||||
out long oldest)
|
||||
{
|
||||
oldest = long.MaxValue;
|
||||
IReadOnlyList<T> ordered = rows;
|
||||
if (tieOrder is not null)
|
||||
{
|
||||
var copy = rows.ToList();
|
||||
copy.Sort((a, b) =>
|
||||
{
|
||||
var c = ticksOf(a).CompareTo(ticksOf(b));
|
||||
return c != 0 ? c : tieOrder.Compare(a, b);
|
||||
});
|
||||
ordered = copy;
|
||||
}
|
||||
|
||||
var count = 0;
|
||||
for (var i = ordered.Count - 1; i >= 0; i--)
|
||||
{
|
||||
var ts = ticksOf(ordered[i]);
|
||||
if (ts <= segLo || ts > segHi) continue;
|
||||
count++;
|
||||
if (ts < oldest) oldest = ts;
|
||||
if (result.Count < limit) result.Add(ordered[i]);
|
||||
}
|
||||
|
||||
return count;
|
||||
}
|
||||
|
||||
private static DateTime Utc(long ticks) => new(ticks, DateTimeKind.Utc);
|
||||
|
||||
private static long AlignDown(long ticks, long step) => ticks - ticks % step;
|
||||
|
||||
private static long AlignUp(long ticks, long step)
|
||||
{
|
||||
var down = AlignDown(ticks, step);
|
||||
if (down == ticks) return ticks;
|
||||
return down > DateTime.MaxValue.Ticks - step ? DateTime.MaxValue.Ticks : down + step;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Orders events within one timestamp by <see cref="HistoricalEvent.EventId"/> (ordinal) — the tie order a walked-back
|
||||
/// events read (#691.4) and its oversized-cluster over-fetch share, because the Gateway orders by event time only.
|
||||
/// </summary>
|
||||
internal sealed class EventIdTieOrder : IComparer<HistoricalEvent>
|
||||
{
|
||||
/// <summary>The shared instance.</summary>
|
||||
public static readonly EventIdTieOrder Instance = new();
|
||||
|
||||
/// <inheritdoc />
|
||||
public int Compare(HistoricalEvent? x, HistoricalEvent? y) => string.CompareOrdinal(x?.EventId, y?.EventId);
|
||||
}
|
||||
@@ -38,8 +38,8 @@ public sealed class SdkServiceLevelPublisher : IServiceLevelPublisher
|
||||
var node = _serverInternal.ServerObject?.ServiceLevel;
|
||||
if (node is null)
|
||||
{
|
||||
_logger.LogWarning("SdkServiceLevelPublisher: ServerObject.ServiceLevel unavailable; skipping write");
|
||||
return;
|
||||
_logger.LogWarning("SdkServiceLevelPublisher: ServerObject.ServiceLevel unavailable");
|
||||
throw new InvalidOperationException("Server.ServiceLevel is unavailable.");
|
||||
}
|
||||
|
||||
try
|
||||
@@ -55,6 +55,7 @@ public sealed class SdkServiceLevelPublisher : IServiceLevelPublisher
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "SdkServiceLevelPublisher: write to Server.ServiceLevel threw");
|
||||
throw;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -82,13 +82,42 @@ public sealed class ServerHistorianOptions
|
||||
/// <c>MaxTieClusterOverfetch + 1</c> ties at that single timestamp (a <c>start == end</c> read) and
|
||||
/// pages through them; this bounds how large a single-timestamp burst the server will buffer in
|
||||
/// memory before it gives up and surfaces <c>BadTooManyOperations</c> for that node (#686; was
|
||||
/// <c>BadHistoryOperationUnsupported</c>). It is also the per-read row budget of a reverse raw read walked
|
||||
/// back from forward reads on a backend that cannot read newest-first (the Gateway): one timestamp holding
|
||||
/// <c>BadHistoryOperationUnsupported</c>). It is also the tie-cluster CEILING of a reverse read walked
|
||||
/// back from forward reads on a backend that cannot read newest-first (the Gateway; segments are sized from the page, #689): one timestamp holding
|
||||
/// more samples than this is refused <c>BadTooManyOperations</c>. The default (65536) comfortably covers
|
||||
/// any realistic same-millisecond burst; a value below 1 is treated as 1.
|
||||
/// </summary>
|
||||
public int MaxTieClusterOverfetch { get; init; } = 65536;
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #691: whether HistoryRead-Raw honours <c>ReadRawModifiedDetails.ReturnBounds</c> (OPC UA Part 11 bounding
|
||||
/// values). Default <c>true</c>. <c>false</c> is the kill switch back to the pre-#691 answer: the flag is ignored
|
||||
/// (stored samples only) and a modified read carrying it is not refused <c>Bad_InvalidArgument</c>. Note the .NET
|
||||
/// SDK's <c>new ReadRawModifiedDetails()</c> defaults <c>ReturnBounds</c> (and <c>IsReadModified</c>) to <c>true</c>.
|
||||
/// </summary>
|
||||
public bool HonorReturnBounds { get; init; } = true;
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #693, <c>Gateway</c> backend only: whether HistoryRead-AtTime is served from RAW gateway reads evaluated with
|
||||
/// the SQL backends' shared AtTime rules (exact stored value at a stored time, linear between usable neighbours with
|
||||
/// the Interpolated bit, held before a Bad neighbour as Uncertain, null + the outage's Bad code inside an outage,
|
||||
/// <c>BadNoData</c> before the first sample). Default <c>true</c>. <c>false</c> is the kill switch back to the
|
||||
/// pre-#693 answer: AVEVA's own interpolation (the gateway's <c>ReadAtTime</c>), passed through unchanged.
|
||||
/// </summary>
|
||||
public bool GatewayAtTimeFromRaw { get; init; } = true;
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #694 (opt-in): HistoryReadEvents never reads events older than <c>now − EventReadHorizon</c> — the earlier
|
||||
/// bound of every fresh events window is raised to it (both directions, every backend; a window wholly older is
|
||||
/// <c>GoodNoData</c>). Default <see cref="TimeSpan.Zero"/> = no horizon: every read is exact to its own window. It
|
||||
/// exists for the Gateway, where proving "no older events" costs a read down to the Unix epoch on
|
||||
/// <c>Runtime.dbo.Events</c> (~3.8 s live per walked reverse / end-only read; an unfiltered one can hit the
|
||||
/// gateway's 30 s SQL timeout) and a forward read from a far-past StartTime costs the same. ⚠️ Lossy by design:
|
||||
/// events older than the horizon are invisible to HistoryReadEvents while it is set. Negative = none (a
|
||||
/// <see cref="Validate"/> warning).
|
||||
/// </summary>
|
||||
public TimeSpan EventReadHorizon { get; init; } = TimeSpan.Zero;
|
||||
|
||||
/// <summary>
|
||||
/// Maximum number of historized nodes served CONCURRENTLY within one HistoryRead batch (arch-review
|
||||
/// 03/S3). The SDK's <c>HistoryRead*</c> override surface is synchronous, so each arm block-bridges
|
||||
@@ -179,6 +208,8 @@ public sealed class ServerHistorianOptions
|
||||
warnings.Add($"ServerHistorian:MaxConcurrentHistoryReadBatches is {MaxConcurrentHistoryReadBatches} — must be > 0; the process-wide HistoryRead batch limiter is disabled.");
|
||||
if (MaxTimeWeightedRawRows <= 0)
|
||||
warnings.Add($"ServerHistorian:MaxTimeWeightedRawRows is {MaxTimeWeightedRawRows} — must be > 0; time-weighted aggregate reads on the SqlServer/QuestDb backends stream an unbounded number of raw samples.");
|
||||
if (EventReadHorizon < TimeSpan.Zero)
|
||||
warnings.Add($"ServerHistorian:EventReadHorizon is {EventReadHorizon} — negative is treated as no horizon (00:00:00); HistoryReadEvents reads every window in full.");
|
||||
if (HistoryReadDeadline <= TimeSpan.Zero)
|
||||
warnings.Add($"ServerHistorian:HistoryReadDeadline is {HistoryReadDeadline} — must be > 0; HistoryRead requests run unbounded (no per-request deadline).");
|
||||
return warnings;
|
||||
|
||||
@@ -970,10 +970,10 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
|
||||
// node starting Detached/role-less (first level = 0) would be dedup'd away, leaving the SDK's
|
||||
// built-in default (255 = full service) standing — a degraded node wrongly advertising 255.
|
||||
if (_publishedAtLeastOnce && msg.ServiceLevel == _lastServiceLevel) return;
|
||||
_lastServiceLevel = msg.ServiceLevel;
|
||||
try
|
||||
{
|
||||
_serviceLevel.Publish(msg.ServiceLevel);
|
||||
_lastServiceLevel = msg.ServiceLevel;
|
||||
_publishedAtLeastOnce = true;
|
||||
OtOpcUaTelemetry.ServiceLevelChange.Add(1,
|
||||
new KeyValuePair<string, object?>("level", msg.ServiceLevel));
|
||||
@@ -981,6 +981,8 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
// A failed write may be partially applied; even a previous successful value must retry.
|
||||
_publishedAtLeastOnce = false;
|
||||
_log.Warning(ex, "OpcUaPublish: ServiceLevel publisher threw at level {Level}", msg.ServiceLevel);
|
||||
}
|
||||
}
|
||||
|
||||
+136
-5
@@ -10,10 +10,10 @@ namespace ZB.MOM.WW.OtOpcUa.Commons.Tests.OpcUa;
|
||||
public class DeferredServiceLevelPublisherTests
|
||||
{
|
||||
[Fact]
|
||||
public void Before_SetInner_Publish_is_a_noop()
|
||||
public void Before_SetInner_Publish_accepts_a_cached_request()
|
||||
{
|
||||
var publisher = new DeferredServiceLevelPublisher();
|
||||
// Must not throw; the NullServiceLevelPublisher absorbs it.
|
||||
// Accept the request before an SDK publisher exists.
|
||||
publisher.Publish(200);
|
||||
}
|
||||
|
||||
@@ -30,7 +30,7 @@ public class DeferredServiceLevelPublisherTests
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SetInner_null_reverts_to_null_publisher()
|
||||
public void SetInner_null_detaches_the_previous_publisher()
|
||||
{
|
||||
var spy = new SpyPublisher();
|
||||
var publisher = new DeferredServiceLevelPublisher();
|
||||
@@ -39,8 +39,139 @@ public class DeferredServiceLevelPublisherTests
|
||||
|
||||
publisher.Publish(100);
|
||||
|
||||
// The spy should NOT have received the publish after revert.
|
||||
spy.LastPublished.ShouldBe((byte)0, "spy should not have been updated after revert to null");
|
||||
// The previous publisher must not receive an unbound request.
|
||||
spy.LastPublished.ShouldBe((byte)0, "the previous publisher should not be updated after unbinding");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Binding_without_a_request_does_not_invent_a_level()
|
||||
{
|
||||
var received = new List<byte>();
|
||||
new DeferredServiceLevelPublisher().SetInner(new CallbackPublisher(received.Add));
|
||||
received.ShouldBeEmpty();
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData((byte)100)]
|
||||
[InlineData((byte)0)]
|
||||
public void Binding_replays_the_latest_request_including_zero(byte latest)
|
||||
{
|
||||
var publisher = new DeferredServiceLevelPublisher();
|
||||
publisher.Publish(240);
|
||||
publisher.Publish(latest);
|
||||
var received = new List<byte>();
|
||||
publisher.SetInner(new CallbackPublisher(received.Add));
|
||||
received.ShouldBe(new[] { latest });
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Unbinding_retains_latest_for_rebinding()
|
||||
{
|
||||
var publisher = new DeferredServiceLevelPublisher();
|
||||
var old = new List<byte>();
|
||||
publisher.SetInner(new CallbackPublisher(old.Add));
|
||||
publisher.Publish(240);
|
||||
publisher.SetInner(null);
|
||||
publisher.Publish(0);
|
||||
var replacement = new List<byte>();
|
||||
publisher.SetInner(new CallbackPublisher(replacement.Add));
|
||||
old.ShouldBe(new byte[] { 240 });
|
||||
replacement.ShouldBe(new byte[] { 0 });
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Failed_publish_retains_the_requested_value_for_replay()
|
||||
{
|
||||
var publisher = new DeferredServiceLevelPublisher();
|
||||
publisher.SetInner(new CallbackPublisher(_ => throw new InvalidOperationException()));
|
||||
Should.Throw<InvalidOperationException>(() => publisher.Publish(0));
|
||||
var received = new List<byte>();
|
||||
publisher.SetInner(new CallbackPublisher(received.Add));
|
||||
received.ShouldBe(new byte[] { 0 });
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Failed_replay_keeps_candidate_bound_for_later_publish()
|
||||
{
|
||||
var publisher = new DeferredServiceLevelPublisher();
|
||||
publisher.Publish(100);
|
||||
var fail = true;
|
||||
var received = new List<byte>();
|
||||
var candidate = new CallbackPublisher(value =>
|
||||
{
|
||||
if (fail) throw new InvalidOperationException();
|
||||
received.Add(value);
|
||||
});
|
||||
Should.Throw<InvalidOperationException>(() => publisher.SetInner(candidate));
|
||||
fail = false;
|
||||
publisher.Publish(0);
|
||||
publisher.SetInner(null);
|
||||
publisher.SetInner(candidate);
|
||||
received.ShouldBe(new byte[] { 0, 0 });
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Failed_replay_retains_latest_for_a_replacement_publisher()
|
||||
{
|
||||
var publisher = new DeferredServiceLevelPublisher();
|
||||
publisher.Publish(100);
|
||||
Should.Throw<InvalidOperationException>(() => publisher.SetInner(
|
||||
new CallbackPublisher(_ => throw new InvalidOperationException())));
|
||||
var received = new List<byte>();
|
||||
publisher.SetInner(new CallbackPublisher(received.Add));
|
||||
received.ShouldBe(new byte[] { 100 });
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Binding_waits_for_inflight_publish_then_replays_latest()
|
||||
{
|
||||
var publisher = new DeferredServiceLevelPublisher();
|
||||
using var entered = new ManualResetEventSlim();
|
||||
using var release = new ManualResetEventSlim();
|
||||
publisher.SetInner(new CallbackPublisher(_ =>
|
||||
{
|
||||
entered.Set();
|
||||
release.Wait(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken).ShouldBeTrue();
|
||||
}));
|
||||
var publish = Task.Run(() => publisher.Publish(100), TestContext.Current.CancellationToken);
|
||||
var received = new List<byte>();
|
||||
using var bindingStarted = new ManualResetEventSlim();
|
||||
Exception? bindingError = null;
|
||||
var binding = new Thread(() =>
|
||||
{
|
||||
bindingStarted.Set();
|
||||
try { publisher.SetInner(new CallbackPublisher(received.Add)); }
|
||||
catch (Exception ex) { bindingError = ex; }
|
||||
});
|
||||
var bindingWasStarted = false;
|
||||
try
|
||||
{
|
||||
entered.Wait(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken).ShouldBeTrue();
|
||||
binding.Start();
|
||||
bindingWasStarted = true;
|
||||
bindingStarted.Wait(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken).ShouldBeTrue();
|
||||
// Past the marker, SetInner is the thread's only potentially blocking operation.
|
||||
// Observe actual contention rather than assuming Task.Run has entered the adapter.
|
||||
SpinWait.SpinUntil(() => !binding.IsAlive ||
|
||||
(binding.ThreadState & System.Threading.ThreadState.WaitSleepJoin) != 0,
|
||||
TimeSpan.FromSeconds(3)).ShouldBeTrue();
|
||||
(binding.ThreadState & System.Threading.ThreadState.WaitSleepJoin).ShouldNotBe(
|
||||
(System.Threading.ThreadState)0, "binding must wait for the in-flight delivery");
|
||||
}
|
||||
finally
|
||||
{
|
||||
release.Set();
|
||||
if (bindingWasStarted) binding.Join(TimeSpan.FromSeconds(5)).ShouldBeTrue();
|
||||
await publish.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken);
|
||||
}
|
||||
bindingError.ShouldBeNull();
|
||||
publisher.Publish(0);
|
||||
received.ShouldBe(new byte[] { 100, 0 });
|
||||
}
|
||||
|
||||
private sealed class CallbackPublisher(Action<byte> publish) : IServiceLevelPublisher
|
||||
{
|
||||
public void Publish(byte serviceLevel) => publish(serviceLevel);
|
||||
}
|
||||
|
||||
// ---- test double ----
|
||||
|
||||
+15
@@ -94,4 +94,19 @@ public sealed class HistorianDataSourceReverseEventsGuardTests
|
||||
$"{type.FullName} must declare SupportedAggregates itself, not inherit the interface default");
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Every_implementation_declares_its_raw_page_limit_itself()
|
||||
{
|
||||
// Gitea #694: the default (0 = no limit) is what a strict-ceiling Gateway must NOT report — a wrapper that inherited
|
||||
// it would bring back BadTooManyOperations for every page larger than the gateway's ceiling.
|
||||
foreach (var type in ProductionImplementations())
|
||||
{
|
||||
var map = type.GetInterfaceMap(typeof(IHistorianDataSource));
|
||||
var getter = map.TargetMethods[Array.FindIndex(map.InterfaceMethods,
|
||||
m => m.Name == "get_" + nameof(IHistorianDataSource.MaxRawValuesPerRead))];
|
||||
getter.DeclaringType.ShouldBe(type,
|
||||
$"{type.FullName} must declare MaxRawValuesPerRead itself, not inherit the interface default");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+7
-5
@@ -68,17 +68,19 @@ public sealed class NullHistorianDataSourceTests
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// #675: the direction-aware overload's default — forward is the five-argument read, reverse is refused
|
||||
/// (the server answers <c>BadHistoryOperationUnsupported</c>).
|
||||
/// #691.4 review: like raw, the null source reads events in either direction (empty) and says so — the server sends
|
||||
/// it the reverse read directly instead of walking back through empty forward reads (it refused reverse under #675).
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task Direction_aware_ReadEventsAsync_forwards_forward_and_refuses_reverse()
|
||||
public async Task Direction_aware_ReadEventsAsync_is_empty_in_both_directions()
|
||||
{
|
||||
IHistorianDataSource source = Source;
|
||||
source.SupportsReverseEvents.ShouldBeTrue();
|
||||
(await source.ReadEventsAsync(null, DateTime.UtcNow.AddHours(-1), DateTime.UtcNow, 0, reverse: false, Ct))
|
||||
.Events.ShouldBeEmpty();
|
||||
await Should.ThrowAsync<NotSupportedException>(() =>
|
||||
source.ReadEventsAsync(null, DateTime.UtcNow, DateTime.UtcNow.AddHours(-1), 0, reverse: true, Ct));
|
||||
var reverse = await source.ReadEventsAsync(null, DateTime.UtcNow, DateTime.UtcNow.AddHours(-1), 0, reverse: true, Ct);
|
||||
reverse.Events.ShouldBeEmpty();
|
||||
reverse.ContinuationPoint.ShouldBeNull();
|
||||
}
|
||||
|
||||
/// <summary>GetHealthSnapshot reports a fully-disabled (zeroed, disconnected, no-node) shape.</summary>
|
||||
|
||||
+78
@@ -0,0 +1,78 @@
|
||||
using System.Text.Json;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian.Tests;
|
||||
|
||||
public sealed class AlarmHistorianEventIdentityTests
|
||||
{
|
||||
internal const string Payload = """{"AlarmId":"A","EquipmentPath":"S","AlarmName":"N","AlarmTypeName":"LimitAlarm","Severity":2,"EventKind":"Activated","Message":"m","User":"system","Comment":null,"TimestampUtc":"2026-01-01T00:00:00Z"}""";
|
||||
internal const string QueueId = "52C8292271C629CCC2BFE4EC8C16A5E70CA7456C7D88BC282ABCEDF636F05901";
|
||||
private static AlarmHistorianEvent Event() => new("A", "S", "N", "LimitAlarm", AlarmSeverity.High,
|
||||
"Activated", "m", "system", null, new DateTime(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc));
|
||||
|
||||
[Theory]
|
||||
[InlineData(QueueId, "48bca092-b99d-8ea4-a2de-8b30d80a481a")]
|
||||
[InlineData("persisted-row-v1", "39473040-002b-876c-a1e6-9a9ad4a60ded")]
|
||||
public void Persisted_keys_have_frozen_versioned_delivery_GUID_vectors(string key, string expected)
|
||||
{
|
||||
Assert.Equal(Guid.Parse(expected), AlarmDeliveryIds.FromQueueId(key));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Delivery_identity_is_nonzero_and_a_copy_preserves_it()
|
||||
{
|
||||
var original = Event();
|
||||
Assert.NotEqual(Guid.Empty, original.DeliveryId);
|
||||
Assert.Equal(original.DeliveryId, (original with { }).DeliveryId);
|
||||
Assert.Equal(original.DeliveryId, (original with { Comment = "copy" }).DeliveryId);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Business_equality_hash_and_historical_JSON_ignore_delivery_identity()
|
||||
{
|
||||
var first = Event();
|
||||
var second = Event();
|
||||
Assert.NotEqual(first.DeliveryId, second.DeliveryId);
|
||||
Assert.Equal(first, second);
|
||||
Assert.True(first == second);
|
||||
Assert.Equal(first.GetHashCode(), second.GetHashCode());
|
||||
Assert.Single(new HashSet<AlarmHistorianEvent> { first, second });
|
||||
Assert.Equal(Payload, JsonSerializer.Serialize(first));
|
||||
Assert.Equal(Payload, JsonSerializer.Serialize(second));
|
||||
Assert.Equal(QueueId, AlarmSfSchema.DeriveId(Payload));
|
||||
Assert.Equal(first, JsonSerializer.Deserialize<AlarmHistorianEvent>(Payload));
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("AlarmId")]
|
||||
[InlineData("EquipmentPath")]
|
||||
[InlineData("AlarmName")]
|
||||
[InlineData("AlarmTypeName")]
|
||||
[InlineData("Severity")]
|
||||
[InlineData("EventKind")]
|
||||
[InlineData("Message")]
|
||||
[InlineData("User")]
|
||||
[InlineData("Comment")]
|
||||
[InlineData("TimestampUtc")]
|
||||
public void Each_business_field_still_participates_in_equality(string field)
|
||||
{
|
||||
var first = Event();
|
||||
var changed = field switch
|
||||
{
|
||||
"AlarmId" => first with { AlarmId = "B" },
|
||||
"EquipmentPath" => first with { EquipmentPath = "T" },
|
||||
"AlarmName" => first with { AlarmName = "Other" },
|
||||
"AlarmTypeName" => first with { AlarmTypeName = "DiscreteAlarm" },
|
||||
"Severity" => first with { Severity = AlarmSeverity.Low },
|
||||
"EventKind" => first with { EventKind = "Cleared" },
|
||||
"Message" => first with { Message = "other" },
|
||||
"User" => first with { User = "operator" },
|
||||
"Comment" => first with { Comment = "note" },
|
||||
"TimestampUtc" => first with { TimestampUtc = first.TimestampUtc.AddTicks(1) },
|
||||
_ => throw new ArgumentOutOfRangeException(nameof(field)),
|
||||
};
|
||||
Assert.NotEqual(first, changed);
|
||||
Assert.False(first == changed);
|
||||
}
|
||||
}
|
||||
+68
@@ -123,6 +123,69 @@ public sealed class LocalDbStoreAndForwardSinkTests : IDisposable
|
||||
Comment: null,
|
||||
TimestampUtc: ts ?? DateTime.UtcNow);
|
||||
|
||||
[Fact]
|
||||
public async Task Queued_delivery_identity_survives_retry_reopen_and_payload_changes()
|
||||
{
|
||||
// The persisted row key is authoritative: reserializing today's payload must never change a retry's identity.
|
||||
var expected = Guid.Parse("48bca092-b99d-8ea4-a2de-8b30d80a481a");
|
||||
var evt = System.Text.Json.JsonSerializer.Deserialize<AlarmHistorianEvent>(AlarmHistorianEventIdentityTests.Payload)!;
|
||||
var writer = new FakeWriter { DefaultOutcome = HistorianWriteOutcome.RetryPlease };
|
||||
using (var sink = new LocalDbStoreAndForwardSink(Db, writer, _log))
|
||||
{
|
||||
await sink.EnqueueAsync(evt, TestContext.Current.CancellationToken);
|
||||
var row = (await Db.QueryAsync("SELECT id, payload_json FROM alarm_sf_events",
|
||||
r => (Id: r.GetString(0), Payload: r.GetString(1)), null, TestContext.Current.CancellationToken)).Single();
|
||||
row.Id.ShouldBe(AlarmHistorianEventIdentityTests.QueueId);
|
||||
row.Payload.ShouldBe(AlarmHistorianEventIdentityTests.Payload);
|
||||
await sink.DrainOnceAsync(TestContext.Current.CancellationToken);
|
||||
await sink.DrainOnceAsync(TestContext.Current.CancellationToken);
|
||||
writer.Batches.ShouldAllBe(batch => batch.Single().DeliveryId == expected);
|
||||
await Db.ExecuteAsync("UPDATE alarm_sf_events SET payload_json = @Payload",
|
||||
new { Payload = System.Text.Json.JsonSerializer.Serialize(evt with { Comment = "edited" }) },
|
||||
TestContext.Current.CancellationToken);
|
||||
}
|
||||
_provider!.Dispose();
|
||||
_provider = null;
|
||||
_db = null;
|
||||
var reopenedWriter = new FakeWriter();
|
||||
using var reopened = new LocalDbStoreAndForwardSink(Db, reopenedWriter, _log);
|
||||
await reopened.DrainOnceAsync(TestContext.Current.CancellationToken);
|
||||
reopenedWriter.Batches.Single().Single().DeliveryId.ShouldBe(expected);
|
||||
reopenedWriter.Batches.Single().Single().Comment.ShouldBe("edited");
|
||||
reopened.GetStatus().QueueDepth.ShouldBe(0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Independently_created_identical_events_keep_the_historical_queue_key()
|
||||
{
|
||||
var first = System.Text.Json.JsonSerializer.Deserialize<AlarmHistorianEvent>(AlarmHistorianEventIdentityTests.Payload)!;
|
||||
var second = System.Text.Json.JsonSerializer.Deserialize<AlarmHistorianEvent>(AlarmHistorianEventIdentityTests.Payload)!;
|
||||
first.DeliveryId.ShouldNotBe(second.DeliveryId);
|
||||
using var sink = new LocalDbStoreAndForwardSink(Db, new FakeWriter(), _log);
|
||||
await sink.EnqueueAsync(first, TestContext.Current.CancellationToken);
|
||||
await sink.EnqueueAsync(second, TestContext.Current.CancellationToken);
|
||||
sink.GetStatus().QueueDepth.ShouldBe(1);
|
||||
(await Db.QueryAsync("SELECT id FROM alarm_sf_events", r => r.GetString(0), null, TestContext.Current.CancellationToken))
|
||||
.ShouldBe([AlarmHistorianEventIdentityTests.QueueId]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Distinct_business_events_at_the_same_time_have_distinct_queued_delivery_ids()
|
||||
{
|
||||
var first = Event("A", new DateTime(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc));
|
||||
var second = first with { EventKind = "Cleared", Comment = "different transition" };
|
||||
var writer = new FakeWriter();
|
||||
using var sink = new LocalDbStoreAndForwardSink(Db, writer, _log);
|
||||
await sink.EnqueueAsync(first, TestContext.Current.CancellationToken);
|
||||
await sink.EnqueueAsync(second, TestContext.Current.CancellationToken);
|
||||
sink.GetStatus().QueueDepth.ShouldBe(2);
|
||||
await sink.DrainOnceAsync(TestContext.Current.CancellationToken);
|
||||
var delivered = writer.Batches.Single();
|
||||
delivered.Count.ShouldBe(2);
|
||||
delivered.ShouldAllBe(evt => evt.DeliveryId != Guid.Empty);
|
||||
delivered.Select(evt => evt.DeliveryId).Distinct().Count().ShouldBe(2);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that acknowledged events are removed from the queue.</summary>
|
||||
[Fact]
|
||||
public async Task EnqueueThenDrain_Ack_removes_row()
|
||||
@@ -297,6 +360,11 @@ public sealed class LocalDbStoreAndForwardSinkTests : IDisposable
|
||||
var status = sink.GetStatus();
|
||||
status.QueueDepth.ShouldBe(1);
|
||||
status.DeadLetterDepth.ShouldBe(0);
|
||||
var originalDeliveryId = writer.Batches.Single().Single().DeliveryId;
|
||||
await sink.DrainOnceAsync(TestContext.Current.CancellationToken);
|
||||
writer.Batches.Count.ShouldBe(2);
|
||||
writer.Batches.Last().Single().DeliveryId.ShouldBe(originalDeliveryId);
|
||||
sink.GetStatus().QueueDepth.ShouldBe(0);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that the exponential backoff ladder caps at 60 seconds.</summary>
|
||||
|
||||
@@ -139,6 +139,7 @@ public abstract class AtTimeConformance : ConformanceBase
|
||||
(At(10), HistorianValue.FromInt64(15), Good));
|
||||
var s = (await B.CreateDataSource().ReadAtTimeAsync("I", [At(5), At(2)], Ct)).Samples;
|
||||
s[0].Value.ShouldBe(13L); // 12.5 ⇒ 13 (away from zero)
|
||||
s[0].Value.ShouldBeOfType<long>(); // ShouldBe compares numbers across types — the type needs its own check (#693)
|
||||
s[1].Value.ShouldBe(11L); // 11.0
|
||||
s[0].StatusCode.ShouldBe(GoodInterpolated);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,100 @@
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Conformance;
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #691: the server composes Part 11 bounding values (HistoryRead-Raw <c>ReturnBounds</c>) from exactly these raw
|
||||
/// reads on a backend that reads newest-first natively (<c>HistoryBounds</c> in the OPC UA server) — so every backend must
|
||||
/// answer them precisely: "at or before T" = a reverse read of <c>(0, T]</c> capped at one, "at or after T" = a forward read
|
||||
/// of <c>[T, MaxValue)</c> capped at one, "strictly after T" = the same from <c>T + resolution</c>. A bound is a stored
|
||||
/// sample of any quality, so a Bad row or an outage marker comes back as the bound.
|
||||
/// </summary>
|
||||
public abstract class BoundingValueConformance : ConformanceBase
|
||||
{
|
||||
private static readonly CancellationToken Ct = CancellationToken.None;
|
||||
private static readonly DateTime Floor = new(0, DateTimeKind.Utc);
|
||||
private static readonly DateTime Ceiling = new(DateTime.MaxValue.Ticks, DateTimeKind.Utc);
|
||||
|
||||
private async Task<DataValueSnapshot?> AtOrBeforeAsync(IHistorianDataSource src, string tag, DateTime t) =>
|
||||
(await src.ReadRawAsync(tag, t, Floor, 1, Ct)).Samples.SingleOrDefault();
|
||||
|
||||
private async Task<DataValueSnapshot?> AtOrAfterAsync(IHistorianDataSource src, string tag, DateTime t) =>
|
||||
(await src.ReadRawAsync(tag, t, Ceiling, 1, Ct)).Samples.SingleOrDefault();
|
||||
|
||||
[Fact]
|
||||
public async Task The_sample_at_or_before_a_time_is_the_latest_one_on_or_before_it()
|
||||
{
|
||||
await ConformanceData.SeedAAsync(B);
|
||||
var src = B.CreateDataSource();
|
||||
|
||||
(await AtOrBeforeAsync(src, "A", At(10)))!.Value.ShouldBe(20d); // exactly on it
|
||||
(await AtOrBeforeAsync(src, "A", At(15)))!.Value.ShouldBe(20d); // between samples
|
||||
(await AtOrBeforeAsync(src, "A", At(1000)))!.Value.ShouldBe(50d); // past the last
|
||||
(await AtOrBeforeAsync(src, "A", At(-1))).ShouldBeNull(); // before the first: no bound
|
||||
(await AtOrBeforeAsync(src, "A", At(15)))!.SourceTimestampUtc!.Value.Ticks.ShouldBe(B.Round(At(10)).Ticks);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task The_sample_at_or_after_a_time_is_the_earliest_one_on_or_after_it()
|
||||
{
|
||||
await ConformanceData.SeedAAsync(B);
|
||||
var src = B.CreateDataSource();
|
||||
|
||||
(await AtOrAfterAsync(src, "A", At(10)))!.Value.ShouldBe(20d); // exactly on it
|
||||
(await AtOrAfterAsync(src, "A", At(15)))!.Value.ShouldBe(30d); // between samples
|
||||
(await AtOrAfterAsync(src, "A", At(-100)))!.Value.ShouldBe(10d); // before the first
|
||||
(await AtOrAfterAsync(src, "A", At(41))).ShouldBeNull(); // past the last: no bound
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task The_sample_strictly_after_a_time_starts_one_storage_resolution_later()
|
||||
{
|
||||
await ConformanceData.SeedAAsync(B);
|
||||
var src = B.CreateDataSource();
|
||||
|
||||
(await AtOrAfterAsync(src, "A", At(10).AddTicks(B.ResolutionTicks)))!.Value.ShouldBe(30d);
|
||||
(await AtOrAfterAsync(src, "A", At(40).AddTicks(B.ResolutionTicks))).ShouldBeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_time_between_storage_ticks_finds_the_sample_the_backend_stores_at_its_truncation()
|
||||
{
|
||||
// The server compares "on the bound" at the storage resolution, so the lookups must agree with the window reads:
|
||||
// a time a fraction of a resolution step after a stored sample is that sample's instant on a coarse backend.
|
||||
await ConformanceData.SeedAAsync(B);
|
||||
var src = B.CreateDataSource();
|
||||
var t = At(10).AddTicks(B.ResolutionTicks - 1); // == At(10) on a 1-tick backend
|
||||
|
||||
(await AtOrBeforeAsync(src, "A", t))!.Value.ShouldBe(20d);
|
||||
var after = await AtOrAfterAsync(src, "A", t);
|
||||
after!.SourceTimestampUtc!.Value.Ticks.ShouldBeGreaterThanOrEqualTo(B.Round(t).Ticks);
|
||||
after.Value.ShouldBe(B.Round(t) == B.Round(At(10)) ? 20d : 30d);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_bad_sample_or_an_outage_marker_is_a_bound_as_stored()
|
||||
{
|
||||
await B.SeedAsync("M",
|
||||
(At(0), HistorianValue.FromDouble(10), Good),
|
||||
(At(5), HistorianValue.Empty, BadNoCommunication),
|
||||
(At(7), HistorianValue.FromDouble(99), Bad));
|
||||
var src = B.CreateDataSource();
|
||||
|
||||
var before = await AtOrBeforeAsync(src, "M", At(6));
|
||||
before!.Value.ShouldBeNull();
|
||||
before.StatusCode.ShouldBe(0x80310000u);
|
||||
var after = await AtOrAfterAsync(src, "M", At(6));
|
||||
after!.Value.ShouldBe(99d);
|
||||
after.StatusCode.ShouldBe(0x80000000u);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task An_unknown_tag_has_no_bounds()
|
||||
{
|
||||
var src = B.CreateDataSource();
|
||||
(await AtOrBeforeAsync(src, "nope", At(10))).ShouldBeNull();
|
||||
(await AtOrAfterAsync(src, "nope", At(10))).ShouldBeNull();
|
||||
}
|
||||
}
|
||||
+60
-4
@@ -12,6 +12,20 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests;
|
||||
/// </summary>
|
||||
public sealed class FakeHistorianGatewayClient : IHistorianGatewayClient
|
||||
{
|
||||
public bool TagExistsResult = true;
|
||||
public int TagExistsCallCount;
|
||||
public Exception? TagExistsThrows;
|
||||
/// <summary>Optional asynchronous catalog response for overlapping presence-cache tests.</summary>
|
||||
public Func<string, CancellationToken, Task<bool>>? TagExistsSource;
|
||||
public Task<bool> TagExistsAsync(string tag, CancellationToken ct)
|
||||
{
|
||||
ct.ThrowIfCancellationRequested();
|
||||
Interlocked.Increment(ref TagExistsCallCount);
|
||||
if (TagExistsSource is { } source) return source(tag, ct);
|
||||
if (TagExistsThrows is { } error) throw error;
|
||||
return Task.FromResult(TagExistsResult);
|
||||
}
|
||||
|
||||
// ---- ReadRaw -------------------------------------------------------------------------------
|
||||
public IReadOnlyList<HistorianSample> RawSamples = Array.Empty<HistorianSample>();
|
||||
public string? LastReadRawTag;
|
||||
@@ -54,6 +68,9 @@ public sealed class FakeHistorianGatewayClient : IHistorianGatewayClient
|
||||
public int LastReadEventsMaxEvents;
|
||||
public int ReadEventsCallCount;
|
||||
|
||||
/// <summary>When set, ReadEvents serves <c>EventsSource(start, end)</c> instead of <see cref="Events"/>.</summary>
|
||||
public Func<DateTime, DateTime, IEnumerable<HistorianEvent>>? EventsSource;
|
||||
|
||||
/// <summary>Thrown (deferred to first enumeration) by every read method when set.</summary>
|
||||
public Exception? ThrowOnRead;
|
||||
|
||||
@@ -100,12 +117,35 @@ public sealed class FakeHistorianGatewayClient : IHistorianGatewayClient
|
||||
LastReadRawStartUtc = startUtc;
|
||||
LastReadRawEndUtc = endUtc;
|
||||
LastReadRawMaxValues = maxValues;
|
||||
ReadRawCallCount++;
|
||||
RawCalls.Add((startUtc, endUtc, maxValues));
|
||||
lock (RawCalls)
|
||||
{
|
||||
ReadRawCallCount++;
|
||||
RawCalls.Add((startUtc, endUtc, maxValues));
|
||||
}
|
||||
if (MaxValuesCeiling is { } ceiling && maxValues > ceiling)
|
||||
{
|
||||
// The real gateway's refusal (HistorianReadService.ReadRaw), raised on the first MoveNext before any row, as
|
||||
// the client package surfaces it: a HistorianGatewayException wrapping the RpcException.
|
||||
var rpc = new Grpc.Core.RpcException(new Grpc.Core.Status(Grpc.Core.StatusCode.InvalidArgument,
|
||||
CeilingRejectionDetail ?? $"max_values exceeds the server ceiling of {ceiling}; read large ranges by time-slicing " +
|
||||
"(split the range and read the slices sequentially), or enable " +
|
||||
"Historian:ReadLimits:AllowStreamingBeyondCeiling to page through server-side."));
|
||||
return ToAsyncStream(Array.Empty<HistorianSample>(),
|
||||
new ZB.MOM.WW.HistorianGateway.Client.HistorianGatewayException(rpc.Status.Detail, rpc), ct);
|
||||
}
|
||||
var items = RawSource is null ? RawSamples : RawSource(startUtc, endUtc, maxValues).ToList();
|
||||
return CountingStream(ToAsyncStream(items, ThrowOnRead, ct));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A strict gateway (<c>Historian:ReadLimits:AllowStreamingBeyondCeiling=false</c>) with this
|
||||
/// <c>MaxValuesCeiling</c>: a ReadRaw asking for more is refused <c>InvalidArgument</c> (#691.3).
|
||||
/// </summary>
|
||||
public int? MaxValuesCeiling;
|
||||
|
||||
/// <summary>Overrides the refusal's detail text (an older/other gateway wording).</summary>
|
||||
public string? CeilingRejectionDetail;
|
||||
|
||||
public IAsyncEnumerable<HistorianAggregateSample> ReadAggregateAsync(
|
||||
string tag,
|
||||
DateTime startUtc,
|
||||
@@ -120,9 +160,25 @@ public sealed class FakeHistorianGatewayClient : IHistorianGatewayClient
|
||||
LastAggregateMode = mode;
|
||||
LastAggregateInterval = interval;
|
||||
ReadAggregateCallCount++;
|
||||
return ToAsyncStream(AggregateSamples, ThrowOnRead, ct);
|
||||
AggregateCalls.Add((startUtc, endUtc, mode, interval));
|
||||
if (AggregateSource is null) return ToAsyncStream(AggregateSamples, ThrowOnRead, ct);
|
||||
if (startUtc > endUtc || interval <= TimeSpan.Zero)
|
||||
{
|
||||
// The real gateway's request validation (ValidateRange; interval must be > 0), raised before any row.
|
||||
var rpc = new Grpc.Core.RpcException(new Grpc.Core.Status(Grpc.Core.StatusCode.InvalidArgument,
|
||||
startUtc > endUtc ? "start_time must be less than or equal to end_time." : "interval must be greater than zero."));
|
||||
return ToAsyncStream(Array.Empty<HistorianAggregateSample>(),
|
||||
new ZB.MOM.WW.HistorianGateway.Client.HistorianGatewayException(rpc.Status.Detail, rpc), ct);
|
||||
}
|
||||
return ToAsyncStream(AggregateSource(startUtc, endUtc, mode, interval).ToList(), ThrowOnRead, ct);
|
||||
}
|
||||
|
||||
/// <summary>When set, ReadAggregate serves <c>AggregateSource(start, end, mode, interval)</c> (and refuses start > end).</summary>
|
||||
public Func<DateTime, DateTime, RetrievalMode, TimeSpan, IEnumerable<HistorianAggregateSample>>? AggregateSource;
|
||||
|
||||
/// <summary>Every ReadAggregate call's (start, end, mode, interval), in order.</summary>
|
||||
public readonly List<(DateTime Start, DateTime End, RetrievalMode Mode, TimeSpan Interval)> AggregateCalls = new();
|
||||
|
||||
public Task<IReadOnlyList<HistorianSample>> ReadAtTimeAsync(
|
||||
string tag,
|
||||
IReadOnlyList<DateTime> timestampsUtc,
|
||||
@@ -148,7 +204,7 @@ public sealed class FakeHistorianGatewayClient : IHistorianGatewayClient
|
||||
LastReadEventsEndUtc = endUtc;
|
||||
LastReadEventsMaxEvents = maxEvents;
|
||||
ReadEventsCallCount++;
|
||||
return ToAsyncStream(Events, ThrowOnRead, ct);
|
||||
return ToAsyncStream(EventsSource is null ? Events : EventsSource(startUtc, endUtc).ToList(), ThrowOnRead, ct);
|
||||
}
|
||||
|
||||
public Task<WriteAck> WriteLiveValuesAsync(
|
||||
|
||||
+27
@@ -18,6 +18,33 @@ public sealed class GatewayAlarmHistorianWriterTests
|
||||
private static GatewayAlarmHistorianWriter Writer(FakeHistorianGatewayClient fake) =>
|
||||
new(fake, NullLogger<GatewayAlarmHistorianWriter>.Instance);
|
||||
|
||||
[Fact]
|
||||
public async Task A_direct_retry_preserves_the_original_delivery_id()
|
||||
{
|
||||
var evt = Evt("A");
|
||||
var fake = new FakeHistorianGatewayClient { SendEventThrows = new RpcException(new Status(StatusCode.Unavailable, "down")) };
|
||||
using var writer = Writer(fake);
|
||||
Assert.Equal(HistorianWriteOutcome.RetryPlease,
|
||||
(await writer.WriteBatchAsync([evt], TestContext.Current.CancellationToken)).Single());
|
||||
var first = fake.LastSendEvent!.Id;
|
||||
fake.SendEventThrows = null;
|
||||
fake.SendEventResult = new WriteAck { Success = true };
|
||||
Assert.Equal(HistorianWriteOutcome.Ack,
|
||||
(await writer.WriteBatchAsync([evt with { }], TestContext.Current.CancellationToken)).Single());
|
||||
Assert.Equal(evt.DeliveryId.ToString("D"), first);
|
||||
Assert.Equal(first, fake.LastSendEvent!.Id);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task An_empty_delivery_id_is_permanent_failure_without_an_RPC()
|
||||
{
|
||||
var fake = new FakeHistorianGatewayClient();
|
||||
using var writer = Writer(fake);
|
||||
Assert.Equal(HistorianWriteOutcome.PermanentFail,
|
||||
(await writer.WriteBatchAsync([Evt("A") with { DeliveryId = Guid.Empty }], TestContext.Current.CancellationToken)).Single());
|
||||
Assert.Equal(0, fake.SendEventCallCount);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task All_acked_when_SendEvent_succeeds()
|
||||
{
|
||||
|
||||
@@ -0,0 +1,746 @@
|
||||
using Google.Protobuf.WellKnownTypes;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.HistorianGateway.Contracts.Grpc;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #693: Gateway HistoryRead AtTime is served from RAW reads and evaluated with the SQL backends' shared
|
||||
/// <c>AtTimeEvaluator</c> (Part 11 §6.5.4) — never AVEVA's own interpolated answer, which the live gate caught returning
|
||||
/// the previous value as Good at a stored NULL's own time, null + Bad at a Good sample right after a NULL, a drifted value
|
||||
/// at an exact stored sample, and no Interpolated bit. The fake is AVEVA-shaped (the real wire shape): ReadRaw over
|
||||
/// <c>[start, end]</c> ascending, capped, led by the synthetic boundary row at the query start (the value in effect there,
|
||||
/// <c>Quality = 133</c>, or a NULL row), NULL rows as <c>NumericValue = 0</c> + <c>q=1 opc=0</c>, millisecond timestamps;
|
||||
/// and it REJECTS what the real gateway rejects (<c>max_values <= 0</c>, <c>end < start</c>).
|
||||
/// </summary>
|
||||
public sealed class GatewayAtTimeTests
|
||||
{
|
||||
private static CancellationToken Ct => TestContext.Current.CancellationToken;
|
||||
|
||||
private const uint Good = 0u;
|
||||
private const uint Bad = 0x80000000u;
|
||||
private const uint BadNoData = 0x809B0000u;
|
||||
private const uint GoodInterpolated = 0x00000402u;
|
||||
private const uint UncertainInterpolated = 0x40A40402u; // UncertainDataSubNormal | InfoType DataValue | Interpolated
|
||||
|
||||
/// <summary>The live gate's tag <c>HistGW.LiveTest.Gate685</c>: 19:00:03 is a stored NULL.</summary>
|
||||
private static readonly DateTime T0 = new(2026, 10, 12, 19, 0, 0, DateTimeKind.Utc);
|
||||
|
||||
private static readonly DateTime Now = T0.AddHours(1);
|
||||
|
||||
private static DateTime At(double seconds) => T0.AddTicks((long)(seconds * TimeSpan.TicksPerSecond));
|
||||
|
||||
private static List<(DateTime T, double V, bool Null)> Gate685() =>
|
||||
[(At(0), 100, false), (At(2), 107, false), (At(3), 0, true), (At(5), 108, false)];
|
||||
|
||||
/// <summary>An AVEVA-shaped store over sorted rows, counting reads and rejecting what the real gateway rejects.</summary>
|
||||
private sealed class AvevaStore
|
||||
{
|
||||
private readonly (DateTime T, double V, bool Null)[] _rows;
|
||||
|
||||
public AvevaStore(IEnumerable<(DateTime T, double V, bool Null)> rows) => _rows = rows.OrderBy(r => r.T).ToArray();
|
||||
|
||||
/// <summary>Also emit a row at the exact query END carrying the value in effect there (unverified AVEVA behaviour).</summary>
|
||||
public bool TerminalRow { get; init; }
|
||||
|
||||
/// <summary>Total rows streamed (cost).</summary>
|
||||
public long RowsStreamed;
|
||||
|
||||
public FakeHistorianGatewayClient Client(int? ceiling = null) => new()
|
||||
{
|
||||
RawSource = Serve,
|
||||
MaxValuesCeiling = ceiling,
|
||||
};
|
||||
|
||||
private static HistorianSample Row(DateTime t, double v, bool isNull, uint quality = 0) => new()
|
||||
{
|
||||
Tag = "T",
|
||||
Timestamp = Timestamp.FromDateTime(t),
|
||||
NumericValue = isNull ? 0 : v,
|
||||
OpcQuality = isNull ? 0u : 192u,
|
||||
Quality = isNull ? 1u : quality,
|
||||
};
|
||||
|
||||
private int FirstAtOrAfter(DateTime t)
|
||||
{
|
||||
int lo = 0, hi = _rows.Length;
|
||||
while (lo < hi)
|
||||
{
|
||||
var mid = (lo + hi) / 2;
|
||||
if (_rows[mid].T < t) lo = mid + 1; else hi = mid;
|
||||
}
|
||||
return lo;
|
||||
}
|
||||
|
||||
private HistorianSample InEffect(DateTime t)
|
||||
{
|
||||
var i = FirstAtOrAfter(t) - 1;
|
||||
if (i < 0) return new HistorianSample { Tag = "T", Timestamp = Timestamp.FromDateTime(t), NumericValue = 0, OpcQuality = 0, Quality = 1 };
|
||||
var p = _rows[i];
|
||||
return Row(t, p.V, p.Null, p.Null ? 1u : 133u);
|
||||
}
|
||||
|
||||
private IEnumerable<HistorianSample> Serve(DateTime start, DateTime end, int max)
|
||||
{
|
||||
if (max <= 0)
|
||||
throw new Grpc.Core.RpcException(new Grpc.Core.Status(Grpc.Core.StatusCode.InvalidArgument, "max_values must be greater than zero."));
|
||||
if (end < start)
|
||||
throw new Grpc.Core.RpcException(new Grpc.Core.Status(Grpc.Core.StatusCode.InvalidArgument,
|
||||
"start_time must be less than or equal to end_time."));
|
||||
if (end > Now.AddDays(2))
|
||||
throw new Grpc.Core.RpcException(new Grpc.Core.Status(Grpc.Core.StatusCode.InvalidArgument,
|
||||
"end_time is outside the historian's time range"));
|
||||
if (start.Ticks % TimeSpan.TicksPerMillisecond != 0 || end.Ticks % TimeSpan.TicksPerMillisecond != 0)
|
||||
throw new InvalidOperationException("AVEVA truncates request times to the millisecond; send millisecond bounds.");
|
||||
|
||||
var rows = new List<HistorianSample>();
|
||||
var i = FirstAtOrAfter(start);
|
||||
if (i >= _rows.Length || _rows[i].T != start) rows.Add(InEffect(start));
|
||||
for (; i < _rows.Length && _rows[i].T <= end && rows.Count < max; i++)
|
||||
rows.Add(Row(_rows[i].T, _rows[i].V, _rows[i].Null));
|
||||
if (TerminalRow && rows.Count < max && (i >= _rows.Length || _rows[i].T > end) && _rows.All(r => r.T != end))
|
||||
rows.Add(InEffect(end));
|
||||
var served = rows.Take(max).ToList();
|
||||
RowsStreamed += served.Count;
|
||||
return served;
|
||||
}
|
||||
}
|
||||
|
||||
private static GatewayHistorianDataSource Source(FakeHistorianGatewayClient fake) =>
|
||||
new(fake, NullLogger<GatewayHistorianDataSource>.Instance, timeProvider: new FixedTime(Now));
|
||||
|
||||
private sealed class FixedTime(DateTime now) : TimeProvider
|
||||
{
|
||||
public override DateTimeOffset GetUtcNow() => new(now);
|
||||
}
|
||||
|
||||
private static async Task<IReadOnlyList<DataValueSnapshot>> AtTime(FakeHistorianGatewayClient fake, params DateTime[] times)
|
||||
{
|
||||
var r = await Source(fake).ReadAtTimeAsync("T", times, Ct);
|
||||
r.Samples.Count.ShouldBe(times.Length);
|
||||
for (var i = 0; i < times.Length; i++) r.Samples[i].SourceTimestampUtc.ShouldBe(times[i]);
|
||||
var refused = fake.MaxValuesCeiling is { } c ? fake.RawCalls.Count(x => x.MaxValues > c) : 0;
|
||||
fake.RawStreamsCompleted.ShouldBe(fake.ReadRawCallCount - refused); // never abandons a gateway stream
|
||||
fake.ReadAtTimeCallCount.ShouldBe(0); // never AVEVA's own interpolation
|
||||
return r.Samples;
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_future_time_on_a_live_tag_finds_the_prior_sample_without_walking_the_empty_future()
|
||||
{
|
||||
var store = new AvevaStore(Enumerable.Range(0, 7201).Select(i => (Now.AddSeconds(i - 7200), (double)i, false)));
|
||||
var fake = store.Client();
|
||||
var result = (await AtTime(fake, Now.AddHours(1))).Single();
|
||||
result.Value.ShouldBe(7200d);
|
||||
result.StatusCode.ShouldBe(UncertainInterpolated);
|
||||
fake.ReadRawCallCount.ShouldBeLessThanOrEqualTo(4);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_future_time_still_uses_a_stored_future_prior_sample()
|
||||
{
|
||||
var fake = new AvevaStore([(Now.AddSeconds(-1), 1d, false), (Now.AddMinutes(30), 2d, false)]).Client();
|
||||
var result = (await AtTime(fake, Now.AddHours(1))).Single();
|
||||
result.Value.ShouldBe(2d);
|
||||
result.StatusCode.ShouldBe(UncertainInterpolated);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_full_future_probe_walks_to_the_latest_stored_future_sample()
|
||||
{
|
||||
var fake = new AvevaStore(Enumerable.Range(0, 7201).Select(i => (Now.AddSeconds(i), (double)i, false))).Client();
|
||||
var result = (await AtTime(fake, Now.AddHours(3))).Single();
|
||||
result.Value.ShouldBe(7200d);
|
||||
result.StatusCode.ShouldBe(UncertainInterpolated);
|
||||
}
|
||||
|
||||
// ---- the four live findings ------------------------------------------------------------------------------------
|
||||
|
||||
[Fact] // live: 19:00:03 (a stored NULL) ⇒ 107 Good
|
||||
public async Task At_a_stored_NULLs_own_time_is_null_and_Bad()
|
||||
{
|
||||
var s = (await AtTime(new AvevaStore(Gate685()).Client(), At(3))).Single();
|
||||
s.Value.ShouldBeNull();
|
||||
s.StatusCode.ShouldBe(Bad);
|
||||
}
|
||||
|
||||
[Fact] // live: 19:00:05 (stored 108, right after the NULL) ⇒ null + Bad
|
||||
public async Task At_a_Good_sample_right_after_a_NULL_is_the_stored_value()
|
||||
{
|
||||
var s = (await AtTime(new AvevaStore(Gate685()).Client(), At(5))).Single();
|
||||
s.Value.ShouldBe(108d);
|
||||
s.StatusCode.ShouldBe(Good);
|
||||
}
|
||||
|
||||
[Fact] // live: 99.99999999944 for a stored 100
|
||||
public async Task At_an_exact_stored_sample_is_the_exact_stored_value_without_the_Interpolated_bit()
|
||||
{
|
||||
var s = await AtTime(new AvevaStore(Gate685()).Client(), At(0), At(2));
|
||||
s[0].Value.ShouldBe(100d);
|
||||
s[0].StatusCode.ShouldBe(Good);
|
||||
s[1].Value.ShouldBe(107d);
|
||||
s[1].StatusCode.ShouldBe(Good);
|
||||
}
|
||||
|
||||
[Fact] // live: no Interpolated bit
|
||||
public async Task Between_two_Good_samples_is_linear_with_the_Interpolated_bit()
|
||||
{
|
||||
var s = (await AtTime(new AvevaStore(Gate685()).Client(), At(1))).Single();
|
||||
s.Value.ShouldBe(103.5d);
|
||||
s.StatusCode.ShouldBe(GoodInterpolated);
|
||||
new Opc.Ua.StatusCode(s.StatusCode).AggregateBits.ShouldBe(Opc.Ua.AggregateBits.Interpolated);
|
||||
}
|
||||
|
||||
// ---- the rest of the AtTime rules --------------------------------------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public async Task Before_a_Bad_neighbour_the_previous_value_is_held_Uncertain_and_Interpolated()
|
||||
{
|
||||
var s = (await AtTime(new AvevaStore(Gate685()).Client(), At(2.5))).Single();
|
||||
s.Value.ShouldBe(107d);
|
||||
s.StatusCode.ShouldBe(UncertainInterpolated);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Inside_an_outage_is_null_with_the_outages_Bad_code()
|
||||
{
|
||||
var s = (await AtTime(new AvevaStore(Gate685()).Client(), At(4))).Single();
|
||||
s.Value.ShouldBeNull();
|
||||
s.StatusCode.ShouldBe(Bad);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Before_the_first_sample_is_BadNoData_not_the_boundary_rows_zero()
|
||||
{
|
||||
var s = await AtTime(new AvevaStore(Gate685()).Client(), At(-1), At(-0.001), At(-3600));
|
||||
s.ShouldAllBe(x => x.StatusCode == BadNoData && x.Value == null);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task After_the_last_sample_is_stepped_Good_until_now_then_Uncertain()
|
||||
{
|
||||
var s = await AtTime(new AvevaStore(Gate685()).Client(), At(10), Now.AddMinutes(5), Now.AddDays(3));
|
||||
s[0].Value.ShouldBe(108d);
|
||||
s[0].StatusCode.ShouldBe(GoodInterpolated);
|
||||
s[1].Value.ShouldBe(108d);
|
||||
s[1].StatusCode.ShouldBe(UncertainInterpolated); // later than now: not known to still hold
|
||||
s[2].Value.ShouldBe(108d);
|
||||
s[2].StatusCode.ShouldBe(UncertainInterpolated); // beyond the horizon: never sent to the gateway (it refuses)
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_mixed_request_keeps_order_and_duplicates()
|
||||
{
|
||||
var s = await AtTime(new AvevaStore(Gate685()).Client(), At(5), At(1), At(-1), At(3), At(1), At(2.5));
|
||||
s.Select(x => x.Value).ShouldBe(new object?[] { 108d, 103.5d, null, null, 103.5d, 107d });
|
||||
s.Select(x => x.StatusCode).ShouldBe(new[] { Good, GoodInterpolated, BadNoData, Bad, GoodInterpolated, UncertainInterpolated });
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task An_empty_request_reads_nothing()
|
||||
{
|
||||
var fake = new AvevaStore(Gate685()).Client();
|
||||
(await Source(fake).ReadAtTimeAsync("T", [], Ct)).Samples.ShouldBeEmpty();
|
||||
fake.ReadRawCallCount.ShouldBe(0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_sub_millisecond_request_time_interpolates_between_the_millisecond_samples()
|
||||
{
|
||||
var s = (await AtTime(new AvevaStore(Gate685()).Client(), At(0.0005))).Single();
|
||||
((double)s.Value!).ShouldBe(100.00175d, 1e-9);
|
||||
s.StatusCode.ShouldBe(GoodInterpolated);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_row_at_the_query_end_never_counts_as_a_stored_sample()
|
||||
{
|
||||
var store = new AvevaStore(Gate685()) { TerminalRow = true };
|
||||
var s = await AtTime(store.Client(), At(1), At(2.5), At(4), At(10));
|
||||
s.Select(x => x.Value).ShouldBe(new object?[] { 103.5d, 107d, null, 108d });
|
||||
s.Select(x => x.StatusCode).ShouldBe(new[] { GoodInterpolated, UncertainInterpolated, Bad, GoodInterpolated });
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_string_tag_is_stepped()
|
||||
{
|
||||
var fake = new FakeHistorianGatewayClient
|
||||
{
|
||||
RawSource = (start, end, max) =>
|
||||
{
|
||||
var rows = new List<HistorianSample>
|
||||
{
|
||||
new() { Tag = "T", Timestamp = Timestamp.FromDateTime(start), StringValue = "", OpcQuality = 0, Quality = 1 },
|
||||
};
|
||||
foreach (var (t, v) in new[] { (At(0), "a"), (At(10), "b") })
|
||||
if (t >= start && t <= end)
|
||||
rows.Add(new HistorianSample { Tag = "T", Timestamp = Timestamp.FromDateTime(t), StringValue = v, OpcQuality = 192 });
|
||||
return rows.Take(max);
|
||||
},
|
||||
};
|
||||
var s = (await AtTime(fake, At(5))).Single();
|
||||
s.Value.ShouldBe("a");
|
||||
s.StatusCode.ShouldBe(GoodInterpolated);
|
||||
}
|
||||
|
||||
// ---- the node's value type (the node manager's hint; AVEVA stores a Boolean as a 0/1 Int2) ------------------------
|
||||
|
||||
/// <summary>Reads through the INTERFACE's hinted overload, as the node manager does.</summary>
|
||||
private static async Task<IReadOnlyList<DataValueSnapshot>> AtTimeTyped(
|
||||
FakeHistorianGatewayClient fake, TypeCode? valueType, params DateTime[] times)
|
||||
{
|
||||
IHistorianDataSource source = Source(fake);
|
||||
var r = await source.ReadAtTimeAsync("T", times, valueType, Ct);
|
||||
r.Samples.Count.ShouldBe(times.Length);
|
||||
fake.ReadAtTimeCallCount.ShouldBe(0);
|
||||
return r.Samples;
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_Boolean_node_is_stepped_never_a_fraction()
|
||||
{
|
||||
var fake = new AvevaStore([(At(0), 0, false), (At(10), 1, false), (At(20), 0, false)]).Client();
|
||||
var s = await AtTimeTyped(fake, TypeCode.Boolean, At(5), At(10), At(15), At(25));
|
||||
s.Select(x => x.Value).ShouldBe(new object?[] { false, true, true, false });
|
||||
s.ShouldAllBe(x => x.Value is bool);
|
||||
s.Select(x => x.StatusCode).ShouldBe(new[] { GoodInterpolated, Good, GoodInterpolated, GoodInterpolated });
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Without_a_hint_a_Boolean_tag_interpolates_as_before()
|
||||
{
|
||||
var fake = new AvevaStore([(At(0), 0, false), (At(10), 1, false)]).Client();
|
||||
(await AtTimeTyped(fake, null, At(5))).Single().Value.ShouldBeOfType<double>().ShouldBe(0.5d);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task An_Int32_node_interpolates_rounds_half_away_from_zero_and_keeps_its_type()
|
||||
{
|
||||
var s = await AtTimeTyped(new AvevaStore(Gate685()).Client(), TypeCode.Int32, At(0), At(0.5), At(1), At(2.5), At(3));
|
||||
s[0].Value.ShouldBe(100); // exact hit, as stored, Int32
|
||||
s[0].StatusCode.ShouldBe(Good);
|
||||
s[1].Value.ShouldBe(102); // 101.75 ⇒ 102
|
||||
s[2].Value.ShouldBe(104); // 103.5 ⇒ 104 (half away from zero)
|
||||
s[1].StatusCode.ShouldBe(GoodInterpolated);
|
||||
s[3].Value.ShouldBe(107); // held before the NULL, Uncertain
|
||||
s[3].StatusCode.ShouldBe(UncertainInterpolated);
|
||||
s[4].Value.ShouldBeNull(); // the NULL itself
|
||||
s[4].StatusCode.ShouldBe(Bad);
|
||||
s.Take(4).Select(x => x.Value + ":" + x.Value!.GetType().Name).ShouldBe(["100:Int32", "102:Int32", "104:Int32", "107:Int32"]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_negative_midpoint_rounds_away_from_zero()
|
||||
{
|
||||
var fake = new AvevaStore([(At(0), -100, false), (At(2), -107, false)]).Client();
|
||||
(await AtTimeTyped(fake, TypeCode.Int16, At(1))).Single().Value.ShouldBeOfType<short>().ShouldBe((short)-104);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Other_integer_nodes_keep_their_own_type()
|
||||
{
|
||||
var fake = new AvevaStore(Gate685()).Client();
|
||||
(await AtTimeTyped(fake, TypeCode.UInt16, At(1))).Single().Value.ShouldBeOfType<ushort>().ShouldBe((ushort)104);
|
||||
(await AtTimeTyped(fake, TypeCode.Int64, At(1))).Single().Value.ShouldBeOfType<long>().ShouldBe(104L);
|
||||
(await AtTimeTyped(fake, TypeCode.Byte, At(1))).Single().Value.ShouldBeOfType<byte>().ShouldBe((byte)104);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Float_and_Double_nodes_interpolate_unchanged()
|
||||
{
|
||||
var fake = new AvevaStore(Gate685()).Client();
|
||||
(await AtTimeTyped(fake, TypeCode.Double, At(1))).Single().Value.ShouldBeOfType<double>().ShouldBe(103.5d);
|
||||
(await AtTimeTyped(fake, TypeCode.Single, At(1))).Single().Value.ShouldBeOfType<double>().ShouldBe(103.5d); // as the SQL backends (Double kind)
|
||||
}
|
||||
|
||||
// ---- cost ------------------------------------------------------------------------------------------------------
|
||||
|
||||
private static IEnumerable<(DateTime T, double V, bool Null)> OneHertz(DateTime from, int seconds) =>
|
||||
Enumerable.Range(0, seconds).Select(i => (from.AddSeconds(i), (double)i, false));
|
||||
|
||||
[Fact]
|
||||
public async Task Ten_exact_hits_on_dense_data_are_one_read()
|
||||
{
|
||||
var fake = new AvevaStore(OneHertz(T0.AddHours(-1), 3600)).Client();
|
||||
var times = Enumerable.Range(0, 10).Select(i => T0.AddMinutes(-30).AddSeconds(i)).ToArray();
|
||||
var s = await AtTime(fake, times);
|
||||
s.Select(x => x.Value).ShouldBe(Enumerable.Range(1800, 10).Select(i => (object?)(double)i));
|
||||
s.ShouldAllBe(x => x.StatusCode == Good);
|
||||
fake.ReadRawCallCount.ShouldBe(1);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Ten_interpolated_times_on_dense_data_are_three_reads()
|
||||
{
|
||||
var fake = new AvevaStore(OneHertz(T0.AddHours(-1), 3600)).Client();
|
||||
var times = Enumerable.Range(0, 10).Select(i => T0.AddMinutes(-30).AddSeconds(i + 0.5)).ToArray();
|
||||
var s = await AtTime(fake, times);
|
||||
s.Select(x => x.Value).ShouldBe(Enumerable.Range(1800, 10).Select(i => (object?)(i + 0.5)));
|
||||
s.ShouldAllBe(x => x.StatusCode == GoodInterpolated);
|
||||
fake.ReadRawCallCount.ShouldBe(3); // the window, the sample before the first time, the sample after the last
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Ten_times_an_hour_apart_on_dense_data_cost_about_one_read_each_not_the_rows_between()
|
||||
{
|
||||
var store = new AvevaStore(OneHertz(T0.AddHours(-12), 12 * 3600));
|
||||
var fake = store.Client();
|
||||
var times = Enumerable.Range(0, 10).Select(i => T0.AddHours(-11).AddHours(i).AddSeconds(0.5)).ToArray();
|
||||
var s = await AtTime(fake, times);
|
||||
s.Select(x => x.Value).ShouldBe(Enumerable.Range(0, 10).Select(i => (object?)(3600.0 * (i + 1) + 0.5)));
|
||||
s.ShouldAllBe(x => x.StatusCode == GoodInterpolated);
|
||||
fake.ReadRawCallCount.ShouldBe(12); // 10 window reads (each finds its own prev) + 1 walk-back + 1 probe
|
||||
store.RowsStreamed.ShouldBeLessThan(10 * 1100); // never the ~32,400 rows between the first and last time
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Sparse_history_far_back_is_found_in_a_bounded_number_of_reads()
|
||||
{
|
||||
var fake = new AvevaStore([(T0.AddDays(-400), 42, false), (T0.AddDays(-400).AddSeconds(10), 43, false)]).Client();
|
||||
var s = (await AtTime(fake, T0)).Single();
|
||||
s.Value.ShouldBe(43d);
|
||||
s.StatusCode.ShouldBe(GoodInterpolated);
|
||||
fake.ReadRawCallCount.ShouldBe(5); // window, 1-min + 4-min segments, the whole remainder, the probe after
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_tag_with_no_history_is_BadNoData_in_a_bounded_number_of_reads()
|
||||
{
|
||||
var fake = new AvevaStore([]).Client();
|
||||
var s = await AtTime(fake, At(1), At(2));
|
||||
s.ShouldAllBe(x => x.StatusCode == BadNoData);
|
||||
fake.ReadRawCallCount.ShouldBe(4); // window, two segments, the whole remainder (no probe: nothing to interpolate)
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Dense_data_far_denser_than_one_read_still_finds_the_neighbours()
|
||||
{
|
||||
// 1 kHz for 10 minutes: a 1-minute walk-back segment holds 60,000 rows (truncated) — it must narrow, not fail.
|
||||
var rows = Enumerable.Range(0, 600_000).Select(i => (T0.AddMinutes(-20).AddMilliseconds(i), (double)i, false));
|
||||
var fake = new AvevaStore(rows).Client();
|
||||
var t = T0.AddMinutes(-15).AddTicks(5000); // 0.5 ms past sample 300,000
|
||||
var s = (await AtTime(fake, t)).Single();
|
||||
s.Value.ShouldBe(300_000.5d);
|
||||
s.StatusCode.ShouldBe(GoodInterpolated);
|
||||
fake.ReadRawCallCount.ShouldBe(2); // the window (holds the prev) + the probe
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_walk_back_into_data_far_denser_than_one_read_narrows_instead_of_paging()
|
||||
{
|
||||
// 1 kHz for 10 minutes, then a 10-minute gap: the walk-back crosses the gap and lands in 600,000 rows.
|
||||
var rows = Enumerable.Range(0, 600_000).Select(i => (T0.AddMinutes(-20).AddMilliseconds(i), (double)i, false));
|
||||
var store = new AvevaStore(rows);
|
||||
var fake = store.Client();
|
||||
var s = (await AtTime(fake, T0.AddMinutes(-5))).Single();
|
||||
s.Value.ShouldBe(599_999d);
|
||||
s.StatusCode.ShouldBe(GoodInterpolated); // stepped from the last sample (before now)
|
||||
fake.ReadRawCallCount.ShouldBe(6); // window, 1m, 4m, remainder (full: seeds the bisection), one aimed segment, probe
|
||||
store.RowsStreamed.ShouldBeLessThan(10_000);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_strict_gateway_with_a_small_ceiling_is_still_served()
|
||||
{
|
||||
var store = new AvevaStore(OneHertz(T0.AddHours(-1), 3600));
|
||||
var fake = store.Client(ceiling: 100);
|
||||
var times = Enumerable.Range(0, 10).Select(i => T0.AddMinutes(-30).AddSeconds(i * 30 + 0.5)).ToArray();
|
||||
var s = await AtTime(fake, times);
|
||||
s.Select(x => x.Value).ShouldBe(Enumerable.Range(0, 10).Select(i => (object?)(1800 + i * 30 + 0.5)));
|
||||
s.ShouldAllBe(x => x.StatusCode == GoodInterpolated);
|
||||
fake.RawCalls.Count(c => c.MaxValues > 100).ShouldBe(1); // learned once, then remembered — a short page is not an error
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Success_is_recorded_once()
|
||||
{
|
||||
var fake = new AvevaStore(Gate685()).Client();
|
||||
var ds = Source(fake);
|
||||
await ds.ReadAtTimeAsync("T", [At(1), At(4)], Ct);
|
||||
var h = ds.GetHealthSnapshot();
|
||||
h.TotalQueries.ShouldBe(1);
|
||||
h.TotalSuccesses.ShouldBe(1);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_gateway_fault_is_a_recorded_failure()
|
||||
{
|
||||
var fake = new AvevaStore(Gate685()).Client();
|
||||
fake.ThrowOnRead = new Grpc.Core.RpcException(new Grpc.Core.Status(Grpc.Core.StatusCode.Unavailable, "down"));
|
||||
var ds = Source(fake);
|
||||
await Should.ThrowAsync<Exception>(() => ds.ReadAtTimeAsync("T", [At(1)], Ct));
|
||||
ds.GetHealthSnapshot().TotalFailures.ShouldBe(1);
|
||||
}
|
||||
|
||||
// ---- long histories (review H1/M1/L1): a generated store, too large to materialise -----------------------------
|
||||
|
||||
/// <summary>
|
||||
/// A regular series (value = sample index) over <c>[from, to]</c> every <paramref name="step"/>, optionally with a
|
||||
/// hole <c>(gapFrom, gapTo)</c>, served in the AVEVA wire shape (boundary row at the query start, ms bounds only,
|
||||
/// <c>max_values > 0</c>, <c>start <= end</c>, nothing past now + 2 days).
|
||||
/// </summary>
|
||||
private sealed class SeriesStore(DateTime from, DateTime to, TimeSpan step, DateTime? gapFrom = null, DateTime? gapTo = null)
|
||||
{
|
||||
public long RowsStreamed;
|
||||
|
||||
private bool InGap(DateTime t) => gapFrom is { } a && gapTo is { } b && t > a && t < b;
|
||||
|
||||
private DateTime? FirstAtOrAfter(DateTime t)
|
||||
{
|
||||
if (t > to) return null;
|
||||
var k = t <= from ? 0 : ((t - from).Ticks + step.Ticks - 1) / step.Ticks;
|
||||
var r = from.AddTicks(k * step.Ticks);
|
||||
if (InGap(r)) return FirstAtOrAfter(gapTo!.Value);
|
||||
return r <= to ? r : null;
|
||||
}
|
||||
|
||||
private DateTime? LastBefore(DateTime t)
|
||||
{
|
||||
if (t <= from) return null;
|
||||
var k = ((Min(t, to.AddTicks(1)) - from).Ticks - 1) / step.Ticks;
|
||||
var r = from.AddTicks(k * step.Ticks);
|
||||
return InGap(r) ? LastBefore(gapFrom!.Value.AddTicks(1)) : r;
|
||||
}
|
||||
|
||||
private static DateTime Min(DateTime a, DateTime b) => a <= b ? a : b;
|
||||
|
||||
/// <summary>The stored row at <paramref name="t"/>, stamped <paramref name="at"/> (the boundary row) when given.</summary>
|
||||
private HistorianSample Row(DateTime t, uint quality = 0, DateTime? at = null) => new()
|
||||
{
|
||||
Tag = "T", Timestamp = Timestamp.FromDateTime(at ?? t), NumericValue = (t - from).Ticks / step.Ticks, OpcQuality = 192,
|
||||
Quality = quality,
|
||||
};
|
||||
|
||||
public FakeHistorianGatewayClient Client(int? ceiling = null) => new() { RawSource = Serve, MaxValuesCeiling = ceiling };
|
||||
|
||||
private IEnumerable<HistorianSample> Serve(DateTime start, DateTime end, int max)
|
||||
{
|
||||
if (max <= 0 || end < start || end > Now.AddDays(2)
|
||||
|| start.Ticks % TimeSpan.TicksPerMillisecond != 0 || end.Ticks % TimeSpan.TicksPerMillisecond != 0)
|
||||
throw new Grpc.Core.RpcException(new Grpc.Core.Status(Grpc.Core.StatusCode.InvalidArgument, "refused"));
|
||||
var rows = new List<HistorianSample>();
|
||||
if (FirstAtOrAfter(start) != start)
|
||||
rows.Add(LastBefore(start) is { } p
|
||||
? Row(p, 133, start)
|
||||
: new HistorianSample { Tag = "T", Timestamp = Timestamp.FromDateTime(start), OpcQuality = 0, Quality = 1, NumericValue = 0 });
|
||||
for (var t = FirstAtOrAfter(start); t is { } u && u <= end && rows.Count < max; t = FirstAtOrAfter(u.AddTicks(1)))
|
||||
rows.Add(Row(u));
|
||||
RowsStreamed += rows.Count;
|
||||
return rows;
|
||||
}
|
||||
}
|
||||
|
||||
[Theory] // H1: a dense tag that stopped reporting — the walk crosses the silence by bisection, not by regrowing segments
|
||||
[InlineData(1.0, 30)]
|
||||
[InlineData(1.0, 365)]
|
||||
[InlineData(10.0, 30)]
|
||||
[InlineData(10.0, 365)]
|
||||
public async Task A_dense_tag_that_stopped_long_ago_is_found_in_a_bounded_number_of_reads(double hz, int stoppedDaysAgo)
|
||||
{
|
||||
var step = TimeSpan.FromTicks((long)(TimeSpan.TicksPerSecond / hz));
|
||||
var end = Now.AddDays(-stoppedDaysAgo);
|
||||
var fake = new SeriesStore(end.AddYears(-5), end, step).Client();
|
||||
var s = (await AtTime(fake, Now.AddMinutes(-1))).Single();
|
||||
s.Value.ShouldBe((double)((end - end.AddYears(-5)).Ticks / step.Ticks)); // the last sample, held
|
||||
s.StatusCode.ShouldBe(GoodInterpolated);
|
||||
fake.ReadRawCallCount.ShouldBeLessThan(40);
|
||||
}
|
||||
|
||||
[Fact] // H1: a time inside a month-long hole in the middle of a dense history
|
||||
public async Task A_time_inside_a_mid_history_gap_interpolates_across_it()
|
||||
{
|
||||
var from = Now.AddYears(-2);
|
||||
var gapFrom = Now.AddDays(-200);
|
||||
var gapTo = gapFrom.AddDays(30);
|
||||
var fake = new SeriesStore(from, Now, TimeSpan.FromSeconds(1), gapFrom, gapTo).Client();
|
||||
var t = gapFrom.AddDays(15);
|
||||
var s = (await AtTime(fake, t)).Single();
|
||||
var before = (gapFrom - from).Ticks / TimeSpan.TicksPerSecond;
|
||||
var after = (gapTo - from).Ticks / TimeSpan.TicksPerSecond;
|
||||
((double)s.Value!).ShouldBe(before + (after - before) * 0.5, 1e-6);
|
||||
s.StatusCode.ShouldBe(GoodInterpolated);
|
||||
fake.ReadRawCallCount.ShouldBeLessThan(40);
|
||||
}
|
||||
|
||||
[Fact] // H1: a jump to a time in a gap — the walk starts from the earlier read's last row, far below
|
||||
public async Task A_jump_into_a_gap_bisects_up_from_the_earlier_reads_last_row()
|
||||
{
|
||||
var from = Now.AddDays(-30);
|
||||
var gapFrom = from.AddDays(2);
|
||||
var gapTo = gapFrom.AddDays(10);
|
||||
var fake = new SeriesStore(from, Now, TimeSpan.FromSeconds(1), gapFrom, gapTo).Client();
|
||||
var t0 = from.AddSeconds(10.5);
|
||||
var t1 = gapFrom.AddDays(5);
|
||||
var s = await AtTime(fake, t0, t1);
|
||||
s[0].Value.ShouldBe(10.5d);
|
||||
var before = (gapFrom - from).Ticks / TimeSpan.TicksPerSecond;
|
||||
var after = (gapTo - from).Ticks / TimeSpan.TicksPerSecond;
|
||||
((double)s[1].Value!).ShouldBe(before + (after - before) * 0.5, 1e-6);
|
||||
fake.ReadRawCallCount.ShouldBeLessThan(40);
|
||||
}
|
||||
|
||||
[Fact] // M1: widely spread times cost one SMALL read each, not one 1,024-row read each
|
||||
public async Task A_thousand_times_over_ten_years_cost_one_small_read_each()
|
||||
{
|
||||
var store = new SeriesStore(Now.AddYears(-10), Now, TimeSpan.FromSeconds(1));
|
||||
var fake = store.Client();
|
||||
var rnd = new Random(1);
|
||||
var times = Enumerable.Range(0, 1000).Select(_ => Now.AddYears(-9).AddTicks((long)(rnd.NextDouble() * TimeSpan.TicksPerDay * 3000))).ToArray();
|
||||
var s = await AtTime(fake, times);
|
||||
s.ShouldAllBe(x => x.StatusCode == GoodInterpolated);
|
||||
fake.ReadRawCallCount.ShouldBeLessThanOrEqualTo(1010);
|
||||
store.RowsStreamed.ShouldBeLessThan(80_000); // ~64 rows a read (was ~1,000,000)
|
||||
}
|
||||
|
||||
[Fact] // L1: a strict-gateway ceiling too small to work with is refused with a message naming it
|
||||
public async Task A_ceiling_below_a_usable_read_is_refused_naming_the_ceiling()
|
||||
{
|
||||
var fake = new SeriesStore(Now.AddDays(-1), Now, TimeSpan.FromSeconds(1)).Client(ceiling: 2);
|
||||
var ex = await Should.ThrowAsync<HistoryReadLimitExceededException>(
|
||||
() => Source(fake).ReadAtTimeAsync("T", [Now.AddHours(-1)], Ct));
|
||||
ex.Message.ShouldContain("row ceiling (2");
|
||||
}
|
||||
|
||||
/// <summary>A bursty tag: 1 Hz for the first hour of every day (value = seconds since <paramref name="from"/>).</summary>
|
||||
private sealed class BurstStore(DateTime from, int days)
|
||||
{
|
||||
public long RowsStreamed;
|
||||
private const long Burst = 3600 * TimeSpan.TicksPerSecond;
|
||||
|
||||
private DateTime? FirstAtOrAfter(DateTime t)
|
||||
{
|
||||
if (t <= from) return from;
|
||||
var d = (t - from).Ticks / TimeSpan.TicksPerDay;
|
||||
var inDay = (t - from).Ticks % TimeSpan.TicksPerDay;
|
||||
var r = inDay >= Burst
|
||||
? from.AddDays(d + 1)
|
||||
: from.AddTicks(d * TimeSpan.TicksPerDay + (inDay + TimeSpan.TicksPerSecond - 1) / TimeSpan.TicksPerSecond * TimeSpan.TicksPerSecond);
|
||||
if ((r - from).Ticks % TimeSpan.TicksPerDay >= Burst) r = from.AddDays(d + 1);
|
||||
return r < from.AddDays(days) ? r : null;
|
||||
}
|
||||
|
||||
private DateTime? LastBefore(DateTime t)
|
||||
{
|
||||
if (t <= from) return null;
|
||||
var x = (t - from).Ticks - 1;
|
||||
var d = x / TimeSpan.TicksPerDay;
|
||||
var inDay = x % TimeSpan.TicksPerDay;
|
||||
var r = inDay >= Burst - TimeSpan.TicksPerSecond
|
||||
? from.AddTicks(d * TimeSpan.TicksPerDay + Burst - TimeSpan.TicksPerSecond)
|
||||
: from.AddTicks(d * TimeSpan.TicksPerDay + inDay / TimeSpan.TicksPerSecond * TimeSpan.TicksPerSecond);
|
||||
var last = from.AddDays(days - 1).AddTicks(Burst - TimeSpan.TicksPerSecond);
|
||||
return r > last ? last : r;
|
||||
}
|
||||
|
||||
private HistorianSample Row(DateTime t, uint quality = 0, DateTime? at = null) => new()
|
||||
{
|
||||
Tag = "T", Timestamp = Timestamp.FromDateTime(at ?? t), NumericValue = (t - from).Ticks / TimeSpan.TicksPerSecond,
|
||||
OpcQuality = 192, Quality = quality,
|
||||
};
|
||||
|
||||
public FakeHistorianGatewayClient Client() => new() { RawSource = Serve };
|
||||
|
||||
private IEnumerable<HistorianSample> Serve(DateTime start, DateTime end, int max)
|
||||
{
|
||||
if (max <= 0 || end < start || end > Now.AddDays(2)
|
||||
|| start.Ticks % TimeSpan.TicksPerMillisecond != 0 || end.Ticks % TimeSpan.TicksPerMillisecond != 0)
|
||||
throw new Grpc.Core.RpcException(new Grpc.Core.Status(Grpc.Core.StatusCode.InvalidArgument, "refused"));
|
||||
var rows = new List<HistorianSample>();
|
||||
if (FirstAtOrAfter(start) != start)
|
||||
rows.Add(LastBefore(start) is { } p
|
||||
? Row(p, 133, start)
|
||||
: new HistorianSample { Tag = "T", Timestamp = Timestamp.FromDateTime(start), OpcQuality = 0, Quality = 1 });
|
||||
for (var t = FirstAtOrAfter(start); t is { } u && u <= end && rows.Count < max; t = FirstAtOrAfter(u.AddTicks(1)))
|
||||
rows.Add(Row(u));
|
||||
RowsStreamed += rows.Count;
|
||||
return rows;
|
||||
}
|
||||
}
|
||||
|
||||
[Theory] // re-review: every time in a gap of a bursty tag costs a jump read + a bisection — the cap must allow that
|
||||
[InlineData(30)]
|
||||
[InlineData(100)]
|
||||
[InlineData(300)]
|
||||
public async Task A_bursty_tag_asked_at_noon_on_consecutive_days_is_served(int days)
|
||||
{
|
||||
var from = Now.Date.AddDays(-days - 1);
|
||||
var fake = new BurstStore(from, days + 1).Client();
|
||||
var times = Enumerable.Range(0, days).Select(d => from.AddDays(d).AddHours(12)).ToArray();
|
||||
var s = await AtTime(fake, times);
|
||||
for (var d = 0; d < days; d++)
|
||||
{
|
||||
double prev = d * 86400 + 3599, next = (d + 1) * 86400;
|
||||
((double)s[d].Value!).ShouldBe(prev + (next - prev) * (12 * 3600 - 3599.0) / (86400 - 3599.0), 1e-6);
|
||||
s[d].StatusCode.ShouldBe(GoodInterpolated);
|
||||
}
|
||||
fake.ReadRawCallCount.ShouldBeLessThanOrEqualTo(7 * days); // ≈ 6 a time; the cap is 64 + 16 a time
|
||||
}
|
||||
|
||||
[Fact] // L1: a small ceiling is served (paging), within the read cap
|
||||
public async Task A_small_ceiling_is_served_within_the_read_cap()
|
||||
{
|
||||
var fake = new SeriesStore(Now.AddYears(-1), Now, TimeSpan.FromSeconds(1)).Client(ceiling: 10);
|
||||
var times = Enumerable.Range(0, 50).Select(i => Now.AddDays(-300 + i * 5).AddSeconds(0.5)).ToArray();
|
||||
var s = await AtTime(fake, times);
|
||||
s.ShouldAllBe(x => x.StatusCode == GoodInterpolated);
|
||||
fake.ReadRawCallCount.ShouldBeLessThanOrEqualTo(GatewayAtTimeReader.BaseReads + GatewayAtTimeReader.ReadsPerTime * 50);
|
||||
}
|
||||
|
||||
[Fact] // L1: the request's read cap stops a walk with a clear error, never a hang (here a cap of 3 vs. a 5-read walk)
|
||||
public async Task The_read_cap_stops_a_request_with_a_clear_error()
|
||||
{
|
||||
var fake = new AvevaStore([(T0.AddDays(-400), 42, false), (T0.AddDays(-400).AddSeconds(10), 43, false)]).Client();
|
||||
var ex = await Should.ThrowAsync<HistoryReadLimitExceededException>(() => GatewayAtTimeReader.ReadAsync(
|
||||
fake, new GatewayRowCeiling(), "T", [T0], Now, Ct, maxReads: 3));
|
||||
ex.Message.ShouldContain("needed more than 3 raw reads");
|
||||
fake.ReadRawCallCount.ShouldBe(3);
|
||||
fake.RawStreamsCompleted.ShouldBe(3);
|
||||
}
|
||||
|
||||
[Fact] // the default cap scales with the request: 64 + 16 per distinct time (a runaway guard; a bursty tag needs ≈ 6)
|
||||
public async Task The_default_read_cap_is_64_plus_4_per_distinct_time()
|
||||
{
|
||||
(GatewayAtTimeReader.BaseReads, GatewayAtTimeReader.ReadsPerTime).ShouldBe((64, 16));
|
||||
await Task.CompletedTask;
|
||||
}
|
||||
|
||||
// ---- type hint edge cases (review L2/L3) ------------------------------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public async Task A_DateTime_node_is_stepped()
|
||||
{
|
||||
var fake = new AvevaStore([(At(0), 0, false), (At(10), 10, false)]).Client();
|
||||
var s = (await AtTimeTyped(fake, TypeCode.DateTime, At(5))).Single();
|
||||
s.Value.ShouldBe(0d);
|
||||
s.StatusCode.ShouldBe(GoodInterpolated);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(double.PositiveInfinity)]
|
||||
[InlineData(double.NaN)]
|
||||
[InlineData(1e30)]
|
||||
public async Task Under_an_integer_hint_a_non_integer_next_value_is_unusable_not_saturated(double next)
|
||||
{
|
||||
var fake = new AvevaStore([(At(0), 100, false), (At(10), next, false)]).Client();
|
||||
var s = (await AtTimeTyped(fake, TypeCode.Int32, At(5))).Single();
|
||||
s.Value.ShouldBeOfType<int>().ShouldBe(100); // held before an unusable neighbour
|
||||
s.StatusCode.ShouldBe(UncertainInterpolated);
|
||||
}
|
||||
|
||||
// ---- kill switch -----------------------------------------------------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public async Task The_kill_switch_restores_AVEVAs_own_answer()
|
||||
{
|
||||
var fake = new AvevaStore(Gate685()).Client();
|
||||
fake.AtTimeSamples = [new HistorianSample { NumericValue = 107, OpcQuality = 192, Timestamp = Timestamp.FromDateTime(At(3)) }];
|
||||
var ds = new GatewayHistorianDataSource(fake, NullLogger<GatewayHistorianDataSource>.Instance, atTimeFromRaw: false);
|
||||
var s = (await ds.ReadAtTimeAsync("T", [At(3)], Ct)).Samples.Single();
|
||||
s.Value.ShouldBe(107d);
|
||||
fake.ReadAtTimeCallCount.ShouldBe(1);
|
||||
fake.ReadRawCallCount.ShouldBe(0);
|
||||
}
|
||||
}
|
||||
+174
@@ -0,0 +1,174 @@
|
||||
using Google.Protobuf.WellKnownTypes;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.HistorianGateway.Contracts.Grpc;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.OpcUaServer;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #691: Part 11 bounding values on the Gateway, composed by the server's <see cref="HistoryBounds"/> over the REAL
|
||||
/// source and an AVEVA-shaped fake (the live gate's wire shape). The start bound of a forward read must be the stored
|
||||
/// sample before StartTime, at ITS OWN time — never AVEVA's synthetic boundary row, which is stamped at the query start
|
||||
/// and carries the value in effect there (<c>Quality = 133</c>). The Gateway reads forward only, so "at or before" is
|
||||
/// served by the walk-back (<see cref="ReverseRawReader"/>), and no reverse window is ever sent.
|
||||
/// </summary>
|
||||
public sealed class GatewayBoundingValueTests
|
||||
{
|
||||
private static CancellationToken Ct => TestContext.Current.CancellationToken;
|
||||
|
||||
private static readonly DateTime T0 = new(2026, 10, 12, 18, 6, 0, DateTimeKind.Utc);
|
||||
private static readonly DateTime Now = T0.AddHours(1);
|
||||
|
||||
/// <summary>The live gate's tag <c>HistGW.LiveTest.Gate681</c> (18:06:40 is a stored NULL).</summary>
|
||||
private static readonly (double Sec, double Value, bool Null)[] Gate681 =
|
||||
[
|
||||
(0, 10, false), (5, 20, false), (20, 30, false), (40, 0, true), (50, 5, false), (60, 1, false), (90, 1, false),
|
||||
(100, 7, false),
|
||||
];
|
||||
|
||||
private static DateTime At(double seconds) => T0.AddTicks((long)(seconds * TimeSpan.TicksPerSecond));
|
||||
|
||||
private static HistorianSample Stored(DateTime t, double v, bool isNull) => new()
|
||||
{
|
||||
Tag = "T",
|
||||
Timestamp = Timestamp.FromDateTime(t),
|
||||
NumericValue = isNull ? 0 : v,
|
||||
OpcQuality = isNull ? 0u : 192u,
|
||||
Quality = isNull ? 1u : 0u,
|
||||
};
|
||||
|
||||
/// <summary>The latest query end any ReadRaw was sent (a far-future end is the #691 review's risk).</summary>
|
||||
private sealed class EndProbe
|
||||
{
|
||||
public DateTime MaxEnd = DateTime.MinValue;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// ReadRaw over <c>[start, end]</c> ascending, capped, led by AVEVA's boundary row at the query start. A query end more
|
||||
/// than two days past <see cref="Now"/> is REFUSED (<c>InvalidArgument</c>), modelling the unverified AVEVA behaviour
|
||||
/// for a year-9999 end (the HistorianGateway only checks <c>start <= end</c>, so it reaches AVEVA unclamped).
|
||||
/// </summary>
|
||||
private static FakeHistorianGatewayClient Aveva(EndProbe? probe = null)
|
||||
{
|
||||
var sorted = Gate681.Select(x => (T: At(x.Sec), V: x.Value, x.Null)).ToList();
|
||||
return new FakeHistorianGatewayClient
|
||||
{
|
||||
RawSource = (start, end, max) =>
|
||||
{
|
||||
if (probe is not null && end > probe.MaxEnd) probe.MaxEnd = end;
|
||||
if (end > Now.AddDays(2))
|
||||
throw new Grpc.Core.RpcException(new Grpc.Core.Status(Grpc.Core.StatusCode.InvalidArgument,
|
||||
"end_time is outside the historian's time range"));
|
||||
var rows = new List<HistorianSample>();
|
||||
if (sorted.All(d => d.T != start))
|
||||
{
|
||||
var prior = sorted.Where(d => d.T < start).Select(d => ((DateTime T, double V, bool Null)?)d).LastOrDefault();
|
||||
HistorianSample boundary;
|
||||
if (prior is { } p)
|
||||
{
|
||||
boundary = Stored(start, p.V, p.Null);
|
||||
if (!p.Null) boundary.Quality = 133;
|
||||
}
|
||||
else
|
||||
{
|
||||
boundary = new HistorianSample { Tag = "T", Timestamp = Timestamp.FromDateTime(start), NumericValue = 0, OpcQuality = 0, Quality = 1 };
|
||||
}
|
||||
|
||||
rows.Add(boundary);
|
||||
}
|
||||
|
||||
rows.AddRange(sorted.Where(d => d.T >= start && d.T <= end).Select(d => Stored(d.T, d.V, d.Null)));
|
||||
return max > 0 ? rows.Take(max) : rows;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
private static GatewayHistorianDataSource Source(FakeHistorianGatewayClient fake) =>
|
||||
new(fake, NullLogger<GatewayHistorianDataSource>.Instance);
|
||||
|
||||
[Fact]
|
||||
public async Task The_start_bound_is_the_stored_prior_sample_at_its_own_time_not_the_synthetic_row()
|
||||
{
|
||||
var bound = await HistoryBounds.StartBoundAsync(Source(Aveva()), "T", At(21), reverse: false, 65536, Now, Ct);
|
||||
|
||||
bound.ShouldNotBeNull();
|
||||
bound.SourceTimestampUtc.ShouldBe(At(20)); // the synthetic row would be stamped At(21)
|
||||
bound.Value.ShouldBe(30d);
|
||||
bound.StatusCode.ShouldBe(0u);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_stored_NULL_prior_sample_is_the_start_bound_as_the_Bad_sample_it_is()
|
||||
{
|
||||
var bound = await HistoryBounds.StartBoundAsync(Source(Aveva()), "T", At(45), reverse: false, 65536, Now, Ct);
|
||||
|
||||
bound!.SourceTimestampUtc.ShouldBe(At(40));
|
||||
bound.StatusCode.ShouldBe(0x80000000u);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Before_the_first_sample_the_start_bound_is_not_found_at_the_start_time()
|
||||
{
|
||||
var bound = await HistoryBounds.StartBoundAsync(Source(Aveva()), "T", At(-30), reverse: false, 65536, Now, Ct);
|
||||
|
||||
bound!.StatusCode.ShouldBe(Opc.Ua.StatusCodes.BadBoundNotFound);
|
||||
bound.SourceTimestampUtc.ShouldBe(At(-30));
|
||||
bound.Value.ShouldBeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_sample_exactly_on_the_start_time_is_the_bound_and_already_the_data()
|
||||
{
|
||||
(await HistoryBounds.StartBoundAsync(Source(Aveva()), "T", At(20), reverse: false, 65536, Now, Ct)).ShouldBeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task The_end_bound_is_the_first_stored_sample_at_or_after_the_end_time()
|
||||
{
|
||||
var source = Source(Aveva());
|
||||
var after = await HistoryBounds.EndBoundAsync(source, "T", HistoryBounds.For(At(0), At(41)), At(40), 65536, Now, Ct);
|
||||
after.SourceTimestampUtc.ShouldBe(At(50));
|
||||
after.Value.ShouldBe(5d);
|
||||
|
||||
var missing = await HistoryBounds.EndBoundAsync(source, "T", HistoryBounds.For(At(0), At(101)), At(100), 65536, Now, Ct);
|
||||
missing.StatusCode.ShouldBe(Opc.Ua.StatusCodes.BadBoundNotFound);
|
||||
missing.SourceTimestampUtc.ShouldBe(At(101));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// #691 review: the at-or-after read never sends an end beyond now + <see cref="ReverseRawReader.LookAhead"/> (the
|
||||
/// historian stores nothing further ahead), and an end bound wanted beyond that horizon is not found without a read.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task The_at_or_after_read_never_asks_past_now_plus_one_day()
|
||||
{
|
||||
var probe = new EndProbe();
|
||||
var fake = Aveva(probe);
|
||||
var source = Source(fake);
|
||||
|
||||
var found = await HistoryBounds.EndBoundAsync(source, "T", HistoryBounds.For(At(0), At(41)), At(40), 65536, Now, Ct);
|
||||
found.SourceTimestampUtc.ShouldBe(At(50));
|
||||
probe.MaxEnd.ShouldBeLessThanOrEqualTo(Now + ReverseRawReader.LookAhead.Add(TimeSpan.FromMilliseconds(1)));
|
||||
|
||||
var calls = fake.ReadRawCallCount;
|
||||
var beyond = Now.AddDays(3);
|
||||
var missing = await HistoryBounds.EndBoundAsync(source, "T", HistoryBounds.For(At(0), beyond), At(100), 65536, Now, Ct);
|
||||
missing.StatusCode.ShouldBe(Opc.Ua.StatusCodes.BadBoundNotFound);
|
||||
missing.SourceTimestampUtc.ShouldBe(beyond);
|
||||
fake.ReadRawCallCount.ShouldBe(calls, "a bound beyond the horizon is not found without a read");
|
||||
source.GetHealthSnapshot().TotalFailures.ShouldBe(0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_reverse_read_end_bound_walks_back_to_the_stored_sample_at_or_before_the_end_time()
|
||||
{
|
||||
var fake = Aveva();
|
||||
var bound = await HistoryBounds.EndBoundAsync(Source(fake), "T", HistoryBounds.For(At(100), At(55)), At(60), 65536, Now, Ct);
|
||||
|
||||
bound.SourceTimestampUtc.ShouldBe(At(50));
|
||||
bound.Value.ShouldBe(5d);
|
||||
}
|
||||
}
|
||||
+560
@@ -0,0 +1,560 @@
|
||||
using Google.Protobuf.WellKnownTypes;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Opc.Ua;
|
||||
using Opc.Ua.Server;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.HistorianGateway.Contracts.Grpc;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.OpcUaServer;
|
||||
using SdkHistoryReadResult = Opc.Ua.HistoryReadResult;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #694 item 1 (live gate F3): a strict gateway (<c>AllowStreamingBeyondCeiling=false</c>) with a LOWERED
|
||||
/// <c>MaxValuesCeiling</c> (1000) refused ordinary page sizes — <c>NumValuesPerNode = 5000</c>, or an uncapped read over
|
||||
/// 3,600 rows, came back <c>BadTooManyOperations</c>. The server now sizes the page to the learned ceiling
|
||||
/// (<see cref="IHistorianDataSource.MaxRawValuesPerRead"/> L = ceiling − 16): the read — page P + boundary ties k + the
|
||||
/// #692 lookahead row — is at most L, so a first page holds L − 1 values and a resumed one L − k − 1, each with a
|
||||
/// continuation point. Driven end to end: a REAL SDK server's node manager → the REAL
|
||||
/// <see cref="GatewayHistorianDataSource"/> → an AVEVA-shaped fake gateway client that refuses an over-ceiling
|
||||
/// <c>max_values</c> exactly as the real gateway does (<c>InvalidArgument</c> "max_values exceeds the server ceiling of N",
|
||||
/// wrapped by the client package) and rejects <c>max_values <= 0</c> as the client package does.
|
||||
/// </summary>
|
||||
public sealed class GatewayCeilingPagingTests : IDisposable
|
||||
{
|
||||
private static CancellationToken Ct => TestContext.Current.CancellationToken;
|
||||
|
||||
private static readonly DateTime T0 = new(2026, 10, 12, 10, 0, 0, DateTimeKind.Utc);
|
||||
|
||||
private const int Ceiling = 1000;
|
||||
private const int L = Ceiling - RawAggregateReader.PreWindowAllowance; // 984
|
||||
private const int FirstPage = L - 1; // 983: page + lookahead fills a read
|
||||
private const int NextPage = L - 2; // 982: + the one boundary tie a resumed page re-reads (distinct timestamps)
|
||||
|
||||
private readonly string _pkiRoot = Path.Combine(Path.GetTempPath(), $"otopcua-gwceiling-{Guid.NewGuid():N}");
|
||||
|
||||
private static DateTime At(int seconds) => T0.AddSeconds(seconds);
|
||||
|
||||
private static HistorianSample Row(DateTime t, double v, uint quality = 0) =>
|
||||
new() { Tag = "T", Timestamp = Timestamp.FromDateTime(t), NumericValue = v, OpcQuality = 192, Quality = quality };
|
||||
|
||||
/// <summary>
|
||||
/// AVEVA-shaped ReadRaw over <paramref name="stored"/>: a boundary row at the query start (the value in effect, or a
|
||||
/// NULL row before the first sample), then <c>[start, end]</c> inclusive ascending, capped at <c>max_values</c>.
|
||||
/// </summary>
|
||||
private static FakeHistorianGatewayClient Gateway(IReadOnlyList<(DateTime T, double V)> stored, int? ceiling) => new()
|
||||
{
|
||||
MaxValuesCeiling = ceiling,
|
||||
RawSource = (start, end, max) =>
|
||||
{
|
||||
if (max <= 0) throw new ArgumentOutOfRangeException(nameof(max), "max_values must be positive"); // the client package
|
||||
var rows = new List<HistorianSample>();
|
||||
if (stored.All(s => s.T != start))
|
||||
{
|
||||
var prior = stored.LastOrDefault(s => s.T < start);
|
||||
rows.Add(prior.T == default
|
||||
? new HistorianSample { Tag = "T", Timestamp = Timestamp.FromDateTime(start), NumericValue = 0, OpcQuality = 0, Quality = 1 }
|
||||
: Row(start, prior.V, 133));
|
||||
}
|
||||
rows.AddRange(stored.Where(s => s.T >= start && s.T <= end).Select(s => Row(s.T, s.V)));
|
||||
return rows.Take(max);
|
||||
},
|
||||
};
|
||||
|
||||
/// <summary>One sample per second, value = seconds, at +0 … +(count − 1).</summary>
|
||||
private static List<(DateTime T, double V)> PerSecond(int count) =>
|
||||
Enumerable.Range(0, count).Select(i => (At(i), (double)i)).ToList();
|
||||
|
||||
private static int Refusals(FakeHistorianGatewayClient fake) =>
|
||||
fake.RawCalls.Count(c => fake.MaxValuesCeiling is { } n && c.MaxValues > n);
|
||||
|
||||
private static void EveryRequestFits(FakeHistorianGatewayClient fake) =>
|
||||
fake.RawCalls.ShouldAllBe(c => c.MaxValues > 0 && c.MaxValues <= Ceiling); // never uncapped, never over the ceiling
|
||||
|
||||
// ---- the data source's page limit ------------------------------------------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public async Task The_page_limit_is_unknown_until_a_refusal_names_the_ceiling_and_forgotten_when_the_gateway_streams()
|
||||
{
|
||||
var fake = Gateway(PerSecond(3600), Ceiling);
|
||||
var ds = new GatewayHistorianDataSource(fake, NullLogger<GatewayHistorianDataSource>.Instance);
|
||||
ds.MaxRawValuesPerRead.ShouldBe(0);
|
||||
|
||||
await ds.ReadRawAsync("T", At(0), At(10), 0, Ct);
|
||||
ds.MaxRawValuesPerRead.ShouldBe(L); // a read of this size asks for exactly the ceiling
|
||||
|
||||
(await ds.ReadRawAsync("T", At(0), At(3600), (uint)L, Ct)).Samples.Count.ShouldBe(L);
|
||||
fake.RawCalls[^1].MaxValues.ShouldBe(Ceiling);
|
||||
Refusals(fake).ShouldBe(1);
|
||||
|
||||
// The operator enabled AllowStreamingBeyondCeiling: a ceiling-full uncapped read re-probes (#691.3) and forgets it.
|
||||
fake.MaxValuesCeiling = null;
|
||||
(await ds.ReadRawAsync("T", At(0), At(3600), 0, Ct)).Samples.Count.ShouldBe(3600);
|
||||
ds.MaxRawValuesPerRead.ShouldBe(0);
|
||||
}
|
||||
|
||||
// ---- through the node manager ----------------------------------------------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public async Task The_cold_first_read_costs_one_refusal_and_its_page_fits_the_learned_ceiling()
|
||||
{
|
||||
var fake = Gateway(PerSecond(3600), Ceiling);
|
||||
var (host, nm, nodeId) = await BootAsync(fake);
|
||||
try
|
||||
{
|
||||
var first = ReadRaw(nm, nodeId, At(0), At(3600), 5000, null);
|
||||
first.Error.StatusCode.Code.ShouldBe(StatusCodes.Good);
|
||||
first.Values.Count.ShouldBe(FirstPage);
|
||||
first.Cp.ShouldNotBeNull();
|
||||
// The budget the client asked for (+ lookahead + allowance, refused naming 1000), the ceiling re-read that cannot
|
||||
// hold the window, then the page sized to the learned ceiling.
|
||||
fake.RawCalls.Select(c => c.MaxValues).ShouldBe([5000 + 1 + RawAggregateReader.PreWindowAllowance, Ceiling, Ceiling]);
|
||||
|
||||
var (rest, _, status) = DrainFrom(nm, nodeId, At(0), At(3600), 5000, first.Cp);
|
||||
status.Code.ShouldBe(StatusCodes.Good);
|
||||
first.Values.Concat(rest).ShouldBe(PerSecond(3600).Select(s => s.Item2));
|
||||
Refusals(fake).ShouldBe(1);
|
||||
}
|
||||
finally { await host.DisposeAsync(); }
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(5000u, false)]
|
||||
[InlineData(0u, false)]
|
||||
[InlineData(5000u, true)]
|
||||
[InlineData(0u, true)]
|
||||
public async Task With_the_ceiling_known_every_page_is_exactly_the_ceiling_page_and_nothing_is_refused(uint n, bool reverse)
|
||||
{
|
||||
var stored = PerSecond(3600);
|
||||
var fake = Gateway(stored, Ceiling);
|
||||
var (host, nm, nodeId) = await BootAsync(fake);
|
||||
try
|
||||
{
|
||||
ReadRaw(nm, nodeId, At(0), At(3600), 5000, null); // learns the ceiling
|
||||
fake.RawCalls.Clear();
|
||||
|
||||
var (values, pages, status) = reverse
|
||||
? DrainFrom(nm, nodeId, At(3600), At(-1), n, null)
|
||||
: DrainFrom(nm, nodeId, At(0), At(3600), n, null);
|
||||
|
||||
status.Code.ShouldBe(StatusCodes.Good);
|
||||
var expected = stored.Select(s => s.Item2);
|
||||
values.ShouldBe(reverse ? expected.Reverse() : expected);
|
||||
pages[0].ShouldBe(FirstPage);
|
||||
pages.Skip(1).SkipLast(1).ShouldAllBe(p => p == NextPage);
|
||||
pages[^1].ShouldBeGreaterThan(0); // no empty trailing page
|
||||
Refusals(fake).ShouldBe(0);
|
||||
EveryRequestFits(fake);
|
||||
if (!reverse) fake.RawCalls.Count.ShouldBe(pages.Count); // one read per page
|
||||
}
|
||||
finally { await host.DisposeAsync(); }
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task An_uncapped_read_that_fits_the_ceiling_is_still_one_page_without_a_continuation_point()
|
||||
{
|
||||
var fake = Gateway(PerSecond(3600), Ceiling);
|
||||
var (host, nm, nodeId) = await BootAsync(fake);
|
||||
try
|
||||
{
|
||||
var (values, pages, _) = DrainFrom(nm, nodeId, At(0), At(10), 0, null);
|
||||
values.Count.ShouldBe(10);
|
||||
pages.Count.ShouldBe(1);
|
||||
fake.RawCalls.Select(c => c.MaxValues).ShouldBe([int.MaxValue, Ceiling]); // #691.3 unchanged
|
||||
|
||||
var again = DrainFrom(nm, nodeId, At(0), At(10), 0, null);
|
||||
again.Pages.Count.ShouldBe(1);
|
||||
fake.RawCalls[^1].MaxValues.ShouldBe(Ceiling); // the learned page size, one read
|
||||
}
|
||||
finally { await host.DisposeAsync(); }
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_default_gateway_is_unchanged_one_read_and_no_continuation_point()
|
||||
{
|
||||
var fake = Gateway(PerSecond(3600), ceiling: null);
|
||||
var (host, nm, nodeId) = await BootAsync(fake);
|
||||
try
|
||||
{
|
||||
var (values, pages, _) = DrainFrom(nm, nodeId, At(0), At(3600), 5000, null);
|
||||
values.Count.ShouldBe(3600);
|
||||
pages.Count.ShouldBe(1);
|
||||
fake.RawCalls.Select(c => c.MaxValues).ShouldBe([5000 + 1 + RawAggregateReader.PreWindowAllowance]);
|
||||
}
|
||||
finally { await host.DisposeAsync(); }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A tie cluster straddling a page boundary, at the sizes where the ceiling arithmetic turns: a read can carry L rows;
|
||||
/// the over-fetch that pages within a cluster reads at most L − 1 (+1 to detect more). Every outcome is exact, or
|
||||
/// <c>BadTooManyOperations</c> after an exact prefix — never an uncapped or over-ceiling request, never an endless read.
|
||||
/// </summary>
|
||||
[Theory]
|
||||
[InlineData(L - 2, 5000u)]
|
||||
[InlineData(L - 1, 5000u)]
|
||||
[InlineData(L, 5000u)]
|
||||
[InlineData(L + 1, 5000u)]
|
||||
[InlineData(Ceiling + 1, 5000u)]
|
||||
[InlineData(L - 2, 0u)]
|
||||
[InlineData(L - 1, 0u)]
|
||||
[InlineData(L + 1, 0u)]
|
||||
[InlineData(Ceiling + 1, 0u)]
|
||||
public async Task A_tie_cluster_at_a_page_boundary_is_exact_or_refused_never_truncated_or_endless(int size, uint n)
|
||||
{
|
||||
var stored = PerSecond(500)
|
||||
.Concat(Enumerable.Range(0, size).Select(i => (At(600), 10_000.0 + i)))
|
||||
.Concat(Enumerable.Range(0, 1500).Select(i => (At(700 + i), 20_000.0 + i)))
|
||||
.ToList();
|
||||
var fake = Gateway(stored, Ceiling);
|
||||
var (host, nm, nodeId) = await BootAsync(fake);
|
||||
try
|
||||
{
|
||||
ReadRaw(nm, nodeId, At(0), At(5), 5000, null); // learns the ceiling
|
||||
fake.RawCalls.Clear();
|
||||
|
||||
var (values, pages, status) = DrainFrom(nm, nodeId, At(0), At(5000), n, null, maxPages: 60);
|
||||
var expected = stored.Select(s => s.Item2).ToList();
|
||||
if (status.Code == StatusCodes.Good) values.ShouldBe(expected);
|
||||
else
|
||||
{
|
||||
status.Code.ShouldBe(StatusCodes.BadTooManyOperations);
|
||||
values.ShouldBe(expected.Take(values.Count)); // an exact prefix
|
||||
size.ShouldBeGreaterThan(L - 2, "a cluster that fits the over-fetch is always served");
|
||||
}
|
||||
pages.Count.ShouldBeLessThan(60);
|
||||
EveryRequestFits(fake);
|
||||
}
|
||||
finally { await host.DisposeAsync(); }
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task An_uncapped_cursor_inside_a_tie_cluster_finishes_exactly_when_the_gateway_stops_being_strict()
|
||||
{
|
||||
// 100 samples, then 983 ties: the first page ends inside the cluster. Then the operator enables streaming and the
|
||||
// data source forgets the ceiling (L back to 0) — the uncapped cursor must still finish exactly, not loop.
|
||||
var stored = PerSecond(100)
|
||||
.Concat(Enumerable.Range(0, L - 1).Select(i => (At(200), 10_000.0 + i)))
|
||||
.Concat(Enumerable.Range(0, 300).Select(i => (At(300 + i), 20_000.0 + i)))
|
||||
.ToList();
|
||||
var fake = Gateway(stored, Ceiling);
|
||||
var ds = new GatewayHistorianDataSource(fake, NullLogger<GatewayHistorianDataSource>.Instance);
|
||||
var (host, nm, nodeId) = await BootAsync(fake, ds);
|
||||
try
|
||||
{
|
||||
ReadRaw(nm, nodeId, At(0), At(5), 5000, null); // learns the ceiling
|
||||
var first = ReadRaw(nm, nodeId, At(0), At(5000), 0, null);
|
||||
first.Values.Count.ShouldBe(FirstPage);
|
||||
first.Cp.ShouldNotBeNull();
|
||||
|
||||
fake.MaxValuesCeiling = null;
|
||||
await ds.ReadRawAsync("T", At(0), At(5000), 0, Ct); // a ceiling-full read re-probes and forgets
|
||||
ds.MaxRawValuesPerRead.ShouldBe(0);
|
||||
|
||||
var (rest, pages, status) = DrainFrom(nm, nodeId, At(0), At(5000), 0, first.Cp, maxPages: 20);
|
||||
status.Code.ShouldBe(StatusCodes.Good);
|
||||
first.Values.Concat(rest).ShouldBe(stored.Select(s => s.Item2));
|
||||
pages.Count.ShouldBeLessThan(20);
|
||||
}
|
||||
finally { await host.DisposeAsync(); }
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_ceiling_lowered_mid_page_through_is_re_learned_and_the_read_stays_exact()
|
||||
{
|
||||
var stored = PerSecond(3600);
|
||||
var fake = Gateway(stored, Ceiling);
|
||||
var (host, nm, nodeId) = await BootAsync(fake);
|
||||
try
|
||||
{
|
||||
var got = new List<double>();
|
||||
var p1 = ReadRaw(nm, nodeId, At(0), At(3600), 5000, null);
|
||||
got.AddRange(p1.Values);
|
||||
var p2 = ReadRaw(nm, nodeId, At(0), At(3600), 5000, p1.Cp);
|
||||
got.AddRange(p2.Values);
|
||||
|
||||
fake.MaxValuesCeiling = 500; // the operator lowered it
|
||||
fake.RawCalls.Clear();
|
||||
var (rest, pages, status) = DrainFrom(nm, nodeId, At(0), At(3600), 5000, p2.Cp, maxPages: 20);
|
||||
status.Code.ShouldBe(StatusCodes.Good);
|
||||
got.AddRange(rest);
|
||||
got.ShouldBe(stored.Select(s => s.Item2));
|
||||
pages.ShouldAllBe(p => p <= 500 - RawAggregateReader.PreWindowAllowance);
|
||||
Refusals(fake).ShouldBe(1); // the one that names the new ceiling
|
||||
}
|
||||
finally { await host.DisposeAsync(); }
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Bounding_values_page_within_the_ceiling()
|
||||
{
|
||||
var stored = PerSecond(3600);
|
||||
var fake = Gateway(stored, Ceiling);
|
||||
var (host, nm, nodeId) = await BootAsync(fake);
|
||||
try
|
||||
{
|
||||
ReadRaw(nm, nodeId, At(0), At(5), 5000, null); // learns the ceiling
|
||||
fake.RawCalls.Clear();
|
||||
|
||||
// Start bound = the sample at +10 (before StartTime), end bound = the sample at +3001 (after EndTime).
|
||||
var start = At(10).AddMilliseconds(500);
|
||||
var end = At(3000).AddMilliseconds(500);
|
||||
var (values, pages, status) = DrainFrom(nm, nodeId, start, end, 5000, null, returnBounds: true);
|
||||
status.Code.ShouldBe(StatusCodes.Good);
|
||||
values.ShouldBe(Enumerable.Range(10, 2992).Select(i => (double)i));
|
||||
pages.ShouldAllBe(p => p <= L);
|
||||
Refusals(fake).ShouldBe(0);
|
||||
}
|
||||
finally { await host.DisposeAsync(); }
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(1000, 100_000u, 10)]
|
||||
[InlineData(1000, 5000u, 10)]
|
||||
[InlineData(20, 3000u, 40)]
|
||||
public async Task A_cold_end_only_read_with_a_huge_page_shrinks_the_page_not_the_walk(int ceiling, uint n, int maxReads)
|
||||
{
|
||||
// Review F2: the walk used to re-run itself at the learned ceiling while keeping the page at N — a cold end-only read
|
||||
// with ceiling 20 and N = 3000 cost 3,023 gateway reads in ONE HistoryRead (ceiling 1000, N = 100,000: 207). The
|
||||
// refusal now reaches the page planner, which shrinks the page to the ceiling.
|
||||
var dense = Enumerable.Range(0, 200_000).Select(i => (T0.AddMilliseconds(i), (double)i)).ToList();
|
||||
var fake = Gateway(dense, ceiling);
|
||||
var (host, nm, nodeId) = await BootAsync(fake);
|
||||
try
|
||||
{
|
||||
var r = ReadRaw(nm, nodeId, DateTime.MinValue, T0.AddMilliseconds(200_000), n, null);
|
||||
r.Error.StatusCode.Code.ShouldBe(StatusCodes.Good);
|
||||
r.Cp.ShouldNotBeNull();
|
||||
var page = ceiling - RawAggregateReader.PreWindowAllowance - 1;
|
||||
r.Values.ShouldBe(Enumerable.Range(200_000 - r.Values.Count, r.Values.Count).Reverse().Select(i => (double)i));
|
||||
r.Values.Count.ShouldBeLessThanOrEqualTo(page);
|
||||
fake.RawCalls.Count.ShouldBeLessThanOrEqualTo(maxReads);
|
||||
}
|
||||
finally { await host.DisposeAsync(); }
|
||||
}
|
||||
|
||||
// ---- F1: a window starting at/before the epoch gets no look-back (review of #694) -------------------------------------
|
||||
|
||||
private static List<(DateTime T, double V)> FromEpoch(int count) =>
|
||||
Enumerable.Range(1, count).Select(i => (DateTime.UnixEpoch.AddSeconds(i), (double)i)).ToList();
|
||||
|
||||
[Fact]
|
||||
public async Task A_ceiling_truncated_read_from_the_epoch_is_never_reported_complete()
|
||||
{
|
||||
// AVEVA's synthetic row lands AT the epoch start (no 1 ms look-back there). It used to count as "past the window
|
||||
// end", so a ceiling-truncated uncapped read answered its first 999 rows, Good, as if that were everything.
|
||||
var fake = Gateway(FromEpoch(3000), Ceiling);
|
||||
var ds = new GatewayHistorianDataSource(fake, NullLogger<GatewayHistorianDataSource>.Instance);
|
||||
await Should.ThrowAsync<HistoryReadLimitExceededException>(() =>
|
||||
ds.ReadRawAsync("T", DateTime.UnixEpoch, DateTime.UnixEpoch.AddDays(1), 0, Ct));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task An_uncapped_read_from_the_epoch_pages_every_sample()
|
||||
{
|
||||
var stored = FromEpoch(3000);
|
||||
var fake = Gateway(stored, Ceiling);
|
||||
var (host, nm, nodeId) = await BootAsync(fake);
|
||||
try
|
||||
{
|
||||
var (values, pages, status) = DrainFrom(nm, nodeId, DateTime.UnixEpoch, DateTime.UnixEpoch.AddDays(1), 0, null);
|
||||
status.Code.ShouldBe(StatusCodes.Good);
|
||||
values.ShouldBe(stored.Select(s => s.Item2));
|
||||
pages.Count.ShouldBeGreaterThan(1);
|
||||
}
|
||||
finally { await host.DisposeAsync(); }
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task An_end_only_read_far_past_epoch_data_returns_the_latest_values()
|
||||
{
|
||||
var stored = FromEpoch(3000);
|
||||
var fake = Gateway(stored, Ceiling);
|
||||
var (host, nm, nodeId) = await BootAsync(fake);
|
||||
try
|
||||
{
|
||||
ReadRaw(nm, nodeId, DateTime.UnixEpoch, DateTime.UnixEpoch.AddSeconds(5), 5000, null); // learns the ceiling
|
||||
var r = ReadRaw(nm, nodeId, DateTime.MinValue, DateTime.UnixEpoch.AddDays(30), 10, null);
|
||||
r.Error.StatusCode.Code.ShouldBe(StatusCodes.Good);
|
||||
r.Values.ShouldBe(Enumerable.Range(2991, 10).Reverse().Select(i => (double)i)); // the LATEST ten, newest first
|
||||
}
|
||||
finally { await host.DisposeAsync(); }
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_stopped_dense_tag_under_a_strict_ceiling_is_found_with_bounded_discovery_cost()
|
||||
{
|
||||
var stored = PerSecond(7200);
|
||||
var fake = Gateway(stored, Ceiling);
|
||||
var ds = new GatewayHistorianDataSource(fake, NullLogger<GatewayHistorianDataSource>.Instance);
|
||||
await ds.ReadRawAsync("T", At(0), At(1), 5000, Ct);
|
||||
fake.RawCalls.Clear();
|
||||
var page = await ReverseRawReader.ReadAsync(ds, "T", At(7200).AddDays(1), DateTime.UnixEpoch,
|
||||
3, 65536, At(7200).AddDays(1), 0, Ct);
|
||||
page.Samples.Select(s => s.Value).ShouldBe(new object?[] { 7199d, 7198d, 7197d });
|
||||
fake.RawCalls.Count.ShouldBeLessThanOrEqualTo(12);
|
||||
EveryRequestFits(fake);
|
||||
}
|
||||
|
||||
// ---- F3: rows crowding the look-back before a page start ---------------------------------------------------------------
|
||||
|
||||
[Theory]
|
||||
[InlineData(5000u)]
|
||||
[InlineData(0u)]
|
||||
[InlineData((uint)L)]
|
||||
[InlineData(900u)]
|
||||
public async Task Ties_just_before_a_page_start_are_read_around_not_refused(uint n)
|
||||
{
|
||||
// 962 samples, 20 ties at X, one sample at X + 1 ms: the first page (983) ends exactly at X + 1 ms. The next page's
|
||||
// 1 ms look-back holds the 20 ties — more than the 16-row allowance, and a ceiling-full page leaves no room for the
|
||||
// #687 re-read, so it used to fail BadTooManyOperations. The source now names the crowding rows and the page is
|
||||
// re-read that much smaller.
|
||||
var x = At(1000);
|
||||
var stored = Enumerable.Range(0, 962).Select(i => (At(i), (double)i))
|
||||
.Concat(Enumerable.Range(0, 20).Select(i => (x, 962.0 + i)))
|
||||
.Append((x.AddMilliseconds(1), 982.0))
|
||||
.Concat(Enumerable.Range(0, 3000).Select(i => (x.AddSeconds(1 + i), 983.0 + i)))
|
||||
.ToList();
|
||||
var fake = Gateway(stored, Ceiling);
|
||||
var (host, nm, nodeId) = await BootAsync(fake);
|
||||
try
|
||||
{
|
||||
ReadRaw(nm, nodeId, At(0), At(5), 5000, null); // learns the ceiling
|
||||
fake.RawCalls.Clear();
|
||||
|
||||
var (values, _, status) = DrainFrom(nm, nodeId, At(0), x.AddDays(1), n, null);
|
||||
status.Code.ShouldBe(StatusCodes.Good);
|
||||
values.ShouldBe(stored.Select(s => s.Item2));
|
||||
Refusals(fake).ShouldBe(0);
|
||||
EveryRequestFits(fake);
|
||||
}
|
||||
finally { await host.DisposeAsync(); }
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(1709, 1366)] // #695 reverse-walk trace
|
||||
[InlineData(1369, 1292)] // #695 bounds trace
|
||||
public async Task A_warm_read_uses_the_last_room_below_the_ceiling_for_pre_window_ties(int ceiling, int cluster)
|
||||
{
|
||||
var start = At(1000);
|
||||
var stored = Enumerable.Range(0, cluster).Select(i => (start.AddMilliseconds(-0.5), (double)i))
|
||||
.Append((start, 9000d)).Append((start.AddSeconds(1), 9001d)).ToList();
|
||||
var fake = Gateway(stored, ceiling);
|
||||
using var ds = new GatewayHistorianDataSource(fake, NullLogger<GatewayHistorianDataSource>.Instance);
|
||||
await ds.ReadRawAsync("T", At(0), At(1), 5000, Ct); // learn without crowding
|
||||
fake.RawCalls.Clear();
|
||||
var result = await ds.ReadRawAsync("T", start, start.AddSeconds(2), 1, Ct);
|
||||
result.Samples.Select(s => s.Value).ShouldBe([9000d]);
|
||||
fake.RawCalls.ShouldAllBe(c => c.MaxValues <= ceiling);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_warm_reverse_segment_can_shrink_again_when_the_pre_window_cluster_grows()
|
||||
{
|
||||
var start = At(1000);
|
||||
var stored = Enumerable.Range(0, 1366).Select(i => (start.AddMilliseconds(-0.5), (double)i))
|
||||
.Concat(Enumerable.Range(0, 500).Select(i => (start.AddMilliseconds(i), 9000d + i))).ToList();
|
||||
var fake = Gateway(stored, 1709);
|
||||
using var ds = new GatewayHistorianDataSource(fake, NullLogger<GatewayHistorianDataSource>.Instance);
|
||||
await ds.ReadRawAsync("T", At(0), At(1), 5000, Ct);
|
||||
var page = await ReverseRawReader.ReadAsync(ds, "T", start.AddMilliseconds(499), start.AddTicks(-1),
|
||||
249, 65536, At(2000), new ReverseWalkHint(499 * TimeSpan.TicksPerMillisecond), Ct);
|
||||
page.Samples.Select(s => (double)s.Value!).ShouldBe(Enumerable.Range(9251, 249).Reverse().Select(i => (double)i));
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(false)]
|
||||
[InlineData(true)]
|
||||
public async Task Gateway_transport_faults_surface_as_BadNoCommunication(bool deadline)
|
||||
{
|
||||
var fake = new FakeHistorianGatewayClient { ThrowOnRead = deadline
|
||||
? new ZB.MOM.WW.HistorianGateway.Client.HistorianGatewayException("deadline", new Grpc.Core.RpcException(new Grpc.Core.Status(Grpc.Core.StatusCode.DeadlineExceeded, "deadline")))
|
||||
: new ZB.MOM.WW.HistorianGateway.Client.HistorianGatewayUnavailableException("unavailable") };
|
||||
var (host, nm, nodeId) = await BootAsync(fake);
|
||||
try
|
||||
{
|
||||
var r = ReadRaw(nm, nodeId, At(0), At(10), 10, null);
|
||||
r.Error.StatusCode.Code.ShouldBe(StatusCodes.BadNoCommunication);
|
||||
}
|
||||
finally { await host.DisposeAsync(); }
|
||||
}
|
||||
|
||||
// ---- helpers ---------------------------------------------------------------------------------------------------------
|
||||
|
||||
private async Task<(OpcUaApplicationHost Host, OtOpcUaNodeManager Nm, NodeId NodeId)> BootAsync(
|
||||
FakeHistorianGatewayClient fake, GatewayHistorianDataSource? ds = null)
|
||||
{
|
||||
var started = await OpcUaHostStart.StartAsync(
|
||||
port => new OpcUaApplicationHost(
|
||||
new OpcUaApplicationHostOptions
|
||||
{
|
||||
ApplicationName = "OtOpcUa.GatewayCeilingPagingTest",
|
||||
ApplicationUri = $"urn:OtOpcUa.GatewayCeilingPagingTest:{Guid.NewGuid():N}",
|
||||
OpcUaPort = port,
|
||||
PublicHostname = "localhost",
|
||||
PkiStoreRoot = _pkiRoot,
|
||||
},
|
||||
NullLogger<OpcUaApplicationHost>.Instance),
|
||||
() => new OtOpcUaSdkServer(), Ct);
|
||||
var nm = started.Server.NodeManager!;
|
||||
nm.HistoryContinuationStore = new InMemoryHistoryContinuationStore();
|
||||
nm.HistorianDataSource = ds ?? new GatewayHistorianDataSource(fake, NullLogger<GatewayHistorianDataSource>.Instance);
|
||||
nm.EnsureVariable("eq-1/pv", parentFolderNodeId: null, displayName: "PV", dataType: "Double",
|
||||
writable: false, historianTagname: "T", realm: AddressSpaceRealm.Uns);
|
||||
return (started.Host, nm, nm.TryGetVariable("eq-1/pv")!.NodeId);
|
||||
}
|
||||
|
||||
/// <summary>Pages until no continuation point or a Bad status; the page sizes and the final status.</summary>
|
||||
private static (List<double> Values, List<int> Pages, StatusCode Status) DrainFrom(
|
||||
OtOpcUaNodeManager nm, NodeId nodeId, DateTime start, DateTime end, uint max, byte[]? cp, int maxPages = 20,
|
||||
bool returnBounds = false)
|
||||
{
|
||||
var values = new List<double>();
|
||||
var pages = new List<int>();
|
||||
ServiceResult error;
|
||||
do
|
||||
{
|
||||
(var page, error, cp) = ReadRaw(nm, nodeId, start, end, max, cp, returnBounds);
|
||||
values.AddRange(page);
|
||||
pages.Add(page.Count);
|
||||
pages.Count.ShouldBeLessThanOrEqualTo(maxPages, "paging must terminate");
|
||||
}
|
||||
while (cp is not null && StatusCode.IsGood(error.StatusCode));
|
||||
return (values, pages, error.StatusCode);
|
||||
}
|
||||
|
||||
private static (IReadOnlyList<double> Values, ServiceResult Error, byte[]? Cp) ReadRaw(
|
||||
OtOpcUaNodeManager nm, NodeId nodeId, DateTime start, DateTime end, uint max, byte[]? inboundCp, bool returnBounds = false)
|
||||
{
|
||||
var context = new OperationContext(new RequestHeader(), secureChannelContext: null, RequestType.HistoryRead, identity: null);
|
||||
var details = new ReadRawModifiedDetails
|
||||
{
|
||||
ReturnBounds = returnBounds,
|
||||
IsReadModified = false,
|
||||
StartTime = start,
|
||||
EndTime = end,
|
||||
NumValuesPerNode = max,
|
||||
};
|
||||
var nodesToRead = new List<HistoryReadValueId> { new() { NodeId = nodeId, ContinuationPoint = inboundCp } };
|
||||
var results = new List<SdkHistoryReadResult> { null! };
|
||||
var errors = new List<ServiceResult> { null! };
|
||||
nm.HistoryRead(context, details, TimestampsToReturn.Both, releaseContinuationPoints: false, nodesToRead, results, errors);
|
||||
|
||||
var values = new List<double>();
|
||||
if (results[0]?.HistoryData is { } ho && !ExtensionObject.IsNull(ho))
|
||||
foreach (var dv in ((HistoryData)ExtensionObject.ToEncodeable(ho)).DataValues)
|
||||
values.Add(Convert.ToDouble(dv.Value));
|
||||
return (values, errors[0], results[0]?.ContinuationPoint);
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
if (Directory.Exists(_pkiRoot))
|
||||
{
|
||||
try { Directory.Delete(_pkiRoot, recursive: true); }
|
||||
catch { /* best-effort */ }
|
||||
}
|
||||
}
|
||||
}
|
||||
+1
-1
@@ -35,7 +35,7 @@ public sealed class GatewayEventBoundsTests
|
||||
{
|
||||
var fake = new FakeHistorianGatewayClient();
|
||||
IHistorianDataSource ds = new GatewayHistorianDataSource(fake, NullLogger<GatewayHistorianDataSource>.Instance);
|
||||
ds.SupportsReverseEvents.ShouldBeFalse("the server must keep end-only reads forward on the default backend");
|
||||
ds.SupportsReverseEvents.ShouldBeFalse("the server must walk reverse reads back from forward reads on the default backend (#691.4)");
|
||||
|
||||
await Should.ThrowAsync<NotSupportedException>(() =>
|
||||
ds.ReadEventsAsync("Pump1", DateTime.UtcNow, DateTime.UtcNow.AddHours(-1), 10, reverse: true, Ct));
|
||||
|
||||
+20
-3
@@ -29,12 +29,14 @@ public sealed class GatewayHistorianDataSourceTests
|
||||
Assert.Equal(100 + RawAggregateReader.PreWindowAllowance, fake.LastReadRawMaxValues);
|
||||
}
|
||||
|
||||
private static readonly DateTime Hour0 = new(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc);
|
||||
|
||||
[Fact]
|
||||
public async Task ReadProcessed_uses_aggregate_mode_mapping()
|
||||
{
|
||||
var fake = new FakeHistorianGatewayClient();
|
||||
var ds = new GatewayHistorianDataSource(fake, NullLogger<GatewayHistorianDataSource>.Instance);
|
||||
await ds.ReadProcessedAsync("T", default, default, TimeSpan.FromSeconds(60), HistoryAggregateType.Total, TestContext.Current.CancellationToken);
|
||||
await ds.ReadProcessedAsync("T", Hour0, Hour0.AddHours(1), TimeSpan.FromSeconds(60), HistoryAggregateType.Total, TestContext.Current.CancellationToken); // an empty window sends nothing
|
||||
Assert.Equal(RetrievalMode.Integral, fake.LastAggregateMode);
|
||||
Assert.Equal(TimeSpan.FromSeconds(60), fake.LastAggregateInterval);
|
||||
}
|
||||
@@ -62,7 +64,7 @@ public sealed class GatewayHistorianDataSourceTests
|
||||
{
|
||||
var fake = new FakeHistorianGatewayClient();
|
||||
var ds = new GatewayHistorianDataSource(fake, NullLogger<GatewayHistorianDataSource>.Instance);
|
||||
await ds.ReadProcessedAsync("T", default, default, TimeSpan.FromSeconds(60), HistoryAggregateType.TimeAverage, TestContext.Current.CancellationToken);
|
||||
await ds.ReadProcessedAsync("T", Hour0, Hour0.AddHours(1), TimeSpan.FromSeconds(60), HistoryAggregateType.TimeAverage, TestContext.Current.CancellationToken); // an empty window sends nothing
|
||||
Assert.Equal(RetrievalMode.TimeWeightedAverage, fake.LastAggregateMode);
|
||||
}
|
||||
|
||||
@@ -104,13 +106,28 @@ public sealed class GatewayHistorianDataSourceTests
|
||||
var t0 = new DateTime(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc);
|
||||
var t1 = t0.AddSeconds(1);
|
||||
fake.AtTimeSamples = new[] { new HistorianSample { NumericValue = 5.0, OpcQuality = 192, Timestamp = Timestamp.FromDateTime(t0) } };
|
||||
var ds = new GatewayHistorianDataSource(fake, NullLogger<GatewayHistorianDataSource>.Instance);
|
||||
var ds = new GatewayHistorianDataSource(fake, NullLogger<GatewayHistorianDataSource>.Instance, atTimeFromRaw: false); // #693 kill switch
|
||||
var r = await ds.ReadAtTimeAsync("T", new[] { t0, t1 }, TestContext.Current.CancellationToken);
|
||||
Assert.Equal(2, r.Samples.Count); // exactly one per requested ts, in order
|
||||
Assert.Equal(5.0, r.Samples[0].Value);
|
||||
Assert.Equal(0x80000000u, r.Samples[1].StatusCode); // gap → Bad at requested ts
|
||||
}
|
||||
|
||||
[Fact] // #690: an at-time NULL (histsdk: value 0 + Bad) is null + Bad, like the SQL backends' AtTime of a Bad sample
|
||||
public async Task ReadAtTime_of_a_NULL_row_carries_no_value()
|
||||
{
|
||||
var fake = new FakeHistorianGatewayClient();
|
||||
var t0 = new DateTime(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc);
|
||||
fake.AtTimeSamples = new[]
|
||||
{
|
||||
new HistorianSample { NumericValue = 0, OpcQuality = 0, Quality = 1, Timestamp = Timestamp.FromDateTime(t0) },
|
||||
};
|
||||
var ds = new GatewayHistorianDataSource(fake, NullLogger<GatewayHistorianDataSource>.Instance, atTimeFromRaw: false); // #693 kill switch
|
||||
var r = await ds.ReadAtTimeAsync("T", new[] { t0 }, TestContext.Current.CancellationToken);
|
||||
Assert.Null(r.Samples[0].Value);
|
||||
Assert.Equal(0x80000000u, r.Samples[0].StatusCode);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Empty_window_is_not_a_fault()
|
||||
{
|
||||
|
||||
+126
@@ -0,0 +1,126 @@
|
||||
using Google.Protobuf.WellKnownTypes;
|
||||
using Grpc.Core;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.HistorianGateway.Contracts.Grpc;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests;
|
||||
|
||||
public sealed class GatewayMissingTagTests
|
||||
{
|
||||
private static CancellationToken Ct => TestContext.Current.CancellationToken;
|
||||
private static readonly DateTime Now = new(2026, 10, 13, 8, 0, 0, DateTimeKind.Utc);
|
||||
private sealed class Clock : TimeProvider
|
||||
{
|
||||
public DateTime Current = Now;
|
||||
public override DateTimeOffset GetUtcNow() => new(Current);
|
||||
}
|
||||
private static GatewayHistorianDataSource Source(FakeHistorianGatewayClient fake, Clock? clock = null, bool native = false) =>
|
||||
new(fake, NullLogger<GatewayHistorianDataSource>.Instance, timeProvider: clock, atTimeFromRaw: !native);
|
||||
|
||||
[Theory]
|
||||
[InlineData("Raw")]
|
||||
[InlineData("AtTime")]
|
||||
[InlineData("NativeAtTime")]
|
||||
[InlineData("Count")]
|
||||
[InlineData("TimeAverage")]
|
||||
public async Task A_missing_tag_returns_no_data_without_a_history_RPC(string kind)
|
||||
{
|
||||
var fake = new FakeHistorianGatewayClient { TagExistsResult = false };
|
||||
var ds = Source(fake, native: kind == "NativeAtTime");
|
||||
var result = kind switch
|
||||
{
|
||||
"Raw" => await ds.ReadRawAsync("Missing", Now.AddMinutes(-1), Now, 10, Ct),
|
||||
"AtTime" or "NativeAtTime" => await ds.ReadAtTimeAsync("Missing", [Now], Ct),
|
||||
_ => await ds.ReadProcessedAsync("Missing", Now.AddMinutes(-1), Now, TimeSpan.FromMinutes(1),
|
||||
kind == "Count" ? HistoryAggregateType.Count : HistoryAggregateType.TimeAverage, Ct),
|
||||
};
|
||||
if (kind == "Raw") result.Samples.ShouldBeEmpty();
|
||||
else
|
||||
{
|
||||
result.Samples.Count.ShouldBe(1);
|
||||
result.Samples[0].Value.ShouldBeNull();
|
||||
(result.Samples[0].StatusCode & 0xFFFF0000u).ShouldBe(0x809B0000u);
|
||||
result.Samples[0].SourceTimestampUtc.ShouldBe(kind.Contains("AtTime") ? Now : Now.AddMinutes(-1));
|
||||
}
|
||||
fake.TagExistsCallCount.ShouldBe(1);
|
||||
(fake.ReadRawCallCount + fake.ReadAggregateCallCount + fake.ReadAtTimeCallCount).ShouldBe(0);
|
||||
ds.GetHealthSnapshot().TotalFailures.ShouldBe(0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Negative_cache_expires_so_a_newly_provisioned_tag_can_be_read()
|
||||
{
|
||||
var fake = new FakeHistorianGatewayClient { TagExistsResult = false };
|
||||
var clock = new Clock(); var ds = Source(fake, clock);
|
||||
await ds.ReadRawAsync("T", Now.AddMinutes(-1), Now, 10, Ct);
|
||||
fake.TagExistsResult = true;
|
||||
fake.RawSamples = [new HistorianSample { Timestamp = Timestamp.FromDateTime(Now.AddSeconds(-1)), NumericValue = 42, OpcQuality = 192 }];
|
||||
(await ds.ReadRawAsync("T", Now.AddMinutes(-1), Now, 10, Ct)).Samples.ShouldBeEmpty();
|
||||
fake.TagExistsCallCount.ShouldBe(1);
|
||||
clock.Current = Now.AddSeconds(5);
|
||||
(await ds.ReadRawAsync("T", Now.AddMinutes(-1), Now, 10, Ct)).Samples.Single().Value.ShouldBe(42d);
|
||||
fake.TagExistsCallCount.ShouldBe(2);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Positive_cache_avoids_checks_per_page_and_expires_after_a_minute()
|
||||
{
|
||||
var fake = new FakeHistorianGatewayClient(); var clock = new Clock(); var ds = Source(fake, clock);
|
||||
await ds.ReadRawAsync("T", Now.AddMinutes(-1), Now, 10, Ct);
|
||||
await ds.ReadRawAsync("T", Now.AddMinutes(-1), Now, 10, Ct);
|
||||
fake.TagExistsCallCount.ShouldBe(1);
|
||||
clock.Current = Now.AddSeconds(60);
|
||||
await ds.ReadRawAsync("T", Now.AddMinutes(-1), Now, 10, Ct);
|
||||
fake.TagExistsCallCount.ShouldBe(2);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task An_explicit_nested_NotFound_is_cached_as_missing()
|
||||
{
|
||||
var fake = new FakeHistorianGatewayClient { TagExistsThrows = new InvalidOperationException("wrapper",
|
||||
new RpcException(new Status(StatusCode.NotFound, "absent"))) };
|
||||
var ds = Source(fake);
|
||||
(await ds.ReadRawAsync("T", Now.AddMinutes(-1), Now, 10, Ct)).Samples.ShouldBeEmpty();
|
||||
(await ds.ReadRawAsync("T", Now.AddMinutes(-1), Now, 10, Ct)).Samples.ShouldBeEmpty();
|
||||
fake.TagExistsCallCount.ShouldBe(1);
|
||||
fake.ReadRawCallCount.ShouldBe(0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task The_presence_cache_is_bounded_and_a_displaced_entry_is_checked_again()
|
||||
{
|
||||
var fake = new FakeHistorianGatewayClient { TagExistsResult = false }; var ds = Source(fake);
|
||||
for (var i = 0; i <= GatewayTagPresenceCache.Capacity; i++)
|
||||
await ds.ReadRawAsync($"T{i}", Now.AddMinutes(-1), Now, 10, Ct);
|
||||
await ds.ReadRawAsync("T0", Now.AddMinutes(-1), Now, 10, Ct);
|
||||
fake.TagExistsCallCount.ShouldBe(GatewayTagPresenceCache.Capacity + 2);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Caller_cancellation_is_not_cached_as_missing()
|
||||
{
|
||||
var fake = new FakeHistorianGatewayClient { TagExistsThrows = new OperationCanceledException() }; var ds = Source(fake);
|
||||
await Should.ThrowAsync<OperationCanceledException>(() => ds.ReadRawAsync("T", Now.AddMinutes(-1), Now, 10, Ct));
|
||||
fake.TagExistsThrows = null;
|
||||
await ds.ReadRawAsync("T", Now.AddMinutes(-1), Now, 10, Ct);
|
||||
fake.TagExistsCallCount.ShouldBe(2);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(StatusCode.Unavailable)]
|
||||
[InlineData(StatusCode.PermissionDenied)]
|
||||
[InlineData(StatusCode.Internal)]
|
||||
public async Task A_metadata_fault_is_not_cached_as_a_missing_tag(StatusCode code)
|
||||
{
|
||||
var fake = new FakeHistorianGatewayClient { TagExistsThrows = new RpcException(new Status(code, "failure")) };
|
||||
var ds = Source(fake);
|
||||
await Should.ThrowAsync<Exception>(() => ds.ReadRawAsync("T", Now.AddMinutes(-1), Now, 10, Ct));
|
||||
fake.TagExistsThrows = null;
|
||||
await ds.ReadRawAsync("T", Now.AddMinutes(-1), Now, 10, Ct);
|
||||
fake.TagExistsCallCount.ShouldBe(2);
|
||||
fake.ReadRawCallCount.ShouldBe(1);
|
||||
}
|
||||
}
|
||||
+216
@@ -0,0 +1,216 @@
|
||||
using Google.Protobuf.WellKnownTypes;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.HistorianGateway.Contracts.Grpc;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// #691.2: a forward or reverse processed read of the AVEVA-native <c>TimeAverage</c> / <c>Total</c> on the SQL
|
||||
/// backends' interval grid. The gateway refuses <c>start > end</c>, so a reverse read is served by FORWARD native
|
||||
/// requests whose bucket grid is anchored at the request's later time (#677: window <c>(end, start]</c>, intervals
|
||||
/// stepping back from <c>start</c>, the clipped interval the earliest, each stamped with its later edge, results newest
|
||||
/// first); a forward read's clipped LATEST interval is read over exactly its span (review). The fake is a native-mode AVEVA
|
||||
/// model: a step-held stored series; for a forward request <c>[s, e)</c> with interval <c>I</c> it answers one row per
|
||||
/// interval of the grid anchored at <c>s</c> (plus AVEVA's extra row for the interval before <c>s</c>, #685), each the
|
||||
/// time-weighted average / integral of exactly that interval — the per-interval behaviour the gateway will have once
|
||||
/// historiangw#27 is fixed.
|
||||
/// </summary>
|
||||
public sealed class GatewayNativeAggregateGridTests
|
||||
{
|
||||
private static readonly CancellationToken Ct = CancellationToken.None;
|
||||
private static readonly DateTime T0 = new(2026, 10, 12, 18, 0, 0, DateTimeKind.Utc);
|
||||
|
||||
private const uint GoodCalc = 0x00000401u;
|
||||
private const uint Partial = 0x4u;
|
||||
private const uint NoDataCalc = 0x809B0401u;
|
||||
|
||||
private static DateTime At(double seconds) => T0.AddTicks((long)(seconds * TimeSpan.TicksPerSecond));
|
||||
|
||||
/// <summary>Step-held series: value changes at these offsets (s); data starts at −100 s.</summary>
|
||||
private static readonly (double T, double V)[] Series = [(-100, 1), (7, 5), (23, 2), (41, 10), (55, 4), (80, 6)];
|
||||
|
||||
/// <summary>∫ v dt (value·s) of the step-held series over [a, b); null when no data covers any of it.</summary>
|
||||
private static double? Integral(DateTime a, DateTime b)
|
||||
{
|
||||
double sum = 0;
|
||||
var covered = false;
|
||||
for (var i = 0; i < Series.Length; i++)
|
||||
{
|
||||
var from = At(Series[i].T);
|
||||
var to = i + 1 < Series.Length ? At(Series[i + 1].T) : DateTime.MaxValue;
|
||||
var lo = from > a ? from : a;
|
||||
var hi = to < b ? to : b;
|
||||
if (hi <= lo) continue;
|
||||
covered = true;
|
||||
sum += Series[i].V * (hi - lo).TotalSeconds;
|
||||
}
|
||||
return covered ? sum : null;
|
||||
}
|
||||
|
||||
private static double? Expected(RetrievalMode mode, DateTime a, DateTime b) =>
|
||||
Integral(a, b) is { } i ? mode == RetrievalMode.Integral ? i : i / (b - a).TotalSeconds : null;
|
||||
|
||||
private static FakeHistorianGatewayClient Aveva() => new()
|
||||
{
|
||||
AggregateSource = (s, e, mode, interval) =>
|
||||
{
|
||||
var rows = new List<HistorianAggregateSample>();
|
||||
for (var lo = s - interval; lo < e; lo += interval) // the pre-window interval first (#685)
|
||||
{
|
||||
var hi = lo + interval;
|
||||
var v = Expected(mode, lo, hi);
|
||||
rows.Add(new HistorianAggregateSample
|
||||
{
|
||||
Tag = "T",
|
||||
StartTime = Timestamp.FromDateTime(lo),
|
||||
EndTime = Timestamp.FromDateTime(hi),
|
||||
Value = v ?? 0,
|
||||
OpcQuality = v is null ? 0u : 192u,
|
||||
RetrievalMode = mode,
|
||||
});
|
||||
}
|
||||
return rows;
|
||||
},
|
||||
};
|
||||
|
||||
private static GatewayHistorianDataSource Source(FakeHistorianGatewayClient fake) =>
|
||||
new(fake, NullLogger<GatewayHistorianDataSource>.Instance);
|
||||
|
||||
[Theory]
|
||||
[InlineData(HistoryAggregateType.TimeAverage, RetrievalMode.TimeWeightedAverage)]
|
||||
[InlineData(HistoryAggregateType.Total, RetrievalMode.Integral)]
|
||||
public async Task A_reverse_read_over_whole_intervals_is_anchored_at_its_later_time(HistoryAggregateType agg, RetrievalMode mode)
|
||||
{
|
||||
var fake = Aveva();
|
||||
var ds = Source(fake);
|
||||
var s = (await ds.ReadProcessedAsync("T", At(60), At(0), TimeSpan.FromSeconds(20), agg, Ct)).Samples;
|
||||
|
||||
// (0, 60] in three intervals, newest first, each stamped with its LATER edge (#677).
|
||||
s.Select(x => x.SourceTimestampUtc).ShouldBe(new DateTime?[] { At(60), At(40), At(20) });
|
||||
s.Select(x => (double)x.Value!).ShouldBe(
|
||||
[Expected(mode, At(40), At(60))!.Value, Expected(mode, At(20), At(40))!.Value, Expected(mode, At(0), At(20))!.Value],
|
||||
tolerance: 1e-9);
|
||||
s.ShouldAllBe(x => x.StatusCode == GoodCalc);
|
||||
fake.AggregateCalls.ShouldBe([(At(0), At(60), mode, TimeSpan.FromSeconds(20))]);
|
||||
ds.GetHealthSnapshot().TotalFailures.ShouldBe(0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_reverse_read_with_a_partial_interval_clips_the_EARLIEST_one_and_marks_it_Partial()
|
||||
{
|
||||
var fake = Aveva();
|
||||
var s = (await Source(fake).ReadProcessedAsync("T", At(70), At(5), TimeSpan.FromSeconds(20),
|
||||
HistoryAggregateType.TimeAverage, Ct)).Samples;
|
||||
|
||||
// (5, 70]: (50,70], (30,50], (10,30], and the clipped (5,10] — the grid steps back from 70, not forward from 5.
|
||||
s.Select(x => x.SourceTimestampUtc).ShouldBe(new DateTime?[] { At(70), At(50), At(30), At(10) });
|
||||
var m = RetrievalMode.TimeWeightedAverage;
|
||||
s.Select(x => (double)x.Value!).ShouldBe(
|
||||
[Expected(m, At(50), At(70))!.Value, Expected(m, At(30), At(50))!.Value, Expected(m, At(10), At(30))!.Value,
|
||||
Expected(m, At(5), At(10))!.Value],
|
||||
tolerance: 1e-9);
|
||||
s.Select(x => x.StatusCode).ShouldBe([GoodCalc, GoodCalc, GoodCalc, GoodCalc | Partial]);
|
||||
|
||||
// Two forward requests, neither reversed: the whole intervals on the later-anchored grid, and the clipped one alone
|
||||
// (its own width as the interval, so AVEVA computes it over exactly (5, 10]).
|
||||
fake.AggregateCalls.ShouldBe([
|
||||
(At(10), At(70), m, TimeSpan.FromSeconds(20)),
|
||||
(At(5), At(10), m, TimeSpan.FromSeconds(5)),
|
||||
]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_reverse_read_narrower_than_one_interval_is_one_interval_stamped_at_its_later_time()
|
||||
{
|
||||
var fake = Aveva();
|
||||
var s = (await Source(fake).ReadProcessedAsync("T", At(30), At(12), TimeSpan.FromSeconds(60),
|
||||
HistoryAggregateType.Total, Ct)).Samples;
|
||||
s.Single().SourceTimestampUtc.ShouldBe(At(30));
|
||||
((double)s.Single().Value!).ShouldBe(Expected(RetrievalMode.Integral, At(12), At(30))!.Value, 1e-9);
|
||||
s.Single().StatusCode.ShouldBe(GoodCalc); // a whole (clamped) interval, not a truncated one
|
||||
fake.AggregateCalls.ShouldBe([(At(12), At(30), RetrievalMode.Integral, TimeSpan.FromSeconds(18))]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_zero_interval_reverse_read_is_one_interval_over_the_window()
|
||||
{
|
||||
var fake = Aveva();
|
||||
var s = (await Source(fake).ReadProcessedAsync("T", At(30), At(0), TimeSpan.Zero,
|
||||
HistoryAggregateType.TimeAverage, Ct)).Samples;
|
||||
s.Single().SourceTimestampUtc.ShouldBe(At(30));
|
||||
fake.AggregateCalls.Single().Interval.ShouldBe(TimeSpan.FromSeconds(30));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_no_data_interval_is_BadNoData_without_the_Partial_bit()
|
||||
{
|
||||
// Data starts at −100 s: (−110, −90] holds some; (−130, −110] and the clipped earliest (−135, −130] hold none.
|
||||
var s = (await Source(Aveva()).ReadProcessedAsync("T", At(-90), At(-135), TimeSpan.FromSeconds(20),
|
||||
HistoryAggregateType.TimeAverage, Ct)).Samples;
|
||||
s.Select(x => x.SourceTimestampUtc).ShouldBe(new DateTime?[] { At(-90), At(-110), At(-130) });
|
||||
s[0].StatusCode.ShouldBe(GoodCalc);
|
||||
s[1].StatusCode.ShouldBe(NoDataCalc);
|
||||
s[2].StatusCode.ShouldBe(NoDataCalc); // the clipped interval, but Bad ⇒ no Partial bit
|
||||
s[2].Value.ShouldBeNull();
|
||||
}
|
||||
|
||||
// ---- forward reads on the same grid (#691.2 review): the clipped LATEST interval is its own exact-span read ----------
|
||||
|
||||
[Theory]
|
||||
[InlineData(HistoryAggregateType.TimeAverage, RetrievalMode.TimeWeightedAverage)]
|
||||
[InlineData(HistoryAggregateType.Total, RetrievalMode.Integral)]
|
||||
public async Task A_forward_read_with_a_partial_interval_clips_the_LATEST_one_and_marks_it_Partial(HistoryAggregateType agg,
|
||||
RetrievalMode mode)
|
||||
{
|
||||
var fake = Aveva();
|
||||
var s = (await Source(fake).ReadProcessedAsync("T", At(5), At(70), TimeSpan.FromSeconds(20), agg, Ct)).Samples;
|
||||
|
||||
// [5, 70): [5,25), [25,45), [45,65) and the clipped [65,70) — stamped with each EARLIER edge, oldest first, as on
|
||||
// SqlServer/QuestDB (BucketPlan forward; the truncated interval Partial unless Bad).
|
||||
s.Select(x => x.SourceTimestampUtc).ShouldBe(new DateTime?[] { At(5), At(25), At(45), At(65) });
|
||||
s.Select(x => (double)x.Value!).ShouldBe(
|
||||
[Expected(mode, At(5), At(25))!.Value, Expected(mode, At(25), At(45))!.Value, Expected(mode, At(45), At(65))!.Value,
|
||||
Expected(mode, At(65), At(70))!.Value],
|
||||
tolerance: 1e-9);
|
||||
s.Select(x => x.StatusCode).ShouldBe([GoodCalc, GoodCalc, GoodCalc, GoodCalc | Partial]);
|
||||
// AVEVA never averages past EndTime: the whole intervals, then the clipped one over exactly its span.
|
||||
fake.AggregateCalls.ShouldBe([
|
||||
(At(5), At(65), mode, TimeSpan.FromSeconds(20)),
|
||||
(At(65), At(70), mode, TimeSpan.FromSeconds(5)),
|
||||
]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_forward_interval_wider_than_the_window_is_sent_as_the_window()
|
||||
{
|
||||
var fake = Aveva();
|
||||
var s = (await Source(fake).ReadProcessedAsync("T", At(12), At(30), TimeSpan.FromSeconds(60),
|
||||
HistoryAggregateType.Total, Ct)).Samples;
|
||||
s.Single().SourceTimestampUtc.ShouldBe(At(12));
|
||||
s.Single().StatusCode.ShouldBe(GoodCalc);
|
||||
fake.AggregateCalls.ShouldBe([(At(12), At(30), RetrievalMode.Integral, TimeSpan.FromSeconds(18))]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_zero_interval_forward_read_is_one_interval_over_the_window_not_a_zero_interval_request()
|
||||
{
|
||||
var fake = Aveva();
|
||||
var s = (await Source(fake).ReadProcessedAsync("T", At(0), At(30), TimeSpan.Zero,
|
||||
HistoryAggregateType.TimeAverage, Ct)).Samples;
|
||||
s.Single().SourceTimestampUtc.ShouldBe(At(0));
|
||||
fake.AggregateCalls.Single().Interval.ShouldBe(TimeSpan.FromSeconds(30));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task The_forward_read_is_unchanged()
|
||||
{
|
||||
var fake = Aveva();
|
||||
var s = (await Source(fake).ReadProcessedAsync("T", At(0), At(60), TimeSpan.FromSeconds(20),
|
||||
HistoryAggregateType.TimeAverage, Ct)).Samples;
|
||||
s.Select(x => x.SourceTimestampUtc).ShouldBe(new DateTime?[] { At(0), At(20), At(40) });
|
||||
fake.AggregateCalls.ShouldBe([(At(0), At(60), RetrievalMode.TimeWeightedAverage, TimeSpan.FromSeconds(20))]);
|
||||
}
|
||||
}
|
||||
+3
-1
@@ -106,9 +106,11 @@ public sealed class GatewayNativeAggregateRowsTests
|
||||
public async Task The_window_filter_is_direction_agnostic()
|
||||
{
|
||||
// A reverse request (start > end) covers the same span (end, start]; a row for the interval before the span is out.
|
||||
// #691.2: served by a forward request on the later-anchored grid, newest first, stamped with each later edge.
|
||||
var wire = new[] { Row(T0 - I20, T0, 9, 192), Row(T0, T0 + I20, 1, 192), Row(T0 + I20, T0 + 2 * I20, 2, 192) };
|
||||
var s = await Read(wire, T0 + 2 * I20, T0);
|
||||
s.Select(x => x.Value).ShouldBe(new object?[] { 1d, 2d });
|
||||
s.Select(x => x.Value).ShouldBe(new object?[] { 2d, 1d });
|
||||
s.Select(x => x.SourceTimestampUtc).ShouldBe(new DateTime?[] { T0 + 2 * I20, T0 + I20 });
|
||||
}
|
||||
|
||||
// ---- D2: an Uncertain row keeps its value ------------------------------------------------------------------------
|
||||
|
||||
+67
-5
@@ -109,6 +109,15 @@ public sealed class GatewayRawBoundaryRowTests
|
||||
r.Samples[0].StatusCode.ShouldBe(0x80000000u);
|
||||
}
|
||||
|
||||
[Fact] // #690: the wire's value 0 on a NULL row is not a value — SqlServer/QuestDB return null + Bad for the same sample
|
||||
public async Task A_stored_NULL_sample_carries_no_value_not_the_wires_zero()
|
||||
{
|
||||
var r = await Source(Gate()).ReadRawAsync("T", At(30), At(55), 0, Ct);
|
||||
r.Samples[0].SourceTimestampUtc.ShouldBe(At(40));
|
||||
r.Samples[0].Value.ShouldBeNull();
|
||||
r.Samples[1].Value.ShouldBe(5d); // a Good neighbour keeps its value
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_stored_sample_exactly_at_the_window_start_is_kept()
|
||||
{
|
||||
@@ -213,20 +222,73 @@ public sealed class GatewayRawBoundaryRowTests
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_margin_still_crowding_the_re_read_fails_the_node_as_a_request_limit()
|
||||
public async Task A_margin_crowding_a_truncated_read_is_re_read_with_growing_room()
|
||||
{
|
||||
// 200 rows in the margin: the single bounded re-read (5 + 21 + 16) is still short of the cap because of them.
|
||||
// #694 F4: 200 rows in the margin. The first read (5 + 16) saw only 21 of them, so a single re-read sized from that
|
||||
// count (5 + 21 + 16) was still short and the node failed — with no ceiling at all. The room now at least doubles per
|
||||
// re-read (21, 42, 84, 168, 336), so the window is served in a bounded number of reads.
|
||||
var margin = Enumerable.Range(0, 200).Select(i => (At(30).AddTicks(-1 - i), (double)i, false));
|
||||
var fake = Aveva(margin.Append((At(35), 1d, false)).ToList());
|
||||
var ds = Source(fake);
|
||||
|
||||
await Should.ThrowAsync<HistoryReadLimitExceededException>(() => ds.ReadRawAsync("T", At(30), At(40), 5, Ct));
|
||||
var r = await ds.ReadRawAsync("T", At(30), At(40), 5, Ct);
|
||||
|
||||
fake.ReadRawCallCount.ShouldBe(2); // one bounded re-read, never more
|
||||
fake.RawStreamsCompleted.ShouldBe(2);
|
||||
r.Samples.Select(s => s.Value).ShouldBe([1d]);
|
||||
fake.ReadRawCallCount.ShouldBe(6);
|
||||
fake.RawStreamsCompleted.ShouldBe(6);
|
||||
ds.GetHealthSnapshot().TotalFailures.ShouldBe(0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_margin_still_crowding_the_bounded_re_reads_fails_the_node_as_a_request_limit()
|
||||
{
|
||||
// 20,000 rows in the margin: past the bounded re-reads (21 · 2^8 ≈ 5,400 rows of room).
|
||||
var margin = Enumerable.Range(0, 20_000).Select(i => (At(30).AddTicks(-1 - i), (double)i, false));
|
||||
var fake = Aveva(margin.Append((At(35), 1d, false)).ToList());
|
||||
var ds = Source(fake);
|
||||
|
||||
var ex = await Should.ThrowAsync<HistoryReadLimitExceededException>(() => ds.ReadRawAsync("T", At(30), At(40), 5, Ct));
|
||||
|
||||
ex.PreWindowRows.ShouldBeGreaterThan(0);
|
||||
fake.ReadRawCallCount.ShouldBe(9); // the read + MaxCrowdedReReads, never more
|
||||
fake.RawStreamsCompleted.ShouldBe(9);
|
||||
ds.GetHealthSnapshot().TotalFailures.ShouldBe(0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task The_crowding_re_reads_never_make_room_for_more_than_the_tie_cluster_limit()
|
||||
{
|
||||
// #694 R2: the room used to double up to 8 re-reads with no cap — on a streaming gateway a pathological 1M-row
|
||||
// single-millisecond cluster streamed ~4M rows for one page where it used to fail at once. The room is capped at
|
||||
// MaxTieClusterOverfetch (here 100); past it the read is a request limit, after a bounded number of bounded reads.
|
||||
var margin = Enumerable.Range(0, 150).Select(i => (At(30).AddTicks(-1 - i), (double)i, false));
|
||||
var fake = Aveva(margin.Append((At(35), 1d, false)).ToList());
|
||||
var ds = new GatewayHistorianDataSource(fake, NullLogger<GatewayHistorianDataSource>.Instance, maxPreWindowRows: 100);
|
||||
|
||||
var ex = await Should.ThrowAsync<HistoryReadLimitExceededException>(() => ds.ReadRawAsync("T", At(30), At(40), 5, Ct));
|
||||
|
||||
ex.PreWindowRows.ShouldBeGreaterThan(100);
|
||||
// Room 21, 42, 84: that read already saw more than 100 rows before the window, so it stops — never asking past the cap.
|
||||
fake.ReadRawCallCount.ShouldBe(4);
|
||||
fake.RawCalls.Max(c => c.MaxValues).ShouldBeLessThanOrEqualTo(5 + 100 + RawAggregateReader.PreWindowAllowance);
|
||||
fake.RawYieldedCount.ShouldBeLessThanOrEqualTo(21 + 42 + 63 + 105);
|
||||
fake.RawStreamsCompleted.ShouldBe(4);
|
||||
ds.GetHealthSnapshot().TotalFailures.ShouldBe(0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Crowding_rows_just_under_the_tie_cluster_limit_are_read_around()
|
||||
{
|
||||
var margin = Enumerable.Range(0, 90).Select(i => (At(30).AddTicks(-1 - i), (double)i, false));
|
||||
var fake = Aveva(margin.Append((At(35), 1d, false)).ToList());
|
||||
var ds = new GatewayHistorianDataSource(fake, NullLogger<GatewayHistorianDataSource>.Instance, maxPreWindowRows: 100);
|
||||
|
||||
var r = await ds.ReadRawAsync("T", At(30), At(40), 5, Ct);
|
||||
|
||||
r.Samples.Select(s => s.Value).ShouldBe([1d]);
|
||||
fake.RawCalls.Max(c => c.MaxValues).ShouldBeLessThanOrEqualTo(5 + 100 + RawAggregateReader.PreWindowAllowance);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_short_read_is_complete_and_is_not_re_read()
|
||||
{
|
||||
|
||||
+76
@@ -0,0 +1,76 @@
|
||||
using Google.Protobuf.WellKnownTypes;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.HistorianGateway.Contracts.Grpc;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.OpcUaServer;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #691.4 review: events read uncapped (and walked back newest-first) through the REAL
|
||||
/// <see cref="GatewayHistorianDataSource"/> over a gateway-shaped fake — inclusive <c>[start, end]</c>, ordered by event time.
|
||||
/// </summary>
|
||||
public sealed class GatewayReverseEventsTests
|
||||
{
|
||||
private static CancellationToken Ct => TestContext.Current.CancellationToken;
|
||||
|
||||
private static readonly DateTime T0 = new(2026, 10, 12, 18, 0, 0, DateTimeKind.Utc);
|
||||
|
||||
private static FakeHistorianGatewayClient Store(int count)
|
||||
{
|
||||
var stored = Enumerable.Range(0, count)
|
||||
.Select(i => new HistorianEvent { Id = $"e{i:0000}", Type = "Alarm", EventTime = Timestamp.FromDateTime(T0.AddSeconds(i)) })
|
||||
.ToList();
|
||||
return new FakeHistorianGatewayClient
|
||||
{
|
||||
EventsSource = (start, end) => stored.Where(e => e.EventTime.ToDateTime() >= start && e.EventTime.ToDateTime() <= end),
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// <c>NumValuesPerNode = 0</c> reaches the source as <c>int.MaxValue</c>; the result list used to be pre-sized to it
|
||||
/// ("Array dimensions exceeded") — every uncapped Gateway events read failed and counted as a backend failure.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task An_uncapped_forward_events_read_is_served()
|
||||
{
|
||||
var fake = Store(30);
|
||||
var ds = new GatewayHistorianDataSource(fake, NullLogger<GatewayHistorianDataSource>.Instance);
|
||||
|
||||
var r = await ds.ReadEventsAsync(null, T0, T0.AddMinutes(1), int.MaxValue, Ct);
|
||||
|
||||
r.Events.Count.ShouldBe(30);
|
||||
r.ContinuationPoint.ShouldBeNull();
|
||||
ds.GetHealthSnapshot().TotalFailures.ShouldBe(0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task An_uncapped_reverse_events_read_is_one_forward_read_newest_first()
|
||||
{
|
||||
var fake = Store(30);
|
||||
var ds = new GatewayHistorianDataSource(fake, NullLogger<GatewayHistorianDataSource>.Instance);
|
||||
|
||||
var page = await ReverseEventsReader.ReadAsync(ds, null, hi: T0.AddSeconds(25), lo: T0.AddSeconds(5), want: 0,
|
||||
maxTieCluster: 65536, nowUtc: T0.AddHours(1), hint: default, Ct);
|
||||
|
||||
page.Events.Select(e => e.EventId).ShouldBe(Enumerable.Range(6, 20).Reverse().Select(i => $"e{i:0000}"));
|
||||
fake.ReadEventsCallCount.ShouldBe(1);
|
||||
ds.GetHealthSnapshot().TotalFailures.ShouldBe(0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_capped_reverse_events_read_walked_back_over_the_gateway_returns_the_latest_newest_first()
|
||||
{
|
||||
var fake = Store(300);
|
||||
var ds = new GatewayHistorianDataSource(fake, NullLogger<GatewayHistorianDataSource>.Instance);
|
||||
|
||||
var page = await ReverseEventsReader.ReadAsync(ds, null, hi: T0.AddSeconds(200), lo: T0.AddSeconds(50), want: 10,
|
||||
maxTieCluster: 65536, nowUtc: T0.AddHours(1), hint: default, Ct);
|
||||
|
||||
page.Events.Select(e => e.EventId).ShouldBe(Enumerable.Range(191, 10).Reverse().Select(i => $"e{i:0000}"));
|
||||
fake.LastReadEventsStartUtc.ShouldBeLessThanOrEqualTo(fake.LastReadEventsEndUtc);
|
||||
ds.GetHealthSnapshot().TotalFailures.ShouldBe(0);
|
||||
}
|
||||
}
|
||||
+314
@@ -0,0 +1,314 @@
|
||||
using Google.Protobuf.WellKnownTypes;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.HistorianGateway.Contracts.Grpc;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// #691.3: a gateway with <c>Historian:ReadLimits:AllowStreamingBeyondCeiling=false</c> refuses any <c>ReadRaw</c> whose
|
||||
/// <c>max_values</c> exceeds its <c>MaxValuesCeiling</c> (before streaming, naming the ceiling). An uncapped raw read
|
||||
/// (<c>NumValuesPerNode = 0</c>, sent as <see cref="int.MaxValue"/>) used to be refused outright — even a 10-row window.
|
||||
/// The read is now re-issued once with the ceiling the refusal named, and a stream that delivered the whole ceiling
|
||||
/// without reaching the window end is never answered from (<c>BadTooManyOperations</c>). The fake is AVEVA-shaped:
|
||||
/// a boundary row at the query start, <c>[start, end]</c> inclusive, capped at <c>max</c>.
|
||||
/// </summary>
|
||||
public sealed class GatewayStrictCeilingTests
|
||||
{
|
||||
private static readonly CancellationToken Ct = CancellationToken.None;
|
||||
private static readonly DateTime T0 = new(2026, 10, 12, 19, 0, 0, DateTimeKind.Utc);
|
||||
private static readonly DateTime Now = T0.AddDays(1);
|
||||
|
||||
private static DateTime At(int seconds) => T0.AddSeconds(seconds);
|
||||
|
||||
private static HistorianSample Row(DateTime t, double v, uint quality = 0) =>
|
||||
new() { Tag = "T", Timestamp = Timestamp.FromDateTime(t), NumericValue = v, OpcQuality = 192, Quality = quality };
|
||||
|
||||
/// <summary>One stored sample per second, value = seconds, at +0 … +(count − 1); strict ceiling when given.</summary>
|
||||
private static FakeHistorianGatewayClient Store(int count, int? ceiling)
|
||||
{
|
||||
var stored = Enumerable.Range(0, count).Select(i => (T: At(i), V: (double)i)).ToList();
|
||||
return new FakeHistorianGatewayClient
|
||||
{
|
||||
MaxValuesCeiling = ceiling,
|
||||
RawSource = (start, end, max) =>
|
||||
{
|
||||
var rows = new List<HistorianSample>();
|
||||
if (stored.All(s => s.T != start))
|
||||
{
|
||||
var prior = stored.LastOrDefault(s => s.T < start);
|
||||
rows.Add(prior.T == default
|
||||
? new HistorianSample { Tag = "T", Timestamp = Timestamp.FromDateTime(start), NumericValue = 0, OpcQuality = 0, Quality = 1 }
|
||||
: Row(start, prior.V, 133));
|
||||
}
|
||||
rows.AddRange(stored.Where(s => s.T >= start && s.T <= end).Select(s => Row(s.T, s.V)));
|
||||
return rows.Take(max);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
private static GatewayHistorianDataSource Source(FakeHistorianGatewayClient fake, int maxRawRows = 1_000_000) =>
|
||||
new(fake, NullLogger<GatewayHistorianDataSource>.Instance, maxRawRows, new FixedClock(Now));
|
||||
|
||||
private sealed class FixedClock(DateTime now) : TimeProvider
|
||||
{
|
||||
public override DateTimeOffset GetUtcNow() => new(now);
|
||||
}
|
||||
|
||||
private sealed class MovingClock : TimeProvider
|
||||
{
|
||||
public DateTime Current = Now;
|
||||
public override DateTimeOffset GetUtcNow() => new(Current);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Aggregate_ceiling_re_probes_wait_thirty_seconds_then_notice_a_raised_limit()
|
||||
{
|
||||
var fake = Store(100, ceiling: 50);
|
||||
var clock = new MovingClock();
|
||||
var ds = new GatewayHistorianDataSource(fake, NullLogger<GatewayHistorianDataSource>.Instance, timeProvider: clock);
|
||||
Task<HistoryReadResult> Read() => ds.ReadProcessedAsync("T", At(0), At(100), TimeSpan.FromSeconds(100), HistoryAggregateType.Count, Ct);
|
||||
await Should.ThrowAsync<HistoryReadLimitExceededException>(Read);
|
||||
await Should.ThrowAsync<HistoryReadLimitExceededException>(Read);
|
||||
Refusals(fake).ShouldBe(1);
|
||||
fake.MaxValuesCeiling = 200;
|
||||
await Should.ThrowAsync<HistoryReadLimitExceededException>(Read);
|
||||
Refusals(fake).ShouldBe(1);
|
||||
clock.Current = Now.AddSeconds(30);
|
||||
(await Read()).Samples.Single().Value.ShouldBe(100);
|
||||
ds.MaxRawValuesPerRead.ShouldBe(200 - RawAggregateReader.PreWindowAllowance);
|
||||
}
|
||||
|
||||
// ---- raw, uncapped ---------------------------------------------------------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public async Task An_uncapped_read_of_a_small_window_on_a_strict_gateway_is_served_with_the_ceiling_as_budget()
|
||||
{
|
||||
var fake = Store(10, ceiling: 1000);
|
||||
var ds = Source(fake);
|
||||
var r = await ds.ReadRawAsync("T", At(0), At(60), 0, Ct);
|
||||
|
||||
r.Samples.Select(s => s.Value).ShouldBe(Enumerable.Range(0, 10).Select(i => (object?)(double)i));
|
||||
fake.RawCalls.Select(c => c.MaxValues).ShouldBe([int.MaxValue, 1000]); // refused, then the ceiling it named
|
||||
fake.RawStreamsCompleted.ShouldBe(1);
|
||||
ds.GetHealthSnapshot().TotalFailures.ShouldBe(0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task An_uncapped_read_on_a_default_gateway_still_sends_one_unbounded_request()
|
||||
{
|
||||
var fake = Store(10, ceiling: null);
|
||||
await Source(fake).ReadRawAsync("T", At(0), At(60), 0, Ct);
|
||||
fake.RawCalls.Select(c => c.MaxValues).ShouldBe([int.MaxValue]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task An_uncapped_window_holding_more_than_the_ceiling_is_a_request_limit_never_a_truncated_answer()
|
||||
{
|
||||
var fake = Store(100, ceiling: 50);
|
||||
var ds = Source(fake);
|
||||
await Should.ThrowAsync<HistoryReadLimitExceededException>(() => ds.ReadRawAsync("T", At(0), At(200), 0, Ct));
|
||||
fake.RawStreamsCompleted.ShouldBe(1); // drained, never abandoned
|
||||
ds.GetHealthSnapshot().TotalFailures.ShouldBe(0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_ceiling_full_stream_that_reached_the_window_end_is_complete()
|
||||
{
|
||||
// 50 rows: the boundary row + +0 … +48; the window end is +48 (the gateway's end is inclusive), so the stream
|
||||
// provably ended at the window end — complete although it delivered the whole ceiling.
|
||||
var fake = Store(100, ceiling: 50);
|
||||
var r = await Source(fake).ReadRawAsync("T", At(0), At(48), 0, Ct);
|
||||
r.Samples.Count.ShouldBe(48); // [+0, +48): the end is exclusive
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_refusal_without_a_parseable_ceiling_retries_at_the_gateways_default_ceiling()
|
||||
{
|
||||
var fake = Store(10, ceiling: 1_000_000);
|
||||
fake.CeilingRejectionDetail = "max_values exceeds the server ceiling; read large ranges by time-slicing.";
|
||||
var r = await Source(fake).ReadRawAsync("T", At(0), At(60), 0, Ct);
|
||||
r.Samples.Count.ShouldBe(10);
|
||||
fake.RawCalls.Select(c => c.MaxValues).ShouldBe([int.MaxValue, RawAggregateReader.GatewayDefaultMaxValuesCeiling]);
|
||||
}
|
||||
|
||||
// ---- the learned ceiling (review: a refused attempt counts as an error in the gateway's own metrics) -----------------
|
||||
|
||||
private static int Refusals(FakeHistorianGatewayClient fake) =>
|
||||
fake.RawCalls.Count(c => fake.MaxValuesCeiling is { } n && c.MaxValues > n);
|
||||
|
||||
[Fact]
|
||||
public async Task The_ceiling_is_learned_once_per_data_source_so_later_reads_are_not_refused()
|
||||
{
|
||||
var fake = Store(10, ceiling: 1000);
|
||||
var ds = Source(fake);
|
||||
await ds.ReadRawAsync("T", At(0), At(60), 0, Ct);
|
||||
await ds.ReadRawAsync("T", At(0), At(60), 0, Ct);
|
||||
await ds.ReadRawAsync("T", At(0), At(60), 995, Ct);
|
||||
await ds.ReadProcessedAsync("T", At(0), At(10), TimeSpan.FromSeconds(10), HistoryAggregateType.Count, Ct);
|
||||
|
||||
fake.RawCalls.Select(c => c.MaxValues).Take(4).ShouldBe([int.MaxValue, 1000, 1000, 1000]);
|
||||
fake.RawCalls[4].MaxValues.ShouldBe(1000); // the per-sample read shares it (its own budget 1,000,000)
|
||||
Refusals(fake).ShouldBe(1);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_gateway_that_now_streams_is_re_probed_when_a_learned_ceiling_read_comes_back_full()
|
||||
{
|
||||
var fake = Store(100, ceiling: 50);
|
||||
var ds = Source(fake);
|
||||
await ds.ReadRawAsync("T", At(0), At(5), 0, Ct); // learns 50
|
||||
fake.MaxValuesCeiling = null; // the operator enabled AllowStreamingBeyondCeiling
|
||||
|
||||
var r = await ds.ReadRawAsync("T", At(0), At(200), 0, Ct);
|
||||
r.Samples.Count.ShouldBe(100); // not BadTooManyOperations from a stale ceiling
|
||||
fake.RawCalls.Skip(2).Select(c => c.MaxValues).ShouldBe([50, int.MaxValue]);
|
||||
|
||||
await ds.ReadRawAsync("T", At(0), At(5), 0, Ct);
|
||||
fake.RawCalls[^1].MaxValues.ShouldBe(int.MaxValue); // forgotten
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_raised_but_still_strict_ceiling_is_re_learned_from_the_re_probe()
|
||||
{
|
||||
var fake = Store(100, ceiling: 50);
|
||||
var ds = Source(fake);
|
||||
await ds.ReadRawAsync("T", At(0), At(5), 0, Ct);
|
||||
fake.MaxValuesCeiling = 500;
|
||||
|
||||
(await ds.ReadRawAsync("T", At(0), At(200), 0, Ct)).Samples.Count.ShouldBe(100);
|
||||
fake.RawCalls.Skip(2).Select(c => c.MaxValues).ShouldBe([50, int.MaxValue, 500]);
|
||||
|
||||
await ds.ReadRawAsync("T", At(0), At(5), 0, Ct);
|
||||
fake.RawCalls[^1].MaxValues.ShouldBe(500);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_genuinely_over_ceiling_read_costs_one_re_probe_and_no_repeat_at_the_same_ceiling()
|
||||
{
|
||||
var fake = Store(100, ceiling: 50);
|
||||
var ds = Source(fake);
|
||||
await ds.ReadRawAsync("T", At(0), At(5), 0, Ct);
|
||||
await Should.ThrowAsync<HistoryReadLimitExceededException>(() => ds.ReadRawAsync("T", At(0), At(200), 0, Ct));
|
||||
fake.RawCalls.Skip(2).Select(c => c.MaxValues).ShouldBe([50, int.MaxValue]);
|
||||
ds.GetHealthSnapshot().TotalFailures.ShouldBe(0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_lowered_ceiling_is_re_learned_from_the_refusal_of_the_learned_one()
|
||||
{
|
||||
var fake = Store(10, ceiling: 1000);
|
||||
var ds = Source(fake);
|
||||
await ds.ReadRawAsync("T", At(0), At(60), 0, Ct);
|
||||
fake.MaxValuesCeiling = 50;
|
||||
|
||||
(await ds.ReadRawAsync("T", At(0), At(60), 0, Ct)).Samples.Count.ShouldBe(10);
|
||||
await ds.ReadRawAsync("T", At(0), At(60), 0, Ct);
|
||||
fake.RawCalls.Skip(2).Select(c => c.MaxValues).ShouldBe([1000, 50, 50]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Concurrent_reads_share_the_learned_ceiling_safely()
|
||||
{
|
||||
var fake = Store(10, ceiling: 1000);
|
||||
var ds = Source(fake);
|
||||
await ds.ReadRawAsync("T", At(0), At(60), 0, Ct);
|
||||
var results = await Task.WhenAll(Enumerable.Range(0, 32).Select(_ => Task.Run(() => ds.ReadRawAsync("T", At(0), At(60), 0, Ct))));
|
||||
results.ShouldAllBe(r => r.Samples.Count == 10);
|
||||
Refusals(fake).ShouldBe(1);
|
||||
}
|
||||
|
||||
// ---- raw, capped -----------------------------------------------------------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public async Task A_cap_whose_allowance_exceeds_the_ceiling_is_served_with_the_ceiling_as_budget()
|
||||
{
|
||||
// cap 995 + 16 = 1011 > 1000: refused, re-asked with 1000 — a small window is served in full.
|
||||
var fake = Store(10, ceiling: 1000);
|
||||
var r = await Source(fake).ReadRawAsync("T", At(0), At(60), 995, Ct);
|
||||
r.Samples.Count.ShouldBe(10);
|
||||
fake.RawCalls.Select(c => c.MaxValues).ShouldBe([995 + RawAggregateReader.PreWindowAllowance, 1000]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_cap_above_the_ceiling_over_a_dense_window_is_a_request_limit_not_a_short_page()
|
||||
{
|
||||
// A short page would end the client's paging early (silent truncation), so it must fail instead.
|
||||
var fake = Store(100, ceiling: 50);
|
||||
var ds = Source(fake);
|
||||
await Should.ThrowAsync<HistoryReadLimitExceededException>(() => ds.ReadRawAsync("T", At(0), At(200), 80, Ct));
|
||||
fake.RawCalls.Count.ShouldBe(2); // the ceiling re-read is final: no third read at the same ceiling
|
||||
ds.GetHealthSnapshot().TotalFailures.ShouldBe(0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_cap_that_fits_the_ceiling_is_unaffected()
|
||||
{
|
||||
var fake = Store(100, ceiling: 50);
|
||||
var r = await Source(fake).ReadRawAsync("T", At(0), At(200), 10, Ct);
|
||||
r.Samples.Count.ShouldBe(10);
|
||||
fake.RawCalls.Select(c => c.MaxValues).ShouldBe([10 + RawAggregateReader.PreWindowAllowance]);
|
||||
}
|
||||
|
||||
// ---- per-sample aggregates (#688 clamp vs a ceiling below 1,000,000) ---------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public async Task A_per_sample_aggregate_on_a_gateway_with_a_lower_ceiling_is_served_when_the_window_fits()
|
||||
{
|
||||
var fake = Store(30, ceiling: 1000); // default cap 1,000,000 ⇒ budget 1,000,000 > 1000
|
||||
var s = (await Source(fake).ReadProcessedAsync("T", At(0), At(30), TimeSpan.FromSeconds(10),
|
||||
HistoryAggregateType.Count, Ct)).Samples;
|
||||
s.Select(x => x.Value).ShouldBe([10, 10, 10]);
|
||||
fake.RawCalls[0].MaxValues.ShouldBe(RawAggregateReader.GatewayDefaultMaxValuesCeiling);
|
||||
fake.RawCalls[1].MaxValues.ShouldBe(1000);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_per_sample_aggregate_whose_window_fills_the_lower_ceiling_is_a_request_limit()
|
||||
{
|
||||
var fake = Store(100, ceiling: 50);
|
||||
var ds = Source(fake);
|
||||
await Should.ThrowAsync<HistoryReadLimitExceededException>(() => ds.ReadProcessedAsync("T", At(0), At(100),
|
||||
TimeSpan.FromSeconds(10), HistoryAggregateType.Count, Ct));
|
||||
ds.GetHealthSnapshot().TotalFailures.ShouldBe(0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task An_unbounded_per_sample_aggregate_on_a_strict_gateway_is_served_with_the_ceiling()
|
||||
{
|
||||
var fake = Store(30, ceiling: 1000);
|
||||
var s = (await Source(fake, maxRawRows: 0).ReadProcessedAsync("T", At(0), At(30), TimeSpan.FromSeconds(10),
|
||||
HistoryAggregateType.Count, Ct)).Samples;
|
||||
s.Select(x => x.Value).ShouldBe([10, 10, 10]);
|
||||
fake.RawCalls.Take(2).Select(c => c.MaxValues).ShouldBe([int.MaxValue, 1000]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task An_unbounded_per_sample_aggregate_filling_the_ceiling_is_a_request_limit()
|
||||
{
|
||||
var fake = Store(100, ceiling: 50);
|
||||
await Should.ThrowAsync<HistoryReadLimitExceededException>(() => Source(fake, maxRawRows: 0).ReadProcessedAsync("T",
|
||||
At(0), At(100), TimeSpan.FromSeconds(10), HistoryAggregateType.Count, Ct));
|
||||
}
|
||||
|
||||
[Theory] // review: the message names the limit that actually bound the read — the gateway's ceiling, not the cap
|
||||
[InlineData(0)] // unbounded cap: used to say "more than 0 raw samples (MaxTimeWeightedRawRows)"
|
||||
[InlineData(1_000_000)] // default cap: used to cite 1,000,000
|
||||
public async Task A_ceiling_bound_per_sample_read_names_the_gateway_ceiling(int maxRawRows)
|
||||
{
|
||||
var ex = await Should.ThrowAsync<HistoryReadLimitExceededException>(() => Source(Store(100, ceiling: 50), maxRawRows)
|
||||
.ReadProcessedAsync("T", At(0), At(100), TimeSpan.FromSeconds(10), HistoryAggregateType.Count, Ct));
|
||||
ex.Message.ShouldContain("row ceiling (50,");
|
||||
ex.Message.ShouldNotContain("MaxTimeWeightedRawRows); the Gateway");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_cap_bound_per_sample_read_still_names_the_cap()
|
||||
{
|
||||
var ex = await Should.ThrowAsync<HistoryReadLimitExceededException>(() => Source(Store(100, ceiling: null), 20)
|
||||
.ReadProcessedAsync("T", At(0), At(100), TimeSpan.FromSeconds(10), HistoryAggregateType.Count, Ct));
|
||||
ex.Message.ShouldContain("more than 20 raw samples");
|
||||
}
|
||||
}
|
||||
+90
@@ -0,0 +1,90 @@
|
||||
using System.Net;
|
||||
using Grpc.Core;
|
||||
using Microsoft.AspNetCore.Builder;
|
||||
using Microsoft.AspNetCore.Hosting;
|
||||
using Microsoft.AspNetCore.Server.Kestrel.Core;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.HistorianGateway.Contracts.Grpc;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Historian;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests;
|
||||
|
||||
public sealed class GatewayTagCatalogWireTests
|
||||
{
|
||||
private static CancellationToken Ct => TestContext.Current.CancellationToken;
|
||||
|
||||
[Theory]
|
||||
[InlineData("Pump1", true)]
|
||||
[InlineData("Missing", false)]
|
||||
[InlineData("Pump*", false)]
|
||||
public async Task Presence_is_proved_by_exact_names_without_using_the_broken_metadata_parser(string tag, bool expected)
|
||||
{
|
||||
var service = new TagsService();
|
||||
await using var server = await StartAsync(service);
|
||||
using var adapter = Create(server.Port);
|
||||
(await adapter.TagExistsAsync(tag, Ct)).ShouldBe(expected);
|
||||
service.Filter.ShouldBe(tag);
|
||||
service.MetadataCalls.ShouldBe(0);
|
||||
service.Completed.ShouldBe(1);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_catalog_stream_fault_after_a_matching_name_is_not_cached_as_presence()
|
||||
{
|
||||
var service = new TagsService { FailAfterRow = true };
|
||||
await using var server = await StartAsync(service);
|
||||
using var adapter = Create(server.Port);
|
||||
var source = new GatewayHistorianDataSource(adapter, NullLogger<GatewayHistorianDataSource>.Instance);
|
||||
var now = DateTime.UtcNow;
|
||||
await Should.ThrowAsync<Exception>(() => source.ReadRawAsync("Pump1", now.AddMinutes(-1), now, 10, Ct));
|
||||
service.FailAfterRow = false; service.Names = [];
|
||||
(await source.ReadRawAsync("Pump1", now.AddMinutes(-1), now, 10, Ct)).Samples.ShouldBeEmpty();
|
||||
service.BrowseCalls.ShouldBe(2);
|
||||
}
|
||||
|
||||
private static HistorianGatewayClientAdapter Create(int port) => HistorianGatewayClientAdapter.Create(
|
||||
new ServerHistorianOptions { Endpoint = $"http://127.0.0.1:{port}", ApiKey = "histgw_test_key", UseTls = false },
|
||||
NullLoggerFactory.Instance);
|
||||
|
||||
private sealed class TagsService : HistorianTags.HistorianTagsBase
|
||||
{
|
||||
public int MetadataCalls, BrowseCalls, Completed;
|
||||
public string? Filter;
|
||||
public bool FailAfterRow;
|
||||
public string[] Names = ["Pump1", "Pump2"];
|
||||
public override Task<GetTagMetadataReply> GetTagMetadata(GetTagMetadataRequest request, ServerCallContext context)
|
||||
{
|
||||
MetadataCalls++;
|
||||
throw new RpcException(new Status(StatusCode.Internal, "GetTagInfo response ended unexpectedly."));
|
||||
}
|
||||
public override async Task BrowseTagNames(BrowseTagNamesRequest request, IServerStreamWriter<TagName> stream, ServerCallContext context)
|
||||
{
|
||||
BrowseCalls++; Filter = request.Filter;
|
||||
foreach (var name in Names)
|
||||
{
|
||||
await stream.WriteAsync(new TagName { Name = name }, context.CancellationToken);
|
||||
if (FailAfterRow) throw new RpcException(new Status(StatusCode.Internal, "catalog failure"));
|
||||
}
|
||||
Completed++;
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class Server(WebApplication app, int port) : IAsyncDisposable
|
||||
{
|
||||
public int Port => port;
|
||||
public async ValueTask DisposeAsync() { await app.StopAsync(); await app.DisposeAsync(); }
|
||||
}
|
||||
private static async Task<Server> StartAsync(TagsService service)
|
||||
{
|
||||
var builder = WebApplication.CreateSlimBuilder();
|
||||
builder.Logging.ClearProviders();
|
||||
builder.WebHost.ConfigureKestrel(k => k.Listen(IPAddress.Loopback, 0, o => o.Protocols = HttpProtocols.Http2));
|
||||
builder.Services.AddGrpc(); builder.Services.AddSingleton(service);
|
||||
var app = builder.Build(); app.MapGrpcService<TagsService>(); await app.StartAsync();
|
||||
return new Server(app, new Uri(app.Urls.First()).Port);
|
||||
}
|
||||
}
|
||||
+209
@@ -0,0 +1,209 @@
|
||||
using Grpc.Core;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests;
|
||||
|
||||
public sealed class GatewayTagPresenceCacheConcurrencyTests(ITestOutputHelper output)
|
||||
{
|
||||
private static CancellationToken Ct => TestContext.Current.CancellationToken;
|
||||
private const int BurstSize = 16;
|
||||
|
||||
private sealed class Clock : TimeProvider
|
||||
{
|
||||
public DateTimeOffset Current = new(2026, 10, 13, 8, 0, 0, TimeSpan.Zero);
|
||||
public override DateTimeOffset GetUtcNow() => Current;
|
||||
}
|
||||
|
||||
private static TaskCompletionSource<bool> Gate() => new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
|
||||
[Theory]
|
||||
[InlineData(false)]
|
||||
[InlineData(true)]
|
||||
public async Task Overlapping_cold_lookups_return_the_catalog_result_and_populate_the_cache(bool exists)
|
||||
{
|
||||
// Catches dropped/corrupted concurrent writes or a result cached with the wrong presence value.
|
||||
var gate = Gate();
|
||||
var fake = new FakeHistorianGatewayClient { TagExistsSource = (_, _) => gate.Task };
|
||||
var cache = new GatewayTagPresenceCache(fake, new Clock());
|
||||
var reads = Enumerable.Range(0, BurstSize).Select(_ => cache.ExistsAsync("T", Ct)).ToArray();
|
||||
reads.ShouldAllBe(read => !read.IsCompleted);
|
||||
var rpcCount = fake.TagExistsCallCount;
|
||||
// Measure amplification without requiring either coalescing or one RPC per caller.
|
||||
rpcCount.ShouldBeInRange(1, BurstSize);
|
||||
output.WriteLine($"Cold {exists}: {BurstSize} overlapping callers, {rpcCount} catalog RPCs.");
|
||||
gate.SetResult(exists);
|
||||
(await Task.WhenAll(reads).WaitAsync(Ct)).ShouldAllBe(value => value == exists);
|
||||
|
||||
fake.TagExistsSource = null;
|
||||
fake.TagExistsThrows = new InvalidOperationException("A warm lookup must not contact the catalog.");
|
||||
(await cache.ExistsAsync("T", Ct)).ShouldBe(exists);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(false, 5)]
|
||||
[InlineData(true, 60)]
|
||||
public async Task Overlapping_expired_lookups_refresh_presence_at_the_TTL_boundary(bool initial, int ttlSeconds)
|
||||
{
|
||||
// Catches an inclusive expiry comparison or failure to replace an expired positive/negative entry.
|
||||
var clock = new Clock();
|
||||
var fake = new FakeHistorianGatewayClient { TagExistsResult = initial };
|
||||
var cache = new GatewayTagPresenceCache(fake, clock);
|
||||
(await cache.ExistsAsync("T", Ct)).ShouldBe(initial);
|
||||
var expires = clock.Current.AddSeconds(ttlSeconds);
|
||||
clock.Current = expires.AddTicks(-1);
|
||||
fake.TagExistsThrows = new InvalidOperationException("The entry must still be warm before expiry.");
|
||||
(await cache.ExistsAsync("T", Ct)).ShouldBe(initial);
|
||||
|
||||
clock.Current = expires;
|
||||
var gate = Gate();
|
||||
fake.TagExistsThrows = null;
|
||||
fake.TagExistsSource = (_, _) => gate.Task;
|
||||
var reads = Enumerable.Range(0, BurstSize).Select(_ => cache.ExistsAsync("T", Ct)).ToArray();
|
||||
reads.ShouldAllBe(read => !read.IsCompleted);
|
||||
var rpcCount = fake.TagExistsCallCount - 1;
|
||||
rpcCount.ShouldBeInRange(1, BurstSize);
|
||||
output.WriteLine($"Expired {initial}: {BurstSize} overlapping callers, {rpcCount} catalog RPCs.");
|
||||
gate.SetResult(!initial);
|
||||
(await Task.WhenAll(reads).WaitAsync(Ct)).ShouldAllBe(value => value == !initial);
|
||||
|
||||
fake.TagExistsSource = null;
|
||||
fake.TagExistsThrows = new InvalidOperationException("A refreshed lookup must stay cached.");
|
||||
(await cache.ExistsAsync("T", Ct)).ShouldBe(!initial);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(false)]
|
||||
[InlineData(true)]
|
||||
public async Task Overlapping_distinct_tag_completions_keep_the_cache_bounded(bool exists)
|
||||
{
|
||||
// Catches racing capacity checks that allow more than Capacity resident entries.
|
||||
var fake = new FakeHistorianGatewayClient { TagExistsResult = exists };
|
||||
var cache = new GatewayTagPresenceCache(fake, new Clock());
|
||||
var tags = Enumerable.Range(0, GatewayTagPresenceCache.Capacity + 31).Select(i => $"T{i}").ToArray();
|
||||
foreach (var tag in tags.Take(GatewayTagPresenceCache.Capacity - 1))
|
||||
(await cache.ExistsAsync(tag, Ct)).ShouldBe(exists);
|
||||
|
||||
var gate = Gate();
|
||||
fake.TagExistsSource = (_, _) => gate.Task;
|
||||
var reads = tags.Skip(GatewayTagPresenceCache.Capacity - 1).Select(tag => cache.ExistsAsync(tag, Ct)).ToArray();
|
||||
reads.ShouldAllBe(read => !read.IsCompleted);
|
||||
gate.SetResult(exists);
|
||||
(await Task.WhenAll(reads).WaitAsync(Ct)).ShouldAllBe(value => value == exists);
|
||||
|
||||
// Failed probes cannot insert or evict entries, so count resident hits through the public behavior.
|
||||
fake.TagExistsSource = null;
|
||||
var uncached = new InvalidOperationException("Uncached catalog probe.");
|
||||
fake.TagExistsThrows = uncached;
|
||||
var hits = 0;
|
||||
foreach (var tag in tags)
|
||||
{
|
||||
try { (await cache.ExistsAsync(tag, Ct)).ShouldBe(exists); hits++; }
|
||||
catch (InvalidOperationException error) when (ReferenceEquals(error, uncached)) { }
|
||||
}
|
||||
hits.ShouldBe(GatewayTagPresenceCache.Capacity);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(StatusCode.Unavailable)]
|
||||
[InlineData(StatusCode.PermissionDenied)]
|
||||
[InlineData(StatusCode.Internal)]
|
||||
[InlineData(StatusCode.Cancelled)]
|
||||
public async Task A_faulting_lookup_cannot_overwrite_an_overlapping_positive_result(StatusCode code)
|
||||
{
|
||||
// Catches converting metadata faults to false, including a fault finishing after a successful lookup.
|
||||
var success = Gate();
|
||||
var failure = Gate();
|
||||
var calls = 0;
|
||||
var fake = new FakeHistorianGatewayClient
|
||||
{
|
||||
TagExistsSource = (_, _) => Interlocked.Increment(ref calls) == 1 ? success.Task : failure.Task,
|
||||
};
|
||||
var cache = new GatewayTagPresenceCache(fake, new Clock());
|
||||
var goodRead = cache.ExistsAsync("T", Ct);
|
||||
var badRead = cache.ExistsAsync("T", Ct);
|
||||
goodRead.IsCompleted.ShouldBeFalse();
|
||||
badRead.IsCompleted.ShouldBeFalse();
|
||||
success.SetResult(true);
|
||||
(await goodRead.WaitAsync(Ct)).ShouldBeTrue();
|
||||
failure.SetException(new RpcException(new Status(code, "Catalog fault.")));
|
||||
(await Should.ThrowAsync<RpcException>(() => badRead.WaitAsync(Ct))).StatusCode.ShouldBe(code);
|
||||
fake.TagExistsSource = null;
|
||||
fake.TagExistsThrows = new InvalidOperationException("The positive entry must survive the failed lookup.");
|
||||
(await cache.ExistsAsync("T", Ct)).ShouldBeTrue();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Cancellation_after_a_catalog_result_cannot_overwrite_an_overlapping_positive_result()
|
||||
{
|
||||
// Catches removal of the post-await token check when the client ignores cancellation and returns false.
|
||||
var success = Gate();
|
||||
var cancelled = Gate();
|
||||
using var caller = CancellationTokenSource.CreateLinkedTokenSource(Ct);
|
||||
var fake = new FakeHistorianGatewayClient
|
||||
{
|
||||
TagExistsSource = (_, token) => token == caller.Token ? cancelled.Task : success.Task,
|
||||
};
|
||||
var cache = new GatewayTagPresenceCache(fake, new Clock());
|
||||
var goodRead = cache.ExistsAsync("T", Ct);
|
||||
var cancelledRead = cache.ExistsAsync("T", caller.Token);
|
||||
success.SetResult(true);
|
||||
(await goodRead.WaitAsync(Ct)).ShouldBeTrue();
|
||||
caller.Cancel();
|
||||
cancelled.SetResult(false);
|
||||
await Should.ThrowAsync<OperationCanceledException>(() => cancelledRead.WaitAsync(Ct));
|
||||
fake.TagExistsSource = null;
|
||||
fake.TagExistsThrows = new InvalidOperationException("The positive entry must survive cancellation.");
|
||||
(await cache.ExistsAsync("T", Ct)).ShouldBeTrue();
|
||||
// Cached results still respect an already-cancelled caller.
|
||||
await Should.ThrowAsync<OperationCanceledException>(() => cache.ExistsAsync("T", caller.Token));
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(false, 5)]
|
||||
[InlineData(true, 60)]
|
||||
public async Task An_expired_entry_is_rechecked_after_a_faulting_refresh(bool initial, int ttlSeconds)
|
||||
{
|
||||
// Catches faults renewing an expired negative or replacing an expired positive with a negative.
|
||||
var clock = new Clock();
|
||||
var fake = new FakeHistorianGatewayClient { TagExistsResult = initial };
|
||||
var cache = new GatewayTagPresenceCache(fake, clock);
|
||||
(await cache.ExistsAsync("T", Ct)).ShouldBe(initial);
|
||||
clock.Current = clock.Current.AddSeconds(ttlSeconds);
|
||||
var gate = Gate();
|
||||
fake.TagExistsSource = (_, _) => gate.Task;
|
||||
var read = cache.ExistsAsync("T", Ct);
|
||||
read.IsCompleted.ShouldBeFalse();
|
||||
gate.SetException(new RpcException(new Status(StatusCode.Unavailable, "Catalog unavailable.")));
|
||||
await Should.ThrowAsync<RpcException>(() => read.WaitAsync(Ct));
|
||||
fake.TagExistsSource = null;
|
||||
fake.TagExistsResult = true;
|
||||
(await cache.ExistsAsync("T", Ct)).ShouldBeTrue();
|
||||
fake.TagExistsCallCount.ShouldBe(3);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(false, 5)]
|
||||
[InlineData(true, 60)]
|
||||
public async Task An_expired_entry_is_rechecked_after_a_cancelled_refresh(bool initial, int ttlSeconds)
|
||||
{
|
||||
// Catches cancellation renewing expired entries when the catalog returns an absence result anyway.
|
||||
var clock = new Clock();
|
||||
var fake = new FakeHistorianGatewayClient { TagExistsResult = initial };
|
||||
var cache = new GatewayTagPresenceCache(fake, clock);
|
||||
(await cache.ExistsAsync("T", Ct)).ShouldBe(initial);
|
||||
clock.Current = clock.Current.AddSeconds(ttlSeconds);
|
||||
var gate = Gate();
|
||||
fake.TagExistsSource = (_, _) => gate.Task;
|
||||
using var caller = CancellationTokenSource.CreateLinkedTokenSource(Ct);
|
||||
var read = cache.ExistsAsync("T", caller.Token);
|
||||
read.IsCompleted.ShouldBeFalse();
|
||||
caller.Cancel();
|
||||
gate.SetResult(false);
|
||||
await Should.ThrowAsync<OperationCanceledException>(() => read.WaitAsync(Ct));
|
||||
fake.TagExistsSource = null;
|
||||
fake.TagExistsResult = true;
|
||||
(await cache.ExistsAsync("T", Ct)).ShouldBeTrue();
|
||||
fake.TagExistsCallCount.ShouldBe(3);
|
||||
}
|
||||
}
|
||||
+27
@@ -117,4 +117,31 @@ public sealed class HistorianGatewayClientAdapterTests
|
||||
var dataSource = GatewayHistorian.CreateDataSource(opts, services);
|
||||
Assert.IsType<GatewayHistorianDataSource>(dataSource);
|
||||
}
|
||||
|
||||
[Theory] // #693: the ServerHistorian:GatewayAtTimeFromRaw kill switch reaches the data source (default on)
|
||||
[InlineData(null, true)]
|
||||
[InlineData(true, true)]
|
||||
[InlineData(false, false)]
|
||||
public void Factory_passes_the_AtTime_kill_switch(bool? configured, bool expected)
|
||||
{
|
||||
var opts = configured is { } c
|
||||
? new ServerHistorianOptions { Enabled = true, Endpoint = "https://localhost:5222", ApiKey = "histgw_x_y", GatewayAtTimeFromRaw = c }
|
||||
: new ServerHistorianOptions { Enabled = true, Endpoint = "https://localhost:5222", ApiKey = "histgw_x_y" };
|
||||
using var services = new ServiceCollection().BuildServiceProvider();
|
||||
var dataSource = Assert.IsType<GatewayHistorianDataSource>(GatewayHistorian.CreateDataSource(opts, services));
|
||||
dataSource.AtTimeFromRaw.ShouldBe(expected);
|
||||
}
|
||||
|
||||
[Theory] // #694 R2: ServerHistorian:MaxTieClusterOverfetch bounds the crowding re-reads (default 65536)
|
||||
[InlineData(null, 65536)]
|
||||
[InlineData(1000, 1000)]
|
||||
public void Factory_passes_the_tie_cluster_limit_to_the_crowding_re_reads(int? configured, int expected)
|
||||
{
|
||||
var opts = configured is { } c
|
||||
? new ServerHistorianOptions { Enabled = true, Endpoint = "https://localhost:5222", ApiKey = "histgw_x_y", MaxTieClusterOverfetch = c }
|
||||
: new ServerHistorianOptions { Enabled = true, Endpoint = "https://localhost:5222", ApiKey = "histgw_x_y" };
|
||||
using var services = new ServiceCollection().BuildServiceProvider();
|
||||
var dataSource = Assert.IsType<GatewayHistorianDataSource>(GatewayHistorian.CreateDataSource(opts, services));
|
||||
dataSource.MaxPreWindowRows.ShouldBe(expected);
|
||||
}
|
||||
}
|
||||
|
||||
+9
-10
@@ -149,9 +149,8 @@ public sealed class GatewayLiveIntegrationTests(GatewayLiveFixture fixture) : IC
|
||||
/// <see cref="GatewayAlarmHistorianWriter"/>, then <c>ReadEvents</c> the recent window for that
|
||||
/// source through the real <see cref="GatewayHistorianDataSource"/> and assert the event is
|
||||
/// present. Requires the gateway running <c>RuntimeDb:EventReadsEnabled=true</c> (the SQL
|
||||
/// alarm-history read path). Presence is asserted as "at least one event for the source surfaced
|
||||
/// in the post-send window" (the data source filters by source); the exact AlarmId / message
|
||||
/// match is surfaced as a diagnostic, since the SQL event store may re-key the row.
|
||||
/// alarm-history read path). The exact delivery GUID must return; an unrelated event for the
|
||||
/// same source cannot satisfy the assertion.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
[Trait("Category", "LiveIntegration")]
|
||||
@@ -196,7 +195,7 @@ public sealed class GatewayLiveIntegrationTests(GatewayLiveFixture fixture) : IC
|
||||
var read = await dataSource.ReadEventsAsync(
|
||||
source, eventUtc - TimeSpan.FromMinutes(5), DateTime.UtcNow + TimeSpan.FromMinutes(1), maxEvents: 0, ct);
|
||||
events = read.Events;
|
||||
if (events.Count > 0) break;
|
||||
if (events.Any(e => Guid.TryParse(e.EventId, out var id) && id == alarm.DeliveryId)) break;
|
||||
await Task.Delay(TimeSpan.FromSeconds(2), ct);
|
||||
}
|
||||
while (DateTime.UtcNow < deadline);
|
||||
@@ -211,18 +210,18 @@ public sealed class GatewayLiveIntegrationTests(GatewayLiveFixture fixture) : IC
|
||||
$"a source-filtered ReadEvents must return the just-sent alarm for source '{source}': the gateway now " +
|
||||
"populates Source_Object from the event SourceName (C4 fix), so OtOpcUa's own alarm sends round-trip.");
|
||||
|
||||
var exactMatch = events.Any(e => string.Equals(e.EventId, alarmId, StringComparison.Ordinal));
|
||||
var exactMatch = events.Any(e => Guid.TryParse(e.EventId, out var id) && id == alarm.DeliveryId);
|
||||
exactMatch.ShouldBeTrue("the just-sent event must retain its nonzero delivery GUID through persistence and readback.");
|
||||
TestContext.Current.SendDiagnosticMessage(
|
||||
$"alarm round-trip: SendEvent source='{source}' id={alarmId} → ReadEvents returned {events.Count} event(s); exact-id match={exactMatch}.");
|
||||
$"alarm round-trip: SendEvent source='{source}' delivery-id={alarm.DeliveryId:D} → exact ID returned among {events.Count} event(s).");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Alarm send contract — the OtOpcUa responsibility in isolation: an <see cref="AlarmHistorianEvent"/>
|
||||
/// maps to a wire event the gateway accepts and <c>SendEvent</c> returns <see cref="HistorianWriteOutcome.Ack"/>.
|
||||
/// This is the half that must always hold (the readback half is historian-gated; see
|
||||
/// <see cref="Alarm_SendEvent_then_ReadEvents"/>). Regression guard for the event-id mapping —
|
||||
/// a client-supplied wire <c>Id</c> makes the gateway's SendEvent handler throw, so the mapper
|
||||
/// must leave it unset.
|
||||
/// <see cref="Alarm_SendEvent_then_ReadEvents"/>). The mapper sends the stable nonzero delivery GUID;
|
||||
/// the earlier assumption that the gateway rejected every client-supplied ID was incorrect (#705).
|
||||
/// </summary>
|
||||
[Fact]
|
||||
[Trait("Category", "LiveIntegration")]
|
||||
@@ -251,7 +250,7 @@ public sealed class GatewayLiveIntegrationTests(GatewayLiveFixture fixture) : IC
|
||||
|
||||
outcomes.ShouldHaveSingleItem().ShouldBe(
|
||||
HistorianWriteOutcome.Ack,
|
||||
"the alarm SendEvent must be acked (the AlarmEventMapper must NOT set the wire event Id — the gateway rejects a client-supplied id).");
|
||||
"the alarm SendEvent carrying its nonzero delivery GUID must be acked.");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
||||
+23
-7
@@ -7,6 +7,17 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests.Mapping;
|
||||
|
||||
public sealed class AlarmEventMapperTests
|
||||
{
|
||||
[Fact]
|
||||
public void A_direct_event_has_a_nonzero_wire_id_that_survives_repeated_mapping()
|
||||
{
|
||||
var alarm = new AlarmHistorianEvent("A", "S", "N", "LimitAlarm", AlarmSeverity.High,
|
||||
"Activated", "m", "system", null, new DateTime(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc));
|
||||
var first = AlarmEventMapper.ToHistorianEvent(alarm).Id;
|
||||
Assert.True(Guid.TryParse(first, out var id), $"Expected a delivery GUID; received '{first}'.");
|
||||
Assert.NotEqual(Guid.Empty, id);
|
||||
Assert.Equal(first, AlarmEventMapper.ToHistorianEvent(alarm).Id);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Maps_source_time_type_and_rich_properties()
|
||||
{
|
||||
@@ -17,9 +28,7 @@ public sealed class AlarmEventMapperTests
|
||||
Assert.Equal("Area/Line/Pump1", e.SourceName);
|
||||
Assert.Equal("LimitAlarm", e.Type);
|
||||
Assert.Equal(new DateTime(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc), e.EventTime.ToDateTime());
|
||||
// The wire Id is deliberately left unset — the gateway's SendEvent rejects a client-supplied
|
||||
// event id. The alarm's own id is carried as a property instead.
|
||||
Assert.Equal(string.Empty, e.Id);
|
||||
Assert.Equal(a.DeliveryId.ToString("D"), e.Id);
|
||||
Assert.Equal("A1", e.Properties["AlarmId"]);
|
||||
Assert.Equal("HiHi", e.Properties["AlarmName"]);
|
||||
Assert.Equal("Activated", e.Properties["EventKind"]);
|
||||
@@ -67,14 +76,21 @@ public sealed class AlarmEventMapperTests
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Wire_id_is_never_set_and_blank_alarm_id_adds_no_property()
|
||||
public void A_blank_condition_id_still_has_delivery_identity_and_adds_no_condition_property()
|
||||
{
|
||||
// A blank AlarmId must NOT fall back to a generated wire Id (the gateway rejects any id) and
|
||||
// must not add an empty AlarmId property.
|
||||
// Condition identity and event delivery identity serve different purposes.
|
||||
var a = new AlarmHistorianEvent("", "S", "N", "DiscreteAlarm", AlarmSeverity.Low, "Cleared", "m", "system", null,
|
||||
new DateTime(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc));
|
||||
var e = AlarmEventMapper.ToHistorianEvent(a);
|
||||
Assert.Equal(string.Empty, e.Id);
|
||||
Assert.Equal(a.DeliveryId.ToString("D"), e.Id);
|
||||
Assert.False(e.Properties.ContainsKey("AlarmId"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void An_explicitly_empty_delivery_id_is_rejected_instead_of_reminted()
|
||||
{
|
||||
var alarm = new AlarmHistorianEvent("A", "S", "N", "LimitAlarm", AlarmSeverity.High,
|
||||
"Activated", "m", "system", null, DateTime.UtcNow) { DeliveryId = Guid.Empty };
|
||||
Assert.Throws<ArgumentException>(() => AlarmEventMapper.ToHistorianEvent(alarm));
|
||||
}
|
||||
}
|
||||
|
||||
+25
@@ -32,6 +32,31 @@ public sealed class SampleMapperTests
|
||||
Assert.Equal(0x80000000u, SampleMapper.ToSnapshot(s).StatusCode);
|
||||
}
|
||||
|
||||
[Theory] // #690: histsdk puts a value on every row, so a NULL row arrives as 0 / "" with a Bad quality
|
||||
[InlineData(0u, 1u)] // the live NULL row: opc_quality 0, AVEVA quality 1
|
||||
[InlineData(24u, 0u)] // any Bad OPC quality (out of service)
|
||||
public void A_Bad_numeric_row_carries_no_value(uint opc, uint quality)
|
||||
{
|
||||
var s = new HistorianSample { Tag = "T", NumericValue = 0, OpcQuality = opc, Quality = quality, Timestamp = Ts(2026, 1, 1, 0, 0, 0) };
|
||||
var snap = SampleMapper.ToSnapshot(s);
|
||||
Assert.True(snap.StatusCode >= 0x80000000u);
|
||||
Assert.Null(snap.Value);
|
||||
}
|
||||
|
||||
[Fact] // #690: a string tag's NULL row arrives as "" + Bad
|
||||
public void A_Bad_string_row_carries_no_value()
|
||||
{
|
||||
var s = new HistorianSample { Tag = "T", StringValue = "", OpcQuality = 0, Quality = 1, Timestamp = Ts(2026, 1, 1, 0, 0, 0) };
|
||||
Assert.Null(SampleMapper.ToSnapshot(s).Value);
|
||||
}
|
||||
|
||||
[Fact] // #690: an Uncertain row keeps its value (only Bad means "no value")
|
||||
public void An_Uncertain_row_keeps_its_value()
|
||||
{
|
||||
var s = new HistorianSample { Tag = "T", NumericValue = 3, OpcQuality = 64, Timestamp = Ts(2026, 1, 1, 0, 0, 0) };
|
||||
Assert.Equal(3d, SampleMapper.ToSnapshot(s).Value);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Aggregate_null_value_is_BadNoData()
|
||||
{
|
||||
|
||||
+5
@@ -31,4 +31,9 @@
|
||||
<ProjectReference Include="..\..\..\src\Drivers\ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway\ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.csproj"/>
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<!-- #694: the node manager's HistoryRead paging over the REAL gateway data source boots a real SDK server. -->
|
||||
<Compile Include="..\..\Server\ZB.MOM.WW.OtOpcUa.OpcUaServer.Tests\OpcUaHostStart.cs" Link="OpcUaHostStart.cs"/>
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
|
||||
+7
@@ -67,3 +67,10 @@ public sealed class QuestDbEventReadConformance(QuestDbFixture fx) : EventReadCo
|
||||
/// <inheritdoc/>
|
||||
protected override Task<IConformanceBackend> CreateBackendAsync() => QuestDbConformanceBackend.CreateAsync(fx);
|
||||
}
|
||||
|
||||
[Collection(QuestDbCollection.Name)]
|
||||
public sealed class QuestDbBoundingValueConformance(QuestDbFixture fx) : BoundingValueConformance
|
||||
{
|
||||
/// <inheritdoc/>
|
||||
protected override Task<IConformanceBackend> CreateBackendAsync() => QuestDbConformanceBackend.CreateAsync(fx);
|
||||
}
|
||||
|
||||
+27
@@ -11,6 +11,33 @@ public sealed class QuestDbHistorianProviderTests
|
||||
ConnectionString = "Host=localhost;Port=18812;Username=admin;Password=quest;Database=qdb;ServerCompatibilityMode=NoTypeLoading",
|
||||
});
|
||||
|
||||
[Theory]
|
||||
[InlineData("Raw")]
|
||||
[InlineData("AtTime")]
|
||||
[InlineData("Processed")]
|
||||
[InlineData("Events")]
|
||||
[InlineData("Modified")]
|
||||
public async Task A_connection_failure_is_exposed_as_a_transport_failure(string kind)
|
||||
{
|
||||
using var provider = new QuestDbHistorianProvider(new QuestDbHistorianOptions {
|
||||
ConnectionString = "Host=127.0.0.1;Port=1;Username=admin;Password=test;Database=qdb;Timeout=1;Pooling=False;ServerCompatibilityMode=NoTypeLoading", AutoCreateSchema = false
|
||||
});
|
||||
using var source = provider.CreateDataSource(new NullSp());
|
||||
var t = new DateTime(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc);
|
||||
Func<Task> read = kind switch
|
||||
{
|
||||
"Raw" => async () => await source.ReadRawAsync("T", t, t.AddHours(1), 10, TestContext.Current.CancellationToken),
|
||||
"AtTime" => async () => await source.ReadAtTimeAsync("T", [t], TestContext.Current.CancellationToken),
|
||||
"Processed" => async () => await source.ReadProcessedAsync("T", t, t.AddHours(1), TimeSpan.FromSeconds(10), HistoryAggregateType.Average, TestContext.Current.CancellationToken),
|
||||
"Events" => async () => await source.ReadEventsAsync("T", t, t.AddHours(1), 10, TestContext.Current.CancellationToken),
|
||||
_ => async () => await source.ReadModifiedAsync("T", t, t.AddHours(1), 10, null, TestContext.Current.CancellationToken)
|
||||
};
|
||||
var error = await Should.ThrowAsync<System.IO.IOException>(read);
|
||||
error.InnerException.ShouldNotBeNull();
|
||||
// A cached bootstrap outage must remain retryable on subsequent reads as well.
|
||||
(await Should.ThrowAsync<System.IO.IOException>(read)).InnerException.ShouldNotBeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Name_and_all_six_kinds()
|
||||
{
|
||||
|
||||
+5
@@ -83,6 +83,11 @@ public sealed class AtTimeEvaluatorTests
|
||||
var n = S(10, HistorianValue.FromInt64(15), Good);
|
||||
AtTimeEvaluator.Evaluate(At(5), p, n, Now, Now).Value.ShouldBe(13L); // 12.5 ⇒ 13
|
||||
AtTimeEvaluator.Evaluate(At(2), p, n, Now, Now).Value.ShouldBe(11L);
|
||||
// #693: ShouldBe compares numbers across types, so the type needs its own assertion — a C# conditional
|
||||
// `long : double` boxed a Double here, and the line above still passed.
|
||||
AtTimeEvaluator.Evaluate(At(5), p, n, Now, Now).Value.ShouldBeOfType<long>();
|
||||
AtTimeEvaluator.Evaluate(At(5), S(0, HistorianValue.FromDouble(10), Good), S(10, HistorianValue.FromDouble(15), Good), Now, Now)
|
||||
.Value.ShouldBeOfType<double>();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
|
||||
+7
@@ -67,3 +67,10 @@ public sealed class SqlServerEventReadConformance(SqlHistorianFixture fx) : Even
|
||||
/// <inheritdoc/>
|
||||
protected override Task<IConformanceBackend> CreateBackendAsync() => SqlServerConformanceBackend.CreateAsync(fx);
|
||||
}
|
||||
|
||||
[Collection(SqlHistorianCollection.Name)]
|
||||
public sealed class SqlServerBoundingValueConformance(SqlHistorianFixture fx) : BoundingValueConformance
|
||||
{
|
||||
/// <inheritdoc/>
|
||||
protected override Task<IConformanceBackend> CreateBackendAsync() => SqlServerConformanceBackend.CreateAsync(fx);
|
||||
}
|
||||
|
||||
+27
@@ -11,6 +11,33 @@ public sealed class SqlServerHistorianProviderTests
|
||||
ConnectionString = "Server=localhost;User Id=sa;Password=x;TrustServerCertificate=True",
|
||||
});
|
||||
|
||||
[Theory]
|
||||
[InlineData("Raw")]
|
||||
[InlineData("AtTime")]
|
||||
[InlineData("Processed")]
|
||||
[InlineData("Events")]
|
||||
[InlineData("Modified")]
|
||||
public async Task A_connection_failure_is_exposed_as_a_transport_failure(string kind)
|
||||
{
|
||||
var provider = new SqlServerHistorianProvider(new SqlServerHistorianOptions {
|
||||
ConnectionString = "Server=127.0.0.1,1;User Id=sa;Password=test;TrustServerCertificate=True;Connect Timeout=1;ConnectRetryCount=0;Pooling=False", AutoCreateSchema = false
|
||||
});
|
||||
using var source = provider.CreateDataSource(new NullSp());
|
||||
var t = new DateTime(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc);
|
||||
Func<Task> read = kind switch
|
||||
{
|
||||
"Raw" => async () => await source.ReadRawAsync("T", t, t.AddHours(1), 10, TestContext.Current.CancellationToken),
|
||||
"AtTime" => async () => await source.ReadAtTimeAsync("T", [t], TestContext.Current.CancellationToken),
|
||||
"Processed" => async () => await source.ReadProcessedAsync("T", t, t.AddHours(1), TimeSpan.FromSeconds(10), HistoryAggregateType.Average, TestContext.Current.CancellationToken),
|
||||
"Events" => async () => await source.ReadEventsAsync("T", t, t.AddHours(1), 10, TestContext.Current.CancellationToken),
|
||||
_ => async () => await source.ReadModifiedAsync("T", t, t.AddHours(1), 10, null, TestContext.Current.CancellationToken)
|
||||
};
|
||||
var error = await Should.ThrowAsync<System.IO.IOException>(read);
|
||||
error.InnerException.ShouldNotBeNull();
|
||||
// A cached bootstrap outage must remain retryable on subsequent reads as well.
|
||||
(await Should.ThrowAsync<System.IO.IOException>(read)).InnerException.ShouldNotBeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Name_and_all_six_kinds()
|
||||
{
|
||||
|
||||
+26
-12
@@ -27,7 +27,15 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.S7.IntegrationTests.S7_1500;
|
||||
/// areas: <b>PASSED</b> — 4 passed, 0 failed, 1 skipped (the device number-addressing test,
|
||||
/// see below). The Timer/Counter batch was served by 4 requests for 7 tags: one DB1 span,
|
||||
/// one 3-item multi-var carrying only the scalars, and one 2-byte area read apiece for
|
||||
/// <c>T0</c> (decoded 456.0 s from S5TIME <c>0x2456</c>) and <c>C0</c> (decoded count 4711).
|
||||
/// <c>T0</c> (decoded 456.0 s from S5TIME <c>0x2456</c>) and <c>C0</c> (surfaced raw word 4711).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Counter evidence limit.</b> The profile's C0=4711 and C7=31337 are synthetic raw
|
||||
/// words, outside classic SIMATIC counts 0..999. The device-0 record proves the fixture's
|
||||
/// raw-word decode and batching exclusion, not classic count semantics or nonzero
|
||||
/// addressing. BuildOptions retains S71500/rack 0/slot 0; a target-specific semantic gate
|
||||
/// still needs independent PLC counts and binary/BCD references paired with captured
|
||||
/// C-area bytes, CPU model/firmware and connection settings (docs/drivers/S7.md, #616).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>What the layout buys.</b> The ten tags are positioned so the planner's rules produce a
|
||||
@@ -73,11 +81,12 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.S7.IntegrationTests.S7_1500;
|
||||
/// returns TM bytes 0..1, and it takes <c>start 80</c> (= 10 × 8) to reach the timer seeded
|
||||
/// at TM byte 10. Device <b>0</b> is the single fixed point of the two conventions
|
||||
/// (<c>0 >> 3 == 2 × 0 == 0</c>), so it is the only device this fixture can pin a
|
||||
/// VALUE for, and that is what the mixed batch uses. The spec-correct <c>2 × N</c> cells
|
||||
/// VALUE for, and that is what the mixed batch uses. The profile's <c>2 × N</c> cells
|
||||
/// are seeded too and
|
||||
/// <see cref="Timer_and_counter_decode_the_seeded_values_at_their_device_numbers"/> asserts
|
||||
/// them — skipped unless <c>S7_TIMER_NUMBER_ADDRESSING=1</c> declares the endpoint a real
|
||||
/// CPU. The device-number offset arithmetic itself stays unit-covered
|
||||
/// CPU. This flag is an operator declaration, not hardware evidence.
|
||||
/// The device-number offset arithmetic itself stays unit-covered
|
||||
/// (<c>S7ScalarBlockTests</c>, <c>S7SpanReadParityTests</c>).
|
||||
/// </para>
|
||||
/// <para>
|
||||
@@ -240,7 +249,7 @@ public sealed class S7_1500SpanReadTests(Snap7ServerFixture sim)
|
||||
snapshot.StatusCode.ShouldBe(0u, $"'{name}' must be Good");
|
||||
|
||||
// (a) Correct decode off real wire. A Timer surfaces as the S5TIME word's duration in
|
||||
// seconds (double) and a Counter as the raw count (int) — two different areas, two
|
||||
// seconds (double) and a Counter as the raw word (int) — two different areas, two
|
||||
// different values, two different CLR types, so a read that reached the wrong area or
|
||||
// decoded through the wrong arm cannot pass.
|
||||
byName["TC_Timer"].Value.ShouldBe(TimerCounterLayout.Device0TimerSeconds);
|
||||
@@ -280,15 +289,19 @@ public sealed class S7_1500SpanReadTests(Snap7ServerFixture sim)
|
||||
|
||||
/// <summary>
|
||||
/// Verifies a Timer and a Counter at NON-zero device numbers decode their seeded values —
|
||||
/// the assertion the shared fixture cannot serve.
|
||||
/// the raw-word assertion the shared fixture cannot serve.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Skipped unless <c>S7_TIMER_NUMBER_ADDRESSING=1</c> declares the endpoint a device
|
||||
/// number-addressing CPU, because python-snap7's server is not one: it resolves the TM/CT
|
||||
/// request address as a bit address, so <c>T5</c> reads TM byte 0 rather than TM byte 10
|
||||
/// (measured 2026-08-17 — see the type remarks). Set the variable alongside
|
||||
/// <c>S7_SIM_ENDPOINT</c> when pointing this suite at a real S7 CPU seeded to match
|
||||
/// <c>Docker/profiles/s7_1500.json</c>'s TM/CT areas.
|
||||
/// (measured 2026-08-17 — see the type remarks). The flag only enables this test;
|
||||
/// BuildOptions still selects S71500, rack 0, slot 0 and the profile's raw C7 word 31337.
|
||||
/// It does not configure a classic CPU or prove classic 0..999 count semantics. A separate
|
||||
/// target-specific gate needs valid counts, independent PLC binary/BCD references and
|
||||
/// captured C-area bytes, plus CPU model/firmware and connection settings (#616).
|
||||
/// See docs/drivers/S7.md for the evidence checklist. The skip message below retains the
|
||||
/// historical fixture instructions; enabling the flag alone does not satisfy this gate.
|
||||
/// </remarks>
|
||||
[Fact]
|
||||
public async Task Timer_and_counter_decode_the_seeded_values_at_their_device_numbers()
|
||||
@@ -408,12 +421,13 @@ public sealed class S7_1500SpanReadTests(Snap7ServerFixture sim)
|
||||
/// <summary>
|
||||
/// The Timer/Counter layouts. Two sets: a mixed batch that provokes all three read shapes at
|
||||
/// once (block span, multi-var, per-tag Timer/Counter), and a device-numbered pair only a
|
||||
/// real CPU can serve. Addresses map 1:1 to the TM0 / CT0 seeds in
|
||||
/// device number-addressing endpoint can serve. These synthetic raw-word expectations
|
||||
/// do not prove classic counter semantics. Addresses map 1:1 to the TM0 / CT0 seeds in
|
||||
/// <c>Docker/profiles/s7_1500.json</c>.
|
||||
/// </summary>
|
||||
private static class TimerCounterLayout
|
||||
{
|
||||
/// <summary>Declares the endpoint a real S7 CPU, which addresses TM/CT by device number.</summary>
|
||||
/// <summary>Operator opt-in for nonzero TM/CT addressing; does not change CPU options or prove encoding.</summary>
|
||||
public const string RealCpuEnvVar = "S7_TIMER_NUMBER_ADDRESSING";
|
||||
|
||||
/// <summary>
|
||||
@@ -425,7 +439,7 @@ public sealed class S7_1500SpanReadTests(Snap7ServerFixture sim)
|
||||
/// <summary>T0's seed 0x2456 — S5TIME base 2 (×1 s) over BCD 456.</summary>
|
||||
public const double Device0TimerSeconds = 456.0d;
|
||||
|
||||
/// <summary>C0's seed — the raw big-endian count word.</summary>
|
||||
/// <summary>C0's synthetic raw big-endian word 4711; not a classic 0..999 count reference.</summary>
|
||||
public const int Device0CounterCount = 4711;
|
||||
|
||||
/// <summary>
|
||||
@@ -441,7 +455,7 @@ public sealed class S7_1500SpanReadTests(Snap7ServerFixture sim)
|
||||
new("TC_Lone_I16", "DB1.DBW600", S7DataType.Int16),
|
||||
new("TC_Lone_Mk", "MW10", S7DataType.UInt16),
|
||||
// Timer tags are typed Float64 (decoded to seconds) and Counter tags Int32 (decoded to
|
||||
// a count) — the init guard rejects any other typing. Writable:false is mandatory, not
|
||||
// a raw word) — the init guard rejects any other typing. Writable:false is mandatory, not
|
||||
// stylistic: S7TagDefinition defaults Writable to TRUE and the same guard rejects a
|
||||
// writable Timer/Counter outright (they are read-only this phase).
|
||||
new("TC_Timer", "T0", S7DataType.Float64, Writable: false),
|
||||
|
||||
+39
-14
@@ -1,6 +1,9 @@
|
||||
using System.Diagnostics.CodeAnalysis;
|
||||
using Akka.Actor;
|
||||
using Akka.Cluster.Tools.PublishSubscribe;
|
||||
using Akka.Event;
|
||||
using Akka.TestKit;
|
||||
using Akka.TestKit.Xunit2;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
@@ -680,7 +683,7 @@ public class CentralCommunicationActorTests : ControlPlaneActorTestBase
|
||||
|
||||
/// <summary>Counts contact loads and fails the first <c>failFirst</c> of them.</summary>
|
||||
private sealed class CountingDbFactory(
|
||||
IDbContextFactory<OtOpcUaConfigDbContext> inner, int failFirst = 0)
|
||||
IDbContextFactory<OtOpcUaConfigDbContext> inner, int failFirst = 0, Action<int>? onLoad = null)
|
||||
: IDbContextFactory<OtOpcUaConfigDbContext>
|
||||
{
|
||||
private int _calls;
|
||||
@@ -689,7 +692,9 @@ public class CentralCommunicationActorTests : ControlPlaneActorTestBase
|
||||
|
||||
public OtOpcUaConfigDbContext CreateDbContext()
|
||||
{
|
||||
if (Interlocked.Increment(ref _calls) <= failFirst)
|
||||
var call = Interlocked.Increment(ref _calls);
|
||||
onLoad?.Invoke(call);
|
||||
if (call <= failFirst)
|
||||
throw new InvalidOperationException("ConfigDb unreachable at boot");
|
||||
return inner.CreateDbContext();
|
||||
}
|
||||
@@ -747,31 +752,51 @@ public class CentralCommunicationActorTests : ControlPlaneActorTestBase
|
||||
[Fact]
|
||||
public void Bootstrap_retries_stop_once_the_gate_is_published()
|
||||
{
|
||||
// The provider throws on PreStart (call 1) and on the first load's resolve (call 2), which
|
||||
// schedules a 1 s retry. A manual refresh then publishes while that retry is still PENDING;
|
||||
// stop-on-publish must cancel it. Without the cancel the timer fires, loads again and
|
||||
// republishes (loads == 3, publications == 2).
|
||||
// Freeze time only for this fact: the cluster harness and every other retry test retain
|
||||
// their real scheduler. gRPC mode needs neither cluster membership nor pub-sub here.
|
||||
using var local = new TestKit(@"akka {
|
||||
loglevel = INFO
|
||||
scheduler.implementation = ""Akka.TestKit.TestScheduler, Akka.TestKit""
|
||||
}");
|
||||
var scheduler = (TestScheduler)local.Sys.Scheduler;
|
||||
var loads = local.CreateTestProbe();
|
||||
var barrier = local.CreateTestProbe();
|
||||
local.Sys.EventStream.Subscribe(barrier.Ref, typeof(UnhandledMessage));
|
||||
var inner = NewInMemoryDbFactory();
|
||||
SeedGrpcCluster(inner, "C1", ("host-a:4053", "host-a", 4056, true, false));
|
||||
var factory = new CountingDbFactory(inner);
|
||||
var factory = new CountingDbFactory(inner, onLoad: n => loads.Ref.Tell(n));
|
||||
var psk = new RecordingPskProvider();
|
||||
var calls = 0;
|
||||
|
||||
// PreStart and the first contact load fail resolution. The latter registers a retry.
|
||||
IMeshPskProvider? PskFactory() =>
|
||||
Interlocked.Increment(ref calls) <= 2 ? throw new InvalidOperationException("race") : psk;
|
||||
|
||||
var actor = Sys.ActorOf(CentralCommunicationActor.Props(
|
||||
var actor = local.Sys.ActorOf(CentralCommunicationActor.Props(
|
||||
factory, GrpcMode(), () => null, () => new RecordingDialer(), PskFactory,
|
||||
bootstrapRetryInterval: TimeSpan.FromSeconds(1)));
|
||||
// A mailbox barrier also waits for PreStart to register the zero-delay periodic tick.
|
||||
var marker = new object();
|
||||
actor.Tell(marker);
|
||||
barrier.ExpectMsg<UnhandledMessage>(TimeSpan.FromSeconds(5)).Message.ShouldBeSameAs(marker);
|
||||
scheduler.Advance(TimeSpan.Zero);
|
||||
loads.ExpectMsg(1, TimeSpan.FromSeconds(5));
|
||||
local.AwaitCondition(() => Volatile.Read(ref calls) == 2, TimeSpan.FromSeconds(5));
|
||||
|
||||
// Positive control: the failed first resolve happened, so a retry is pending.
|
||||
AwaitCondition(() => Volatile.Read(ref calls) >= 2, TimeSpan.FromSeconds(5));
|
||||
Thread.Sleep(100);
|
||||
actor.Tell(new CentralCommunicationActor.RefreshContacts());
|
||||
AwaitCondition(() => psk.KnownClusterPublications.Count >= 1, TimeSpan.FromSeconds(5));
|
||||
// This mailbox marker is handled after the failed ContactsLoaded handler returns.
|
||||
// Unlike observing the provider call alone, it proves retry registration is complete.
|
||||
actor.Tell(marker);
|
||||
barrier.ExpectMsg<UnhandledMessage>(TimeSpan.FromSeconds(5)).Message.ShouldBeSameAs(marker);
|
||||
|
||||
Thread.Sleep(2500);
|
||||
local.EventFilter.Info(contains: "Known-cluster gate published").ExpectOne(
|
||||
TimeSpan.FromSeconds(5), () => actor.Tell(new CentralCommunicationActor.RefreshContacts()));
|
||||
loads.ExpectMsg(2, TimeSpan.FromSeconds(5));
|
||||
|
||||
// Cross the pending retry deadline, leaving the 60-second periodic tick untouched.
|
||||
// LoadContactsFromDb uses Task.Run, so observe its boundary for the original absence
|
||||
// budget rather than asserting before a mistakenly retained retry can start its read.
|
||||
scheduler.Advance(TimeSpan.FromMilliseconds(2500));
|
||||
loads.ExpectNoMsg(TimeSpan.FromMilliseconds(2500));
|
||||
factory.Calls.ShouldBe(2);
|
||||
psk.KnownClusterPublications.Count.ShouldBe(1);
|
||||
}
|
||||
|
||||
+21
-5
@@ -41,10 +41,14 @@ public sealed class AlarmSfConvergenceTests
|
||||
/// <summary>A writer that never acknowledges — a historian outage, which is when the buffer matters.</summary>
|
||||
private sealed class OutageWriter : IAlarmHistorianWriter
|
||||
{
|
||||
public List<AlarmHistorianEvent> Events { get; } = [];
|
||||
public Task<IReadOnlyList<HistorianWriteOutcome>> WriteBatchAsync(
|
||||
IReadOnlyList<AlarmHistorianEvent> batch, CancellationToken cancellationToken) =>
|
||||
Task.FromResult<IReadOnlyList<HistorianWriteOutcome>>(
|
||||
IReadOnlyList<AlarmHistorianEvent> batch, CancellationToken cancellationToken)
|
||||
{
|
||||
Events.AddRange(batch);
|
||||
return Task.FromResult<IReadOnlyList<HistorianWriteOutcome>>(
|
||||
batch.Select(_ => HistorianWriteOutcome.RetryPlease).ToList());
|
||||
}
|
||||
}
|
||||
|
||||
private static AlarmHistorianEvent Event(string alarmId, int second) => new(
|
||||
@@ -170,11 +174,15 @@ public sealed class AlarmSfConvergenceTests
|
||||
await pair.StartAsync();
|
||||
|
||||
var shared = Event("eq/shared", 1);
|
||||
using var sinkA = new LocalDbStoreAndForwardSink(pair.A, new OutageWriter(), Log);
|
||||
using var sinkB = new LocalDbStoreAndForwardSink(pair.B, new OutageWriter(), Log);
|
||||
var sharedOnB = Event("eq/shared", 1);
|
||||
shared.DeliveryId.ShouldNotBe(sharedOnB.DeliveryId);
|
||||
var writerA = new OutageWriter();
|
||||
var writerB = new OutageWriter();
|
||||
using var sinkA = new LocalDbStoreAndForwardSink(pair.A, writerA, Log);
|
||||
using var sinkB = new LocalDbStoreAndForwardSink(pair.B, writerB, Log);
|
||||
|
||||
await sinkA.EnqueueAsync(shared, TestContext.Current.CancellationToken);
|
||||
await sinkB.EnqueueAsync(shared, TestContext.Current.CancellationToken);
|
||||
await sinkB.EnqueueAsync(sharedOnB, TestContext.Current.CancellationToken);
|
||||
|
||||
// A second, DISTINCT event on B. Without it the scenario would pass with replication
|
||||
// switched off entirely — each node would hold its own single copy and "count == 1 on
|
||||
@@ -193,5 +201,13 @@ public sealed class AlarmSfConvergenceTests
|
||||
async () => await CountAsync(pair.A) > 2 || await CountAsync(pair.B) > 2,
|
||||
TimeSpan.FromSeconds(3)))
|
||||
.ShouldBeFalse("the duplicate must stay collapsed");
|
||||
|
||||
// Either node can retry after promotion, with the same GUID despite independent input identities.
|
||||
await sinkA.DrainOnceAsync(TestContext.Current.CancellationToken);
|
||||
await sinkB.DrainOnceAsync(TestContext.Current.CancellationToken);
|
||||
var deliveryA = writerA.Events.Single(e => e.AlarmId == "eq/shared").DeliveryId;
|
||||
var deliveryB = writerB.Events.Single(e => e.AlarmId == "eq/shared").DeliveryId;
|
||||
deliveryA.ShouldNotBe(Guid.Empty);
|
||||
deliveryB.ShouldBe(deliveryA);
|
||||
}
|
||||
}
|
||||
|
||||
+32
@@ -36,6 +36,38 @@ public sealed class AlarmSfLegacyMigratorTests : IDisposable
|
||||
|
||||
private ServiceProvider? _provider;
|
||||
|
||||
private sealed class CapturingRetryWriter : IAlarmHistorianWriter
|
||||
{
|
||||
public List<AlarmHistorianEvent> Events { get; } = [];
|
||||
public Task<IReadOnlyList<HistorianWriteOutcome>> WriteBatchAsync(
|
||||
IReadOnlyList<AlarmHistorianEvent> batch, CancellationToken cancellationToken)
|
||||
{
|
||||
Events.AddRange(batch);
|
||||
return Task.FromResult<IReadOnlyList<HistorianWriteOutcome>>(batch.Select(_ => HistorianWriteOutcome.RetryPlease).ToArray());
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_legacy_payload_keeps_its_key_and_stable_delivery_identity_across_drains()
|
||||
{
|
||||
var payload = Payload("eq/a");
|
||||
SeedLegacy((1, "eq/a", payload, 0, dead: 0));
|
||||
var db = BuildDb();
|
||||
AlarmSfLegacyMigrator.Migrate(db, ConfigWithLegacyPath());
|
||||
var row = (await db.QueryAsync("SELECT id, payload_json FROM alarm_sf_events",
|
||||
r => (Id: r.GetString(0), Payload: r.GetString(1)), null, TestContext.Current.CancellationToken)).Single();
|
||||
row.Id.ShouldBe("B1D80B7207C3F3B6CB31105DBF1A6E0D6FE300DA1B3F877AB746B36535310A60");
|
||||
row.Payload.ShouldBe(payload);
|
||||
var writer = new CapturingRetryWriter();
|
||||
using var log = new Serilog.LoggerConfiguration().CreateLogger();
|
||||
using (var sink = new LocalDbStoreAndForwardSink(db, writer, log))
|
||||
await sink.DrainOnceAsync(TestContext.Current.CancellationToken);
|
||||
using (var sink = new LocalDbStoreAndForwardSink(db, writer, log))
|
||||
await sink.DrainOnceAsync(TestContext.Current.CancellationToken);
|
||||
writer.Events.Count.ShouldBe(2);
|
||||
writer.Events.ShouldAllBe(evt => evt.DeliveryId == Guid.Parse("55372883-eaec-8dd7-a1a4-6f98f416a272"));
|
||||
}
|
||||
|
||||
// ---- the cases ---------------------------------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
|
||||
+45
@@ -0,0 +1,45 @@
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Host.OpcUa;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Host.Tests.OpcUa;
|
||||
|
||||
public sealed class FailedServiceLevelBindingCleanupTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task Successful_cleanup_observes_stop_before_disposal_and_releases_ownership()
|
||||
{
|
||||
var events = new List<string>();
|
||||
var released = await FailedServiceLevelBindingCleanup.StopAndDisposeAsync(
|
||||
() => { events.Add("stop"); return Task.CompletedTask; },
|
||||
() => { events.Add("dispose"); return ValueTask.CompletedTask; },
|
||||
NullLogger.Instance);
|
||||
events.ShouldBe(new[] { "stop", "dispose" });
|
||||
released.ShouldBeTrue();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Failed_stop_still_disposes_and_retains_ownership_for_retry()
|
||||
{
|
||||
var events = new List<string>();
|
||||
var released = await FailedServiceLevelBindingCleanup.StopAndDisposeAsync(
|
||||
() => { events.Add("stop"); throw new InvalidOperationException("controlled stop failure"); },
|
||||
() => { events.Add("dispose"); return ValueTask.CompletedTask; },
|
||||
NullLogger.Instance);
|
||||
events.ShouldBe(new[] { "stop", "dispose" });
|
||||
released.ShouldBeFalse();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Failed_disposal_retains_ownership_without_escaping_the_binding_failure_handler()
|
||||
{
|
||||
var events = new List<string>();
|
||||
var released = await FailedServiceLevelBindingCleanup.StopAndDisposeAsync(
|
||||
() => { events.Add("stop"); return Task.CompletedTask; },
|
||||
() => { events.Add("dispose"); throw new InvalidOperationException("controlled disposal failure"); },
|
||||
NullLogger.Instance);
|
||||
events.ShouldBe(new[] { "stop", "dispose" });
|
||||
released.ShouldBeFalse();
|
||||
}
|
||||
}
|
||||
+103
@@ -0,0 +1,103 @@
|
||||
using System.Net;
|
||||
using System.Net.Sockets;
|
||||
using Akka.Hosting;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Authorization;
|
||||
using ZB.MOM.WW.OtOpcUa.Host.OpcUa;
|
||||
using ZB.MOM.WW.OtOpcUa.OpcUaServer;
|
||||
using ZB.MOM.WW.OtOpcUa.OpcUaServer.Security;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Host.Tests.OpcUa;
|
||||
|
||||
public sealed class OtOpcUaServerHostedServiceBindingFailureTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task Binding_failure_closes_started_endpoint_and_allows_other_host_services_to_start()
|
||||
{
|
||||
using var deadline = new CancellationTokenSource(TimeSpan.FromSeconds(15));
|
||||
using var probe = new TcpListener(IPAddress.Loopback, 0);
|
||||
probe.Start();
|
||||
var port = ((IPEndPoint)probe.LocalEndpoint).Port;
|
||||
probe.Stop();
|
||||
var pki = Path.Combine(Path.GetTempPath(), $"otopcua-binding-{Guid.NewGuid():N}");
|
||||
var deferred = new DeferredServiceLevelPublisher();
|
||||
deferred.Publish(0);
|
||||
var bindingReached = false;
|
||||
var listenerObserved = false;
|
||||
OtOpcUaServerHostedService? service = null;
|
||||
var factory = new FaultInjectingLoggerFactory(() =>
|
||||
{
|
||||
using var connected = new TcpClient();
|
||||
connected.Connect(IPAddress.Loopback, port);
|
||||
listenerObserved = connected.Connected;
|
||||
bindingReached = true;
|
||||
throw new InvalidOperationException("controlled binding factory failure");
|
||||
});
|
||||
service = new OtOpcUaServerHostedService(
|
||||
Options.Create(new OpcUaApplicationHostOptions
|
||||
{
|
||||
ApplicationName = "OtOpcUa.BindingFailure",
|
||||
ApplicationUri = $"urn:OtOpcUa.BindingFailure:{Guid.NewGuid():N}",
|
||||
OpcUaPort = port, PublicHostname = "localhost", PkiStoreRoot = pki,
|
||||
}), new DeferredAddressSpaceSink(), deferred,
|
||||
NullOpcUaUserAuthenticator.Instance, NullHistorianDataSource.Instance,
|
||||
EmptyDataPlaneAuthoritySource.Instance, new ConfigurationBuilder().Build(),
|
||||
() => throw new NotSupportedException("no actor system required"), new ActorRegistry(),
|
||||
null, Options.Create(new AclOptions()), TimeProvider.System, factory);
|
||||
var admin = new AdminBootSentinel();
|
||||
using var host = new HostBuilder().ConfigureServices(services =>
|
||||
{
|
||||
services.AddSingleton<IHostedService>(service);
|
||||
services.AddSingleton<IHostedService>(admin);
|
||||
}).Build();
|
||||
try
|
||||
{
|
||||
await host.StartAsync(deadline.Token);
|
||||
listenerObserved.ShouldBeTrue("the SDK endpoint was actually listening before replay");
|
||||
bindingReached.ShouldBeTrue();
|
||||
admin.Started.ShouldBeTrue();
|
||||
using var client = new TcpClient();
|
||||
await Should.ThrowAsync<SocketException>(async () =>
|
||||
await client.ConnectAsync(IPAddress.Loopback, port, deadline.Token));
|
||||
// After construction failure, subsequent requests are accepted into the unbound cache.
|
||||
deferred.Publish(100);
|
||||
await host.StopAsync(deadline.Token);
|
||||
}
|
||||
finally
|
||||
{
|
||||
await service.DisposeAsync();
|
||||
if (Directory.Exists(pki)) Directory.Delete(pki, recursive: true);
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class FaultInjectingLoggerFactory(Action installFault) : ILoggerFactory
|
||||
{
|
||||
public ILogger CreateLogger(string categoryName)
|
||||
{
|
||||
if (categoryName == typeof(SdkServiceLevelPublisher).FullName) installFault();
|
||||
return NullLogger.Instance;
|
||||
}
|
||||
public void AddProvider(ILoggerProvider provider) { }
|
||||
public void Dispose() { }
|
||||
}
|
||||
|
||||
private sealed class AdminBootSentinel : IHostedService
|
||||
{
|
||||
public bool Started { get; private set; }
|
||||
public Task StartAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
Started = true;
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
public Task StopAsync(CancellationToken cancellationToken) => Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
+24
@@ -72,6 +72,30 @@ public sealed class OtOpcUaServerHostedServiceHistoryUpdateTests
|
||||
Service(view: null).BuildHistoryUpdatePosture().MaxFutureSkew.ShouldBe(TimeSpan.FromDays(1));
|
||||
}
|
||||
|
||||
/// <summary>#691: the ReturnBounds kill switch is bound from <c>ServerHistorian:HonorReturnBounds</c>, default on.</summary>
|
||||
[Fact]
|
||||
public void The_return_bounds_kill_switch_comes_from_options_and_defaults_on()
|
||||
{
|
||||
Service(view: null).HonorReturnBounds.ShouldBeTrue();
|
||||
var config = new ConfigurationBuilder().AddInMemoryCollection(new Dictionary<string, string?>
|
||||
{
|
||||
["ServerHistorian:HonorReturnBounds"] = "false",
|
||||
}).Build();
|
||||
Service(view: null, config: config).HonorReturnBounds.ShouldBeFalse();
|
||||
}
|
||||
|
||||
/// <summary>#694: the events horizon is bound from <c>ServerHistorian:EventReadHorizon</c>, default none.</summary>
|
||||
[Fact]
|
||||
public void The_event_read_horizon_comes_from_options_and_defaults_off()
|
||||
{
|
||||
Service(view: null).EventReadHorizon.ShouldBe(TimeSpan.Zero);
|
||||
var config = new ConfigurationBuilder().AddInMemoryCollection(new Dictionary<string, string?>
|
||||
{
|
||||
["ServerHistorian:EventReadHorizon"] = "365.00:00:00",
|
||||
}).Build();
|
||||
Service(view: null, config: config).EventReadHorizon.ShouldBe(TimeSpan.FromDays(365));
|
||||
}
|
||||
|
||||
private static OtOpcUaServerHostedService Service(
|
||||
IHistoryUpdateRoleView? view, IDriverPrimaryRoleSource? primary = null, IConfiguration? config = null,
|
||||
TimeProvider? clock = null) => new(
|
||||
|
||||
+1
@@ -97,6 +97,7 @@ public sealed class HistoryUpdateOverTheWireTests
|
||||
|
||||
var details = new ReadRawModifiedDetails
|
||||
{
|
||||
ReturnBounds = false,
|
||||
StartTime = T1.AddHours(-1), EndTime = T1.AddHours(1), NumValuesPerNode = 100, IsReadModified = true,
|
||||
};
|
||||
var response = await session.HistoryReadAsync(
|
||||
|
||||
+16
-6
@@ -13,8 +13,8 @@ namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests;
|
||||
/// comms-loss watchdog trips on). Recovery is the watchdog's sequence, replayed here as <c>DriverInstanceActor</c>
|
||||
/// runs it: ShutdownAsync, InitializeAsync, re-subscribe — after which the value comes back Good. There is no
|
||||
/// in-driver reconnect any more (#670: the SDK reconnect handler kept everything Good and Healthy forever).
|
||||
/// Since #676 a gracefully stopped server is a DEFINITIVE loss (<c>BadServerHalted</c>), so the driver also raises
|
||||
/// <c>ConnectionLost</c> and the host runs that sequence at once instead of after the watchdog's silence window.
|
||||
/// Since #676 definitive session loss also raises <c>ConnectionLost</c>, so the host runs that sequence at once
|
||||
/// instead of after the watchdog's silence window. Shutdown timing determines the SDK's loss status.
|
||||
/// </summary>
|
||||
[Trait("Category", "Integration")]
|
||||
public sealed class OpcUaClientDriverReconnectWireTests
|
||||
@@ -72,12 +72,22 @@ public sealed class OpcUaClientDriverReconnectWireTests
|
||||
await Task.Delay(TimeSpan.FromSeconds(3), ct); // three keep-alive intervals
|
||||
drv.GetHealth().LastSuccessfulRead.ShouldBe(frozen);
|
||||
|
||||
// #676: a server that HALTED (graceful stop) is a definitive loss — measured here, the SDK reports
|
||||
// BadServerHalted on every keep-alive and never re-opens the channel, so the session can never recover in
|
||||
// place. The driver says so at once, so the host re-initialises it instead of waiting out the ~90 s watchdog.
|
||||
// #676/#702: shutdown can report a halted server, an invalid/closed session, or a sustained closed
|
||||
// channel. Each proves definitive loss; #702 reproduced BadSessionIdInvalid during this graceful stop.
|
||||
// Deterministic ConnectionLossSignal tests cover the individual statuses and the sustained-channel gate.
|
||||
await WaitAsync(() => { lock (lost) return lost.Count > 0; },
|
||||
"a halted upstream must raise ConnectionLost", TimeSpan.FromSeconds(10));
|
||||
lock (lost) lost.Single().ShouldContain("BadServerHalted");
|
||||
lock (lost)
|
||||
{
|
||||
var reason = lost.Single();
|
||||
string[] definitiveStatuses =
|
||||
[
|
||||
"BadSessionIdInvalid", "BadSessionClosed", "BadSessionNotActivated", "BadServerHalted",
|
||||
"BadSecureChannelClosed", "BadConnectionClosed",
|
||||
];
|
||||
definitiveStatuses.ShouldContain(status => reason.EndsWith(status, StringComparison.Ordinal),
|
||||
$"the restart must report definitive loss, but reported: {reason}");
|
||||
}
|
||||
});
|
||||
|
||||
// The restarted server knows nothing of the old session, so the driver stays down on its own …
|
||||
|
||||
+71
@@ -0,0 +1,71 @@
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Opc.Ua;
|
||||
using Opc.Ua.Client;
|
||||
using Opc.Ua.Server;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests;
|
||||
|
||||
/// <summary>Local loopback evidence that late binding replaces the SDK default on the wire.</summary>
|
||||
public sealed class ServiceLevelBindingReplayTests
|
||||
{
|
||||
[Theory]
|
||||
[InlineData((byte)100)]
|
||||
[InlineData((byte)0)]
|
||||
public async Task Late_binding_replays_latest_request_to_standard_ServiceLevel_read(byte latest)
|
||||
{
|
||||
using var deadline = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken);
|
||||
deadline.CancelAfter(TimeSpan.FromSeconds(15));
|
||||
var ct = deadline.Token;
|
||||
var pki = Path.Combine(Path.GetTempPath(), $"otopcua-replay-{Guid.NewGuid():N}");
|
||||
var clientPki = TestClientSecurity.AllocatePkiRoot();
|
||||
var port = 0;
|
||||
try
|
||||
{
|
||||
var deferred = new DeferredServiceLevelPublisher();
|
||||
deferred.Publish(240);
|
||||
deferred.Publish(latest);
|
||||
var started = await OpcUaHostStart.StartAsync(p =>
|
||||
{
|
||||
port = p;
|
||||
return new OpcUaApplicationHost(new OpcUaApplicationHostOptions
|
||||
{
|
||||
ApplicationName = "OtOpcUa.Replay",
|
||||
ApplicationUri = $"urn:OtOpcUa.Replay:{Path.GetFileName(pki)}",
|
||||
OpcUaPort = p, PublicHostname = "localhost", PkiStoreRoot = pki,
|
||||
}, NullLogger<OpcUaApplicationHost>.Instance);
|
||||
}, () => new StandardServer(), ct);
|
||||
await using var host = started.Host;
|
||||
deferred.SetInner(new SdkServiceLevelPublisher(started.Server.CurrentInstance,
|
||||
NullLogger<SdkServiceLevelPublisher>.Instance));
|
||||
var config = new ApplicationConfiguration
|
||||
{
|
||||
ApplicationName = "OtOpcUa.ReplayClient",
|
||||
ApplicationUri = $"urn:OtOpcUa.ReplayClient:{Guid.NewGuid():N}",
|
||||
ApplicationType = ApplicationType.Client,
|
||||
SecurityConfiguration = TestClientSecurity.Build(clientPki),
|
||||
ClientConfiguration = new ClientConfiguration { DefaultSessionTimeout = 10_000 },
|
||||
TransportQuotas = new TransportQuotas { OperationTimeout = 5_000 },
|
||||
};
|
||||
await config.ValidateAsync(ApplicationType.Client, ct);
|
||||
config.CertificateValidator.CertificateValidation += (_, e) => e.Accept = true;
|
||||
var endpoint = await CoreClientUtils.SelectEndpointAsync(config,
|
||||
$"opc.tcp://localhost:{port}/OtOpcUa", false, DefaultTelemetry.Create(_ => { }), ct);
|
||||
using var session = await new DefaultSessionFactory(DefaultTelemetry.Create(_ => { })).CreateAsync(
|
||||
config, new ConfiguredEndpoint(null, endpoint, EndpointConfiguration.Create(config)),
|
||||
updateBeforeConnect: false, checkDomain: false, sessionName: "ServiceLevelReplay",
|
||||
sessionTimeout: 10_000, identity: new UserIdentity(new AnonymousIdentityToken()),
|
||||
preferredLocales: null, ct: ct);
|
||||
var read = await session.ReadValueAsync(new Opc.Ua.NodeId(2267u), ct);
|
||||
read.StatusCode.ShouldBe(StatusCodes.Good);
|
||||
read.Value.ShouldBe(latest);
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (Directory.Exists(pki)) Directory.Delete(pki, recursive: true);
|
||||
if (Directory.Exists(clientPki)) Directory.Delete(clientPki, recursive: true);
|
||||
}
|
||||
}
|
||||
}
|
||||
+4
-4
@@ -6,9 +6,9 @@ namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.Tests;
|
||||
|
||||
public sealed class DeferredServiceLevelPublisherTests
|
||||
{
|
||||
/// <summary>Verifies that publish before SetInner is a safe no-op.</summary>
|
||||
/// <summary>Verifies that publish before SetInner safely accepts a cached request.</summary>
|
||||
[Fact]
|
||||
public void Publish_before_SetInner_is_a_safe_noop()
|
||||
public void Publish_before_SetInner_accepts_a_cached_request()
|
||||
{
|
||||
var deferred = new DeferredServiceLevelPublisher();
|
||||
|
||||
@@ -28,9 +28,9 @@ public sealed class DeferredServiceLevelPublisherTests
|
||||
recording.LastValue.ShouldBe((byte)200);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that SetInner with null reverts to the null publisher.</summary>
|
||||
/// <summary>Verifies that SetInner with null detaches the previous publisher.</summary>
|
||||
[Fact]
|
||||
public void SetInner_null_reverts_to_Null_publisher()
|
||||
public void SetInner_null_detaches_the_previous_publisher()
|
||||
{
|
||||
var recording = new RecordingPublisher();
|
||||
var deferred = new DeferredServiceLevelPublisher();
|
||||
|
||||
@@ -0,0 +1,63 @@
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.Tests;
|
||||
|
||||
/// <summary>Gitea #694: <see cref="EventHorizon.TryClamp"/> — the pure window arithmetic of <c>ServerHistorian:EventReadHorizon</c>.</summary>
|
||||
public sealed class EventHorizonTests
|
||||
{
|
||||
private static readonly DateTime Now = new(2026, 10, 13, 12, 0, 0, DateTimeKind.Utc);
|
||||
private static readonly TimeSpan H = TimeSpan.FromDays(30);
|
||||
private static readonly DateTime Floor = Now - H;
|
||||
|
||||
[Fact]
|
||||
public void No_horizon_changes_nothing()
|
||||
{
|
||||
var (s, e) = (DateTime.MinValue, Now);
|
||||
EventHorizon.TryClamp(ref s, ref e, reverse: false, TimeSpan.Zero, Now).ShouldBeTrue();
|
||||
(s, e).ShouldBe((DateTime.MinValue, Now));
|
||||
EventHorizon.TryClamp(ref s, ref e, reverse: false, TimeSpan.FromDays(-1), Now).ShouldBeTrue();
|
||||
(s, e).ShouldBe((DateTime.MinValue, Now));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_forward_window_starts_at_the_floor()
|
||||
{
|
||||
var (s, e) = (DateTime.UnixEpoch, Now);
|
||||
EventHorizon.TryClamp(ref s, ref e, reverse: false, H, Now).ShouldBeTrue();
|
||||
(s, e).ShouldBe((Floor, Now));
|
||||
|
||||
var (s2, e2) = (Floor.AddDays(1), Now);
|
||||
EventHorizon.TryClamp(ref s2, ref e2, reverse: false, H, Now).ShouldBeTrue();
|
||||
s2.ShouldBe(Floor.AddDays(1)); // already inside: unchanged
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_reverse_window_keeps_the_floor_instant()
|
||||
{
|
||||
var (s, e) = (Now, new DateTime(0, DateTimeKind.Utc));
|
||||
EventHorizon.TryClamp(ref s, ref e, reverse: true, H, Now).ShouldBeTrue();
|
||||
(s, e).ShouldBe((Now, Floor.AddTicks(-1))); // (end, start] — the floor itself is read
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_window_wholly_before_the_floor_is_empty()
|
||||
{
|
||||
var (s, e) = (Floor.AddDays(-2), Floor);
|
||||
EventHorizon.TryClamp(ref s, ref e, reverse: false, H, Now).ShouldBeFalse(); // [start, end) ends at the floor
|
||||
|
||||
var (rs, re) = (Floor.AddTicks(-1), Floor.AddDays(-2));
|
||||
EventHorizon.TryClamp(ref rs, ref re, reverse: true, H, Now).ShouldBeFalse();
|
||||
|
||||
var (ok, oe) = (Floor, Floor.AddDays(-2));
|
||||
EventHorizon.TryClamp(ref ok, ref oe, reverse: true, H, Now).ShouldBeTrue(); // (…, floor] holds the floor
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_horizon_longer_than_the_calendar_is_no_horizon()
|
||||
{
|
||||
var (s, e) = (DateTime.MinValue, Now);
|
||||
EventHorizon.TryClamp(ref s, ref e, reverse: false, TimeSpan.MaxValue, Now).ShouldBeTrue();
|
||||
s.ShouldBe(DateTime.MinValue);
|
||||
}
|
||||
}
|
||||
@@ -384,6 +384,7 @@ public sealed class NodeManagerAclEnforcementTests : IDisposable
|
||||
context,
|
||||
new ReadRawModifiedDetails
|
||||
{
|
||||
ReturnBounds = false,
|
||||
StartTime = DateTime.UtcNow.AddHours(-1),
|
||||
EndTime = DateTime.UtcNow,
|
||||
NumValuesPerNode = 10,
|
||||
|
||||
+523
@@ -0,0 +1,523 @@
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Opc.Ua;
|
||||
using Opc.Ua.Server;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using HistorianRead = ZB.MOM.WW.OtOpcUa.Core.Abstractions.HistoryReadResult;
|
||||
using SdkHistoryReadResult = Opc.Ua.HistoryReadResult;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.Tests;
|
||||
|
||||
/// <summary>Boots ONE real SDK server for the whole bounds suite (the cases swap the historian source per read).</summary>
|
||||
public sealed class BoundsServerFixture : IAsyncLifetime
|
||||
{
|
||||
private readonly string _pkiRoot = Path.Combine(Path.GetTempPath(), $"otopcua-bounds-{Guid.NewGuid():N}");
|
||||
private OpcUaApplicationHost? _host;
|
||||
|
||||
/// <summary>The node manager under test.</summary>
|
||||
public OtOpcUaNodeManager Nm { get; private set; } = null!;
|
||||
|
||||
/// <summary>The historized variable every case reads.</summary>
|
||||
public NodeId NodeId { get; private set; } = null!;
|
||||
|
||||
/// <summary>Serialises the cases (they share the node manager's source).</summary>
|
||||
public SemaphoreSlim Gate { get; } = new(1, 1);
|
||||
|
||||
public async ValueTask InitializeAsync()
|
||||
{
|
||||
var started = await OpcUaHostStart.StartAsync(
|
||||
port => new OpcUaApplicationHost(
|
||||
new OpcUaApplicationHostOptions
|
||||
{
|
||||
ApplicationName = "OtOpcUa.BoundsTest",
|
||||
ApplicationUri = $"urn:OtOpcUa.BoundsTest:{Guid.NewGuid():N}",
|
||||
OpcUaPort = port,
|
||||
PublicHostname = "localhost",
|
||||
PkiStoreRoot = _pkiRoot,
|
||||
},
|
||||
NullLogger<OpcUaApplicationHost>.Instance),
|
||||
() => new OtOpcUaSdkServer(), CancellationToken.None);
|
||||
_host = started.Host;
|
||||
Nm = started.Server.NodeManager!;
|
||||
Nm.HistoryContinuationStore = new InMemoryHistoryContinuationStore();
|
||||
Nm.EnsureVariable("eq-1/bounds", parentFolderNodeId: null, displayName: "Bounds", dataType: "Double",
|
||||
writable: false, historianTagname: "WW.Bounds", realm: AddressSpaceRealm.Uns);
|
||||
NodeId = Nm.TryGetVariable("eq-1/bounds")!.NodeId;
|
||||
}
|
||||
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
if (_host is not null) await _host.DisposeAsync();
|
||||
if (Directory.Exists(_pkiRoot))
|
||||
{
|
||||
try { Directory.Delete(_pkiRoot, recursive: true); }
|
||||
catch { /* best-effort */ }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #691: HistoryRead-Raw <c>ReturnBounds</c> (OPC UA Part 11 §4.6 bounding values, §6.5.3.2). The oracle is Part 11
|
||||
/// Table 1 itself — every row, over its data set (samples at 5:00, 5:02, 5:03, 5:05, 5:06) — run against three backend
|
||||
/// shapes: one that reads newest-first natively (SqlServer-like), one that reads forward only (Gateway-like, walked back
|
||||
/// by <see cref="ReverseRawReader"/>) and a microsecond one (QuestDB-like,
|
||||
/// truncating its window bounds).
|
||||
/// </summary>
|
||||
public sealed class NodeManagerHistoryReadBoundsTests(BoundsServerFixture fx) : IClassFixture<BoundsServerFixture>
|
||||
{
|
||||
private static readonly DateTime Day = new(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc);
|
||||
private static readonly DateTime Unspecified = DateTime.MinValue;
|
||||
|
||||
private static DateTime T(string hhmm)
|
||||
{
|
||||
var parts = hhmm.Split(':');
|
||||
return Day.AddHours(int.Parse(parts[0])).AddMinutes(int.Parse(parts[1]));
|
||||
}
|
||||
|
||||
/// <summary>Part 11 Table 1's data: value = minutes past 5:00.</summary>
|
||||
private static DataValueSnapshot[] TableData() =>
|
||||
new[] { "5:00", "5:02", "5:03", "5:05", "5:06" }
|
||||
.Select(s => new DataValueSnapshot((double)(T(s) - T("5:00")).TotalMinutes, StatusCodes.Good, T(s), T(s)))
|
||||
.ToArray();
|
||||
|
||||
public enum Shape { Reverse, ForwardOnly, Microsecond }
|
||||
|
||||
private static IHistorianDataSource Source(Shape shape, IReadOnlyList<DataValueSnapshot> series) => shape switch
|
||||
{
|
||||
Shape.Reverse => new SeriesSource(series, reverse: true, inclusiveForwardEnd: false, resolution: 1),
|
||||
Shape.ForwardOnly => new SeriesSource(series, reverse: false, inclusiveForwardEnd: false, resolution: 1),
|
||||
_ => new SeriesSource(series, reverse: true, inclusiveForwardEnd: false, resolution: 10),
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// Part 11 Table 1, every row, verbatim. "FIRST"/"LAST" are <c>Bad_BoundNotFound</c> placeholders (null value) at the
|
||||
/// StartTime/EndTime; a/b footnotes: one second after/before the previous returned value when that side is unspecified.
|
||||
/// </summary>
|
||||
public static TheoryData<string, string, uint, bool, string> Table1 => new()
|
||||
{
|
||||
{ "5:00", "5:05", 0, true, "5:00, 5:02, 5:03, 5:05" },
|
||||
{ "5:00", "5:05", 0, false, "5:00, 5:02, 5:03" },
|
||||
{ "5:01", "5:04", 0, true, "5:00, 5:02, 5:03, 5:05" },
|
||||
{ "5:01", "5:04", 0, false, "5:02, 5:03" },
|
||||
{ "5:05", "5:00", 0, true, "5:05, 5:03, 5:02, 5:00" },
|
||||
{ "5:05", "5:00", 0, false, "5:05, 5:03, 5:02" },
|
||||
{ "5:04", "5:01", 0, true, "5:05, 5:03, 5:02, 5:00" },
|
||||
{ "5:04", "5:01", 0, false, "5:03, 5:02" },
|
||||
{ "4:59", "5:05", 0, true, "FIRST, 5:00, 5:02, 5:03, 5:05" },
|
||||
{ "4:59", "5:05", 0, false, "5:00, 5:02, 5:03" },
|
||||
{ "5:01", "5:07", 0, true, "5:00, 5:02, 5:03, 5:05, 5:06, LAST" },
|
||||
{ "5:01", "5:07", 0, false, "5:02, 5:03, 5:05, 5:06" },
|
||||
{ "5:00", "5:05", 3, true, "5:00, 5:02, 5:03" },
|
||||
{ "5:00", "5:05", 3, false, "5:00, 5:02, 5:03" },
|
||||
{ "5:01", "5:04", 3, true, "5:00, 5:02, 5:03" },
|
||||
{ "5:01", "5:04", 3, false, "5:02, 5:03" },
|
||||
{ "5:05", "5:00", 3, true, "5:05, 5:03, 5:02" },
|
||||
{ "5:05", "5:00", 3, false, "5:05, 5:03, 5:02" },
|
||||
{ "5:04", "5:01", 3, true, "5:05, 5:03, 5:02" },
|
||||
{ "5:04", "5:01", 3, false, "5:03, 5:02" },
|
||||
{ "4:59", "5:05", 3, true, "FIRST, 5:00, 5:02" },
|
||||
{ "4:59", "5:05", 3, false, "5:00, 5:02, 5:03" },
|
||||
{ "5:01", "5:07", 3, true, "5:00, 5:02, 5:03" },
|
||||
{ "5:01", "5:07", 3, false, "5:02, 5:03, 5:05" },
|
||||
{ "5:00", "", 3, true, "5:00, 5:02, 5:03" },
|
||||
{ "5:00", "", 3, false, "5:00, 5:02, 5:03" },
|
||||
{ "5:00", "", 6, true, "5:00, 5:02, 5:03, 5:05, 5:06, LAST+" },
|
||||
{ "5:00", "", 6, false, "5:00, 5:02, 5:03, 5:05, 5:06" },
|
||||
{ "5:07", "", 6, true, "5:06, LAST+" },
|
||||
{ "5:07", "", 6, false, "" },
|
||||
{ "", "5:06", 3, true, "5:06, 5:05, 5:03" },
|
||||
{ "", "5:06", 3, false, "5:06, 5:05, 5:03" },
|
||||
{ "", "5:06", 6, true, "5:06, 5:05, 5:03, 5:02, 5:00, FIRST-" },
|
||||
{ "", "5:06", 6, false, "5:06, 5:05, 5:03, 5:02, 5:00" },
|
||||
{ "", "4:48", 6, true, "5:00, FIRST-" },
|
||||
{ "", "4:48", 6, false, "" },
|
||||
{ "4:48", "4:48", 0, true, "FIRST, 5:00" },
|
||||
{ "4:48", "4:48", 0, false, "" },
|
||||
{ "4:48", "4:48", 1, true, "FIRST" },
|
||||
{ "4:48", "4:48", 1, false, "" },
|
||||
{ "4:48", "4:48", 2, true, "FIRST, 5:00" },
|
||||
{ "5:00", "5:00", 0, true, "5:00, 5:02" },
|
||||
{ "5:00", "5:00", 0, false, "5:00" },
|
||||
{ "5:00", "5:00", 1, true, "5:00" },
|
||||
{ "5:00", "5:00", 1, false, "5:00" },
|
||||
{ "5:01", "5:01", 0, true, "5:00, 5:02" },
|
||||
{ "5:01", "5:01", 0, false, "" },
|
||||
{ "5:01", "5:01", 1, true, "5:00" },
|
||||
{ "5:01", "5:01", 1, false, "" },
|
||||
};
|
||||
|
||||
[Theory]
|
||||
[MemberData(nameof(Table1))]
|
||||
public async Task Part11_table1_first_page_on_a_reverse_capable_backend(string start, string end, uint max, bool bounds, string expected) =>
|
||||
await AssertFirstPage(Shape.Reverse, start, end, max, bounds, expected);
|
||||
|
||||
[Theory]
|
||||
[MemberData(nameof(Table1))]
|
||||
public async Task Part11_table1_first_page_on_a_forward_only_backend(string start, string end, uint max, bool bounds, string expected) =>
|
||||
await AssertFirstPage(Shape.ForwardOnly, start, end, max, bounds, expected);
|
||||
|
||||
[Theory]
|
||||
[MemberData(nameof(Table1))]
|
||||
public async Task Part11_table1_first_page_on_a_microsecond_backend(string start, string end, uint max, bool bounds, string expected) =>
|
||||
await AssertFirstPage(Shape.Microsecond, start, end, max, bounds, expected);
|
||||
|
||||
private async Task AssertFirstPage(Shape shape, string start, string end, uint max, bool bounds, string expected)
|
||||
{
|
||||
var s = start.Length == 0 ? Unspecified : T(start);
|
||||
var e = end.Length == 0 ? Unspecified : T(end);
|
||||
var source = Source(shape, TableData());
|
||||
var (values, error, _) = await ReadAsync(source, s, e, max, bounds, cp: null);
|
||||
|
||||
error.StatusCode.Code.ShouldBe(StatusCodes.Good);
|
||||
Render(values, s, e).ShouldBe(expected);
|
||||
(source as SeriesSource)?.ReverseWindowsOnForwardOnly.ShouldBe(0);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Part 11: bounds count toward NumValuesPerNode, the start bound comes with the first page and the end bound with the
|
||||
/// last — so paging a bounded two-sided read at any page size concatenates to the unpaged (N = 0) answer.
|
||||
/// </summary>
|
||||
[Theory]
|
||||
[InlineData(Shape.Reverse)]
|
||||
[InlineData(Shape.ForwardOnly)]
|
||||
[InlineData(Shape.Microsecond)]
|
||||
public async Task Paging_a_bounded_read_concatenates_to_the_unpaged_answer(Shape shape)
|
||||
{
|
||||
var windows = new[]
|
||||
{
|
||||
("5:00", "5:05"), ("5:01", "5:04"), ("5:05", "5:00"), ("5:04", "5:01"), ("4:59", "5:05"), ("5:01", "5:07"),
|
||||
("4:48", "4:48"), ("5:00", "5:00"), ("5:01", "5:01"), ("5:07", "4:00"), ("5:06", "5:06"),
|
||||
};
|
||||
foreach (var (start, end) in windows)
|
||||
{
|
||||
var source = Source(shape, TableData());
|
||||
var all = await ReadAsync(source, T(start), T(end), 0, true, cp: null);
|
||||
all.Cp.ShouldBeNull();
|
||||
var expected = Render(all.Values, T(start), T(end));
|
||||
for (uint n = 1; n <= 6; n++)
|
||||
{
|
||||
var paged = await DrainAsync(source, T(start), T(end), n, bounds: true);
|
||||
Render(paged, T(start), T(end)).ShouldBe(expected, $"{start}->{end} paged by {n}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>One-sided reads page to the open placeholder exactly once (footnotes a/b), never past it.</summary>
|
||||
[Theory]
|
||||
[InlineData(Shape.Reverse)]
|
||||
[InlineData(Shape.ForwardOnly)]
|
||||
[InlineData(Shape.Microsecond)]
|
||||
public async Task Paging_a_one_sided_bounded_read_ends_with_the_open_placeholder(Shape shape)
|
||||
{
|
||||
for (uint n = 1; n <= 7; n++)
|
||||
{
|
||||
var fwd = await DrainAsync(Source(shape, TableData()), T("5:01"), Unspecified, n, bounds: true);
|
||||
Render(fwd, T("5:01"), Unspecified).ShouldBe("5:00, 5:02, 5:03, 5:05, 5:06, LAST+", $"forward by {n}");
|
||||
var rev = await DrainAsync(Source(shape, TableData()), Unspecified, T("5:04"), n, bounds: true);
|
||||
Render(rev, Unspecified, T("5:04")).ShouldBe("5:05, 5:03, 5:02, 5:00, FIRST-", $"reverse by {n}");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>A bound is a stored sample of any quality: a Bad sample before StartTime is the start bound, as stored.</summary>
|
||||
[Theory]
|
||||
[InlineData(Shape.Reverse)]
|
||||
[InlineData(Shape.ForwardOnly)]
|
||||
[InlineData(Shape.Microsecond)]
|
||||
public async Task A_bad_sample_is_returned_as_the_bound_as_stored(Shape shape)
|
||||
{
|
||||
var series = TableData();
|
||||
series[0] = new DataValueSnapshot(null, StatusCodes.BadNoCommunication, T("5:00"), T("5:00"));
|
||||
var (values, _, _) = await ReadAsync(Source(shape, series), T("5:01"), T("5:04"), 0, true, cp: null);
|
||||
|
||||
values[0].StatusCode.Code.ShouldBe(StatusCodes.BadNoCommunication);
|
||||
values[0].SourceTimestamp.ShouldBe(T("5:00"));
|
||||
values.Count.ShouldBe(4);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Ties on a bound: the start bound before a forward window is the LAST of a tie cluster (the sample immediately
|
||||
/// before StartTime in the read's order); the end bound after it is the FIRST of one; and ties ON StartTime are all
|
||||
/// data (the bound is the first of them, so nothing extra is added).
|
||||
/// </summary>
|
||||
[Theory]
|
||||
[InlineData(Shape.Reverse)]
|
||||
[InlineData(Shape.ForwardOnly)]
|
||||
[InlineData(Shape.Microsecond)]
|
||||
public async Task Tie_clusters_on_either_side_yield_one_bound_each(Shape shape)
|
||||
{
|
||||
var series = new List<DataValueSnapshot>();
|
||||
void Add(string t, double v) => series.Add(new DataValueSnapshot(v, StatusCodes.Good, T(t), T(t)));
|
||||
Add("5:00", 1); Add("5:00", 2); Add("5:00", 3);
|
||||
Add("5:02", 4);
|
||||
Add("5:05", 5); Add("5:05", 6);
|
||||
|
||||
var (fwd, _, _) = await ReadAsync(Source(shape, series), T("5:01"), T("5:04"), 0, true, cp: null);
|
||||
fwd.Select(v => (double)v.Value).ShouldBe([3d, 4d, 5d]);
|
||||
|
||||
var (rev, _, _) = await ReadAsync(Source(shape, series), T("5:04"), T("5:01"), 0, true, cp: null);
|
||||
rev.Select(v => (double)v.Value).ShouldBe([5d, 4d, 3d]);
|
||||
|
||||
var (onStart, _, _) = await ReadAsync(Source(shape, series), T("5:00"), T("5:03"), 0, true, cp: null);
|
||||
onStart.Select(v => (double)v.Value).ShouldBe([1d, 2d, 3d, 4d, 5d]);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// QuestDB truncates the window bounds it is sent to the microsecond: a sample at ⌊StartTime⌋ is inside a window that
|
||||
/// starts a few ticks later, so it is data — never returned a second time as the start bound.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task A_microsecond_backend_does_not_return_a_sample_at_the_truncated_start_twice()
|
||||
{
|
||||
var series = TableData();
|
||||
var (values, _, _) = await ReadAsync(Source(Shape.Microsecond, series), T("5:00").AddTicks(7), T("5:04"), 0, true, cp: null);
|
||||
|
||||
values.Select(v => (double)v.Value).ShouldBe([0d, 2d, 3d, 5d]);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A backend whose forward end is inclusive (the gateway's own <c>ReadRaw</c>), or a resumed read that steps onto
|
||||
/// EndTime, returns the sample AT EndTime as data — it must come back once, as the end bound.
|
||||
/// </summary>
|
||||
[Theory]
|
||||
[InlineData(0u)]
|
||||
[InlineData(1u)]
|
||||
[InlineData(2u)]
|
||||
[InlineData(3u)]
|
||||
public async Task An_inclusive_forward_end_does_not_return_the_end_bound_twice(uint max)
|
||||
{
|
||||
var source = new SeriesSource(TableData(), reverse: false, inclusiveForwardEnd: true, resolution: 1);
|
||||
var values = await DrainAsync(source, T("5:00"), T("5:05"), max, bounds: true);
|
||||
Render(values, T("5:00"), T("5:05")).ShouldBe("5:00, 5:02, 5:03, 5:05");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// #691 review: no data AND no bound found ⇒ <c>GoodNoData</c> with no values — the answer a tag with no history (and a
|
||||
/// disabled historian) gave before bounds (§6.5.3.2: bounds make the result Good only "if one or both exist"). Only
|
||||
/// when the WHOLE read is empty: a single bound found keeps the placeholders (Table 1: "4:48 4:48 1 ⇒ FIRST").
|
||||
/// </summary>
|
||||
[Theory]
|
||||
[InlineData(Shape.Reverse)]
|
||||
[InlineData(Shape.ForwardOnly)]
|
||||
[InlineData(Shape.Microsecond)]
|
||||
public async Task No_data_and_no_bound_found_is_GoodNoData_with_no_values(Shape shape)
|
||||
{
|
||||
var windows = new (string S, string E)[] { ("5:00", "5:05"), ("5:05", "5:00"), ("5:00", "5:00"), ("5:00", ""), ("", "5:05") };
|
||||
foreach (var (start, end) in windows)
|
||||
{
|
||||
var s = start.Length == 0 ? Unspecified : T(start);
|
||||
var e = end.Length == 0 ? Unspecified : T(end);
|
||||
foreach (var n in new uint[] { 0, 1, 2, 3 })
|
||||
{
|
||||
if (n == 0 && (s == Unspecified || e == Unspecified)) continue; // a one-sided read needs a cap
|
||||
var (values, error, cp) = await ReadResultAsync(Source(shape, []), s, e, n);
|
||||
values.ShouldBeEmpty($"{start}->{end} by {n}");
|
||||
cp.ShouldBeNull();
|
||||
error.StatusCode.Code.ShouldBe(StatusCodes.Good);
|
||||
LastNodeStatus.Code.ShouldBe(StatusCodes.GoodNoData, $"{start}->{end} by {n}");
|
||||
}
|
||||
}
|
||||
|
||||
// Positive control: one bound found keeps the missing one's placeholder.
|
||||
var single = await DrainAsync(Source(shape, TableData()), T("4:48"), T("4:48"), 1, bounds: true);
|
||||
Render(single, T("4:48"), T("4:48")).ShouldBe("FIRST, 5:00");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// #691 review: <c>ServerHistorian:HonorReturnBounds = false</c> restores the pre-#691 answer — ReturnBounds is ignored
|
||||
/// (Table 1's "No" rows for every "Yes" row) and a modified read with ReturnBounds is not refused for it.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task The_kill_switch_ignores_return_bounds()
|
||||
{
|
||||
await fx.Gate.WaitAsync(TestContext.Current.CancellationToken);
|
||||
try
|
||||
{
|
||||
fx.Nm.HonorReturnBounds = false;
|
||||
fx.Nm.HistorianDataSource = Source(Shape.Reverse, TableData());
|
||||
var (values, _, _) = Read(T("5:01"), T("5:07"), 0, bounds: true, cp: null, modified: false);
|
||||
Render(values, T("5:01"), T("5:07")).ShouldBe("5:02, 5:03, 5:05, 5:06");
|
||||
|
||||
var (_, modErr, _) = Read(T("5:00"), T("5:05"), 0, bounds: true, cp: null, modified: true);
|
||||
modErr.StatusCode.Code.ShouldBe(StatusCodes.BadHistoryOperationUnsupported, "no trail on this fake — not InvalidArgument");
|
||||
}
|
||||
finally
|
||||
{
|
||||
fx.Nm.HonorReturnBounds = true;
|
||||
fx.Gate.Release();
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Bounds_are_honoured_by_default()
|
||||
{
|
||||
await fx.Gate.WaitAsync(TestContext.Current.CancellationToken);
|
||||
try { fx.Nm.HonorReturnBounds.ShouldBeTrue(); }
|
||||
finally { fx.Gate.Release(); }
|
||||
}
|
||||
|
||||
/// <summary>Part 11 §6.5.3.3: a modified read with ReturnBounds is <c>Bad_InvalidArgument</c>.</summary>
|
||||
[Fact]
|
||||
public async Task A_modified_read_with_return_bounds_is_an_invalid_argument()
|
||||
{
|
||||
await fx.Gate.WaitAsync(TestContext.Current.CancellationToken);
|
||||
try
|
||||
{
|
||||
fx.Nm.HistorianDataSource = Source(Shape.Reverse, TableData());
|
||||
var (_, error, cp) = Read(T("5:00"), T("5:05"), 0, bounds: true, cp: null, modified: true);
|
||||
error.StatusCode.Code.ShouldBe(StatusCodes.BadInvalidArgument);
|
||||
cp.ShouldBeNull();
|
||||
}
|
||||
finally { fx.Gate.Release(); }
|
||||
}
|
||||
|
||||
// --- helpers --------------------------------------------------------------------------------
|
||||
|
||||
private static string Render(IReadOnlyList<DataValue> values, DateTime start, DateTime end)
|
||||
{
|
||||
var parts = new List<string>();
|
||||
for (var i = 0; i < values.Count; i++)
|
||||
{
|
||||
var v = values[i];
|
||||
if (v.StatusCode.Code != StatusCodes.BadBoundNotFound)
|
||||
{
|
||||
v.Value.ShouldNotBeNull();
|
||||
parts.Add($"{v.SourceTimestamp.Hour}:{v.SourceTimestamp.Minute:00}");
|
||||
continue;
|
||||
}
|
||||
|
||||
v.Value.ShouldBeNull("a missing bound carries a null value");
|
||||
var isFirst = i == 0 && start != Unspecified && v.SourceTimestamp == start;
|
||||
var isLast = end != Unspecified && v.SourceTimestamp == end;
|
||||
if (isFirst) { parts.Add("FIRST"); continue; }
|
||||
if (isLast) { parts.Add("LAST"); continue; }
|
||||
// Open side (footnotes a/b): one second from the previous returned value.
|
||||
i.ShouldBeGreaterThan(0);
|
||||
var prev = values[i - 1].SourceTimestamp;
|
||||
if (v.SourceTimestamp == prev.AddSeconds(1)) parts.Add("LAST+");
|
||||
else if (v.SourceTimestamp == prev.AddSeconds(-1)) parts.Add("FIRST-");
|
||||
else parts.Add($"?{v.SourceTimestamp:O}");
|
||||
}
|
||||
|
||||
return string.Join(", ", parts);
|
||||
}
|
||||
|
||||
private async Task<List<DataValue>> DrainAsync(IHistorianDataSource source, DateTime start, DateTime end, uint max, bool bounds)
|
||||
{
|
||||
var all = new List<DataValue>();
|
||||
byte[]? cp = null;
|
||||
var pages = 0;
|
||||
do
|
||||
{
|
||||
var (values, error, next) = await ReadAsync(source, start, end, max, bounds, cp);
|
||||
error.StatusCode.Code.ShouldBe(StatusCodes.Good);
|
||||
if (max > 0) values.Count.ShouldBeLessThanOrEqualTo((int)max, "bounds count toward NumValuesPerNode");
|
||||
all.AddRange(values);
|
||||
cp = next;
|
||||
(++pages).ShouldBeLessThan(50, "paging must terminate");
|
||||
}
|
||||
while (cp is not null);
|
||||
|
||||
return all;
|
||||
}
|
||||
|
||||
/// <summary>The node-level StatusCode of the last <see cref="Read"/> (GoodNoData vs Good).</summary>
|
||||
private StatusCode LastNodeStatus;
|
||||
|
||||
private async Task<(IReadOnlyList<DataValue> Values, ServiceResult Error, byte[]? Cp)> ReadResultAsync(
|
||||
IHistorianDataSource source, DateTime start, DateTime end, uint max) =>
|
||||
await ReadAsync(source, start, end, max, bounds: true, cp: null);
|
||||
|
||||
private async Task<(IReadOnlyList<DataValue> Values, ServiceResult Error, byte[]? Cp)> ReadAsync(
|
||||
IHistorianDataSource source, DateTime start, DateTime end, uint max, bool bounds, byte[]? cp)
|
||||
{
|
||||
await fx.Gate.WaitAsync(TestContext.Current.CancellationToken);
|
||||
try
|
||||
{
|
||||
fx.Nm.HistorianDataSource = source;
|
||||
return Read(start, end, max, bounds, cp, modified: false);
|
||||
}
|
||||
finally { fx.Gate.Release(); }
|
||||
}
|
||||
|
||||
private (IReadOnlyList<DataValue> Values, ServiceResult Error, byte[]? Cp) Read(
|
||||
DateTime start, DateTime end, uint max, bool bounds, byte[]? cp, bool modified)
|
||||
{
|
||||
var context = new OperationContext(new RequestHeader(), secureChannelContext: null, RequestType.HistoryRead, identity: null);
|
||||
var details = new ReadRawModifiedDetails
|
||||
{
|
||||
StartTime = start, EndTime = end, NumValuesPerNode = max, ReturnBounds = bounds, IsReadModified = modified,
|
||||
};
|
||||
var nodesToRead = new List<HistoryReadValueId> { new() { NodeId = fx.NodeId, ContinuationPoint = cp } };
|
||||
var results = new List<SdkHistoryReadResult> { null! };
|
||||
var errors = new List<ServiceResult> { null! };
|
||||
fx.Nm.HistoryRead(context, details, TimestampsToReturn.Both, releaseContinuationPoints: false, nodesToRead, results, errors);
|
||||
LastNodeStatus = results[0]?.StatusCode ?? StatusCodes.Bad;
|
||||
|
||||
var values = new List<DataValue>();
|
||||
if (results[0]?.HistoryData is { } ho && !ExtensionObject.IsNull(ho))
|
||||
values.AddRange(((HistoryData)ExtensionObject.ToEncodeable(ho)).DataValues);
|
||||
return (values, errors[0], results[0]?.ContinuationPoint);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A series-backed historian. <paramref name="reverse"/>: serves <c>start > end</c> as <c>(end, start]</c>
|
||||
/// newest-first (SqlServer/QuestDB); otherwise rejects it (the Gateway) and counts that. The forward read is
|
||||
/// <c>[start, end)</c>, or <c>[start, end]</c> when <paramref name="inclusiveForwardEnd"/> (the gateway's own end);
|
||||
/// a point read is the instant. <paramref name="resolution"/> > 1 truncates every window bound to it first (QuestDB
|
||||
/// µs); the series must already be stored at that resolution.
|
||||
/// </summary>
|
||||
private sealed class SeriesSource(IReadOnlyList<DataValueSnapshot> series, bool reverse, bool inclusiveForwardEnd, long resolution)
|
||||
: IHistorianDataSource
|
||||
{
|
||||
public int ReverseWindowsOnForwardOnly;
|
||||
|
||||
public bool SupportsReverseRaw => reverse;
|
||||
|
||||
public long TimestampResolutionTicks => resolution;
|
||||
|
||||
private DateTime Trunc(DateTime t) => new(t.Ticks - t.Ticks % resolution, DateTimeKind.Utc);
|
||||
|
||||
public Task<HistorianRead> ReadRawAsync(
|
||||
string fullReference, DateTime startUtc, DateTime endUtc, uint maxValuesPerNode, CancellationToken cancellationToken)
|
||||
{
|
||||
static DateTime Ts(DataValueSnapshot s) => s.SourceTimestampUtc ?? DateTime.MinValue;
|
||||
startUtc = Trunc(startUtc);
|
||||
endUtc = Trunc(endUtc);
|
||||
List<DataValueSnapshot> window;
|
||||
if (startUtc > endUtc)
|
||||
{
|
||||
if (!reverse)
|
||||
{
|
||||
Interlocked.Increment(ref ReverseWindowsOnForwardOnly);
|
||||
throw new InvalidOperationException("start_time must be less than or equal to end_time. (InvalidArgument)");
|
||||
}
|
||||
|
||||
window = series.Where(s => Ts(s) > endUtc && Ts(s) <= startUtc).Reverse().ToList();
|
||||
}
|
||||
else if (startUtc == endUtc) window = series.Where(s => Ts(s) == startUtc).ToList();
|
||||
else window = series.Where(s => Ts(s) >= startUtc && (inclusiveForwardEnd ? Ts(s) <= endUtc : Ts(s) < endUtc)).ToList();
|
||||
|
||||
var page = maxValuesPerNode == 0 ? window : window.Take((int)maxValuesPerNode).ToList();
|
||||
return Task.FromResult(new HistorianRead(page, null));
|
||||
}
|
||||
|
||||
public Task<HistorianRead> ReadProcessedAsync(
|
||||
string fullReference, DateTime startUtc, DateTime endUtc, TimeSpan interval,
|
||||
HistoryAggregateType aggregate, CancellationToken cancellationToken) =>
|
||||
Task.FromResult(new HistorianRead(Array.Empty<DataValueSnapshot>(), null));
|
||||
|
||||
public Task<HistorianRead> ReadAtTimeAsync(
|
||||
string fullReference, IReadOnlyList<DateTime> timestampsUtc, CancellationToken cancellationToken) =>
|
||||
Task.FromResult(new HistorianRead(Array.Empty<DataValueSnapshot>(), null));
|
||||
|
||||
public Task<HistoricalEventsResult> ReadEventsAsync(
|
||||
string? sourceName, DateTime startUtc, DateTime endUtc, int maxEvents, CancellationToken cancellationToken) =>
|
||||
Task.FromResult(new HistoricalEventsResult(Array.Empty<HistoricalEvent>(), null));
|
||||
|
||||
public HistorianHealthSnapshot GetHealthSnapshot() => NullHistorianDataSource.Instance.GetHealthSnapshot();
|
||||
|
||||
public void Dispose() { }
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user