Compare commits
114 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| dccab57d39 | |||
| a935e21bf8 | |||
| 3ec6006f57 | |||
| 618d98e822 | |||
| 5897764024 | |||
| 97668d26d3 | |||
| 66a2b0c3e9 | |||
| de8bc7a12c | |||
| 3c3513ea3e | |||
| a5a0496176 | |||
| 2609c5462e | |||
| 68a1f31397 | |||
| aa923cea83 | |||
| e04e1bf127 | |||
| 0d249f3ee9 | |||
| 3d7eea6280 | |||
| 8dfa165110 | |||
| 37390139fe | |||
| 15ccd9b722 | |||
| 942ca7bc14 | |||
| df0c1f3513 | |||
| ba46f4f47a | |||
| 934909b430 | |||
| ebc6b8d4d9 | |||
| f6681544e7 | |||
| 92aab7282c | |||
| 60febbf2dc | |||
| 88141d6ed5 | |||
| 9bb1363cd6 | |||
| 574c3aa1b0 | |||
| d28e3e64f1 | |||
| ccda27d220 | |||
| d277f34a96 | |||
| 186b7d5ea5 | |||
| d33992034d | |||
| 71d031fcd0 | |||
| f0d064a68c | |||
| 830b47f624 | |||
| 7b86059daf | |||
| ab5030174a | |||
| 0e47e887de | |||
| dbffb6bdb4 | |||
| 39552e15b9 | |||
| 1f5d310071 | |||
| a3ee917fb4 | |||
| 8114f13a28 | |||
| 6796ba4126 | |||
| 9e47532283 | |||
| 44a03bedc0 | |||
| 9ced6455ad | |||
| 2d03268975 | |||
| cf1af55720 | |||
| be2f0cc14d | |||
| 410c8d8976 | |||
| 70fb1f1fda | |||
| ce3369ebca | |||
| fe644d7eda | |||
| 4a3b12564a | |||
| 910befc84c | |||
| cc0cc8cd89 | |||
| 549d2d68b3 | |||
| 197246c3c1 | |||
| 9d5eef658f | |||
| 5f225f0b44 | |||
| 78f604e0cd | |||
| dbdf29c8fc | |||
| c0d6bacd9c | |||
| a11ab0dc08 | |||
| 113f1eaf7c | |||
| c019d0fc2b | |||
| ede8610804 | |||
| 6ea3ba44da | |||
| 8ebee2ac40 | |||
| d266ff83f4 | |||
| 2ceed561ad | |||
| 11f792538e | |||
| b5e3e5108c | |||
| b8344bb151 | |||
| dbfe2fc4de | |||
| f6a2628656 | |||
| 0d206a0884 | |||
| b61975f7d7 | |||
| 19a4c093be | |||
| fb5ec80fe6 | |||
| 6d5c1c768b | |||
| beb1c583b1 | |||
| 945ec2c069 | |||
| 7349260231 | |||
| 50c09e21d5 | |||
| 27bf4826d6 | |||
| 029a916288 | |||
| 232c03ed1b | |||
| 7594120a55 | |||
| 93d6c1da67 | |||
| 30a9d5f094 | |||
| fccffd0a0f | |||
| 9120182314 | |||
| 27ca6cd791 | |||
| 8e6ac4b344 | |||
| e59ab69398 | |||
| 8d154c6a81 | |||
| 0658d244d0 | |||
| 69190b14f1 | |||
| 1375f9349e | |||
| 045a21da14 | |||
| f292cf3beb | |||
| 3067146be3 | |||
| fe919f8b08 | |||
| f0b409f999 | |||
| f9ab2f7293 | |||
| 1e343b0c52 | |||
| 42eb96af9e | |||
| f41df6710c | |||
| 825393299e |
@@ -321,6 +321,13 @@ non-`Healthy` state **and** its `LastSuccessfulRead` has been frozen for 90 s. T
|
||||
`DefaultReconnectInterval`. Unlike the #523 drift detector this generates **no** traffic on a healthy
|
||||
system — it is a local `GetHealth()` read, and only reaches the network once a driver has already stopped
|
||||
reading.
|
||||
- **Fast path for DEFINITIVE losses (#676):** a driver implementing the opt-in `IDriverConnectionLossSource` raises
|
||||
`ConnectionLost` (once per connection) when its protocol proves the connection can never recover in place — OpcUaClient:
|
||||
keep-alive `BadSessionIdInvalid`/`BadSessionClosed`/`BadSessionNotActivated`/`BadServerHalted`, a SUSTAINED closed
|
||||
connection (2nd `BadSecureChannelClosed`/`BadConnectionClosed` in one outage — what a real process stop produces; the
|
||||
SDK never reopens it) or a subscription the SERVER reported dropped (its publishing-stall heuristic does not raise) — and `DriverInstanceActor` (attach PreStart, detach PostStop) turns it into the same disconnect only
|
||||
while Connected; ambiguous failures (timeouts, `BadNoCommunication`, a stalled socket) must never raise and stay with
|
||||
the watchdog's conjunction. `docs/DriverLifecycle.md`.
|
||||
|
||||
- **What comms loss now does downstream — one contract, six seams (archreview R3-02).** "A value consumer
|
||||
never reports a quality better than its source's" is stated once in `docs/DriverLifecycle.md` §"The
|
||||
@@ -1343,19 +1350,23 @@ historian tagname, which is the tag's driver `FullName`.
|
||||
### Pluggable backends, SQL Server and HistoryUpdate (2026-10)
|
||||
|
||||
- **Selection:** `ServerHistorian:Backend` (`Gateway` default | `SqlServer` | `QuestDb`; unknown fails host start). `HistorianBackendRegistry` → `IHistorianBackendProvider` builds all five seams; the gateway keys are only required for `Gateway`. `SqlServer` uses `ServerHistorian:SqlServer:{ConnectionString (required), Schema=hist, CommandTimeout=30s, AutoCreateSchema=true}`.
|
||||
- **SQL Server** (`Driver.Historian.SqlServer`): `hist.Sample` PK `(TagId, SourceTimeUtc)` — writes are insert-if-absent, so outbox redelivery is idempotent. Stores every value kind with its **real status**; per-sample aggregates (12, simple definitions) are pushed down to T-SQL with bucket math in **100 ns ticks**; the Part 13 time-weighted five (TimeAverage/TimeAverage2/Total/Total2/Interpolative, #671 — ⚠️ `Total` is now the time-weighted integral, no longer `SUM`) stream window + 4 bound rows into the shared `TimeWeightedAggregator` (both SQL backends, identical results; client `AggregateConfiguration` ignored; status = the SDK `AggregateCalculator` time-based rule, Uncertain time counts as Good; reverse `Interpolative` anchored at the later time; window capped by `ServerHistorian:MaxTimeWeightedRawRows` ⇒ `BadTooManyOperations`). Docker-dev recipe, schema and aggregate semantics: `docs/Historian.md` §"SQL Server backend". Integration tests need `HISTORIAN_SQL_TEST_CONNECTION`. **Retention (#667):** opt-in `SqlServer:RetentionDays` (+ `PurgeInterval` 1h, `PurgeBatchSize` 4000) -> `HistorianRetentionService` purges old `hist.Sample` (per tag) / `hist.Event` in `DELETE TOP(n)` batches, Primary-gated on `IHistorizationRoleView`, one purger per DB via `sp_getapplock`; counter `otopcua.historian.retention.purged{table}`. Optional `IHistorianBackendProvider.CreateRetention` (default null). `docs/Historian.md` §Retention.
|
||||
- **SQL Server** (`Driver.Historian.SqlServer`): `hist.Sample` PK `(TagId, SourceTimeUtc)` — writes are insert-if-absent, so outbox redelivery is idempotent. Stores every value kind with its **real status**; per-sample aggregates (10, simple definitions) are pushed down to T-SQL with bucket math in **100 ns ticks**; the Part 13 time-weighted seven (TimeAverage/TimeAverage2/Total/Total2/Interpolative, #671 — ⚠️ `Total` is now the time-weighted integral, no longer `SUM` — plus the **duration-based** PercentGood/PercentBad, #680) stream window + 4 bound rows into the shared `TimeWeightedAggregator` (both SQL backends, identical results; reverse `Interpolative` anchored at the later time; window capped by `ServerHistorian:MaxTimeWeightedRawRows` ⇒ `BadTooManyOperations`, percentages included). ⚠️ **#680 status rules (client-visible):** TimeAverage/Total are Uncertain if any part of the interval uses an Uncertain value, is bridged across a Bad sample, is extrapolated past the last sample (⇒ a live tag's latest interval is Uncertain) or is not covered; TimeAverage2/Total2 keep the SDK time-based rule (Uncertain time counts as Good) but never extrapolate past the last sample — that interval is Partial, later ones `BadNoData` (a report-by-exception tag with nothing after its last sample reads `BadNoData` until its next sample arrives). `docs/Historian.md` §"Time-weighted aggregates". ⚠️ **#680 per-sample (client-visible):** `Count` counts **Good** samples only (value-based status; an all-Bad interval is generic Bad, value null) and an empty interval **inside the tag's data range** is `0`/Good (outside it still `BadNoData`); `Start`/`End` return the **raw** first/last sample of any quality — its own timestamp and status, value null when Bad; `Delta` is Uncertain only when the interval's first or last sample is unusable; ⚠️ #684: `Average`/`Minimum`/`Maximum`/`Range`/`*ActualTime` use **Good samples only** (was Good + Uncertain; an Uncertain-only interval is `BadNoData`), Min/Max-family status Uncertain on a Bad sample, an Uncertain value beyond the Good extreme (Part 13 — the SDK ignores that) or an Uncertain non-numeric sample, and only a **Good** String/DateTime sample refuses them (`BadAggregateInvalidInputs`; Delta still refuses any usable one); and every per-sample aggregate but `Average` sets **end-of-data Partial** on the interval holding the last sample (forward incl. its earlier edge, reverse strictly inside), never on a Bad result. The data-range lookup is a key seek on SQL Server and a `tag` symbol-index seek on QuestDB (#683 — `{p}sample.tag` is `SYMBOL INDEX`; the bootstrapper ADDs it to a pre-#683 table, idempotent), read BEFORE the bucket statement so a racing append can never yield a Good `Count` of 0 (a concurrent HistoryUpdate delete is the mirror case and still can, transiently). `docs/Historian.md` §"Aggregates". Docker-dev recipe, schema and aggregate semantics: `docs/Historian.md` §"SQL Server backend". Integration tests need `HISTORIAN_SQL_TEST_CONNECTION`. **Retention (#667):** opt-in `SqlServer:RetentionDays` (+ `PurgeInterval` 1h, `PurgeBatchSize` 4000) -> `HistorianRetentionService` purges old `hist.Sample` (per tag) / `hist.Event` in `DELETE TOP(n)` batches, Primary-gated on `IHistorizationRoleView`, one purger per DB via `sp_getapplock`; counter `otopcua.historian.retention.purged{table}`. Optional `IHistorianBackendProvider.CreateRetention` (default null). `docs/Historian.md` §Retention.
|
||||
- **QuestDB** (`Driver.Historian.QuestDb`, `ServerHistorian:QuestDb:*`, Npgsql over PG wire; docker-dev overlay `docker-compose.questdb.yml`): same kinds/status/aggregates/HistoryUpdate as SqlServer, soft delete by tombstone (INSERT only — never `UPDATE`/`DELETE`; every read filters `NOT deleted`). ⚠️ **Timestamps are µs** (`QuestDbConnection.Round`/`BindTimestamp`; `IHistorianDataSource.TimestampResolutionTicks` = 10 steps HistoryRead paging), **pre-1970 cannot be stored** (writer drops, HistoryUpdate `BadOutOfRange`), every write waits for WAL visibility, TTL is enforced on write commits only, set Npgsql `Maximum Pool Size` < QuestDB's 64. `docs/Historian.md` §"QuestDB backend".
|
||||
- ⚠️ **Reverse processed reads are Part 13/SDK-anchored (#677, client-visible):** every aggregate on SqlServer/QuestDB now anchors a reverse read at its later time — window `(end, start]`, partial interval earliest, each stamped with its later edge (was: earlier-anchored, stamped earlier edge); on forward reads Partial also marks the interval the data starts inside (not Average — the SDK's rule, forward-only); a non-default client `AggregateConfiguration` ⇒ `BadAggregateConfigurationRejected`; Gateway provisioning sends `IntegralDivisor = 1`. SDK-parity tests: `SdkAggregateParityTests`. `docs/Historian.md` §Aggregates.
|
||||
- ⚠️ **Reverse / end-only RAW reads work on Gateway (#686, client-visible):** `IHistorianDataSource.SupportsReverseRaw` (SqlServer/QuestDB/Null `true`, Gateway `false`, interface default `false`, guard-tested) — the node manager sends `start > end` only to a backend that declares it; otherwise `ReverseRawReader` serves it from FORWARD reads: uncapped ⇒ ONE forward read reversed in memory; capped ⇒ a bounded walk back in segments (ms-aligned request bounds, width aimed from observed density at ≈ N + 8 rows and carried page to page in the continuation point — #689: it used to aim at half `MaxTieClusterOverfetch`, ~32k rows per page on a dense tag; a read asks for `2N + 18` rows (a discovery segment ≤ 1,026), `MaxTieClusterOverfetch` is only the tie-cluster ceiling — a full read is re-read, never answered truncated; only more than max(1, `MaxTieClusterOverfetch`) samples within ONE MILLISECOND ⇒ `BadTooManyOperations` (per ms, not per timestamp — requests are ms-rounded, and letting a segment narrow below 1 ms stepped a page through one ms a tick per read, ~10k reads); walks no later than now + 1 day and never before the Unix epoch; #691: after two empty segments it asks once for the whole remainder, and the stretch a page proved empty rides the continuation point — the last pages of a deep page-through no longer each walk ~14 reads to 1970). The raw/events tie-path oversized-cluster refusal is now `BadTooManyOperations` too (was `BadHistoryOperationUnsupported`). Uncapped Gateway raw reads (forward too) used to send `max_values = 0`, which the client/gateway reject — now unbounded (`int.MaxValue`; a ceiling refusal ⇒ `BadTooManyOperations`), and a cancelled read is no longer a backend failure. Since #669 such reads failed the node on Gateway and counted as a backend fault (the gateway rejects start > end — `GatewayHistorianDataSource.ReadRawAsync` now refuses one unsent); before #669 an end-only Gateway read returned the OLDEST N ascending — now the latest N at/before EndTime, newest first. Events follow since #691.4: a reverse/end-only capped events read on Gateway (`SupportsReverseEvents` false; Null now reports `true` and answers empty) is walked back the same way (ties sorted by EventId — the gateway orders by time only; one extra event found ⇒ continuation point) — it used to be refused `BadHistoryOperationUnsupported` (explicit reverse) or answered with the OLDEST N ascending (end-only); needs the gateway's `RuntimeDb:EventReadMaxRows` 0/unlimited (it truncates silently). Uncapped Gateway events reads (forward too) used to throw "Array dimensions exceeded" (result list pre-sized to `int.MaxValue`) — fixed. Above a truncated read the walk gallops up from the newest row seen, then bisects (never below 1 ms): a burst that ended 100 days ago ≈ 18 reads, a dense tag that stopped days ago ≈ 26. `docs/Historian.md` §"Open-ended and reverse raw reads".
|
||||
- **Advertised aggregates (#682):** `IHistorianDataSource.SupportedAggregates` (Gateway 4 via `GatewayHistorianDataSource.ServedAggregates`, SqlServer/QuestDB 17 via `AggregateFamilies.Served`, Null **none**; a null set ⇒ all) is the one authority for `HistoryServerCapabilities/AggregateFunctions` (i=11201 — only its folder references are trimmed) and the up-front `BadAggregateNotSupported` in HistoryRead-Processed. ⚠️ `ServerCapabilities/AggregateFunctions` (i=2997) is deliberately NOT trimmed: it keeps the SDK's 37 because live `AggregateFilter` subscriptions still compute them all. ⚠️ Historian disabled ⇒ Processed is now `BadAggregateNotSupported`, not `GoodNoData`. `docs/Historian.md` §Advertised aggregates.
|
||||
- **Raw `ReturnBounds` is honoured on every backend (#691, client-visible):** Part 11 bounding values (§4.6 Table 1 is the test oracle) are composed in the server by `HistoryBounds` from each backend's raw reads — "at or before" = capped reverse read (native on SQL backends, the #686 `ReverseRawReader` walk-back on Gateway, never AVEVA's synthetic boundary row), "at or after" = capped forward read. Bounds are stored samples of any quality, count toward `NumValuesPerNode`, start bound on the first page / end bound on the last (pending end bound rides the continuation point); missing ⇒ `Bad_BoundNotFound` (null, at the bound time; a one-sided read's open side ±1 s from the previous value). ⚠️ **Breaking for SDK-default clients:** the .NET SDK's `new ReadRawModifiedDetails()` defaults BOTH `ReturnBounds = true` AND `IsReadModified = true` — a client that cleared only `IsReadModified` now gets bounds, and one that set neither is a modified read with bounds ⇒ `Bad_InvalidArgument` (§6.5.3.3); set both explicitly (Client.Shared does; tests building details must too). No data AND no bound found ⇒ `GoodNoData`, no values (the pre-bounds answer). On Gateway the at-or-after lookup ends at now + 1 day (never a year-9999 end to AVEVA). Kill switch `ServerHistorian:HonorReturnBounds` (default `true`; `false` = pre-#691). `docs/Historian.md` §Bounding values.
|
||||
- **Real status fix:** the recorder used to stamp every sample Good (`192`); it now carries the driver's status. ⚠️ On the **Gateway** backend this changes nothing stored: the gateway discards quality (C-002), so AVEVA still stores Good.
|
||||
- **Comms loss is historized as an outage marker** (`Empty` + `BadNoCommunication`, one per tag per outage) on SqlServer; dropped (`DroppedUnsupportedKind`) on Gateway. Outbox records are **format v2** — drain or delete `ContinuousHistorization:OutboxPath` before downgrading a node.
|
||||
- **HistoryUpdate** (SqlServer only; Gateway ⇒ `BadHistoryOperationUnsupported`): Insert/Replace/Update/DeleteRaw/DeleteAtTime, gated in both ACL modes — flag off needs the `HistoryUpdate` role from a **Cluster-scoped** grant (+ key re-check), flag on uses `DataPlaneAuthorizer`'s HistoryUpdate operation; `WriteOperate` never implies it. The `HistoryWrite` AccessLevel bit is set only when an updater is wired at materialise time. `docs/Historian.md` §HistoryUpdate, `docs/security.md`.
|
||||
- **HistoryUpdate** (SqlServer + QuestDb; Gateway ⇒ `BadHistoryOperationUnsupported`): Insert/Replace/Update/DeleteRaw/DeleteAtTime, **Primary-only** since #674 (off-Primary ⇒ `BadInvalidState`, after authorization AND after planning — an unservable request (Remove / `IsDeleteModified` / not historized) stays `BadHistoryOperationUnsupported` off-Primary; the verdict is `IHistoryUpdateRoleView` = the data-plane verdict but **closed until this node's first redundancy snapshot**, so a pair cannot both accept in the boot window; admin-only nodes register no view ⇒ no gate; reads ungated), items with a `SourceTimestamp` past now + `ServerHistorian:HistoryUpdateMaxFutureSkew` (default 1 day, ≤0 disables) ⇒ `BadOutOfRange` in the node manager (every backend), gated in both ACL modes — flag off needs the `HistoryUpdate` role from a **Cluster-scoped** grant (+ key re-check), flag on uses `DataPlaneAuthorizer`'s HistoryUpdate operation; `WriteOperate` never implies it. The `HistoryWrite` AccessLevel bit is set only when an updater is wired at materialise time. `docs/Historian.md` §HistoryUpdate, `docs/security.md`. ⚠️ **SqlServer schema v1 is accepted (#674)** with `AutoCreateSchema=false`: writes/reads/events/retention work; modified reads + HistoryUpdate ⇒ node-level `BadHistoryOperationUnsupported` on every arm (the updater throws `NotSupportedException`, nothing sent; the `HistoryWrite` bit is still advertised — not knowable at materialise time), retention skips `SampleModified`, one Warning per boot (`SqlServerSchemaBootstrapper.SupportsModifiedTrail`; `RequiredVersion` = 1, `ModifiedTrailVersion` = 2). QuestDB: keep `cairo.ttl.use.wall.clock=true`; a QuestDB trail INSERT retried after a client timeout duplicates (no DEDUP key).
|
||||
- **Modified-value history (#668, Part 11):** every applied HistoryUpdate item leaves a trail row (prior value, principal from the session's API key / `anonymous`, `HistoryModificationType`, time) — SqlServer `hist.SampleModified` (**schema v2**, written by `OUTPUT … INTO` in the same statement ⇒ atomic; ⚠️ no FK/CHECK/trigger on it, and `AutoCreateSchema=false` installs must run `002_sample_modified.sql` before upgrading), QuestDB `{p}sample_modified` (append-only, written **before** the change ⇒ a crash leaves an orphan trail row, never an untracked change). `IsReadModified=true` HistoryRead serves it as `HistoryModifiedData` via `IHistorianDataSource.ReadModifiedAsync` (default ⇒ `NotSupported` ⇒ `BadHistoryOperationUnsupported` on Gateway/Null), paged on `(SourceTime, ModifiedId)`, gated by plain HistoryRead. Live writes and retention never write it; retention/TTL purge it on source time. `IHistorianDataUpdater` methods take a `HistoryUpdateContext`. CLI `historyread --modified`. `docs/Historian.md` §Modified-value history.
|
||||
|
||||
### Read path (`ServerHistorian` section)
|
||||
|
||||
The server dispatches all OPC UA HistoryRead to the registered `IHistorianDataSource` — the
|
||||
`GatewayHistorianDataSource` read client when enabled, else the `NullHistorianDataSource` default
|
||||
(historized nodes return `GoodNoData`, never an error). Supported variants: Raw, Processed
|
||||
(Average/Minimum/Maximum/Total/Count aggregates), and AtTime over historized variable nodes; Events over
|
||||
(historized nodes return `GoodNoData` for Raw/AtTime — Processed is `BadAggregateNotSupported` since #682). Supported variants: Raw (⚠️ #687, client-visible: Gateway raw reads no longer return AVEVA's synthetic start-boundary row — asked from 1 ms early (epoch-floored) with `NumValuesPerNode + 16` rows (unclamped; 0 ⇒ `int.MaxValue`) and filtered by timestamp — so an empty window is `GoodNoData`, not a fabricated value at StartTime; the forward end is now exclusive (a point read inclusive); pre-window rows that crowd the budget trigger ONE re-read with room for them, else `BadTooManyOperations`; `ReturnBounds` is ignored on every backend; ⚠️ #690: a Gateway Bad row — a stored NULL, which histsdk sends as value 0 — now reads value **null** + Bad on Raw and AtTime, as on the SQL backends; ⚠️ #691.3: a strict gateway (`AllowStreamingBeyondCeiling=false`) that refuses a raw budget is re-asked ONCE at the ceiling its refusal names — raw and per-sample reads alike; the ceiling is then remembered per data source (`GatewayRowCeiling`: refusals count as gateway errors), re-learned from a refusal when lowered, and re-probed once at the full budget when a ceiling-bounded read comes back full (raised / streaming) — so an uncapped (`NumValuesPerNode = 0`) read of a small window is served; a ceiling-bounded stream that is not provably complete is `BadTooManyOperations`, never a truncated answer), Processed
|
||||
(backend-dependent: SqlServer/QuestDb serve seventeen; Gateway twelve = `GatewayHistorianDataSource.ServedAggregates` — ⚠️ #681, client-visible: the ten per-sample ones (Average, Count, Min/Max, Range, Start/End/Delta, Min/MaxActualTime) are evaluated from RAW gateway reads by the shared `Sql.Common` evaluator — identical to the SQL backends, Calculated bits, interval-start stamps, #677 reverse; Min/Max are no longer AVEVA modes, the read moves every raw sample of the window (a month of 1 Hz ⇒ `BadTooManyOperations` at the default `MaxTimeWeightedRawRows` cap), and every gateway stream is consumed to completion (an abandoned one makes the gateway evict its shared historian session) — plus TimeAverage/Total via AVEVA modes (interval-start stamped + Calculated; ⚠️ #685: AVEVA's extra pre-window row is dropped ⇒ one row per interval in `[Start, End)`, and an Uncertain (opc 64) interval keeps its value as `UncertainDataSubNormal` — AVEVA's TWA excludes NULL-gap time where the SQL backends bridge it; per-interval values still need gateway fix historiangw#27; ⚠️ #691.2: native reads use the SQL backends' grid in both directions via forward requests only — a REVERSE read (refused by the gateway before) is anchored at its later time (#677), and the clipped interval (earliest reverse, LATEST forward — which used to be averaged past EndTime) is its own exact-span read marked Partial unless Bad; an interval wider than the window is sent as the window, interval 0 as one interval; more than 100,000 intervals is refused up front (`BadHistoryOperationUnsupported`, as on the SQL backends — forward used to be served up to the gateway's 1,000,000); until historiangw#27 lands both directions return Good-status wrong values); the other time-weighted five stay `BadAggregateNotSupported` because the gateway has no prior-sample read for Part 13 bounds; `docs/Historian.md` §Aggregate set), and AtTime over historized variable nodes; Events over
|
||||
alarm-owning equipment-folder event-notifier nodes. Reads are ungated (served from any redundancy node);
|
||||
authorization uses the standard `AccessLevels.HistoryRead` bit set at materialization.
|
||||
|
||||
@@ -1371,9 +1382,10 @@ authorization uses the standard `AccessLevels.HistoryRead` bit set at materializ
|
||||
| `CaCertificatePath` | `null` | PEM CA file pinning the gateway TLS chain; null/empty uses the OS trust store |
|
||||
| `CallTimeout` | `00:00:30` | Per-call deadline for each unary gateway read |
|
||||
| `MaxTieClusterOverfetch` | `65536` | Bounded over-fetch the HistoryRead-Raw paging uses to page within an oversized same-timestamp tie cluster (retained from the prior backend) |
|
||||
| `HonorReturnBounds` | `true` | #691 kill switch: `false` ignores HistoryRead-Raw `ReturnBounds` as before (no bounding values, no modified-read `Bad_InvalidArgument`) |
|
||||
| `HistoryReadBatchParallelism` | `4` | Max nodes served concurrently within one HistoryRead batch (03/S3); `⌈N/P⌉ × CallTimeout` worst case instead of sequential |
|
||||
| `MaxConcurrentHistoryReadBatches` | `16` | Process-wide concurrent HistoryRead batch cap (03/S3); flood degrades to `BadTooManyOperations`. Total in-flight reads capped at 16 × 4 = 64 |
|
||||
| `MaxTimeWeightedRawRows` | `1000000` | `SqlServer`/`QuestDb` only (#671): raw in-window rows one time-weighted aggregate read (incl. `Total`) may stream into the C# evaluator; window query capped at this + 1, past it ⇒ `BadTooManyOperations` for that node. ≤ 0 = unbounded |
|
||||
| `MaxTimeWeightedRawRows` | `1000000` | `SqlServer`/`QuestDb` (#671): raw in-window rows one time-weighted aggregate read (incl. `Total` and, since #680, PercentGood/PercentBad) may stream into the C# evaluator — and, since #681, every `Gateway` per-sample read (raw-evaluated); window query capped at this + 1, past it ⇒ `BadTooManyOperations` for that node. ≤ 0 = unbounded. Gateway (#688): the `cap + 18` budget is clamped at the gateway's default `MaxValuesCeiling` (1,000,000) while the cap is ≤ it, so the default no longer trips a strict (`AllowStreamingBeyondCeiling=false`) gateway |
|
||||
| `HistoryReadDeadline` | `00:01:00` | Per-request HistoryRead wall-clock deadline (03/S3); a node still in flight surfaces `BadTimeout`. Non-positive = unbounded |
|
||||
|
||||
### Alarm-history path (`AlarmHistorian` section)
|
||||
@@ -1445,6 +1457,9 @@ The target HistorianGateway OtOpcUa points at **must** run with:
|
||||
|
||||
- `RuntimeDb:Enabled=true` — enables the `WriteLiveValues` SQL live path (continuous historization).
|
||||
- `RuntimeDb:EventReadsEnabled=true` — enables `ReadEvents` from `Runtime.dbo.Events` (alarm-history reads).
|
||||
- `RuntimeDb:EventReadMaxRows=0` (its default, unlimited) — or larger than the reverse-events walk's biggest read
|
||||
(`2·MaxTieClusterOverfetch + 4`). The cap truncates **silently**, so a walked-back reverse events read (#691.4) would
|
||||
lose the newest events of a segment it cut.
|
||||
- An API key carrying scopes **`historian:read`**, **`historian:write`**, **`historian:tags:write`**.
|
||||
|
||||
### Migration note (deployments upgrading from the Wonderware backend)
|
||||
|
||||
@@ -207,7 +207,9 @@ The `-v` drops the SQL volume; remove it to keep ConfigDb state across restarts.
|
||||
|
||||
The full single-mesh stack (`central-1`/`central-2` + the four site nodes) can OOM-kill `central-1` on a loaded host. Two settings in the compose file guard against that:
|
||||
|
||||
- **EF Core + ASP.NET Core logs are pinned to `Warning`** on every host node (`Serilog__MinimumLevel__Override__Microsoft.EntityFrameworkCore` / `…Microsoft.AspNetCore` = `Warning`). The host logs via Serilog (`AddZbSerilog` → `ReadFrom.Configuration`), and in `Development` the default level is `Debug` — without these overrides every Deployment-poll emits an `Executed DbCommand` / `SELECT … FROM [Deployment]` line, flooding the Serilog pipeline and starving the Akka cluster heartbeat thread. Application + Akka log levels are left untouched, so this only silences the per-poll SQL chatter. To temporarily restore the SQL log flood for debugging, drop those two env vars (or set them back to `Information`) on the node you're inspecting.
|
||||
- **EF Core + ASP.NET Core logs are pinned to `Warning`** on every host node (`Serilog__MinimumLevel__Override__Microsoft.EntityFrameworkCore` / `…Microsoft.AspNetCore` = `Warning`). The host logs via Serilog (`AddZbSerilog` → `ReadFrom.Configuration`), and a `Development` environment would default to `Debug` (`appsettings.Development.json`) — the rig does NOT set `ASPNETCORE_ENVIRONMENT`, so nodes actually run at the base `Information` default, but these pins keep the SQL chatter out should the rig ever run as Development; without them a Debug/Information-level EF pipeline emits an `Executed DbCommand` / `SELECT … FROM [Deployment]` line, flooding the Serilog pipeline and starving the Akka cluster heartbeat thread. Application + Akka log levels are left untouched, so this only silences the per-poll SQL chatter. To temporarily restore the SQL log flood for debugging, drop those two env vars (or set them back to `Information`) on the node you're inspecting.
|
||||
|
||||
To raise Debug logging for chosen categories on `central-1`/`central-2` (e.g. `ZB.MOM.WW.OtOpcUa.Runtime`), use the overlay: `docker compose -f docker-dev/docker-compose.yml -f docker-dev/docker-compose.debug.yml up -d --no-deps central-1 central-2` — it sets `Serilog__MinimumLevel__Override__<Namespace>=Debug`; edit its category list to taste.
|
||||
- **Each host node has `mem_limit: 1g`** (`mem_reservation: 512m`). A quiet solo `central-1` measures ~357 MiB; the limit leaves headroom for the deploy/UI load and per-cluster driver subscriptions that push a fully-loaded node higher. The limit/reservation live on the `&otopcua-host` anchor, so all six host services inherit them; `sql`, `traefik`, and the one-shot `migrator`/`cluster-seed` are left unbounded.
|
||||
|
||||
The full six-node host stack therefore needs roughly **6 GiB** of Docker Desktop VM memory just for the host nodes (plus SQL Server's own footprint on top). On a constrained host, either raise the Docker Desktop VM memory or run fewer host services (e.g. just `central-1` + `central-2`, or a single central node) rather than the full mesh.
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
# docker-dev/docker-compose.debug.yml — optional overlay: raise Serilog levels per category on the central pair.
|
||||
#
|
||||
# The rig nodes run with ASPNETCORE_ENVIRONMENT unset (Production), so there is NO appsettings.Development.json and the
|
||||
# Serilog default level is Information (the base appsettings.json sets none). Debug is therefore OFF by default; this
|
||||
# overlay turns it on for chosen namespaces only, via Serilog:MinimumLevel:Override:<Namespace> (env: double underscore
|
||||
# for ':', and the namespace's own dots stay dots).
|
||||
#
|
||||
# Up (recreates only the two central nodes with the extra env):
|
||||
# docker compose -f docker-dev/docker-compose.yml -f docker-dev/docker-compose.debug.yml up -d --no-deps central-1 central-2
|
||||
# Back to normal: re-run `up -d --no-deps central-1 central-2` with the base file alone.
|
||||
#
|
||||
# Edit the list below to the categories you care about. Useful ones:
|
||||
# ZB.MOM.WW.OtOpcUa.Runtime actors: DriverHostActor, OpcUaPublishActor, redundancy, deploys
|
||||
# ZB.MOM.WW.OtOpcUa.Runtime.Drivers.DriverHostActor one actor only (e.g. the #672 historization verdict logs)
|
||||
# ZB.MOM.WW.OtOpcUa.Driver.OpcUaClient a single driver
|
||||
# Opc.Ua OPC Foundation stack (very chatty)
|
||||
# Akka cluster/remoting internals (chatty; base is Information)
|
||||
# Leave Microsoft.EntityFrameworkCore / Microsoft.AspNetCore at Warning (see README "Resource limits & dev logging"):
|
||||
# a Debug SQL flood starves the Akka heartbeat thread.
|
||||
services:
|
||||
central-1:
|
||||
environment:
|
||||
Serilog__MinimumLevel__Override__ZB.MOM.WW.OtOpcUa.Runtime: "Debug"
|
||||
central-2:
|
||||
environment:
|
||||
Serilog__MinimumLevel__Override__ZB.MOM.WW.OtOpcUa.Runtime: "Debug"
|
||||
+16
-4
@@ -57,6 +57,11 @@ otopcua-cli write -u opc.tcp://localhost:4840 -n "ns=2;s=MyNode" -v 42 -U operat
|
||||
|
||||
### Failover
|
||||
|
||||
**Mapped ports and internal hostnames.** Endpoint discovery rewrites the host *and port* of the endpoint the server
|
||||
advertises to the URL you passed (when the scheme matches), so connecting through a port mapping works even though the
|
||||
server advertises its internal address (for example the docker-dev rig: `-u opc.tcp://localhost:4841` reaches the
|
||||
node mapped to 4841, although every container advertises `:4840`).
|
||||
|
||||
When `-F` is provided, the shared service tries the primary URL first, then each failover URL in order. For long-running commands (`subscribe`, `alarms`), the service monitors the session via keep-alive and automatically reconnects to the next available server on failure.
|
||||
|
||||
```bash
|
||||
@@ -205,7 +210,8 @@ otopcua-cli historyread -u opc.tcp://localhost:4840/OtOpcUa \
|
||||
-n "ns=1;s=TestMachine_001.TestHistoryValue" \
|
||||
--start "2026-03-25T00:00:00Z" --end "2026-03-30T00:00:00Z"
|
||||
|
||||
# Aggregate: 1-hour average
|
||||
# Aggregate: 1-hour average (every backend; on Gateway it is computed from raw samples since #681 —
|
||||
# capped by ServerHistorian:MaxTimeWeightedRawRows, so read long dense windows in slices)
|
||||
otopcua-cli historyread -u opc.tcp://localhost:4840/OtOpcUa \
|
||||
-n "ns=1;s=TestMachine_001.TestHistoryValue" \
|
||||
--start "2026-03-25T00:00:00Z" --end "2026-03-30T00:00:00Z" \
|
||||
@@ -229,7 +235,7 @@ continuation points up to `--max`. A server on the `Gateway` backend (no modific
|
||||
| `--start` | Start time in ISO 8601 UTC format, e.g. `2026-01-15T08:00:00Z` (default: 24 hours ago) |
|
||||
| `--end` | End time in ISO 8601 UTC format, e.g. `2026-01-15T09:00:00Z` (default: now) |
|
||||
| `--max` | Maximum number of values (default: 1000) |
|
||||
| `--aggregate` | Aggregate function: Average, Minimum, Maximum, Count, Start, End, StandardDeviation |
|
||||
| `--aggregate` | Aggregate function: Average, Minimum, Maximum, Count, Start, End, StandardDeviation, TimeAverage, TimeAverage2, Total, Total2, Interpolative |
|
||||
| `--interval` | Processing interval in milliseconds for aggregates (default: 3600000) |
|
||||
| `--modified` | Read modified-value history (Part 11 `IsReadModified`) instead of raw samples |
|
||||
|
||||
@@ -270,8 +276,9 @@ node-level code (`Good`, or `BadNoData` if the range held nothing). A node that
|
||||
updater, so no `HistoryWrite` access bit), answers `BadHistoryOperationUnsupported`; a `NaN`/`Infinity` double is `BadInvalidArgument`. The session's API key
|
||||
(`-U apikey -P otua_<keyId>_<secret>`) needs the `HistoryUpdate` permission — via a `Cluster`-scoped grant when `Security:Acl:Enabled` is off, or a scoped
|
||||
grant on the node when it is on; a key with only `WriteOperate` gets `BadUserAccessDenied`. See [Historian.md §HistoryUpdate](Historian.md#historyupdate) and
|
||||
[security.md](security.md#historyupdate-historian-writes). `historyread --aggregate` only knows the seven names in the table below; the SQL backend's other
|
||||
aggregates (Total, Range, Delta, PercentGood, …) need a different client.
|
||||
[security.md](security.md#historyupdate-historian-writes). `historyread --aggregate` only knows the twelve names in the table below; the SQL backend's other
|
||||
aggregates (Range, Delta, PercentGood, …) need a different client. A refused history read (for example
|
||||
`BadHistoryOperationUnsupported`) prints one error line and exits non-zero; it does not dump a stack trace.
|
||||
|
||||
`delete-raw` is sent with `IsDeleteModified=false`. Invalid or inconsistent options (unknown mode, unpaired
|
||||
value/time, unparsable value, options that do not apply to the mode) fail before any connection is made.
|
||||
@@ -287,6 +294,11 @@ value/time, unparsable value, options that do not apply to the mode) fail before
|
||||
| `Start` | `first` | `AggregateFunction_Start` |
|
||||
| `End` | `last` | `AggregateFunction_End` |
|
||||
| `StandardDeviation` | `stddev`, `stdev` | `AggregateFunction_StandardDeviationPopulation` |
|
||||
| `TimeAverage` | `timeavg` | `AggregateFunction_TimeAverage` |
|
||||
| `TimeAverage2` | `timeavg2` | `AggregateFunction_TimeAverage2` |
|
||||
| `Total` | | `AggregateFunction_Total` |
|
||||
| `Total2` | | `AggregateFunction_Total2` |
|
||||
| `Interpolative` | `interp` | `AggregateFunction_Interpolative` |
|
||||
|
||||
### alarms
|
||||
|
||||
|
||||
+1
-1
@@ -206,7 +206,7 @@ All times are in UTC. Invalid input turns red on blur.
|
||||
|
||||
| Option | Description |
|
||||
|--------|-------------|
|
||||
| Aggregate | Raw (default), Average, Minimum, Maximum, Count, Start, End, Standard Deviation |
|
||||
| Aggregate | Raw (default), Average, Minimum, Maximum, Count, Start, End, Standard Deviation, Time Average, Time Average 2, Total, Total 2, Interpolative |
|
||||
| Interval (ms) | Processing interval for aggregate queries (shown only for aggregates) |
|
||||
| Max Values | Maximum number of raw values to return (default 1000) |
|
||||
|
||||
|
||||
+26
-5
@@ -22,14 +22,30 @@ Two related concerns get their own dedicated pages and are **only summarised + l
|
||||
|
||||
The Host (`Program.cs`) loads `appsettings.json`, then overlays a **per-role** file chosen from the cluster roles:
|
||||
|
||||
- A single role → `appsettings.{role}.json` (e.g. `appsettings.driver.json`, `appsettings.admin.json`).
|
||||
- Both roles → `appsettings.admin-driver.json` (roles joined with `-`, ordinal-sorted).
|
||||
- A single plain role → `appsettings.{role}.json` (e.g. `appsettings.driver.json`, `appsettings.admin.json`).
|
||||
- Both roles → `appsettings.admin-driver.json` (plain roles joined with `-`, ordinal-sorted).
|
||||
- `cluster-{ClusterId}` roles are **ignored** when naming the overlay (#678.6) — `[driver, cluster-SITE-A]` loads `appsettings.driver.json`.
|
||||
- `appsettings.{ASPNETCORE_ENVIRONMENT}.json` (e.g. `appsettings.Development.json`) is layered on by the host builder.
|
||||
|
||||
All role overlays are **optional** — the base `appsettings.json` plus the Options-class C# defaults are enough to boot. The roles themselves come from `EffectiveRoles.Resolve` (`ZB.MOM.WW.OtOpcUa.Cluster`) — `Cluster:Roles`, falling back to `OTOPCUA_ROLES` when that is empty (see [`ServiceHosting.md`](ServiceHosting.md) and the table below).
|
||||
|
||||
The checked-in `appsettings*.json` files are deliberately thin: they carry only `Serilog` and the `Security:Ldap` overlay. Everything else (`OpcUa`, `Cluster`, `ConnectionStrings`/`ConfigDb`) binds from the Options-class defaults documented below unless an operator adds the section explicitly or supplies the corresponding environment variable.
|
||||
|
||||
<a id="role-overlay-audit"></a>
|
||||
#### Role overlay audit (#678.6)
|
||||
|
||||
Until #678.6 the overlay never loaded on a node carrying a `cluster-*` role (the cluster role leaked into the filename). Fixing that **activates** the shipped overlays on those nodes (the docker-dev rig and any per-cluster-mesh production node), so each key was audited against base `appsettings.json` / the Options defaults. Env vars and command-line args are re-appended after the overlay, so any value a deployment sets explicitly (the rig sets every `Security__Ldap__*` key) still wins.
|
||||
|
||||
| Key | Overlay(s) | Base / default | Effect on activation |
|
||||
|---|---|---|---|
|
||||
| `Serilog:MinimumLevel:Default` = `Information` | all three | Serilog default `Information` | no-op (in `Development` it outranks `appsettings.Development.json`'s `Debug`, by design) |
|
||||
| `…Override:Akka` = `Information` | all three | inherits `Default` (`Information`) | no-op |
|
||||
| `…Override:Microsoft.AspNetCore` = `Warning` | `admin`, `admin-driver` | unset (`Information`) | quieter ASP.NET Core logs; log-level only, matches what the rig already pins |
|
||||
| `…Override:Opc.Ua` = `Information` | `admin-driver`, `driver` | inherits `Default` | no-op. **`driver` shipped `Debug` — changed to `Information` in #678.6**: activating a Debug OPC UA stack trace on every production driver node would flood Serilog (the same pipeline starvation the rig's EF/AspNetCore pins guard against). Raise it per node with `Serilog__MinimumLevel__Override__Opc.Ua=Debug`. |
|
||||
| `Security:Ldap:DevStubMode` = `false` | all three | `false` | no-op; the overlay's point is that it outranks a developer's `appsettings.Development.json` (`true`) |
|
||||
| `Security:Ldap:Transport` = `Ldaps` | all three | `LdapOptions.Transport` default `None` | **behaviour-changing, kept deliberately** (documented secure production default). A cluster-role node that configures `Security:Ldap:Server`/`Port` for a plaintext LDAP but never sets `Transport` would now dial LDAPS and fail AdminUI sign-in; set `Security__Ldap__Transport` (the rig already sets `None` explicitly). Data-plane auth is API-key based and unaffected; driver-only nodes have no AdminUI. |
|
||||
|
||||
|
||||
---
|
||||
|
||||
## `appsettings` sections
|
||||
@@ -38,7 +54,7 @@ The checked-in `appsettings*.json` files are deliberately thin: they carry only
|
||||
|
||||
- **Purpose:** logging. Console + rolling daily file sink, layered with the shared `ZB.MOM.WW.Telemetry` enrichers (`AddZbSerilog` in `Program.cs`).
|
||||
- **Where bound:** `builder.AddZbSerilog(...)` reads `Serilog` from configuration (`ReadFrom.Configuration`).
|
||||
- **Checked-in shape** (`appsettings.json`): `Using` = `[ "Serilog.Sinks.Console", "Serilog.Sinks.File" ]`, `WriteTo` = a `Console` sink and a `File` sink (`path: logs/otopcua-.log`, `rollingInterval: Day`). Role overlays add `MinimumLevel` / `Override` blocks (e.g. `Opc.Ua: Debug`, `Akka: Information`).
|
||||
- **Checked-in shape** (`appsettings.json`): `Using` = `[ "Serilog.Sinks.Console", "Serilog.Sinks.File" ]`, `WriteTo` = a `Console` sink and a `File` sink (`path: logs/otopcua-.log`, `rollingInterval: Day`). Role overlays add `MinimumLevel` / `Override` blocks (e.g. `Opc.Ua: Information`, `Akka: Information`).
|
||||
|
||||
### `OpcUa`
|
||||
|
||||
@@ -410,7 +426,7 @@ Validated at host start (driver-role nodes) when `ServerHistorian:Enabled` **or*
|
||||
| `ServerHistorian:SqlServer:ConnectionString` | `""` | **Required.** Historian database connection string (not the ConfigDb). Supply via `${secret:OT-HISTORIAN-SQL}` or env `ServerHistorian__SqlServer__ConnectionString`; the database must already exist. |
|
||||
| `ServerHistorian:SqlServer:Schema` | `hist` | Must match `^[A-Za-z_][A-Za-z0-9_]{0,127}$` (spliced into DDL/DML). |
|
||||
| `ServerHistorian:SqlServer:CommandTimeout` | `00:00:30` | Must be `> 0`; rounded up to whole seconds. |
|
||||
| `ServerHistorian:SqlServer:AutoCreateSchema` | `true` | `true` applies the embedded schema script under an app-lock on first use; `false` only verifies `SchemaVersion >= 1` and fails every historian call (health `LastError`, no crash) if behind. |
|
||||
| `ServerHistorian:SqlServer:AutoCreateSchema` | `true` | `true` applies the embedded schema scripts under an app-lock on first use (upgrading a v1 schema to v2); `false` only verifies `SchemaVersion >= 1` and fails every historian call (health `LastError`, no crash) if behind. A verified **v1** schema (no `002_sample_modified.sql`) is served with modified-value reads and HistoryUpdate refused `BadHistoryOperationUnsupported`, retention skipping `SampleModified`, and one Warning per boot (#674). |
|
||||
| `ServerHistorian:SqlServer:RetentionDays` | unset | Days of history kept (source time), `>= 1` when set; unset keeps everything. Enables the Primary-gated purge of `hist.Sample`/`hist.Event` (see [Historian.md](Historian.md#retention-retentiondays)). |
|
||||
| `ServerHistorian:SqlServer:PurgeInterval` | `01:00:00` | Purge cadence; must be `> 0`. |
|
||||
| `ServerHistorian:SqlServer:PurgeBatchSize` | `4000` | Rows per `DELETE TOP (n)` batch; `>= 1`. Keep under ~5000 to avoid lock escalation. |
|
||||
@@ -430,7 +446,12 @@ when `ServerHistorian:Enabled` **or** `AlarmHistorian:Enabled` is true (`QuestDb
|
||||
| `ServerHistorian:QuestDb:VisibilityTimeout` | `00:00:05` | Must be `>= 0`. How long a write waits for its rows to become visible to queries (WAL apply); `0` disables the wait. A timeout only logs a Warning. |
|
||||
|
||||
HistoryUpdate (SqlServer and QuestDb only; QuestDb deletes are soft — tombstones) additionally needs `ServerHistorian:Enabled=true` and a key
|
||||
whose principal holds the `HistoryUpdate` permission — see [`docs/security.md`](security.md#historyupdate-historian-writes). Full semantics,
|
||||
whose principal holds the `HistoryUpdate` permission — see [`docs/security.md`](security.md#historyupdate-historian-writes). It is served
|
||||
only on the redundancy **Primary** (`BadInvalidState` elsewhere, #674) and is bounded by one backend-independent key:
|
||||
|
||||
| Key | Default | Meaning |
|
||||
|---|---|---|
|
||||
| `ServerHistorian:HistoryUpdateMaxFutureSkew` | `1.00:00:00` (1 day) | TimeSpan. A HistoryUpdate (Insert / Replace / Update) item whose `SourceTimestamp` is later than server-clock now + this is refused `BadOutOfRange` before any backend sees it (#674 — a far-future typo could otherwise age out a QuestDB TTL table or poison the history). Applies to every backend; the live recorder is not bounded. Non-positive disables the bound. | Full semantics,
|
||||
schema, aggregates and the docker-dev recipe: [`docs/Historian.md`](Historian.md#sql-server-backend) (SqlServer) and
|
||||
[`docs/Historian.md`](Historian.md#questdb-backend) (QuestDb: the µs timestamp rules, pre-1970 limits, TTL and the docker-dev overlay).
|
||||
|
||||
|
||||
@@ -441,6 +441,40 @@ Two consequences worth knowing:
|
||||
**notices**, so a silent-success termination path — a stream that ends cleanly, a poll loop that exits
|
||||
without throwing — bypasses it entirely.
|
||||
|
||||
**A driver that KNOWS its connection is gone can skip the wait: `IDriverConnectionLossSource` (Gitea #676).** The
|
||||
watchdog has to prove a loss from silence, so it costs ~90 s even when the protocol already said "this connection
|
||||
will never come back". An opt-in `Core.Abstractions` interface lets a driver say so:
|
||||
`event EventHandler<DriverConnectionLossEventArgs> ConnectionLost` (a reason string). `DriverInstanceActor` attaches
|
||||
in `PreStart` and detaches in `PostStop` — the same lifetime as `IRediscoverable`, for the same reason (the `IDriver`
|
||||
outlives the actor on a respawn) — marshals each raise to the actor thread (`Self.Tell`; actor state is never touched
|
||||
on the driver's thread), and acts on it **only while `Connected`**, through the very handler the watchdog's
|
||||
`DisconnectObserved` uses (detach, record a fault, `Reconnecting`, teardown-then-init on the retry). In
|
||||
`Connecting`/`Reconnecting`/`Stubbed` it is ignored, not deferred: a re-initialise is already pending, and acting on a
|
||||
stale raise later could tear down the connection that attempt is building. The two are complementary, not
|
||||
alternatives:
|
||||
|
||||
- **The signal is for DEFINITIVE losses only.** A driver must raise only when no in-place recovery is possible, at most
|
||||
once per connection, re-armed by a re-initialise. An ambiguous failure — a timeout, a stalled socket, "no
|
||||
communication" — must not raise: a stalled link that resumes keeps its session, subscriptions and in-place recovery,
|
||||
and tearing it down would cost all three. Those stay with the watchdog, whose conjunction is the honest test for them.
|
||||
- **Today only `OpcUaClientDriver` implements it:** keep-alive `BadSessionIdInvalid` / `BadSessionClosed` /
|
||||
`BadSessionNotActivated` / `BadServerHalted`; a CLOSED connection (`BadSecureChannelClosed` /
|
||||
`BadConnectionClosed`) once sustained — the second closed-channel keep-alive in one outage, which is what a real
|
||||
process stop actually produces (live gate A1), and the SDK never reopens a closed connection by itself; or a
|
||||
subscription the server REPORTED dropped (not the driver's
|
||||
publishing-stall heuristic, which is an inference from silence and stays with the watchdog; see
|
||||
[OpcUaClient.md](drivers/OpcUaClient.md) §"Upstream outage and recovery" for why `BadServerHalted` is on the list —
|
||||
it was measured, the SDK never re-opens a halted channel). A restarted, stopped or killed upstream therefore
|
||||
recovers in about one keep-alive plus one reconnect interval instead of ~100 s; a stall/pause
|
||||
(`BadRequestTimeout`/`BadNoCommunication`) still goes through the watchdog and keeps its in-place recovery.
|
||||
- ⚠️ **Known gap:** "only while `Connected`" plus "once per connection" means a raise the host receives while not yet
|
||||
Connected (e.g. the protocol proves the fresh connection dead before `InitializeSucceeded` is processed) is swallowed,
|
||||
and that connection falls back to the watchdog — the pre-#676 recovery time, in a narrow window.
|
||||
- Pinned by `DriverInstanceActorConnectionLossSignalTests` (Reconnecting within seconds while the watchdog is set to an
|
||||
hour; a falsification control with no raise that stays Connected; a raise mid-connect ignored; a stray watchdog
|
||||
`DisconnectObserved` in `Reconnecting` is a no-op; detach on stop) and `OpcUaClientConnectionLossSignalTests`
|
||||
(definitive raises once, ambiguous and the stall heuristic never, re-armed after re-init).
|
||||
|
||||
**The reconnect retry now tears down before re-initialising.** `Reconnecting`'s `RetryConnect` calls
|
||||
`ShutdownAsync` before `InitializeAsync` — the sequence every driver's own `ReinitializeAsync` already
|
||||
uses — because several drivers **overwrite** an `IAsyncDisposable` transport field inside `InitializeAsync`
|
||||
|
||||
+1034
-103
File diff suppressed because it is too large
Load Diff
@@ -32,13 +32,13 @@ Single-node dev: `OTOPCUA_ROLES=admin,driver` (or the `Cluster:Roles` equivalent
|
||||
|
||||
### Per-role configuration overlays
|
||||
|
||||
`Program.cs` builds a role suffix by joining the resolved `EffectiveRoles.Gossiped` roles **alphabetically** with `-` and loads `appsettings.{roleSuffix}.json` as an optional overlay on top of base `appsettings.json`. A plain role (`admin`/`driver`/`dev`) is always lower-case in `Gossiped` regardless of how the operator spelled it, so the overlay filename for those is stable (`appsettings.driver.json`); if a `cluster-{ClusterId}` role is also configured, its suffix carries into the filename **verbatim** (e.g. `Cluster:Roles: ["driver", "cluster-SITE-A"]` loads `appsettings.cluster-SITE-A-driver.json`), since `Gossiped` preserves that spelling by design (see [Configuration.md § `Roles`](Configuration.md#appsettings-sections)). Three overlays ship in `src/Server/ZB.MOM.WW.OtOpcUa.Host/`:
|
||||
`Program.cs` builds a role suffix (`RoleOverlay.Suffix`) by joining the resolved `EffectiveRoles.Gossiped` **plain** roles (`admin`/`driver`/`dev`) alphabetically with `-` and loads `appsettings.{roleSuffix}.json` as an optional overlay on top of base `appsettings.json`. A plain role is always lower-case in `Gossiped` regardless of how the operator spelled it, so the overlay filename is stable (`appsettings.driver.json`). **`cluster-{ClusterId}` roles are NOT part of the filename** (#678.6): they are mesh-membership tags, so `Cluster:Roles: ["driver", "cluster-SITE-A"]` loads `appsettings.driver.json` and `["admin", "driver", "cluster-MAIN"]` loads `appsettings.admin-driver.json`. (Before #678.6 the cluster role was folded into the name, producing e.g. `appsettings.admin-cluster-MAIN-driver.json`, which never exists — so on every node with a cluster role no shipped overlay ever loaded.) Three overlays ship in `src/Server/ZB.MOM.WW.OtOpcUa.Host/`:
|
||||
|
||||
- `appsettings.admin.json` — admin-only nodes
|
||||
- `appsettings.driver.json` — driver-only nodes
|
||||
- `appsettings.admin-driver.json` — fused single-node dev / small deployments
|
||||
|
||||
All three carry Serilog log-level overrides + `Security:Ldap:DevStubMode = false`. Configuration loading order (lowest to highest precedence) is:
|
||||
All three carry Serilog log-level overrides + `Security:Ldap:DevStubMode = false` + `Security:Ldap:Transport = Ldaps` (audit table: [Configuration.md § Role overlays](Configuration.md#role-overlay-audit)). Configuration loading order (lowest to highest precedence) is:
|
||||
|
||||
`appsettings.json` < `appsettings.{Environment}.json` < `appsettings.{role}.json` < environment variables < command-line args
|
||||
|
||||
|
||||
@@ -50,7 +50,19 @@ provides ordered **failover** — the driver tries each candidate in turn at eve
|
||||
4840 / 4841). See
|
||||
`src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.OpcUaClient.Contracts/OpcUaClientDriverOptions.cs`
|
||||
for every field (security policy/mode, auth type, session timeout, keep-alive,
|
||||
browse root, node/depth caps). `ReconnectPeriod` is still parsed but no longer used (#670).
|
||||
browse root, node/depth caps). `ReconnectPeriod` was **deleted** (#676 — unused since #670 removed the SDK reconnect
|
||||
handler, and the AdminUI no longer shows it); stored configs that still carry `reconnectPeriod` keep parsing in the
|
||||
factory, the Test-Connect probe and the browse picker (unknown keys are skipped), and the form drops it on the next save.
|
||||
|
||||
### Endpoint address: the configured host and port win (#676)
|
||||
|
||||
GetEndpoints returns each endpoint with the URL the **server** believes it lives at. Behind NAT, a port map, a
|
||||
container port publish or a proxy that address is unreachable from the client — or reachable but bypassing the proxy.
|
||||
The driver therefore rewrites the selected endpoint's **host and port** to the configured URL's whenever the schemes
|
||||
match, keeping the advertised path (the SDK's own `CoreClientUtils.SelectEndpoint` rule, which the driver's
|
||||
policy-aware endpoint selection used to bypass). A configured URL with **no port** keeps the advertised port (copying
|
||||
the port-less `-1` would have sent the session to 4840). A different scheme is left alone; an unparseable advertised
|
||||
URL falls back to the configured one. `OpcUaClientDriver.RewriteEndpointUrl`.
|
||||
|
||||
### Session lifetime
|
||||
|
||||
@@ -83,12 +95,67 @@ The session keep-alive (`KeepAliveInterval`, default 5 s) is the liveness signal
|
||||
without moving the mark. Outside that window every notification is
|
||||
accepted whatever its timestamp, so an upstream clock that steps backwards (NTP, a PLC clock reset) never
|
||||
freezes a node. The outage Bad never moves the comparison; a re-initialise resets it.
|
||||
⚠️ **By design, inside the window a notification whose source time EQUALS the last accepted one is dropped** — so a
|
||||
status-only change that keeps the old source timestamp (not what OPC UA servers are supposed to do) is lost during
|
||||
the window only (the trailing re-read below repairs it). When the window closes — or a second recovery re-arms it —
|
||||
the driver logs ONE line, `post-recovery replay window closed — dropped N late notifications across M refs after
|
||||
recovery`, at **Information when N > 0** and Debug otherwise, because the per-drop line is Debug and the
|
||||
`stale_notification_dropped` counter is OTel-only, which made the guard unobservable on the docker-dev rig (#676).
|
||||
A re-arm starts the count afresh.
|
||||
- **Trailing re-read after an in-place recovery (#676).** Found by the TCP-stall wire test: a change the server
|
||||
published just after the stall ended rode a Publish response that had been in flight across the stall; the SDK
|
||||
timed that request out (`BadTimeout`), discarded the response and never republished the message (the following
|
||||
keep-alives moved the sequence number past it), so the node stayed **Good on the pre-change value indefinitely**.
|
||||
`Timeout + KeepAliveInterval` after the recovery — when every request in flight across the stall has completed or
|
||||
timed out — every subscribed ref is read once more and published when its **(value, status) differs** from the last
|
||||
value delivered for that ref **and is not older** than it. Any status counts: a Good→Bad/Uncertain change lost in the
|
||||
pipeline is as stale as a lost value. "Older" compares source timestamps only when both carry one; a read with no
|
||||
source timestamp (a server that does not send them) is the server's current state and counts as newer. An **equal**
|
||||
source timestamp with different content IS published — it is the server's current state, not a replay, and it repairs
|
||||
the equal-timestamp status-only change the replay guard drops by design. An identical (value, status) is never
|
||||
republished. A ref that received ANY delivery while the read was in flight is skipped (a per-ref delivery counter is
|
||||
captured before the read is issued): without source timestamps the read's sample cannot be ordered against that
|
||||
notification, and publishing it would regress the node. An Information line names how many refs it had to repair. The trailing read and the replay-guard disarm
|
||||
timer run on the driver clock and are cancelled when the session is released (re-initialise / shutdown / dispose).
|
||||
- **No older value is ever delivered after a newer one.** The paths that publish upstream data — live notifications,
|
||||
the recovery read, the trailing read and the comms-loss Bad fan — make their "may I publish this?" decision and the
|
||||
publish inside one driver-wide critical section. A single lock rather than one per ref: it also serialises delivery
|
||||
across subscriptions (the SDK dispatches each subscription's messages on its own worker), which is cheap because each
|
||||
section is a dictionary update plus a subscriber `Tell` and nothing awaits under it. Subscribers run inside it — the
|
||||
host's handler only `Tell`s an actor — so a subscriber must never block on, or call back into, the driver.
|
||||
- **The session does not come back** (an upstream restart: the server forgot the session) ⇒ the driver stays
|
||||
`Degraded` with its read stamp frozen, which is exactly the #527 comms-loss watchdog's trip condition. After 90 s
|
||||
`DriverInstanceActor` runs `ShutdownAsync` + `InitializeAsync` (retried every 10 s while the server is down) and
|
||||
re-subscribes data + alarms; the new subscription's initial values come back Good. Alarm last-told state survives
|
||||
the re-initialise (#619), so the post-subscribe `ConditionRefresh` raises no un-ack flicker; the host's reconnect
|
||||
annotation restores condition quality.
|
||||
`Degraded` with its read stamp frozen, which is the #527 comms-loss watchdog's trip condition; `DriverInstanceActor`
|
||||
runs `ShutdownAsync` + `InitializeAsync` (retried every 10 s while the server is down) and re-subscribes data +
|
||||
alarms; the new subscription's initial values come back Good. Alarm last-told state survives the re-initialise
|
||||
(#619), so the post-subscribe `ConditionRefresh` raises no un-ack flicker; the host's reconnect annotation restores
|
||||
condition quality.
|
||||
- **Fast recovery for a DEFINITIVELY lost session (#676).** The driver implements `IDriverConnectionLossSource` and
|
||||
raises `ConnectionLost` — **once per session**, re-armed by a re-initialise — when the loss is certain, so the host
|
||||
re-initialises at once instead of after the watchdog's 90 s. Definitive means: a keep-alive status of
|
||||
`BadSessionIdInvalid` / `BadSessionClosed` / `BadSessionNotActivated` (the server forgot, closed or never activated
|
||||
the session) or `BadServerHalted`, or a subscription the **server** reported gone (below). `BadServerHalted` is on the list
|
||||
because it was **measured**: on the restart wire test a gracefully stopped server reports `BadServerHalted` (and the
|
||||
keep-alives alternate it with `BadNoCommunication`), and the SDK never re-opens a channel the server halted — 15 s
|
||||
after the server was back, the same session still failed every request with `BadServerHalted`; `BadSessionIdInvalid`
|
||||
never arrived.
|
||||
**A CLOSED connection is definitive too, once sustained (live gate A1).** Against a real opc-plc on docker-dev, none of
|
||||
`docker restart`, a graceful SIGINT stop (opc-plc itself logged `BadServerHalted`) or `docker kill` delivered a
|
||||
session status to the client: the keep-alives reported `BadSecureChannelClosed` (0x80860000) once, then
|
||||
`BadConnectionClosed` (0x80AE0000) interleaved with `BadNoCommunication` for the whole outage, so the first cut of the
|
||||
fast path never fired and recovery took 91–127 s. Reproduced in-process with `TcpStallProxy.HardClose` (every
|
||||
connection reset, new ones refused, the server AND its session still alive behind it): the same sequence, and the SDK
|
||||
1.5.378 session **never opens a new connection by itself** — 48 s after the proxy accepted connections again, not one
|
||||
attempt. So a closed channel can never recover in place; on the **second** closed-channel keep-alive failure in one
|
||||
outage (`IsClosedChannel`; a good keep-alive resets the count, so a single transient does not count) the driver raises
|
||||
`ConnectionLost` — about one keep-alive interval after the connection dropped. Everything else —
|
||||
`BadNoCommunication`, `BadNotConnected`, `BadRequestTimeout`/`BadTimeout` (what a stalled or paused link produces) —
|
||||
is **ambiguous** (the link may resume and keep its session, see the stall test) and stays with the watchdog.
|
||||
Consequence: a restarted, stopped or killed upstream is re-initialised within roughly one keep-alive plus one
|
||||
reconnect interval instead of ~100 s; a pause (`docker pause`, a congested link) still recovers in place. ⚠️ **Known gap:** the host acts on the signal only while the driver is
|
||||
`Connected`, and the driver raises it once per session — so a definitive loss raised while the host is not
|
||||
Connected (e.g. `BadServerHalted` on the fresh session before the host has processed the successful initialise) is
|
||||
swallowed and that session falls back to the watchdog. Re-raising on later keep-alives would close it but was left out
|
||||
to keep the once-per-session contract simple; the cost is the pre-#676 recovery time, in a narrow window.
|
||||
- **The subscription dies but the session lives** (an outage longer than the subscription lifetime —
|
||||
`LifetimeCount` x publishing interval, often well under the session timeout — or a server that drops it) ⇒ keep-alives
|
||||
stay Good, so they cannot be the signal. A `PublishError` of `BadNoSubscription` (while subscriptions are tracked) or
|
||||
@@ -96,7 +163,10 @@ The session keep-alive (`KeepAliveInterval`, default 5 s) is the liveness signal
|
||||
persists for one more subscription keep-alive window + 5 s across good keep-alives, declares it lost: `Degraded`
|
||||
(`"subscription lost: …"`), the outage published once, and — sticky until the session is released — **nothing**
|
||||
advances `LastSuccessfulRead` any more (not good keep-alives, not a sibling subscription's notifications), so the
|
||||
watchdog's re-initialise recreates it. Both detectors are wire-tested independently.
|
||||
watchdog's re-initialise recreates it. Both detectors are wire-tested independently. A subscription the **server**
|
||||
reported gone (`BadNoSubscription` / `BadSubscriptionIdInvalid`) never comes back on its session, so that also raises
|
||||
`ConnectionLost` (#676) and the re-initialise happens at once; the `PublishingStopped` **heuristic** is an inference
|
||||
from silence and keeps going through the watchdog.
|
||||
- An in-place recovery is bound to the session whose keep-alive recovered: one overtaken by a re-initialise neither
|
||||
publishes, nor degrades, nor re-arms the outage latch of the new session.
|
||||
|
||||
@@ -126,7 +196,7 @@ initialise (the AdminUI form drops them on the next save). Pinned by
|
||||
|
||||
## Capability surface
|
||||
|
||||
`OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWritable, ISubscribable, IHostConnectivityProbe, IAlarmSource`
|
||||
`OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWritable, ISubscribable, IHostConnectivityProbe, IAlarmSource, IDriverConnectionLossSource`
|
||||
(`src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.OpcUaClient/OpcUaClientDriver.cs:31`).
|
||||
|
||||
| Capability | Path | Notes |
|
||||
@@ -253,6 +323,17 @@ the reconnect handler, #670). They are not part of `DriverHealth` or the
|
||||
- **Integration fixture** — exercises the driver against a reference OPC UA
|
||||
server (opc-plc) on the shared docker host; see
|
||||
[OpcUaClient-Test-Fixture.md](OpcUaClient-Test-Fixture.md) for the coverage map.
|
||||
- **Wire tests against the in-process server** — `tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/`
|
||||
`OpcUaClientDriver*WireTests` (no Docker). `OpcUaClientDriverStallWireTests` (#676) covers the in-place blip that
|
||||
used to be live-gate-only: the driver connects through `TcpStallProxy`, an in-test TCP forwarder whose pause holds
|
||||
every byte without closing anything (which only works because of the endpoint rewrite above); it asserts one Bad,
|
||||
a frozen read stamp, recovery on the SAME session, the identical value republished Good, a change published right
|
||||
after recovery still arriving, and no source-time regression once the SDK's late backlog flush has arrived. Its
|
||||
second test uses `HardClose` (reset every connection, refuse new ones) — the shape of a real process stop — and
|
||||
asserts the live-gate status sequence, that the SDK never reconnects on its own once the proxy accepts again,
|
||||
`ConnectionLost` exactly once, and the re-initialise bringing the identical value back Good.
|
||||
`OpcUaClientDriverReconnectWireTests` covers the server restart (incl. the `BadServerHalted` → `ConnectionLost` signal)
|
||||
and the subscription-lost case.
|
||||
|
||||
## Further reading
|
||||
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
# Follow-up batch #674–#678 — design (2026-10-11)
|
||||
|
||||
Follow-ups filed after the #667–#673 batch. Base: master `bc03adb1`. User-approved; recommended choices throughout.
|
||||
Investigation found three new defects (client endpoint port not rewritten; role overlays never load on
|
||||
cluster-role nodes; Gateway Count/Average mode mapping) — the first two are fixed here, the third is filed.
|
||||
|
||||
| Lane | Scope | Model |
|
||||
|---|---|---|
|
||||
| L1 | #674 HistoryUpdate posture | opus |
|
||||
| L2 | #675 reverse event paging | opus |
|
||||
| L3 | #677 aggregates | opus |
|
||||
| L4 | client (#677.5, #678.5/.6, #674.6) | sonnet |
|
||||
| L5 | #676 OpcUaClient | opus |
|
||||
| L6 | #678.1–.4 + role-overlay bug | sonnet |
|
||||
|
||||
## L1 — #674
|
||||
- **674.1:** spike on QuestDB 10.0.1 showed TTL is judged against `min(maxTs, wall clock)` (`cairo.ttl.use.wall.clock=true`
|
||||
default) — a 2066 row evicted nothing. Add a QuestDB integration test pinning that + a docs note. HistoryUpdate refuses a
|
||||
SourceTimestamp beyond `now + ServerHistorian:HistoryUpdateMaxFutureSkew` (default 1 day) with `BadOutOfRange`; the
|
||||
recorder is not bounded.
|
||||
- **674.2:** SqlServer base schema requirement back to v1; `SupportsModifiedTrail` (v ≥ 2). On v1: modified reads and
|
||||
HistoryUpdate ⇒ `BadHistoryOperationUnsupported` (nothing sent), retention skips `SampleModified`, one Warning per boot.
|
||||
- **674.3:** HistoryUpdate Primary-gated (off-Primary ⇒ `BadInvalidState`, same shape as `NativeAlarmAckPrimaryGate`).
|
||||
Retry duplicates on QuestDB documented; no DEDUP key.
|
||||
- 674.4/674.5 documented only.
|
||||
|
||||
## L2 — #675
|
||||
- Reverse event reads: `reverse` flag on `IHistorianDataSource.ReadEventsAsync`; SqlServer/QuestDB order DESC and take
|
||||
the top N; direction-aware cursor (reuse raw `SliceTieCluster(reverse:)`); Gateway ⇒ `BadHistoryOperationUnsupported`
|
||||
for reverse.
|
||||
- End-only event read ⇒ backward (mirrors raw).
|
||||
- Offline test pinning `DateTime.MaxValue` through the Gateway proto/FILETIME encoding; live AVEVA leg stays open (VPN).
|
||||
|
||||
## L3 — #677
|
||||
- Reverse processed reads per SDK/Part 13: buckets anchored at the later (start) time stepping back, partial bucket
|
||||
earliest, bucket timestamp = later edge. Parity test vs SDK `AggregateCalculator`. Client-visible, release-noted.
|
||||
- Partial bit also when the data starts inside a bucket (no bounding prior).
|
||||
- `AggregateConfiguration`: `UseServerCapabilitiesDefaults` or values equal to the server defaults ⇒ served; otherwise
|
||||
per-node `BadAggregateConfigurationRejected`.
|
||||
- Gateway provisioner sends `IntegralDivisor = 1` (proto3 default 0). Live magnitude check stays open (VPN).
|
||||
|
||||
## L4 — client
|
||||
- `EndpointSelector.SelectBest` rewrites host **and** port of the advertised endpoint (fixes rig `:4841` cert error).
|
||||
- Client aggregate enum/mapper/CLI/UI gain TimeAverage, TimeAverage2, Total, Total2, Interpolative.
|
||||
- History read refusal ⇒ one-line `CommandException`, no stack trace.
|
||||
- `NodeManagerHistoryReadTests` non-historized case sets `IsReadModified=false`.
|
||||
|
||||
## L5 — #676
|
||||
- Driver endpoint host+port rewrite; in-process TCP-stall proxy wire test (in-place blip, same session, Good republish,
|
||||
no regression).
|
||||
- Remove dead `ReconnectPeriod` (options, AdminUI form, docs, tests); stored JSON with it still parses.
|
||||
- New opt-in `IDriverConnectionLossSource` (Core.Abstractions): raised once per session by OpcUaClient only on a
|
||||
definitive loss (`BadSessionIdInvalid`/`BadSessionClosed`/`BadSessionNotActivated`/subscription lost);
|
||||
`DriverInstanceActor` (attach PreStart, detach PostStop) turns it into `DisconnectObserved` only while Connected.
|
||||
Ambiguous codes keep going through the #527 watchdog.
|
||||
- Information-level per-recovery replay-drop summary. Equal-timestamp drop documented.
|
||||
|
||||
## L6 — small
|
||||
- Retention: mid-command cancel test + `ClearPool` fallback test with falsification control.
|
||||
- #672 tests: initial-state unreachable path; strengthen `Self_unreachable_is_ignored`; edge-triggered Debug log.
|
||||
- FOCAS `WireBackendTests`: wait for motor speeds too; `WaitFor` fails on timeout.
|
||||
- QuestDB TTL-cleared log wording; `docker-dev/docker-compose.debug.yml` Debug recipe.
|
||||
- Role overlay suffix built from plain roles only — after auditing each `appsettings.<role>.json` and stripping anything
|
||||
that would change production behaviour on activation.
|
||||
|
||||
## Live gates (docker-dev)
|
||||
1. opc-plc `docker restart` recovers in seconds (was ~100 s); pause-blip re-check.
|
||||
2. Client.CLI connects to `:4841`–`:4845`.
|
||||
3. HistoryUpdate refused on the Secondary, accepted on the Primary; v1-schema DB keeps historizing.
|
||||
4. Reverse event paging and reverse processed reads over the wire, SqlServer + QuestDB.
|
||||
@@ -0,0 +1,9 @@
|
||||
# Gateway tag-presence concurrency coverage — 2026-10-13
|
||||
|
||||
Approved tests-only fix for #701. Added 15 deterministic cases for overlapping positive/negative cold lookups, expiry refreshes, capacity crossing, successful-result isolation from faults/cancellation, and cancellation after asynchronous completion. Fake presence calls support an async hook and atomic counter. Production cache is unchanged; no single-flight promise.
|
||||
|
||||
The synthetic 16-caller cold/expiry bursts produced 16 catalog RPCs. Assertions permit one through16; this records amplification without making it a requirement or a live-cost claim. Newclass15/15 and Gateway449/449 passed. Independent review found no actionable defects; diff check clean.
|
||||
|
||||
First full solution run:14,614passed/609skipped/1failed across62suites. Existing restart/reconnect wire test failed, then isolated retry passed1/1. Full repeat with detailed console logging passed14,615/609skip/0fail across62suites, exit0, including OPC UA integration122/122 and host integration268passed/12skip. #702 tracks the unresolved intermittent failure; it is not closed merely because the repeat passed.
|
||||
|
||||
Both full runs excluded configured live fixture variables from the environment; environment-gated tests skipped. Existing NU1510/xUnit duplicate-InlineData warnings remain pre-existing. No production behavior or deployment changed by this patch.
|
||||
@@ -0,0 +1,36 @@
|
||||
# Missing-tag and Gateway performance fixes — 2026-10-13
|
||||
|
||||
Follow-up to pushed `97668d26` / PR #699. Implements #653 and the remaining #696 items 2/3 under the approved bounded design.
|
||||
|
||||
## Changes
|
||||
|
||||
- **#653:** Raw, Processed and AtTime check tag presence before history RPCs. The bounded 4,096-name cache uses a 5-second negative and 60-second positive lifetime. Only explicit absence or NotFound is cached as missing; failures and cancellation propagate. Raw is empty; AtTime and Processed retain requested/bucket timestamps with null/BadNoData. Unsupported aggregate validation runs first.
|
||||
- **Live compatibility adjustment:** the deployed gateway misparses missing `GetTagMetadata` replies as `Internal` (`InvalidDataException: GetTagInfo response ended unexpectedly`). The adapter therefore checks the filtered tag catalog and compares exact names, fully draining the stream before declaring presence/absence. It never turns the parser error into a missing tag. Wildcard matches alone do not prove presence, and a trailing stream fault cannot enter the cache.
|
||||
- **#696 item 2:** a truncated remainder probe seeds strict-ceiling reverse discovery with its observed rows/density; gap searches use the available discovery budget. Complete segments alone contribute to the newest-first page. Regression: a 7,200-row stopped 1 Hz tag, ceiling 1,000, requested latest three one day later, previously 17 raw calls; now meets a maximum of 12 with exactly 7199/7198/7197 returned. Ordinary page sizing and event discovery behavior are unchanged.
|
||||
- **#696 item 3:** per-sample aggregates reuse a ceiling refusal for 30 seconds instead of immediately re-probing on every full stream. The next incomplete aggregate read after expiry notices a raised ceiling; a lowered ceiling is re-learned immediately. Raw and AtTime re-probes remain immediate. During the cooldown an oversized aggregate remains BadTooManyOperations, never a truncated result.
|
||||
|
||||
## Verification evidence
|
||||
|
||||
New regressions failed before fixes: all initial 12 cases failed (missing preflight/caching, 17 discovery reads, duplicate aggregate refusal). The four catalog wire cases failed with the metadata adapter and pass with the catalog adapter. Final Gateway suite: 434 passed, six environment-gated skips.
|
||||
|
||||
Independent read-only review found no important defects. Concurrent cold requests can duplicate a metadata/catalog check; cache access and capacity mutations are locked. No single-flight guarantee was introduced. Catalog RPC contract streams all matches; the upstream SDK pages the native tag-query results, with no configured browse result ceiling.
|
||||
|
||||
## Live #653 confirmation
|
||||
|
||||
Measured the updated GatewayHistorianDataSource through the real adapter and local gateway against the existing remote AVEVA connection. Temporary historian:read key had a two-hour expiry and was revoked after the run. No historian writes/provisioning and no gateway deployment changes.
|
||||
|
||||
Nonexistent tag: `OtOpcUa.Issue653.Nonexistent.20261013`.
|
||||
|
||||
| Operation, five consecutive runs | Elapsed milliseconds | Result |
|
||||
|---|---|---|
|
||||
| Raw | 228, 0, 0, 0, 0 | empty |
|
||||
| Count processed | 2, 0, 0, 0, 0 | six null/BadNoData buckets |
|
||||
| AtTime | 1, 0, 0, 0, 0 | null/BadNoData at requested time |
|
||||
|
||||
These used one data source/cache: the first Raw call is cold, subsequent calls are cached. The prior direct-client measurement took ~30.6 seconds for missing-tag Raw/Aggregate and ~61.3 seconds for AtTime. Direct client RPCs are not modified; the fast-fail protection is in OtOpcUa.
|
||||
|
||||
**Control limitation:** SysTimeSec catalog presence succeeded, but its Raw RPC twice timed out upstream with DeadlineExceeded at ~30 seconds. The data source propagated IOException/communication failure rather than returning no data. A fresh control using the original package client (no data-source presence checks) also timed out with DeadlineExceeded after 30,656 ms, confirming this timeout is independent of the new checks. The earlier direct-client control had succeeded. This run therefore verifies missing-tag fast-fail and failure propagation, not healthy-tag live history behavior. No end-to-end deployed OPC UA claim is made.
|
||||
|
||||
#694 remains open for native aggregate rerun after historiangw#27. No sibling gateway code changed.
|
||||
|
||||
Full final-tree solution: 14,600 passed, 609 skipped, 0 failed across 62 suites. `dotnet test ZB.MOM.WW.OtOpcUa.slnx --verbosity quiet`, exit 0. Includes 122 passing OPC UA integration tests and 268 passing host integration tests (12 host skips). Environment-gated suites are included in the skip count.
|
||||
@@ -0,0 +1,28 @@
|
||||
# Historian follow-up results — 2026-10-13
|
||||
|
||||
Scope: approved bounded fixes for #695–#698, plus live measurement of #653. Base: `66a2b0c3`.
|
||||
|
||||
- #695: spend the remaining learned-ceiling room on crowding reads; reduce crowded reverse-read windows using the observed pre-window count. Regression cases cover ceilings 1,709 and 1,369, and reverse paging through a crowded millisecond.
|
||||
- #696 item 1: probe the future interval before starting the prior-sample search near now. Preserve stored future samples, requested timestamps and Uncertain status. Dense future probes still use the existing bounded walk. Items 2 and 3 remain open (stopped-tag strict-ceiling cost and aggregate ceiling re-probes).
|
||||
- #697: actual Gateway, SQL Server and QuestDB transport failures become `BadNoCommunication` on every HistoryRead arm. Cancellation remains `BadTimeout`; unsupported operations retain `BadHistoryOperationUnsupported`. Tests include real refused database connections, repeated reads and nested gRPC failures.
|
||||
- #698: normalize Raw, Modified and AtTime scalar values to the node DataType. Conversion failure gives per-sample null/`BadTypeMismatch`; arrays, audit metadata and timestamps survive. Processed results retain aggregate-specific types.
|
||||
- #694 item 3 remains blocked on historiangw#27. No upstream code was changed.
|
||||
|
||||
## Live #653 measurement
|
||||
|
||||
Five sequential calls per operation and tag, using client 0.3.0 against the running local gateway (`localhost:5221`) with its existing remote AVEVA connection. Commissioned a two-hour `historian:read` key, then revoked it after measurement. No historian writes or provisioning performed.
|
||||
|
||||
| Operation | SysTimeSec, elapsed ms | Nonexistent tag, elapsed ms |
|
||||
|---|---|---|
|
||||
| Metadata | 299, 85, 81, 82, 48 (success) | 89, 53, 41, 77, 76 (gateway exception) |
|
||||
| Raw | 143, 99, 145, 133, 159 (success) | 30260, 30634, 30675, 30754, 30628 (gateway exception) |
|
||||
| Aggregate | 271, 275, 309, 249, 304 (success) | 30564, 30605, 30583, 30575, 30806 (gateway exception) |
|
||||
| AtTime | 140, 88, 89, 127, 92 (success) | 61074, 61330, 61601, 61249, 61271 (gateway exception) |
|
||||
|
||||
Nonexistent tag: `OtOpcUa.Issue653.Nonexistent.20261013`. The delay is still present. Recommend a metadata pre-check with a short negative cache, caching only explicit missing-tag outcomes and never authentication, communication or arbitrary errors. This is follow-up implementation; #653 remains open. Direct gateway AtTime above differs from OtOpcUa's raw-based AtTime evaluator.
|
||||
|
||||
## Consumer and validation notes
|
||||
|
||||
ScadaBridge source and tests contain no HistoryRead consumer; no consumer change required. Independent review found a future-stored-sample edge case, which was fixed and covered before final verification. No deployment or live end-to-end OPC UA verification of these new fixes is claimed; the live run measures #653 through the gateway client.
|
||||
|
||||
Full solution: 14,581 passed, 609 skipped, 0 failed across 62 test suites. Command: `dotnet test ZB.MOM.WW.OtOpcUa.slnx --verbosity quiet`, exit 0. Skips include environment-gated integration suites.
|
||||
@@ -0,0 +1,9 @@
|
||||
# Restart loss reason test correction — #702
|
||||
|
||||
The full integration reproduction failed at the restart wire test reason assertion: it required BadServerHalted but correctly received BadSessionIdInvalid. Production already supports that definitive loss reason.
|
||||
|
||||
The test now accepts an explicit semantic set: four direct definitive session statuses and two sustained closed-channel statuses. All BadNoCommunication, retained-value, unhealthy-state, frozen-read-stamp, one-notification and explicit reinitialization/recovery assertions remain unchanged. Only test expectation/comments changed; production unchanged. Existing deterministic driver tests independently enforce direct-status ordering/count and the sustained-channel threshold.
|
||||
|
||||
Verification: isolated restart1/1, deterministic status routes30/30, full OPC UA integration122/122. Full offline solution: 14,613 passed / 611 skipped / 0 failed, exit0, 61 reported suite summaries. Host integration266 passed /14 skipped; unavailable environment/fixture tests remain skips. Independent review found no important defects; git diff --check passed. Actual prior red run121pass/1fail, detailed console/TRX confirms the exact reason mismatch.
|
||||
|
||||
Logs/TRX retained in the Codex thread workspace under work/issue702-*. No Gateway or deployment behavior changed by this test correction. Separate live gates remain open.
|
||||
@@ -884,6 +884,7 @@ HistoryUpdate rewrites recorded history, so unlike HistoryRead it is gated **wha
|
||||
- **Flag off (default):** the session's API key is re-checked on the call (`DataPlaneAuthorizer.AuthorizeKeyUsable`, Gitea #563 — an expired or disabled key loses its activation-time roles), then the session must carry the **`HistoryUpdate` role** (`OpcUaDataPlaneRoles.HistoryUpdate`). `ApiKeyInterimRoleMapper` confers it **only from a `Cluster`-scoped grant** carrying `NodePermissions.HistoryUpdate` — narrower-scoped grants confer nothing, for the same reason as `WriteOperate` (honouring a flag without honouring its scope would widen it). A key holding only `WriteOperate` is denied `BadUserAccessDenied`.
|
||||
- **Flag on:** `DataPlaneAuthorizer.Authorize(identity, OpcUaOperation.HistoryUpdate, nodeId, realm)` decides per node and **replaces** the role check (never chained) — a session with no roles succeeds on the strength of a scoped `HistoryUpdate` grant, and a `WriteOperate` grant does not help.
|
||||
- **Availability is separate from authorization.** A node accepts HistoryUpdate only if it is historized **and** the selected historian backend has an updater when the node materialises (the `HistoryWrite` AccessLevel bit). `Backend=Gateway` has none, so it answers `BadHistoryOperationUnsupported` for every node whatever the grant. Operations, result codes and unsupported cases: [`docs/Historian.md` §HistoryUpdate](Historian.md#historyupdate).
|
||||
- **Primary-only, after authorization (Gitea #674).** An authorized HistoryUpdate on a node that is not the redundancy Primary is refused `BadInvalidState` (the native-alarm-ack verdict, #597); an unauthorized one is told `BadUserAccessDenied` first, so the refusal never reveals the node's role to a caller who may not update.
|
||||
- **Every applied change is attributed (Part 11 modified-value history, Gitea #668).** The session's principal (the API key's
|
||||
`NodeAcl` principal, or the reserved `anonymous`) and key id are passed to the backend with every HistoryUpdate and recorded in the
|
||||
modification trail next to the prior value; a modified-value HistoryRead (`IsReadModified = true`) returns it as
|
||||
|
||||
@@ -815,7 +815,6 @@ Already used by Client.Shared in the existing codebase.
|
||||
| `Password` | string? | — | For Username auth |
|
||||
| `SessionTimeoutMs` | int | 120000 | Server-negotiated session timeout |
|
||||
| `KeepAliveIntervalMs` | int | 5000 | Session keep-alive interval |
|
||||
| `ReconnectPeriodMs` | int | 5000 | Initial reconnect delay |
|
||||
| `BrowseStrategy` | enum | Full | `Full` (browse all at startup), `Lazy`, `Hybrid` |
|
||||
| `BrowseRoot` | string? | — | Optional root NodeId to mirror (default: ObjectsFolder) |
|
||||
| `AutoAcceptCertificates` | bool | false | Accept untrusted server certificates |
|
||||
|
||||
@@ -47,7 +47,7 @@ public class HistoryReadCommand : CommandBase
|
||||
/// <summary>
|
||||
/// Gets the optional aggregate name used when the operator wants processed history instead of raw samples.
|
||||
/// </summary>
|
||||
[CommandOption("aggregate", Description = "Aggregate function: Average, Minimum, Maximum, Count, Start, End, StandardDeviation")]
|
||||
[CommandOption("aggregate", Description = "Aggregate function: Average, Minimum, Maximum, Count, Start, End, StandardDeviation, TimeAverage, TimeAverage2, Total, Total2, Interpolative")]
|
||||
public string? Aggregate { get; init; }
|
||||
|
||||
/// <summary>
|
||||
@@ -183,6 +183,12 @@ public class HistoryReadCommand : CommandBase
|
||||
await console.Output.WriteLineAsync();
|
||||
await console.Output.WriteLineAsync($"{values.Count} values returned.");
|
||||
}
|
||||
catch (ServiceResultException ex) when (ex.InnerException is null)
|
||||
{
|
||||
// e.g. HistoryReadStatus.ThrowIfRefused: a refusal is an operator-facing outcome, not a crash.
|
||||
// A wrapped failure (certificate validation, identity rejection on connect) keeps its full detail.
|
||||
throw new CommandException(ex.Message);
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (service != null)
|
||||
@@ -204,8 +210,13 @@ public class HistoryReadCommand : CommandBase
|
||||
"start" or "first" => AggregateType.Start,
|
||||
"end" or "last" => AggregateType.End,
|
||||
"standarddeviation" or "stddev" or "stdev" => AggregateType.StandardDeviation,
|
||||
"timeaverage" or "timeavg" => AggregateType.TimeAverage,
|
||||
"timeaverage2" or "timeavg2" => AggregateType.TimeAverage2,
|
||||
"total" => AggregateType.Total,
|
||||
"total2" => AggregateType.Total2,
|
||||
"interpolative" or "interp" => AggregateType.Interpolative,
|
||||
_ => throw new ArgumentException(
|
||||
$"Unknown aggregate: '{name}'. Supported: Average, Minimum, Maximum, Count, Start, End, StandardDeviation")
|
||||
$"Unknown aggregate: '{name}'. Supported: Average, Minimum, Maximum, Count, Start, End, StandardDeviation, TimeAverage, TimeAverage2, Total, Total2, Interpolative")
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
@@ -85,17 +85,22 @@ internal static class EndpointSelector
|
||||
$"No endpoint found with security mode '{requestedMode}'. Available endpoints: {available}");
|
||||
}
|
||||
|
||||
// Rewrite endpoint URL hostname to match user-supplied hostname. Necessary
|
||||
// when the OPC UA server returns a discovery URL using a different hostname
|
||||
// (e.g. internal DNS name) than the one the operator routed to.
|
||||
// Rewrite the endpoint URL host AND port to the user-supplied ones (as the SDK's
|
||||
// CoreClientUtils.SelectEndpoint does). A server behind a port mapping or internal DNS
|
||||
// advertises its own host:port (docker-dev containers all say :4840); following that would
|
||||
// open the session on a different server and fail the secure-channel certificate check.
|
||||
// Only when the scheme matches: a different scheme (https vs opc.tcp) is a different transport.
|
||||
var serverUri = new Uri(best.EndpointUrl);
|
||||
var requestedUri = new Uri(endpointUrl);
|
||||
if (serverUri.Host != requestedUri.Host)
|
||||
// A URL typed without a port (opc.tcp has no registered default, so Port is -1) keeps the advertised port.
|
||||
var port = requestedUri.Port > 0 ? requestedUri.Port : serverUri.Port;
|
||||
if (string.Equals(serverUri.Scheme, requestedUri.Scheme, StringComparison.OrdinalIgnoreCase)
|
||||
&& (serverUri.Host != requestedUri.Host || serverUri.Port != port))
|
||||
{
|
||||
var builder = new UriBuilder(best.EndpointUrl) { Host = requestedUri.Host };
|
||||
var builder = new UriBuilder(best.EndpointUrl) { Host = requestedUri.Host, Port = port };
|
||||
best.EndpointUrl = builder.ToString();
|
||||
Logger.Debug("Rewrote endpoint host from {ServerHost} to {RequestedHost}", serverUri.Host,
|
||||
requestedUri.Host);
|
||||
Logger.Debug("Rewrote endpoint from {ServerHost}:{ServerPort} to {RequestedHost}:{RequestedPort}",
|
||||
serverUri.Host, serverUri.Port, requestedUri.Host, port);
|
||||
}
|
||||
|
||||
return best;
|
||||
|
||||
@@ -24,6 +24,11 @@ public static class AggregateTypeMapper
|
||||
AggregateType.Start => ObjectIds.AggregateFunction_Start,
|
||||
AggregateType.End => ObjectIds.AggregateFunction_End,
|
||||
AggregateType.StandardDeviation => ObjectIds.AggregateFunction_StandardDeviationPopulation,
|
||||
AggregateType.TimeAverage => ObjectIds.AggregateFunction_TimeAverage,
|
||||
AggregateType.TimeAverage2 => ObjectIds.AggregateFunction_TimeAverage2,
|
||||
AggregateType.Total => ObjectIds.AggregateFunction_Total,
|
||||
AggregateType.Total2 => ObjectIds.AggregateFunction_Total2,
|
||||
AggregateType.Interpolative => ObjectIds.AggregateFunction_Interpolative,
|
||||
_ => throw new ArgumentOutOfRangeException(nameof(aggregate), aggregate, "Unknown AggregateType value.")
|
||||
};
|
||||
}
|
||||
|
||||
@@ -24,5 +24,20 @@ public enum AggregateType
|
||||
End,
|
||||
|
||||
/// <summary>Population standard deviation of values in the interval.</summary>
|
||||
StandardDeviation
|
||||
StandardDeviation,
|
||||
|
||||
/// <summary>Time-weighted average, interpolated bounds (Part 13 TimeAverage).</summary>
|
||||
TimeAverage,
|
||||
|
||||
/// <summary>Time-weighted average, stepped bounds (Part 13 TimeAverage2).</summary>
|
||||
TimeAverage2,
|
||||
|
||||
/// <summary>Time-weighted integral over the interval (Part 13 Total).</summary>
|
||||
Total,
|
||||
|
||||
/// <summary>Time-weighted integral, stepped bounds (Part 13 Total2).</summary>
|
||||
Total2,
|
||||
|
||||
/// <summary>Interpolated value at the interval start (Part 13 Interpolative).</summary>
|
||||
Interpolative
|
||||
}
|
||||
@@ -53,7 +53,12 @@ public partial class HistoryViewModel : ObservableObject
|
||||
AggregateType.Count,
|
||||
AggregateType.Start,
|
||||
AggregateType.End,
|
||||
AggregateType.StandardDeviation
|
||||
AggregateType.StandardDeviation,
|
||||
AggregateType.TimeAverage,
|
||||
AggregateType.TimeAverage2,
|
||||
AggregateType.Total,
|
||||
AggregateType.Total2,
|
||||
AggregateType.Interpolative
|
||||
];
|
||||
|
||||
/// <summary>Gets a value indicating whether an aggregate read is selected.</summary>
|
||||
|
||||
@@ -55,6 +55,18 @@ public enum HistoryAggregateType
|
||||
Total2,
|
||||
}
|
||||
|
||||
/// <summary>Shared <see cref="HistoryAggregateType"/> sets for <see cref="IHistorianDataSource.SupportedAggregates"/> (#682).</summary>
|
||||
public static class HistoryAggregateTypes
|
||||
{
|
||||
/// <summary>Every defined <see cref="HistoryAggregateType"/> — the interface default.</summary>
|
||||
public static readonly IReadOnlySet<HistoryAggregateType> All =
|
||||
System.Collections.Frozen.FrozenSet.ToFrozenSet(Enum.GetValues<HistoryAggregateType>());
|
||||
|
||||
/// <summary>No aggregates — a source with no history to aggregate (the Null historian).</summary>
|
||||
public static readonly IReadOnlySet<HistoryAggregateType> None =
|
||||
System.Collections.Frozen.FrozenSet<HistoryAggregateType>.Empty;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Thrown by a history backend when it cannot serve the requested <see cref="HistoryAggregateType"/>.
|
||||
/// The Server HistoryRead path translates it to <c>BadAggregateNotSupported</c> for that node only.
|
||||
@@ -81,6 +93,14 @@ public sealed class HistoryReadLimitExceededException : InvalidOperationExceptio
|
||||
/// <summary>The configured limit that was exceeded.</summary>
|
||||
public long Limit { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #694: a raw read refused because rows BEFORE its window (inside the backend's look-back margin — a tie
|
||||
/// cluster just before a page start, the HistorianGateway's boundary row) crowded a ceiling-bounded row budget: how
|
||||
/// many such rows the read saw (a lower bound when the read was itself truncated). <c>0</c> = not a crowding refusal.
|
||||
/// The server re-reads such a page once, smaller by this many values (<c>RawPageLimit</c>).
|
||||
/// </summary>
|
||||
public int PreWindowRows { get; init; }
|
||||
|
||||
/// <summary>Creates the exception.</summary>
|
||||
/// <param name="limit">The configured limit.</param>
|
||||
/// <param name="message">Operator-facing description.</param>
|
||||
@@ -88,7 +108,7 @@ public sealed class HistoryReadLimitExceededException : InvalidOperationExceptio
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// One row returned by <see cref="IHistorianDataSource.ReadEventsAsync"/> — a historical
|
||||
/// One row returned by <see cref="IHistorianDataSource.ReadEventsAsync(string?, DateTime, DateTime, int, CancellationToken)"/> — a historical
|
||||
/// alarm/event record, not the OPC UA live-event stream. Fields match the minimum set the
|
||||
/// Server needs to populate a <c>HistoryEventFieldList</c> for HistoryReadEvents responses.
|
||||
/// </summary>
|
||||
@@ -106,8 +126,8 @@ public sealed record HistoricalEvent(
|
||||
string? Message,
|
||||
ushort Severity);
|
||||
|
||||
/// <summary>Result of a <see cref="IHistorianDataSource.ReadEventsAsync"/> call.</summary>
|
||||
/// <param name="Events">Events in chronological order by <c>EventTimeUtc</c>.</param>
|
||||
/// <summary>Result of a <see cref="IHistorianDataSource.ReadEventsAsync(string?, DateTime, DateTime, int, CancellationToken)"/> call.</summary>
|
||||
/// <param name="Events">Events ordered by <c>EventTimeUtc</c> in the read's direction — ascending, or descending for a reverse read (#675).</param>
|
||||
/// <param name="ContinuationPoint">Opaque token for the next call when more events are available; null when complete.</param>
|
||||
public sealed record HistoricalEventsResult(
|
||||
IReadOnlyList<HistoricalEvent> Events,
|
||||
|
||||
@@ -17,6 +17,12 @@ public interface IHistorianDataSource : IDisposable
|
||||
/// <summary>
|
||||
/// Read raw historical samples for a single tag over a time range.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Forward (<c>startUtc < endUtc</c>) is <c>[start, end)</c> ascending; <c>startUtc == endUtc</c> is a point read.
|
||||
/// <c>startUtc > endUtc</c> is the Part 11 reverse read <c>(end, start]</c> newest-first — but only a backend that
|
||||
/// declares <see cref="SupportsReverseRaw"/> is ever sent one (Gitea #686); the server serves a reverse read on any
|
||||
/// other backend from bounded forward reads.
|
||||
/// </remarks>
|
||||
/// <param name="fullReference">The full reference of the tag to read.</param>
|
||||
/// <param name="startUtc">The start of the time range in UTC.</param>
|
||||
/// <param name="endUtc">The end of the time range in UTC.</param>
|
||||
@@ -30,6 +36,28 @@ public interface IHistorianDataSource : IDisposable
|
||||
uint maxValuesPerNode,
|
||||
CancellationToken cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// Whether this backend can serve a reverse (newest-first) raw read itself — <see cref="ReadRawAsync"/> with
|
||||
/// <c>startUtc > endUtc</c> (Gitea #686). The server checks it before every raw read. When <c>false</c> it never
|
||||
/// sends <c>start > end</c>: it serves the reverse read — an explicit one, a resumed reverse page, and the
|
||||
/// end-only capped read Part 11 reads backward from EndTime — from a series of bounded FORWARD reads walking back
|
||||
/// from the later bound (<c>ReverseRawReader</c> in the OPC UA server). Default <c>false</c>, the safe direction: a
|
||||
/// source that only reads forward is still answered correctly. A backend declares it itself (a guard test pins that).
|
||||
/// </summary>
|
||||
bool SupportsReverseRaw => false;
|
||||
|
||||
/// <summary>
|
||||
/// The most values one capped <see cref="ReadRawAsync"/> can currently return without failing
|
||||
/// <see cref="HistoryReadLimitExceededException"/> on a BACKEND row ceiling (Gitea #694); <c>0</c> = no known limit.
|
||||
/// The server sizes its raw pages to it — a page is at most this many values and carries a continuation point when
|
||||
/// full (Part 11 allows fewer than <c>NumValuesPerNode</c> with a continuation point), instead of failing a page size
|
||||
/// the backend cannot read in one go. It may change at runtime (a strict HistorianGateway's ceiling is learned from
|
||||
/// its first refusal, #691.3), so it is read per read. A backend whose reads are never ceiling-bound returns <c>0</c>.
|
||||
/// The source still never truncates: a read asking for more than this is answered in full or refused. Default
|
||||
/// <c>0</c>; a backend declares it itself (a guard test pins that).
|
||||
/// </summary>
|
||||
int MaxRawValuesPerRead => 0;
|
||||
|
||||
/// <summary>
|
||||
/// Read processed (interval-bucketed) samples — average / min / max / count / etc.
|
||||
/// A bucket with no source data returns a sample whose
|
||||
@@ -65,6 +93,31 @@ public interface IHistorianDataSource : IDisposable
|
||||
IReadOnlyList<DateTime> timestampsUtc,
|
||||
CancellationToken cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #693: <see cref="ReadAtTimeAsync(string, IReadOnlyList{DateTime}, CancellationToken)"/> with the value type of
|
||||
/// the node being read, for a backend whose stored values do not carry their type (the Gateway: AVEVA stores every
|
||||
/// historized type as a numeric analog, so a Boolean comes back as a 0/1 Double). Such a backend uses the hint to apply
|
||||
/// the AtTime rules exactly as the typed backends do — a Boolean (or String / DateTime) node is stepped, an integer node
|
||||
/// is interpolated, rounded half away from zero and returned as the node's integer type, a floating-point node is
|
||||
/// interpolated. A backend that stores typed values ignores it: the default forwards to the hint-less overload, so
|
||||
/// SqlServer, QuestDB and the Null source need no change.
|
||||
/// </summary>
|
||||
/// <param name="fullReference">The full reference of the tag to read.</param>
|
||||
/// <param name="timestampsUtc">The list of timestamps to read values at.</param>
|
||||
/// <param name="valueType">
|
||||
/// The scalar value type of the node (<see cref="TypeCode.Boolean"/>, the integer codes, <see cref="TypeCode.Single"/>,
|
||||
/// <see cref="TypeCode.Double"/>, <see cref="TypeCode.String"/>, <see cref="TypeCode.DateTime"/>), or <c>null</c> when
|
||||
/// unknown, not scalar, or not one of those — the backend's own policy then applies.
|
||||
/// </param>
|
||||
/// <param name="cancellationToken">A cancellation token that can be used to cancel the operation.</param>
|
||||
/// <returns>One sample per requested timestamp, in the same order as <paramref name="timestampsUtc"/>.</returns>
|
||||
Task<HistoryReadResult> ReadAtTimeAsync(
|
||||
string fullReference,
|
||||
IReadOnlyList<DateTime> timestampsUtc,
|
||||
TypeCode? valueType,
|
||||
CancellationToken cancellationToken) =>
|
||||
ReadAtTimeAsync(fullReference, timestampsUtc, cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// Read historical alarm / event records — OPC UA HistoryReadEvents service.
|
||||
/// Distinct from any live event stream; sources here come from the historian's
|
||||
@@ -79,7 +132,7 @@ public interface IHistorianDataSource : IDisposable
|
||||
/// <c>uint maxValuesPerNode</c> because their OPC UA HistoryRead surface has no
|
||||
/// equivalent "use default" sentinel.
|
||||
///
|
||||
/// This surface declares <see cref="ReadAtTimeAsync"/> and <see cref="ReadEventsAsync"/>
|
||||
/// This surface declares <see cref="ReadAtTimeAsync(string, IReadOnlyList{DateTime}, CancellationToken)"/> and <see cref="ReadEventsAsync(string?, DateTime, DateTime, int, CancellationToken)"/>
|
||||
/// as required members — a server-side historian owns the full read surface.
|
||||
/// </remarks>
|
||||
/// <param name="sourceName">The source name to filter events, or null to return events from all sources.</param>
|
||||
@@ -101,6 +154,57 @@ public interface IHistorianDataSource : IDisposable
|
||||
int maxEvents,
|
||||
CancellationToken cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// Whether this backend can serve a reverse (newest-first) events read — the reverse overload of
|
||||
/// <see cref="ReadEventsAsync(string?, DateTime, DateTime, int, bool, CancellationToken)"/> (Gitea #675). The server
|
||||
/// checks it BEFORE calling the source: a reverse read (explicit <c>StartTime > EndTime</c>, or an end-only capped
|
||||
/// read, which Part 11 reads backward) on a backend that cannot is never sent the reverse overload — since Gitea
|
||||
/// #691.4 the server walks it back from bounded FORWARD reads instead (#675 refused it). Default <c>false</c>; an
|
||||
/// implementation (or a wrapper) declares it itself — a guard test pins that.
|
||||
/// </summary>
|
||||
bool SupportsReverseEvents => false;
|
||||
|
||||
/// <summary>
|
||||
/// Read historical alarm / event records in a chosen direction — the overload the server's HistoryReadEvents
|
||||
/// paging calls (Gitea #675).
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// The bounds may be given in either order; let <c>lo</c>/<c>hi</c> be the earlier/later of them. Forward
|
||||
/// (<paramref name="reverse"/> <c>false</c>) is exactly <see cref="ReadEventsAsync(string?, DateTime, DateTime, int, CancellationToken)"/>:
|
||||
/// <c>[lo, hi)</c> ascending by <c>(EventTimeUtc, EventId)</c>. Reverse is the Part 11 reverse window
|
||||
/// <c>(lo, hi]</c> in DESCENDING <c>(EventTimeUtc, EventId)</c> order, so a cap keeps the LATEST
|
||||
/// <paramref name="maxEvents"/> events — the mirror of forward, with the same tie-breaker reversed, so a
|
||||
/// resumed reverse page that re-reads its boundary timestamp sees the boundary ties in the same order.
|
||||
/// <paramref name="maxEvents"/> and the continuation-point contract are as for the forward read.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// The default forwards a forward read to the five-argument overload and throws
|
||||
/// <see cref="NotSupportedException"/> for a reverse one — a backend that cannot read newest-first (the
|
||||
/// Gateway). The server never relies on that throw: it consults <see cref="SupportsReverseEvents"/> first and walks
|
||||
/// such a read back from forward reads (#691.4), never returning the earliest events in ascending order as if
|
||||
/// they were the answer. A backend that serves reverse natively must override BOTH members.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
/// <param name="sourceName">The source name to filter events, or null to return events from all sources.</param>
|
||||
/// <param name="startUtc">One bound of the window (UTC).</param>
|
||||
/// <param name="endUtc">The other bound of the window (UTC).</param>
|
||||
/// <param name="maxEvents">The cap; non-positive uses the backend default (see the five-argument overload).</param>
|
||||
/// <param name="reverse"><c>true</c> to read newest-first over <c>(lo, hi]</c>.</param>
|
||||
/// <param name="cancellationToken">Cancellation.</param>
|
||||
/// <returns>The matching events in the read's direction, with a continuation point if truncated.</returns>
|
||||
/// <exception cref="NotSupportedException">A reverse read on a backend that cannot read newest-first.</exception>
|
||||
Task<HistoricalEventsResult> ReadEventsAsync(
|
||||
string? sourceName,
|
||||
DateTime startUtc,
|
||||
DateTime endUtc,
|
||||
int maxEvents,
|
||||
bool reverse,
|
||||
CancellationToken cancellationToken) =>
|
||||
reverse
|
||||
? throw new NotSupportedException("This historian backend cannot read events newest-first (reverse).")
|
||||
: ReadEventsAsync(sourceName, startUtc, endUtc, maxEvents, cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// Point-in-time health snapshot for diagnostics and dashboards. Pure
|
||||
/// observation; never blocks on backend I/O.
|
||||
@@ -123,6 +227,25 @@ public interface IHistorianDataSource : IDisposable
|
||||
/// </remarks>
|
||||
long TimestampResolutionTicks => 1;
|
||||
|
||||
/// <summary>
|
||||
/// The Part 13 aggregates this backend serves for a processed HistoryRead (Gitea #682) — the ONE authority for
|
||||
/// both what the server advertises under <c>HistoryServerCapabilities/AggregateFunctions</c> (i=11201) and what
|
||||
/// its HistoryRead-Processed answers <c>BadAggregateNotSupported</c> up front without calling the backend.
|
||||
/// (<c>ServerCapabilities/AggregateFunctions</c>, i=2997, is not affected: it lists every aggregate the server
|
||||
/// computes, including for live <c>AggregateFilter</c> subscriptions.)
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The default is <see cref="HistoryAggregateTypes.All"/> — "every aggregate the server can name; a member I cannot
|
||||
/// serve throws <see cref="HistoryAggregateNotSupportedException"/> at read time", which is exactly the pre-#682
|
||||
/// behaviour, so a test fake or an out-of-tree source keeps working unchanged. It is a concrete set rather than a
|
||||
/// <c>null</c> "all" sentinel so every consumer handles one shape. Every PRODUCTION backend declares its set itself
|
||||
/// (a reflection guard pins that): a wrapper that forgot to forward this would otherwise advertise every aggregate.
|
||||
/// The set is read once when the source is wired (the backend is fixed at boot by <c>ServerHistorian:Backend</c>)
|
||||
/// and again on every processed read, so it must be cheap and stable — return a static frozen set. A source that
|
||||
/// returns <c>null</c> anyway is treated as the default (all) by both consumers.
|
||||
/// </remarks>
|
||||
IReadOnlySet<HistoryAggregateType> SupportedAggregates => HistoryAggregateTypes.All;
|
||||
|
||||
/// <summary>
|
||||
/// Read modified-value history — OPC UA HistoryReadRaw with <c>IsReadModified = true</c> (Part 11, Gitea #668):
|
||||
/// every modification a HistoryUpdate applied to the tag inside the window, each carrying the PRIOR value.
|
||||
|
||||
@@ -23,6 +23,12 @@ public enum HistoryUpdateMode
|
||||
/// <see cref="HistoryUpdateContext.Principal"/>, the modification time and the <see cref="HistoryModificationType"/>,
|
||||
/// readable through <see cref="IHistorianDataSource.ReadModifiedAsync"/>. A refused item (any non-Good code) leaves
|
||||
/// no record. Live recorder writes and retention purges never leave one.
|
||||
/// <para>
|
||||
/// <b>Cannot update at all</b> (Gitea #674): an implementation that is wired but cannot perform any update right
|
||||
/// now — the SqlServer backend on a schema without the modification trail — throws
|
||||
/// <see cref="NotSupportedException"/> from every method without touching storage. The server reports it as a
|
||||
/// node-level <c>BadHistoryOperationUnsupported</c>, the same as <see cref="IHistorianDataSource.ReadModifiedAsync"/>.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public interface IHistorianDataUpdater
|
||||
{
|
||||
|
||||
@@ -55,6 +55,40 @@ public sealed class NullHistorianDataSource : IHistorianDataSource
|
||||
int maxEvents,
|
||||
CancellationToken cancellationToken) => Task.FromResult(EmptyEvents);
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks><c>true</c> (Gitea #686): there is no history, so the empty result is the correct answer in either
|
||||
/// direction — no need for the server to walk back through forward reads to find nothing.</remarks>
|
||||
public bool SupportsReverseRaw => true;
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks><c>0</c>: no history, no ceiling (Gitea #694).</remarks>
|
||||
public int MaxRawValuesPerRead => 0;
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks><c>true</c>, like <see cref="SupportsReverseRaw"/> (Gitea #691.4 review; <c>false</c> under #675): there is no
|
||||
/// history to read in either direction — no need for the server to walk back through empty forward reads.</remarks>
|
||||
public bool SupportsReverseEvents => true;
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>The empty result in either direction.</remarks>
|
||||
public Task<HistoricalEventsResult> ReadEventsAsync(
|
||||
string? sourceName,
|
||||
DateTime startUtc,
|
||||
DateTime endUtc,
|
||||
int maxEvents,
|
||||
bool reverse,
|
||||
CancellationToken cancellationToken) =>
|
||||
Task.FromResult(EmptyEvents);
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>
|
||||
/// EMPTY (Gitea #682): with no historian configured the server advertises no aggregate functions, and a processed
|
||||
/// HistoryRead answers <c>BadAggregateNotSupported</c> per node rather than <c>GoodNoData</c> — an aggregate is a
|
||||
/// statistic the server cannot compute without a history, and advertising the SDK's full list on a server with no
|
||||
/// historian misled capability discovery. Raw / AtTime / Events reads still return the empty <c>GoodNoData</c> result.
|
||||
/// </remarks>
|
||||
public IReadOnlySet<HistoryAggregateType> SupportedAggregates => HistoryAggregateTypes.None;
|
||||
|
||||
/// <inheritdoc />
|
||||
public HistorianHealthSnapshot GetHealthSnapshot() => new(
|
||||
TotalQueries: 0,
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
namespace ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
|
||||
/// <summary>
|
||||
/// Optional driver capability (Gitea #676) — a driver that can tell, from its protocol, that its connection is
|
||||
/// DEFINITIVELY gone (it can never recover in place, only a re-initialise brings it back) raises
|
||||
/// <see cref="ConnectionLost"/> so the host re-initialises it at once instead of waiting out the #527 comms-loss
|
||||
/// watchdog (~90 s of frozen <see cref="DriverHealth.LastSuccessfulRead"/>).
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>Definitive only.</b> An AMBIGUOUS failure — a timeout, a stalled socket, "no communication" — must NOT
|
||||
/// raise: the connection may yet resume in place (a <c>docker pause</c>, a congested link), and tearing it
|
||||
/// down would cost the session, its subscriptions and the in-place recovery. Ambiguous failures keep going
|
||||
/// through the watchdog, whose conjunction (non-Healthy AND a frozen read stamp) is the honest test for them.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// The host (<c>DriverInstanceActor</c>) acts on a raise only while the driver is Connected — in any other
|
||||
/// state a re-initialise is already pending or running — and it subscribes for its own lifetime, not per
|
||||
/// connect, because the <see cref="IDriver"/> instance can outlive the actor. A driver should raise at most
|
||||
/// once per connection and re-arm when it is re-initialised.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public interface IDriverConnectionLossSource
|
||||
{
|
||||
/// <summary>Raised (on the driver's own thread) when the driver's connection is definitively lost.</summary>
|
||||
event EventHandler<DriverConnectionLossEventArgs>? ConnectionLost;
|
||||
}
|
||||
|
||||
/// <summary>Event payload for <see cref="IDriverConnectionLossSource.ConnectionLost"/>.</summary>
|
||||
/// <param name="Reason">Driver-supplied reason for the diagnostic log (e.g. the protocol status that proved the loss).</param>
|
||||
public sealed record DriverConnectionLossEventArgs(string Reason);
|
||||
@@ -0,0 +1,514 @@
|
||||
using ZB.MOM.WW.HistorianGateway.Contracts.Grpc;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Mapping;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway;
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #693: HistoryRead AtTime on the Gateway backend, served from RAW gateway reads and evaluated with the SQL backends'
|
||||
/// shared <see cref="AtTimeEvaluator"/> (Part 11 §6.5.4) — as #681 did for the per-sample aggregates — so the Gateway
|
||||
/// answers exactly what SqlServer and QuestDB answer for the same stored samples. AVEVA's own interpolated answer (the
|
||||
/// gateway's <c>ReadAtTime</c>) is not used: live, it returned the previous value as Good at a stored NULL's own time,
|
||||
/// null + Bad at a Good sample right after a NULL, a drifted value (99.99999999944) at an exact stored 100, and no
|
||||
/// Interpolated bit.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>What each requested time needs.</b> <c>prev</c> = the latest stored sample at or before it, <c>next</c> = the
|
||||
/// first stored sample after it. <c>next</c> is only read when the evaluator uses it (a usable <c>prev</c> that is not
|
||||
/// an exact hit).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Read plan — O(1) reads for a dense batch, never two round-trips per timestamp by default.</b> The distinct
|
||||
/// requested times are walked in ascending order with ONE forward window read <c>[t − 1 ms, max + 1 ms]</c> of at most
|
||||
/// <see cref="ReadRows"/> rows, which yields every in-window neighbour. The sample before the first time comes from a
|
||||
/// walk-back (below), the sample after the last time from one small forward probe. A window read that fills its budget
|
||||
/// (dense data, widely spread times) resolves the times before its last row; the next unresolved time either
|
||||
/// continues from that row (when the rows in between are expected to fit one read) or jumps there with a read that
|
||||
/// starts a density-sized margin before it (so its <c>prev</c> is normally in the same read). Ten times one second
|
||||
/// apart on a 1 Hz tag cost 1 read (all exact hits) or 3 (interpolated); ten times an hour apart about one read each.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Walk-back</b> (the gateway reads forward only), in two phases. <i>Gallop:</i> segments <c>(lo, hi]</c> stepping
|
||||
/// back from <c>hi</c>, each one forward read, the first sized from the density the window read showed (else
|
||||
/// <see cref="InitialWalkWidth"/>), an empty one growing the next ×4; after two empty segments the whole remainder down
|
||||
/// to the epoch is asked once — a complete answer ends the walk, a FULL one seeds the bisection with its last row.
|
||||
/// <i>Bisect</i> (once a stored row is known below <c>hi</c> — a full segment's last row, or the last row of an earlier
|
||||
/// window read): read <c>(mid, hi]</c>; empty ⇒ <c>hi = mid</c>; full ⇒ the known row moves up to its last row;
|
||||
/// complete with rows ⇒ its last row is the answer. Right after a full read the next segment is aimed at about a
|
||||
/// quarter of a read at the density it showed (one try — an empty aimed segment falls back to halving), and below
|
||||
/// 2 ms the segment is the whole stretch from the known row (never a 100 ns step).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Bounded.</b> One request may take at most <c>64 + 16 × distinct times</c> raw reads (<see cref="BaseReads"/>,
|
||||
/// <see cref="ReadsPerTime"/>); past it — or with a strict-gateway ceiling below <see cref="MinBudget"/> rows — the
|
||||
/// node fails <c>BadTooManyOperations</c> with a message naming the limit. A jump read (to a time far past the previous
|
||||
/// read) asks for only <see cref="JumpRows"/> rows, so a thousand times spread over ten years cost about a thousand
|
||||
/// small reads, not a thousand 1,024-row reads.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Wire shape.</b> Every request bound is a whole millisecond (AVEVA stores milliseconds and truncates request
|
||||
/// times); a row counts only strictly INSIDE its request — AVEVA's synthetic boundary row at the query start (the value
|
||||
/// in effect there, <c>Quality = 133</c>, or a NULL row) can never be taken for a stored sample, nor a possible row at
|
||||
/// the query end; the epoch and before hold nothing. A Bad row (a stored NULL arrives as value 0, <c>q=1 opc=0</c>,
|
||||
/// #690) is value-less. Every stream is drained, never abandoned (a cancelled gateway stream evicts its pooled
|
||||
/// historian session). Nothing is asked past <c>now + </c><see cref="RawAggregateReader.LookAheadHorizon"/> (the
|
||||
/// historian stores nothing further ahead, and the gateway's AVEVA may refuse a far-future end); a strict gateway's
|
||||
/// row ceiling (#691.3) is learned and honoured — a short read is ordinary paging here, never re-probed.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Values.</b> A gateway value is a Double or a String whatever the node's data type (AVEVA stores every historized
|
||||
/// type as a numeric analog — a Boolean as a 0/1 <c>Int2</c>). The node's type hint converts each row BEFORE the
|
||||
/// shared evaluator runs, so its own type rules apply exactly as on the SQL backends: Boolean (value ≠ 0) and String
|
||||
/// are stepped; an integer type is an Int64 — interpolated, rounded half away from zero — and the result is returned
|
||||
/// as the node's integer type; Float / Double (and no hint) interpolate linearly as Double.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
internal static class GatewayAtTimeReader
|
||||
{
|
||||
/// <summary>The row budget of a window read (the first, or one continuing from a full one) and a walk-back segment.</summary>
|
||||
internal const int ReadRows = 1024;
|
||||
|
||||
/// <summary>The row budget of a jump read (a window read restarting at a time far past the previous read's last row).</summary>
|
||||
internal const int JumpRows = 64;
|
||||
|
||||
/// <summary>The row budget of the forward probe for the sample after the last time (boundary row + ties + one).</summary>
|
||||
internal const int ProbeRows = 2 + RawAggregateReader.PreWindowAllowance;
|
||||
|
||||
/// <summary>The smallest row budget a read can work with (the boundary row, a tie, one sample beyond).</summary>
|
||||
internal const int MinBudget = 4;
|
||||
|
||||
/// <summary>Raw reads one request may always take …</summary>
|
||||
internal const int BaseReads = 64;
|
||||
|
||||
/// <summary>
|
||||
/// … plus this many per distinct requested time. A runaway guard, not a budget (<c>HistoryReadDeadline</c> bounds the
|
||||
/// wall clock): a time inside a gap of a bursty tag legitimately costs a jump read plus a bisection back across the gap
|
||||
/// (≈ 6 reads — a 1 h/day tag asked at noon on consecutive days), so 4 refused requests AVEVA's own AtTime served.
|
||||
/// </summary>
|
||||
internal const int ReadsPerTime = 16;
|
||||
|
||||
/// <summary>The first walk-back segment's width when the window read showed no density.</summary>
|
||||
internal static readonly TimeSpan InitialWalkWidth = TimeSpan.FromMinutes(1);
|
||||
|
||||
/// <summary>The widest walk-back segment sized from density, before ×4 growth.</summary>
|
||||
private static readonly TimeSpan MaxSeededWidth = TimeSpan.FromDays(1);
|
||||
|
||||
private const long Ms = TimeSpan.TicksPerMillisecond;
|
||||
|
||||
private static readonly TimeSpan OneMs = TimeSpan.FromMilliseconds(1);
|
||||
|
||||
private static readonly DateTime Floor = RawAggregateReader.ProbeFloor;
|
||||
|
||||
/// <summary>One drained read: the rows strictly inside the request (ascending) and whether the stream ended short.</summary>
|
||||
private readonly record struct Read(List<StoredSample> Rows, bool Complete);
|
||||
|
||||
/// <summary>Evaluates AtTime for <paramref name="timestampsUtc"/> (same length and order, duplicates included).</summary>
|
||||
/// <param name="client">The gateway client.</param>
|
||||
/// <param name="ceiling">The data source's learned gateway row ceiling (#691.3).</param>
|
||||
/// <param name="tag">Historian tag name.</param>
|
||||
/// <param name="timestampsUtc">Requested times (any kind; treated as UTC, as on the SQL backends).</param>
|
||||
/// <param name="now">Server timestamp, the beyond-now rule and the look-ahead horizon.</param>
|
||||
/// <param name="ct">Cancellation.</param>
|
||||
/// <param name="valueType">The node's scalar value type (the hinted <c>IHistorianDataSource.ReadAtTimeAsync</c>), or null.</param>
|
||||
/// <param name="maxReads">The request's raw-read cap; null = <see cref="BaseReads"/> + <see cref="ReadsPerTime"/> per distinct time.</param>
|
||||
/// <returns>One value per requested time, stamped with it.</returns>
|
||||
/// <exception cref="HistoryReadLimitExceededException">More than <see cref="ReadRows"/> rows within one millisecond, the
|
||||
/// read cap, or a strict gateway ceiling below <see cref="MinBudget"/>.</exception>
|
||||
public static async Task<IReadOnlyList<DataValueSnapshot>> ReadAsync(IHistorianGatewayClient client, GatewayRowCeiling ceiling,
|
||||
string tag, IReadOnlyList<DateTime> timestampsUtc, DateTime now, CancellationToken ct, TypeCode? valueType = null,
|
||||
int? maxReads = null)
|
||||
{
|
||||
var requested = timestampsUtc.Select(t => DateTime.SpecifyKind(t, DateTimeKind.Utc)).ToArray();
|
||||
if (requested.Length == 0) return [];
|
||||
var times = requested.Distinct().Order().ToArray();
|
||||
var ctx = new Ctx(client, ceiling, tag, ct, valueType, maxReads ?? BaseReads + ReadsPerTime * times.Length);
|
||||
var neighbours = await ResolveAsync(ctx, times, now).ConfigureAwait(false);
|
||||
var result = new DataValueSnapshot[requested.Length];
|
||||
for (var i = 0; i < requested.Length; i++)
|
||||
{
|
||||
var t = requested[i];
|
||||
var (prev, next) = neighbours[t.Ticks];
|
||||
// A DateTime node is stepped (as on the SQL backends): a usable non-numeric next sends the evaluator down its
|
||||
// stepped branch, with the next's own status.
|
||||
if (valueType == TypeCode.DateTime && next is { IsUsable: true } n)
|
||||
next = n with { Value = HistorianValue.FromString(string.Empty) };
|
||||
result[i] = ToNodeType(AtTimeEvaluator.Evaluate(t, prev, next, now, now), valueType);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/// <summary>One request's read context: the client, the type hint, and the read count against its cap.</summary>
|
||||
private sealed class Ctx(IHistorianGatewayClient client, GatewayRowCeiling ceiling, string tag, CancellationToken ct,
|
||||
TypeCode? valueType, int maxReads)
|
||||
{
|
||||
public IHistorianGatewayClient Client { get; } = client;
|
||||
public GatewayRowCeiling Ceiling { get; } = ceiling;
|
||||
public string Tag { get; } = tag;
|
||||
public CancellationToken Ct { get; } = ct;
|
||||
public TypeCode? ValueType { get; } = valueType;
|
||||
public int MaxReads { get; } = maxReads;
|
||||
public int Reads { get; set; }
|
||||
|
||||
/// <summary>The smallest budget a read was served with (a strict gateway's ceiling shows here).</summary>
|
||||
public int SmallestBudget { get; set; } = int.MaxValue;
|
||||
}
|
||||
|
||||
/// <summary>True for the integer type codes (evaluated as Int64, returned as the node's own type).</summary>
|
||||
private static bool IsInteger(TypeCode? t) =>
|
||||
t is TypeCode.SByte or TypeCode.Byte or TypeCode.Int16 or TypeCode.UInt16 or TypeCode.Int32 or TypeCode.UInt32
|
||||
or TypeCode.Int64 or TypeCode.UInt64;
|
||||
|
||||
/// <summary>
|
||||
/// An evaluated Int64 (integer hint) returned as the node's integer type. A value the type cannot hold (only possible
|
||||
/// for a stored value already outside the node's range) stays the evaluator's Int64.
|
||||
/// </summary>
|
||||
private static DataValueSnapshot ToNodeType(DataValueSnapshot v, TypeCode? valueType)
|
||||
{
|
||||
if (v.Value is not long l || !IsInteger(valueType) || valueType == TypeCode.Int64) return v;
|
||||
try
|
||||
{
|
||||
return v with { Value = Convert.ChangeType(l, valueType!.Value, System.Globalization.CultureInfo.InvariantCulture) };
|
||||
}
|
||||
catch (OverflowException)
|
||||
{
|
||||
return v;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>prev / next for every time of <paramref name="times"/> (distinct, ascending).</summary>
|
||||
private static async Task<Dictionary<long, (StoredSample? Prev, StoredSample? Next)>> ResolveAsync(
|
||||
Ctx ctx, DateTime[] times, DateTime now)
|
||||
{
|
||||
var result = new Dictionary<long, (StoredSample?, StoredSample?)>(times.Length);
|
||||
var horizon = MsFloor(DateTime.MaxValue - now > RawAggregateReader.LookAheadHorizon
|
||||
? DateTime.SpecifyKind(now + RawAggregateReader.LookAheadHorizon, DateTimeKind.Utc)
|
||||
: DateTime.MaxValue.AddTicks(-Ms));
|
||||
|
||||
// Nothing is stored at or before the epoch.
|
||||
var i = 0;
|
||||
for (; i < times.Length && times[i] <= Floor; i++) result[times[i].Ticks] = (null, null);
|
||||
if (i == times.Length) return result;
|
||||
|
||||
// Reads are planned on times clamped to the horizon (nothing is stored beyond it, and no read ends past it).
|
||||
var qe = Min(MsFloor(Min(times[^1], horizon)) + OneMs, horizon);
|
||||
var qs = RequestStart(Min(times[i], horizon));
|
||||
var budget = ReadRows;
|
||||
StoredSample? before = null; // the latest stored sample at or before qs …
|
||||
var beforeKnown = false; // … once known
|
||||
DateTime? walkKnown = null; // a stored sample exists here (at or before qs): the walk-back bisects above it
|
||||
var walkWidth = InitialWalkWidth;
|
||||
var probeFrom = -1; // first index whose next lies at or after qe (complete last read)
|
||||
|
||||
while (true)
|
||||
{
|
||||
var read = await ReadTrustedAsync(ctx, qs, qe, budget).ConfigureAwait(false);
|
||||
var rows = read.Rows;
|
||||
if (!read.Complete && rows.Count == 0)
|
||||
throw TooDense(ctx, qs);
|
||||
if (rows.Count >= 2 && !beforeKnown && walkKnown is null)
|
||||
walkWidth = Clamp(TimeSpan.FromTicks(4 * (rows[^1].TimeUtc - rows[0].TimeUtc).Ticks / (rows.Count - 1)));
|
||||
|
||||
// A short stream saw every row up to qe; a full one every row before its last row's time x.
|
||||
var x = read.Complete ? qe : rows[^1].TimeUtc;
|
||||
var j = -1; // index of the last row at or before the current time
|
||||
var k = i;
|
||||
for (; k < times.Length; k++)
|
||||
{
|
||||
var t = times[k];
|
||||
var tr = Min(t, horizon);
|
||||
if (!read.Complete && tr >= x) break;
|
||||
while (j + 1 < rows.Count && rows[j + 1].TimeUtc <= tr) j++;
|
||||
|
||||
StoredSample? prev;
|
||||
if (j >= 0) prev = rows[j];
|
||||
else
|
||||
{
|
||||
if (!beforeKnown)
|
||||
{
|
||||
// #696: prove the future stretch empty before jumping to now. Historian clocks or imports
|
||||
// can place real rows there; a full bounded probe seeds the original backward search.
|
||||
var searchEnd = qs;
|
||||
if (qs > now && walkKnown is null)
|
||||
{
|
||||
var future = await ReadTrustedAsync(ctx, MsFloor(now),
|
||||
Min(MsFloor(qs) + OneMs, horizon), ReadRows).ConfigureAwait(false);
|
||||
StoredSample? latest = null;
|
||||
foreach (var row in future.Rows)
|
||||
if (row.TimeUtc <= qs) latest = row;
|
||||
if (future.Complete)
|
||||
{
|
||||
before = latest;
|
||||
beforeKnown = latest is not null;
|
||||
searchEnd = now;
|
||||
}
|
||||
else if (latest is { } found)
|
||||
walkKnown = found.TimeUtc;
|
||||
}
|
||||
if (!beforeKnown)
|
||||
before = await WalkBackAsync(ctx, searchEnd, walkKnown, walkWidth).ConfigureAwait(false);
|
||||
beforeKnown = true;
|
||||
}
|
||||
prev = before;
|
||||
}
|
||||
|
||||
StoredSample? next = null;
|
||||
if (NeedsNext(t, prev) && t < horizon)
|
||||
{
|
||||
if (j + 1 < rows.Count) next = rows[j + 1];
|
||||
else if (probeFrom < 0) probeFrom = k; // after every row of a complete read: the probe answers it
|
||||
}
|
||||
result[t.Ticks] = (prev, next);
|
||||
}
|
||||
|
||||
if (k == times.Length) break;
|
||||
|
||||
// Truncated before times[k]: continue from the last row, or jump to times[k] with a density-sized margin and a
|
||||
// small budget (it needs that time's neighbours, and whatever later times follow closely).
|
||||
var tn = Min(times[k], horizon);
|
||||
var spanTicks = Math.Max(1, (x - rows[0].TimeUtc).Ticks);
|
||||
var density = (double)rows.Count / spanTicks; // rows per tick
|
||||
var gapRows = density * (tn - x).Ticks;
|
||||
var margin = Clamp(TimeSpan.FromTicks((long)Math.Min(TimeSpan.MaxValue.Ticks / 2.0, JumpRows / 4.0 / Math.Max(density, 1e-18))));
|
||||
var resume = MsFloor(x) - OneMs;
|
||||
var jump = gapRows >= ReadRows / 2.0;
|
||||
var nextQs = jump ? Max(RequestStart(tn - Min(margin, tn - Floor)), resume) : resume;
|
||||
if (nextQs <= qs) throw TooDense(ctx, x);
|
||||
|
||||
if (nextQs < x)
|
||||
{
|
||||
// Every row before x was seen: the latest at or before nextQs is known.
|
||||
var lastIndex = rows.FindLastIndex(r => r.TimeUtc <= nextQs);
|
||||
if (lastIndex >= 0)
|
||||
{
|
||||
before = rows[lastIndex];
|
||||
beforeKnown = true;
|
||||
}
|
||||
walkKnown = null;
|
||||
}
|
||||
else
|
||||
{
|
||||
beforeKnown = false;
|
||||
before = null;
|
||||
walkKnown = x;
|
||||
walkWidth = margin;
|
||||
}
|
||||
budget = jump ? JumpRows : ReadRows;
|
||||
qs = nextQs;
|
||||
i = k;
|
||||
}
|
||||
|
||||
if (probeFrom >= 0 && qe < horizon)
|
||||
{
|
||||
var after = await FirstAfterAsync(ctx, qe - OneMs, horizon, Min(times[^1], horizon)).ConfigureAwait(false);
|
||||
if (after is not null)
|
||||
for (var k = probeFrom; k < times.Length; k++)
|
||||
{
|
||||
var t = times[k];
|
||||
var (prev, next) = result[t.Ticks];
|
||||
if (next is null && t < horizon && NeedsNext(t, prev) && after.Value.TimeUtc > t)
|
||||
result[t.Ticks] = (prev, after);
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/// <summary>The evaluator reads <c>next</c> only for a usable <c>prev</c> that is not an exact hit.</summary>
|
||||
private static bool NeedsNext(DateTime t, StoredSample? prev) => prev is { IsUsable: true } p && p.TimeUtc != t;
|
||||
|
||||
/// <summary>One walk-back segment <c>(lo, hi]</c>: its oldest and newest rows, and whether the read was complete.</summary>
|
||||
private static async Task<(StoredSample? First, StoredSample? Last, bool Complete)> SegmentAsync(
|
||||
Ctx ctx, DateTime lo, DateTime hi)
|
||||
{
|
||||
var read = await ReadTrustedAsync(ctx, MsFloor(lo), MsFloor(hi) + OneMs, ReadRows).ConfigureAwait(false);
|
||||
StoredSample? first = null, last = null;
|
||||
foreach (var r in read.Rows)
|
||||
if (r.TimeUtc > lo && r.TimeUtc <= hi)
|
||||
{
|
||||
first ??= r;
|
||||
last = r;
|
||||
}
|
||||
return (first, last, read.Complete);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The latest stored sample with time in <c>(Floor, hi]</c> (see the type remarks: gallop, then bisect).
|
||||
/// <paramref name="known"/>: a stored sample exists at that time (≤ <paramref name="hi"/>) — the walk then bisects
|
||||
/// above it straight away, its first segment aimed at <paramref name="width"/>.
|
||||
/// </summary>
|
||||
private static async Task<StoredSample?> WalkBackAsync(Ctx ctx, DateTime hi, DateTime? known, TimeSpan width)
|
||||
{
|
||||
TimeSpan? aim = known is null ? null : width;
|
||||
|
||||
// Gallop: no stored row known below hi yet.
|
||||
var empties = 0;
|
||||
while (known is null)
|
||||
{
|
||||
if (hi <= Floor) return null;
|
||||
var rest = empties >= 2;
|
||||
var lo = rest || (hi - Floor).Ticks <= width.Ticks ? Floor : hi - width;
|
||||
var seg = await SegmentAsync(ctx, lo, hi).ConfigureAwait(false);
|
||||
if (seg.Complete)
|
||||
{
|
||||
if (seg.Last is { } found) return found;
|
||||
if (lo <= Floor) return null;
|
||||
hi = lo;
|
||||
empties++;
|
||||
width = Grow(width);
|
||||
continue;
|
||||
}
|
||||
// Full — a whole-remainder read of deep history too: its last row is a stored row below hi. Bisect above it.
|
||||
if (seg.Last is not { } seen) throw TooDense(ctx, lo);
|
||||
known = seen.TimeUtc;
|
||||
aim = AimFrom(seg.First!.Value.TimeUtc, seen.TimeUtc);
|
||||
}
|
||||
|
||||
// Bisect: a stored row exists at `known`, nothing newer than `hi` matters; the answer is the newest row in [known, hi].
|
||||
var aimMissed = false;
|
||||
while (true)
|
||||
{
|
||||
var k = known.Value;
|
||||
DateTime lo;
|
||||
var aimed = false;
|
||||
if (hi - k < TimeSpan.FromMilliseconds(2))
|
||||
lo = k.AddTicks(-1); // the whole stretch from the known row: never a sub-millisecond step
|
||||
else
|
||||
{
|
||||
lo = k + TimeSpan.FromTicks((hi - k).Ticks / 2);
|
||||
if (aim is { } a && hi - a > lo)
|
||||
{
|
||||
lo = hi - a; // an aimed try right after a full read — until one comes back empty (a gap: halve from then on)
|
||||
aimed = true;
|
||||
}
|
||||
}
|
||||
aim = null;
|
||||
|
||||
var seg = await SegmentAsync(ctx, lo, hi).ConfigureAwait(false);
|
||||
if (seg.Complete)
|
||||
{
|
||||
if (seg.Last is { } found) return found;
|
||||
if (lo < k) return null; // the known row is gone (deleted meanwhile): nothing is left to find
|
||||
hi = lo;
|
||||
aimMissed |= aimed;
|
||||
continue;
|
||||
}
|
||||
if (seg.Last is not { } seen || seen.TimeUtc <= k) throw TooDense(ctx, k);
|
||||
known = seen.TimeUtc;
|
||||
if (!aimMissed) aim = AimFrom(seg.First!.Value.TimeUtc, seen.TimeUtc);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>About a quarter of a read at the density a full read showed (its rows from <paramref name="first"/> to <paramref name="last"/>).</summary>
|
||||
private static TimeSpan AimFrom(DateTime first, DateTime last) =>
|
||||
TimeSpan.FromTicks(Math.Max(Ms, (last - first).Ticks / 4));
|
||||
|
||||
/// <summary>The first stored sample after <paramref name="after"/> in <c>(qs, end)</c>, from a small forward read.</summary>
|
||||
private static async Task<StoredSample?> FirstAfterAsync(Ctx ctx, DateTime qs, DateTime end, DateTime after)
|
||||
{
|
||||
foreach (var budget in new[] { ProbeRows, ReadRows })
|
||||
{
|
||||
var read = await ReadTrustedAsync(ctx, qs, end, budget).ConfigureAwait(false);
|
||||
foreach (var r in read.Rows)
|
||||
if (r.TimeUtc > after) return r;
|
||||
if (read.Complete) return null;
|
||||
}
|
||||
throw TooDense(ctx, qs);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// One forward read <c>[qs, qe]</c> (whole milliseconds), drained, keeping only rows strictly inside it and after the
|
||||
/// epoch — counted against the request's read cap. A strict gateway's ceiling is learned on refusal and then asked for
|
||||
/// directly; a full read is not re-probed (the caller pages on, it does not need the rest of this read).
|
||||
/// </summary>
|
||||
private static async Task<Read> ReadTrustedAsync(Ctx ctx, DateTime qs, DateTime qe, int budget)
|
||||
{
|
||||
if (++ctx.Reads > ctx.MaxReads)
|
||||
throw new HistoryReadLimitExceededException(ctx.MaxReads,
|
||||
$"The AtTime read of '{ctx.Tag}' needed more than {ctx.MaxReads} raw reads ({BaseReads} + {ReadsPerTime} per " +
|
||||
"requested time)" + CeilingNote(ctx) + "; the Gateway backend serves AtTime from raw reads — request fewer times.");
|
||||
var (read, used) = await ctx.Ceiling.ReadAsync(budget, b => PassAsync(ctx, qs, qe, b), (_, _) => false)
|
||||
.ConfigureAwait(false);
|
||||
if (used < ctx.SmallestBudget) ctx.SmallestBudget = used;
|
||||
if (used < MinBudget)
|
||||
throw new HistoryReadLimitExceededException(used,
|
||||
$"The HistorianGateway's row ceiling ({used}, its Historian:ReadLimits:MaxValuesCeiling with " +
|
||||
$"AllowStreamingBeyondCeiling=false) is below the {MinBudget} rows an AtTime raw read of '{ctx.Tag}' needs.");
|
||||
return read;
|
||||
}
|
||||
|
||||
private static string CeilingNote(Ctx ctx) =>
|
||||
ctx.SmallestBudget < JumpRows
|
||||
? $", with reads bounded by the HistorianGateway's row ceiling of {ctx.SmallestBudget} (Historian:ReadLimits:MaxValuesCeiling)"
|
||||
: string.Empty;
|
||||
|
||||
private static async Task<Read> PassAsync(Ctx ctx, DateTime qs, DateTime qe, int budget)
|
||||
{
|
||||
var rows = new List<StoredSample>();
|
||||
var received = 0;
|
||||
await foreach (var raw in ctx.Client.ReadRawAsync(ctx.Tag, qs, qe, budget, ctx.Ct).ConfigureAwait(false))
|
||||
{
|
||||
received++; // drained to completion, never broken out of
|
||||
if (raw.Timestamp is null) continue;
|
||||
var t = raw.Timestamp.ToDateTime();
|
||||
if (t > qs && t < qe && t > Floor) rows.Add(ToStored(raw, t, ctx.ValueType));
|
||||
}
|
||||
return new Read(rows, received < budget);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A gateway row as a stored sample of the node's type (see the type remarks); a Bad row carries no value (#690: a
|
||||
/// stored NULL arrives as value 0).
|
||||
/// </summary>
|
||||
private static StoredSample ToStored(HistorianSample raw, DateTime t, TypeCode? valueType)
|
||||
{
|
||||
var status = SampleMapper.ToStatus(raw);
|
||||
var value = StatusBits.IsBad(status) ? HistorianValue.Empty
|
||||
: raw.HasNumericValue ? Typed(raw.NumericValue, valueType)
|
||||
: raw.HasStringValue ? HistorianValue.FromString(raw.StringValue)
|
||||
: HistorianValue.Empty;
|
||||
return StoredSample.FromStored(t, value, status);
|
||||
}
|
||||
|
||||
/// <summary>A gateway number as the hinted kind, so the shared evaluator's own type branch decides stepped vs linear.</summary>
|
||||
private static HistorianValue Typed(double v, TypeCode? valueType) => valueType switch
|
||||
{
|
||||
TypeCode.Boolean => HistorianValue.FromBoolean(v != 0),
|
||||
TypeCode.String => HistorianValue.FromString(v.ToString("R", System.Globalization.CultureInfo.InvariantCulture)),
|
||||
// In Int64 range only (2^63 is exactly representable); NaN, ±∞ or out of range is no integer at all — Empty, which
|
||||
// StoredSample.FromStored makes BadNoData (unusable), never a saturated Good value.
|
||||
_ when IsInteger(valueType) => v >= -9.2233720368547758E18 && v < 9.2233720368547758E18
|
||||
? HistorianValue.FromInt64((long)Math.Round(v, MidpointRounding.AwayFromZero))
|
||||
: HistorianValue.Empty,
|
||||
_ => HistorianValue.FromDouble(v),
|
||||
};
|
||||
|
||||
/// <summary>The window read's start for a time: a millisecond before its millisecond (never before the epoch).</summary>
|
||||
private static DateTime RequestStart(DateTime t)
|
||||
{
|
||||
var s = MsFloor(t) - TimeSpan.FromMilliseconds(1);
|
||||
return s < Floor ? Floor : s;
|
||||
}
|
||||
|
||||
private static DateTime MsFloor(DateTime t) => new(t.Ticks - t.Ticks % Ms, DateTimeKind.Utc);
|
||||
|
||||
private static DateTime Min(DateTime a, DateTime b) => a <= b ? a : b;
|
||||
|
||||
private static DateTime Max(DateTime a, DateTime b) => a >= b ? a : b;
|
||||
|
||||
private static TimeSpan Min(TimeSpan a, TimeSpan b) => a <= b ? a : b;
|
||||
|
||||
private static TimeSpan Clamp(TimeSpan w) =>
|
||||
w < TimeSpan.FromMilliseconds(1) ? TimeSpan.FromMilliseconds(1) : w > MaxSeededWidth ? MaxSeededWidth : w;
|
||||
|
||||
private static TimeSpan Grow(TimeSpan w) => w.Ticks > TimeSpan.MaxValue.Ticks / 4 ? TimeSpan.MaxValue : w * 4;
|
||||
|
||||
private static HistoryReadLimitExceededException TooDense(Ctx ctx, DateTime at)
|
||||
{
|
||||
var rows = Math.Min(ReadRows, ctx.SmallestBudget);
|
||||
return new(rows,
|
||||
$"The AtTime read of '{ctx.Tag}' met more than {rows} stored samples within about one millisecond of {at:O}" +
|
||||
(rows < ReadRows ? $" (reads are bounded by the HistorianGateway's row ceiling of {rows})" : string.Empty) +
|
||||
"; the Gateway backend serves AtTime from raw reads.");
|
||||
}
|
||||
}
|
||||
+487
-20
@@ -1,7 +1,10 @@
|
||||
using System.Collections.Frozen;
|
||||
using Grpc.Core;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using ZB.MOM.WW.HistorianGateway.Contracts.Grpc;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Mapping;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway;
|
||||
|
||||
@@ -23,8 +26,50 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway;
|
||||
/// </remarks>
|
||||
public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDisposable
|
||||
{
|
||||
/// <summary>
|
||||
/// The processed (aggregate) functions the Gateway backend serves — the single source of truth for this backend's
|
||||
/// aggregate set (#681): the per-sample aggregates evaluated from raw reads (every member of the
|
||||
/// <see cref="AggregateFamily.Simple"/> / <see cref="AggregateFamily.StartEnd"/> /
|
||||
/// <see cref="AggregateFamily.ActualTime"/> families of <see cref="AggregateFamilies"/>: Average, Count, Minimum,
|
||||
/// Maximum, Range, Start, End, Delta, MinimumActualTime, MaximumActualTime) plus the two answered natively by AVEVA
|
||||
/// (TimeAverage, Total — <see cref="AggregateModeMapper"/>). Any other aggregate throws
|
||||
/// <see cref="HistoryAggregateNotSupportedException"/> without calling the gateway. It is what
|
||||
/// <see cref="SupportedAggregates"/> returns, so the advertised <c>AggregateFunctions</c> folders and the server's
|
||||
/// up-front <c>BadAggregateNotSupported</c> read it too (#682).
|
||||
/// </summary>
|
||||
public static IReadOnlySet<HistoryAggregateType> ServedAggregates { get; } =
|
||||
Enum.GetValues<HistoryAggregateType>()
|
||||
.Where(a => RawAggregateReader.Serves(a) || AggregateModeMapper.NativeAggregates.Contains(a))
|
||||
.ToFrozenSet();
|
||||
|
||||
private readonly IHistorianGatewayClient _client;
|
||||
private readonly ILogger<GatewayHistorianDataSource> _logger;
|
||||
private readonly int _maxRawRows;
|
||||
private readonly TimeProvider _time;
|
||||
private readonly bool _atTimeFromRaw;
|
||||
|
||||
/// <summary>#693: AtTime from raw reads (true) or AVEVA's own interpolation (the kill switch).</summary>
|
||||
internal bool AtTimeFromRaw => _atTimeFromRaw;
|
||||
|
||||
/// <summary>The most #687 crowding re-reads one raw read makes (#694 F4): each at least doubles the pre-window room.</summary>
|
||||
private const int MaxCrowdedReReads = 8;
|
||||
|
||||
/// <summary>The default of <c>maxPreWindowRows</c>: <c>ServerHistorian:MaxTieClusterOverfetch</c>'s default.</summary>
|
||||
public const int DefaultMaxPreWindowRows = 65536;
|
||||
|
||||
/// <summary>
|
||||
/// The most rows before a raw window (inside the 1 ms look-back) a crowding re-read makes room for (#694 R2) — the
|
||||
/// server's tie-cluster limit (<c>ServerHistorian:MaxTieClusterOverfetch</c>), which every other tie path treats as
|
||||
/// the limit. More ⇒ <see cref="HistoryReadLimitExceededException"/> with <c>PreWindowRows</c> set.
|
||||
/// </summary>
|
||||
private readonly int _maxPreWindowRows;
|
||||
|
||||
/// <summary>The tie-cluster limit the crowding re-reads honour (#694 R2) — exposed so the factory wiring is testable.</summary>
|
||||
internal int MaxPreWindowRows => _maxPreWindowRows;
|
||||
|
||||
/// <summary>A strict gateway's row ceiling, learned from its first refusal (#691.3).</summary>
|
||||
private readonly GatewayRowCeiling _ceiling;
|
||||
private readonly GatewayTagPresenceCache _tags;
|
||||
|
||||
private readonly object _healthLock = new();
|
||||
private DateTime? _lastSuccessUtc;
|
||||
@@ -38,79 +83,440 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
/// <summary>Creates a gateway-backed historian data source.</summary>
|
||||
/// <param name="client">The gateway client seam used for all reads.</param>
|
||||
/// <param name="logger">Diagnostic logger; failures are recorded without leaking tag/host detail.</param>
|
||||
public GatewayHistorianDataSource(IHistorianGatewayClient client, ILogger<GatewayHistorianDataSource> logger)
|
||||
/// <param name="maxRawRows">
|
||||
/// In-window raw-row cap of one raw-evaluated processed read (<c>ServerHistorian:MaxTimeWeightedRawRows</c>;
|
||||
/// <= 0 = unbounded). Past it the node fails <c>BadTooManyOperations</c>.
|
||||
/// </param>
|
||||
/// <param name="timeProvider">Clock for result server timestamps and the raw look-ahead horizon; null = system.</param>
|
||||
/// <param name="atTimeFromRaw">
|
||||
/// #693: serve AtTime from raw reads with the shared AtTime rules (<see cref="GatewayAtTimeReader"/>; default);
|
||||
/// <c>false</c> = AVEVA's own interpolation passed through (<c>ServerHistorian:GatewayAtTimeFromRaw</c> kill switch).
|
||||
/// </param>
|
||||
/// <param name="maxPreWindowRows">
|
||||
/// The most rows before a raw window a crowding re-read makes room for (#694 R2; <c>ServerHistorian:MaxTieClusterOverfetch</c>,
|
||||
/// values below 1 count as 1). More ⇒ <see cref="HistoryReadLimitExceededException"/> (<c>BadTooManyOperations</c>).
|
||||
/// </param>
|
||||
public GatewayHistorianDataSource(IHistorianGatewayClient client, ILogger<GatewayHistorianDataSource> logger,
|
||||
int maxRawRows = TimeWeightedAggregator.DefaultMaxWindowRows, TimeProvider? timeProvider = null,
|
||||
bool atTimeFromRaw = true, int maxPreWindowRows = DefaultMaxPreWindowRows)
|
||||
{
|
||||
_maxPreWindowRows = Math.Max(1, maxPreWindowRows);
|
||||
_atTimeFromRaw = atTimeFromRaw;
|
||||
ArgumentNullException.ThrowIfNull(client);
|
||||
ArgumentNullException.ThrowIfNull(logger);
|
||||
_client = client;
|
||||
_logger = logger;
|
||||
_maxRawRows = maxRawRows;
|
||||
_time = timeProvider ?? TimeProvider.System;
|
||||
_ceiling = new GatewayRowCeiling(_time);
|
||||
_tags = new GatewayTagPresenceCache(_client, _time);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>
|
||||
/// A reverse window (<c>start > end</c>) is refused with <see cref="NotSupportedException"/> before anything is sent
|
||||
/// and is not recorded as a health failure (Gitea #686): the gateway rejects it <c>InvalidArgument</c>, and the server
|
||||
/// never asks — <see cref="SupportsReverseRaw"/> is <c>false</c>, so it walks a reverse read back with forward reads.
|
||||
/// Stored samples only (#687 — Part 11 Raw): AVEVA's synthetic start-boundary row is never returned, see
|
||||
/// <see cref="ReadRawWindowAsync"/>.
|
||||
/// </remarks>
|
||||
public async Task<HistoryReadResult> ReadRawAsync(
|
||||
string fullReference, DateTime startUtc, DateTime endUtc, uint maxValuesPerNode,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (startUtc > endUtc)
|
||||
throw new NotSupportedException("The HistorianGateway reads raw history oldest-first only (start > end).");
|
||||
|
||||
try
|
||||
{
|
||||
// The gateway seam caps with an int; OPC UA hands us a uint, so clamp to int range.
|
||||
var maxValues = (int)Math.Min(maxValuesPerNode, int.MaxValue);
|
||||
var samples = new List<HistorianSample>();
|
||||
await foreach (var sample in _client
|
||||
.ReadRawAsync(fullReference, startUtc, endUtc, maxValues, cancellationToken)
|
||||
.ConfigureAwait(false))
|
||||
if (!await _tags.ExistsAsync(fullReference, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
samples.Add(sample);
|
||||
RecordOutcome(success: true, error: null);
|
||||
return new HistoryReadResult([], ContinuationPoint: null);
|
||||
}
|
||||
|
||||
var samples = await ReadRawWindowAsync(fullReference, startUtc, endUtc, maxValuesPerNode, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
var snapshots = SampleMapper.ToSnapshots(samples);
|
||||
RecordOutcome(success: true, error: null);
|
||||
return new HistoryReadResult(snapshots, ContinuationPoint: null);
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
throw; // the caller (client cancel / HistoryReadDeadline) asked for this — not a backend fault
|
||||
}
|
||||
catch (Exception ex) when (IsMaxValuesCeilingRejection(ex))
|
||||
{
|
||||
// An unbounded read over Historian:ReadLimits:MaxValuesCeiling with AllowStreamingBeyondCeiling=false: a
|
||||
// client-request limit (BadTooManyOperations), not a backend fault.
|
||||
throw new HistoryReadLimitExceededException(maxValuesPerNode,
|
||||
$"The HistorianGateway refused the row budget of the raw read of '{fullReference}' (its " +
|
||||
"Historian:ReadLimits:MaxValuesCeiling); set NumValuesPerNode or narrow the window.");
|
||||
}
|
||||
catch (HistoryReadLimitExceededException)
|
||||
{
|
||||
// A client-request limit (#687: the pre-window margin swallowed the row budget), not a backend fault.
|
||||
throw;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
RecordReadFailure(ex);
|
||||
if (IsTransportFailure(ex)) throw new System.IO.IOException("HistorianGateway read transport is unavailable.", ex);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The stored samples of <c>[startUtc, endUtc)</c> (a point read <c>start == end</c>: of that instant), oldest first,
|
||||
/// at most <paramref name="maxValuesPerNode"/> (<c>0</c> = all) — without AVEVA's synthetic rows (#687).
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// An AVEVA <c>Full</c> raw read starts with a row stamped exactly at the query start: the value in effect there
|
||||
/// (<c>Quality = 133</c>, the prior sample's value — never stored at that time) or, before the tag's first sample,
|
||||
/// a NULL row (value 0, Bad). Part 11 Raw with <c>ReturnBounds = false</c> wants stored values only, and the
|
||||
/// server's paging (#669) counts every returned row toward the page and resumes at the last returned timestamp,
|
||||
/// so a fabricated row would also be counted as a sample and re-fabricated at every resume point. The read
|
||||
/// therefore uses the same request start as the raw-evaluated aggregates (#681): it is requested from
|
||||
/// <see cref="RawAggregateReader.RequestStartFor"/> (1 ms early) and only rows
|
||||
/// <see cref="RawAggregateReader.IsWindowSample"/> are kept — the decision is by TIMESTAMP, because a stored NULL
|
||||
/// sample and the no-data boundary row look the same on the wire (both carry a value 0 with a Bad quality). The
|
||||
/// forward end is exclusive (the interface contract; the gateway's own end is inclusive), a point read inclusive.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Row budget.</b> <c>0</c> ("all values") asks for <see cref="int.MaxValue"/> (the client package and the
|
||||
/// gateway reject <c>max_values <= 0</c>, #686). A cap asks for <c>cap + </c>
|
||||
/// <see cref="RawAggregateReader.PreWindowAllowance"/> rows — NOT clamped at the gateway's default ceiling (#688
|
||||
/// clamps only the per-sample aggregate budget): AVEVA always emits the boundary row, so a clamped budget would make
|
||||
/// every page size within the allowance of the ceiling come back short. A strict gateway
|
||||
/// (<c>AllowStreamingBeyondCeiling=false</c>) refuses an over-ceiling budget and names its ceiling; the read is
|
||||
/// then re-issued ONCE with the ceiling as budget (#691.3, <see cref="GatewayReadLimits"/>) — so an uncapped read
|
||||
/// of a small window is served — and a ceiling-bounded read that is not provably complete (it delivered the
|
||||
/// whole ceiling without reaching the window end, or a page short of its cap) fails
|
||||
/// <see cref="HistoryReadLimitExceededException"/> (<c>BadTooManyOperations</c>), never a truncated answer.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Completeness.</b> A full page must hold exactly the cap of stored samples — a short page ends the client's
|
||||
/// paging. A read is complete when the stream came back short, filled the cap, or reached the window end. If rows
|
||||
/// BEFORE the window (the boundary row and stored samples inside the 1 ms margin, e.g. a tie cluster at
|
||||
/// <c>start − 1 ms</c>) used up the budget, the window is read ONCE more asking for <c>cap + m + </c>
|
||||
/// <see cref="RawAggregateReader.PreWindowAllowance"/> (m = the pre-window rows the first read saw); only if that
|
||||
/// re-read is still short because of pre-window rows does the read fail with
|
||||
/// <see cref="HistoryReadLimitExceededException"/>. Every stream is consumed to completion (never broken out of —
|
||||
/// a cancelled gateway stream evicts its pooled historian session). <c>ReturnBounds</c> is not plumbed to the
|
||||
/// data source on any backend, so bounding values are never synthesised here.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
private async Task<List<HistorianSample>> ReadRawWindowAsync(
|
||||
string fullReference, DateTime startUtc, DateTime endUtc, uint maxValuesPerNode,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
// The gateway seam caps with an int; OPC UA hands us a uint, so clamp to int range.
|
||||
var cap = (int)Math.Min(maxValuesPerNode, int.MaxValue);
|
||||
if (cap == 0)
|
||||
{
|
||||
var all = await PassAsync(int.MaxValue).ConfigureAwait(false);
|
||||
if (all.Result.Complete) return all.Result.Samples;
|
||||
// Only reachable on a strict gateway (#691.3): the ceiling re-read delivered the whole ceiling without reaching
|
||||
// the window end — the window may hold more, and "all values" is never answered from a truncated read.
|
||||
throw new HistoryReadLimitExceededException(cap,
|
||||
$"The raw read of '{fullReference}' holds more than the HistorianGateway's row ceiling ({all.Budget}, its " +
|
||||
"Historian:ReadLimits:MaxValuesCeiling with AllowStreamingBeyondCeiling=false); set NumValuesPerNode to page, " +
|
||||
"or narrow the window.")
|
||||
{ PreWindowRows = Crowding(all.Result.PreWindowRows) };
|
||||
}
|
||||
|
||||
var asked = (int)Math.Min(int.MaxValue, (long)cap + RawAggregateReader.PreWindowAllowance);
|
||||
var pass = await PassAsync(asked).ConfigureAwait(false);
|
||||
if (pass.Result.Complete) return pass.Result.Samples;
|
||||
|
||||
// A strict gateway's ceiling already bounded that read (#691.3): a larger budget would only be refused again. The
|
||||
// refusal carries the pre-window rows it saw (#694): the server re-reads the page that many values smaller.
|
||||
if (pass.Budget < asked)
|
||||
throw new HistoryReadLimitExceededException(cap,
|
||||
$"The raw read of '{fullReference}' could not be read completely within the HistorianGateway's row ceiling " +
|
||||
$"({pass.Budget}, its Historian:ReadLimits:MaxValuesCeiling with AllowStreamingBeyondCeiling=false); lower " +
|
||||
"NumValuesPerNode or narrow the window.")
|
||||
{ PreWindowRows = Crowding(pass.Result.PreWindowRows) };
|
||||
|
||||
// The margin crowded the window out: re-read with room for the pre-window rows. A read that was itself cut short saw
|
||||
// only some of them (#694 F4: a tie cluster of hundreds just before a resume point used to fail even with no
|
||||
// ceiling), so the room at least doubles per re-read — bounded by MaxCrowdedReReads, and never past a known ceiling.
|
||||
// #694 R2: the room is capped at the tie-cluster limit (MaxTieClusterOverfetch) — a pathological single-millisecond
|
||||
// cluster on a streaming gateway would otherwise be streamed several times over for one page.
|
||||
var preWindow = pass.Result.PreWindowRows;
|
||||
var room = Math.Min(Math.Max(1, preWindow), _maxPreWindowRows);
|
||||
for (var attempt = 0; attempt < MaxCrowdedReReads && preWindow <= _maxPreWindowRows; attempt++)
|
||||
{
|
||||
var budget = (int)Math.Min(int.MaxValue, (long)cap + room + RawAggregateReader.PreWindowAllowance);
|
||||
// #695: doubling can leap over the ceiling while the cluster still fits. Spend the remaining room once.
|
||||
if (_ceiling.Learned is var known and > 0 && budget > known)
|
||||
{
|
||||
var available = known - cap - RawAggregateReader.PreWindowAllowance;
|
||||
if (available <= 0) break;
|
||||
room = Math.Min(room, available);
|
||||
budget = cap + room + RawAggregateReader.PreWindowAllowance;
|
||||
}
|
||||
if (budget <= pass.Budget) break; // no more room, never retry the same full read
|
||||
pass = await PassAsync(budget).ConfigureAwait(false);
|
||||
if (pass.Result.Complete) return pass.Result.Samples;
|
||||
preWindow = Math.Max(preWindow, pass.Result.PreWindowRows);
|
||||
if (pass.Budget < budget) break; // bounded by a ceiling learned meanwhile
|
||||
if (room >= _maxPreWindowRows) break; // the limit's worth of room did not suffice
|
||||
room = (int)Math.Min(_maxPreWindowRows, Math.Max(preWindow, 2L * room));
|
||||
}
|
||||
|
||||
throw new HistoryReadLimitExceededException(cap,
|
||||
$"The raw read of '{fullReference}' could not be read completely: more than {pass.Result.PreWindowRows} rows lie in " +
|
||||
$"the {RawAggregateReader.BoundaryMargin.TotalMilliseconds} ms before the window and exhausted the row budget.")
|
||||
{ PreWindowRows = preWindow };
|
||||
|
||||
// Pre-window rows the allowance did not cover are what crowded the read; fewer are not the cause (#694).
|
||||
static int Crowding(int preWindowRows) => preWindowRows > RawAggregateReader.PreWindowAllowance ? preWindowRows : 0;
|
||||
|
||||
// One drained pass; a strict gateway's refusal of the budget is retried once at the ceiling it names (#691.3).
|
||||
Task<((List<HistorianSample> Samples, int PreWindowRows, bool Complete) Result, int Budget)> PassAsync(int budget) =>
|
||||
_ceiling.ReadAsync(budget,
|
||||
b => ReadRawPassAsync(fullReference, startUtc, endUtc, cap, b, cancellationToken),
|
||||
(r, _) => !r.Complete);
|
||||
}
|
||||
|
||||
/// <summary>One forward raw read of a window, drained, with its pre-window row count and whether it is complete.</summary>
|
||||
private async Task<(List<HistorianSample> Samples, int PreWindowRows, bool Complete)> ReadRawPassAsync(
|
||||
string fullReference, DateTime startUtc, DateTime endUtc, int cap, int budget, CancellationToken cancellationToken)
|
||||
{
|
||||
var samples = new List<HistorianSample>();
|
||||
long received = 0;
|
||||
var preWindow = 0;
|
||||
var reachedEnd = false;
|
||||
await foreach (var sample in _client
|
||||
.ReadRawAsync(fullReference, RawAggregateReader.RequestStartFor(startUtc), endUtc, budget, cancellationToken)
|
||||
.ConfigureAwait(false))
|
||||
{
|
||||
received++;
|
||||
// Filter while draining — the rest of the (bounded) stream is consumed, never abandoned.
|
||||
if (RawAggregateReader.IsWindowSample(sample, startUtc, endUtc))
|
||||
{
|
||||
if (cap == 0 || samples.Count < cap) samples.Add(sample);
|
||||
}
|
||||
// A row before the window — or, for a window starting at/before the epoch (no look-back), AVEVA's synthetic row at
|
||||
// the query start, which is no sample either (#694 F1: it used to set reachedEnd, so a ceiling-truncated read
|
||||
// passed as complete) — is a pre-window row.
|
||||
else if (sample.Timestamp is { } ts
|
||||
&& (ts.ToDateTime() < startUtc || !RawAggregateReader.IsAfterFloor(ts.ToDateTime()))) preWindow++;
|
||||
else if (sample.Timestamp is not null) reachedEnd = true;
|
||||
}
|
||||
|
||||
// cap == 0 ("all values") is complete only when the stream proves it: short of the budget (always, unless a strict
|
||||
// gateway's ceiling became the budget — #691.3) or past the window end.
|
||||
var complete = received < budget || reachedEnd || (cap > 0 && samples.Count >= cap);
|
||||
return (samples, preWindow, complete);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>Per-sample aggregates (#681)</b> — Average, Count, Minimum, Maximum, Range, Start, End, Delta,
|
||||
/// Minimum/MaximumActualTime — are evaluated from a raw read with the SQL backends' shared evaluator
|
||||
/// (<see cref="RawAggregateReader"/>): identical values, status, Calculated/Partial bits and timestamps to
|
||||
/// SqlServer/QuestDB for the same samples, forward and reverse (#677). The in-window raw rows are capped by
|
||||
/// <c>ServerHistorian:MaxTimeWeightedRawRows</c> (<see cref="HistoryReadLimitExceededException"/>, not counted as
|
||||
/// a backend failure).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>TimeAverage / Total</b> stay native AVEVA retrieval modes (<see cref="AggregateModeMapper"/>), stamped with
|
||||
/// the interval start and the Calculated bit; AVEVA's extra row for the interval before the window is dropped and
|
||||
/// an Uncertain interval keeps its value (#685). Both directions use the SQL backends' interval grid with forward
|
||||
/// requests only — a reverse read (which the gateway refuses) anchored at its later time (#677), the clipped interval
|
||||
/// read over exactly its span and marked Partial (#691.2, <see cref="NativeAggregatePlan"/>). Everything else is <see cref="HistoryAggregateNotSupportedException"/>
|
||||
/// (see <see cref="ServedAggregates"/>).
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
/// <exception cref="ArgumentOutOfRangeException">A raw-evaluated read whose window holds more than
|
||||
/// <see cref="BucketPlan.MaxBuckets"/> intervals.</exception>
|
||||
public async Task<HistoryReadResult> ReadProcessedAsync(
|
||||
string fullReference, DateTime startUtc, DateTime endUtc, TimeSpan interval,
|
||||
HistoryAggregateType aggregate, CancellationToken cancellationToken)
|
||||
{
|
||||
if (RawAggregateReader.Serves(aggregate))
|
||||
return await ReadProcessedFromRawAsync(fullReference, startUtc, endUtc, interval, aggregate, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
// Mapped OUTSIDE the failure-recording try: an aggregate the gateway has no mode for is a client
|
||||
// request the server answers BadAggregateNotSupported, not a backend fault to count (#285).
|
||||
var mode = AggregateModeMapper.ToRetrievalMode(aggregate);
|
||||
// Planned OUTSIDE the failure-recording try: too many intervals is a client request limit, not a backend fault.
|
||||
return await ReadNativeAsync(fullReference, BucketPlan.Create(startUtc, endUtc, interval), mode, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A native TimeAverage / Total read on the SQL backends' interval grid, either direction (#691.2): forward native
|
||||
/// requests only, the clipped interval read over exactly its span. See <see cref="NativeAggregatePlan"/>.
|
||||
/// </summary>
|
||||
private async Task<HistoryReadResult> ReadNativeAsync(string fullReference, BucketPlan plan, RetrievalMode mode,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (plan.BucketCount == 0) return new HistoryReadResult([], ContinuationPoint: null);
|
||||
try
|
||||
{
|
||||
// Total/Count are now native gateway retrieval modes — no client-side scaling
|
||||
// (unlike the Wonderware path that derived Total as Average × interval-seconds).
|
||||
var buckets = new List<HistorianAggregateSample>();
|
||||
await foreach (var bucket in _client
|
||||
.ReadAggregateAsync(fullReference, startUtc, endUtc, mode, interval, cancellationToken)
|
||||
.ConfigureAwait(false))
|
||||
if (!await _tags.ExistsAsync(fullReference, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
buckets.Add(bucket);
|
||||
RecordOutcome(success: true, error: null);
|
||||
return new HistoryReadResult(plan.Assemble(new Dictionary<long, DataValueSnapshot>(), t => AggregateEvaluator.NoData(t, _time.GetUtcNow().UtcDateTime)), ContinuationPoint: null);
|
||||
}
|
||||
var byBucket = new Dictionary<long, DataValueSnapshot>();
|
||||
foreach (var request in NativeAggregatePlan.Requests(plan))
|
||||
{
|
||||
var buckets = new List<HistorianAggregateSample>();
|
||||
await foreach (var bucket in _client
|
||||
.ReadAggregateAsync(fullReference, request.StartUtc, request.EndUtc, mode, request.Interval, cancellationToken)
|
||||
.ConfigureAwait(false))
|
||||
{
|
||||
buckets.Add(bucket); // drained to completion, mapped afterwards
|
||||
}
|
||||
NativeAggregatePlan.Place(plan, request, buckets, byBucket);
|
||||
}
|
||||
|
||||
var snapshots = SampleMapper.ToAggregateSnapshots(buckets);
|
||||
RecordOutcome(success: true, error: null);
|
||||
return new HistoryReadResult(snapshots, ContinuationPoint: null);
|
||||
return new HistoryReadResult(NativeAggregatePlan.Assemble(plan, byBucket), ContinuationPoint: null);
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
throw; // caller-requested, not a backend fault
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
RecordReadFailure(ex);
|
||||
if (IsTransportFailure(ex)) throw new System.IO.IOException("HistorianGateway read transport is unavailable.", ex);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Serves a per-sample aggregate from a raw read (#681); see <see cref="RawAggregateReader"/>.</summary>
|
||||
private async Task<HistoryReadResult> ReadProcessedFromRawAsync(
|
||||
string fullReference, DateTime startUtc, DateTime endUtc, TimeSpan interval,
|
||||
HistoryAggregateType aggregate, CancellationToken cancellationToken)
|
||||
{
|
||||
// Planned OUTSIDE the failure-recording try (as on the SQL backends): too many intervals is a client request
|
||||
// limit, not a backend fault.
|
||||
var plan = BucketPlan.Create(startUtc, endUtc, interval);
|
||||
try
|
||||
{
|
||||
var now = _time.GetUtcNow().UtcDateTime;
|
||||
if (plan.BucketCount > 0 && !await _tags.ExistsAsync(fullReference, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
RecordOutcome(success: true, error: null);
|
||||
return new HistoryReadResult(plan.Assemble(new Dictionary<long, DataValueSnapshot>(), t => AggregateEvaluator.NoData(t, now)), ContinuationPoint: null);
|
||||
}
|
||||
var values = await RawAggregateReader
|
||||
.ReadAsync(_client, _ceiling, fullReference, aggregate, plan, _maxRawRows, now, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
RecordOutcome(success: true, error: null);
|
||||
return new HistoryReadResult(values, ContinuationPoint: null);
|
||||
}
|
||||
catch (HistoryReadLimitExceededException)
|
||||
{
|
||||
// A client-request limit (the raw-row cap), not a backend fault: not counted as a failure.
|
||||
throw;
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
// The caller (client cancel / HistoryReadDeadline) asked for this — not a backend fault.
|
||||
throw;
|
||||
}
|
||||
catch (Exception ex) when (IsMaxValuesCeilingRejection(ex))
|
||||
{
|
||||
// The gateway refused the row budget (Historian:ReadLimits:MaxValuesCeiling with
|
||||
// AllowStreamingBeyondCeiling=false): the same client-request limit as the cap, not a backend fault.
|
||||
throw new HistoryReadLimitExceededException(_maxRawRows,
|
||||
$"The HistorianGateway refused the raw row budget of the processed read of '{fullReference}' (its " +
|
||||
"Historian:ReadLimits:MaxValuesCeiling); lower ServerHistorian:MaxTimeWeightedRawRows below that ceiling " +
|
||||
"or narrow the window.");
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
RecordReadFailure(ex);
|
||||
if (IsTransportFailure(ex)) throw new System.IO.IOException("HistorianGateway read transport is unavailable.", ex);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// True when <paramref name="ex"/> (or an inner exception — the client package wraps the <see cref="RpcException"/>)
|
||||
/// is the gateway's <c>InvalidArgument</c> "max_values exceeds the server ceiling" rejection.
|
||||
/// </summary>
|
||||
internal static bool IsMaxValuesCeilingRejection(Exception ex) => GatewayReadLimits.IsCeilingRejection(ex);
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>
|
||||
/// #693: served from RAW gateway reads — each requested time's stored sample at-or-before and after it — evaluated with
|
||||
/// the SQL backends' shared <see cref="AtTimeEvaluator"/> (Part 11 §6.5.4), so the Gateway answers what SqlServer and
|
||||
/// QuestDB answer for the same stored samples; see <see cref="GatewayAtTimeReader"/> for the read plan and its cost.
|
||||
/// AVEVA's own interpolated answer (the gateway's <c>ReadAtTime</c>) returned the previous value as Good at a stored
|
||||
/// NULL's time, null + Bad at a Good sample right after a NULL, a drifted value at an exact stored sample, and no
|
||||
/// Interpolated bit; it is only used when <c>ServerHistorian:GatewayAtTimeFromRaw = false</c> (kill switch).
|
||||
/// A gateway value is a Double (or a String) whatever the node's type (AVEVA stores a Boolean as a 0/1 <c>Int2</c>), so
|
||||
/// without a type hint every numeric tag is interpolated linearly; the node manager passes the node's type through
|
||||
/// <see cref="ReadAtTimeAsync(string, IReadOnlyList{DateTime}, TypeCode?, CancellationToken)"/>.
|
||||
/// </remarks>
|
||||
public Task<HistoryReadResult> ReadAtTimeAsync(
|
||||
string fullReference, IReadOnlyList<DateTime> timestampsUtc, CancellationToken cancellationToken) =>
|
||||
ReadAtTimeAsync(fullReference, timestampsUtc, valueType: null, cancellationToken);
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>
|
||||
/// #693: the hint makes the AtTime rules type-exact, as on the SQL backends — the gateway's Double rows are converted to
|
||||
/// the node's type BEFORE the shared evaluator runs (Boolean = value ≠ 0 ⇒ stepped; an integer type ⇒ Int64 ⇒
|
||||
/// interpolated and rounded half away from zero, then returned as the node's integer type; String ⇒ stepped; Float /
|
||||
/// Double / no hint ⇒ linear Double). Ignored with the kill switch (AVEVA's own answer).
|
||||
/// </remarks>
|
||||
public async Task<HistoryReadResult> ReadAtTimeAsync(
|
||||
string fullReference, IReadOnlyList<DateTime> timestampsUtc, TypeCode? valueType, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(timestampsUtc);
|
||||
if (!_atTimeFromRaw) return await ReadAtTimeNativeAsync(fullReference, timestampsUtc, cancellationToken).ConfigureAwait(false);
|
||||
try
|
||||
{
|
||||
var now = _time.GetUtcNow().UtcDateTime;
|
||||
if (timestampsUtc.Count > 0 && !await _tags.ExistsAsync(fullReference, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
RecordOutcome(success: true, error: null);
|
||||
return MissingAtTime(timestampsUtc, now);
|
||||
}
|
||||
var values = await GatewayAtTimeReader
|
||||
.ReadAsync(_client, _ceiling, fullReference, timestampsUtc, now, cancellationToken, valueType)
|
||||
.ConfigureAwait(false);
|
||||
RecordOutcome(success: true, error: null);
|
||||
return new HistoryReadResult(values, ContinuationPoint: null);
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
throw; // caller-requested, not a backend fault
|
||||
}
|
||||
catch (HistoryReadLimitExceededException)
|
||||
{
|
||||
throw; // a client-request limit (a pathological tie cluster), not a backend fault
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
RecordReadFailure(ex);
|
||||
if (IsTransportFailure(ex)) throw new System.IO.IOException("HistorianGateway read transport is unavailable.", ex);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>The pre-#693 AtTime: AVEVA's own interpolation, aligned to the request (kill switch only).</summary>
|
||||
private async Task<HistoryReadResult> ReadAtTimeNativeAsync(
|
||||
string fullReference, IReadOnlyList<DateTime> timestampsUtc, CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (timestampsUtc.Count > 0 && !await _tags.ExistsAsync(fullReference, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
RecordOutcome(success: true, error: null);
|
||||
return MissingAtTime(timestampsUtc, _time.GetUtcNow().UtcDateTime);
|
||||
}
|
||||
|
||||
var samples = await _client
|
||||
.ReadAtTimeAsync(fullReference, timestampsUtc, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
@@ -121,10 +527,56 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
catch (Exception ex)
|
||||
{
|
||||
RecordReadFailure(ex);
|
||||
if (IsTransportFailure(ex)) throw new System.IO.IOException("HistorianGateway read transport is unavailable.", ex);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>
|
||||
/// <c>false</c> (Gitea #686): the gateway's <c>ReadRaw</c> streams ascending only and rejects <c>end < start</c>
|
||||
/// (<c>InvalidArgument</c>). The server serves reverse and end-only capped raw reads from bounded forward reads
|
||||
/// instead, and never sends this source a reverse window (<see cref="ReadRawAsync"/> refuses one unsent).
|
||||
/// </remarks>
|
||||
public bool SupportsReverseRaw => false;
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>
|
||||
/// Gitea #694: <c>0</c> until a strict gateway (<c>AllowStreamingBeyondCeiling=false</c>) has refused a budget and
|
||||
/// named its ceiling (#691.3, <see cref="GatewayRowCeiling"/>); then that ceiling less the
|
||||
/// <see cref="RawAggregateReader.PreWindowAllowance"/> a capped read adds for AVEVA's boundary row — so a page of this
|
||||
/// size asks for exactly the ceiling. Back to <c>0</c> when a re-probe finds the gateway streaming again.
|
||||
/// </remarks>
|
||||
public int MaxRawValuesPerRead => _ceiling.Learned is var c and > 0
|
||||
? Math.Max(1, c - RawAggregateReader.PreWindowAllowance)
|
||||
: 0;
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>
|
||||
/// <c>false</c>: the gateway's <c>ReadEvents</c> streams ascending only (Gitea #675). The server never sends it a
|
||||
/// reverse read; since #691.4 it walks one back from bounded forward reads (ties sorted by EventId — the gateway
|
||||
/// orders by event time only).
|
||||
/// </remarks>
|
||||
public bool SupportsReverseEvents => false;
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks><see cref="ServedAggregates"/>; every other member answers <c>BadAggregateNotSupported</c> and is not advertised.</remarks>
|
||||
public IReadOnlySet<HistoryAggregateType> SupportedAggregates => ServedAggregates;
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>
|
||||
/// A reverse (newest-first) read is refused with <see cref="NotSupportedException"/> (Gitea #675) — the gateway's
|
||||
/// <c>ReadEvents</c> streams ascending only and rejects <c>end < start</c>. The server never calls this with
|
||||
/// <c>reverse</c>: it walks a reverse read back from bounded forward reads (#691.4). Nothing is sent.
|
||||
/// </remarks>
|
||||
public Task<HistoricalEventsResult> ReadEventsAsync(
|
||||
string? sourceName, DateTime startUtc, DateTime endUtc, int maxEvents, bool reverse,
|
||||
CancellationToken cancellationToken) =>
|
||||
reverse
|
||||
? throw new NotSupportedException(
|
||||
"The HistorianGateway backend reads events oldest-first only; a reverse (newest-first) events read is not supported.")
|
||||
: ReadEventsAsync(sourceName, startUtc, endUtc, maxEvents, cancellationToken);
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<HistoricalEventsResult> ReadEventsAsync(
|
||||
string? sourceName, DateTime startUtc, DateTime endUtc, int maxEvents,
|
||||
@@ -133,7 +585,8 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
try
|
||||
{
|
||||
var hasCap = maxEvents > 0;
|
||||
var collected = new List<HistoricalEvent>(hasCap ? maxEvents : 0);
|
||||
// Bounded initial capacity: an uncapped read arrives as int.MaxValue ("Array dimensions exceeded", #691.4 review).
|
||||
var collected = new List<HistoricalEvent>(hasCap ? Math.Min(maxEvents, 1024) : 0);
|
||||
var truncated = false;
|
||||
|
||||
await foreach (var wireEvent in _client
|
||||
@@ -170,6 +623,7 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
catch (Exception ex)
|
||||
{
|
||||
RecordReadFailure(ex);
|
||||
if (IsTransportFailure(ex)) throw new System.IO.IOException("HistorianGateway read transport is unavailable.", ex);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
@@ -201,7 +655,7 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
|
||||
/// <summary>
|
||||
/// Reconciles a gateway at-time reply against the requested timestamps to honour the
|
||||
/// <see cref="IHistorianDataSource.ReadAtTimeAsync"/> contract: exactly one snapshot per
|
||||
/// <see cref="IHistorianDataSource.ReadAtTimeAsync(string, IReadOnlyList{DateTime}, CancellationToken)"/> contract: exactly one snapshot per
|
||||
/// requested timestamp, in request order. Returned samples are indexed by timestamp ticks;
|
||||
/// any requested timestamp the gateway did not return is filled with a Bad-quality
|
||||
/// (<c>0x80000000</c>) snapshot stamped at the requested time rather than positionally
|
||||
@@ -244,6 +698,19 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
return result;
|
||||
}
|
||||
|
||||
/// <summary>No-data values for a missing tag, stamped with the client's requested times.</summary>
|
||||
private static HistoryReadResult MissingAtTime(IReadOnlyList<DateTime> times, DateTime now) =>
|
||||
new(times.Select(t => AtTimeEvaluator.Evaluate(t, null, null, now, now)).ToArray(), ContinuationPoint: null);
|
||||
|
||||
/// <summary>Classifies actual gRPC/package transport failures without treating refusals as outages (#697).</summary>
|
||||
private static bool IsTransportFailure(Exception ex) => ex switch
|
||||
{
|
||||
ZB.MOM.WW.HistorianGateway.Client.HistorianGatewayUnavailableException => true,
|
||||
RpcException rpc => rpc.StatusCode is Grpc.Core.StatusCode.Unavailable or Grpc.Core.StatusCode.DeadlineExceeded,
|
||||
TimeoutException => true,
|
||||
_ => ex.InnerException is { } inner && IsTransportFailure(inner),
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// Records a failed read: bumps the health counters and logs a generic, redaction-safe
|
||||
/// debug line (no tag, host, or value). The exception itself is rethrown by the caller.
|
||||
|
||||
+7
-1
@@ -38,7 +38,13 @@ public static class GatewayHistorian
|
||||
|
||||
return new GatewayHistorianDataSource(
|
||||
HistorianGatewayClientAdapter.Create(options, loggerFactory),
|
||||
logger);
|
||||
logger,
|
||||
// #681: the raw-evaluated per-sample aggregates share the SQL backends' raw-row cap.
|
||||
maxRawRows: options.MaxTimeWeightedRawRows,
|
||||
// #693: AtTime from raw reads + the shared evaluator; false = AVEVA's own interpolation (kill switch).
|
||||
atTimeFromRaw: options.GatewayAtTimeFromRaw,
|
||||
// #694 R2: a crowding re-read never makes room for more pre-window rows than the tie-cluster limit.
|
||||
maxPreWindowRows: options.MaxTieClusterOverfetch);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
||||
@@ -0,0 +1,148 @@
|
||||
using System.Globalization;
|
||||
using System.Text.RegularExpressions;
|
||||
using Grpc.Core;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway;
|
||||
|
||||
/// <summary>
|
||||
/// The HistorianGateway's raw-read row ceiling (<c>Historian:ReadLimits:MaxValuesCeiling</c>, default 1,000,000) as seen
|
||||
/// from this side of the wire. With <c>AllowStreamingBeyondCeiling=false</c> (a "strict" gateway) the gateway refuses
|
||||
/// any <c>ReadRaw</c> whose <c>max_values</c> exceeds the ceiling with <c>InvalidArgument</c> — BEFORE it streams a row
|
||||
/// — and names the ceiling in the refusal (<c>"max_values exceeds the server ceiling of {n}; …"</c>). The gateway has
|
||||
/// no RPC that reports the ceiling, so the refusal is how it is learned (#691.3).
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// A read whose budget the gateway refuses is re-issued ONCE with the ceiling as its budget, and the ceiling is
|
||||
/// remembered per data source (<see cref="GatewayRowCeiling"/>); the caller treats a stream that delivered the whole
|
||||
/// ceiling as possibly truncated (never answered from).
|
||||
/// </remarks>
|
||||
internal static class GatewayReadLimits
|
||||
{
|
||||
private const string CeilingRejectionText = "max_values exceeds the server ceiling";
|
||||
|
||||
private static readonly Regex CeilingNumber = new(@"max_values exceeds the server ceiling of (\d+)",
|
||||
RegexOptions.CultureInvariant | RegexOptions.Compiled);
|
||||
|
||||
/// <summary>
|
||||
/// True when <paramref name="ex"/> (or an inner exception — the client package wraps the <see cref="RpcException"/>)
|
||||
/// is the gateway's <c>InvalidArgument</c> "max_values exceeds the server ceiling" rejection.
|
||||
/// </summary>
|
||||
/// <param name="ex">The exception.</param>
|
||||
/// <returns>Whether it is the ceiling rejection.</returns>
|
||||
public static bool IsCeilingRejection(Exception ex) => Rejection(ex) is not null;
|
||||
|
||||
/// <summary>
|
||||
/// The ceiling a ceiling rejection names, or — when <paramref name="ex"/> is a ceiling rejection that does not
|
||||
/// carry a parseable number — the gateway's default (<see cref="RawAggregateReader.GatewayDefaultMaxValuesCeiling"/>).
|
||||
/// </summary>
|
||||
/// <param name="ex">The exception.</param>
|
||||
/// <param name="ceiling">The ceiling (> 0) when this returns true.</param>
|
||||
/// <returns>Whether <paramref name="ex"/> is a ceiling rejection.</returns>
|
||||
public static bool TryGetCeiling(Exception ex, out int ceiling)
|
||||
{
|
||||
ceiling = 0;
|
||||
if (Rejection(ex) is not { } rpc) return false;
|
||||
var m = CeilingNumber.Match(rpc.Status.Detail);
|
||||
ceiling = m.Success && int.TryParse(m.Groups[1].Value, NumberStyles.None, CultureInfo.InvariantCulture, out var n) && n > 0
|
||||
? n
|
||||
: RawAggregateReader.GatewayDefaultMaxValuesCeiling;
|
||||
return true;
|
||||
}
|
||||
|
||||
private static RpcException? Rejection(Exception ex)
|
||||
{
|
||||
for (var e = ex; e is not null; e = e.InnerException)
|
||||
if (e is RpcException { StatusCode: StatusCode.InvalidArgument } rpc
|
||||
&& rpc.Status.Detail.Contains(CeilingRejectionText, StringComparison.Ordinal))
|
||||
return rpc;
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A strict gateway's row ceiling as learned by ONE data source (#691.3 review): every refused attempt counts as an
|
||||
/// error in the gateway's own outcome metrics, so the ceiling a refusal names is remembered and later reads ask for at
|
||||
/// most it — no refusal per read in the steady state. Thread-safe (one <see cref="int"/>, read/written atomically; a
|
||||
/// race costs at most one extra refusal or re-probe, never a wrong answer).
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>Rules.</b> (1) Unknown ceiling: ask for the full budget; a refusal naming <c>N</c> < budget learns <c>N</c> and
|
||||
/// the read is re-issued once at <c>N</c>. (2) Learned ceiling <c>C</c> below the budget: ask for <c>C</c> — a refusal
|
||||
/// there (the gateway was LOWERED) re-learns its <c>N</c> as in (1). (3) If that <c>C</c>-bounded read came back
|
||||
/// full but not provably complete, the gateway may have been RAISED (or switched to streaming) since <c>C</c> was
|
||||
/// learned, and answering <c>BadTooManyOperations</c> from a stale ceiling would be wrong: the read is re-probed ONCE
|
||||
/// at the full budget — served ⇒ the ceiling is forgotten; refused naming a larger <c>N</c> ⇒ <c>N</c> is learned and
|
||||
/// the read is re-issued at <c>N</c>; refused naming <c>C</c> again ⇒ the <c>C</c>-bounded result stands (no repeat).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Re-probe timing (#696).</b> Raw and AtTime reads re-probe incomplete streams immediately. Per-sample
|
||||
/// aggregate reads opt into a 30-second cooldown after a refusal; a full stream during it remains a request
|
||||
/// limit, never a truncated answer. After the cooldown the next incomplete read detects a raised ceiling.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
internal sealed class GatewayRowCeiling(TimeProvider? timeProvider = null)
|
||||
{
|
||||
private readonly TimeProvider _time = timeProvider ?? TimeProvider.System;
|
||||
private long _lastRefusalTicks;
|
||||
private int _learned; // 0 = unknown / not strict
|
||||
|
||||
/// <summary>The learned ceiling (0 = none).</summary>
|
||||
public int Learned => Volatile.Read(ref _learned);
|
||||
|
||||
/// <summary>Runs a drained read under the rules above.</summary>
|
||||
/// <typeparam name="T">The pass result.</typeparam>
|
||||
/// <param name="budget">The <c>max_values</c> the read wants.</param>
|
||||
/// <param name="pass">One drained read with the given budget.</param>
|
||||
/// <param name="needsMore">True when a pass with the given budget delivered it all without proving completeness.</param>
|
||||
/// <param name="deferRecentReprobe">Aggregate-only opt-in: reuse a refusal for 30 seconds before re-probing an incomplete read.</param>
|
||||
/// <returns>The result and the budget it was read with.</returns>
|
||||
public async Task<(T Result, int Budget)> ReadAsync<T>(int budget, Func<int, Task<T>> pass, Func<T, int, bool> needsMore, bool deferRecentReprobe = false)
|
||||
{
|
||||
var learned = Learned;
|
||||
if (learned <= 0 || learned >= budget) return await AttemptAsync(budget, pass).ConfigureAwait(false);
|
||||
|
||||
var bounded = await AttemptAsync(learned, pass).ConfigureAwait(false);
|
||||
if (bounded.Budget < learned || !needsMore(bounded.Result, bounded.Budget)) return bounded;
|
||||
|
||||
// #696: aggregate reads may reuse a recent refusal for 30 seconds. Raw paging keeps immediate re-probes.
|
||||
if (deferRecentReprobe && _time.GetUtcNow().UtcTicks - Volatile.Read(ref _lastRefusalTicks) < TimeSpan.FromSeconds(30).Ticks)
|
||||
return bounded;
|
||||
|
||||
// Rule 3: full at a possibly stale ceiling — re-probe once at the full budget.
|
||||
int fresh;
|
||||
try
|
||||
{
|
||||
var full = await pass(budget).ConfigureAwait(false);
|
||||
Interlocked.CompareExchange(ref _learned, 0, learned);
|
||||
return (full, budget);
|
||||
}
|
||||
catch (Exception ex) when (GatewayReadLimits.TryGetCeiling(ex, out fresh) && fresh < budget)
|
||||
{
|
||||
// fall through (outside the catch)
|
||||
}
|
||||
Learn(fresh);
|
||||
return fresh > learned ? (await pass(fresh).ConfigureAwait(false), fresh) : bounded;
|
||||
}
|
||||
|
||||
private void Learn(int ceiling)
|
||||
{
|
||||
Volatile.Write(ref _lastRefusalTicks, _time.GetUtcNow().UtcTicks);
|
||||
Volatile.Write(ref _learned, ceiling);
|
||||
}
|
||||
|
||||
private async Task<(T Result, int Budget)> AttemptAsync<T>(int budget, Func<int, Task<T>> pass)
|
||||
{
|
||||
int ceiling;
|
||||
try
|
||||
{
|
||||
return (await pass(budget).ConfigureAwait(false), budget);
|
||||
}
|
||||
catch (Exception ex) when (GatewayReadLimits.TryGetCeiling(ex, out ceiling) && ceiling < budget)
|
||||
{
|
||||
// fall through to the retry (outside the catch, so a second refusal is not nested in this one)
|
||||
}
|
||||
Learn(ceiling);
|
||||
return (await pass(ceiling).ConfigureAwait(false), ceiling);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
using Grpc.Core;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway;
|
||||
|
||||
/// <summary>Bounded metadata cache (#653). Missing tags expire quickly; failures never become missing tags.</summary>
|
||||
internal sealed class GatewayTagPresenceCache(IHistorianGatewayClient client, TimeProvider time)
|
||||
{
|
||||
internal const int Capacity = 4096;
|
||||
private readonly object _sync = new();
|
||||
private readonly Dictionary<string, (bool Exists, DateTimeOffset Expires)> _entries = new(StringComparer.Ordinal);
|
||||
|
||||
public async Task<bool> ExistsAsync(string tag, CancellationToken ct)
|
||||
{
|
||||
ct.ThrowIfCancellationRequested();
|
||||
lock (_sync)
|
||||
if (_entries.TryGetValue(tag, out var entry) && time.GetUtcNow() < entry.Expires) return entry.Exists;
|
||||
|
||||
bool exists;
|
||||
try { exists = await client.TagExistsAsync(tag, ct).ConfigureAwait(false); }
|
||||
catch (Exception ex) when (IsNotFound(ex)) { exists = false; }
|
||||
ct.ThrowIfCancellationRequested();
|
||||
lock (_sync)
|
||||
{
|
||||
if (!_entries.ContainsKey(tag) && _entries.Count >= Capacity) _entries.Remove(_entries.Keys.First());
|
||||
_entries[tag] = (exists, time.GetUtcNow().AddSeconds(exists ? 60 : 5));
|
||||
}
|
||||
return exists;
|
||||
}
|
||||
|
||||
private static bool IsNotFound(Exception ex)
|
||||
{
|
||||
for (Exception? e = ex; e is not null; e = e.InnerException)
|
||||
if (e is RpcException { StatusCode: StatusCode.NotFound }) return true;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@@ -29,6 +29,14 @@ public sealed class GatewayTagProvisioner : IHistorianProvisioning, IAsyncDispos
|
||||
/// </summary>
|
||||
private const uint DefaultStorageRateMs = 1000;
|
||||
|
||||
/// <summary>
|
||||
/// Integral divisor stamped on every provisioned analog tag (#677). AVEVA's <c>Total</c> integral is
|
||||
/// <c>∫ value dt (seconds) ÷ IntegralDivisor</c>; the proto's <c>double</c> default of 0 would reach the gateway as
|
||||
/// "unset" (or a divide-by-zero), so 1 is sent explicitly — the value·seconds unit the SQL backends' <c>Total</c>
|
||||
/// uses. (The live magnitude check against AVEVA is still open: it needs the VPN.)
|
||||
/// </summary>
|
||||
private const double DefaultIntegralDivisor = 1.0;
|
||||
|
||||
private readonly IHistorianGatewayClient _client;
|
||||
private readonly ILogger<GatewayTagProvisioner> _logger;
|
||||
|
||||
@@ -70,6 +78,8 @@ public sealed class GatewayTagProvisioner : IHistorianProvisioning, IAsyncDispos
|
||||
Description = request.Description ?? string.Empty,
|
||||
// Required: a 0 storage rate makes the gateway's EnsureTags throw (see the constant).
|
||||
StorageRateMs = DefaultStorageRateMs,
|
||||
// Explicit: the proto3 double default (0) is not "1" (see the constant).
|
||||
IntegralDivisor = DefaultIntegralDivisor,
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
+11
@@ -81,6 +81,17 @@ public sealed class HistorianGatewayClientAdapter : IHistorianGatewayClient, IDi
|
||||
return new HistorianGatewayClientAdapter(HistorianGatewayClient.Create(clientOptions));
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<bool> TagExistsAsync(string tag, CancellationToken ct)
|
||||
{
|
||||
// #653: older gateway SDKs misparse a missing GetTagInfo response as Internal. A catalog query proves
|
||||
// absence without guessing from an error. Even a glob-like tag is checked by exact name, never by row count.
|
||||
var found = false;
|
||||
await foreach (var name in _inner.BrowseTagNamesAsync(tag, ct).ConfigureAwait(false))
|
||||
found |= string.Equals(name, tag, StringComparison.OrdinalIgnoreCase);
|
||||
return found; // drain first: a trailing stream fault must propagate and must not become a cache entry
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public IAsyncEnumerable<HistorianSample> ReadRawAsync(
|
||||
string tag, DateTime startUtc, DateTime endUtc, int maxValues, CancellationToken ct) =>
|
||||
|
||||
@@ -11,6 +11,12 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway;
|
||||
/// </summary>
|
||||
public interface IHistorianGatewayClient : IAsyncDisposable
|
||||
{
|
||||
/// <summary>Checks the tag catalog; false means explicitly absent, while failures propagate.</summary>
|
||||
/// <param name="tag">Historian tag name.</param>
|
||||
/// <param name="ct">Cancellation token.</param>
|
||||
/// <returns>Whether the historian contains the tag.</returns>
|
||||
Task<bool> TagExistsAsync(string tag, CancellationToken ct);
|
||||
|
||||
/// <summary>Streams raw historian samples for a tag over a time window.</summary>
|
||||
/// <param name="tag">The historian tag name to read.</param>
|
||||
/// <param name="startUtc">Inclusive start of the requested time window (UTC).</param>
|
||||
|
||||
+35
-17
@@ -1,3 +1,4 @@
|
||||
using System.Collections.Frozen;
|
||||
using ZB.MOM.WW.HistorianGateway.Contracts.Grpc;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
|
||||
@@ -5,40 +6,57 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Mapping;
|
||||
|
||||
/// <summary>
|
||||
/// Maps the driver-agnostic <see cref="HistoryAggregateType"/> (OPC UA Part 13 aggregate) onto the
|
||||
/// gateway's native <see cref="RetrievalMode"/>.
|
||||
/// gateway's native <see cref="RetrievalMode"/> — for the aggregates still answered by AVEVA itself.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// Average/Minimum/Maximum line up with the legacy Wonderware client's aggregate mapping. The two
|
||||
/// remaining members are now served by <b>native</b> gateway retrieval modes:
|
||||
/// <see cref="HistoryAggregateType.Total"/> → <see cref="RetrievalMode.Integral"/> and
|
||||
/// <see cref="HistoryAggregateType.Count"/> → <see cref="RetrievalMode.Counter"/>.
|
||||
/// Native (AVEVA-computed): <see cref="HistoryAggregateType.TimeAverage"/> → <see cref="RetrievalMode.TimeWeightedAverage"/>
|
||||
/// and <see cref="HistoryAggregateType.Total"/> → <see cref="RetrievalMode.Integral"/>. They stay native because an exact
|
||||
/// Part 13 time-weighted evaluation needs the real bounding samples on both sides of the window (time, value and
|
||||
/// quality of the latest sample before it), and the gateway has no "previous stored sample" read — a raw read streams
|
||||
/// forward from its start and AVEVA replaces whatever lies before with an initial-value row stamped at the query start,
|
||||
/// losing the original time a sloped segment across the window start needs (#681).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// This replaces the Wonderware-era client-side workarounds (Total derived as Average × interval,
|
||||
/// Count approximated from a value count): no client-side scaling is performed any more, so the
|
||||
/// gateway path is a strict improvement.
|
||||
/// Since #681 the per-sample aggregates (Average, Count, Minimum, Maximum, Range, Start, End, Delta,
|
||||
/// Minimum/MaximumActualTime) are NOT native: <see cref="GatewayHistorianDataSource"/> evaluates them from raw reads
|
||||
/// with the SQL backends' shared evaluator (Minimum/Maximum used to map to AVEVA's <c>MinimumWithTime</c> /
|
||||
/// <c>MaximumWithTime</c>). Every other member — incl. TimeAverage2, Total2, Interpolative, PercentGood, PercentBad — is
|
||||
/// <see cref="HistoryAggregateNotSupportedException"/> here.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
internal static class AggregateModeMapper
|
||||
{
|
||||
/// <summary>
|
||||
/// The aggregates answered by a native AVEVA retrieval mode — every defined member <see cref="TryMap"/> maps, derived
|
||||
/// from it so the dispatch and the declared set can never disagree. Part of
|
||||
/// <see cref="GatewayHistorianDataSource.ServedAggregates"/> (#681/#682).
|
||||
/// </summary>
|
||||
public static readonly FrozenSet<HistoryAggregateType> NativeAggregates =
|
||||
Enum.GetValues<HistoryAggregateType>().Where(a => TryMap(a) is not null).ToFrozenSet();
|
||||
|
||||
/// <summary>Maps an aggregate function to the gateway retrieval mode.</summary>
|
||||
/// <param name="aggregate">The driver-agnostic aggregate function.</param>
|
||||
/// <returns>The matching gateway <see cref="RetrievalMode"/>.</returns>
|
||||
/// <exception cref="HistoryAggregateNotSupportedException">A defined member the gateway has no retrieval mode for.</exception>
|
||||
/// <exception cref="ArgumentOutOfRangeException">An undefined enum value.</exception>
|
||||
public static RetrievalMode ToRetrievalMode(HistoryAggregateType aggregate) => aggregate switch
|
||||
public static RetrievalMode ToRetrievalMode(HistoryAggregateType aggregate)
|
||||
{
|
||||
HistoryAggregateType.Average => RetrievalMode.TimeWeightedAverage,
|
||||
HistoryAggregateType.Minimum => RetrievalMode.MinimumWithTime,
|
||||
HistoryAggregateType.Maximum => RetrievalMode.MaximumWithTime,
|
||||
HistoryAggregateType.Total => RetrievalMode.Integral,
|
||||
HistoryAggregateType.Count => RetrievalMode.Counter,
|
||||
if (TryMap(aggregate) is { } mode) return mode;
|
||||
|
||||
// Part 13 aggregates the gateway has no retrieval mode for (#285) — an UNDEFINED enum value is a
|
||||
// programming error, but a defined-yet-unserved member is a normal client request the server
|
||||
// answers with BadAggregateNotSupported.
|
||||
_ when Enum.IsDefined(aggregate) => throw new HistoryAggregateNotSupportedException(aggregate),
|
||||
_ => throw new ArgumentOutOfRangeException(
|
||||
nameof(aggregate), aggregate, "Unmapped HistoryAggregateType — add a RetrievalMode mapping."),
|
||||
if (Enum.IsDefined(aggregate)) throw new HistoryAggregateNotSupportedException(aggregate);
|
||||
throw new ArgumentOutOfRangeException(
|
||||
nameof(aggregate), aggregate, "Unmapped HistoryAggregateType — add a RetrievalMode mapping.");
|
||||
}
|
||||
|
||||
/// <summary>The single mapping table: the retrieval mode for a natively served aggregate, or null.</summary>
|
||||
private static RetrievalMode? TryMap(HistoryAggregateType aggregate) => aggregate switch
|
||||
{
|
||||
HistoryAggregateType.TimeAverage => RetrievalMode.TimeWeightedAverage,
|
||||
HistoryAggregateType.Total => RetrievalMode.Integral,
|
||||
_ => null,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
using ZB.MOM.WW.HistorianGateway.Contracts.Grpc;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Mapping;
|
||||
|
||||
@@ -16,30 +17,50 @@ internal static class SampleMapper
|
||||
/// <summary>OPC DA "Good" family floor — a quality byte at/above this carries usable data.</summary>
|
||||
private const byte GoodQualityFloor = 192;
|
||||
|
||||
/// <summary>OPC DA "Uncertain" family floor (64–191).</summary>
|
||||
private const byte UncertainQualityFloor = 64;
|
||||
|
||||
/// <summary>Maps a single raw sample to a value snapshot.</summary>
|
||||
/// <param name="sample">The gateway raw sample.</param>
|
||||
/// <returns>The driver-agnostic snapshot.</returns>
|
||||
/// <remarks>
|
||||
/// #690: a Bad row carries no value. histsdk puts a value on EVERY row (<c>NumericValue = 0</c> on a numeric row,
|
||||
/// <c>StringValue = ""</c> on a string one), so a stored NULL arrives as <c>0</c> / <c>""</c> with a Bad OPC quality
|
||||
/// (live: <c>q=1 opc=0</c>) — that 0 is not a value, and AVEVA stores a Bad-quality sample as NULL, so a Bad row never
|
||||
/// carries a real one. The value is therefore dropped whenever the mapped status is Bad, matching the SqlServer/QuestDB
|
||||
/// backends (an Empty sample is null + Bad) and the aggregate mapper's no-data rows. Uncertain rows keep their value.
|
||||
/// </remarks>
|
||||
public static DataValueSnapshot ToSnapshot(HistorianSample sample)
|
||||
{
|
||||
var status = ToStatus(sample);
|
||||
// proto3 explicit presence: prefer the numeric value, else the string value, else null.
|
||||
object? value;
|
||||
if (sample.HasNumericValue)
|
||||
if (StatusBits.IsBad(status))
|
||||
value = null;
|
||||
else if (sample.HasNumericValue)
|
||||
value = sample.NumericValue; // boxes as System.Double
|
||||
else if (sample.HasStringValue)
|
||||
value = sample.StringValue;
|
||||
else
|
||||
value = null;
|
||||
|
||||
return new DataValueSnapshot(
|
||||
Value: value,
|
||||
StatusCode: status,
|
||||
SourceTimestampUtc: sample.Timestamp?.ToDateTime(), // Utc kind
|
||||
ServerTimestampUtc: DateTime.UtcNow);
|
||||
}
|
||||
|
||||
/// <summary>The OPC UA status of a raw sample (shared by raw reads and the #681 raw-evaluated aggregates).</summary>
|
||||
/// <param name="sample">The gateway raw sample.</param>
|
||||
/// <returns>The mapped status code.</returns>
|
||||
public static uint ToStatus(HistorianSample sample)
|
||||
{
|
||||
// Prefer the OPC DA quality byte (opc_quality); the gateway populates it directly from the
|
||||
// SDK's OpcQuality, so it is the authoritative byte for GatewayQualityMapper. Fall back to
|
||||
// the historian quality field only when opc_quality is unset (0).
|
||||
byte qualityByte = sample.OpcQuality != 0 ? (byte)sample.OpcQuality : (byte)sample.Quality;
|
||||
|
||||
return new DataValueSnapshot(
|
||||
Value: value,
|
||||
StatusCode: GatewayQualityMapper.Map(qualityByte),
|
||||
SourceTimestampUtc: sample.Timestamp?.ToDateTime(), // Utc kind
|
||||
ServerTimestampUtc: DateTime.UtcNow);
|
||||
return GatewayQualityMapper.Map(qualityByte);
|
||||
}
|
||||
|
||||
/// <summary>Maps a batch of raw samples to value snapshots, in order.</summary>
|
||||
@@ -57,31 +78,78 @@ internal static class SampleMapper
|
||||
/// <param name="aggregate">The gateway aggregate sample.</param>
|
||||
/// <returns>The driver-agnostic snapshot.</returns>
|
||||
/// <remarks>
|
||||
/// Unlike the legacy Wonderware DTO (a nullable <c>double?</c>), the gateway proto carries a
|
||||
/// non-optional <c>double value</c>, so an unavailable (no-data) bucket cannot be signalled by a
|
||||
/// null value. Instead it is signalled by a non-Good <c>opc_quality</c>: a Good bucket
|
||||
/// (<c>opc_quality >= 192</c>) yields its value with a Good status, anything else maps to
|
||||
/// <c>BadNoData</c> with a null value — preserving the Wonderware aggregate contract (binary
|
||||
/// Good-with-value / BadNoData-null).
|
||||
/// <para>
|
||||
/// The gateway proto carries a non-optional <c>double value</c>, so an unavailable (no-data) bucket cannot be signalled
|
||||
/// by a null value; it is signalled by the OPC DA quality family of <c>opc_quality</c> (its low byte, as the raw mapping
|
||||
/// reads it):
|
||||
/// </para>
|
||||
/// <list type="bullet">
|
||||
/// <item>Good (≥ 192) ⇒ the value, <c>Good</c>.</item>
|
||||
/// <item>Uncertain (64–191) ⇒ the value, <c>UncertainDataSubNormal</c> (#685): AVEVA marks an interval that holds a
|
||||
/// NULL/Bad sample <c>opc_quality = 64</c> and still answers its value (computed over the interval's good time only —
|
||||
/// see docs/Historian.md). It used to map to <c>BadNoData</c> with the value discarded. The sub-code is the one the SQL
|
||||
/// backends' aggregates give an interval that includes non-good data.</item>
|
||||
/// <item>Bad (< 64) ⇒ <c>BadNoData</c>, null value (the wire's 0 is not a value).</item>
|
||||
/// </list>
|
||||
/// <para>
|
||||
/// #681: every status carries the Calculated info bits (<c>0x401</c>) like every other backend's processed value, and
|
||||
/// the bucket is stamped with its interval START (Part 13; it was the interval end), falling back to the end only when
|
||||
/// the start is absent (verified on live AVEVA 2026-10-12: <c>start_time</c> is the interval start).
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public static DataValueSnapshot ToAggregateSnapshot(HistorianAggregateSample aggregate)
|
||||
{
|
||||
bool hasData = aggregate.OpcQuality >= GoodQualityFloor;
|
||||
var q = (byte)aggregate.OpcQuality;
|
||||
var status = q >= GoodQualityFloor ? StatusGood
|
||||
: q >= UncertainQualityFloor ? Sql.Common.HistorianStatusCodes.UncertainDataSubNormal
|
||||
: StatusBadNoData;
|
||||
return new DataValueSnapshot(
|
||||
Value: hasData ? aggregate.Value : null, // boxes as System.Double when present
|
||||
StatusCode: hasData ? StatusGood : StatusBadNoData,
|
||||
SourceTimestampUtc: (aggregate.EndTime ?? aggregate.StartTime)?.ToDateTime(), // bucket timestamp
|
||||
Value: status == StatusBadNoData ? null : aggregate.Value, // boxes as System.Double when present
|
||||
StatusCode: HistorianInfoBits.WithCalculated(status),
|
||||
SourceTimestampUtc: (aggregate.StartTime ?? aggregate.EndTime)?.ToDateTime(), // bucket timestamp = interval start
|
||||
ServerTimestampUtc: DateTime.UtcNow);
|
||||
}
|
||||
|
||||
/// <summary>Maps a batch of aggregate buckets to value snapshots, in order.</summary>
|
||||
/// <summary>
|
||||
/// True when the aggregate row's interval lies in the requested window — the span between <paramref name="startUtc"/>
|
||||
/// and <paramref name="endUtc"/> in either order (a reverse request covers the same span). AVEVA answers one extra row
|
||||
/// for the interval BEFORE the window (<c>[start − interval, start)</c>, #685), which Part 11/13 has no place for: an
|
||||
/// interval belongs to the window when its start lies in <c>[lo, hi)</c>; a row without a start time (the mapper then
|
||||
/// stamps it with its end) belongs when its end lies in <c>(lo, hi]</c>.
|
||||
/// </summary>
|
||||
/// <param name="aggregate">The gateway aggregate sample.</param>
|
||||
/// <param name="startUtc">The request's start time.</param>
|
||||
/// <param name="endUtc">The request's end time.</param>
|
||||
/// <returns>Whether the row is one of the window's intervals.</returns>
|
||||
public static bool IsInWindow(HistorianAggregateSample aggregate, DateTime startUtc, DateTime endUtc)
|
||||
{
|
||||
var lo = startUtc <= endUtc ? startUtc : endUtc;
|
||||
var hi = startUtc <= endUtc ? endUtc : startUtc;
|
||||
if (aggregate.StartTime is { } s)
|
||||
{
|
||||
var t = s.ToDateTime();
|
||||
return t >= lo && t < hi;
|
||||
}
|
||||
if (aggregate.EndTime is { } e)
|
||||
{
|
||||
var t = e.ToDateTime();
|
||||
return t > lo && t <= hi;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/// <summary>Maps the aggregate buckets of a window to value snapshots, in order, dropping rows outside it (#685).</summary>
|
||||
/// <param name="aggregates">The gateway aggregate samples.</param>
|
||||
/// <returns>The driver-agnostic snapshots.</returns>
|
||||
public static IReadOnlyList<DataValueSnapshot> ToAggregateSnapshots(IEnumerable<HistorianAggregateSample> aggregates)
|
||||
/// <param name="startUtc">The request's start time.</param>
|
||||
/// <param name="endUtc">The request's end time.</param>
|
||||
/// <returns>The driver-agnostic snapshots, one per in-window interval row.</returns>
|
||||
public static IReadOnlyList<DataValueSnapshot> ToAggregateSnapshots(IEnumerable<HistorianAggregateSample> aggregates,
|
||||
DateTime startUtc, DateTime endUtc)
|
||||
{
|
||||
var result = new List<DataValueSnapshot>();
|
||||
foreach (var aggregate in aggregates)
|
||||
result.Add(ToAggregateSnapshot(aggregate));
|
||||
if (IsInWindow(aggregate, startUtc, endUtc))
|
||||
result.Add(ToAggregateSnapshot(aggregate));
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,112 @@
|
||||
using ZB.MOM.WW.HistorianGateway.Contracts.Grpc;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Mapping;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway;
|
||||
|
||||
/// <summary>
|
||||
/// Serves a processed read of the AVEVA-native <c>TimeAverage</c> / <c>Total</c> on the SQL backends' interval grid
|
||||
/// (<see cref="BucketPlan"/>), in either direction: forward — window <c>[start, end)</c>, intervals stepping forward
|
||||
/// from <c>start</c>, the clipped interval the LATEST, each stamped with its earlier edge, oldest first; reverse (#677,
|
||||
/// #691.2) — window <c>(end, start]</c>, intervals stepping back from <c>start</c>, the clipped interval the EARLIEST,
|
||||
/// each stamped with its later edge, newest first. The gateway refuses <c>start > end</c> and <c>interval <= 0</c>,
|
||||
/// so every request it sees is forward with a positive interval.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>Forward requests on the plan's grid.</b> AVEVA anchors a native read's interval grid at the request start (live,
|
||||
/// 2026-10-12: rows at <c>start + j·interval</c>, plus one extra row for the interval before it, #685). So the plan's
|
||||
/// whole intervals are requested as ONE forward read spanning exactly them with the interval <c>I</c>, and the
|
||||
/// clipped interval (when the window is not a whole number of intervals) as a second forward read over exactly its
|
||||
/// span with its own width as the interval — AVEVA never averages time outside the window (before it on a reverse
|
||||
/// read, past <c>EndTime</c> on a forward one). An interval wider than the window is sent as the window, and
|
||||
/// <c>interval <= 0</c> as one interval over the whole window (it used to reach the gateway as 0 and be refused).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Soundness of reverse.</b> A time-weighted average / integral over <c>[a, b)</c> and over <c>(a, b]</c> integrate
|
||||
/// the same function over sets differing by two instants, so a forward native row IS the value of the reverse
|
||||
/// interval with the same edges. The one place the half-open side could matter is AVEVA's status for an interval
|
||||
/// whose only non-good sample lies exactly on an edge — accepted, as on every native row.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Mapping.</b> Each row is mapped as before (<see cref="SampleMapper.ToAggregateSnapshot"/>: Uncertain keeps its
|
||||
/// value, no data ⇒ <c>BadNoData</c>, Calculated bits), placed by its interval start (else end), restamped with
|
||||
/// <see cref="BucketPlan.BucketTimestamp"/>, and the clipped interval carries the Partial bit unless Bad (Part 13
|
||||
/// §5.3.3.2 Case 3, as the SQL backends' time-weighted aggregates — which additionally mark the intervals the data
|
||||
/// starts/ends inside; AVEVA's rows carry no data range, so those are not reproduced). An interval AVEVA answers no
|
||||
/// row for is absent (not a fabricated <c>BadNoData</c>, unlike the SQL backends).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// ⚠️ Today historiangw#27 (the interval reaches AVEVA in ticks where it reads milliseconds) makes AVEVA answer ONE
|
||||
/// Good row per request — the whole-intervals read lands on its earliest interval with a whole-span value and the rest
|
||||
/// are absent — in both directions. The mapping needs no change once the gateway is fixed.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
internal static class NativeAggregatePlan
|
||||
{
|
||||
/// <summary>One forward native request and the ascending bucket index of its first interval.</summary>
|
||||
/// <param name="StartUtc">Forward request start (inclusive).</param>
|
||||
/// <param name="EndUtc">Forward request end.</param>
|
||||
/// <param name="Interval">The request's interval (> 0).</param>
|
||||
/// <param name="FirstBucket">Ascending plan bucket of the request's first interval.</param>
|
||||
/// <param name="BucketCount">Number of plan buckets the request covers.</param>
|
||||
internal readonly record struct Request(DateTime StartUtc, DateTime EndUtc, TimeSpan Interval, long FirstBucket, long BucketCount);
|
||||
|
||||
/// <summary>The forward requests serving a plan: the whole intervals first, then the clipped one (if any).</summary>
|
||||
/// <param name="plan">A plan with at least one bucket.</param>
|
||||
/// <returns>One or two forward requests.</returns>
|
||||
public static IReadOnlyList<Request> Requests(BucketPlan plan)
|
||||
{
|
||||
var interval = TimeSpan.FromTicks(plan.IntervalTicks);
|
||||
var n = plan.BucketCount;
|
||||
var clipped = plan.IsTruncated(0) ? 0L : plan.IsTruncated(n - 1) ? n - 1 : -1L;
|
||||
if (clipped < 0) return [new Request(plan.StartUtc, plan.EndUtc, interval, 0, n)];
|
||||
|
||||
var clipLo = new DateTime(plan.LowerTicks(clipped), DateTimeKind.Utc);
|
||||
var clipHi = new DateTime(plan.UpperTicks(clipped), DateTimeKind.Utc);
|
||||
var requests = new List<Request>(2);
|
||||
if (n > 1)
|
||||
requests.Add(clipped == 0
|
||||
? new Request(clipHi, plan.EndUtc, interval, 1, n - 1) // reverse: the clipped one is the earliest
|
||||
: new Request(plan.StartUtc, clipLo, interval, 0, n - 1)); // forward: the clipped one is the latest
|
||||
requests.Add(new Request(clipLo, clipHi, clipHi - clipLo, clipped, 1));
|
||||
return requests;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Maps a request's rows onto the plan's buckets: rows outside the request window (AVEVA's pre-window row, #685)
|
||||
/// are dropped; each in-window row is placed by its interval start (or, without one, its end).
|
||||
/// </summary>
|
||||
/// <param name="plan">The plan.</param>
|
||||
/// <param name="request">The forward request the rows answer.</param>
|
||||
/// <param name="rows">The drained rows.</param>
|
||||
/// <param name="byBucket">Results by ascending bucket index (first row per bucket wins).</param>
|
||||
public static void Place(BucketPlan plan, Request request, IEnumerable<HistorianAggregateSample> rows,
|
||||
Dictionary<long, DataValueSnapshot> byBucket)
|
||||
{
|
||||
var interval = request.Interval.Ticks;
|
||||
foreach (var row in rows)
|
||||
{
|
||||
if (!SampleMapper.IsInWindow(row, request.StartUtc, request.EndUtc)) continue;
|
||||
long offset = row.StartTime is { } s
|
||||
? (s.ToDateTime().Ticks - request.StartUtc.Ticks) / interval
|
||||
: (row.EndTime.ToDateTime().Ticks - request.StartUtc.Ticks - 1) / interval; // IsInWindow ⇒ end in (lo, hi]
|
||||
if (offset < 0 || offset >= request.BucketCount) continue;
|
||||
var index = request.FirstBucket + offset;
|
||||
if (byBucket.ContainsKey(index)) continue;
|
||||
|
||||
var snapshot = SampleMapper.ToAggregateSnapshot(row);
|
||||
var status = snapshot.StatusCode;
|
||||
if (plan.IsTruncated(index) && !StatusBits.IsBad(status)) status |= HistorianInfoBits.HistorianPartial;
|
||||
byBucket[index] = snapshot with { StatusCode = status, SourceTimestampUtc = plan.BucketTimestamp(index) };
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>The placed results in the read's order (ascending forward, descending reverse).</summary>
|
||||
/// <param name="plan">The plan.</param>
|
||||
/// <param name="byBucket">Results by ascending bucket index.</param>
|
||||
/// <returns>The results.</returns>
|
||||
public static IReadOnlyList<DataValueSnapshot> Assemble(BucketPlan plan, IReadOnlyDictionary<long, DataValueSnapshot> byBucket) =>
|
||||
(plan.Reversed ? byBucket.OrderByDescending(kv => kv.Key) : byBucket.OrderBy(kv => kv.Key)).Select(kv => kv.Value).ToList();
|
||||
}
|
||||
@@ -0,0 +1,415 @@
|
||||
using ZB.MOM.WW.HistorianGateway.Contracts.Grpc;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Mapping;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway;
|
||||
|
||||
/// <summary>
|
||||
/// #681: serves the per-sample processed aggregates (the <see cref="AggregateFamily.Simple"/>,
|
||||
/// <see cref="AggregateFamily.StartEnd"/> and <see cref="AggregateFamily.ActualTime"/> families) on the Gateway backend by
|
||||
/// streaming the window's RAW samples through the gateway and evaluating them with the shared
|
||||
/// <c>Driver.Historian.Sql.Common</c> code — <see cref="BucketPlan"/> for the bucket math (forward and reverse, #677) and
|
||||
/// <see cref="AggregateEvaluator.EvaluateAll"/> for values, status and the Partial/Calculated bits — so the Gateway answers
|
||||
/// exactly what SqlServer and QuestDB answer for the same stored samples.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>Forward raw reads, each consumed to completion.</b> The gateway's <c>ReadRaw</c> streams oldest-first over an
|
||||
/// inclusive <c>[start, end]</c> and refuses <c>end < start</c>, so a reverse read streams the same ascending range
|
||||
/// and <see cref="BucketPlan"/> assembles the result newest-first. The window stream is classified sample by sample
|
||||
/// with <see cref="BucketPlan.IndexOf"/> (<c>[start, end)</c> forward, <c>(start, end]</c> reverse) — memory is
|
||||
/// O(buckets), never O(samples). No gateway stream is ever abandoned mid-way: a client-side break cancels the call,
|
||||
/// and the gateway currently classifies that cancellation as a connection fault and evicts its shared historian
|
||||
/// session (HistorianGateway <c>HistorianRpcFaults.IsConnectionFault</c>: any <c>RpcException</c> ⇒ true).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>The AVEVA start-boundary row.</b> An AVEVA <c>Full</c> raw read emits an initial-value row stamped exactly at
|
||||
/// the query start (histsdk reverse-engineering: <c>Quality = 133</c>, <c>OpcQuality = 192</c>) carrying the value
|
||||
/// in effect there, not a stored sample. The window stream is therefore requested from <see cref="BoundaryMargin"/>
|
||||
/// (1 ms) BEFORE the window, so that row lands outside every bucket and can never be counted as a sample. A row
|
||||
/// before the window counts only as evidence that the tag has earlier data, and only when its OPC quality is not
|
||||
/// Bad — a "no earlier data" NULL row still arrives WITH a value (histsdk sets <c>NumericValue = 0</c> on every
|
||||
/// numeric row, <c>StringValue = ""</c> on a string one) but with a Bad OPC quality (captured: 1, no-data).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>The data range.</b> The start/end-of-data Partial rules and Count's in-range empty interval need the tag's
|
||||
/// earliest sample at or before the window's later edge and latest at or after its earlier edge
|
||||
/// (<see cref="AggregateEvaluator.NeedsDataRange"/>; a plain <c>Average</c> needs neither). Earlier data is a non-Bad
|
||||
/// pre-window row, or — when none arrived — a <see cref="ProbeRows"/>-row read <c>[</c><see cref="ProbeFloor"/><c>,
|
||||
/// start)</c>. Later data is a sample exactly at the window end (forward) from the window read, or else a
|
||||
/// <see cref="ProbeRows"/>-row read <c>[end, now + </c><see cref="LookAheadHorizon"/><c>]</c> (the historian stores
|
||||
/// nothing further in the future). In both small reads a row stamped exactly at the query start (a boundary row) or
|
||||
/// at the query end (a possible terminal row) is not a stored sample, whatever its value.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Bounded.</b> The in-window rows are capped by <c>ServerHistorian:MaxTimeWeightedRawRows</c> (the same knob the
|
||||
/// SQL backends' time-weighted reads use): past it the node fails with
|
||||
/// <see cref="HistoryReadLimitExceededException"/> (⇒ <c>BadTooManyOperations</c>) once the stream — bounded by the
|
||||
/// requested row budget — has been drained. The gateway is asked for at most <c>cap + 2 + </c>
|
||||
/// <see cref="PreWindowAllowance"/> rows, clamped at <see cref="GatewayDefaultMaxValuesCeiling"/> while the cap is at
|
||||
/// or under it (#688, <see cref="ClampToGatewayCeiling"/>); a stream that delivers all of them cannot be proven
|
||||
/// complete and fails the same way rather than answering from a truncated window. A strict gateway whose ceiling is
|
||||
/// below that budget (a ceiling under 1,000,000, or an unbounded cap) refuses it naming its ceiling; the window is
|
||||
/// then re-read once with the ceiling as budget (#691.3, <see cref="GatewayReadLimits"/>), and the same
|
||||
/// "delivered the whole budget ⇒ cannot be proven complete" rule applies.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Values and status.</b> A sample's value is its numeric value (Double), else its string value, else Empty; its
|
||||
/// status is <see cref="SampleMapper.ToStatus"/> — the same quality mapping a raw HistoryRead serves (C-002: the
|
||||
/// gateway stores no quality, so AVEVA history is normally all Good). An Empty sample is clamped Bad
|
||||
/// (<see cref="StoredSample.FromStored"/>), exactly as on the SQL backends.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
internal static class RawAggregateReader
|
||||
{
|
||||
/// <summary>How far before the window the raw stream starts, so the AVEVA initial-value row lands outside it.</summary>
|
||||
internal static readonly TimeSpan BoundaryMargin = TimeSpan.FromMilliseconds(1);
|
||||
|
||||
/// <summary>How far past "now" the stream may look for a sample after the window (end-of-data rules).</summary>
|
||||
internal static readonly TimeSpan LookAheadHorizon = TimeSpan.FromDays(1);
|
||||
|
||||
/// <summary>Rows the stream may spend before the window (the boundary row plus samples inside the margin).</summary>
|
||||
internal const int PreWindowAllowance = 16;
|
||||
|
||||
/// <summary>Start of the "is there data before the window?" probe (the Unix epoch — before any AVEVA history).</summary>
|
||||
internal static readonly DateTime ProbeFloor = DateTime.UnixEpoch;
|
||||
|
||||
/// <summary>Rows the earlier-data probe asks for (a possible boundary row plus the first stored sample).</summary>
|
||||
internal const int ProbeRows = 2;
|
||||
|
||||
/// <summary>
|
||||
/// The <c>ReadRaw</c> start for a window that begins at <paramref name="windowStartUtc"/>: <see cref="BoundaryMargin"/>
|
||||
/// earlier, so the AVEVA start-boundary row — stamped exactly at the QUERY start, carrying the value in effect there
|
||||
/// (<c>Quality = 133</c>) or, with no earlier data, a NULL row (value 0, Bad OPC quality) — lands BEFORE the window
|
||||
/// (<see cref="IsWindowSample"/>). The one place both the raw-evaluated aggregates (#681) and a raw HistoryRead
|
||||
/// (#687) decide what the request start is. Never before <see cref="ProbeFloor"/> (the Unix epoch — the start of
|
||||
/// the historian's time encoding, the same floor the server's reverse walk-back uses, #686): a window starting less
|
||||
/// than one margin after it is requested from the epoch itself.
|
||||
/// </summary>
|
||||
/// <param name="windowStartUtc">The window's (inclusive) start.</param>
|
||||
/// <returns>The start to request.</returns>
|
||||
public static DateTime RequestStartFor(DateTime windowStartUtc) =>
|
||||
windowStartUtc - ProbeFloor > BoundaryMargin ? windowStartUtc - BoundaryMargin
|
||||
: windowStartUtc > ProbeFloor ? ProbeFloor
|
||||
: windowStartUtc;
|
||||
|
||||
/// <summary>
|
||||
/// True when a row of a forward raw stream requested from <see cref="RequestStartFor"/> is a stored sample of the raw
|
||||
/// window <c>[windowStartUtc, windowEndUtc)</c> — the <see cref="IHistorianDataSource.ReadRawAsync"/> contract
|
||||
/// (exclusive end, as on the SQL backends; the gateway's own <c>ReadRaw</c> end is inclusive) — or, for a point read
|
||||
/// (<c>start == end</c>, the server's tie-cluster over-fetch), of that one instant. A row before the start is the AVEVA
|
||||
/// boundary row (whatever its value or quality — a stored NULL looks the same on the wire, so the timestamp is the
|
||||
/// evidence) or a stored sample inside the margin; a row without a timestamp is never a sample.
|
||||
/// </summary>
|
||||
/// <param name="raw">The gateway row.</param>
|
||||
/// <param name="windowStartUtc">The window's (inclusive) start.</param>
|
||||
/// <param name="windowEndUtc">The window's end: exclusive, or inclusive when it equals the start.</param>
|
||||
/// <returns>Whether the row belongs to the window.</returns>
|
||||
public static bool IsWindowSample(HistorianSample raw, DateTime windowStartUtc, DateTime windowEndUtc) =>
|
||||
raw.Timestamp is { } ts && ts.ToDateTime() is var t && IsAfterFloor(t) && t >= windowStartUtc
|
||||
&& (t < windowEndUtc || (t == windowEndUtc && windowStartUtc == windowEndUtc));
|
||||
|
||||
/// <summary>
|
||||
/// A row stamped at or before <see cref="ProbeFloor"/> (the Unix epoch) is never a stored sample: AVEVA stores nothing
|
||||
/// there (the server's #686 walk-back assumes the same), and a window starting at or before the epoch gets no 1 ms
|
||||
/// margin (<see cref="RequestStartFor"/>), so AVEVA's synthetic boundary / no-data row lands exactly at its start.
|
||||
/// </summary>
|
||||
/// <param name="t">The row's timestamp.</param>
|
||||
/// <returns>Whether the row can be a stored sample.</returns>
|
||||
internal static bool IsAfterFloor(DateTime t) => t > ProbeFloor;
|
||||
|
||||
/// <summary>True for the aggregates this reader serves: the per-sample families of <see cref="AggregateFamilies"/>.</summary>
|
||||
/// <param name="aggregate">Requested aggregate.</param>
|
||||
/// <returns>Whether the aggregate is evaluated from raw samples.</returns>
|
||||
public static bool Serves(HistoryAggregateType aggregate) =>
|
||||
AggregateFamilies.FamilyOf(aggregate) is AggregateFamily.Simple or AggregateFamily.StartEnd or AggregateFamily.ActualTime;
|
||||
|
||||
/// <summary>The gateway <c>max_values</c> for a configured window cap (<see cref="int.MaxValue"/> when unbounded).</summary>
|
||||
/// <param name="maxWindowRows">The configured cap (<= 0 = unbounded).</param>
|
||||
/// <returns>The row limit to request.</returns>
|
||||
public static int MaxValuesFor(int maxWindowRows) =>
|
||||
maxWindowRows <= 0 ? int.MaxValue : ClampToGatewayCeiling((long)maxWindowRows + 2 + PreWindowAllowance, maxWindowRows);
|
||||
|
||||
/// <summary>
|
||||
/// The HistorianGateway's default <c>Historian:ReadLimits:MaxValuesCeiling</c>. With
|
||||
/// <c>AllowStreamingBeyondCeiling=false</c> the gateway refuses any <c>max_values</c> above its ceiling.
|
||||
/// </summary>
|
||||
internal const int GatewayDefaultMaxValuesCeiling = 1_000_000;
|
||||
|
||||
/// <summary>
|
||||
/// #688: a <c>ReadRaw</c> row budget of <paramref name="rows"/> plus this backend's own allowance
|
||||
/// (<paramref name="requested"/>) clamped at <see cref="GatewayDefaultMaxValuesCeiling"/> — but never below
|
||||
/// <paramref name="rows"/> itself. The allowance (the boundary row, the margin, the completeness row) must not push a
|
||||
/// budget that fits the gateway's default ceiling over it: the default <c>MaxTimeWeightedRawRows</c> (1,000,000) + 18
|
||||
/// used to exceed it, so a gateway with <c>AllowStreamingBeyondCeiling=false</c> refused EVERY per-sample read. A
|
||||
/// count above the ceiling is the operator's (or client's) deliberate choice for a streaming gateway and keeps its
|
||||
/// allowance. The cost of clamping is that a window holding within the allowance of the ceiling cannot be proven
|
||||
/// complete and fails as over the cap — the same outcome it would get one row later.
|
||||
/// </summary>
|
||||
/// <param name="requested">The budget including the allowance.</param>
|
||||
/// <param name="rows">The caller's own row count (the cap, or <c>NumValuesPerNode</c>), > 0.</param>
|
||||
/// <returns>The <c>max_values</c> to request.</returns>
|
||||
internal static int ClampToGatewayCeiling(long requested, long rows) =>
|
||||
(int)Math.Min(int.MaxValue, rows > GatewayDefaultMaxValuesCeiling ? requested : Math.Min(requested, GatewayDefaultMaxValuesCeiling));
|
||||
|
||||
/// <summary>Maps one gateway raw sample to a stored sample (value + mapped status, Empty clamped Bad).</summary>
|
||||
/// <param name="sample">The gateway sample; its timestamp must be set.</param>
|
||||
/// <returns>The stored sample.</returns>
|
||||
public static StoredSample ToStored(HistorianSample sample)
|
||||
{
|
||||
var value = sample.HasNumericValue ? HistorianValue.FromDouble(sample.NumericValue)
|
||||
: sample.HasStringValue ? HistorianValue.FromString(sample.StringValue)
|
||||
: HistorianValue.Empty;
|
||||
return StoredSample.FromStored(sample.Timestamp.ToDateTime(), value, SampleMapper.ToStatus(sample));
|
||||
}
|
||||
|
||||
/// <summary>Reads and evaluates one per-sample processed read.</summary>
|
||||
/// <param name="client">The gateway client.</param>
|
||||
/// <param name="ceiling">The data source's learned gateway row ceiling (#691.3).</param>
|
||||
/// <param name="tag">Historian tag name.</param>
|
||||
/// <param name="aggregate">A per-sample aggregate (<see cref="Serves"/>).</param>
|
||||
/// <param name="plan">The bucket plan.</param>
|
||||
/// <param name="maxWindowRows">The in-window raw-row cap (<= 0 = unbounded).</param>
|
||||
/// <param name="now">Server timestamp, and the anchor of the look-ahead horizon.</param>
|
||||
/// <param name="ct">Cancellation token.</param>
|
||||
/// <returns>One value per interval, in the plan's order.</returns>
|
||||
/// <exception cref="HistoryReadLimitExceededException">The window holds more than <paramref name="maxWindowRows"/> rows,
|
||||
/// or the stream could not be read to completion within the requested row budget.</exception>
|
||||
public static async Task<IReadOnlyList<DataValueSnapshot>> ReadAsync(IHistorianGatewayClient client, GatewayRowCeiling ceiling, string tag,
|
||||
HistoryAggregateType aggregate, BucketPlan plan, int maxWindowRows, DateTime now, CancellationToken ct)
|
||||
{
|
||||
if (plan.BucketCount == 0) return [];
|
||||
var needsRange = AggregateEvaluator.NeedsDataRange(aggregate, plan);
|
||||
var requestStart = RequestStartFor(plan.StartUtc);
|
||||
|
||||
// A strict gateway that refuses the budget is re-asked once at the ceiling it names (#691.3; e.g. a ceiling below
|
||||
// the 1,000,000 #688 clamps at, or an unbounded cap) — and a stream that fills that ceiling is never evaluated.
|
||||
var requested = MaxValuesFor(maxWindowRows);
|
||||
var (pass, maxValues) = await ceiling.ReadAsync(requested,
|
||||
budget => ReadWindowAsync(client, tag, aggregate, plan, requestStart, budget, maxWindowRows, ct),
|
||||
(r, budget) => !r.OverCap && r.Received >= budget, deferRecentReprobe: true).ConfigureAwait(false);
|
||||
var (buckets, earliest, latest, received, overCap) = pass;
|
||||
if (overCap) throw LimitExceeded(tag, maxWindowRows);
|
||||
if (received >= maxValues) throw BudgetExhausted(tag, maxWindowRows, requested, maxValues);
|
||||
|
||||
if (needsRange)
|
||||
{
|
||||
// Earlier data: no non-Bad pre-window row ⇒ ask explicitly (normally only when the tag's history really starts
|
||||
// inside the window).
|
||||
var earlierSeen = earliest is { } e && (e < plan.StartUtc || (plan.Reversed && e == plan.StartUtc));
|
||||
if (!earlierSeen && plan.StartUtc.AddTicks(-1) > ProbeFloor)
|
||||
if (await FirstSampleAsync(client, tag, ProbeFloor, plan.StartUtc.AddTicks(-1), ct).ConfigureAwait(false) is { } before)
|
||||
earliest = before;
|
||||
|
||||
// Later data: the main read covers a sample exactly at the window end (forward: that is already later data);
|
||||
// otherwise a small separate read from the end to now + horizon.
|
||||
var laterSeen = latest is { } l && (plan.Reversed ? l > plan.EndUtc : l >= plan.EndUtc);
|
||||
var horizon = DateTime.MaxValue - now > LookAheadHorizon
|
||||
? DateTime.SpecifyKind(now + LookAheadHorizon, DateTimeKind.Utc)
|
||||
: DateTime.MaxValue;
|
||||
if (!laterSeen && horizon > plan.EndUtc)
|
||||
if (await FirstSampleAsync(client, tag, plan.EndUtc, horizon, ct).ConfigureAwait(false) is { } after)
|
||||
latest = after;
|
||||
}
|
||||
|
||||
return AggregateEvaluator.EvaluateAll(aggregate, plan, buckets.Rows(), earliest, latest, now);
|
||||
}
|
||||
|
||||
/// <summary>The window read of <see cref="ReadAsync"/> with one row budget, drained.</summary>
|
||||
private static async Task<(RawBucketAccumulator Buckets, DateTime? Earliest, DateTime? Latest, long Received, bool OverCap)>
|
||||
ReadWindowAsync(IHistorianGatewayClient client, string tag, HistoryAggregateType aggregate, BucketPlan plan,
|
||||
DateTime requestStart, int maxValues, int maxWindowRows, CancellationToken ct)
|
||||
{
|
||||
var buckets = new RawBucketAccumulator(aggregate == HistoryAggregateType.MaximumActualTime);
|
||||
DateTime? earliest = null, latest = null;
|
||||
long windowRows = 0, received = 0;
|
||||
var overCap = false;
|
||||
// Consumed to COMPLETION (never broken out of): abandoning a gateway stream cancels the call server-side, and the
|
||||
// gateway currently treats that cancellation as a connection fault and evicts its shared historian session. Past
|
||||
// the cap the rest is drained (bounded by maxValues) without being evaluated.
|
||||
await foreach (var raw in client.ReadRawAsync(tag, requestStart, plan.EndUtc, maxValues, ct).ConfigureAwait(false))
|
||||
{
|
||||
received++;
|
||||
if (overCap || raw.Timestamp is null) continue;
|
||||
var s = ToStored(raw);
|
||||
var t = s.TimeUtc;
|
||||
if (!IsAfterFloor(t)) continue; // the boundary row of a window starting at/before the epoch, never a sample
|
||||
if (plan.IndexOf(t) is { } bucket)
|
||||
{
|
||||
if (maxWindowRows > 0 && ++windowRows > maxWindowRows)
|
||||
{
|
||||
overCap = true;
|
||||
continue;
|
||||
}
|
||||
buckets.Add(bucket, s);
|
||||
}
|
||||
// The data range as the SQL backends define it: earliest stored sample at or before the window's later edge,
|
||||
// latest at or after its earlier edge. A row before the window (the AVEVA boundary row, or a stored sample in
|
||||
// the margin) proves earlier data only when its OPC quality is not Bad: AVEVA's boundary row for a tag with no
|
||||
// earlier data is a NULL row, which arrives WITH a value (histsdk sets NumericValue = 0 / StringValue "") and
|
||||
// a Bad OPC quality (captured: 1, no-data).
|
||||
if (t >= plan.StartUtc || IsNotBadOnTheWire(raw))
|
||||
{
|
||||
if (t <= plan.EndUtc && (earliest is null || t < earliest)) earliest = t;
|
||||
if (t >= plan.StartUtc && (latest is null || t > latest)) latest = t;
|
||||
}
|
||||
}
|
||||
return (buckets, earliest, latest, received, overCap);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The time of the first stored sample (any quality) strictly between <paramref name="startUtc"/> and
|
||||
/// <paramref name="endUtc"/>, read as a <see cref="ProbeRows"/>-row raw read consumed to completion. A row stamped
|
||||
/// exactly at the query start is the historian's boundary row (the value in effect there, or a NULL row when there is
|
||||
/// none) and a row exactly at the query end may be a terminal row — neither is a stored sample, whatever its value.
|
||||
/// </summary>
|
||||
private static async Task<DateTime?> FirstSampleAsync(IHistorianGatewayClient client, string tag, DateTime startUtc,
|
||||
DateTime endUtc, CancellationToken ct)
|
||||
{
|
||||
DateTime? found = null;
|
||||
await foreach (var raw in client.ReadRawAsync(tag, startUtc, endUtc, ProbeRows, ct).ConfigureAwait(false))
|
||||
{
|
||||
if (found is not null || raw.Timestamp is null) continue; // drain, never break (see ReadAsync)
|
||||
var t = raw.Timestamp.ToDateTime();
|
||||
if (t > startUtc && t < endUtc) found = t;
|
||||
}
|
||||
return found;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The row's OPC DA quality alone (<c>opc_quality</c> ≥ 64: Good or Uncertain) — deliberately without
|
||||
/// <see cref="SampleMapper.ToStatus"/>'s fallback to the AVEVA <c>quality</c> field when <c>opc_quality</c> is 0, which
|
||||
/// is also the OPC DA Bad value a NULL row carries.
|
||||
/// </summary>
|
||||
private static bool IsNotBadOnTheWire(HistorianSample raw) => (byte)raw.OpcQuality >= 64; // low byte, as SampleMapper.ToStatus reads it
|
||||
|
||||
/// <summary>
|
||||
/// The stream delivered its whole row budget, so the window cannot be proven complete. Names the limit that actually
|
||||
/// bound the budget: the gateway's ceiling (a strict gateway refused the requested budget and named it, #691.3), the
|
||||
/// #688 clamp at the gateway's default ceiling, or the configured cap.
|
||||
/// </summary>
|
||||
private static HistoryReadLimitExceededException BudgetExhausted(string tag, int maxWindowRows, int requested, int budget)
|
||||
{
|
||||
if (budget < requested)
|
||||
return new HistoryReadLimitExceededException(budget,
|
||||
$"The processed read of '{tag}' covers more raw samples than the HistorianGateway's row ceiling ({budget}, its " +
|
||||
"Historian:ReadLimits:MaxValuesCeiling with AllowStreamingBeyondCeiling=false); the Gateway backend evaluates " +
|
||||
"per-sample aggregates from raw reads — narrow the window.");
|
||||
if (maxWindowRows > 0 && budget < (long)maxWindowRows + 2 + PreWindowAllowance)
|
||||
return new HistoryReadLimitExceededException(budget,
|
||||
$"The processed read of '{tag}' covers more raw samples than the HistorianGateway's default row ceiling " +
|
||||
$"({budget}) the request is clamped at; the Gateway backend evaluates per-sample aggregates from raw reads — " +
|
||||
"narrow the window.");
|
||||
return LimitExceeded(tag, maxWindowRows);
|
||||
}
|
||||
|
||||
private static HistoryReadLimitExceededException LimitExceeded(string tag, int maxWindowRows) =>
|
||||
new(maxWindowRows,
|
||||
$"The processed read of '{tag}' covers more than {maxWindowRows} raw samples " +
|
||||
"(ServerHistorian:MaxTimeWeightedRawRows); the Gateway backend evaluates per-sample aggregates from raw reads — " +
|
||||
"narrow the window.");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Builds the per-bucket statistics (<see cref="BucketRow"/>) the SQL backends' aggregate queries produce, from a stream
|
||||
/// of raw samples — the C# statement of <c>SqlAggregateQueries</c>' columns: counts by class (usable = non-Bad with a
|
||||
/// value; good = usable with Good severity; NonNumeric = usable String/DateTime; GoodNonNumeric = good String/DateTime),
|
||||
/// Avg/Min/Max/Sum over the GOOD numeric values and UncertainMin/UncertainMax over the usable-but-not-Good numeric values
|
||||
/// (#684), first/last usable value by time, the extreme GOOD numeric value with its time (ties ⇒ earliest), and the
|
||||
/// first/last sample of any quality.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Keep in step with the SQL backends' per-sample value columns: a change to which samples feed a statistic there
|
||||
/// (e.g. Uncertain handling) must be repeated here, or the Gateway backend silently diverges.
|
||||
/// </remarks>
|
||||
internal sealed class RawBucketAccumulator
|
||||
{
|
||||
private readonly bool _maximum;
|
||||
private readonly Dictionary<long, State> _buckets = new();
|
||||
|
||||
/// <summary>Creates an accumulator.</summary>
|
||||
/// <param name="maximum">Extreme = the maximum (MaximumActualTime); otherwise the minimum.</param>
|
||||
public RawBucketAccumulator(bool maximum) => _maximum = maximum;
|
||||
|
||||
private sealed class State
|
||||
{
|
||||
public int Total, Usable, Good, NonNumeric, GoodNonNumeric, GoodNumeric, UncertainNumeric;
|
||||
public double Sum, Min = double.PositiveInfinity, Max = double.NegativeInfinity;
|
||||
public double UncertainMin = double.PositiveInfinity, UncertainMax = double.NegativeInfinity;
|
||||
public StoredSample? First, Last, FirstAny, LastAny, Extreme;
|
||||
}
|
||||
|
||||
/// <summary>Adds one in-window sample to its bucket. Order-independent (picks compare by time).</summary>
|
||||
/// <param name="bucket">Ascending bucket index (<see cref="BucketPlan.IndexOf"/>).</param>
|
||||
/// <param name="s">The sample.</param>
|
||||
public void Add(long bucket, StoredSample s)
|
||||
{
|
||||
if (!_buckets.TryGetValue(bucket, out var st)) _buckets[bucket] = st = new State();
|
||||
st.Total++;
|
||||
if (st.FirstAny is not { } fa || s.TimeUtc < fa.TimeUtc) st.FirstAny = s;
|
||||
if (st.LastAny is not { } la || s.TimeUtc >= la.TimeUtc) st.LastAny = s;
|
||||
if (!s.IsUsable) return;
|
||||
st.Usable++;
|
||||
if (s.IsGood) st.Good++;
|
||||
if (st.First is not { } f || s.TimeUtc < f.TimeUtc) st.First = s;
|
||||
if (st.Last is not { } l || s.TimeUtc >= l.TimeUtc) st.Last = s;
|
||||
if (!s.Value.TryGetDouble(out var v))
|
||||
{
|
||||
st.NonNumeric++;
|
||||
if (s.IsGood) st.GoodNonNumeric++;
|
||||
return;
|
||||
}
|
||||
if (!s.IsGood)
|
||||
{
|
||||
// #684: an Uncertain value is not part of the value set; it only feeds the Min/Max status rule.
|
||||
st.UncertainNumeric++;
|
||||
if (v < st.UncertainMin) st.UncertainMin = v;
|
||||
if (v > st.UncertainMax) st.UncertainMax = v;
|
||||
return;
|
||||
}
|
||||
st.GoodNumeric++;
|
||||
st.Sum += v;
|
||||
if (v < st.Min) st.Min = v;
|
||||
if (v > st.Max) st.Max = v;
|
||||
if (st.Extreme is not { } e) st.Extreme = s;
|
||||
else
|
||||
{
|
||||
var ev = e.Value.Number;
|
||||
var better = _maximum ? v > ev : v < ev;
|
||||
if (better || (v == ev && s.TimeUtc < e.TimeUtc)) st.Extreme = s;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>The non-empty buckets' rows, ascending by bucket index.</summary>
|
||||
/// <returns>The rows.</returns>
|
||||
public IEnumerable<BucketRow> Rows() =>
|
||||
_buckets.OrderBy(kv => kv.Key).Select(kv =>
|
||||
{
|
||||
var st = kv.Value;
|
||||
var hasNumeric = st.GoodNumeric > 0;
|
||||
var hasUncertain = st.UncertainNumeric > 0;
|
||||
return new BucketRow
|
||||
{
|
||||
Bucket = kv.Key,
|
||||
Total = st.Total,
|
||||
Usable = st.Usable,
|
||||
Good = st.Good,
|
||||
Bad = st.Total - st.Usable,
|
||||
NonNumeric = st.NonNumeric,
|
||||
GoodNonNumeric = st.GoodNonNumeric,
|
||||
Avg = hasNumeric ? st.Sum / st.GoodNumeric : null,
|
||||
Min = hasNumeric ? st.Min : null,
|
||||
Max = hasNumeric ? st.Max : null,
|
||||
Sum = hasNumeric ? st.Sum : null,
|
||||
UncertainMin = hasUncertain ? st.UncertainMin : null,
|
||||
UncertainMax = hasUncertain ? st.UncertainMax : null,
|
||||
First = st.First?.Value ?? HistorianValue.Empty,
|
||||
Last = st.Last?.Value ?? HistorianValue.Empty,
|
||||
Extreme = st.Extreme?.Value.Number,
|
||||
ExtremeTime = st.Extreme?.TimeUtc,
|
||||
FirstAny = st.FirstAny,
|
||||
LastAny = st.LastAny,
|
||||
};
|
||||
});
|
||||
}
|
||||
+5
@@ -11,6 +11,11 @@
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\..\Core\ZB.MOM.WW.OtOpcUa.Core.Abstractions\ZB.MOM.WW.OtOpcUa.Core.Abstractions.csproj"/>
|
||||
<ProjectReference Include="..\..\Core\ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian\ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian.csproj"/>
|
||||
<!-- #681: the per-sample processed aggregates are evaluated from raw reads with the SAME database-independent code the
|
||||
SQL backends use (BucketPlan, BucketRow, AggregateEvaluator), so Gateway matches SqlServer/QuestDB exactly.
|
||||
Sql.Common references only Core.Abstractions + Core.AlarmHistorian (both already referenced here) and no
|
||||
driver, so this adds no cycle. -->
|
||||
<ProjectReference Include="..\ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common\ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common.csproj"/>
|
||||
<!-- Runtime owns ServerHistorianOptions (the bound ServerHistorian config). The read-cutover
|
||||
factory (GatewayHistorian.CreateDataSource / HistorianGatewayClientAdapter.Create) maps
|
||||
those options onto the package client, so the driver references it. Runtime references no
|
||||
|
||||
+120
-41
@@ -9,21 +9,23 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb;
|
||||
/// per <see cref="AggregateFamily"/>, mirroring the SQL Server backend's <c>SqlAggregateQueries</c> family split and
|
||||
/// column order. Every query takes five positional parameters:
|
||||
/// <list type="number">
|
||||
/// <item><c>$1</c> — the (rounded) window start in 100 ns ticks since the Unix epoch
|
||||
/// (<see cref="BucketPlan.StartUnixTicks"/>, <c>Bigint</c>);</item>
|
||||
/// <item><c>$1</c> — the bucket base in 100 ns ticks since the Unix epoch
|
||||
/// (<see cref="BucketPlan.BucketBaseUnixTicks"/>, <c>Bigint</c>: the rounded window start forward; re-anchored at
|
||||
/// the later time for a reverse read, #677);</item>
|
||||
/// <item><c>$2</c> — the bucket width in ticks (<see cref="BucketPlan.IntervalTicks"/>, <c>Bigint</c>);</item>
|
||||
/// <item><c>$3</c> — the tag (<c>Varchar</c>);</item>
|
||||
/// <item><c>$4</c>/<c>$5</c> — the forward window <c>[start, end)</c> (<c>TimestampTz</c> via
|
||||
/// <see cref="QuestDbConnection.BindTimestamp"/>).</item>
|
||||
/// <item><c>$4</c>/<c>$5</c> — the earlier / later window edges (<c>TimestampTz</c> via
|
||||
/// <see cref="QuestDbConnection.BindTimestamp"/>): <c>[$4, $5)</c> forward, <c>($4, $5]</c> reverse.</item>
|
||||
/// </list>
|
||||
/// Each returns one row per <b>non-empty</b> bucket; the caller fills the empty ones.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>Bucketing.</b> <c>b = (cast(ts AS LONG) * 10 − $1) / $2</c>. <c>cast(ts AS LONG)</c> is µs since 1970, so
|
||||
/// <c>× 10</c> is ticks since 1970; both it and <c>$1</c> are µs-aligned (R6), so the integer division is
|
||||
/// start-inclusive / end-exclusive to the tick, exactly like SQL Server's. Every row is at or after the start,
|
||||
/// so the dividend is never negative (no truncation-toward-zero surprise), and nothing overflows before 9999.
|
||||
/// <c>× 10</c> is ticks since 1970, so the integer division is exact to the tick, exactly like SQL Server's
|
||||
/// (<see cref="BucketPlan.IndexOf"/> is the C# statement of the math). Every in-window row is at or after the base
|
||||
/// (forward) or strictly after the window start with the base one tick past the grid origin (reverse), so the
|
||||
/// dividend is never negative (no truncation-toward-zero surprise), and nothing overflows before 9999.
|
||||
/// <c>SAMPLE BY</c> is deliberately not used: its alignment rules differ from ours.
|
||||
/// </para>
|
||||
/// <para>
|
||||
@@ -55,54 +57,101 @@ internal static class QuestDbAggregateQueries
|
||||
private static readonly string EmptyKind = ((byte)HistorianValueKind.Empty).ToString(CultureInfo.InvariantCulture);
|
||||
|
||||
/// <summary>
|
||||
/// Average / Minimum / Maximum / Total / Count / Range / PercentGood / PercentBad.
|
||||
/// Family columns: <c>6 avg_value, 7 min_value, 8 max_value, 9 sum_value</c> (DOUBLE, NULL when no usable sample).
|
||||
/// Average / Minimum / Maximum / Count / Range (PercentGood / PercentBad and Total are time-weighted, not here).
|
||||
/// Family columns: <c>6 avg_value, 7 min_value, 8 max_value, 9 sum_value</c> over the <b>Good</b> numeric samples
|
||||
/// (DOUBLE, NULL when none — #684, Part 13 / the SDK), then <c>10 unc_min, 11 unc_max</c> over the Uncertain ones (the
|
||||
/// Part 13 Minimum/Maximum/Range status rule) and <c>12 good_non_numeric_count</c>.
|
||||
/// </summary>
|
||||
/// <param name="sample">Prefixed sample table name.</param>
|
||||
/// <returns>The SQL text.</returns>
|
||||
public static string Simple(string sample) =>
|
||||
/// <param name="reverse">True for a reverse read (window <c>($4, $5]</c>).</param>
|
||||
public static string Simple(string sample, bool reverse = false) =>
|
||||
"SELECT b, " + CommonCounts() + ", " +
|
||||
"avg(CASE WHEN u = 1 THEN num_value END) AS avg_value, " +
|
||||
"min(CASE WHEN u = 1 THEN num_value END) AS min_value, " +
|
||||
"max(CASE WHEN u = 1 THEN num_value END) AS max_value, " +
|
||||
"sum(CASE WHEN u = 1 THEN num_value END) AS sum_value " +
|
||||
"FROM (" + Classified(sample) + ") GROUP BY b ORDER BY b";
|
||||
"avg(CASE WHEN g = 1 THEN num_value END) AS avg_value, " +
|
||||
"min(CASE WHEN g = 1 THEN num_value END) AS min_value, " +
|
||||
"max(CASE WHEN g = 1 THEN num_value END) AS max_value, " +
|
||||
"sum(CASE WHEN g = 1 THEN num_value END) AS sum_value, " +
|
||||
UncertainExtremes() + " " +
|
||||
"FROM (" + Classified(sample, reverse) + ") GROUP BY b ORDER BY b";
|
||||
|
||||
/// <summary>Ordinal of the StartEnd family's first-sample-of-any-quality columns (<see cref="StartEnd"/>).</summary>
|
||||
public const int FirstAnyColumn = FirstFamilyColumn + 10;
|
||||
|
||||
/// <summary>
|
||||
/// Start / End / Delta: the first and last usable sample of each bucket by <c>ts</c>.
|
||||
/// Family columns: <c>6..10</c> first sample (<c>kind, num_value, int_value, str_value, ts</c>), <c>11..15</c> last
|
||||
/// sample (same order). A bucket without a usable sample has NULL <c>ts</c> but <c>kind = 0</c> (BYTE has no NULL),
|
||||
/// so the reader keys absence on the NULL time.
|
||||
/// Start / End / Delta: the first and last usable sample of each bucket by <c>ts</c> (Delta's value), and the first
|
||||
/// and last sample of <b>any</b> quality (Start / End, and Delta's status — #680).
|
||||
/// Family columns: <c>6..10</c> first usable sample (<c>kind, num_value, int_value, str_value, ts</c>), <c>11..15</c>
|
||||
/// last usable sample (same order) — a bucket without a usable sample has NULL <c>ts</c> but <c>kind = 0</c> (BYTE
|
||||
/// has no NULL), so the reader keys absence on the NULL time; then <c>16..21</c> the first sample of any quality
|
||||
/// (<c>ts, kind, num_value, int_value, str_value, status</c>) and <c>22..27</c> the last (same order), present for
|
||||
/// every returned (non-empty) bucket.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>One scan, one GROUP BY (live-gate 8 F1).</b> The counts and all four picked samples come from a single read
|
||||
/// of the sample table. The first/last sample of any quality are QuestDB's <c>first()</c>/<c>last()</c> (the
|
||||
/// first/last row of the group in designated-timestamp order — null values included, so every column comes from
|
||||
/// the same row); the first/last <i>usable</i> sample are <c>arg_min</c>/<c>arg_max</c> keyed on <c>uts</c>, the
|
||||
/// row's time when usable and NULL otherwise (a NULL key is skipped, a NULL value is returned as is, and
|
||||
/// <c>(ts, tag)</c> is unique — DEDUP — so there are no ties). <c>arg_min</c> has no VARCHAR form, so the usable
|
||||
/// picks carry no <c>str_value</c>: only Delta reads them, and only when the bucket has no usable non-numeric
|
||||
/// sample.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// The previous form referenced its CTE four times (counts + three self-joins); QuestDB opens a separate table
|
||||
/// reader per reference, so a row committed between two of them produced a bucket whose counts included a sample
|
||||
/// its first/last join could not find (a NULL first-of-any-quality time that threw) — ~1 in 90k reads of a tag
|
||||
/// being written. One reference means one reader at one table version. A windowed (<c>row_number()</c>)
|
||||
/// single-scan form was also correct but ~2× slower; this one is not slower than the old query.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// ⚠️ <c>first()</c>/<c>last()</c> following timestamp order is QuestDB 10.0.1 IMPLEMENTATION behaviour (rows reach
|
||||
/// the GROUP BY in row-id order, and the parallel merge keeps row id), not a documented guarantee. It was verified
|
||||
/// on 20M rows with O3 inserts, single-threaded and parallel. Adding a JOIN, <c>ORDER BY</c> or <c>UNION</c> inside
|
||||
/// <c>Classified</c>, or upgrading QuestDB, can silently break Start/End — re-verify <c>first(ts) = min(ts)</c>.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
/// <param name="sample">Prefixed sample table name.</param>
|
||||
/// <returns>The SQL text.</returns>
|
||||
public static string StartEnd(string sample) =>
|
||||
"WITH s AS (" + Classified(sample) + "), " +
|
||||
"st AS (SELECT b, " + CommonCounts() + " FROM s GROUP BY b), " +
|
||||
"f AS (SELECT b, kind, num_value, int_value, str_value, ts, " +
|
||||
"row_number() OVER (PARTITION BY b ORDER BY ts ASC) AS rf, " +
|
||||
"row_number() OVER (PARTITION BY b ORDER BY ts DESC) AS rl FROM s WHERE u = 1) " +
|
||||
"SELECT st.*, " +
|
||||
"x.kind, x.num_value, x.int_value, x.str_value, x.ts, " +
|
||||
"y.kind, y.num_value, y.int_value, y.str_value, y.ts " +
|
||||
"FROM st LEFT JOIN (SELECT * FROM f WHERE rf = 1) x ON x.b = st.b " +
|
||||
"LEFT JOIN (SELECT * FROM f WHERE rl = 1) y ON y.b = st.b ORDER BY st.b";
|
||||
/// <param name="reverse">True for a reverse read (window <c>($4, $5]</c>).</param>
|
||||
public static string StartEnd(string sample, bool reverse = false)
|
||||
{
|
||||
// The usable picks: kind, num_value, int_value, str_value (not available — see remarks), ts.
|
||||
static string Usable(string f, string g) =>
|
||||
$"cast({f}(kind, uts) AS SHORT), {f}(num_value, uts), {f}(int_value, uts), cast(NULL AS VARCHAR), {g}(uts)";
|
||||
// The any-quality picks: ts, kind, num_value, int_value, str_value, status — all from the group's first / last row.
|
||||
static string Any(string f) =>
|
||||
$"{f}(ts), cast({f}(kind) AS SHORT), {f}(num_value), {f}(int_value), {f}(str_value), {f}(status)";
|
||||
return "SELECT b, " + CommonCounts() + ", " +
|
||||
Usable("arg_min", "min") + ", " + Usable("arg_max", "max") + ", " + Any("first") + ", " + Any("last") + " " +
|
||||
"FROM (SELECT b, ts, kind, num_value, int_value, str_value, status, u, g, CASE WHEN u = 1 THEN ts END AS uts " +
|
||||
"FROM (" + Classified(sample, reverse) + ")) GROUP BY b ORDER BY b";
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// MinimumActualTime / MaximumActualTime: the usable sample with the smallest (largest) <c>num_value</c>, ties
|
||||
/// broken by the earliest <c>ts</c>. Family columns: <c>6 num_value, 7 ts</c> (NULL when the bucket has no usable
|
||||
/// numeric sample).
|
||||
/// MinimumActualTime / MaximumActualTime: the <b>Good</b> sample (#684) with the smallest (largest) <c>num_value</c>,
|
||||
/// ties broken by the earliest <c>ts</c>. Family columns: <c>6 num_value, 7 ts</c> (NULL when the bucket has no Good
|
||||
/// numeric sample), then <c>8 unc_min, 9 unc_max, 10 good_non_numeric_count</c> as in <see cref="Simple"/>.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Deliberately still two references to the classified rows (counts + the ranked extreme), unlike
|
||||
/// <see cref="StartEnd"/>: the single-scan <c>row_number()</c> form measured ~2× slower, and QuestDB's
|
||||
/// <c>arg_max</c>/<c>arg_min</c> cannot express the earliest-time tie-break. Live-gate 8 F1's inconsistency (a row
|
||||
/// committed between the two readers — counted but not ranked) is absorbed by the reader and evaluator instead: a
|
||||
/// NULL extreme in a bucket with usable samples evaluates to <c>BadNoData</c>, never a value-less Good.
|
||||
/// </remarks>
|
||||
/// <param name="sample">Prefixed sample table name.</param>
|
||||
/// <param name="maximum">True for MaximumActualTime.</param>
|
||||
/// <returns>The SQL text.</returns>
|
||||
public static string ActualTime(string sample, bool maximum) =>
|
||||
"WITH s AS (" + Classified(sample) + "), " +
|
||||
"st AS (SELECT b, " + CommonCounts() + " FROM s GROUP BY b), " +
|
||||
/// <param name="reverse">True for a reverse read (window <c>($4, $5]</c>).</param>
|
||||
public static string ActualTime(string sample, bool maximum, bool reverse = false) =>
|
||||
"WITH s AS (" + Classified(sample, reverse) + "), " +
|
||||
"st AS (SELECT b, " + CommonCounts() + ", " + UncertainExtremes() + " FROM s GROUP BY b), " +
|
||||
"r AS (SELECT b, num_value, ts, " +
|
||||
"row_number() OVER (PARTITION BY b ORDER BY num_value " + (maximum ? "DESC" : "ASC") + ", ts ASC) AS rn " +
|
||||
"FROM s WHERE u = 1 AND num_value IS NOT NULL) " +
|
||||
"SELECT st.*, r.num_value, r.ts " +
|
||||
"FROM s WHERE g = 1 AND num_value IS NOT NULL) " +
|
||||
"SELECT st.b, st.total_count, st.usable_count, st.good_count, st.bad_count, st.non_numeric_count, " +
|
||||
"r.num_value, r.ts, st.unc_min, st.unc_max, st.good_non_numeric_count " +
|
||||
"FROM st LEFT JOIN (SELECT * FROM r WHERE rn = 1) r ON r.b = st.b ORDER BY st.b";
|
||||
|
||||
/// <summary>
|
||||
@@ -144,12 +193,42 @@ internal static class QuestDbAggregateQueries
|
||||
/// <summary>The bucket-index expression (exposed for tests).</summary>
|
||||
internal const string BucketExpression = "(cast(ts AS LONG) * 10 - $1) / $2";
|
||||
|
||||
/// <summary>
|
||||
/// The tag's earliest live sample (any quality) at or before the window's later edge — half of the tag's data range
|
||||
/// (the start-of-data Partial rule, #677, and Count's in-range empty interval, #680; see
|
||||
/// <see cref="AggregateEvaluator.NeedsDataRange"/>). Parameters <c>$1</c> tag, <c>$2</c> the later edge. Column:
|
||||
/// <c>ts</c>; no row when the tag has none — the data range then does not reach the window.
|
||||
/// </summary>
|
||||
/// <param name="sample">Prefixed sample table name.</param>
|
||||
/// <returns>The SQL text.</returns>
|
||||
public static string EarliestSample(string sample) =>
|
||||
$"SELECT ts FROM {sample} WHERE tag = $1 AND ts <= $2 AND NOT deleted ORDER BY ts ASC LIMIT 1";
|
||||
|
||||
/// <summary>
|
||||
/// The tag's latest live sample (any quality) at or after the window's earlier edge — the other half of the data
|
||||
/// range (the end-of-data Partial rule and Count's in-range empty interval, #680). Parameters <c>$1</c> tag,
|
||||
/// <c>$2</c> the earlier edge. Column: <c>ts</c>; no row when the tag has none.
|
||||
/// </summary>
|
||||
/// <param name="sample">Prefixed sample table name.</param>
|
||||
/// <returns>The SQL text.</returns>
|
||||
public static string LatestSample(string sample) =>
|
||||
$"SELECT ts FROM {sample} WHERE tag = $1 AND ts >= $2 AND NOT deleted ORDER BY ts DESC LIMIT 1";
|
||||
|
||||
/// <summary>The classified subquery <c>s</c>: the window's live samples with bucket index and the usable/good flags.</summary>
|
||||
private static string Classified(string sample) =>
|
||||
"SELECT " + BucketExpression + " AS b, ts, kind, num_value, int_value, str_value, " +
|
||||
private static string Classified(string sample, bool reverse) =>
|
||||
"SELECT " + BucketExpression + " AS b, ts, kind, num_value, int_value, str_value, status, " +
|
||||
$"CASE WHEN status < {NotBadLimit} AND kind <> {EmptyKind} THEN 1 ELSE 0 END AS u, " +
|
||||
$"CASE WHEN status < {GoodLimit} AND kind <> {EmptyKind} THEN 1 ELSE 0 END AS g " +
|
||||
$"FROM {sample} WHERE tag = $3 AND ts >= $4 AND ts < $5 AND NOT deleted";
|
||||
$"FROM {sample} WHERE tag = $3 AND " + (reverse ? "ts > $4 AND ts <= $5" : "ts >= $4 AND ts < $5") + " AND NOT deleted";
|
||||
|
||||
/// <summary>
|
||||
/// The minimum and maximum <b>Uncertain</b> (usable, not Good) numeric value of the bucket — <c>unc_min, unc_max</c>
|
||||
/// (#684). Not result values: Part 13 makes Minimum/Maximum/Range Uncertain when one lies beyond the Good extreme.
|
||||
/// Then <c>good_non_numeric_count</c>, the Good String/DateTime samples (the Good-value aggregates' refusal check).
|
||||
/// </summary>
|
||||
private static string UncertainExtremes() =>
|
||||
"min(CASE WHEN u = 1 AND g = 0 THEN num_value END) AS unc_min, max(CASE WHEN u = 1 AND g = 0 THEN num_value END) AS unc_max, " +
|
||||
$"sum(CASE WHEN g = 1 AND kind IN ({NonNumericKinds}) THEN 1 ELSE 0 END) AS good_non_numeric_count";
|
||||
|
||||
/// <summary>The five common count columns (after <c>b</c>).</summary>
|
||||
private static string CommonCounts() =>
|
||||
|
||||
@@ -201,14 +201,14 @@ internal sealed class QuestDbConnection : IAsyncDisposable, IDisposable
|
||||
/// <param name="r">Reader positioned on a row.</param>
|
||||
/// <param name="i">Column ordinal.</param>
|
||||
/// <returns>The timestamp with <c>Kind=Utc</c>.</returns>
|
||||
public static DateTime ReadUtc(NpgsqlDataReader r, int i) =>
|
||||
public static DateTime ReadUtc(System.Data.Common.DbDataReader r, int i) =>
|
||||
DateTime.SpecifyKind(r.GetDateTime(i), DateTimeKind.Utc);
|
||||
|
||||
/// <summary>Reads a <c>BYTE</c> column (QuestDB returns <c>smallint</c>); NULL (never stored) reads as 0 (Empty).</summary>
|
||||
/// <param name="r">Reader positioned on a row.</param>
|
||||
/// <param name="i">Column ordinal.</param>
|
||||
/// <returns>The byte value.</returns>
|
||||
public static byte ReadKind(NpgsqlDataReader r, int i) => r.IsDBNull(i) ? (byte)0 : (byte)r.GetInt16(i);
|
||||
public static byte ReadKind(System.Data.Common.DbDataReader r, int i) => r.IsDBNull(i) ? (byte)0 : (byte)r.GetInt16(i);
|
||||
|
||||
// ---- error classification --------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
+118
-29
@@ -1,3 +1,4 @@
|
||||
using System.Data.Common;
|
||||
using System.Globalization;
|
||||
using System.Text;
|
||||
using Microsoft.Extensions.Logging;
|
||||
@@ -136,6 +137,7 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource
|
||||
catch (Exception ex)
|
||||
{
|
||||
RecordFailure(ex);
|
||||
if (IsTransportFailure(ex)) throw new System.IO.IOException("Historian read transport is unavailable.", ex);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
@@ -231,6 +233,7 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource
|
||||
catch (Exception ex)
|
||||
{
|
||||
RecordFailure(ex);
|
||||
if (IsTransportFailure(ex)) throw new System.IO.IOException("Historian read transport is unavailable.", ex);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
@@ -315,6 +318,7 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource
|
||||
catch (Exception ex)
|
||||
{
|
||||
RecordFailure(ex);
|
||||
if (IsTransportFailure(ex)) throw new System.IO.IOException("Historian read transport is unavailable.", ex);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
@@ -354,6 +358,23 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource
|
||||
|
||||
// ------------------------------------------------------------------------------------------ Events
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>Reverse raw reads are served natively (<c>(end, start]</c>, <c>ORDER BY … DESC</c>; #669/#686).</remarks>
|
||||
public bool SupportsReverseRaw => true;
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks><c>0</c>: a capped raw read is one bounded statement — no backend row ceiling (Gitea #694).</remarks>
|
||||
public int MaxRawValuesPerRead => 0;
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>Reverse events reads are served natively (<c>ORDER BY … DESC</c>, Gitea #675).</remarks>
|
||||
public bool SupportsReverseEvents => true;
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>The seventeen <see cref="AggregateFamilies.Served"/> — shared with the other SQL backend (#682).</remarks>
|
||||
public IReadOnlySet<HistoryAggregateType> SupportedAggregates => AggregateFamilies.Served;
|
||||
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>
|
||||
/// <c>[start, end)</c> by event time (a reversed window is swapped), ordered by <c>(ts, event_id)</c> ascending.
|
||||
@@ -361,8 +382,18 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource
|
||||
/// <c>maxEvents + 1</c> rows are fetched; when the extra row arrives it is dropped and the continuation point is
|
||||
/// the UTF-8 of <see cref="EventContinuation.Encode"/> for that first not-returned event.
|
||||
/// </remarks>
|
||||
public Task<HistoricalEventsResult> ReadEventsAsync(
|
||||
string? sourceName, DateTime startUtc, DateTime endUtc, int maxEvents, CancellationToken cancellationToken) =>
|
||||
ReadEventsAsync(sourceName, startUtc, endUtc, maxEvents, reverse: false, cancellationToken);
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>
|
||||
/// Forward is the five-argument read. Reverse (Gitea #675) is the (rounded) <c>(lo, hi]</c> ordered by
|
||||
/// <c>(ts, event_id)</c> DESCENDING, so <c>LIMIT</c> keeps the latest events; truncation and the continuation point
|
||||
/// work as forward.
|
||||
/// </remarks>
|
||||
public async Task<HistoricalEventsResult> ReadEventsAsync(
|
||||
string? sourceName, DateTime startUtc, DateTime endUtc, int maxEvents, CancellationToken cancellationToken)
|
||||
string? sourceName, DateTime startUtc, DateTime endUtc, int maxEvents, bool reverse, CancellationToken cancellationToken)
|
||||
{
|
||||
// No stored event can carry a source longer than the limit, so an over-long filter matches nothing.
|
||||
if (sourceName is not null && sourceName.Length > HistorianTagNames.MaxLength)
|
||||
@@ -376,9 +407,9 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource
|
||||
var bounded = maxEvents > 0 && maxEvents < int.MaxValue;
|
||||
var sql =
|
||||
$"SELECT event_id, source, ts, received_utc, message, severity FROM {_conn.EventTable} " +
|
||||
"WHERE ts >= $1 AND ts < $2" +
|
||||
(reverse ? "WHERE ts > $1 AND ts <= $2" : "WHERE ts >= $1 AND ts < $2") +
|
||||
(sourceName is null ? "" : " AND source = $3") +
|
||||
" ORDER BY ts ASC, event_id ASC" +
|
||||
(reverse ? " ORDER BY ts DESC, event_id DESC" : " ORDER BY ts ASC, event_id ASC") +
|
||||
(bounded ? (sourceName is null ? " LIMIT $3" : " LIMIT $4") : "");
|
||||
|
||||
await using var conn = await _conn.OpenAsync(cancellationToken).ConfigureAwait(false);
|
||||
@@ -415,6 +446,7 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource
|
||||
catch (Exception ex)
|
||||
{
|
||||
RecordFailure(ex);
|
||||
if (IsTransportFailure(ex)) throw new System.IO.IOException("Historian read transport is unavailable.", ex);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
@@ -425,12 +457,15 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource
|
||||
/// <remarks>
|
||||
/// The (rounded) window is planned by <see cref="BucketPlan.Create"/>: bucket <c>i</c> =
|
||||
/// <c>[start + i·interval, start + (i+1)·interval)</c>, start-inclusive / end-exclusive, the last bucket truncated
|
||||
/// at <c>end</c>, interval <= 0 ⇒ one bucket, <c>start == end</c> ⇒ none, a reversed window computed forward
|
||||
/// and returned descending, at most <see cref="BucketPlan.MaxBuckets"/> buckets. One statement per node returns
|
||||
/// the <see cref="BucketRow"/> statistics of each non-empty bucket (bucket index
|
||||
/// <c>(cast(ts AS LONG)*10 − StartUnixTicks) / IntervalTicks</c>, never <c>SAMPLE BY</c>);
|
||||
/// <see cref="AggregateEvaluator.Evaluate"/> produces each value and <see cref="BucketPlan.Assemble"/> fills the
|
||||
/// empty buckets with <c>BadNoData</c>. The time-weighted family (#671) instead streams the window plus four bounding
|
||||
/// at <c>end</c>, interval <= 0 ⇒ one bucket, <c>start == end</c> ⇒ none, at most
|
||||
/// <see cref="BucketPlan.MaxBuckets"/> buckets. A reversed window (#677) is anchored at its later time per Part 13 /
|
||||
/// the SDK: window <c>(end, start]</c>, buckets stepping back from <c>start</c>, the partial one earliest, each stamped
|
||||
/// with its later edge, returned latest first. One statement per node returns the <see cref="BucketRow"/> statistics
|
||||
/// of each non-empty bucket (bucket index <c>(cast(ts AS LONG)*10 − BucketBaseUnixTicks) / IntervalTicks</c>, never
|
||||
/// <c>SAMPLE BY</c>); before it, when <see cref="AggregateEvaluator.NeedsDataRange"/>, two tag-index seeks fetch the
|
||||
/// tag's data range (earliest / latest live sample) for the start/end-of-data Partial bits and Count's in-range
|
||||
/// empty interval; and
|
||||
/// <see cref="AggregateEvaluator.EvaluateAll"/> produces the values and fills the empty buckets with <c>BadNoData</c>. The time-weighted family (#671) instead streams the window plus four bounding
|
||||
/// rows into <see cref="TimeWeightedAggregator"/>, exactly as the SQL Server backend does.
|
||||
/// </remarks>
|
||||
/// <exception cref="HistoryAggregateNotSupportedException">The aggregate is not one of the seventeen served.</exception>
|
||||
@@ -446,7 +481,6 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource
|
||||
try
|
||||
{
|
||||
var now = _time.GetUtcNow().UtcDateTime;
|
||||
var evaluated = new Dictionary<long, DataValueSnapshot>();
|
||||
if (family == AggregateFamily.TimeWeighted)
|
||||
{
|
||||
var tw = new TimeWeightedAggregator(aggregate, plan);
|
||||
@@ -456,31 +490,46 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource
|
||||
RecordSuccess();
|
||||
return new HistoryReadResult(twResult, ContinuationPoint: null);
|
||||
}
|
||||
var rows = new List<BucketRow>();
|
||||
DateTime? earliest = null, latest = null;
|
||||
if (plan.BucketCount > 0 && await ResolveAsync(fullReference, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
var sample = _conn.SampleTable;
|
||||
var sql = family switch
|
||||
{
|
||||
AggregateFamily.Simple => QuestDbAggregateQueries.Simple(sample),
|
||||
AggregateFamily.StartEnd => QuestDbAggregateQueries.StartEnd(sample),
|
||||
_ => QuestDbAggregateQueries.ActualTime(sample, aggregate == HistoryAggregateType.MaximumActualTime),
|
||||
AggregateFamily.Simple => QuestDbAggregateQueries.Simple(sample, plan.Reversed),
|
||||
AggregateFamily.StartEnd => QuestDbAggregateQueries.StartEnd(sample, plan.Reversed),
|
||||
_ => QuestDbAggregateQueries.ActualTime(sample, aggregate == HistoryAggregateType.MaximumActualTime, plan.Reversed),
|
||||
};
|
||||
await using var conn = await _conn.OpenAsync(cancellationToken).ConfigureAwait(false);
|
||||
await using var cmd = _conn.CreateCommand(conn, sql);
|
||||
QuestDbConnection.BindLong(cmd, plan.StartUnixTicks);
|
||||
QuestDbConnection.BindLong(cmd, plan.IntervalTicks);
|
||||
QuestDbConnection.BindText(cmd, fullReference);
|
||||
QuestDbConnection.BindTimestamp(cmd, plan.StartUtc);
|
||||
QuestDbConnection.BindTimestamp(cmd, plan.EndUtc);
|
||||
await using var reader = await cmd.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false);
|
||||
while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false))
|
||||
// The tag's data range (earliest + latest live sample): the start/end-of-data Partial rules (#677/#680 —
|
||||
// any per-sample aggregate but Average, either direction) and Count's in-range empty interval (#680); skipped
|
||||
// otherwise. Index seeks on tag (#683). Read BEFORE the bucket statement, which is a separate table reader:
|
||||
// a sample appended in between (the common race — a live tag read up to now) then reaches the buckets but
|
||||
// not the range — at worst a stale end-of-data Partial or a BadNoData where a 0 belongs — instead of a
|
||||
// range covering a bucket the rows do not show, which reported a Good Count of 0 for an interval that has
|
||||
// samples. (A concurrent HistoryUpdate delete is the mirror case and can still do that, transiently.)
|
||||
if (AggregateEvaluator.NeedsDataRange(aggregate, plan))
|
||||
{
|
||||
var row = ReadBucket(reader, family);
|
||||
evaluated[row.Bucket] = AggregateEvaluator.Evaluate(aggregate, row, plan.BucketStart(row.Bucket), now);
|
||||
earliest = await ReadBoundAsync(conn, QuestDbAggregateQueries.EarliestSample(sample), fullReference,
|
||||
plan.EndUtc, cancellationToken).ConfigureAwait(false);
|
||||
latest = await ReadBoundAsync(conn, QuestDbAggregateQueries.LatestSample(sample), fullReference,
|
||||
plan.StartUtc, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
await using (var cmd = _conn.CreateCommand(conn, sql))
|
||||
{
|
||||
QuestDbConnection.BindLong(cmd, plan.BucketBaseUnixTicks);
|
||||
QuestDbConnection.BindLong(cmd, plan.IntervalTicks);
|
||||
QuestDbConnection.BindText(cmd, fullReference);
|
||||
QuestDbConnection.BindTimestamp(cmd, plan.StartUtc);
|
||||
QuestDbConnection.BindTimestamp(cmd, plan.EndUtc);
|
||||
await using var reader = await cmd.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false);
|
||||
while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false))
|
||||
rows.Add(ReadBucket(reader, family));
|
||||
}
|
||||
}
|
||||
|
||||
var result = plan.Assemble(evaluated, bucketStart => AggregateEvaluator.NoData(bucketStart, now));
|
||||
var result = AggregateEvaluator.EvaluateAll(aggregate, plan, rows, earliest, latest, now);
|
||||
|
||||
RecordSuccess();
|
||||
return new HistoryReadResult(result, ContinuationPoint: null);
|
||||
@@ -493,6 +542,7 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource
|
||||
catch (Exception ex)
|
||||
{
|
||||
RecordFailure(ex);
|
||||
if (IsTransportFailure(ex)) throw new System.IO.IOException("Historian read transport is unavailable.", ex);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
@@ -538,12 +588,32 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource
|
||||
if (bounds[3] is { } usableAfter) tw.Add(usableAfter);
|
||||
}
|
||||
|
||||
private static BucketRow ReadBucket(NpgsqlDataReader reader, AggregateFamily family)
|
||||
/// <summary>Runs a one-column <c>ts</c> seek (<see cref="QuestDbAggregateQueries.EarliestSample"/> / <c>LatestSample</c>).</summary>
|
||||
private async Task<DateTime?> ReadBoundAsync(NpgsqlConnection conn, string sql, string tag, DateTime edge, CancellationToken ct)
|
||||
{
|
||||
static double? D(NpgsqlDataReader r, int i) => r.IsDBNull(i) ? null : r.GetDouble(i);
|
||||
static int N(NpgsqlDataReader r, int i) => r.IsDBNull(i) ? 0 : Convert.ToInt32(r.GetValue(i), CultureInfo.InvariantCulture);
|
||||
await using var cmd = _conn.CreateCommand(conn, sql);
|
||||
QuestDbConnection.BindText(cmd, tag);
|
||||
QuestDbConnection.BindTimestamp(cmd, edge);
|
||||
await using var reader = await cmd.ExecuteReaderAsync(ct).ConfigureAwait(false);
|
||||
return await reader.ReadAsync(ct).ConfigureAwait(false) && !reader.IsDBNull(0) ? QuestDbConnection.ReadUtc(reader, 0) : null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Maps one per-sample aggregate row (<see cref="QuestDbAggregateQueries"/> column layout) to a
|
||||
/// <see cref="BucketRow"/>. Every picked sample — first/last usable, first/last of any quality, the extreme — is
|
||||
/// optional: a NULL time reads as "no such sample", never an exception (live-gate 8 F1: a row whose counts and
|
||||
/// picks disagreed used to throw out of <see cref="ReadStored"/> and fail the node's whole read; the evaluator
|
||||
/// answers such a bucket from what is present, or <c>BadNoData</c>).
|
||||
/// </summary>
|
||||
/// <param name="reader">Reader positioned on a bucket row.</param>
|
||||
/// <param name="family">The query's aggregate family.</param>
|
||||
/// <returns>The bucket statistics.</returns>
|
||||
internal static BucketRow ReadBucket(DbDataReader reader, AggregateFamily family)
|
||||
{
|
||||
static double? D(DbDataReader r, int i) => r.IsDBNull(i) ? null : r.GetDouble(i);
|
||||
static int N(DbDataReader r, int i) => r.IsDBNull(i) ? 0 : Convert.ToInt32(r.GetValue(i), CultureInfo.InvariantCulture);
|
||||
// A bucket without a usable sample has NULL times but kind 0 (BYTE has no NULL): absence is keyed on the time.
|
||||
static HistorianValue V(NpgsqlDataReader r, int i) =>
|
||||
static HistorianValue V(DbDataReader r, int i) =>
|
||||
r.IsDBNull(i + 4)
|
||||
? HistorianValue.Empty
|
||||
: ToValue(QuestDbConnection.ReadKind(r, i), D(r, i + 1), r.IsDBNull(i + 2) ? null : r.GetInt64(i + 2),
|
||||
@@ -566,14 +636,22 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource
|
||||
row.Min = D(reader, f + 1);
|
||||
row.Max = D(reader, f + 2);
|
||||
row.Sum = D(reader, f + 3);
|
||||
row.UncertainMin = D(reader, f + 4);
|
||||
row.UncertainMax = D(reader, f + 5);
|
||||
row.GoodNonNumeric = N(reader, f + 6);
|
||||
break;
|
||||
case AggregateFamily.StartEnd:
|
||||
row.First = V(reader, f);
|
||||
row.Last = V(reader, f + 5);
|
||||
row.FirstAny = ReadStoredOrNull(reader, QuestDbAggregateQueries.FirstAnyColumn);
|
||||
row.LastAny = ReadStoredOrNull(reader, QuestDbAggregateQueries.FirstAnyColumn + 6);
|
||||
break;
|
||||
default:
|
||||
row.Extreme = D(reader, f);
|
||||
row.ExtremeTime = reader.IsDBNull(f + 1) ? null : QuestDbConnection.ReadUtc(reader, f + 1);
|
||||
row.UncertainMin = D(reader, f + 2);
|
||||
row.UncertainMax = D(reader, f + 3);
|
||||
row.GoodNonNumeric = N(reader, f + 4);
|
||||
break;
|
||||
}
|
||||
return row;
|
||||
@@ -585,7 +663,7 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource
|
||||
/// Reads <c>ts, kind, num_value, int_value, str_value, status</c> starting at <paramref name="o"/> and applies the
|
||||
/// shared quality clamp (<see cref="StoredSample.FromStored"/>).
|
||||
/// </summary>
|
||||
private static StoredSample ReadStored(NpgsqlDataReader r, int o)
|
||||
private static StoredSample ReadStored(DbDataReader r, int o)
|
||||
{
|
||||
var value = ToValue(
|
||||
QuestDbConnection.ReadKind(r, o + 1),
|
||||
@@ -597,6 +675,12 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource
|
||||
return StoredSample.FromStored(QuestDbConnection.ReadUtc(r, o), value, status);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// <see cref="ReadStored"/> for an optional picked sample: a NULL <c>ts</c> means the row carries no such sample
|
||||
/// (live-gate 8 F1), so the result is null rather than an <see cref="InvalidCastException"/>.
|
||||
/// </summary>
|
||||
private static StoredSample? ReadStoredOrNull(DbDataReader r, int o) => r.IsDBNull(o) ? null : ReadStored(r, o);
|
||||
|
||||
/// <summary>
|
||||
/// Rebuilds a value from its stored columns via the shared <see cref="SampleColumnMapper.FromColumns"/>, after the
|
||||
/// RC2 repair: QuestDB reads a <c>LONG</c> holding <see cref="long.MinValue"/> back as NULL, so an Int64 row with
|
||||
@@ -622,6 +706,11 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource
|
||||
|
||||
private void RecordSuccess() => _health.RecordSuccess();
|
||||
|
||||
/// <summary>Classifies provider transport/timeouts, keeping SQL/query errors distinct (#697).</summary>
|
||||
private static bool IsTransportFailure(Exception ex) => ex is NpgsqlException { IsTransient: true } and not PostgresException
|
||||
|| ex is PostgresException pg && (pg.SqlState.StartsWith("08", StringComparison.Ordinal)
|
||||
|| pg.SqlState is "57P01" or "57P02" or "57P03");
|
||||
|
||||
private void RecordFailure(Exception ex)
|
||||
{
|
||||
// A cancelled read (deadline / client abort) is not a backend fault (the counters ignore it too).
|
||||
|
||||
+53
-2
@@ -1,3 +1,4 @@
|
||||
using System.Globalization;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Npgsql;
|
||||
@@ -15,10 +16,17 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb;
|
||||
/// <item><b>TTL (RC3):</b> <c>tables()</c> normalises units (7 days reads back as <c>1 WEEK</c>) so values are compared
|
||||
/// in hours, and an ALTER becomes visible only after the WAL applies it, so the bootstrapper never re-reads to verify
|
||||
/// its own ALTER. A boot whose TTL already matches issues no ALTER.</item>
|
||||
/// <item><b>Tag index (#683):</b> a sample table created before the <c>tag</c> symbol index existed gains it with
|
||||
/// <see cref="QuestDbTables.AddTagIndex"/> (idempotent; a boot that finds it indexed issues no ALTER, and like the TTL
|
||||
/// the ALTER is a WAL transaction the bootstrapper does not re-read). Building it is a one-off WAL apply — ~3 s for
|
||||
/// 68M rows / 63 daily partitions measured — during which writes queue and reads keep scanning. A refused ALTER is
|
||||
/// logged as a Warning, never fails the bootstrap (the index is optional); a failure at WAL-apply time would suspend
|
||||
/// the table unseen, the same blind spot as the TTL ALTER.</item>
|
||||
/// <item><b>Failure:</b> not latched. Within <see cref="BootstrapFailureCache.DefaultDuration"/> the last failure is
|
||||
/// re-thrown without touching QuestDB.</item>
|
||||
/// <item><c>AutoCreateSchema=false</c> only verifies the tables exist (no TTL changes) and throws
|
||||
/// <see cref="HistorianSchemaMissingException"/> otherwise.</item>
|
||||
/// <see cref="HistorianSchemaMissingException"/> otherwise; a missing tag index is only logged (a Warning naming
|
||||
/// the ALTER to run), since reads stay correct without it.</item>
|
||||
/// </list>
|
||||
/// </summary>
|
||||
internal sealed class QuestDbSchemaBootstrapper
|
||||
@@ -63,10 +71,12 @@ internal sealed class QuestDbSchemaBootstrapper
|
||||
{
|
||||
await CreateTablesAsync(conn, ct).ConfigureAwait(false);
|
||||
await ReconcileTtlAsync(conn, ct).ConfigureAwait(false);
|
||||
await EnsureTagIndexAsync(conn, apply: true, ct).ConfigureAwait(false);
|
||||
}
|
||||
else
|
||||
{
|
||||
await VerifyAsync(conn, ct).ConfigureAwait(false);
|
||||
await EnsureTagIndexAsync(conn, apply: false, ct).ConfigureAwait(false);
|
||||
}
|
||||
_ready = true;
|
||||
_failure.Clear();
|
||||
@@ -117,6 +127,44 @@ internal sealed class QuestDbSchemaBootstrapper
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// #683: indexes the sample table's <c>tag</c> column when it is not (a table created before the index existed).
|
||||
/// <paramref name="apply"/> false (AutoCreateSchema off) only logs the statement to run.
|
||||
/// </summary>
|
||||
private async Task EnsureTagIndexAsync(NpgsqlConnection conn, bool apply, CancellationToken ct)
|
||||
{
|
||||
var table = _conn.SampleTable;
|
||||
await using (var cmd = _conn.CreateCommand(conn, QuestDbTables.TagIndexedQuery(table)))
|
||||
{
|
||||
var indexed = await cmd.ExecuteScalarAsync(ct).ConfigureAwait(false);
|
||||
if (indexed is null or DBNull || Convert.ToBoolean(indexed, CultureInfo.InvariantCulture)) return;
|
||||
}
|
||||
if (!apply)
|
||||
{
|
||||
_log.LogWarning("QuestDB historian table {Table} has no index on its tag column, so the data-range lookups of " +
|
||||
"processed reads scan partitions instead of seeking (#683). AutoCreateSchema is false; run: {Statement}",
|
||||
table, QuestDbTables.AddTagIndex(table));
|
||||
return;
|
||||
}
|
||||
// A WAL transaction: table_columns() keeps reporting false until it applies, so do NOT re-read to verify.
|
||||
// The index is optional (reads are correct without it), so a refused ALTER (e.g. no DDL rights) must not fail the
|
||||
// bootstrap: it is logged and retried on the next boot. A failure at WAL-APPLY time is not seen here (QuestDB would
|
||||
// suspend the table) — the same blind spot as the TTL ALTER.
|
||||
try
|
||||
{
|
||||
await using var alter = _conn.CreateCommand(conn, QuestDbTables.AddTagIndex(table));
|
||||
await alter.ExecuteNonQueryAsync(ct).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex) when (ex is not OperationCanceledException || !ct.IsCancellationRequested)
|
||||
{
|
||||
_log.LogWarning("Could not add the tag index to QuestDB table {Table} ({Exception}: {Message}); data-range lookups " +
|
||||
"of processed reads scan partitions instead of seeking (#683). Run: {Statement}",
|
||||
table, ex.GetType().Name, ex.Message, QuestDbTables.AddTagIndex(table));
|
||||
return;
|
||||
}
|
||||
_log.LogInformation("Added the tag index to QuestDB table {Table} (#683).", table);
|
||||
}
|
||||
|
||||
private async Task ReconcileTtlAsync(NpgsqlConnection conn, CancellationToken ct)
|
||||
{
|
||||
var wantHours = _options.RetentionDays is { } d ? d * 24L : 0L;
|
||||
@@ -137,7 +185,10 @@ internal sealed class QuestDbSchemaBootstrapper
|
||||
// The ALTER is a WAL transaction: tables() keeps the old value until it applies, so do NOT re-read to verify.
|
||||
await using (var alter = _conn.CreateCommand(conn, QuestDbTables.AlterTtl(table, _options.RetentionDays)))
|
||||
await alter.ExecuteNonQueryAsync(ct).ConfigureAwait(false);
|
||||
_log.LogInformation("Applied TTL {Days} day(s) to QuestDB table {Table}.", _options.RetentionDays, table);
|
||||
if (_options.RetentionDays is { } days)
|
||||
_log.LogInformation("Applied TTL {Days} day(s) to QuestDB table {Table}.", days, table);
|
||||
else
|
||||
_log.LogInformation("Cleared TTL on QuestDB table {Table}.", table);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -39,9 +39,15 @@ internal static class QuestDbTables
|
||||
/// </summary>
|
||||
public static string[] Expiring(string prefix) => [Sample(prefix), Event(prefix), SampleModified(prefix)];
|
||||
|
||||
/// <summary><c>CREATE TABLE IF NOT EXISTS</c> for the sample table.</summary>
|
||||
/// <summary>
|
||||
/// <c>CREATE TABLE IF NOT EXISTS</c> for the sample table. <c>tag</c> carries a symbol (bitmap) index (#683): every
|
||||
/// read filters on one tag, and without the index QuestDB answers a <c>tag = x</c> seek by scanning the tag column of
|
||||
/// every partition in the time range — the data-range lookups (<see cref="QuestDbAggregateQueries.EarliestSample"/> /
|
||||
/// <c>LatestSample</c>) of a stopped or late-starting tag then scan every partition on one side of the window.
|
||||
/// A table created before #683 gains the index through <see cref="AddTagIndex"/>.
|
||||
/// </summary>
|
||||
public static string CreateSample(string prefix, int? retentionDays) =>
|
||||
$"CREATE TABLE IF NOT EXISTS {Sample(prefix)} (ts TIMESTAMP, tag SYMBOL, kind BYTE, num_value DOUBLE, " +
|
||||
$"CREATE TABLE IF NOT EXISTS {Sample(prefix)} (ts TIMESTAMP, tag SYMBOL INDEX, kind BYTE, num_value DOUBLE, " +
|
||||
"int_value LONG, str_value VARCHAR, status LONG, deleted BOOLEAN, server_ts TIMESTAMP) " +
|
||||
$"TIMESTAMP(ts) PARTITION BY DAY{Ttl(retentionDays)} WAL DEDUP UPSERT KEYS(ts, tag)";
|
||||
|
||||
@@ -74,6 +80,21 @@ internal static class QuestDbTables
|
||||
[CreateSample(prefix, retentionDays), CreateEvent(prefix, retentionDays), CreateTag(prefix),
|
||||
CreateSampleModified(prefix, retentionDays)];
|
||||
|
||||
/// <summary>
|
||||
/// <c>ALTER TABLE t ALTER COLUMN tag ADD INDEX</c> — the #683 migration of a sample table created without the tag
|
||||
/// index. A WAL transaction (visible once applied); re-issuing it on an indexed column is a harmless no-op (the table
|
||||
/// is not suspended), so two nodes booting together may both send it.
|
||||
/// </summary>
|
||||
public static string AddTagIndex(string table) => $"ALTER TABLE {table} ALTER COLUMN tag ADD INDEX";
|
||||
|
||||
/// <summary>
|
||||
/// Whether <paramref name="table"/>'s <c>tag</c> column is indexed: one row, one BOOLEAN column, no row when the
|
||||
/// table or column is missing. The name is inlined (<c>table_columns()</c> takes a constant); it is always a
|
||||
/// prefix-validated name built here.
|
||||
/// </summary>
|
||||
public static string TagIndexedQuery(string table) =>
|
||||
$"SELECT indexed FROM table_columns('{table}') WHERE \"column\" = 'tag'";
|
||||
|
||||
/// <summary><c>ALTER TABLE t SET TTL n DAYS</c>; <c>null</c> removes the TTL (<c>SET TTL 0 DAYS</c>).</summary>
|
||||
public static string AlterTtl(string table, int? retentionDays) =>
|
||||
$"ALTER TABLE {table} SET TTL {(retentionDays ?? 0).ToString(CultureInfo.InvariantCulture)} DAYS";
|
||||
|
||||
@@ -7,12 +7,32 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common;
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// Per interval: no samples ⇒ <c>BadNoData</c>. PercentGood / PercentBad ⇒ <c>100 × good|bad / all</c> over
|
||||
/// every sample, status Good. Otherwise no usable sample ⇒ <c>BadNoData</c>; a numeric aggregate with a usable
|
||||
/// String/DateTime sample ⇒ <c>BadAggregateInvalidInputs</c>; else the value, Good when every sample is Good,
|
||||
/// <c>UncertainDataSubNormal</c> when any is Bad or Uncertain. Every status — including <c>BadNoData</c> —
|
||||
/// carries the Calculated info bits. Timestamps are the bucket start, except Minimum/MaximumActualTime
|
||||
/// (the sample's time; ties ⇒ earliest).
|
||||
/// Per interval: no samples ⇒ <c>BadNoData</c> (Count: 0 inside the data range). Otherwise no usable sample ⇒
|
||||
/// <c>BadNoData</c>; a numeric aggregate with a usable String/DateTime sample ⇒ <c>BadAggregateInvalidInputs</c>;
|
||||
/// else the value. #684: Average / Minimum / Maximum / Range / *ActualTime are computed over <b>Good</b> values only
|
||||
/// (Part 13 and the SDK — an Uncertain sample is never part of the value; an interval with no Good value is
|
||||
/// <c>BadNoData</c>). Status: Average value-based (Good only when every sample is Good, else
|
||||
/// <c>UncertainDataSubNormal</c>); Minimum / Maximum / Range / *ActualTime per their Part 13 tables — Uncertain when a
|
||||
/// Bad sample exists or an Uncertain value lies beyond the reported extreme (<see cref="MinMaxStatus"/>).
|
||||
/// Every calculated status — including <c>BadNoData</c> — carries the Calculated info bits. Timestamps are the
|
||||
/// bucket's <see cref="BucketPlan.BucketTimestamp"/> (its start forward, its later edge reverse), except
|
||||
/// Minimum/MaximumActualTime (the sample's time; ties ⇒ earliest) and Start/End. Every aggregate but
|
||||
/// <c>Average</c> also carries the Partial bit on the bucket the data starts (forward) or ends inside
|
||||
/// (<see cref="EvaluateAll"/>). PercentGood / PercentBad are duration-based since #680 and evaluated by
|
||||
/// <see cref="TimeWeightedAggregator"/>, not here.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// #680 (Part 13 / the SDK's calculators): <b>Count</b> counts Good samples only, value-based status (all Good ⇒
|
||||
/// Good, all Bad ⇒ the generic Bad, else Uncertain), and is 0/Good on an empty bucket inside the tag's data range;
|
||||
/// <b>Start / End</b> are the raw first / last sample of any quality (own value — null when Bad —, own status
|
||||
/// without the Calculated bit, own timestamp); <b>Delta</b> is Uncertain only when the bucket's first or last
|
||||
/// sample is not usable.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Inconsistent rows</b> (live-gate 8 F1): a row whose counts say a bucket holds samples while a picked sample
|
||||
/// (first/last of any quality, first/last usable, the extreme) is absent is answered from what is present —
|
||||
/// Start/End/Delta/ActualTime without the sample they need are <c>BadNoData</c> — never an exception. The backends'
|
||||
/// queries read counts and picks in one scan so this should not arise; the evaluator does not rely on it.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Values: Count is <see cref="int"/>; Start/End keep the sample's type; every other aggregate is
|
||||
@@ -26,7 +46,7 @@ public static class AggregateEvaluator
|
||||
/// <summary>Evaluates <paramref name="aggregate"/> over one non-empty bucket.</summary>
|
||||
/// <param name="aggregate">Requested aggregate.</param>
|
||||
/// <param name="r">The bucket's statistics.</param>
|
||||
/// <param name="bucketStart">The bucket's start time (the result timestamp).</param>
|
||||
/// <param name="bucketStart">The bucket's result timestamp (<see cref="BucketPlan.BucketTimestamp"/>).</param>
|
||||
/// <param name="now">Server timestamp.</param>
|
||||
/// <returns>The calculated value.</returns>
|
||||
/// <exception cref="HistoryAggregateNotSupportedException">The aggregate is not one of the per-sample (Simple /
|
||||
@@ -36,40 +56,168 @@ public static class AggregateEvaluator
|
||||
DataValueSnapshot Calc(object? value, uint status, DateTime? t = null) =>
|
||||
new(value, HistorianInfoBits.WithCalculated(status), t ?? bucketStart, now);
|
||||
|
||||
// Refuse a non-per-sample aggregate up front, whatever the bucket holds (an all-Bad bucket would otherwise answer
|
||||
// BadNoData before the switch's default is reached).
|
||||
if (AggregateFamilies.FamilyOf(aggregate) is not (AggregateFamily.Simple or AggregateFamily.StartEnd or AggregateFamily.ActualTime))
|
||||
throw new HistoryAggregateNotSupportedException(aggregate);
|
||||
if (r.Total == 0) return NoData(bucketStart, now);
|
||||
switch (aggregate)
|
||||
{
|
||||
case HistoryAggregateType.PercentGood: return Calc(100d * r.Good / r.Total, HistorianStatusCodes.Good);
|
||||
case HistoryAggregateType.PercentBad: return Calc(100d * r.Bad / r.Total, HistorianStatusCodes.Good);
|
||||
// #680 (Part 13 Tables 73/74; SDK ComputeStartEnd): the raw first / last sample of ANY quality — its own value
|
||||
// (null when Bad), its own status (Raw: no Calculated bit) and its own timestamp.
|
||||
case HistoryAggregateType.Start: return Raw(r.FirstAny, bucketStart, now);
|
||||
case HistoryAggregateType.End: return Raw(r.LastAny, bucketStart, now);
|
||||
// #680 (Part 13 §5.4.3.21; SDK ComputeCount): Good samples only; value-based status — Good when every sample
|
||||
// is Good, the generic Bad when every one is Bad (value null), else Uncertain.
|
||||
case HistoryAggregateType.Count:
|
||||
if (r.Bad == r.Total) return Calc(null, HistorianStatusCodes.Bad);
|
||||
return Calc(r.Good, r.Good == r.Total ? HistorianStatusCodes.Good : HistorianStatusCodes.UncertainDataSubNormal);
|
||||
}
|
||||
if (r.Usable == 0) return NoData(bucketStart, now);
|
||||
|
||||
var anyText = aggregate is HistoryAggregateType.Count or HistoryAggregateType.Start or HistoryAggregateType.End;
|
||||
if (!anyText && r.NonNumeric > 0)
|
||||
// #684 review: Delta reads usable picks, so any usable String/DateTime sample refuses it. The Good-value aggregates
|
||||
// refuse only a GOOD non-numeric sample — an Uncertain one is not part of their value set (it only degrades the
|
||||
// status, as the SDK's skipped/uncastable value does).
|
||||
if ((aggregate == HistoryAggregateType.Delta ? r.NonNumeric : r.GoodNonNumericOrAll) > 0)
|
||||
return Calc(null, HistorianStatusCodes.BadAggregateInvalidInputs);
|
||||
|
||||
var status = r.Good == r.Total ? HistorianStatusCodes.Good : HistorianStatusCodes.UncertainDataSubNormal;
|
||||
switch (aggregate)
|
||||
{
|
||||
case HistoryAggregateType.Count: return Calc(r.Usable, status);
|
||||
case HistoryAggregateType.Average: return Calc(r.Avg, status);
|
||||
case HistoryAggregateType.Minimum: return Calc(r.Min, status);
|
||||
case HistoryAggregateType.Maximum: return Calc(r.Max, status);
|
||||
case HistoryAggregateType.Range: return Calc(r.Max - r.Min, status);
|
||||
case HistoryAggregateType.Start: return Calc(r.First.ToObject(), status);
|
||||
case HistoryAggregateType.End: return Calc(r.Last.ToObject(), status);
|
||||
// #684 (Part 13 §5.4.3.5; SDK ComputeAverage): the mean of the GOOD values — an interval with none (only
|
||||
// Uncertain / Bad) is BadNoData; value-based status: Good only when every sample is Good, else Uncertain.
|
||||
case HistoryAggregateType.Average:
|
||||
if (r.Avg is null) return NoData(bucketStart, now);
|
||||
return Calc(r.Avg, r.Good == r.Total ? HistorianStatusCodes.Good : HistorianStatusCodes.UncertainDataSubNormal);
|
||||
// #684 (Part 13 §5.4.3.10/11/14; SDK ComputeMinMax): the extreme(s) of the GOOD values; Uncertain when a Bad
|
||||
// sample exists or an Uncertain value lies beyond the reported extreme (see MinMaxStatus).
|
||||
case HistoryAggregateType.Minimum:
|
||||
if (r.Min is null) return NoData(bucketStart, now);
|
||||
return Calc(r.Min, MinMaxStatus(r, r.Min, r.Max, low: true, high: false));
|
||||
case HistoryAggregateType.Maximum:
|
||||
if (r.Max is null) return NoData(bucketStart, now);
|
||||
return Calc(r.Max, MinMaxStatus(r, r.Min, r.Max, low: false, high: true));
|
||||
case HistoryAggregateType.Range:
|
||||
if (r.Min is null || r.Max is null) return NoData(bucketStart, now);
|
||||
return Calc(r.Max - r.Min, MinMaxStatus(r, r.Min, r.Max, low: true, high: true));
|
||||
case HistoryAggregateType.Delta:
|
||||
return r.First.TryGetDouble(out var a) && r.Last.TryGetDouble(out var b)
|
||||
? Calc(b - a, status)
|
||||
: Calc(null, HistorianStatusCodes.BadAggregateInvalidInputs);
|
||||
// #680 (SDK ComputeDelta): last usable − first usable; Uncertain only when the interval's first or last
|
||||
// sample was skipped (not usable) — a Bad sample in the middle does not degrade it.
|
||||
// Live-gate 8 F1: a row whose counts promise a usable sample its picks do not carry (a backend that read
|
||||
// the counts and the picks from two table versions) has no consistent first/last — no data, not a throw
|
||||
// and not a wrong-input verdict.
|
||||
if (r.First.Kind == HistorianValueKind.Empty || r.Last.Kind == HistorianValueKind.Empty)
|
||||
return NoData(bucketStart, now);
|
||||
if (!r.First.TryGetDouble(out var a) || !r.Last.TryGetDouble(out var b))
|
||||
return Calc(null, HistorianStatusCodes.BadAggregateInvalidInputs);
|
||||
var bounded = r.FirstAny is { IsUsable: true } && r.LastAny is { IsUsable: true };
|
||||
return Calc(b - a, bounded ? HistorianStatusCodes.Good : HistorianStatusCodes.UncertainDataSubNormal);
|
||||
case HistoryAggregateType.MinimumActualTime:
|
||||
case HistoryAggregateType.MaximumActualTime:
|
||||
return Calc(r.Extreme, status, r.ExtremeTime);
|
||||
// Live-gate 8 F1: usable numeric samples counted but no extreme picked (an inconsistent row) is no data,
|
||||
// never a value-less Good.
|
||||
if (r.Extreme is null || r.ExtremeTime is null) return NoData(bucketStart, now);
|
||||
var min = aggregate == HistoryAggregateType.MinimumActualTime;
|
||||
return Calc(r.Extreme, MinMaxStatus(r, r.Extreme, r.Extreme, low: min, high: !min), r.ExtremeTime);
|
||||
default:
|
||||
throw new HistoryAggregateNotSupportedException(aggregate);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// #684 — the Part 13 status of Minimum / Maximum / Range / *ActualTime (Tables 57–62): <c>Uncertain_DataSubNormal</c>
|
||||
/// when a Bad sample exists in the interval, or when an Uncertain value is less than the minimum Good value
|
||||
/// (<paramref name="low"/>) / greater than the maximum Good value (<paramref name="high"/>); else Good. An Uncertain
|
||||
/// value inside the Good range leaves it Good. An Uncertain String/DateTime sample (no order against the Good values)
|
||||
/// also makes it Uncertain — the SDK's <c>ComputeMinMax</c> flags a value it cannot cast exactly that way.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The SDK's <c>ComputeMinMax</c> (1.5.378) applies only the Bad half: it collects the Uncertain extremes and never
|
||||
/// reads them, so it answers Good where Part 13 says Uncertain. Ours follows the specification there — the more
|
||||
/// conservative of the two, and never a status better than the inputs warrant. The comparison is strict, so an
|
||||
/// Uncertain value EQUAL to the Good extreme does not degrade it.
|
||||
/// </remarks>
|
||||
private static uint MinMaxStatus(BucketRow r, double? goodMin, double? goodMax, bool low, bool high) =>
|
||||
r.Bad > 0 || r.NonNumeric > r.GoodNonNumericOrAll
|
||||
|| (low && r.UncertainMin < goodMin) || (high && r.UncertainMax > goodMax)
|
||||
? HistorianStatusCodes.UncertainDataSubNormal
|
||||
: HistorianStatusCodes.Good;
|
||||
|
||||
/// <summary>
|
||||
/// True for the per-sample aggregates the SDK sets the start-of-data Partial bit on (every calculator but
|
||||
/// <c>AverageAggregateCalculator</c>'s plain <c>Average</c>, which turns <c>SetPartialBit</c> off).
|
||||
/// </summary>
|
||||
/// <param name="aggregate">Requested aggregate.</param>
|
||||
/// <returns>Whether the start-of-data Partial rule applies.</returns>
|
||||
public static bool SetsStartOfDataPartial(HistoryAggregateType aggregate) => aggregate != HistoryAggregateType.Average;
|
||||
|
||||
/// <summary>
|
||||
/// True when a per-sample read must fetch the tag's data range — its earliest and latest stored samples of any
|
||||
/// quality — because the answer can change a result: the start/end-of-data Partial rules for an aggregate other
|
||||
/// than <c>Average</c> (either direction: the end-of-data rule applies to reverse reads too), and Count (an empty
|
||||
/// interval inside the range is 0, outside it <c>BadNoData</c>, #680). Only a per-sample <c>Average</c> read skips
|
||||
/// both queries.
|
||||
/// </summary>
|
||||
/// <param name="aggregate">Requested aggregate.</param>
|
||||
/// <param name="plan">The bucket plan.</param>
|
||||
/// <returns>Whether the earliest and latest samples are needed.</returns>
|
||||
public static bool NeedsDataRange(HistoryAggregateType aggregate, BucketPlan plan)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(plan);
|
||||
return plan.BucketCount > 0 && (aggregate == HistoryAggregateType.Count || SetsStartOfDataPartial(aggregate));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Evaluates every non-empty bucket row and assembles one result per interval in the plan's order (descending for a
|
||||
/// reverse read): each value stamped with <see cref="BucketPlan.BucketTimestamp"/> (the earlier edge forward, the
|
||||
/// later edge reverse) — Start/End/ActualTime with their sample's own time; empty buckets <c>BadNoData</c>, except a
|
||||
/// Count inside the tag's data range (0, Good — #680); and, for every aggregate but <c>Average</c>, the Partial bit
|
||||
/// on a non-Bad result of the bucket the tag's data starts strictly inside (forward only — #677, the SDK's
|
||||
/// <c>m_startOfData</c> rule) or ends inside (#680 — <see cref="BucketPlan.DataEndsInside"/>: forward the SDK's
|
||||
/// <c>m_endOfData</c>, earlier edge included; reverse strictly inside, the SDK's <c>m_startOfData</c> keyed on the
|
||||
/// first value a reverse read is fed — the latest). A Bad result never carries Partial.
|
||||
/// </summary>
|
||||
/// <param name="aggregate">A per-sample (Simple / StartEnd / ActualTime) aggregate.</param>
|
||||
/// <param name="plan">The bucket plan.</param>
|
||||
/// <param name="rows">The backend's non-empty bucket rows (ascending bucket index).</param>
|
||||
/// <param name="earliestSampleUtc">The tag's earliest stored sample of any quality, or null when none.</param>
|
||||
/// <param name="latestSampleUtc">The tag's latest stored sample of any quality, or null when none.</param>
|
||||
/// <param name="now">Server timestamp.</param>
|
||||
/// <returns><see cref="BucketPlan.BucketCount"/> results.</returns>
|
||||
public static IReadOnlyList<DataValueSnapshot> EvaluateAll(HistoryAggregateType aggregate, BucketPlan plan,
|
||||
IEnumerable<BucketRow> rows, DateTime? earliestSampleUtc, DateTime? latestSampleUtc, DateTime now)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(plan);
|
||||
ArgumentNullException.ThrowIfNull(rows);
|
||||
var evaluated = new Dictionary<long, DataValueSnapshot>();
|
||||
var startRule = !plan.Reversed && SetsStartOfDataPartial(aggregate);
|
||||
var endRule = SetsStartOfDataPartial(aggregate);
|
||||
foreach (var row in rows)
|
||||
{
|
||||
var v = Evaluate(aggregate, row, plan.BucketTimestamp(row.Bucket), now);
|
||||
if (!StatusBits.IsBad(v.StatusCode)
|
||||
&& ((startRule && plan.DataStartsInside(row.Bucket, earliestSampleUtc))
|
||||
|| (endRule && plan.DataEndsInside(row.Bucket, latestSampleUtc))))
|
||||
// A raw Start/End sample gains the DataValue InfoType with its first aggregate bit (as the SDK's
|
||||
// SetAggregateBits does); a calculated status already carries it.
|
||||
v = v with { StatusCode = v.StatusCode | HistorianInfoBits.InfoTypeDataValue | HistorianInfoBits.HistorianPartial };
|
||||
evaluated[row.Bucket] = v;
|
||||
}
|
||||
if (aggregate == HistoryAggregateType.Count)
|
||||
{
|
||||
// #680 (Part 13 §5.4.3.21; SDK GetValues): an empty interval inside [earliest, latest] is a Count of 0, Good.
|
||||
for (long i = 0; i < plan.BucketCount; i++)
|
||||
if (!evaluated.ContainsKey(i) && plan.WithinDataRange(i, earliestSampleUtc, latestSampleUtc))
|
||||
evaluated[i] = new DataValueSnapshot(0, HistorianInfoBits.WithCalculated(HistorianStatusCodes.Good),
|
||||
plan.BucketTimestamp(i), now);
|
||||
}
|
||||
return plan.Assemble(evaluated, t => NoData(t, now));
|
||||
}
|
||||
|
||||
/// <summary>A raw stored sample as an aggregate result (Start / End): value null when Bad, its own status and time.</summary>
|
||||
private static DataValueSnapshot Raw(StoredSample? sample, DateTime bucketStart, DateTime now) =>
|
||||
sample is not { } s
|
||||
? NoData(bucketStart, now)
|
||||
: new(StatusBits.IsBad(s.Status) ? null : s.Value.ToObject(), s.Status, s.TimeUtc, now);
|
||||
|
||||
/// <summary>The result for a bucket with no samples (or no usable sample): <c>BadNoData</c> + Calculated.</summary>
|
||||
/// <param name="t">The bucket's start time.</param>
|
||||
/// <param name="now">Server timestamp.</param>
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
using System.Collections.Frozen;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common;
|
||||
@@ -5,7 +6,7 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common;
|
||||
/// <summary>The query shape that serves an aggregate. One query per family.</summary>
|
||||
public enum AggregateFamily
|
||||
{
|
||||
/// <summary>Average / Minimum / Maximum / Count / Range / PercentGood / PercentBad: one GROUP BY.</summary>
|
||||
/// <summary>Average / Minimum / Maximum / Count / Range: one GROUP BY.</summary>
|
||||
Simple,
|
||||
|
||||
/// <summary>Start / End / Delta: first + last usable sample per bucket.</summary>
|
||||
@@ -15,7 +16,8 @@ public enum AggregateFamily
|
||||
ActualTime,
|
||||
|
||||
/// <summary>
|
||||
/// TimeAverage / TimeAverage2 / Total / Total2 / Interpolative: Part 13 time-weighted, with bounding values.
|
||||
/// TimeAverage / TimeAverage2 / Total / Total2 / Interpolative, and the duration-based PercentGood / PercentBad
|
||||
/// (#680): Part 13 time-weighted, with bounding values.
|
||||
/// The backend streams the window's samples plus the bounding rows and <see cref="TimeWeightedAggregator"/>
|
||||
/// evaluates them (identically for every backend).
|
||||
/// </summary>
|
||||
@@ -31,12 +33,20 @@ public static class AggregateFamilies
|
||||
public static AggregateFamily? FamilyOf(HistoryAggregateType aggregate) => aggregate switch
|
||||
{
|
||||
HistoryAggregateType.Average or HistoryAggregateType.Minimum or HistoryAggregateType.Maximum
|
||||
or HistoryAggregateType.Count or HistoryAggregateType.Range
|
||||
or HistoryAggregateType.PercentGood or HistoryAggregateType.PercentBad => AggregateFamily.Simple,
|
||||
or HistoryAggregateType.Count or HistoryAggregateType.Range => AggregateFamily.Simple,
|
||||
HistoryAggregateType.Start or HistoryAggregateType.End or HistoryAggregateType.Delta => AggregateFamily.StartEnd,
|
||||
HistoryAggregateType.MinimumActualTime or HistoryAggregateType.MaximumActualTime => AggregateFamily.ActualTime,
|
||||
HistoryAggregateType.TimeAverage or HistoryAggregateType.TimeAverage2 or HistoryAggregateType.Total
|
||||
or HistoryAggregateType.Total2 or HistoryAggregateType.Interpolative => AggregateFamily.TimeWeighted,
|
||||
or HistoryAggregateType.Total2 or HistoryAggregateType.Interpolative
|
||||
or HistoryAggregateType.PercentGood or HistoryAggregateType.PercentBad => AggregateFamily.TimeWeighted,
|
||||
_ => null,
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// The seventeen aggregates the SQL backends serve — every member <see cref="FamilyOf"/> assigns a family, derived
|
||||
/// from it so the set and the dispatch can never disagree. Both <c>SqlServerHistorianDataSource</c> and
|
||||
/// <c>QuestDbHistorianDataSource</c> return it as <see cref="IHistorianDataSource.SupportedAggregates"/> (#682).
|
||||
/// </summary>
|
||||
public static readonly IReadOnlySet<HistoryAggregateType> Served =
|
||||
Enum.GetValues<HistoryAggregateType>().Where(a => FamilyOf(a) is not null).ToFrozenSet();
|
||||
}
|
||||
|
||||
@@ -49,8 +49,10 @@ public static class AtTimeEvaluator
|
||||
{
|
||||
var fraction = (double)(t.Ticks - p.TimeUtc.Ticks) / (n.TimeUtc.Ticks - p.TimeUtc.Ticks);
|
||||
var v = p.Value.Number + (nv - p.Value.Number) * fraction;
|
||||
// Each arm boxed on its own: `cond ? (long)x : v` is a DOUBLE expression in C# (the long widens), which returned
|
||||
// every interpolated Int64 as a Double until #693.
|
||||
object value = p.Value.Kind == HistorianValueKind.Int64
|
||||
? (long)Math.Round(v, MidpointRounding.AwayFromZero)
|
||||
? (object)(long)Math.Round(v, MidpointRounding.AwayFromZero)
|
||||
: v;
|
||||
var status = p.IsGood && n.IsGood ? HistorianStatusCodes.Good : HistorianStatusCodes.UncertainDataSubNormal;
|
||||
return new DataValueSnapshot(value, HistorianInfoBits.WithInterpolated(status), t, serverTime);
|
||||
|
||||
@@ -3,12 +3,29 @@ using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common;
|
||||
|
||||
/// <summary>
|
||||
/// The bucket layout of a processed (aggregate) read. Bucket <c>i</c> = <c>[start + i·interval,
|
||||
/// start + (i+1)·interval)</c> (start-inclusive, end-exclusive to the tick; the last bucket is truncated at
|
||||
/// <c>end</c>). An interval <= 0 ⇒ one interval <c>[start, end)</c>. <c>start == end</c> ⇒ no intervals.
|
||||
/// <c>end < start</c> (reverse) ⇒ the window is swapped, computed forward, and assembled in descending bucket
|
||||
/// order (buckets stay anchored at the earlier time).
|
||||
/// The bucket layout of a processed (aggregate) read, per OPC UA Part 13 / the SDK's <c>AggregateCalculator</c>.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>Forward</b> (<c>start < end</c>): the window is <c>[start, end)</c>; bucket <c>i</c> =
|
||||
/// <c>[start + i·interval, start + (i+1)·interval)</c>, the last bucket truncated at <c>end</c> (the partial one is
|
||||
/// the <i>latest</i>); each bucket's timestamp is its start (earlier edge); results ascend.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Reverse</b> (<c>end < start</c>, #677): the read is anchored at the request's start — its <i>later</i>
|
||||
/// time — and steps back. The window is <c>(end, start]</c> (Part 11: the request's start is inclusive, its end
|
||||
/// exclusive, so a reverse read includes its later time and excludes its earlier one — the same window as a reverse
|
||||
/// raw read); result <c>k</c> = <c>(start − (k+1)·interval, start − k·interval]</c>, clipped at <c>end</c> (the
|
||||
/// partial bucket is the <i>earliest</i>); each bucket's timestamp is its later edge <c>start − k·interval</c>;
|
||||
/// results descend. This is exactly the SDK's <c>CreateSlice</c> / <c>GetTimestamp</c> / backward
|
||||
/// <c>GetValues</c>.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Internally buckets are indexed in ascending time (<c>0</c> = earliest) for both directions; <see cref="Assemble"/>
|
||||
/// reverses the list for a reverse read. <see cref="StartUtc"/>/<see cref="EndUtc"/> are always the earlier / later
|
||||
/// window edges. An interval <= 0 ⇒ one interval over the whole window. <c>start == end</c> ⇒ no intervals.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public sealed class BucketPlan
|
||||
{
|
||||
/// <summary>Spec §9.7: a processed read covering more intervals than this is refused.</summary>
|
||||
@@ -21,12 +38,15 @@ public sealed class BucketPlan
|
||||
Reversed = reversed;
|
||||
IntervalTicks = intervalTicks;
|
||||
BucketCount = bucketCount;
|
||||
// Forward buckets are anchored at the earlier edge; reverse ones at the later edge, so the grid origin sits a
|
||||
// whole number of intervals before it (possibly before the window start — the earliest bucket is clipped).
|
||||
OriginTicks = reversed ? endUtc.Ticks - bucketCount * intervalTicks : startUtc.Ticks;
|
||||
}
|
||||
|
||||
/// <summary>The earlier end of the window (the forward start).</summary>
|
||||
/// <summary>The earlier edge of the window.</summary>
|
||||
public DateTime StartUtc { get; }
|
||||
|
||||
/// <summary>The later end of the window (the forward, exclusive end).</summary>
|
||||
/// <summary>The later edge of the window.</summary>
|
||||
public DateTime EndUtc { get; }
|
||||
|
||||
/// <summary>True when the request came in reversed (<c>end < start</c>).</summary>
|
||||
@@ -38,9 +58,30 @@ public sealed class BucketPlan
|
||||
/// <summary>Number of buckets (0 for an empty window).</summary>
|
||||
public long BucketCount { get; }
|
||||
|
||||
/// <summary>
|
||||
/// The tick at which the (unclipped) bucket grid starts: <see cref="StartUtc"/> forward, <c>EndUtc − n·interval</c>
|
||||
/// reverse. May precede <see cref="StartUtc"/> (and even <see cref="DateTime.MinValue"/>), so it is a raw tick count.
|
||||
/// </summary>
|
||||
public long OriginTicks { get; }
|
||||
|
||||
/// <summary><see cref="StartUtc"/> as 100 ns ticks since the Unix epoch.</summary>
|
||||
public long StartUnixTicks => (StartUtc - DateTime.UnixEpoch).Ticks;
|
||||
|
||||
/// <summary>
|
||||
/// The constant a backend adds to <c>elapsed = t − StartUtc</c> (ticks) before the integer division by
|
||||
/// <see cref="IntervalTicks"/> to get a sample's ascending bucket index: <c>0</c> forward (<c>[lo, hi)</c> buckets);
|
||||
/// <c>(StartUtc − OriginTicks) − 1</c> reverse, where buckets are <c>(lo, hi]</c> — index <c>⌈(t − origin)/I⌉ − 1
|
||||
/// = (t − origin − 1) / I</c>. Never makes the dividend negative for an in-window sample (reverse excludes
|
||||
/// <c>t == StartUtc</c>).
|
||||
/// </summary>
|
||||
public long BucketIndexAdjustTicks => Reversed ? StartUtc.Ticks - OriginTicks - 1 : 0;
|
||||
|
||||
/// <summary>
|
||||
/// <see cref="StartUnixTicks"/> minus <see cref="BucketIndexAdjustTicks"/>: the base a backend subtracts from a
|
||||
/// sample's Unix-epoch ticks to get the same dividend (<c>(t − base) / IntervalTicks</c>).
|
||||
/// </summary>
|
||||
public long BucketBaseUnixTicks => StartUnixTicks - BucketIndexAdjustTicks;
|
||||
|
||||
/// <summary>Plans the buckets of a processed read.</summary>
|
||||
/// <param name="startUtc">Requested start.</param>
|
||||
/// <param name="endUtc">Requested end (earlier than start for a reverse read).</param>
|
||||
@@ -64,15 +105,86 @@ public sealed class BucketPlan
|
||||
return new BucketPlan(start, end, reverse, intervalTicks, bucketCount);
|
||||
}
|
||||
|
||||
/// <summary>The start time of bucket <paramref name="index"/> (UTC kind).</summary>
|
||||
/// <param name="index">Zero-based bucket index.</param>
|
||||
/// <returns>The bucket start.</returns>
|
||||
public DateTime BucketStart(long index) =>
|
||||
DateTime.SpecifyKind(new DateTime(StartUtc.Ticks + index * IntervalTicks), DateTimeKind.Utc);
|
||||
/// <summary>The earlier edge of bucket <paramref name="index"/> (ascending index), in ticks, clipped to the window.</summary>
|
||||
/// <param name="index">Zero-based ascending bucket index.</param>
|
||||
/// <returns>The earlier edge.</returns>
|
||||
public long LowerTicks(long index) => Math.Max(OriginTicks + index * IntervalTicks, StartUtc.Ticks);
|
||||
|
||||
/// <summary>One snapshot per bucket in window order; missing buckets get <paramref name="empty"/>; reversed if the window was.</summary>
|
||||
/// <param name="byBucket">Evaluated snapshots of the non-empty buckets, by bucket index.</param>
|
||||
/// <param name="empty">Builds the snapshot of a bucket with no row, from its start time.</param>
|
||||
/// <summary>The later edge of bucket <paramref name="index"/> (ascending index), in ticks, clipped to the window.</summary>
|
||||
/// <param name="index">Zero-based ascending bucket index.</param>
|
||||
/// <returns>The later edge.</returns>
|
||||
public long UpperTicks(long index) => Math.Min(OriginTicks + (index + 1) * IntervalTicks, EndUtc.Ticks);
|
||||
|
||||
/// <summary>
|
||||
/// The result timestamp of bucket <paramref name="index"/> (UTC kind): its earlier edge forward, its later edge
|
||||
/// reverse (the SDK's <c>GetTimestamp</c>).
|
||||
/// </summary>
|
||||
/// <param name="index">Zero-based ascending bucket index.</param>
|
||||
/// <returns>The bucket's timestamp.</returns>
|
||||
public DateTime BucketTimestamp(long index) =>
|
||||
new(Reversed ? UpperTicks(index) : LowerTicks(index), DateTimeKind.Utc);
|
||||
|
||||
/// <summary>True when bucket <paramref name="index"/> is shorter than the interval (the clipped one).</summary>
|
||||
/// <param name="index">Zero-based ascending bucket index.</param>
|
||||
/// <returns>Whether the bucket is truncated.</returns>
|
||||
public bool IsTruncated(long index) => UpperTicks(index) - LowerTicks(index) < IntervalTicks;
|
||||
|
||||
/// <summary>
|
||||
/// The ascending bucket index holding a sample at <paramref name="t"/>, or null when it is outside the window
|
||||
/// (<c>[start, end)</c> forward, <c>(start, end]</c> reverse). The C# statement of the bucket math every backend's
|
||||
/// SQL mirrors.
|
||||
/// </summary>
|
||||
/// <param name="t">Sample time.</param>
|
||||
/// <returns>The bucket index, or null.</returns>
|
||||
public long? IndexOf(DateTime t)
|
||||
{
|
||||
var x = t.Ticks;
|
||||
if (BucketCount == 0) return null;
|
||||
var inside = Reversed ? x > StartUtc.Ticks && x <= EndUtc.Ticks : x >= StartUtc.Ticks && x < EndUtc.Ticks;
|
||||
if (!inside) return null;
|
||||
return (x - StartUtc.Ticks + BucketIndexAdjustTicks) / IntervalTicks;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// True when the earliest stored sample of the tag lies strictly inside bucket <paramref name="index"/> — the data
|
||||
/// starts part-way through the interval (no bounding value before it), so Part 13 / the SDK set the Partial bit.
|
||||
/// </summary>
|
||||
/// <param name="index">Zero-based ascending bucket index.</param>
|
||||
/// <param name="earliestSampleUtc">The tag's earliest stored sample (any quality), or null when it has none.</param>
|
||||
/// <returns>Whether the data starts inside the bucket.</returns>
|
||||
public bool DataStartsInside(long index, DateTime? earliestSampleUtc) =>
|
||||
earliestSampleUtc is { } e && e.Ticks > LowerTicks(index) && e.Ticks < UpperTicks(index);
|
||||
|
||||
/// <summary>
|
||||
/// True when the latest stored sample of the tag lies in bucket <paramref name="index"/>: the data ends part-way
|
||||
/// through the interval, so Part 13 §5.3.3.2 / the SDK set the Partial bit (#680). Forward the earlier edge is
|
||||
/// included (the SDK's <c>m_endOfData >= startTime && m_endOfData < endTime</c>); reverse the sample
|
||||
/// must lie strictly inside (the SDK's reverse read keys <c>m_startOfData</c> — a strict test — on the first value it
|
||||
/// is fed, the latest).
|
||||
/// </summary>
|
||||
/// <param name="index">Zero-based ascending bucket index.</param>
|
||||
/// <param name="latestSampleUtc">The tag's latest stored sample (any quality), or null when it has none.</param>
|
||||
/// <returns>Whether the data ends inside the bucket.</returns>
|
||||
public bool DataEndsInside(long index, DateTime? latestSampleUtc) =>
|
||||
latestSampleUtc is { } l
|
||||
&& (Reversed ? l.Ticks > LowerTicks(index) : l.Ticks >= LowerTicks(index)) && l.Ticks < UpperTicks(index);
|
||||
|
||||
/// <summary>
|
||||
/// True when bucket <paramref name="index"/> overlaps the tag's data range <c>[earliest, latest]</c> — neither
|
||||
/// wholly before the first sample nor wholly after the last (the SDK's <c>GetValues</c> null test:
|
||||
/// <c>start > last || end < first</c> ⇒ no data, both edges inclusive in either direction). An empty bucket
|
||||
/// inside the range is a Count of 0, not <c>BadNoData</c> (#680, Part 13 §5.4.3.21).
|
||||
/// </summary>
|
||||
/// <param name="index">Zero-based ascending bucket index.</param>
|
||||
/// <param name="earliestSampleUtc">The tag's earliest stored sample (any quality), or null.</param>
|
||||
/// <param name="latestSampleUtc">The tag's latest stored sample (any quality), or null.</param>
|
||||
/// <returns>Whether the bucket is within the data range.</returns>
|
||||
public bool WithinDataRange(long index, DateTime? earliestSampleUtc, DateTime? latestSampleUtc) =>
|
||||
earliestSampleUtc is { } e && latestSampleUtc is { } l && LowerTicks(index) <= l.Ticks && UpperTicks(index) >= e.Ticks;
|
||||
|
||||
/// <summary>One snapshot per bucket in result order; missing buckets get <paramref name="empty"/>; descending if reversed.</summary>
|
||||
/// <param name="byBucket">Evaluated snapshots of the non-empty buckets, by ascending bucket index.</param>
|
||||
/// <param name="empty">Builds the snapshot of a bucket with no row, from its <see cref="BucketTimestamp"/>.</param>
|
||||
/// <returns><see cref="BucketCount"/> snapshots.</returns>
|
||||
public IReadOnlyList<DataValueSnapshot> Assemble(IReadOnlyDictionary<long, DataValueSnapshot> byBucket,
|
||||
Func<DateTime, DataValueSnapshot> empty)
|
||||
@@ -81,10 +193,7 @@ public sealed class BucketPlan
|
||||
ArgumentNullException.ThrowIfNull(empty);
|
||||
var result = new List<DataValueSnapshot>((int)BucketCount);
|
||||
for (long i = 0; i < BucketCount; i++)
|
||||
{
|
||||
var bucketStart = BucketStart(i);
|
||||
result.Add(byBucket.TryGetValue(i, out var v) ? v : empty(bucketStart));
|
||||
}
|
||||
result.Add(byBucket.TryGetValue(i, out var v) ? v : empty(BucketTimestamp(i)));
|
||||
if (Reversed) result.Reverse();
|
||||
return result;
|
||||
}
|
||||
|
||||
@@ -9,7 +9,11 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common;
|
||||
/// <remarks>
|
||||
/// Sample classes: <i>usable</i> = severity Good or Uncertain AND a stored value (an Empty sample is an outage marker
|
||||
/// and counts as Bad whatever its code); <i>good</i> = usable with severity Good; <i>bad</i> = not usable.
|
||||
/// Every value field is computed over usable samples only.
|
||||
/// <see cref="Avg"/> / <see cref="Min"/> / <see cref="Max"/> / <see cref="Sum"/> / <see cref="Extreme"/> are computed over
|
||||
/// <b>Good</b> numeric samples only (#684 — Part 13 defines Average / Minimum / Maximum / Range / *ActualTime over Good
|
||||
/// raw values, as the SDK's calculators do); <see cref="UncertainMin"/> / <see cref="UncertainMax"/> over the Uncertain
|
||||
/// ones (the Part 13 Minimum/Maximum/Range status rule); <see cref="First"/> / <see cref="Last"/> over usable samples;
|
||||
/// <see cref="FirstAny"/> / <see cref="LastAny"/> over every sample.
|
||||
/// </remarks>
|
||||
public sealed class BucketRow
|
||||
{
|
||||
@@ -28,19 +32,30 @@ public sealed class BucketRow
|
||||
/// <summary>Bad (not usable) samples.</summary>
|
||||
public int Bad { get; set; }
|
||||
|
||||
/// <summary>Usable String / DateTime samples.</summary>
|
||||
/// <summary>Usable String / DateTime samples (Delta's refusal check).</summary>
|
||||
public int NonNumeric { get; set; }
|
||||
|
||||
/// <summary>Average of the usable numeric values (null when none).</summary>
|
||||
/// <summary>
|
||||
/// <b>Good</b> String / DateTime samples — the refusal check of the Good-value aggregates (Average / Minimum / Maximum /
|
||||
/// Range / *ActualTime, #684 review): an Uncertain non-numeric sample is not part of their value set. Filled by the
|
||||
/// Simple and ActualTime queries; null (a row that does not carry it) falls back to <see cref="NonNumeric"/>, the
|
||||
/// pre-#684 (stricter) check.
|
||||
/// </summary>
|
||||
public int? GoodNonNumeric { get; set; }
|
||||
|
||||
/// <summary><see cref="GoodNonNumeric"/>, or <see cref="NonNumeric"/> when the row does not carry it.</summary>
|
||||
internal int GoodNonNumericOrAll => GoodNonNumeric ?? NonNumeric;
|
||||
|
||||
/// <summary>Average of the Good numeric values (null when none — #684).</summary>
|
||||
public double? Avg { get; set; }
|
||||
|
||||
/// <summary>Minimum of the usable numeric values (null when none).</summary>
|
||||
/// <summary>Minimum of the Good numeric values (null when none — #684).</summary>
|
||||
public double? Min { get; set; }
|
||||
|
||||
/// <summary>Maximum of the usable numeric values (null when none).</summary>
|
||||
/// <summary>Maximum of the Good numeric values (null when none — #684).</summary>
|
||||
public double? Max { get; set; }
|
||||
|
||||
/// <summary>Sum of the usable numeric values (null when none).</summary>
|
||||
/// <summary>Sum of the Good numeric values (null when none — #684).</summary>
|
||||
public double? Sum { get; set; }
|
||||
|
||||
/// <summary>First usable sample's value by time (Empty when none).</summary>
|
||||
@@ -49,9 +64,31 @@ public sealed class BucketRow
|
||||
/// <summary>Last usable sample's value by time (Empty when none).</summary>
|
||||
public HistorianValue Last { get; set; }
|
||||
|
||||
/// <summary>Extreme (min or max) usable numeric value (null when none).</summary>
|
||||
/// <summary>Extreme (min or max) Good numeric value (null when none — #684).</summary>
|
||||
public double? Extreme { get; set; }
|
||||
|
||||
/// <summary>Time of <see cref="Extreme"/> (ties ⇒ earliest; null when none).</summary>
|
||||
public DateTime? ExtremeTime { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Minimum of the bucket's <b>Uncertain</b> (usable, not Good) numeric values (null when none). Not a result value:
|
||||
/// Part 13 makes Minimum / MinimumActualTime / Range <c>Uncertain_DataSubNormal</c> when an Uncertain value is less
|
||||
/// than the minimum Good value (#684).
|
||||
/// </summary>
|
||||
public double? UncertainMin { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Maximum of the bucket's Uncertain numeric values (null when none): Maximum / MaximumActualTime / Range are
|
||||
/// Uncertain when one is greater than the maximum Good value (#684).
|
||||
/// </summary>
|
||||
public double? UncertainMax { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// The bucket's first sample by time of <b>any</b> quality (null when the bucket is empty): Start returns it raw, and
|
||||
/// Delta is Uncertain when it is not usable (#680, Part 13 Tables 73/74 — the SDK's <c>StartEndAggregateCalculator</c>).
|
||||
/// </summary>
|
||||
public StoredSample? FirstAny { get; set; }
|
||||
|
||||
/// <summary>The bucket's last sample by time of any quality (null when the bucket is empty): End, Delta (#680).</summary>
|
||||
public StoredSample? LastAny { get; set; }
|
||||
}
|
||||
|
||||
+219
-49
@@ -4,8 +4,8 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common;
|
||||
|
||||
/// <summary>
|
||||
/// The OPC UA Part 13 time-weighted aggregates (<see cref="AggregateFamily.TimeWeighted"/>): <c>TimeAverage</c>,
|
||||
/// <c>TimeAverage2</c>, <c>Total</c>, <c>Total2</c> and <c>Interpolative</c>, evaluated over a time-ordered stream of
|
||||
/// stored samples. Both SQL backends fetch the same rows and feed them here, so the two produce identical results
|
||||
/// <c>TimeAverage2</c>, <c>Total</c>, <c>Total2</c>, <c>Interpolative</c> and the duration-based <c>PercentGood</c> /
|
||||
/// <c>PercentBad</c> (#680), evaluated over a time-ordered stream of stored samples. Both SQL backends fetch the same rows and feed them here, so the two produce identical results
|
||||
/// by construction.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
@@ -18,16 +18,18 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common;
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>The signal (server defaults: TreatUncertainAsBad = false, PercentDataBad = PercentDataGood = 100,
|
||||
/// UseSlopedExtrapolation = false; a client <c>AggregateConfiguration</c> is not honoured).</b> Between a usable
|
||||
/// UseSlopedExtrapolation = false; the node manager refuses any other client <c>AggregateConfiguration</c>).</b> Between a usable
|
||||
/// sample <c>p</c> and the next sample <c>s</c>:
|
||||
/// <list type="bullet">
|
||||
/// <item><c>s</c> usable, both Double/Int64 ⇒ <b>sloped</b> (linear <c>p → s</c>); quality = worse of the two.</item>
|
||||
/// <item><c>s</c> usable, either Boolean ⇒ <b>stepped</b> (<c>p</c> held); quality = <c>p</c>'s.</item>
|
||||
/// <item><c>s</c> Bad ⇒ <c>p</c> held up to the outage (never interpolated into it, as AtTime does); Uncertain.</item>
|
||||
/// <item>no later sample at all ⇒ <c>p</c> held to the window end (stepped extrapolation); Uncertain.</item>
|
||||
/// <item>no later sample at all (nothing at or after the window end) ⇒ the data has ENDED at <c>p</c>: no
|
||||
/// extrapolation (Part 13 "After End of Data", #680).</item>
|
||||
/// </list>
|
||||
/// From a Bad sample (incl. an <c>Empty</c> outage marker) to the next sample, and before the first sample, the
|
||||
/// signal is Bad (no data). This is the <i>simple bounding</i> signal.
|
||||
/// From a Bad sample (incl. an <c>Empty</c> outage marker) to the next sample the signal is Bad (outage) — to the
|
||||
/// window end when nothing follows it; before the first sample there is no data. This is the <i>simple bounding</i>
|
||||
/// signal.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// The <i>interpolated bounding</i> signal (Part 13 "interpolated bounding values") skips Bad samples: it joins
|
||||
@@ -36,30 +38,58 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common;
|
||||
/// </para>
|
||||
/// <list type="table">
|
||||
/// <item><term>TimeAverage</term><description>∫ interpolated-bounding signal ÷ its defined duration in the interval.</description></item>
|
||||
/// <item><term>Total</term><description><c>TimeAverage × interval seconds</c> (value·s) — Part 13 §5.4.3.8.</description></item>
|
||||
/// <item><term>Total</term><description>∫ interpolated-bounding signal over its defined part, in value·s — Part 13
|
||||
/// §5.4.3.8 (= <c>TimeAverage × interval seconds</c> on a covered interval; only the covered part when the data
|
||||
/// starts inside it, #680).</description></item>
|
||||
/// <item><term>TimeAverage2</term><description>∫ simple-bounding signal over its non-Bad parts ÷ non-Bad duration (Bad intervals excluded).</description></item>
|
||||
/// <item><term>Total2</term><description>∫ simple-bounding signal over its non-Bad parts, in value·s (= TimeAverage2 × non-Bad seconds).</description></item>
|
||||
/// <item><term>PercentGood / PercentBad</term><description><c>100 ×</c> Good / Bad region time ÷ interval duration
|
||||
/// (Part 13 §5.4.3.31-34): stepped regions, each with the status of the sample that starts it; the region before an
|
||||
/// interval's first own sample has the simple bounding value's status (Bad after a Bad sample; Uncertain on a
|
||||
/// segment running into an outage or sloping to/from an Uncertain sample; else the earlier sample's — the SDK's
|
||||
/// <c>GetSimpleBound</c>); time before the first sample is Bad; at the end of the data the last region runs to the
|
||||
/// end of its own interval (Part 13; the SDK gives it 1 ms) and later intervals are <c>BadNoData</c>; an interval
|
||||
/// ending before the first sample ⇒ <c>BadNoData</c>. Status Good; any value kind.</description></item>
|
||||
/// <item><term>Interpolative</term><description>The interpolated-bounding signal's value at each interval start —
|
||||
/// for a reverse read, at <c>End − i·interval</c> (Part 13 anchors a reverse read at its later time), emitted
|
||||
/// latest first.</description></item>
|
||||
/// </list>
|
||||
/// <para>
|
||||
/// <b>Status (time-weighted four)</b> — the SDK <c>AggregateCalculator.GetTimeBasedStatusCode</c> rule with
|
||||
/// TreatUncertainAsBad = false, so Uncertain time (Uncertain samples, a value held up to an outage, stepped
|
||||
/// extrapolation past the last sample) counts as Good time: every instant of the interval non-Bad ⇒
|
||||
/// <c>Good</c>; no non-Bad time but some Bad (outage) time ⇒ <c>Bad</c>; no data at all ⇒ <c>BadNoData</c>;
|
||||
/// otherwise (PercentDataGood = PercentDataBad = 100) <c>UncertainDataSubNormal</c>. Time before the first sample
|
||||
/// counts as Bad (no data) time. A usable String/DateTime sample contributing to the interval ⇒
|
||||
/// <c>BadAggregateInvalidInputs</c>. Every status carries the Calculated bit, plus the Partial bit on a non-Bad
|
||||
/// result of a truncated last interval; the timestamp is the interval start; the value is a
|
||||
/// <see cref="double"/> (Boolean = 0/1, Int64 as double).
|
||||
/// <b>Status (TimeAverage / Total, #680)</b> — Part 13 ("Uncertain if Bad values are skipped or Uncertain values are
|
||||
/// used") and the SDK's <c>AverageAggregateCalculator</c>: no defined value in the interval ⇒ <c>BadNoData</c>; any
|
||||
/// part of it on a bridged segment with an Uncertain end (or Uncertain start when stepped), bridged across a Bad
|
||||
/// sample, extrapolated past the last usable sample, or not covered at all ⇒ <c>UncertainDataSubNormal</c> (so the
|
||||
/// latest interval of a live tag is Uncertain, and an interval wholly inside an outage has the bridged value,
|
||||
/// Uncertain); else <c>Good</c>.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Status (TimeAverage2 / Total2)</b> — the SDK <c>AggregateCalculator.GetTimeBasedStatusCode</c> rule with
|
||||
/// TreatUncertainAsBad = false, so Uncertain time (Uncertain samples, a value held up to an outage) counts as Good
|
||||
/// time, over the interval cut at the end of the data: after the end of the data ⇒ <c>BadNoData</c>; the data
|
||||
/// ending exactly on the interval start ⇒ the last sample's value (Total2 0) with its status; every covered instant
|
||||
/// non-Bad ⇒ <c>Good</c>; no non-Bad time but some Bad (outage) time ⇒ <c>Bad</c> (Part 13's PercentDataBad rule;
|
||||
/// the SDK answers <c>BadNoData</c>); no data at all ⇒ <c>BadNoData</c>; otherwise (PercentDataGood = PercentDataBad
|
||||
/// = 100) <c>UncertainDataSubNormal</c>. Time before the first sample counts as Bad (no data) time.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// For the four, a usable String/DateTime sample contributing to the interval ⇒ <c>BadAggregateInvalidInputs</c>.
|
||||
/// Every status of this family but Interpolative's carries the Calculated bit, plus — on a non-Bad result only — the
|
||||
/// Partial bit on: the truncated interval (Part 13 §5.3.3.2 Case 3; the SDK does not set it — deliberate); the
|
||||
/// interval the data starts inside in the read direction (forward: the first sample fed lies strictly within it,
|
||||
/// #677; reverse: the latest sample fed lies strictly within it, nothing after — the SDK's <c>m_startOfData</c>,
|
||||
/// keyed on the first value it is fed, #680); and, forward, the end-of-data interval of TimeAverage2 / Total2 /
|
||||
/// PercentGood / PercentBad (the SDK's <c>m_endOfData</c>, which it skips for an extrapolated TimeAverage / Total).
|
||||
/// The timestamp is the interval's <see cref="BucketPlan.BucketTimestamp"/> (its start forward, its later edge
|
||||
/// reverse); the value is a <see cref="double"/> (Boolean = 0/1, Int64 as double).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Status (Interpolative).</b> A usable sample exactly at the point ⇒ its value with its own status
|
||||
/// (raw, no historian bit). Otherwise no usable sample at or before ⇒ <c>BadNoData</c>; else the interpolated
|
||||
/// value with the Interpolated bit and the status of the simple-bounding signal at that instant: <c>Good</c> on
|
||||
/// a Good segment, <c>UncertainDataSubNormal</c> on an Uncertain one, inside an outage (interpolated across it)
|
||||
/// or past the last sample (extrapolated).
|
||||
/// or past the last sample (extrapolated). A reverse read returns the raw sample at a point exactly on a sample
|
||||
/// too — the SDK's reverse answer (Interpolated, from the segment beyond) differs from its own forward one; that
|
||||
/// asymmetry is the SDK's defect, deliberately not copied.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public sealed class TimeWeightedAggregator
|
||||
@@ -106,12 +136,19 @@ public sealed class TimeWeightedAggregator
|
||||
// Interpolated-bounding (bridged) signal, per bucket.
|
||||
private readonly double[] _bridgedArea;
|
||||
private readonly long[] _bridgedTicks;
|
||||
// A bridged segment touching the bucket was non-Good: an Uncertain end, a Bad sample skipped, or extrapolated (#680).
|
||||
private readonly bool[] _bridgedNonGood;
|
||||
private readonly bool[] _invalid;
|
||||
// PercentGood / PercentBad (#680): stepped status regions, per bucket.
|
||||
private readonly long[] _goodRegionTicks;
|
||||
private readonly long[] _badRegionTicks;
|
||||
// Interpolative, per point (index = ascending time; see PointTicks).
|
||||
private readonly DataValueSnapshot?[] _points;
|
||||
|
||||
private StoredSample? _prevAny;
|
||||
private StoredSample? _prevUsable;
|
||||
private DateTime? _firstSampleUtc; // the earliest sample fed (any quality): where the data starts
|
||||
private long? _dataEndTicks; // the last sample fed, when usable and nothing at/after the end followed
|
||||
private long _nextPoint; // next point (ascending time) not yet classified
|
||||
private readonly List<PendingPoint> _pending = []; // points awaiting the next usable sample
|
||||
private bool _finished;
|
||||
@@ -143,7 +180,10 @@ public sealed class TimeWeightedAggregator
|
||||
_badTicks = new long[n];
|
||||
_bridgedArea = new double[n];
|
||||
_bridgedTicks = new long[n];
|
||||
_bridgedNonGood = new bool[n];
|
||||
_invalid = new bool[n];
|
||||
_goodRegionTicks = new long[n];
|
||||
_badRegionTicks = new long[n];
|
||||
_points = new DataValueSnapshot?[n];
|
||||
}
|
||||
|
||||
@@ -169,6 +209,11 @@ public sealed class TimeWeightedAggregator
|
||||
if (_finished) throw new InvalidOperationException("The aggregation is already finished.");
|
||||
var t = s.TimeUtc.Ticks;
|
||||
if (_prevAny is { } last && t <= last.TimeUtc.Ticks) return;
|
||||
if (_firstSampleUtc is null)
|
||||
{
|
||||
_firstSampleUtc = s.TimeUtc;
|
||||
AddRegion(_badRegionTicks, _start, t); // PercentBad: the time before the first sample is Bad (no data)
|
||||
}
|
||||
|
||||
// Interpolative: classify every bucket start before this sample by the simple-bounding signal there.
|
||||
ClassifyPointsBefore(t);
|
||||
@@ -184,6 +229,7 @@ public sealed class TimeWeightedAggregator
|
||||
{
|
||||
AddBad(bad, t);
|
||||
}
|
||||
if (_prevAny is { } r) AddStatusRegion(r, s, t);
|
||||
|
||||
if (!s.IsUsable) DegradePending(); // the segment the pending points sit on runs into an outage
|
||||
|
||||
@@ -192,7 +238,10 @@ public sealed class TimeWeightedAggregator
|
||||
if (_prevUsable is { } pu)
|
||||
{
|
||||
var sloped = IsSloped(pu, s);
|
||||
AddBridged(pu, t, sloped ? s.Value.Number : pu.Value.Number);
|
||||
// Part 13: Uncertain when a Bad value was skipped or an Uncertain value is used (the end sample only
|
||||
// contributes its value to a sloped segment).
|
||||
var nonGood = !pu.IsGood || (sloped && !s.IsGood) || _prevAny is { IsUsable: false };
|
||||
AddBridged(pu, t, sloped ? s.Value.Number : pu.Value.Number, nonGood);
|
||||
}
|
||||
ResolvePending(_prevUsable, s);
|
||||
_prevUsable = s;
|
||||
@@ -211,11 +260,21 @@ public sealed class TimeWeightedAggregator
|
||||
_finished = true;
|
||||
ClassifyPointsBefore(long.MaxValue);
|
||||
if (_prevAny is { IsUsable: true } p && p.TimeUtc.Ticks < _end)
|
||||
AddRaw(p, _end, p.Value.Number); // stepped extrapolation
|
||||
{
|
||||
// No sample at or after the window end: the data ENDS here (Part 13 "After End of Data"). The simple-
|
||||
// bounding signal is not extrapolated (TimeAverage2/Total2 cover up to it; later intervals BadNoData), and
|
||||
// the status region it starts runs to the end of its own interval only (PercentGood/PercentBad).
|
||||
_dataEndTicks = p.TimeUtc.Ticks;
|
||||
if (p.TimeUtc.Ticks >= _start && _plan.BucketCount > 0)
|
||||
AddRegion(RegionTicks(p), p.TimeUtc.Ticks, BucketEnd(FirstBucketAt(p.TimeUtc.Ticks)));
|
||||
}
|
||||
else if (_prevAny is { } bad && bad.TimeUtc.Ticks < _end)
|
||||
AddBad(bad, _end); // the outage runs to the window end
|
||||
{
|
||||
AddBad(bad, _end); // an outage marker last: the outage runs to the window end
|
||||
AddRegion(_badRegionTicks, bad.TimeUtc.Ticks, _end);
|
||||
}
|
||||
if (_prevUsable is { } pu && pu.TimeUtc.Ticks < _end)
|
||||
AddBridged(pu, _end, pu.Value.Number);
|
||||
AddBridged(pu, _end, pu.Value.Number, nonGood: true); // stepped extrapolation ⇒ Uncertain
|
||||
ResolvePending(_prevUsable, next: null);
|
||||
}
|
||||
|
||||
@@ -234,29 +293,83 @@ public sealed class TimeWeightedAggregator
|
||||
? point with { ServerTimestampUtc = now }
|
||||
: new DataValueSnapshot(null, HistorianStatusCodes.BadNoData, PointTime(i), now);
|
||||
|
||||
var bucketStart = _plan.BucketStart(i);
|
||||
var duration = BucketEnd(i) - BucketStartTicks(i);
|
||||
var partial = duration < _plan.IntervalTicks; // only the truncated last interval
|
||||
DataValueSnapshot Calc(object? value, uint status)
|
||||
var stamp = _plan.BucketTimestamp(i);
|
||||
var lo = BucketStartTicks(i);
|
||||
var hi = BucketEnd(i);
|
||||
var duration = hi - lo;
|
||||
// Partial: the truncated interval (the latest forward, the earliest reverse — Part 13 §5.3.3.2 Case 3, which the
|
||||
// SDK does not implement), or the one the data starts inside IN THE READ DIRECTION (no bounding value on that
|
||||
// side — the SDK's m_startOfData rule, #677): forward, the earliest sample strictly inside; reverse, the latest
|
||||
// sample strictly inside (nothing after it — the SDK keys m_startOfData on the first value it is fed, #680).
|
||||
// Forward, the end-of-data interval (the latest sample inside it, nothing after) is Partial too — never on an
|
||||
// extrapolated TimeAverage/Total (the SDK's m_endOfData rule, #680). Never on a Bad result.
|
||||
var latestInside = _plan.Reversed && _prevAny is { } latest
|
||||
&& latest.TimeUtc.Ticks > lo && latest.TimeUtc.Ticks < hi;
|
||||
var partial = _plan.IsTruncated(i) || latestInside
|
||||
|| (!_plan.Reversed && _plan.DataStartsInside(i, _firstSampleUtc));
|
||||
var endOfData = !_plan.Reversed && _dataEndTicks is { } de && de >= lo && de < hi;
|
||||
DataValueSnapshot Calc(object? value, uint status, bool endPartial = false)
|
||||
{
|
||||
var code = HistorianInfoBits.WithCalculated(status);
|
||||
if (partial && !StatusBits.IsBad(status)) code |= HistorianInfoBits.HistorianPartial;
|
||||
return new DataValueSnapshot(value, code, bucketStart, now);
|
||||
if ((partial || endPartial) && !StatusBits.IsBad(status)) code |= HistorianInfoBits.HistorianPartial;
|
||||
return new DataValueSnapshot(value, code, stamp, now);
|
||||
}
|
||||
|
||||
if (_invalid[i]) return Calc(null, HistorianStatusCodes.BadAggregateInvalidInputs);
|
||||
if (_usableTicks[i] == 0)
|
||||
return _badTicks[i] > 0 ? Calc(null, HistorianStatusCodes.Bad) : AggregateEvaluator.NoData(bucketStart, now);
|
||||
|
||||
var status = _usableTicks[i] == duration ? HistorianStatusCodes.Good : HistorianStatusCodes.UncertainDataSubNormal;
|
||||
return _aggregate switch
|
||||
switch (_aggregate)
|
||||
{
|
||||
HistoryAggregateType.TimeAverage => Calc(_bridgedArea[i] / _bridgedTicks[i], status),
|
||||
HistoryAggregateType.Total => Calc(_bridgedArea[i] / _bridgedTicks[i] * (duration / TicksPerSecond), status),
|
||||
HistoryAggregateType.TimeAverage2 => Calc(_rawArea[i] / _usableTicks[i], status),
|
||||
HistoryAggregateType.Total2 => Calc(_rawArea[i] / TicksPerSecond, status),
|
||||
_ => throw new HistoryAggregateNotSupportedException(_aggregate),
|
||||
};
|
||||
case HistoryAggregateType.TimeAverage or HistoryAggregateType.Total:
|
||||
{
|
||||
// Interpolated bounding: Uncertain when any part of the interval was bridged across a Bad value, used an
|
||||
// Uncertain value, was extrapolated, or is not covered at all (Part 13; the SDK's AverageAggregateCalculator).
|
||||
if (_invalid[i]) return Calc(null, HistorianStatusCodes.BadAggregateInvalidInputs);
|
||||
if (_bridgedTicks[i] == 0) return AggregateEvaluator.NoData(stamp, now);
|
||||
var status = _bridgedNonGood[i] || _bridgedTicks[i] < duration
|
||||
? HistorianStatusCodes.UncertainDataSubNormal
|
||||
: HistorianStatusCodes.Good;
|
||||
return _aggregate == HistoryAggregateType.TimeAverage
|
||||
? Calc(_bridgedArea[i] / _bridgedTicks[i], status)
|
||||
: Calc(_bridgedArea[i] / TicksPerSecond, status); // ∫ over the covered part only (Part 13 partial width)
|
||||
}
|
||||
case HistoryAggregateType.TimeAverage2 or HistoryAggregateType.Total2:
|
||||
{
|
||||
if (_invalid[i]) return Calc(null, HistorianStatusCodes.BadAggregateInvalidInputs);
|
||||
if (_dataEndTicks is { } dataEnd)
|
||||
{
|
||||
// After the end of the data: nothing to aggregate (Part 13 "After End of Data: Bad_NoData").
|
||||
if (lo > dataEnd) return AggregateEvaluator.NoData(stamp, now);
|
||||
if (_usableTicks[i] == 0 && dataEnd < hi && _prevAny is { } last)
|
||||
{
|
||||
// The data ends inside this interval and no non-Bad time precedes its last sample here (the data
|
||||
// ends on the interval's start; or the last sample is the first usable one after an outage — the
|
||||
// comms-loss recovery shape — or the tag's only sample): nothing covered, so the value at that
|
||||
// instant (Total2 0). Good only when the data ends exactly on the interval start with a Good
|
||||
// sample (nothing before it in the interval); otherwise the interval also holds Bad / no-data time
|
||||
// ⇒ Uncertain (the SDK answers the same, modulo its 1 ms artifact).
|
||||
if (!IsNumeric(last)) return Calc(null, HistorianStatusCodes.BadAggregateInvalidInputs);
|
||||
var st = lo == dataEnd && last.IsGood ? HistorianStatusCodes.Good : HistorianStatusCodes.UncertainDataSubNormal;
|
||||
return Calc(_aggregate == HistoryAggregateType.TimeAverage2 ? last.Value.Number : 0d, st, endOfData);
|
||||
}
|
||||
duration = Math.Min(hi, dataEnd) - lo;
|
||||
}
|
||||
if (_usableTicks[i] == 0)
|
||||
return _badTicks[i] > 0 ? Calc(null, HistorianStatusCodes.Bad) : AggregateEvaluator.NoData(stamp, now);
|
||||
var status = _usableTicks[i] == duration ? HistorianStatusCodes.Good : HistorianStatusCodes.UncertainDataSubNormal;
|
||||
return _aggregate == HistoryAggregateType.TimeAverage2
|
||||
? Calc(_rawArea[i] / _usableTicks[i], status, endOfData)
|
||||
: Calc(_rawArea[i] / TicksPerSecond, status, endOfData);
|
||||
}
|
||||
case HistoryAggregateType.PercentGood or HistoryAggregateType.PercentBad:
|
||||
{
|
||||
// No data in or around the interval (it ends before the first sample — the SDK's out-of-range test — or
|
||||
// starts after the end of the data) ⇒ BadNoData; otherwise the share of the interval's duration.
|
||||
if (_firstSampleUtc is not { } first || hi < first.Ticks || (_dataEndTicks is { } de2 && lo > de2))
|
||||
return AggregateEvaluator.NoData(stamp, now);
|
||||
var ticks = _aggregate == HistoryAggregateType.PercentGood ? _goodRegionTicks[i] : _badRegionTicks[i];
|
||||
return Calc(100d * ticks / duration, HistorianStatusCodes.Good, endOfData);
|
||||
}
|
||||
default:
|
||||
throw new HistoryAggregateNotSupportedException(_aggregate);
|
||||
}
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------------------------------ Segments
|
||||
@@ -269,9 +382,12 @@ public sealed class TimeWeightedAggregator
|
||||
a.Value.Kind is HistorianValueKind.Double or HistorianValueKind.Int64
|
||||
&& b.Value.Kind is HistorianValueKind.Double or HistorianValueKind.Int64;
|
||||
|
||||
private long BucketStartTicks(long i) => _start + i * _plan.IntervalTicks;
|
||||
private long BucketStartTicks(long i) => _plan.LowerTicks(i);
|
||||
|
||||
private long BucketEnd(long i) => Math.Min(BucketStartTicks(i) + _plan.IntervalTicks, _end);
|
||||
private long BucketEnd(long i) => _plan.UpperTicks(i);
|
||||
|
||||
/// <summary>The ascending index of the bucket whose (half-open) span holds tick <paramref name="t"/> (t ≥ window start).</summary>
|
||||
private long FirstBucketAt(long t) => Math.Max(0, (t - _plan.OriginTicks) / _plan.IntervalTicks);
|
||||
|
||||
private void AddRaw(StoredSample from, long toTicks, double toValue) =>
|
||||
Distribute(from, toTicks, toValue, (i, area, ticks) =>
|
||||
@@ -286,7 +402,7 @@ public sealed class TimeWeightedAggregator
|
||||
var lo = Math.Max(from.TimeUtc.Ticks, _start);
|
||||
var hi = Math.Min(toTicks, _end);
|
||||
if (lo >= hi || _plan.BucketCount == 0) return;
|
||||
for (var i = (lo - _start) / _plan.IntervalTicks; i < _plan.BucketCount; i++)
|
||||
for (var i = FirstBucketAt(lo); i < _plan.BucketCount; i++)
|
||||
{
|
||||
var bs = BucketStartTicks(i);
|
||||
if (bs >= hi) break;
|
||||
@@ -296,13 +412,68 @@ public sealed class TimeWeightedAggregator
|
||||
}
|
||||
}
|
||||
|
||||
private void AddBridged(StoredSample from, long toTicks, double toValue) =>
|
||||
private void AddBridged(StoredSample from, long toTicks, double toValue, bool nonGood) =>
|
||||
Distribute(from, toTicks, toValue, (i, area, ticks) =>
|
||||
{
|
||||
_bridgedArea[i] += area;
|
||||
_bridgedTicks[i] += ticks;
|
||||
if (nonGood) _bridgedNonGood[i] = true;
|
||||
});
|
||||
|
||||
// ------------------------------------------------------------------------------------------ PercentGood / PercentBad
|
||||
|
||||
/// <summary>The region array a stepped region of <paramref name="s"/>'s status counts toward (null: Uncertain, neither).</summary>
|
||||
private long[]? RegionTicks(StoredSample s) =>
|
||||
s.IsGood ? _goodRegionTicks : !s.IsUsable ? _badRegionTicks : null;
|
||||
|
||||
/// <summary>
|
||||
/// The status regions between sample <paramref name="p"/> and the next sample <paramref name="s"/> (at
|
||||
/// <paramref name="t"/>). Inside the interval <paramref name="p"/> lies in, the region has <paramref name="p"/>'s
|
||||
/// status (stepped, Part 13 §5.4.3.31-34). In a later interval the region starts at that interval's simple bounding
|
||||
/// value, whose status is the simple-bounding signal's: Bad after a Bad <paramref name="p"/>; Uncertain when the
|
||||
/// segment runs into an outage or a sloped one has a non-Good end; else <paramref name="p"/>'s (the SDK's
|
||||
/// <c>GetSimpleBound</c>).
|
||||
/// </summary>
|
||||
private void AddStatusRegion(StoredSample p, StoredSample s, long t)
|
||||
{
|
||||
var lo = Math.Max(p.TimeUtc.Ticks, _start);
|
||||
var hi = Math.Min(t, _end);
|
||||
if (lo >= hi || _plan.BucketCount == 0) return;
|
||||
long[]? bound;
|
||||
if (!p.IsUsable) bound = _badRegionTicks;
|
||||
else if (!s.IsUsable) bound = null; // held into an outage ⇒ Uncertain
|
||||
else if (IsSloped(p, s)) bound = p.IsGood && s.IsGood ? _goodRegionTicks : null;
|
||||
else bound = RegionTicks(p);
|
||||
var own = RegionTicks(p);
|
||||
for (var i = FirstBucketAt(lo); i < _plan.BucketCount; i++)
|
||||
{
|
||||
var bs = BucketStartTicks(i);
|
||||
if (bs >= hi) break;
|
||||
var a = Math.Max(lo, bs);
|
||||
var b = Math.Min(hi, BucketEnd(i));
|
||||
if (a >= b) continue;
|
||||
var target = p.TimeUtc.Ticks >= bs ? own : bound;
|
||||
if (target is not null) target[i] += b - a;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Adds <c>[from, to)</c>, clipped to the window, to <paramref name="target"/> per bucket.</summary>
|
||||
private void AddRegion(long[]? target, long from, long to)
|
||||
{
|
||||
if (target is null) return;
|
||||
var lo = Math.Max(from, _start);
|
||||
var hi = Math.Min(to, _end);
|
||||
if (lo >= hi || _plan.BucketCount == 0) return;
|
||||
for (var i = FirstBucketAt(lo); i < _plan.BucketCount; i++)
|
||||
{
|
||||
var bs = BucketStartTicks(i);
|
||||
if (bs >= hi) break;
|
||||
var a = Math.Max(lo, bs);
|
||||
var b = Math.Min(hi, BucketEnd(i));
|
||||
if (a < b) target[i] += b - a;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Splits the linear segment <c>[from.t, toTicks)</c> (value <c>from.v → toValue</c>) at the bucket edges, clipped to
|
||||
/// the window, and hands each bucket its trapezoid area (value·ticks) and duration. A non-numeric start sample
|
||||
@@ -320,7 +491,7 @@ public sealed class TimeWeightedAggregator
|
||||
var span = (double)(toTicks - t0);
|
||||
double ValueAt(long x) => toValue == v0 ? v0 : v0 + (toValue - v0) * ((x - t0) / span);
|
||||
|
||||
for (var i = (lo - _start) / _plan.IntervalTicks; i < _plan.BucketCount; i++)
|
||||
for (var i = FirstBucketAt(lo); i < _plan.BucketCount; i++)
|
||||
{
|
||||
var bs = BucketStartTicks(i);
|
||||
if (bs >= hi) break;
|
||||
@@ -335,13 +506,12 @@ public sealed class TimeWeightedAggregator
|
||||
// ------------------------------------------------------------------------------------------ Interpolative
|
||||
|
||||
/// <summary>
|
||||
/// The time of Interpolative point <paramref name="j"/> (ascending time). Forward: the bucket starts
|
||||
/// <c>start + j·interval</c>. Reverse (Part 13 anchors a reverse read at its later time, the request's start):
|
||||
/// <c>End − (n−1−j)·interval</c>, i.e. <c>End, End − interval, …</c> once <see cref="BucketPlan.Assemble"/> has
|
||||
/// The time of Interpolative point <paramref name="j"/> (ascending time): the bucket's timestamp. Forward: the
|
||||
/// bucket starts <c>start + j·interval</c>. Reverse (Part 13 anchors a reverse read at its later time, the
|
||||
/// request's start): the later edges <c>End, End − interval, …</c> once <see cref="BucketPlan.Assemble"/> has
|
||||
/// reversed the order — so a reverse read 10:00 → 09:00 at 1 h yields the value at 10:00.
|
||||
/// </summary>
|
||||
private long PointTicks(long j) =>
|
||||
_plan.Reversed ? _end - (_plan.BucketCount - 1 - j) * _plan.IntervalTicks : BucketStartTicks(j);
|
||||
private long PointTicks(long j) => _plan.BucketTimestamp(j).Ticks;
|
||||
|
||||
private DateTime PointTime(long j) => new(PointTicks(j), DateTimeKind.Utc);
|
||||
|
||||
|
||||
@@ -5,13 +5,18 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer;
|
||||
/// <summary>
|
||||
/// Push-down T-SQL for processed (aggregate) reads (spec §4 "Processed"), one query per
|
||||
/// <see cref="AggregateFamily"/> (<see cref="AggregateFamilies.FamilyOf"/> picks it). Every query takes
|
||||
/// <c>@id</c> (int TagId), <c>@start</c>/<c>@end</c> (<c>datetime2(7)</c>, the forward window
|
||||
/// <c>[start, end)</c>) and <c>@intervalTicks</c> (bigint bucket width in 100 ns ticks), and returns one row
|
||||
/// per <b>non-empty</b> bucket. The caller generates every interval and fills the empty ones.
|
||||
/// <c>@id</c> (int TagId), <c>@start</c>/<c>@end</c> (<c>datetime2(7)</c>, the earlier / later window edges —
|
||||
/// <see cref="BucketPlan.StartUtc"/>/<see cref="BucketPlan.EndUtc"/>), <c>@intervalTicks</c> (bigint bucket width in
|
||||
/// 100 ns ticks) and <c>@bucketAdjustTicks</c> (bigint, <see cref="BucketPlan.BucketIndexAdjustTicks"/>), and returns
|
||||
/// one row per <b>non-empty</b> bucket (ascending bucket index). The caller generates every interval and fills the
|
||||
/// empty ones.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>Bucketing.</b> <c>b = elapsedTicks / @intervalTicks</c>, where
|
||||
/// <b>Bucketing.</b> <c>b = (elapsedTicks + @bucketAdjustTicks) / @intervalTicks</c> (#677: the adjust is 0 for a
|
||||
/// forward read and re-anchors a reverse read's grid at its later time — <see cref="BucketPlan.IndexOf"/> is the C#
|
||||
/// statement of the same math). The window is <c>[@start, @end)</c> forward and <c>(@start, @end]</c> reverse (a
|
||||
/// reverse read includes its later time), selected by the <c>reverse</c> argument. <c>elapsedTicks</c> is
|
||||
/// <c>elapsedTicks = DATEDIFF_BIG(microsecond, @start, t) * 10 + tick-of-microsecond(t) − tick-of-microsecond(@start)</c>
|
||||
/// and <c>tick-of-microsecond(x) = (DATEPART(nanosecond, x) / 100) % 10</c>. The spec sketched
|
||||
/// <c>millisecond</c>, but <c>DATEDIFF</c> counts unit <i>boundaries crossed</i>, not elapsed time, so with a
|
||||
@@ -27,7 +32,8 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer;
|
||||
/// and bit 30): <i>usable</i> = severity Good or Uncertain (<c>StatusCode >= 0</c>) AND a stored value
|
||||
/// (<c>ValueKind <> 0</c>) — an <c>Empty</c> sample is an outage marker and counts as Bad whatever its
|
||||
/// code; <i>good</i> = usable with severity Good (<c>StatusCode < 0x40000000</c>); <i>bad</i> = not
|
||||
/// usable. Every value column below is computed over usable samples only.
|
||||
/// usable. Average/Minimum/Maximum/Range/*ActualTime value columns are computed over <i>good</i> samples only
|
||||
/// (#684); Start/End/Delta picks over usable samples (or every sample).
|
||||
/// </para>
|
||||
/// Common result columns (ordinals 0–5): <c>Bucket, TotalCount, UsableCount, GoodCount, BadCount,
|
||||
/// NonNumericCount</c> (usable String/DateTime samples). Family columns follow.
|
||||
@@ -41,54 +47,119 @@ internal static class SqlAggregateQueries
|
||||
private const string NonNumericKinds = "4, 5";
|
||||
|
||||
/// <summary>
|
||||
/// Average / Minimum / Maximum / Total / Count / Range / PercentGood / PercentBad.
|
||||
/// Family columns: <c>6 AvgValue, 7 MinValue, 8 MaxValue, 9 SumValue</c> (float, NULL when no usable
|
||||
/// numeric sample).
|
||||
/// Average / Minimum / Maximum / Count / Range (PercentGood / PercentBad and Total are time-weighted, not here).
|
||||
/// Family columns: <c>6 AvgValue, 7 MinValue, 8 MaxValue, 9 SumValue</c> over the <b>Good</b> numeric samples (float,
|
||||
/// NULL when none — #684: Part 13 and the SDK compute these over Good values only), then <c>10 UncMin, 11 UncMax</c>
|
||||
/// over the Uncertain ones (the Part 13 Minimum/Maximum/Range status rule, <see cref="AggregateEvaluator"/>), and
|
||||
/// <c>12 GoodNonNumericCount</c>.
|
||||
/// </summary>
|
||||
/// <param name="sample">Schema-qualified <c>Sample</c> table name.</param>
|
||||
/// <returns>The SQL text.</returns>
|
||||
public static string Simple(string sample) =>
|
||||
Classified(sample) +
|
||||
/// <param name="reverse">True for a reverse read (window <c>(@start, @end]</c>).</param>
|
||||
public static string Simple(string sample, bool reverse = false) =>
|
||||
Classified(sample, reverse) +
|
||||
"SELECT b, " + CommonCounts() + ", " +
|
||||
"AVG(CASE WHEN u = 1 THEN NumValue END) AS AvgValue, " +
|
||||
"MIN(CASE WHEN u = 1 THEN NumValue END) AS MinValue, " +
|
||||
"MAX(CASE WHEN u = 1 THEN NumValue END) AS MaxValue, " +
|
||||
"SUM(CASE WHEN u = 1 THEN NumValue END) AS SumValue " +
|
||||
"AVG(CASE WHEN g = 1 THEN NumValue END) AS AvgValue, " +
|
||||
"MIN(CASE WHEN g = 1 THEN NumValue END) AS MinValue, " +
|
||||
"MAX(CASE WHEN g = 1 THEN NumValue END) AS MaxValue, " +
|
||||
"SUM(CASE WHEN g = 1 THEN NumValue END) AS SumValue, " +
|
||||
UncertainExtremes() + " " +
|
||||
"FROM s GROUP BY b ORDER BY b;";
|
||||
|
||||
/// <summary>
|
||||
/// Start / End / Delta: the first and last usable sample of each bucket by <c>SourceTimeUtc</c>.
|
||||
/// Family columns: <c>6..10</c> first sample (<c>ValueKind, NumValue, IntValue, StrValue, SourceTimeUtc</c>),
|
||||
/// <c>11..15</c> last sample (same order); all NULL when the bucket has no usable sample.
|
||||
/// </summary>
|
||||
/// <param name="sample">Schema-qualified <c>Sample</c> table name.</param>
|
||||
/// <returns>The SQL text.</returns>
|
||||
public static string StartEnd(string sample) =>
|
||||
Classified(sample) +
|
||||
", st AS (SELECT b, " + CommonCounts() + " FROM s GROUP BY b)" +
|
||||
", f AS (SELECT b, ValueKind, NumValue, IntValue, StrValue, SourceTimeUtc, " +
|
||||
"ROW_NUMBER() OVER (PARTITION BY b ORDER BY SourceTimeUtc ASC) AS rf, " +
|
||||
"ROW_NUMBER() OVER (PARTITION BY b ORDER BY SourceTimeUtc DESC) AS rl FROM s WHERE u = 1) " +
|
||||
"SELECT st.*, " +
|
||||
"x.ValueKind, x.NumValue, x.IntValue, x.StrValue, x.SourceTimeUtc, " +
|
||||
"y.ValueKind, y.NumValue, y.IntValue, y.StrValue, y.SourceTimeUtc " +
|
||||
"FROM st LEFT JOIN f x ON x.b = st.b AND x.rf = 1 LEFT JOIN f y ON y.b = st.b AND y.rl = 1 ORDER BY st.b;";
|
||||
/// <summary>Ordinal of the StartEnd family's first-sample-of-any-quality columns (<see cref="StartEnd"/>).</summary>
|
||||
public const int FirstAnyColumn = FirstFamilyColumn + 10;
|
||||
|
||||
/// <summary>
|
||||
/// MinimumActualTime / MaximumActualTime: the usable sample with the smallest (largest) <c>NumValue</c>,
|
||||
/// ties broken by the earliest <c>SourceTimeUtc</c>. Family columns: <c>6 NumValue, 7 SourceTimeUtc</c>
|
||||
/// (NULL when the bucket has no usable numeric sample).
|
||||
/// Start / End / Delta: the first and last usable sample of each bucket by <c>SourceTimeUtc</c> (Delta's value), and
|
||||
/// the first and last sample of <b>any</b> quality (Start / End, and Delta's status — #680).
|
||||
/// Family columns: <c>6..10</c> first usable sample (<c>ValueKind, NumValue, IntValue, StrValue, SourceTimeUtc</c>),
|
||||
/// <c>11..15</c> last usable sample (same order) — all NULL when the bucket has no usable sample; then
|
||||
/// <c>16..21</c> the first sample of any quality (<c>SourceTimeUtc, ValueKind, NumValue, IntValue, StrValue,
|
||||
/// StatusCode</c>) and <c>22..27</c> the last (same order) — present for every returned (non-empty) bucket.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>Aggregate, then join back on the key (#683).</b> One <c>GROUP BY</c> over the window computes the counts and
|
||||
/// each bucket's four pick TIMES — <c>MIN</c>/<c>MAX(SourceTimeUtc)</c> over every sample (any quality) and over
|
||||
/// the usable ones — and each pick's columns are then fetched by a seek on the clustered key
|
||||
/// <c>(TagId, SourceTimeUtc)</c> (unique, so one row per pick; a NULL usable time joins nothing ⇒ NULL columns,
|
||||
/// as before). Output is identical to the previous windowed form (one <c>ROW_NUMBER</c> pass + conditional
|
||||
/// <c>MAX</c>, live-gate 8 F1) — <c>EXCEPT</c> both ways is empty on 6M rows, forward and reverse — without that
|
||||
/// form's four window sorts, which spilled: on 6M rows / 30 days, 30 d @ 1 h ~3.4 s vs 18–32 s, 7 d @ 1 d
|
||||
/// ~0.8 s vs 5.2–6.1 s, 1 d @ 1 min ~0.16 s vs 0.31–0.40 s (docs/Historian.md).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// ⚠️ The key seeks are separate reads of the table within one statement, so under READ COMMITTED a sample
|
||||
/// deleted (HistoryUpdate DeleteRaw/DeleteAtTime) between the aggregate and its seek leaves that pick NULL while
|
||||
/// the counts still include it. The reader maps a NULL pick to "absent" (<c>ReadStoredOrNull</c>, live-gate 8 F1)
|
||||
/// and the evaluator answers that bucket's Start/End/Delta <c>BadNoData</c> — transient, until the next read. A
|
||||
/// sample replaced in between is picked with its new value (same time) — except that a usable pick re-checks
|
||||
/// usability in its join (the exact <c>u</c> predicate), so a Replace that turned it Bad/Empty reads as absent
|
||||
/// (<c>BadNoData</c>), never as a Bad row's value under Delta. A sample inserted in between is in neither half
|
||||
/// (the pick times come from the aggregate).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Every pick is an exact-key seek, by hint (#683 live gate 9 F1).</b> Each pick join carries
|
||||
/// <c>WITH (FORCESEEK (PK_Sample (TagId, SourceTimeUtc)))</c>, so it can only be a nested-loops seek on BOTH key
|
||||
/// columns — one row per bucket per pick — whatever the statistics say. Without it the plan was the optimizer's
|
||||
/// choice, and both alternatives it found were pathological: keyed on the pick time alone, a hash/merge join whose
|
||||
/// inner input was the tag's whole history (review of 910befc8: 43,657 vs 361 logical reads); bounded to the window
|
||||
/// as well (4a3b1256), a nested-loops RANGE seek over the window with the pick time as a residual, once per bucket
|
||||
/// — O(buckets × window rows), chosen under understated statistics and then reused from the plan cache (gate 9:
|
||||
/// 1 d @ 1 min 20 s; here 47 s, 3.6M reads, vs 0.25 s). With the hint neither shape exists (a forced
|
||||
/// <c>HASH</c>/<c>MERGE JOIN</c> is refused, error 8622), so the window bound is no longer needed in the join.
|
||||
/// ⚠️ The hint names the clustered primary key <c>PK_Sample</c> created by <c>001_initial.sql</c>; a hand-built
|
||||
/// schema must keep that name.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
/// <param name="sample">Schema-qualified <c>Sample</c> table name.</param>
|
||||
/// <returns>The SQL text.</returns>
|
||||
/// <param name="reverse">True for a reverse read (window <c>(@start, @end]</c>).</param>
|
||||
public static string StartEnd(string sample, bool reverse = false)
|
||||
{
|
||||
// A pick's columns from its joined row, in the established order: usable picks (kind, num, int, str, time), any-
|
||||
// quality picks (time, kind, num, int, str, status).
|
||||
static string Usable(string a) => $"{a}.ValueKind, {a}.NumValue, {a}.IntValue, {a}.StrValue, {a}.SourceTimeUtc";
|
||||
static string Any(string a) => $"{a}.SourceTimeUtc, {a}.ValueKind, {a}.NumValue, {a}.IntValue, {a}.StrValue, {a}.StatusCode";
|
||||
// An exact-key seek per pick, forced (see remarks). A usable pick also re-checks usability (the exact `u`
|
||||
// predicate), so a sample a HistoryUpdate Replace turned Bad/Empty in between joins nothing ⇒ absent ⇒ BadNoData,
|
||||
// never a Bad row's value.
|
||||
string Join(string a, string time, bool usable) =>
|
||||
$"LEFT JOIN {sample} {a} WITH (FORCESEEK (PK_Sample (TagId, SourceTimeUtc))) ON {a}.TagId = @id AND {a}.SourceTimeUtc = st.{time}" +
|
||||
(usable ? $" AND {a}.StatusCode >= 0 AND {a}.ValueKind <> 0" : "") + " ";
|
||||
return Classified(sample, reverse) +
|
||||
", st AS (SELECT b, " + CommonCounts() + ", " +
|
||||
"MIN(CASE WHEN u = 1 THEN SourceTimeUtc END) AS fu, MAX(CASE WHEN u = 1 THEN SourceTimeUtc END) AS lu, " +
|
||||
"MIN(SourceTimeUtc) AS fa, MAX(SourceTimeUtc) AS la FROM s GROUP BY b) " +
|
||||
"SELECT st.b, st.TotalCount, st.UsableCount, st.GoodCount, st.BadCount, st.NonNumericCount, " +
|
||||
Usable("x") + ", " + Usable("y") + ", " + Any("p") + ", " + Any("q") + " FROM st " +
|
||||
Join("x", "fu", usable: true) + Join("y", "lu", usable: true) + Join("p", "fa", usable: false) + Join("q", "la", usable: false) + "ORDER BY st.b;";
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// MinimumActualTime / MaximumActualTime: the <b>Good</b> sample (#684) with the smallest (largest) <c>NumValue</c>,
|
||||
/// ties broken by the earliest <c>SourceTimeUtc</c>. Family columns: <c>6 NumValue, 7 SourceTimeUtc</c>
|
||||
/// (NULL when the bucket has no Good numeric sample), then <c>8 UncMin, 9 UncMax, 10 GoodNonNumericCount</c> as in <see cref="Simple"/>.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Deliberately still two references to <c>s</c> (counts + the ranked extreme), unlike <see cref="StartEnd"/>:
|
||||
/// every single-scan form measured (a <c>ROW_NUMBER</c> or windowed <c>MIN</c>/<c>MAX</c> over every row) was
|
||||
/// 1.2–2× slower on day-plus windows than ranking only the filtered usable rows. Live-gate 8 F1's inconsistency —
|
||||
/// a sample committed between the two scans, counted but not ranked — is instead absorbed by the reader: a
|
||||
/// NULL extreme in a bucket with usable samples evaluates to <c>BadNoData</c> (never a value-less Good).
|
||||
/// </remarks>
|
||||
/// <param name="sample">Schema-qualified <c>Sample</c> table name.</param>
|
||||
/// <param name="maximum">True for MaximumActualTime.</param>
|
||||
/// <returns>The SQL text.</returns>
|
||||
public static string ActualTime(string sample, bool maximum) =>
|
||||
Classified(sample) +
|
||||
", st AS (SELECT b, " + CommonCounts() + " FROM s GROUP BY b)" +
|
||||
/// <param name="reverse">True for a reverse read (window <c>(@start, @end]</c>).</param>
|
||||
public static string ActualTime(string sample, bool maximum, bool reverse = false) =>
|
||||
Classified(sample, reverse) +
|
||||
", st AS (SELECT b, " + CommonCounts() + ", " + UncertainExtremes() + " FROM s GROUP BY b)" +
|
||||
", r AS (SELECT b, NumValue, SourceTimeUtc, " +
|
||||
"ROW_NUMBER() OVER (PARTITION BY b ORDER BY NumValue " + (maximum ? "DESC" : "ASC") + ", SourceTimeUtc ASC) AS rn " +
|
||||
"FROM s WHERE u = 1 AND NumValue IS NOT NULL) " +
|
||||
"SELECT st.*, r.NumValue, r.SourceTimeUtc " +
|
||||
"FROM s WHERE g = 1 AND NumValue IS NOT NULL) " +
|
||||
"SELECT st.b, st.TotalCount, st.UsableCount, st.GoodCount, st.BadCount, st.NonNumericCount, " +
|
||||
"r.NumValue, r.SourceTimeUtc, st.UncMin, st.UncMax, st.GoodNonNumericCount " +
|
||||
"FROM st LEFT JOIN r ON r.b = st.b AND r.rn = 1 ORDER BY st.b;";
|
||||
|
||||
/// <summary>
|
||||
@@ -121,14 +192,40 @@ internal static class SqlAggregateQueries
|
||||
$"SELECT TOP 1 {cols} {key}SourceTimeUtc >= @end{usable} ORDER BY SourceTimeUtc ASC;";
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The tag's data range as one row of two columns (each <c>NULL</c> when none): <c>0</c> the earliest stored sample
|
||||
/// of any quality at or before the window's later edge, <c>1</c> the latest at or after its earlier edge. Used by the
|
||||
/// start-of-data (#677) and end-of-data (#680) Partial rules and by Count's in-range empty interval (#680) — only when
|
||||
/// <see cref="AggregateEvaluator.NeedsDataRange"/> says the answer can change a result. Each is a <c>MIN</c> /
|
||||
/// <c>MAX</c> over the <c>(TagId, SourceTimeUtc)</c> key — a single seek — bounded by the window (a sample past the
|
||||
/// later edge cannot start the data inside it, nor one before the earlier edge end it there; a NULL then means "the
|
||||
/// data range does not reach the window").
|
||||
/// </summary>
|
||||
/// <param name="sample">Schema-qualified <c>Sample</c> table name.</param>
|
||||
/// <returns>The SQL text.</returns>
|
||||
public static string DataRange(string sample) =>
|
||||
$"SELECT (SELECT MIN(SourceTimeUtc) FROM {sample} WHERE TagId = @id AND SourceTimeUtc <= @end), " +
|
||||
$"(SELECT MAX(SourceTimeUtc) FROM {sample} WHERE TagId = @id AND SourceTimeUtc >= @start);";
|
||||
|
||||
/// <summary>The <c>s</c> CTE: the window's samples with bucket index and the usable/good flags.</summary>
|
||||
private static string Classified(string sample) =>
|
||||
private static string Classified(string sample, bool reverse) =>
|
||||
"WITH s AS (SELECT (DATEDIFF_BIG(microsecond, @start, SourceTimeUtc) * 10 " +
|
||||
"+ (DATEPART(nanosecond, SourceTimeUtc) / 100) % 10 - (DATEPART(nanosecond, @start) / 100) % 10) / @intervalTicks AS b, " +
|
||||
"SourceTimeUtc, ValueKind, NumValue, IntValue, StrValue, " +
|
||||
"+ (DATEPART(nanosecond, SourceTimeUtc) / 100) % 10 - (DATEPART(nanosecond, @start) / 100) % 10 " +
|
||||
"+ @bucketAdjustTicks) / @intervalTicks AS b, " +
|
||||
"SourceTimeUtc, ValueKind, NumValue, IntValue, StrValue, StatusCode, " +
|
||||
"CASE WHEN StatusCode >= 0 AND ValueKind <> 0 THEN 1 ELSE 0 END AS u, " +
|
||||
"CASE WHEN StatusCode >= 0 AND StatusCode < 1073741824 AND ValueKind <> 0 THEN 1 ELSE 0 END AS g " +
|
||||
$"FROM {sample} WHERE TagId = @id AND SourceTimeUtc >= @start AND SourceTimeUtc < @end) ";
|
||||
$"FROM {sample} WHERE TagId = @id AND " +
|
||||
(reverse ? "SourceTimeUtc > @start AND SourceTimeUtc <= @end) " : "SourceTimeUtc >= @start AND SourceTimeUtc < @end) ");
|
||||
|
||||
/// <summary>
|
||||
/// The minimum and maximum <b>Uncertain</b> (usable, not Good) numeric value of the bucket — <c>UncMin, UncMax</c>
|
||||
/// (#684). Not result values: Part 13 makes Minimum/Maximum/Range Uncertain when one lies beyond the Good extreme.
|
||||
/// Then <c>GoodNonNumericCount</c>, the Good String/DateTime samples (the Good-value aggregates' refusal check).
|
||||
/// </summary>
|
||||
private static string UncertainExtremes() =>
|
||||
"MIN(CASE WHEN u = 1 AND g = 0 THEN NumValue END) AS UncMin, MAX(CASE WHEN u = 1 AND g = 0 THEN NumValue END) AS UncMax, " +
|
||||
$"SUM(CASE WHEN g = 1 AND ValueKind IN ({NonNumericKinds}) THEN 1 ELSE 0 END) AS GoodNonNumericCount";
|
||||
|
||||
/// <summary>The five common count columns (after <c>b</c>).</summary>
|
||||
private static string CommonCounts() =>
|
||||
|
||||
@@ -106,7 +106,7 @@ internal sealed class SqlHistorianConnection
|
||||
/// <param name="reader">Reader positioned on a row.</param>
|
||||
/// <param name="ordinal">Column ordinal.</param>
|
||||
/// <returns>UTC timestamp.</returns>
|
||||
public static DateTime GetUtc(SqlDataReader reader, int ordinal) =>
|
||||
public static DateTime GetUtc(System.Data.Common.DbDataReader reader, int ordinal) =>
|
||||
DateTime.SpecifyKind(reader.GetDateTime(ordinal), DateTimeKind.Utc);
|
||||
|
||||
/// <summary>True for connection/timeout/deadlock/throttling errors worth retrying.</summary>
|
||||
|
||||
+115
-27
@@ -1,4 +1,5 @@
|
||||
using System.Data;
|
||||
using System.Data.Common;
|
||||
using System.Text;
|
||||
using Microsoft.Data.SqlClient;
|
||||
using Microsoft.Extensions.Logging;
|
||||
@@ -33,7 +34,7 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer;
|
||||
/// carries a value; an <c>Empty</c>-kind sample is an outage marker (comms-loss <c>BadNoCommunication</c>
|
||||
/// or a driver's Bad publish) and is Bad whatever its code. Raw returns every sample as stored, outage
|
||||
/// markers included. Aggregates exclude Bad samples from the math, report the interval Uncertain when
|
||||
/// any non-Good sample is present, and count them in PercentBad. AtTime never interpolates across a Bad
|
||||
/// any non-Good sample is present, and count their duration as Bad time in PercentBad. AtTime never interpolates across a Bad
|
||||
/// neighbour and reports a time inside an outage with the outage's Bad code (see <see cref="ReadAtTimeAsync"/>).
|
||||
/// <b>An Empty sample is always served Bad:</b> a row with no value whose stored status is not Bad (only
|
||||
/// possible for rows written before the writer forced Empty samples Bad) is read back as <c>BadNoData</c>,
|
||||
@@ -131,6 +132,7 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
|
||||
catch (Exception ex)
|
||||
{
|
||||
RecordFailure(ex);
|
||||
if (IsTransportFailure(ex)) throw new System.IO.IOException("Historian read transport is unavailable.", ex);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
@@ -154,6 +156,12 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
|
||||
var bounded = maxValues > 0 && maxValues < int.MaxValue;
|
||||
try
|
||||
{
|
||||
// #674: a v1 schema has no SampleModified. NotSupportedException is the interface's "no trail" signal (the
|
||||
// node manager reports BadHistoryOperationUnsupported); it is not a backend failure and sends no query.
|
||||
await _bootstrap.EnsureAsync(cancellationToken).ConfigureAwait(false);
|
||||
if (!_bootstrap.SupportsModifiedTrail)
|
||||
throw new NotSupportedException(
|
||||
$"Historian schema [{_conn.Schema}] is below version {SqlServerSchemaBootstrapper.ModifiedTrailVersion}: no modified-value history.");
|
||||
var values = new List<HistoryModifiedValue>();
|
||||
var hasMore = false;
|
||||
var tagId = await ResolveAsync(fullReference, cancellationToken).ConfigureAwait(false);
|
||||
@@ -219,9 +227,10 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
|
||||
RecordSuccess();
|
||||
return new HistoryModifiedReadResult(values, hasMore);
|
||||
}
|
||||
catch (Exception ex)
|
||||
catch (Exception ex) when (ex is not NotSupportedException)
|
||||
{
|
||||
RecordFailure(ex);
|
||||
if (IsTransportFailure(ex)) throw new System.IO.IOException("Historian read transport is unavailable.", ex);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
@@ -299,12 +308,30 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
|
||||
catch (Exception ex)
|
||||
{
|
||||
RecordFailure(ex);
|
||||
if (IsTransportFailure(ex)) throw new System.IO.IOException("Historian read transport is unavailable.", ex);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------------------------------ Events
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>Reverse raw reads are served natively (<c>(end, start]</c>, <c>ORDER BY … DESC</c>; #669/#686).</remarks>
|
||||
public bool SupportsReverseRaw => true;
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks><c>0</c>: a capped raw read is one bounded statement — no backend row ceiling (Gitea #694).</remarks>
|
||||
public int MaxRawValuesPerRead => 0;
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>Reverse events reads are served natively (<c>ORDER BY … DESC</c>, Gitea #675).</remarks>
|
||||
public bool SupportsReverseEvents => true;
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>The seventeen <see cref="AggregateFamilies.Served"/> — shared with the other SQL backend (#682).</remarks>
|
||||
public IReadOnlySet<HistoryAggregateType> SupportedAggregates => AggregateFamilies.Served;
|
||||
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>
|
||||
/// <c>[start, end)</c> by <c>EventTimeUtc</c> (a reversed window is swapped), ordered by
|
||||
@@ -314,8 +341,18 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
|
||||
/// of <c>"{EventTimeUtc.Ticks}|{EventId}"</c> of that first not-returned event (opaque to callers; only
|
||||
/// its presence is contractual).
|
||||
/// </remarks>
|
||||
public Task<HistoricalEventsResult> ReadEventsAsync(
|
||||
string? sourceName, DateTime startUtc, DateTime endUtc, int maxEvents, CancellationToken cancellationToken) =>
|
||||
ReadEventsAsync(sourceName, startUtc, endUtc, maxEvents, reverse: false, cancellationToken);
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>
|
||||
/// Forward is the five-argument read. Reverse (Gitea #675) is <c>(lo, hi]</c> ordered by
|
||||
/// <c>(EventTimeUtc, EventId)</c> DESCENDING — the clustered index scanned backwards — so <c>TOP</c> keeps the latest
|
||||
/// events; truncation and the continuation point work as forward.
|
||||
/// </remarks>
|
||||
public async Task<HistoricalEventsResult> ReadEventsAsync(
|
||||
string? sourceName, DateTime startUtc, DateTime endUtc, int maxEvents, CancellationToken cancellationToken)
|
||||
string? sourceName, DateTime startUtc, DateTime endUtc, int maxEvents, bool reverse, CancellationToken cancellationToken)
|
||||
{
|
||||
// No stored event can carry a source longer than the column (the alarm writer dead-letters it rather than
|
||||
// truncating), so an over-long filter matches nothing — never a truncated prefix of someone else's source.
|
||||
@@ -328,9 +365,10 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
|
||||
var bounded = maxEvents > 0 && maxEvents < int.MaxValue;
|
||||
var sql =
|
||||
$"SELECT {(bounded ? "TOP (@take) " : "")}EventId, SourceName, EventTimeUtc, ReceivedTimeUtc, Message, Severity " +
|
||||
$"FROM {_conn.Q("Event")} WHERE EventTimeUtc >= @lo AND EventTimeUtc < @hi" +
|
||||
$"FROM {_conn.Q("Event")} WHERE " +
|
||||
(reverse ? "EventTimeUtc > @lo AND EventTimeUtc <= @hi" : "EventTimeUtc >= @lo AND EventTimeUtc < @hi") +
|
||||
(sourceName is null ? "" : " AND SourceName = @source") +
|
||||
" ORDER BY EventTimeUtc ASC, EventId ASC";
|
||||
(reverse ? " ORDER BY EventTimeUtc DESC, EventId DESC" : " ORDER BY EventTimeUtc ASC, EventId ASC");
|
||||
|
||||
await using var conn = await _conn.OpenAsync(cancellationToken).ConfigureAwait(false);
|
||||
await using var cmd = _conn.CreateCommand(conn, sql);
|
||||
@@ -366,6 +404,7 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
|
||||
catch (Exception ex)
|
||||
{
|
||||
RecordFailure(ex);
|
||||
if (IsTransportFailure(ex)) throw new System.IO.IOException("Historian read transport is unavailable.", ex);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
@@ -378,16 +417,21 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
|
||||
/// One result per interval of <c>[start, end)</c>, bucket <c>i</c> = <c>[start + i·interval,
|
||||
/// start + (i+1)·interval)</c> (start-inclusive, end-exclusive to the tick; the last bucket is truncated at
|
||||
/// <c>end</c>). <paramref name="interval"/> <= 0 ⇒ one interval <c>[start, end)</c>. <c>start == end</c>
|
||||
/// ⇒ no intervals. <c>end < start</c> (reverse) ⇒ the window is swapped, computed forward, and returned
|
||||
/// in descending bucket order (buckets stay anchored at the earlier time).
|
||||
/// ⇒ no intervals. <c>end < start</c> (reverse, #677) ⇒ Part 13 / SDK anchoring: window <c>(end, start]</c>,
|
||||
/// buckets stepping back from <c>start</c> (the partial one is the earliest), each stamped with its later edge,
|
||||
/// returned latest first (<see cref="BucketPlan"/>).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Per interval: no samples ⇒ <c>BadNoData</c>. PercentGood / PercentBad ⇒ <c>100 × good|bad / all</c> over
|
||||
/// every sample, status Good. Otherwise no usable sample ⇒ <c>BadNoData</c>; a numeric aggregate with a usable
|
||||
/// String/DateTime sample ⇒ <c>BadAggregateInvalidInputs</c>; else the value, Good when every sample is Good,
|
||||
/// <c>UncertainDataSubNormal</c> when any is Bad or Uncertain. Every status — including <c>BadNoData</c> —
|
||||
/// carries the Calculated info bits. Timestamps are the bucket start, except Minimum/MaximumActualTime
|
||||
/// (the sample's time; ties ⇒ earliest).
|
||||
/// Per-sample aggregates, per interval: no samples ⇒ <c>BadNoData</c>. Otherwise no usable sample ⇒
|
||||
/// <c>BadNoData</c>; a numeric aggregate with a usable String/DateTime sample ⇒ <c>BadAggregateInvalidInputs</c>;
|
||||
/// else the value, Good when every sample is Good, <c>UncertainDataSubNormal</c> when any is Bad or Uncertain.
|
||||
/// Every calculated status — including <c>BadNoData</c> — carries the Calculated info bits. Timestamps are the
|
||||
/// bucket start (its later edge on a reverse read), except Minimum/MaximumActualTime (the sample's time; ties ⇒
|
||||
/// earliest) and Start/End. Every aggregate but Average sets the Partial bit on the bucket the tag's earliest
|
||||
/// sample lies strictly inside (forward, #677) or its latest sample lies in (#680; reverse: strictly inside).
|
||||
/// PercentGood / PercentBad are duration-based since #680 (time-weighted family, below). #680: Count counts Good samples (value-based status; 0/Good on an empty bucket inside the tag's
|
||||
/// data range), Start/End are the raw first/last sample of any quality, Delta is Uncertain only when the bucket's
|
||||
/// first or last sample is not usable (see <see cref="AggregateEvaluator"/>).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Values: Count is <see cref="int"/>; Start/End keep the sample's type; every other aggregate is
|
||||
@@ -410,7 +454,6 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
|
||||
try
|
||||
{
|
||||
var now = _time.GetUtcNow().UtcDateTime;
|
||||
var evaluated = new Dictionary<long, DataValueSnapshot>();
|
||||
var tagId = plan.BucketCount == 0 ? null : await ResolveAsync(fullReference, cancellationToken).ConfigureAwait(false);
|
||||
if (family == AggregateFamily.TimeWeighted)
|
||||
{
|
||||
@@ -420,30 +463,47 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
|
||||
RecordSuccess();
|
||||
return new HistoryReadResult(twResult, ContinuationPoint: null);
|
||||
}
|
||||
var rows = new List<BucketRow>();
|
||||
DateTime? earliest = null, latest = null;
|
||||
if (tagId is { } id)
|
||||
{
|
||||
var sample = _conn.Q("Sample");
|
||||
var sql = family switch
|
||||
{
|
||||
AggregateFamily.Simple => SqlAggregateQueries.Simple(sample),
|
||||
AggregateFamily.StartEnd => SqlAggregateQueries.StartEnd(sample),
|
||||
_ => SqlAggregateQueries.ActualTime(sample, aggregate == HistoryAggregateType.MaximumActualTime),
|
||||
AggregateFamily.Simple => SqlAggregateQueries.Simple(sample, plan.Reversed),
|
||||
AggregateFamily.StartEnd => SqlAggregateQueries.StartEnd(sample, plan.Reversed),
|
||||
_ => SqlAggregateQueries.ActualTime(sample, aggregate == HistoryAggregateType.MaximumActualTime, plan.Reversed),
|
||||
};
|
||||
// FIRST result set when needed: the tag's data range (earliest + latest sample) — the start/end-of-data Partial
|
||||
// rules (#677/#680 — any per-sample aggregate but Average, either direction) and Count's in-range empty interval
|
||||
// (#680). Read before the bucket statement (#683): under READ COMMITTED the two statements can see different
|
||||
// rows, and a sample appended in between must reach the buckets rather than only the range — the opposite
|
||||
// order turned such a bucket into a Good Count of 0.
|
||||
var needsRange = AggregateEvaluator.NeedsDataRange(aggregate, plan);
|
||||
if (needsRange) sql = SqlAggregateQueries.DataRange(sample) + " " + sql;
|
||||
await using var conn = await _conn.OpenAsync(cancellationToken).ConfigureAwait(false);
|
||||
await using var cmd = _conn.CreateCommand(conn, sql);
|
||||
cmd.Parameters.Add(new SqlParameter("@id", SqlDbType.Int) { Value = id });
|
||||
SqlHistorianConnection.AddDateTime2(cmd, "@start", DateTime.SpecifyKind(plan.StartUtc, DateTimeKind.Utc));
|
||||
SqlHistorianConnection.AddDateTime2(cmd, "@end", DateTime.SpecifyKind(plan.EndUtc, DateTimeKind.Utc));
|
||||
cmd.Parameters.Add(new SqlParameter("@intervalTicks", SqlDbType.BigInt) { Value = plan.IntervalTicks });
|
||||
cmd.Parameters.Add(new SqlParameter("@bucketAdjustTicks", SqlDbType.BigInt) { Value = plan.BucketIndexAdjustTicks });
|
||||
await using var reader = await cmd.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false);
|
||||
while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false))
|
||||
if (needsRange)
|
||||
{
|
||||
var row = ReadBucket(reader, family);
|
||||
evaluated[row.Bucket] = AggregateEvaluator.Evaluate(aggregate, row, plan.BucketStart(row.Bucket), now);
|
||||
if (await reader.ReadAsync(cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
if (!reader.IsDBNull(0)) earliest = SqlHistorianConnection.GetUtc(reader, 0);
|
||||
if (!reader.IsDBNull(1)) latest = SqlHistorianConnection.GetUtc(reader, 1);
|
||||
}
|
||||
if (!await reader.NextResultAsync(cancellationToken).ConfigureAwait(false))
|
||||
throw new InvalidOperationException("The processed-read batch returned no bucket result set.");
|
||||
}
|
||||
while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false))
|
||||
rows.Add(ReadBucket(reader, family));
|
||||
}
|
||||
|
||||
var result = plan.Assemble(evaluated, bucketStart => AggregateEvaluator.NoData(bucketStart, now));
|
||||
var result = AggregateEvaluator.EvaluateAll(aggregate, plan, rows, earliest, latest, now);
|
||||
|
||||
RecordSuccess();
|
||||
return new HistoryReadResult(result, ContinuationPoint: null);
|
||||
@@ -456,6 +516,7 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
|
||||
catch (Exception ex)
|
||||
{
|
||||
RecordFailure(ex);
|
||||
if (IsTransportFailure(ex)) throw new System.IO.IOException("Historian read transport is unavailable.", ex);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
@@ -488,12 +549,22 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
|
||||
} while (await reader.NextResultAsync(ct).ConfigureAwait(false));
|
||||
}
|
||||
|
||||
private static BucketRow ReadBucket(SqlDataReader reader, AggregateFamily family)
|
||||
/// <summary>
|
||||
/// Maps one per-sample aggregate row (<see cref="SqlAggregateQueries"/> column layout) to a <see cref="BucketRow"/>.
|
||||
/// Every picked sample — first/last usable, first/last of any quality, the extreme — is optional: a NULL reads as
|
||||
/// "no such sample", never an exception (live-gate 8 F1: on QuestDB a row whose counts and picks disagreed threw
|
||||
/// out of <see cref="ReadStored"/> and failed the node's whole read; the evaluator answers such a bucket from what
|
||||
/// is present, or <c>BadNoData</c>).
|
||||
/// </summary>
|
||||
/// <param name="reader">Reader positioned on a bucket row.</param>
|
||||
/// <param name="family">The query's aggregate family.</param>
|
||||
/// <returns>The bucket statistics.</returns>
|
||||
internal static BucketRow ReadBucket(DbDataReader reader, AggregateFamily family)
|
||||
{
|
||||
static double? D(SqlDataReader r, int i) => r.IsDBNull(i) ? null : r.GetDouble(i);
|
||||
static long? L(SqlDataReader r, int i) => r.IsDBNull(i) ? null : r.GetInt64(i);
|
||||
static string? S(SqlDataReader r, int i) => r.IsDBNull(i) ? null : r.GetString(i);
|
||||
static HistorianValue V(SqlDataReader r, int i) =>
|
||||
static double? D(DbDataReader r, int i) => r.IsDBNull(i) ? null : r.GetDouble(i);
|
||||
static long? L(DbDataReader r, int i) => r.IsDBNull(i) ? null : r.GetInt64(i);
|
||||
static string? S(DbDataReader r, int i) => r.IsDBNull(i) ? null : r.GetString(i);
|
||||
static HistorianValue V(DbDataReader r, int i) =>
|
||||
r.IsDBNull(i) ? HistorianValue.Empty : SampleColumnMapper.FromColumns(r.GetByte(i), D(r, i + 1), L(r, i + 2), S(r, i + 3));
|
||||
|
||||
var row = new BucketRow
|
||||
@@ -513,14 +584,22 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
|
||||
row.Min = D(reader, f + 1);
|
||||
row.Max = D(reader, f + 2);
|
||||
row.Sum = D(reader, f + 3);
|
||||
row.UncertainMin = D(reader, f + 4);
|
||||
row.UncertainMax = D(reader, f + 5);
|
||||
row.GoodNonNumeric = reader.IsDBNull(f + 6) ? 0 : reader.GetInt32(f + 6);
|
||||
break;
|
||||
case AggregateFamily.StartEnd:
|
||||
row.First = V(reader, f);
|
||||
row.Last = V(reader, f + 5);
|
||||
row.FirstAny = ReadStoredOrNull(reader, SqlAggregateQueries.FirstAnyColumn);
|
||||
row.LastAny = ReadStoredOrNull(reader, SqlAggregateQueries.FirstAnyColumn + 6);
|
||||
break;
|
||||
default:
|
||||
row.Extreme = D(reader, f);
|
||||
row.ExtremeTime = reader.IsDBNull(f + 1) ? null : SqlHistorianConnection.GetUtc(reader, f + 1);
|
||||
row.UncertainMin = D(reader, f + 2);
|
||||
row.UncertainMax = D(reader, f + 3);
|
||||
row.GoodNonNumeric = reader.IsDBNull(f + 4) ? 0 : reader.GetInt32(f + 4);
|
||||
break;
|
||||
}
|
||||
return row;
|
||||
@@ -534,7 +613,7 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
|
||||
/// (<see cref="StoredSample.FromStored"/>) — defence in depth for rows written before the writer forced every
|
||||
/// Empty sample Bad.
|
||||
/// </summary>
|
||||
private static StoredSample ReadStored(SqlDataReader r, int o)
|
||||
private static StoredSample ReadStored(DbDataReader r, int o)
|
||||
{
|
||||
var value = SampleColumnMapper.FromColumns(
|
||||
r.GetByte(o + 1),
|
||||
@@ -545,6 +624,12 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
|
||||
return StoredSample.FromStored(SqlHistorianConnection.GetUtc(r, o), value, status);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// <see cref="ReadStored"/> for an optional picked sample: a NULL <c>SourceTimeUtc</c> means the row carries no such
|
||||
/// sample (live-gate 8 F1), so the result is null rather than an exception.
|
||||
/// </summary>
|
||||
private static StoredSample? ReadStoredOrNull(DbDataReader r, int o) => r.IsDBNull(o) ? null : ReadStored(r, o);
|
||||
|
||||
/// <summary>Read-only tag lookup (never creates a row); runs the schema bootstrap on a cache miss.</summary>
|
||||
private Task<int?> ResolveAsync(string name, CancellationToken ct) => _catalog.TryGetTagIdAsync(name, ct);
|
||||
|
||||
@@ -555,6 +640,9 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
|
||||
|
||||
private void RecordSuccess() => _health.RecordSuccess();
|
||||
|
||||
/// <summary>Classifies provider transport/timeouts, keeping SQL/query errors distinct (#697).</summary>
|
||||
private static bool IsTransportFailure(Exception ex) => ex is SqlException sql && sql.Number is -2 or 0 or 53 or 64 or 233 or 258 or 10053 or 10054 or 10060 or 10061 or 10065 or 11001 or 40197 or 40613;
|
||||
|
||||
private void RecordFailure(Exception ex)
|
||||
{
|
||||
// A cancelled read (deadline / client abort) is not a backend fault (the counters ignore it too).
|
||||
|
||||
+27
-1
@@ -27,21 +27,44 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer;
|
||||
/// value from <c>deleted.*</c> (NULL for an insert), the caller's <see cref="HistoryUpdateContext"/>, the
|
||||
/// <see cref="HistoryModificationType"/> and <c>SYSUTCDATETIME()</c>. One statement means the sample change and its
|
||||
/// audit row commit or roll back together: a failed update leaves no audit row.</para>
|
||||
/// <para><b>Schema v1 (Gitea #674).</b> Without the trail (<see cref="SqlServerSchemaBootstrapper.SupportsModifiedTrail"/>
|
||||
/// false) a change could not be audited, so every call throws <see cref="NotSupportedException"/> — which the node
|
||||
/// manager reports as a node-level <c>BadHistoryOperationUnsupported</c>, on all three arms alike — and nothing is
|
||||
/// sent to the database beyond the one-time schema check.</para>
|
||||
/// </summary>
|
||||
internal sealed class SqlServerHistorianDataUpdater : IHistorianDataUpdater
|
||||
{
|
||||
private readonly SqlHistorianConnection _conn;
|
||||
private readonly SqlServerTagCatalog _tags;
|
||||
private readonly SqlServerSchemaBootstrapper _bootstrap;
|
||||
|
||||
/// <summary>Creates the updater.</summary>
|
||||
/// <param name="conn">Connection helper.</param>
|
||||
/// <param name="bootstrap">Schema bootstrapper — decides whether the modification trail exists (#674).</param>
|
||||
/// <param name="tags">Tag catalog (runs the schema bootstrap itself).</param>
|
||||
public SqlServerHistorianDataUpdater(SqlHistorianConnection conn, SqlServerTagCatalog tags)
|
||||
public SqlServerHistorianDataUpdater(SqlHistorianConnection conn, SqlServerSchemaBootstrapper bootstrap,
|
||||
SqlServerTagCatalog tags)
|
||||
{
|
||||
_conn = conn;
|
||||
_bootstrap = bootstrap;
|
||||
_tags = tags;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// #674: ensures the schema and refuses the call when it has no Part 11 trail (no SQL after the first success).
|
||||
/// </summary>
|
||||
/// <param name="ct">Cancellation.</param>
|
||||
/// <returns>A task completing when the trail is available.</returns>
|
||||
/// <exception cref="NotSupportedException">The schema is below <see cref="SqlServerSchemaBootstrapper.ModifiedTrailVersion"/>.</exception>
|
||||
private async Task RequireTrailAsync(CancellationToken ct)
|
||||
{
|
||||
await _bootstrap.EnsureAsync(ct).ConfigureAwait(false);
|
||||
if (!_bootstrap.SupportsModifiedTrail)
|
||||
throw new NotSupportedException(
|
||||
$"Historian schema [{_conn.Schema}] is below version {SqlServerSchemaBootstrapper.ModifiedTrailVersion}: " +
|
||||
"HistoryUpdate needs the Part 11 modification trail (apply 002_sample_modified.sql and restart).");
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public async Task<IReadOnlyList<uint>> UpdateDataAsync(string tag, HistoryUpdateMode mode,
|
||||
IReadOnlyList<HistorizationValue> values, HistoryUpdateContext context, CancellationToken ct)
|
||||
@@ -49,6 +72,7 @@ internal sealed class SqlServerHistorianDataUpdater : IHistorianDataUpdater
|
||||
ArgumentNullException.ThrowIfNull(tag);
|
||||
ArgumentNullException.ThrowIfNull(values);
|
||||
ArgumentNullException.ThrowIfNull(context);
|
||||
await RequireTrailAsync(ct).ConfigureAwait(false);
|
||||
var plan = HistoryUpdatePlanner.Plan(tag, mode, values);
|
||||
if (plan.Winners.Count == 0) return HistoryUpdatePlanner.Complete(plan, mode, new Dictionary<long, UpsertOutcome>());
|
||||
|
||||
@@ -68,6 +92,7 @@ internal sealed class SqlServerHistorianDataUpdater : IHistorianDataUpdater
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(tag);
|
||||
ArgumentNullException.ThrowIfNull(context);
|
||||
await RequireTrailAsync(ct).ConfigureAwait(false);
|
||||
if (!HistorianTagNames.IsValid(tag)) return HistorianStatusCodes.BadInvalidArgument;
|
||||
var tagId = await _tags.TryGetTagIdAsync(tag, ct).ConfigureAwait(false);
|
||||
if (tagId is null) return HistoryUpdatePlanner.DeleteRawResult(tagKnown: false, 0);
|
||||
@@ -95,6 +120,7 @@ internal sealed class SqlServerHistorianDataUpdater : IHistorianDataUpdater
|
||||
ArgumentNullException.ThrowIfNull(tag);
|
||||
ArgumentNullException.ThrowIfNull(timesUtc);
|
||||
ArgumentNullException.ThrowIfNull(context);
|
||||
await RequireTrailAsync(ct).ConfigureAwait(false);
|
||||
if (!HistorianTagNames.IsValid(tag))
|
||||
{
|
||||
var refused = new uint[timesUtc.Count];
|
||||
|
||||
+4
-2
@@ -18,8 +18,10 @@ public sealed partial class SqlServerHistorianOptions
|
||||
|
||||
/// <summary>
|
||||
/// <c>true</c> creates the schema and applies missing scripts on first use; <c>false</c> only verifies
|
||||
/// that the required schema version (<c>SqlServerSchemaBootstrapper.RequiredVersion</c>, currently 2) exists and
|
||||
/// fails every historian call with a clear error otherwise.
|
||||
/// that the base schema version (<c>SqlServerSchemaBootstrapper.RequiredVersion</c>, 1) exists and fails every
|
||||
/// historian call with a clear error otherwise. A verified v1 schema (no Part 11 modification trail, script 002
|
||||
/// not applied) is served with modified reads and HistoryUpdate refused <c>BadHistoryOperationUnsupported</c>
|
||||
/// and one Warning per boot (Gitea #674).
|
||||
/// </summary>
|
||||
public bool AutoCreateSchema { get; set; } = true;
|
||||
|
||||
|
||||
+1
-1
@@ -99,7 +99,7 @@ public sealed class SqlServerHistorianProvider : IHistorianBackendProvider
|
||||
public IHistorianDataUpdater? CreateUpdater(IServiceProvider sp)
|
||||
{
|
||||
var s = Get(sp);
|
||||
return new SqlServerHistorianDataUpdater(s.Conn, s.Catalog);
|
||||
return new SqlServerHistorianDataUpdater(s.Conn, s.Bootstrap, s.Catalog);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
|
||||
+26
-12
@@ -11,7 +11,8 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer;
|
||||
/// <c>DELETE TOP(n)</c> batches (under the lock-escalation threshold). Samples are purged per tag so each
|
||||
/// delete is a seek on the clustered key <c>(TagId, SourceTimeUtc)</c>; events seek on <c>CX_Event_Time</c>.
|
||||
/// A session <c>sp_getapplock</c> (timeout 0) serialises purgers that share one database; a loser skips.
|
||||
/// To purge another table later, add it to <see cref="PurgeAsync"/>.
|
||||
/// To purge another table later, add it to <see cref="PurgeAsync"/>. On a v1 schema (no Part 11 trail, Gitea #674)
|
||||
/// the <c>SampleModified</c> purge is skipped and absent from the results.
|
||||
/// </summary>
|
||||
internal sealed class SqlServerHistorianRetention : IHistorianRetention
|
||||
{
|
||||
@@ -42,6 +43,9 @@ internal sealed class SqlServerHistorianRetention : IHistorianRetention
|
||||
/// <summary>Test seam for <see cref="SqlConnection.ClearPool"/>.</summary>
|
||||
internal Action<SqlConnection> ClearPool { get; set; } = SqlConnection.ClearPool;
|
||||
|
||||
/// <summary>Test seam: the release statement. Tests point it at a failing statement to exercise the pool-clear fallback.</summary>
|
||||
internal string ReleaseSql { get; set; } = "EXEC sp_releaseapplock @Resource = @res, @LockOwner = 'Session';";
|
||||
|
||||
public async Task<IReadOnlyList<RetentionPurgeResult>> PurgeAsync(CancellationToken ct)
|
||||
{
|
||||
await _bootstrap.EnsureAsync(ct).ConfigureAwait(false);
|
||||
@@ -69,6 +73,8 @@ internal sealed class SqlServerHistorianRetention : IHistorianRetention
|
||||
return [];
|
||||
}
|
||||
|
||||
// #674: a v1 schema has no SampleModified table — skip its purge rather than fail the whole pass.
|
||||
var trail = _bootstrap.SupportsModifiedTrail;
|
||||
long samples = 0, modified = 0;
|
||||
var tagIds = new List<int>();
|
||||
await using (var cmd = _conn.CreateCommand(conn, $"SELECT TagId FROM {_conn.Q("Tag")}"))
|
||||
@@ -82,21 +88,30 @@ internal sealed class SqlServerHistorianRetention : IHistorianRetention
|
||||
cutoff, id, ct).ConfigureAwait(false);
|
||||
// The Part 11 modification trail (#668) ages out on the SAME source-time cutoff as the samples it
|
||||
// describes; a purge never writes to it (no trigger, no OUTPUT INTO).
|
||||
modified += await DeleteLoopAsync(conn,
|
||||
$"DELETE TOP (@n) FROM {_conn.Q("SampleModified")} WHERE TagId = @id AND SourceTimeUtc < @cutoff",
|
||||
cutoff, id, ct).ConfigureAwait(false);
|
||||
if (trail)
|
||||
{
|
||||
modified += await DeleteLoopAsync(conn,
|
||||
$"DELETE TOP (@n) FROM {_conn.Q("SampleModified")} WHERE TagId = @id AND SourceTimeUtc < @cutoff",
|
||||
cutoff, id, ct).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
var events = await DeleteLoopAsync(conn,
|
||||
$"DELETE TOP (@n) FROM {_conn.Q("Event")} WHERE EventTimeUtc < @cutoff",
|
||||
cutoff, null, ct).ConfigureAwait(false);
|
||||
|
||||
return
|
||||
[
|
||||
new RetentionPurgeResult("Sample", samples),
|
||||
new RetentionPurgeResult("SampleModified", modified),
|
||||
new RetentionPurgeResult("Event", events),
|
||||
];
|
||||
return trail
|
||||
?
|
||||
[
|
||||
new RetentionPurgeResult("Sample", samples),
|
||||
new RetentionPurgeResult("SampleModified", modified),
|
||||
new RetentionPurgeResult("Event", events),
|
||||
]
|
||||
:
|
||||
[
|
||||
new RetentionPurgeResult("Sample", samples),
|
||||
new RetentionPurgeResult("Event", events),
|
||||
];
|
||||
}
|
||||
finally
|
||||
{
|
||||
@@ -137,8 +152,7 @@ internal sealed class SqlServerHistorianRetention : IHistorianRetention
|
||||
{
|
||||
try
|
||||
{
|
||||
await using var cmd = _conn.CreateCommand(conn,
|
||||
"EXEC sp_releaseapplock @Resource = @res, @LockOwner = 'Session';");
|
||||
await using var cmd = _conn.CreateCommand(conn, ReleaseSql);
|
||||
cmd.Parameters.Add(new SqlParameter("@res", SqlDbType.NVarChar, 255) { Value = LockResource });
|
||||
await cmd.ExecuteNonQueryAsync(CancellationToken.None).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
+30
-4
@@ -15,15 +15,27 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer;
|
||||
/// SQL, so a missing schema (or a SQL outage) does not serialise every write, read and alarm event behind one
|
||||
/// semaphore and one round-trip each.
|
||||
/// With <c>AutoCreateSchema=false</c> it only verifies <c>MAX(Version) >= </c><see cref="RequiredVersion"/>.
|
||||
/// <para><b>Schema v1 is accepted (Gitea #674).</b> A database at version 1 (no <c>SampleModified</c>) still serves
|
||||
/// live writes, raw / processed / at-time reads, alarm events and retention; only the Part 11 modification trail is
|
||||
/// unavailable — <see cref="SupportsModifiedTrail"/> is false, modified reads and HistoryUpdate report
|
||||
/// <c>BadHistoryOperationUnsupported</c> without touching SQL, and one Warning is logged per instance (= per boot: the
|
||||
/// provider shares one bootstrapper). Before #674 a v1 database refused EVERY historian call, continuous-historization
|
||||
/// drains included, until script 002 was applied by hand.</para>
|
||||
/// </summary>
|
||||
internal sealed partial class SqlServerSchemaBootstrapper
|
||||
{
|
||||
/// <summary>
|
||||
/// The schema version this build requires: 2 = <c>002_sample_modified.sql</c> (Part 11 modified-value
|
||||
/// history, Gitea #668). An existing v1 schema is upgraded in place on the next bootstrap when
|
||||
/// <c>AutoCreateSchema</c> is on; otherwise the script must be applied by hand.
|
||||
/// The schema version this build requires to serve at all: 1 = <c>001_initial.sql</c>. Gitea #674 lowered it from
|
||||
/// 2 — a missing modification trail disables only the features that need it (<see cref="ModifiedTrailVersion"/>).
|
||||
/// </summary>
|
||||
public const int RequiredVersion = 2;
|
||||
public const int RequiredVersion = 1;
|
||||
|
||||
/// <summary>
|
||||
/// The schema version that carries the Part 11 modification trail: 2 = <c>002_sample_modified.sql</c> (Gitea #668).
|
||||
/// An auto-creating bootstrap always upgrades to it; with <c>AutoCreateSchema=false</c> the script must be applied by
|
||||
/// hand (and the node restarted — the verdict is taken once per instance).
|
||||
/// </summary>
|
||||
public const int ModifiedTrailVersion = 2;
|
||||
|
||||
private const string LockResource = "OtOpcUaHistorianSchema";
|
||||
private const int LockTimeoutMs = 30_000;
|
||||
@@ -34,6 +46,13 @@ internal sealed partial class SqlServerSchemaBootstrapper
|
||||
private readonly SemaphoreSlim _gate = new(1, 1);
|
||||
private readonly BootstrapFailureCache _failure; // guarded by _gate
|
||||
private volatile bool _ready;
|
||||
private volatile bool _supportsModifiedTrail;
|
||||
|
||||
/// <summary>
|
||||
/// Whether the schema carries the Part 11 modification trail (version >= <see cref="ModifiedTrailVersion"/>).
|
||||
/// Meaningful only after <see cref="EnsureAsync"/> has succeeded (false before). Taken once per instance.
|
||||
/// </summary>
|
||||
public bool SupportsModifiedTrail => _supportsModifiedTrail;
|
||||
|
||||
/// <summary>Creates the bootstrapper.</summary>
|
||||
/// <param name="conn">Connection helper.</param>
|
||||
@@ -99,6 +118,13 @@ internal sealed partial class SqlServerSchemaBootstrapper
|
||||
throw new HistorianSchemaMissingException(
|
||||
$"Historian schema [{_conn.Schema}] is missing version {RequiredVersion} (found {v}) and AutoCreateSchema is false. " +
|
||||
"Apply the docs/historian/sqlserver/NNN_*.sql scripts above version " + v + " in order, or enable AutoCreateSchema.");
|
||||
_supportsModifiedTrail = v >= ModifiedTrailVersion;
|
||||
if (!_supportsModifiedTrail)
|
||||
_log.LogWarning("Historian schema [{Schema}] is at version {Version}, below {TrailVersion}, and AutoCreateSchema is false: " +
|
||||
"the Part 11 modification trail is unavailable, so modified-value history reads and HistoryUpdate report " +
|
||||
"BadHistoryOperationUnsupported and retention skips SampleModified. Live historization, reads, events and retention " +
|
||||
"are unaffected. Apply docs/historian/sqlserver/002_sample_modified.sql and restart the node to enable them.",
|
||||
_conn.Schema, v, ModifiedTrailVersion);
|
||||
}
|
||||
|
||||
private async Task ApplyAsync(SqlConnection conn, CancellationToken ct)
|
||||
|
||||
@@ -101,12 +101,6 @@ public sealed record OpcUaClientDriverOptions
|
||||
/// <summary>Client-side keep-alive interval.</summary>
|
||||
public TimeSpan KeepAliveInterval { get; init; } = TimeSpan.FromSeconds(5);
|
||||
|
||||
/// <summary>
|
||||
/// Retained for config compatibility only — no longer used (Gitea #670). The driver has no in-driver reconnect
|
||||
/// handler; a lost session is recovered by the host's comms-loss watchdog re-initialising the driver.
|
||||
/// </summary>
|
||||
public TimeSpan ReconnectPeriod { get; init; } = TimeSpan.FromSeconds(5);
|
||||
|
||||
/// <summary>
|
||||
/// When <c>true</c>, the driver accepts any self-signed / untrusted server certificate.
|
||||
/// Dev-only — must be <c>false</c> in production so MITM attacks against the opc.tcp
|
||||
|
||||
@@ -29,7 +29,7 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.OpcUaClient;
|
||||
/// back on its own recovers in place.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWritable, ISubscribable, IHostConnectivityProbe, IAlarmSource, ISkippedTagReporter, IDisposable, IAsyncDisposable
|
||||
public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWritable, ISubscribable, IHostConnectivityProbe, IAlarmSource, ISkippedTagReporter, IDriverConnectionLossSource, IDisposable, IAsyncDisposable
|
||||
{
|
||||
/// <summary>Authored tags whose TagConfig carried no usable <c>nodeId</c>, so there is no upstream node
|
||||
/// to dial (Gitea #541). This driver's skip path was the quietest of the twelve: <c>NamespaceMap</c>
|
||||
@@ -83,6 +83,7 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
|
||||
_logger = logger ?? NullLogger<OpcUaClientDriver>.Instance;
|
||||
_secretResolver = secretResolver ?? NullSecretResolver.Instance;
|
||||
_sdkTelemetry = logger is null ? SdkTelemetry.Context : SdkTelemetry.ForLogger(logger);
|
||||
PostRecoveryReconcileDelay = () => _options.Timeout + _options.KeepAliveInterval;
|
||||
}
|
||||
|
||||
/// <summary>Mutable so <see cref="InitializeAsync"/> can adopt a re-parsed config on reinitialize
|
||||
@@ -135,6 +136,14 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
|
||||
/// <summary>Occurs when the host connectivity status of the remote OPC UA server changes.</summary>
|
||||
public event EventHandler<HostStatusChangedEventArgs>? OnHostStatusChanged;
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>Gitea #676 — raised at most once per session, and only on a DEFINITIVE loss
|
||||
/// (<see cref="IsDefinitiveSessionLoss"/>, or a tracked subscription the SERVER reported dropped — never the
|
||||
/// publishing-stall heuristic). Re-armed when the session is released (re-initialise / shutdown). Known gap: the host
|
||||
/// acts on it only while Connected, so a raise it receives before it has processed the successful initialise is
|
||||
/// swallowed and that session falls back to the #527 watchdog.</remarks>
|
||||
public event EventHandler<DriverConnectionLossEventArgs>? ConnectionLost;
|
||||
|
||||
// OPC UA StatusCode constants the driver surfaces for local-side faults. Upstream-server
|
||||
// StatusCodes are passed through verbatim per driver-specs.md §8 "cascading quality" —
|
||||
// downstream clients need to distinguish 'remote source down' from 'local driver failure'.
|
||||
@@ -164,6 +173,9 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
|
||||
/// <summary>Test-only seam (#670): the most recent in-place recovery task.</summary>
|
||||
internal Task RecoveryTaskForTest => _recoveryTask;
|
||||
|
||||
/// <summary>Test seam (#676): the trailing reconcile read scheduled by the last in-place recovery.</summary>
|
||||
internal Task ReconcileTaskForTest => _reconcileTask;
|
||||
|
||||
/// <summary>Test-only seam (#670 review M1): raise the session's PublishError as the SDK publish loop does.</summary>
|
||||
/// <param name="status">The publish error status.</param>
|
||||
/// <param name="subscriptionId">The subscription id the error names (0 = none).</param>
|
||||
@@ -282,6 +294,33 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
|
||||
/// <summary>The last in-place recovery started (#670). Test seam: lets a test await it deterministically.</summary>
|
||||
private Task _recoveryTask = Task.CompletedTask;
|
||||
|
||||
/// <summary>#676 — 1 once <see cref="ConnectionLost"/> has been raised for the CURRENT session (once per session);
|
||||
/// cleared when the session is released, so a re-initialised driver reports its next loss.</summary>
|
||||
private int _connectionLostRaised;
|
||||
|
||||
/// <summary>#676 live gate A1 — closed-channel keep-alive failures (<see cref="IsClosedChannel"/>) in the current
|
||||
/// outage; reset by a good keep-alive and when the session is released. Two make a closed channel "sustained".</summary>
|
||||
private int _closedChannelFailures;
|
||||
|
||||
/// <summary>Closed-channel keep-alive failures in one outage that make the loss definitive (a single one may be a
|
||||
/// transient that the next good keep-alive contradicts).</summary>
|
||||
private const int SustainedClosedChannelFailures = 2;
|
||||
|
||||
/// <summary>#676 — the trailing reconcile read scheduled by the last in-place recovery. Test seam.</summary>
|
||||
private Task _reconcileTask = Task.CompletedTask;
|
||||
|
||||
/// <summary>#676 — late notifications the replay guard dropped since it was armed, and the refs they were for;
|
||||
/// summarised in ONE Information line when the window disarms. Guarded by <see cref="_replayGuardLock"/>.</summary>
|
||||
private int _replayDrops;
|
||||
private readonly HashSet<string> _replayDropRefs = new(StringComparer.Ordinal);
|
||||
|
||||
/// <summary>
|
||||
/// Test seam (#676): how long after an in-place recovery the trailing reconcile read runs. Production: the
|
||||
/// operation timeout plus one keep-alive interval — by then every request that was in flight across the stall
|
||||
/// (including the Publish responses whose notifications the SDK can lose) has completed or timed out.
|
||||
/// </summary>
|
||||
internal Func<TimeSpan> PostRecoveryReconcileDelay { get; set; }
|
||||
|
||||
/// <summary>Test seam (#670 review M1): whether a subscription has stopped publishing. Production asks the SDK.</summary>
|
||||
internal Func<Subscription, bool> IsPublishingStopped { get; set; } = static s => s.Created && s.PublishingStopped;
|
||||
|
||||
@@ -326,6 +365,37 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
|
||||
/// <summary>Serialises arming against expiry, so an expiry racing a re-arm cannot wipe the new seeds.</summary>
|
||||
private readonly object _replayGuardLock = new();
|
||||
|
||||
/// <summary>
|
||||
/// Review of 3b88056f — serialises "decide whether this upstream value may be published" with "publish it" across
|
||||
/// the three paths that publish upstream data: live notifications, the recovery read and the trailing reconcile
|
||||
/// read. Without it a path could pass its newer-than check, lose the CPU to another path publishing a NEWER value,
|
||||
/// and then deliver its older one last — regressing the node. It also covers the comms-loss Bad fan
|
||||
/// (<see cref="PublishCommsLoss"/>). One driver-wide lock rather than one per ref: it also serialises delivery
|
||||
/// ACROSS subscriptions — with several subscriptions the SDK dispatches each subscription's messages on its own
|
||||
/// worker, so their notifications no longer run in parallel. That is the cost, and it is small: each critical
|
||||
/// section is a dictionary update plus a subscriber call that only <c>Tell</c>s an actor (tens of nanoseconds to
|
||||
/// a few microseconds), nothing awaits under it, and per-subscription message order is unaffected. Because
|
||||
/// subscribers are invoked under it, a subscriber must never block on, or call back into, this driver.
|
||||
/// </summary>
|
||||
private readonly Lock _deliveryLock = new();
|
||||
|
||||
/// <summary>
|
||||
/// Re-review of 30a9d5f0 — per RawPath, a counter bumped (under <see cref="_deliveryLock"/>) every time a value is
|
||||
/// delivered for it. The trailing re-read captures it before issuing its read and skips any ref delivered to since:
|
||||
/// without source timestamps the read's value cannot be ordered against a notification that landed while the read
|
||||
/// was in flight, and publishing it would regress the node to the value the read sampled.
|
||||
/// </summary>
|
||||
private readonly System.Collections.Concurrent.ConcurrentDictionary<string, long> _deliverySeq =
|
||||
new(StringComparer.Ordinal);
|
||||
|
||||
/// <summary>Review of 3b88056f — cancels the post-recovery background work (trailing read, replay-guard disarm timer)
|
||||
/// of the current session; cancelled and replaced whenever the session is released, and on dispose.</summary>
|
||||
private CancellationTokenSource _sessionWorkCts = new();
|
||||
|
||||
/// <summary>Test seam (review of 3b88056f): invoked inside the trailing read's critical section, after it decided
|
||||
/// to publish a ref and before it does — where a racing notification would otherwise overtake it.</summary>
|
||||
internal Action<string>? BeforeReconcilePublishForTest { get; set; }
|
||||
|
||||
/// <summary>Floor of the replay-guard window; the live gate saw the late replay 4–5 s after recovery.</summary>
|
||||
private static readonly TimeSpan ReplayGuardMinimum = TimeSpan.FromSeconds(30);
|
||||
|
||||
@@ -883,9 +953,40 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
|
||||
$"No endpoint at '{endpointUrl}' matches SecurityPolicy={policy} + SecurityMode={mode}. " +
|
||||
$"Server advertises: {advertised}");
|
||||
}
|
||||
// #676: connect where the operator pointed us, not where the server says it lives.
|
||||
match.EndpointUrl = RewriteEndpointUrl(match.EndpointUrl, endpointUrl);
|
||||
return match;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #676 — the address to open the session on. A server reports its endpoints with ITS view of its own
|
||||
/// address; behind NAT, a port map, a container port publish or a proxy that address is unreachable from here
|
||||
/// (or bypasses the proxy). When the advertised URL's scheme matches the configured one, the configured host
|
||||
/// AND port (when it names one) replace the advertised ones and the advertised path is kept — the SDK's own
|
||||
/// <c>CoreClientUtils.SelectEndpoint</c> rule, which this driver's policy-aware selection bypassed. A different
|
||||
/// scheme is a different transport and is left alone; an unparseable advertised URL falls back to the
|
||||
/// configured one.
|
||||
/// </summary>
|
||||
/// <param name="advertisedUrl">The <c>EndpointUrl</c> the server returned from GetEndpoints.</param>
|
||||
/// <param name="configuredUrl">The URL the driver was configured with (the one GetEndpoints was sent to).</param>
|
||||
/// <returns>The URL to connect the session to.</returns>
|
||||
internal static string RewriteEndpointUrl(string? advertisedUrl, string configuredUrl)
|
||||
{
|
||||
if (!Uri.TryCreate(configuredUrl, UriKind.Absolute, out var configured)) return advertisedUrl ?? configuredUrl;
|
||||
if (string.IsNullOrWhiteSpace(advertisedUrl)
|
||||
|| !Uri.TryCreate(advertisedUrl, UriKind.Absolute, out var advertised))
|
||||
return configuredUrl;
|
||||
if (!string.Equals(advertised.Scheme, configured.Scheme, StringComparison.OrdinalIgnoreCase)) return advertisedUrl;
|
||||
// A port-less configured URL (Uri.Port == -1 for opc.tcp) keeps the advertised port — copying -1 would drop it
|
||||
// and the SDK would fall back to 4840 (review of 3b88056f; same fix as the client's EndpointSelector).
|
||||
var builder = new UriBuilder(advertised)
|
||||
{
|
||||
Host = configured.DnsSafeHost,
|
||||
Port = configured.Port > 0 ? configured.Port : advertised.Port,
|
||||
};
|
||||
return builder.Uri.ToString();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Build a <see cref="UserIdentity"/> carrying a client user-authentication
|
||||
/// certificate loaded from <see cref="OpcUaClientDriverOptions.UserCertificatePath"/>.
|
||||
@@ -986,6 +1087,21 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
|
||||
Interlocked.Exchange(ref _sessionLost, 0);
|
||||
Interlocked.Exchange(ref _subscriptionLost, 0);
|
||||
Interlocked.Exchange(ref _publishingStoppedSinceTicks, 0);
|
||||
// #676: re-arm the definitive-loss signal for the next session.
|
||||
Interlocked.Exchange(ref _connectionLostRaised, 0);
|
||||
Interlocked.Exchange(ref _closedChannelFailures, 0);
|
||||
// Review of 3b88056f: the released session's post-recovery timers must not outlive it.
|
||||
CancelSessionWork();
|
||||
}
|
||||
|
||||
/// <summary>Cancel the current session's background work and start a fresh token for the next one.</summary>
|
||||
private void CancelSessionWork()
|
||||
{
|
||||
// Cancelled but deliberately NOT disposed (re-review of 30a9d5f0): the keep-alive thread may be reading the old
|
||||
// source's Token concurrently, and a disposed source throws ObjectDisposedException there. Nothing here is
|
||||
// timer-backed (no CancelAfter), so an undisposed source holds no unmanaged resource.
|
||||
var old = Interlocked.Exchange(ref _sessionWorkCts, new CancellationTokenSource());
|
||||
old.Cancel();
|
||||
}
|
||||
|
||||
/// <summary>Upper bound on the best-effort wire calls a teardown makes against a possibly-dead upstream.</summary>
|
||||
@@ -1021,8 +1137,13 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
|
||||
// previous connection's value bookkeeping applies to them.
|
||||
_lastValues.Clear();
|
||||
_lastGoodSourceTicks.Clear();
|
||||
_replayGuard.Clear();
|
||||
Interlocked.Exchange(ref _replayGuardUntilTicks, 0);
|
||||
lock (_replayGuardLock)
|
||||
{
|
||||
_replayGuard.Clear();
|
||||
Interlocked.Exchange(ref _replayGuardUntilTicks, 0);
|
||||
_replayDrops = 0;
|
||||
_replayDropRefs.Clear();
|
||||
}
|
||||
_awaitingRecovery.Clear();
|
||||
// #619 re-review: the last-told alarm states (and ack targets) deliberately SURVIVE a shutdown. A
|
||||
// ReinitializeAsync (comms-loss watchdog / Force Reconnect) is followed by the host re-subscribing the
|
||||
@@ -1762,20 +1883,33 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
|
||||
StatusCode: dv.StatusCode.Code,
|
||||
SourceTimestampUtc: dv.SourceTimestamp == DateTime.MinValue ? null : dv.SourceTimestamp,
|
||||
ServerTimestampUtc: dv.ServerTimestamp == DateTime.MinValue ? DateTime.UtcNow : dv.ServerTimestamp);
|
||||
// F2: inside the post-recovery replay window only, a late OLDER notification must not regress the node.
|
||||
if (IsLateReplay(fullRef, snapshot.SourceTimestampUtc, out var newest))
|
||||
// Review of 3b88056f: the replay check and the publish are one critical section shared with the recovery and
|
||||
// trailing reads, so an older value can never be delivered after a newer one (see _deliveryLock).
|
||||
lock (_deliveryLock)
|
||||
{
|
||||
_diag.Record(OpcUaClientDiagnostics.StaleNotificationDropped);
|
||||
_logger.LogDebug(
|
||||
"OpcUaClientDriver '{DriverInstanceId}': dropped a late replayed notification for {Ref} after a recovery " +
|
||||
"(source {Source:O} is not newer than {Newest:O}).", _driverInstanceId, fullRef, snapshot.SourceTimestampUtc,
|
||||
new DateTime(newest, DateTimeKind.Utc));
|
||||
return;
|
||||
// F2: inside the post-recovery replay window only, a late OLDER notification must not regress the node.
|
||||
if (IsLateReplay(fullRef, snapshot.SourceTimestampUtc, out var newest))
|
||||
{
|
||||
_diag.Record(OpcUaClientDiagnostics.StaleNotificationDropped);
|
||||
// #676: the OTel counter and the Debug line below are invisible on a default-configured node; the
|
||||
// per-recovery Information summary logged at disarm is what makes the guard observable.
|
||||
lock (_replayGuardLock)
|
||||
{
|
||||
_replayDrops++;
|
||||
_replayDropRefs.Add(fullRef);
|
||||
}
|
||||
_logger.LogDebug(
|
||||
"OpcUaClientDriver '{DriverInstanceId}': dropped a late replayed notification for {Ref} after a recovery " +
|
||||
"(source {Source:O} is not newer than {Newest:O}).", _driverInstanceId, fullRef, snapshot.SourceTimestampUtc,
|
||||
new DateTime(newest, DateTimeKind.Utc));
|
||||
return;
|
||||
}
|
||||
RecordUpstreamGood(fullRef, snapshot);
|
||||
_lastValues[fullRef] = snapshot;
|
||||
_deliverySeq.AddOrUpdate(fullRef, 1, (_, n) => n + 1);
|
||||
_awaitingRecovery.TryRemove(fullRef, out _);
|
||||
OnDataChange?.Invoke(this, new DataChangeEventArgs(handle, fullRef, snapshot));
|
||||
}
|
||||
RecordUpstreamGood(fullRef, snapshot);
|
||||
_lastValues[fullRef] = snapshot;
|
||||
_awaitingRecovery.TryRemove(fullRef, out _);
|
||||
OnDataChange?.Invoke(this, new DataChangeEventArgs(handle, fullRef, snapshot));
|
||||
}
|
||||
|
||||
/// <summary>#670 F2 — remember the last accepted Good upstream source time for <paramref name="fullRef"/>.</summary>
|
||||
@@ -1798,14 +1932,7 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
|
||||
if (until == 0) return false;
|
||||
if (Clock.GetUtcNow().UtcDateTime.Ticks >= until)
|
||||
{
|
||||
lock (_replayGuardLock)
|
||||
{
|
||||
if (Interlocked.Read(ref _replayGuardUntilTicks) == until)
|
||||
{
|
||||
Interlocked.Exchange(ref _replayGuardUntilTicks, 0);
|
||||
_replayGuard.Clear();
|
||||
}
|
||||
}
|
||||
DisarmReplayGuard(until);
|
||||
return false;
|
||||
}
|
||||
if (source is not { } src) return false;
|
||||
@@ -1821,11 +1948,40 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Expire the replay guard armed until <paramref name="until"/> — a no-op when it was re-armed or already disarmed
|
||||
/// since — and log ONE Information summary of what it dropped (#676). The guard's OTel counter and per-drop Debug
|
||||
/// line are invisible on a default-configured node, so before this a recovery's drops left no trace at all.
|
||||
/// </summary>
|
||||
/// <param name="until">The expiry the caller observed; only that arming is disarmed.</param>
|
||||
private void DisarmReplayGuard(long until)
|
||||
{
|
||||
int drops, refs;
|
||||
lock (_replayGuardLock)
|
||||
{
|
||||
if (Interlocked.Read(ref _replayGuardUntilTicks) != until) return;
|
||||
Interlocked.Exchange(ref _replayGuardUntilTicks, 0);
|
||||
_replayGuard.Clear();
|
||||
drops = _replayDrops;
|
||||
refs = _replayDropRefs.Count;
|
||||
_replayDrops = 0;
|
||||
_replayDropRefs.Clear();
|
||||
}
|
||||
LogReplaySummary(drops, refs);
|
||||
}
|
||||
|
||||
/// <summary>One line per replay window: Information when the guard dropped something, Debug otherwise.</summary>
|
||||
private void LogReplaySummary(int drops, int refs) =>
|
||||
_logger.Log(drops > 0 ? LogLevel.Information : LogLevel.Debug,
|
||||
"OpcUaClientDriver '{DriverInstanceId}': post-recovery replay window closed — dropped {Drops} late " +
|
||||
"notifications across {Refs} refs after recovery.", _driverInstanceId, drops, refs);
|
||||
|
||||
/// <summary>
|
||||
/// #670 F2 — arm the replay guard on a same-session recovery: every ref with a known Good upstream source time,
|
||||
/// for max(30 s, 2 × the longest tracked subscription keep-alive window).
|
||||
/// </summary>
|
||||
private void ArmReplayGuard(DateTime now)
|
||||
/// <returns>The window's expiry in clock ticks.</returns>
|
||||
private long ArmReplayGuard(DateTime now)
|
||||
{
|
||||
var window = ReplayGuardMinimum;
|
||||
foreach (var sub in _subscriptions.Values.Select(r => r.Subscription))
|
||||
@@ -1833,12 +1989,39 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
|
||||
var keepAliveWindow = TimeSpan.FromMilliseconds(sub.CurrentPublishingInterval * (double)sub.CurrentKeepAliveCount);
|
||||
if (keepAliveWindow * 2 > window) window = keepAliveWindow * 2;
|
||||
}
|
||||
int supersededDrops, supersededRefs;
|
||||
lock (_replayGuardLock)
|
||||
{
|
||||
// A re-arm inside an open window ends that window: report what it dropped and count afresh.
|
||||
supersededDrops = _replayDrops;
|
||||
supersededRefs = _replayDropRefs.Count;
|
||||
_replayDrops = 0;
|
||||
_replayDropRefs.Clear();
|
||||
_replayGuard.Clear();
|
||||
foreach (var kv in _lastGoodSourceTicks) _replayGuard[kv.Key] = kv.Value;
|
||||
Interlocked.Exchange(ref _replayGuardUntilTicks, (now + window).Ticks);
|
||||
}
|
||||
if (supersededDrops > 0) LogReplaySummary(supersededDrops, supersededRefs);
|
||||
return (now + window).Ticks;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// #676 — disarm the replay guard (and log its summary) when its window ends, even if no notification arrives to
|
||||
/// disarm it lazily in <see cref="IsLateReplay"/> — a quiet subscription would otherwise never log the summary.
|
||||
/// </summary>
|
||||
private async Task DisarmReplayGuardWhenDueAsync(long until, CancellationToken ct)
|
||||
{
|
||||
try
|
||||
{
|
||||
var wait = new DateTime(until, DateTimeKind.Utc) - Clock.GetUtcNow().UtcDateTime;
|
||||
if (wait > TimeSpan.Zero) await Task.Delay(wait + TimeSpan.FromMilliseconds(100), Clock, ct).ConfigureAwait(false);
|
||||
if (Clock.GetUtcNow().UtcDateTime.Ticks >= until) DisarmReplayGuard(until);
|
||||
}
|
||||
catch (OperationCanceledException) when (ct.IsCancellationRequested) { /* session released */ }
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogDebug(ex, "OpcUaClientDriver '{DriverInstanceId}': replay-guard disarm timer faulted.", _driverInstanceId);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -2683,7 +2866,9 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
|
||||
/// session-recreate path cloned subscriptions the driver never re-tracked, and it kept health Healthy while
|
||||
/// it did so — the #670 symptom. A session that does not come back keeps health Degraded with the read stamp
|
||||
/// frozen, which is exactly what the host's #527 comms-loss watchdog trips on: ShutdownAsync + InitializeAsync
|
||||
/// + re-subscribe, whose initial values come back Good.
|
||||
/// + re-subscribe, whose initial values come back Good. When the keep-alive status proves the session is gone
|
||||
/// for good (<see cref="IsDefinitiveSessionLoss"/>) the driver also raises <see cref="ConnectionLost"/> (#676),
|
||||
/// so the host runs that sequence at once instead of after the watchdog's silence window.
|
||||
/// </remarks>
|
||||
private void OnKeepAlive(ISession sender, KeepAliveEventArgs e)
|
||||
{
|
||||
@@ -2693,6 +2878,7 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
|
||||
|
||||
if (!ServiceResult.IsBad(e.Status))
|
||||
{
|
||||
Interlocked.Exchange(ref _closedChannelFailures, 0);
|
||||
TransitionTo(HostState.Running);
|
||||
// M1: the session answers but a subscription is dead — no stamp, no recovery; the watchdog reinit is the fix.
|
||||
if (Volatile.Read(ref _subscriptionLost) == 1)
|
||||
@@ -2706,18 +2892,23 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
|
||||
UpdateHealth(_ => new DriverHealth(DriverState.Healthy, now, null));
|
||||
// A fresh grace: every subscription reports stopped right after an outage until its next message.
|
||||
Interlocked.Exchange(ref _publishingStoppedSinceTicks, 0);
|
||||
ArmReplayGuard(now);
|
||||
var guardUntil = ArmReplayGuard(now);
|
||||
_diag.Record(OpcUaClientDiagnostics.KeepAliveRecovered);
|
||||
_logger.LogInformation(
|
||||
"OpcUaClientDriver '{DriverInstanceId}': upstream keep-alive recovered on the same session; " +
|
||||
"re-reading the refs published Bad during the outage.", _driverInstanceId);
|
||||
_recoveryTask = Task.Run(() => RecoverAfterOutageAsync(sender));
|
||||
var recovery = Task.Run(() => RecoverAfterOutageAsync(sender));
|
||||
_recoveryTask = recovery;
|
||||
var sessionWork = _sessionWorkCts.Token;
|
||||
_reconcileTask = ReconcileAfterRecoveryAsync(sender, recovery, PostRecoveryReconcileDelay(), sessionWork);
|
||||
_ = DisarmReplayGuardWhenDueAsync(guardUntil, sessionWork);
|
||||
return;
|
||||
}
|
||||
|
||||
if (FindStalledSubscription(now) is { } stalled)
|
||||
{
|
||||
DeclareSubscriptionLost($"subscription '{stalled}' stopped publishing while the session stayed up", now);
|
||||
DeclareSubscriptionLost($"subscription '{stalled}' stopped publishing while the session stayed up", now,
|
||||
serverReported: false);
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -2729,14 +2920,76 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
|
||||
TransitionTo(HostState.Stopped);
|
||||
_diag.Record(OpcUaClientDiagnostics.KeepAliveMiss);
|
||||
UpdateHealth(h => new DriverHealth(DriverState.Degraded, h.LastSuccessfulRead, $"keep-alive: {e.Status}"));
|
||||
if (Interlocked.Exchange(ref _sessionLost, 1) == 1) return;
|
||||
if (Volatile.Read(ref _subscriptionLost) == 1) return; // already published Bad for this connection
|
||||
// #676: the first bad keep-alive is often ambiguous and a definitive one follows, so this is checked on EVERY
|
||||
// bad keep-alive — but only after the outage below has been published, so the Bad reaches subscribers first.
|
||||
var keepAliveCode = e.Status?.StatusCode ?? StatusCodes.Good;
|
||||
var definitive = IsDefinitiveSessionLoss(keepAliveCode)
|
||||
|| (IsClosedChannel(keepAliveCode)
|
||||
&& Interlocked.Increment(ref _closedChannelFailures) >= SustainedClosedChannelFailures);
|
||||
// A subscription loss already published Bad for this connection.
|
||||
if (Interlocked.Exchange(ref _sessionLost, 1) == 0 && Volatile.Read(ref _subscriptionLost) == 0)
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"OpcUaClientDriver '{DriverInstanceId}': upstream session keep-alive failed ({Status}); subscribed values " +
|
||||
"are BadNoCommunication until it recovers or the host's comms-loss watchdog reconnects the driver.",
|
||||
_driverInstanceId, e.Status);
|
||||
PublishCommsLoss(now);
|
||||
}
|
||||
if (definitive)
|
||||
RaiseConnectionLost($"upstream session keep-alive reported {StatusCodes.GetBrowseName(keepAliveCode.Code)}");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// #676 live gate A1 — the connection under the session is CLOSED (not merely silent). A real upstream process stop
|
||||
/// (docker restart, SIGINT, docker kill) never produced a session status on the docker-dev gate: the keep-alives
|
||||
/// reported <c>BadSecureChannelClosed</c> once, then <c>BadConnectionClosed</c> interleaved with
|
||||
/// <c>BadNoCommunication</c> for the whole outage. Reproduced in-process (a proxy that resets every connection
|
||||
/// while the server and its session stay alive): the SDK 1.5.378 session never opens a new connection on its
|
||||
/// own — over 48 s with the server reachable again, not one connection attempt — so a closed channel can never
|
||||
/// recover in place. It is treated as definitive once SUSTAINED (<see cref="SustainedClosedChannelFailures"/> in
|
||||
/// one outage), so a single transient contradicted by the next good keep-alive does not count. A stall (a paused
|
||||
/// upstream, a congested link) produces <c>BadRequestTimeout</c>/<c>BadNoCommunication</c> instead and keeps
|
||||
/// its in-place recovery.
|
||||
/// </summary>
|
||||
/// <param name="status">A keep-alive status.</param>
|
||||
/// <returns>True for a closed secure channel or connection.</returns>
|
||||
internal static bool IsClosedChannel(StatusCode status) =>
|
||||
status == StatusCodes.BadSecureChannelClosed || status == StatusCodes.BadConnectionClosed;
|
||||
|
||||
/// <summary>
|
||||
/// #676 — keep-alive statuses that prove the session can NEVER recover in place, so waiting out the #527
|
||||
/// watchdog would only delay the inevitable re-initialise. The server forgot the session
|
||||
/// (<c>BadSessionIdInvalid</c> — what a re-opened channel to a restarted server answers), closed it
|
||||
/// (<c>BadSessionClosed</c>) or never activated it (<c>BadSessionNotActivated</c>); or the server HALTED
|
||||
/// (<c>BadServerHalted</c>) — measured on the #676 restart wire test, the SDK never re-opens a channel the
|
||||
/// server halted: 15 s after the server was back the same session still failed every request with
|
||||
/// <c>BadServerHalted</c>, so a halted session is dead for good. A closed connection is definitive too, but only
|
||||
/// once sustained — see <see cref="IsClosedChannel"/>. Everything else — <c>BadNoCommunication</c>,
|
||||
/// <c>BadNotConnected</c>, request timeouts — is AMBIGUOUS (a stalled link that resumes keeps its session, see the
|
||||
/// stall wire test) and stays with the watchdog.
|
||||
/// </summary>
|
||||
/// <param name="status">A keep-alive status.</param>
|
||||
/// <returns>True when the session is definitively gone.</returns>
|
||||
internal static bool IsDefinitiveSessionLoss(StatusCode status) =>
|
||||
status == StatusCodes.BadSessionIdInvalid
|
||||
|| status == StatusCodes.BadSessionClosed
|
||||
|| status == StatusCodes.BadSessionNotActivated
|
||||
|| status == StatusCodes.BadServerHalted;
|
||||
|
||||
/// <summary>#676 — raise <see cref="ConnectionLost"/> once for the current session, isolating a faulting subscriber.</summary>
|
||||
/// <param name="reason">Why the connection is definitively gone, for the host's log.</param>
|
||||
private void RaiseConnectionLost(string reason)
|
||||
{
|
||||
if (Interlocked.Exchange(ref _connectionLostRaised, 1) == 1) return;
|
||||
_logger.LogWarning(
|
||||
"OpcUaClientDriver '{DriverInstanceId}': upstream session keep-alive failed ({Status}); subscribed values " +
|
||||
"are BadNoCommunication until it recovers or the host's comms-loss watchdog reconnects the driver.",
|
||||
_driverInstanceId, e.Status);
|
||||
PublishCommsLoss(now);
|
||||
"OpcUaClientDriver '{DriverInstanceId}': connection definitively lost ({Reason}); asking the host to " +
|
||||
"re-initialise now instead of waiting for the comms-loss watchdog.", _driverInstanceId, reason);
|
||||
try { ConnectionLost?.Invoke(this, new DriverConnectionLossEventArgs(reason)); }
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "OpcUaClientDriver '{DriverInstanceId}': a ConnectionLost subscriber faulted.",
|
||||
_driverInstanceId);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -2771,14 +3024,23 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
|
||||
/// frozen (sticky until the session is released), and the outage published once, so the host's watchdog
|
||||
/// re-initialises and re-subscribes.
|
||||
/// </summary>
|
||||
private void DeclareSubscriptionLost(string reason, DateTime now)
|
||||
/// <param name="reason">Why, for the log and health.</param>
|
||||
/// <param name="now">Detection time.</param>
|
||||
/// <param name="serverReported">True when the SERVER said the subscription is gone (BadNoSubscription /
|
||||
/// BadSubscriptionIdInvalid) — definitive, so <see cref="ConnectionLost"/> is raised. False for the publishing-stall
|
||||
/// heuristic, an inference from silence that stays with the #527 watchdog (review of 3b88056f).</param>
|
||||
private void DeclareSubscriptionLost(string reason, DateTime now, bool serverReported)
|
||||
{
|
||||
if (Interlocked.Exchange(ref _subscriptionLost, 1) == 1) return;
|
||||
UpdateHealth(h => new DriverHealth(DriverState.Degraded, h.LastSuccessfulRead, $"subscription lost: {reason}"));
|
||||
_logger.LogWarning(
|
||||
"OpcUaClientDriver '{DriverInstanceId}': {Reason}; subscribed values are BadNoCommunication until the " +
|
||||
"host's comms-loss watchdog re-initialises the driver and re-subscribes.", _driverInstanceId, reason);
|
||||
"host re-initialises the driver and re-subscribes.", _driverInstanceId, reason);
|
||||
if (Volatile.Read(ref _sessionLost) == 0) PublishCommsLoss(now);
|
||||
// #676: a subscription the SERVER says is gone never comes back on this session — only the re-subscribe after a
|
||||
// re-initialise recreates it — so ask the host now rather than after the watchdog's 90 s. The stall heuristic is
|
||||
// an inference, not a server statement, and keeps going through the watchdog.
|
||||
if (serverReported) RaiseConnectionLost($"subscription lost: {reason}");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -2801,7 +3063,8 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
|
||||
else
|
||||
return;
|
||||
if (tracked)
|
||||
DeclareSubscriptionLost($"server reported {code} for a tracked subscription", Clock.GetUtcNow().UtcDateTime);
|
||||
DeclareSubscriptionLost($"server reported {code} for a tracked subscription", Clock.GetUtcNow().UtcDateTime,
|
||||
serverReported: true);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -2811,18 +3074,23 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
|
||||
/// </summary>
|
||||
private void PublishCommsLoss(DateTime now)
|
||||
{
|
||||
foreach (var rs in _subscriptions.Values)
|
||||
// Under the delivery lock (re-review of 30a9d5f0) so the Bad cannot interleave with a recovery publish.
|
||||
lock (_deliveryLock)
|
||||
{
|
||||
foreach (var pair in rs.ItemHandlers)
|
||||
foreach (var rs in _subscriptions.Values)
|
||||
{
|
||||
var fullRef = RefOf(pair.Item);
|
||||
_lastValues.TryGetValue(fullRef, out var last);
|
||||
_awaitingRecovery[fullRef] = 0;
|
||||
// F1: stamped with the DETECTION time, as the host's own comms-loss fan does. The recorder historizes
|
||||
// SourceTimestampUtc ?? ServerTimestampUtc, so the last Good's source time here put the Bad at the
|
||||
// Good row's (TagId, SourceTimeUtc) key, where the insert-if-absent historian dropped it.
|
||||
RaiseDataChange(new DataChangeEventArgs(rs.Handle, fullRef, new DataValueSnapshot(
|
||||
last?.Value, StatusBadNoCommunication, now, now)));
|
||||
foreach (var pair in rs.ItemHandlers)
|
||||
{
|
||||
var fullRef = RefOf(pair.Item);
|
||||
_lastValues.TryGetValue(fullRef, out var last);
|
||||
_awaitingRecovery[fullRef] = 0;
|
||||
_deliverySeq.AddOrUpdate(fullRef, 1, (_, n) => n + 1);
|
||||
// F1: stamped with the DETECTION time, as the host's own comms-loss fan does. The recorder historizes
|
||||
// SourceTimestampUtc ?? ServerTimestampUtc, so the last Good's source time here put the Bad at the
|
||||
// Good row's (TagId, SourceTimeUtc) key, where the insert-if-absent historian dropped it.
|
||||
RaiseDataChange(new DataChangeEventArgs(rs.Handle, fullRef, new DataValueSnapshot(
|
||||
last?.Value, StatusBadNoCommunication, now, now)));
|
||||
}
|
||||
}
|
||||
}
|
||||
AnnotateAlarmQuality(StatusBadNoCommunication, now);
|
||||
@@ -2856,15 +3124,13 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
|
||||
for (var i = 0; i < refs.Count; i++)
|
||||
{
|
||||
if (!Current()) return;
|
||||
if (!_awaitingRecovery.TryRemove(refs[i], out _)) continue; // a notification got there first
|
||||
_lastValues[refs[i]] = results[i];
|
||||
RecordUpstreamGood(refs[i], results[i]);
|
||||
// The read is upstream data published inside the replay window: a replay older than it is late too.
|
||||
if (results[i].SourceTimestampUtc is { } readAt && StatusCode.IsGood(results[i].StatusCode))
|
||||
_replayGuard.AddOrUpdate(refs[i], readAt.Ticks, (_, prior) => Math.Max(prior, readAt.Ticks));
|
||||
foreach (var rs in _subscriptions.Values)
|
||||
if (rs.ItemHandlers.Any(p => RefOf(p.Item) == refs[i]))
|
||||
RaiseDataChange(new DataChangeEventArgs(rs.Handle, refs[i], results[i]));
|
||||
// One critical section with the live notification path (_deliveryLock): a notification that
|
||||
// removed the ref first wins, and none can overtake this publish once it has decided.
|
||||
lock (_deliveryLock)
|
||||
{
|
||||
if (!_awaitingRecovery.TryRemove(refs[i], out _)) continue; // a notification got there first
|
||||
PublishUpstreamLocked(refs[i], results[i]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2880,6 +3146,108 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Publish an upstream value read (not notified) for <paramref name="fullRef"/> to every subscription carrying it,
|
||||
/// recording it as the last delivered value and — when Good with a source time — as the replay guard's mark, so a
|
||||
/// replay older than the read is late too. Caller holds <see cref="_deliveryLock"/>.
|
||||
/// </summary>
|
||||
private void PublishUpstreamLocked(string fullRef, DataValueSnapshot value)
|
||||
{
|
||||
_lastValues[fullRef] = value;
|
||||
_deliverySeq.AddOrUpdate(fullRef, 1, (_, n) => n + 1);
|
||||
RecordUpstreamGood(fullRef, value);
|
||||
if (value.SourceTimestampUtc is { } readAt && StatusCode.IsGood(value.StatusCode)
|
||||
&& Interlocked.Read(ref _replayGuardUntilTicks) != 0)
|
||||
_replayGuard.AddOrUpdate(fullRef, readAt.Ticks, (_, prior) => Math.Max(prior, readAt.Ticks));
|
||||
foreach (var rs in _subscriptions.Values)
|
||||
if (rs.ItemHandlers.Any(p => RefOf(p.Item) == fullRef))
|
||||
RaiseDataChange(new DataChangeEventArgs(rs.Handle, fullRef, value));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// #676 — the trailing reconcile read after an in-place recovery. Found by the TCP-stall wire test: a value change
|
||||
/// the server published just after a stall ended rode a Publish response that had been in flight across the
|
||||
/// stall; the SDK timed that request out (<c>BadTimeout</c>), discarded the response, and never republished the
|
||||
/// message (the following keep-alives advanced the sequence number past it). The node stayed on the value from
|
||||
/// the recovery read — stale, indefinitely, for a value that does not change again. Once every request that was
|
||||
/// in flight across the stall has completed or timed out (<see cref="PostRecoveryReconcileDelay"/>), every
|
||||
/// subscribed ref is read once more and published when <see cref="IsReconcileRepair"/> says the read differs from
|
||||
/// what was last delivered and is not older than it. The decision and the publish are one critical section with
|
||||
/// the live notification path (<see cref="_deliveryLock"/>), so a newer notification can never be overtaken.
|
||||
/// Bound to the session's work token: a re-initialise or shutdown cancels it.
|
||||
/// </summary>
|
||||
/// <param name="session">The session that recovered; nothing happens if it is no longer current or is in outage.</param>
|
||||
/// <param name="recovery">The recovery read, which runs first so this read is compared against its result.</param>
|
||||
/// <param name="delay">How long to wait before reading (on <see cref="Clock"/>).</param>
|
||||
/// <param name="ct">The session's background-work token.</param>
|
||||
private async Task ReconcileAfterRecoveryAsync(ISession session, Task recovery, TimeSpan delay, CancellationToken ct)
|
||||
{
|
||||
bool Current() => !ct.IsCancellationRequested
|
||||
&& ReferenceEquals(session, Session)
|
||||
&& Volatile.Read(ref _sessionLost) == 0
|
||||
&& Volatile.Read(ref _subscriptionLost) == 0;
|
||||
try
|
||||
{
|
||||
if (delay > TimeSpan.Zero) await Task.Delay(delay, Clock, ct).ConfigureAwait(false);
|
||||
await recovery.WaitAsync(ct).ConfigureAwait(false);
|
||||
if (!Current()) return;
|
||||
var refs = _subscriptions.Values
|
||||
.SelectMany(rs => rs.ItemHandlers.Select(p => RefOf(p.Item)))
|
||||
.Distinct(StringComparer.Ordinal)
|
||||
.ToList();
|
||||
if (refs.Count == 0) return;
|
||||
// Captured BEFORE the read is issued: a ref delivered to while the read is in flight has a value at least as
|
||||
// new as anything the read could have sampled, so the read must not overwrite it.
|
||||
var seqAtIssue = refs.Select(r => _deliverySeq.GetValueOrDefault(r)).ToArray();
|
||||
var (results, transportOk) = await ReadCoreAsync(refs, ct, session).ConfigureAwait(false);
|
||||
if (!transportOk) return; // a new outage — the keep-alive reports it
|
||||
var published = 0;
|
||||
for (var i = 0; i < refs.Count; i++)
|
||||
{
|
||||
if (!Current()) return;
|
||||
lock (_deliveryLock)
|
||||
{
|
||||
if (_deliverySeq.GetValueOrDefault(refs[i]) != seqAtIssue[i]) continue;
|
||||
if (!IsReconcileRepair(refs[i], results[i])) continue;
|
||||
BeforeReconcilePublishForTest?.Invoke(refs[i]);
|
||||
_awaitingRecovery.TryRemove(refs[i], out _);
|
||||
PublishUpstreamLocked(refs[i], results[i]);
|
||||
}
|
||||
published++;
|
||||
}
|
||||
if (published > 0)
|
||||
_logger.LogInformation(
|
||||
"OpcUaClientDriver '{DriverInstanceId}': trailing re-read after recovery found {Count} of {Total} refs " +
|
||||
"whose latest change never arrived through the subscription; published them.",
|
||||
_driverInstanceId, published, refs.Count);
|
||||
}
|
||||
catch (OperationCanceledException) when (ct.IsCancellationRequested) { /* session released */ }
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "OpcUaClientDriver '{DriverInstanceId}': trailing re-read after a recovery failed.",
|
||||
_driverInstanceId);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Whether the trailing read's <paramref name="read"/> repairs <paramref name="fullRef"/>: its (value, status)
|
||||
/// differs from the last value delivered for the ref, and it is not OLDER than it. Any status counts — a Good→Bad
|
||||
/// or Good→Uncertain change lost in the stalled pipeline is as stale as a lost value. "Older" is judged on source
|
||||
/// timestamps only when both carry one; a read with no source timestamp (a server that does not send them) is the
|
||||
/// server's CURRENT state, read after the stalled pipeline drained, so it counts as newer. An EQUAL source
|
||||
/// timestamp with different content is published too: it is not a replay but the server's current state, and it
|
||||
/// repairs the one change the replay guard drops by design (a status-only change that kept its source time inside
|
||||
/// the window). Caller holds <see cref="_deliveryLock"/>.
|
||||
/// </summary>
|
||||
private bool IsReconcileRepair(string fullRef, DataValueSnapshot read)
|
||||
{
|
||||
if (!_lastValues.TryGetValue(fullRef, out var last)) return true;
|
||||
if (last.StatusCode == read.StatusCode && Utils.IsEqual(last.Value, read.Value)) return false;
|
||||
return read.SourceTimestampUtc is not { } src
|
||||
|| last.SourceTimestampUtc is not { } lastSrc
|
||||
|| src >= lastSrc;
|
||||
}
|
||||
|
||||
/// <summary>Quality-only annotation (#626 pattern) of every subscribed alarm condition.</summary>
|
||||
private void AnnotateAlarmQuality(uint quality, DateTime now)
|
||||
{
|
||||
@@ -2981,6 +3349,7 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
|
||||
}
|
||||
catch { /* timeout or already disposed — proceed */ }
|
||||
_gate.Dispose();
|
||||
try { _sessionWorkCts.Cancel(); } catch (ObjectDisposedException) { }
|
||||
SdkTelemetry.Release(_sdkTelemetry);
|
||||
}
|
||||
|
||||
@@ -3010,6 +3379,7 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
|
||||
}
|
||||
catch { /* timeout or already disposed */ }
|
||||
_gate.Dispose();
|
||||
try { _sessionWorkCts.Cancel(); } catch (ObjectDisposedException) { }
|
||||
SdkTelemetry.Release(_sdkTelemetry);
|
||||
}
|
||||
}
|
||||
|
||||
-8
@@ -53,11 +53,6 @@
|
||||
<InputNumber @bind-Value="_form.OpcUa.KeepAliveIntervalSeconds" @bind-Value:after="EmitAsync" class="form-control form-control-sm" />
|
||||
<div class="form-text">Default 5 s.</div>
|
||||
</div>
|
||||
<div class="col-md-3">
|
||||
<label class="form-label">Reconnect period (s)</label>
|
||||
<InputNumber @bind-Value="_form.OpcUa.ReconnectPeriodSeconds" @bind-Value:after="EmitAsync" class="form-control form-control-sm" />
|
||||
<div class="form-text">Initial delay after session drop. Default 5 s.</div>
|
||||
</div>
|
||||
<div class="col-md-3">
|
||||
<label class="form-label">Max discovered nodes</label>
|
||||
<InputNumber @bind-Value="_form.OpcUa.MaxDiscoveredNodes" @bind-Value:after="EmitAsync" class="form-control form-control-sm" />
|
||||
@@ -330,7 +325,6 @@
|
||||
public int TimeoutSeconds { get; set; } = 10;
|
||||
public int SessionTimeoutSeconds { get; set; } = 120;
|
||||
public int KeepAliveIntervalSeconds { get; set; } = 5;
|
||||
public int ReconnectPeriodSeconds { get; set; } = 5;
|
||||
public int MaxDiscoveredNodes { get; set; } = 10_000;
|
||||
public int MaxBrowseDepth { get; set; } = 10;
|
||||
|
||||
@@ -370,7 +364,6 @@
|
||||
TimeoutSeconds = (int)r.Timeout.TotalSeconds,
|
||||
SessionTimeoutSeconds = (int)r.SessionTimeout.TotalSeconds,
|
||||
KeepAliveIntervalSeconds = (int)r.KeepAliveInterval.TotalSeconds,
|
||||
ReconnectPeriodSeconds = (int)r.ReconnectPeriod.TotalSeconds,
|
||||
MaxDiscoveredNodes = r.MaxDiscoveredNodes,
|
||||
MaxBrowseDepth = r.MaxBrowseDepth,
|
||||
SecurityMode = r.SecurityMode,
|
||||
@@ -404,7 +397,6 @@
|
||||
Timeout = TimeSpan.FromSeconds(TimeoutSeconds),
|
||||
SessionTimeout = TimeSpan.FromSeconds(SessionTimeoutSeconds),
|
||||
KeepAliveInterval = TimeSpan.FromSeconds(KeepAliveIntervalSeconds),
|
||||
ReconnectPeriod = TimeSpan.FromSeconds(ReconnectPeriodSeconds),
|
||||
MaxDiscoveredNodes = MaxDiscoveredNodes,
|
||||
MaxBrowseDepth = MaxBrowseDepth,
|
||||
SecurityMode = SecurityMode,
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
using ZB.MOM.WW.OtOpcUa.Cluster;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Host.Configuration;
|
||||
|
||||
/// <summary>
|
||||
/// Names the per-role appsettings overlay (<c>appsettings.{suffix}.json</c>) for a node.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The suffix is built from the node's PLAIN roles only. <c>cluster-{ClusterId}</c> roles are mesh-membership
|
||||
/// tags (and carry a verbatim, operator-chosen ClusterId), not configuration profiles — folding one into the
|
||||
/// filename produced names like <c>appsettings.admin-cluster-MAIN-driver.json</c> that never exist, so on every
|
||||
/// node with a cluster role the shipped overlays silently never loaded.
|
||||
/// </remarks>
|
||||
internal static class RoleOverlay
|
||||
{
|
||||
/// <summary>
|
||||
/// The overlay filename suffix: the plain (non-cluster) roles, ordinal-sorted and joined with <c>-</c>, or
|
||||
/// <see langword="null"/> when the node has none.
|
||||
/// </summary>
|
||||
public static string? Suffix(IReadOnlyList<string> gossipedRoles)
|
||||
{
|
||||
var plain = gossipedRoles
|
||||
.Where(r => !string.IsNullOrWhiteSpace(r) && !RoleParser.IsClusterRole(r))
|
||||
.OrderBy(r => r, StringComparer.Ordinal)
|
||||
.ToList();
|
||||
return plain.Count == 0 ? null : string.Join('-', plain);
|
||||
}
|
||||
}
|
||||
@@ -13,6 +13,7 @@ using ZB.MOM.WW.OtOpcUa.OpcUaServer.Security;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Historian;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Redundancy;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.ScriptedAlarms;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Host.OpcUa;
|
||||
@@ -46,6 +47,7 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
|
||||
private readonly ILogger<OtOpcUaServerHostedService> _logger;
|
||||
private readonly IDriverPrimaryRoleSource? _primaryRoleSource;
|
||||
private readonly HistorianUpdaterHolder _historianUpdater;
|
||||
private readonly IHistoryUpdateRoleView? _historyUpdateRoleView;
|
||||
|
||||
private OpcUaApplicationHost? _appHost;
|
||||
private OtOpcUaSdkServer? _server;
|
||||
@@ -98,6 +100,9 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
|
||||
/// <param name="historianUpdater">The selected historian backend's optional HistoryUpdate capability.
|
||||
/// Optional: when unregistered it resolves to <see cref="HistorianUpdaterHolder.None"/> and HistoryUpdate is
|
||||
/// reported unsupported on every node.</param>
|
||||
/// <param name="historyUpdateRoleView">Gitea #674 — the HistoryUpdate verdict <c>DriverHostActor</c> publishes (the
|
||||
/// data-plane Primary verdict, closed until this node's first redundancy snapshot). Optional: registered on every
|
||||
/// driver-role node; an admin-only node has none and wires no HistoryUpdate Primary gate (allow).</param>
|
||||
public OtOpcUaServerHostedService(
|
||||
IOptions<OpcUaApplicationHostOptions> options,
|
||||
DeferredAddressSpaceSink deferredSink,
|
||||
@@ -113,7 +118,8 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
|
||||
TimeProvider time,
|
||||
ILoggerFactory loggerFactory,
|
||||
IDriverPrimaryRoleSource? primaryRoleSource = null,
|
||||
HistorianUpdaterHolder? historianUpdater = null)
|
||||
HistorianUpdaterHolder? historianUpdater = null,
|
||||
IHistoryUpdateRoleView? historyUpdateRoleView = null)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(aclOptions);
|
||||
ArgumentNullException.ThrowIfNull(time);
|
||||
@@ -137,6 +143,7 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
|
||||
_loggerFactory = loggerFactory;
|
||||
_primaryRoleSource = primaryRoleSource;
|
||||
_historianUpdater = historianUpdater ?? HistorianUpdaterHolder.None;
|
||||
_historyUpdateRoleView = historyUpdateRoleView;
|
||||
_logger = loggerFactory.CreateLogger<OtOpcUaServerHostedService>();
|
||||
}
|
||||
|
||||
@@ -310,6 +317,12 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
|
||||
// section is absent; NodeManager is non-null here (guarded above).
|
||||
_server.NodeManager.MaxTieClusterOverfetch = _serverHistorianOptions.MaxTieClusterOverfetch;
|
||||
|
||||
// Gitea #691: the ReturnBounds kill switch (default on).
|
||||
_server.NodeManager.HonorReturnBounds = HonorReturnBounds;
|
||||
|
||||
// Gitea #694: the opt-in HistoryReadEvents horizon (default off = exact to each read's own window).
|
||||
_server.NodeManager.EventReadHorizon = EventReadHorizon;
|
||||
|
||||
// Propagate the S3 HistoryRead concurrency bounds (arch-review 03/S3): bounded per-node fan-out
|
||||
// within a batch, the process-wide concurrent-batch limiter, and the per-request deadline. Defaults
|
||||
// (4 / 16 / 60 s) survive when the ServerHistorian section is absent.
|
||||
@@ -317,6 +330,14 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
|
||||
_server.NodeManager.MaxConcurrentHistoryReadBatches = _serverHistorianOptions.MaxConcurrentHistoryReadBatches;
|
||||
_server.NodeManager.HistoryReadDeadline = _serverHistorianOptions.HistoryReadDeadline;
|
||||
|
||||
// Gitea #674: HistoryUpdate is Primary-only (BadInvalidState elsewhere, and closed until this node's first
|
||||
// redundancy snapshot) and refuses a SourceTimestamp beyond now + HistoryUpdateMaxFutureSkew (BadOutOfRange),
|
||||
// judged against the host clock.
|
||||
var historyUpdate = BuildHistoryUpdatePosture();
|
||||
_server.NodeManager.HistoryUpdatePrimaryGate = historyUpdate.PrimaryGate;
|
||||
_server.NodeManager.HistoryUpdateMaxFutureSkew = historyUpdate.MaxFutureSkew;
|
||||
_server.NodeManager.Clock = historyUpdate.Clock;
|
||||
|
||||
// ServiceLevel publisher needs IServerInternal — only available after Start.
|
||||
if (_server.CurrentInstance is { } serverInternal)
|
||||
{
|
||||
@@ -332,6 +353,40 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
|
||||
_logger.LogInformation("OtOpcUaServerHostedService: SDK started, address-space + ServiceLevel sinks bound");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #691 — the <c>ServerHistorian:HonorReturnBounds</c> kill switch <c>StartAsync</c> applies to the node manager
|
||||
/// (default <c>true</c> when the section or key is absent). Exposed so the wiring is testable without booting the SDK.
|
||||
/// </summary>
|
||||
internal bool HonorReturnBounds => _serverHistorianOptions.HonorReturnBounds;
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #694 — the <c>ServerHistorian:EventReadHorizon</c> <c>StartAsync</c> applies to the node manager (default
|
||||
/// <see cref="TimeSpan.Zero"/> = none). Exposed so the wiring is testable without booting the SDK.
|
||||
/// </summary>
|
||||
internal TimeSpan EventReadHorizon => _serverHistorianOptions.EventReadHorizon;
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #674 — what <c>StartAsync</c> wires onto the node manager for HistoryUpdate: the Primary gate (read from
|
||||
/// the <see cref="IHistoryUpdateRoleView"/> on every call; null when none is registered, i.e. allow), the
|
||||
/// future-timestamp bound from <c>ServerHistorian:HistoryUpdateMaxFutureSkew</c>, and the host clock it is judged
|
||||
/// against. Separate from <c>StartAsync</c> so the wiring is testable without booting the SDK.
|
||||
/// </summary>
|
||||
/// <returns>The posture to apply.</returns>
|
||||
internal HistoryUpdatePosture BuildHistoryUpdatePosture()
|
||||
{
|
||||
Func<bool>? gate = _historyUpdateRoleView is { } view ? () => view.ShouldServeHistoryUpdate : null;
|
||||
if (gate is null)
|
||||
_logger.LogInformation(
|
||||
"OtOpcUaServerHostedService: no IHistoryUpdateRoleView registered (no redundancy manager on this node) — HistoryUpdate is not Primary-gated");
|
||||
return new HistoryUpdatePosture(gate, _serverHistorianOptions.HistoryUpdateMaxFutureSkew, _time);
|
||||
}
|
||||
|
||||
/// <summary>Gitea #674 — the HistoryUpdate settings <see cref="BuildHistoryUpdatePosture"/> produces.</summary>
|
||||
/// <param name="PrimaryGate">The node manager's HistoryUpdate Primary gate; null allows.</param>
|
||||
/// <param name="MaxFutureSkew">The future-timestamp bound; non-positive disables it.</param>
|
||||
/// <param name="Clock">The clock the bound is judged against.</param>
|
||||
internal sealed record HistoryUpdatePosture(Func<bool>? PrimaryGate, TimeSpan MaxFutureSkew, TimeProvider Clock);
|
||||
|
||||
/// <summary>
|
||||
/// Arms the clock-driven monitored-item revalidation tick (arch-review 03/S19).
|
||||
/// </summary>
|
||||
@@ -382,6 +437,7 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
|
||||
// Clear the native-alarm-ack router so a late ack doesn't Tell a stopping DriverHostActor.
|
||||
_server?.SetNativeAlarmAckRouter(null);
|
||||
_server?.SetNativeAlarmAckPrimaryGate(null);
|
||||
_server?.SetHistoryUpdatePrimaryGate(null);
|
||||
// Restore the Null historian so a late HistoryRead doesn't hit a disposed read client.
|
||||
_server?.SetHistorianDataSource(null);
|
||||
_server?.SetHistorianDataUpdater(null);
|
||||
|
||||
@@ -90,8 +90,9 @@ builder.WebHost.UseStaticWebAssets();
|
||||
builder.Host.UseWindowsService(options => options.ServiceName = "OtOpcUaHost");
|
||||
|
||||
// Per-role appsettings overlay: appsettings.{role}.json (single role) or appsettings.admin-driver.json
|
||||
// (both). Optional — base appsettings.json carries enough to boot if these don't exist.
|
||||
var roleSuffix = roles.Count == 0 ? null : string.Join('-', roles.OrderBy(r => r, StringComparer.Ordinal));
|
||||
// (both). Optional — base appsettings.json carries enough to boot if these don't exist. Built from the
|
||||
// PLAIN roles only (#678.6): a cluster-{ClusterId} role in the name pointed at a file that never exists.
|
||||
var roleSuffix = RoleOverlay.Suffix(roles);
|
||||
if (roleSuffix is not null)
|
||||
{
|
||||
builder.Configuration.AddJsonFile($"appsettings.{roleSuffix}.json", optional: true, reloadOnChange: true);
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
"MinimumLevel": {
|
||||
"Default": "Information",
|
||||
"Override": {
|
||||
"Opc.Ua": "Debug",
|
||||
"Opc.Ua": "Information",
|
||||
"Akka": "Information"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -35,7 +35,9 @@
|
||||
"AllowUntrustedServerCertificate": false,
|
||||
"CaCertificatePath": null,
|
||||
"CallTimeout": "00:00:30",
|
||||
"MaxTieClusterOverfetch": 65536
|
||||
"MaxTieClusterOverfetch": 65536,
|
||||
"EventReadHorizon": "00:00:00",
|
||||
"_EventReadHorizonComment": "Gitea #694, opt-in. 00:00:00 = no horizon: HistoryReadEvents is exact to each read's window (on the Gateway a reverse/end-only events read proves 'nothing older' with one read down to 1970 - seconds on a large Runtime.dbo.Events). A positive span (e.g. 365.00:00:00) never reads events older than now minus it, in either direction - LOSSY: older events become invisible to HistoryReadEvents."
|
||||
},
|
||||
"ContinuousHistorization": {
|
||||
"_comment": "Continuous historization of driver (non-Galaxy) tag values: a crash-safe FasterLog outbox + recorder draining to the ServerHistorian gateway's WriteLiveValues. Disabled => no recorder is spawned. Requires ServerHistorian to be configured; the gateway connection (endpoint/key/TLS) is sourced from the ServerHistorian section, not here.",
|
||||
|
||||
@@ -0,0 +1,91 @@
|
||||
using System.Reflection;
|
||||
using Opc.Ua;
|
||||
using Opc.Ua.Server;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.OpcUaServer;
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #682 — make <c>HistoryServerCapabilities/AggregateFunctions</c> (i=11201) list exactly the aggregates the
|
||||
/// wired historian backend serves for HistoryRead (<see cref="IHistorianDataSource.SupportedAggregates"/>), instead of
|
||||
/// the SDK's full registered set.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// ⚠️ <c>ServerCapabilities/AggregateFunctions</c> (i=2997) is deliberately NOT trimmed. Part 5 defines it as every
|
||||
/// aggregate the SERVER supports, and the server still supports all the SDK's registered aggregates for live
|
||||
/// data: a MonitoredItem <c>AggregateFilter</c> is validated and computed by the SDK's <c>AggregateManager</c>,
|
||||
/// not the historian. The SDK itself models the split — <c>DiagnosticsNodeManager.AddAggregateFunction</c> always
|
||||
/// adds to i=2997 and adds to i=11201 only when the aggregate is historical. i=11201 is the history surface, so it
|
||||
/// is the one that follows the backend.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Only the folder ⇄ aggregate <c>Organizes</c> reference pair is removed or restored (both directions); the
|
||||
/// calculator factories and the aggregate-function object nodes stay. Idempotent and reversible, so re-wiring a
|
||||
/// different source re-syncs the folder.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
internal static class AdvertisedAggregateFunctions
|
||||
{
|
||||
/// <summary>Every standard aggregate-function NodeId the SDK's nodeset knows (<c>ObjectIds.AggregateFunction_*</c>).</summary>
|
||||
private static readonly NodeId[] StandardAggregateIds = typeof(ObjectIds)
|
||||
.GetFields(BindingFlags.Public | BindingFlags.Static)
|
||||
.Where(f => f.FieldType == typeof(NodeId) && f.Name.StartsWith("AggregateFunction_", StringComparison.Ordinal))
|
||||
.Select(f => (NodeId)f.GetValue(null)!)
|
||||
.ToArray();
|
||||
|
||||
/// <summary>The trimmed folder — the HISTORY aggregate list only (see remarks for why not i=2997).</summary>
|
||||
internal static readonly NodeId Folder = ObjectIds.HistoryServerCapabilities_AggregateFunctions;
|
||||
|
||||
/// <summary>
|
||||
/// The set a source serves, treating a <c>null</c> <see cref="IHistorianDataSource.SupportedAggregates"/> (an
|
||||
/// out-of-tree source that returns null despite the non-nullable contract) as <see cref="HistoryAggregateTypes.All"/>,
|
||||
/// the interface default — the ONE null rule both the advertisement and HistoryRead-Processed use.
|
||||
/// </summary>
|
||||
/// <param name="source">The historian source.</param>
|
||||
/// <returns>Its served set, never null.</returns>
|
||||
public static IReadOnlySet<HistoryAggregateType> ServedBy(IHistorianDataSource source) =>
|
||||
source.SupportedAggregates ?? HistoryAggregateTypes.All;
|
||||
|
||||
/// <summary>
|
||||
/// Sync i=11201 to <paramref name="served"/>: an aggregate-function node is organized under it iff the server maps
|
||||
/// it (<see cref="AggregateFunctionMap"/>) AND <paramref name="served"/> contains it.
|
||||
/// </summary>
|
||||
/// <param name="server">The running server.</param>
|
||||
/// <param name="served">The backend's served aggregates.</param>
|
||||
/// <returns>The advertised NodeIds, or <c>null</c> when the diagnostics node manager or folder is not available.</returns>
|
||||
public static IReadOnlyList<NodeId>? Apply(IServerInternal server, IReadOnlySet<HistoryAggregateType> served)
|
||||
{
|
||||
if (server?.DiagnosticsNodeManager is not CustomNodeManager2 diagnostics) return null;
|
||||
|
||||
var advertised = new List<NodeId>();
|
||||
lock (diagnostics.Lock)
|
||||
{
|
||||
var folder = diagnostics.FindPredefinedNode(Folder, typeof(BaseObjectState));
|
||||
if (folder is null) return null;
|
||||
|
||||
foreach (var aggregateId in StandardAggregateIds)
|
||||
{
|
||||
var aggregateNode = diagnostics.FindPredefinedNode(aggregateId, null);
|
||||
if (aggregateNode is null) continue;
|
||||
|
||||
var advertise = AggregateFunctionMap.Map(aggregateId) is { } aggregate && served.Contains(aggregate);
|
||||
if (advertise) advertised.Add(aggregateId);
|
||||
|
||||
var linked = folder.ReferenceExists(ReferenceTypeIds.Organizes, false, aggregateId);
|
||||
if (advertise && !linked)
|
||||
{
|
||||
folder.AddReference(ReferenceTypeIds.Organizes, false, aggregateId);
|
||||
aggregateNode.AddReference(ReferenceTypeIds.Organizes, true, folder.NodeId);
|
||||
}
|
||||
else if (!advertise && linked)
|
||||
{
|
||||
folder.RemoveReference(ReferenceTypeIds.Organizes, false, aggregateId);
|
||||
aggregateNode.RemoveReference(ReferenceTypeIds.Organizes, true, folder.NodeId);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return advertised;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
using System.Collections.Frozen;
|
||||
using Opc.Ua;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.OpcUaServer;
|
||||
|
||||
/// <summary>
|
||||
/// The one table between OPC UA Part 13 standard aggregate-function NodeIds (<c>ObjectIds.AggregateFunction_*</c>)
|
||||
/// and <see cref="HistoryAggregateType"/>. HistoryRead-Processed maps a request's aggregate through it, and the
|
||||
/// advertised <c>AggregateFunctions</c> folders (#682) map a backend's served set back through it — one table, so a
|
||||
/// NodeId the server can dispatch is exactly a NodeId it can advertise.
|
||||
/// </summary>
|
||||
internal static class AggregateFunctionMap
|
||||
{
|
||||
/// <summary>NodeId ⇒ aggregate, for every aggregate the server can name to a backend.</summary>
|
||||
public static readonly FrozenDictionary<NodeId, HistoryAggregateType> ByNodeId = new Dictionary<NodeId, HistoryAggregateType>
|
||||
{
|
||||
[ObjectIds.AggregateFunction_Average] = HistoryAggregateType.Average,
|
||||
[ObjectIds.AggregateFunction_Minimum] = HistoryAggregateType.Minimum,
|
||||
[ObjectIds.AggregateFunction_Maximum] = HistoryAggregateType.Maximum,
|
||||
[ObjectIds.AggregateFunction_Total] = HistoryAggregateType.Total,
|
||||
[ObjectIds.AggregateFunction_Count] = HistoryAggregateType.Count,
|
||||
[ObjectIds.AggregateFunction_Interpolative] = HistoryAggregateType.Interpolative,
|
||||
[ObjectIds.AggregateFunction_TimeAverage] = HistoryAggregateType.TimeAverage,
|
||||
[ObjectIds.AggregateFunction_Start] = HistoryAggregateType.Start,
|
||||
[ObjectIds.AggregateFunction_End] = HistoryAggregateType.End,
|
||||
[ObjectIds.AggregateFunction_Delta] = HistoryAggregateType.Delta,
|
||||
[ObjectIds.AggregateFunction_Range] = HistoryAggregateType.Range,
|
||||
[ObjectIds.AggregateFunction_MinimumActualTime] = HistoryAggregateType.MinimumActualTime,
|
||||
[ObjectIds.AggregateFunction_MaximumActualTime] = HistoryAggregateType.MaximumActualTime,
|
||||
[ObjectIds.AggregateFunction_StartBound] = HistoryAggregateType.StartBound,
|
||||
[ObjectIds.AggregateFunction_EndBound] = HistoryAggregateType.EndBound,
|
||||
[ObjectIds.AggregateFunction_DeltaBounds] = HistoryAggregateType.DeltaBounds,
|
||||
[ObjectIds.AggregateFunction_DurationGood] = HistoryAggregateType.DurationGood,
|
||||
[ObjectIds.AggregateFunction_DurationBad] = HistoryAggregateType.DurationBad,
|
||||
[ObjectIds.AggregateFunction_PercentGood] = HistoryAggregateType.PercentGood,
|
||||
[ObjectIds.AggregateFunction_PercentBad] = HistoryAggregateType.PercentBad,
|
||||
[ObjectIds.AggregateFunction_WorstQuality] = HistoryAggregateType.WorstQuality,
|
||||
[ObjectIds.AggregateFunction_NumberOfTransitions] = HistoryAggregateType.NumberOfTransitions,
|
||||
[ObjectIds.AggregateFunction_AnnotationCount] = HistoryAggregateType.AnnotationCount,
|
||||
[ObjectIds.AggregateFunction_StandardDeviationPopulation] = HistoryAggregateType.StandardDeviationPopulation,
|
||||
[ObjectIds.AggregateFunction_VariancePopulation] = HistoryAggregateType.VariancePopulation,
|
||||
[ObjectIds.AggregateFunction_TimeAverage2] = HistoryAggregateType.TimeAverage2,
|
||||
[ObjectIds.AggregateFunction_Total2] = HistoryAggregateType.Total2,
|
||||
}.ToFrozenDictionary();
|
||||
|
||||
/// <summary>
|
||||
/// Map a request's aggregate-function NodeId to <see cref="HistoryAggregateType"/>; <c>null</c> for one the server
|
||||
/// cannot name to any backend (the caller answers <c>BadAggregateNotSupported</c>).
|
||||
/// </summary>
|
||||
/// <param name="aggregateNodeId">The per-node aggregate-function NodeId from the request.</param>
|
||||
/// <returns>The mapped aggregate, or <c>null</c>.</returns>
|
||||
public static HistoryAggregateType? Map(NodeId? aggregateNodeId) =>
|
||||
aggregateNodeId is not null && ByNodeId.TryGetValue(aggregateNodeId, out var aggregate) ? aggregate : null;
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
namespace ZB.MOM.WW.OtOpcUa.OpcUaServer;
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #694 item 2: the opt-in <c>ServerHistorian:EventReadHorizon</c> — HistoryReadEvents never reads events older than
|
||||
/// <c>now − horizon</c>. Off (<see cref="TimeSpan.Zero"/>, the default) a read is exact down to whatever its window says,
|
||||
/// which on the Gateway means a reverse / end-only read walked back with forward reads proves "nothing older" with one
|
||||
/// read down to the Unix epoch (#691) — ~3.8 s live on <c>Runtime.dbo.Events</c>, and an unfiltered one can hit the
|
||||
/// gateway's 30 s SQL timeout. A forward read whose StartTime is far in the past costs the same on that table. The
|
||||
/// horizon bounds both, at a stated price: events older than it are not returned, in either direction, on any backend.
|
||||
/// </summary>
|
||||
internal static class EventHorizon
|
||||
{
|
||||
/// <summary>
|
||||
/// Raises the EARLIER bound of a fresh events window to the horizon's floor (<c>now − horizon</c>, inclusive): the
|
||||
/// start of a forward <c>[start, end)</c> window, the exclusive lower end of a reverse <c>(end, start]</c> one.
|
||||
/// </summary>
|
||||
/// <param name="startUtc">The window start (the later bound of a reverse read).</param>
|
||||
/// <param name="endUtc">The window end (the exclusive earlier bound of a reverse read).</param>
|
||||
/// <param name="reverse">Whether the read is newest-first (<c>start > end</c>).</param>
|
||||
/// <param name="horizon">The horizon; non-positive = none (nothing changes).</param>
|
||||
/// <param name="nowUtc">Now.</param>
|
||||
/// <returns><c>false</c> when the whole window lies before the floor — the read is empty and must not be sent.</returns>
|
||||
public static bool TryClamp(ref DateTime startUtc, ref DateTime endUtc, bool reverse, TimeSpan horizon, DateTime nowUtc)
|
||||
{
|
||||
if (horizon <= TimeSpan.Zero) return true;
|
||||
var floor = nowUtc.Ticks - DateTime.MinValue.Ticks > horizon.Ticks
|
||||
? new DateTime(nowUtc.Ticks - horizon.Ticks, DateTimeKind.Utc)
|
||||
: new DateTime(0, DateTimeKind.Utc);
|
||||
if (floor.Ticks == 0) return true;
|
||||
|
||||
if (reverse)
|
||||
{
|
||||
if (startUtc < floor) return false; // (end, start] lies wholly before the floor
|
||||
var below = floor.AddTicks(-1); // exclusive lower end ⇒ the floor instant itself is read
|
||||
if (endUtc < below) endUtc = below;
|
||||
return true;
|
||||
}
|
||||
|
||||
if (endUtc <= floor) return false; // [start, end) lies wholly before the floor
|
||||
if (startUtc < floor) startUtc = floor;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,241 @@
|
||||
using Opc.Ua;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.OpcUaServer;
|
||||
|
||||
/// <summary>
|
||||
/// The end bound still owed to a HistoryRead-Raw read with <c>ReturnBounds = true</c> (Gitea #691). It rides the
|
||||
/// continuation point, because Part 11 returns the end bound with the LAST page only.
|
||||
/// </summary>
|
||||
/// <param name="AtUtc">The request's end time (the later bound of a forward read, the earlier of a reverse one).</param>
|
||||
/// <param name="Reverse">The read is newest-first (<c>StartTime > EndTime</c>, including an end-only read).</param>
|
||||
/// <param name="Inclusive">
|
||||
/// <c>false</c> only for <c>StartTime == EndTime</c>: the samples AT that instant are the data, so the end bound is the
|
||||
/// first sample strictly after it (Part 11 Table 1 footnote c). Otherwise the end bound is the sample at or beyond the
|
||||
/// end time, and the data window excludes the end time.
|
||||
/// </param>
|
||||
/// <param name="Open">The end time was not given (a one-sided read's sentinel): there is no end bound to look up.</param>
|
||||
/// <param name="DataExhausted">The data is drained and the end bound did not fit on the previous page.</param>
|
||||
/// <param name="LastReturnedUtc">The last value's time returned on an earlier page (stamps an open bound's placeholder).</param>
|
||||
/// <param name="Resolved">
|
||||
/// Gitea #692: the end bound an earlier page already looked up (a stored sample — never a placeholder), so the page that
|
||||
/// returns it does not read the backend again. <c>null</c> = not looked up yet.
|
||||
/// </param>
|
||||
internal sealed record HistoryEndBound(
|
||||
DateTime AtUtc, bool Reverse, bool Inclusive, bool Open, bool DataExhausted = false, DateTime? LastReturnedUtc = null,
|
||||
DataValueSnapshot? Resolved = null);
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #691: OPC UA Part 11 bounding values for HistoryRead-Raw (<c>ReadRawModifiedDetails.ReturnBounds</c>), composed
|
||||
/// in the server from every backend's existing raw read — the ONE authority, so SqlServer, QuestDB, Gateway and Null
|
||||
/// answer identically.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>What a bound is</b> (Part 11 §4.6, Table 1). Forward: the start bound is the stored sample AT or immediately
|
||||
/// BEFORE StartTime, the end bound the one AT or immediately AFTER EndTime. Reverse mirrors it: the start bound is at
|
||||
/// or after StartTime, the end bound at or before EndTime. A sample exactly on a bound IS the bound: on the start it
|
||||
/// is already the first sample of the data (the window includes StartTime), so nothing extra is added; on the end it is
|
||||
/// added (the window excludes EndTime). <c>StartTime == EndTime</c> reads forward, and its end bound is the first
|
||||
/// sample strictly after the instant. A bound is a stored sample of any quality — a Bad sample is returned as stored.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Missing bound.</b> <c>Bad_BoundNotFound</c>, a null value, stamped with the bound's time. A one-sided read's
|
||||
/// missing side (no EndTime / no StartTime) is never looked up; it is <c>Bad_BoundNotFound</c> stamped one second after
|
||||
/// (forward) or before (reverse) the previous returned value (Table 1 footnotes a and b).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Lookups.</b> "At or before T" is a newest-first read of <c>(0, T]</c> capped at one — native on a backend that
|
||||
/// declares <see cref="IHistorianDataSource.SupportsReverseRaw"/>, otherwise <see cref="ReverseRawReader"/>'s walk-back
|
||||
/// from bounded forward reads (the Gateway: NOT AVEVA's synthetic boundary row, which is stamped at the query start
|
||||
/// rather than at the prior sample's own time). "At or after T" is a forward read of <c>[T, MaxValue)</c> capped at one (on a
|
||||
/// forward-only backend ending at now + <see cref="ReverseRawReader.LookAhead"/> instead — #691 review);
|
||||
/// "strictly after T" starts one storage resolution later (lossless: nothing is stored in between). Inside a tie
|
||||
/// cluster the bound is the cluster's first sample in the read's direction — the forward read's first, the reverse
|
||||
/// read's first (the forward order's last).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Resolution.</b> "On the bound" compares at the backend's storage resolution, because a microsecond backend
|
||||
/// (QuestDB) truncates the window bounds it is sent: a sample at <c>⌊T⌋</c> is inside a window starting at <c>T</c>
|
||||
/// there, so it must not also be returned as the start bound.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
internal static class HistoryBounds
|
||||
{
|
||||
private static readonly TimeSpan OpenBoundStep = TimeSpan.FromSeconds(1);
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #692: the placeholders this server synthesised (<see cref="NotFound"/>), by reference. A placeholder is known by
|
||||
/// this marker, never by its shape: a stored row can legitimately carry <c>Bad_BoundNotFound</c> with a null value (a
|
||||
/// client may write one through HistoryUpdate), and that row is data.
|
||||
/// </summary>
|
||||
private static readonly System.Runtime.CompilerServices.ConditionalWeakTable<DataValueSnapshot, object> Placeholders = new();
|
||||
|
||||
private static readonly object PlaceholderMark = new();
|
||||
|
||||
/// <summary>The end bound owed by a fresh read over the normalised window.</summary>
|
||||
/// <param name="startUtc">The (normalised) start: the later time of a reverse read.</param>
|
||||
/// <param name="endUtc">The (normalised) end; a one-sided read's sentinel (tick 0 / MaxValue) is an open bound.</param>
|
||||
/// <returns>The pending end bound.</returns>
|
||||
public static HistoryEndBound For(DateTime startUtc, DateTime endUtc) =>
|
||||
new(endUtc, Reverse: startUtc > endUtc, Inclusive: startUtc != endUtc, Open: IsOpen(endUtc));
|
||||
|
||||
/// <summary>
|
||||
/// Whether a returned value is a <see cref="NotFound"/> placeholder this server synthesised rather than a stored sample
|
||||
/// — by an explicit marker (#692), so a stored row whose status happens to be <c>Bad_BoundNotFound</c> is not taken for one.
|
||||
/// </summary>
|
||||
/// <param name="value">The value.</param>
|
||||
/// <returns><c>true</c> only for an instance <see cref="NotFound"/> produced.</returns>
|
||||
public static bool IsPlaceholder(DataValueSnapshot value) => Placeholders.TryGetValue(value, out _);
|
||||
|
||||
/// <summary>The pending end bound to store behind a continuation point, remembering the page's last returned time.</summary>
|
||||
/// <param name="bound">The pending end bound (null = bounds not requested).</param>
|
||||
/// <param name="page">The page being returned.</param>
|
||||
/// <returns>The bound to carry, or null.</returns>
|
||||
public static HistoryEndBound? Carry(HistoryEndBound? bound, IReadOnlyList<DataValueSnapshot> page) =>
|
||||
bound is null || page.Count == 0 || page[^1].SourceTimestampUtc is not { } last
|
||||
? bound
|
||||
: bound with { LastReturnedUtc = last };
|
||||
|
||||
/// <summary>The <c>Bad_BoundNotFound</c> placeholder (null value) stamped at <paramref name="atUtc"/>.</summary>
|
||||
/// <param name="atUtc">The bound's time.</param>
|
||||
/// <returns>The placeholder.</returns>
|
||||
public static DataValueSnapshot NotFound(DateTime atUtc)
|
||||
{
|
||||
var placeholder = new DataValueSnapshot(null, StatusCodes.BadBoundNotFound, atUtc, atUtc);
|
||||
Placeholders.AddOrUpdate(placeholder, PlaceholderMark);
|
||||
return placeholder;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The start bound to return ahead of the data, or <c>null</c> when a stored sample sits on StartTime (that sample is
|
||||
/// the bound, and it is already the data's first sample).
|
||||
/// </summary>
|
||||
public static async Task<DataValueSnapshot?> StartBoundAsync(
|
||||
IHistorianDataSource source, string tagname, DateTime startUtc, bool reverse, int maxTieCluster, DateTime nowUtc,
|
||||
CancellationToken ct)
|
||||
{
|
||||
if (IsOpen(startUtc)) return NotFound(startUtc);
|
||||
var found = reverse
|
||||
? await EarliestAtOrAfterAsync(source, tagname, startUtc, nowUtc, ct).ConfigureAwait(false)
|
||||
: await LatestAtOrBeforeAsync(source, tagname, startUtc, maxTieCluster, nowUtc, ct).ConfigureAwait(false);
|
||||
if (found is null) return NotFound(startUtc);
|
||||
return SameInstant(Ticks(found), startUtc.Ticks, Resolution(source)) ? null : found;
|
||||
}
|
||||
|
||||
/// <summary>The end bound: the stored sample on or beyond the end time, else the placeholder.</summary>
|
||||
/// <param name="source">The backend.</param>
|
||||
/// <param name="tagname">Historian tag.</param>
|
||||
/// <param name="bound">The pending end bound.</param>
|
||||
/// <param name="previousReturnedUtc">The last value's time already returned (stamps an open bound's placeholder).</param>
|
||||
/// <param name="maxTieCluster">The walk-back's row budget (forward-only backends).</param>
|
||||
/// <param name="nowUtc">Now (the walk-back's look-ahead clamp).</param>
|
||||
/// <param name="ct">Cancellation.</param>
|
||||
public static async Task<DataValueSnapshot> EndBoundAsync(
|
||||
IHistorianDataSource source, string tagname, HistoryEndBound bound, DateTime previousReturnedUtc,
|
||||
int maxTieCluster, DateTime nowUtc, CancellationToken ct)
|
||||
{
|
||||
if (bound.Open)
|
||||
{
|
||||
var ticks = bound.Reverse
|
||||
? Math.Max(0L, previousReturnedUtc.Ticks - OpenBoundStep.Ticks)
|
||||
: Math.Min(DateTime.MaxValue.Ticks, previousReturnedUtc.Ticks + OpenBoundStep.Ticks);
|
||||
return NotFound(new DateTime(ticks, DateTimeKind.Utc));
|
||||
}
|
||||
|
||||
if (bound.Resolved is { } resolved) return resolved; // #692: an earlier page already looked it up
|
||||
|
||||
DataValueSnapshot? found;
|
||||
if (bound.Reverse)
|
||||
found = await LatestAtOrBeforeAsync(source, tagname, bound.AtUtc, maxTieCluster, nowUtc, ct).ConfigureAwait(false);
|
||||
else if (bound.Inclusive)
|
||||
found = await EarliestAtOrAfterAsync(source, tagname, bound.AtUtc, nowUtc, ct).ConfigureAwait(false);
|
||||
else
|
||||
{
|
||||
var resolution = Resolution(source);
|
||||
found = bound.AtUtc.Ticks > DateTime.MaxValue.Ticks - resolution
|
||||
? null
|
||||
: await EarliestAtOrAfterAsync(source, tagname, bound.AtUtc.AddTicks(resolution), nowUtc, ct).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
return found ?? NotFound(bound.AtUtc);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Whether a data sample lies inside the data window of a bounded read — a forward window excludes the end time (a
|
||||
/// sample there is the end bound), compared at the storage resolution. A backend whose forward end is inclusive, or a
|
||||
/// resumed read that steps onto the end time, would otherwise return the end bound twice.
|
||||
/// </summary>
|
||||
public static bool InDataWindow(DataValueSnapshot sample, HistoryEndBound bound, long resolution)
|
||||
{
|
||||
if (!bound.Inclusive || bound.Open) return true;
|
||||
var ts = Align(Ticks(sample), resolution);
|
||||
var end = Align(bound.AtUtc.Ticks, resolution);
|
||||
return bound.Reverse ? ts > end : ts < end;
|
||||
}
|
||||
|
||||
/// <summary>Whether a resumed read's next start is already past the data window (only the end bound remains).</summary>
|
||||
public static bool DataWindowDrained(DateTime nextStartUtc, HistoryEndBound bound, long resolution)
|
||||
{
|
||||
if (!bound.Inclusive || bound.Open) return false;
|
||||
var next = Align(nextStartUtc.Ticks, resolution);
|
||||
var end = Align(bound.AtUtc.Ticks, resolution);
|
||||
return bound.Reverse ? next <= end : next >= end;
|
||||
}
|
||||
|
||||
/// <summary>The latest stored sample at or before <paramref name="atUtc"/>, or null.</summary>
|
||||
internal static async Task<DataValueSnapshot?> LatestAtOrBeforeAsync(
|
||||
IHistorianDataSource source, string tagname, DateTime atUtc, int maxTieCluster, DateTime nowUtc, CancellationToken ct)
|
||||
{
|
||||
if (atUtc.Ticks == 0) return null;
|
||||
var floor = new DateTime(0, DateTimeKind.Utc);
|
||||
IReadOnlyList<DataValueSnapshot> rows = source.SupportsReverseRaw
|
||||
? (await source.ReadRawAsync(tagname, atUtc, floor, 1, ct).ConfigureAwait(false)).Samples
|
||||
: (await ReverseRawReader.ReadAsync(source, tagname, atUtc, floor, 1, maxTieCluster, nowUtc, 0, ct)
|
||||
.ConfigureAwait(false)).Samples;
|
||||
var limit = Align(atUtc.Ticks, Resolution(source));
|
||||
foreach (var row in rows)
|
||||
if (Align(Ticks(row), Resolution(source)) <= limit)
|
||||
return row;
|
||||
return null;
|
||||
}
|
||||
|
||||
/// <summary>The earliest stored sample at or after <paramref name="atUtc"/>, or null.</summary>
|
||||
/// <remarks>
|
||||
/// #691 review: on a backend that reads forward only (the Gateway) the read's end is clamped to now +
|
||||
/// <see cref="ReverseRawReader.LookAhead"/> — the same horizon the walk-back uses — and a time beyond it is "not found"
|
||||
/// without a read. The HistorianGateway only checks <c>start <= end</c>, so a <c>DateTime.MaxValue</c> end would
|
||||
/// reach AVEVA as a year-9999 query end (unverified: a rejection, or a fabricated row). The clamp is NOT applied to a
|
||||
/// backend that reads newest-first natively (SqlServer / QuestDB): they bind <c>MaxValue</c> safely (pinned by the
|
||||
/// <c>Extreme_bounds</c> conformance test) and can legitimately hold samples further ahead (HistoryUpdate's
|
||||
/// <c>HistoryUpdateMaxFutureSkew</c> is configurable, and driver-stamped live values carry device time) — clamping
|
||||
/// there would turn a real end bound into <c>Bad_BoundNotFound</c>.
|
||||
/// </remarks>
|
||||
internal static async Task<DataValueSnapshot?> EarliestAtOrAfterAsync(
|
||||
IHistorianDataSource source, string tagname, DateTime atUtc, DateTime nowUtc, CancellationToken ct)
|
||||
{
|
||||
var endTicks = DateTime.MaxValue.Ticks;
|
||||
if (!source.SupportsReverseRaw)
|
||||
endTicks = nowUtc.Ticks > DateTime.MaxValue.Ticks - ReverseRawReader.LookAhead.Ticks
|
||||
? DateTime.MaxValue.Ticks
|
||||
: nowUtc.Ticks + ReverseRawReader.LookAhead.Ticks;
|
||||
if (atUtc.Ticks >= endTicks) return null;
|
||||
var rows = (await source.ReadRawAsync(tagname, atUtc, new DateTime(endTicks, DateTimeKind.Utc), 1, ct)
|
||||
.ConfigureAwait(false)).Samples;
|
||||
var limit = Align(atUtc.Ticks, Resolution(source));
|
||||
foreach (var row in rows)
|
||||
if (Align(Ticks(row), Resolution(source)) >= limit)
|
||||
return row;
|
||||
return null;
|
||||
}
|
||||
|
||||
private static bool IsOpen(DateTime t) => t.Ticks == 0 || t.Ticks == DateTime.MaxValue.Ticks;
|
||||
|
||||
private static bool SameInstant(long a, long b, long resolution) => Align(a, resolution) == Align(b, resolution);
|
||||
|
||||
private static long Resolution(IHistorianDataSource source) => Math.Max(1L, source.TimestampResolutionTicks);
|
||||
|
||||
private static long Ticks(DataValueSnapshot s) => (s.SourceTimestampUtc ?? DateTime.MinValue).Ticks;
|
||||
|
||||
private static long Align(long ticks, long resolution) => ticks - ticks % resolution;
|
||||
}
|
||||
@@ -46,6 +46,31 @@ namespace ZB.MOM.WW.OtOpcUa.OpcUaServer;
|
||||
/// <see cref="HistoryCursorKind.Modified"/> only: the normalised window start, so a resumed page keeps the original
|
||||
/// window and direction (the modified read resumes by key, not by moving the window start).
|
||||
/// </param>
|
||||
/// <param name="Reverse">
|
||||
/// <see cref="HistoryCursorKind.Events"/> only (Gitea #675): the read is newest-first over
|
||||
/// <c>(EndUtc, NextStartUtc]</c>. A resumed events read takes its direction from here, never from the resume
|
||||
/// request's details. Raw keeps deriving it from the window (<c>NextStartUtc > EndUtc</c>).
|
||||
/// </param>
|
||||
/// <param name="SegmentWidthTicks">
|
||||
/// <see cref="HistoryCursorKind.Raw"/> reverse reads on a forward-only backend only (Gitea #686): the segment width the
|
||||
/// walk-back ended on, so the next page starts at the observed density instead of re-discovering it. 0 = none.
|
||||
/// </param>
|
||||
/// <param name="EmptyFromTicks">
|
||||
/// Walked reverse reads only (Gitea #691): with <paramref name="EmptyToTicks"/>, a stretch <c>(from, to)</c> the previous
|
||||
/// page's walk proved holds no row, so the next page steps over it rather than re-walking it (to the 1970 floor, ~14
|
||||
/// reads, at the end of a deep page-through). 0/0 = none. The cursor lives in the session's in-memory store (never
|
||||
/// serialized), so a new field with a "none" default is all the versioning it needs.
|
||||
/// </param>
|
||||
/// <param name="EmptyToTicks">The exclusive upper end of that stretch.</param>
|
||||
/// <param name="PendingEndBound">
|
||||
/// <see cref="HistoryCursorKind.Raw"/> with <c>ReturnBounds = true</c> only (Gitea #691): the end bound still owed —
|
||||
/// Part 11 returns it with the last page. <c>null</c> = bounds not requested.
|
||||
/// </param>
|
||||
/// <param name="WithinTieCluster">
|
||||
/// <see cref="HistoryCursorKind.Raw"/> only (Gitea #692): the previous page was a slice of an oversized tie cluster at
|
||||
/// <paramref name="NextStartUtc"/> that still has un-emitted ties — the next page over-fetches the cluster directly
|
||||
/// instead of first re-reading <c>N + skip + 1</c> rows that are known to be ties.
|
||||
/// </param>
|
||||
internal sealed record HistoryContinuationState(
|
||||
string Tagname,
|
||||
DateTime NextStartUtc,
|
||||
@@ -54,7 +79,17 @@ internal sealed record HistoryContinuationState(
|
||||
uint NumValuesPerNode,
|
||||
HistoryCursorKind Kind = HistoryCursorKind.Raw,
|
||||
long LastModifiedId = 0,
|
||||
DateTime WindowStartUtc = default);
|
||||
DateTime WindowStartUtc = default,
|
||||
bool Reverse = false,
|
||||
long SegmentWidthTicks = 0,
|
||||
long EmptyFromTicks = 0,
|
||||
long EmptyToTicks = 0,
|
||||
HistoryEndBound? PendingEndBound = null,
|
||||
bool WithinTieCluster = false)
|
||||
{
|
||||
/// <summary>The walk-back hint this cursor carries (#686/#689/#691).</summary>
|
||||
public ReverseWalkHint WalkHint => new(SegmentWidthTicks, EmptyFromTicks, EmptyToTicks);
|
||||
}
|
||||
|
||||
/// <summary>The HistoryRead kind that issued a <see cref="HistoryContinuationState"/>.</summary>
|
||||
internal enum HistoryCursorKind
|
||||
@@ -223,12 +258,11 @@ internal static class HistoryPaging
|
||||
/// <param name="cap">The client's per-page cap (<c>NumValuesPerNode</c>); must be > 0 to make progress.</param>
|
||||
/// <param name="boundaryT">The single SourceTimestamp every tie in the cluster shares.</param>
|
||||
/// <param name="endUtc">The (inclusive) upper bound of the read window; unchanged across pages.
|
||||
/// OpcUaServer-003, accepted by design: the backend contract
|
||||
/// (<c>IHistorianDataSource.ReadRawAsync</c>) treats <c>endUtc</c> as EXCLUSIVE, so the
|
||||
/// <c>next <= endUtc</c> test here is deliberately the conservative side — at worst one extra
|
||||
/// empty resume page (<c>[endUtc, endUtc)</c> → GoodNoData, no continuation point), never a
|
||||
/// dropped or duplicated sample. Tightening to <c><</c> would be correct only if the live
|
||||
/// backend's boundary is confirmed exclusive; it is not worth the opposite off-by-one risk.</param>
|
||||
/// The forward window is end-EXCLUSIVE (Part 11; the <c>IHistorianDataSource.ReadRawAsync</c> contract), so a drained
|
||||
/// cluster continues only while <c>next</c> is strictly before <paramref name="endUtc"/> truncated to
|
||||
/// <paramref name="resolutionTicks"/>. (OpcUaServer-003 had kept <c>next <= endUtc</c> as "the conservative side",
|
||||
/// but it was not: resuming AT EndTime makes the next read a <c>start == end</c> point read, which returns the samples
|
||||
/// at EndTime as data — #692 review F1; on a microsecond backend EndTime is truncated first.)</param>
|
||||
/// <param name="sliceStart">The index into the cluster the emitted slice starts at (clamped to the count).</param>
|
||||
/// <param name="sliceCount">How many ties the emitted slice contains (may be <c>0</c> on a stale-skip self-heal).</param>
|
||||
/// <param name="nextStartUtc">The next page's inclusive start: <paramref name="boundaryT"/> while the
|
||||
@@ -271,7 +305,9 @@ internal static class HistoryPaging
|
||||
else
|
||||
{
|
||||
var next = boundaryT.AddTicks(resolutionTicks);
|
||||
if (next <= endUtc) { nextStartUtc = next; nextSkip = 0; }
|
||||
// The forward window is end-EXCLUSIVE, at the backend's resolution: resuming AT EndTime would turn the next read
|
||||
// into a start == end point read, which returns the samples at EndTime as data (#692 review F1).
|
||||
if (next.Ticks < endUtc.Ticks - endUtc.Ticks % resolutionTicks) { nextStartUtc = next; nextSkip = 0; }
|
||||
else { nextStartUtc = null; nextSkip = 0; }
|
||||
}
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -83,6 +83,21 @@ public sealed class OtOpcUaSdkServer : StandardServer
|
||||
return true;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #674 — wire the redundancy verdict that decides whether this node serves an OPC UA HistoryUpdate
|
||||
/// (<see cref="OtOpcUaNodeManager.HistoryUpdatePrimaryGate"/>); on any other node every authorized node of a
|
||||
/// HistoryUpdate is refused <c>BadInvalidState</c>. Passing <c>null</c> clears it (allow). No-op (returns
|
||||
/// <c>false</c>) when the node manager has not been created yet (mirrors <see cref="SetNativeAlarmAckPrimaryGate"/>).
|
||||
/// </summary>
|
||||
/// <param name="isPrimary">A cheap, non-blocking read of the data-plane Primary verdict; null to allow.</param>
|
||||
/// <returns><c>true</c> when set on a live node manager; <c>false</c> when no node manager exists yet.</returns>
|
||||
public bool SetHistoryUpdatePrimaryGate(Func<bool>? isPrimary)
|
||||
{
|
||||
if (_otOpcUaNodeManager is null) return false;
|
||||
_otOpcUaNodeManager.HistoryUpdatePrimaryGate = isPrimary;
|
||||
return true;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Wire the reverse-path gateway for inbound operator writes to writable equipment-tag nodes onto the
|
||||
/// created <see cref="OtOpcUaNodeManager"/>. The host calls this after start with an
|
||||
@@ -185,6 +200,22 @@ public sealed class OtOpcUaSdkServer : StandardServer
|
||||
return true;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #682 — keep the SDK's full set of calculator factories (live-data <c>AggregateFilter</c> subscriptions
|
||||
/// need them, and <c>ServerCapabilities/AggregateFunctions</c> keeps listing all of them), then trim
|
||||
/// <c>HistoryServerCapabilities/AggregateFunctions</c> to the node manager's current historian source. At this
|
||||
/// point that is the Null default (no history aggregates); the Host's <see cref="SetHistorianDataSource"/>
|
||||
/// re-syncs to the configured backend's set right after start.
|
||||
/// </summary>
|
||||
/// <inheritdoc />
|
||||
protected override AggregateManager CreateAggregateManager(
|
||||
IServerInternal server, ApplicationConfiguration configuration)
|
||||
{
|
||||
var manager = base.CreateAggregateManager(server, configuration);
|
||||
_otOpcUaNodeManager?.SyncAdvertisedAggregates();
|
||||
return manager;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override MasterNodeManager CreateMasterNodeManager(
|
||||
IServerInternal server, ApplicationConfiguration configuration)
|
||||
|
||||
@@ -0,0 +1,97 @@
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.OpcUaServer;
|
||||
|
||||
/// <summary>How one HistoryRead-Raw page reads (<see cref="RawPageLimit.Plan"/>).</summary>
|
||||
/// <param name="Limit">The backend's <see cref="IHistorianDataSource.MaxRawValuesPerRead"/> the plan was made under (0 = none).</param>
|
||||
/// <param name="PageCap">The page: at most this many values are returned (0 = all values — only without a limit).</param>
|
||||
/// <param name="RequestCap">What the read asks for: page + the boundary ties it will trim + one lookahead row (0 = all).</param>
|
||||
/// <param name="NoRoom">The boundary ties alone fill a read — nothing can be read past them; serve the tie cluster.</param>
|
||||
internal readonly record struct RawPagePlan(int Limit, uint PageCap, uint RequestCap, bool NoRoom);
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #694: sizes a HistoryRead-Raw page to what the backend can return in one read
|
||||
/// (<see cref="IHistorianDataSource.MaxRawValuesPerRead"/> = L — a strict HistorianGateway's learned row ceiling, less
|
||||
/// its boundary-row allowance). With a limit the read — page P + boundary ties k + the #692 lookahead row — is at most L,
|
||||
/// so P = min(want, L − k − 1) (all values: P = L − k − 1); a full page carries a continuation point like any other, as
|
||||
/// Part 11 allows fewer values than <c>NumValuesPerNode</c> with one. Before #694 a cap above a strict gateway's ceiling
|
||||
/// (or an uncapped read of a window holding more) failed <c>BadTooManyOperations</c>.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>Never a silent truncation.</b> The backend still answers a read in full or refuses it; only the server's page is
|
||||
/// smaller, and a page is judged full against what was asked for. A read the backend cannot serve at all (a tie
|
||||
/// cluster larger than one read) stays <c>BadTooManyOperations</c>.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>One re-read, planned again.</b> A refused read (<see cref="HistoryReadLimitExceededException"/>) is re-planned and
|
||||
/// re-issued ONCE when (a) the backend learned a different limit while it ran — the first read after start-up teaches
|
||||
/// the ceiling — or (b) it was refused because rows before the window crowded the budget
|
||||
/// (<see cref="HistoryReadLimitExceededException.PreWindowRows"/> m): the page is then shrunk by m so that the
|
||||
/// backend's look-back fits beside it. Anything else is the answer.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
internal static class RawPageLimit
|
||||
{
|
||||
/// <summary>Plans one page.</summary>
|
||||
/// <param name="want">The page the client asked for (N, less a start bound; 0 = all values).</param>
|
||||
/// <param name="boundaryTies">Ties at the resume timestamp already returned, which the read re-reads and trims (k).</param>
|
||||
/// <param name="limit">The backend's limit L (<= 0 = none).</param>
|
||||
/// <param name="sizeLimit">The limit to size the page by (L, or L − m after a crowded refusal); defaults to <paramref name="limit"/>.</param>
|
||||
/// <returns>The plan.</returns>
|
||||
public static RawPagePlan Plan(uint want, int boundaryTies, int limit, int? sizeLimit = null)
|
||||
{
|
||||
var k = Math.Max(0, boundaryTies);
|
||||
if (limit <= 0)
|
||||
return new RawPagePlan(0, want, want == 0 ? 0u : (uint)Math.Min(uint.MaxValue, (long)want + k + 1L), false);
|
||||
|
||||
var room = (long)(sizeLimit ?? limit) - k - 1;
|
||||
if (room < 1) return new RawPagePlan(limit, 0, 0, true);
|
||||
var page = want == 0 ? room : Math.Min(want, room);
|
||||
return new RawPagePlan(limit, (uint)page, (uint)(page + k + 1), false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reads one page under the backend's current limit, re-planning and re-reading ONCE as described in the remarks.
|
||||
/// A <see cref="RawPagePlan.NoRoom"/> plan is returned without a read.
|
||||
/// </summary>
|
||||
/// <typeparam name="T">The page.</typeparam>
|
||||
/// <param name="source">The backend.</param>
|
||||
/// <param name="want">The page the client asked for (0 = all values).</param>
|
||||
/// <param name="boundaryTies">The boundary ties the read re-reads and trims.</param>
|
||||
/// <param name="read">One read with the given plan (it asks for <see cref="RawPagePlan.RequestCap"/>).</param>
|
||||
/// <returns>The page (null when <see cref="RawPagePlan.NoRoom"/>) and the plan it was read with.</returns>
|
||||
public static async Task<(T? Page, RawPagePlan Plan)> ReadAsync<T>(
|
||||
IHistorianDataSource source, uint want, int boundaryTies, Func<RawPagePlan, Task<T>> read)
|
||||
{
|
||||
var before = source.MaxRawValuesPerRead;
|
||||
var plan = Plan(want, boundaryTies, before);
|
||||
if (plan.NoRoom) return (default, plan);
|
||||
|
||||
HistoryReadLimitExceededException refused;
|
||||
try
|
||||
{
|
||||
return (await read(plan).ConfigureAwait(false), plan);
|
||||
}
|
||||
catch (HistoryReadLimitExceededException ex) when (source.MaxRawValuesPerRead is var now and > 0
|
||||
&& (now != before || ex.PreWindowRows > 0))
|
||||
{
|
||||
refused = ex; // re-read outside the catch: the second refusal is the answer
|
||||
}
|
||||
|
||||
var limit = source.MaxRawValuesPerRead;
|
||||
var shrink = Math.Max(0, refused.PreWindowRows);
|
||||
plan = Plan(want, boundaryTies, limit, (int)Math.Max(0L, (long)limit - shrink));
|
||||
// No room left once the crowding rows are allowed for: those rows are not (necessarily) the boundary ties, so this is
|
||||
// NOT "the cursor sits in a tie cluster" — the refusal is the answer (stepping the cursor through it a tick per page
|
||||
// would never end).
|
||||
// (A limit that changed under an un-crowded read leaves the boundary ties alone filling a read: that IS the tie
|
||||
// cluster at the cursor, served by the over-fetch.)
|
||||
if (plan.NoRoom)
|
||||
{
|
||||
if (shrink > 0) throw refused;
|
||||
return (default, plan);
|
||||
}
|
||||
return (await read(plan).ConfigureAwait(false), plan);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.OpcUaServer;
|
||||
|
||||
/// <summary>One walked-back reverse events page (<see cref="ReverseEventsReader"/>).</summary>
|
||||
/// <param name="Events">At most the requested count, newest first by <c>(EventTimeUtc, EventId)</c>.</param>
|
||||
/// <param name="Hint">What the next page of the same read resumes with.</param>
|
||||
internal readonly record struct ReverseEventsPage(List<HistoricalEvent> Events, ReverseWalkHint Hint);
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #691.4: serves a Part 11 reverse events read — <c>(lo, hi]</c> newest-first — on a backend that reads events
|
||||
/// forward only (<see cref="IHistorianDataSource.SupportsReverseEvents"/> = <c>false</c>: the Gateway) with the
|
||||
/// <see cref="ReverseWalk"/>. A segment read is cut short when the source flags truncation (its continuation point); ties
|
||||
/// are ordered by <see cref="EventIdTieOrder"/> because the gateway orders by event time only.
|
||||
/// </summary>
|
||||
internal static class ReverseEventsReader
|
||||
{
|
||||
/// <summary>Reads <c>(lo, hi]</c> newest-first from forward event reads.</summary>
|
||||
/// <param name="source">A backend that reads events forward.</param>
|
||||
/// <param name="sourceName">Event-source filter (null = all).</param>
|
||||
/// <param name="hi">The later (inclusive) bound.</param>
|
||||
/// <param name="lo">The earlier (exclusive) bound.</param>
|
||||
/// <param name="want">How many events; <c>0</c> = all in the window (one forward read).</param>
|
||||
/// <param name="maxTieCluster">The tie budget (<c>MaxTieClusterOverfetch</c>) — a ceiling only.</param>
|
||||
/// <param name="nowUtc">Now, for the look-ahead clamp.</param>
|
||||
/// <param name="hint">What the previous page carried (default = none).</param>
|
||||
/// <param name="ct">Cancellation.</param>
|
||||
/// <returns>The events, newest first, and the hint for the next page.</returns>
|
||||
/// <exception cref="HistoryReadLimitExceededException">One timestamp holds more than the tie budget.</exception>
|
||||
public static async Task<ReverseEventsPage> ReadAsync(
|
||||
IHistorianDataSource source, string? sourceName, DateTime hi, DateTime lo, uint want, int maxTieCluster,
|
||||
DateTime nowUtc, ReverseWalkHint hint, CancellationToken ct)
|
||||
{
|
||||
var budget = Math.Max(1, maxTieCluster);
|
||||
var (rows, next) = await ReverseWalk.WalkAsync<HistoricalEvent>(
|
||||
async (start, end, cap, token) =>
|
||||
{
|
||||
// cap 0 = the whole window: int.MaxValue is the events contract's "no cap" (a non-positive value would
|
||||
// mean the backend's DEFAULT cap).
|
||||
var r = await source.ReadEventsAsync(sourceName, start, end, cap == 0 ? int.MaxValue : cap, reverse: false, token)
|
||||
.ConfigureAwait(false);
|
||||
return new ForwardSegmentRead<HistoricalEvent>(r.Events, r.ContinuationPoint is not null);
|
||||
},
|
||||
ev => ev.EventTimeUtc.Ticks, EventIdTieOrder.Instance, hi, lo, want, budget, source.TimestampResolutionTicks,
|
||||
nowUtc, hint,
|
||||
at => new HistoryReadLimitExceededException(budget,
|
||||
$"A reverse events read of '{sourceName}' at {at:O} found more than {budget} events within one millisecond; the " +
|
||||
"backend reads forward only, so the latest of them cannot be selected. Increase " +
|
||||
"ServerHistorian:MaxTieClusterOverfetch."),
|
||||
ct)
|
||||
.ConfigureAwait(false);
|
||||
return new ReverseEventsPage(rows, next);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,136 @@
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.OpcUaServer;
|
||||
|
||||
/// <summary>One walked-back reverse raw page (<see cref="ReverseRawReader"/>).</summary>
|
||||
/// <param name="Samples">At most the requested cap, newest first.</param>
|
||||
/// <param name="WidthHintTicks">The segment width the walk ended on — the next page's starting width (0 = none).</param>
|
||||
internal readonly record struct ReverseRawPage(IReadOnlyList<DataValueSnapshot> Samples, long WidthHintTicks)
|
||||
{
|
||||
/// <summary>Everything the next page of the same read should resume with (its width is <see cref="WidthHintTicks"/>).</summary>
|
||||
public ReverseWalkHint Hint { get; init; } = new(WidthHintTicks);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #686: serves a Part 11 reverse raw read — <c>(lo, hi]</c> newest-first, capped — on a historian backend that can
|
||||
/// only read forward (<see cref="IHistorianDataSource.SupportsReverseRaw"/> = <c>false</c>: the Gateway, whose
|
||||
/// <c>ReadRaw</c> streams ascending and rejects <c>start > end</c>). The walk itself — segment sizing (#689), the
|
||||
/// truncation rules, edges, ties and bounds — is <see cref="ReverseWalk"/>; this adapter supplies the raw read (a read is
|
||||
/// cut short when it returned its full cap) and keeps the backend's tie order.
|
||||
/// </summary>
|
||||
internal static class ReverseRawReader
|
||||
{
|
||||
/// <summary>The first segment's width when no hint is carried.</summary>
|
||||
internal static readonly TimeSpan InitialSegment = ReverseWalk.InitialSegment;
|
||||
|
||||
/// <summary>The walk's exclusive lower limit.</summary>
|
||||
internal static readonly DateTime Floor = ReverseWalk.Floor;
|
||||
|
||||
/// <summary>How far past "now" the walk starts at most.</summary>
|
||||
internal static readonly TimeSpan LookAhead = ReverseWalk.LookAhead;
|
||||
|
||||
/// <summary>Reads <c>(lo, hi]</c> newest-first from forward reads.</summary>
|
||||
/// <param name="source">A backend that reads forward.</param>
|
||||
/// <param name="tagname">Historian tag.</param>
|
||||
/// <param name="hi">The later (inclusive) bound — the request's StartTime or a reverse resume point.</param>
|
||||
/// <param name="lo">The earlier (exclusive) bound.</param>
|
||||
/// <param name="maxValues">Cap; <c>0</c> = every sample in the window (one forward read).</param>
|
||||
/// <param name="maxTieCluster">The tie budget (<c>MaxTieClusterOverfetch</c>; values below 1 count as 1) — a ceiling only.</param>
|
||||
/// <param name="nowUtc">Now, for the look-ahead clamp.</param>
|
||||
/// <param name="widthHintTicks">Starting segment width (the previous page's <see cref="ReverseRawPage.WidthHintTicks"/>), 0 = none.</param>
|
||||
/// <param name="ct">Cancellation.</param>
|
||||
/// <returns>The page, newest first, and the width to start the next page with.</returns>
|
||||
/// <exception cref="HistoryReadLimitExceededException">One timestamp holds more than the tie budget.</exception>
|
||||
public static Task<ReverseRawPage> ReadAsync(
|
||||
IHistorianDataSource source, string tagname, DateTime hi, DateTime lo, uint maxValues, int maxTieCluster,
|
||||
DateTime nowUtc, long widthHintTicks, CancellationToken ct) =>
|
||||
ReadAsync(source, tagname, hi, lo, maxValues, maxTieCluster, nowUtc, new ReverseWalkHint(widthHintTicks), ct);
|
||||
|
||||
/// <summary>Reads <c>(lo, hi]</c> newest-first from forward reads, resuming with the previous page's full hint.</summary>
|
||||
/// <param name="source">A backend that reads forward.</param>
|
||||
/// <param name="tagname">Historian tag.</param>
|
||||
/// <param name="hi">The later (inclusive) bound.</param>
|
||||
/// <param name="lo">The earlier (exclusive) bound.</param>
|
||||
/// <param name="maxValues">Cap; <c>0</c> = every sample in the window (one forward read).</param>
|
||||
/// <param name="maxTieCluster">The tie budget — a ceiling only.</param>
|
||||
/// <param name="nowUtc">Now, for the look-ahead clamp.</param>
|
||||
/// <param name="hint">What the previous page carried (<see cref="ReverseRawPage.Hint"/>), default = none.</param>
|
||||
/// <param name="ct">Cancellation.</param>
|
||||
/// <returns>The page, newest first, and the hint for the next page.</returns>
|
||||
/// <exception cref="HistoryReadLimitExceededException">One timestamp holds more than the tie budget.</exception>
|
||||
/// <remarks>
|
||||
/// Gitea #694: every segment read fits the backend's <see cref="IHistorianDataSource.MaxRawValuesPerRead"/> (a strict
|
||||
/// gateway's learned ceiling) — the walk's tie budget is bounded by it so its segments are sized for it, and a
|
||||
/// crowded re-read asking more is cut to it (and counts as truncated, never as complete). A walk that TEACHES the
|
||||
/// backend its limit (the first read after start-up) and is refused for it is re-run once with segments sized for the
|
||||
/// new limit — unless <paramref name="retryOnLearnedLimit"/> is false: the HistoryRead page path passes false, so the
|
||||
/// refusal reaches <see cref="RawPageLimit"/>, which shrinks the PAGE to the limit as well (re-running the walk for a
|
||||
/// page of N far above the limit cost thousands of segment reads). The bound lookups (a page of one) keep the re-run.
|
||||
/// The page size itself is never cut here: a short answer would read as the last page. A segment refused because rows
|
||||
/// just before it crowded the backend's budget (<see cref="HistoryReadLimitExceededException.PreWindowRows"/>) is
|
||||
/// re-read once that many rows smaller and counts as truncated when full. An uncapped walk (one forward read of the
|
||||
/// whole window) is never cut: it is answered in full or refused.
|
||||
/// </remarks>
|
||||
/// <param name="retryOnLearnedLimit">Re-run the walk once when it taught the backend its limit (see the remarks).</param>
|
||||
public static async Task<ReverseRawPage> ReadAsync(
|
||||
IHistorianDataSource source, string tagname, DateTime hi, DateTime lo, uint maxValues, int maxTieCluster,
|
||||
DateTime nowUtc, ReverseWalkHint hint, CancellationToken ct, bool retryOnLearnedLimit = true)
|
||||
{
|
||||
var before = source.MaxRawValuesPerRead;
|
||||
try
|
||||
{
|
||||
return await WalkAsync(before).ConfigureAwait(false);
|
||||
}
|
||||
catch (HistoryReadLimitExceededException) when (retryOnLearnedLimit && maxValues > 0
|
||||
&& source.MaxRawValuesPerRead is var now and > 0 && now != before)
|
||||
{
|
||||
// fall through (outside the catch): the walk taught the backend its limit — re-run it sized for that limit once.
|
||||
// Not an uncapped walk: it is one read of the whole window, which the limit does not change.
|
||||
}
|
||||
|
||||
return await WalkAsync(source.MaxRawValuesPerRead).ConfigureAwait(false);
|
||||
|
||||
async Task<ReverseRawPage> WalkAsync(int limit)
|
||||
{
|
||||
// A segment asks for at most its budget + 2 rows (ReverseWalk), so the budget is bounded 2 under the limit.
|
||||
var budget = Math.Max(1, maxTieCluster);
|
||||
if (limit > 0) budget = Math.Min(budget, Math.Max(1, limit - 2));
|
||||
var (rows, next) = await ReverseWalk.WalkAsync(
|
||||
async (start, end, cap, token) =>
|
||||
{
|
||||
var asked = limit > 0 && cap > limit ? limit : cap; // cap 0 (the whole window) is never cut
|
||||
IReadOnlyList<DataValueSnapshot> samples;
|
||||
for (var attempt = 0; ; attempt++)
|
||||
{
|
||||
try
|
||||
{
|
||||
samples = (await source.ReadRawAsync(tagname, start, end, (uint)asked, token).ConfigureAwait(false)).Samples;
|
||||
break;
|
||||
}
|
||||
catch (HistoryReadLimitExceededException ex) when (asked > 0 && ex.PreWindowRows > 0 && attempt < 2)
|
||||
{
|
||||
// #695: the discovery request may be smaller than the pre-window cluster. Size against
|
||||
// the ceiling, not that request; a second pass can discover more crowding rows.
|
||||
var ceilingRoom = source.MaxRawValuesPerRead is var learned and > 0 ? learned : asked;
|
||||
var smaller = Math.Min(asked - 1, ceilingRoom - ex.PreWindowRows - 1);
|
||||
if (smaller < 1) throw;
|
||||
asked = smaller;
|
||||
}
|
||||
}
|
||||
return new ForwardSegmentRead<DataValueSnapshot>(samples, asked > 0 && samples.Count >= asked);
|
||||
},
|
||||
Ticks, tieOrder: null, hi, lo, maxValues, budget, source.TimestampResolutionTicks, nowUtc, hint,
|
||||
at => new HistoryReadLimitExceededException(budget,
|
||||
$"A reverse raw read of '{tagname}' at {at:O} found more than {budget} samples within one millisecond; the " +
|
||||
"backend reads forward only, so the latest of them cannot be selected. Increase " +
|
||||
(budget < Math.Max(1, maxTieCluster)
|
||||
? "the backend's row ceiling (HistorianGateway Historian:ReadLimits:MaxValuesCeiling)."
|
||||
: "ServerHistorian:MaxTieClusterOverfetch.")),
|
||||
ct, useDiscoveryBudgetForGaps: limit > 0)
|
||||
.ConfigureAwait(false);
|
||||
return new ReverseRawPage(rows, next.WidthTicks) { Hint = next };
|
||||
}
|
||||
}
|
||||
|
||||
private static long Ticks(DataValueSnapshot s) => (s.SourceTimestampUtc ?? DateTime.MinValue).Ticks;
|
||||
}
|
||||
@@ -0,0 +1,454 @@
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.OpcUaServer;
|
||||
|
||||
/// <summary>What one walked-back page tells the next page of the same read (carried in the continuation point).</summary>
|
||||
/// <param name="WidthTicks">The segment width the walk ended on, aimed at the page's density — the next page's first width (0 = none).</param>
|
||||
/// <param name="EmptyFromTicks">
|
||||
/// With <paramref name="EmptyToTicks"/>: a stretch <c>(EmptyFrom, EmptyTo)</c> (both exclusive) a complete segment read
|
||||
/// proved holds no row (#691) — typically from the oldest row the page saw down to the walk's floor. A resumed page
|
||||
/// steps straight over it instead of re-walking it. Both 0 = none.
|
||||
/// </param>
|
||||
/// <param name="EmptyToTicks">The exclusive upper end of the empty stretch.</param>
|
||||
internal readonly record struct ReverseWalkHint(long WidthTicks, long EmptyFromTicks = 0, long EmptyToTicks = 0)
|
||||
{
|
||||
/// <summary>Whether an empty stretch is carried.</summary>
|
||||
public bool HasEmptyStretch => EmptyToTicks > EmptyFromTicks && EmptyToTicks > 0;
|
||||
}
|
||||
|
||||
/// <summary>One forward read of a segment: the rows (ascending by time) and whether a cap cut the read short.</summary>
|
||||
/// <typeparam name="T">Row type.</typeparam>
|
||||
/// <param name="Rows">Rows in ascending time order (ties in any order).</param>
|
||||
/// <param name="Truncated">A cap stopped the read — more rows may follow the last one.</param>
|
||||
internal readonly record struct ForwardSegmentRead<T>(IReadOnlyList<T> Rows, bool Truncated);
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #686/#689: the walk-back that serves a Part 11 reverse read — <c>(lo, hi]</c> newest-first, capped — from a
|
||||
/// backend that can only read FORWARD. Shared by raw values (<see cref="ReverseRawReader"/>) and events; the row type
|
||||
/// only has to expose a timestamp.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>Uncapped</b> (<c>want = 0</c>): ONE forward read of the whole window, filtered and reversed in memory.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Capped: walk back in segments.</b> Starting at <c>hi</c>, read one segment <c>(segLo, segHi]</c> FORWARD, keep
|
||||
/// it newest-first, and step to the next older segment until the cap is met or <c>lo</c> is reached. The first
|
||||
/// segment is <see cref="InitialSegment"/> wide, or the width the previous page of the same read ended on (carried in
|
||||
/// the continuation point). After each read the next width is re-aimed from the density just observed at about half
|
||||
/// the PAGE budget <c>2N + 16</c> (≈ N + 8 rows), growing at most ×4 per step (#689 — it used to aim at half the tie
|
||||
/// budget, ~32k rows, whatever N was) — so a live tag's "latest N" is usually one read of about N rows, and sparse
|
||||
/// history is found in a few dozen tiny reads.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Read sizes.</b> A segment read asks for its budget + 2 rows (room for a row an AVEVA read fabricates at its query
|
||||
/// start, and one to see past the segment). The budget is the page budget, except a <b>discovery</b> segment — the
|
||||
/// first of a page with no carried width, or one after an empty segment (whose ×4 growth may overshoot into dense
|
||||
/// data) — which may read up to <see cref="DiscoveryRows"/>. The tie budget <c>M = max(1, MaxTieClusterOverfetch)</c>
|
||||
/// is only a CEILING: a single timestamp holding more rows than the segment budget escalates to <c>M</c>.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Never answer from a truncated read.</b> Rows stream ascending, so a read is complete for its segment when it was
|
||||
/// not cut short OR its last row is already past <c>segHi</c>. An incomplete read is discarded: when rows BEFORE the
|
||||
/// segment (a neighbouring tie cluster inside the alignment slack, a fabricated row) crowded it out it is re-read once
|
||||
/// with room for them (at most twice the budget + 4); otherwise the segment is narrowed — to the density the truncated
|
||||
/// read showed, and never again below the newest row it saw (<c>known</c>: rows exist there, so walking past it before
|
||||
/// finding what lies above it would re-read the same dense stretch over and over; the gap above it is bisected
|
||||
/// instead). Only more than <c>M</c> rows within one millisecond — a segment no wider than the 1 ms request
|
||||
/// granularity that still does not fit at the tie budget — is refused (per millisecond, not per timestamp: narrower
|
||||
/// requests are the same request, and the Gateway stores milliseconds).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Proving the rest empty (#691).</b> Walking down to the floor through empty history used to take one read per ×4
|
||||
/// growth step (~14 reads from a minute to the epoch) — on every page that ran out of rows. After two consecutive empty
|
||||
/// segments the walk asks once per page for the WHOLE remainder <c>(lo, segHi]</c> at the page budget: a complete
|
||||
/// answer ends the walk in one read; a truncated one (deep history) is discarded and the walk carries on as before. And
|
||||
/// the stretch a page's last complete segment proved empty below its oldest row rides the continuation point
|
||||
/// (<see cref="ReverseWalkHint.EmptyFromTicks"/>), so the next page steps over it — the rows of a page are often the
|
||||
/// last ones before a stretch the previous page already read.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Edges.</b> A segment is requested as <c>[⌊segLo⌋, ⌈segHi + 1 tick⌉]</c>, both rounded outward to the millisecond
|
||||
/// (the gateway's / AVEVA's resolution), and filtered to <c>(segLo, segHi]</c> on the rows' own timestamps — exact
|
||||
/// whether the backend's forward end is exclusive or inclusive and whether it truncates its bounds; a row fabricated at
|
||||
/// the (millisecond) query start can never land inside the segment (#687). Adjacent segments share no timestamp.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Ties</b> come back in the backend's forward order reversed, or — with a tie order given (events, whose gateway
|
||||
/// order within a timestamp is arbitrary) — sorted by it and reversed, so a resumed page's boundary-tie skip lines up.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Bounds.</b> The walk never asks for data after now + <see cref="LookAhead"/> nor before <see cref="Floor"/> (the
|
||||
/// Unix epoch — AVEVA holds nothing earlier). A walked-back read returns only rows strictly after the epoch and no
|
||||
/// later than now + one day.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
internal static class ReverseWalk
|
||||
{
|
||||
/// <summary>The first segment's width when no hint is carried.</summary>
|
||||
internal static readonly TimeSpan InitialSegment = TimeSpan.FromMinutes(1);
|
||||
|
||||
/// <summary>The walk's exclusive lower limit.</summary>
|
||||
internal static readonly DateTime Floor = DateTime.UnixEpoch;
|
||||
|
||||
/// <summary>How far past "now" the walk starts at most.</summary>
|
||||
internal static readonly TimeSpan LookAhead = TimeSpan.FromDays(1);
|
||||
|
||||
/// <summary>Slack added to a page's row budget (#689): room for density variation between segments.</summary>
|
||||
internal const int PageSlack = 16;
|
||||
|
||||
/// <summary>The most rows a discovery segment (see the remarks) may read before it counts as truncated (#689).</summary>
|
||||
internal const int DiscoveryRows = 1024;
|
||||
|
||||
/// <summary>Request bounds are rounded outward to this (the gateway / AVEVA timestamp resolution).</summary>
|
||||
private const long Alignment = TimeSpan.TicksPerMillisecond;
|
||||
|
||||
/// <summary>The rows a segment is sized for: <c>2N + </c><see cref="PageSlack"/>, never above the tie budget.</summary>
|
||||
/// <param name="want">The page cap N (> 0).</param>
|
||||
/// <param name="tieBudget">The tie budget M (>= 1).</param>
|
||||
/// <returns>The page budget.</returns>
|
||||
internal static int PageBudget(uint want, int tieBudget) => (int)Math.Clamp(2L * want + PageSlack, 1L, Math.Max(1, tieBudget));
|
||||
|
||||
/// <summary>Reads <c>(lo, hi]</c> newest-first from forward reads.</summary>
|
||||
/// <typeparam name="T">Row type.</typeparam>
|
||||
/// <param name="read">One forward read <c>[start, end]</c> asking for at most <c>cap</c> rows (<c>0</c> = all).</param>
|
||||
/// <param name="ticksOf">A row's timestamp, in ticks.</param>
|
||||
/// <param name="tieOrder">Order within one timestamp (null = the backend's forward order).</param>
|
||||
/// <param name="hi">The later (inclusive) bound.</param>
|
||||
/// <param name="lo">The earlier (exclusive) bound.</param>
|
||||
/// <param name="want">Cap; <c>0</c> = every row in the window (one forward read).</param>
|
||||
/// <param name="tieBudget">The tie budget M (values below 1 count as 1).</param>
|
||||
/// <param name="resolutionTicks">The backend's timestamp resolution (>= 1).</param>
|
||||
/// <param name="nowUtc">Now, for the look-ahead clamp.</param>
|
||||
/// <param name="hint">What the previous page of the same read carried (default = none).</param>
|
||||
/// <param name="tooManyAtOneTimestamp">Builds the exception for a timestamp holding more than M rows.</param>
|
||||
/// <param name="ct">Cancellation.</param>
|
||||
/// <param name="useDiscoveryBudgetForGaps">Use ceiling-sized discovery and retain truncated remainder rows (#696, strict raw backends only).</param>
|
||||
/// <returns>At most <paramref name="want"/> rows, newest first, and the hint for the next page.</returns>
|
||||
public static async Task<(List<T> Rows, ReverseWalkHint Next)> WalkAsync<T>(
|
||||
Func<DateTime, DateTime, int, CancellationToken, Task<ForwardSegmentRead<T>>> read,
|
||||
Func<T, long> ticksOf,
|
||||
IComparer<T>? tieOrder,
|
||||
DateTime hi, DateTime lo, uint want, int tieBudget, long resolutionTicks, DateTime nowUtc,
|
||||
ReverseWalkHint hint, Func<DateTime, Exception> tooManyAtOneTimestamp, CancellationToken ct, bool useDiscoveryBudgetForGaps = false)
|
||||
{
|
||||
var resolution = Math.Max(1L, resolutionTicks);
|
||||
var granularity = Math.Max(Alignment, resolution);
|
||||
var budget = Math.Max(1, tieBudget);
|
||||
var result = new List<T>();
|
||||
|
||||
var horizon = nowUtc.Ticks > DateTime.MaxValue.Ticks - LookAhead.Ticks ? DateTime.MaxValue.Ticks : nowUtc.Ticks + LookAhead.Ticks;
|
||||
var hiTicks = Math.Min(hi.Ticks, horizon);
|
||||
var stopTicks = Math.Max(lo.Ticks, Floor.Ticks);
|
||||
if (hiTicks <= stopTicks) return (result, hint);
|
||||
|
||||
if (want == 0)
|
||||
{
|
||||
// One forward read of the whole window, reversed — the uncapped forward read's own cost and bound.
|
||||
var all = (await read(Utc(AlignDown(stopTicks, granularity)), Utc(AlignUp(hiTicks + 1, granularity)), 0, ct)
|
||||
.ConfigureAwait(false)).Rows;
|
||||
AppendNewestFirst(all, ticksOf, tieOrder, stopTicks, hiTicks, result, int.MaxValue);
|
||||
return (result, hint);
|
||||
}
|
||||
|
||||
var maxWidth = DateTime.MaxValue.Ticks;
|
||||
var discovering = hint.WidthTicks <= 0;
|
||||
var width = discovering ? Math.Max(resolution, InitialSegment.Ticks) : Math.Clamp(hint.WidthTicks, resolution, maxWidth);
|
||||
var pageBudget = PageBudget(want, budget);
|
||||
var discoveryBudget = Math.Min(budget, Math.Max(pageBudget, DiscoveryRows));
|
||||
|
||||
// `known` > 0: a truncated read saw rows up to this timestamp (below segHi), and nothing is known of (known, segHi].
|
||||
// Segments stay above it until that stretch is read; then `knownWidth` (the density the truncated read showed)
|
||||
// sizes the segment that ends on it. Searching the gap: first a top-anchored segment at that density (data running
|
||||
// right up to segHi); if it is empty, the data stops somewhere above `known`, so GALLOP up from it — read
|
||||
// (known + lowerBy, segHi] with lowerBy growing ×4, `known` following what each truncated read saw — until a read
|
||||
// comes back empty (the top is now bracketed below its segLo), then BISECT that bracket. Any complete read holding
|
||||
// rows ends the search. A burst that ended long before segHi is found in a few reads instead of bisecting the whole
|
||||
// gap down to the millisecond, and a long dense stretch is crossed geometrically, not a budget at a time (#691.4
|
||||
// review). `noGallop` falls back to plain top-anchored halving if a galloping read is crowded by rows before it.
|
||||
var known = 0L;
|
||||
var knownWidth = 0L;
|
||||
var lowerBy = 0L;
|
||||
var bisecting = false;
|
||||
var noGallop = false;
|
||||
var segHi = hiTicks;
|
||||
|
||||
// #691: the remainder probe (once per page, after two empty segments) and the empty stretches.
|
||||
var emptyRun = 0;
|
||||
var probed = false;
|
||||
var carried = hint.HasEmptyStretch ? (hint.EmptyFromTicks, hint.EmptyToTicks) : (0L, 0L);
|
||||
(long From, long To) proven = (0L, 0L);
|
||||
|
||||
while (segHi > stopTicks && result.Count < want)
|
||||
{
|
||||
if (carried.Item2 > carried.Item1 && segHi > carried.Item1 && segHi < carried.Item2)
|
||||
{
|
||||
segHi = carried.Item1; // (from, segHi] lies inside a stretch a previous page proved empty
|
||||
continue;
|
||||
}
|
||||
|
||||
var probe = !probed && known == 0 && emptyRun >= 2 && segHi - stopTicks > width;
|
||||
var w = probe ? segHi - stopTicks : width;
|
||||
var lowered = false;
|
||||
var bisected = false;
|
||||
if (known > 0)
|
||||
{
|
||||
var gap = segHi - known;
|
||||
lowered = lowerBy > 0 && lowerBy < gap;
|
||||
bisected = !lowered && (bisecting || lowerBy > 0) && gap > 2 * Math.Max(granularity, knownWidth);
|
||||
// Never narrower than the request granularity: requests are rounded outward to it, so a narrower segment
|
||||
// re-sends the same request — and a width left at one tick (a tie cluster's) stepped segHi down a tick per
|
||||
// read, ~10,000 identical reads (#691.4 re-review). Keep the Min: a truncated read crowded by rows before
|
||||
// the segment must still shrink the segment.
|
||||
w = lowered ? gap - lowerBy
|
||||
: bisected ? gap / 2
|
||||
: Math.Min(Math.Max(w, granularity), gap <= 2 * granularity ? gap : gap / 2);
|
||||
}
|
||||
var segLo = segHi - stopTicks <= w ? stopTicks : segHi - w;
|
||||
var start = Utc(AlignDown(segLo, granularity));
|
||||
var end = Utc(AlignUp(segHi + 1, granularity));
|
||||
var segBudget = (discovering && !probe) || (useDiscoveryBudgetForGaps && (probe || known > 0))
|
||||
? discoveryBudget : pageBudget;
|
||||
var cap = segBudget + 2;
|
||||
IReadOnlyList<T> rows;
|
||||
var narrowed = false;
|
||||
|
||||
while (true)
|
||||
{
|
||||
ct.ThrowIfCancellationRequested();
|
||||
var segment = await read(start, end, cap, ct).ConfigureAwait(false);
|
||||
rows = segment.Rows;
|
||||
if (!segment.Truncated || rows.Count == 0 || ticksOf(rows[^1]) > segHi) break; // complete for (segLo, segHi]
|
||||
|
||||
if (probe)
|
||||
{
|
||||
// #696: strict-ceiling discovery retains the rows/density instead of discarding the remainder probe.
|
||||
probed = true;
|
||||
if (!useDiscoveryBudgetForGaps)
|
||||
{
|
||||
narrowed = true;
|
||||
break;
|
||||
}
|
||||
probe = false;
|
||||
}
|
||||
|
||||
var leading = 0;
|
||||
var inner = 0;
|
||||
var firstInner = 0L;
|
||||
var lastInner = 0L;
|
||||
foreach (var r in rows)
|
||||
{
|
||||
var ts = ticksOf(r);
|
||||
if (ts <= segLo) leading++;
|
||||
else if (ts <= segHi)
|
||||
{
|
||||
if (inner++ == 0) firstInner = ts;
|
||||
lastInner = ts;
|
||||
}
|
||||
}
|
||||
|
||||
// Crowded out by rows before the segment, with the segment itself possibly within budget: re-read once
|
||||
// with room for them.
|
||||
var roomier = leading + segBudget + 2;
|
||||
if (leading > 0 && inner <= segBudget && roomier > cap && roomier <= 2 * segBudget + 4)
|
||||
{
|
||||
cap = roomier;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (segHi - segLo <= granularity)
|
||||
{
|
||||
// A segment no wider than the request granularity (1 ms — narrower requests are the same request) still
|
||||
// over its budget: a tie cluster, or a burst inside one millisecond. Escalate to the tie budget M before
|
||||
// refusing — up to M rows within one millisecond are served (#686); more are refused, per millisecond
|
||||
// (#691.4 re-review — exact for the millisecond-resolution Gateway).
|
||||
if (segBudget < budget)
|
||||
{
|
||||
segBudget = budget;
|
||||
cap = segBudget + 2;
|
||||
continue;
|
||||
}
|
||||
|
||||
throw tooManyAtOneTimestamp(Utc(segHi));
|
||||
}
|
||||
|
||||
// Narrow — the segment shrinks every time, so this terminates.
|
||||
if (inner > 0 && lastInner == segHi)
|
||||
{
|
||||
width = resolution; // a tie cluster at the top of the segment: read that one timestamp
|
||||
lowerBy = 0;
|
||||
bisecting = false;
|
||||
}
|
||||
else if (inner > 0 && (lowered || bisected))
|
||||
{
|
||||
// Data reaches past this segment's bottom: move `known` up to what this read saw; a gallop goes further.
|
||||
known = lastInner;
|
||||
if (lowered) lowerBy = lowerBy > maxWidth / 4 ? maxWidth : lowerBy * 4;
|
||||
}
|
||||
else if (inner > 0)
|
||||
{
|
||||
// Rows exist at lastInner; what lies above it is unknown. Aim at about half the page budget at the
|
||||
// density the truncated rows showed (their span, not the segment's — it may open on an empty stretch).
|
||||
var aimed = inner > 1 && lastInner > firstInner
|
||||
? (long)Math.Min((double)(lastInner - firstInner) * ((useDiscoveryBudgetForGaps ? segBudget : pageBudget) / 2.0) / (inner - 1), maxWidth)
|
||||
: resolution;
|
||||
known = lastInner;
|
||||
knownWidth = Math.Max(resolution, aimed);
|
||||
width = Math.Max(resolution, Math.Min(aimed, segHi - segLo - 1));
|
||||
lowerBy = 0;
|
||||
bisecting = false;
|
||||
}
|
||||
else
|
||||
{
|
||||
// Only rows before the segment: shrink it — top-anchored from here on (a gallop would re-send the same
|
||||
// crowded read).
|
||||
width = Math.Max(resolution, Math.Min(width, segHi - segLo) / 4);
|
||||
if (lowered || bisected) noGallop = true;
|
||||
lowerBy = 0;
|
||||
bisecting = false;
|
||||
}
|
||||
|
||||
discovering = false;
|
||||
narrowed = true;
|
||||
break;
|
||||
}
|
||||
|
||||
if (narrowed) continue; // re-read the same segHi, narrower
|
||||
|
||||
var count = AppendNewestFirst(rows, ticksOf, tieOrder, segLo, segHi, result, (int)Math.Min(int.MaxValue, want),
|
||||
out var oldest);
|
||||
// This complete read proves (segLo, oldest) empty — or the whole segment when it held nothing.
|
||||
proven = count > 0 ? (segLo, oldest) : (segLo, segHi + 1);
|
||||
if (probe)
|
||||
{
|
||||
probed = true;
|
||||
segHi = segLo; // the whole remainder is read
|
||||
continue;
|
||||
}
|
||||
|
||||
emptyRun = count == 0 ? emptyRun + 1 : 0;
|
||||
var grown = width > maxWidth / 4 ? maxWidth : width * 4;
|
||||
if (count == 0)
|
||||
{
|
||||
discovering = true; // ×4 growth may overshoot into dense data
|
||||
if (known > 0)
|
||||
{
|
||||
// An empty top-anchored segment: gallop up from `known`. An empty galloping read: the top is bracketed
|
||||
// in (known, segLo] — bisect it (an empty bisecting read keeps bisecting).
|
||||
if (lowered || bisected)
|
||||
{
|
||||
lowerBy = 0;
|
||||
bisecting = true;
|
||||
}
|
||||
else if (noGallop)
|
||||
{
|
||||
width = maxWidth; // halve the gap, top-anchored
|
||||
}
|
||||
else
|
||||
{
|
||||
lowerBy = Math.Max(knownWidth, granularity);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
width = grown;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
// Re-aim at about half the page budget from the density just seen.
|
||||
width = (long)Math.Clamp((double)(segHi - segLo) * Math.Max(1.0, pageBudget / 2.0) / count, resolution, grown);
|
||||
discovering = false;
|
||||
known = 0;
|
||||
lowerBy = 0;
|
||||
bisecting = false;
|
||||
noGallop = false;
|
||||
}
|
||||
|
||||
segHi = segLo;
|
||||
if (known > 0 && segHi <= known)
|
||||
{
|
||||
// Everything above the newest row the truncated read saw is read: the next segment ends on it, at its density.
|
||||
known = 0;
|
||||
lowerBy = 0;
|
||||
bisecting = false;
|
||||
noGallop = false;
|
||||
width = knownWidth;
|
||||
discovering = false;
|
||||
}
|
||||
}
|
||||
|
||||
// Carry the stretch the last complete read proved empty, joined to the one carried in when they touch; when they
|
||||
// do not, keep the one reaching the floor (it saves a whole walk), else the newer.
|
||||
var next = carried;
|
||||
if (proven.To > proven.From)
|
||||
{
|
||||
if (carried.Item2 > carried.Item1 && proven.From < carried.Item2 && carried.Item1 < proven.To)
|
||||
next = (Math.Min(proven.From, carried.Item1), Math.Max(proven.To, carried.Item2));
|
||||
else if (!(carried.Item2 > carried.Item1 && carried.Item1 <= stopTicks) || proven.From <= stopTicks)
|
||||
next = proven;
|
||||
}
|
||||
|
||||
return (result, new ReverseWalkHint(width, next.Item1, next.Item2));
|
||||
}
|
||||
|
||||
/// <summary>Appends the rows in <c>(segLo, segHi]</c> newest-first (until <paramref name="limit"/>) and counts them.</summary>
|
||||
private static int AppendNewestFirst<T>(
|
||||
IReadOnlyList<T> rows, Func<T, long> ticksOf, IComparer<T>? tieOrder, long segLo, long segHi, List<T> result, int limit) =>
|
||||
AppendNewestFirst(rows, ticksOf, tieOrder, segLo, segHi, result, limit, out _);
|
||||
|
||||
/// <summary>As above, also returning the oldest in-range timestamp (meaningless when none).</summary>
|
||||
private static int AppendNewestFirst<T>(
|
||||
IReadOnlyList<T> rows, Func<T, long> ticksOf, IComparer<T>? tieOrder, long segLo, long segHi, List<T> result, int limit,
|
||||
out long oldest)
|
||||
{
|
||||
oldest = long.MaxValue;
|
||||
IReadOnlyList<T> ordered = rows;
|
||||
if (tieOrder is not null)
|
||||
{
|
||||
var copy = rows.ToList();
|
||||
copy.Sort((a, b) =>
|
||||
{
|
||||
var c = ticksOf(a).CompareTo(ticksOf(b));
|
||||
return c != 0 ? c : tieOrder.Compare(a, b);
|
||||
});
|
||||
ordered = copy;
|
||||
}
|
||||
|
||||
var count = 0;
|
||||
for (var i = ordered.Count - 1; i >= 0; i--)
|
||||
{
|
||||
var ts = ticksOf(ordered[i]);
|
||||
if (ts <= segLo || ts > segHi) continue;
|
||||
count++;
|
||||
if (ts < oldest) oldest = ts;
|
||||
if (result.Count < limit) result.Add(ordered[i]);
|
||||
}
|
||||
|
||||
return count;
|
||||
}
|
||||
|
||||
private static DateTime Utc(long ticks) => new(ticks, DateTimeKind.Utc);
|
||||
|
||||
private static long AlignDown(long ticks, long step) => ticks - ticks % step;
|
||||
|
||||
private static long AlignUp(long ticks, long step)
|
||||
{
|
||||
var down = AlignDown(ticks, step);
|
||||
if (down == ticks) return ticks;
|
||||
return down > DateTime.MaxValue.Ticks - step ? DateTime.MaxValue.Ticks : down + step;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Orders events within one timestamp by <see cref="HistoricalEvent.EventId"/> (ordinal) — the tie order a walked-back
|
||||
/// events read (#691.4) and its oversized-cluster over-fetch share, because the Gateway orders by event time only.
|
||||
/// </summary>
|
||||
internal sealed class EventIdTieOrder : IComparer<HistoricalEvent>
|
||||
{
|
||||
/// <summary>The shared instance.</summary>
|
||||
public static readonly EventIdTieOrder Instance = new();
|
||||
|
||||
/// <inheritdoc />
|
||||
public int Compare(HistoricalEvent? x, HistoricalEvent? y) => string.CompareOrdinal(x?.EventId, y?.EventId);
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
using Opc.Ua;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.OpcUaServer;
|
||||
|
||||
/// <summary>
|
||||
/// The Part 13 <see cref="AggregateConfiguration"/> processed reads are evaluated under (#677), and the rule for a
|
||||
/// client-supplied one in <c>ReadProcessedDetails</c>.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// The aggregate evaluators (<c>AggregateEvaluator</c> / <c>TimeWeightedAggregator</c> in
|
||||
/// <c>Driver.Historian.Sql.Common</c>) are written against fixed settings, not parameterised: Uncertain time counts as
|
||||
/// Good (<see cref="TreatUncertainAsBad"/> = false) in the time-weighted family; the per-sample Average / Minimum /
|
||||
/// Maximum / Range / *ActualTime / Count are defined by Part 13 over Good values whatever this setting (#684), as the
|
||||
/// SDK computes them under it; an interval is Good only when all of
|
||||
/// it is (<see cref="PercentDataGood"/> = 100) and Bad only when none of it is non-Bad
|
||||
/// (<see cref="PercentDataBad"/> = 100), otherwise <c>UncertainDataSubNormal</c>; the value past the last sample
|
||||
/// is held (stepped, <see cref="UseSlopedExtrapolation"/> = false). These are also the SDK's own server defaults
|
||||
/// (<c>AggregateManager.GetDefaultConfiguration</c>), which a test pins.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// A request asking for <c>UseServerCapabilitiesDefaults</c>, or spelling out exactly these values, is served;
|
||||
/// any other configuration is refused per node with <c>BadAggregateConfigurationRejected</c> rather than silently
|
||||
/// answered under settings the client did not ask for (it used to be ignored).
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public static class ServerAggregateConfiguration
|
||||
{
|
||||
/// <summary>Uncertain data is not treated as Bad.</summary>
|
||||
public const bool TreatUncertainAsBad = false;
|
||||
|
||||
/// <summary>An interval is Bad only when 100 % of it is Bad.</summary>
|
||||
public const byte PercentDataBad = 100;
|
||||
|
||||
/// <summary>An interval is Good only when 100 % of it is Good.</summary>
|
||||
public const byte PercentDataGood = 100;
|
||||
|
||||
/// <summary>The value past the last sample is held (stepped), not sloped.</summary>
|
||||
public const bool UseSlopedExtrapolation = false;
|
||||
|
||||
/// <summary>The server defaults as an <see cref="AggregateConfiguration"/> (explicit values, not "use defaults").</summary>
|
||||
public static AggregateConfiguration Defaults => new()
|
||||
{
|
||||
UseServerCapabilitiesDefaults = false,
|
||||
TreatUncertainAsBad = TreatUncertainAsBad,
|
||||
PercentDataBad = PercentDataBad,
|
||||
PercentDataGood = PercentDataGood,
|
||||
UseSlopedExtrapolation = UseSlopedExtrapolation,
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// True when a processed read with <paramref name="requested"/> can be served: none supplied,
|
||||
/// <c>UseServerCapabilitiesDefaults</c>, or every setting equal to the server's.
|
||||
/// </summary>
|
||||
/// <param name="requested">The request's configuration (may be null).</param>
|
||||
/// <returns>Whether to serve; false ⇒ <c>BadAggregateConfigurationRejected</c>.</returns>
|
||||
public static bool IsServable(AggregateConfiguration? requested) =>
|
||||
requested is null
|
||||
|| requested.UseServerCapabilitiesDefaults
|
||||
|| (requested.TreatUncertainAsBad == TreatUncertainAsBad
|
||||
&& requested.PercentDataBad == PercentDataBad
|
||||
&& requested.PercentDataGood == PercentDataGood
|
||||
&& requested.UseSlopedExtrapolation == UseSlopedExtrapolation);
|
||||
}
|
||||
@@ -35,6 +35,8 @@
|
||||
<!-- Gitea #562: the over-the-wire event-revalidation test asserts the server-side fan-out, because a
|
||||
deleted item resurrected onto a notifier is invisible to the client that deleted it. -->
|
||||
<InternalsVisibleTo Include="ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests"/>
|
||||
<!-- Gitea #686: the reverse-raw walk-back is driven over the real Gateway source + its fake gateway client. -->
|
||||
<InternalsVisibleTo Include="ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests"/>
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
|
||||
@@ -452,6 +452,13 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
|
||||
/// </summary>
|
||||
private readonly IHistorizationRoleView? _historizationRoleView;
|
||||
|
||||
/// <summary>Gitea #674 review: where the HistoryUpdate verdict is published. Null on nodes that wire no redundancy manager.</summary>
|
||||
private readonly IHistoryUpdateRoleView? _historyUpdateRoleView;
|
||||
|
||||
/// <summary>Gitea #674 review: whether any redundancy snapshot has reached this host — the HistoryUpdate gate stays
|
||||
/// closed until one has (see <see cref="PrimaryGatePolicy.ShouldServeHistoryUpdate"/>).</summary>
|
||||
private bool _redundancySnapshotReceived;
|
||||
|
||||
/// <summary>
|
||||
/// Tests whether a node named in a redundancy snapshot serves this node's address space — i.e.
|
||||
/// is a per-cluster mesh partner recording the very same historized refs. Null (the default)
|
||||
@@ -689,7 +696,8 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
|
||||
TimeSpan? commsRecoveryAuditDelay = null,
|
||||
Func<CommonsNodeId, bool>? sharesAlarmQueueWithLocal = null,
|
||||
TimeSpan? applyOutcomeTimeout = null,
|
||||
bool captureWhilePrimaryUnreachable = true) =>
|
||||
bool captureWhilePrimaryUnreachable = true,
|
||||
IHistoryUpdateRoleView? historyUpdateRoleView = null) =>
|
||||
// WARNING: this forwarding list is POSITIONAL, and Props.Create compiles it into an
|
||||
// expression tree. Six IActorRef? parameters and several interface-typed ones mean a
|
||||
// mis-ordered argument is usually type-compatible and therefore compiles clean, then binds
|
||||
@@ -702,7 +710,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
|
||||
deploymentArtifactCache, redundancyRoleView, replicationPeerHost, fetchAndCacheMode, artifactFetcher,
|
||||
alarmStateStore, telemetryHub, primaryRoleSource, historizationRoleView,
|
||||
sharesHistorizationScopeWithLocal, applyWedgeTimeout, reprovisionBackoff, commsRecoveryAuditDelay,
|
||||
sharesAlarmQueueWithLocal, applyOutcomeTimeout, captureWhilePrimaryUnreachable));
|
||||
sharesAlarmQueueWithLocal, applyOutcomeTimeout, captureWhilePrimaryUnreachable, historyUpdateRoleView));
|
||||
|
||||
/// <summary>Initializes a new DriverHostActor with the specified dependencies.</summary>
|
||||
/// <param name="dbFactory">Database context factory for configuration database access.</param>
|
||||
@@ -775,6 +783,10 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
|
||||
/// reported UNREACHABLE by the cluster failure detector, stop deferring to it and capture — rather than waiting
|
||||
/// for auto-down + a fresh redundancy snapshot. Affects ONLY the historization capture verdict; the alarm drain
|
||||
/// and the Primary data-plane verdict are unchanged. <c>false</c> restores the pre-#672 behaviour.</param>
|
||||
/// <param name="historyUpdateRoleView">Gitea #674 review: where the OPC UA HistoryUpdate verdict
|
||||
/// (<see cref="PrimaryGatePolicy.ShouldServeHistoryUpdate"/> — the data-plane verdict, but closed until the first
|
||||
/// redundancy snapshot) is published for the node manager's HistoryUpdate Primary gate. Null on nodes that wire no
|
||||
/// redundancy manager.</param>
|
||||
public DriverHostActor(
|
||||
IDbContextFactory<OtOpcUaConfigDbContext>? dbFactory,
|
||||
CommonsNodeId localNode,
|
||||
@@ -807,9 +819,11 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
|
||||
TimeSpan? commsRecoveryAuditDelay = null,
|
||||
Func<CommonsNodeId, bool>? sharesAlarmQueueWithLocal = null,
|
||||
TimeSpan? applyOutcomeTimeout = null,
|
||||
bool captureWhilePrimaryUnreachable = true)
|
||||
bool captureWhilePrimaryUnreachable = true,
|
||||
IHistoryUpdateRoleView? historyUpdateRoleView = null)
|
||||
{
|
||||
_captureWhilePrimaryUnreachable = captureWhilePrimaryUnreachable;
|
||||
_historyUpdateRoleView = historyUpdateRoleView;
|
||||
_applyOutcomeTimeout = applyOutcomeTimeout ?? DefaultApplyOutcomeTimeout;
|
||||
_sharesAlarmQueueWithLocal = sharesAlarmQueueWithLocal;
|
||||
_applyWedgeTimeout = applyWedgeTimeout ?? DefaultApplyWedgeTimeout;
|
||||
@@ -909,6 +923,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
|
||||
/// </summary>
|
||||
private void StartPrimaryVerdictPublishing()
|
||||
{
|
||||
PublishHistoryUpdateVerdict();
|
||||
if (_primaryRoleSource is null) return;
|
||||
PublishPrimaryVerdict();
|
||||
try
|
||||
@@ -948,6 +963,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
|
||||
/// The snapshot path publishes on every snapshot, as the other verdicts do.</summary>
|
||||
private void PublishPrimaryVerdict()
|
||||
{
|
||||
PublishHistoryUpdateVerdict();
|
||||
if (_primaryRoleSource is null) return;
|
||||
var verdict = ShouldServiceAsPrimary();
|
||||
_lastPublishedPrimaryVerdict = verdict;
|
||||
@@ -958,10 +974,18 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
|
||||
/// role information, so republishing an unchanged verdict on each would be noise.</summary>
|
||||
private void PublishPrimaryVerdictIfChanged()
|
||||
{
|
||||
PublishHistoryUpdateVerdict();
|
||||
if (_primaryRoleSource is null || ShouldServiceAsPrimary() == _lastPublishedPrimaryVerdict) return;
|
||||
PublishPrimaryVerdict();
|
||||
}
|
||||
|
||||
/// <summary>Gitea #674 review: publishes <see cref="PrimaryGatePolicy.ShouldServeHistoryUpdate"/> (no-op when no view
|
||||
/// is wired). Rides every Primary-verdict publish point — host start, every snapshot, membership changes — so it is
|
||||
/// closed from host start and opens only once a snapshot has arrived and the data-plane verdict allows.</summary>
|
||||
private void PublishHistoryUpdateVerdict() =>
|
||||
_historyUpdateRoleView?.PublishHistoryUpdateRole(
|
||||
PrimaryGatePolicy.ShouldServeHistoryUpdate(_redundancySnapshotReceived, _localRole, DriverMemberCount()));
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #672: follow cluster reachability so the historization capture gate stops deferring to a recording
|
||||
/// Primary the failure detector already considers dead. Its OWN subscription, deliberately not folded into
|
||||
@@ -2270,6 +2294,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
|
||||
/// <c>ScriptedAlarmHostActor.OnRedundancyStateChanged</c> / <c>OpcUaPublishActor</c>.</summary>
|
||||
private void OnRedundancyStateChanged(RedundancyStateChanged msg)
|
||||
{
|
||||
_redundancySnapshotReceived = true;
|
||||
var local = msg.Nodes.FirstOrDefault(n => n.NodeId == _localNode);
|
||||
if (local is not null)
|
||||
{
|
||||
|
||||
@@ -153,6 +153,11 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
|
||||
/// behaviour for the same reason as <see cref="RediscoveryRaised"/> — a probe tick can land while the
|
||||
/// driver is between connects.</summary>
|
||||
private sealed record HostStatusRaised;
|
||||
/// <summary>Self-sent when the wrapped driver raises <see cref="IDriverConnectionLossSource.ConnectionLost"/> (#676)
|
||||
/// — it KNOWS its connection is gone for good. Marshals the event off the driver's thread. Acted on only in
|
||||
/// Connected (it becomes a disconnect at once, instead of after the #527 watchdog's silence window); every other
|
||||
/// behaviour is already reconnecting or connecting and ignores it.</summary>
|
||||
private sealed record ConnectionLossRaised(DriverConnectionLossEventArgs Args);
|
||||
|
||||
public sealed class RetryConnect
|
||||
{
|
||||
@@ -233,6 +238,7 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
|
||||
private EventHandler<AlarmEventArgs>? _alarmEventHandler;
|
||||
private EventHandler<RediscoveryEventArgs>? _rediscoveryHandler;
|
||||
private EventHandler<HostStatusChangedEventArgs>? _hostStatusHandler;
|
||||
private EventHandler<DriverConnectionLossEventArgs>? _connectionLossHandler;
|
||||
|
||||
/// <summary>When the driver last raised <see cref="IRediscoverable.OnRediscoveryNeeded"/>, and the reason
|
||||
/// it gave. Null until the first raise. Carried on every subsequent health snapshot so the AdminUI can
|
||||
@@ -400,6 +406,7 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
|
||||
// has no connection affinity, and a driver can observe a remote change while disconnected.
|
||||
AttachRediscoverySource();
|
||||
AttachHostStatusSource();
|
||||
AttachConnectionLossSource();
|
||||
PublishHealthSnapshot();
|
||||
Timers.StartPeriodicTimer("health-poll", HealthPollTick.Instance, _healthPollInterval);
|
||||
}
|
||||
@@ -427,6 +434,7 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
|
||||
// Stubbed drivers have no upstream alarm system — swallow the ack (it's fire-and-forget, no reply).
|
||||
Receive<RouteAlarmAck>(_ => { /* stubbed drivers have no alarm backend */ });
|
||||
Receive<DisconnectObserved>(_ => { /* stubbed drivers don't disconnect */ });
|
||||
Receive<ConnectionLossRaised>(IgnoreConnectionLoss);
|
||||
Receive<ForceReconnect>(_ => { /* stubbed drivers don't reconnect */ });
|
||||
Receive<SetDesiredSubscriptions>(StoreDesiredSubscriptions);
|
||||
// Stubbed drivers never enter Connected, so they never kick discovery; swallow defensively in case a
|
||||
@@ -468,6 +476,7 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
|
||||
});
|
||||
Receive<SetDesiredSubscriptions>(StoreDesiredSubscriptions);
|
||||
Receive<ForceReconnect>(_ => { /* already connecting — no-op */ });
|
||||
Receive<ConnectionLossRaised>(IgnoreConnectionLoss);
|
||||
// ResubscribeDesired self-Tells Subscribe; HandleSubscribeAsync replies SubscriptionEstablished to the
|
||||
// sender, which on the self-resubscribe path is Self. Swallow it (trace only) so it doesn't dead-letter.
|
||||
Receive<SubscriptionEstablished>(msg =>
|
||||
@@ -500,15 +509,11 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
|
||||
// reconnects but never manages a read is measured from THIS connect, not from a stale earlier one.
|
||||
_connectedSinceUtc = DateTime.UtcNow;
|
||||
|
||||
Receive<DisconnectObserved>(msg =>
|
||||
{
|
||||
_log.Warning("DriverInstance {Id}: disconnect observed ({Reason}); reconnecting",
|
||||
_driverInstanceId, msg.Reason);
|
||||
DetachSubscription();
|
||||
RecordFault();
|
||||
Become(Reconnecting);
|
||||
PublishHealthSnapshot();
|
||||
});
|
||||
Receive<DisconnectObserved>(msg => HandleDisconnectObserved(msg.Reason));
|
||||
// #676 — the driver KNOWS its connection is gone (e.g. OpcUaClient: the upstream server halted or forgot the
|
||||
// session). Same handling as the watchdog's disconnect, minus the ~90 s of silence it would need to prove it.
|
||||
// Handled inline rather than re-told as DisconnectObserved (Reconnecting also swallows a stray DisconnectObserved).
|
||||
Receive<ConnectionLossRaised>(msg => HandleDisconnectObserved($"driver reported connection lost: {msg.Args.Reason}"));
|
||||
Receive<ForceReconnect>(_ =>
|
||||
{
|
||||
_log.Info("DriverInstance {Id}: ForceReconnect requested by admin; re-entering Reconnecting", _driverInstanceId);
|
||||
@@ -764,6 +769,10 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
|
||||
Receive<InitializeFailed>(_ => { /* keep retrying via timer */ });
|
||||
Receive<SetDesiredSubscriptions>(StoreDesiredSubscriptions);
|
||||
Receive<ForceReconnect>(_ => { /* already reconnecting — no-op */ });
|
||||
Receive<ConnectionLossRaised>(IgnoreConnectionLoss);
|
||||
// A watchdog DisconnectObserved self-sent on the Connected tick can still be queued when a ConnectionLossRaised
|
||||
// (or another trip) has already moved us here — already reconnecting, so it is a no-op, not an Unhandled.
|
||||
Receive<DisconnectObserved>(_ => { /* already reconnecting — no-op */ });
|
||||
// ResubscribeDesired self-Tells Subscribe; HandleSubscribeAsync replies SubscriptionEstablished to the
|
||||
// sender, which on the self-resubscribe path is Self. Swallow it (trace only) so it doesn't dead-letter.
|
||||
Receive<SubscriptionEstablished>(msg =>
|
||||
@@ -1101,6 +1110,50 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
|
||||
_rediscoveryHandler = null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Subscribe the driver's <see cref="IDriverConnectionLossSource.ConnectionLost"/> (#676), if it is one,
|
||||
/// marshalling each raise to the actor thread — actor state is never touched on the driver's thread. Attached
|
||||
/// once in <c>PreStart</c> and detached in <c>PostStop</c>, like <see cref="AttachRediscoverySource"/>: the
|
||||
/// <see cref="IDriver"/> instance can outlive this actor, so a per-connect attach would leak a handler (holding
|
||||
/// a dead <c>Self</c>) per respawn.
|
||||
/// </summary>
|
||||
private void AttachConnectionLossSource()
|
||||
{
|
||||
if (_driver is not IDriverConnectionLossSource src || _connectionLossHandler is not null) return;
|
||||
var self = Self;
|
||||
_connectionLossHandler = (_, e) => self.Tell(new ConnectionLossRaised(e));
|
||||
src.ConnectionLost += _connectionLossHandler;
|
||||
}
|
||||
|
||||
/// <summary>Symmetric teardown, called from PostStop — same leak argument as <see cref="DetachRediscoverySource"/>.</summary>
|
||||
private void DetachConnectionLossSource()
|
||||
{
|
||||
if (_driver is IDriverConnectionLossSource src && _connectionLossHandler is not null)
|
||||
src.ConnectionLost -= _connectionLossHandler;
|
||||
_connectionLossHandler = null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A connection loss reported while NOT Connected (#676): a (re)initialise is already pending or running, so
|
||||
/// there is nothing to tear down — and acting on it could tear down the connection that attempt is building.
|
||||
/// </summary>
|
||||
private void IgnoreConnectionLoss(ConnectionLossRaised msg) =>
|
||||
_log.Debug("DriverInstance {Id}: driver reported connection lost ({Reason}) while not connected — ignored",
|
||||
_driverInstanceId, msg.Args.Reason);
|
||||
|
||||
/// <summary>
|
||||
/// Connected → Reconnecting on an observed disconnect — from the #527 watchdog or a driver's definitive
|
||||
/// <see cref="IDriverConnectionLossSource.ConnectionLost"/> (#676).
|
||||
/// </summary>
|
||||
private void HandleDisconnectObserved(string reason)
|
||||
{
|
||||
_log.Warning("DriverInstance {Id}: disconnect observed ({Reason}); reconnecting", _driverInstanceId, reason);
|
||||
DetachSubscription();
|
||||
RecordFault();
|
||||
Become(Reconnecting);
|
||||
PublishHealthSnapshot();
|
||||
}
|
||||
|
||||
/// <summary>Subscribe the driver's <see cref="IHostConnectivityProbe.OnHostStatusChanged"/> (if it is
|
||||
/// one), marshaling each transition to the actor thread. Idempotent; mirrors
|
||||
/// <see cref="AttachRediscoverySource"/>, including the PreStart-not-per-connect placement — a probe
|
||||
@@ -1497,6 +1550,7 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
|
||||
// same driver object), so a missing unsubscribe accumulates a handler per respawn holding a dead Self.
|
||||
DetachRediscoverySource();
|
||||
DetachHostStatusSource();
|
||||
DetachConnectionLossSource();
|
||||
try { _driver.ShutdownAsync(CancellationToken.None).GetAwaiter().GetResult(); }
|
||||
catch (Exception ex) { _log.Warning(ex, "DriverInstance {Id}: ShutdownAsync threw on PostStop", _driverInstanceId); }
|
||||
OtOpcUaTelemetry.DriverInstanceLifecycle.Add(1,
|
||||
|
||||
@@ -26,6 +26,36 @@ public static class PrimaryGatePolicy
|
||||
_ => driverMemberCount <= 1, // unknown: allow only when no driver peer exists
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #674 review: decide whether this node may serve an OPC UA <b>HistoryUpdate</b>. The
|
||||
/// <see cref="ShouldServiceAsPrimary"/> verdict, but CLOSED until this node has received its first
|
||||
/// redundancy snapshot.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <see cref="ShouldServiceAsPrimary"/> deliberately allows an unknown role while no driver peer is
|
||||
/// visible, which covers the pre-join boot window: a node spawned before it has joined its mesh sees a
|
||||
/// member count of 0. For a live data-plane operation (a device write, an ack) that is the right trade —
|
||||
/// denying would stall a genuine single node during its own join. HistoryUpdate is not live: a client can
|
||||
/// retry it, and both nodes of a pair accepting one concurrently during that window is exactly the
|
||||
/// wrong-prior-value race the Primary gate exists for. So it waits for a real verdict.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// "No snapshot yet" is told apart from "single node" by the snapshot itself: the redundancy singleton runs
|
||||
/// on every driver node (one per mesh, a single node included) and publishes on every change and every
|
||||
/// 10 s heartbeat, so a single node is named Primary within seconds of joining. Once a snapshot has
|
||||
/// arrived this is exactly <see cref="ShouldServiceAsPrimary"/> (including its unknown-role member-count
|
||||
/// rule for a snapshot that omits this node). A node with no redundancy manager at all (admin-only, or a
|
||||
/// harness) wires no gate and so allows.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
/// <param name="redundancySnapshotReceived">Whether any redundancy snapshot has reached this node yet.</param>
|
||||
/// <param name="localRole">This node's last-known redundancy role, or <c>null</c> when unknown.</param>
|
||||
/// <param name="driverMemberCount">Count of Up cluster members carrying the <c>driver</c> role.</param>
|
||||
/// <returns><c>true</c> to serve HistoryUpdate; <c>false</c> to refuse it (<c>BadInvalidState</c>).</returns>
|
||||
public static bool ShouldServeHistoryUpdate(bool redundancySnapshotReceived, RedundancyRole? localRole, int driverMemberCount) =>
|
||||
redundancySnapshotReceived && ShouldServiceAsPrimary(localRole, driverMemberCount);
|
||||
|
||||
/// <summary>
|
||||
/// Decide whether this node should drain the replicated alarm store-and-forward queue.
|
||||
/// </summary>
|
||||
|
||||
@@ -81,11 +81,43 @@ public sealed class ServerHistorianOptions
|
||||
/// cap). When a resume read stalls on such a cluster, the node manager over-fetches up to
|
||||
/// <c>MaxTieClusterOverfetch + 1</c> ties at that single timestamp (a <c>start == end</c> read) and
|
||||
/// pages through them; this bounds how large a single-timestamp burst the server will buffer in
|
||||
/// memory before it gives up and surfaces <c>BadHistoryOperationUnsupported</c> for that node. The
|
||||
/// default (65536) comfortably covers any realistic same-millisecond burst.
|
||||
/// memory before it gives up and surfaces <c>BadTooManyOperations</c> for that node (#686; was
|
||||
/// <c>BadHistoryOperationUnsupported</c>). It is also the tie-cluster CEILING of a reverse read walked
|
||||
/// back from forward reads on a backend that cannot read newest-first (the Gateway; segments are sized from the page, #689): one timestamp holding
|
||||
/// more samples than this is refused <c>BadTooManyOperations</c>. The default (65536) comfortably covers
|
||||
/// any realistic same-millisecond burst; a value below 1 is treated as 1.
|
||||
/// </summary>
|
||||
public int MaxTieClusterOverfetch { get; init; } = 65536;
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #691: whether HistoryRead-Raw honours <c>ReadRawModifiedDetails.ReturnBounds</c> (OPC UA Part 11 bounding
|
||||
/// values). Default <c>true</c>. <c>false</c> is the kill switch back to the pre-#691 answer: the flag is ignored
|
||||
/// (stored samples only) and a modified read carrying it is not refused <c>Bad_InvalidArgument</c>. Note the .NET
|
||||
/// SDK's <c>new ReadRawModifiedDetails()</c> defaults <c>ReturnBounds</c> (and <c>IsReadModified</c>) to <c>true</c>.
|
||||
/// </summary>
|
||||
public bool HonorReturnBounds { get; init; } = true;
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #693, <c>Gateway</c> backend only: whether HistoryRead-AtTime is served from RAW gateway reads evaluated with
|
||||
/// the SQL backends' shared AtTime rules (exact stored value at a stored time, linear between usable neighbours with
|
||||
/// the Interpolated bit, held before a Bad neighbour as Uncertain, null + the outage's Bad code inside an outage,
|
||||
/// <c>BadNoData</c> before the first sample). Default <c>true</c>. <c>false</c> is the kill switch back to the
|
||||
/// pre-#693 answer: AVEVA's own interpolation (the gateway's <c>ReadAtTime</c>), passed through unchanged.
|
||||
/// </summary>
|
||||
public bool GatewayAtTimeFromRaw { get; init; } = true;
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #694 (opt-in): HistoryReadEvents never reads events older than <c>now − EventReadHorizon</c> — the earlier
|
||||
/// bound of every fresh events window is raised to it (both directions, every backend; a window wholly older is
|
||||
/// <c>GoodNoData</c>). Default <see cref="TimeSpan.Zero"/> = no horizon: every read is exact to its own window. It
|
||||
/// exists for the Gateway, where proving "no older events" costs a read down to the Unix epoch on
|
||||
/// <c>Runtime.dbo.Events</c> (~3.8 s live per walked reverse / end-only read; an unfiltered one can hit the
|
||||
/// gateway's 30 s SQL timeout) and a forward read from a far-past StartTime costs the same. ⚠️ Lossy by design:
|
||||
/// events older than the horizon are invisible to HistoryReadEvents while it is set. Negative = none (a
|
||||
/// <see cref="Validate"/> warning).
|
||||
/// </summary>
|
||||
public TimeSpan EventReadHorizon { get; init; } = TimeSpan.Zero;
|
||||
|
||||
/// <summary>
|
||||
/// Maximum number of historized nodes served CONCURRENTLY within one HistoryRead batch (arch-review
|
||||
/// 03/S3). The SDK's <c>HistoryRead*</c> override surface is synchronous, so each arm block-bridges
|
||||
@@ -122,6 +154,16 @@ public sealed class ServerHistorianOptions
|
||||
/// </summary>
|
||||
public int MaxTimeWeightedRawRows { get; init; } = 1_000_000;
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #674: how far into the future an OPC UA HistoryUpdate (Insert / Replace / Update) item's
|
||||
/// <c>SourceTimestamp</c> may lie, measured from the server clock. An item beyond <c>now + skew</c> is refused
|
||||
/// <c>BadOutOfRange</c> by the node manager before any backend sees it — a far-future row is never a legitimate
|
||||
/// history edit, and on a backend that judges retention against the newest timestamp it could age out every
|
||||
/// older partition. Applies to every backend. The live recorder is not bounded. Non-positive disables the bound.
|
||||
/// Default 1 day.
|
||||
/// </summary>
|
||||
public TimeSpan HistoryUpdateMaxFutureSkew { get; init; } = TimeSpan.FromDays(1);
|
||||
|
||||
/// <summary>Returns operator-facing misconfiguration warnings for an <c>Enabled</c> historian
|
||||
/// (empty when disabled or correctly configured). Pure — the registration logs each entry.
|
||||
/// <para>
|
||||
@@ -157,7 +199,7 @@ public sealed class ServerHistorianOptions
|
||||
// the over-fetch code path only runs when a real IHistorianDataSource is wired in,
|
||||
// so a zero/negative value is harmless (and noise-free) when the historian is disabled.
|
||||
if (MaxTieClusterOverfetch <= 0)
|
||||
warnings.Add($"ServerHistorian:MaxTieClusterOverfetch is {MaxTieClusterOverfetch} — must be > 0; HistoryRead-Raw cannot page within an oversized tie cluster and will surface BadHistoryOperationUnsupported for those reads.");
|
||||
warnings.Add($"ServerHistorian:MaxTieClusterOverfetch is {MaxTieClusterOverfetch} — must be > 0 (treated as 1); HistoryRead-Raw and -Events cannot page within a tie cluster larger than that, and a Gateway reverse raw read cannot serve one timestamp holding more samples — those reads surface BadTooManyOperations.");
|
||||
// S3 HistoryRead concurrency knobs — a non-positive value disables the corresponding bound, so warn
|
||||
// (these gate the per-node fan-out / batch limiter that only run when a real data source is wired).
|
||||
if (HistoryReadBatchParallelism <= 0)
|
||||
@@ -166,6 +208,8 @@ public sealed class ServerHistorianOptions
|
||||
warnings.Add($"ServerHistorian:MaxConcurrentHistoryReadBatches is {MaxConcurrentHistoryReadBatches} — must be > 0; the process-wide HistoryRead batch limiter is disabled.");
|
||||
if (MaxTimeWeightedRawRows <= 0)
|
||||
warnings.Add($"ServerHistorian:MaxTimeWeightedRawRows is {MaxTimeWeightedRawRows} — must be > 0; time-weighted aggregate reads on the SqlServer/QuestDb backends stream an unbounded number of raw samples.");
|
||||
if (EventReadHorizon < TimeSpan.Zero)
|
||||
warnings.Add($"ServerHistorian:EventReadHorizon is {EventReadHorizon} — negative is treated as no horizon (00:00:00); HistoryReadEvents reads every window in full.");
|
||||
if (HistoryReadDeadline <= TimeSpan.Zero)
|
||||
warnings.Add($"ServerHistorian:HistoryReadDeadline is {HistoryReadDeadline} — must be > 0; HistoryRead requests run unbounded (no per-request deadline).");
|
||||
return warnings;
|
||||
|
||||
@@ -75,9 +75,29 @@ public interface IHistorizationRoleView
|
||||
void PublishHistorizationRole(bool shouldRecordHistorizedValues);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #674 review: a singleton snapshot of the OPC UA <b>HistoryUpdate</b> verdict
|
||||
/// (<see cref="PrimaryGatePolicy.ShouldServeHistoryUpdate"/>), read by the node manager's HistoryUpdate Primary gate.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// A separate verdict from <see cref="IDriverPrimaryRoleSource.ShouldActAsPrimary"/> because it is closed until the
|
||||
/// first redundancy snapshot, where that one mirrors the live data-plane gate and is open through the pre-join boot
|
||||
/// window. Registered unconditionally on every driver-role node (same object as the other views); a node that does
|
||||
/// not register it wires no gate.
|
||||
/// </remarks>
|
||||
public interface IHistoryUpdateRoleView
|
||||
{
|
||||
/// <summary>Whether this node may serve HistoryUpdate right now.</summary>
|
||||
bool ShouldServeHistoryUpdate { get; }
|
||||
|
||||
/// <summary>Records a fresh decision. Called by <c>DriverHostActor</c> whenever it publishes its Primary verdict.</summary>
|
||||
/// <param name="shouldServeHistoryUpdate">The decision <see cref="PrimaryGatePolicy.ShouldServeHistoryUpdate"/> produced.</param>
|
||||
void PublishHistoryUpdateRole(bool shouldServeHistoryUpdate);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Thread-safe <see cref="IRedundancyRoleView"/> + <see cref="IDriverPrimaryRoleSource"/> +
|
||||
/// <see cref="IHistorizationRoleView"/>, each backed by its own volatile field.
|
||||
/// <see cref="IHistorizationRoleView"/> + <see cref="IHistoryUpdateRoleView"/>, each backed by its own volatile field.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
@@ -90,8 +110,15 @@ public interface IHistorizationRoleView
|
||||
/// them is the bug that live gate already found once.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public sealed class RedundancyRoleView : IRedundancyRoleView, IDriverPrimaryRoleSource, IHistorizationRoleView
|
||||
public sealed class RedundancyRoleView : IRedundancyRoleView, IDriverPrimaryRoleSource, IHistorizationRoleView,
|
||||
IHistoryUpdateRoleView
|
||||
{
|
||||
// Gitea #674 review: seeded CLOSED, unlike the data-plane verdict. HistoryUpdate waits for a real redundancy verdict
|
||||
// (PrimaryGatePolicy.ShouldServeHistoryUpdate); a node that never publishes here is not a driver node with a
|
||||
// redundancy manager, and such a node does not register this view as its gate source.
|
||||
private volatile bool _shouldServeHistoryUpdate =
|
||||
PrimaryGatePolicy.ShouldServeHistoryUpdate(redundancySnapshotReceived: false, localRole: null, driverMemberCount: 1);
|
||||
|
||||
// Seeded with the decision for "role unknown" rather than a bare false. An unpublished view and a
|
||||
// node whose role nothing ever reports are genuinely the same situation, and they must behave the
|
||||
// same: a deployment that runs no redundancy at all never publishes here, and defaulting closed
|
||||
@@ -121,6 +148,12 @@ public sealed class RedundancyRoleView : IRedundancyRoleView, IDriverPrimaryRole
|
||||
/// <inheritdoc/>
|
||||
public bool ShouldActAsPrimary => _shouldActAsPrimary;
|
||||
|
||||
/// <inheritdoc/>
|
||||
public bool ShouldServeHistoryUpdate => _shouldServeHistoryUpdate;
|
||||
|
||||
/// <inheritdoc/>
|
||||
public void PublishHistoryUpdateRole(bool shouldServeHistoryUpdate) => _shouldServeHistoryUpdate = shouldServeHistoryUpdate;
|
||||
|
||||
/// <inheritdoc/>
|
||||
public void Publish(bool shouldDrainAlarmHistory) => _shouldDrainAlarmHistory = shouldDrainAlarmHistory;
|
||||
|
||||
|
||||
@@ -98,6 +98,11 @@ public static class ServiceCollectionExtensions
|
||||
// with no gate at all, and both nodes would write every sample twice with nothing to say so.
|
||||
services.TryAddSingleton<IHistorizationRoleView>(sp => sp.GetRequiredService<RedundancyRoleView>());
|
||||
|
||||
// Gitea #674 review: the OPC UA HistoryUpdate verdict — the data-plane verdict, but closed until this node's first
|
||||
// redundancy snapshot. Same instance; registered here (driver-role nodes only) so an admin-only node, which has no
|
||||
// redundancy manager, wires no HistoryUpdate gate at all.
|
||||
services.TryAddSingleton<IHistoryUpdateRoleView>(sp => sp.GetRequiredService<RedundancyRoleView>());
|
||||
|
||||
// The node→central ack transport under MeshTransport:Mode = Grpc (mesh gRPC command plane,
|
||||
// Task 1B.2). Registered unconditionally but resolved only by the Grpc arm of
|
||||
// WithOtOpcUaRuntimeActors — the mode is not knowable here (this runs before configuration is
|
||||
@@ -631,7 +636,9 @@ public static class ServiceCollectionExtensions
|
||||
// Review R6 MINOR-4: the passive half of a LocalDb pair has no PeerAddress, so it is told
|
||||
// its queue-sharing peer as its mesh partner instead — otherwise it never stands down.
|
||||
sharesAlarmQueueWithLocal: BuildAlarmQueuePeerPredicate(
|
||||
resolver.GetService<IConfiguration>(), roleInfo, replicationPeerHost)),
|
||||
resolver.GetService<IConfiguration>(), roleInfo, replicationPeerHost),
|
||||
// Gitea #674 review: the HistoryUpdate verdict (closed until the first redundancy snapshot).
|
||||
historyUpdateRoleView: resolver.GetService<IHistoryUpdateRoleView>()),
|
||||
DriverHostActorName);
|
||||
registry.Register<DriverHostActorKey>(driverHost);
|
||||
|
||||
|
||||
@@ -108,6 +108,9 @@ public sealed class FakeOpcUaClientService : IOpcUaClientService
|
||||
/// </summary>
|
||||
public Dictionary<string, IReadOnlyList<BrowseResult>> BrowseResultsByParent { get; } = new();
|
||||
|
||||
/// <summary>Gets or sets an exception thrown by HistoryReadRawAsync (simulates a refused read).</summary>
|
||||
public Exception? HistoryReadException { get; set; }
|
||||
|
||||
/// <summary>Gets or sets the history read result returned by HistoryReadRawAsync and HistoryReadAggregateAsync.</summary>
|
||||
public IReadOnlyList<DataValue> HistoryReadResult { get; set; } = new List<DataValue>
|
||||
{
|
||||
@@ -323,6 +326,7 @@ public sealed class FakeOpcUaClientService : IOpcUaClientService
|
||||
NodeId nodeId, DateTime startTime, DateTime endTime, int maxValues = 1000, CancellationToken ct = default)
|
||||
{
|
||||
HistoryReadRawCalls.Add((nodeId, startTime, endTime, maxValues));
|
||||
if (HistoryReadException != null) throw HistoryReadException;
|
||||
return Task.FromResult(HistoryReadResult);
|
||||
}
|
||||
|
||||
|
||||
@@ -197,4 +197,53 @@ public class HistoryReadCommandTests
|
||||
using var console = TestConsoleHelper.CreateConsole();
|
||||
await Should.ThrowAsync<CliFx.Exceptions.CommandException>(async () => await command.ExecuteAsync(console));
|
||||
}
|
||||
|
||||
/// <summary>#677.5: the new aggregate names parse (and the aliases).</summary>
|
||||
[Theory]
|
||||
[InlineData("TimeAverage", AggregateType.TimeAverage)]
|
||||
[InlineData("timeaverage2", AggregateType.TimeAverage2)]
|
||||
[InlineData("Total", AggregateType.Total)]
|
||||
[InlineData("total2", AggregateType.Total2)]
|
||||
[InlineData("Interpolative", AggregateType.Interpolative)]
|
||||
[InlineData("interp", AggregateType.Interpolative)]
|
||||
public async Task Execute_NewAggregates_Parse(string name, AggregateType expected)
|
||||
{
|
||||
var fakeService = new FakeOpcUaClientService();
|
||||
var command = new HistoryReadCommand(new FakeOpcUaClientServiceFactory(fakeService))
|
||||
{
|
||||
Url = "opc.tcp://localhost:4840",
|
||||
NodeId = "ns=2;s=HistNode",
|
||||
Aggregate = name,
|
||||
};
|
||||
|
||||
using var console = TestConsoleHelper.CreateConsole();
|
||||
await command.ExecuteAsync(console);
|
||||
|
||||
fakeService.HistoryReadAggregateCalls.ShouldHaveSingleItem().Aggregate.ShouldBe(expected);
|
||||
}
|
||||
|
||||
/// <summary>#674.6: a refused read is a one-line CommandException (non-zero exit), not a stack trace.</summary>
|
||||
[Fact]
|
||||
public async Task Execute_RefusedRead_ThrowsCommandException_WithOneLineMessage()
|
||||
{
|
||||
var fakeService = new FakeOpcUaClientService
|
||||
{
|
||||
HistoryReadException = new ServiceResultException(StatusCodes.BadHistoryOperationUnsupported,
|
||||
"HistoryRead of ns=2;s=HistNode failed: BadHistoryOperationUnsupported (0x80B40000)."),
|
||||
};
|
||||
var command = new HistoryReadCommand(new FakeOpcUaClientServiceFactory(fakeService))
|
||||
{
|
||||
Url = "opc.tcp://localhost:4840",
|
||||
NodeId = "ns=2;s=HistNode",
|
||||
};
|
||||
|
||||
using var console = TestConsoleHelper.CreateConsole();
|
||||
var ex = await Should.ThrowAsync<CliFx.Exceptions.CommandException>(async () => await command.ExecuteAsync(console));
|
||||
|
||||
ex.Message.ShouldContain("BadHistoryOperationUnsupported");
|
||||
ex.Message.ShouldNotContain("\n");
|
||||
ex.Message.ShouldNotContain(" at ");
|
||||
ex.ExitCode.ShouldNotBe(0);
|
||||
fakeService.DisconnectCalled.ShouldBeTrue();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -160,4 +160,67 @@ public class EndpointSelectorTests
|
||||
Should.Throw<ArgumentException>(() =>
|
||||
EndpointSelector.SelectBest(Array.Empty<EndpointDescription>(), "", MessageSecurityMode.None));
|
||||
}
|
||||
|
||||
/// <summary>#678.5: a server advertising its internal port must not redirect the session off the mapped port.</summary>
|
||||
[Fact]
|
||||
public void SelectBest_RewritesHostAndPort_WhenSchemeMatches()
|
||||
{
|
||||
var endpoints = new[] { Ep(MessageSecurityMode.None, SecurityPolicies.None, "opc.tcp://internal-host:4840/OtOpcUa") };
|
||||
|
||||
var best = EndpointSelector.SelectBest(endpoints, "opc.tcp://localhost:4841", MessageSecurityMode.None);
|
||||
|
||||
var uri = new Uri(best.EndpointUrl);
|
||||
uri.Host.ShouldBe("localhost");
|
||||
uri.Port.ShouldBe(4841);
|
||||
uri.AbsolutePath.ShouldBe("/OtOpcUa");
|
||||
}
|
||||
|
||||
/// <summary>A different scheme (e.g. https endpoint advertised to an opc.tcp caller) is left untouched.</summary>
|
||||
[Fact]
|
||||
public void SelectBest_LeavesEndpointUntouched_WhenSchemeDiffers()
|
||||
{
|
||||
var endpoints = new[] { Ep(MessageSecurityMode.None, SecurityPolicies.None, "https://internal-host:4843/x") };
|
||||
|
||||
var best = EndpointSelector.SelectBest(endpoints, "opc.tcp://localhost:4841", MessageSecurityMode.None);
|
||||
|
||||
new Uri(best.EndpointUrl).Host.ShouldBe("internal-host");
|
||||
new Uri(best.EndpointUrl).Port.ShouldBe(4843);
|
||||
}
|
||||
|
||||
/// <summary>Identical host, port and scheme: nothing to rewrite.</summary>
|
||||
[Fact]
|
||||
public void SelectBest_Unchanged_WhenHostPortAndSchemeMatch()
|
||||
{
|
||||
var endpoints = new[] { Ep(MessageSecurityMode.None, SecurityPolicies.None, "opc.tcp://server:4840") };
|
||||
|
||||
var best = EndpointSelector.SelectBest(endpoints, "opc.tcp://server:4840", MessageSecurityMode.None);
|
||||
|
||||
best.EndpointUrl.ShouldBe("opc.tcp://server:4840");
|
||||
}
|
||||
|
||||
/// <summary>A URL typed without a port keeps the advertised port (opc.tcp has no default; Uri.Port is -1).</summary>
|
||||
[Fact]
|
||||
public void SelectBest_KeepsAdvertisedPort_WhenRequestedUrlHasNoPort()
|
||||
{
|
||||
var endpoints = new[] { Ep(MessageSecurityMode.None, SecurityPolicies.None, "opc.tcp://internal-host:4840/OtOpcUa") };
|
||||
|
||||
var best = EndpointSelector.SelectBest(endpoints, "opc.tcp://localhost", MessageSecurityMode.None);
|
||||
|
||||
var uri = new Uri(best.EndpointUrl);
|
||||
uri.Host.ShouldBe("localhost");
|
||||
uri.Port.ShouldBe(4840);
|
||||
}
|
||||
|
||||
/// <summary>An IPv6 host is rewritten with its brackets intact.</summary>
|
||||
[Fact]
|
||||
public void SelectBest_RewritesIpv6Host()
|
||||
{
|
||||
var endpoints = new[] { Ep(MessageSecurityMode.None, SecurityPolicies.None, "opc.tcp://internal-host:4840/OtOpcUa") };
|
||||
|
||||
var best = EndpointSelector.SelectBest(endpoints, "opc.tcp://[::1]:4841", MessageSecurityMode.None);
|
||||
|
||||
var uri = new Uri(best.EndpointUrl);
|
||||
uri.Host.ShouldBe("[::1]");
|
||||
uri.Port.ShouldBe(4841);
|
||||
}
|
||||
}
|
||||
|
||||
+16
@@ -18,6 +18,11 @@ public class AggregateTypeMapperTests
|
||||
[InlineData(AggregateType.Start)]
|
||||
[InlineData(AggregateType.End)]
|
||||
[InlineData(AggregateType.StandardDeviation)]
|
||||
[InlineData(AggregateType.TimeAverage)]
|
||||
[InlineData(AggregateType.TimeAverage2)]
|
||||
[InlineData(AggregateType.Total)]
|
||||
[InlineData(AggregateType.Total2)]
|
||||
[InlineData(AggregateType.Interpolative)]
|
||||
public void ToNodeId_ReturnsNonNullForAllValues(AggregateType aggregate)
|
||||
{
|
||||
var nodeId = AggregateTypeMapper.ToNodeId(aggregate);
|
||||
@@ -82,4 +87,15 @@ public class AggregateTypeMapperTests
|
||||
Should.Throw<ArgumentOutOfRangeException>(() =>
|
||||
AggregateTypeMapper.ToNodeId((AggregateType)99));
|
||||
}
|
||||
|
||||
/// <summary>#677.5: the time-weighted Part 13 aggregates map to their ObjectIds.</summary>
|
||||
[Fact]
|
||||
public void ToNodeId_TimeWeightedAggregates_MapCorrectly()
|
||||
{
|
||||
AggregateTypeMapper.ToNodeId(AggregateType.TimeAverage).ShouldBe(ObjectIds.AggregateFunction_TimeAverage);
|
||||
AggregateTypeMapper.ToNodeId(AggregateType.TimeAverage2).ShouldBe(ObjectIds.AggregateFunction_TimeAverage2);
|
||||
AggregateTypeMapper.ToNodeId(AggregateType.Total).ShouldBe(ObjectIds.AggregateFunction_Total);
|
||||
AggregateTypeMapper.ToNodeId(AggregateType.Total2).ShouldBe(ObjectIds.AggregateFunction_Total2);
|
||||
AggregateTypeMapper.ToNodeId(AggregateType.Interpolative).ShouldBe(ObjectIds.AggregateFunction_Interpolative);
|
||||
}
|
||||
}
|
||||
@@ -136,6 +136,6 @@ public class ModelConstructionTests
|
||||
[Fact]
|
||||
public void AggregateType_Enum_HasExpectedValues()
|
||||
{
|
||||
Enum.GetValues<AggregateType>().Length.ShouldBe(7);
|
||||
Enum.GetValues<AggregateType>().Length.ShouldBe(12);
|
||||
}
|
||||
}
|
||||
@@ -141,7 +141,7 @@ public class HistoryViewModelTests
|
||||
public void AggregateTypes_ContainsNullForRaw()
|
||||
{
|
||||
_vm.AggregateTypes.ShouldContain((AggregateType?)null);
|
||||
_vm.AggregateTypes.Count.ShouldBe(8); // null + 7 enum values
|
||||
_vm.AggregateTypes.Count.ShouldBe(13); // null + 12 enum values
|
||||
}
|
||||
|
||||
/// <summary>Verifies that the Clear method resets the view model state.</summary>
|
||||
|
||||
+112
@@ -0,0 +1,112 @@
|
||||
using System.Reflection;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Core.Abstractions.Tests.Historian;
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #675 review: the reverse-events surface is two interface members with defaults
|
||||
/// (<see cref="IHistorianDataSource.SupportsReverseEvents"/> = false and the direction-aware
|
||||
/// <c>ReadEventsAsync(…, bool reverse, …)</c>). A future wrapper / decorator that forwards only the five-argument read
|
||||
/// and the other members would compile and silently fall back to the defaults — turning every reverse read into
|
||||
/// <c>BadHistoryOperationUnsupported</c> on a backend that supports it. So every production implementation must declare
|
||||
/// BOTH members itself.
|
||||
/// </summary>
|
||||
public sealed class HistorianDataSourceReverseEventsGuardTests
|
||||
{
|
||||
private static IReadOnlyList<Type> ProductionImplementations()
|
||||
{
|
||||
// Every ZB.MOM.WW.OtOpcUa production assembly next to the tests (this project references every historian backend).
|
||||
var dir = AppContext.BaseDirectory;
|
||||
var assemblies = Directory.GetFiles(dir, "ZB.MOM.WW.OtOpcUa.*.dll")
|
||||
.Where(f => !Path.GetFileName(f).Contains(".Tests", StringComparison.Ordinal)
|
||||
&& !Path.GetFileName(f).Contains("IntegrationTests", StringComparison.Ordinal))
|
||||
.Select(f => Assembly.LoadFrom(f));
|
||||
return assemblies
|
||||
.SelectMany(a =>
|
||||
{
|
||||
try { return a.GetTypes(); }
|
||||
catch (ReflectionTypeLoadException e) { return e.Types.Where(t => t is not null).Cast<Type>(); }
|
||||
})
|
||||
.Where(t => t is { IsClass: true, IsAbstract: false } && typeof(IHistorianDataSource).IsAssignableFrom(t))
|
||||
.ToList();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void The_scan_finds_every_known_backend()
|
||||
{
|
||||
// Positive control: a scan that finds nothing would pass the guard below vacuously.
|
||||
var names = ProductionImplementations().Select(t => t.Name).ToList();
|
||||
names.ShouldContain("NullHistorianDataSource");
|
||||
names.ShouldContain("GatewayHistorianDataSource");
|
||||
names.ShouldContain("SqlServerHistorianDataSource");
|
||||
names.ShouldContain("QuestDbHistorianDataSource");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Every_implementation_declares_the_reverse_overload_and_the_capability_itself()
|
||||
{
|
||||
foreach (var type in ProductionImplementations())
|
||||
{
|
||||
var map = type.GetInterfaceMap(typeof(IHistorianDataSource));
|
||||
MethodInfo Impl(Func<MethodInfo, bool> pick) =>
|
||||
map.TargetMethods[Array.FindIndex(map.InterfaceMethods, m => pick(m))];
|
||||
|
||||
var reverseRead = Impl(m => m.Name == nameof(IHistorianDataSource.ReadEventsAsync)
|
||||
&& m.GetParameters().Any(p => p.Name == "reverse"));
|
||||
reverseRead.DeclaringType.ShouldBe(type,
|
||||
$"{type.FullName} must declare ReadEventsAsync(…, bool reverse, …) itself, not inherit the interface default");
|
||||
|
||||
var capability = Impl(m => m.Name == "get_" + nameof(IHistorianDataSource.SupportsReverseEvents));
|
||||
capability.DeclaringType.ShouldBe(type,
|
||||
$"{type.FullName} must declare SupportsReverseEvents itself, not inherit the interface default");
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Every_implementation_declares_its_reverse_raw_capability_itself()
|
||||
{
|
||||
// Gitea #686: the default (false) is safe — the server walks back with forward reads — but a backend that serves
|
||||
// reverse natively and inherited it would silently lose its native path, and a wrapper of the Gateway that claimed
|
||||
// true would send start > end to a gateway that rejects it. Declare it, both ways.
|
||||
foreach (var type in ProductionImplementations())
|
||||
{
|
||||
var map = type.GetInterfaceMap(typeof(IHistorianDataSource));
|
||||
var getter = map.TargetMethods[Array.FindIndex(map.InterfaceMethods,
|
||||
m => m.Name == "get_" + nameof(IHistorianDataSource.SupportsReverseRaw))];
|
||||
getter.DeclaringType.ShouldBe(type,
|
||||
$"{type.FullName} must declare SupportsReverseRaw itself, not inherit the interface default");
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Every_implementation_declares_its_supported_aggregates_itself()
|
||||
{
|
||||
// Gitea #682: the interface default is "every aggregate" — a production backend (or a future wrapper) that
|
||||
// inherited it would advertise all of them under AggregateFunctions and forward reads it cannot serve.
|
||||
foreach (var type in ProductionImplementations())
|
||||
{
|
||||
var map = type.GetInterfaceMap(typeof(IHistorianDataSource));
|
||||
var getter = map.TargetMethods[Array.FindIndex(map.InterfaceMethods,
|
||||
m => m.Name == "get_" + nameof(IHistorianDataSource.SupportedAggregates))];
|
||||
getter.DeclaringType.ShouldBe(type,
|
||||
$"{type.FullName} must declare SupportedAggregates itself, not inherit the interface default");
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Every_implementation_declares_its_raw_page_limit_itself()
|
||||
{
|
||||
// Gitea #694: the default (0 = no limit) is what a strict-ceiling Gateway must NOT report — a wrapper that inherited
|
||||
// it would bring back BadTooManyOperations for every page larger than the gateway's ceiling.
|
||||
foreach (var type in ProductionImplementations())
|
||||
{
|
||||
var map = type.GetInterfaceMap(typeof(IHistorianDataSource));
|
||||
var getter = map.TargetMethods[Array.FindIndex(map.InterfaceMethods,
|
||||
m => m.Name == "get_" + nameof(IHistorianDataSource.MaxRawValuesPerRead))];
|
||||
getter.DeclaringType.ShouldBe(type,
|
||||
$"{type.FullName} must declare MaxRawValuesPerRead itself, not inherit the interface default");
|
||||
}
|
||||
}
|
||||
}
|
||||
+25
-14
@@ -45,9 +45,10 @@ public sealed class IHistorianDataSourceContractTests
|
||||
[InlineData("ReadEventsAsync", typeof(Task<HistoricalEventsResult>))]
|
||||
public void ReadMethods_ReturnExpectedTaskShape(string methodName, Type expectedReturnType)
|
||||
{
|
||||
var method = typeof(IHistorianDataSource).GetMethod(methodName);
|
||||
method.ShouldNotBeNull();
|
||||
method!.ReturnType.ShouldBe(expectedReturnType);
|
||||
// ReadEventsAsync is overloaded (the #675 direction-aware overload) — every overload must keep the shape.
|
||||
var methods = Overloads(methodName);
|
||||
methods.ShouldNotBeEmpty();
|
||||
foreach (var method in methods) method.ReturnType.ShouldBe(expectedReturnType);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that GetHealthSnapshot is synchronous.</summary>
|
||||
@@ -146,11 +147,14 @@ public sealed class IHistorianDataSourceContractTests
|
||||
[InlineData("ReadEventsAsync", "maxEvents", typeof(int))]
|
||||
public void HistoryRead_MaxParameter_TypePinned(string methodName, string parameterName, Type expectedType)
|
||||
{
|
||||
var method = typeof(IHistorianDataSource).GetMethod(methodName);
|
||||
method.ShouldNotBeNull();
|
||||
var parameter = method!.GetParameters().FirstOrDefault(p => p.Name == parameterName);
|
||||
parameter.ShouldNotBeNull($"Method {methodName} should expose a '{parameterName}' parameter.");
|
||||
parameter!.ParameterType.ShouldBe(expectedType);
|
||||
var methods = Overloads(methodName);
|
||||
methods.ShouldNotBeEmpty();
|
||||
foreach (var method in methods)
|
||||
{
|
||||
var parameter = method.GetParameters().FirstOrDefault(p => p.Name == parameterName);
|
||||
parameter.ShouldNotBeNull($"Method {methodName} should expose a '{parameterName}' parameter.");
|
||||
parameter!.ParameterType.ShouldBe(expectedType);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -163,13 +167,17 @@ public sealed class IHistorianDataSourceContractTests
|
||||
[Fact]
|
||||
public void ReadEventsAsync_sentinel_and_continuation_contract_types_pinned()
|
||||
{
|
||||
var method = typeof(IHistorianDataSource).GetMethod("ReadEventsAsync");
|
||||
method.ShouldNotBeNull();
|
||||
var methods = Overloads("ReadEventsAsync");
|
||||
methods.Count.ShouldBe(2, "the five-argument read plus the #675 direction-aware overload");
|
||||
methods.ShouldContain(m => m.GetParameters().Any(p => p.Name == "reverse" && p.ParameterType == typeof(bool)));
|
||||
|
||||
var maxEventsParam = method!.GetParameters().FirstOrDefault(p => p.Name == "maxEvents");
|
||||
maxEventsParam.ShouldNotBeNull("maxEvents parameter must exist for the sentinel contract");
|
||||
maxEventsParam!.ParameterType.ShouldBe(typeof(int),
|
||||
"maxEvents is int (not uint) so callers can pass <=0 as a 'use backend default cap' sentinel");
|
||||
foreach (var method in methods)
|
||||
{
|
||||
var maxEventsParam = method.GetParameters().FirstOrDefault(p => p.Name == "maxEvents");
|
||||
maxEventsParam.ShouldNotBeNull("maxEvents parameter must exist for the sentinel contract");
|
||||
maxEventsParam!.ParameterType.ShouldBe(typeof(int),
|
||||
"maxEvents is int (not uint) so callers can pass <=0 as a 'use backend default cap' sentinel");
|
||||
}
|
||||
|
||||
// HistoricalEventsResult must carry a nullable ContinuationPoint so implementations
|
||||
// can signal 'more results exist' (non-null) vs 'all results returned' (null).
|
||||
@@ -179,4 +187,7 @@ public sealed class IHistorianDataSourceContractTests
|
||||
continuationProp!.PropertyType.ShouldBe(typeof(byte[]),
|
||||
"ContinuationPoint is byte[]? — null means no more pages, non-null means caller must page");
|
||||
}
|
||||
|
||||
private static List<System.Reflection.MethodInfo> Overloads(string name) =>
|
||||
typeof(IHistorianDataSource).GetMethods().Where(m => m.Name == name).ToList();
|
||||
}
|
||||
|
||||
+16
@@ -67,6 +67,22 @@ public sealed class NullHistorianDataSourceTests
|
||||
result.ContinuationPoint.ShouldBeNull();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// #691.4 review: like raw, the null source reads events in either direction (empty) and says so — the server sends
|
||||
/// it the reverse read directly instead of walking back through empty forward reads (it refused reverse under #675).
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task Direction_aware_ReadEventsAsync_is_empty_in_both_directions()
|
||||
{
|
||||
IHistorianDataSource source = Source;
|
||||
source.SupportsReverseEvents.ShouldBeTrue();
|
||||
(await source.ReadEventsAsync(null, DateTime.UtcNow.AddHours(-1), DateTime.UtcNow, 0, reverse: false, Ct))
|
||||
.Events.ShouldBeEmpty();
|
||||
var reverse = await source.ReadEventsAsync(null, DateTime.UtcNow, DateTime.UtcNow.AddHours(-1), 0, reverse: true, Ct);
|
||||
reverse.Events.ShouldBeEmpty();
|
||||
reverse.ContinuationPoint.ShouldBeNull();
|
||||
}
|
||||
|
||||
/// <summary>GetHealthSnapshot reports a fully-disabled (zeroed, disconnected, no-node) shape.</summary>
|
||||
[Fact]
|
||||
public void GetHealthSnapshot_is_disabled()
|
||||
|
||||
@@ -139,6 +139,7 @@ public abstract class AtTimeConformance : ConformanceBase
|
||||
(At(10), HistorianValue.FromInt64(15), Good));
|
||||
var s = (await B.CreateDataSource().ReadAtTimeAsync("I", [At(5), At(2)], Ct)).Samples;
|
||||
s[0].Value.ShouldBe(13L); // 12.5 ⇒ 13 (away from zero)
|
||||
s[0].Value.ShouldBeOfType<long>(); // ShouldBe compares numbers across types — the type needs its own check (#693)
|
||||
s[1].Value.ShouldBe(11L); // 11.0
|
||||
s[0].StatusCode.ShouldBe(GoodInterpolated);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,100 @@
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Conformance;
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #691: the server composes Part 11 bounding values (HistoryRead-Raw <c>ReturnBounds</c>) from exactly these raw
|
||||
/// reads on a backend that reads newest-first natively (<c>HistoryBounds</c> in the OPC UA server) — so every backend must
|
||||
/// answer them precisely: "at or before T" = a reverse read of <c>(0, T]</c> capped at one, "at or after T" = a forward read
|
||||
/// of <c>[T, MaxValue)</c> capped at one, "strictly after T" = the same from <c>T + resolution</c>. A bound is a stored
|
||||
/// sample of any quality, so a Bad row or an outage marker comes back as the bound.
|
||||
/// </summary>
|
||||
public abstract class BoundingValueConformance : ConformanceBase
|
||||
{
|
||||
private static readonly CancellationToken Ct = CancellationToken.None;
|
||||
private static readonly DateTime Floor = new(0, DateTimeKind.Utc);
|
||||
private static readonly DateTime Ceiling = new(DateTime.MaxValue.Ticks, DateTimeKind.Utc);
|
||||
|
||||
private async Task<DataValueSnapshot?> AtOrBeforeAsync(IHistorianDataSource src, string tag, DateTime t) =>
|
||||
(await src.ReadRawAsync(tag, t, Floor, 1, Ct)).Samples.SingleOrDefault();
|
||||
|
||||
private async Task<DataValueSnapshot?> AtOrAfterAsync(IHistorianDataSource src, string tag, DateTime t) =>
|
||||
(await src.ReadRawAsync(tag, t, Ceiling, 1, Ct)).Samples.SingleOrDefault();
|
||||
|
||||
[Fact]
|
||||
public async Task The_sample_at_or_before_a_time_is_the_latest_one_on_or_before_it()
|
||||
{
|
||||
await ConformanceData.SeedAAsync(B);
|
||||
var src = B.CreateDataSource();
|
||||
|
||||
(await AtOrBeforeAsync(src, "A", At(10)))!.Value.ShouldBe(20d); // exactly on it
|
||||
(await AtOrBeforeAsync(src, "A", At(15)))!.Value.ShouldBe(20d); // between samples
|
||||
(await AtOrBeforeAsync(src, "A", At(1000)))!.Value.ShouldBe(50d); // past the last
|
||||
(await AtOrBeforeAsync(src, "A", At(-1))).ShouldBeNull(); // before the first: no bound
|
||||
(await AtOrBeforeAsync(src, "A", At(15)))!.SourceTimestampUtc!.Value.Ticks.ShouldBe(B.Round(At(10)).Ticks);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task The_sample_at_or_after_a_time_is_the_earliest_one_on_or_after_it()
|
||||
{
|
||||
await ConformanceData.SeedAAsync(B);
|
||||
var src = B.CreateDataSource();
|
||||
|
||||
(await AtOrAfterAsync(src, "A", At(10)))!.Value.ShouldBe(20d); // exactly on it
|
||||
(await AtOrAfterAsync(src, "A", At(15)))!.Value.ShouldBe(30d); // between samples
|
||||
(await AtOrAfterAsync(src, "A", At(-100)))!.Value.ShouldBe(10d); // before the first
|
||||
(await AtOrAfterAsync(src, "A", At(41))).ShouldBeNull(); // past the last: no bound
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task The_sample_strictly_after_a_time_starts_one_storage_resolution_later()
|
||||
{
|
||||
await ConformanceData.SeedAAsync(B);
|
||||
var src = B.CreateDataSource();
|
||||
|
||||
(await AtOrAfterAsync(src, "A", At(10).AddTicks(B.ResolutionTicks)))!.Value.ShouldBe(30d);
|
||||
(await AtOrAfterAsync(src, "A", At(40).AddTicks(B.ResolutionTicks))).ShouldBeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_time_between_storage_ticks_finds_the_sample_the_backend_stores_at_its_truncation()
|
||||
{
|
||||
// The server compares "on the bound" at the storage resolution, so the lookups must agree with the window reads:
|
||||
// a time a fraction of a resolution step after a stored sample is that sample's instant on a coarse backend.
|
||||
await ConformanceData.SeedAAsync(B);
|
||||
var src = B.CreateDataSource();
|
||||
var t = At(10).AddTicks(B.ResolutionTicks - 1); // == At(10) on a 1-tick backend
|
||||
|
||||
(await AtOrBeforeAsync(src, "A", t))!.Value.ShouldBe(20d);
|
||||
var after = await AtOrAfterAsync(src, "A", t);
|
||||
after!.SourceTimestampUtc!.Value.Ticks.ShouldBeGreaterThanOrEqualTo(B.Round(t).Ticks);
|
||||
after.Value.ShouldBe(B.Round(t) == B.Round(At(10)) ? 20d : 30d);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_bad_sample_or_an_outage_marker_is_a_bound_as_stored()
|
||||
{
|
||||
await B.SeedAsync("M",
|
||||
(At(0), HistorianValue.FromDouble(10), Good),
|
||||
(At(5), HistorianValue.Empty, BadNoCommunication),
|
||||
(At(7), HistorianValue.FromDouble(99), Bad));
|
||||
var src = B.CreateDataSource();
|
||||
|
||||
var before = await AtOrBeforeAsync(src, "M", At(6));
|
||||
before!.Value.ShouldBeNull();
|
||||
before.StatusCode.ShouldBe(0x80310000u);
|
||||
var after = await AtOrAfterAsync(src, "M", At(6));
|
||||
after!.Value.ShouldBe(99d);
|
||||
after.StatusCode.ShouldBe(0x80000000u);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task An_unknown_tag_has_no_bounds()
|
||||
{
|
||||
var src = B.CreateDataSource();
|
||||
(await AtOrBeforeAsync(src, "nope", At(10))).ShouldBeNull();
|
||||
(await AtOrAfterAsync(src, "nope", At(10))).ShouldBeNull();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,142 @@
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Conformance;
|
||||
|
||||
/// <summary>
|
||||
/// The direction-aware event read (Gitea #675): forward is <c>[lo, hi)</c> ascending by <c>(EventTimeUtc, EventId)</c>;
|
||||
/// reverse is the Part 11 reverse window <c>(lo, hi]</c> DESCENDING by the same key, so a capped reverse read returns the
|
||||
/// LATEST events and flags truncation. Paging a reverse read the way the server does — resume at the last returned
|
||||
/// time inclusive and drop the boundary ties already emitted — must visit every event exactly once, newest-first,
|
||||
/// through a same-timestamp tie cluster.
|
||||
/// </summary>
|
||||
public abstract class EventReadConformance : ConformanceBase
|
||||
{
|
||||
private static readonly CancellationToken Ct = CancellationToken.None;
|
||||
|
||||
/// <summary>e0@0, e1@1, then the ties t0..t4@2, then e5@3 and e6@4 (all source "s"), plus o0@2 from another source.</summary>
|
||||
private async Task<IHistorianDataSource> SeedAsync()
|
||||
{
|
||||
await B.SeedEventsAsync(
|
||||
(At(0), "e0", "s"), (At(1), "e1", "s"),
|
||||
(At(2), "t0", "s"), (At(2), "t1", "s"), (At(2), "t2", "s"), (At(2), "t3", "s"), (At(2), "t4", "s"),
|
||||
(At(3), "e5", "s"), (At(4), "e6", "s"),
|
||||
(At(2), "o0", "other"));
|
||||
return B.CreateDataSource();
|
||||
}
|
||||
|
||||
private static readonly string[] NewestFirst = ["e6", "e5", "t4", "t3", "t2", "t1", "t0", "e1", "e0"];
|
||||
|
||||
[Fact]
|
||||
public async Task Reverse_read_is_newest_first_by_time_then_EventId_descending()
|
||||
{
|
||||
using var src = await SeedAsync();
|
||||
var r = await src.ReadEventsAsync("s", At(-1), At(10), 0, reverse: true, Ct);
|
||||
r.Events.Select(e => e.EventId).ShouldBe(NewestFirst);
|
||||
r.ContinuationPoint.ShouldBeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Reverse_window_is_end_exclusive_start_inclusive_whichever_order_the_bounds_come_in()
|
||||
{
|
||||
using var src = await SeedAsync();
|
||||
// (At(1), At(3)]: e1 at the lower bound is excluded, e5 at the upper bound included.
|
||||
(await src.ReadEventsAsync("s", At(3), At(1), 0, reverse: true, Ct)).Events.Select(e => e.EventId)
|
||||
.ShouldBe(["e5", "t4", "t3", "t2", "t1", "t0"]);
|
||||
(await src.ReadEventsAsync("s", At(1), At(3), 0, reverse: true, Ct)).Events.Select(e => e.EventId)
|
||||
.ShouldBe(["e5", "t4", "t3", "t2", "t1", "t0"]);
|
||||
// Forward is unchanged: [At(1), At(3)) ascending.
|
||||
(await src.ReadEventsAsync("s", At(1), At(3), 0, reverse: false, Ct)).Events.Select(e => e.EventId)
|
||||
.ShouldBe(["e1", "t0", "t1", "t2", "t3", "t4"]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_capped_reverse_read_keeps_the_LATEST_events_and_flags_truncation()
|
||||
{
|
||||
using var src = await SeedAsync();
|
||||
var r = await src.ReadEventsAsync("s", At(10), At(-1), 3, reverse: true, Ct);
|
||||
r.Events.Select(e => e.EventId).ShouldBe(["e6", "e5", "t4"]);
|
||||
r.ContinuationPoint.ShouldNotBeNull();
|
||||
// An exact fit is complete.
|
||||
(await src.ReadEventsAsync("s", At(10), At(-1), 9, reverse: true, Ct)).ContinuationPoint.ShouldBeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Paging_a_reverse_read_through_a_tie_cluster_visits_every_event_once_newest_first()
|
||||
{
|
||||
using var src = await SeedAsync();
|
||||
foreach (var cap in new[] { 1, 2, 3, 4 })
|
||||
{
|
||||
var all = new List<string>();
|
||||
var lo = At(-1);
|
||||
var from = At(10);
|
||||
var skip = 0;
|
||||
for (var page = 0; ; page++)
|
||||
{
|
||||
page.ShouldBeLessThan(40, "paging must terminate");
|
||||
var r = await src.ReadEventsAsync("s", from, lo, cap + skip, reverse: true, Ct);
|
||||
var events = r.Events.ToList();
|
||||
var drop = 0;
|
||||
while (drop < skip && drop < events.Count && events[drop].EventTimeUtc == from) drop++;
|
||||
events = events.Skip(drop).ToList();
|
||||
// The server asks for cap + skip rows (it re-reads the boundary ties it then drops), so a page is never
|
||||
// emptied by the trim while more remains.
|
||||
if (r.ContinuationPoint is not null) events.Count.ShouldBe(cap);
|
||||
|
||||
all.AddRange(events.Select(e => e.EventId));
|
||||
if (r.ContinuationPoint is null) break;
|
||||
var boundary = events[^1].EventTimeUtc;
|
||||
var ties = events.AsEnumerable().Reverse().TakeWhile(e => e.EventTimeUtc == boundary).Count();
|
||||
skip = boundary == from ? skip + ties : ties;
|
||||
from = boundary;
|
||||
}
|
||||
|
||||
all.SequenceEqual(NewestFirst).ShouldBeTrue($"cap {cap}: {string.Join(",", all)}");
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void The_backend_reports_reverse_support()
|
||||
{
|
||||
using var src = B.CreateDataSource();
|
||||
src.SupportsReverseEvents.ShouldBeTrue();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task An_oversized_reverse_tie_cluster_sliced_the_way_the_server_does_matches_the_reverse_read_order()
|
||||
{
|
||||
// The server pages WITHIN a tie cluster larger than the page by over-fetching it FORWARD over [T, T + resolution)
|
||||
// and reversing it for a reverse read (OtOpcUaNodeManager.ServeEventsAsync + HistoryPaging.SliceTieCluster). That is
|
||||
// only correct if this backend's forward cluster, reversed, is exactly the order its reverse read emits the ties in
|
||||
// — the tie-breaker must be the same key in both directions. Drive both through the real backend.
|
||||
await B.SeedEventsAsync(Enumerable.Range(0, 12).Select(i => (At(2), $"c{i:00}", "s")).Append((At(3), "z", "s")).ToArray());
|
||||
using var src = B.CreateDataSource();
|
||||
|
||||
var reverseOrder = (await src.ReadEventsAsync("s", At(10), At(0), 0, reverse: true, Ct)).Events
|
||||
.Where(e => e.EventTimeUtc == B.Round(At(2))).Select(e => e.EventId).ToList();
|
||||
var clusterReversed = (await src.ReadEventsAsync("s", B.Round(At(2)), B.Round(At(2)).AddTicks(B.ResolutionTicks), 1000, reverse: false, Ct))
|
||||
.Events.Select(e => e.EventId).Reverse().ToList();
|
||||
|
||||
clusterReversed.Count.ShouldBe(12);
|
||||
clusterReversed.ShouldBe(reverseOrder);
|
||||
|
||||
// And slicing it at a cap of 5 (skip 0, 5, 10) walks the cluster newest-first with no duplicate or gap.
|
||||
var slices = new List<string>();
|
||||
for (var skip = 0; skip < clusterReversed.Count; skip += 5) slices.AddRange(clusterReversed.Skip(skip).Take(5));
|
||||
slices.ShouldBe(Enumerable.Range(0, 12).Reverse().Select(i => $"c{i:00}"));
|
||||
|
||||
// A capped reverse read stopping inside the cluster flags truncation, so the server's cluster branch is reachable.
|
||||
var capped = await src.ReadEventsAsync("s", At(10), At(0), 4, reverse: true, Ct);
|
||||
capped.Events.Select(e => e.EventId).ShouldBe(["z", "c11", "c10", "c09"]);
|
||||
capped.ContinuationPoint.ShouldNotBeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Reverse_read_with_a_null_source_spans_every_source()
|
||||
{
|
||||
using var src = await SeedAsync();
|
||||
var r = await src.ReadEventsAsync(null, At(2.5), At(1.5), 0, reverse: true, Ct);
|
||||
r.Events.Select(e => e.EventId).ShouldBe(["t4", "t3", "t2", "t1", "t0", "o0"]);
|
||||
}
|
||||
}
|
||||
@@ -41,6 +41,13 @@ public interface IConformanceBackend : IAsyncDisposable
|
||||
/// <param name="samples">The rows to store.</param>
|
||||
Task SeedAsync(string tag, params (DateTime T, HistorianValue V, uint Status)[] samples);
|
||||
|
||||
/// <summary>
|
||||
/// Writes historical events directly in the backend's event table (never through the alarm writer): event id,
|
||||
/// source and time as given (rounded to the storage resolution), severity 500, received time = event time.
|
||||
/// </summary>
|
||||
/// <param name="events">The events to store.</param>
|
||||
Task SeedEventsAsync(params (DateTime T, string Id, string Source)[] events);
|
||||
|
||||
/// <summary>
|
||||
/// Live (non-deleted) rows for a tag, ascending by time, read directly from storage. The status is returned
|
||||
/// VERBATIM as stored — there is no read-side clamp (an Empty row stored non-Bad reads back non-Bad here).
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Conformance;
|
||||
|
||||
@@ -20,6 +21,7 @@ public abstract class ProcessedConformance : ConformanceBase
|
||||
private const uint UncertainCalc = 0x40A40401u; // UncertainDataSubNormal | 0x401
|
||||
private const uint NoDataCalc = 0x809B0401u; // BadNoData | 0x401
|
||||
private const uint InvalidInputsCalc = 0x80D60401u; // BadAggregateInvalidInputs | 0x401
|
||||
private const uint Partial = 0x4u; // the Partial aggregate bit (#677 start-of-data, #680 end-of-data)
|
||||
|
||||
private async Task<HistoryReadResult> ReadA(HistoryAggregateType agg) =>
|
||||
await B.CreateDataSource().ReadProcessedAsync("A", At(0), At(80), Twenty, agg, Ct);
|
||||
@@ -39,8 +41,6 @@ public abstract class ProcessedConformance : ConformanceBase
|
||||
[InlineData(HistoryAggregateType.Average, 15d, 40d, 50d)]
|
||||
[InlineData(HistoryAggregateType.Minimum, 10d, 40d, 50d)]
|
||||
[InlineData(HistoryAggregateType.Maximum, 20d, 40d, 50d)]
|
||||
[InlineData(HistoryAggregateType.Start, 10d, 40d, 50d)]
|
||||
[InlineData(HistoryAggregateType.End, 20d, 40d, 50d)]
|
||||
[InlineData(HistoryAggregateType.Delta, 10d, 0d, 0d)]
|
||||
[InlineData(HistoryAggregateType.Range, 10d, 0d, 0d)]
|
||||
public async Task Numeric_aggregates_over_the_fixture(HistoryAggregateType agg, double b0, double b1, double b2)
|
||||
@@ -50,30 +50,81 @@ public abstract class ProcessedConformance : ConformanceBase
|
||||
|
||||
s.Count.ShouldBe(4);
|
||||
s.Select(x => x.Value).ShouldBe([b0, b1, b2, null]);
|
||||
// B1 holds a Bad sample alongside a usable one ⇒ Uncertain; the empty B3 ⇒ BadNoData.
|
||||
s.Select(x => x.StatusCode).ShouldBe([GoodCalc, UncertainCalc, GoodCalc, NoDataCalc]);
|
||||
// B1 holds a Bad sample alongside a usable one ⇒ Uncertain (Delta: its first sample is the Bad one); the empty B3,
|
||||
// after the last sample, ⇒ BadNoData. #680: B2 holds the last sample (+40) ⇒ end-of-data Partial on a forward
|
||||
// read, for every aggregate but Average.
|
||||
var b2Status = agg == HistoryAggregateType.Average ? GoodCalc : GoodCalc | Partial;
|
||||
s.Select(x => x.StatusCode).ShouldBe([GoodCalc, UncertainCalc, b2Status, NoDataCalc]);
|
||||
s.Select(x => x.SourceTimestampUtc).ShouldBe([B.Round(At(0)), B.Round(At(20)), B.Round(At(40)), B.Round(At(60))]); // bucket starts
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Count_counts_usable_samples_as_int()
|
||||
public async Task Start_and_End_return_the_raw_first_and_last_sample_of_any_quality()
|
||||
{
|
||||
// #680 (Part 13 Tables 73/74; SDK ComputeStartEnd): the interval's raw first / last sample — its own value (null
|
||||
// when Bad), its own status (Raw: no Calculated bit) and its own timestamp. B1's Start is the Bad sample at +20.
|
||||
await ConformanceData.SeedAAsync(B);
|
||||
var start = (await ReadA(HistoryAggregateType.Start)).Samples;
|
||||
start.Select(x => x.Value).ShouldBe([10d, null, 50d, null]);
|
||||
start.Select(x => x.StatusCode).ShouldBe([Good, Bad, Good | 0x400u | Partial, NoDataCalc]);
|
||||
start.Select(x => x.SourceTimestampUtc).ShouldBe([B.Round(At(0)), B.Round(At(20)), B.Round(At(40)), B.Round(At(60))]);
|
||||
var end = (await ReadA(HistoryAggregateType.End)).Samples;
|
||||
end.Select(x => x.Value).ShouldBe([20d, 40d, 50d, null]);
|
||||
end.Select(x => x.StatusCode).ShouldBe([Good, Good, Good | 0x400u | Partial, NoDataCalc]);
|
||||
end.Select(x => x.SourceTimestampUtc).ShouldBe([B.Round(At(10)), B.Round(At(30)), B.Round(At(40)), B.Round(At(60))]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Count_counts_good_samples_as_int()
|
||||
{
|
||||
// #680 (Part 13 §5.4.3.21): Good samples only, value-based status; end-of-data Partial on B2.
|
||||
await ConformanceData.SeedAAsync(B);
|
||||
var s = (await ReadA(HistoryAggregateType.Count)).Samples;
|
||||
s.Select(x => x.Value).ShouldBe([(object?)2, 1, 1, null]);
|
||||
s.Select(x => x.StatusCode).ShouldBe([GoodCalc, UncertainCalc, GoodCalc, NoDataCalc]);
|
||||
s.Select(x => x.StatusCode).ShouldBe([GoodCalc, UncertainCalc, GoodCalc | Partial, NoDataCalc]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Count_is_zero_on_an_empty_interval_inside_the_data_range_in_both_directions()
|
||||
{
|
||||
// #680: +1 G, +2 U, +25 Bad, +45 G, +70 G. Forward [0,60) at 10 s: [10,20), [30,40) and [50,60) are empty but
|
||||
// inside [first, last] ⇒ 0, Good; [20,30) holds only a Bad sample ⇒ generic Bad. Reverse (60 → 0) likewise, and a
|
||||
// window wholly after the data ⇒ BadNoData — through the real SQL (the data range is two pushed-down seeks).
|
||||
await B.SeedAsync("CZ",
|
||||
(At(1), HistorianValue.FromDouble(10), Good),
|
||||
(At(2), HistorianValue.FromDouble(10), Uncertain),
|
||||
(At(25), HistorianValue.FromDouble(10), Bad),
|
||||
(At(45), HistorianValue.FromDouble(10), Good),
|
||||
(At(70), HistorianValue.FromDouble(1), Good));
|
||||
var src = B.CreateDataSource();
|
||||
var ten = TimeSpan.FromSeconds(10);
|
||||
var fwd = (await src.ReadProcessedAsync("CZ", At(0), At(60), ten, HistoryAggregateType.Count, Ct)).Samples;
|
||||
fwd.Select(x => x.Value).ShouldBe([(object?)1, 0, null, 0, 1, 0]);
|
||||
fwd.Select(x => x.StatusCode).ShouldBe(
|
||||
[UncertainCalc | Partial, GoodCalc, BadCalc, GoodCalc, GoodCalc, GoodCalc]);
|
||||
var rev = (await src.ReadProcessedAsync("CZ", At(60), At(0), ten, HistoryAggregateType.Count, Ct)).Samples;
|
||||
rev.Select(x => x.Value).ShouldBe([(object?)0, 1, 0, null, 0, 1]);
|
||||
rev.Select(x => x.StatusCode).ShouldBe([GoodCalc, GoodCalc, GoodCalc, BadCalc, GoodCalc, UncertainCalc]);
|
||||
(await src.ReadProcessedAsync("CZ", At(80), At(100), ten, HistoryAggregateType.Count, Ct)).Samples
|
||||
.ShouldAllBe(x => x.StatusCode == NoDataCalc);
|
||||
(await src.ReadProcessedAsync("CZ", At(-20), At(0), ten, HistoryAggregateType.Count, Ct)).Samples
|
||||
.ShouldAllBe(x => x.StatusCode == NoDataCalc);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(HistoryAggregateType.PercentGood, 100d, 50d, 100d)]
|
||||
[InlineData(HistoryAggregateType.PercentBad, 0d, 50d, 0d)]
|
||||
public async Task Percent_aggregates_count_every_sample(HistoryAggregateType agg, double b0, double b1, double b2)
|
||||
public async Task Percent_aggregates_are_shares_of_the_interval_duration(HistoryAggregateType agg, double b0, double b1,
|
||||
double b2)
|
||||
{
|
||||
// #680 (Part 13 §5.4.3.31-34): stepped regions, each with the status of the sample starting it. B1: [20,30) Bad,
|
||||
// [30,40) Good. B2 holds the end of the data (+40, nothing after) ⇒ its region runs to the interval end, Partial;
|
||||
// B3 is after the end of the data ⇒ BadNoData.
|
||||
await ConformanceData.SeedAAsync(B);
|
||||
var s = (await ReadA(agg)).Samples;
|
||||
s.Select(x => x.Value).ShouldBe([b0, b1, b2, null]);
|
||||
// A percentage is a statement about quality, so it is not itself degraded by the Bad input.
|
||||
s.Select(x => x.StatusCode).ShouldBe([GoodCalc, GoodCalc, GoodCalc, NoDataCalc]);
|
||||
s.Select(x => x.StatusCode).ShouldBe([GoodCalc, GoodCalc, GoodCalc | 0x4u, NoDataCalc]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -83,7 +134,8 @@ public abstract class ProcessedConformance : ConformanceBase
|
||||
var min = (await ReadA(HistoryAggregateType.MinimumActualTime)).Samples;
|
||||
min.Select(x => x.Value).ShouldBe([10d, 40d, 50d, null]);
|
||||
min.Select(x => x.SourceTimestampUtc).ShouldBe([B.Round(At(0)), B.Round(At(30)), B.Round(At(40)), B.Round(At(60))]);
|
||||
min.Select(x => x.StatusCode).ShouldBe([GoodCalc, UncertainCalc, GoodCalc, NoDataCalc]);
|
||||
// #680: B2 holds the last sample (+40) ⇒ end-of-data Partial.
|
||||
min.Select(x => x.StatusCode).ShouldBe([GoodCalc, UncertainCalc, GoodCalc | Partial, NoDataCalc]);
|
||||
var max = (await ReadA(HistoryAggregateType.MaximumActualTime)).Samples;
|
||||
max.Select(x => x.Value).ShouldBe([20d, 40d, 50d, null]);
|
||||
max.Select(x => x.SourceTimestampUtc).ShouldBe([B.Round(At(10)), B.Round(At(30)), B.Round(At(40)), B.Round(At(60))]);
|
||||
@@ -168,10 +220,12 @@ public abstract class ProcessedConformance : ConformanceBase
|
||||
// An interval wider than the window (here TimeSpan.MaxValue) is the same single interval, no overflow.
|
||||
(await src.ReadProcessedAsync("A", start, end, TimeSpan.MaxValue, HistoryAggregateType.Count, Ct))
|
||||
.Samples.Single().Value.ShouldBe(4);
|
||||
// The time-weighted path over the whole range: ≈ the extrapolated 50 that dominates eight millennia.
|
||||
// The time-weighted path over the whole range. TimeAverage2 is not extrapolated past the end of the data (+40, #680):
|
||||
// (150 + 200 + 450) value·s over the 30 non-Bad seconds = 80/3.
|
||||
var tw = (await src.ReadProcessedAsync("A", start, end, TimeSpan.Zero, HistoryAggregateType.TimeAverage2, Ct)).Samples.Single();
|
||||
((double)tw.Value!).ShouldBe(50d, 1e-6);
|
||||
tw.StatusCode.ShouldBe(UncertainCalc);
|
||||
((double)tw.Value!).ShouldBe(80d / 3d, 1e-6);
|
||||
// The data starts (+0) strictly inside the one interval ⇒ Partial as well (#677, the SDK's m_startOfData rule).
|
||||
tw.StatusCode.ShouldBe(UncertainCalc | 0x4u);
|
||||
src.GetHealthSnapshot().TotalFailures.ShouldBe(0);
|
||||
}
|
||||
|
||||
@@ -196,12 +250,209 @@ public abstract class ProcessedConformance : ConformanceBase
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_reverse_window_is_computed_forward_and_returned_descending()
|
||||
public async Task A_reverse_window_is_anchored_at_its_later_time()
|
||||
{
|
||||
// #677 (Part 13 / SDK): At(80) → At(0) is the window (0,80] in intervals (60,80], (40,60], (20,40], (0,20], each
|
||||
// stamped with its LATER edge, latest first. A sample on an edge belongs to the interval it ends: +40 is in
|
||||
// (20,40], +20 (Bad) in (0,20], and +0 is outside the window. (The pre-#677 layout answered [60,80) … [0,20)
|
||||
// stamped 60/40/20/0 with values null, 50, 40, 15.)
|
||||
await ConformanceData.SeedAAsync(B);
|
||||
var s = (await B.CreateDataSource().ReadProcessedAsync("A", At(80), At(0), Twenty, HistoryAggregateType.Average, Ct)).Samples;
|
||||
s.Select(x => x.SourceTimestampUtc).ShouldBe([B.Round(At(60)), B.Round(At(40)), B.Round(At(20)), B.Round(At(0))]);
|
||||
s.Select(x => x.Value).ShouldBe([null, 50d, 40d, 15d]);
|
||||
s.Select(x => x.SourceTimestampUtc).ShouldBe([B.Round(At(80)), B.Round(At(60)), B.Round(At(40)), B.Round(At(20))]);
|
||||
s.Select(x => x.Value).ShouldBe([null, null, 45d, 20d]);
|
||||
s.Select(x => x.StatusCode).ShouldBe([NoDataCalc, NoDataCalc, GoodCalc, UncertainCalc]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_reverse_window_truncates_its_earliest_interval()
|
||||
{
|
||||
// At(50) → At(0), 20 s over tag A (+0=10, +10=20, +20=Bad, +30=40, +40=50): (30,50] = {+40} → 50 (+30 is its
|
||||
// earlier edge, excluded); (10,30] = {+20 Bad, +30} → 40, Uncertain; (0,10] = {+10} → 20 — the truncated interval
|
||||
// is the EARLIEST one. (Pre-#677: [0,20), [20,40), [40,50) stamped 40/20/0 with values 50, 40, 15.)
|
||||
await ConformanceData.SeedAAsync(B);
|
||||
var s = (await B.CreateDataSource().ReadProcessedAsync("A", At(50), At(0), Twenty, HistoryAggregateType.Average, Ct)).Samples;
|
||||
s.Select(x => x.SourceTimestampUtc).ShouldBe([B.Round(At(50)), B.Round(At(30)), B.Round(At(10))]);
|
||||
s.Select(x => x.Value).ShouldBe([50d, 40d, 20d]);
|
||||
s.Select(x => x.StatusCode).ShouldBe([GoodCalc, UncertainCalc, GoodCalc]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Reverse_bucket_edges_are_later_edge_inclusive_to_the_tick()
|
||||
{
|
||||
// The mirror of the forward edge test: At(40) → At(0), 20 s ⇒ (20,40], (0,20]. The bucket math is pushed down,
|
||||
// so this pins the SQL, not just the C# plan.
|
||||
var r = B.ResolutionTicks;
|
||||
await B.SeedAsync("ER",
|
||||
(At(0), HistorianValue.FromDouble(1000), Good), // == earlier edge: excluded
|
||||
(At(0).AddTicks(r), HistorianValue.FromDouble(1), Good), // first tick of (0,20]
|
||||
(At(20), HistorianValue.FromDouble(2), Good), // == edge: the LATER edge of (0,20]
|
||||
(At(20).AddTicks(r), HistorianValue.FromDouble(4), Good), // first tick of (20,40]
|
||||
(At(40), HistorianValue.FromDouble(8), Good), // == request start: included, in (20,40]
|
||||
(At(40).AddTicks(r), HistorianValue.FromDouble(2000), Good)); // after the window: excluded
|
||||
var s = (await B.CreateDataSource().ReadProcessedAsync("ER", At(40), At(0), Twenty, HistoryAggregateType.Average, Ct)).Samples;
|
||||
s.Select(x => x.SourceTimestampUtc).ShouldBe([B.Round(At(40)), B.Round(At(20))]);
|
||||
s.Select(x => x.Value).ShouldBe([6d, 1.5d]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task The_interval_the_data_starts_inside_carries_the_Partial_bit()
|
||||
{
|
||||
// #677 (SDK m_startOfData): the tag's first sample is +5, inside [0,20) ⇒ Partial there for every aggregate but
|
||||
// Average; [20,40) is fully covered. With an earlier sample stored (even outside the window) nothing is Partial.
|
||||
await B.SeedAsync("LS",
|
||||
(At(5), HistorianValue.FromDouble(10), Good),
|
||||
(At(15), HistorianValue.FromDouble(30), Good),
|
||||
(At(25), HistorianValue.FromDouble(20), Good),
|
||||
(At(60), HistorianValue.FromDouble(20), Good));
|
||||
var src = B.CreateDataSource();
|
||||
foreach (var agg in new[] { HistoryAggregateType.Count, HistoryAggregateType.Minimum, HistoryAggregateType.TimeAverage2 })
|
||||
{
|
||||
var s = (await src.ReadProcessedAsync("LS", At(0), At(40), Twenty, agg, Ct)).Samples;
|
||||
(s[0].StatusCode & 0x4u).ShouldBe(0x4u, agg.ToString());
|
||||
(s[1].StatusCode & 0x4u).ShouldBe(0u, agg.ToString());
|
||||
}
|
||||
(await src.ReadProcessedAsync("LS", At(0), At(40), Twenty, HistoryAggregateType.Average, Ct)).Samples
|
||||
.ShouldAllBe(x => (x.StatusCode & 0x4u) == 0);
|
||||
|
||||
await B.SeedAsync("LS2",
|
||||
(At(-100), HistorianValue.FromDouble(10), Good),
|
||||
(At(5), HistorianValue.FromDouble(10), Good),
|
||||
(At(15), HistorianValue.FromDouble(30), Good),
|
||||
(At(25), HistorianValue.FromDouble(20), Good));
|
||||
// ... but LS2's data ENDS (+25) inside [20,40) ⇒ Partial there (#680, the SDK's m_endOfData rule).
|
||||
(await src.ReadProcessedAsync("LS2", At(0), At(40), Twenty, HistoryAggregateType.Count, Ct)).Samples
|
||||
.Select(x => x.StatusCode & Partial).ShouldBe([0u, Partial]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_reverse_read_sets_no_start_of_data_Partial_and_matches_the_SDK()
|
||||
{
|
||||
// Review F1: the SDK applies the start-of-data rule to FORWARD reads only. Reverse 40 → 0 at 10 s over data that
|
||||
// starts at +7: (0,10] holds the data's start but is not Partial — checked through the real SQL and the SDK oracle.
|
||||
var rows = Enumerable.Range(0, 20)
|
||||
.Select(i => (At(7 + i * 5), HistorianValue.FromDouble(i % 2 == 0 ? 20 : 40), Good)).ToArray();
|
||||
await B.SeedAsync("RLS", rows);
|
||||
var sdkRows = rows.Select(r => (B.Round(r.Item1), r.Item2, r.Item3)).ToArray();
|
||||
var src = B.CreateDataSource();
|
||||
var ten = TimeSpan.FromSeconds(10);
|
||||
foreach (var agg in new[]
|
||||
{
|
||||
HistoryAggregateType.Count, HistoryAggregateType.Minimum, HistoryAggregateType.Range,
|
||||
HistoryAggregateType.Delta, HistoryAggregateType.TimeAverage, HistoryAggregateType.TimeAverage2,
|
||||
HistoryAggregateType.Total2,
|
||||
})
|
||||
{
|
||||
var ours = (await src.ReadProcessedAsync("RLS", At(40), At(0), ten, agg, Ct)).Samples;
|
||||
ours.ShouldAllBe(x => (x.StatusCode & 0x4u) == 0, agg.ToString());
|
||||
var tw = AggregateFamilies.FamilyOf(agg) == AggregateFamily.TimeWeighted;
|
||||
SdkAggregateOracle.ShouldMatch(ours, SdkAggregateOracle.Run(agg, B.Round(At(40)), B.Round(At(0)), ten, sdkRows),
|
||||
$"{agg} reverse late start", compareCalculatedInterpolatedBits: tw);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Dense: every 2.5 s from −10 to +100, values cycling, one Bad sample at +22.5 (no empty interval).</summary>
|
||||
private Task SeedDenseAsync(bool uncertain = false) => B.SeedAsync(uncertain ? "PU" : "P", DenseRows(uncertain));
|
||||
|
||||
/// <summary>
|
||||
/// #684 <paramref name="uncertain"/>: every fifth sample (other than the Bad one) Uncertain at 16 — inside every
|
||||
/// interval's Good range (the cycling values alternate around 16), so the min/max family matches the SDK unmasked
|
||||
/// (SdkAggregateParityTests.DenseUncertain is the same data).
|
||||
/// </summary>
|
||||
private static (DateTime T, HistorianValue V, uint Status)[] DenseRows(bool uncertain = false) => Enumerable.Range(0, 45)
|
||||
.Select(i => uncertain && i % 5 == 0 && i != 13
|
||||
? (At(-10 + i * 2.5), HistorianValue.FromDouble(16), Uncertain)
|
||||
: (At(-10 + i * 2.5), HistorianValue.FromDouble(10 + (i * 7 % 13)), i == 13 ? Bad : Good))
|
||||
.ToArray();
|
||||
|
||||
[Theory]
|
||||
[InlineData(0, 80, 20)]
|
||||
[InlineData(80, 0, 20)]
|
||||
[InlineData(77, 3, 20)]
|
||||
[InlineData(3, 77, 20)]
|
||||
public async Task Per_sample_processed_reads_over_uncertain_samples_match_the_SDK(double from, double to, double intervalS)
|
||||
{
|
||||
// #684: the Good-only value columns through the real SQL against the SDK's calculators, over Dense with Uncertain
|
||||
// samples — values, timestamps, code bits and Partial, no mask.
|
||||
await SeedDenseAsync(uncertain: true);
|
||||
var rows = DenseRows(uncertain: true).Select(r => (B.Round(r.T), r.V, r.Status)).ToArray();
|
||||
var src = B.CreateDataSource();
|
||||
var interval = TimeSpan.FromSeconds(intervalS);
|
||||
var (start, end) = (B.Round(At(from)), B.Round(At(to)));
|
||||
foreach (var agg in new[]
|
||||
{
|
||||
HistoryAggregateType.Average, HistoryAggregateType.Minimum, HistoryAggregateType.Maximum,
|
||||
HistoryAggregateType.Count, HistoryAggregateType.Range, HistoryAggregateType.MinimumActualTime,
|
||||
HistoryAggregateType.MaximumActualTime,
|
||||
})
|
||||
{
|
||||
var ours = (await src.ReadProcessedAsync("PU", At(from), At(to), interval, agg, Ct)).Samples.ToList();
|
||||
SdkAggregateOracle.ShouldMatch(ours, SdkAggregateOracle.Run(agg, start, end, interval, rows),
|
||||
$"{agg} uncertain {from}→{to}/{intervalS}s", compareCalculatedInterpolatedBits: false);
|
||||
}
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(0, 80, 20)]
|
||||
[InlineData(80, 0, 20)]
|
||||
[InlineData(77, 3, 20)]
|
||||
[InlineData(3, 77, 20)]
|
||||
public async Task Processed_reads_match_the_SDK_aggregate_calculators(double from, double to, double intervalS)
|
||||
{
|
||||
// #677: the real backend (pushed-down SQL bucket math + the shared evaluators) against the pinned SDK's own Part 13
|
||||
// calculators over the same stored rows — timestamps, values, status code bits and the Partial bit, forward and
|
||||
// reverse. See SdkAggregateParityTests (Sql.Common.Tests) for the documented divergences this avoids/masks.
|
||||
await SeedDenseAsync();
|
||||
var rows = DenseRows().Select(r => (B.Round(r.T), r.V, r.Status)).ToArray();
|
||||
var src = B.CreateDataSource();
|
||||
var interval = TimeSpan.FromSeconds(intervalS);
|
||||
// The SDK sees the window the backend actually serves (QuestDB rounds the bounds to µs).
|
||||
var (start, end) = (B.Round(At(from)), B.Round(At(to)));
|
||||
foreach (var agg in new[]
|
||||
{
|
||||
HistoryAggregateType.Average, HistoryAggregateType.Minimum, HistoryAggregateType.Maximum,
|
||||
HistoryAggregateType.Count, HistoryAggregateType.TimeAverage, HistoryAggregateType.TimeAverage2,
|
||||
HistoryAggregateType.Total, HistoryAggregateType.Total2, HistoryAggregateType.Interpolative,
|
||||
HistoryAggregateType.Range, HistoryAggregateType.Start, HistoryAggregateType.End,
|
||||
HistoryAggregateType.Delta, HistoryAggregateType.MinimumActualTime, HistoryAggregateType.MaximumActualTime,
|
||||
HistoryAggregateType.PercentGood, HistoryAggregateType.PercentBad,
|
||||
})
|
||||
{
|
||||
var ours = (await src.ReadProcessedAsync("P", At(from), At(to), interval, agg, Ct)).Samples.ToList();
|
||||
var tw = AggregateFamilies.FamilyOf(agg) == AggregateFamily.TimeWeighted;
|
||||
if (tw && agg != HistoryAggregateType.Interpolative)
|
||||
{
|
||||
// Documented divergence: ours flags the truncated interval Partial, the SDK does not.
|
||||
var plan = BucketPlan.Create(start, end, interval);
|
||||
for (long i = 0; i < plan.BucketCount; i++)
|
||||
if (plan.IsTruncated(i))
|
||||
{
|
||||
var k = (int)(plan.Reversed ? plan.BucketCount - 1 - i : i);
|
||||
ours[k] = ours[k] with { StatusCode = ours[k].StatusCode & ~0x4u };
|
||||
}
|
||||
}
|
||||
var reverseInterpolative = agg == HistoryAggregateType.Interpolative && from > to;
|
||||
var sdk = SdkAggregateOracle.Run(agg, start, end, interval, rows);
|
||||
if (from > to && agg is HistoryAggregateType.TimeAverage or HistoryAggregateType.Total)
|
||||
{
|
||||
// Documented SDK divergence (#680, see SdkAggregateParityTests.MaskDocumented): on a reverse read the SDK files
|
||||
// a sample exactly on an interval's later edge as its early bound, so a TimeAverage/Total interval ending on a
|
||||
// sample takes the status of the segment beyond it (Dense: +20 next to the Bad +22.5). Only that interval,
|
||||
// only Good (ours) vs Uncertain (SDK), is aligned.
|
||||
var plan = BucketPlan.Create(start, end, interval);
|
||||
for (long i = 0; i < plan.BucketCount; i++)
|
||||
{
|
||||
var k = (int)(plan.BucketCount - 1 - i);
|
||||
if (k < sdk.Count && rows.Any(r => r.Item1.Ticks == plan.UpperTicks(i))
|
||||
&& (ours[k].StatusCode & 0xFFFF0000u) == 0
|
||||
&& (sdk[k].StatusCode & 0xFFFF0000u) == 0x40A40000u)
|
||||
ours[k] = ours[k] with { StatusCode = ours[k].StatusCode | 0x40A40000u };
|
||||
}
|
||||
}
|
||||
SdkAggregateOracle.ShouldMatch(ours, sdk,
|
||||
$"{agg} {from}→{to}/{intervalS}s",
|
||||
compareCalculatedInterpolatedBits: tw && !reverseInterpolative,
|
||||
compareCodeBits: !reverseInterpolative);
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -228,7 +479,8 @@ public abstract class ProcessedConformance : ConformanceBase
|
||||
avg.StatusCode.ShouldBe(UncertainCalc);
|
||||
(await One(HistoryAggregateType.Count)).Value.ShouldBe(2);
|
||||
(await One(HistoryAggregateType.Start)).Value.ShouldBe(10d);
|
||||
((double)(await One(HistoryAggregateType.PercentBad)).Value!).ShouldBe(33.333333333333336, 1e-9); // 100 × 1/3
|
||||
// #680: duration-based — [0,5) Good, [5,10) Bad, [10,20) Good (no end bound: the last region runs to the interval end).
|
||||
((double)(await One(HistoryAggregateType.PercentBad)).Value!).ShouldBe(25d, 1e-9);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -242,21 +494,158 @@ public abstract class ProcessedConformance : ConformanceBase
|
||||
(await src.ReadProcessedAsync("Z", At(0), At(20), Twenty, a, Ct)).Samples.Single();
|
||||
|
||||
(await One(HistoryAggregateType.Average)).StatusCode.ShouldBe(NoDataCalc);
|
||||
(await One(HistoryAggregateType.Count)).StatusCode.ShouldBe(NoDataCalc);
|
||||
// #680 (SDK value-based status): an interval holding only Bad samples counts as the generic Bad, value null.
|
||||
var count = await One(HistoryAggregateType.Count);
|
||||
count.StatusCode.ShouldBe(BadCalc);
|
||||
count.Value.ShouldBeNull();
|
||||
// Start is the raw first sample: the outage marker itself, with its own status and time.
|
||||
var start = await One(HistoryAggregateType.Start);
|
||||
start.StatusCode.ShouldBe(BadNoCommunication);
|
||||
start.SourceTimestampUtc.ShouldBe(B.Round(At(1)));
|
||||
start.Value.ShouldBeNull();
|
||||
var pb = await One(HistoryAggregateType.PercentBad);
|
||||
pb.Value.ShouldBe(100d);
|
||||
pb.StatusCode.ShouldBe(GoodCalc);
|
||||
// The tag's data starts (+1) inside the interval ⇒ Partial on the non-Bad percentage (#677).
|
||||
pb.StatusCode.ShouldBe(GoodCalc | 0x4u);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Uncertain_samples_are_used_but_make_the_interval_uncertain()
|
||||
public async Task Uncertain_samples_are_left_out_of_the_value_and_degrade_the_status_per_Part_13()
|
||||
{
|
||||
// #684 (Part 13 §5.4.3.5/10-14; the SDK's calculators): Average / Minimum / Maximum / Range / *ActualTime use GOOD
|
||||
// values only. One interval [0,20): Good 6 @+2, Good 8 @+3, Uncertain 4 @+1 (below the Good minimum) and
|
||||
// Uncertain 7 @+4 (inside the Good range); a Good sample before (-5) and after (+30) the window keep the data range
|
||||
// around it (no Partial). Pre-#684 the backends averaged 6.25 and answered Minimum 4.
|
||||
await B.SeedAsync("U",
|
||||
(At(-5), HistorianValue.FromDouble(100), Good),
|
||||
(At(1), HistorianValue.FromDouble(4), Uncertain),
|
||||
(At(2), HistorianValue.FromDouble(6), Good));
|
||||
var s = (await B.CreateDataSource().ReadProcessedAsync("U", At(0), At(20), Twenty, HistoryAggregateType.Average, Ct)).Samples.Single();
|
||||
s.Value.ShouldBe(5d);
|
||||
s.StatusCode.ShouldBe(UncertainCalc);
|
||||
(At(2), HistorianValue.FromDouble(6), Good),
|
||||
(At(3), HistorianValue.FromDouble(8), Good),
|
||||
(At(4), HistorianValue.FromDouble(7), Uncertain),
|
||||
(At(30), HistorianValue.FromDouble(100), Good));
|
||||
var src = B.CreateDataSource();
|
||||
async Task<DataValueSnapshot> One(HistoryAggregateType a) =>
|
||||
(await src.ReadProcessedAsync("U", At(0), At(20), Twenty, a, Ct)).Samples.Single();
|
||||
|
||||
// Average: value-based status — not every sample is Good ⇒ Uncertain.
|
||||
var avg = await One(HistoryAggregateType.Average);
|
||||
avg.Value.ShouldBe(7d);
|
||||
avg.StatusCode.ShouldBe(UncertainCalc);
|
||||
// Minimum: Uncertain 4 is below the Good minimum 6 ⇒ Uncertain (Part 13 Table 57), value still 6.
|
||||
var min = await One(HistoryAggregateType.Minimum);
|
||||
min.Value.ShouldBe(6d);
|
||||
min.StatusCode.ShouldBe(UncertainCalc);
|
||||
// Maximum: no Uncertain value above 8 and no Bad sample ⇒ Good.
|
||||
var max = await One(HistoryAggregateType.Maximum);
|
||||
max.Value.ShouldBe(8d);
|
||||
max.StatusCode.ShouldBe(GoodCalc);
|
||||
var range = await One(HistoryAggregateType.Range);
|
||||
range.Value.ShouldBe(2d);
|
||||
range.StatusCode.ShouldBe(UncertainCalc);
|
||||
var minAt = await One(HistoryAggregateType.MinimumActualTime);
|
||||
minAt.Value.ShouldBe(6d);
|
||||
minAt.SourceTimestampUtc.ShouldBe(B.Round(At(2)));
|
||||
minAt.StatusCode.ShouldBe(UncertainCalc);
|
||||
var maxAt = await One(HistoryAggregateType.MaximumActualTime);
|
||||
maxAt.Value.ShouldBe(8d);
|
||||
maxAt.SourceTimestampUtc.ShouldBe(B.Round(At(3)));
|
||||
maxAt.StatusCode.ShouldBe(GoodCalc);
|
||||
// Count (#680, unchanged): the two Good samples, Uncertain.
|
||||
(await One(HistoryAggregateType.Count)).Value.ShouldBe(2);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task An_uncertain_value_above_the_good_maximum_makes_the_maximum_family_uncertain()
|
||||
{
|
||||
// #684 review: pins the Uncertain-MAXIMUM column (UncMax / unc_max) through the real SQL — Good 6 @+2, Good 8 @+3,
|
||||
// Uncertain 9 @+4 (above the Good maximum, nothing below the minimum). Part 13 Tables 58/60/62 ⇒ Maximum,
|
||||
// MaximumActualTime and Range Uncertain (values still from the Good samples); Minimum Good.
|
||||
await B.SeedAsync("UA",
|
||||
(At(-5), HistorianValue.FromDouble(7), Good),
|
||||
(At(2), HistorianValue.FromDouble(6), Good),
|
||||
(At(3), HistorianValue.FromDouble(8), Good),
|
||||
(At(4), HistorianValue.FromDouble(9), Uncertain),
|
||||
(At(30), HistorianValue.FromDouble(7), Good));
|
||||
var src = B.CreateDataSource();
|
||||
async Task<DataValueSnapshot> One(HistoryAggregateType a) =>
|
||||
(await src.ReadProcessedAsync("UA", At(0), At(20), Twenty, a, Ct)).Samples.Single();
|
||||
|
||||
var max = await One(HistoryAggregateType.Maximum);
|
||||
max.Value.ShouldBe(8d);
|
||||
max.StatusCode.ShouldBe(UncertainCalc);
|
||||
var maxAt = await One(HistoryAggregateType.MaximumActualTime);
|
||||
maxAt.Value.ShouldBe(8d);
|
||||
maxAt.SourceTimestampUtc.ShouldBe(B.Round(At(3)));
|
||||
maxAt.StatusCode.ShouldBe(UncertainCalc);
|
||||
var range = await One(HistoryAggregateType.Range);
|
||||
range.Value.ShouldBe(2d);
|
||||
range.StatusCode.ShouldBe(UncertainCalc);
|
||||
var min = await One(HistoryAggregateType.Minimum);
|
||||
min.Value.ShouldBe(6d);
|
||||
min.StatusCode.ShouldBe(GoodCalc);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task An_uncertain_string_sample_does_not_refuse_the_good_value_aggregates()
|
||||
{
|
||||
// #684 review: the Good-value aggregates refuse only a GOOD String/DateTime sample (the good-non-numeric count the
|
||||
// SQL now returns); an Uncertain one is skipped with an Uncertain status — the SDK's behaviour. Delta keeps its
|
||||
// usable-sample refusal.
|
||||
await B.SeedAsync("US",
|
||||
(At(-5), HistorianValue.FromDouble(7), Good),
|
||||
(At(2), HistorianValue.FromDouble(6), Good),
|
||||
(At(3), HistorianValue.FromString("x"), Uncertain),
|
||||
(At(4), HistorianValue.FromDouble(8), Good),
|
||||
(At(30), HistorianValue.FromDouble(7), Good));
|
||||
var src = B.CreateDataSource();
|
||||
async Task<DataValueSnapshot> One(HistoryAggregateType a) =>
|
||||
(await src.ReadProcessedAsync("US", At(0), At(20), Twenty, a, Ct)).Samples.Single();
|
||||
|
||||
foreach (var (agg, value) in new[]
|
||||
{
|
||||
(HistoryAggregateType.Average, 7d), (HistoryAggregateType.Minimum, 6d), (HistoryAggregateType.Maximum, 8d),
|
||||
(HistoryAggregateType.Range, 2d), (HistoryAggregateType.MinimumActualTime, 6d),
|
||||
(HistoryAggregateType.MaximumActualTime, 8d),
|
||||
})
|
||||
{
|
||||
var v = await One(agg);
|
||||
v.Value.ShouldBe(value, agg.ToString());
|
||||
v.StatusCode.ShouldBe(UncertainCalc, agg.ToString());
|
||||
}
|
||||
(await One(HistoryAggregateType.Delta)).StatusCode.ShouldBe(InvalidInputsCalc);
|
||||
|
||||
// A GOOD string in the same shape still refuses them.
|
||||
await B.SeedAsync("GS",
|
||||
(At(-5), HistorianValue.FromDouble(7), Good),
|
||||
(At(2), HistorianValue.FromDouble(6), Good),
|
||||
(At(3), HistorianValue.FromString("x"), Good),
|
||||
(At(30), HistorianValue.FromDouble(7), Good));
|
||||
(await src.ReadProcessedAsync("GS", At(0), At(20), Twenty, HistoryAggregateType.Maximum, Ct)).Samples.Single()
|
||||
.StatusCode.ShouldBe(InvalidInputsCalc);
|
||||
(await src.ReadProcessedAsync("GS", At(0), At(20), Twenty, HistoryAggregateType.MinimumActualTime, Ct)).Samples
|
||||
.Single().StatusCode.ShouldBe(InvalidInputsCalc);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task An_interval_of_only_uncertain_samples_is_BadNoData_for_the_good_value_aggregates()
|
||||
{
|
||||
// #684: Part 13 defines these over Good values; the SDK returns no-data when an interval has none.
|
||||
await B.SeedAsync("UO",
|
||||
(At(-5), HistorianValue.FromDouble(1), Good),
|
||||
(At(1), HistorianValue.FromDouble(4), Uncertain),
|
||||
(At(2), HistorianValue.FromDouble(6), Uncertain),
|
||||
(At(30), HistorianValue.FromDouble(1), Good));
|
||||
var src = B.CreateDataSource();
|
||||
foreach (var agg in new[]
|
||||
{
|
||||
HistoryAggregateType.Average, HistoryAggregateType.Minimum, HistoryAggregateType.Maximum,
|
||||
HistoryAggregateType.Range, HistoryAggregateType.MinimumActualTime, HistoryAggregateType.MaximumActualTime,
|
||||
})
|
||||
{
|
||||
var v = (await src.ReadProcessedAsync("UO", At(0), At(20), Twenty, agg, Ct)).Samples.Single();
|
||||
v.Value.ShouldBeNull(agg.ToString());
|
||||
v.StatusCode.ShouldBe(NoDataCalc, agg.ToString());
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -455,6 +844,9 @@ public abstract class ProcessedConformance : ConformanceBase
|
||||
var ex = await Should.ThrowAsync<HistoryReadLimitExceededException>(
|
||||
() => under.ReadProcessedAsync("A", At(0), At(80), Twenty, HistoryAggregateType.Total, Ct));
|
||||
ex.Limit.ShouldBe(4);
|
||||
// #680: PercentGood/PercentBad are duration-based, served by the same window stream ⇒ capped too.
|
||||
(await Should.ThrowAsync<HistoryReadLimitExceededException>(
|
||||
() => under.ReadProcessedAsync("A", At(0), At(80), Twenty, HistoryAggregateType.PercentGood, Ct))).Limit.ShouldBe(4);
|
||||
// Pushed-down (per-sample) aggregates are not capped; the bound rows outside the window do not count.
|
||||
(await under.ReadProcessedAsync("A", At(0), At(80), Twenty, HistoryAggregateType.Average, Ct)).Samples.Count.ShouldBe(4);
|
||||
(await B.CreateDataSource(3).ReadProcessedAsync("W", At(0), At(80), Twenty, HistoryAggregateType.TimeAverage, Ct))
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user