Compare commits
139 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 5897764024 | |||
| 97668d26d3 | |||
| 66a2b0c3e9 | |||
| de8bc7a12c | |||
| 3c3513ea3e | |||
| a5a0496176 | |||
| 2609c5462e | |||
| 68a1f31397 | |||
| aa923cea83 | |||
| e04e1bf127 | |||
| 0d249f3ee9 | |||
| 3d7eea6280 | |||
| 8dfa165110 | |||
| 37390139fe | |||
| 15ccd9b722 | |||
| 942ca7bc14 | |||
| df0c1f3513 | |||
| ba46f4f47a | |||
| 934909b430 | |||
| ebc6b8d4d9 | |||
| f6681544e7 | |||
| 92aab7282c | |||
| 60febbf2dc | |||
| 88141d6ed5 | |||
| 9bb1363cd6 | |||
| 574c3aa1b0 | |||
| d28e3e64f1 | |||
| ccda27d220 | |||
| d277f34a96 | |||
| 186b7d5ea5 | |||
| d33992034d | |||
| 71d031fcd0 | |||
| f0d064a68c | |||
| 830b47f624 | |||
| 7b86059daf | |||
| ab5030174a | |||
| 0e47e887de | |||
| dbffb6bdb4 | |||
| 39552e15b9 | |||
| 1f5d310071 | |||
| a3ee917fb4 | |||
| 8114f13a28 | |||
| 6796ba4126 | |||
| 9e47532283 | |||
| 44a03bedc0 | |||
| 9ced6455ad | |||
| 2d03268975 | |||
| cf1af55720 | |||
| be2f0cc14d | |||
| 410c8d8976 | |||
| 70fb1f1fda | |||
| ce3369ebca | |||
| fe644d7eda | |||
| 4a3b12564a | |||
| 910befc84c | |||
| cc0cc8cd89 | |||
| 549d2d68b3 | |||
| 197246c3c1 | |||
| 9d5eef658f | |||
| 5f225f0b44 | |||
| 78f604e0cd | |||
| dbdf29c8fc | |||
| c0d6bacd9c | |||
| a11ab0dc08 | |||
| 113f1eaf7c | |||
| c019d0fc2b | |||
| ede8610804 | |||
| 6ea3ba44da | |||
| 8ebee2ac40 | |||
| d266ff83f4 | |||
| 2ceed561ad | |||
| 11f792538e | |||
| b5e3e5108c | |||
| b8344bb151 | |||
| dbfe2fc4de | |||
| f6a2628656 | |||
| 0d206a0884 | |||
| b61975f7d7 | |||
| 19a4c093be | |||
| fb5ec80fe6 | |||
| 6d5c1c768b | |||
| beb1c583b1 | |||
| 945ec2c069 | |||
| 7349260231 | |||
| 50c09e21d5 | |||
| 27bf4826d6 | |||
| 029a916288 | |||
| 232c03ed1b | |||
| 7594120a55 | |||
| 93d6c1da67 | |||
| 30a9d5f094 | |||
| fccffd0a0f | |||
| 9120182314 | |||
| 27ca6cd791 | |||
| 8e6ac4b344 | |||
| e59ab69398 | |||
| 8d154c6a81 | |||
| 0658d244d0 | |||
| 69190b14f1 | |||
| 1375f9349e | |||
| 045a21da14 | |||
| f292cf3beb | |||
| 3067146be3 | |||
| fe919f8b08 | |||
| f0b409f999 | |||
| f9ab2f7293 | |||
| 1e343b0c52 | |||
| 42eb96af9e | |||
| f41df6710c | |||
| 825393299e | |||
| bc03adb1fd | |||
| 099bc1b6e0 | |||
| 57deaeba16 | |||
| 7c80d36eb6 | |||
| 062086f9a6 | |||
| cfd403710e | |||
| 699871040a | |||
| 87e8aff786 | |||
| 984e069f5d | |||
| e583b068a1 | |||
| 99c13e4459 | |||
| 3b739b982b | |||
| 56bb4855bb | |||
| 1f88669304 | |||
| db6446ce73 | |||
| b028322568 | |||
| d803e0eab4 | |||
| 50b504d293 | |||
| acce99de9a | |||
| 91434e7f61 | |||
| b3db64af6c | |||
| e4123ada91 | |||
| 2a8f9af5e4 | |||
| 1b2c36eb87 | |||
| 924eff8d3c | |||
| f9fd495292 | |||
| 7d75478f10 | |||
| e5d12d391a | |||
| db6a53036b |
@@ -302,7 +302,9 @@ non-`Healthy` state **and** its `LastSuccessfulRead` has been frozen for 90 s. T
|
||||
success. Each conjunct has a test that goes red when the *other* is deleted.
|
||||
- **The mechanism rests on every driver reaching a non-Healthy state on sustained failure.** Surveyed across
|
||||
all 12: nine poll drivers via `Degraded`, MQTT via live connection state, Galaxy via its reconnect
|
||||
supervisor. `CalculationDriver` reports `Healthy` while running and so can never trip — correct, as it has
|
||||
supervisor, OpcUaClient via a bad session keep-alive (#670 — it used to stay `Healthy` forever; its SDK
|
||||
`SessionReconnectHandler` is deleted, a lost session publishes `BadNoCommunication` once per outage and the
|
||||
watchdog's re-init is the recovery; good keep-alives + notifications advance `LastSuccessfulRead`). `CalculationDriver` reports `Healthy` while running and so can never trip — correct, as it has
|
||||
no transport. Re-check this if a new driver lands.
|
||||
- **The reconnect retry MUST tear down before re-initialising — FIXED in both drivers, keep it that way.**
|
||||
`ModbusDriver` (`_transport`) and `MqttDriver` (`_connection`) used to overwrite an `IAsyncDisposable`
|
||||
@@ -319,6 +321,13 @@ non-`Healthy` state **and** its `LastSuccessfulRead` has been frozen for 90 s. T
|
||||
`DefaultReconnectInterval`. Unlike the #523 drift detector this generates **no** traffic on a healthy
|
||||
system — it is a local `GetHealth()` read, and only reaches the network once a driver has already stopped
|
||||
reading.
|
||||
- **Fast path for DEFINITIVE losses (#676):** a driver implementing the opt-in `IDriverConnectionLossSource` raises
|
||||
`ConnectionLost` (once per connection) when its protocol proves the connection can never recover in place — OpcUaClient:
|
||||
keep-alive `BadSessionIdInvalid`/`BadSessionClosed`/`BadSessionNotActivated`/`BadServerHalted`, a SUSTAINED closed
|
||||
connection (2nd `BadSecureChannelClosed`/`BadConnectionClosed` in one outage — what a real process stop produces; the
|
||||
SDK never reopens it) or a subscription the SERVER reported dropped (its publishing-stall heuristic does not raise) — and `DriverInstanceActor` (attach PreStart, detach PostStop) turns it into the same disconnect only
|
||||
while Connected; ambiguous failures (timeouts, `BadNoCommunication`, a stalled socket) must never raise and stay with
|
||||
the watchdog's conjunction. `docs/DriverLifecycle.md`.
|
||||
|
||||
- **What comms loss now does downstream — one contract, six seams (archreview R3-02).** "A value consumer
|
||||
never reports a quality better than its source's" is stated once in `docs/DriverLifecycle.md` §"The
|
||||
@@ -1341,18 +1350,23 @@ historian tagname, which is the tag's driver `FullName`.
|
||||
### Pluggable backends, SQL Server and HistoryUpdate (2026-10)
|
||||
|
||||
- **Selection:** `ServerHistorian:Backend` (`Gateway` default | `SqlServer` | `QuestDb`; unknown fails host start). `HistorianBackendRegistry` → `IHistorianBackendProvider` builds all five seams; the gateway keys are only required for `Gateway`. `SqlServer` uses `ServerHistorian:SqlServer:{ConnectionString (required), Schema=hist, CommandTimeout=30s, AutoCreateSchema=true}`.
|
||||
- **SQL Server** (`Driver.Historian.SqlServer`): `hist.Sample` PK `(TagId, SourceTimeUtc)` — writes are insert-if-absent, so outbox redelivery is idempotent. Stores every value kind with its **real status**; aggregates (13, simple per-sample definitions, not Part 13) are pushed down to T-SQL with bucket math in **100 ns ticks**. Docker-dev recipe, schema and aggregate semantics: `docs/Historian.md` §"SQL Server backend". Integration tests need `HISTORIAN_SQL_TEST_CONNECTION`.
|
||||
- **SQL Server** (`Driver.Historian.SqlServer`): `hist.Sample` PK `(TagId, SourceTimeUtc)` — writes are insert-if-absent, so outbox redelivery is idempotent. Stores every value kind with its **real status**; per-sample aggregates (10, simple definitions) are pushed down to T-SQL with bucket math in **100 ns ticks**; the Part 13 time-weighted seven (TimeAverage/TimeAverage2/Total/Total2/Interpolative, #671 — ⚠️ `Total` is now the time-weighted integral, no longer `SUM` — plus the **duration-based** PercentGood/PercentBad, #680) stream window + 4 bound rows into the shared `TimeWeightedAggregator` (both SQL backends, identical results; reverse `Interpolative` anchored at the later time; window capped by `ServerHistorian:MaxTimeWeightedRawRows` ⇒ `BadTooManyOperations`, percentages included). ⚠️ **#680 status rules (client-visible):** TimeAverage/Total are Uncertain if any part of the interval uses an Uncertain value, is bridged across a Bad sample, is extrapolated past the last sample (⇒ a live tag's latest interval is Uncertain) or is not covered; TimeAverage2/Total2 keep the SDK time-based rule (Uncertain time counts as Good) but never extrapolate past the last sample — that interval is Partial, later ones `BadNoData` (a report-by-exception tag with nothing after its last sample reads `BadNoData` until its next sample arrives). `docs/Historian.md` §"Time-weighted aggregates". ⚠️ **#680 per-sample (client-visible):** `Count` counts **Good** samples only (value-based status; an all-Bad interval is generic Bad, value null) and an empty interval **inside the tag's data range** is `0`/Good (outside it still `BadNoData`); `Start`/`End` return the **raw** first/last sample of any quality — its own timestamp and status, value null when Bad; `Delta` is Uncertain only when the interval's first or last sample is unusable; ⚠️ #684: `Average`/`Minimum`/`Maximum`/`Range`/`*ActualTime` use **Good samples only** (was Good + Uncertain; an Uncertain-only interval is `BadNoData`), Min/Max-family status Uncertain on a Bad sample, an Uncertain value beyond the Good extreme (Part 13 — the SDK ignores that) or an Uncertain non-numeric sample, and only a **Good** String/DateTime sample refuses them (`BadAggregateInvalidInputs`; Delta still refuses any usable one); and every per-sample aggregate but `Average` sets **end-of-data Partial** on the interval holding the last sample (forward incl. its earlier edge, reverse strictly inside), never on a Bad result. The data-range lookup is a key seek on SQL Server and a `tag` symbol-index seek on QuestDB (#683 — `{p}sample.tag` is `SYMBOL INDEX`; the bootstrapper ADDs it to a pre-#683 table, idempotent), read BEFORE the bucket statement so a racing append can never yield a Good `Count` of 0 (a concurrent HistoryUpdate delete is the mirror case and still can, transiently). `docs/Historian.md` §"Aggregates". Docker-dev recipe, schema and aggregate semantics: `docs/Historian.md` §"SQL Server backend". Integration tests need `HISTORIAN_SQL_TEST_CONNECTION`. **Retention (#667):** opt-in `SqlServer:RetentionDays` (+ `PurgeInterval` 1h, `PurgeBatchSize` 4000) -> `HistorianRetentionService` purges old `hist.Sample` (per tag) / `hist.Event` in `DELETE TOP(n)` batches, Primary-gated on `IHistorizationRoleView`, one purger per DB via `sp_getapplock`; counter `otopcua.historian.retention.purged{table}`. Optional `IHistorianBackendProvider.CreateRetention` (default null). `docs/Historian.md` §Retention.
|
||||
- **QuestDB** (`Driver.Historian.QuestDb`, `ServerHistorian:QuestDb:*`, Npgsql over PG wire; docker-dev overlay `docker-compose.questdb.yml`): same kinds/status/aggregates/HistoryUpdate as SqlServer, soft delete by tombstone (INSERT only — never `UPDATE`/`DELETE`; every read filters `NOT deleted`). ⚠️ **Timestamps are µs** (`QuestDbConnection.Round`/`BindTimestamp`; `IHistorianDataSource.TimestampResolutionTicks` = 10 steps HistoryRead paging), **pre-1970 cannot be stored** (writer drops, HistoryUpdate `BadOutOfRange`), every write waits for WAL visibility, TTL is enforced on write commits only, set Npgsql `Maximum Pool Size` < QuestDB's 64. `docs/Historian.md` §"QuestDB backend".
|
||||
- ⚠️ **Reverse processed reads are Part 13/SDK-anchored (#677, client-visible):** every aggregate on SqlServer/QuestDB now anchors a reverse read at its later time — window `(end, start]`, partial interval earliest, each stamped with its later edge (was: earlier-anchored, stamped earlier edge); on forward reads Partial also marks the interval the data starts inside (not Average — the SDK's rule, forward-only); a non-default client `AggregateConfiguration` ⇒ `BadAggregateConfigurationRejected`; Gateway provisioning sends `IntegralDivisor = 1`. SDK-parity tests: `SdkAggregateParityTests`. `docs/Historian.md` §Aggregates.
|
||||
- ⚠️ **Reverse / end-only RAW reads work on Gateway (#686, client-visible):** `IHistorianDataSource.SupportsReverseRaw` (SqlServer/QuestDB/Null `true`, Gateway `false`, interface default `false`, guard-tested) — the node manager sends `start > end` only to a backend that declares it; otherwise `ReverseRawReader` serves it from FORWARD reads: uncapped ⇒ ONE forward read reversed in memory; capped ⇒ a bounded walk back in segments (ms-aligned request bounds, width aimed from observed density at ≈ N + 8 rows and carried page to page in the continuation point — #689: it used to aim at half `MaxTieClusterOverfetch`, ~32k rows per page on a dense tag; a read asks for `2N + 18` rows (a discovery segment ≤ 1,026), `MaxTieClusterOverfetch` is only the tie-cluster ceiling — a full read is re-read, never answered truncated; only more than max(1, `MaxTieClusterOverfetch`) samples within ONE MILLISECOND ⇒ `BadTooManyOperations` (per ms, not per timestamp — requests are ms-rounded, and letting a segment narrow below 1 ms stepped a page through one ms a tick per read, ~10k reads); walks no later than now + 1 day and never before the Unix epoch; #691: after two empty segments it asks once for the whole remainder, and the stretch a page proved empty rides the continuation point — the last pages of a deep page-through no longer each walk ~14 reads to 1970). The raw/events tie-path oversized-cluster refusal is now `BadTooManyOperations` too (was `BadHistoryOperationUnsupported`). Uncapped Gateway raw reads (forward too) used to send `max_values = 0`, which the client/gateway reject — now unbounded (`int.MaxValue`; a ceiling refusal ⇒ `BadTooManyOperations`), and a cancelled read is no longer a backend failure. Since #669 such reads failed the node on Gateway and counted as a backend fault (the gateway rejects start > end — `GatewayHistorianDataSource.ReadRawAsync` now refuses one unsent); before #669 an end-only Gateway read returned the OLDEST N ascending — now the latest N at/before EndTime, newest first. Events follow since #691.4: a reverse/end-only capped events read on Gateway (`SupportsReverseEvents` false; Null now reports `true` and answers empty) is walked back the same way (ties sorted by EventId — the gateway orders by time only; one extra event found ⇒ continuation point) — it used to be refused `BadHistoryOperationUnsupported` (explicit reverse) or answered with the OLDEST N ascending (end-only); needs the gateway's `RuntimeDb:EventReadMaxRows` 0/unlimited (it truncates silently). Uncapped Gateway events reads (forward too) used to throw "Array dimensions exceeded" (result list pre-sized to `int.MaxValue`) — fixed. Above a truncated read the walk gallops up from the newest row seen, then bisects (never below 1 ms): a burst that ended 100 days ago ≈ 18 reads, a dense tag that stopped days ago ≈ 26. `docs/Historian.md` §"Open-ended and reverse raw reads".
|
||||
- **Advertised aggregates (#682):** `IHistorianDataSource.SupportedAggregates` (Gateway 4 via `GatewayHistorianDataSource.ServedAggregates`, SqlServer/QuestDB 17 via `AggregateFamilies.Served`, Null **none**; a null set ⇒ all) is the one authority for `HistoryServerCapabilities/AggregateFunctions` (i=11201 — only its folder references are trimmed) and the up-front `BadAggregateNotSupported` in HistoryRead-Processed. ⚠️ `ServerCapabilities/AggregateFunctions` (i=2997) is deliberately NOT trimmed: it keeps the SDK's 37 because live `AggregateFilter` subscriptions still compute them all. ⚠️ Historian disabled ⇒ Processed is now `BadAggregateNotSupported`, not `GoodNoData`. `docs/Historian.md` §Advertised aggregates.
|
||||
- **Raw `ReturnBounds` is honoured on every backend (#691, client-visible):** Part 11 bounding values (§4.6 Table 1 is the test oracle) are composed in the server by `HistoryBounds` from each backend's raw reads — "at or before" = capped reverse read (native on SQL backends, the #686 `ReverseRawReader` walk-back on Gateway, never AVEVA's synthetic boundary row), "at or after" = capped forward read. Bounds are stored samples of any quality, count toward `NumValuesPerNode`, start bound on the first page / end bound on the last (pending end bound rides the continuation point); missing ⇒ `Bad_BoundNotFound` (null, at the bound time; a one-sided read's open side ±1 s from the previous value). ⚠️ **Breaking for SDK-default clients:** the .NET SDK's `new ReadRawModifiedDetails()` defaults BOTH `ReturnBounds = true` AND `IsReadModified = true` — a client that cleared only `IsReadModified` now gets bounds, and one that set neither is a modified read with bounds ⇒ `Bad_InvalidArgument` (§6.5.3.3); set both explicitly (Client.Shared does; tests building details must too). No data AND no bound found ⇒ `GoodNoData`, no values (the pre-bounds answer). On Gateway the at-or-after lookup ends at now + 1 day (never a year-9999 end to AVEVA). Kill switch `ServerHistorian:HonorReturnBounds` (default `true`; `false` = pre-#691). `docs/Historian.md` §Bounding values.
|
||||
- **Real status fix:** the recorder used to stamp every sample Good (`192`); it now carries the driver's status. ⚠️ On the **Gateway** backend this changes nothing stored: the gateway discards quality (C-002), so AVEVA still stores Good.
|
||||
- **Comms loss is historized as an outage marker** (`Empty` + `BadNoCommunication`, one per tag per outage) on SqlServer; dropped (`DroppedUnsupportedKind`) on Gateway. Outbox records are **format v2** — drain or delete `ContinuousHistorization:OutboxPath` before downgrading a node.
|
||||
- **HistoryUpdate** (SqlServer only; Gateway ⇒ `BadHistoryOperationUnsupported`): Insert/Replace/Update/DeleteRaw/DeleteAtTime, gated in both ACL modes — flag off needs the `HistoryUpdate` role from a **Cluster-scoped** grant (+ key re-check), flag on uses `DataPlaneAuthorizer`'s HistoryUpdate operation; `WriteOperate` never implies it. The `HistoryWrite` AccessLevel bit is set only when an updater is wired at materialise time. `docs/Historian.md` §HistoryUpdate, `docs/security.md`.
|
||||
- **HistoryUpdate** (SqlServer + QuestDb; Gateway ⇒ `BadHistoryOperationUnsupported`): Insert/Replace/Update/DeleteRaw/DeleteAtTime, **Primary-only** since #674 (off-Primary ⇒ `BadInvalidState`, after authorization AND after planning — an unservable request (Remove / `IsDeleteModified` / not historized) stays `BadHistoryOperationUnsupported` off-Primary; the verdict is `IHistoryUpdateRoleView` = the data-plane verdict but **closed until this node's first redundancy snapshot**, so a pair cannot both accept in the boot window; admin-only nodes register no view ⇒ no gate; reads ungated), items with a `SourceTimestamp` past now + `ServerHistorian:HistoryUpdateMaxFutureSkew` (default 1 day, ≤0 disables) ⇒ `BadOutOfRange` in the node manager (every backend), gated in both ACL modes — flag off needs the `HistoryUpdate` role from a **Cluster-scoped** grant (+ key re-check), flag on uses `DataPlaneAuthorizer`'s HistoryUpdate operation; `WriteOperate` never implies it. The `HistoryWrite` AccessLevel bit is set only when an updater is wired at materialise time. `docs/Historian.md` §HistoryUpdate, `docs/security.md`. ⚠️ **SqlServer schema v1 is accepted (#674)** with `AutoCreateSchema=false`: writes/reads/events/retention work; modified reads + HistoryUpdate ⇒ node-level `BadHistoryOperationUnsupported` on every arm (the updater throws `NotSupportedException`, nothing sent; the `HistoryWrite` bit is still advertised — not knowable at materialise time), retention skips `SampleModified`, one Warning per boot (`SqlServerSchemaBootstrapper.SupportsModifiedTrail`; `RequiredVersion` = 1, `ModifiedTrailVersion` = 2). QuestDB: keep `cairo.ttl.use.wall.clock=true`; a QuestDB trail INSERT retried after a client timeout duplicates (no DEDUP key).
|
||||
- **Modified-value history (#668, Part 11):** every applied HistoryUpdate item leaves a trail row (prior value, principal from the session's API key / `anonymous`, `HistoryModificationType`, time) — SqlServer `hist.SampleModified` (**schema v2**, written by `OUTPUT … INTO` in the same statement ⇒ atomic; ⚠️ no FK/CHECK/trigger on it, and `AutoCreateSchema=false` installs must run `002_sample_modified.sql` before upgrading), QuestDB `{p}sample_modified` (append-only, written **before** the change ⇒ a crash leaves an orphan trail row, never an untracked change). `IsReadModified=true` HistoryRead serves it as `HistoryModifiedData` via `IHistorianDataSource.ReadModifiedAsync` (default ⇒ `NotSupported` ⇒ `BadHistoryOperationUnsupported` on Gateway/Null), paged on `(SourceTime, ModifiedId)`, gated by plain HistoryRead. Live writes and retention never write it; retention/TTL purge it on source time. `IHistorianDataUpdater` methods take a `HistoryUpdateContext`. CLI `historyread --modified`. `docs/Historian.md` §Modified-value history.
|
||||
|
||||
### Read path (`ServerHistorian` section)
|
||||
|
||||
The server dispatches all OPC UA HistoryRead to the registered `IHistorianDataSource` — the
|
||||
`GatewayHistorianDataSource` read client when enabled, else the `NullHistorianDataSource` default
|
||||
(historized nodes return `GoodNoData`, never an error). Supported variants: Raw, Processed
|
||||
(Average/Minimum/Maximum/Total/Count aggregates), and AtTime over historized variable nodes; Events over
|
||||
(historized nodes return `GoodNoData` for Raw/AtTime — Processed is `BadAggregateNotSupported` since #682). Supported variants: Raw (⚠️ #687, client-visible: Gateway raw reads no longer return AVEVA's synthetic start-boundary row — asked from 1 ms early (epoch-floored) with `NumValuesPerNode + 16` rows (unclamped; 0 ⇒ `int.MaxValue`) and filtered by timestamp — so an empty window is `GoodNoData`, not a fabricated value at StartTime; the forward end is now exclusive (a point read inclusive); pre-window rows that crowd the budget trigger ONE re-read with room for them, else `BadTooManyOperations`; `ReturnBounds` is ignored on every backend; ⚠️ #690: a Gateway Bad row — a stored NULL, which histsdk sends as value 0 — now reads value **null** + Bad on Raw and AtTime, as on the SQL backends; ⚠️ #691.3: a strict gateway (`AllowStreamingBeyondCeiling=false`) that refuses a raw budget is re-asked ONCE at the ceiling its refusal names — raw and per-sample reads alike; the ceiling is then remembered per data source (`GatewayRowCeiling`: refusals count as gateway errors), re-learned from a refusal when lowered, and re-probed once at the full budget when a ceiling-bounded read comes back full (raised / streaming) — so an uncapped (`NumValuesPerNode = 0`) read of a small window is served; a ceiling-bounded stream that is not provably complete is `BadTooManyOperations`, never a truncated answer), Processed
|
||||
(backend-dependent: SqlServer/QuestDb serve seventeen; Gateway twelve = `GatewayHistorianDataSource.ServedAggregates` — ⚠️ #681, client-visible: the ten per-sample ones (Average, Count, Min/Max, Range, Start/End/Delta, Min/MaxActualTime) are evaluated from RAW gateway reads by the shared `Sql.Common` evaluator — identical to the SQL backends, Calculated bits, interval-start stamps, #677 reverse; Min/Max are no longer AVEVA modes, the read moves every raw sample of the window (a month of 1 Hz ⇒ `BadTooManyOperations` at the default `MaxTimeWeightedRawRows` cap), and every gateway stream is consumed to completion (an abandoned one makes the gateway evict its shared historian session) — plus TimeAverage/Total via AVEVA modes (interval-start stamped + Calculated; ⚠️ #685: AVEVA's extra pre-window row is dropped ⇒ one row per interval in `[Start, End)`, and an Uncertain (opc 64) interval keeps its value as `UncertainDataSubNormal` — AVEVA's TWA excludes NULL-gap time where the SQL backends bridge it; per-interval values still need gateway fix historiangw#27; ⚠️ #691.2: native reads use the SQL backends' grid in both directions via forward requests only — a REVERSE read (refused by the gateway before) is anchored at its later time (#677), and the clipped interval (earliest reverse, LATEST forward — which used to be averaged past EndTime) is its own exact-span read marked Partial unless Bad; an interval wider than the window is sent as the window, interval 0 as one interval; more than 100,000 intervals is refused up front (`BadHistoryOperationUnsupported`, as on the SQL backends — forward used to be served up to the gateway's 1,000,000); until historiangw#27 lands both directions return Good-status wrong values); the other time-weighted five stay `BadAggregateNotSupported` because the gateway has no prior-sample read for Part 13 bounds; `docs/Historian.md` §Aggregate set), and AtTime over historized variable nodes; Events over
|
||||
alarm-owning equipment-folder event-notifier nodes. Reads are ungated (served from any redundancy node);
|
||||
authorization uses the standard `AccessLevels.HistoryRead` bit set at materialization.
|
||||
|
||||
@@ -1368,8 +1382,10 @@ authorization uses the standard `AccessLevels.HistoryRead` bit set at materializ
|
||||
| `CaCertificatePath` | `null` | PEM CA file pinning the gateway TLS chain; null/empty uses the OS trust store |
|
||||
| `CallTimeout` | `00:00:30` | Per-call deadline for each unary gateway read |
|
||||
| `MaxTieClusterOverfetch` | `65536` | Bounded over-fetch the HistoryRead-Raw paging uses to page within an oversized same-timestamp tie cluster (retained from the prior backend) |
|
||||
| `HonorReturnBounds` | `true` | #691 kill switch: `false` ignores HistoryRead-Raw `ReturnBounds` as before (no bounding values, no modified-read `Bad_InvalidArgument`) |
|
||||
| `HistoryReadBatchParallelism` | `4` | Max nodes served concurrently within one HistoryRead batch (03/S3); `⌈N/P⌉ × CallTimeout` worst case instead of sequential |
|
||||
| `MaxConcurrentHistoryReadBatches` | `16` | Process-wide concurrent HistoryRead batch cap (03/S3); flood degrades to `BadTooManyOperations`. Total in-flight reads capped at 16 × 4 = 64 |
|
||||
| `MaxTimeWeightedRawRows` | `1000000` | `SqlServer`/`QuestDb` (#671): raw in-window rows one time-weighted aggregate read (incl. `Total` and, since #680, PercentGood/PercentBad) may stream into the C# evaluator — and, since #681, every `Gateway` per-sample read (raw-evaluated); window query capped at this + 1, past it ⇒ `BadTooManyOperations` for that node. ≤ 0 = unbounded. Gateway (#688): the `cap + 18` budget is clamped at the gateway's default `MaxValuesCeiling` (1,000,000) while the cap is ≤ it, so the default no longer trips a strict (`AllowStreamingBeyondCeiling=false`) gateway |
|
||||
| `HistoryReadDeadline` | `00:01:00` | Per-request HistoryRead wall-clock deadline (03/S3); a node still in flight surfaces `BadTimeout`. Non-positive = unbounded |
|
||||
|
||||
### Alarm-history path (`AlarmHistorian` section)
|
||||
@@ -1408,6 +1424,7 @@ only the recorder + outbox knobs:
|
||||
| `MaxWriteAttempts` | `5` | #538: consecutive failed drain passes one tag may accumulate — counted **only across passes where other tags wrote successfully** — before its queued entries are abandoned so the FIFO head can advance |
|
||||
| `MinBackoffSeconds` | `1` | Initial retry backoff after a failed drain pass |
|
||||
| `MaxBackoffSeconds` | `30` | Cap on the exponential retry backoff |
|
||||
| `CaptureWhilePrimaryUnreachable` | `true` | #672: a standby stood down for its mesh Primary (#537) captures as soon as that Primary is reported UNREACHABLE, instead of after auto-down — crash-failover gap ~30 s ⇒ ~10–12 s, at the cost of duplicate windows (see #537/#672 below). `false` = pre-#672 |
|
||||
|
||||
### Tag auto-provisioning (`IHistorianProvisioning` EnsureTags hook)
|
||||
|
||||
@@ -1440,6 +1457,9 @@ The target HistorianGateway OtOpcUa points at **must** run with:
|
||||
|
||||
- `RuntimeDb:Enabled=true` — enables the `WriteLiveValues` SQL live path (continuous historization).
|
||||
- `RuntimeDb:EventReadsEnabled=true` — enables `ReadEvents` from `Runtime.dbo.Events` (alarm-history reads).
|
||||
- `RuntimeDb:EventReadMaxRows=0` (its default, unlimited) — or larger than the reverse-events walk's biggest read
|
||||
(`2·MaxTieClusterOverfetch + 4`). The cap truncates **silently**, so a walked-back reverse events read (#691.4) would
|
||||
lose the newest events of a segment it cut.
|
||||
- An API key carrying scopes **`historian:read`**, **`historian:write`**, **`historian:tags:write`**.
|
||||
|
||||
### Migration note (deployments upgrading from the Wonderware backend)
|
||||
@@ -1547,6 +1567,21 @@ tag); evidence in the same plan doc. Two things to know before touching either.*
|
||||
`RedundancyRoleView` behind **`IHistorizationRoleView`**, registered unconditionally on driver nodes
|
||||
(NOT alongside the alarm `IRedundancyRoleView`, which only exists when that section is enabled — the
|
||||
#511 registration accident).
|
||||
- **#672 — an UNREACHABLE recording Primary does not count (crash-failover gap).** A SIGKILLed Primary
|
||||
stays a member (unreachable, not downed) for 10 s failure detection + 15 s auto-down, and the
|
||||
redundancy singleton died with it, so the standby's cached snapshot kept naming it Primary and
|
||||
*nobody* captured for ~30 s. `DriverHostActor` now holds its **own** `ReachabilityEvent` (+`MemberUp`)
|
||||
subscription — not inside `StartPrimaryVerdictPublishing`, which early-returns with no primary-role
|
||||
source — handled in **all three** behaviours, caches the last snapshot, and on a reachability change
|
||||
re-derives and republishes **only** the historization verdict with unreachable peers excluded. Gap ⇒ the
|
||||
failure-detector window (~10–12 s). Gated by `ContinuousHistorization:CaptureWhilePrimaryUnreachable`
|
||||
(default `true`). ⚠️ **The alarm drain and `ShouldServiceAsPrimary`/`PublishPrimaryVerdict` must NOT
|
||||
adopt this** (a false allow there is a dual claimant, not a duplicate) — the tests assert both stay
|
||||
put. ⚠️ A **removed** member deliberately stays marked unreachable (only `MemberUp` at that address
|
||||
clears it): un-marking it would re-derive from a cached snapshot still naming it Primary and close the
|
||||
gate on a dead node. Cost: duplicates while a live Primary is merely unreachable and under a real
|
||||
partition — idempotent on SqlServer/QuestDB only for source-timestamped drivers (Modbus stamps each
|
||||
node's own poll time ⇒ duplicate rows on every backend); Gateway stores duplicates.
|
||||
|
||||
Re-confirmed live in 2026-06 and **since fixed (#600.1)**: `EnsureTags` provisioning used to fire only for
|
||||
newly-ADDED historized tags, so flipping an existing tag's `historianTagname` (or historization, or DataType)
|
||||
|
||||
@@ -207,7 +207,9 @@ The `-v` drops the SQL volume; remove it to keep ConfigDb state across restarts.
|
||||
|
||||
The full single-mesh stack (`central-1`/`central-2` + the four site nodes) can OOM-kill `central-1` on a loaded host. Two settings in the compose file guard against that:
|
||||
|
||||
- **EF Core + ASP.NET Core logs are pinned to `Warning`** on every host node (`Serilog__MinimumLevel__Override__Microsoft.EntityFrameworkCore` / `…Microsoft.AspNetCore` = `Warning`). The host logs via Serilog (`AddZbSerilog` → `ReadFrom.Configuration`), and in `Development` the default level is `Debug` — without these overrides every Deployment-poll emits an `Executed DbCommand` / `SELECT … FROM [Deployment]` line, flooding the Serilog pipeline and starving the Akka cluster heartbeat thread. Application + Akka log levels are left untouched, so this only silences the per-poll SQL chatter. To temporarily restore the SQL log flood for debugging, drop those two env vars (or set them back to `Information`) on the node you're inspecting.
|
||||
- **EF Core + ASP.NET Core logs are pinned to `Warning`** on every host node (`Serilog__MinimumLevel__Override__Microsoft.EntityFrameworkCore` / `…Microsoft.AspNetCore` = `Warning`). The host logs via Serilog (`AddZbSerilog` → `ReadFrom.Configuration`), and a `Development` environment would default to `Debug` (`appsettings.Development.json`) — the rig does NOT set `ASPNETCORE_ENVIRONMENT`, so nodes actually run at the base `Information` default, but these pins keep the SQL chatter out should the rig ever run as Development; without them a Debug/Information-level EF pipeline emits an `Executed DbCommand` / `SELECT … FROM [Deployment]` line, flooding the Serilog pipeline and starving the Akka cluster heartbeat thread. Application + Akka log levels are left untouched, so this only silences the per-poll SQL chatter. To temporarily restore the SQL log flood for debugging, drop those two env vars (or set them back to `Information`) on the node you're inspecting.
|
||||
|
||||
To raise Debug logging for chosen categories on `central-1`/`central-2` (e.g. `ZB.MOM.WW.OtOpcUa.Runtime`), use the overlay: `docker compose -f docker-dev/docker-compose.yml -f docker-dev/docker-compose.debug.yml up -d --no-deps central-1 central-2` — it sets `Serilog__MinimumLevel__Override__<Namespace>=Debug`; edit its category list to taste.
|
||||
- **Each host node has `mem_limit: 1g`** (`mem_reservation: 512m`). A quiet solo `central-1` measures ~357 MiB; the limit leaves headroom for the deploy/UI load and per-cluster driver subscriptions that push a fully-loaded node higher. The limit/reservation live on the `&otopcua-host` anchor, so all six host services inherit them; `sql`, `traefik`, and the one-shot `migrator`/`cluster-seed` are left unbounded.
|
||||
|
||||
The full six-node host stack therefore needs roughly **6 GiB** of Docker Desktop VM memory just for the host nodes (plus SQL Server's own footprint on top). On a constrained host, either raise the Docker Desktop VM memory or run fewer host services (e.g. just `central-1` + `central-2`, or a single central node) rather than the full mesh.
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
# docker-dev/docker-compose.debug.yml — optional overlay: raise Serilog levels per category on the central pair.
|
||||
#
|
||||
# The rig nodes run with ASPNETCORE_ENVIRONMENT unset (Production), so there is NO appsettings.Development.json and the
|
||||
# Serilog default level is Information (the base appsettings.json sets none). Debug is therefore OFF by default; this
|
||||
# overlay turns it on for chosen namespaces only, via Serilog:MinimumLevel:Override:<Namespace> (env: double underscore
|
||||
# for ':', and the namespace's own dots stay dots).
|
||||
#
|
||||
# Up (recreates only the two central nodes with the extra env):
|
||||
# docker compose -f docker-dev/docker-compose.yml -f docker-dev/docker-compose.debug.yml up -d --no-deps central-1 central-2
|
||||
# Back to normal: re-run `up -d --no-deps central-1 central-2` with the base file alone.
|
||||
#
|
||||
# Edit the list below to the categories you care about. Useful ones:
|
||||
# ZB.MOM.WW.OtOpcUa.Runtime actors: DriverHostActor, OpcUaPublishActor, redundancy, deploys
|
||||
# ZB.MOM.WW.OtOpcUa.Runtime.Drivers.DriverHostActor one actor only (e.g. the #672 historization verdict logs)
|
||||
# ZB.MOM.WW.OtOpcUa.Driver.OpcUaClient a single driver
|
||||
# Opc.Ua OPC Foundation stack (very chatty)
|
||||
# Akka cluster/remoting internals (chatty; base is Information)
|
||||
# Leave Microsoft.EntityFrameworkCore / Microsoft.AspNetCore at Warning (see README "Resource limits & dev logging"):
|
||||
# a Debug SQL flood starves the Akka heartbeat thread.
|
||||
services:
|
||||
central-1:
|
||||
environment:
|
||||
Serilog__MinimumLevel__Override__ZB.MOM.WW.OtOpcUa.Runtime: "Debug"
|
||||
central-2:
|
||||
environment:
|
||||
Serilog__MinimumLevel__Override__ZB.MOM.WW.OtOpcUa.Runtime: "Debug"
|
||||
+28
-4
@@ -57,6 +57,11 @@ otopcua-cli write -u opc.tcp://localhost:4840 -n "ns=2;s=MyNode" -v 42 -U operat
|
||||
|
||||
### Failover
|
||||
|
||||
**Mapped ports and internal hostnames.** Endpoint discovery rewrites the host *and port* of the endpoint the server
|
||||
advertises to the URL you passed (when the scheme matches), so connecting through a port mapping works even though the
|
||||
server advertises its internal address (for example the docker-dev rig: `-u opc.tcp://localhost:4841` reaches the
|
||||
node mapped to 4841, although every container advertises `:4840`).
|
||||
|
||||
When `-F` is provided, the shared service tries the primary URL first, then each failover URL in order. For long-running commands (`subscribe`, `alarms`), the service monitors the session via keep-alive and automatically reconnects to the next available server on failure.
|
||||
|
||||
```bash
|
||||
@@ -205,21 +210,34 @@ otopcua-cli historyread -u opc.tcp://localhost:4840/OtOpcUa \
|
||||
-n "ns=1;s=TestMachine_001.TestHistoryValue" \
|
||||
--start "2026-03-25T00:00:00Z" --end "2026-03-30T00:00:00Z"
|
||||
|
||||
# Aggregate: 1-hour average
|
||||
# Aggregate: 1-hour average (every backend; on Gateway it is computed from raw samples since #681 —
|
||||
# capped by ServerHistorian:MaxTimeWeightedRawRows, so read long dense windows in slices)
|
||||
otopcua-cli historyread -u opc.tcp://localhost:4840/OtOpcUa \
|
||||
-n "ns=1;s=TestMachine_001.TestHistoryValue" \
|
||||
--start "2026-03-25T00:00:00Z" --end "2026-03-30T00:00:00Z" \
|
||||
--aggregate Average --interval 3600000
|
||||
|
||||
# Modified-value history (what HistoryUpdate changed): prior value, update type, user, modification time
|
||||
otopcua-cli historyread -u opc.tcp://localhost:4840/OtOpcUa -U apikey -P otua_<keyId>_<secret> \
|
||||
-n "ns=2;s=Plant/Modbus/dev1/Speed" \
|
||||
--start "2026-01-15T00:00:00Z" --end "2026-01-16T00:00:00Z" --modified
|
||||
```
|
||||
|
||||
`--modified` sends `ReadRawModifiedDetails` with `IsReadModified = true` (OPC UA Part 11, Gitea #668) and prints one line per
|
||||
recorded modification: the source timestamp, the **prior** value (`(none)` for an insert, which had none), its status, the update type
|
||||
(`Insert` / `Replace` / `Update` / `Delete`), the user (the API key's principal, or `anonymous`) and the modification time. It follows
|
||||
continuation points up to `--max`. A server on the `Gateway` backend (no modification trail) answers `BadHistoryOperationUnsupported`.
|
||||
`--modified` cannot be combined with `--aggregate`. See [Historian.md §Modified-value history](Historian.md#modified-value-history-part-11).
|
||||
|
||||
| Flag | Description |
|
||||
|------|-------------|
|
||||
| `-n` / `--node` | Node ID to read history for (required) |
|
||||
| `--start` | Start time in ISO 8601 UTC format, e.g. `2026-01-15T08:00:00Z` (default: 24 hours ago) |
|
||||
| `--end` | End time in ISO 8601 UTC format, e.g. `2026-01-15T09:00:00Z` (default: now) |
|
||||
| `--max` | Maximum number of values (default: 1000) |
|
||||
| `--aggregate` | Aggregate function: Average, Minimum, Maximum, Count, Start, End, StandardDeviation |
|
||||
| `--aggregate` | Aggregate function: Average, Minimum, Maximum, Count, Start, End, StandardDeviation, TimeAverage, TimeAverage2, Total, Total2, Interpolative |
|
||||
| `--interval` | Processing interval in milliseconds for aggregates (default: 3600000) |
|
||||
| `--modified` | Read modified-value history (Part 11 `IsReadModified`) instead of raw samples |
|
||||
|
||||
### historyupdate
|
||||
|
||||
@@ -258,8 +276,9 @@ node-level code (`Good`, or `BadNoData` if the range held nothing). A node that
|
||||
updater, so no `HistoryWrite` access bit), answers `BadHistoryOperationUnsupported`; a `NaN`/`Infinity` double is `BadInvalidArgument`. The session's API key
|
||||
(`-U apikey -P otua_<keyId>_<secret>`) needs the `HistoryUpdate` permission — via a `Cluster`-scoped grant when `Security:Acl:Enabled` is off, or a scoped
|
||||
grant on the node when it is on; a key with only `WriteOperate` gets `BadUserAccessDenied`. See [Historian.md §HistoryUpdate](Historian.md#historyupdate) and
|
||||
[security.md](security.md#historyupdate-historian-writes). `historyread --aggregate` only knows the seven names in the table below; the SQL backend's other
|
||||
aggregates (Total, Range, Delta, PercentGood, …) need a different client.
|
||||
[security.md](security.md#historyupdate-historian-writes). `historyread --aggregate` only knows the twelve names in the table below; the SQL backend's other
|
||||
aggregates (Range, Delta, PercentGood, …) need a different client. A refused history read (for example
|
||||
`BadHistoryOperationUnsupported`) prints one error line and exits non-zero; it does not dump a stack trace.
|
||||
|
||||
`delete-raw` is sent with `IsDeleteModified=false`. Invalid or inconsistent options (unknown mode, unpaired
|
||||
value/time, unparsable value, options that do not apply to the mode) fail before any connection is made.
|
||||
@@ -275,6 +294,11 @@ value/time, unparsable value, options that do not apply to the mode) fail before
|
||||
| `Start` | `first` | `AggregateFunction_Start` |
|
||||
| `End` | `last` | `AggregateFunction_End` |
|
||||
| `StandardDeviation` | `stddev`, `stdev` | `AggregateFunction_StandardDeviationPopulation` |
|
||||
| `TimeAverage` | `timeavg` | `AggregateFunction_TimeAverage` |
|
||||
| `TimeAverage2` | `timeavg2` | `AggregateFunction_TimeAverage2` |
|
||||
| `Total` | | `AggregateFunction_Total` |
|
||||
| `Total2` | | `AggregateFunction_Total2` |
|
||||
| `Interpolative` | `interp` | `AggregateFunction_Interpolative` |
|
||||
|
||||
### alarms
|
||||
|
||||
|
||||
+1
-1
@@ -206,7 +206,7 @@ All times are in UTC. Invalid input turns red on blur.
|
||||
|
||||
| Option | Description |
|
||||
|--------|-------------|
|
||||
| Aggregate | Raw (default), Average, Minimum, Maximum, Count, Start, End, Standard Deviation |
|
||||
| Aggregate | Raw (default), Average, Minimum, Maximum, Count, Start, End, Standard Deviation, Time Average, Time Average 2, Total, Total 2, Interpolative |
|
||||
| Interval (ms) | Processing interval for aggregate queries (shown only for aggregates) |
|
||||
| Max Values | Maximum number of raw values to return (default 1000) |
|
||||
|
||||
|
||||
+30
-5
@@ -22,14 +22,30 @@ Two related concerns get their own dedicated pages and are **only summarised + l
|
||||
|
||||
The Host (`Program.cs`) loads `appsettings.json`, then overlays a **per-role** file chosen from the cluster roles:
|
||||
|
||||
- A single role → `appsettings.{role}.json` (e.g. `appsettings.driver.json`, `appsettings.admin.json`).
|
||||
- Both roles → `appsettings.admin-driver.json` (roles joined with `-`, ordinal-sorted).
|
||||
- A single plain role → `appsettings.{role}.json` (e.g. `appsettings.driver.json`, `appsettings.admin.json`).
|
||||
- Both roles → `appsettings.admin-driver.json` (plain roles joined with `-`, ordinal-sorted).
|
||||
- `cluster-{ClusterId}` roles are **ignored** when naming the overlay (#678.6) — `[driver, cluster-SITE-A]` loads `appsettings.driver.json`.
|
||||
- `appsettings.{ASPNETCORE_ENVIRONMENT}.json` (e.g. `appsettings.Development.json`) is layered on by the host builder.
|
||||
|
||||
All role overlays are **optional** — the base `appsettings.json` plus the Options-class C# defaults are enough to boot. The roles themselves come from `EffectiveRoles.Resolve` (`ZB.MOM.WW.OtOpcUa.Cluster`) — `Cluster:Roles`, falling back to `OTOPCUA_ROLES` when that is empty (see [`ServiceHosting.md`](ServiceHosting.md) and the table below).
|
||||
|
||||
The checked-in `appsettings*.json` files are deliberately thin: they carry only `Serilog` and the `Security:Ldap` overlay. Everything else (`OpcUa`, `Cluster`, `ConnectionStrings`/`ConfigDb`) binds from the Options-class defaults documented below unless an operator adds the section explicitly or supplies the corresponding environment variable.
|
||||
|
||||
<a id="role-overlay-audit"></a>
|
||||
#### Role overlay audit (#678.6)
|
||||
|
||||
Until #678.6 the overlay never loaded on a node carrying a `cluster-*` role (the cluster role leaked into the filename). Fixing that **activates** the shipped overlays on those nodes (the docker-dev rig and any per-cluster-mesh production node), so each key was audited against base `appsettings.json` / the Options defaults. Env vars and command-line args are re-appended after the overlay, so any value a deployment sets explicitly (the rig sets every `Security__Ldap__*` key) still wins.
|
||||
|
||||
| Key | Overlay(s) | Base / default | Effect on activation |
|
||||
|---|---|---|---|
|
||||
| `Serilog:MinimumLevel:Default` = `Information` | all three | Serilog default `Information` | no-op (in `Development` it outranks `appsettings.Development.json`'s `Debug`, by design) |
|
||||
| `…Override:Akka` = `Information` | all three | inherits `Default` (`Information`) | no-op |
|
||||
| `…Override:Microsoft.AspNetCore` = `Warning` | `admin`, `admin-driver` | unset (`Information`) | quieter ASP.NET Core logs; log-level only, matches what the rig already pins |
|
||||
| `…Override:Opc.Ua` = `Information` | `admin-driver`, `driver` | inherits `Default` | no-op. **`driver` shipped `Debug` — changed to `Information` in #678.6**: activating a Debug OPC UA stack trace on every production driver node would flood Serilog (the same pipeline starvation the rig's EF/AspNetCore pins guard against). Raise it per node with `Serilog__MinimumLevel__Override__Opc.Ua=Debug`. |
|
||||
| `Security:Ldap:DevStubMode` = `false` | all three | `false` | no-op; the overlay's point is that it outranks a developer's `appsettings.Development.json` (`true`) |
|
||||
| `Security:Ldap:Transport` = `Ldaps` | all three | `LdapOptions.Transport` default `None` | **behaviour-changing, kept deliberately** (documented secure production default). A cluster-role node that configures `Security:Ldap:Server`/`Port` for a plaintext LDAP but never sets `Transport` would now dial LDAPS and fail AdminUI sign-in; set `Security__Ldap__Transport` (the rig already sets `None` explicitly). Data-plane auth is API-key based and unaffected; driver-only nodes have no AdminUI. |
|
||||
|
||||
|
||||
---
|
||||
|
||||
## `appsettings` sections
|
||||
@@ -38,7 +54,7 @@ The checked-in `appsettings*.json` files are deliberately thin: they carry only
|
||||
|
||||
- **Purpose:** logging. Console + rolling daily file sink, layered with the shared `ZB.MOM.WW.Telemetry` enrichers (`AddZbSerilog` in `Program.cs`).
|
||||
- **Where bound:** `builder.AddZbSerilog(...)` reads `Serilog` from configuration (`ReadFrom.Configuration`).
|
||||
- **Checked-in shape** (`appsettings.json`): `Using` = `[ "Serilog.Sinks.Console", "Serilog.Sinks.File" ]`, `WriteTo` = a `Console` sink and a `File` sink (`path: logs/otopcua-.log`, `rollingInterval: Day`). Role overlays add `MinimumLevel` / `Override` blocks (e.g. `Opc.Ua: Debug`, `Akka: Information`).
|
||||
- **Checked-in shape** (`appsettings.json`): `Using` = `[ "Serilog.Sinks.Console", "Serilog.Sinks.File" ]`, `WriteTo` = a `Console` sink and a `File` sink (`path: logs/otopcua-.log`, `rollingInterval: Day`). Role overlays add `MinimumLevel` / `Override` blocks (e.g. `Opc.Ua: Information`, `Akka: Information`).
|
||||
|
||||
### `OpcUa`
|
||||
|
||||
@@ -410,7 +426,10 @@ Validated at host start (driver-role nodes) when `ServerHistorian:Enabled` **or*
|
||||
| `ServerHistorian:SqlServer:ConnectionString` | `""` | **Required.** Historian database connection string (not the ConfigDb). Supply via `${secret:OT-HISTORIAN-SQL}` or env `ServerHistorian__SqlServer__ConnectionString`; the database must already exist. |
|
||||
| `ServerHistorian:SqlServer:Schema` | `hist` | Must match `^[A-Za-z_][A-Za-z0-9_]{0,127}$` (spliced into DDL/DML). |
|
||||
| `ServerHistorian:SqlServer:CommandTimeout` | `00:00:30` | Must be `> 0`; rounded up to whole seconds. |
|
||||
| `ServerHistorian:SqlServer:AutoCreateSchema` | `true` | `true` applies the embedded schema script under an app-lock on first use; `false` only verifies `SchemaVersion >= 1` and fails every historian call (health `LastError`, no crash) if behind. |
|
||||
| `ServerHistorian:SqlServer:AutoCreateSchema` | `true` | `true` applies the embedded schema scripts under an app-lock on first use (upgrading a v1 schema to v2); `false` only verifies `SchemaVersion >= 1` and fails every historian call (health `LastError`, no crash) if behind. A verified **v1** schema (no `002_sample_modified.sql`) is served with modified-value reads and HistoryUpdate refused `BadHistoryOperationUnsupported`, retention skipping `SampleModified`, and one Warning per boot (#674). |
|
||||
| `ServerHistorian:SqlServer:RetentionDays` | unset | Days of history kept (source time), `>= 1` when set; unset keeps everything. Enables the Primary-gated purge of `hist.Sample`/`hist.Event` (see [Historian.md](Historian.md#retention-retentiondays)). |
|
||||
| `ServerHistorian:SqlServer:PurgeInterval` | `01:00:00` | Purge cadence; must be `> 0`. |
|
||||
| `ServerHistorian:SqlServer:PurgeBatchSize` | `4000` | Rows per `DELETE TOP (n)` batch; `>= 1`. Keep under ~5000 to avoid lock escalation. |
|
||||
|
||||
#### `ServerHistorian:QuestDb` (`Backend=QuestDb`)
|
||||
|
||||
@@ -427,7 +446,12 @@ when `ServerHistorian:Enabled` **or** `AlarmHistorian:Enabled` is true (`QuestDb
|
||||
| `ServerHistorian:QuestDb:VisibilityTimeout` | `00:00:05` | Must be `>= 0`. How long a write waits for its rows to become visible to queries (WAL apply); `0` disables the wait. A timeout only logs a Warning. |
|
||||
|
||||
HistoryUpdate (SqlServer and QuestDb only; QuestDb deletes are soft — tombstones) additionally needs `ServerHistorian:Enabled=true` and a key
|
||||
whose principal holds the `HistoryUpdate` permission — see [`docs/security.md`](security.md#historyupdate-historian-writes). Full semantics,
|
||||
whose principal holds the `HistoryUpdate` permission — see [`docs/security.md`](security.md#historyupdate-historian-writes). It is served
|
||||
only on the redundancy **Primary** (`BadInvalidState` elsewhere, #674) and is bounded by one backend-independent key:
|
||||
|
||||
| Key | Default | Meaning |
|
||||
|---|---|---|
|
||||
| `ServerHistorian:HistoryUpdateMaxFutureSkew` | `1.00:00:00` (1 day) | TimeSpan. A HistoryUpdate (Insert / Replace / Update) item whose `SourceTimestamp` is later than server-clock now + this is refused `BadOutOfRange` before any backend sees it (#674 — a far-future typo could otherwise age out a QuestDB TTL table or poison the history). Applies to every backend; the live recorder is not bounded. Non-positive disables the bound. | Full semantics,
|
||||
schema, aggregates and the docker-dev recipe: [`docs/Historian.md`](Historian.md#sql-server-backend) (SqlServer) and
|
||||
[`docs/Historian.md`](Historian.md#questdb-backend) (QuestDb: the µs timestamp rules, pre-1970 limits, TTL and the docker-dev overlay).
|
||||
|
||||
@@ -447,6 +471,7 @@ Only the knobs; the backend connection comes from `ServerHistorian`. Outbox reco
|
||||
| `ContinuousHistorization:DrainConcurrency` | `8` | Historian tags one drain pass writes **concurrently** — a batch is grouped per tag and each group is its own gateway round-trip, so this turns a sequential `tags × latency` pass into `⌈tags / concurrency⌉ × latency` while bounding outstanding round-trips. `1` = fully sequential; `< 1` is clamped to 1 with a `Validate()` warning. The ack is unchanged (all-or-nothing after every group reports), as is the #538 quarantine rule. |
|
||||
| `ContinuousHistorization:Capacity` | `500000` | Max un-acked entries before drop-oldest; `0` = unbounded. The #538 disk backstop for a historian refusing **every** write — the one case the per-tag quarantine cannot resolve. |
|
||||
| `ContinuousHistorization:MaxWriteAttempts` | `5` | #538 poison-tag budget: consecutive failed passes one tag may accumulate — **counted only across passes where *other* tags wrote successfully** — before its entries are abandoned so the FIFO head can advance. |
|
||||
| `ContinuousHistorization:CaptureWhilePrimaryUnreachable` | `true` | #672: a standby stood down for its mesh-partner Primary (#537) resumes capture as soon as the failure detector reports that Primary **unreachable**, rather than after auto-down + a fresh snapshot — a crash-failover gap of ~25–30 s becomes the failure-detector window (~10–12 s). Cost: duplicate capture while a live Primary is merely unreachable and under a real partition (absorbed idempotently on SqlServer/QuestDB only for source-timestamped drivers; Modbus-style poll-time stamps and the Gateway backend store duplicates). Affects only the capture verdict. `false` = pre-#672 behaviour. |
|
||||
| `ContinuousHistorization:MinBackoffSeconds` / `MaxBackoffSeconds` | `1` / `30` | Initial and capped exponential retry backoff after a failed drain pass. |
|
||||
|
||||
### `LocalDb` (pair-local config cache — driver role only)
|
||||
|
||||
+38
-1
@@ -422,7 +422,10 @@ Two consequences worth knowing:
|
||||
- **`CalculationDriver` can never trip it, correctly.** It reports `Healthy` whenever it is running and has
|
||||
no transport to be degraded about. Every other driver reaches a non-Healthy state on sustained failure:
|
||||
the nine poll drivers via `Degraded`, MQTT via its live connection state, Galaxy via its reconnect
|
||||
supervisor. This was surveyed across all 12 before relying on it — it is the assumption the whole
|
||||
supervisor, OpcUaClient via a bad session keep-alive (`Degraded`, since Gitea #670 — before that its health
|
||||
only moved on `ReadAsync`, which a subscription-only driver never calls, so it stayed `Healthy` through any
|
||||
outage and this survey was wrong about it; good keep-alives and notifications now advance its read stamp, so
|
||||
an idle-but-alive subscription does not look dead either). This was surveyed across all 12 before relying on it — it is the assumption the whole
|
||||
mechanism rests on.
|
||||
- **A device that accepts a connection but serves no data will cycle**, reconnecting once per threshold.
|
||||
That is the correct reading of a broken device, it is rate-limited, and it is visible in the health
|
||||
@@ -438,6 +441,40 @@ Two consequences worth knowing:
|
||||
**notices**, so a silent-success termination path — a stream that ends cleanly, a poll loop that exits
|
||||
without throwing — bypasses it entirely.
|
||||
|
||||
**A driver that KNOWS its connection is gone can skip the wait: `IDriverConnectionLossSource` (Gitea #676).** The
|
||||
watchdog has to prove a loss from silence, so it costs ~90 s even when the protocol already said "this connection
|
||||
will never come back". An opt-in `Core.Abstractions` interface lets a driver say so:
|
||||
`event EventHandler<DriverConnectionLossEventArgs> ConnectionLost` (a reason string). `DriverInstanceActor` attaches
|
||||
in `PreStart` and detaches in `PostStop` — the same lifetime as `IRediscoverable`, for the same reason (the `IDriver`
|
||||
outlives the actor on a respawn) — marshals each raise to the actor thread (`Self.Tell`; actor state is never touched
|
||||
on the driver's thread), and acts on it **only while `Connected`**, through the very handler the watchdog's
|
||||
`DisconnectObserved` uses (detach, record a fault, `Reconnecting`, teardown-then-init on the retry). In
|
||||
`Connecting`/`Reconnecting`/`Stubbed` it is ignored, not deferred: a re-initialise is already pending, and acting on a
|
||||
stale raise later could tear down the connection that attempt is building. The two are complementary, not
|
||||
alternatives:
|
||||
|
||||
- **The signal is for DEFINITIVE losses only.** A driver must raise only when no in-place recovery is possible, at most
|
||||
once per connection, re-armed by a re-initialise. An ambiguous failure — a timeout, a stalled socket, "no
|
||||
communication" — must not raise: a stalled link that resumes keeps its session, subscriptions and in-place recovery,
|
||||
and tearing it down would cost all three. Those stay with the watchdog, whose conjunction is the honest test for them.
|
||||
- **Today only `OpcUaClientDriver` implements it:** keep-alive `BadSessionIdInvalid` / `BadSessionClosed` /
|
||||
`BadSessionNotActivated` / `BadServerHalted`; a CLOSED connection (`BadSecureChannelClosed` /
|
||||
`BadConnectionClosed`) once sustained — the second closed-channel keep-alive in one outage, which is what a real
|
||||
process stop actually produces (live gate A1), and the SDK never reopens a closed connection by itself; or a
|
||||
subscription the server REPORTED dropped (not the driver's
|
||||
publishing-stall heuristic, which is an inference from silence and stays with the watchdog; see
|
||||
[OpcUaClient.md](drivers/OpcUaClient.md) §"Upstream outage and recovery" for why `BadServerHalted` is on the list —
|
||||
it was measured, the SDK never re-opens a halted channel). A restarted, stopped or killed upstream therefore
|
||||
recovers in about one keep-alive plus one reconnect interval instead of ~100 s; a stall/pause
|
||||
(`BadRequestTimeout`/`BadNoCommunication`) still goes through the watchdog and keeps its in-place recovery.
|
||||
- ⚠️ **Known gap:** "only while `Connected`" plus "once per connection" means a raise the host receives while not yet
|
||||
Connected (e.g. the protocol proves the fresh connection dead before `InitializeSucceeded` is processed) is swallowed,
|
||||
and that connection falls back to the watchdog — the pre-#676 recovery time, in a narrow window.
|
||||
- Pinned by `DriverInstanceActorConnectionLossSignalTests` (Reconnecting within seconds while the watchdog is set to an
|
||||
hour; a falsification control with no raise that stays Connected; a raise mid-connect ignored; a stray watchdog
|
||||
`DisconnectObserved` in `Reconnecting` is a no-op; detach on stop) and `OpcUaClientConnectionLossSignalTests`
|
||||
(definitive raises once, ambiguous and the stall heuristic never, re-armed after re-init).
|
||||
|
||||
**The reconnect retry now tears down before re-initialising.** `Reconnecting`'s `RetryConnect` calls
|
||||
`ShutdownAsync` before `InitializeAsync` — the sequence every driver's own `ReinitializeAsync` already
|
||||
uses — because several drivers **overwrite** an `IAsyncDisposable` transport field inside `InitializeAsync`
|
||||
|
||||
+1257
-86
File diff suppressed because it is too large
Load Diff
+10
-1
@@ -546,7 +546,16 @@ Under warm/hot redundancy both cluster nodes run `ScriptedAlarmHostActor` and ev
|
||||
node. ⚠️ **The peer it stands down for must record the same refs** — a Primary in its own per-cluster
|
||||
mesh — not merely hold the same rows; a node with no cluster-scoped role never stands down, because a
|
||||
fleet-wide election's Primary usually runs a different cluster's configuration and deferring to it would
|
||||
end this cluster's historization silently. See [Historian.md](Historian.md).
|
||||
end this cluster's historization silently. **Crash failover (Gitea #672):** a SIGKILLed Primary stays an
|
||||
unreachable-but-not-downed member until auto-down (10 s failure detector + 15 s), and the redundancy
|
||||
singleton usually died with it, so the standby's cached snapshot kept naming it Primary and nobody
|
||||
captured for ~30 s. `DriverHostActor` now follows `ClusterEvent.ReachabilityEvent` and excludes an
|
||||
**unreachable** peer from the historization peer-is-Primary question only (re-deriving the capture verdict
|
||||
from the cached snapshot), shrinking the gap to the failure-detector window; gated by
|
||||
`ContinuousHistorization:CaptureWhilePrimaryUnreachable` (default `true`). The alarm drain and the Primary
|
||||
data-plane verdict are **unchanged** — they still wait for a snapshot. The cost is duplicate capture while a
|
||||
live Primary is merely unreachable, and from detection onward under a real partition. See
|
||||
[Historian.md](Historian.md) §"A crashed Primary: capture gap (#672)".
|
||||
|
||||
Net effect: each alarm transition appears **once** on `/alerts` and would historize once, not once per node.
|
||||
|
||||
|
||||
@@ -32,13 +32,13 @@ Single-node dev: `OTOPCUA_ROLES=admin,driver` (or the `Cluster:Roles` equivalent
|
||||
|
||||
### Per-role configuration overlays
|
||||
|
||||
`Program.cs` builds a role suffix by joining the resolved `EffectiveRoles.Gossiped` roles **alphabetically** with `-` and loads `appsettings.{roleSuffix}.json` as an optional overlay on top of base `appsettings.json`. A plain role (`admin`/`driver`/`dev`) is always lower-case in `Gossiped` regardless of how the operator spelled it, so the overlay filename for those is stable (`appsettings.driver.json`); if a `cluster-{ClusterId}` role is also configured, its suffix carries into the filename **verbatim** (e.g. `Cluster:Roles: ["driver", "cluster-SITE-A"]` loads `appsettings.cluster-SITE-A-driver.json`), since `Gossiped` preserves that spelling by design (see [Configuration.md § `Roles`](Configuration.md#appsettings-sections)). Three overlays ship in `src/Server/ZB.MOM.WW.OtOpcUa.Host/`:
|
||||
`Program.cs` builds a role suffix (`RoleOverlay.Suffix`) by joining the resolved `EffectiveRoles.Gossiped` **plain** roles (`admin`/`driver`/`dev`) alphabetically with `-` and loads `appsettings.{roleSuffix}.json` as an optional overlay on top of base `appsettings.json`. A plain role is always lower-case in `Gossiped` regardless of how the operator spelled it, so the overlay filename is stable (`appsettings.driver.json`). **`cluster-{ClusterId}` roles are NOT part of the filename** (#678.6): they are mesh-membership tags, so `Cluster:Roles: ["driver", "cluster-SITE-A"]` loads `appsettings.driver.json` and `["admin", "driver", "cluster-MAIN"]` loads `appsettings.admin-driver.json`. (Before #678.6 the cluster role was folded into the name, producing e.g. `appsettings.admin-cluster-MAIN-driver.json`, which never exists — so on every node with a cluster role no shipped overlay ever loaded.) Three overlays ship in `src/Server/ZB.MOM.WW.OtOpcUa.Host/`:
|
||||
|
||||
- `appsettings.admin.json` — admin-only nodes
|
||||
- `appsettings.driver.json` — driver-only nodes
|
||||
- `appsettings.admin-driver.json` — fused single-node dev / small deployments
|
||||
|
||||
All three carry Serilog log-level overrides + `Security:Ldap:DevStubMode = false`. Configuration loading order (lowest to highest precedence) is:
|
||||
All three carry Serilog log-level overrides + `Security:Ldap:DevStubMode = false` + `Security:Ldap:Transport = Ldaps` (audit table: [Configuration.md § Role overlays](Configuration.md#role-overlay-audit)). Configuration loading order (lowest to highest precedence) is:
|
||||
|
||||
`appsettings.json` < `appsettings.{Environment}.json` < `appsettings.{role}.json` < environment variables < command-line args
|
||||
|
||||
|
||||
+142
-12
@@ -45,22 +45,139 @@ out-of-process isolation.
|
||||
```
|
||||
|
||||
`EndpointUrls` (a list) takes precedence over the single-URL `EndpointUrl` and
|
||||
provides ordered **failover** — the driver tries each candidate in turn at init
|
||||
and on session drop, and the first to connect wins (e.g. a hot-standby pair on
|
||||
provides ordered **failover** — the driver tries each candidate in turn at every
|
||||
`InitializeAsync` (including the comms-loss watchdog's re-initialise), and the first to connect wins (e.g. a hot-standby pair on
|
||||
4840 / 4841). See
|
||||
`src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.OpcUaClient.Contracts/OpcUaClientDriverOptions.cs`
|
||||
for every field (security policy/mode, auth type, session timeout, keep-alive,
|
||||
reconnect period, browse root, node/depth caps).
|
||||
browse root, node/depth caps). `ReconnectPeriod` was **deleted** (#676 — unused since #670 removed the SDK reconnect
|
||||
handler, and the AdminUI no longer shows it); stored configs that still carry `reconnectPeriod` keep parsing in the
|
||||
factory, the Test-Connect probe and the browse picker (unknown keys are skipped), and the form drops it on the next save.
|
||||
|
||||
### Endpoint address: the configured host and port win (#676)
|
||||
|
||||
GetEndpoints returns each endpoint with the URL the **server** believes it lives at. Behind NAT, a port map, a
|
||||
container port publish or a proxy that address is unreachable from the client — or reachable but bypassing the proxy.
|
||||
The driver therefore rewrites the selected endpoint's **host and port** to the configured URL's whenever the schemes
|
||||
match, keeping the advertised path (the SDK's own `CoreClientUtils.SelectEndpoint` rule, which the driver's
|
||||
policy-aware endpoint selection used to bypass). A configured URL with **no port** keeps the advertised port (copying
|
||||
the port-less `-1` would have sent the session to 4840). A different scheme is left alone; an unparseable advertised
|
||||
URL falls back to the configured one. `OpcUaClientDriver.RewriteEndpointUrl`.
|
||||
|
||||
### Session lifetime
|
||||
|
||||
A single `Session` per driver instance; subscriptions multiplex onto it. The
|
||||
SDK reconnect handler takes the session down and brings it back on remote-server
|
||||
restart, re-sending subscriptions on reconnect so monitored-item handles don't
|
||||
dangle. Stored NodeIds embed the server-stable namespace **URI** (not the
|
||||
A single `Session` per driver instance; subscriptions multiplex onto it. Stored NodeIds embed the server-stable namespace **URI** (not the
|
||||
session-relative `ns=N` index) so a remote namespace-table reorder across a
|
||||
restart doesn't silently re-point references at the wrong namespace.
|
||||
|
||||
### Upstream outage and recovery (Gitea #670)
|
||||
|
||||
The session keep-alive (`KeepAliveInterval`, default 5 s) is the liveness signal:
|
||||
|
||||
- **Good keep-alive / any data or event notification** advances `DriverHealth.LastSuccessfulRead` — a
|
||||
subscription-only driver never calls `ReadAsync`, so without this its read stamp froze at connect time.
|
||||
- **Bad keep-alive** ⇒ health `Degraded` (`LastError = "keep-alive: <status>"`, read stamp preserved), host status
|
||||
`Stopped`, and — **once per outage**, not per keep-alive — every subscribed ref is published `BadNoCommunication`
|
||||
with its last value retained and **source + server timestamp = the detection time** (live-gate F1: stamped with the
|
||||
last Good's source time it collided with that row's `(TagId, SourceTimeUtc)` history key and was never stored), plus a quality-only `BadNoCommunication` annotation (#626 pattern, no alarm-state
|
||||
change, no `/alerts` row) on every subscribed alarm condition.
|
||||
- **The session survives** (a network blip; the keep-alive turns Good again) ⇒ recovered in place: health `Healthy`,
|
||||
the refs still Bad are re-read once and published Good (a server whose subscription survived does not resend an
|
||||
unchanged value, and a `(value, status)` dedup downstream lets the identical value through), alarm quality goes
|
||||
back Good and a `ConditionRefresh` re-reports anything missed. A notification that lands first wins over the read.
|
||||
For a bounded **replay window** after such a recovery — max(30 s, 2 x the longest subscription keep-alive window) —
|
||||
each ref is guarded by a high-water mark — its last Good upstream source time, raised by the recovery read and
|
||||
advanced by every notification accepted in the window — and a notification whose source time is at or BELOW it
|
||||
(equal included: OPC UA moves the source timestamp on every value or status change, so an equal one is a replay)
|
||||
is dropped: the SDK was seen delivering a missed one 4–6 s late, which regressed the node (live-gate F2). The
|
||||
guard holds for the WHOLE window — fresh notifications arrive ~1 s after recovery, before the late replay, so a
|
||||
ref is never disarmed early (the F2 re-gate failed on exactly that). A notification with no source time passes
|
||||
without moving the mark. Outside that window every notification is
|
||||
accepted whatever its timestamp, so an upstream clock that steps backwards (NTP, a PLC clock reset) never
|
||||
freezes a node. The outage Bad never moves the comparison; a re-initialise resets it.
|
||||
⚠️ **By design, inside the window a notification whose source time EQUALS the last accepted one is dropped** — so a
|
||||
status-only change that keeps the old source timestamp (not what OPC UA servers are supposed to do) is lost during
|
||||
the window only (the trailing re-read below repairs it). When the window closes — or a second recovery re-arms it —
|
||||
the driver logs ONE line, `post-recovery replay window closed — dropped N late notifications across M refs after
|
||||
recovery`, at **Information when N > 0** and Debug otherwise, because the per-drop line is Debug and the
|
||||
`stale_notification_dropped` counter is OTel-only, which made the guard unobservable on the docker-dev rig (#676).
|
||||
A re-arm starts the count afresh.
|
||||
- **Trailing re-read after an in-place recovery (#676).** Found by the TCP-stall wire test: a change the server
|
||||
published just after the stall ended rode a Publish response that had been in flight across the stall; the SDK
|
||||
timed that request out (`BadTimeout`), discarded the response and never republished the message (the following
|
||||
keep-alives moved the sequence number past it), so the node stayed **Good on the pre-change value indefinitely**.
|
||||
`Timeout + KeepAliveInterval` after the recovery — when every request in flight across the stall has completed or
|
||||
timed out — every subscribed ref is read once more and published when its **(value, status) differs** from the last
|
||||
value delivered for that ref **and is not older** than it. Any status counts: a Good→Bad/Uncertain change lost in the
|
||||
pipeline is as stale as a lost value. "Older" compares source timestamps only when both carry one; a read with no
|
||||
source timestamp (a server that does not send them) is the server's current state and counts as newer. An **equal**
|
||||
source timestamp with different content IS published — it is the server's current state, not a replay, and it repairs
|
||||
the equal-timestamp status-only change the replay guard drops by design. An identical (value, status) is never
|
||||
republished. A ref that received ANY delivery while the read was in flight is skipped (a per-ref delivery counter is
|
||||
captured before the read is issued): without source timestamps the read's sample cannot be ordered against that
|
||||
notification, and publishing it would regress the node. An Information line names how many refs it had to repair. The trailing read and the replay-guard disarm
|
||||
timer run on the driver clock and are cancelled when the session is released (re-initialise / shutdown / dispose).
|
||||
- **No older value is ever delivered after a newer one.** The paths that publish upstream data — live notifications,
|
||||
the recovery read, the trailing read and the comms-loss Bad fan — make their "may I publish this?" decision and the
|
||||
publish inside one driver-wide critical section. A single lock rather than one per ref: it also serialises delivery
|
||||
across subscriptions (the SDK dispatches each subscription's messages on its own worker), which is cheap because each
|
||||
section is a dictionary update plus a subscriber `Tell` and nothing awaits under it. Subscribers run inside it — the
|
||||
host's handler only `Tell`s an actor — so a subscriber must never block on, or call back into, the driver.
|
||||
- **The session does not come back** (an upstream restart: the server forgot the session) ⇒ the driver stays
|
||||
`Degraded` with its read stamp frozen, which is the #527 comms-loss watchdog's trip condition; `DriverInstanceActor`
|
||||
runs `ShutdownAsync` + `InitializeAsync` (retried every 10 s while the server is down) and re-subscribes data +
|
||||
alarms; the new subscription's initial values come back Good. Alarm last-told state survives the re-initialise
|
||||
(#619), so the post-subscribe `ConditionRefresh` raises no un-ack flicker; the host's reconnect annotation restores
|
||||
condition quality.
|
||||
- **Fast recovery for a DEFINITIVELY lost session (#676).** The driver implements `IDriverConnectionLossSource` and
|
||||
raises `ConnectionLost` — **once per session**, re-armed by a re-initialise — when the loss is certain, so the host
|
||||
re-initialises at once instead of after the watchdog's 90 s. Definitive means: a keep-alive status of
|
||||
`BadSessionIdInvalid` / `BadSessionClosed` / `BadSessionNotActivated` (the server forgot, closed or never activated
|
||||
the session) or `BadServerHalted`, or a subscription the **server** reported gone (below). `BadServerHalted` is on the list
|
||||
because it was **measured**: on the restart wire test a gracefully stopped server reports `BadServerHalted` (and the
|
||||
keep-alives alternate it with `BadNoCommunication`), and the SDK never re-opens a channel the server halted — 15 s
|
||||
after the server was back, the same session still failed every request with `BadServerHalted`; `BadSessionIdInvalid`
|
||||
never arrived.
|
||||
**A CLOSED connection is definitive too, once sustained (live gate A1).** Against a real opc-plc on docker-dev, none of
|
||||
`docker restart`, a graceful SIGINT stop (opc-plc itself logged `BadServerHalted`) or `docker kill` delivered a
|
||||
session status to the client: the keep-alives reported `BadSecureChannelClosed` (0x80860000) once, then
|
||||
`BadConnectionClosed` (0x80AE0000) interleaved with `BadNoCommunication` for the whole outage, so the first cut of the
|
||||
fast path never fired and recovery took 91–127 s. Reproduced in-process with `TcpStallProxy.HardClose` (every
|
||||
connection reset, new ones refused, the server AND its session still alive behind it): the same sequence, and the SDK
|
||||
1.5.378 session **never opens a new connection by itself** — 48 s after the proxy accepted connections again, not one
|
||||
attempt. So a closed channel can never recover in place; on the **second** closed-channel keep-alive failure in one
|
||||
outage (`IsClosedChannel`; a good keep-alive resets the count, so a single transient does not count) the driver raises
|
||||
`ConnectionLost` — about one keep-alive interval after the connection dropped. Everything else —
|
||||
`BadNoCommunication`, `BadNotConnected`, `BadRequestTimeout`/`BadTimeout` (what a stalled or paused link produces) —
|
||||
is **ambiguous** (the link may resume and keep its session, see the stall test) and stays with the watchdog.
|
||||
Consequence: a restarted, stopped or killed upstream is re-initialised within roughly one keep-alive plus one
|
||||
reconnect interval instead of ~100 s; a pause (`docker pause`, a congested link) still recovers in place. ⚠️ **Known gap:** the host acts on the signal only while the driver is
|
||||
`Connected`, and the driver raises it once per session — so a definitive loss raised while the host is not
|
||||
Connected (e.g. `BadServerHalted` on the fresh session before the host has processed the successful initialise) is
|
||||
swallowed and that session falls back to the watchdog. Re-raising on later keep-alives would close it but was left out
|
||||
to keep the once-per-session contract simple; the cost is the pre-#676 recovery time, in a narrow window.
|
||||
- **The subscription dies but the session lives** (an outage longer than the subscription lifetime —
|
||||
`LifetimeCount` x publishing interval, often well under the session timeout — or a server that drops it) ⇒ keep-alives
|
||||
stay Good, so they cannot be the signal. A `PublishError` of `BadNoSubscription` (while subscriptions are tracked) or
|
||||
`BadSubscriptionIdInvalid` naming a tracked subscription, or a tracked subscription whose SDK `PublishingStopped`
|
||||
persists for one more subscription keep-alive window + 5 s across good keep-alives, declares it lost: `Degraded`
|
||||
(`"subscription lost: …"`), the outage published once, and — sticky until the session is released — **nothing**
|
||||
advances `LastSuccessfulRead` any more (not good keep-alives, not a sibling subscription's notifications), so the
|
||||
watchdog's re-initialise recreates it. Both detectors are wire-tested independently. A subscription the **server**
|
||||
reported gone (`BadNoSubscription` / `BadSubscriptionIdInvalid`) never comes back on its session, so that also raises
|
||||
`ConnectionLost` (#676) and the re-initialise happens at once; the `PublishingStopped` **heuristic** is an inference
|
||||
from silence and keeps going through the watchdog.
|
||||
- An in-place recovery is bound to the session whose keep-alive recovered: one overtaken by a re-initialise neither
|
||||
publishes, nor degrades, nor re-arms the outage latch of the new session.
|
||||
|
||||
There is **no in-driver reconnect** any more. The SDK's `SessionReconnectHandler` (removed) retried forever in SDK
|
||||
1.5.378 (its "exhausted" path was unreachable), recreated sessions by cloning subscriptions the driver never
|
||||
re-tracked, and kept health `Healthy` throughout — so on docker-dev a stopped opc-plc left nodes Good on their last
|
||||
value indefinitely and they never recovered. A teardown against a lost session skips the wire deletes and `CloseSession`
|
||||
(bounded to 5 s otherwise). SDK warnings (keep-alive / publish / channel errors) now reach the driver's own logger,
|
||||
rate-limited to 20 per minute with a suppressed-count note; the per-driver SDK logger factory is disposed with the
|
||||
driver.
|
||||
|
||||
### Namespace assignment (retired, Gitea #648)
|
||||
|
||||
Earlier versions required a per-instance `TargetNamespaceKind` (`Equipment` +
|
||||
@@ -79,7 +196,7 @@ initialise (the AdminUI form drops them on the next save). Pinned by
|
||||
|
||||
## Capability surface
|
||||
|
||||
`OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWritable, ISubscribable, IHostConnectivityProbe, IAlarmSource`
|
||||
`OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWritable, ISubscribable, IHostConnectivityProbe, IAlarmSource, IDriverConnectionLossSource`
|
||||
(`src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.OpcUaClient/OpcUaClientDriver.cs:31`).
|
||||
|
||||
| Capability | Path | Notes |
|
||||
@@ -107,11 +224,11 @@ initialise (the AdminUI form drops them on the next save). Pinned by
|
||||
> from whatever the host retained: an inactive+acked one goes out as `Clear` + `Acknowledge` (#657; it used to be an
|
||||
> `Unspecified` the host wrote as "Activated", and a retained-Active host condition stayed Active). So the host writes no
|
||||
> duplicate row when its condition already shows that state (see `docs/AlarmTracking.md`). Condition **branch** events (non-null `BranchId`) are ignored.
|
||||
> **`ConditionRefresh`** is called after the event item is created, after every in-place reconnect, and on a
|
||||
> **`ConditionRefresh`** is called after the event item is created, after a keep-alive outage the session survived, and on a
|
||||
> `RefreshRequired` event, so a condition already active upstream is reported at subscribe; `RefreshStart`/`RefreshEnd`
|
||||
> markers are dropped. A server without it (`BadNotSupported`/`BadMethodInvalid`/…) is logged once and tolerated.
|
||||
> Last-told state survives a re-initialise (the host re-subscribes the same set over nodes it did not rebuild) and is
|
||||
> forgotten when the subscribed set changes (the host rebuilt, recreating the nodes inactive). Index keys are server-stable (`nsu=`), so a namespace reorder across a reconnect keeps
|
||||
> forgotten when the subscribed set changes (the host rebuilt, recreating the nodes inactive). Index keys are server-stable (`nsu=`), so a namespace reorder across a re-initialise keeps
|
||||
> matching. docker-dev seeds an opc-plc driver (`MAIN-opcua`); the alarm tag is
|
||||
> created only when `@OpcPlcAlarmSourceNodeId` is set in `docker-dev/seed/seed-clusters.sql`.
|
||||
|
||||
@@ -192,8 +309,10 @@ every `InitializeAsync`, including comms-loss reconnects). The discovery channel
|
||||
Session/publish events are published on the shared OTel meter as the counter
|
||||
`otopcua.driver.opcuaclient.events`, tagged `driver_instance` (the driver instance id) and `event` =
|
||||
`publish_response`, `notification` (items), `data_change`, `bad_status_value`, `publish_error`,
|
||||
`keepalive_miss`, `reconnect_attempt`, `reconnect_success`, `reconnect_exhausted`. A climbing `keepalive_miss` /
|
||||
`reconnect_attempt` rate is the signature of a flapping upstream. They are not part of `DriverHealth` or the
|
||||
`keepalive_miss`, `keepalive_recovered` (a bad keep-alive followed by a good one on the same session),
|
||||
`stale_notification_dropped` (a late replay dropped inside the post-recovery window, see above). A climbing
|
||||
`keepalive_miss` / `keepalive_recovered` rate is the signature of a flapping upstream (`reconnect_*` were removed with
|
||||
the reconnect handler, #670). They are not part of `DriverHealth` or the
|
||||
`/hosts` panel.
|
||||
|
||||
## Testing
|
||||
@@ -204,6 +323,17 @@ Session/publish events are published on the shared OTel meter as the counter
|
||||
- **Integration fixture** — exercises the driver against a reference OPC UA
|
||||
server (opc-plc) on the shared docker host; see
|
||||
[OpcUaClient-Test-Fixture.md](OpcUaClient-Test-Fixture.md) for the coverage map.
|
||||
- **Wire tests against the in-process server** — `tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/`
|
||||
`OpcUaClientDriver*WireTests` (no Docker). `OpcUaClientDriverStallWireTests` (#676) covers the in-place blip that
|
||||
used to be live-gate-only: the driver connects through `TcpStallProxy`, an in-test TCP forwarder whose pause holds
|
||||
every byte without closing anything (which only works because of the endpoint rewrite above); it asserts one Bad,
|
||||
a frozen read stamp, recovery on the SAME session, the identical value republished Good, a change published right
|
||||
after recovery still arriving, and no source-time regression once the SDK's late backlog flush has arrived. Its
|
||||
second test uses `HardClose` (reset every connection, refuse new ones) — the shape of a real process stop — and
|
||||
asserts the live-gate status sequence, that the SDK never reconnects on its own once the proxy accepts again,
|
||||
`ConnectionLost` exactly once, and the re-initialise bringing the identical value back Good.
|
||||
`OpcUaClientDriverReconnectWireTests` covers the server restart (incl. the `BadServerHalted` → `ConnectionLost` signal)
|
||||
and the subscription-lost case.
|
||||
|
||||
## Further reading
|
||||
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
-- OtOpcUa SQL Server historian backend - schema version 2: OPC UA Part 11 modified-value history (Gitea #668).
|
||||
-- $(Schema) is substituted by SqlServerSchemaBootstrapper (identifier-validated) BEFORE the script is split on GO lines.
|
||||
-- Every statement is guarded so a re-run is a no-op.
|
||||
--
|
||||
-- One row per sample a HistoryUpdate inserted, replaced or deleted, carrying the PRIOR value (all NULL for an
|
||||
-- insert). Rows are written by the SAME statement that changes hist.Sample (OUTPUT ... INTO), so a failed update
|
||||
-- leaves no audit row and a committed one always has its row.
|
||||
-- WARNING: OUTPUT ... INTO forbids, on its target table: enabled triggers, either side of a FOREIGN KEY, and CHECK
|
||||
-- constraints. Do not add any of those to SampleModified, or every HistoryUpdate fails.
|
||||
IF OBJECT_ID(N'[$(Schema)].[SampleModified]', N'U') IS NULL
|
||||
CREATE TABLE [$(Schema)].[SampleModified] (
|
||||
TagId int NOT NULL,
|
||||
SourceTimeUtc datetime2(7) NOT NULL,
|
||||
ModifiedId bigint IDENTITY(1,1) NOT NULL,
|
||||
ModificationTimeUtc datetime2(7) NOT NULL,
|
||||
UpdateType tinyint NOT NULL,
|
||||
Principal nvarchar(200) NOT NULL,
|
||||
ApiKeyId nvarchar(200) NULL,
|
||||
ValueKind tinyint NULL,
|
||||
NumValue float NULL,
|
||||
IntValue bigint NULL,
|
||||
StrValue nvarchar(max) NULL,
|
||||
StatusCode int NULL,
|
||||
ServerTimeUtc datetime2(7) NULL,
|
||||
CONSTRAINT PK_SampleModified PRIMARY KEY CLUSTERED (TagId, SourceTimeUtc, ModifiedId))
|
||||
GO
|
||||
IF NOT EXISTS (SELECT 1 FROM [$(Schema)].[SchemaVersion] WHERE Version = 2)
|
||||
INSERT [$(Schema)].[SchemaVersion] (Version, AppliedUtc) VALUES (2, SYSUTCDATETIME())
|
||||
GO
|
||||
@@ -0,0 +1,70 @@
|
||||
# Issue batch #667–#673 — design (2026-10-11)
|
||||
|
||||
All seven open issues ≥ #667 are testable with available infra (docker-dev rig, SQL Server, opc-plc fixture on
|
||||
10.100.0.35). Base: master `ee67b637`. Recommended choices taken throughout (user-approved).
|
||||
|
||||
## Lanes
|
||||
|
||||
| Lane | Issue | Model | Wave |
|
||||
|---|---|---|---|
|
||||
| L1 | #673 FOCAS flake | sonnet | 1 |
|
||||
| L2 | #670 OpcUaClient session loss | opus | 1 |
|
||||
| L3 | #672 crash-failover capture gap | opus | 1 |
|
||||
| L4 | #669 event HistoryRead continuation points | sonnet | 1 |
|
||||
| L5 | #667 SqlServer retention/purge | sonnet | 1 |
|
||||
| L6 | #671 time-weighted Part 13 aggregates | opus | 1 |
|
||||
| L7 | #668 Part 11 modified values | opus | 2 (after L4 + L5 merge) |
|
||||
|
||||
Each lane: isolated worktree reset to master, TDD, independent opus review, merge, full suite, docker-dev live gate.
|
||||
|
||||
## #673
|
||||
Root cause: `FocasDriver.FixedTreeLoopAsync` publishes `LastStatus` before `FixedTreeCache` / `LastFixedPollUtc`;
|
||||
the test waits on `LastStatus` only, and an authored `FIXED:Status/*` read returns `BadCommunicationError` while
|
||||
`LastFixedPollUtc` is null. Fix: test waits on all three; driver writes `LastStatus` last via `Volatile.Write`.
|
||||
|
||||
## #670
|
||||
Root causes: bad keep-alive never touches `_health` nor publishes; `LastSuccessfulRead` never advances on
|
||||
notifications/keep-alives; the SDK `SessionReconnectHandler` retries forever (exhausted path unreachable) and on a
|
||||
session recreate clones subscriptions the driver's dictionaries no longer track.
|
||||
Fix (Option B): bad keep-alive ⇒ `Degraded` (stamp preserved) + one `BadNoCommunication` publish per subscribed ref
|
||||
per outage (value retained) + quality-only alarm update; good keep-alive / notifications advance
|
||||
`LastSuccessfulRead`; a self-healed short outage re-reads and republishes Good (beats the (value,status) dedup);
|
||||
sustained loss is recovered by the #527 watchdog (shutdown + init + resubscribe). Delete `SessionReconnectHandler`
|
||||
wiring, `OnReconnectComplete`, `TryRearmReconnect`, the `reconnect_exhausted` counter. Wire SDK telemetry logging
|
||||
at Warning. Correct `docs/DriverLifecycle.md` survey.
|
||||
|
||||
## #672
|
||||
Standby opens the historization capture gate when its recording Primary is cluster-UNREACHABLE (local
|
||||
`ClusterEvent.ReachabilityEvent`, recomputed from the cached redundancy snapshot in `DriverHostActor`). Alarm drain
|
||||
gate and data-plane Primary gate unchanged (falsification tests). Knob
|
||||
`ContinuousHistorization:CaptureWhilePrimaryUnreachable` (default true). No backend capability flag. Gap shrinks to
|
||||
the failure-detector window (~10–12 s). Documented: drivers without a source timestamp (Modbus) duplicate rows in
|
||||
the overlap window.
|
||||
|
||||
## #669
|
||||
Event HistoryRead issues/resumes continuation points: cursor = last event time + trailing tie count; oversized tie
|
||||
cluster over-fetch via `[T, T+resolution)` bounded by `MaxTieClusterOverfetch`; generic `SliceTieCluster`; store
|
||||
state widened to carry an event cursor; unknown CP ⇒ `BadContinuationPointInvalid`. Reverse event reads out of
|
||||
scope. Ties exact on SqlServer/QuestDB, best-effort on Gateway (documented). Adds an over-the-wire event HistoryRead
|
||||
test.
|
||||
|
||||
## #667
|
||||
`ServerHistorian:SqlServer:{RetentionDays (null=forever), PurgeInterval=1h, PurgeBatchSize=4000}`; optional
|
||||
`IHistorianBackendProvider.CreateRetention`; `HistorianRetentionService` gated on `IHistorizationRoleView`
|
||||
plus `sp_getapplock` (timeout 0); per-tag batched `DELETE TOP(n)` on the clustered key, events by time; cutoff on
|
||||
source time; metric `otopcua.historian.retention.purged{table}`. QuestDB: no change (TTL). Partition switching
|
||||
deferred/documented.
|
||||
|
||||
## #671
|
||||
New time-weighted family: TimeAverage, TimeAverage2, Interpolative, Total (now the Part 13 time-weighted integral,
|
||||
matching Gateway — client-visible, release-noted), Total2. Bounding values outside the window + `LEAD()` in-bucket
|
||||
areas in SQL; C# splits bucket-crossing segments. Sloped for numeric, stepped for Boolean. Server-default
|
||||
AggregateConfiguration (client's ignored). QuestDB mirrored after a lag/lead support spike. Hand-computed literals
|
||||
in the shared conformance suite.
|
||||
|
||||
## #668
|
||||
`hist.SampleModified` (schema v2), prior values via `OUTPUT deleted.* INTO` in the same statement;
|
||||
`HistoryUpdateContext(Principal, ApiKeyId)` threaded into updater methods; Insert operations logged (empty prior);
|
||||
retention purges never logged but #667 purges `SampleModified`; `ReadModifiedAsync` serves `IsReadModified=true`
|
||||
with continuation points, authorized by the HistoryRead gate. QuestDB: append-only `sample_modified` written before
|
||||
the change (orphan-audit-row-on-crash, documented deviation).
|
||||
@@ -0,0 +1,70 @@
|
||||
# Follow-up batch #674–#678 — design (2026-10-11)
|
||||
|
||||
Follow-ups filed after the #667–#673 batch. Base: master `bc03adb1`. User-approved; recommended choices throughout.
|
||||
Investigation found three new defects (client endpoint port not rewritten; role overlays never load on
|
||||
cluster-role nodes; Gateway Count/Average mode mapping) — the first two are fixed here, the third is filed.
|
||||
|
||||
| Lane | Scope | Model |
|
||||
|---|---|---|
|
||||
| L1 | #674 HistoryUpdate posture | opus |
|
||||
| L2 | #675 reverse event paging | opus |
|
||||
| L3 | #677 aggregates | opus |
|
||||
| L4 | client (#677.5, #678.5/.6, #674.6) | sonnet |
|
||||
| L5 | #676 OpcUaClient | opus |
|
||||
| L6 | #678.1–.4 + role-overlay bug | sonnet |
|
||||
|
||||
## L1 — #674
|
||||
- **674.1:** spike on QuestDB 10.0.1 showed TTL is judged against `min(maxTs, wall clock)` (`cairo.ttl.use.wall.clock=true`
|
||||
default) — a 2066 row evicted nothing. Add a QuestDB integration test pinning that + a docs note. HistoryUpdate refuses a
|
||||
SourceTimestamp beyond `now + ServerHistorian:HistoryUpdateMaxFutureSkew` (default 1 day) with `BadOutOfRange`; the
|
||||
recorder is not bounded.
|
||||
- **674.2:** SqlServer base schema requirement back to v1; `SupportsModifiedTrail` (v ≥ 2). On v1: modified reads and
|
||||
HistoryUpdate ⇒ `BadHistoryOperationUnsupported` (nothing sent), retention skips `SampleModified`, one Warning per boot.
|
||||
- **674.3:** HistoryUpdate Primary-gated (off-Primary ⇒ `BadInvalidState`, same shape as `NativeAlarmAckPrimaryGate`).
|
||||
Retry duplicates on QuestDB documented; no DEDUP key.
|
||||
- 674.4/674.5 documented only.
|
||||
|
||||
## L2 — #675
|
||||
- Reverse event reads: `reverse` flag on `IHistorianDataSource.ReadEventsAsync`; SqlServer/QuestDB order DESC and take
|
||||
the top N; direction-aware cursor (reuse raw `SliceTieCluster(reverse:)`); Gateway ⇒ `BadHistoryOperationUnsupported`
|
||||
for reverse.
|
||||
- End-only event read ⇒ backward (mirrors raw).
|
||||
- Offline test pinning `DateTime.MaxValue` through the Gateway proto/FILETIME encoding; live AVEVA leg stays open (VPN).
|
||||
|
||||
## L3 — #677
|
||||
- Reverse processed reads per SDK/Part 13: buckets anchored at the later (start) time stepping back, partial bucket
|
||||
earliest, bucket timestamp = later edge. Parity test vs SDK `AggregateCalculator`. Client-visible, release-noted.
|
||||
- Partial bit also when the data starts inside a bucket (no bounding prior).
|
||||
- `AggregateConfiguration`: `UseServerCapabilitiesDefaults` or values equal to the server defaults ⇒ served; otherwise
|
||||
per-node `BadAggregateConfigurationRejected`.
|
||||
- Gateway provisioner sends `IntegralDivisor = 1` (proto3 default 0). Live magnitude check stays open (VPN).
|
||||
|
||||
## L4 — client
|
||||
- `EndpointSelector.SelectBest` rewrites host **and** port of the advertised endpoint (fixes rig `:4841` cert error).
|
||||
- Client aggregate enum/mapper/CLI/UI gain TimeAverage, TimeAverage2, Total, Total2, Interpolative.
|
||||
- History read refusal ⇒ one-line `CommandException`, no stack trace.
|
||||
- `NodeManagerHistoryReadTests` non-historized case sets `IsReadModified=false`.
|
||||
|
||||
## L5 — #676
|
||||
- Driver endpoint host+port rewrite; in-process TCP-stall proxy wire test (in-place blip, same session, Good republish,
|
||||
no regression).
|
||||
- Remove dead `ReconnectPeriod` (options, AdminUI form, docs, tests); stored JSON with it still parses.
|
||||
- New opt-in `IDriverConnectionLossSource` (Core.Abstractions): raised once per session by OpcUaClient only on a
|
||||
definitive loss (`BadSessionIdInvalid`/`BadSessionClosed`/`BadSessionNotActivated`/subscription lost);
|
||||
`DriverInstanceActor` (attach PreStart, detach PostStop) turns it into `DisconnectObserved` only while Connected.
|
||||
Ambiguous codes keep going through the #527 watchdog.
|
||||
- Information-level per-recovery replay-drop summary. Equal-timestamp drop documented.
|
||||
|
||||
## L6 — small
|
||||
- Retention: mid-command cancel test + `ClearPool` fallback test with falsification control.
|
||||
- #672 tests: initial-state unreachable path; strengthen `Self_unreachable_is_ignored`; edge-triggered Debug log.
|
||||
- FOCAS `WireBackendTests`: wait for motor speeds too; `WaitFor` fails on timeout.
|
||||
- QuestDB TTL-cleared log wording; `docker-dev/docker-compose.debug.yml` Debug recipe.
|
||||
- Role overlay suffix built from plain roles only — after auditing each `appsettings.<role>.json` and stripping anything
|
||||
that would change production behaviour on activation.
|
||||
|
||||
## Live gates (docker-dev)
|
||||
1. opc-plc `docker restart` recovers in seconds (was ~100 s); pause-blip re-check.
|
||||
2. Client.CLI connects to `:4841`–`:4845`.
|
||||
3. HistoryUpdate refused on the Secondary, accepted on the Primary; v1-schema DB keeps historizing.
|
||||
4. Reverse event paging and reverse processed reads over the wire, SqlServer + QuestDB.
|
||||
@@ -0,0 +1,36 @@
|
||||
# Missing-tag and Gateway performance fixes — 2026-10-13
|
||||
|
||||
Follow-up to pushed `97668d26` / PR #699. Implements #653 and the remaining #696 items 2/3 under the approved bounded design.
|
||||
|
||||
## Changes
|
||||
|
||||
- **#653:** Raw, Processed and AtTime check tag presence before history RPCs. The bounded 4,096-name cache uses a 5-second negative and 60-second positive lifetime. Only explicit absence or NotFound is cached as missing; failures and cancellation propagate. Raw is empty; AtTime and Processed retain requested/bucket timestamps with null/BadNoData. Unsupported aggregate validation runs first.
|
||||
- **Live compatibility adjustment:** the deployed gateway misparses missing `GetTagMetadata` replies as `Internal` (`InvalidDataException: GetTagInfo response ended unexpectedly`). The adapter therefore checks the filtered tag catalog and compares exact names, fully draining the stream before declaring presence/absence. It never turns the parser error into a missing tag. Wildcard matches alone do not prove presence, and a trailing stream fault cannot enter the cache.
|
||||
- **#696 item 2:** a truncated remainder probe seeds strict-ceiling reverse discovery with its observed rows/density; gap searches use the available discovery budget. Complete segments alone contribute to the newest-first page. Regression: a 7,200-row stopped 1 Hz tag, ceiling 1,000, requested latest three one day later, previously 17 raw calls; now meets a maximum of 12 with exactly 7199/7198/7197 returned. Ordinary page sizing and event discovery behavior are unchanged.
|
||||
- **#696 item 3:** per-sample aggregates reuse a ceiling refusal for 30 seconds instead of immediately re-probing on every full stream. The next incomplete aggregate read after expiry notices a raised ceiling; a lowered ceiling is re-learned immediately. Raw and AtTime re-probes remain immediate. During the cooldown an oversized aggregate remains BadTooManyOperations, never a truncated result.
|
||||
|
||||
## Verification evidence
|
||||
|
||||
New regressions failed before fixes: all initial 12 cases failed (missing preflight/caching, 17 discovery reads, duplicate aggregate refusal). The four catalog wire cases failed with the metadata adapter and pass with the catalog adapter. Final Gateway suite: 434 passed, six environment-gated skips.
|
||||
|
||||
Independent read-only review found no important defects. Concurrent cold requests can duplicate a metadata/catalog check; cache access and capacity mutations are locked. No single-flight guarantee was introduced. Catalog RPC contract streams all matches; the upstream SDK pages the native tag-query results, with no configured browse result ceiling.
|
||||
|
||||
## Live #653 confirmation
|
||||
|
||||
Measured the updated GatewayHistorianDataSource through the real adapter and local gateway against the existing remote AVEVA connection. Temporary historian:read key had a two-hour expiry and was revoked after the run. No historian writes/provisioning and no gateway deployment changes.
|
||||
|
||||
Nonexistent tag: `OtOpcUa.Issue653.Nonexistent.20261013`.
|
||||
|
||||
| Operation, five consecutive runs | Elapsed milliseconds | Result |
|
||||
|---|---|---|
|
||||
| Raw | 228, 0, 0, 0, 0 | empty |
|
||||
| Count processed | 2, 0, 0, 0, 0 | six null/BadNoData buckets |
|
||||
| AtTime | 1, 0, 0, 0, 0 | null/BadNoData at requested time |
|
||||
|
||||
These used one data source/cache: the first Raw call is cold, subsequent calls are cached. The prior direct-client measurement took ~30.6 seconds for missing-tag Raw/Aggregate and ~61.3 seconds for AtTime. Direct client RPCs are not modified; the fast-fail protection is in OtOpcUa.
|
||||
|
||||
**Control limitation:** SysTimeSec catalog presence succeeded, but its Raw RPC twice timed out upstream with DeadlineExceeded at ~30 seconds. The data source propagated IOException/communication failure rather than returning no data. A fresh control using the original package client (no data-source presence checks) also timed out with DeadlineExceeded after 30,656 ms, confirming this timeout is independent of the new checks. The earlier direct-client control had succeeded. This run therefore verifies missing-tag fast-fail and failure propagation, not healthy-tag live history behavior. No end-to-end deployed OPC UA claim is made.
|
||||
|
||||
#694 remains open for native aggregate rerun after historiangw#27. No sibling gateway code changed.
|
||||
|
||||
Full final-tree solution: 14,600 passed, 609 skipped, 0 failed across 62 suites. `dotnet test ZB.MOM.WW.OtOpcUa.slnx --verbosity quiet`, exit 0. Includes 122 passing OPC UA integration tests and 268 passing host integration tests (12 host skips). Environment-gated suites are included in the skip count.
|
||||
@@ -0,0 +1,28 @@
|
||||
# Historian follow-up results — 2026-10-13
|
||||
|
||||
Scope: approved bounded fixes for #695–#698, plus live measurement of #653. Base: `66a2b0c3`.
|
||||
|
||||
- #695: spend the remaining learned-ceiling room on crowding reads; reduce crowded reverse-read windows using the observed pre-window count. Regression cases cover ceilings 1,709 and 1,369, and reverse paging through a crowded millisecond.
|
||||
- #696 item 1: probe the future interval before starting the prior-sample search near now. Preserve stored future samples, requested timestamps and Uncertain status. Dense future probes still use the existing bounded walk. Items 2 and 3 remain open (stopped-tag strict-ceiling cost and aggregate ceiling re-probes).
|
||||
- #697: actual Gateway, SQL Server and QuestDB transport failures become `BadNoCommunication` on every HistoryRead arm. Cancellation remains `BadTimeout`; unsupported operations retain `BadHistoryOperationUnsupported`. Tests include real refused database connections, repeated reads and nested gRPC failures.
|
||||
- #698: normalize Raw, Modified and AtTime scalar values to the node DataType. Conversion failure gives per-sample null/`BadTypeMismatch`; arrays, audit metadata and timestamps survive. Processed results retain aggregate-specific types.
|
||||
- #694 item 3 remains blocked on historiangw#27. No upstream code was changed.
|
||||
|
||||
## Live #653 measurement
|
||||
|
||||
Five sequential calls per operation and tag, using client 0.3.0 against the running local gateway (`localhost:5221`) with its existing remote AVEVA connection. Commissioned a two-hour `historian:read` key, then revoked it after measurement. No historian writes or provisioning performed.
|
||||
|
||||
| Operation | SysTimeSec, elapsed ms | Nonexistent tag, elapsed ms |
|
||||
|---|---|---|
|
||||
| Metadata | 299, 85, 81, 82, 48 (success) | 89, 53, 41, 77, 76 (gateway exception) |
|
||||
| Raw | 143, 99, 145, 133, 159 (success) | 30260, 30634, 30675, 30754, 30628 (gateway exception) |
|
||||
| Aggregate | 271, 275, 309, 249, 304 (success) | 30564, 30605, 30583, 30575, 30806 (gateway exception) |
|
||||
| AtTime | 140, 88, 89, 127, 92 (success) | 61074, 61330, 61601, 61249, 61271 (gateway exception) |
|
||||
|
||||
Nonexistent tag: `OtOpcUa.Issue653.Nonexistent.20261013`. The delay is still present. Recommend a metadata pre-check with a short negative cache, caching only explicit missing-tag outcomes and never authentication, communication or arbitrary errors. This is follow-up implementation; #653 remains open. Direct gateway AtTime above differs from OtOpcUa's raw-based AtTime evaluator.
|
||||
|
||||
## Consumer and validation notes
|
||||
|
||||
ScadaBridge source and tests contain no HistoryRead consumer; no consumer change required. Independent review found a future-stored-sample edge case, which was fixed and covered before final verification. No deployment or live end-to-end OPC UA verification of these new fixes is claimed; the live run measures #653 through the gateway client.
|
||||
|
||||
Full solution: 14,581 passed, 609 skipped, 0 failed across 62 test suites. Command: `dotnet test ZB.MOM.WW.OtOpcUa.slnx --verbosity quiet`, exit 0. Skips include environment-gated integration suites.
|
||||
@@ -884,6 +884,13 @@ HistoryUpdate rewrites recorded history, so unlike HistoryRead it is gated **wha
|
||||
- **Flag off (default):** the session's API key is re-checked on the call (`DataPlaneAuthorizer.AuthorizeKeyUsable`, Gitea #563 — an expired or disabled key loses its activation-time roles), then the session must carry the **`HistoryUpdate` role** (`OpcUaDataPlaneRoles.HistoryUpdate`). `ApiKeyInterimRoleMapper` confers it **only from a `Cluster`-scoped grant** carrying `NodePermissions.HistoryUpdate` — narrower-scoped grants confer nothing, for the same reason as `WriteOperate` (honouring a flag without honouring its scope would widen it). A key holding only `WriteOperate` is denied `BadUserAccessDenied`.
|
||||
- **Flag on:** `DataPlaneAuthorizer.Authorize(identity, OpcUaOperation.HistoryUpdate, nodeId, realm)` decides per node and **replaces** the role check (never chained) — a session with no roles succeeds on the strength of a scoped `HistoryUpdate` grant, and a `WriteOperate` grant does not help.
|
||||
- **Availability is separate from authorization.** A node accepts HistoryUpdate only if it is historized **and** the selected historian backend has an updater when the node materialises (the `HistoryWrite` AccessLevel bit). `Backend=Gateway` has none, so it answers `BadHistoryOperationUnsupported` for every node whatever the grant. Operations, result codes and unsupported cases: [`docs/Historian.md` §HistoryUpdate](Historian.md#historyupdate).
|
||||
- **Primary-only, after authorization (Gitea #674).** An authorized HistoryUpdate on a node that is not the redundancy Primary is refused `BadInvalidState` (the native-alarm-ack verdict, #597); an unauthorized one is told `BadUserAccessDenied` first, so the refusal never reveals the node's role to a caller who may not update.
|
||||
- **Every applied change is attributed (Part 11 modified-value history, Gitea #668).** The session's principal (the API key's
|
||||
`NodeAcl` principal, or the reserved `anonymous`) and key id are passed to the backend with every HistoryUpdate and recorded in the
|
||||
modification trail next to the prior value; a modified-value HistoryRead (`IsReadModified = true`) returns it as
|
||||
`ModificationInfo.UserName`. Reading the trail is gated by the **same `HistoryRead` decision** as a raw read — there is no separate
|
||||
permission — so with `Security:Acl:Enabled` off it is as ungated as any HistoryRead, and anyone who may read a node's history can see
|
||||
who rewrote it and what it held before. See [`docs/Historian.md` §Modified-value history](Historian.md#modified-value-history-part-11).
|
||||
|
||||
### Evaluator — `PermissionTrie`
|
||||
|
||||
|
||||
@@ -815,7 +815,6 @@ Already used by Client.Shared in the existing codebase.
|
||||
| `Password` | string? | — | For Username auth |
|
||||
| `SessionTimeoutMs` | int | 120000 | Server-negotiated session timeout |
|
||||
| `KeepAliveIntervalMs` | int | 5000 | Session keep-alive interval |
|
||||
| `ReconnectPeriodMs` | int | 5000 | Initial reconnect delay |
|
||||
| `BrowseStrategy` | enum | Full | `Full` (browse all at startup), `Lazy`, `Hybrid` |
|
||||
| `BrowseRoot` | string? | — | Optional root NodeId to mirror (default: ObjectsFolder) |
|
||||
| `AutoAcceptCertificates` | bool | false | Accept untrusted server certificates |
|
||||
|
||||
@@ -47,7 +47,7 @@ public class HistoryReadCommand : CommandBase
|
||||
/// <summary>
|
||||
/// Gets the optional aggregate name used when the operator wants processed history instead of raw samples.
|
||||
/// </summary>
|
||||
[CommandOption("aggregate", Description = "Aggregate function: Average, Minimum, Maximum, Count, Start, End, StandardDeviation")]
|
||||
[CommandOption("aggregate", Description = "Aggregate function: Average, Minimum, Maximum, Count, Start, End, StandardDeviation, TimeAverage, TimeAverage2, Total, Total2, Interpolative")]
|
||||
public string? Aggregate { get; init; }
|
||||
|
||||
/// <summary>
|
||||
@@ -56,6 +56,13 @@ public class HistoryReadCommand : CommandBase
|
||||
[CommandOption("interval", Description = "Processing interval in milliseconds for aggregates")]
|
||||
public double IntervalMs { get; init; } = 3600000;
|
||||
|
||||
/// <summary>
|
||||
/// Gets a value indicating whether to read modified-value history (Part 11 IsReadModified) instead of raw samples:
|
||||
/// the prior value each HistoryUpdate replaced or deleted, with its update type, user and modification time.
|
||||
/// </summary>
|
||||
[CommandOption("modified", Description = "Read modified-value history (what HistoryUpdate changed: prior value, update type, user, time) instead of raw samples")]
|
||||
public bool Modified { get; init; }
|
||||
|
||||
/// <inheritdoc />
|
||||
public override async ValueTask ExecuteAsync(IConsole console)
|
||||
{
|
||||
@@ -64,6 +71,8 @@ public class HistoryReadCommand : CommandBase
|
||||
throw new CommandException($"--max must be greater than 0 (was {MaxValues}).");
|
||||
if (!string.IsNullOrEmpty(Aggregate) && IntervalMs <= 0)
|
||||
throw new CommandException($"--interval must be greater than 0 (was {IntervalMs}).");
|
||||
if (Modified && !string.IsNullOrEmpty(Aggregate))
|
||||
throw new CommandException("--modified and --aggregate cannot be combined.");
|
||||
|
||||
NodeId nodeId;
|
||||
try
|
||||
@@ -119,6 +128,30 @@ public class HistoryReadCommand : CommandBase
|
||||
var ct = console.RegisterCancellationHandler();
|
||||
(service, _) = await CreateServiceAndConnectAsync(ct);
|
||||
|
||||
if (Modified)
|
||||
{
|
||||
await console.Output.WriteLineAsync(
|
||||
$"Modified history for {NodeId} ({start:yyyy-MM-dd HH:mm} -> {end:yyyy-MM-dd HH:mm})");
|
||||
var entries = await service.HistoryReadModifiedAsync(nodeId, start, end, MaxValues, ct);
|
||||
await console.Output.WriteLineAsync();
|
||||
await console.Output.WriteLineAsync(
|
||||
$"{"Timestamp",-35} {"Prior value",-15} {"Status",-10} {"Update",-8} {"User",-20} {"Modified at"}");
|
||||
foreach (var e in entries)
|
||||
{
|
||||
var dv = e.Value;
|
||||
var status = StatusCode.IsGood(dv.StatusCode) ? "Good"
|
||||
: StatusCode.IsBad(dv.StatusCode) ? "Bad"
|
||||
: "Uncertain";
|
||||
await console.Output.WriteLineAsync(
|
||||
$"{dv.SourceTimestamp.ToString("O"),-35} {dv.Value?.ToString() ?? "(none)",-15} {status,-10} " +
|
||||
$"{e.UpdateType,-8} {e.UserName ?? "",-20} {e.ModificationTime:O}");
|
||||
}
|
||||
|
||||
await console.Output.WriteLineAsync();
|
||||
await console.Output.WriteLineAsync($"{entries.Count} modifications returned.");
|
||||
return;
|
||||
}
|
||||
|
||||
IReadOnlyList<DataValue> values;
|
||||
|
||||
if (string.IsNullOrEmpty(Aggregate))
|
||||
@@ -150,6 +183,12 @@ public class HistoryReadCommand : CommandBase
|
||||
await console.Output.WriteLineAsync();
|
||||
await console.Output.WriteLineAsync($"{values.Count} values returned.");
|
||||
}
|
||||
catch (ServiceResultException ex) when (ex.InnerException is null)
|
||||
{
|
||||
// e.g. HistoryReadStatus.ThrowIfRefused: a refusal is an operator-facing outcome, not a crash.
|
||||
// A wrapped failure (certificate validation, identity rejection on connect) keeps its full detail.
|
||||
throw new CommandException(ex.Message);
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (service != null)
|
||||
@@ -171,8 +210,13 @@ public class HistoryReadCommand : CommandBase
|
||||
"start" or "first" => AggregateType.Start,
|
||||
"end" or "last" => AggregateType.End,
|
||||
"standarddeviation" or "stddev" or "stdev" => AggregateType.StandardDeviation,
|
||||
"timeaverage" or "timeavg" => AggregateType.TimeAverage,
|
||||
"timeaverage2" or "timeavg2" => AggregateType.TimeAverage2,
|
||||
"total" => AggregateType.Total,
|
||||
"total2" => AggregateType.Total2,
|
||||
"interpolative" or "interp" => AggregateType.Interpolative,
|
||||
_ => throw new ArgumentException(
|
||||
$"Unknown aggregate: '{name}'. Supported: Average, Minimum, Maximum, Count, Start, End, StandardDeviation")
|
||||
$"Unknown aggregate: '{name}'. Supported: Average, Minimum, Maximum, Count, Start, End, StandardDeviation, TimeAverage, TimeAverage2, Total, Total2, Interpolative")
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
@@ -85,17 +85,22 @@ internal static class EndpointSelector
|
||||
$"No endpoint found with security mode '{requestedMode}'. Available endpoints: {available}");
|
||||
}
|
||||
|
||||
// Rewrite endpoint URL hostname to match user-supplied hostname. Necessary
|
||||
// when the OPC UA server returns a discovery URL using a different hostname
|
||||
// (e.g. internal DNS name) than the one the operator routed to.
|
||||
// Rewrite the endpoint URL host AND port to the user-supplied ones (as the SDK's
|
||||
// CoreClientUtils.SelectEndpoint does). A server behind a port mapping or internal DNS
|
||||
// advertises its own host:port (docker-dev containers all say :4840); following that would
|
||||
// open the session on a different server and fail the secure-channel certificate check.
|
||||
// Only when the scheme matches: a different scheme (https vs opc.tcp) is a different transport.
|
||||
var serverUri = new Uri(best.EndpointUrl);
|
||||
var requestedUri = new Uri(endpointUrl);
|
||||
if (serverUri.Host != requestedUri.Host)
|
||||
// A URL typed without a port (opc.tcp has no registered default, so Port is -1) keeps the advertised port.
|
||||
var port = requestedUri.Port > 0 ? requestedUri.Port : serverUri.Port;
|
||||
if (string.Equals(serverUri.Scheme, requestedUri.Scheme, StringComparison.OrdinalIgnoreCase)
|
||||
&& (serverUri.Host != requestedUri.Host || serverUri.Port != port))
|
||||
{
|
||||
var builder = new UriBuilder(best.EndpointUrl) { Host = requestedUri.Host };
|
||||
var builder = new UriBuilder(best.EndpointUrl) { Host = requestedUri.Host, Port = port };
|
||||
best.EndpointUrl = builder.ToString();
|
||||
Logger.Debug("Rewrote endpoint host from {ServerHost} to {RequestedHost}", serverUri.Host,
|
||||
requestedUri.Host);
|
||||
Logger.Debug("Rewrote endpoint from {ServerHost}:{ServerPort} to {RequestedHost}:{RequestedPort}",
|
||||
serverUri.Host, serverUri.Port, requestedUri.Host, port);
|
||||
}
|
||||
|
||||
return best;
|
||||
|
||||
@@ -206,6 +206,74 @@ internal sealed class DefaultSessionAdapter : ISessionAdapter
|
||||
return allValues;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<IReadOnlyList<Models.HistoryModifiedEntry>> HistoryReadModifiedAsync(
|
||||
NodeId nodeId, DateTime startTime, DateTime endTime, int maxValues, CancellationToken ct)
|
||||
{
|
||||
var details = new ReadRawModifiedDetails
|
||||
{
|
||||
StartTime = startTime,
|
||||
EndTime = endTime,
|
||||
NumValuesPerNode = (uint)maxValues,
|
||||
IsReadModified = true,
|
||||
ReturnBounds = false
|
||||
};
|
||||
|
||||
var nodesToRead = new HistoryReadValueIdCollection
|
||||
{
|
||||
new HistoryReadValueId { NodeId = nodeId }
|
||||
};
|
||||
|
||||
var all = new List<Models.HistoryModifiedEntry>();
|
||||
byte[]? continuationPoint = null;
|
||||
|
||||
do
|
||||
{
|
||||
if (continuationPoint != null)
|
||||
nodesToRead[0].ContinuationPoint = continuationPoint;
|
||||
|
||||
var response = await _session.HistoryReadAsync(
|
||||
null,
|
||||
new ExtensionObject(details),
|
||||
TimestampsToReturn.Both,
|
||||
false,
|
||||
nodesToRead,
|
||||
ct).ConfigureAwait(false);
|
||||
|
||||
var results = response.Results;
|
||||
if (results == null || results.Count == 0)
|
||||
break;
|
||||
|
||||
var result = results[0];
|
||||
HistoryReadStatus.ThrowIfRefused(result.StatusCode, nodeId);
|
||||
|
||||
if (result.HistoryData is ExtensionObject ext && ext.Body is HistoryModifiedData modified)
|
||||
{
|
||||
for (var i = 0; i < modified.DataValues.Count; i++)
|
||||
{
|
||||
var info = i < modified.ModificationInfos.Count ? modified.ModificationInfos[i] : null;
|
||||
all.Add(new Models.HistoryModifiedEntry(
|
||||
modified.DataValues[i],
|
||||
info?.ModificationTime ?? DateTime.MinValue,
|
||||
info?.UpdateType ?? default,
|
||||
info?.UserName));
|
||||
}
|
||||
}
|
||||
|
||||
continuationPoint = result.ContinuationPoint;
|
||||
} while (continuationPoint != null && continuationPoint.Length > 0 && all.Count < maxValues);
|
||||
|
||||
// A read stopped by the client's own cap leaves a server cursor behind; release it.
|
||||
if (continuationPoint is { Length: > 0 })
|
||||
{
|
||||
nodesToRead[0].ContinuationPoint = continuationPoint;
|
||||
await _session.HistoryReadAsync(null, new ExtensionObject(details), TimestampsToReturn.Both, true,
|
||||
nodesToRead, ct).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
return all;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<IReadOnlyList<DataValue>> HistoryReadAggregateAsync(
|
||||
NodeId nodeId, DateTime startTime, DateTime endTime, NodeId aggregateId, double intervalMs,
|
||||
|
||||
@@ -110,6 +110,20 @@ internal interface ISessionAdapter : IDisposable
|
||||
Task<IReadOnlyList<DataValue>> HistoryReadRawAsync(NodeId nodeId, DateTime startTime, DateTime endTime,
|
||||
int maxValues, CancellationToken ct = default);
|
||||
|
||||
/// <summary>
|
||||
/// Reads modified-value history (Part 11, <c>IsReadModified = true</c>): the prior values and modification info
|
||||
/// the server recorded for every HistoryUpdate in the window, following continuation points.
|
||||
/// </summary>
|
||||
/// <param name="nodeId">The historized node.</param>
|
||||
/// <param name="startTime">Window start.</param>
|
||||
/// <param name="endTime">Window end.</param>
|
||||
/// <param name="maxValues">The maximum number of entries to return.</param>
|
||||
/// <param name="ct">The cancellation token.</param>
|
||||
/// <returns>The modification entries in server order.</returns>
|
||||
Task<IReadOnlyList<Models.HistoryModifiedEntry>> HistoryReadModifiedAsync(NodeId nodeId, DateTime startTime,
|
||||
DateTime endTime, int maxValues, CancellationToken ct = default) =>
|
||||
throw new NotSupportedException("This session adapter does not support modified-value history reads.");
|
||||
|
||||
/// <summary>
|
||||
/// Sends a HistoryUpdate request for the supplied details.
|
||||
/// </summary>
|
||||
|
||||
@@ -24,6 +24,11 @@ public static class AggregateTypeMapper
|
||||
AggregateType.Start => ObjectIds.AggregateFunction_Start,
|
||||
AggregateType.End => ObjectIds.AggregateFunction_End,
|
||||
AggregateType.StandardDeviation => ObjectIds.AggregateFunction_StandardDeviationPopulation,
|
||||
AggregateType.TimeAverage => ObjectIds.AggregateFunction_TimeAverage,
|
||||
AggregateType.TimeAverage2 => ObjectIds.AggregateFunction_TimeAverage2,
|
||||
AggregateType.Total => ObjectIds.AggregateFunction_Total,
|
||||
AggregateType.Total2 => ObjectIds.AggregateFunction_Total2,
|
||||
AggregateType.Interpolative => ObjectIds.AggregateFunction_Interpolative,
|
||||
_ => throw new ArgumentOutOfRangeException(nameof(aggregate), aggregate, "Unknown AggregateType value.")
|
||||
};
|
||||
}
|
||||
|
||||
@@ -193,6 +193,20 @@ public interface IOpcUaClientService : IDisposable
|
||||
Task<IReadOnlyList<DataValue>> HistoryReadRawAsync(NodeId nodeId, DateTime startTime, DateTime endTime,
|
||||
int maxValues = 1000, CancellationToken ct = default);
|
||||
|
||||
/// <summary>
|
||||
/// Reads modified-value history (Part 11, <c>IsReadModified = true</c>): what each HistoryUpdate in the window
|
||||
/// replaced or deleted, with the server's modification time, update type and user name.
|
||||
/// </summary>
|
||||
/// <param name="nodeId">The historized node.</param>
|
||||
/// <param name="startTime">Window start.</param>
|
||||
/// <param name="endTime">Window end.</param>
|
||||
/// <param name="maxValues">The maximum number of entries to return.</param>
|
||||
/// <param name="ct">The cancellation token.</param>
|
||||
/// <returns>The modification entries in server order.</returns>
|
||||
Task<IReadOnlyList<HistoryModifiedEntry>> HistoryReadModifiedAsync(NodeId nodeId, DateTime startTime,
|
||||
DateTime endTime, int maxValues = 1000, CancellationToken ct = default) =>
|
||||
throw new NotSupportedException("This client service does not support modified-value history reads.");
|
||||
|
||||
/// <summary>
|
||||
/// Sends a HistoryUpdate request (insert/replace/update or delete) for the supplied details.
|
||||
/// </summary>
|
||||
|
||||
@@ -24,5 +24,20 @@ public enum AggregateType
|
||||
End,
|
||||
|
||||
/// <summary>Population standard deviation of values in the interval.</summary>
|
||||
StandardDeviation
|
||||
StandardDeviation,
|
||||
|
||||
/// <summary>Time-weighted average, interpolated bounds (Part 13 TimeAverage).</summary>
|
||||
TimeAverage,
|
||||
|
||||
/// <summary>Time-weighted average, stepped bounds (Part 13 TimeAverage2).</summary>
|
||||
TimeAverage2,
|
||||
|
||||
/// <summary>Time-weighted integral over the interval (Part 13 Total).</summary>
|
||||
Total,
|
||||
|
||||
/// <summary>Time-weighted integral, stepped bounds (Part 13 Total2).</summary>
|
||||
Total2,
|
||||
|
||||
/// <summary>Interpolated value at the interval start (Part 13 Interpolative).</summary>
|
||||
Interpolative
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
using Opc.Ua;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Client.Shared.Models;
|
||||
|
||||
/// <summary>
|
||||
/// One entry of an OPC UA Part 11 modified-value history read (HistoryReadRaw with <c>IsReadModified = true</c>): the
|
||||
/// value as it was BEFORE the modification plus the server's <see cref="ModificationInfo"/>.
|
||||
/// </summary>
|
||||
/// <param name="Value">The prior value at its source timestamp (an insert had none: null value, Bad status).</param>
|
||||
/// <param name="ModificationTime">When the modification was applied.</param>
|
||||
/// <param name="UpdateType">The Part 11 update type (Insert / Replace / Update / Delete).</param>
|
||||
/// <param name="UserName">Who made the modification (the server's recorded principal).</param>
|
||||
public sealed record HistoryModifiedEntry(
|
||||
DataValue Value,
|
||||
DateTime ModificationTime,
|
||||
HistoryUpdateType UpdateType,
|
||||
string? UserName);
|
||||
@@ -559,6 +559,15 @@ public sealed class OpcUaClientService : IOpcUaClientService
|
||||
return await _session!.HistoryReadRawAsync(nodeId, startTime, endTime, maxValues, ct);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<IReadOnlyList<HistoryModifiedEntry>> HistoryReadModifiedAsync(
|
||||
NodeId nodeId, DateTime startTime, DateTime endTime, int maxValues = 1000, CancellationToken ct = default)
|
||||
{
|
||||
ThrowIfDisposed();
|
||||
ThrowIfNotConnected();
|
||||
return await _session!.HistoryReadModifiedAsync(nodeId, startTime, endTime, maxValues, ct);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<HistoryUpdateOutcome> HistoryUpdateAsync(HistoryUpdateDetails details, CancellationToken ct = default)
|
||||
{
|
||||
|
||||
@@ -53,7 +53,12 @@ public partial class HistoryViewModel : ObservableObject
|
||||
AggregateType.Count,
|
||||
AggregateType.Start,
|
||||
AggregateType.End,
|
||||
AggregateType.StandardDeviation
|
||||
AggregateType.StandardDeviation,
|
||||
AggregateType.TimeAverage,
|
||||
AggregateType.TimeAverage2,
|
||||
AggregateType.Total,
|
||||
AggregateType.Total2,
|
||||
AggregateType.Interpolative
|
||||
];
|
||||
|
||||
/// <summary>Gets a value indicating whether an aggregate read is selected.</summary>
|
||||
|
||||
@@ -80,11 +80,11 @@ public static class OtOpcUaTelemetry
|
||||
/// Session/publish-level events of an OPC UA Client driver instance (#276): the operator's view of a
|
||||
/// flapping upstream. Tagged <see cref="PollOverrunDriverInstanceTag"/> (<c>driver_instance</c>) +
|
||||
/// <c>event</c> = <c>publish_response|notification|data_change|bad_status_value|publish_error|
|
||||
/// keepalive_miss|reconnect_attempt|reconnect_success|reconnect_exhausted</c>.
|
||||
/// keepalive_miss|keepalive_recovered|stale_notification_dropped</c>.
|
||||
/// </summary>
|
||||
public static readonly Counter<long> OpcUaClientEvents =
|
||||
Meter.CreateCounter<long>("otopcua.driver.opcuaclient.events", unit: "{event}",
|
||||
description: "OPC UA Client driver session/publish events (event=publish_response|notification|data_change|bad_status_value|publish_error|keepalive_miss|reconnect_attempt|reconnect_success|reconnect_exhausted).");
|
||||
description: "OPC UA Client driver session/publish events (event=publish_response|notification|data_change|bad_status_value|publish_error|keepalive_miss|keepalive_recovered|stale_notification_dropped).");
|
||||
|
||||
// ---------------- VirtualTag / ScriptedAlarm engines ----------------
|
||||
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
namespace ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
|
||||
// OPC UA Part 11 modified-value history (Gitea #668): every HistoryUpdate a backend applies leaves an audit row
|
||||
// carrying the PRIOR value, who made the change and when; HistoryRead with IsReadModified=true serves them.
|
||||
|
||||
/// <summary>Who is performing a HistoryUpdate. Threaded into every <see cref="IHistorianDataUpdater"/> call.</summary>
|
||||
/// <param name="Principal">
|
||||
/// The session's authenticated principal (an API key's principal, or the reserved <c>anonymous</c>). Recorded as
|
||||
/// the modification's user name.
|
||||
/// </param>
|
||||
/// <param name="ApiKeyId">The API key id the session authenticated with, or <c>null</c> (anonymous / non-key).</param>
|
||||
public sealed record HistoryUpdateContext(string Principal, string? ApiKeyId)
|
||||
{
|
||||
/// <summary>The longest principal / key id a backend stores; longer values are truncated (never refused).</summary>
|
||||
public const int MaxIdentityLength = 200;
|
||||
|
||||
/// <summary>The principal clamped to <see cref="MaxIdentityLength"/> characters.</summary>
|
||||
public string StoredPrincipal => Clamp(Principal) ?? string.Empty;
|
||||
|
||||
/// <summary>The key id clamped to <see cref="MaxIdentityLength"/> characters.</summary>
|
||||
public string? StoredApiKeyId => Clamp(ApiKeyId);
|
||||
|
||||
private static string? Clamp(string? s) => s is { Length: > MaxIdentityLength } ? s[..MaxIdentityLength] : s;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The OPC UA Part 11 <c>HistoryUpdateType</c> of a recorded modification. The numeric values are the spec's and
|
||||
/// are persisted (SQL <c>tinyint</c>, QuestDB <c>BYTE</c>) — never renumber.
|
||||
/// </summary>
|
||||
public enum HistoryModificationType : byte
|
||||
{
|
||||
/// <summary>A sample was inserted where none existed (no prior value).</summary>
|
||||
Insert = 1,
|
||||
|
||||
/// <summary>An existing sample was replaced.</summary>
|
||||
Replace = 2,
|
||||
|
||||
/// <summary>An upsert (<c>PerformUpdateType.Update</c>): replaced a sample, or inserted one (then no prior value).</summary>
|
||||
Update = 3,
|
||||
|
||||
/// <summary>A sample was deleted (DeleteAtTime / DeleteRaw).</summary>
|
||||
Delete = 4,
|
||||
}
|
||||
|
||||
/// <summary>One recorded modification: the value as it was BEFORE the change, plus who changed it and when.</summary>
|
||||
/// <param name="Value">
|
||||
/// The prior value at its source timestamp. An insert had no prior value: <c>Value = null</c>,
|
||||
/// <c>StatusCode = BadNoData</c>.
|
||||
/// </param>
|
||||
/// <param name="ModificationTimeUtc">When the modification was applied.</param>
|
||||
/// <param name="UpdateType">The kind of modification.</param>
|
||||
/// <param name="UserName">The principal that made it.</param>
|
||||
/// <param name="ModifiedId">The backend's ordering key within one source timestamp (paging cursor).</param>
|
||||
/// <param name="SourceTimeUtc">
|
||||
/// The modified sample's source time as stored on the trail row — the paging cursor's time component. Explicit
|
||||
/// because an insert's <see cref="Value"/> has no prior value and so cannot be relied on to carry it.
|
||||
/// </param>
|
||||
public sealed record HistoryModifiedValue(
|
||||
DataValueSnapshot Value,
|
||||
DateTime ModificationTimeUtc,
|
||||
HistoryModificationType UpdateType,
|
||||
string UserName,
|
||||
long ModifiedId,
|
||||
DateTime SourceTimeUtc);
|
||||
|
||||
/// <summary>Resume position for a paged modified-value read: strictly after (or before, reversed) this key.</summary>
|
||||
/// <param name="SourceTimeUtc">The last returned modification's source time.</param>
|
||||
/// <param name="ModifiedId">The last returned modification's id.</param>
|
||||
public sealed record HistoryModifiedCursor(DateTime SourceTimeUtc, long ModifiedId);
|
||||
|
||||
/// <summary>Result of <see cref="IHistorianDataSource.ReadModifiedAsync"/>.</summary>
|
||||
/// <param name="Values">Modifications ordered by <c>(SourceTimeUtc, ModifiedId)</c> (descending for a reverse read).</param>
|
||||
/// <param name="HasMore"><c>true</c> when the cap truncated the result and more modifications remain in the window.</param>
|
||||
public sealed record HistoryModifiedReadResult(IReadOnlyList<HistoryModifiedValue> Values, bool HasMore);
|
||||
@@ -47,6 +47,24 @@ public enum HistoryAggregateType
|
||||
AnnotationCount,
|
||||
StandardDeviationPopulation,
|
||||
VariancePopulation,
|
||||
|
||||
/// <summary>Part 13 TimeAverage2: time-weighted average over the interval's non-Bad data only (#671).</summary>
|
||||
TimeAverage2,
|
||||
|
||||
/// <summary>Part 13 Total2: time-weighted integral (value·seconds) over the interval's non-Bad data only (#671).</summary>
|
||||
Total2,
|
||||
}
|
||||
|
||||
/// <summary>Shared <see cref="HistoryAggregateType"/> sets for <see cref="IHistorianDataSource.SupportedAggregates"/> (#682).</summary>
|
||||
public static class HistoryAggregateTypes
|
||||
{
|
||||
/// <summary>Every defined <see cref="HistoryAggregateType"/> — the interface default.</summary>
|
||||
public static readonly IReadOnlySet<HistoryAggregateType> All =
|
||||
System.Collections.Frozen.FrozenSet.ToFrozenSet(Enum.GetValues<HistoryAggregateType>());
|
||||
|
||||
/// <summary>No aggregates — a source with no history to aggregate (the Null historian).</summary>
|
||||
public static readonly IReadOnlySet<HistoryAggregateType> None =
|
||||
System.Collections.Frozen.FrozenSet<HistoryAggregateType>.Empty;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -66,7 +84,31 @@ public sealed class HistoryAggregateNotSupportedException : NotSupportedExceptio
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// One row returned by <see cref="IHistorianDataSource.ReadEventsAsync"/> — a historical
|
||||
/// Thrown by a history backend when a read would exceed a configured size limit (e.g. the raw-row cap of the SQL
|
||||
/// backends' time-weighted aggregates, #671). The Server HistoryRead path translates it to
|
||||
/// <c>BadTooManyOperations</c> for that node only; it is a client-request limit, not a backend fault.
|
||||
/// </summary>
|
||||
public sealed class HistoryReadLimitExceededException : InvalidOperationException
|
||||
{
|
||||
/// <summary>The configured limit that was exceeded.</summary>
|
||||
public long Limit { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #694: a raw read refused because rows BEFORE its window (inside the backend's look-back margin — a tie
|
||||
/// cluster just before a page start, the HistorianGateway's boundary row) crowded a ceiling-bounded row budget: how
|
||||
/// many such rows the read saw (a lower bound when the read was itself truncated). <c>0</c> = not a crowding refusal.
|
||||
/// The server re-reads such a page once, smaller by this many values (<c>RawPageLimit</c>).
|
||||
/// </summary>
|
||||
public int PreWindowRows { get; init; }
|
||||
|
||||
/// <summary>Creates the exception.</summary>
|
||||
/// <param name="limit">The configured limit.</param>
|
||||
/// <param name="message">Operator-facing description.</param>
|
||||
public HistoryReadLimitExceededException(long limit, string message) : base(message) => Limit = limit;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// One row returned by <see cref="IHistorianDataSource.ReadEventsAsync(string?, DateTime, DateTime, int, CancellationToken)"/> — a historical
|
||||
/// alarm/event record, not the OPC UA live-event stream. Fields match the minimum set the
|
||||
/// Server needs to populate a <c>HistoryEventFieldList</c> for HistoryReadEvents responses.
|
||||
/// </summary>
|
||||
@@ -84,8 +126,8 @@ public sealed record HistoricalEvent(
|
||||
string? Message,
|
||||
ushort Severity);
|
||||
|
||||
/// <summary>Result of a <see cref="IHistorianDataSource.ReadEventsAsync"/> call.</summary>
|
||||
/// <param name="Events">Events in chronological order by <c>EventTimeUtc</c>.</param>
|
||||
/// <summary>Result of a <see cref="IHistorianDataSource.ReadEventsAsync(string?, DateTime, DateTime, int, CancellationToken)"/> call.</summary>
|
||||
/// <param name="Events">Events ordered by <c>EventTimeUtc</c> in the read's direction — ascending, or descending for a reverse read (#675).</param>
|
||||
/// <param name="ContinuationPoint">Opaque token for the next call when more events are available; null when complete.</param>
|
||||
public sealed record HistoricalEventsResult(
|
||||
IReadOnlyList<HistoricalEvent> Events,
|
||||
|
||||
@@ -17,6 +17,12 @@ public interface IHistorianDataSource : IDisposable
|
||||
/// <summary>
|
||||
/// Read raw historical samples for a single tag over a time range.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Forward (<c>startUtc < endUtc</c>) is <c>[start, end)</c> ascending; <c>startUtc == endUtc</c> is a point read.
|
||||
/// <c>startUtc > endUtc</c> is the Part 11 reverse read <c>(end, start]</c> newest-first — but only a backend that
|
||||
/// declares <see cref="SupportsReverseRaw"/> is ever sent one (Gitea #686); the server serves a reverse read on any
|
||||
/// other backend from bounded forward reads.
|
||||
/// </remarks>
|
||||
/// <param name="fullReference">The full reference of the tag to read.</param>
|
||||
/// <param name="startUtc">The start of the time range in UTC.</param>
|
||||
/// <param name="endUtc">The end of the time range in UTC.</param>
|
||||
@@ -30,6 +36,28 @@ public interface IHistorianDataSource : IDisposable
|
||||
uint maxValuesPerNode,
|
||||
CancellationToken cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// Whether this backend can serve a reverse (newest-first) raw read itself — <see cref="ReadRawAsync"/> with
|
||||
/// <c>startUtc > endUtc</c> (Gitea #686). The server checks it before every raw read. When <c>false</c> it never
|
||||
/// sends <c>start > end</c>: it serves the reverse read — an explicit one, a resumed reverse page, and the
|
||||
/// end-only capped read Part 11 reads backward from EndTime — from a series of bounded FORWARD reads walking back
|
||||
/// from the later bound (<c>ReverseRawReader</c> in the OPC UA server). Default <c>false</c>, the safe direction: a
|
||||
/// source that only reads forward is still answered correctly. A backend declares it itself (a guard test pins that).
|
||||
/// </summary>
|
||||
bool SupportsReverseRaw => false;
|
||||
|
||||
/// <summary>
|
||||
/// The most values one capped <see cref="ReadRawAsync"/> can currently return without failing
|
||||
/// <see cref="HistoryReadLimitExceededException"/> on a BACKEND row ceiling (Gitea #694); <c>0</c> = no known limit.
|
||||
/// The server sizes its raw pages to it — a page is at most this many values and carries a continuation point when
|
||||
/// full (Part 11 allows fewer than <c>NumValuesPerNode</c> with a continuation point), instead of failing a page size
|
||||
/// the backend cannot read in one go. It may change at runtime (a strict HistorianGateway's ceiling is learned from
|
||||
/// its first refusal, #691.3), so it is read per read. A backend whose reads are never ceiling-bound returns <c>0</c>.
|
||||
/// The source still never truncates: a read asking for more than this is answered in full or refused. Default
|
||||
/// <c>0</c>; a backend declares it itself (a guard test pins that).
|
||||
/// </summary>
|
||||
int MaxRawValuesPerRead => 0;
|
||||
|
||||
/// <summary>
|
||||
/// Read processed (interval-bucketed) samples — average / min / max / count / etc.
|
||||
/// A bucket with no source data returns a sample whose
|
||||
@@ -65,6 +93,31 @@ public interface IHistorianDataSource : IDisposable
|
||||
IReadOnlyList<DateTime> timestampsUtc,
|
||||
CancellationToken cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #693: <see cref="ReadAtTimeAsync(string, IReadOnlyList{DateTime}, CancellationToken)"/> with the value type of
|
||||
/// the node being read, for a backend whose stored values do not carry their type (the Gateway: AVEVA stores every
|
||||
/// historized type as a numeric analog, so a Boolean comes back as a 0/1 Double). Such a backend uses the hint to apply
|
||||
/// the AtTime rules exactly as the typed backends do — a Boolean (or String / DateTime) node is stepped, an integer node
|
||||
/// is interpolated, rounded half away from zero and returned as the node's integer type, a floating-point node is
|
||||
/// interpolated. A backend that stores typed values ignores it: the default forwards to the hint-less overload, so
|
||||
/// SqlServer, QuestDB and the Null source need no change.
|
||||
/// </summary>
|
||||
/// <param name="fullReference">The full reference of the tag to read.</param>
|
||||
/// <param name="timestampsUtc">The list of timestamps to read values at.</param>
|
||||
/// <param name="valueType">
|
||||
/// The scalar value type of the node (<see cref="TypeCode.Boolean"/>, the integer codes, <see cref="TypeCode.Single"/>,
|
||||
/// <see cref="TypeCode.Double"/>, <see cref="TypeCode.String"/>, <see cref="TypeCode.DateTime"/>), or <c>null</c> when
|
||||
/// unknown, not scalar, or not one of those — the backend's own policy then applies.
|
||||
/// </param>
|
||||
/// <param name="cancellationToken">A cancellation token that can be used to cancel the operation.</param>
|
||||
/// <returns>One sample per requested timestamp, in the same order as <paramref name="timestampsUtc"/>.</returns>
|
||||
Task<HistoryReadResult> ReadAtTimeAsync(
|
||||
string fullReference,
|
||||
IReadOnlyList<DateTime> timestampsUtc,
|
||||
TypeCode? valueType,
|
||||
CancellationToken cancellationToken) =>
|
||||
ReadAtTimeAsync(fullReference, timestampsUtc, cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// Read historical alarm / event records — OPC UA HistoryReadEvents service.
|
||||
/// Distinct from any live event stream; sources here come from the historian's
|
||||
@@ -79,7 +132,7 @@ public interface IHistorianDataSource : IDisposable
|
||||
/// <c>uint maxValuesPerNode</c> because their OPC UA HistoryRead surface has no
|
||||
/// equivalent "use default" sentinel.
|
||||
///
|
||||
/// This surface declares <see cref="ReadAtTimeAsync"/> and <see cref="ReadEventsAsync"/>
|
||||
/// This surface declares <see cref="ReadAtTimeAsync(string, IReadOnlyList{DateTime}, CancellationToken)"/> and <see cref="ReadEventsAsync(string?, DateTime, DateTime, int, CancellationToken)"/>
|
||||
/// as required members — a server-side historian owns the full read surface.
|
||||
/// </remarks>
|
||||
/// <param name="sourceName">The source name to filter events, or null to return events from all sources.</param>
|
||||
@@ -101,6 +154,57 @@ public interface IHistorianDataSource : IDisposable
|
||||
int maxEvents,
|
||||
CancellationToken cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// Whether this backend can serve a reverse (newest-first) events read — the reverse overload of
|
||||
/// <see cref="ReadEventsAsync(string?, DateTime, DateTime, int, bool, CancellationToken)"/> (Gitea #675). The server
|
||||
/// checks it BEFORE calling the source: a reverse read (explicit <c>StartTime > EndTime</c>, or an end-only capped
|
||||
/// read, which Part 11 reads backward) on a backend that cannot is never sent the reverse overload — since Gitea
|
||||
/// #691.4 the server walks it back from bounded FORWARD reads instead (#675 refused it). Default <c>false</c>; an
|
||||
/// implementation (or a wrapper) declares it itself — a guard test pins that.
|
||||
/// </summary>
|
||||
bool SupportsReverseEvents => false;
|
||||
|
||||
/// <summary>
|
||||
/// Read historical alarm / event records in a chosen direction — the overload the server's HistoryReadEvents
|
||||
/// paging calls (Gitea #675).
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// The bounds may be given in either order; let <c>lo</c>/<c>hi</c> be the earlier/later of them. Forward
|
||||
/// (<paramref name="reverse"/> <c>false</c>) is exactly <see cref="ReadEventsAsync(string?, DateTime, DateTime, int, CancellationToken)"/>:
|
||||
/// <c>[lo, hi)</c> ascending by <c>(EventTimeUtc, EventId)</c>. Reverse is the Part 11 reverse window
|
||||
/// <c>(lo, hi]</c> in DESCENDING <c>(EventTimeUtc, EventId)</c> order, so a cap keeps the LATEST
|
||||
/// <paramref name="maxEvents"/> events — the mirror of forward, with the same tie-breaker reversed, so a
|
||||
/// resumed reverse page that re-reads its boundary timestamp sees the boundary ties in the same order.
|
||||
/// <paramref name="maxEvents"/> and the continuation-point contract are as for the forward read.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// The default forwards a forward read to the five-argument overload and throws
|
||||
/// <see cref="NotSupportedException"/> for a reverse one — a backend that cannot read newest-first (the
|
||||
/// Gateway). The server never relies on that throw: it consults <see cref="SupportsReverseEvents"/> first and walks
|
||||
/// such a read back from forward reads (#691.4), never returning the earliest events in ascending order as if
|
||||
/// they were the answer. A backend that serves reverse natively must override BOTH members.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
/// <param name="sourceName">The source name to filter events, or null to return events from all sources.</param>
|
||||
/// <param name="startUtc">One bound of the window (UTC).</param>
|
||||
/// <param name="endUtc">The other bound of the window (UTC).</param>
|
||||
/// <param name="maxEvents">The cap; non-positive uses the backend default (see the five-argument overload).</param>
|
||||
/// <param name="reverse"><c>true</c> to read newest-first over <c>(lo, hi]</c>.</param>
|
||||
/// <param name="cancellationToken">Cancellation.</param>
|
||||
/// <returns>The matching events in the read's direction, with a continuation point if truncated.</returns>
|
||||
/// <exception cref="NotSupportedException">A reverse read on a backend that cannot read newest-first.</exception>
|
||||
Task<HistoricalEventsResult> ReadEventsAsync(
|
||||
string? sourceName,
|
||||
DateTime startUtc,
|
||||
DateTime endUtc,
|
||||
int maxEvents,
|
||||
bool reverse,
|
||||
CancellationToken cancellationToken) =>
|
||||
reverse
|
||||
? throw new NotSupportedException("This historian backend cannot read events newest-first (reverse).")
|
||||
: ReadEventsAsync(sourceName, startUtc, endUtc, maxEvents, cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// Point-in-time health snapshot for diagnostics and dashboards. Pure
|
||||
/// observation; never blocks on backend I/O.
|
||||
@@ -122,4 +226,51 @@ public interface IHistorianDataSource : IDisposable
|
||||
/// Same-timestamp ties are unaffected (they are paged within the timestamp, never stepped over).
|
||||
/// </remarks>
|
||||
long TimestampResolutionTicks => 1;
|
||||
|
||||
/// <summary>
|
||||
/// The Part 13 aggregates this backend serves for a processed HistoryRead (Gitea #682) — the ONE authority for
|
||||
/// both what the server advertises under <c>HistoryServerCapabilities/AggregateFunctions</c> (i=11201) and what
|
||||
/// its HistoryRead-Processed answers <c>BadAggregateNotSupported</c> up front without calling the backend.
|
||||
/// (<c>ServerCapabilities/AggregateFunctions</c>, i=2997, is not affected: it lists every aggregate the server
|
||||
/// computes, including for live <c>AggregateFilter</c> subscriptions.)
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The default is <see cref="HistoryAggregateTypes.All"/> — "every aggregate the server can name; a member I cannot
|
||||
/// serve throws <see cref="HistoryAggregateNotSupportedException"/> at read time", which is exactly the pre-#682
|
||||
/// behaviour, so a test fake or an out-of-tree source keeps working unchanged. It is a concrete set rather than a
|
||||
/// <c>null</c> "all" sentinel so every consumer handles one shape. Every PRODUCTION backend declares its set itself
|
||||
/// (a reflection guard pins that): a wrapper that forgot to forward this would otherwise advertise every aggregate.
|
||||
/// The set is read once when the source is wired (the backend is fixed at boot by <c>ServerHistorian:Backend</c>)
|
||||
/// and again on every processed read, so it must be cheap and stable — return a static frozen set. A source that
|
||||
/// returns <c>null</c> anyway is treated as the default (all) by both consumers.
|
||||
/// </remarks>
|
||||
IReadOnlySet<HistoryAggregateType> SupportedAggregates => HistoryAggregateTypes.All;
|
||||
|
||||
/// <summary>
|
||||
/// Read modified-value history — OPC UA HistoryReadRaw with <c>IsReadModified = true</c> (Part 11, Gitea #668):
|
||||
/// every modification a HistoryUpdate applied to the tag inside the window, each carrying the PRIOR value.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Window: forward (<c>start < end</c>) <c>[start, end)</c> ascending by <c>(SourceTimeUtc, ModifiedId)</c>;
|
||||
/// reverse (<c>end < start</c>) <c>(end, start]</c> descending; <c>start == end</c> the modifications at
|
||||
/// exactly that time. <paramref name="resumeAfter"/> continues strictly past that key in the read's direction.
|
||||
/// The default throws <see cref="NotSupportedException"/> — a backend that keeps no modification trail
|
||||
/// (Gateway, Null) — which the server reports as <c>BadHistoryOperationUnsupported</c>.
|
||||
/// </remarks>
|
||||
/// <param name="fullReference">The historian tagname.</param>
|
||||
/// <param name="startUtc">Window start (UTC).</param>
|
||||
/// <param name="endUtc">Window end (UTC).</param>
|
||||
/// <param name="maxValues">Cap on returned modifications; <c>0</c> is unbounded.</param>
|
||||
/// <param name="resumeAfter">Resume cursor from a previous page, or <c>null</c> for the first page.</param>
|
||||
/// <param name="cancellationToken">Cancellation.</param>
|
||||
/// <returns>The modifications, with <see cref="HistoryModifiedReadResult.HasMore"/> set when truncated.</returns>
|
||||
/// <exception cref="NotSupportedException">The backend keeps no modified-value history.</exception>
|
||||
Task<HistoryModifiedReadResult> ReadModifiedAsync(
|
||||
string fullReference,
|
||||
DateTime startUtc,
|
||||
DateTime endUtc,
|
||||
uint maxValues,
|
||||
HistoryModifiedCursor? resumeAfter,
|
||||
CancellationToken cancellationToken) =>
|
||||
throw new NotSupportedException("This historian backend keeps no modified-value history.");
|
||||
}
|
||||
|
||||
@@ -17,12 +17,26 @@ public enum HistoryUpdateMode
|
||||
/// Optional historian capability behind OPC UA HistoryUpdate. A backend that cannot modify history
|
||||
/// supplies no updater (the server then reports HistoryUpdate as unsupported).
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <b>Modified-value history (Part 11, Gitea #668).</b> Every sample an implementation inserts, replaces or deletes
|
||||
/// MUST leave one modification record carrying the prior value (none for an insert), the
|
||||
/// <see cref="HistoryUpdateContext.Principal"/>, the modification time and the <see cref="HistoryModificationType"/>,
|
||||
/// readable through <see cref="IHistorianDataSource.ReadModifiedAsync"/>. A refused item (any non-Good code) leaves
|
||||
/// no record. Live recorder writes and retention purges never leave one.
|
||||
/// <para>
|
||||
/// <b>Cannot update at all</b> (Gitea #674): an implementation that is wired but cannot perform any update right
|
||||
/// now — the SqlServer backend on a schema without the modification trail — throws
|
||||
/// <see cref="NotSupportedException"/> from every method without touching storage. The server reports it as a
|
||||
/// node-level <c>BadHistoryOperationUnsupported</c>, the same as <see cref="IHistorianDataSource.ReadModifiedAsync"/>.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public interface IHistorianDataUpdater
|
||||
{
|
||||
/// <summary>Inserts, replaces or upserts samples for one tag.</summary>
|
||||
/// <param name="tag">Historian tag name.</param>
|
||||
/// <param name="mode">Update mode.</param>
|
||||
/// <param name="values">Samples to apply.</param>
|
||||
/// <param name="context">Who is making the change (recorded in the modification trail).</param>
|
||||
/// <param name="ct">Cancellation token.</param>
|
||||
/// <returns>
|
||||
/// One status code per value, in order: Insert gives <c>GoodEntryInserted</c> or <c>BadEntryExists</c>;
|
||||
@@ -30,20 +44,24 @@ public interface IHistorianDataUpdater
|
||||
/// <c>GoodEntryInserted</c> or <c>GoodEntryReplaced</c>; a null timestamp gives <c>BadInvalidTimestamp</c>.
|
||||
/// </returns>
|
||||
Task<IReadOnlyList<uint>> UpdateDataAsync(string tag, HistoryUpdateMode mode,
|
||||
IReadOnlyList<HistorizationValue> values, CancellationToken ct);
|
||||
IReadOnlyList<HistorizationValue> values, HistoryUpdateContext context, CancellationToken ct);
|
||||
|
||||
/// <summary>Deletes raw samples in a time range.</summary>
|
||||
/// <param name="tag">Historian tag name.</param>
|
||||
/// <param name="startUtc">Range start (UTC).</param>
|
||||
/// <param name="endUtc">Range end (UTC).</param>
|
||||
/// <param name="context">Who is making the change (recorded in the modification trail).</param>
|
||||
/// <param name="ct">Cancellation token.</param>
|
||||
/// <returns><c>Good</c>, or <c>BadNoData</c> when the range held nothing.</returns>
|
||||
Task<uint> DeleteRawAsync(string tag, DateTime startUtc, DateTime endUtc, CancellationToken ct);
|
||||
Task<uint> DeleteRawAsync(string tag, DateTime startUtc, DateTime endUtc, HistoryUpdateContext context,
|
||||
CancellationToken ct);
|
||||
|
||||
/// <summary>Deletes the samples at exact timestamps.</summary>
|
||||
/// <param name="tag">Historian tag name.</param>
|
||||
/// <param name="timesUtc">Timestamps to delete (UTC).</param>
|
||||
/// <param name="context">Who is making the change (recorded in the modification trail).</param>
|
||||
/// <param name="ct">Cancellation token.</param>
|
||||
/// <returns>One status code per time, in order: <c>Good</c> or <c>BadNoEntryExists</c>.</returns>
|
||||
Task<IReadOnlyList<uint>> DeleteAtTimeAsync(string tag, IReadOnlyList<DateTime> timesUtc, CancellationToken ct);
|
||||
Task<IReadOnlyList<uint>> DeleteAtTimeAsync(string tag, IReadOnlyList<DateTime> timesUtc, HistoryUpdateContext context,
|
||||
CancellationToken ct);
|
||||
}
|
||||
|
||||
@@ -55,6 +55,40 @@ public sealed class NullHistorianDataSource : IHistorianDataSource
|
||||
int maxEvents,
|
||||
CancellationToken cancellationToken) => Task.FromResult(EmptyEvents);
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks><c>true</c> (Gitea #686): there is no history, so the empty result is the correct answer in either
|
||||
/// direction — no need for the server to walk back through forward reads to find nothing.</remarks>
|
||||
public bool SupportsReverseRaw => true;
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks><c>0</c>: no history, no ceiling (Gitea #694).</remarks>
|
||||
public int MaxRawValuesPerRead => 0;
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks><c>true</c>, like <see cref="SupportsReverseRaw"/> (Gitea #691.4 review; <c>false</c> under #675): there is no
|
||||
/// history to read in either direction — no need for the server to walk back through empty forward reads.</remarks>
|
||||
public bool SupportsReverseEvents => true;
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>The empty result in either direction.</remarks>
|
||||
public Task<HistoricalEventsResult> ReadEventsAsync(
|
||||
string? sourceName,
|
||||
DateTime startUtc,
|
||||
DateTime endUtc,
|
||||
int maxEvents,
|
||||
bool reverse,
|
||||
CancellationToken cancellationToken) =>
|
||||
Task.FromResult(EmptyEvents);
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>
|
||||
/// EMPTY (Gitea #682): with no historian configured the server advertises no aggregate functions, and a processed
|
||||
/// HistoryRead answers <c>BadAggregateNotSupported</c> per node rather than <c>GoodNoData</c> — an aggregate is a
|
||||
/// statistic the server cannot compute without a history, and advertising the SDK's full list on a server with no
|
||||
/// historian misled capability discovery. Raw / AtTime / Events reads still return the empty <c>GoodNoData</c> result.
|
||||
/// </remarks>
|
||||
public IReadOnlySet<HistoryAggregateType> SupportedAggregates => HistoryAggregateTypes.None;
|
||||
|
||||
/// <inheritdoc />
|
||||
public HistorianHealthSnapshot GetHealthSnapshot() => new(
|
||||
TotalQueries: 0,
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
namespace ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
|
||||
/// <summary>
|
||||
/// Optional driver capability (Gitea #676) — a driver that can tell, from its protocol, that its connection is
|
||||
/// DEFINITIVELY gone (it can never recover in place, only a re-initialise brings it back) raises
|
||||
/// <see cref="ConnectionLost"/> so the host re-initialises it at once instead of waiting out the #527 comms-loss
|
||||
/// watchdog (~90 s of frozen <see cref="DriverHealth.LastSuccessfulRead"/>).
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>Definitive only.</b> An AMBIGUOUS failure — a timeout, a stalled socket, "no communication" — must NOT
|
||||
/// raise: the connection may yet resume in place (a <c>docker pause</c>, a congested link), and tearing it
|
||||
/// down would cost the session, its subscriptions and the in-place recovery. Ambiguous failures keep going
|
||||
/// through the watchdog, whose conjunction (non-Healthy AND a frozen read stamp) is the honest test for them.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// The host (<c>DriverInstanceActor</c>) acts on a raise only while the driver is Connected — in any other
|
||||
/// state a re-initialise is already pending or running — and it subscribes for its own lifetime, not per
|
||||
/// connect, because the <see cref="IDriver"/> instance can outlive the actor. A driver should raise at most
|
||||
/// once per connection and re-arm when it is re-initialised.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public interface IDriverConnectionLossSource
|
||||
{
|
||||
/// <summary>Raised (on the driver's own thread) when the driver's connection is definitively lost.</summary>
|
||||
event EventHandler<DriverConnectionLossEventArgs>? ConnectionLost;
|
||||
}
|
||||
|
||||
/// <summary>Event payload for <see cref="IDriverConnectionLossSource.ConnectionLost"/>.</summary>
|
||||
/// <param name="Reason">Driver-supplied reason for the diagnostic log (e.g. the protocol status that proved the loss).</param>
|
||||
public sealed record DriverConnectionLossEventArgs(string Reason);
|
||||
@@ -1232,13 +1232,16 @@ public sealed class FocasDriver : IDriver, IReadable, IWritable, ITagDiscovery,
|
||||
ProgramInfo: programInfo is not null,
|
||||
Timers: timer is not null,
|
||||
Status: statusProbe is not null);
|
||||
state.LastStatus = statusProbe;
|
||||
|
||||
state.FixedTreeCache = new FocasFixedTreeCache(sys, [.. axes], [.. spindles], caps);
|
||||
// The bootstrap IS an answer from the device, and Identity comes entirely from it —
|
||||
// so stamp liveness here as well as per tick, or a CNC whose axis poll never
|
||||
// succeeds would serve a perfectly valid sysinfo as stale.
|
||||
state.LastFixedPollUtc = DateTime.UtcNow;
|
||||
// Published LAST (release write): LastStatus is the field callers watch for "the first
|
||||
// answer is in", and an authored FIXED:Status/* read is only servable once
|
||||
// FixedTreeCache and LastFixedPollUtc are set (TryReadFixedTreeOnDevice). Publishing it
|
||||
// first let a reader see a status while the read still returned BadCommunicationError (#673).
|
||||
state.LastStatus = statusProbe;
|
||||
}
|
||||
catch (OperationCanceledException) when (ct.IsCancellationRequested) { return; }
|
||||
catch (Exception ex)
|
||||
@@ -1998,7 +2001,12 @@ public sealed class FocasDriver : IDriver, IReadable, IWritable, ITagDiscovery,
|
||||
/// </summary>
|
||||
public ConcurrentDictionary<string, byte> CoalesceRejected { get; } = new(StringComparer.Ordinal);
|
||||
/// <summary>Gets or sets the last <c>cnc_statinfo</c> snapshot (#257); null until the first answer.</summary>
|
||||
public FocasStatus? LastStatus { get; set; }
|
||||
public FocasStatus? LastStatus
|
||||
{
|
||||
get => Volatile.Read(ref _lastStatus);
|
||||
set => Volatile.Write(ref _lastStatus, value);
|
||||
}
|
||||
private FocasStatus? _lastStatus;
|
||||
/// <summary>Gets or sets the last program information snapshot.</summary>
|
||||
public FocasProgramInfo? LastProgramInfo { get; set; }
|
||||
/// <summary>Gets or sets the cached first-axis dynamic snapshot — feeds Program/Number, /MainNumber, /Sequence.</summary>
|
||||
|
||||
@@ -0,0 +1,514 @@
|
||||
using ZB.MOM.WW.HistorianGateway.Contracts.Grpc;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Mapping;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway;
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #693: HistoryRead AtTime on the Gateway backend, served from RAW gateway reads and evaluated with the SQL backends'
|
||||
/// shared <see cref="AtTimeEvaluator"/> (Part 11 §6.5.4) — as #681 did for the per-sample aggregates — so the Gateway
|
||||
/// answers exactly what SqlServer and QuestDB answer for the same stored samples. AVEVA's own interpolated answer (the
|
||||
/// gateway's <c>ReadAtTime</c>) is not used: live, it returned the previous value as Good at a stored NULL's own time,
|
||||
/// null + Bad at a Good sample right after a NULL, a drifted value (99.99999999944) at an exact stored 100, and no
|
||||
/// Interpolated bit.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>What each requested time needs.</b> <c>prev</c> = the latest stored sample at or before it, <c>next</c> = the
|
||||
/// first stored sample after it. <c>next</c> is only read when the evaluator uses it (a usable <c>prev</c> that is not
|
||||
/// an exact hit).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Read plan — O(1) reads for a dense batch, never two round-trips per timestamp by default.</b> The distinct
|
||||
/// requested times are walked in ascending order with ONE forward window read <c>[t − 1 ms, max + 1 ms]</c> of at most
|
||||
/// <see cref="ReadRows"/> rows, which yields every in-window neighbour. The sample before the first time comes from a
|
||||
/// walk-back (below), the sample after the last time from one small forward probe. A window read that fills its budget
|
||||
/// (dense data, widely spread times) resolves the times before its last row; the next unresolved time either
|
||||
/// continues from that row (when the rows in between are expected to fit one read) or jumps there with a read that
|
||||
/// starts a density-sized margin before it (so its <c>prev</c> is normally in the same read). Ten times one second
|
||||
/// apart on a 1 Hz tag cost 1 read (all exact hits) or 3 (interpolated); ten times an hour apart about one read each.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Walk-back</b> (the gateway reads forward only), in two phases. <i>Gallop:</i> segments <c>(lo, hi]</c> stepping
|
||||
/// back from <c>hi</c>, each one forward read, the first sized from the density the window read showed (else
|
||||
/// <see cref="InitialWalkWidth"/>), an empty one growing the next ×4; after two empty segments the whole remainder down
|
||||
/// to the epoch is asked once — a complete answer ends the walk, a FULL one seeds the bisection with its last row.
|
||||
/// <i>Bisect</i> (once a stored row is known below <c>hi</c> — a full segment's last row, or the last row of an earlier
|
||||
/// window read): read <c>(mid, hi]</c>; empty ⇒ <c>hi = mid</c>; full ⇒ the known row moves up to its last row;
|
||||
/// complete with rows ⇒ its last row is the answer. Right after a full read the next segment is aimed at about a
|
||||
/// quarter of a read at the density it showed (one try — an empty aimed segment falls back to halving), and below
|
||||
/// 2 ms the segment is the whole stretch from the known row (never a 100 ns step).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Bounded.</b> One request may take at most <c>64 + 16 × distinct times</c> raw reads (<see cref="BaseReads"/>,
|
||||
/// <see cref="ReadsPerTime"/>); past it — or with a strict-gateway ceiling below <see cref="MinBudget"/> rows — the
|
||||
/// node fails <c>BadTooManyOperations</c> with a message naming the limit. A jump read (to a time far past the previous
|
||||
/// read) asks for only <see cref="JumpRows"/> rows, so a thousand times spread over ten years cost about a thousand
|
||||
/// small reads, not a thousand 1,024-row reads.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Wire shape.</b> Every request bound is a whole millisecond (AVEVA stores milliseconds and truncates request
|
||||
/// times); a row counts only strictly INSIDE its request — AVEVA's synthetic boundary row at the query start (the value
|
||||
/// in effect there, <c>Quality = 133</c>, or a NULL row) can never be taken for a stored sample, nor a possible row at
|
||||
/// the query end; the epoch and before hold nothing. A Bad row (a stored NULL arrives as value 0, <c>q=1 opc=0</c>,
|
||||
/// #690) is value-less. Every stream is drained, never abandoned (a cancelled gateway stream evicts its pooled
|
||||
/// historian session). Nothing is asked past <c>now + </c><see cref="RawAggregateReader.LookAheadHorizon"/> (the
|
||||
/// historian stores nothing further ahead, and the gateway's AVEVA may refuse a far-future end); a strict gateway's
|
||||
/// row ceiling (#691.3) is learned and honoured — a short read is ordinary paging here, never re-probed.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Values.</b> A gateway value is a Double or a String whatever the node's data type (AVEVA stores every historized
|
||||
/// type as a numeric analog — a Boolean as a 0/1 <c>Int2</c>). The node's type hint converts each row BEFORE the
|
||||
/// shared evaluator runs, so its own type rules apply exactly as on the SQL backends: Boolean (value ≠ 0) and String
|
||||
/// are stepped; an integer type is an Int64 — interpolated, rounded half away from zero — and the result is returned
|
||||
/// as the node's integer type; Float / Double (and no hint) interpolate linearly as Double.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
internal static class GatewayAtTimeReader
|
||||
{
|
||||
/// <summary>The row budget of a window read (the first, or one continuing from a full one) and a walk-back segment.</summary>
|
||||
internal const int ReadRows = 1024;
|
||||
|
||||
/// <summary>The row budget of a jump read (a window read restarting at a time far past the previous read's last row).</summary>
|
||||
internal const int JumpRows = 64;
|
||||
|
||||
/// <summary>The row budget of the forward probe for the sample after the last time (boundary row + ties + one).</summary>
|
||||
internal const int ProbeRows = 2 + RawAggregateReader.PreWindowAllowance;
|
||||
|
||||
/// <summary>The smallest row budget a read can work with (the boundary row, a tie, one sample beyond).</summary>
|
||||
internal const int MinBudget = 4;
|
||||
|
||||
/// <summary>Raw reads one request may always take …</summary>
|
||||
internal const int BaseReads = 64;
|
||||
|
||||
/// <summary>
|
||||
/// … plus this many per distinct requested time. A runaway guard, not a budget (<c>HistoryReadDeadline</c> bounds the
|
||||
/// wall clock): a time inside a gap of a bursty tag legitimately costs a jump read plus a bisection back across the gap
|
||||
/// (≈ 6 reads — a 1 h/day tag asked at noon on consecutive days), so 4 refused requests AVEVA's own AtTime served.
|
||||
/// </summary>
|
||||
internal const int ReadsPerTime = 16;
|
||||
|
||||
/// <summary>The first walk-back segment's width when the window read showed no density.</summary>
|
||||
internal static readonly TimeSpan InitialWalkWidth = TimeSpan.FromMinutes(1);
|
||||
|
||||
/// <summary>The widest walk-back segment sized from density, before ×4 growth.</summary>
|
||||
private static readonly TimeSpan MaxSeededWidth = TimeSpan.FromDays(1);
|
||||
|
||||
private const long Ms = TimeSpan.TicksPerMillisecond;
|
||||
|
||||
private static readonly TimeSpan OneMs = TimeSpan.FromMilliseconds(1);
|
||||
|
||||
private static readonly DateTime Floor = RawAggregateReader.ProbeFloor;
|
||||
|
||||
/// <summary>One drained read: the rows strictly inside the request (ascending) and whether the stream ended short.</summary>
|
||||
private readonly record struct Read(List<StoredSample> Rows, bool Complete);
|
||||
|
||||
/// <summary>Evaluates AtTime for <paramref name="timestampsUtc"/> (same length and order, duplicates included).</summary>
|
||||
/// <param name="client">The gateway client.</param>
|
||||
/// <param name="ceiling">The data source's learned gateway row ceiling (#691.3).</param>
|
||||
/// <param name="tag">Historian tag name.</param>
|
||||
/// <param name="timestampsUtc">Requested times (any kind; treated as UTC, as on the SQL backends).</param>
|
||||
/// <param name="now">Server timestamp, the beyond-now rule and the look-ahead horizon.</param>
|
||||
/// <param name="ct">Cancellation.</param>
|
||||
/// <param name="valueType">The node's scalar value type (the hinted <c>IHistorianDataSource.ReadAtTimeAsync</c>), or null.</param>
|
||||
/// <param name="maxReads">The request's raw-read cap; null = <see cref="BaseReads"/> + <see cref="ReadsPerTime"/> per distinct time.</param>
|
||||
/// <returns>One value per requested time, stamped with it.</returns>
|
||||
/// <exception cref="HistoryReadLimitExceededException">More than <see cref="ReadRows"/> rows within one millisecond, the
|
||||
/// read cap, or a strict gateway ceiling below <see cref="MinBudget"/>.</exception>
|
||||
public static async Task<IReadOnlyList<DataValueSnapshot>> ReadAsync(IHistorianGatewayClient client, GatewayRowCeiling ceiling,
|
||||
string tag, IReadOnlyList<DateTime> timestampsUtc, DateTime now, CancellationToken ct, TypeCode? valueType = null,
|
||||
int? maxReads = null)
|
||||
{
|
||||
var requested = timestampsUtc.Select(t => DateTime.SpecifyKind(t, DateTimeKind.Utc)).ToArray();
|
||||
if (requested.Length == 0) return [];
|
||||
var times = requested.Distinct().Order().ToArray();
|
||||
var ctx = new Ctx(client, ceiling, tag, ct, valueType, maxReads ?? BaseReads + ReadsPerTime * times.Length);
|
||||
var neighbours = await ResolveAsync(ctx, times, now).ConfigureAwait(false);
|
||||
var result = new DataValueSnapshot[requested.Length];
|
||||
for (var i = 0; i < requested.Length; i++)
|
||||
{
|
||||
var t = requested[i];
|
||||
var (prev, next) = neighbours[t.Ticks];
|
||||
// A DateTime node is stepped (as on the SQL backends): a usable non-numeric next sends the evaluator down its
|
||||
// stepped branch, with the next's own status.
|
||||
if (valueType == TypeCode.DateTime && next is { IsUsable: true } n)
|
||||
next = n with { Value = HistorianValue.FromString(string.Empty) };
|
||||
result[i] = ToNodeType(AtTimeEvaluator.Evaluate(t, prev, next, now, now), valueType);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/// <summary>One request's read context: the client, the type hint, and the read count against its cap.</summary>
|
||||
private sealed class Ctx(IHistorianGatewayClient client, GatewayRowCeiling ceiling, string tag, CancellationToken ct,
|
||||
TypeCode? valueType, int maxReads)
|
||||
{
|
||||
public IHistorianGatewayClient Client { get; } = client;
|
||||
public GatewayRowCeiling Ceiling { get; } = ceiling;
|
||||
public string Tag { get; } = tag;
|
||||
public CancellationToken Ct { get; } = ct;
|
||||
public TypeCode? ValueType { get; } = valueType;
|
||||
public int MaxReads { get; } = maxReads;
|
||||
public int Reads { get; set; }
|
||||
|
||||
/// <summary>The smallest budget a read was served with (a strict gateway's ceiling shows here).</summary>
|
||||
public int SmallestBudget { get; set; } = int.MaxValue;
|
||||
}
|
||||
|
||||
/// <summary>True for the integer type codes (evaluated as Int64, returned as the node's own type).</summary>
|
||||
private static bool IsInteger(TypeCode? t) =>
|
||||
t is TypeCode.SByte or TypeCode.Byte or TypeCode.Int16 or TypeCode.UInt16 or TypeCode.Int32 or TypeCode.UInt32
|
||||
or TypeCode.Int64 or TypeCode.UInt64;
|
||||
|
||||
/// <summary>
|
||||
/// An evaluated Int64 (integer hint) returned as the node's integer type. A value the type cannot hold (only possible
|
||||
/// for a stored value already outside the node's range) stays the evaluator's Int64.
|
||||
/// </summary>
|
||||
private static DataValueSnapshot ToNodeType(DataValueSnapshot v, TypeCode? valueType)
|
||||
{
|
||||
if (v.Value is not long l || !IsInteger(valueType) || valueType == TypeCode.Int64) return v;
|
||||
try
|
||||
{
|
||||
return v with { Value = Convert.ChangeType(l, valueType!.Value, System.Globalization.CultureInfo.InvariantCulture) };
|
||||
}
|
||||
catch (OverflowException)
|
||||
{
|
||||
return v;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>prev / next for every time of <paramref name="times"/> (distinct, ascending).</summary>
|
||||
private static async Task<Dictionary<long, (StoredSample? Prev, StoredSample? Next)>> ResolveAsync(
|
||||
Ctx ctx, DateTime[] times, DateTime now)
|
||||
{
|
||||
var result = new Dictionary<long, (StoredSample?, StoredSample?)>(times.Length);
|
||||
var horizon = MsFloor(DateTime.MaxValue - now > RawAggregateReader.LookAheadHorizon
|
||||
? DateTime.SpecifyKind(now + RawAggregateReader.LookAheadHorizon, DateTimeKind.Utc)
|
||||
: DateTime.MaxValue.AddTicks(-Ms));
|
||||
|
||||
// Nothing is stored at or before the epoch.
|
||||
var i = 0;
|
||||
for (; i < times.Length && times[i] <= Floor; i++) result[times[i].Ticks] = (null, null);
|
||||
if (i == times.Length) return result;
|
||||
|
||||
// Reads are planned on times clamped to the horizon (nothing is stored beyond it, and no read ends past it).
|
||||
var qe = Min(MsFloor(Min(times[^1], horizon)) + OneMs, horizon);
|
||||
var qs = RequestStart(Min(times[i], horizon));
|
||||
var budget = ReadRows;
|
||||
StoredSample? before = null; // the latest stored sample at or before qs …
|
||||
var beforeKnown = false; // … once known
|
||||
DateTime? walkKnown = null; // a stored sample exists here (at or before qs): the walk-back bisects above it
|
||||
var walkWidth = InitialWalkWidth;
|
||||
var probeFrom = -1; // first index whose next lies at or after qe (complete last read)
|
||||
|
||||
while (true)
|
||||
{
|
||||
var read = await ReadTrustedAsync(ctx, qs, qe, budget).ConfigureAwait(false);
|
||||
var rows = read.Rows;
|
||||
if (!read.Complete && rows.Count == 0)
|
||||
throw TooDense(ctx, qs);
|
||||
if (rows.Count >= 2 && !beforeKnown && walkKnown is null)
|
||||
walkWidth = Clamp(TimeSpan.FromTicks(4 * (rows[^1].TimeUtc - rows[0].TimeUtc).Ticks / (rows.Count - 1)));
|
||||
|
||||
// A short stream saw every row up to qe; a full one every row before its last row's time x.
|
||||
var x = read.Complete ? qe : rows[^1].TimeUtc;
|
||||
var j = -1; // index of the last row at or before the current time
|
||||
var k = i;
|
||||
for (; k < times.Length; k++)
|
||||
{
|
||||
var t = times[k];
|
||||
var tr = Min(t, horizon);
|
||||
if (!read.Complete && tr >= x) break;
|
||||
while (j + 1 < rows.Count && rows[j + 1].TimeUtc <= tr) j++;
|
||||
|
||||
StoredSample? prev;
|
||||
if (j >= 0) prev = rows[j];
|
||||
else
|
||||
{
|
||||
if (!beforeKnown)
|
||||
{
|
||||
// #696: prove the future stretch empty before jumping to now. Historian clocks or imports
|
||||
// can place real rows there; a full bounded probe seeds the original backward search.
|
||||
var searchEnd = qs;
|
||||
if (qs > now && walkKnown is null)
|
||||
{
|
||||
var future = await ReadTrustedAsync(ctx, MsFloor(now),
|
||||
Min(MsFloor(qs) + OneMs, horizon), ReadRows).ConfigureAwait(false);
|
||||
StoredSample? latest = null;
|
||||
foreach (var row in future.Rows)
|
||||
if (row.TimeUtc <= qs) latest = row;
|
||||
if (future.Complete)
|
||||
{
|
||||
before = latest;
|
||||
beforeKnown = latest is not null;
|
||||
searchEnd = now;
|
||||
}
|
||||
else if (latest is { } found)
|
||||
walkKnown = found.TimeUtc;
|
||||
}
|
||||
if (!beforeKnown)
|
||||
before = await WalkBackAsync(ctx, searchEnd, walkKnown, walkWidth).ConfigureAwait(false);
|
||||
beforeKnown = true;
|
||||
}
|
||||
prev = before;
|
||||
}
|
||||
|
||||
StoredSample? next = null;
|
||||
if (NeedsNext(t, prev) && t < horizon)
|
||||
{
|
||||
if (j + 1 < rows.Count) next = rows[j + 1];
|
||||
else if (probeFrom < 0) probeFrom = k; // after every row of a complete read: the probe answers it
|
||||
}
|
||||
result[t.Ticks] = (prev, next);
|
||||
}
|
||||
|
||||
if (k == times.Length) break;
|
||||
|
||||
// Truncated before times[k]: continue from the last row, or jump to times[k] with a density-sized margin and a
|
||||
// small budget (it needs that time's neighbours, and whatever later times follow closely).
|
||||
var tn = Min(times[k], horizon);
|
||||
var spanTicks = Math.Max(1, (x - rows[0].TimeUtc).Ticks);
|
||||
var density = (double)rows.Count / spanTicks; // rows per tick
|
||||
var gapRows = density * (tn - x).Ticks;
|
||||
var margin = Clamp(TimeSpan.FromTicks((long)Math.Min(TimeSpan.MaxValue.Ticks / 2.0, JumpRows / 4.0 / Math.Max(density, 1e-18))));
|
||||
var resume = MsFloor(x) - OneMs;
|
||||
var jump = gapRows >= ReadRows / 2.0;
|
||||
var nextQs = jump ? Max(RequestStart(tn - Min(margin, tn - Floor)), resume) : resume;
|
||||
if (nextQs <= qs) throw TooDense(ctx, x);
|
||||
|
||||
if (nextQs < x)
|
||||
{
|
||||
// Every row before x was seen: the latest at or before nextQs is known.
|
||||
var lastIndex = rows.FindLastIndex(r => r.TimeUtc <= nextQs);
|
||||
if (lastIndex >= 0)
|
||||
{
|
||||
before = rows[lastIndex];
|
||||
beforeKnown = true;
|
||||
}
|
||||
walkKnown = null;
|
||||
}
|
||||
else
|
||||
{
|
||||
beforeKnown = false;
|
||||
before = null;
|
||||
walkKnown = x;
|
||||
walkWidth = margin;
|
||||
}
|
||||
budget = jump ? JumpRows : ReadRows;
|
||||
qs = nextQs;
|
||||
i = k;
|
||||
}
|
||||
|
||||
if (probeFrom >= 0 && qe < horizon)
|
||||
{
|
||||
var after = await FirstAfterAsync(ctx, qe - OneMs, horizon, Min(times[^1], horizon)).ConfigureAwait(false);
|
||||
if (after is not null)
|
||||
for (var k = probeFrom; k < times.Length; k++)
|
||||
{
|
||||
var t = times[k];
|
||||
var (prev, next) = result[t.Ticks];
|
||||
if (next is null && t < horizon && NeedsNext(t, prev) && after.Value.TimeUtc > t)
|
||||
result[t.Ticks] = (prev, after);
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/// <summary>The evaluator reads <c>next</c> only for a usable <c>prev</c> that is not an exact hit.</summary>
|
||||
private static bool NeedsNext(DateTime t, StoredSample? prev) => prev is { IsUsable: true } p && p.TimeUtc != t;
|
||||
|
||||
/// <summary>One walk-back segment <c>(lo, hi]</c>: its oldest and newest rows, and whether the read was complete.</summary>
|
||||
private static async Task<(StoredSample? First, StoredSample? Last, bool Complete)> SegmentAsync(
|
||||
Ctx ctx, DateTime lo, DateTime hi)
|
||||
{
|
||||
var read = await ReadTrustedAsync(ctx, MsFloor(lo), MsFloor(hi) + OneMs, ReadRows).ConfigureAwait(false);
|
||||
StoredSample? first = null, last = null;
|
||||
foreach (var r in read.Rows)
|
||||
if (r.TimeUtc > lo && r.TimeUtc <= hi)
|
||||
{
|
||||
first ??= r;
|
||||
last = r;
|
||||
}
|
||||
return (first, last, read.Complete);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The latest stored sample with time in <c>(Floor, hi]</c> (see the type remarks: gallop, then bisect).
|
||||
/// <paramref name="known"/>: a stored sample exists at that time (≤ <paramref name="hi"/>) — the walk then bisects
|
||||
/// above it straight away, its first segment aimed at <paramref name="width"/>.
|
||||
/// </summary>
|
||||
private static async Task<StoredSample?> WalkBackAsync(Ctx ctx, DateTime hi, DateTime? known, TimeSpan width)
|
||||
{
|
||||
TimeSpan? aim = known is null ? null : width;
|
||||
|
||||
// Gallop: no stored row known below hi yet.
|
||||
var empties = 0;
|
||||
while (known is null)
|
||||
{
|
||||
if (hi <= Floor) return null;
|
||||
var rest = empties >= 2;
|
||||
var lo = rest || (hi - Floor).Ticks <= width.Ticks ? Floor : hi - width;
|
||||
var seg = await SegmentAsync(ctx, lo, hi).ConfigureAwait(false);
|
||||
if (seg.Complete)
|
||||
{
|
||||
if (seg.Last is { } found) return found;
|
||||
if (lo <= Floor) return null;
|
||||
hi = lo;
|
||||
empties++;
|
||||
width = Grow(width);
|
||||
continue;
|
||||
}
|
||||
// Full — a whole-remainder read of deep history too: its last row is a stored row below hi. Bisect above it.
|
||||
if (seg.Last is not { } seen) throw TooDense(ctx, lo);
|
||||
known = seen.TimeUtc;
|
||||
aim = AimFrom(seg.First!.Value.TimeUtc, seen.TimeUtc);
|
||||
}
|
||||
|
||||
// Bisect: a stored row exists at `known`, nothing newer than `hi` matters; the answer is the newest row in [known, hi].
|
||||
var aimMissed = false;
|
||||
while (true)
|
||||
{
|
||||
var k = known.Value;
|
||||
DateTime lo;
|
||||
var aimed = false;
|
||||
if (hi - k < TimeSpan.FromMilliseconds(2))
|
||||
lo = k.AddTicks(-1); // the whole stretch from the known row: never a sub-millisecond step
|
||||
else
|
||||
{
|
||||
lo = k + TimeSpan.FromTicks((hi - k).Ticks / 2);
|
||||
if (aim is { } a && hi - a > lo)
|
||||
{
|
||||
lo = hi - a; // an aimed try right after a full read — until one comes back empty (a gap: halve from then on)
|
||||
aimed = true;
|
||||
}
|
||||
}
|
||||
aim = null;
|
||||
|
||||
var seg = await SegmentAsync(ctx, lo, hi).ConfigureAwait(false);
|
||||
if (seg.Complete)
|
||||
{
|
||||
if (seg.Last is { } found) return found;
|
||||
if (lo < k) return null; // the known row is gone (deleted meanwhile): nothing is left to find
|
||||
hi = lo;
|
||||
aimMissed |= aimed;
|
||||
continue;
|
||||
}
|
||||
if (seg.Last is not { } seen || seen.TimeUtc <= k) throw TooDense(ctx, k);
|
||||
known = seen.TimeUtc;
|
||||
if (!aimMissed) aim = AimFrom(seg.First!.Value.TimeUtc, seen.TimeUtc);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>About a quarter of a read at the density a full read showed (its rows from <paramref name="first"/> to <paramref name="last"/>).</summary>
|
||||
private static TimeSpan AimFrom(DateTime first, DateTime last) =>
|
||||
TimeSpan.FromTicks(Math.Max(Ms, (last - first).Ticks / 4));
|
||||
|
||||
/// <summary>The first stored sample after <paramref name="after"/> in <c>(qs, end)</c>, from a small forward read.</summary>
|
||||
private static async Task<StoredSample?> FirstAfterAsync(Ctx ctx, DateTime qs, DateTime end, DateTime after)
|
||||
{
|
||||
foreach (var budget in new[] { ProbeRows, ReadRows })
|
||||
{
|
||||
var read = await ReadTrustedAsync(ctx, qs, end, budget).ConfigureAwait(false);
|
||||
foreach (var r in read.Rows)
|
||||
if (r.TimeUtc > after) return r;
|
||||
if (read.Complete) return null;
|
||||
}
|
||||
throw TooDense(ctx, qs);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// One forward read <c>[qs, qe]</c> (whole milliseconds), drained, keeping only rows strictly inside it and after the
|
||||
/// epoch — counted against the request's read cap. A strict gateway's ceiling is learned on refusal and then asked for
|
||||
/// directly; a full read is not re-probed (the caller pages on, it does not need the rest of this read).
|
||||
/// </summary>
|
||||
private static async Task<Read> ReadTrustedAsync(Ctx ctx, DateTime qs, DateTime qe, int budget)
|
||||
{
|
||||
if (++ctx.Reads > ctx.MaxReads)
|
||||
throw new HistoryReadLimitExceededException(ctx.MaxReads,
|
||||
$"The AtTime read of '{ctx.Tag}' needed more than {ctx.MaxReads} raw reads ({BaseReads} + {ReadsPerTime} per " +
|
||||
"requested time)" + CeilingNote(ctx) + "; the Gateway backend serves AtTime from raw reads — request fewer times.");
|
||||
var (read, used) = await ctx.Ceiling.ReadAsync(budget, b => PassAsync(ctx, qs, qe, b), (_, _) => false)
|
||||
.ConfigureAwait(false);
|
||||
if (used < ctx.SmallestBudget) ctx.SmallestBudget = used;
|
||||
if (used < MinBudget)
|
||||
throw new HistoryReadLimitExceededException(used,
|
||||
$"The HistorianGateway's row ceiling ({used}, its Historian:ReadLimits:MaxValuesCeiling with " +
|
||||
$"AllowStreamingBeyondCeiling=false) is below the {MinBudget} rows an AtTime raw read of '{ctx.Tag}' needs.");
|
||||
return read;
|
||||
}
|
||||
|
||||
private static string CeilingNote(Ctx ctx) =>
|
||||
ctx.SmallestBudget < JumpRows
|
||||
? $", with reads bounded by the HistorianGateway's row ceiling of {ctx.SmallestBudget} (Historian:ReadLimits:MaxValuesCeiling)"
|
||||
: string.Empty;
|
||||
|
||||
private static async Task<Read> PassAsync(Ctx ctx, DateTime qs, DateTime qe, int budget)
|
||||
{
|
||||
var rows = new List<StoredSample>();
|
||||
var received = 0;
|
||||
await foreach (var raw in ctx.Client.ReadRawAsync(ctx.Tag, qs, qe, budget, ctx.Ct).ConfigureAwait(false))
|
||||
{
|
||||
received++; // drained to completion, never broken out of
|
||||
if (raw.Timestamp is null) continue;
|
||||
var t = raw.Timestamp.ToDateTime();
|
||||
if (t > qs && t < qe && t > Floor) rows.Add(ToStored(raw, t, ctx.ValueType));
|
||||
}
|
||||
return new Read(rows, received < budget);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A gateway row as a stored sample of the node's type (see the type remarks); a Bad row carries no value (#690: a
|
||||
/// stored NULL arrives as value 0).
|
||||
/// </summary>
|
||||
private static StoredSample ToStored(HistorianSample raw, DateTime t, TypeCode? valueType)
|
||||
{
|
||||
var status = SampleMapper.ToStatus(raw);
|
||||
var value = StatusBits.IsBad(status) ? HistorianValue.Empty
|
||||
: raw.HasNumericValue ? Typed(raw.NumericValue, valueType)
|
||||
: raw.HasStringValue ? HistorianValue.FromString(raw.StringValue)
|
||||
: HistorianValue.Empty;
|
||||
return StoredSample.FromStored(t, value, status);
|
||||
}
|
||||
|
||||
/// <summary>A gateway number as the hinted kind, so the shared evaluator's own type branch decides stepped vs linear.</summary>
|
||||
private static HistorianValue Typed(double v, TypeCode? valueType) => valueType switch
|
||||
{
|
||||
TypeCode.Boolean => HistorianValue.FromBoolean(v != 0),
|
||||
TypeCode.String => HistorianValue.FromString(v.ToString("R", System.Globalization.CultureInfo.InvariantCulture)),
|
||||
// In Int64 range only (2^63 is exactly representable); NaN, ±∞ or out of range is no integer at all — Empty, which
|
||||
// StoredSample.FromStored makes BadNoData (unusable), never a saturated Good value.
|
||||
_ when IsInteger(valueType) => v >= -9.2233720368547758E18 && v < 9.2233720368547758E18
|
||||
? HistorianValue.FromInt64((long)Math.Round(v, MidpointRounding.AwayFromZero))
|
||||
: HistorianValue.Empty,
|
||||
_ => HistorianValue.FromDouble(v),
|
||||
};
|
||||
|
||||
/// <summary>The window read's start for a time: a millisecond before its millisecond (never before the epoch).</summary>
|
||||
private static DateTime RequestStart(DateTime t)
|
||||
{
|
||||
var s = MsFloor(t) - TimeSpan.FromMilliseconds(1);
|
||||
return s < Floor ? Floor : s;
|
||||
}
|
||||
|
||||
private static DateTime MsFloor(DateTime t) => new(t.Ticks - t.Ticks % Ms, DateTimeKind.Utc);
|
||||
|
||||
private static DateTime Min(DateTime a, DateTime b) => a <= b ? a : b;
|
||||
|
||||
private static DateTime Max(DateTime a, DateTime b) => a >= b ? a : b;
|
||||
|
||||
private static TimeSpan Min(TimeSpan a, TimeSpan b) => a <= b ? a : b;
|
||||
|
||||
private static TimeSpan Clamp(TimeSpan w) =>
|
||||
w < TimeSpan.FromMilliseconds(1) ? TimeSpan.FromMilliseconds(1) : w > MaxSeededWidth ? MaxSeededWidth : w;
|
||||
|
||||
private static TimeSpan Grow(TimeSpan w) => w.Ticks > TimeSpan.MaxValue.Ticks / 4 ? TimeSpan.MaxValue : w * 4;
|
||||
|
||||
private static HistoryReadLimitExceededException TooDense(Ctx ctx, DateTime at)
|
||||
{
|
||||
var rows = Math.Min(ReadRows, ctx.SmallestBudget);
|
||||
return new(rows,
|
||||
$"The AtTime read of '{ctx.Tag}' met more than {rows} stored samples within about one millisecond of {at:O}" +
|
||||
(rows < ReadRows ? $" (reads are bounded by the HistorianGateway's row ceiling of {rows})" : string.Empty) +
|
||||
"; the Gateway backend serves AtTime from raw reads.");
|
||||
}
|
||||
}
|
||||
+487
-20
@@ -1,7 +1,10 @@
|
||||
using System.Collections.Frozen;
|
||||
using Grpc.Core;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using ZB.MOM.WW.HistorianGateway.Contracts.Grpc;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Mapping;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway;
|
||||
|
||||
@@ -23,8 +26,50 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway;
|
||||
/// </remarks>
|
||||
public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDisposable
|
||||
{
|
||||
/// <summary>
|
||||
/// The processed (aggregate) functions the Gateway backend serves — the single source of truth for this backend's
|
||||
/// aggregate set (#681): the per-sample aggregates evaluated from raw reads (every member of the
|
||||
/// <see cref="AggregateFamily.Simple"/> / <see cref="AggregateFamily.StartEnd"/> /
|
||||
/// <see cref="AggregateFamily.ActualTime"/> families of <see cref="AggregateFamilies"/>: Average, Count, Minimum,
|
||||
/// Maximum, Range, Start, End, Delta, MinimumActualTime, MaximumActualTime) plus the two answered natively by AVEVA
|
||||
/// (TimeAverage, Total — <see cref="AggregateModeMapper"/>). Any other aggregate throws
|
||||
/// <see cref="HistoryAggregateNotSupportedException"/> without calling the gateway. It is what
|
||||
/// <see cref="SupportedAggregates"/> returns, so the advertised <c>AggregateFunctions</c> folders and the server's
|
||||
/// up-front <c>BadAggregateNotSupported</c> read it too (#682).
|
||||
/// </summary>
|
||||
public static IReadOnlySet<HistoryAggregateType> ServedAggregates { get; } =
|
||||
Enum.GetValues<HistoryAggregateType>()
|
||||
.Where(a => RawAggregateReader.Serves(a) || AggregateModeMapper.NativeAggregates.Contains(a))
|
||||
.ToFrozenSet();
|
||||
|
||||
private readonly IHistorianGatewayClient _client;
|
||||
private readonly ILogger<GatewayHistorianDataSource> _logger;
|
||||
private readonly int _maxRawRows;
|
||||
private readonly TimeProvider _time;
|
||||
private readonly bool _atTimeFromRaw;
|
||||
|
||||
/// <summary>#693: AtTime from raw reads (true) or AVEVA's own interpolation (the kill switch).</summary>
|
||||
internal bool AtTimeFromRaw => _atTimeFromRaw;
|
||||
|
||||
/// <summary>The most #687 crowding re-reads one raw read makes (#694 F4): each at least doubles the pre-window room.</summary>
|
||||
private const int MaxCrowdedReReads = 8;
|
||||
|
||||
/// <summary>The default of <c>maxPreWindowRows</c>: <c>ServerHistorian:MaxTieClusterOverfetch</c>'s default.</summary>
|
||||
public const int DefaultMaxPreWindowRows = 65536;
|
||||
|
||||
/// <summary>
|
||||
/// The most rows before a raw window (inside the 1 ms look-back) a crowding re-read makes room for (#694 R2) — the
|
||||
/// server's tie-cluster limit (<c>ServerHistorian:MaxTieClusterOverfetch</c>), which every other tie path treats as
|
||||
/// the limit. More ⇒ <see cref="HistoryReadLimitExceededException"/> with <c>PreWindowRows</c> set.
|
||||
/// </summary>
|
||||
private readonly int _maxPreWindowRows;
|
||||
|
||||
/// <summary>The tie-cluster limit the crowding re-reads honour (#694 R2) — exposed so the factory wiring is testable.</summary>
|
||||
internal int MaxPreWindowRows => _maxPreWindowRows;
|
||||
|
||||
/// <summary>A strict gateway's row ceiling, learned from its first refusal (#691.3).</summary>
|
||||
private readonly GatewayRowCeiling _ceiling;
|
||||
private readonly GatewayTagPresenceCache _tags;
|
||||
|
||||
private readonly object _healthLock = new();
|
||||
private DateTime? _lastSuccessUtc;
|
||||
@@ -38,79 +83,440 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
/// <summary>Creates a gateway-backed historian data source.</summary>
|
||||
/// <param name="client">The gateway client seam used for all reads.</param>
|
||||
/// <param name="logger">Diagnostic logger; failures are recorded without leaking tag/host detail.</param>
|
||||
public GatewayHistorianDataSource(IHistorianGatewayClient client, ILogger<GatewayHistorianDataSource> logger)
|
||||
/// <param name="maxRawRows">
|
||||
/// In-window raw-row cap of one raw-evaluated processed read (<c>ServerHistorian:MaxTimeWeightedRawRows</c>;
|
||||
/// <= 0 = unbounded). Past it the node fails <c>BadTooManyOperations</c>.
|
||||
/// </param>
|
||||
/// <param name="timeProvider">Clock for result server timestamps and the raw look-ahead horizon; null = system.</param>
|
||||
/// <param name="atTimeFromRaw">
|
||||
/// #693: serve AtTime from raw reads with the shared AtTime rules (<see cref="GatewayAtTimeReader"/>; default);
|
||||
/// <c>false</c> = AVEVA's own interpolation passed through (<c>ServerHistorian:GatewayAtTimeFromRaw</c> kill switch).
|
||||
/// </param>
|
||||
/// <param name="maxPreWindowRows">
|
||||
/// The most rows before a raw window a crowding re-read makes room for (#694 R2; <c>ServerHistorian:MaxTieClusterOverfetch</c>,
|
||||
/// values below 1 count as 1). More ⇒ <see cref="HistoryReadLimitExceededException"/> (<c>BadTooManyOperations</c>).
|
||||
/// </param>
|
||||
public GatewayHistorianDataSource(IHistorianGatewayClient client, ILogger<GatewayHistorianDataSource> logger,
|
||||
int maxRawRows = TimeWeightedAggregator.DefaultMaxWindowRows, TimeProvider? timeProvider = null,
|
||||
bool atTimeFromRaw = true, int maxPreWindowRows = DefaultMaxPreWindowRows)
|
||||
{
|
||||
_maxPreWindowRows = Math.Max(1, maxPreWindowRows);
|
||||
_atTimeFromRaw = atTimeFromRaw;
|
||||
ArgumentNullException.ThrowIfNull(client);
|
||||
ArgumentNullException.ThrowIfNull(logger);
|
||||
_client = client;
|
||||
_logger = logger;
|
||||
_maxRawRows = maxRawRows;
|
||||
_time = timeProvider ?? TimeProvider.System;
|
||||
_ceiling = new GatewayRowCeiling(_time);
|
||||
_tags = new GatewayTagPresenceCache(_client, _time);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>
|
||||
/// A reverse window (<c>start > end</c>) is refused with <see cref="NotSupportedException"/> before anything is sent
|
||||
/// and is not recorded as a health failure (Gitea #686): the gateway rejects it <c>InvalidArgument</c>, and the server
|
||||
/// never asks — <see cref="SupportsReverseRaw"/> is <c>false</c>, so it walks a reverse read back with forward reads.
|
||||
/// Stored samples only (#687 — Part 11 Raw): AVEVA's synthetic start-boundary row is never returned, see
|
||||
/// <see cref="ReadRawWindowAsync"/>.
|
||||
/// </remarks>
|
||||
public async Task<HistoryReadResult> ReadRawAsync(
|
||||
string fullReference, DateTime startUtc, DateTime endUtc, uint maxValuesPerNode,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (startUtc > endUtc)
|
||||
throw new NotSupportedException("The HistorianGateway reads raw history oldest-first only (start > end).");
|
||||
|
||||
try
|
||||
{
|
||||
// The gateway seam caps with an int; OPC UA hands us a uint, so clamp to int range.
|
||||
var maxValues = (int)Math.Min(maxValuesPerNode, int.MaxValue);
|
||||
var samples = new List<HistorianSample>();
|
||||
await foreach (var sample in _client
|
||||
.ReadRawAsync(fullReference, startUtc, endUtc, maxValues, cancellationToken)
|
||||
.ConfigureAwait(false))
|
||||
if (!await _tags.ExistsAsync(fullReference, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
samples.Add(sample);
|
||||
RecordOutcome(success: true, error: null);
|
||||
return new HistoryReadResult([], ContinuationPoint: null);
|
||||
}
|
||||
|
||||
var samples = await ReadRawWindowAsync(fullReference, startUtc, endUtc, maxValuesPerNode, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
var snapshots = SampleMapper.ToSnapshots(samples);
|
||||
RecordOutcome(success: true, error: null);
|
||||
return new HistoryReadResult(snapshots, ContinuationPoint: null);
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
throw; // the caller (client cancel / HistoryReadDeadline) asked for this — not a backend fault
|
||||
}
|
||||
catch (Exception ex) when (IsMaxValuesCeilingRejection(ex))
|
||||
{
|
||||
// An unbounded read over Historian:ReadLimits:MaxValuesCeiling with AllowStreamingBeyondCeiling=false: a
|
||||
// client-request limit (BadTooManyOperations), not a backend fault.
|
||||
throw new HistoryReadLimitExceededException(maxValuesPerNode,
|
||||
$"The HistorianGateway refused the row budget of the raw read of '{fullReference}' (its " +
|
||||
"Historian:ReadLimits:MaxValuesCeiling); set NumValuesPerNode or narrow the window.");
|
||||
}
|
||||
catch (HistoryReadLimitExceededException)
|
||||
{
|
||||
// A client-request limit (#687: the pre-window margin swallowed the row budget), not a backend fault.
|
||||
throw;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
RecordReadFailure(ex);
|
||||
if (IsTransportFailure(ex)) throw new System.IO.IOException("HistorianGateway read transport is unavailable.", ex);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The stored samples of <c>[startUtc, endUtc)</c> (a point read <c>start == end</c>: of that instant), oldest first,
|
||||
/// at most <paramref name="maxValuesPerNode"/> (<c>0</c> = all) — without AVEVA's synthetic rows (#687).
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// An AVEVA <c>Full</c> raw read starts with a row stamped exactly at the query start: the value in effect there
|
||||
/// (<c>Quality = 133</c>, the prior sample's value — never stored at that time) or, before the tag's first sample,
|
||||
/// a NULL row (value 0, Bad). Part 11 Raw with <c>ReturnBounds = false</c> wants stored values only, and the
|
||||
/// server's paging (#669) counts every returned row toward the page and resumes at the last returned timestamp,
|
||||
/// so a fabricated row would also be counted as a sample and re-fabricated at every resume point. The read
|
||||
/// therefore uses the same request start as the raw-evaluated aggregates (#681): it is requested from
|
||||
/// <see cref="RawAggregateReader.RequestStartFor"/> (1 ms early) and only rows
|
||||
/// <see cref="RawAggregateReader.IsWindowSample"/> are kept — the decision is by TIMESTAMP, because a stored NULL
|
||||
/// sample and the no-data boundary row look the same on the wire (both carry a value 0 with a Bad quality). The
|
||||
/// forward end is exclusive (the interface contract; the gateway's own end is inclusive), a point read inclusive.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Row budget.</b> <c>0</c> ("all values") asks for <see cref="int.MaxValue"/> (the client package and the
|
||||
/// gateway reject <c>max_values <= 0</c>, #686). A cap asks for <c>cap + </c>
|
||||
/// <see cref="RawAggregateReader.PreWindowAllowance"/> rows — NOT clamped at the gateway's default ceiling (#688
|
||||
/// clamps only the per-sample aggregate budget): AVEVA always emits the boundary row, so a clamped budget would make
|
||||
/// every page size within the allowance of the ceiling come back short. A strict gateway
|
||||
/// (<c>AllowStreamingBeyondCeiling=false</c>) refuses an over-ceiling budget and names its ceiling; the read is
|
||||
/// then re-issued ONCE with the ceiling as budget (#691.3, <see cref="GatewayReadLimits"/>) — so an uncapped read
|
||||
/// of a small window is served — and a ceiling-bounded read that is not provably complete (it delivered the
|
||||
/// whole ceiling without reaching the window end, or a page short of its cap) fails
|
||||
/// <see cref="HistoryReadLimitExceededException"/> (<c>BadTooManyOperations</c>), never a truncated answer.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Completeness.</b> A full page must hold exactly the cap of stored samples — a short page ends the client's
|
||||
/// paging. A read is complete when the stream came back short, filled the cap, or reached the window end. If rows
|
||||
/// BEFORE the window (the boundary row and stored samples inside the 1 ms margin, e.g. a tie cluster at
|
||||
/// <c>start − 1 ms</c>) used up the budget, the window is read ONCE more asking for <c>cap + m + </c>
|
||||
/// <see cref="RawAggregateReader.PreWindowAllowance"/> (m = the pre-window rows the first read saw); only if that
|
||||
/// re-read is still short because of pre-window rows does the read fail with
|
||||
/// <see cref="HistoryReadLimitExceededException"/>. Every stream is consumed to completion (never broken out of —
|
||||
/// a cancelled gateway stream evicts its pooled historian session). <c>ReturnBounds</c> is not plumbed to the
|
||||
/// data source on any backend, so bounding values are never synthesised here.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
private async Task<List<HistorianSample>> ReadRawWindowAsync(
|
||||
string fullReference, DateTime startUtc, DateTime endUtc, uint maxValuesPerNode,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
// The gateway seam caps with an int; OPC UA hands us a uint, so clamp to int range.
|
||||
var cap = (int)Math.Min(maxValuesPerNode, int.MaxValue);
|
||||
if (cap == 0)
|
||||
{
|
||||
var all = await PassAsync(int.MaxValue).ConfigureAwait(false);
|
||||
if (all.Result.Complete) return all.Result.Samples;
|
||||
// Only reachable on a strict gateway (#691.3): the ceiling re-read delivered the whole ceiling without reaching
|
||||
// the window end — the window may hold more, and "all values" is never answered from a truncated read.
|
||||
throw new HistoryReadLimitExceededException(cap,
|
||||
$"The raw read of '{fullReference}' holds more than the HistorianGateway's row ceiling ({all.Budget}, its " +
|
||||
"Historian:ReadLimits:MaxValuesCeiling with AllowStreamingBeyondCeiling=false); set NumValuesPerNode to page, " +
|
||||
"or narrow the window.")
|
||||
{ PreWindowRows = Crowding(all.Result.PreWindowRows) };
|
||||
}
|
||||
|
||||
var asked = (int)Math.Min(int.MaxValue, (long)cap + RawAggregateReader.PreWindowAllowance);
|
||||
var pass = await PassAsync(asked).ConfigureAwait(false);
|
||||
if (pass.Result.Complete) return pass.Result.Samples;
|
||||
|
||||
// A strict gateway's ceiling already bounded that read (#691.3): a larger budget would only be refused again. The
|
||||
// refusal carries the pre-window rows it saw (#694): the server re-reads the page that many values smaller.
|
||||
if (pass.Budget < asked)
|
||||
throw new HistoryReadLimitExceededException(cap,
|
||||
$"The raw read of '{fullReference}' could not be read completely within the HistorianGateway's row ceiling " +
|
||||
$"({pass.Budget}, its Historian:ReadLimits:MaxValuesCeiling with AllowStreamingBeyondCeiling=false); lower " +
|
||||
"NumValuesPerNode or narrow the window.")
|
||||
{ PreWindowRows = Crowding(pass.Result.PreWindowRows) };
|
||||
|
||||
// The margin crowded the window out: re-read with room for the pre-window rows. A read that was itself cut short saw
|
||||
// only some of them (#694 F4: a tie cluster of hundreds just before a resume point used to fail even with no
|
||||
// ceiling), so the room at least doubles per re-read — bounded by MaxCrowdedReReads, and never past a known ceiling.
|
||||
// #694 R2: the room is capped at the tie-cluster limit (MaxTieClusterOverfetch) — a pathological single-millisecond
|
||||
// cluster on a streaming gateway would otherwise be streamed several times over for one page.
|
||||
var preWindow = pass.Result.PreWindowRows;
|
||||
var room = Math.Min(Math.Max(1, preWindow), _maxPreWindowRows);
|
||||
for (var attempt = 0; attempt < MaxCrowdedReReads && preWindow <= _maxPreWindowRows; attempt++)
|
||||
{
|
||||
var budget = (int)Math.Min(int.MaxValue, (long)cap + room + RawAggregateReader.PreWindowAllowance);
|
||||
// #695: doubling can leap over the ceiling while the cluster still fits. Spend the remaining room once.
|
||||
if (_ceiling.Learned is var known and > 0 && budget > known)
|
||||
{
|
||||
var available = known - cap - RawAggregateReader.PreWindowAllowance;
|
||||
if (available <= 0) break;
|
||||
room = Math.Min(room, available);
|
||||
budget = cap + room + RawAggregateReader.PreWindowAllowance;
|
||||
}
|
||||
if (budget <= pass.Budget) break; // no more room, never retry the same full read
|
||||
pass = await PassAsync(budget).ConfigureAwait(false);
|
||||
if (pass.Result.Complete) return pass.Result.Samples;
|
||||
preWindow = Math.Max(preWindow, pass.Result.PreWindowRows);
|
||||
if (pass.Budget < budget) break; // bounded by a ceiling learned meanwhile
|
||||
if (room >= _maxPreWindowRows) break; // the limit's worth of room did not suffice
|
||||
room = (int)Math.Min(_maxPreWindowRows, Math.Max(preWindow, 2L * room));
|
||||
}
|
||||
|
||||
throw new HistoryReadLimitExceededException(cap,
|
||||
$"The raw read of '{fullReference}' could not be read completely: more than {pass.Result.PreWindowRows} rows lie in " +
|
||||
$"the {RawAggregateReader.BoundaryMargin.TotalMilliseconds} ms before the window and exhausted the row budget.")
|
||||
{ PreWindowRows = preWindow };
|
||||
|
||||
// Pre-window rows the allowance did not cover are what crowded the read; fewer are not the cause (#694).
|
||||
static int Crowding(int preWindowRows) => preWindowRows > RawAggregateReader.PreWindowAllowance ? preWindowRows : 0;
|
||||
|
||||
// One drained pass; a strict gateway's refusal of the budget is retried once at the ceiling it names (#691.3).
|
||||
Task<((List<HistorianSample> Samples, int PreWindowRows, bool Complete) Result, int Budget)> PassAsync(int budget) =>
|
||||
_ceiling.ReadAsync(budget,
|
||||
b => ReadRawPassAsync(fullReference, startUtc, endUtc, cap, b, cancellationToken),
|
||||
(r, _) => !r.Complete);
|
||||
}
|
||||
|
||||
/// <summary>One forward raw read of a window, drained, with its pre-window row count and whether it is complete.</summary>
|
||||
private async Task<(List<HistorianSample> Samples, int PreWindowRows, bool Complete)> ReadRawPassAsync(
|
||||
string fullReference, DateTime startUtc, DateTime endUtc, int cap, int budget, CancellationToken cancellationToken)
|
||||
{
|
||||
var samples = new List<HistorianSample>();
|
||||
long received = 0;
|
||||
var preWindow = 0;
|
||||
var reachedEnd = false;
|
||||
await foreach (var sample in _client
|
||||
.ReadRawAsync(fullReference, RawAggregateReader.RequestStartFor(startUtc), endUtc, budget, cancellationToken)
|
||||
.ConfigureAwait(false))
|
||||
{
|
||||
received++;
|
||||
// Filter while draining — the rest of the (bounded) stream is consumed, never abandoned.
|
||||
if (RawAggregateReader.IsWindowSample(sample, startUtc, endUtc))
|
||||
{
|
||||
if (cap == 0 || samples.Count < cap) samples.Add(sample);
|
||||
}
|
||||
// A row before the window — or, for a window starting at/before the epoch (no look-back), AVEVA's synthetic row at
|
||||
// the query start, which is no sample either (#694 F1: it used to set reachedEnd, so a ceiling-truncated read
|
||||
// passed as complete) — is a pre-window row.
|
||||
else if (sample.Timestamp is { } ts
|
||||
&& (ts.ToDateTime() < startUtc || !RawAggregateReader.IsAfterFloor(ts.ToDateTime()))) preWindow++;
|
||||
else if (sample.Timestamp is not null) reachedEnd = true;
|
||||
}
|
||||
|
||||
// cap == 0 ("all values") is complete only when the stream proves it: short of the budget (always, unless a strict
|
||||
// gateway's ceiling became the budget — #691.3) or past the window end.
|
||||
var complete = received < budget || reachedEnd || (cap > 0 && samples.Count >= cap);
|
||||
return (samples, preWindow, complete);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>Per-sample aggregates (#681)</b> — Average, Count, Minimum, Maximum, Range, Start, End, Delta,
|
||||
/// Minimum/MaximumActualTime — are evaluated from a raw read with the SQL backends' shared evaluator
|
||||
/// (<see cref="RawAggregateReader"/>): identical values, status, Calculated/Partial bits and timestamps to
|
||||
/// SqlServer/QuestDB for the same samples, forward and reverse (#677). The in-window raw rows are capped by
|
||||
/// <c>ServerHistorian:MaxTimeWeightedRawRows</c> (<see cref="HistoryReadLimitExceededException"/>, not counted as
|
||||
/// a backend failure).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>TimeAverage / Total</b> stay native AVEVA retrieval modes (<see cref="AggregateModeMapper"/>), stamped with
|
||||
/// the interval start and the Calculated bit; AVEVA's extra row for the interval before the window is dropped and
|
||||
/// an Uncertain interval keeps its value (#685). Both directions use the SQL backends' interval grid with forward
|
||||
/// requests only — a reverse read (which the gateway refuses) anchored at its later time (#677), the clipped interval
|
||||
/// read over exactly its span and marked Partial (#691.2, <see cref="NativeAggregatePlan"/>). Everything else is <see cref="HistoryAggregateNotSupportedException"/>
|
||||
/// (see <see cref="ServedAggregates"/>).
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
/// <exception cref="ArgumentOutOfRangeException">A raw-evaluated read whose window holds more than
|
||||
/// <see cref="BucketPlan.MaxBuckets"/> intervals.</exception>
|
||||
public async Task<HistoryReadResult> ReadProcessedAsync(
|
||||
string fullReference, DateTime startUtc, DateTime endUtc, TimeSpan interval,
|
||||
HistoryAggregateType aggregate, CancellationToken cancellationToken)
|
||||
{
|
||||
if (RawAggregateReader.Serves(aggregate))
|
||||
return await ReadProcessedFromRawAsync(fullReference, startUtc, endUtc, interval, aggregate, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
// Mapped OUTSIDE the failure-recording try: an aggregate the gateway has no mode for is a client
|
||||
// request the server answers BadAggregateNotSupported, not a backend fault to count (#285).
|
||||
var mode = AggregateModeMapper.ToRetrievalMode(aggregate);
|
||||
// Planned OUTSIDE the failure-recording try: too many intervals is a client request limit, not a backend fault.
|
||||
return await ReadNativeAsync(fullReference, BucketPlan.Create(startUtc, endUtc, interval), mode, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A native TimeAverage / Total read on the SQL backends' interval grid, either direction (#691.2): forward native
|
||||
/// requests only, the clipped interval read over exactly its span. See <see cref="NativeAggregatePlan"/>.
|
||||
/// </summary>
|
||||
private async Task<HistoryReadResult> ReadNativeAsync(string fullReference, BucketPlan plan, RetrievalMode mode,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (plan.BucketCount == 0) return new HistoryReadResult([], ContinuationPoint: null);
|
||||
try
|
||||
{
|
||||
// Total/Count are now native gateway retrieval modes — no client-side scaling
|
||||
// (unlike the Wonderware path that derived Total as Average × interval-seconds).
|
||||
var buckets = new List<HistorianAggregateSample>();
|
||||
await foreach (var bucket in _client
|
||||
.ReadAggregateAsync(fullReference, startUtc, endUtc, mode, interval, cancellationToken)
|
||||
.ConfigureAwait(false))
|
||||
if (!await _tags.ExistsAsync(fullReference, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
buckets.Add(bucket);
|
||||
RecordOutcome(success: true, error: null);
|
||||
return new HistoryReadResult(plan.Assemble(new Dictionary<long, DataValueSnapshot>(), t => AggregateEvaluator.NoData(t, _time.GetUtcNow().UtcDateTime)), ContinuationPoint: null);
|
||||
}
|
||||
var byBucket = new Dictionary<long, DataValueSnapshot>();
|
||||
foreach (var request in NativeAggregatePlan.Requests(plan))
|
||||
{
|
||||
var buckets = new List<HistorianAggregateSample>();
|
||||
await foreach (var bucket in _client
|
||||
.ReadAggregateAsync(fullReference, request.StartUtc, request.EndUtc, mode, request.Interval, cancellationToken)
|
||||
.ConfigureAwait(false))
|
||||
{
|
||||
buckets.Add(bucket); // drained to completion, mapped afterwards
|
||||
}
|
||||
NativeAggregatePlan.Place(plan, request, buckets, byBucket);
|
||||
}
|
||||
|
||||
var snapshots = SampleMapper.ToAggregateSnapshots(buckets);
|
||||
RecordOutcome(success: true, error: null);
|
||||
return new HistoryReadResult(snapshots, ContinuationPoint: null);
|
||||
return new HistoryReadResult(NativeAggregatePlan.Assemble(plan, byBucket), ContinuationPoint: null);
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
throw; // caller-requested, not a backend fault
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
RecordReadFailure(ex);
|
||||
if (IsTransportFailure(ex)) throw new System.IO.IOException("HistorianGateway read transport is unavailable.", ex);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Serves a per-sample aggregate from a raw read (#681); see <see cref="RawAggregateReader"/>.</summary>
|
||||
private async Task<HistoryReadResult> ReadProcessedFromRawAsync(
|
||||
string fullReference, DateTime startUtc, DateTime endUtc, TimeSpan interval,
|
||||
HistoryAggregateType aggregate, CancellationToken cancellationToken)
|
||||
{
|
||||
// Planned OUTSIDE the failure-recording try (as on the SQL backends): too many intervals is a client request
|
||||
// limit, not a backend fault.
|
||||
var plan = BucketPlan.Create(startUtc, endUtc, interval);
|
||||
try
|
||||
{
|
||||
var now = _time.GetUtcNow().UtcDateTime;
|
||||
if (plan.BucketCount > 0 && !await _tags.ExistsAsync(fullReference, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
RecordOutcome(success: true, error: null);
|
||||
return new HistoryReadResult(plan.Assemble(new Dictionary<long, DataValueSnapshot>(), t => AggregateEvaluator.NoData(t, now)), ContinuationPoint: null);
|
||||
}
|
||||
var values = await RawAggregateReader
|
||||
.ReadAsync(_client, _ceiling, fullReference, aggregate, plan, _maxRawRows, now, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
RecordOutcome(success: true, error: null);
|
||||
return new HistoryReadResult(values, ContinuationPoint: null);
|
||||
}
|
||||
catch (HistoryReadLimitExceededException)
|
||||
{
|
||||
// A client-request limit (the raw-row cap), not a backend fault: not counted as a failure.
|
||||
throw;
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
// The caller (client cancel / HistoryReadDeadline) asked for this — not a backend fault.
|
||||
throw;
|
||||
}
|
||||
catch (Exception ex) when (IsMaxValuesCeilingRejection(ex))
|
||||
{
|
||||
// The gateway refused the row budget (Historian:ReadLimits:MaxValuesCeiling with
|
||||
// AllowStreamingBeyondCeiling=false): the same client-request limit as the cap, not a backend fault.
|
||||
throw new HistoryReadLimitExceededException(_maxRawRows,
|
||||
$"The HistorianGateway refused the raw row budget of the processed read of '{fullReference}' (its " +
|
||||
"Historian:ReadLimits:MaxValuesCeiling); lower ServerHistorian:MaxTimeWeightedRawRows below that ceiling " +
|
||||
"or narrow the window.");
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
RecordReadFailure(ex);
|
||||
if (IsTransportFailure(ex)) throw new System.IO.IOException("HistorianGateway read transport is unavailable.", ex);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// True when <paramref name="ex"/> (or an inner exception — the client package wraps the <see cref="RpcException"/>)
|
||||
/// is the gateway's <c>InvalidArgument</c> "max_values exceeds the server ceiling" rejection.
|
||||
/// </summary>
|
||||
internal static bool IsMaxValuesCeilingRejection(Exception ex) => GatewayReadLimits.IsCeilingRejection(ex);
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>
|
||||
/// #693: served from RAW gateway reads — each requested time's stored sample at-or-before and after it — evaluated with
|
||||
/// the SQL backends' shared <see cref="AtTimeEvaluator"/> (Part 11 §6.5.4), so the Gateway answers what SqlServer and
|
||||
/// QuestDB answer for the same stored samples; see <see cref="GatewayAtTimeReader"/> for the read plan and its cost.
|
||||
/// AVEVA's own interpolated answer (the gateway's <c>ReadAtTime</c>) returned the previous value as Good at a stored
|
||||
/// NULL's time, null + Bad at a Good sample right after a NULL, a drifted value at an exact stored sample, and no
|
||||
/// Interpolated bit; it is only used when <c>ServerHistorian:GatewayAtTimeFromRaw = false</c> (kill switch).
|
||||
/// A gateway value is a Double (or a String) whatever the node's type (AVEVA stores a Boolean as a 0/1 <c>Int2</c>), so
|
||||
/// without a type hint every numeric tag is interpolated linearly; the node manager passes the node's type through
|
||||
/// <see cref="ReadAtTimeAsync(string, IReadOnlyList{DateTime}, TypeCode?, CancellationToken)"/>.
|
||||
/// </remarks>
|
||||
public Task<HistoryReadResult> ReadAtTimeAsync(
|
||||
string fullReference, IReadOnlyList<DateTime> timestampsUtc, CancellationToken cancellationToken) =>
|
||||
ReadAtTimeAsync(fullReference, timestampsUtc, valueType: null, cancellationToken);
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>
|
||||
/// #693: the hint makes the AtTime rules type-exact, as on the SQL backends — the gateway's Double rows are converted to
|
||||
/// the node's type BEFORE the shared evaluator runs (Boolean = value ≠ 0 ⇒ stepped; an integer type ⇒ Int64 ⇒
|
||||
/// interpolated and rounded half away from zero, then returned as the node's integer type; String ⇒ stepped; Float /
|
||||
/// Double / no hint ⇒ linear Double). Ignored with the kill switch (AVEVA's own answer).
|
||||
/// </remarks>
|
||||
public async Task<HistoryReadResult> ReadAtTimeAsync(
|
||||
string fullReference, IReadOnlyList<DateTime> timestampsUtc, TypeCode? valueType, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(timestampsUtc);
|
||||
if (!_atTimeFromRaw) return await ReadAtTimeNativeAsync(fullReference, timestampsUtc, cancellationToken).ConfigureAwait(false);
|
||||
try
|
||||
{
|
||||
var now = _time.GetUtcNow().UtcDateTime;
|
||||
if (timestampsUtc.Count > 0 && !await _tags.ExistsAsync(fullReference, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
RecordOutcome(success: true, error: null);
|
||||
return MissingAtTime(timestampsUtc, now);
|
||||
}
|
||||
var values = await GatewayAtTimeReader
|
||||
.ReadAsync(_client, _ceiling, fullReference, timestampsUtc, now, cancellationToken, valueType)
|
||||
.ConfigureAwait(false);
|
||||
RecordOutcome(success: true, error: null);
|
||||
return new HistoryReadResult(values, ContinuationPoint: null);
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
throw; // caller-requested, not a backend fault
|
||||
}
|
||||
catch (HistoryReadLimitExceededException)
|
||||
{
|
||||
throw; // a client-request limit (a pathological tie cluster), not a backend fault
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
RecordReadFailure(ex);
|
||||
if (IsTransportFailure(ex)) throw new System.IO.IOException("HistorianGateway read transport is unavailable.", ex);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>The pre-#693 AtTime: AVEVA's own interpolation, aligned to the request (kill switch only).</summary>
|
||||
private async Task<HistoryReadResult> ReadAtTimeNativeAsync(
|
||||
string fullReference, IReadOnlyList<DateTime> timestampsUtc, CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (timestampsUtc.Count > 0 && !await _tags.ExistsAsync(fullReference, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
RecordOutcome(success: true, error: null);
|
||||
return MissingAtTime(timestampsUtc, _time.GetUtcNow().UtcDateTime);
|
||||
}
|
||||
|
||||
var samples = await _client
|
||||
.ReadAtTimeAsync(fullReference, timestampsUtc, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
@@ -121,10 +527,56 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
catch (Exception ex)
|
||||
{
|
||||
RecordReadFailure(ex);
|
||||
if (IsTransportFailure(ex)) throw new System.IO.IOException("HistorianGateway read transport is unavailable.", ex);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>
|
||||
/// <c>false</c> (Gitea #686): the gateway's <c>ReadRaw</c> streams ascending only and rejects <c>end < start</c>
|
||||
/// (<c>InvalidArgument</c>). The server serves reverse and end-only capped raw reads from bounded forward reads
|
||||
/// instead, and never sends this source a reverse window (<see cref="ReadRawAsync"/> refuses one unsent).
|
||||
/// </remarks>
|
||||
public bool SupportsReverseRaw => false;
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>
|
||||
/// Gitea #694: <c>0</c> until a strict gateway (<c>AllowStreamingBeyondCeiling=false</c>) has refused a budget and
|
||||
/// named its ceiling (#691.3, <see cref="GatewayRowCeiling"/>); then that ceiling less the
|
||||
/// <see cref="RawAggregateReader.PreWindowAllowance"/> a capped read adds for AVEVA's boundary row — so a page of this
|
||||
/// size asks for exactly the ceiling. Back to <c>0</c> when a re-probe finds the gateway streaming again.
|
||||
/// </remarks>
|
||||
public int MaxRawValuesPerRead => _ceiling.Learned is var c and > 0
|
||||
? Math.Max(1, c - RawAggregateReader.PreWindowAllowance)
|
||||
: 0;
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>
|
||||
/// <c>false</c>: the gateway's <c>ReadEvents</c> streams ascending only (Gitea #675). The server never sends it a
|
||||
/// reverse read; since #691.4 it walks one back from bounded forward reads (ties sorted by EventId — the gateway
|
||||
/// orders by event time only).
|
||||
/// </remarks>
|
||||
public bool SupportsReverseEvents => false;
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks><see cref="ServedAggregates"/>; every other member answers <c>BadAggregateNotSupported</c> and is not advertised.</remarks>
|
||||
public IReadOnlySet<HistoryAggregateType> SupportedAggregates => ServedAggregates;
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>
|
||||
/// A reverse (newest-first) read is refused with <see cref="NotSupportedException"/> (Gitea #675) — the gateway's
|
||||
/// <c>ReadEvents</c> streams ascending only and rejects <c>end < start</c>. The server never calls this with
|
||||
/// <c>reverse</c>: it walks a reverse read back from bounded forward reads (#691.4). Nothing is sent.
|
||||
/// </remarks>
|
||||
public Task<HistoricalEventsResult> ReadEventsAsync(
|
||||
string? sourceName, DateTime startUtc, DateTime endUtc, int maxEvents, bool reverse,
|
||||
CancellationToken cancellationToken) =>
|
||||
reverse
|
||||
? throw new NotSupportedException(
|
||||
"The HistorianGateway backend reads events oldest-first only; a reverse (newest-first) events read is not supported.")
|
||||
: ReadEventsAsync(sourceName, startUtc, endUtc, maxEvents, cancellationToken);
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<HistoricalEventsResult> ReadEventsAsync(
|
||||
string? sourceName, DateTime startUtc, DateTime endUtc, int maxEvents,
|
||||
@@ -133,7 +585,8 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
try
|
||||
{
|
||||
var hasCap = maxEvents > 0;
|
||||
var collected = new List<HistoricalEvent>(hasCap ? maxEvents : 0);
|
||||
// Bounded initial capacity: an uncapped read arrives as int.MaxValue ("Array dimensions exceeded", #691.4 review).
|
||||
var collected = new List<HistoricalEvent>(hasCap ? Math.Min(maxEvents, 1024) : 0);
|
||||
var truncated = false;
|
||||
|
||||
await foreach (var wireEvent in _client
|
||||
@@ -170,6 +623,7 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
catch (Exception ex)
|
||||
{
|
||||
RecordReadFailure(ex);
|
||||
if (IsTransportFailure(ex)) throw new System.IO.IOException("HistorianGateway read transport is unavailable.", ex);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
@@ -201,7 +655,7 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
|
||||
/// <summary>
|
||||
/// Reconciles a gateway at-time reply against the requested timestamps to honour the
|
||||
/// <see cref="IHistorianDataSource.ReadAtTimeAsync"/> contract: exactly one snapshot per
|
||||
/// <see cref="IHistorianDataSource.ReadAtTimeAsync(string, IReadOnlyList{DateTime}, CancellationToken)"/> contract: exactly one snapshot per
|
||||
/// requested timestamp, in request order. Returned samples are indexed by timestamp ticks;
|
||||
/// any requested timestamp the gateway did not return is filled with a Bad-quality
|
||||
/// (<c>0x80000000</c>) snapshot stamped at the requested time rather than positionally
|
||||
@@ -244,6 +698,19 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
return result;
|
||||
}
|
||||
|
||||
/// <summary>No-data values for a missing tag, stamped with the client's requested times.</summary>
|
||||
private static HistoryReadResult MissingAtTime(IReadOnlyList<DateTime> times, DateTime now) =>
|
||||
new(times.Select(t => AtTimeEvaluator.Evaluate(t, null, null, now, now)).ToArray(), ContinuationPoint: null);
|
||||
|
||||
/// <summary>Classifies actual gRPC/package transport failures without treating refusals as outages (#697).</summary>
|
||||
private static bool IsTransportFailure(Exception ex) => ex switch
|
||||
{
|
||||
ZB.MOM.WW.HistorianGateway.Client.HistorianGatewayUnavailableException => true,
|
||||
RpcException rpc => rpc.StatusCode is Grpc.Core.StatusCode.Unavailable or Grpc.Core.StatusCode.DeadlineExceeded,
|
||||
TimeoutException => true,
|
||||
_ => ex.InnerException is { } inner && IsTransportFailure(inner),
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// Records a failed read: bumps the health counters and logs a generic, redaction-safe
|
||||
/// debug line (no tag, host, or value). The exception itself is rethrown by the caller.
|
||||
|
||||
+7
-1
@@ -38,7 +38,13 @@ public static class GatewayHistorian
|
||||
|
||||
return new GatewayHistorianDataSource(
|
||||
HistorianGatewayClientAdapter.Create(options, loggerFactory),
|
||||
logger);
|
||||
logger,
|
||||
// #681: the raw-evaluated per-sample aggregates share the SQL backends' raw-row cap.
|
||||
maxRawRows: options.MaxTimeWeightedRawRows,
|
||||
// #693: AtTime from raw reads + the shared evaluator; false = AVEVA's own interpolation (kill switch).
|
||||
atTimeFromRaw: options.GatewayAtTimeFromRaw,
|
||||
// #694 R2: a crowding re-read never makes room for more pre-window rows than the tie-cluster limit.
|
||||
maxPreWindowRows: options.MaxTieClusterOverfetch);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
||||
@@ -0,0 +1,148 @@
|
||||
using System.Globalization;
|
||||
using System.Text.RegularExpressions;
|
||||
using Grpc.Core;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway;
|
||||
|
||||
/// <summary>
|
||||
/// The HistorianGateway's raw-read row ceiling (<c>Historian:ReadLimits:MaxValuesCeiling</c>, default 1,000,000) as seen
|
||||
/// from this side of the wire. With <c>AllowStreamingBeyondCeiling=false</c> (a "strict" gateway) the gateway refuses
|
||||
/// any <c>ReadRaw</c> whose <c>max_values</c> exceeds the ceiling with <c>InvalidArgument</c> — BEFORE it streams a row
|
||||
/// — and names the ceiling in the refusal (<c>"max_values exceeds the server ceiling of {n}; …"</c>). The gateway has
|
||||
/// no RPC that reports the ceiling, so the refusal is how it is learned (#691.3).
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// A read whose budget the gateway refuses is re-issued ONCE with the ceiling as its budget, and the ceiling is
|
||||
/// remembered per data source (<see cref="GatewayRowCeiling"/>); the caller treats a stream that delivered the whole
|
||||
/// ceiling as possibly truncated (never answered from).
|
||||
/// </remarks>
|
||||
internal static class GatewayReadLimits
|
||||
{
|
||||
private const string CeilingRejectionText = "max_values exceeds the server ceiling";
|
||||
|
||||
private static readonly Regex CeilingNumber = new(@"max_values exceeds the server ceiling of (\d+)",
|
||||
RegexOptions.CultureInvariant | RegexOptions.Compiled);
|
||||
|
||||
/// <summary>
|
||||
/// True when <paramref name="ex"/> (or an inner exception — the client package wraps the <see cref="RpcException"/>)
|
||||
/// is the gateway's <c>InvalidArgument</c> "max_values exceeds the server ceiling" rejection.
|
||||
/// </summary>
|
||||
/// <param name="ex">The exception.</param>
|
||||
/// <returns>Whether it is the ceiling rejection.</returns>
|
||||
public static bool IsCeilingRejection(Exception ex) => Rejection(ex) is not null;
|
||||
|
||||
/// <summary>
|
||||
/// The ceiling a ceiling rejection names, or — when <paramref name="ex"/> is a ceiling rejection that does not
|
||||
/// carry a parseable number — the gateway's default (<see cref="RawAggregateReader.GatewayDefaultMaxValuesCeiling"/>).
|
||||
/// </summary>
|
||||
/// <param name="ex">The exception.</param>
|
||||
/// <param name="ceiling">The ceiling (> 0) when this returns true.</param>
|
||||
/// <returns>Whether <paramref name="ex"/> is a ceiling rejection.</returns>
|
||||
public static bool TryGetCeiling(Exception ex, out int ceiling)
|
||||
{
|
||||
ceiling = 0;
|
||||
if (Rejection(ex) is not { } rpc) return false;
|
||||
var m = CeilingNumber.Match(rpc.Status.Detail);
|
||||
ceiling = m.Success && int.TryParse(m.Groups[1].Value, NumberStyles.None, CultureInfo.InvariantCulture, out var n) && n > 0
|
||||
? n
|
||||
: RawAggregateReader.GatewayDefaultMaxValuesCeiling;
|
||||
return true;
|
||||
}
|
||||
|
||||
private static RpcException? Rejection(Exception ex)
|
||||
{
|
||||
for (var e = ex; e is not null; e = e.InnerException)
|
||||
if (e is RpcException { StatusCode: StatusCode.InvalidArgument } rpc
|
||||
&& rpc.Status.Detail.Contains(CeilingRejectionText, StringComparison.Ordinal))
|
||||
return rpc;
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A strict gateway's row ceiling as learned by ONE data source (#691.3 review): every refused attempt counts as an
|
||||
/// error in the gateway's own outcome metrics, so the ceiling a refusal names is remembered and later reads ask for at
|
||||
/// most it — no refusal per read in the steady state. Thread-safe (one <see cref="int"/>, read/written atomically; a
|
||||
/// race costs at most one extra refusal or re-probe, never a wrong answer).
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>Rules.</b> (1) Unknown ceiling: ask for the full budget; a refusal naming <c>N</c> < budget learns <c>N</c> and
|
||||
/// the read is re-issued once at <c>N</c>. (2) Learned ceiling <c>C</c> below the budget: ask for <c>C</c> — a refusal
|
||||
/// there (the gateway was LOWERED) re-learns its <c>N</c> as in (1). (3) If that <c>C</c>-bounded read came back
|
||||
/// full but not provably complete, the gateway may have been RAISED (or switched to streaming) since <c>C</c> was
|
||||
/// learned, and answering <c>BadTooManyOperations</c> from a stale ceiling would be wrong: the read is re-probed ONCE
|
||||
/// at the full budget — served ⇒ the ceiling is forgotten; refused naming a larger <c>N</c> ⇒ <c>N</c> is learned and
|
||||
/// the read is re-issued at <c>N</c>; refused naming <c>C</c> again ⇒ the <c>C</c>-bounded result stands (no repeat).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Re-probe timing (#696).</b> Raw and AtTime reads re-probe incomplete streams immediately. Per-sample
|
||||
/// aggregate reads opt into a 30-second cooldown after a refusal; a full stream during it remains a request
|
||||
/// limit, never a truncated answer. After the cooldown the next incomplete read detects a raised ceiling.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
internal sealed class GatewayRowCeiling(TimeProvider? timeProvider = null)
|
||||
{
|
||||
private readonly TimeProvider _time = timeProvider ?? TimeProvider.System;
|
||||
private long _lastRefusalTicks;
|
||||
private int _learned; // 0 = unknown / not strict
|
||||
|
||||
/// <summary>The learned ceiling (0 = none).</summary>
|
||||
public int Learned => Volatile.Read(ref _learned);
|
||||
|
||||
/// <summary>Runs a drained read under the rules above.</summary>
|
||||
/// <typeparam name="T">The pass result.</typeparam>
|
||||
/// <param name="budget">The <c>max_values</c> the read wants.</param>
|
||||
/// <param name="pass">One drained read with the given budget.</param>
|
||||
/// <param name="needsMore">True when a pass with the given budget delivered it all without proving completeness.</param>
|
||||
/// <param name="deferRecentReprobe">Aggregate-only opt-in: reuse a refusal for 30 seconds before re-probing an incomplete read.</param>
|
||||
/// <returns>The result and the budget it was read with.</returns>
|
||||
public async Task<(T Result, int Budget)> ReadAsync<T>(int budget, Func<int, Task<T>> pass, Func<T, int, bool> needsMore, bool deferRecentReprobe = false)
|
||||
{
|
||||
var learned = Learned;
|
||||
if (learned <= 0 || learned >= budget) return await AttemptAsync(budget, pass).ConfigureAwait(false);
|
||||
|
||||
var bounded = await AttemptAsync(learned, pass).ConfigureAwait(false);
|
||||
if (bounded.Budget < learned || !needsMore(bounded.Result, bounded.Budget)) return bounded;
|
||||
|
||||
// #696: aggregate reads may reuse a recent refusal for 30 seconds. Raw paging keeps immediate re-probes.
|
||||
if (deferRecentReprobe && _time.GetUtcNow().UtcTicks - Volatile.Read(ref _lastRefusalTicks) < TimeSpan.FromSeconds(30).Ticks)
|
||||
return bounded;
|
||||
|
||||
// Rule 3: full at a possibly stale ceiling — re-probe once at the full budget.
|
||||
int fresh;
|
||||
try
|
||||
{
|
||||
var full = await pass(budget).ConfigureAwait(false);
|
||||
Interlocked.CompareExchange(ref _learned, 0, learned);
|
||||
return (full, budget);
|
||||
}
|
||||
catch (Exception ex) when (GatewayReadLimits.TryGetCeiling(ex, out fresh) && fresh < budget)
|
||||
{
|
||||
// fall through (outside the catch)
|
||||
}
|
||||
Learn(fresh);
|
||||
return fresh > learned ? (await pass(fresh).ConfigureAwait(false), fresh) : bounded;
|
||||
}
|
||||
|
||||
private void Learn(int ceiling)
|
||||
{
|
||||
Volatile.Write(ref _lastRefusalTicks, _time.GetUtcNow().UtcTicks);
|
||||
Volatile.Write(ref _learned, ceiling);
|
||||
}
|
||||
|
||||
private async Task<(T Result, int Budget)> AttemptAsync<T>(int budget, Func<int, Task<T>> pass)
|
||||
{
|
||||
int ceiling;
|
||||
try
|
||||
{
|
||||
return (await pass(budget).ConfigureAwait(false), budget);
|
||||
}
|
||||
catch (Exception ex) when (GatewayReadLimits.TryGetCeiling(ex, out ceiling) && ceiling < budget)
|
||||
{
|
||||
// fall through to the retry (outside the catch, so a second refusal is not nested in this one)
|
||||
}
|
||||
Learn(ceiling);
|
||||
return (await pass(ceiling).ConfigureAwait(false), ceiling);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
using Grpc.Core;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway;
|
||||
|
||||
/// <summary>Bounded metadata cache (#653). Missing tags expire quickly; failures never become missing tags.</summary>
|
||||
internal sealed class GatewayTagPresenceCache(IHistorianGatewayClient client, TimeProvider time)
|
||||
{
|
||||
internal const int Capacity = 4096;
|
||||
private readonly object _sync = new();
|
||||
private readonly Dictionary<string, (bool Exists, DateTimeOffset Expires)> _entries = new(StringComparer.Ordinal);
|
||||
|
||||
public async Task<bool> ExistsAsync(string tag, CancellationToken ct)
|
||||
{
|
||||
ct.ThrowIfCancellationRequested();
|
||||
lock (_sync)
|
||||
if (_entries.TryGetValue(tag, out var entry) && time.GetUtcNow() < entry.Expires) return entry.Exists;
|
||||
|
||||
bool exists;
|
||||
try { exists = await client.TagExistsAsync(tag, ct).ConfigureAwait(false); }
|
||||
catch (Exception ex) when (IsNotFound(ex)) { exists = false; }
|
||||
ct.ThrowIfCancellationRequested();
|
||||
lock (_sync)
|
||||
{
|
||||
if (!_entries.ContainsKey(tag) && _entries.Count >= Capacity) _entries.Remove(_entries.Keys.First());
|
||||
_entries[tag] = (exists, time.GetUtcNow().AddSeconds(exists ? 60 : 5));
|
||||
}
|
||||
return exists;
|
||||
}
|
||||
|
||||
private static bool IsNotFound(Exception ex)
|
||||
{
|
||||
for (Exception? e = ex; e is not null; e = e.InnerException)
|
||||
if (e is RpcException { StatusCode: StatusCode.NotFound }) return true;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@@ -29,6 +29,14 @@ public sealed class GatewayTagProvisioner : IHistorianProvisioning, IAsyncDispos
|
||||
/// </summary>
|
||||
private const uint DefaultStorageRateMs = 1000;
|
||||
|
||||
/// <summary>
|
||||
/// Integral divisor stamped on every provisioned analog tag (#677). AVEVA's <c>Total</c> integral is
|
||||
/// <c>∫ value dt (seconds) ÷ IntegralDivisor</c>; the proto's <c>double</c> default of 0 would reach the gateway as
|
||||
/// "unset" (or a divide-by-zero), so 1 is sent explicitly — the value·seconds unit the SQL backends' <c>Total</c>
|
||||
/// uses. (The live magnitude check against AVEVA is still open: it needs the VPN.)
|
||||
/// </summary>
|
||||
private const double DefaultIntegralDivisor = 1.0;
|
||||
|
||||
private readonly IHistorianGatewayClient _client;
|
||||
private readonly ILogger<GatewayTagProvisioner> _logger;
|
||||
|
||||
@@ -70,6 +78,8 @@ public sealed class GatewayTagProvisioner : IHistorianProvisioning, IAsyncDispos
|
||||
Description = request.Description ?? string.Empty,
|
||||
// Required: a 0 storage rate makes the gateway's EnsureTags throw (see the constant).
|
||||
StorageRateMs = DefaultStorageRateMs,
|
||||
// Explicit: the proto3 double default (0) is not "1" (see the constant).
|
||||
IntegralDivisor = DefaultIntegralDivisor,
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
+11
@@ -81,6 +81,17 @@ public sealed class HistorianGatewayClientAdapter : IHistorianGatewayClient, IDi
|
||||
return new HistorianGatewayClientAdapter(HistorianGatewayClient.Create(clientOptions));
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<bool> TagExistsAsync(string tag, CancellationToken ct)
|
||||
{
|
||||
// #653: older gateway SDKs misparse a missing GetTagInfo response as Internal. A catalog query proves
|
||||
// absence without guessing from an error. Even a glob-like tag is checked by exact name, never by row count.
|
||||
var found = false;
|
||||
await foreach (var name in _inner.BrowseTagNamesAsync(tag, ct).ConfigureAwait(false))
|
||||
found |= string.Equals(name, tag, StringComparison.OrdinalIgnoreCase);
|
||||
return found; // drain first: a trailing stream fault must propagate and must not become a cache entry
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public IAsyncEnumerable<HistorianSample> ReadRawAsync(
|
||||
string tag, DateTime startUtc, DateTime endUtc, int maxValues, CancellationToken ct) =>
|
||||
|
||||
@@ -11,6 +11,12 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway;
|
||||
/// </summary>
|
||||
public interface IHistorianGatewayClient : IAsyncDisposable
|
||||
{
|
||||
/// <summary>Checks the tag catalog; false means explicitly absent, while failures propagate.</summary>
|
||||
/// <param name="tag">Historian tag name.</param>
|
||||
/// <param name="ct">Cancellation token.</param>
|
||||
/// <returns>Whether the historian contains the tag.</returns>
|
||||
Task<bool> TagExistsAsync(string tag, CancellationToken ct);
|
||||
|
||||
/// <summary>Streams raw historian samples for a tag over a time window.</summary>
|
||||
/// <param name="tag">The historian tag name to read.</param>
|
||||
/// <param name="startUtc">Inclusive start of the requested time window (UTC).</param>
|
||||
|
||||
+35
-17
@@ -1,3 +1,4 @@
|
||||
using System.Collections.Frozen;
|
||||
using ZB.MOM.WW.HistorianGateway.Contracts.Grpc;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
|
||||
@@ -5,40 +6,57 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Mapping;
|
||||
|
||||
/// <summary>
|
||||
/// Maps the driver-agnostic <see cref="HistoryAggregateType"/> (OPC UA Part 13 aggregate) onto the
|
||||
/// gateway's native <see cref="RetrievalMode"/>.
|
||||
/// gateway's native <see cref="RetrievalMode"/> — for the aggregates still answered by AVEVA itself.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// Average/Minimum/Maximum line up with the legacy Wonderware client's aggregate mapping. The two
|
||||
/// remaining members are now served by <b>native</b> gateway retrieval modes:
|
||||
/// <see cref="HistoryAggregateType.Total"/> → <see cref="RetrievalMode.Integral"/> and
|
||||
/// <see cref="HistoryAggregateType.Count"/> → <see cref="RetrievalMode.Counter"/>.
|
||||
/// Native (AVEVA-computed): <see cref="HistoryAggregateType.TimeAverage"/> → <see cref="RetrievalMode.TimeWeightedAverage"/>
|
||||
/// and <see cref="HistoryAggregateType.Total"/> → <see cref="RetrievalMode.Integral"/>. They stay native because an exact
|
||||
/// Part 13 time-weighted evaluation needs the real bounding samples on both sides of the window (time, value and
|
||||
/// quality of the latest sample before it), and the gateway has no "previous stored sample" read — a raw read streams
|
||||
/// forward from its start and AVEVA replaces whatever lies before with an initial-value row stamped at the query start,
|
||||
/// losing the original time a sloped segment across the window start needs (#681).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// This replaces the Wonderware-era client-side workarounds (Total derived as Average × interval,
|
||||
/// Count approximated from a value count): no client-side scaling is performed any more, so the
|
||||
/// gateway path is a strict improvement.
|
||||
/// Since #681 the per-sample aggregates (Average, Count, Minimum, Maximum, Range, Start, End, Delta,
|
||||
/// Minimum/MaximumActualTime) are NOT native: <see cref="GatewayHistorianDataSource"/> evaluates them from raw reads
|
||||
/// with the SQL backends' shared evaluator (Minimum/Maximum used to map to AVEVA's <c>MinimumWithTime</c> /
|
||||
/// <c>MaximumWithTime</c>). Every other member — incl. TimeAverage2, Total2, Interpolative, PercentGood, PercentBad — is
|
||||
/// <see cref="HistoryAggregateNotSupportedException"/> here.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
internal static class AggregateModeMapper
|
||||
{
|
||||
/// <summary>
|
||||
/// The aggregates answered by a native AVEVA retrieval mode — every defined member <see cref="TryMap"/> maps, derived
|
||||
/// from it so the dispatch and the declared set can never disagree. Part of
|
||||
/// <see cref="GatewayHistorianDataSource.ServedAggregates"/> (#681/#682).
|
||||
/// </summary>
|
||||
public static readonly FrozenSet<HistoryAggregateType> NativeAggregates =
|
||||
Enum.GetValues<HistoryAggregateType>().Where(a => TryMap(a) is not null).ToFrozenSet();
|
||||
|
||||
/// <summary>Maps an aggregate function to the gateway retrieval mode.</summary>
|
||||
/// <param name="aggregate">The driver-agnostic aggregate function.</param>
|
||||
/// <returns>The matching gateway <see cref="RetrievalMode"/>.</returns>
|
||||
/// <exception cref="HistoryAggregateNotSupportedException">A defined member the gateway has no retrieval mode for.</exception>
|
||||
/// <exception cref="ArgumentOutOfRangeException">An undefined enum value.</exception>
|
||||
public static RetrievalMode ToRetrievalMode(HistoryAggregateType aggregate) => aggregate switch
|
||||
public static RetrievalMode ToRetrievalMode(HistoryAggregateType aggregate)
|
||||
{
|
||||
HistoryAggregateType.Average => RetrievalMode.TimeWeightedAverage,
|
||||
HistoryAggregateType.Minimum => RetrievalMode.MinimumWithTime,
|
||||
HistoryAggregateType.Maximum => RetrievalMode.MaximumWithTime,
|
||||
HistoryAggregateType.Total => RetrievalMode.Integral,
|
||||
HistoryAggregateType.Count => RetrievalMode.Counter,
|
||||
if (TryMap(aggregate) is { } mode) return mode;
|
||||
|
||||
// Part 13 aggregates the gateway has no retrieval mode for (#285) — an UNDEFINED enum value is a
|
||||
// programming error, but a defined-yet-unserved member is a normal client request the server
|
||||
// answers with BadAggregateNotSupported.
|
||||
_ when Enum.IsDefined(aggregate) => throw new HistoryAggregateNotSupportedException(aggregate),
|
||||
_ => throw new ArgumentOutOfRangeException(
|
||||
nameof(aggregate), aggregate, "Unmapped HistoryAggregateType — add a RetrievalMode mapping."),
|
||||
if (Enum.IsDefined(aggregate)) throw new HistoryAggregateNotSupportedException(aggregate);
|
||||
throw new ArgumentOutOfRangeException(
|
||||
nameof(aggregate), aggregate, "Unmapped HistoryAggregateType — add a RetrievalMode mapping.");
|
||||
}
|
||||
|
||||
/// <summary>The single mapping table: the retrieval mode for a natively served aggregate, or null.</summary>
|
||||
private static RetrievalMode? TryMap(HistoryAggregateType aggregate) => aggregate switch
|
||||
{
|
||||
HistoryAggregateType.TimeAverage => RetrievalMode.TimeWeightedAverage,
|
||||
HistoryAggregateType.Total => RetrievalMode.Integral,
|
||||
_ => null,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
using ZB.MOM.WW.HistorianGateway.Contracts.Grpc;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Mapping;
|
||||
|
||||
@@ -16,30 +17,50 @@ internal static class SampleMapper
|
||||
/// <summary>OPC DA "Good" family floor — a quality byte at/above this carries usable data.</summary>
|
||||
private const byte GoodQualityFloor = 192;
|
||||
|
||||
/// <summary>OPC DA "Uncertain" family floor (64–191).</summary>
|
||||
private const byte UncertainQualityFloor = 64;
|
||||
|
||||
/// <summary>Maps a single raw sample to a value snapshot.</summary>
|
||||
/// <param name="sample">The gateway raw sample.</param>
|
||||
/// <returns>The driver-agnostic snapshot.</returns>
|
||||
/// <remarks>
|
||||
/// #690: a Bad row carries no value. histsdk puts a value on EVERY row (<c>NumericValue = 0</c> on a numeric row,
|
||||
/// <c>StringValue = ""</c> on a string one), so a stored NULL arrives as <c>0</c> / <c>""</c> with a Bad OPC quality
|
||||
/// (live: <c>q=1 opc=0</c>) — that 0 is not a value, and AVEVA stores a Bad-quality sample as NULL, so a Bad row never
|
||||
/// carries a real one. The value is therefore dropped whenever the mapped status is Bad, matching the SqlServer/QuestDB
|
||||
/// backends (an Empty sample is null + Bad) and the aggregate mapper's no-data rows. Uncertain rows keep their value.
|
||||
/// </remarks>
|
||||
public static DataValueSnapshot ToSnapshot(HistorianSample sample)
|
||||
{
|
||||
var status = ToStatus(sample);
|
||||
// proto3 explicit presence: prefer the numeric value, else the string value, else null.
|
||||
object? value;
|
||||
if (sample.HasNumericValue)
|
||||
if (StatusBits.IsBad(status))
|
||||
value = null;
|
||||
else if (sample.HasNumericValue)
|
||||
value = sample.NumericValue; // boxes as System.Double
|
||||
else if (sample.HasStringValue)
|
||||
value = sample.StringValue;
|
||||
else
|
||||
value = null;
|
||||
|
||||
return new DataValueSnapshot(
|
||||
Value: value,
|
||||
StatusCode: status,
|
||||
SourceTimestampUtc: sample.Timestamp?.ToDateTime(), // Utc kind
|
||||
ServerTimestampUtc: DateTime.UtcNow);
|
||||
}
|
||||
|
||||
/// <summary>The OPC UA status of a raw sample (shared by raw reads and the #681 raw-evaluated aggregates).</summary>
|
||||
/// <param name="sample">The gateway raw sample.</param>
|
||||
/// <returns>The mapped status code.</returns>
|
||||
public static uint ToStatus(HistorianSample sample)
|
||||
{
|
||||
// Prefer the OPC DA quality byte (opc_quality); the gateway populates it directly from the
|
||||
// SDK's OpcQuality, so it is the authoritative byte for GatewayQualityMapper. Fall back to
|
||||
// the historian quality field only when opc_quality is unset (0).
|
||||
byte qualityByte = sample.OpcQuality != 0 ? (byte)sample.OpcQuality : (byte)sample.Quality;
|
||||
|
||||
return new DataValueSnapshot(
|
||||
Value: value,
|
||||
StatusCode: GatewayQualityMapper.Map(qualityByte),
|
||||
SourceTimestampUtc: sample.Timestamp?.ToDateTime(), // Utc kind
|
||||
ServerTimestampUtc: DateTime.UtcNow);
|
||||
return GatewayQualityMapper.Map(qualityByte);
|
||||
}
|
||||
|
||||
/// <summary>Maps a batch of raw samples to value snapshots, in order.</summary>
|
||||
@@ -57,31 +78,78 @@ internal static class SampleMapper
|
||||
/// <param name="aggregate">The gateway aggregate sample.</param>
|
||||
/// <returns>The driver-agnostic snapshot.</returns>
|
||||
/// <remarks>
|
||||
/// Unlike the legacy Wonderware DTO (a nullable <c>double?</c>), the gateway proto carries a
|
||||
/// non-optional <c>double value</c>, so an unavailable (no-data) bucket cannot be signalled by a
|
||||
/// null value. Instead it is signalled by a non-Good <c>opc_quality</c>: a Good bucket
|
||||
/// (<c>opc_quality >= 192</c>) yields its value with a Good status, anything else maps to
|
||||
/// <c>BadNoData</c> with a null value — preserving the Wonderware aggregate contract (binary
|
||||
/// Good-with-value / BadNoData-null).
|
||||
/// <para>
|
||||
/// The gateway proto carries a non-optional <c>double value</c>, so an unavailable (no-data) bucket cannot be signalled
|
||||
/// by a null value; it is signalled by the OPC DA quality family of <c>opc_quality</c> (its low byte, as the raw mapping
|
||||
/// reads it):
|
||||
/// </para>
|
||||
/// <list type="bullet">
|
||||
/// <item>Good (≥ 192) ⇒ the value, <c>Good</c>.</item>
|
||||
/// <item>Uncertain (64–191) ⇒ the value, <c>UncertainDataSubNormal</c> (#685): AVEVA marks an interval that holds a
|
||||
/// NULL/Bad sample <c>opc_quality = 64</c> and still answers its value (computed over the interval's good time only —
|
||||
/// see docs/Historian.md). It used to map to <c>BadNoData</c> with the value discarded. The sub-code is the one the SQL
|
||||
/// backends' aggregates give an interval that includes non-good data.</item>
|
||||
/// <item>Bad (< 64) ⇒ <c>BadNoData</c>, null value (the wire's 0 is not a value).</item>
|
||||
/// </list>
|
||||
/// <para>
|
||||
/// #681: every status carries the Calculated info bits (<c>0x401</c>) like every other backend's processed value, and
|
||||
/// the bucket is stamped with its interval START (Part 13; it was the interval end), falling back to the end only when
|
||||
/// the start is absent (verified on live AVEVA 2026-10-12: <c>start_time</c> is the interval start).
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public static DataValueSnapshot ToAggregateSnapshot(HistorianAggregateSample aggregate)
|
||||
{
|
||||
bool hasData = aggregate.OpcQuality >= GoodQualityFloor;
|
||||
var q = (byte)aggregate.OpcQuality;
|
||||
var status = q >= GoodQualityFloor ? StatusGood
|
||||
: q >= UncertainQualityFloor ? Sql.Common.HistorianStatusCodes.UncertainDataSubNormal
|
||||
: StatusBadNoData;
|
||||
return new DataValueSnapshot(
|
||||
Value: hasData ? aggregate.Value : null, // boxes as System.Double when present
|
||||
StatusCode: hasData ? StatusGood : StatusBadNoData,
|
||||
SourceTimestampUtc: (aggregate.EndTime ?? aggregate.StartTime)?.ToDateTime(), // bucket timestamp
|
||||
Value: status == StatusBadNoData ? null : aggregate.Value, // boxes as System.Double when present
|
||||
StatusCode: HistorianInfoBits.WithCalculated(status),
|
||||
SourceTimestampUtc: (aggregate.StartTime ?? aggregate.EndTime)?.ToDateTime(), // bucket timestamp = interval start
|
||||
ServerTimestampUtc: DateTime.UtcNow);
|
||||
}
|
||||
|
||||
/// <summary>Maps a batch of aggregate buckets to value snapshots, in order.</summary>
|
||||
/// <summary>
|
||||
/// True when the aggregate row's interval lies in the requested window — the span between <paramref name="startUtc"/>
|
||||
/// and <paramref name="endUtc"/> in either order (a reverse request covers the same span). AVEVA answers one extra row
|
||||
/// for the interval BEFORE the window (<c>[start − interval, start)</c>, #685), which Part 11/13 has no place for: an
|
||||
/// interval belongs to the window when its start lies in <c>[lo, hi)</c>; a row without a start time (the mapper then
|
||||
/// stamps it with its end) belongs when its end lies in <c>(lo, hi]</c>.
|
||||
/// </summary>
|
||||
/// <param name="aggregate">The gateway aggregate sample.</param>
|
||||
/// <param name="startUtc">The request's start time.</param>
|
||||
/// <param name="endUtc">The request's end time.</param>
|
||||
/// <returns>Whether the row is one of the window's intervals.</returns>
|
||||
public static bool IsInWindow(HistorianAggregateSample aggregate, DateTime startUtc, DateTime endUtc)
|
||||
{
|
||||
var lo = startUtc <= endUtc ? startUtc : endUtc;
|
||||
var hi = startUtc <= endUtc ? endUtc : startUtc;
|
||||
if (aggregate.StartTime is { } s)
|
||||
{
|
||||
var t = s.ToDateTime();
|
||||
return t >= lo && t < hi;
|
||||
}
|
||||
if (aggregate.EndTime is { } e)
|
||||
{
|
||||
var t = e.ToDateTime();
|
||||
return t > lo && t <= hi;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/// <summary>Maps the aggregate buckets of a window to value snapshots, in order, dropping rows outside it (#685).</summary>
|
||||
/// <param name="aggregates">The gateway aggregate samples.</param>
|
||||
/// <returns>The driver-agnostic snapshots.</returns>
|
||||
public static IReadOnlyList<DataValueSnapshot> ToAggregateSnapshots(IEnumerable<HistorianAggregateSample> aggregates)
|
||||
/// <param name="startUtc">The request's start time.</param>
|
||||
/// <param name="endUtc">The request's end time.</param>
|
||||
/// <returns>The driver-agnostic snapshots, one per in-window interval row.</returns>
|
||||
public static IReadOnlyList<DataValueSnapshot> ToAggregateSnapshots(IEnumerable<HistorianAggregateSample> aggregates,
|
||||
DateTime startUtc, DateTime endUtc)
|
||||
{
|
||||
var result = new List<DataValueSnapshot>();
|
||||
foreach (var aggregate in aggregates)
|
||||
result.Add(ToAggregateSnapshot(aggregate));
|
||||
if (IsInWindow(aggregate, startUtc, endUtc))
|
||||
result.Add(ToAggregateSnapshot(aggregate));
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,112 @@
|
||||
using ZB.MOM.WW.HistorianGateway.Contracts.Grpc;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Mapping;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway;
|
||||
|
||||
/// <summary>
|
||||
/// Serves a processed read of the AVEVA-native <c>TimeAverage</c> / <c>Total</c> on the SQL backends' interval grid
|
||||
/// (<see cref="BucketPlan"/>), in either direction: forward — window <c>[start, end)</c>, intervals stepping forward
|
||||
/// from <c>start</c>, the clipped interval the LATEST, each stamped with its earlier edge, oldest first; reverse (#677,
|
||||
/// #691.2) — window <c>(end, start]</c>, intervals stepping back from <c>start</c>, the clipped interval the EARLIEST,
|
||||
/// each stamped with its later edge, newest first. The gateway refuses <c>start > end</c> and <c>interval <= 0</c>,
|
||||
/// so every request it sees is forward with a positive interval.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>Forward requests on the plan's grid.</b> AVEVA anchors a native read's interval grid at the request start (live,
|
||||
/// 2026-10-12: rows at <c>start + j·interval</c>, plus one extra row for the interval before it, #685). So the plan's
|
||||
/// whole intervals are requested as ONE forward read spanning exactly them with the interval <c>I</c>, and the
|
||||
/// clipped interval (when the window is not a whole number of intervals) as a second forward read over exactly its
|
||||
/// span with its own width as the interval — AVEVA never averages time outside the window (before it on a reverse
|
||||
/// read, past <c>EndTime</c> on a forward one). An interval wider than the window is sent as the window, and
|
||||
/// <c>interval <= 0</c> as one interval over the whole window (it used to reach the gateway as 0 and be refused).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Soundness of reverse.</b> A time-weighted average / integral over <c>[a, b)</c> and over <c>(a, b]</c> integrate
|
||||
/// the same function over sets differing by two instants, so a forward native row IS the value of the reverse
|
||||
/// interval with the same edges. The one place the half-open side could matter is AVEVA's status for an interval
|
||||
/// whose only non-good sample lies exactly on an edge — accepted, as on every native row.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Mapping.</b> Each row is mapped as before (<see cref="SampleMapper.ToAggregateSnapshot"/>: Uncertain keeps its
|
||||
/// value, no data ⇒ <c>BadNoData</c>, Calculated bits), placed by its interval start (else end), restamped with
|
||||
/// <see cref="BucketPlan.BucketTimestamp"/>, and the clipped interval carries the Partial bit unless Bad (Part 13
|
||||
/// §5.3.3.2 Case 3, as the SQL backends' time-weighted aggregates — which additionally mark the intervals the data
|
||||
/// starts/ends inside; AVEVA's rows carry no data range, so those are not reproduced). An interval AVEVA answers no
|
||||
/// row for is absent (not a fabricated <c>BadNoData</c>, unlike the SQL backends).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// ⚠️ Today historiangw#27 (the interval reaches AVEVA in ticks where it reads milliseconds) makes AVEVA answer ONE
|
||||
/// Good row per request — the whole-intervals read lands on its earliest interval with a whole-span value and the rest
|
||||
/// are absent — in both directions. The mapping needs no change once the gateway is fixed.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
internal static class NativeAggregatePlan
|
||||
{
|
||||
/// <summary>One forward native request and the ascending bucket index of its first interval.</summary>
|
||||
/// <param name="StartUtc">Forward request start (inclusive).</param>
|
||||
/// <param name="EndUtc">Forward request end.</param>
|
||||
/// <param name="Interval">The request's interval (> 0).</param>
|
||||
/// <param name="FirstBucket">Ascending plan bucket of the request's first interval.</param>
|
||||
/// <param name="BucketCount">Number of plan buckets the request covers.</param>
|
||||
internal readonly record struct Request(DateTime StartUtc, DateTime EndUtc, TimeSpan Interval, long FirstBucket, long BucketCount);
|
||||
|
||||
/// <summary>The forward requests serving a plan: the whole intervals first, then the clipped one (if any).</summary>
|
||||
/// <param name="plan">A plan with at least one bucket.</param>
|
||||
/// <returns>One or two forward requests.</returns>
|
||||
public static IReadOnlyList<Request> Requests(BucketPlan plan)
|
||||
{
|
||||
var interval = TimeSpan.FromTicks(plan.IntervalTicks);
|
||||
var n = plan.BucketCount;
|
||||
var clipped = plan.IsTruncated(0) ? 0L : plan.IsTruncated(n - 1) ? n - 1 : -1L;
|
||||
if (clipped < 0) return [new Request(plan.StartUtc, plan.EndUtc, interval, 0, n)];
|
||||
|
||||
var clipLo = new DateTime(plan.LowerTicks(clipped), DateTimeKind.Utc);
|
||||
var clipHi = new DateTime(plan.UpperTicks(clipped), DateTimeKind.Utc);
|
||||
var requests = new List<Request>(2);
|
||||
if (n > 1)
|
||||
requests.Add(clipped == 0
|
||||
? new Request(clipHi, plan.EndUtc, interval, 1, n - 1) // reverse: the clipped one is the earliest
|
||||
: new Request(plan.StartUtc, clipLo, interval, 0, n - 1)); // forward: the clipped one is the latest
|
||||
requests.Add(new Request(clipLo, clipHi, clipHi - clipLo, clipped, 1));
|
||||
return requests;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Maps a request's rows onto the plan's buckets: rows outside the request window (AVEVA's pre-window row, #685)
|
||||
/// are dropped; each in-window row is placed by its interval start (or, without one, its end).
|
||||
/// </summary>
|
||||
/// <param name="plan">The plan.</param>
|
||||
/// <param name="request">The forward request the rows answer.</param>
|
||||
/// <param name="rows">The drained rows.</param>
|
||||
/// <param name="byBucket">Results by ascending bucket index (first row per bucket wins).</param>
|
||||
public static void Place(BucketPlan plan, Request request, IEnumerable<HistorianAggregateSample> rows,
|
||||
Dictionary<long, DataValueSnapshot> byBucket)
|
||||
{
|
||||
var interval = request.Interval.Ticks;
|
||||
foreach (var row in rows)
|
||||
{
|
||||
if (!SampleMapper.IsInWindow(row, request.StartUtc, request.EndUtc)) continue;
|
||||
long offset = row.StartTime is { } s
|
||||
? (s.ToDateTime().Ticks - request.StartUtc.Ticks) / interval
|
||||
: (row.EndTime.ToDateTime().Ticks - request.StartUtc.Ticks - 1) / interval; // IsInWindow ⇒ end in (lo, hi]
|
||||
if (offset < 0 || offset >= request.BucketCount) continue;
|
||||
var index = request.FirstBucket + offset;
|
||||
if (byBucket.ContainsKey(index)) continue;
|
||||
|
||||
var snapshot = SampleMapper.ToAggregateSnapshot(row);
|
||||
var status = snapshot.StatusCode;
|
||||
if (plan.IsTruncated(index) && !StatusBits.IsBad(status)) status |= HistorianInfoBits.HistorianPartial;
|
||||
byBucket[index] = snapshot with { StatusCode = status, SourceTimestampUtc = plan.BucketTimestamp(index) };
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>The placed results in the read's order (ascending forward, descending reverse).</summary>
|
||||
/// <param name="plan">The plan.</param>
|
||||
/// <param name="byBucket">Results by ascending bucket index.</param>
|
||||
/// <returns>The results.</returns>
|
||||
public static IReadOnlyList<DataValueSnapshot> Assemble(BucketPlan plan, IReadOnlyDictionary<long, DataValueSnapshot> byBucket) =>
|
||||
(plan.Reversed ? byBucket.OrderByDescending(kv => kv.Key) : byBucket.OrderBy(kv => kv.Key)).Select(kv => kv.Value).ToList();
|
||||
}
|
||||
@@ -0,0 +1,415 @@
|
||||
using ZB.MOM.WW.HistorianGateway.Contracts.Grpc;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Mapping;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway;
|
||||
|
||||
/// <summary>
|
||||
/// #681: serves the per-sample processed aggregates (the <see cref="AggregateFamily.Simple"/>,
|
||||
/// <see cref="AggregateFamily.StartEnd"/> and <see cref="AggregateFamily.ActualTime"/> families) on the Gateway backend by
|
||||
/// streaming the window's RAW samples through the gateway and evaluating them with the shared
|
||||
/// <c>Driver.Historian.Sql.Common</c> code — <see cref="BucketPlan"/> for the bucket math (forward and reverse, #677) and
|
||||
/// <see cref="AggregateEvaluator.EvaluateAll"/> for values, status and the Partial/Calculated bits — so the Gateway answers
|
||||
/// exactly what SqlServer and QuestDB answer for the same stored samples.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>Forward raw reads, each consumed to completion.</b> The gateway's <c>ReadRaw</c> streams oldest-first over an
|
||||
/// inclusive <c>[start, end]</c> and refuses <c>end < start</c>, so a reverse read streams the same ascending range
|
||||
/// and <see cref="BucketPlan"/> assembles the result newest-first. The window stream is classified sample by sample
|
||||
/// with <see cref="BucketPlan.IndexOf"/> (<c>[start, end)</c> forward, <c>(start, end]</c> reverse) — memory is
|
||||
/// O(buckets), never O(samples). No gateway stream is ever abandoned mid-way: a client-side break cancels the call,
|
||||
/// and the gateway currently classifies that cancellation as a connection fault and evicts its shared historian
|
||||
/// session (HistorianGateway <c>HistorianRpcFaults.IsConnectionFault</c>: any <c>RpcException</c> ⇒ true).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>The AVEVA start-boundary row.</b> An AVEVA <c>Full</c> raw read emits an initial-value row stamped exactly at
|
||||
/// the query start (histsdk reverse-engineering: <c>Quality = 133</c>, <c>OpcQuality = 192</c>) carrying the value
|
||||
/// in effect there, not a stored sample. The window stream is therefore requested from <see cref="BoundaryMargin"/>
|
||||
/// (1 ms) BEFORE the window, so that row lands outside every bucket and can never be counted as a sample. A row
|
||||
/// before the window counts only as evidence that the tag has earlier data, and only when its OPC quality is not
|
||||
/// Bad — a "no earlier data" NULL row still arrives WITH a value (histsdk sets <c>NumericValue = 0</c> on every
|
||||
/// numeric row, <c>StringValue = ""</c> on a string one) but with a Bad OPC quality (captured: 1, no-data).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>The data range.</b> The start/end-of-data Partial rules and Count's in-range empty interval need the tag's
|
||||
/// earliest sample at or before the window's later edge and latest at or after its earlier edge
|
||||
/// (<see cref="AggregateEvaluator.NeedsDataRange"/>; a plain <c>Average</c> needs neither). Earlier data is a non-Bad
|
||||
/// pre-window row, or — when none arrived — a <see cref="ProbeRows"/>-row read <c>[</c><see cref="ProbeFloor"/><c>,
|
||||
/// start)</c>. Later data is a sample exactly at the window end (forward) from the window read, or else a
|
||||
/// <see cref="ProbeRows"/>-row read <c>[end, now + </c><see cref="LookAheadHorizon"/><c>]</c> (the historian stores
|
||||
/// nothing further in the future). In both small reads a row stamped exactly at the query start (a boundary row) or
|
||||
/// at the query end (a possible terminal row) is not a stored sample, whatever its value.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Bounded.</b> The in-window rows are capped by <c>ServerHistorian:MaxTimeWeightedRawRows</c> (the same knob the
|
||||
/// SQL backends' time-weighted reads use): past it the node fails with
|
||||
/// <see cref="HistoryReadLimitExceededException"/> (⇒ <c>BadTooManyOperations</c>) once the stream — bounded by the
|
||||
/// requested row budget — has been drained. The gateway is asked for at most <c>cap + 2 + </c>
|
||||
/// <see cref="PreWindowAllowance"/> rows, clamped at <see cref="GatewayDefaultMaxValuesCeiling"/> while the cap is at
|
||||
/// or under it (#688, <see cref="ClampToGatewayCeiling"/>); a stream that delivers all of them cannot be proven
|
||||
/// complete and fails the same way rather than answering from a truncated window. A strict gateway whose ceiling is
|
||||
/// below that budget (a ceiling under 1,000,000, or an unbounded cap) refuses it naming its ceiling; the window is
|
||||
/// then re-read once with the ceiling as budget (#691.3, <see cref="GatewayReadLimits"/>), and the same
|
||||
/// "delivered the whole budget ⇒ cannot be proven complete" rule applies.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Values and status.</b> A sample's value is its numeric value (Double), else its string value, else Empty; its
|
||||
/// status is <see cref="SampleMapper.ToStatus"/> — the same quality mapping a raw HistoryRead serves (C-002: the
|
||||
/// gateway stores no quality, so AVEVA history is normally all Good). An Empty sample is clamped Bad
|
||||
/// (<see cref="StoredSample.FromStored"/>), exactly as on the SQL backends.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
internal static class RawAggregateReader
|
||||
{
|
||||
/// <summary>How far before the window the raw stream starts, so the AVEVA initial-value row lands outside it.</summary>
|
||||
internal static readonly TimeSpan BoundaryMargin = TimeSpan.FromMilliseconds(1);
|
||||
|
||||
/// <summary>How far past "now" the stream may look for a sample after the window (end-of-data rules).</summary>
|
||||
internal static readonly TimeSpan LookAheadHorizon = TimeSpan.FromDays(1);
|
||||
|
||||
/// <summary>Rows the stream may spend before the window (the boundary row plus samples inside the margin).</summary>
|
||||
internal const int PreWindowAllowance = 16;
|
||||
|
||||
/// <summary>Start of the "is there data before the window?" probe (the Unix epoch — before any AVEVA history).</summary>
|
||||
internal static readonly DateTime ProbeFloor = DateTime.UnixEpoch;
|
||||
|
||||
/// <summary>Rows the earlier-data probe asks for (a possible boundary row plus the first stored sample).</summary>
|
||||
internal const int ProbeRows = 2;
|
||||
|
||||
/// <summary>
|
||||
/// The <c>ReadRaw</c> start for a window that begins at <paramref name="windowStartUtc"/>: <see cref="BoundaryMargin"/>
|
||||
/// earlier, so the AVEVA start-boundary row — stamped exactly at the QUERY start, carrying the value in effect there
|
||||
/// (<c>Quality = 133</c>) or, with no earlier data, a NULL row (value 0, Bad OPC quality) — lands BEFORE the window
|
||||
/// (<see cref="IsWindowSample"/>). The one place both the raw-evaluated aggregates (#681) and a raw HistoryRead
|
||||
/// (#687) decide what the request start is. Never before <see cref="ProbeFloor"/> (the Unix epoch — the start of
|
||||
/// the historian's time encoding, the same floor the server's reverse walk-back uses, #686): a window starting less
|
||||
/// than one margin after it is requested from the epoch itself.
|
||||
/// </summary>
|
||||
/// <param name="windowStartUtc">The window's (inclusive) start.</param>
|
||||
/// <returns>The start to request.</returns>
|
||||
public static DateTime RequestStartFor(DateTime windowStartUtc) =>
|
||||
windowStartUtc - ProbeFloor > BoundaryMargin ? windowStartUtc - BoundaryMargin
|
||||
: windowStartUtc > ProbeFloor ? ProbeFloor
|
||||
: windowStartUtc;
|
||||
|
||||
/// <summary>
|
||||
/// True when a row of a forward raw stream requested from <see cref="RequestStartFor"/> is a stored sample of the raw
|
||||
/// window <c>[windowStartUtc, windowEndUtc)</c> — the <see cref="IHistorianDataSource.ReadRawAsync"/> contract
|
||||
/// (exclusive end, as on the SQL backends; the gateway's own <c>ReadRaw</c> end is inclusive) — or, for a point read
|
||||
/// (<c>start == end</c>, the server's tie-cluster over-fetch), of that one instant. A row before the start is the AVEVA
|
||||
/// boundary row (whatever its value or quality — a stored NULL looks the same on the wire, so the timestamp is the
|
||||
/// evidence) or a stored sample inside the margin; a row without a timestamp is never a sample.
|
||||
/// </summary>
|
||||
/// <param name="raw">The gateway row.</param>
|
||||
/// <param name="windowStartUtc">The window's (inclusive) start.</param>
|
||||
/// <param name="windowEndUtc">The window's end: exclusive, or inclusive when it equals the start.</param>
|
||||
/// <returns>Whether the row belongs to the window.</returns>
|
||||
public static bool IsWindowSample(HistorianSample raw, DateTime windowStartUtc, DateTime windowEndUtc) =>
|
||||
raw.Timestamp is { } ts && ts.ToDateTime() is var t && IsAfterFloor(t) && t >= windowStartUtc
|
||||
&& (t < windowEndUtc || (t == windowEndUtc && windowStartUtc == windowEndUtc));
|
||||
|
||||
/// <summary>
|
||||
/// A row stamped at or before <see cref="ProbeFloor"/> (the Unix epoch) is never a stored sample: AVEVA stores nothing
|
||||
/// there (the server's #686 walk-back assumes the same), and a window starting at or before the epoch gets no 1 ms
|
||||
/// margin (<see cref="RequestStartFor"/>), so AVEVA's synthetic boundary / no-data row lands exactly at its start.
|
||||
/// </summary>
|
||||
/// <param name="t">The row's timestamp.</param>
|
||||
/// <returns>Whether the row can be a stored sample.</returns>
|
||||
internal static bool IsAfterFloor(DateTime t) => t > ProbeFloor;
|
||||
|
||||
/// <summary>True for the aggregates this reader serves: the per-sample families of <see cref="AggregateFamilies"/>.</summary>
|
||||
/// <param name="aggregate">Requested aggregate.</param>
|
||||
/// <returns>Whether the aggregate is evaluated from raw samples.</returns>
|
||||
public static bool Serves(HistoryAggregateType aggregate) =>
|
||||
AggregateFamilies.FamilyOf(aggregate) is AggregateFamily.Simple or AggregateFamily.StartEnd or AggregateFamily.ActualTime;
|
||||
|
||||
/// <summary>The gateway <c>max_values</c> for a configured window cap (<see cref="int.MaxValue"/> when unbounded).</summary>
|
||||
/// <param name="maxWindowRows">The configured cap (<= 0 = unbounded).</param>
|
||||
/// <returns>The row limit to request.</returns>
|
||||
public static int MaxValuesFor(int maxWindowRows) =>
|
||||
maxWindowRows <= 0 ? int.MaxValue : ClampToGatewayCeiling((long)maxWindowRows + 2 + PreWindowAllowance, maxWindowRows);
|
||||
|
||||
/// <summary>
|
||||
/// The HistorianGateway's default <c>Historian:ReadLimits:MaxValuesCeiling</c>. With
|
||||
/// <c>AllowStreamingBeyondCeiling=false</c> the gateway refuses any <c>max_values</c> above its ceiling.
|
||||
/// </summary>
|
||||
internal const int GatewayDefaultMaxValuesCeiling = 1_000_000;
|
||||
|
||||
/// <summary>
|
||||
/// #688: a <c>ReadRaw</c> row budget of <paramref name="rows"/> plus this backend's own allowance
|
||||
/// (<paramref name="requested"/>) clamped at <see cref="GatewayDefaultMaxValuesCeiling"/> — but never below
|
||||
/// <paramref name="rows"/> itself. The allowance (the boundary row, the margin, the completeness row) must not push a
|
||||
/// budget that fits the gateway's default ceiling over it: the default <c>MaxTimeWeightedRawRows</c> (1,000,000) + 18
|
||||
/// used to exceed it, so a gateway with <c>AllowStreamingBeyondCeiling=false</c> refused EVERY per-sample read. A
|
||||
/// count above the ceiling is the operator's (or client's) deliberate choice for a streaming gateway and keeps its
|
||||
/// allowance. The cost of clamping is that a window holding within the allowance of the ceiling cannot be proven
|
||||
/// complete and fails as over the cap — the same outcome it would get one row later.
|
||||
/// </summary>
|
||||
/// <param name="requested">The budget including the allowance.</param>
|
||||
/// <param name="rows">The caller's own row count (the cap, or <c>NumValuesPerNode</c>), > 0.</param>
|
||||
/// <returns>The <c>max_values</c> to request.</returns>
|
||||
internal static int ClampToGatewayCeiling(long requested, long rows) =>
|
||||
(int)Math.Min(int.MaxValue, rows > GatewayDefaultMaxValuesCeiling ? requested : Math.Min(requested, GatewayDefaultMaxValuesCeiling));
|
||||
|
||||
/// <summary>Maps one gateway raw sample to a stored sample (value + mapped status, Empty clamped Bad).</summary>
|
||||
/// <param name="sample">The gateway sample; its timestamp must be set.</param>
|
||||
/// <returns>The stored sample.</returns>
|
||||
public static StoredSample ToStored(HistorianSample sample)
|
||||
{
|
||||
var value = sample.HasNumericValue ? HistorianValue.FromDouble(sample.NumericValue)
|
||||
: sample.HasStringValue ? HistorianValue.FromString(sample.StringValue)
|
||||
: HistorianValue.Empty;
|
||||
return StoredSample.FromStored(sample.Timestamp.ToDateTime(), value, SampleMapper.ToStatus(sample));
|
||||
}
|
||||
|
||||
/// <summary>Reads and evaluates one per-sample processed read.</summary>
|
||||
/// <param name="client">The gateway client.</param>
|
||||
/// <param name="ceiling">The data source's learned gateway row ceiling (#691.3).</param>
|
||||
/// <param name="tag">Historian tag name.</param>
|
||||
/// <param name="aggregate">A per-sample aggregate (<see cref="Serves"/>).</param>
|
||||
/// <param name="plan">The bucket plan.</param>
|
||||
/// <param name="maxWindowRows">The in-window raw-row cap (<= 0 = unbounded).</param>
|
||||
/// <param name="now">Server timestamp, and the anchor of the look-ahead horizon.</param>
|
||||
/// <param name="ct">Cancellation token.</param>
|
||||
/// <returns>One value per interval, in the plan's order.</returns>
|
||||
/// <exception cref="HistoryReadLimitExceededException">The window holds more than <paramref name="maxWindowRows"/> rows,
|
||||
/// or the stream could not be read to completion within the requested row budget.</exception>
|
||||
public static async Task<IReadOnlyList<DataValueSnapshot>> ReadAsync(IHistorianGatewayClient client, GatewayRowCeiling ceiling, string tag,
|
||||
HistoryAggregateType aggregate, BucketPlan plan, int maxWindowRows, DateTime now, CancellationToken ct)
|
||||
{
|
||||
if (plan.BucketCount == 0) return [];
|
||||
var needsRange = AggregateEvaluator.NeedsDataRange(aggregate, plan);
|
||||
var requestStart = RequestStartFor(plan.StartUtc);
|
||||
|
||||
// A strict gateway that refuses the budget is re-asked once at the ceiling it names (#691.3; e.g. a ceiling below
|
||||
// the 1,000,000 #688 clamps at, or an unbounded cap) — and a stream that fills that ceiling is never evaluated.
|
||||
var requested = MaxValuesFor(maxWindowRows);
|
||||
var (pass, maxValues) = await ceiling.ReadAsync(requested,
|
||||
budget => ReadWindowAsync(client, tag, aggregate, plan, requestStart, budget, maxWindowRows, ct),
|
||||
(r, budget) => !r.OverCap && r.Received >= budget, deferRecentReprobe: true).ConfigureAwait(false);
|
||||
var (buckets, earliest, latest, received, overCap) = pass;
|
||||
if (overCap) throw LimitExceeded(tag, maxWindowRows);
|
||||
if (received >= maxValues) throw BudgetExhausted(tag, maxWindowRows, requested, maxValues);
|
||||
|
||||
if (needsRange)
|
||||
{
|
||||
// Earlier data: no non-Bad pre-window row ⇒ ask explicitly (normally only when the tag's history really starts
|
||||
// inside the window).
|
||||
var earlierSeen = earliest is { } e && (e < plan.StartUtc || (plan.Reversed && e == plan.StartUtc));
|
||||
if (!earlierSeen && plan.StartUtc.AddTicks(-1) > ProbeFloor)
|
||||
if (await FirstSampleAsync(client, tag, ProbeFloor, plan.StartUtc.AddTicks(-1), ct).ConfigureAwait(false) is { } before)
|
||||
earliest = before;
|
||||
|
||||
// Later data: the main read covers a sample exactly at the window end (forward: that is already later data);
|
||||
// otherwise a small separate read from the end to now + horizon.
|
||||
var laterSeen = latest is { } l && (plan.Reversed ? l > plan.EndUtc : l >= plan.EndUtc);
|
||||
var horizon = DateTime.MaxValue - now > LookAheadHorizon
|
||||
? DateTime.SpecifyKind(now + LookAheadHorizon, DateTimeKind.Utc)
|
||||
: DateTime.MaxValue;
|
||||
if (!laterSeen && horizon > plan.EndUtc)
|
||||
if (await FirstSampleAsync(client, tag, plan.EndUtc, horizon, ct).ConfigureAwait(false) is { } after)
|
||||
latest = after;
|
||||
}
|
||||
|
||||
return AggregateEvaluator.EvaluateAll(aggregate, plan, buckets.Rows(), earliest, latest, now);
|
||||
}
|
||||
|
||||
/// <summary>The window read of <see cref="ReadAsync"/> with one row budget, drained.</summary>
|
||||
private static async Task<(RawBucketAccumulator Buckets, DateTime? Earliest, DateTime? Latest, long Received, bool OverCap)>
|
||||
ReadWindowAsync(IHistorianGatewayClient client, string tag, HistoryAggregateType aggregate, BucketPlan plan,
|
||||
DateTime requestStart, int maxValues, int maxWindowRows, CancellationToken ct)
|
||||
{
|
||||
var buckets = new RawBucketAccumulator(aggregate == HistoryAggregateType.MaximumActualTime);
|
||||
DateTime? earliest = null, latest = null;
|
||||
long windowRows = 0, received = 0;
|
||||
var overCap = false;
|
||||
// Consumed to COMPLETION (never broken out of): abandoning a gateway stream cancels the call server-side, and the
|
||||
// gateway currently treats that cancellation as a connection fault and evicts its shared historian session. Past
|
||||
// the cap the rest is drained (bounded by maxValues) without being evaluated.
|
||||
await foreach (var raw in client.ReadRawAsync(tag, requestStart, plan.EndUtc, maxValues, ct).ConfigureAwait(false))
|
||||
{
|
||||
received++;
|
||||
if (overCap || raw.Timestamp is null) continue;
|
||||
var s = ToStored(raw);
|
||||
var t = s.TimeUtc;
|
||||
if (!IsAfterFloor(t)) continue; // the boundary row of a window starting at/before the epoch, never a sample
|
||||
if (plan.IndexOf(t) is { } bucket)
|
||||
{
|
||||
if (maxWindowRows > 0 && ++windowRows > maxWindowRows)
|
||||
{
|
||||
overCap = true;
|
||||
continue;
|
||||
}
|
||||
buckets.Add(bucket, s);
|
||||
}
|
||||
// The data range as the SQL backends define it: earliest stored sample at or before the window's later edge,
|
||||
// latest at or after its earlier edge. A row before the window (the AVEVA boundary row, or a stored sample in
|
||||
// the margin) proves earlier data only when its OPC quality is not Bad: AVEVA's boundary row for a tag with no
|
||||
// earlier data is a NULL row, which arrives WITH a value (histsdk sets NumericValue = 0 / StringValue "") and
|
||||
// a Bad OPC quality (captured: 1, no-data).
|
||||
if (t >= plan.StartUtc || IsNotBadOnTheWire(raw))
|
||||
{
|
||||
if (t <= plan.EndUtc && (earliest is null || t < earliest)) earliest = t;
|
||||
if (t >= plan.StartUtc && (latest is null || t > latest)) latest = t;
|
||||
}
|
||||
}
|
||||
return (buckets, earliest, latest, received, overCap);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The time of the first stored sample (any quality) strictly between <paramref name="startUtc"/> and
|
||||
/// <paramref name="endUtc"/>, read as a <see cref="ProbeRows"/>-row raw read consumed to completion. A row stamped
|
||||
/// exactly at the query start is the historian's boundary row (the value in effect there, or a NULL row when there is
|
||||
/// none) and a row exactly at the query end may be a terminal row — neither is a stored sample, whatever its value.
|
||||
/// </summary>
|
||||
private static async Task<DateTime?> FirstSampleAsync(IHistorianGatewayClient client, string tag, DateTime startUtc,
|
||||
DateTime endUtc, CancellationToken ct)
|
||||
{
|
||||
DateTime? found = null;
|
||||
await foreach (var raw in client.ReadRawAsync(tag, startUtc, endUtc, ProbeRows, ct).ConfigureAwait(false))
|
||||
{
|
||||
if (found is not null || raw.Timestamp is null) continue; // drain, never break (see ReadAsync)
|
||||
var t = raw.Timestamp.ToDateTime();
|
||||
if (t > startUtc && t < endUtc) found = t;
|
||||
}
|
||||
return found;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The row's OPC DA quality alone (<c>opc_quality</c> ≥ 64: Good or Uncertain) — deliberately without
|
||||
/// <see cref="SampleMapper.ToStatus"/>'s fallback to the AVEVA <c>quality</c> field when <c>opc_quality</c> is 0, which
|
||||
/// is also the OPC DA Bad value a NULL row carries.
|
||||
/// </summary>
|
||||
private static bool IsNotBadOnTheWire(HistorianSample raw) => (byte)raw.OpcQuality >= 64; // low byte, as SampleMapper.ToStatus reads it
|
||||
|
||||
/// <summary>
|
||||
/// The stream delivered its whole row budget, so the window cannot be proven complete. Names the limit that actually
|
||||
/// bound the budget: the gateway's ceiling (a strict gateway refused the requested budget and named it, #691.3), the
|
||||
/// #688 clamp at the gateway's default ceiling, or the configured cap.
|
||||
/// </summary>
|
||||
private static HistoryReadLimitExceededException BudgetExhausted(string tag, int maxWindowRows, int requested, int budget)
|
||||
{
|
||||
if (budget < requested)
|
||||
return new HistoryReadLimitExceededException(budget,
|
||||
$"The processed read of '{tag}' covers more raw samples than the HistorianGateway's row ceiling ({budget}, its " +
|
||||
"Historian:ReadLimits:MaxValuesCeiling with AllowStreamingBeyondCeiling=false); the Gateway backend evaluates " +
|
||||
"per-sample aggregates from raw reads — narrow the window.");
|
||||
if (maxWindowRows > 0 && budget < (long)maxWindowRows + 2 + PreWindowAllowance)
|
||||
return new HistoryReadLimitExceededException(budget,
|
||||
$"The processed read of '{tag}' covers more raw samples than the HistorianGateway's default row ceiling " +
|
||||
$"({budget}) the request is clamped at; the Gateway backend evaluates per-sample aggregates from raw reads — " +
|
||||
"narrow the window.");
|
||||
return LimitExceeded(tag, maxWindowRows);
|
||||
}
|
||||
|
||||
private static HistoryReadLimitExceededException LimitExceeded(string tag, int maxWindowRows) =>
|
||||
new(maxWindowRows,
|
||||
$"The processed read of '{tag}' covers more than {maxWindowRows} raw samples " +
|
||||
"(ServerHistorian:MaxTimeWeightedRawRows); the Gateway backend evaluates per-sample aggregates from raw reads — " +
|
||||
"narrow the window.");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Builds the per-bucket statistics (<see cref="BucketRow"/>) the SQL backends' aggregate queries produce, from a stream
|
||||
/// of raw samples — the C# statement of <c>SqlAggregateQueries</c>' columns: counts by class (usable = non-Bad with a
|
||||
/// value; good = usable with Good severity; NonNumeric = usable String/DateTime; GoodNonNumeric = good String/DateTime),
|
||||
/// Avg/Min/Max/Sum over the GOOD numeric values and UncertainMin/UncertainMax over the usable-but-not-Good numeric values
|
||||
/// (#684), first/last usable value by time, the extreme GOOD numeric value with its time (ties ⇒ earliest), and the
|
||||
/// first/last sample of any quality.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Keep in step with the SQL backends' per-sample value columns: a change to which samples feed a statistic there
|
||||
/// (e.g. Uncertain handling) must be repeated here, or the Gateway backend silently diverges.
|
||||
/// </remarks>
|
||||
internal sealed class RawBucketAccumulator
|
||||
{
|
||||
private readonly bool _maximum;
|
||||
private readonly Dictionary<long, State> _buckets = new();
|
||||
|
||||
/// <summary>Creates an accumulator.</summary>
|
||||
/// <param name="maximum">Extreme = the maximum (MaximumActualTime); otherwise the minimum.</param>
|
||||
public RawBucketAccumulator(bool maximum) => _maximum = maximum;
|
||||
|
||||
private sealed class State
|
||||
{
|
||||
public int Total, Usable, Good, NonNumeric, GoodNonNumeric, GoodNumeric, UncertainNumeric;
|
||||
public double Sum, Min = double.PositiveInfinity, Max = double.NegativeInfinity;
|
||||
public double UncertainMin = double.PositiveInfinity, UncertainMax = double.NegativeInfinity;
|
||||
public StoredSample? First, Last, FirstAny, LastAny, Extreme;
|
||||
}
|
||||
|
||||
/// <summary>Adds one in-window sample to its bucket. Order-independent (picks compare by time).</summary>
|
||||
/// <param name="bucket">Ascending bucket index (<see cref="BucketPlan.IndexOf"/>).</param>
|
||||
/// <param name="s">The sample.</param>
|
||||
public void Add(long bucket, StoredSample s)
|
||||
{
|
||||
if (!_buckets.TryGetValue(bucket, out var st)) _buckets[bucket] = st = new State();
|
||||
st.Total++;
|
||||
if (st.FirstAny is not { } fa || s.TimeUtc < fa.TimeUtc) st.FirstAny = s;
|
||||
if (st.LastAny is not { } la || s.TimeUtc >= la.TimeUtc) st.LastAny = s;
|
||||
if (!s.IsUsable) return;
|
||||
st.Usable++;
|
||||
if (s.IsGood) st.Good++;
|
||||
if (st.First is not { } f || s.TimeUtc < f.TimeUtc) st.First = s;
|
||||
if (st.Last is not { } l || s.TimeUtc >= l.TimeUtc) st.Last = s;
|
||||
if (!s.Value.TryGetDouble(out var v))
|
||||
{
|
||||
st.NonNumeric++;
|
||||
if (s.IsGood) st.GoodNonNumeric++;
|
||||
return;
|
||||
}
|
||||
if (!s.IsGood)
|
||||
{
|
||||
// #684: an Uncertain value is not part of the value set; it only feeds the Min/Max status rule.
|
||||
st.UncertainNumeric++;
|
||||
if (v < st.UncertainMin) st.UncertainMin = v;
|
||||
if (v > st.UncertainMax) st.UncertainMax = v;
|
||||
return;
|
||||
}
|
||||
st.GoodNumeric++;
|
||||
st.Sum += v;
|
||||
if (v < st.Min) st.Min = v;
|
||||
if (v > st.Max) st.Max = v;
|
||||
if (st.Extreme is not { } e) st.Extreme = s;
|
||||
else
|
||||
{
|
||||
var ev = e.Value.Number;
|
||||
var better = _maximum ? v > ev : v < ev;
|
||||
if (better || (v == ev && s.TimeUtc < e.TimeUtc)) st.Extreme = s;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>The non-empty buckets' rows, ascending by bucket index.</summary>
|
||||
/// <returns>The rows.</returns>
|
||||
public IEnumerable<BucketRow> Rows() =>
|
||||
_buckets.OrderBy(kv => kv.Key).Select(kv =>
|
||||
{
|
||||
var st = kv.Value;
|
||||
var hasNumeric = st.GoodNumeric > 0;
|
||||
var hasUncertain = st.UncertainNumeric > 0;
|
||||
return new BucketRow
|
||||
{
|
||||
Bucket = kv.Key,
|
||||
Total = st.Total,
|
||||
Usable = st.Usable,
|
||||
Good = st.Good,
|
||||
Bad = st.Total - st.Usable,
|
||||
NonNumeric = st.NonNumeric,
|
||||
GoodNonNumeric = st.GoodNonNumeric,
|
||||
Avg = hasNumeric ? st.Sum / st.GoodNumeric : null,
|
||||
Min = hasNumeric ? st.Min : null,
|
||||
Max = hasNumeric ? st.Max : null,
|
||||
Sum = hasNumeric ? st.Sum : null,
|
||||
UncertainMin = hasUncertain ? st.UncertainMin : null,
|
||||
UncertainMax = hasUncertain ? st.UncertainMax : null,
|
||||
First = st.First?.Value ?? HistorianValue.Empty,
|
||||
Last = st.Last?.Value ?? HistorianValue.Empty,
|
||||
Extreme = st.Extreme?.Value.Number,
|
||||
ExtremeTime = st.Extreme?.TimeUtc,
|
||||
FirstAny = st.FirstAny,
|
||||
LastAny = st.LastAny,
|
||||
};
|
||||
});
|
||||
}
|
||||
+5
@@ -11,6 +11,11 @@
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\..\Core\ZB.MOM.WW.OtOpcUa.Core.Abstractions\ZB.MOM.WW.OtOpcUa.Core.Abstractions.csproj"/>
|
||||
<ProjectReference Include="..\..\Core\ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian\ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian.csproj"/>
|
||||
<!-- #681: the per-sample processed aggregates are evaluated from raw reads with the SAME database-independent code the
|
||||
SQL backends use (BucketPlan, BucketRow, AggregateEvaluator), so Gateway matches SqlServer/QuestDB exactly.
|
||||
Sql.Common references only Core.Abstractions + Core.AlarmHistorian (both already referenced here) and no
|
||||
driver, so this adds no cycle. -->
|
||||
<ProjectReference Include="..\ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common\ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common.csproj"/>
|
||||
<!-- Runtime owns ServerHistorianOptions (the bound ServerHistorian config). The read-cutover
|
||||
factory (GatewayHistorian.CreateDataSource / HistorianGatewayClientAdapter.Create) maps
|
||||
those options onto the package client, so the driver references it. Runtime references no
|
||||
|
||||
+156
-41
@@ -9,21 +9,23 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb;
|
||||
/// per <see cref="AggregateFamily"/>, mirroring the SQL Server backend's <c>SqlAggregateQueries</c> family split and
|
||||
/// column order. Every query takes five positional parameters:
|
||||
/// <list type="number">
|
||||
/// <item><c>$1</c> — the (rounded) window start in 100 ns ticks since the Unix epoch
|
||||
/// (<see cref="BucketPlan.StartUnixTicks"/>, <c>Bigint</c>);</item>
|
||||
/// <item><c>$1</c> — the bucket base in 100 ns ticks since the Unix epoch
|
||||
/// (<see cref="BucketPlan.BucketBaseUnixTicks"/>, <c>Bigint</c>: the rounded window start forward; re-anchored at
|
||||
/// the later time for a reverse read, #677);</item>
|
||||
/// <item><c>$2</c> — the bucket width in ticks (<see cref="BucketPlan.IntervalTicks"/>, <c>Bigint</c>);</item>
|
||||
/// <item><c>$3</c> — the tag (<c>Varchar</c>);</item>
|
||||
/// <item><c>$4</c>/<c>$5</c> — the forward window <c>[start, end)</c> (<c>TimestampTz</c> via
|
||||
/// <see cref="QuestDbConnection.BindTimestamp"/>).</item>
|
||||
/// <item><c>$4</c>/<c>$5</c> — the earlier / later window edges (<c>TimestampTz</c> via
|
||||
/// <see cref="QuestDbConnection.BindTimestamp"/>): <c>[$4, $5)</c> forward, <c>($4, $5]</c> reverse.</item>
|
||||
/// </list>
|
||||
/// Each returns one row per <b>non-empty</b> bucket; the caller fills the empty ones.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>Bucketing.</b> <c>b = (cast(ts AS LONG) * 10 − $1) / $2</c>. <c>cast(ts AS LONG)</c> is µs since 1970, so
|
||||
/// <c>× 10</c> is ticks since 1970; both it and <c>$1</c> are µs-aligned (R6), so the integer division is
|
||||
/// start-inclusive / end-exclusive to the tick, exactly like SQL Server's. Every row is at or after the start,
|
||||
/// so the dividend is never negative (no truncation-toward-zero surprise), and nothing overflows before 9999.
|
||||
/// <c>× 10</c> is ticks since 1970, so the integer division is exact to the tick, exactly like SQL Server's
|
||||
/// (<see cref="BucketPlan.IndexOf"/> is the C# statement of the math). Every in-window row is at or after the base
|
||||
/// (forward) or strictly after the window start with the base one tick past the grid origin (reverse), so the
|
||||
/// dividend is never negative (no truncation-toward-zero surprise), and nothing overflows before 9999.
|
||||
/// <c>SAMPLE BY</c> is deliberately not used: its alignment rules differ from ours.
|
||||
/// </para>
|
||||
/// <para>
|
||||
@@ -55,65 +57,178 @@ internal static class QuestDbAggregateQueries
|
||||
private static readonly string EmptyKind = ((byte)HistorianValueKind.Empty).ToString(CultureInfo.InvariantCulture);
|
||||
|
||||
/// <summary>
|
||||
/// Average / Minimum / Maximum / Total / Count / Range / PercentGood / PercentBad.
|
||||
/// Family columns: <c>6 avg_value, 7 min_value, 8 max_value, 9 sum_value</c> (DOUBLE, NULL when no usable sample).
|
||||
/// Average / Minimum / Maximum / Count / Range (PercentGood / PercentBad and Total are time-weighted, not here).
|
||||
/// Family columns: <c>6 avg_value, 7 min_value, 8 max_value, 9 sum_value</c> over the <b>Good</b> numeric samples
|
||||
/// (DOUBLE, NULL when none — #684, Part 13 / the SDK), then <c>10 unc_min, 11 unc_max</c> over the Uncertain ones (the
|
||||
/// Part 13 Minimum/Maximum/Range status rule) and <c>12 good_non_numeric_count</c>.
|
||||
/// </summary>
|
||||
/// <param name="sample">Prefixed sample table name.</param>
|
||||
/// <returns>The SQL text.</returns>
|
||||
public static string Simple(string sample) =>
|
||||
/// <param name="reverse">True for a reverse read (window <c>($4, $5]</c>).</param>
|
||||
public static string Simple(string sample, bool reverse = false) =>
|
||||
"SELECT b, " + CommonCounts() + ", " +
|
||||
"avg(CASE WHEN u = 1 THEN num_value END) AS avg_value, " +
|
||||
"min(CASE WHEN u = 1 THEN num_value END) AS min_value, " +
|
||||
"max(CASE WHEN u = 1 THEN num_value END) AS max_value, " +
|
||||
"sum(CASE WHEN u = 1 THEN num_value END) AS sum_value " +
|
||||
"FROM (" + Classified(sample) + ") GROUP BY b ORDER BY b";
|
||||
"avg(CASE WHEN g = 1 THEN num_value END) AS avg_value, " +
|
||||
"min(CASE WHEN g = 1 THEN num_value END) AS min_value, " +
|
||||
"max(CASE WHEN g = 1 THEN num_value END) AS max_value, " +
|
||||
"sum(CASE WHEN g = 1 THEN num_value END) AS sum_value, " +
|
||||
UncertainExtremes() + " " +
|
||||
"FROM (" + Classified(sample, reverse) + ") GROUP BY b ORDER BY b";
|
||||
|
||||
/// <summary>Ordinal of the StartEnd family's first-sample-of-any-quality columns (<see cref="StartEnd"/>).</summary>
|
||||
public const int FirstAnyColumn = FirstFamilyColumn + 10;
|
||||
|
||||
/// <summary>
|
||||
/// Start / End / Delta: the first and last usable sample of each bucket by <c>ts</c>.
|
||||
/// Family columns: <c>6..10</c> first sample (<c>kind, num_value, int_value, str_value, ts</c>), <c>11..15</c> last
|
||||
/// sample (same order). A bucket without a usable sample has NULL <c>ts</c> but <c>kind = 0</c> (BYTE has no NULL),
|
||||
/// so the reader keys absence on the NULL time.
|
||||
/// Start / End / Delta: the first and last usable sample of each bucket by <c>ts</c> (Delta's value), and the first
|
||||
/// and last sample of <b>any</b> quality (Start / End, and Delta's status — #680).
|
||||
/// Family columns: <c>6..10</c> first usable sample (<c>kind, num_value, int_value, str_value, ts</c>), <c>11..15</c>
|
||||
/// last usable sample (same order) — a bucket without a usable sample has NULL <c>ts</c> but <c>kind = 0</c> (BYTE
|
||||
/// has no NULL), so the reader keys absence on the NULL time; then <c>16..21</c> the first sample of any quality
|
||||
/// (<c>ts, kind, num_value, int_value, str_value, status</c>) and <c>22..27</c> the last (same order), present for
|
||||
/// every returned (non-empty) bucket.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>One scan, one GROUP BY (live-gate 8 F1).</b> The counts and all four picked samples come from a single read
|
||||
/// of the sample table. The first/last sample of any quality are QuestDB's <c>first()</c>/<c>last()</c> (the
|
||||
/// first/last row of the group in designated-timestamp order — null values included, so every column comes from
|
||||
/// the same row); the first/last <i>usable</i> sample are <c>arg_min</c>/<c>arg_max</c> keyed on <c>uts</c>, the
|
||||
/// row's time when usable and NULL otherwise (a NULL key is skipped, a NULL value is returned as is, and
|
||||
/// <c>(ts, tag)</c> is unique — DEDUP — so there are no ties). <c>arg_min</c> has no VARCHAR form, so the usable
|
||||
/// picks carry no <c>str_value</c>: only Delta reads them, and only when the bucket has no usable non-numeric
|
||||
/// sample.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// The previous form referenced its CTE four times (counts + three self-joins); QuestDB opens a separate table
|
||||
/// reader per reference, so a row committed between two of them produced a bucket whose counts included a sample
|
||||
/// its first/last join could not find (a NULL first-of-any-quality time that threw) — ~1 in 90k reads of a tag
|
||||
/// being written. One reference means one reader at one table version. A windowed (<c>row_number()</c>)
|
||||
/// single-scan form was also correct but ~2× slower; this one is not slower than the old query.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// ⚠️ <c>first()</c>/<c>last()</c> following timestamp order is QuestDB 10.0.1 IMPLEMENTATION behaviour (rows reach
|
||||
/// the GROUP BY in row-id order, and the parallel merge keeps row id), not a documented guarantee. It was verified
|
||||
/// on 20M rows with O3 inserts, single-threaded and parallel. Adding a JOIN, <c>ORDER BY</c> or <c>UNION</c> inside
|
||||
/// <c>Classified</c>, or upgrading QuestDB, can silently break Start/End — re-verify <c>first(ts) = min(ts)</c>.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
/// <param name="sample">Prefixed sample table name.</param>
|
||||
/// <returns>The SQL text.</returns>
|
||||
public static string StartEnd(string sample) =>
|
||||
"WITH s AS (" + Classified(sample) + "), " +
|
||||
"st AS (SELECT b, " + CommonCounts() + " FROM s GROUP BY b), " +
|
||||
"f AS (SELECT b, kind, num_value, int_value, str_value, ts, " +
|
||||
"row_number() OVER (PARTITION BY b ORDER BY ts ASC) AS rf, " +
|
||||
"row_number() OVER (PARTITION BY b ORDER BY ts DESC) AS rl FROM s WHERE u = 1) " +
|
||||
"SELECT st.*, " +
|
||||
"x.kind, x.num_value, x.int_value, x.str_value, x.ts, " +
|
||||
"y.kind, y.num_value, y.int_value, y.str_value, y.ts " +
|
||||
"FROM st LEFT JOIN (SELECT * FROM f WHERE rf = 1) x ON x.b = st.b " +
|
||||
"LEFT JOIN (SELECT * FROM f WHERE rl = 1) y ON y.b = st.b ORDER BY st.b";
|
||||
/// <param name="reverse">True for a reverse read (window <c>($4, $5]</c>).</param>
|
||||
public static string StartEnd(string sample, bool reverse = false)
|
||||
{
|
||||
// The usable picks: kind, num_value, int_value, str_value (not available — see remarks), ts.
|
||||
static string Usable(string f, string g) =>
|
||||
$"cast({f}(kind, uts) AS SHORT), {f}(num_value, uts), {f}(int_value, uts), cast(NULL AS VARCHAR), {g}(uts)";
|
||||
// The any-quality picks: ts, kind, num_value, int_value, str_value, status — all from the group's first / last row.
|
||||
static string Any(string f) =>
|
||||
$"{f}(ts), cast({f}(kind) AS SHORT), {f}(num_value), {f}(int_value), {f}(str_value), {f}(status)";
|
||||
return "SELECT b, " + CommonCounts() + ", " +
|
||||
Usable("arg_min", "min") + ", " + Usable("arg_max", "max") + ", " + Any("first") + ", " + Any("last") + " " +
|
||||
"FROM (SELECT b, ts, kind, num_value, int_value, str_value, status, u, g, CASE WHEN u = 1 THEN ts END AS uts " +
|
||||
"FROM (" + Classified(sample, reverse) + ")) GROUP BY b ORDER BY b";
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// MinimumActualTime / MaximumActualTime: the usable sample with the smallest (largest) <c>num_value</c>, ties
|
||||
/// broken by the earliest <c>ts</c>. Family columns: <c>6 num_value, 7 ts</c> (NULL when the bucket has no usable
|
||||
/// numeric sample).
|
||||
/// MinimumActualTime / MaximumActualTime: the <b>Good</b> sample (#684) with the smallest (largest) <c>num_value</c>,
|
||||
/// ties broken by the earliest <c>ts</c>. Family columns: <c>6 num_value, 7 ts</c> (NULL when the bucket has no Good
|
||||
/// numeric sample), then <c>8 unc_min, 9 unc_max, 10 good_non_numeric_count</c> as in <see cref="Simple"/>.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Deliberately still two references to the classified rows (counts + the ranked extreme), unlike
|
||||
/// <see cref="StartEnd"/>: the single-scan <c>row_number()</c> form measured ~2× slower, and QuestDB's
|
||||
/// <c>arg_max</c>/<c>arg_min</c> cannot express the earliest-time tie-break. Live-gate 8 F1's inconsistency (a row
|
||||
/// committed between the two readers — counted but not ranked) is absorbed by the reader and evaluator instead: a
|
||||
/// NULL extreme in a bucket with usable samples evaluates to <c>BadNoData</c>, never a value-less Good.
|
||||
/// </remarks>
|
||||
/// <param name="sample">Prefixed sample table name.</param>
|
||||
/// <param name="maximum">True for MaximumActualTime.</param>
|
||||
/// <returns>The SQL text.</returns>
|
||||
public static string ActualTime(string sample, bool maximum) =>
|
||||
"WITH s AS (" + Classified(sample) + "), " +
|
||||
"st AS (SELECT b, " + CommonCounts() + " FROM s GROUP BY b), " +
|
||||
/// <param name="reverse">True for a reverse read (window <c>($4, $5]</c>).</param>
|
||||
public static string ActualTime(string sample, bool maximum, bool reverse = false) =>
|
||||
"WITH s AS (" + Classified(sample, reverse) + "), " +
|
||||
"st AS (SELECT b, " + CommonCounts() + ", " + UncertainExtremes() + " FROM s GROUP BY b), " +
|
||||
"r AS (SELECT b, num_value, ts, " +
|
||||
"row_number() OVER (PARTITION BY b ORDER BY num_value " + (maximum ? "DESC" : "ASC") + ", ts ASC) AS rn " +
|
||||
"FROM s WHERE u = 1 AND num_value IS NOT NULL) " +
|
||||
"SELECT st.*, r.num_value, r.ts " +
|
||||
"FROM s WHERE g = 1 AND num_value IS NOT NULL) " +
|
||||
"SELECT st.b, st.total_count, st.usable_count, st.good_count, st.bad_count, st.non_numeric_count, " +
|
||||
"r.num_value, r.ts, st.unc_min, st.unc_max, st.good_non_numeric_count " +
|
||||
"FROM st LEFT JOIN (SELECT * FROM r WHERE rn = 1) r ON r.b = st.b ORDER BY st.b";
|
||||
|
||||
/// <summary>
|
||||
/// Time-weighted family (#671), bounds statement — <b>not</b> pushed down (see <see cref="TimeWeightedWindow"/>).
|
||||
/// Four <c>LIMIT 1</c> seeks <c>UNION ALL</c>-ed, each tagged with <c>k</c>: 0 = latest usable sample before the
|
||||
/// start, 1 = latest sample of any quality before the start, 2 = first sample of any quality at or after the end,
|
||||
/// 3 = first usable one at or after the end. Parameters: <c>$1</c> tag, <c>$2</c>/<c>$3</c> the forward window
|
||||
/// start / end. Columns: <c>k</c>, then <c>ts, kind, num_value, int_value, str_value, status, server_ts</c>.
|
||||
/// </summary>
|
||||
/// <param name="sample">Prefixed sample table name.</param>
|
||||
/// <returns>The SQL text.</returns>
|
||||
public static string TimeWeightedBounds(string sample)
|
||||
{
|
||||
var usable = $" AND status < {NotBadLimit} AND kind <> {EmptyKind}";
|
||||
string Seek(int k, string cmp, string filter, string order) =>
|
||||
$"SELECT * FROM (SELECT {k.ToString(CultureInfo.InvariantCulture)} AS k, {SampleColumns} FROM {sample} " +
|
||||
$"WHERE tag = $1 AND ts {cmp}{filter} AND NOT deleted ORDER BY ts {order} LIMIT 1)";
|
||||
return Seek(0, "< $2", usable, "DESC") + " UNION ALL " + Seek(1, "< $2", "", "DESC") + " UNION ALL " +
|
||||
Seek(2, ">= $3", "", "ASC") + " UNION ALL " + Seek(3, ">= $3", usable, "ASC");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Time-weighted family (#671), window statement: the live samples in <c>[$2, $3)</c> for tag <c>$1</c>, ascending,
|
||||
/// at most <c>$4</c> (the raw-row cap + 1).
|
||||
/// Columns: <c>ts, kind, num_value, int_value, str_value, status, server_ts</c>. The caller feeds the before-bounds,
|
||||
/// these rows and the after-bounds, in that order, into <see cref="TimeWeightedAggregator"/> — the same evaluator
|
||||
/// the SQL Server backend uses, so the two agree exactly. Trade-off: the window's rows travel to the server rather
|
||||
/// than one row per bucket; Part 13's segment interpolation and bucket-edge splitting are then written once
|
||||
/// instead of twice in two SQL dialects that must stay bit-identical.
|
||||
/// </summary>
|
||||
/// <param name="sample">Prefixed sample table name.</param>
|
||||
/// <returns>The SQL text.</returns>
|
||||
public static string TimeWeightedWindow(string sample) =>
|
||||
$"SELECT {SampleColumns} FROM {sample} WHERE tag = $1 AND ts >= $2 AND ts < $3 AND NOT deleted ORDER BY ts ASC LIMIT $4";
|
||||
|
||||
/// <summary>The row columns the time-weighted statements return (after <c>k</c> for the bounds).</summary>
|
||||
private const string SampleColumns = "ts, kind, num_value, int_value, str_value, status, server_ts";
|
||||
|
||||
/// <summary>The bucket-index expression (exposed for tests).</summary>
|
||||
internal const string BucketExpression = "(cast(ts AS LONG) * 10 - $1) / $2";
|
||||
|
||||
/// <summary>
|
||||
/// The tag's earliest live sample (any quality) at or before the window's later edge — half of the tag's data range
|
||||
/// (the start-of-data Partial rule, #677, and Count's in-range empty interval, #680; see
|
||||
/// <see cref="AggregateEvaluator.NeedsDataRange"/>). Parameters <c>$1</c> tag, <c>$2</c> the later edge. Column:
|
||||
/// <c>ts</c>; no row when the tag has none — the data range then does not reach the window.
|
||||
/// </summary>
|
||||
/// <param name="sample">Prefixed sample table name.</param>
|
||||
/// <returns>The SQL text.</returns>
|
||||
public static string EarliestSample(string sample) =>
|
||||
$"SELECT ts FROM {sample} WHERE tag = $1 AND ts <= $2 AND NOT deleted ORDER BY ts ASC LIMIT 1";
|
||||
|
||||
/// <summary>
|
||||
/// The tag's latest live sample (any quality) at or after the window's earlier edge — the other half of the data
|
||||
/// range (the end-of-data Partial rule and Count's in-range empty interval, #680). Parameters <c>$1</c> tag,
|
||||
/// <c>$2</c> the earlier edge. Column: <c>ts</c>; no row when the tag has none.
|
||||
/// </summary>
|
||||
/// <param name="sample">Prefixed sample table name.</param>
|
||||
/// <returns>The SQL text.</returns>
|
||||
public static string LatestSample(string sample) =>
|
||||
$"SELECT ts FROM {sample} WHERE tag = $1 AND ts >= $2 AND NOT deleted ORDER BY ts DESC LIMIT 1";
|
||||
|
||||
/// <summary>The classified subquery <c>s</c>: the window's live samples with bucket index and the usable/good flags.</summary>
|
||||
private static string Classified(string sample) =>
|
||||
"SELECT " + BucketExpression + " AS b, ts, kind, num_value, int_value, str_value, " +
|
||||
private static string Classified(string sample, bool reverse) =>
|
||||
"SELECT " + BucketExpression + " AS b, ts, kind, num_value, int_value, str_value, status, " +
|
||||
$"CASE WHEN status < {NotBadLimit} AND kind <> {EmptyKind} THEN 1 ELSE 0 END AS u, " +
|
||||
$"CASE WHEN status < {GoodLimit} AND kind <> {EmptyKind} THEN 1 ELSE 0 END AS g " +
|
||||
$"FROM {sample} WHERE tag = $3 AND ts >= $4 AND ts < $5 AND NOT deleted";
|
||||
$"FROM {sample} WHERE tag = $3 AND " + (reverse ? "ts > $4 AND ts <= $5" : "ts >= $4 AND ts < $5") + " AND NOT deleted";
|
||||
|
||||
/// <summary>
|
||||
/// The minimum and maximum <b>Uncertain</b> (usable, not Good) numeric value of the bucket — <c>unc_min, unc_max</c>
|
||||
/// (#684). Not result values: Part 13 makes Minimum/Maximum/Range Uncertain when one lies beyond the Good extreme.
|
||||
/// Then <c>good_non_numeric_count</c>, the Good String/DateTime samples (the Good-value aggregates' refusal check).
|
||||
/// </summary>
|
||||
private static string UncertainExtremes() =>
|
||||
"min(CASE WHEN u = 1 AND g = 0 THEN num_value END) AS unc_min, max(CASE WHEN u = 1 AND g = 0 THEN num_value END) AS unc_max, " +
|
||||
$"sum(CASE WHEN g = 1 AND kind IN ({NonNumericKinds}) THEN 1 ELSE 0 END) AS good_non_numeric_count";
|
||||
|
||||
/// <summary>The five common count columns (after <c>b</c>).</summary>
|
||||
private static string CommonCounts() =>
|
||||
|
||||
@@ -83,6 +83,8 @@ internal sealed class QuestDbConnection : IAsyncDisposable, IDisposable
|
||||
public string EventTable => T(QuestDbTables.EventBase);
|
||||
/// <summary>The prefixed tag table.</summary>
|
||||
public string TagTable => T(QuestDbTables.TagBase);
|
||||
/// <summary>The prefixed Part 11 modification-trail table (#668).</summary>
|
||||
public string SampleModifiedTable => T(QuestDbTables.SampleModifiedBase);
|
||||
|
||||
/// <summary>Opens a pooled connection (no bootstrap; callers run the bootstrapper first).</summary>
|
||||
/// <param name="ct">Cancellation.</param>
|
||||
@@ -199,14 +201,14 @@ internal sealed class QuestDbConnection : IAsyncDisposable, IDisposable
|
||||
/// <param name="r">Reader positioned on a row.</param>
|
||||
/// <param name="i">Column ordinal.</param>
|
||||
/// <returns>The timestamp with <c>Kind=Utc</c>.</returns>
|
||||
public static DateTime ReadUtc(NpgsqlDataReader r, int i) =>
|
||||
public static DateTime ReadUtc(System.Data.Common.DbDataReader r, int i) =>
|
||||
DateTime.SpecifyKind(r.GetDateTime(i), DateTimeKind.Utc);
|
||||
|
||||
/// <summary>Reads a <c>BYTE</c> column (QuestDB returns <c>smallint</c>); NULL (never stored) reads as 0 (Empty).</summary>
|
||||
/// <param name="r">Reader positioned on a row.</param>
|
||||
/// <param name="i">Column ordinal.</param>
|
||||
/// <returns>The byte value.</returns>
|
||||
public static byte ReadKind(NpgsqlDataReader r, int i) => r.IsDBNull(i) ? (byte)0 : (byte)r.GetInt16(i);
|
||||
public static byte ReadKind(System.Data.Common.DbDataReader r, int i) => r.IsDBNull(i) ? (byte)0 : (byte)r.GetInt16(i);
|
||||
|
||||
// ---- error classification --------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
+274
-31
@@ -1,3 +1,4 @@
|
||||
using System.Data.Common;
|
||||
using System.Globalization;
|
||||
using System.Text;
|
||||
using Microsoft.Extensions.Logging;
|
||||
@@ -64,6 +65,7 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource
|
||||
private readonly ILogger _log;
|
||||
private readonly TimeProvider _time;
|
||||
private readonly HistorianHealthCounters _health = new();
|
||||
private readonly int _maxTimeWeightedRawRows;
|
||||
|
||||
/// <summary>Creates the data source over the provider's shared connection, bootstrapper and tag catalog.</summary>
|
||||
/// <param name="conn">Connection helper.</param>
|
||||
@@ -71,10 +73,12 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource
|
||||
/// <param name="catalog">Tag catalog (reads use only its read-only <see cref="QuestDbTagCatalog.ExistsAsync"/>).</param>
|
||||
/// <param name="log">Optional logger.</param>
|
||||
/// <param name="timeProvider">Clock for "now" (AtTime beyond the last sample, server timestamps); system by default.</param>
|
||||
/// <param name="maxTimeWeightedRawRows">Raw in-window row cap of one time-weighted read (<= 0 = unbounded).</param>
|
||||
internal QuestDbHistorianDataSource(
|
||||
QuestDbConnection conn, QuestDbSchemaBootstrapper bootstrap, QuestDbTagCatalog catalog, ILogger? log = null,
|
||||
TimeProvider? timeProvider = null)
|
||||
TimeProvider? timeProvider = null, int maxTimeWeightedRawRows = TimeWeightedAggregator.DefaultMaxWindowRows)
|
||||
{
|
||||
_maxTimeWeightedRawRows = maxTimeWeightedRawRows;
|
||||
_conn = conn ?? throw new ArgumentNullException(nameof(conn));
|
||||
_bootstrap = bootstrap ?? throw new ArgumentNullException(nameof(bootstrap));
|
||||
_catalog = catalog ?? throw new ArgumentNullException(nameof(catalog));
|
||||
@@ -133,6 +137,103 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource
|
||||
catch (Exception ex)
|
||||
{
|
||||
RecordFailure(ex);
|
||||
if (IsTransportFailure(ex)) throw new System.IO.IOException("Historian read transport is unavailable.", ex);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------------------------------ Modified (Part 11, #668)
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>
|
||||
/// Reads <c>{prefix}sample_modified</c>, the append-only trail <see cref="QuestDbHistorianDataUpdater"/> writes
|
||||
/// BEFORE each change. Windows follow <see cref="ReadRawAsync"/> (bounds rounded to µs); rows are ordered by
|
||||
/// <c>(ts, modified_id)</c>; <c>maxValues + 1</c> rows are fetched to report
|
||||
/// <see cref="HistoryModifiedReadResult.HasMore"/> exactly. A row whose <c>status</c> is NULL had no prior value
|
||||
/// (an insert) and is served <c>null</c> / <c>BadNoData</c>. Because the trail row lands first, a crash between it and
|
||||
/// the sample write leaves an orphan row describing a change that never applied (documented).
|
||||
/// </remarks>
|
||||
public async Task<HistoryModifiedReadResult> ReadModifiedAsync(
|
||||
string fullReference, DateTime startUtc, DateTime endUtc, uint maxValues, HistoryModifiedCursor? resumeAfter,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(fullReference);
|
||||
var bounded = maxValues > 0 && maxValues < int.MaxValue;
|
||||
try
|
||||
{
|
||||
var values = new List<HistoryModifiedValue>();
|
||||
var hasMore = false;
|
||||
if (await ResolveAsync(fullReference, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
var start = QuestDbConnection.Round(startUtc);
|
||||
var end = QuestDbConnection.Round(endUtc);
|
||||
var reverse = start > end;
|
||||
string where;
|
||||
if (start == end) where = "ts = $2 AND ts = $3";
|
||||
else if (!reverse) where = "ts >= $2 AND ts < $3";
|
||||
else where = "ts > $2 AND ts <= $3";
|
||||
var next = 4;
|
||||
if (resumeAfter is not null)
|
||||
{
|
||||
where += reverse
|
||||
? " AND (ts < $4 OR (ts = $4 AND modified_id < $5))"
|
||||
: " AND (ts > $4 OR (ts = $4 AND modified_id > $5))";
|
||||
next = 6;
|
||||
}
|
||||
var order = reverse ? "DESC" : "ASC";
|
||||
var (lo, hi) = start <= end ? (start, end) : (end, start);
|
||||
|
||||
await using var conn = await _conn.OpenAsync(cancellationToken).ConfigureAwait(false);
|
||||
await using var cmd = _conn.CreateCommand(conn,
|
||||
$"SELECT {SampleColumns}, modified_id, modification_ts, update_type, principal FROM {_conn.SampleModifiedTable} " +
|
||||
$"WHERE tag = $1 AND {where} ORDER BY ts {order}, modified_id {order}" + (bounded ? $" LIMIT ${next}" : ""));
|
||||
QuestDbConnection.BindText(cmd, fullReference);
|
||||
QuestDbConnection.BindTimestamp(cmd, lo);
|
||||
QuestDbConnection.BindTimestamp(cmd, hi);
|
||||
if (resumeAfter is not null)
|
||||
{
|
||||
QuestDbConnection.BindTimestamp(cmd, resumeAfter.SourceTimeUtc);
|
||||
QuestDbConnection.BindLong(cmd, resumeAfter.ModifiedId);
|
||||
}
|
||||
if (bounded)
|
||||
cmd.Parameters.Add(new NpgsqlParameter { NpgsqlDbType = NpgsqlDbType.Integer, Value = (int)maxValues + 1 });
|
||||
|
||||
await using var reader = await cmd.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false);
|
||||
while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
if (bounded && values.Count == maxValues)
|
||||
{
|
||||
hasMore = true;
|
||||
break;
|
||||
}
|
||||
|
||||
var sourceTime = QuestDbConnection.ReadUtc(reader, 0);
|
||||
var modifiedAt = QuestDbConnection.ReadUtc(reader, 8);
|
||||
DataValueSnapshot prior;
|
||||
if (reader.IsDBNull(5))
|
||||
{
|
||||
prior = new DataValueSnapshot(null, HistorianStatusCodes.BadNoData, sourceTime, modifiedAt);
|
||||
}
|
||||
else
|
||||
{
|
||||
var s = ReadStored(reader, 0);
|
||||
prior = new DataValueSnapshot(s.Value.ToObject(), s.Status, s.TimeUtc, ReadServerTime(reader, 6, modifiedAt));
|
||||
}
|
||||
|
||||
values.Add(new HistoryModifiedValue(prior, modifiedAt,
|
||||
(HistoryModificationType)QuestDbConnection.ReadKind(reader, 9),
|
||||
reader.IsDBNull(10) ? string.Empty : reader.GetString(10),
|
||||
reader.GetInt64(7), sourceTime));
|
||||
}
|
||||
}
|
||||
|
||||
RecordSuccess();
|
||||
return new HistoryModifiedReadResult(values, hasMore);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
RecordFailure(ex);
|
||||
if (IsTransportFailure(ex)) throw new System.IO.IOException("Historian read transport is unavailable.", ex);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
@@ -217,6 +318,7 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource
|
||||
catch (Exception ex)
|
||||
{
|
||||
RecordFailure(ex);
|
||||
if (IsTransportFailure(ex)) throw new System.IO.IOException("Historian read transport is unavailable.", ex);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
@@ -256,6 +358,23 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource
|
||||
|
||||
// ------------------------------------------------------------------------------------------ Events
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>Reverse raw reads are served natively (<c>(end, start]</c>, <c>ORDER BY … DESC</c>; #669/#686).</remarks>
|
||||
public bool SupportsReverseRaw => true;
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks><c>0</c>: a capped raw read is one bounded statement — no backend row ceiling (Gitea #694).</remarks>
|
||||
public int MaxRawValuesPerRead => 0;
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>Reverse events reads are served natively (<c>ORDER BY … DESC</c>, Gitea #675).</remarks>
|
||||
public bool SupportsReverseEvents => true;
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>The seventeen <see cref="AggregateFamilies.Served"/> — shared with the other SQL backend (#682).</remarks>
|
||||
public IReadOnlySet<HistoryAggregateType> SupportedAggregates => AggregateFamilies.Served;
|
||||
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>
|
||||
/// <c>[start, end)</c> by event time (a reversed window is swapped), ordered by <c>(ts, event_id)</c> ascending.
|
||||
@@ -263,8 +382,18 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource
|
||||
/// <c>maxEvents + 1</c> rows are fetched; when the extra row arrives it is dropped and the continuation point is
|
||||
/// the UTF-8 of <see cref="EventContinuation.Encode"/> for that first not-returned event.
|
||||
/// </remarks>
|
||||
public Task<HistoricalEventsResult> ReadEventsAsync(
|
||||
string? sourceName, DateTime startUtc, DateTime endUtc, int maxEvents, CancellationToken cancellationToken) =>
|
||||
ReadEventsAsync(sourceName, startUtc, endUtc, maxEvents, reverse: false, cancellationToken);
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>
|
||||
/// Forward is the five-argument read. Reverse (Gitea #675) is the (rounded) <c>(lo, hi]</c> ordered by
|
||||
/// <c>(ts, event_id)</c> DESCENDING, so <c>LIMIT</c> keeps the latest events; truncation and the continuation point
|
||||
/// work as forward.
|
||||
/// </remarks>
|
||||
public async Task<HistoricalEventsResult> ReadEventsAsync(
|
||||
string? sourceName, DateTime startUtc, DateTime endUtc, int maxEvents, CancellationToken cancellationToken)
|
||||
string? sourceName, DateTime startUtc, DateTime endUtc, int maxEvents, bool reverse, CancellationToken cancellationToken)
|
||||
{
|
||||
// No stored event can carry a source longer than the limit, so an over-long filter matches nothing.
|
||||
if (sourceName is not null && sourceName.Length > HistorianTagNames.MaxLength)
|
||||
@@ -278,9 +407,9 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource
|
||||
var bounded = maxEvents > 0 && maxEvents < int.MaxValue;
|
||||
var sql =
|
||||
$"SELECT event_id, source, ts, received_utc, message, severity FROM {_conn.EventTable} " +
|
||||
"WHERE ts >= $1 AND ts < $2" +
|
||||
(reverse ? "WHERE ts > $1 AND ts <= $2" : "WHERE ts >= $1 AND ts < $2") +
|
||||
(sourceName is null ? "" : " AND source = $3") +
|
||||
" ORDER BY ts ASC, event_id ASC" +
|
||||
(reverse ? " ORDER BY ts DESC, event_id DESC" : " ORDER BY ts ASC, event_id ASC") +
|
||||
(bounded ? (sourceName is null ? " LIMIT $3" : " LIMIT $4") : "");
|
||||
|
||||
await using var conn = await _conn.OpenAsync(cancellationToken).ConfigureAwait(false);
|
||||
@@ -317,6 +446,7 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource
|
||||
catch (Exception ex)
|
||||
{
|
||||
RecordFailure(ex);
|
||||
if (IsTransportFailure(ex)) throw new System.IO.IOException("Historian read transport is unavailable.", ex);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
@@ -327,14 +457,18 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource
|
||||
/// <remarks>
|
||||
/// The (rounded) window is planned by <see cref="BucketPlan.Create"/>: bucket <c>i</c> =
|
||||
/// <c>[start + i·interval, start + (i+1)·interval)</c>, start-inclusive / end-exclusive, the last bucket truncated
|
||||
/// at <c>end</c>, interval <= 0 ⇒ one bucket, <c>start == end</c> ⇒ none, a reversed window computed forward
|
||||
/// and returned descending, at most <see cref="BucketPlan.MaxBuckets"/> buckets. One statement per node returns
|
||||
/// the <see cref="BucketRow"/> statistics of each non-empty bucket (bucket index
|
||||
/// <c>(cast(ts AS LONG)*10 − StartUnixTicks) / IntervalTicks</c>, never <c>SAMPLE BY</c>);
|
||||
/// <see cref="AggregateEvaluator.Evaluate"/> produces each value and <see cref="BucketPlan.Assemble"/> fills the
|
||||
/// empty buckets with <c>BadNoData</c>.
|
||||
/// at <c>end</c>, interval <= 0 ⇒ one bucket, <c>start == end</c> ⇒ none, at most
|
||||
/// <see cref="BucketPlan.MaxBuckets"/> buckets. A reversed window (#677) is anchored at its later time per Part 13 /
|
||||
/// the SDK: window <c>(end, start]</c>, buckets stepping back from <c>start</c>, the partial one earliest, each stamped
|
||||
/// with its later edge, returned latest first. One statement per node returns the <see cref="BucketRow"/> statistics
|
||||
/// of each non-empty bucket (bucket index <c>(cast(ts AS LONG)*10 − BucketBaseUnixTicks) / IntervalTicks</c>, never
|
||||
/// <c>SAMPLE BY</c>); before it, when <see cref="AggregateEvaluator.NeedsDataRange"/>, two tag-index seeks fetch the
|
||||
/// tag's data range (earliest / latest live sample) for the start/end-of-data Partial bits and Count's in-range
|
||||
/// empty interval; and
|
||||
/// <see cref="AggregateEvaluator.EvaluateAll"/> produces the values and fills the empty buckets with <c>BadNoData</c>. The time-weighted family (#671) instead streams the window plus four bounding
|
||||
/// rows into <see cref="TimeWeightedAggregator"/>, exactly as the SQL Server backend does.
|
||||
/// </remarks>
|
||||
/// <exception cref="HistoryAggregateNotSupportedException">The aggregate is not one of the thirteen served.</exception>
|
||||
/// <exception cref="HistoryAggregateNotSupportedException">The aggregate is not one of the seventeen served.</exception>
|
||||
/// <exception cref="ArgumentOutOfRangeException">The window holds more than <see cref="BucketPlan.MaxBuckets"/> intervals.</exception>
|
||||
public async Task<HistoryReadResult> ReadProcessedAsync(
|
||||
string fullReference, DateTime startUtc, DateTime endUtc, TimeSpan interval,
|
||||
@@ -347,49 +481,139 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource
|
||||
try
|
||||
{
|
||||
var now = _time.GetUtcNow().UtcDateTime;
|
||||
var evaluated = new Dictionary<long, DataValueSnapshot>();
|
||||
if (family == AggregateFamily.TimeWeighted)
|
||||
{
|
||||
var tw = new TimeWeightedAggregator(aggregate, plan);
|
||||
if (plan.BucketCount > 0 && await ResolveAsync(fullReference, cancellationToken).ConfigureAwait(false))
|
||||
await StreamTimeWeightedAsync(fullReference, plan, tw, cancellationToken).ConfigureAwait(false);
|
||||
var twResult = tw.Finish(now);
|
||||
RecordSuccess();
|
||||
return new HistoryReadResult(twResult, ContinuationPoint: null);
|
||||
}
|
||||
var rows = new List<BucketRow>();
|
||||
DateTime? earliest = null, latest = null;
|
||||
if (plan.BucketCount > 0 && await ResolveAsync(fullReference, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
var sample = _conn.SampleTable;
|
||||
var sql = family switch
|
||||
{
|
||||
AggregateFamily.Simple => QuestDbAggregateQueries.Simple(sample),
|
||||
AggregateFamily.StartEnd => QuestDbAggregateQueries.StartEnd(sample),
|
||||
_ => QuestDbAggregateQueries.ActualTime(sample, aggregate == HistoryAggregateType.MaximumActualTime),
|
||||
AggregateFamily.Simple => QuestDbAggregateQueries.Simple(sample, plan.Reversed),
|
||||
AggregateFamily.StartEnd => QuestDbAggregateQueries.StartEnd(sample, plan.Reversed),
|
||||
_ => QuestDbAggregateQueries.ActualTime(sample, aggregate == HistoryAggregateType.MaximumActualTime, plan.Reversed),
|
||||
};
|
||||
await using var conn = await _conn.OpenAsync(cancellationToken).ConfigureAwait(false);
|
||||
await using var cmd = _conn.CreateCommand(conn, sql);
|
||||
QuestDbConnection.BindLong(cmd, plan.StartUnixTicks);
|
||||
QuestDbConnection.BindLong(cmd, plan.IntervalTicks);
|
||||
QuestDbConnection.BindText(cmd, fullReference);
|
||||
QuestDbConnection.BindTimestamp(cmd, plan.StartUtc);
|
||||
QuestDbConnection.BindTimestamp(cmd, plan.EndUtc);
|
||||
await using var reader = await cmd.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false);
|
||||
while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false))
|
||||
// The tag's data range (earliest + latest live sample): the start/end-of-data Partial rules (#677/#680 —
|
||||
// any per-sample aggregate but Average, either direction) and Count's in-range empty interval (#680); skipped
|
||||
// otherwise. Index seeks on tag (#683). Read BEFORE the bucket statement, which is a separate table reader:
|
||||
// a sample appended in between (the common race — a live tag read up to now) then reaches the buckets but
|
||||
// not the range — at worst a stale end-of-data Partial or a BadNoData where a 0 belongs — instead of a
|
||||
// range covering a bucket the rows do not show, which reported a Good Count of 0 for an interval that has
|
||||
// samples. (A concurrent HistoryUpdate delete is the mirror case and can still do that, transiently.)
|
||||
if (AggregateEvaluator.NeedsDataRange(aggregate, plan))
|
||||
{
|
||||
var row = ReadBucket(reader, family);
|
||||
evaluated[row.Bucket] = AggregateEvaluator.Evaluate(aggregate, row, plan.BucketStart(row.Bucket), now);
|
||||
earliest = await ReadBoundAsync(conn, QuestDbAggregateQueries.EarliestSample(sample), fullReference,
|
||||
plan.EndUtc, cancellationToken).ConfigureAwait(false);
|
||||
latest = await ReadBoundAsync(conn, QuestDbAggregateQueries.LatestSample(sample), fullReference,
|
||||
plan.StartUtc, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
await using (var cmd = _conn.CreateCommand(conn, sql))
|
||||
{
|
||||
QuestDbConnection.BindLong(cmd, plan.BucketBaseUnixTicks);
|
||||
QuestDbConnection.BindLong(cmd, plan.IntervalTicks);
|
||||
QuestDbConnection.BindText(cmd, fullReference);
|
||||
QuestDbConnection.BindTimestamp(cmd, plan.StartUtc);
|
||||
QuestDbConnection.BindTimestamp(cmd, plan.EndUtc);
|
||||
await using var reader = await cmd.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false);
|
||||
while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false))
|
||||
rows.Add(ReadBucket(reader, family));
|
||||
}
|
||||
}
|
||||
|
||||
var result = plan.Assemble(evaluated, bucketStart => AggregateEvaluator.NoData(bucketStart, now));
|
||||
var result = AggregateEvaluator.EvaluateAll(aggregate, plan, rows, earliest, latest, now);
|
||||
|
||||
RecordSuccess();
|
||||
return new HistoryReadResult(result, ContinuationPoint: null);
|
||||
}
|
||||
catch (HistoryReadLimitExceededException)
|
||||
{
|
||||
// A client-request limit (the raw-row cap), not a backend fault: not counted as a failure.
|
||||
throw;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
RecordFailure(ex);
|
||||
if (IsTransportFailure(ex)) throw new System.IO.IOException("Historian read transport is unavailable.", ex);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
private static BucketRow ReadBucket(NpgsqlDataReader reader, AggregateFamily family)
|
||||
/// <summary>
|
||||
/// Streams the time-weighted bounds (<see cref="QuestDbAggregateQueries.TimeWeightedBounds"/>) and window
|
||||
/// (<see cref="QuestDbAggregateQueries.TimeWeightedWindow"/>) into <paramref name="tw"/> in time order: the two
|
||||
/// before-bounds, the window, the two after-bounds (duplicates are dropped by the aggregator).
|
||||
/// </summary>
|
||||
private async Task StreamTimeWeightedAsync(string tag, BucketPlan plan, TimeWeightedAggregator tw, CancellationToken ct)
|
||||
{
|
||||
static double? D(NpgsqlDataReader r, int i) => r.IsDBNull(i) ? null : r.GetDouble(i);
|
||||
static int N(NpgsqlDataReader r, int i) => r.IsDBNull(i) ? 0 : Convert.ToInt32(r.GetValue(i), CultureInfo.InvariantCulture);
|
||||
var bounds = new StoredSample?[4];
|
||||
await using var conn = await _conn.OpenAsync(ct).ConfigureAwait(false);
|
||||
await using (var cmd = _conn.CreateCommand(conn, QuestDbAggregateQueries.TimeWeightedBounds(_conn.SampleTable)))
|
||||
{
|
||||
QuestDbConnection.BindText(cmd, tag);
|
||||
QuestDbConnection.BindTimestamp(cmd, plan.StartUtc);
|
||||
QuestDbConnection.BindTimestamp(cmd, plan.EndUtc);
|
||||
await using var reader = await cmd.ExecuteReaderAsync(ct).ConfigureAwait(false);
|
||||
while (await reader.ReadAsync(ct).ConfigureAwait(false))
|
||||
bounds[Convert.ToInt32(reader.GetValue(0), CultureInfo.InvariantCulture)] = ReadStored(reader, 1);
|
||||
}
|
||||
|
||||
if (bounds[0] is { } usableBefore) tw.Add(usableBefore);
|
||||
if (bounds[1] is { } anyBefore) tw.Add(anyBefore);
|
||||
await using (var cmd = _conn.CreateCommand(conn, QuestDbAggregateQueries.TimeWeightedWindow(_conn.SampleTable)))
|
||||
{
|
||||
QuestDbConnection.BindText(cmd, tag);
|
||||
QuestDbConnection.BindTimestamp(cmd, plan.StartUtc);
|
||||
QuestDbConnection.BindTimestamp(cmd, plan.EndUtc);
|
||||
cmd.Parameters.Add(new NpgsqlParameter
|
||||
{ NpgsqlDbType = NpgsqlDbType.Integer, Value = TimeWeightedAggregator.WindowFetchLimit(_maxTimeWeightedRawRows) });
|
||||
await using var reader = await cmd.ExecuteReaderAsync(ct).ConfigureAwait(false);
|
||||
long windowRows = 0;
|
||||
while (await reader.ReadAsync(ct).ConfigureAwait(false))
|
||||
{
|
||||
// LIMIT $4 = cap + 1: one row past the cap fails the node without reading on.
|
||||
TimeWeightedAggregator.CheckWindowRows(++windowRows, _maxTimeWeightedRawRows, tag);
|
||||
tw.Add(ReadStored(reader, 0));
|
||||
}
|
||||
}
|
||||
if (bounds[2] is { } anyAfter) tw.Add(anyAfter);
|
||||
if (bounds[3] is { } usableAfter) tw.Add(usableAfter);
|
||||
}
|
||||
|
||||
/// <summary>Runs a one-column <c>ts</c> seek (<see cref="QuestDbAggregateQueries.EarliestSample"/> / <c>LatestSample</c>).</summary>
|
||||
private async Task<DateTime?> ReadBoundAsync(NpgsqlConnection conn, string sql, string tag, DateTime edge, CancellationToken ct)
|
||||
{
|
||||
await using var cmd = _conn.CreateCommand(conn, sql);
|
||||
QuestDbConnection.BindText(cmd, tag);
|
||||
QuestDbConnection.BindTimestamp(cmd, edge);
|
||||
await using var reader = await cmd.ExecuteReaderAsync(ct).ConfigureAwait(false);
|
||||
return await reader.ReadAsync(ct).ConfigureAwait(false) && !reader.IsDBNull(0) ? QuestDbConnection.ReadUtc(reader, 0) : null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Maps one per-sample aggregate row (<see cref="QuestDbAggregateQueries"/> column layout) to a
|
||||
/// <see cref="BucketRow"/>. Every picked sample — first/last usable, first/last of any quality, the extreme — is
|
||||
/// optional: a NULL time reads as "no such sample", never an exception (live-gate 8 F1: a row whose counts and
|
||||
/// picks disagreed used to throw out of <see cref="ReadStored"/> and fail the node's whole read; the evaluator
|
||||
/// answers such a bucket from what is present, or <c>BadNoData</c>).
|
||||
/// </summary>
|
||||
/// <param name="reader">Reader positioned on a bucket row.</param>
|
||||
/// <param name="family">The query's aggregate family.</param>
|
||||
/// <returns>The bucket statistics.</returns>
|
||||
internal static BucketRow ReadBucket(DbDataReader reader, AggregateFamily family)
|
||||
{
|
||||
static double? D(DbDataReader r, int i) => r.IsDBNull(i) ? null : r.GetDouble(i);
|
||||
static int N(DbDataReader r, int i) => r.IsDBNull(i) ? 0 : Convert.ToInt32(r.GetValue(i), CultureInfo.InvariantCulture);
|
||||
// A bucket without a usable sample has NULL times but kind 0 (BYTE has no NULL): absence is keyed on the time.
|
||||
static HistorianValue V(NpgsqlDataReader r, int i) =>
|
||||
static HistorianValue V(DbDataReader r, int i) =>
|
||||
r.IsDBNull(i + 4)
|
||||
? HistorianValue.Empty
|
||||
: ToValue(QuestDbConnection.ReadKind(r, i), D(r, i + 1), r.IsDBNull(i + 2) ? null : r.GetInt64(i + 2),
|
||||
@@ -412,14 +636,22 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource
|
||||
row.Min = D(reader, f + 1);
|
||||
row.Max = D(reader, f + 2);
|
||||
row.Sum = D(reader, f + 3);
|
||||
row.UncertainMin = D(reader, f + 4);
|
||||
row.UncertainMax = D(reader, f + 5);
|
||||
row.GoodNonNumeric = N(reader, f + 6);
|
||||
break;
|
||||
case AggregateFamily.StartEnd:
|
||||
row.First = V(reader, f);
|
||||
row.Last = V(reader, f + 5);
|
||||
row.FirstAny = ReadStoredOrNull(reader, QuestDbAggregateQueries.FirstAnyColumn);
|
||||
row.LastAny = ReadStoredOrNull(reader, QuestDbAggregateQueries.FirstAnyColumn + 6);
|
||||
break;
|
||||
default:
|
||||
row.Extreme = D(reader, f);
|
||||
row.ExtremeTime = reader.IsDBNull(f + 1) ? null : QuestDbConnection.ReadUtc(reader, f + 1);
|
||||
row.UncertainMin = D(reader, f + 2);
|
||||
row.UncertainMax = D(reader, f + 3);
|
||||
row.GoodNonNumeric = N(reader, f + 4);
|
||||
break;
|
||||
}
|
||||
return row;
|
||||
@@ -431,7 +663,7 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource
|
||||
/// Reads <c>ts, kind, num_value, int_value, str_value, status</c> starting at <paramref name="o"/> and applies the
|
||||
/// shared quality clamp (<see cref="StoredSample.FromStored"/>).
|
||||
/// </summary>
|
||||
private static StoredSample ReadStored(NpgsqlDataReader r, int o)
|
||||
private static StoredSample ReadStored(DbDataReader r, int o)
|
||||
{
|
||||
var value = ToValue(
|
||||
QuestDbConnection.ReadKind(r, o + 1),
|
||||
@@ -443,6 +675,12 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource
|
||||
return StoredSample.FromStored(QuestDbConnection.ReadUtc(r, o), value, status);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// <see cref="ReadStored"/> for an optional picked sample: a NULL <c>ts</c> means the row carries no such sample
|
||||
/// (live-gate 8 F1), so the result is null rather than an <see cref="InvalidCastException"/>.
|
||||
/// </summary>
|
||||
private static StoredSample? ReadStoredOrNull(DbDataReader r, int o) => r.IsDBNull(o) ? null : ReadStored(r, o);
|
||||
|
||||
/// <summary>
|
||||
/// Rebuilds a value from its stored columns via the shared <see cref="SampleColumnMapper.FromColumns"/>, after the
|
||||
/// RC2 repair: QuestDB reads a <c>LONG</c> holding <see cref="long.MinValue"/> back as NULL, so an Int64 row with
|
||||
@@ -468,6 +706,11 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource
|
||||
|
||||
private void RecordSuccess() => _health.RecordSuccess();
|
||||
|
||||
/// <summary>Classifies provider transport/timeouts, keeping SQL/query errors distinct (#697).</summary>
|
||||
private static bool IsTransportFailure(Exception ex) => ex is NpgsqlException { IsTransient: true } and not PostgresException
|
||||
|| ex is PostgresException pg && (pg.SqlState.StartsWith("08", StringComparison.Ordinal)
|
||||
|| pg.SqlState is "57P01" or "57P02" or "57P03");
|
||||
|
||||
private void RecordFailure(Exception ex)
|
||||
{
|
||||
// A cancelled read (deadline / client abort) is not a backend fault (the counters ignore it too).
|
||||
|
||||
+184
-17
@@ -25,6 +25,13 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb;
|
||||
/// 1970 (read bounds are not clamped).</para>
|
||||
/// <para><b>Read-your-writes:</b> after writing, <see cref="QuestDbWalVisibility"/> waits for the rows to become
|
||||
/// visible, bounded by <c>VisibilityTimeout</c>; a timeout only logs a Warning, never changes a code.</para>
|
||||
/// <para><b>Modified-value history (Part 11, #668).</b> Every sample this updater inserts, replaces or tombstones first
|
||||
/// gets a row in <c>{prefix}sample_modified</c> carrying its PRIOR live value (the existence read already fetches it;
|
||||
/// none for an insert — a tombstone is "absent", so an insert over one records no prior value), the caller's
|
||||
/// <see cref="HistoryUpdateContext"/>, the <see cref="HistoryModificationType"/> and the modification time. QuestDB has
|
||||
/// no multi-table transaction, so the trail row is written <b>BEFORE</b> the change: a crash (or a failed sample INSERT)
|
||||
/// between the two leaves an orphan trail row describing a change that never landed — never a change without its
|
||||
/// trail row. This is the documented deviation from SQL Server, whose trail commits atomically with the change.</para>
|
||||
/// </summary>
|
||||
internal sealed class QuestDbHistorianDataUpdater : IHistorianDataUpdater
|
||||
{
|
||||
@@ -35,6 +42,7 @@ internal sealed class QuestDbHistorianDataUpdater : IHistorianDataUpdater
|
||||
private const int LockStripes = 64;
|
||||
private readonly SemaphoreSlim[] _locks = Enumerable.Range(0, LockStripes).Select(_ => new SemaphoreSlim(1, 1)).ToArray();
|
||||
private long _liveRowsRead;
|
||||
private static long _lastModifiedId;
|
||||
private readonly QuestDbConnection _conn;
|
||||
private readonly QuestDbTagCatalog _tags;
|
||||
private readonly QuestDbWalVisibility _wal;
|
||||
@@ -71,10 +79,11 @@ internal sealed class QuestDbHistorianDataUpdater : IHistorianDataUpdater
|
||||
|
||||
/// <inheritdoc/>
|
||||
public async Task<IReadOnlyList<uint>> UpdateDataAsync(string tag, HistoryUpdateMode mode,
|
||||
IReadOnlyList<HistorizationValue> values, CancellationToken ct)
|
||||
IReadOnlyList<HistorizationValue> values, HistoryUpdateContext context, CancellationToken ct)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(tag);
|
||||
ArgumentNullException.ThrowIfNull(values);
|
||||
ArgumentNullException.ThrowIfNull(context);
|
||||
|
||||
// RC1: a valid-timestamp item before 1970 can never be stored; refuse it and plan the rest.
|
||||
var final = new uint[values.Count];
|
||||
@@ -100,7 +109,7 @@ internal sealed class QuestDbHistorianDataUpdater : IHistorianDataUpdater
|
||||
? await _tags.ExistsAsync(tag, ct).ConfigureAwait(false)
|
||||
: await EnsureAsync(tag, plan.Winners[0].Row.Value.Kind, ct).ConfigureAwait(false);
|
||||
var outcome = known
|
||||
? await ApplyAsync(tag, mode, plan.Winners.Select(w => w.Row).ToList(), ct).ConfigureAwait(false)
|
||||
? await ApplyAsync(tag, mode, plan.Winners.Select(w => w.Row).ToList(), context, ct).ConfigureAwait(false)
|
||||
: new Dictionary<long, UpsertOutcome>();
|
||||
planned = HistoryUpdatePlanner.Complete(plan, mode, outcome);
|
||||
}
|
||||
@@ -109,9 +118,11 @@ internal sealed class QuestDbHistorianDataUpdater : IHistorianDataUpdater
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public async Task<uint> DeleteRawAsync(string tag, DateTime startUtc, DateTime endUtc, CancellationToken ct)
|
||||
public async Task<uint> DeleteRawAsync(string tag, DateTime startUtc, DateTime endUtc, HistoryUpdateContext context,
|
||||
CancellationToken ct)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(tag);
|
||||
ArgumentNullException.ThrowIfNull(context);
|
||||
if (!HistorianTagNames.IsValid(tag)) return HistorianStatusCodes.BadInvalidArgument;
|
||||
if (!await _tags.ExistsAsync(tag, ct).ConfigureAwait(false))
|
||||
return HistoryUpdatePlanner.DeleteRawResult(tagKnown: false, 0);
|
||||
@@ -123,18 +134,29 @@ internal sealed class QuestDbHistorianDataUpdater : IHistorianDataUpdater
|
||||
while (cursor < end)
|
||||
{
|
||||
var page = new List<DateTime>(BatchSize);
|
||||
var priors = new List<LiveRow>(BatchSize);
|
||||
await using (var conn = await _conn.OpenAsync(ct).ConfigureAwait(false))
|
||||
await using (var cmd = _conn.CreateCommand(conn,
|
||||
$"SELECT ts FROM {_conn.SampleTable} WHERE tag = $1 AND ts >= $2 AND ts < $3 AND NOT deleted ORDER BY ts LIMIT {BatchSize}"))
|
||||
$"SELECT {LiveColumns} FROM {_conn.SampleTable} WHERE tag = $1 AND ts >= $2 AND ts < $3 AND NOT deleted ORDER BY ts LIMIT {BatchSize}"))
|
||||
{
|
||||
QuestDbConnection.BindText(cmd, tag);
|
||||
QuestDbConnection.BindTimestamp(cmd, cursor);
|
||||
QuestDbConnection.BindTimestamp(cmd, end);
|
||||
await using var rd = await cmd.ExecuteReaderAsync(ct).ConfigureAwait(false);
|
||||
while (await rd.ReadAsync(ct).ConfigureAwait(false)) page.Add(QuestDbConnection.ReadUtc(rd, 0));
|
||||
while (await rd.ReadAsync(ct).ConfigureAwait(false))
|
||||
{
|
||||
var row = ReadLiveRow(rd);
|
||||
page.Add(row.TimeUtc);
|
||||
priors.Add(row);
|
||||
}
|
||||
}
|
||||
if (page.Count == 0) break;
|
||||
await WriteTombstonesAsync(tag, page, ct).ConfigureAwait(false);
|
||||
// Trail BEFORE the change (no multi-table transaction): a crash here leaves an orphan trail row, never a
|
||||
// tombstone without its trail row.
|
||||
await WriteTrailAsync(tag, HistoryModificationType.Delete,
|
||||
priors.Select(r => (r.TimeUtc, (LiveRow?)r)).ToList(), context, ct).ConfigureAwait(false);
|
||||
await AfterTrailAsync(tag, page[0], page[^1], page.Count,
|
||||
() => WriteTombstonesAsync(tag, page, ct)).ConfigureAwait(false);
|
||||
// The wait between pages is required: without it the next page could re-read rows just tombstoned.
|
||||
await _wal.WaitAsync(_conn.SampleTable, ct).ConfigureAwait(false);
|
||||
total += page.Count;
|
||||
@@ -145,10 +167,12 @@ internal sealed class QuestDbHistorianDataUpdater : IHistorianDataUpdater
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public async Task<IReadOnlyList<uint>> DeleteAtTimeAsync(string tag, IReadOnlyList<DateTime> timesUtc, CancellationToken ct)
|
||||
public async Task<IReadOnlyList<uint>> DeleteAtTimeAsync(string tag, IReadOnlyList<DateTime> timesUtc,
|
||||
HistoryUpdateContext context, CancellationToken ct)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(tag);
|
||||
ArgumentNullException.ThrowIfNull(timesUtc);
|
||||
ArgumentNullException.ThrowIfNull(context);
|
||||
if (!HistorianTagNames.IsValid(tag))
|
||||
{
|
||||
var refused = new uint[timesUtc.Count];
|
||||
@@ -168,9 +192,16 @@ internal sealed class QuestDbHistorianDataUpdater : IHistorianDataUpdater
|
||||
|
||||
using var _ = await LockAsync(tag, ct).ConfigureAwait(false);
|
||||
var live = await ReadLiveAsync(tag, candidates.Select(t => new DateTime(t, DateTimeKind.Utc)).ToList(), ct).ConfigureAwait(false);
|
||||
var toDelete = candidates.Where(live.Contains).OrderBy(x => x).ToList();
|
||||
var toDelete = candidates.Where(live.ContainsKey).OrderBy(x => x).ToList();
|
||||
foreach (var chunk in toDelete.Chunk(BatchSize))
|
||||
await WriteTombstonesAsync(tag, chunk.Select(t => new DateTime(t, DateTimeKind.Utc)).ToList(), ct).ConfigureAwait(false);
|
||||
{
|
||||
// Trail BEFORE the change (see the class remarks).
|
||||
await WriteTrailAsync(tag, HistoryModificationType.Delete,
|
||||
chunk.Select(t => (live[t].TimeUtc, (LiveRow?)live[t])).ToList(), context, ct).ConfigureAwait(false);
|
||||
var chunkTimes = chunk.Select(t => new DateTime(t, DateTimeKind.Utc)).ToList();
|
||||
await AfterTrailAsync(tag, chunkTimes[0], chunkTimes[^1], chunkTimes.Count,
|
||||
() => WriteTombstonesAsync(tag, chunkTimes, ct)).ConfigureAwait(false);
|
||||
}
|
||||
if (toDelete.Count > 0) await _wal.WaitAsync(_conn.SampleTable, ct).ConfigureAwait(false);
|
||||
deleted.UnionWith(toDelete);
|
||||
return HistoryUpdatePlanner.DeleteAtTimeResults(rounded, deleted, tagKnown: true);
|
||||
@@ -183,48 +214,184 @@ internal sealed class QuestDbHistorianDataUpdater : IHistorianDataUpdater
|
||||
}
|
||||
|
||||
private async Task<Dictionary<long, UpsertOutcome>> ApplyAsync(string tag, HistoryUpdateMode mode,
|
||||
List<HistorianSampleRow> rows, CancellationToken ct)
|
||||
List<HistorianSampleRow> rows, HistoryUpdateContext context, CancellationToken ct)
|
||||
{
|
||||
// Key = the planner's tick (what Complete looks up); the stored time is Round(of the same row).
|
||||
var live = await ReadLiveAsync(tag, rows.Select(r => QuestDbConnection.Round(r.TimeUtc)).ToList(), ct).ConfigureAwait(false);
|
||||
var outcome = new Dictionary<long, UpsertOutcome>();
|
||||
var toWrite = new List<HistorianSampleRow>();
|
||||
var trail = new List<(DateTime, LiveRow?)>();
|
||||
foreach (var r in rows)
|
||||
{
|
||||
var present = live.Contains(QuestDbConnection.Round(r.TimeUtc).Ticks);
|
||||
var stored = QuestDbConnection.Round(r.TimeUtc);
|
||||
var present = live.TryGetValue(stored.Ticks, out var prior);
|
||||
if (mode == HistoryUpdateMode.Insert && present) continue;
|
||||
if (mode == HistoryUpdateMode.Replace && !present) continue;
|
||||
outcome[r.TimeUtc.Ticks] = present ? UpsertOutcome.Replaced : UpsertOutcome.Inserted;
|
||||
toWrite.Add(r);
|
||||
trail.Add((stored, present ? prior : null));
|
||||
}
|
||||
// Trail BEFORE the change (see the class remarks).
|
||||
await WriteTrailAsync(tag, ToModificationType(mode), trail, context, ct).ConfigureAwait(false);
|
||||
foreach (var chunk in toWrite.Chunk(BatchSize))
|
||||
await WriteRowsAsync(tag, chunk, ct).ConfigureAwait(false);
|
||||
await AfterTrailAsync(tag, chunk[0].TimeUtc, chunk[^1].TimeUtc, chunk.Length,
|
||||
() => WriteRowsAsync(tag, chunk, ct)).ConfigureAwait(false);
|
||||
if (toWrite.Count > 0) await _wal.WaitAsync(_conn.SampleTable, ct).ConfigureAwait(false);
|
||||
return outcome;
|
||||
}
|
||||
|
||||
/// <summary>Ticks of the live (present, not deleted) rows near the given times: one range query per cluster (I-2).</summary>
|
||||
private async Task<HashSet<long>> ReadLiveAsync(string tag, IReadOnlyList<DateTime> times, CancellationToken ct)
|
||||
/// <summary>The live (present, not deleted) rows near the given times, keyed by stored ticks: one range query per cluster (I-2).</summary>
|
||||
private async Task<Dictionary<long, LiveRow>> ReadLiveAsync(string tag, IReadOnlyList<DateTime> times, CancellationToken ct)
|
||||
{
|
||||
var live = new HashSet<long>();
|
||||
var live = new Dictionary<long, LiveRow>();
|
||||
await using var conn = await _conn.OpenAsync(ct).ConfigureAwait(false);
|
||||
foreach (var (lo, hi) in QuestDbTimeClusters.Cluster(times, QuestDbTimeClusters.ExistenceGap))
|
||||
{
|
||||
await using var cmd = _conn.CreateCommand(conn,
|
||||
$"SELECT ts FROM {_conn.SampleTable} WHERE tag = $1 AND ts >= $2 AND ts <= $3 AND NOT deleted");
|
||||
$"SELECT {LiveColumns} FROM {_conn.SampleTable} WHERE tag = $1 AND ts >= $2 AND ts <= $3 AND NOT deleted");
|
||||
QuestDbConnection.BindText(cmd, tag);
|
||||
QuestDbConnection.BindTimestamp(cmd, lo);
|
||||
QuestDbConnection.BindTimestamp(cmd, hi);
|
||||
await using var rd = await cmd.ExecuteReaderAsync(ct).ConfigureAwait(false);
|
||||
while (await rd.ReadAsync(ct).ConfigureAwait(false))
|
||||
{
|
||||
live.Add(QuestDbConnection.ReadUtc(rd, 0).Ticks);
|
||||
var row = ReadLiveRow(rd);
|
||||
live[row.TimeUtc.Ticks] = row;
|
||||
Interlocked.Increment(ref _liveRowsRead);
|
||||
}
|
||||
}
|
||||
return live;
|
||||
}
|
||||
|
||||
/// <summary>The columns of a live sample the trail needs (prior value + status + server time).</summary>
|
||||
private const string LiveColumns = "ts, kind, num_value, int_value, str_value, status, server_ts";
|
||||
|
||||
/// <summary>A live sample as it was before a modification.</summary>
|
||||
private readonly record struct LiveRow(DateTime TimeUtc, HistorianValue Value, uint Status, DateTime? ServerTimeUtc);
|
||||
|
||||
private static LiveRow ReadLiveRow(NpgsqlDataReader rd)
|
||||
{
|
||||
var value = QuestDbHistorianDataSource.ToValue(
|
||||
QuestDbConnection.ReadKind(rd, 1),
|
||||
rd.IsDBNull(2) ? null : rd.GetDouble(2),
|
||||
rd.IsDBNull(3) ? null : rd.GetInt64(3),
|
||||
rd.IsDBNull(4) ? null : rd.GetString(4));
|
||||
var status = rd.IsDBNull(5) ? HistorianStatusCodes.BadNoData : SampleColumnMapper.FromLongStatus(rd.GetInt64(5));
|
||||
return new LiveRow(QuestDbConnection.ReadUtc(rd, 0), value, status,
|
||||
rd.IsDBNull(6) ? null : QuestDbConnection.ReadUtc(rd, 6));
|
||||
}
|
||||
|
||||
/// <summary>The Part 11 modification type recorded for an update mode.</summary>
|
||||
private static HistoryModificationType ToModificationType(HistoryUpdateMode mode) => mode switch
|
||||
{
|
||||
HistoryUpdateMode.Insert => HistoryModificationType.Insert,
|
||||
HistoryUpdateMode.Replace => HistoryModificationType.Replace,
|
||||
_ => HistoryModificationType.Update,
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// A process-wide, strictly increasing modification id (≈ UTC ticks, bumped past the last issued value). It orders a
|
||||
/// tag's trail within one source time; ids from two nodes interleave by wall clock, which is the modification order.
|
||||
/// </summary>
|
||||
internal static long NextModifiedId()
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
var last = Interlocked.Read(ref _lastModifiedId);
|
||||
var next = Math.Max(DateTime.UtcNow.Ticks, last + 1);
|
||||
if (Interlocked.CompareExchange(ref _lastModifiedId, next, last) == last) return next;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Appends one modification-trail row per entry (<c>prior = null</c> ⇒ an insert: kind Empty, NULL value and
|
||||
/// status), then waits for the trail's WAL visibility so a modified read right after the update sees it.
|
||||
/// </summary>
|
||||
private async Task WriteTrailAsync(string tag, HistoryModificationType type, IReadOnlyList<(DateTime TimeUtc, LiveRow? Prior)> entries,
|
||||
HistoryUpdateContext context, CancellationToken ct)
|
||||
{
|
||||
if (entries.Count == 0) return;
|
||||
var now = DateTime.UtcNow;
|
||||
foreach (var chunk in entries.Chunk(BatchSize))
|
||||
{
|
||||
await using var conn = await _conn.OpenAsync(ct).ConfigureAwait(false);
|
||||
await using var cmd = _conn.CreateCommand(conn, "");
|
||||
var sb = new System.Text.StringBuilder(
|
||||
$"INSERT INTO {_conn.SampleModifiedTable} (ts, tag, modified_id, modification_ts, update_type, principal, " +
|
||||
"api_key_id, kind, num_value, int_value, str_value, status, server_ts) VALUES ");
|
||||
for (var i = 0; i < chunk.Length; i++)
|
||||
{
|
||||
var (time, prior) = chunk[i];
|
||||
QuestDbConnection.BindTimestamp(cmd, time, clampToEpoch: true);
|
||||
QuestDbConnection.BindText(cmd, tag);
|
||||
QuestDbConnection.BindLong(cmd, NextModifiedId());
|
||||
QuestDbConnection.BindTimestamp(cmd, now);
|
||||
QuestDbConnection.BindByte(cmd, (byte)type);
|
||||
QuestDbConnection.BindText(cmd, context.StoredPrincipal);
|
||||
QuestDbConnection.BindText(cmd, context.StoredApiKeyId);
|
||||
if (prior is { } p)
|
||||
{
|
||||
var c = SampleColumnMapper.ToColumns(p.Value);
|
||||
QuestDbConnection.BindByte(cmd, c.ValueKind);
|
||||
Typed(cmd, NpgsqlDbType.Double, c.NumValue);
|
||||
Typed(cmd, NpgsqlDbType.Bigint, c.IntValue);
|
||||
Typed(cmd, NpgsqlDbType.Varchar, c.StrValue);
|
||||
QuestDbConnection.BindLong(cmd, SampleColumnMapper.ToLongStatus(p.Status));
|
||||
Typed(cmd, NpgsqlDbType.TimestampTz, p.ServerTimeUtc);
|
||||
}
|
||||
else
|
||||
{
|
||||
// No prior value: QuestDB BYTE has no NULL, so kind is Empty (0) and the NULL status marks "none".
|
||||
QuestDbConnection.BindByte(cmd, (byte)HistorianValueKind.Empty);
|
||||
Typed(cmd, NpgsqlDbType.Double, null);
|
||||
Typed(cmd, NpgsqlDbType.Bigint, null);
|
||||
Typed(cmd, NpgsqlDbType.Varchar, null);
|
||||
Typed(cmd, NpgsqlDbType.Bigint, null);
|
||||
Typed(cmd, NpgsqlDbType.TimestampTz, null);
|
||||
}
|
||||
|
||||
var b = i * 13;
|
||||
if (i > 0) sb.Append(',');
|
||||
sb.Append('(');
|
||||
for (var q = 1; q <= 13; q++) sb.Append(q > 1 ? "," : "").Append('$').Append(b + q);
|
||||
sb.Append(')');
|
||||
}
|
||||
cmd.CommandText = sb.ToString();
|
||||
try
|
||||
{
|
||||
await cmd.ExecuteNonQueryAsync(ct).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex) when (ex is not OperationCanceledException)
|
||||
{
|
||||
_log.LogWarning(ex, "QuestDB modification-trail INSERT of {Count} rows into {Table} failed; the update was not applied.",
|
||||
chunk.Length, _conn.SampleModifiedTable);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
await _wal.WaitAsync(_conn.SampleModifiedTable, ct).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Runs the sample write that follows an already-written trail batch. QuestDB has no multi-table transaction, so a
|
||||
/// failure here leaves an ORPHAN trail row (a recorded modification that never landed); it is logged with the tag
|
||||
/// and time range so an operator can reconcile it, then rethrown unchanged.
|
||||
/// </summary>
|
||||
private async Task AfterTrailAsync(string tag, DateTime from, DateTime to, int count, Func<Task> write)
|
||||
{
|
||||
try
|
||||
{
|
||||
await write().ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_log.LogWarning(ex,
|
||||
"QuestDB sample write failed AFTER its {Table} trail rows were written; {Count} orphan audit row(s) for tag {Tag} " +
|
||||
"in [{From:O}, {To:O}] describe a modification that did not land.",
|
||||
_conn.SampleModifiedTable, count, tag, from, to);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
private Task WriteRowsAsync(string tag, IReadOnlyList<HistorianSampleRow> rows, CancellationToken ct) =>
|
||||
InsertAsync(rows.Count, ct, (cmd, i) =>
|
||||
{
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common;
|
||||
using System.Collections.Frozen;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
@@ -33,11 +34,16 @@ public sealed class QuestDbHistorianProvider : IHistorianBackendProvider, IAsync
|
||||
|
||||
/// <summary>Creates the provider over the <c>ServerHistorian:QuestDb</c> options (validated elsewhere).</summary>
|
||||
/// <param name="options">The <c>ServerHistorian:QuestDb</c> options.</param>
|
||||
public QuestDbHistorianProvider(QuestDbHistorianOptions options)
|
||||
/// <param name="maxTimeWeightedRawRows">The shared <c>ServerHistorian:MaxTimeWeightedRawRows</c> cap (<= 0 = unbounded).</param>
|
||||
public QuestDbHistorianProvider(QuestDbHistorianOptions options,
|
||||
int maxTimeWeightedRawRows = TimeWeightedAggregator.DefaultMaxWindowRows)
|
||||
{
|
||||
_options = options ?? throw new ArgumentNullException(nameof(options));
|
||||
_maxTimeWeightedRawRows = maxTimeWeightedRawRows;
|
||||
}
|
||||
|
||||
private readonly int _maxTimeWeightedRawRows;
|
||||
|
||||
/// <inheritdoc />
|
||||
public string Name => "QuestDb";
|
||||
|
||||
@@ -69,7 +75,7 @@ public sealed class QuestDbHistorianProvider : IHistorianBackendProvider, IAsync
|
||||
{
|
||||
var s = Get(sp);
|
||||
return new QuestDbHistorianDataSource(s.Conn, s.Bootstrap, s.Catalog,
|
||||
Loggers(sp).CreateLogger<QuestDbHistorianDataSource>());
|
||||
Loggers(sp).CreateLogger<QuestDbHistorianDataSource>(), maxTimeWeightedRawRows: _maxTimeWeightedRawRows);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
|
||||
+54
-3
@@ -1,3 +1,4 @@
|
||||
using System.Globalization;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Npgsql;
|
||||
@@ -6,7 +7,7 @@ using ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common;
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb;
|
||||
|
||||
/// <summary>
|
||||
/// Lazily creates the three tables once per instance (<c>CREATE TABLE IF NOT EXISTS</c>), then reconciles the TTL
|
||||
/// Lazily creates the four tables once per instance (<c>CREATE TABLE IF NOT EXISTS</c>), then reconciles the TTL
|
||||
/// of the sample and event tables with <see cref="QuestDbHistorianOptions.RetentionDays"/>.
|
||||
/// <list type="bullet">
|
||||
/// <item><b>Concurrency:</b> two nodes starting together are safe. If a CREATE still fails with a server error
|
||||
@@ -15,10 +16,17 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb;
|
||||
/// <item><b>TTL (RC3):</b> <c>tables()</c> normalises units (7 days reads back as <c>1 WEEK</c>) so values are compared
|
||||
/// in hours, and an ALTER becomes visible only after the WAL applies it, so the bootstrapper never re-reads to verify
|
||||
/// its own ALTER. A boot whose TTL already matches issues no ALTER.</item>
|
||||
/// <item><b>Tag index (#683):</b> a sample table created before the <c>tag</c> symbol index existed gains it with
|
||||
/// <see cref="QuestDbTables.AddTagIndex"/> (idempotent; a boot that finds it indexed issues no ALTER, and like the TTL
|
||||
/// the ALTER is a WAL transaction the bootstrapper does not re-read). Building it is a one-off WAL apply — ~3 s for
|
||||
/// 68M rows / 63 daily partitions measured — during which writes queue and reads keep scanning. A refused ALTER is
|
||||
/// logged as a Warning, never fails the bootstrap (the index is optional); a failure at WAL-apply time would suspend
|
||||
/// the table unseen, the same blind spot as the TTL ALTER.</item>
|
||||
/// <item><b>Failure:</b> not latched. Within <see cref="BootstrapFailureCache.DefaultDuration"/> the last failure is
|
||||
/// re-thrown without touching QuestDB.</item>
|
||||
/// <item><c>AutoCreateSchema=false</c> only verifies the tables exist (no TTL changes) and throws
|
||||
/// <see cref="HistorianSchemaMissingException"/> otherwise.</item>
|
||||
/// <see cref="HistorianSchemaMissingException"/> otherwise; a missing tag index is only logged (a Warning naming
|
||||
/// the ALTER to run), since reads stay correct without it.</item>
|
||||
/// </list>
|
||||
/// </summary>
|
||||
internal sealed class QuestDbSchemaBootstrapper
|
||||
@@ -63,10 +71,12 @@ internal sealed class QuestDbSchemaBootstrapper
|
||||
{
|
||||
await CreateTablesAsync(conn, ct).ConfigureAwait(false);
|
||||
await ReconcileTtlAsync(conn, ct).ConfigureAwait(false);
|
||||
await EnsureTagIndexAsync(conn, apply: true, ct).ConfigureAwait(false);
|
||||
}
|
||||
else
|
||||
{
|
||||
await VerifyAsync(conn, ct).ConfigureAwait(false);
|
||||
await EnsureTagIndexAsync(conn, apply: false, ct).ConfigureAwait(false);
|
||||
}
|
||||
_ready = true;
|
||||
_failure.Clear();
|
||||
@@ -117,6 +127,44 @@ internal sealed class QuestDbSchemaBootstrapper
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// #683: indexes the sample table's <c>tag</c> column when it is not (a table created before the index existed).
|
||||
/// <paramref name="apply"/> false (AutoCreateSchema off) only logs the statement to run.
|
||||
/// </summary>
|
||||
private async Task EnsureTagIndexAsync(NpgsqlConnection conn, bool apply, CancellationToken ct)
|
||||
{
|
||||
var table = _conn.SampleTable;
|
||||
await using (var cmd = _conn.CreateCommand(conn, QuestDbTables.TagIndexedQuery(table)))
|
||||
{
|
||||
var indexed = await cmd.ExecuteScalarAsync(ct).ConfigureAwait(false);
|
||||
if (indexed is null or DBNull || Convert.ToBoolean(indexed, CultureInfo.InvariantCulture)) return;
|
||||
}
|
||||
if (!apply)
|
||||
{
|
||||
_log.LogWarning("QuestDB historian table {Table} has no index on its tag column, so the data-range lookups of " +
|
||||
"processed reads scan partitions instead of seeking (#683). AutoCreateSchema is false; run: {Statement}",
|
||||
table, QuestDbTables.AddTagIndex(table));
|
||||
return;
|
||||
}
|
||||
// A WAL transaction: table_columns() keeps reporting false until it applies, so do NOT re-read to verify.
|
||||
// The index is optional (reads are correct without it), so a refused ALTER (e.g. no DDL rights) must not fail the
|
||||
// bootstrap: it is logged and retried on the next boot. A failure at WAL-APPLY time is not seen here (QuestDB would
|
||||
// suspend the table) — the same blind spot as the TTL ALTER.
|
||||
try
|
||||
{
|
||||
await using var alter = _conn.CreateCommand(conn, QuestDbTables.AddTagIndex(table));
|
||||
await alter.ExecuteNonQueryAsync(ct).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex) when (ex is not OperationCanceledException || !ct.IsCancellationRequested)
|
||||
{
|
||||
_log.LogWarning("Could not add the tag index to QuestDB table {Table} ({Exception}: {Message}); data-range lookups " +
|
||||
"of processed reads scan partitions instead of seeking (#683). Run: {Statement}",
|
||||
table, ex.GetType().Name, ex.Message, QuestDbTables.AddTagIndex(table));
|
||||
return;
|
||||
}
|
||||
_log.LogInformation("Added the tag index to QuestDB table {Table} (#683).", table);
|
||||
}
|
||||
|
||||
private async Task ReconcileTtlAsync(NpgsqlConnection conn, CancellationToken ct)
|
||||
{
|
||||
var wantHours = _options.RetentionDays is { } d ? d * 24L : 0L;
|
||||
@@ -137,7 +185,10 @@ internal sealed class QuestDbSchemaBootstrapper
|
||||
// The ALTER is a WAL transaction: tables() keeps the old value until it applies, so do NOT re-read to verify.
|
||||
await using (var alter = _conn.CreateCommand(conn, QuestDbTables.AlterTtl(table, _options.RetentionDays)))
|
||||
await alter.ExecuteNonQueryAsync(ct).ConfigureAwait(false);
|
||||
_log.LogInformation("Applied TTL {Days} day(s) to QuestDB table {Table}.", _options.RetentionDays, table);
|
||||
if (_options.RetentionDays is { } days)
|
||||
_log.LogInformation("Applied TTL {Days} day(s) to QuestDB table {Table}.", days, table);
|
||||
else
|
||||
_log.LogInformation("Cleared TTL on QuestDB table {Table}.", table);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -15,6 +15,8 @@ internal static class QuestDbTables
|
||||
public const string EventBase = "event";
|
||||
/// <summary>Base name of the tag catalog table.</summary>
|
||||
public const string TagBase = "tag";
|
||||
/// <summary>Base name of the Part 11 modification-trail table (#668).</summary>
|
||||
public const string SampleModifiedBase = "sample_modified";
|
||||
|
||||
/// <summary>The fixed designated timestamp of every <c>tag</c> row, which makes the dedup key an upsert by name.</summary>
|
||||
public static readonly DateTime TagEpoch = DateTime.UnixEpoch;
|
||||
@@ -25,16 +27,27 @@ internal static class QuestDbTables
|
||||
public static string Event(string prefix) => Name(prefix, EventBase);
|
||||
/// <summary>The prefixed tag table name.</summary>
|
||||
public static string Tag(string prefix) => Name(prefix, TagBase);
|
||||
/// <summary>The prefixed modification-trail table name.</summary>
|
||||
public static string SampleModified(string prefix) => Name(prefix, SampleModifiedBase);
|
||||
|
||||
/// <summary>All three table names in creation order.</summary>
|
||||
public static string[] All(string prefix) => [Sample(prefix), Event(prefix), Tag(prefix)];
|
||||
/// <summary>All four table names in creation order.</summary>
|
||||
public static string[] All(string prefix) => [Sample(prefix), Event(prefix), Tag(prefix), SampleModified(prefix)];
|
||||
|
||||
/// <summary>The tables that carry a TTL (sample and event; the tag catalog never expires).</summary>
|
||||
public static string[] Expiring(string prefix) => [Sample(prefix), Event(prefix)];
|
||||
/// <summary>
|
||||
/// The tables that carry a TTL (sample, event and the modification trail, which ages out with the samples it
|
||||
/// describes; the tag catalog never expires).
|
||||
/// </summary>
|
||||
public static string[] Expiring(string prefix) => [Sample(prefix), Event(prefix), SampleModified(prefix)];
|
||||
|
||||
/// <summary><c>CREATE TABLE IF NOT EXISTS</c> for the sample table.</summary>
|
||||
/// <summary>
|
||||
/// <c>CREATE TABLE IF NOT EXISTS</c> for the sample table. <c>tag</c> carries a symbol (bitmap) index (#683): every
|
||||
/// read filters on one tag, and without the index QuestDB answers a <c>tag = x</c> seek by scanning the tag column of
|
||||
/// every partition in the time range — the data-range lookups (<see cref="QuestDbAggregateQueries.EarliestSample"/> /
|
||||
/// <c>LatestSample</c>) of a stopped or late-starting tag then scan every partition on one side of the window.
|
||||
/// A table created before #683 gains the index through <see cref="AddTagIndex"/>.
|
||||
/// </summary>
|
||||
public static string CreateSample(string prefix, int? retentionDays) =>
|
||||
$"CREATE TABLE IF NOT EXISTS {Sample(prefix)} (ts TIMESTAMP, tag SYMBOL, kind BYTE, num_value DOUBLE, " +
|
||||
$"CREATE TABLE IF NOT EXISTS {Sample(prefix)} (ts TIMESTAMP, tag SYMBOL INDEX, kind BYTE, num_value DOUBLE, " +
|
||||
"int_value LONG, str_value VARCHAR, status LONG, deleted BOOLEAN, server_ts TIMESTAMP) " +
|
||||
$"TIMESTAMP(ts) PARTITION BY DAY{Ttl(retentionDays)} WAL DEDUP UPSERT KEYS(ts, tag)";
|
||||
|
||||
@@ -50,9 +63,37 @@ internal static class QuestDbTables
|
||||
$"CREATE TABLE IF NOT EXISTS {Tag(prefix)} (ts TIMESTAMP, name SYMBOL, data_type BYTE, eng_unit VARCHAR, " +
|
||||
"description VARCHAR, updated_utc TIMESTAMP) TIMESTAMP(ts) PARTITION BY YEAR WAL DEDUP UPSERT KEYS(ts, name)";
|
||||
|
||||
/// <summary>The three CREATE statements in creation order.</summary>
|
||||
/// <summary>
|
||||
/// <c>CREATE TABLE IF NOT EXISTS</c> for the Part 11 modification trail (#668): one row per sample a HistoryUpdate
|
||||
/// inserted, replaced or deleted, carrying the PRIOR value (NULL kind/value/status for an insert). <c>ts</c> is the
|
||||
/// sample's source time (so the TTL ages it out with the samples); <c>modified_id</c> orders the trail within one
|
||||
/// source time. Append-only: no DEDUP, never updated or deleted.
|
||||
/// </summary>
|
||||
public static string CreateSampleModified(string prefix, int? retentionDays) =>
|
||||
$"CREATE TABLE IF NOT EXISTS {SampleModified(prefix)} (ts TIMESTAMP, tag SYMBOL, modified_id LONG, " +
|
||||
"modification_ts TIMESTAMP, update_type BYTE, principal VARCHAR, api_key_id VARCHAR, kind BYTE, num_value DOUBLE, " +
|
||||
"int_value LONG, str_value VARCHAR, status LONG, server_ts TIMESTAMP) " +
|
||||
$"TIMESTAMP(ts) PARTITION BY DAY{Ttl(retentionDays)} WAL";
|
||||
|
||||
/// <summary>The four CREATE statements in creation order (parallel to <see cref="All"/>).</summary>
|
||||
public static string[] CreateAll(string prefix, int? retentionDays) =>
|
||||
[CreateSample(prefix, retentionDays), CreateEvent(prefix, retentionDays), CreateTag(prefix)];
|
||||
[CreateSample(prefix, retentionDays), CreateEvent(prefix, retentionDays), CreateTag(prefix),
|
||||
CreateSampleModified(prefix, retentionDays)];
|
||||
|
||||
/// <summary>
|
||||
/// <c>ALTER TABLE t ALTER COLUMN tag ADD INDEX</c> — the #683 migration of a sample table created without the tag
|
||||
/// index. A WAL transaction (visible once applied); re-issuing it on an indexed column is a harmless no-op (the table
|
||||
/// is not suspended), so two nodes booting together may both send it.
|
||||
/// </summary>
|
||||
public static string AddTagIndex(string table) => $"ALTER TABLE {table} ALTER COLUMN tag ADD INDEX";
|
||||
|
||||
/// <summary>
|
||||
/// Whether <paramref name="table"/>'s <c>tag</c> column is indexed: one row, one BOOLEAN column, no row when the
|
||||
/// table or column is missing. The name is inlined (<c>table_columns()</c> takes a constant); it is always a
|
||||
/// prefix-validated name built here.
|
||||
/// </summary>
|
||||
public static string TagIndexedQuery(string table) =>
|
||||
$"SELECT indexed FROM table_columns('{table}') WHERE \"column\" = 'tag'";
|
||||
|
||||
/// <summary><c>ALTER TABLE t SET TTL n DAYS</c>; <c>null</c> removes the TTL (<c>SET TTL 0 DAYS</c>).</summary>
|
||||
public static string AlterTtl(string table, int? retentionDays) =>
|
||||
|
||||
@@ -7,17 +7,38 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common;
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// Per interval: no samples ⇒ <c>BadNoData</c>. PercentGood / PercentBad ⇒ <c>100 × good|bad / all</c> over
|
||||
/// every sample, status Good. Otherwise no usable sample ⇒ <c>BadNoData</c>; a numeric aggregate with a usable
|
||||
/// String/DateTime sample ⇒ <c>BadAggregateInvalidInputs</c>; else the value, Good when every sample is Good,
|
||||
/// <c>UncertainDataSubNormal</c> when any is Bad or Uncertain. Every status — including <c>BadNoData</c> —
|
||||
/// carries the Calculated info bits. Timestamps are the bucket start, except Minimum/MaximumActualTime
|
||||
/// (the sample's time; ties ⇒ earliest).
|
||||
/// Per interval: no samples ⇒ <c>BadNoData</c> (Count: 0 inside the data range). Otherwise no usable sample ⇒
|
||||
/// <c>BadNoData</c>; a numeric aggregate with a usable String/DateTime sample ⇒ <c>BadAggregateInvalidInputs</c>;
|
||||
/// else the value. #684: Average / Minimum / Maximum / Range / *ActualTime are computed over <b>Good</b> values only
|
||||
/// (Part 13 and the SDK — an Uncertain sample is never part of the value; an interval with no Good value is
|
||||
/// <c>BadNoData</c>). Status: Average value-based (Good only when every sample is Good, else
|
||||
/// <c>UncertainDataSubNormal</c>); Minimum / Maximum / Range / *ActualTime per their Part 13 tables — Uncertain when a
|
||||
/// Bad sample exists or an Uncertain value lies beyond the reported extreme (<see cref="MinMaxStatus"/>).
|
||||
/// Every calculated status — including <c>BadNoData</c> — carries the Calculated info bits. Timestamps are the
|
||||
/// bucket's <see cref="BucketPlan.BucketTimestamp"/> (its start forward, its later edge reverse), except
|
||||
/// Minimum/MaximumActualTime (the sample's time; ties ⇒ earliest) and Start/End. Every aggregate but
|
||||
/// <c>Average</c> also carries the Partial bit on the bucket the data starts (forward) or ends inside
|
||||
/// (<see cref="EvaluateAll"/>). PercentGood / PercentBad are duration-based since #680 and evaluated by
|
||||
/// <see cref="TimeWeightedAggregator"/>, not here.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// #680 (Part 13 / the SDK's calculators): <b>Count</b> counts Good samples only, value-based status (all Good ⇒
|
||||
/// Good, all Bad ⇒ the generic Bad, else Uncertain), and is 0/Good on an empty bucket inside the tag's data range;
|
||||
/// <b>Start / End</b> are the raw first / last sample of any quality (own value — null when Bad —, own status
|
||||
/// without the Calculated bit, own timestamp); <b>Delta</b> is Uncertain only when the bucket's first or last
|
||||
/// sample is not usable.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Inconsistent rows</b> (live-gate 8 F1): a row whose counts say a bucket holds samples while a picked sample
|
||||
/// (first/last of any quality, first/last usable, the extreme) is absent is answered from what is present —
|
||||
/// Start/End/Delta/ActualTime without the sample they need are <c>BadNoData</c> — never an exception. The backends'
|
||||
/// queries read counts and picks in one scan so this should not arise; the evaluator does not rely on it.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Values: Count is <see cref="int"/>; Start/End keep the sample's type; every other aggregate is
|
||||
/// <see cref="double"/> computed over the numeric value (Boolean = 0/1). These are the simple per-sample
|
||||
/// definitions, not Part 13's interval-boundary (interpolated bound / time-weighted) semantics.
|
||||
/// definitions; the Part 13 time-weighted aggregates (TimeAverage, TimeAverage2, Total, Total2, Interpolative)
|
||||
/// are evaluated by <see cref="TimeWeightedAggregator"/>, not here.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public static class AggregateEvaluator
|
||||
@@ -25,50 +46,178 @@ public static class AggregateEvaluator
|
||||
/// <summary>Evaluates <paramref name="aggregate"/> over one non-empty bucket.</summary>
|
||||
/// <param name="aggregate">Requested aggregate.</param>
|
||||
/// <param name="r">The bucket's statistics.</param>
|
||||
/// <param name="bucketStart">The bucket's start time (the result timestamp).</param>
|
||||
/// <param name="bucketStart">The bucket's result timestamp (<see cref="BucketPlan.BucketTimestamp"/>).</param>
|
||||
/// <param name="now">Server timestamp.</param>
|
||||
/// <returns>The calculated value.</returns>
|
||||
/// <exception cref="HistoryAggregateNotSupportedException">The aggregate is not one of the thirteen served.</exception>
|
||||
/// <exception cref="HistoryAggregateNotSupportedException">The aggregate is not one of the per-sample (Simple /
|
||||
/// StartEnd / ActualTime family) aggregates.</exception>
|
||||
public static DataValueSnapshot Evaluate(HistoryAggregateType aggregate, BucketRow r, DateTime bucketStart, DateTime now)
|
||||
{
|
||||
DataValueSnapshot Calc(object? value, uint status, DateTime? t = null) =>
|
||||
new(value, HistorianInfoBits.WithCalculated(status), t ?? bucketStart, now);
|
||||
|
||||
// Refuse a non-per-sample aggregate up front, whatever the bucket holds (an all-Bad bucket would otherwise answer
|
||||
// BadNoData before the switch's default is reached).
|
||||
if (AggregateFamilies.FamilyOf(aggregate) is not (AggregateFamily.Simple or AggregateFamily.StartEnd or AggregateFamily.ActualTime))
|
||||
throw new HistoryAggregateNotSupportedException(aggregate);
|
||||
if (r.Total == 0) return NoData(bucketStart, now);
|
||||
switch (aggregate)
|
||||
{
|
||||
case HistoryAggregateType.PercentGood: return Calc(100d * r.Good / r.Total, HistorianStatusCodes.Good);
|
||||
case HistoryAggregateType.PercentBad: return Calc(100d * r.Bad / r.Total, HistorianStatusCodes.Good);
|
||||
// #680 (Part 13 Tables 73/74; SDK ComputeStartEnd): the raw first / last sample of ANY quality — its own value
|
||||
// (null when Bad), its own status (Raw: no Calculated bit) and its own timestamp.
|
||||
case HistoryAggregateType.Start: return Raw(r.FirstAny, bucketStart, now);
|
||||
case HistoryAggregateType.End: return Raw(r.LastAny, bucketStart, now);
|
||||
// #680 (Part 13 §5.4.3.21; SDK ComputeCount): Good samples only; value-based status — Good when every sample
|
||||
// is Good, the generic Bad when every one is Bad (value null), else Uncertain.
|
||||
case HistoryAggregateType.Count:
|
||||
if (r.Bad == r.Total) return Calc(null, HistorianStatusCodes.Bad);
|
||||
return Calc(r.Good, r.Good == r.Total ? HistorianStatusCodes.Good : HistorianStatusCodes.UncertainDataSubNormal);
|
||||
}
|
||||
if (r.Usable == 0) return NoData(bucketStart, now);
|
||||
|
||||
var anyText = aggregate is HistoryAggregateType.Count or HistoryAggregateType.Start or HistoryAggregateType.End;
|
||||
if (!anyText && r.NonNumeric > 0)
|
||||
// #684 review: Delta reads usable picks, so any usable String/DateTime sample refuses it. The Good-value aggregates
|
||||
// refuse only a GOOD non-numeric sample — an Uncertain one is not part of their value set (it only degrades the
|
||||
// status, as the SDK's skipped/uncastable value does).
|
||||
if ((aggregate == HistoryAggregateType.Delta ? r.NonNumeric : r.GoodNonNumericOrAll) > 0)
|
||||
return Calc(null, HistorianStatusCodes.BadAggregateInvalidInputs);
|
||||
|
||||
var status = r.Good == r.Total ? HistorianStatusCodes.Good : HistorianStatusCodes.UncertainDataSubNormal;
|
||||
switch (aggregate)
|
||||
{
|
||||
case HistoryAggregateType.Count: return Calc(r.Usable, status);
|
||||
case HistoryAggregateType.Average: return Calc(r.Avg, status);
|
||||
case HistoryAggregateType.Minimum: return Calc(r.Min, status);
|
||||
case HistoryAggregateType.Maximum: return Calc(r.Max, status);
|
||||
case HistoryAggregateType.Total: return Calc(r.Sum, status);
|
||||
case HistoryAggregateType.Range: return Calc(r.Max - r.Min, status);
|
||||
case HistoryAggregateType.Start: return Calc(r.First.ToObject(), status);
|
||||
case HistoryAggregateType.End: return Calc(r.Last.ToObject(), status);
|
||||
// #684 (Part 13 §5.4.3.5; SDK ComputeAverage): the mean of the GOOD values — an interval with none (only
|
||||
// Uncertain / Bad) is BadNoData; value-based status: Good only when every sample is Good, else Uncertain.
|
||||
case HistoryAggregateType.Average:
|
||||
if (r.Avg is null) return NoData(bucketStart, now);
|
||||
return Calc(r.Avg, r.Good == r.Total ? HistorianStatusCodes.Good : HistorianStatusCodes.UncertainDataSubNormal);
|
||||
// #684 (Part 13 §5.4.3.10/11/14; SDK ComputeMinMax): the extreme(s) of the GOOD values; Uncertain when a Bad
|
||||
// sample exists or an Uncertain value lies beyond the reported extreme (see MinMaxStatus).
|
||||
case HistoryAggregateType.Minimum:
|
||||
if (r.Min is null) return NoData(bucketStart, now);
|
||||
return Calc(r.Min, MinMaxStatus(r, r.Min, r.Max, low: true, high: false));
|
||||
case HistoryAggregateType.Maximum:
|
||||
if (r.Max is null) return NoData(bucketStart, now);
|
||||
return Calc(r.Max, MinMaxStatus(r, r.Min, r.Max, low: false, high: true));
|
||||
case HistoryAggregateType.Range:
|
||||
if (r.Min is null || r.Max is null) return NoData(bucketStart, now);
|
||||
return Calc(r.Max - r.Min, MinMaxStatus(r, r.Min, r.Max, low: true, high: true));
|
||||
case HistoryAggregateType.Delta:
|
||||
return r.First.TryGetDouble(out var a) && r.Last.TryGetDouble(out var b)
|
||||
? Calc(b - a, status)
|
||||
: Calc(null, HistorianStatusCodes.BadAggregateInvalidInputs);
|
||||
// #680 (SDK ComputeDelta): last usable − first usable; Uncertain only when the interval's first or last
|
||||
// sample was skipped (not usable) — a Bad sample in the middle does not degrade it.
|
||||
// Live-gate 8 F1: a row whose counts promise a usable sample its picks do not carry (a backend that read
|
||||
// the counts and the picks from two table versions) has no consistent first/last — no data, not a throw
|
||||
// and not a wrong-input verdict.
|
||||
if (r.First.Kind == HistorianValueKind.Empty || r.Last.Kind == HistorianValueKind.Empty)
|
||||
return NoData(bucketStart, now);
|
||||
if (!r.First.TryGetDouble(out var a) || !r.Last.TryGetDouble(out var b))
|
||||
return Calc(null, HistorianStatusCodes.BadAggregateInvalidInputs);
|
||||
var bounded = r.FirstAny is { IsUsable: true } && r.LastAny is { IsUsable: true };
|
||||
return Calc(b - a, bounded ? HistorianStatusCodes.Good : HistorianStatusCodes.UncertainDataSubNormal);
|
||||
case HistoryAggregateType.MinimumActualTime:
|
||||
case HistoryAggregateType.MaximumActualTime:
|
||||
return Calc(r.Extreme, status, r.ExtremeTime);
|
||||
// Live-gate 8 F1: usable numeric samples counted but no extreme picked (an inconsistent row) is no data,
|
||||
// never a value-less Good.
|
||||
if (r.Extreme is null || r.ExtremeTime is null) return NoData(bucketStart, now);
|
||||
var min = aggregate == HistoryAggregateType.MinimumActualTime;
|
||||
return Calc(r.Extreme, MinMaxStatus(r, r.Extreme, r.Extreme, low: min, high: !min), r.ExtremeTime);
|
||||
default:
|
||||
throw new HistoryAggregateNotSupportedException(aggregate);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// #684 — the Part 13 status of Minimum / Maximum / Range / *ActualTime (Tables 57–62): <c>Uncertain_DataSubNormal</c>
|
||||
/// when a Bad sample exists in the interval, or when an Uncertain value is less than the minimum Good value
|
||||
/// (<paramref name="low"/>) / greater than the maximum Good value (<paramref name="high"/>); else Good. An Uncertain
|
||||
/// value inside the Good range leaves it Good. An Uncertain String/DateTime sample (no order against the Good values)
|
||||
/// also makes it Uncertain — the SDK's <c>ComputeMinMax</c> flags a value it cannot cast exactly that way.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The SDK's <c>ComputeMinMax</c> (1.5.378) applies only the Bad half: it collects the Uncertain extremes and never
|
||||
/// reads them, so it answers Good where Part 13 says Uncertain. Ours follows the specification there — the more
|
||||
/// conservative of the two, and never a status better than the inputs warrant. The comparison is strict, so an
|
||||
/// Uncertain value EQUAL to the Good extreme does not degrade it.
|
||||
/// </remarks>
|
||||
private static uint MinMaxStatus(BucketRow r, double? goodMin, double? goodMax, bool low, bool high) =>
|
||||
r.Bad > 0 || r.NonNumeric > r.GoodNonNumericOrAll
|
||||
|| (low && r.UncertainMin < goodMin) || (high && r.UncertainMax > goodMax)
|
||||
? HistorianStatusCodes.UncertainDataSubNormal
|
||||
: HistorianStatusCodes.Good;
|
||||
|
||||
/// <summary>
|
||||
/// True for the per-sample aggregates the SDK sets the start-of-data Partial bit on (every calculator but
|
||||
/// <c>AverageAggregateCalculator</c>'s plain <c>Average</c>, which turns <c>SetPartialBit</c> off).
|
||||
/// </summary>
|
||||
/// <param name="aggregate">Requested aggregate.</param>
|
||||
/// <returns>Whether the start-of-data Partial rule applies.</returns>
|
||||
public static bool SetsStartOfDataPartial(HistoryAggregateType aggregate) => aggregate != HistoryAggregateType.Average;
|
||||
|
||||
/// <summary>
|
||||
/// True when a per-sample read must fetch the tag's data range — its earliest and latest stored samples of any
|
||||
/// quality — because the answer can change a result: the start/end-of-data Partial rules for an aggregate other
|
||||
/// than <c>Average</c> (either direction: the end-of-data rule applies to reverse reads too), and Count (an empty
|
||||
/// interval inside the range is 0, outside it <c>BadNoData</c>, #680). Only a per-sample <c>Average</c> read skips
|
||||
/// both queries.
|
||||
/// </summary>
|
||||
/// <param name="aggregate">Requested aggregate.</param>
|
||||
/// <param name="plan">The bucket plan.</param>
|
||||
/// <returns>Whether the earliest and latest samples are needed.</returns>
|
||||
public static bool NeedsDataRange(HistoryAggregateType aggregate, BucketPlan plan)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(plan);
|
||||
return plan.BucketCount > 0 && (aggregate == HistoryAggregateType.Count || SetsStartOfDataPartial(aggregate));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Evaluates every non-empty bucket row and assembles one result per interval in the plan's order (descending for a
|
||||
/// reverse read): each value stamped with <see cref="BucketPlan.BucketTimestamp"/> (the earlier edge forward, the
|
||||
/// later edge reverse) — Start/End/ActualTime with their sample's own time; empty buckets <c>BadNoData</c>, except a
|
||||
/// Count inside the tag's data range (0, Good — #680); and, for every aggregate but <c>Average</c>, the Partial bit
|
||||
/// on a non-Bad result of the bucket the tag's data starts strictly inside (forward only — #677, the SDK's
|
||||
/// <c>m_startOfData</c> rule) or ends inside (#680 — <see cref="BucketPlan.DataEndsInside"/>: forward the SDK's
|
||||
/// <c>m_endOfData</c>, earlier edge included; reverse strictly inside, the SDK's <c>m_startOfData</c> keyed on the
|
||||
/// first value a reverse read is fed — the latest). A Bad result never carries Partial.
|
||||
/// </summary>
|
||||
/// <param name="aggregate">A per-sample (Simple / StartEnd / ActualTime) aggregate.</param>
|
||||
/// <param name="plan">The bucket plan.</param>
|
||||
/// <param name="rows">The backend's non-empty bucket rows (ascending bucket index).</param>
|
||||
/// <param name="earliestSampleUtc">The tag's earliest stored sample of any quality, or null when none.</param>
|
||||
/// <param name="latestSampleUtc">The tag's latest stored sample of any quality, or null when none.</param>
|
||||
/// <param name="now">Server timestamp.</param>
|
||||
/// <returns><see cref="BucketPlan.BucketCount"/> results.</returns>
|
||||
public static IReadOnlyList<DataValueSnapshot> EvaluateAll(HistoryAggregateType aggregate, BucketPlan plan,
|
||||
IEnumerable<BucketRow> rows, DateTime? earliestSampleUtc, DateTime? latestSampleUtc, DateTime now)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(plan);
|
||||
ArgumentNullException.ThrowIfNull(rows);
|
||||
var evaluated = new Dictionary<long, DataValueSnapshot>();
|
||||
var startRule = !plan.Reversed && SetsStartOfDataPartial(aggregate);
|
||||
var endRule = SetsStartOfDataPartial(aggregate);
|
||||
foreach (var row in rows)
|
||||
{
|
||||
var v = Evaluate(aggregate, row, plan.BucketTimestamp(row.Bucket), now);
|
||||
if (!StatusBits.IsBad(v.StatusCode)
|
||||
&& ((startRule && plan.DataStartsInside(row.Bucket, earliestSampleUtc))
|
||||
|| (endRule && plan.DataEndsInside(row.Bucket, latestSampleUtc))))
|
||||
// A raw Start/End sample gains the DataValue InfoType with its first aggregate bit (as the SDK's
|
||||
// SetAggregateBits does); a calculated status already carries it.
|
||||
v = v with { StatusCode = v.StatusCode | HistorianInfoBits.InfoTypeDataValue | HistorianInfoBits.HistorianPartial };
|
||||
evaluated[row.Bucket] = v;
|
||||
}
|
||||
if (aggregate == HistoryAggregateType.Count)
|
||||
{
|
||||
// #680 (Part 13 §5.4.3.21; SDK GetValues): an empty interval inside [earliest, latest] is a Count of 0, Good.
|
||||
for (long i = 0; i < plan.BucketCount; i++)
|
||||
if (!evaluated.ContainsKey(i) && plan.WithinDataRange(i, earliestSampleUtc, latestSampleUtc))
|
||||
evaluated[i] = new DataValueSnapshot(0, HistorianInfoBits.WithCalculated(HistorianStatusCodes.Good),
|
||||
plan.BucketTimestamp(i), now);
|
||||
}
|
||||
return plan.Assemble(evaluated, t => NoData(t, now));
|
||||
}
|
||||
|
||||
/// <summary>A raw stored sample as an aggregate result (Start / End): value null when Bad, its own status and time.</summary>
|
||||
private static DataValueSnapshot Raw(StoredSample? sample, DateTime bucketStart, DateTime now) =>
|
||||
sample is not { } s
|
||||
? NoData(bucketStart, now)
|
||||
: new(StatusBits.IsBad(s.Status) ? null : s.Value.ToObject(), s.Status, s.TimeUtc, now);
|
||||
|
||||
/// <summary>The result for a bucket with no samples (or no usable sample): <c>BadNoData</c> + Calculated.</summary>
|
||||
/// <param name="t">The bucket's start time.</param>
|
||||
/// <param name="now">Server timestamp.</param>
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
using System.Collections.Frozen;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common;
|
||||
@@ -5,7 +6,7 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common;
|
||||
/// <summary>The query shape that serves an aggregate. One query per family.</summary>
|
||||
public enum AggregateFamily
|
||||
{
|
||||
/// <summary>Average / Minimum / Maximum / Total / Count / Range / PercentGood / PercentBad: one GROUP BY.</summary>
|
||||
/// <summary>Average / Minimum / Maximum / Count / Range: one GROUP BY.</summary>
|
||||
Simple,
|
||||
|
||||
/// <summary>Start / End / Delta: first + last usable sample per bucket.</summary>
|
||||
@@ -13,6 +14,14 @@ public enum AggregateFamily
|
||||
|
||||
/// <summary>MinimumActualTime / MaximumActualTime: extreme usable sample per bucket, ties ⇒ earliest.</summary>
|
||||
ActualTime,
|
||||
|
||||
/// <summary>
|
||||
/// TimeAverage / TimeAverage2 / Total / Total2 / Interpolative, and the duration-based PercentGood / PercentBad
|
||||
/// (#680): Part 13 time-weighted, with bounding values.
|
||||
/// The backend streams the window's samples plus the bounding rows and <see cref="TimeWeightedAggregator"/>
|
||||
/// evaluates them (identically for every backend).
|
||||
/// </summary>
|
||||
TimeWeighted,
|
||||
}
|
||||
|
||||
/// <summary>Which aggregates the SQL historian backends serve, and by which family.</summary>
|
||||
@@ -24,10 +33,20 @@ public static class AggregateFamilies
|
||||
public static AggregateFamily? FamilyOf(HistoryAggregateType aggregate) => aggregate switch
|
||||
{
|
||||
HistoryAggregateType.Average or HistoryAggregateType.Minimum or HistoryAggregateType.Maximum
|
||||
or HistoryAggregateType.Total or HistoryAggregateType.Count or HistoryAggregateType.Range
|
||||
or HistoryAggregateType.PercentGood or HistoryAggregateType.PercentBad => AggregateFamily.Simple,
|
||||
or HistoryAggregateType.Count or HistoryAggregateType.Range => AggregateFamily.Simple,
|
||||
HistoryAggregateType.Start or HistoryAggregateType.End or HistoryAggregateType.Delta => AggregateFamily.StartEnd,
|
||||
HistoryAggregateType.MinimumActualTime or HistoryAggregateType.MaximumActualTime => AggregateFamily.ActualTime,
|
||||
HistoryAggregateType.TimeAverage or HistoryAggregateType.TimeAverage2 or HistoryAggregateType.Total
|
||||
or HistoryAggregateType.Total2 or HistoryAggregateType.Interpolative
|
||||
or HistoryAggregateType.PercentGood or HistoryAggregateType.PercentBad => AggregateFamily.TimeWeighted,
|
||||
_ => null,
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// The seventeen aggregates the SQL backends serve — every member <see cref="FamilyOf"/> assigns a family, derived
|
||||
/// from it so the set and the dispatch can never disagree. Both <c>SqlServerHistorianDataSource</c> and
|
||||
/// <c>QuestDbHistorianDataSource</c> return it as <see cref="IHistorianDataSource.SupportedAggregates"/> (#682).
|
||||
/// </summary>
|
||||
public static readonly IReadOnlySet<HistoryAggregateType> Served =
|
||||
Enum.GetValues<HistoryAggregateType>().Where(a => FamilyOf(a) is not null).ToFrozenSet();
|
||||
}
|
||||
|
||||
@@ -49,8 +49,10 @@ public static class AtTimeEvaluator
|
||||
{
|
||||
var fraction = (double)(t.Ticks - p.TimeUtc.Ticks) / (n.TimeUtc.Ticks - p.TimeUtc.Ticks);
|
||||
var v = p.Value.Number + (nv - p.Value.Number) * fraction;
|
||||
// Each arm boxed on its own: `cond ? (long)x : v` is a DOUBLE expression in C# (the long widens), which returned
|
||||
// every interpolated Int64 as a Double until #693.
|
||||
object value = p.Value.Kind == HistorianValueKind.Int64
|
||||
? (long)Math.Round(v, MidpointRounding.AwayFromZero)
|
||||
? (object)(long)Math.Round(v, MidpointRounding.AwayFromZero)
|
||||
: v;
|
||||
var status = p.IsGood && n.IsGood ? HistorianStatusCodes.Good : HistorianStatusCodes.UncertainDataSubNormal;
|
||||
return new DataValueSnapshot(value, HistorianInfoBits.WithInterpolated(status), t, serverTime);
|
||||
|
||||
@@ -3,12 +3,29 @@ using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common;
|
||||
|
||||
/// <summary>
|
||||
/// The bucket layout of a processed (aggregate) read. Bucket <c>i</c> = <c>[start + i·interval,
|
||||
/// start + (i+1)·interval)</c> (start-inclusive, end-exclusive to the tick; the last bucket is truncated at
|
||||
/// <c>end</c>). An interval <= 0 ⇒ one interval <c>[start, end)</c>. <c>start == end</c> ⇒ no intervals.
|
||||
/// <c>end < start</c> (reverse) ⇒ the window is swapped, computed forward, and assembled in descending bucket
|
||||
/// order (buckets stay anchored at the earlier time).
|
||||
/// The bucket layout of a processed (aggregate) read, per OPC UA Part 13 / the SDK's <c>AggregateCalculator</c>.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>Forward</b> (<c>start < end</c>): the window is <c>[start, end)</c>; bucket <c>i</c> =
|
||||
/// <c>[start + i·interval, start + (i+1)·interval)</c>, the last bucket truncated at <c>end</c> (the partial one is
|
||||
/// the <i>latest</i>); each bucket's timestamp is its start (earlier edge); results ascend.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Reverse</b> (<c>end < start</c>, #677): the read is anchored at the request's start — its <i>later</i>
|
||||
/// time — and steps back. The window is <c>(end, start]</c> (Part 11: the request's start is inclusive, its end
|
||||
/// exclusive, so a reverse read includes its later time and excludes its earlier one — the same window as a reverse
|
||||
/// raw read); result <c>k</c> = <c>(start − (k+1)·interval, start − k·interval]</c>, clipped at <c>end</c> (the
|
||||
/// partial bucket is the <i>earliest</i>); each bucket's timestamp is its later edge <c>start − k·interval</c>;
|
||||
/// results descend. This is exactly the SDK's <c>CreateSlice</c> / <c>GetTimestamp</c> / backward
|
||||
/// <c>GetValues</c>.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Internally buckets are indexed in ascending time (<c>0</c> = earliest) for both directions; <see cref="Assemble"/>
|
||||
/// reverses the list for a reverse read. <see cref="StartUtc"/>/<see cref="EndUtc"/> are always the earlier / later
|
||||
/// window edges. An interval <= 0 ⇒ one interval over the whole window. <c>start == end</c> ⇒ no intervals.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public sealed class BucketPlan
|
||||
{
|
||||
/// <summary>Spec §9.7: a processed read covering more intervals than this is refused.</summary>
|
||||
@@ -21,12 +38,15 @@ public sealed class BucketPlan
|
||||
Reversed = reversed;
|
||||
IntervalTicks = intervalTicks;
|
||||
BucketCount = bucketCount;
|
||||
// Forward buckets are anchored at the earlier edge; reverse ones at the later edge, so the grid origin sits a
|
||||
// whole number of intervals before it (possibly before the window start — the earliest bucket is clipped).
|
||||
OriginTicks = reversed ? endUtc.Ticks - bucketCount * intervalTicks : startUtc.Ticks;
|
||||
}
|
||||
|
||||
/// <summary>The earlier end of the window (the forward start).</summary>
|
||||
/// <summary>The earlier edge of the window.</summary>
|
||||
public DateTime StartUtc { get; }
|
||||
|
||||
/// <summary>The later end of the window (the forward, exclusive end).</summary>
|
||||
/// <summary>The later edge of the window.</summary>
|
||||
public DateTime EndUtc { get; }
|
||||
|
||||
/// <summary>True when the request came in reversed (<c>end < start</c>).</summary>
|
||||
@@ -38,9 +58,30 @@ public sealed class BucketPlan
|
||||
/// <summary>Number of buckets (0 for an empty window).</summary>
|
||||
public long BucketCount { get; }
|
||||
|
||||
/// <summary>
|
||||
/// The tick at which the (unclipped) bucket grid starts: <see cref="StartUtc"/> forward, <c>EndUtc − n·interval</c>
|
||||
/// reverse. May precede <see cref="StartUtc"/> (and even <see cref="DateTime.MinValue"/>), so it is a raw tick count.
|
||||
/// </summary>
|
||||
public long OriginTicks { get; }
|
||||
|
||||
/// <summary><see cref="StartUtc"/> as 100 ns ticks since the Unix epoch.</summary>
|
||||
public long StartUnixTicks => (StartUtc - DateTime.UnixEpoch).Ticks;
|
||||
|
||||
/// <summary>
|
||||
/// The constant a backend adds to <c>elapsed = t − StartUtc</c> (ticks) before the integer division by
|
||||
/// <see cref="IntervalTicks"/> to get a sample's ascending bucket index: <c>0</c> forward (<c>[lo, hi)</c> buckets);
|
||||
/// <c>(StartUtc − OriginTicks) − 1</c> reverse, where buckets are <c>(lo, hi]</c> — index <c>⌈(t − origin)/I⌉ − 1
|
||||
/// = (t − origin − 1) / I</c>. Never makes the dividend negative for an in-window sample (reverse excludes
|
||||
/// <c>t == StartUtc</c>).
|
||||
/// </summary>
|
||||
public long BucketIndexAdjustTicks => Reversed ? StartUtc.Ticks - OriginTicks - 1 : 0;
|
||||
|
||||
/// <summary>
|
||||
/// <see cref="StartUnixTicks"/> minus <see cref="BucketIndexAdjustTicks"/>: the base a backend subtracts from a
|
||||
/// sample's Unix-epoch ticks to get the same dividend (<c>(t − base) / IntervalTicks</c>).
|
||||
/// </summary>
|
||||
public long BucketBaseUnixTicks => StartUnixTicks - BucketIndexAdjustTicks;
|
||||
|
||||
/// <summary>Plans the buckets of a processed read.</summary>
|
||||
/// <param name="startUtc">Requested start.</param>
|
||||
/// <param name="endUtc">Requested end (earlier than start for a reverse read).</param>
|
||||
@@ -64,15 +105,86 @@ public sealed class BucketPlan
|
||||
return new BucketPlan(start, end, reverse, intervalTicks, bucketCount);
|
||||
}
|
||||
|
||||
/// <summary>The start time of bucket <paramref name="index"/> (UTC kind).</summary>
|
||||
/// <param name="index">Zero-based bucket index.</param>
|
||||
/// <returns>The bucket start.</returns>
|
||||
public DateTime BucketStart(long index) =>
|
||||
DateTime.SpecifyKind(new DateTime(StartUtc.Ticks + index * IntervalTicks), DateTimeKind.Utc);
|
||||
/// <summary>The earlier edge of bucket <paramref name="index"/> (ascending index), in ticks, clipped to the window.</summary>
|
||||
/// <param name="index">Zero-based ascending bucket index.</param>
|
||||
/// <returns>The earlier edge.</returns>
|
||||
public long LowerTicks(long index) => Math.Max(OriginTicks + index * IntervalTicks, StartUtc.Ticks);
|
||||
|
||||
/// <summary>One snapshot per bucket in window order; missing buckets get <paramref name="empty"/>; reversed if the window was.</summary>
|
||||
/// <param name="byBucket">Evaluated snapshots of the non-empty buckets, by bucket index.</param>
|
||||
/// <param name="empty">Builds the snapshot of a bucket with no row, from its start time.</param>
|
||||
/// <summary>The later edge of bucket <paramref name="index"/> (ascending index), in ticks, clipped to the window.</summary>
|
||||
/// <param name="index">Zero-based ascending bucket index.</param>
|
||||
/// <returns>The later edge.</returns>
|
||||
public long UpperTicks(long index) => Math.Min(OriginTicks + (index + 1) * IntervalTicks, EndUtc.Ticks);
|
||||
|
||||
/// <summary>
|
||||
/// The result timestamp of bucket <paramref name="index"/> (UTC kind): its earlier edge forward, its later edge
|
||||
/// reverse (the SDK's <c>GetTimestamp</c>).
|
||||
/// </summary>
|
||||
/// <param name="index">Zero-based ascending bucket index.</param>
|
||||
/// <returns>The bucket's timestamp.</returns>
|
||||
public DateTime BucketTimestamp(long index) =>
|
||||
new(Reversed ? UpperTicks(index) : LowerTicks(index), DateTimeKind.Utc);
|
||||
|
||||
/// <summary>True when bucket <paramref name="index"/> is shorter than the interval (the clipped one).</summary>
|
||||
/// <param name="index">Zero-based ascending bucket index.</param>
|
||||
/// <returns>Whether the bucket is truncated.</returns>
|
||||
public bool IsTruncated(long index) => UpperTicks(index) - LowerTicks(index) < IntervalTicks;
|
||||
|
||||
/// <summary>
|
||||
/// The ascending bucket index holding a sample at <paramref name="t"/>, or null when it is outside the window
|
||||
/// (<c>[start, end)</c> forward, <c>(start, end]</c> reverse). The C# statement of the bucket math every backend's
|
||||
/// SQL mirrors.
|
||||
/// </summary>
|
||||
/// <param name="t">Sample time.</param>
|
||||
/// <returns>The bucket index, or null.</returns>
|
||||
public long? IndexOf(DateTime t)
|
||||
{
|
||||
var x = t.Ticks;
|
||||
if (BucketCount == 0) return null;
|
||||
var inside = Reversed ? x > StartUtc.Ticks && x <= EndUtc.Ticks : x >= StartUtc.Ticks && x < EndUtc.Ticks;
|
||||
if (!inside) return null;
|
||||
return (x - StartUtc.Ticks + BucketIndexAdjustTicks) / IntervalTicks;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// True when the earliest stored sample of the tag lies strictly inside bucket <paramref name="index"/> — the data
|
||||
/// starts part-way through the interval (no bounding value before it), so Part 13 / the SDK set the Partial bit.
|
||||
/// </summary>
|
||||
/// <param name="index">Zero-based ascending bucket index.</param>
|
||||
/// <param name="earliestSampleUtc">The tag's earliest stored sample (any quality), or null when it has none.</param>
|
||||
/// <returns>Whether the data starts inside the bucket.</returns>
|
||||
public bool DataStartsInside(long index, DateTime? earliestSampleUtc) =>
|
||||
earliestSampleUtc is { } e && e.Ticks > LowerTicks(index) && e.Ticks < UpperTicks(index);
|
||||
|
||||
/// <summary>
|
||||
/// True when the latest stored sample of the tag lies in bucket <paramref name="index"/>: the data ends part-way
|
||||
/// through the interval, so Part 13 §5.3.3.2 / the SDK set the Partial bit (#680). Forward the earlier edge is
|
||||
/// included (the SDK's <c>m_endOfData >= startTime && m_endOfData < endTime</c>); reverse the sample
|
||||
/// must lie strictly inside (the SDK's reverse read keys <c>m_startOfData</c> — a strict test — on the first value it
|
||||
/// is fed, the latest).
|
||||
/// </summary>
|
||||
/// <param name="index">Zero-based ascending bucket index.</param>
|
||||
/// <param name="latestSampleUtc">The tag's latest stored sample (any quality), or null when it has none.</param>
|
||||
/// <returns>Whether the data ends inside the bucket.</returns>
|
||||
public bool DataEndsInside(long index, DateTime? latestSampleUtc) =>
|
||||
latestSampleUtc is { } l
|
||||
&& (Reversed ? l.Ticks > LowerTicks(index) : l.Ticks >= LowerTicks(index)) && l.Ticks < UpperTicks(index);
|
||||
|
||||
/// <summary>
|
||||
/// True when bucket <paramref name="index"/> overlaps the tag's data range <c>[earliest, latest]</c> — neither
|
||||
/// wholly before the first sample nor wholly after the last (the SDK's <c>GetValues</c> null test:
|
||||
/// <c>start > last || end < first</c> ⇒ no data, both edges inclusive in either direction). An empty bucket
|
||||
/// inside the range is a Count of 0, not <c>BadNoData</c> (#680, Part 13 §5.4.3.21).
|
||||
/// </summary>
|
||||
/// <param name="index">Zero-based ascending bucket index.</param>
|
||||
/// <param name="earliestSampleUtc">The tag's earliest stored sample (any quality), or null.</param>
|
||||
/// <param name="latestSampleUtc">The tag's latest stored sample (any quality), or null.</param>
|
||||
/// <returns>Whether the bucket is within the data range.</returns>
|
||||
public bool WithinDataRange(long index, DateTime? earliestSampleUtc, DateTime? latestSampleUtc) =>
|
||||
earliestSampleUtc is { } e && latestSampleUtc is { } l && LowerTicks(index) <= l.Ticks && UpperTicks(index) >= e.Ticks;
|
||||
|
||||
/// <summary>One snapshot per bucket in result order; missing buckets get <paramref name="empty"/>; descending if reversed.</summary>
|
||||
/// <param name="byBucket">Evaluated snapshots of the non-empty buckets, by ascending bucket index.</param>
|
||||
/// <param name="empty">Builds the snapshot of a bucket with no row, from its <see cref="BucketTimestamp"/>.</param>
|
||||
/// <returns><see cref="BucketCount"/> snapshots.</returns>
|
||||
public IReadOnlyList<DataValueSnapshot> Assemble(IReadOnlyDictionary<long, DataValueSnapshot> byBucket,
|
||||
Func<DateTime, DataValueSnapshot> empty)
|
||||
@@ -81,10 +193,7 @@ public sealed class BucketPlan
|
||||
ArgumentNullException.ThrowIfNull(empty);
|
||||
var result = new List<DataValueSnapshot>((int)BucketCount);
|
||||
for (long i = 0; i < BucketCount; i++)
|
||||
{
|
||||
var bucketStart = BucketStart(i);
|
||||
result.Add(byBucket.TryGetValue(i, out var v) ? v : empty(bucketStart));
|
||||
}
|
||||
result.Add(byBucket.TryGetValue(i, out var v) ? v : empty(BucketTimestamp(i)));
|
||||
if (Reversed) result.Reverse();
|
||||
return result;
|
||||
}
|
||||
|
||||
@@ -9,7 +9,11 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common;
|
||||
/// <remarks>
|
||||
/// Sample classes: <i>usable</i> = severity Good or Uncertain AND a stored value (an Empty sample is an outage marker
|
||||
/// and counts as Bad whatever its code); <i>good</i> = usable with severity Good; <i>bad</i> = not usable.
|
||||
/// Every value field is computed over usable samples only.
|
||||
/// <see cref="Avg"/> / <see cref="Min"/> / <see cref="Max"/> / <see cref="Sum"/> / <see cref="Extreme"/> are computed over
|
||||
/// <b>Good</b> numeric samples only (#684 — Part 13 defines Average / Minimum / Maximum / Range / *ActualTime over Good
|
||||
/// raw values, as the SDK's calculators do); <see cref="UncertainMin"/> / <see cref="UncertainMax"/> over the Uncertain
|
||||
/// ones (the Part 13 Minimum/Maximum/Range status rule); <see cref="First"/> / <see cref="Last"/> over usable samples;
|
||||
/// <see cref="FirstAny"/> / <see cref="LastAny"/> over every sample.
|
||||
/// </remarks>
|
||||
public sealed class BucketRow
|
||||
{
|
||||
@@ -28,19 +32,30 @@ public sealed class BucketRow
|
||||
/// <summary>Bad (not usable) samples.</summary>
|
||||
public int Bad { get; set; }
|
||||
|
||||
/// <summary>Usable String / DateTime samples.</summary>
|
||||
/// <summary>Usable String / DateTime samples (Delta's refusal check).</summary>
|
||||
public int NonNumeric { get; set; }
|
||||
|
||||
/// <summary>Average of the usable numeric values (null when none).</summary>
|
||||
/// <summary>
|
||||
/// <b>Good</b> String / DateTime samples — the refusal check of the Good-value aggregates (Average / Minimum / Maximum /
|
||||
/// Range / *ActualTime, #684 review): an Uncertain non-numeric sample is not part of their value set. Filled by the
|
||||
/// Simple and ActualTime queries; null (a row that does not carry it) falls back to <see cref="NonNumeric"/>, the
|
||||
/// pre-#684 (stricter) check.
|
||||
/// </summary>
|
||||
public int? GoodNonNumeric { get; set; }
|
||||
|
||||
/// <summary><see cref="GoodNonNumeric"/>, or <see cref="NonNumeric"/> when the row does not carry it.</summary>
|
||||
internal int GoodNonNumericOrAll => GoodNonNumeric ?? NonNumeric;
|
||||
|
||||
/// <summary>Average of the Good numeric values (null when none — #684).</summary>
|
||||
public double? Avg { get; set; }
|
||||
|
||||
/// <summary>Minimum of the usable numeric values (null when none).</summary>
|
||||
/// <summary>Minimum of the Good numeric values (null when none — #684).</summary>
|
||||
public double? Min { get; set; }
|
||||
|
||||
/// <summary>Maximum of the usable numeric values (null when none).</summary>
|
||||
/// <summary>Maximum of the Good numeric values (null when none — #684).</summary>
|
||||
public double? Max { get; set; }
|
||||
|
||||
/// <summary>Sum of the usable numeric values (null when none).</summary>
|
||||
/// <summary>Sum of the Good numeric values (null when none — #684).</summary>
|
||||
public double? Sum { get; set; }
|
||||
|
||||
/// <summary>First usable sample's value by time (Empty when none).</summary>
|
||||
@@ -49,9 +64,31 @@ public sealed class BucketRow
|
||||
/// <summary>Last usable sample's value by time (Empty when none).</summary>
|
||||
public HistorianValue Last { get; set; }
|
||||
|
||||
/// <summary>Extreme (min or max) usable numeric value (null when none).</summary>
|
||||
/// <summary>Extreme (min or max) Good numeric value (null when none — #684).</summary>
|
||||
public double? Extreme { get; set; }
|
||||
|
||||
/// <summary>Time of <see cref="Extreme"/> (ties ⇒ earliest; null when none).</summary>
|
||||
public DateTime? ExtremeTime { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Minimum of the bucket's <b>Uncertain</b> (usable, not Good) numeric values (null when none). Not a result value:
|
||||
/// Part 13 makes Minimum / MinimumActualTime / Range <c>Uncertain_DataSubNormal</c> when an Uncertain value is less
|
||||
/// than the minimum Good value (#684).
|
||||
/// </summary>
|
||||
public double? UncertainMin { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Maximum of the bucket's Uncertain numeric values (null when none): Maximum / MaximumActualTime / Range are
|
||||
/// Uncertain when one is greater than the maximum Good value (#684).
|
||||
/// </summary>
|
||||
public double? UncertainMax { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// The bucket's first sample by time of <b>any</b> quality (null when the bucket is empty): Start returns it raw, and
|
||||
/// Delta is Uncertain when it is not usable (#680, Part 13 Tables 73/74 — the SDK's <c>StartEndAggregateCalculator</c>).
|
||||
/// </summary>
|
||||
public StoredSample? FirstAny { get; set; }
|
||||
|
||||
/// <summary>The bucket's last sample by time of any quality (null when the bucket is empty): End, Delta (#680).</summary>
|
||||
public StoredSample? LastAny { get; set; }
|
||||
}
|
||||
|
||||
@@ -12,6 +12,12 @@ public static class HistorianInfoBits
|
||||
/// <summary>The value was interpolated.</summary>
|
||||
public const uint HistorianInterpolated = 0x2u;
|
||||
|
||||
/// <summary>
|
||||
/// Partial (Part 4 §7.39 / <c>Opc.Ua.AggregateBits.Partial</c>): the calculated value covers an interval shorter
|
||||
/// than the processing interval (the truncated last interval of a processed read).
|
||||
/// </summary>
|
||||
public const uint HistorianPartial = 0x4u;
|
||||
|
||||
/// <summary>
|
||||
/// Marks <paramref name="status"/> as a Calculated data value. Any historian bits the status already carries
|
||||
/// (e.g. a stored Interpolated sample) are cleared first, so the result is never both.
|
||||
|
||||
@@ -10,6 +10,9 @@ public static class HistorianStatusCodes
|
||||
public const uint GoodEntryInserted = 0x00A20000u;
|
||||
public const uint GoodEntryReplaced = 0x00A30000u;
|
||||
public const uint BadNoData = 0x809B0000u;
|
||||
|
||||
/// <summary>Generic Bad: a time-weighted interval holding data but no non-Bad time (the SDK aggregate rule).</summary>
|
||||
public const uint Bad = 0x80000000u;
|
||||
public const uint BadEntryExists = 0x809F0000u;
|
||||
public const uint BadNoEntryExists = 0x80A00000u;
|
||||
public const uint BadInvalidTimestamp = 0x80230000u;
|
||||
|
||||
@@ -0,0 +1,600 @@
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common;
|
||||
|
||||
/// <summary>
|
||||
/// The OPC UA Part 13 time-weighted aggregates (<see cref="AggregateFamily.TimeWeighted"/>): <c>TimeAverage</c>,
|
||||
/// <c>TimeAverage2</c>, <c>Total</c>, <c>Total2</c>, <c>Interpolative</c> and the duration-based <c>PercentGood</c> /
|
||||
/// <c>PercentBad</c> (#680), evaluated over a time-ordered stream of stored samples. Both SQL backends fetch the same rows and feed them here, so the two produce identical results
|
||||
/// by construction.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>Input.</b> Samples in strictly ascending time: the bounding rows before the window (the latest
|
||||
/// <i>usable</i> sample before <c>start</c>, then the latest sample of any quality before <c>start</c> — they differ
|
||||
/// when an outage straddles the start), every in-window sample, then the bounding rows at/after the window end
|
||||
/// (the first sample of any quality at or after <c>end</c>, then the first <i>usable</i> one). <see cref="Add"/>
|
||||
/// ignores a sample at or before the last one, so a bound fetched twice is harmless. Memory is O(buckets), not O(samples).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>The signal (server defaults: TreatUncertainAsBad = false, PercentDataBad = PercentDataGood = 100,
|
||||
/// UseSlopedExtrapolation = false; the node manager refuses any other client <c>AggregateConfiguration</c>).</b> Between a usable
|
||||
/// sample <c>p</c> and the next sample <c>s</c>:
|
||||
/// <list type="bullet">
|
||||
/// <item><c>s</c> usable, both Double/Int64 ⇒ <b>sloped</b> (linear <c>p → s</c>); quality = worse of the two.</item>
|
||||
/// <item><c>s</c> usable, either Boolean ⇒ <b>stepped</b> (<c>p</c> held); quality = <c>p</c>'s.</item>
|
||||
/// <item><c>s</c> Bad ⇒ <c>p</c> held up to the outage (never interpolated into it, as AtTime does); Uncertain.</item>
|
||||
/// <item>no later sample at all (nothing at or after the window end) ⇒ the data has ENDED at <c>p</c>: no
|
||||
/// extrapolation (Part 13 "After End of Data", #680).</item>
|
||||
/// </list>
|
||||
/// From a Bad sample (incl. an <c>Empty</c> outage marker) to the next sample the signal is Bad (outage) — to the
|
||||
/// window end when nothing follows it; before the first sample there is no data. This is the <i>simple bounding</i>
|
||||
/// signal.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// The <i>interpolated bounding</i> signal (Part 13 "interpolated bounding values") skips Bad samples: it joins
|
||||
/// consecutive <b>usable</b> samples the same way (sloped / stepped) straight across any outage, and holds the
|
||||
/// last usable value to the window end. It is undefined before the first usable sample.
|
||||
/// </para>
|
||||
/// <list type="table">
|
||||
/// <item><term>TimeAverage</term><description>∫ interpolated-bounding signal ÷ its defined duration in the interval.</description></item>
|
||||
/// <item><term>Total</term><description>∫ interpolated-bounding signal over its defined part, in value·s — Part 13
|
||||
/// §5.4.3.8 (= <c>TimeAverage × interval seconds</c> on a covered interval; only the covered part when the data
|
||||
/// starts inside it, #680).</description></item>
|
||||
/// <item><term>TimeAverage2</term><description>∫ simple-bounding signal over its non-Bad parts ÷ non-Bad duration (Bad intervals excluded).</description></item>
|
||||
/// <item><term>Total2</term><description>∫ simple-bounding signal over its non-Bad parts, in value·s (= TimeAverage2 × non-Bad seconds).</description></item>
|
||||
/// <item><term>PercentGood / PercentBad</term><description><c>100 ×</c> Good / Bad region time ÷ interval duration
|
||||
/// (Part 13 §5.4.3.31-34): stepped regions, each with the status of the sample that starts it; the region before an
|
||||
/// interval's first own sample has the simple bounding value's status (Bad after a Bad sample; Uncertain on a
|
||||
/// segment running into an outage or sloping to/from an Uncertain sample; else the earlier sample's — the SDK's
|
||||
/// <c>GetSimpleBound</c>); time before the first sample is Bad; at the end of the data the last region runs to the
|
||||
/// end of its own interval (Part 13; the SDK gives it 1 ms) and later intervals are <c>BadNoData</c>; an interval
|
||||
/// ending before the first sample ⇒ <c>BadNoData</c>. Status Good; any value kind.</description></item>
|
||||
/// <item><term>Interpolative</term><description>The interpolated-bounding signal's value at each interval start —
|
||||
/// for a reverse read, at <c>End − i·interval</c> (Part 13 anchors a reverse read at its later time), emitted
|
||||
/// latest first.</description></item>
|
||||
/// </list>
|
||||
/// <para>
|
||||
/// <b>Status (TimeAverage / Total, #680)</b> — Part 13 ("Uncertain if Bad values are skipped or Uncertain values are
|
||||
/// used") and the SDK's <c>AverageAggregateCalculator</c>: no defined value in the interval ⇒ <c>BadNoData</c>; any
|
||||
/// part of it on a bridged segment with an Uncertain end (or Uncertain start when stepped), bridged across a Bad
|
||||
/// sample, extrapolated past the last usable sample, or not covered at all ⇒ <c>UncertainDataSubNormal</c> (so the
|
||||
/// latest interval of a live tag is Uncertain, and an interval wholly inside an outage has the bridged value,
|
||||
/// Uncertain); else <c>Good</c>.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Status (TimeAverage2 / Total2)</b> — the SDK <c>AggregateCalculator.GetTimeBasedStatusCode</c> rule with
|
||||
/// TreatUncertainAsBad = false, so Uncertain time (Uncertain samples, a value held up to an outage) counts as Good
|
||||
/// time, over the interval cut at the end of the data: after the end of the data ⇒ <c>BadNoData</c>; the data
|
||||
/// ending exactly on the interval start ⇒ the last sample's value (Total2 0) with its status; every covered instant
|
||||
/// non-Bad ⇒ <c>Good</c>; no non-Bad time but some Bad (outage) time ⇒ <c>Bad</c> (Part 13's PercentDataBad rule;
|
||||
/// the SDK answers <c>BadNoData</c>); no data at all ⇒ <c>BadNoData</c>; otherwise (PercentDataGood = PercentDataBad
|
||||
/// = 100) <c>UncertainDataSubNormal</c>. Time before the first sample counts as Bad (no data) time.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// For the four, a usable String/DateTime sample contributing to the interval ⇒ <c>BadAggregateInvalidInputs</c>.
|
||||
/// Every status of this family but Interpolative's carries the Calculated bit, plus — on a non-Bad result only — the
|
||||
/// Partial bit on: the truncated interval (Part 13 §5.3.3.2 Case 3; the SDK does not set it — deliberate); the
|
||||
/// interval the data starts inside in the read direction (forward: the first sample fed lies strictly within it,
|
||||
/// #677; reverse: the latest sample fed lies strictly within it, nothing after — the SDK's <c>m_startOfData</c>,
|
||||
/// keyed on the first value it is fed, #680); and, forward, the end-of-data interval of TimeAverage2 / Total2 /
|
||||
/// PercentGood / PercentBad (the SDK's <c>m_endOfData</c>, which it skips for an extrapolated TimeAverage / Total).
|
||||
/// The timestamp is the interval's <see cref="BucketPlan.BucketTimestamp"/> (its start forward, its later edge
|
||||
/// reverse); the value is a <see cref="double"/> (Boolean = 0/1, Int64 as double).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Status (Interpolative).</b> A usable sample exactly at the point ⇒ its value with its own status
|
||||
/// (raw, no historian bit). Otherwise no usable sample at or before ⇒ <c>BadNoData</c>; else the interpolated
|
||||
/// value with the Interpolated bit and the status of the simple-bounding signal at that instant: <c>Good</c> on
|
||||
/// a Good segment, <c>UncertainDataSubNormal</c> on an Uncertain one, inside an outage (interpolated across it)
|
||||
/// or past the last sample (extrapolated). A reverse read returns the raw sample at a point exactly on a sample
|
||||
/// too — the SDK's reverse answer (Interpolated, from the segment beyond) differs from its own forward one; that
|
||||
/// asymmetry is the SDK's defect, deliberately not copied.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public sealed class TimeWeightedAggregator
|
||||
{
|
||||
private const double TicksPerSecond = TimeSpan.TicksPerSecond;
|
||||
|
||||
/// <summary>
|
||||
/// Default cap on the raw in-window rows one time-weighted read may stream (<c>ServerHistorian:MaxTimeWeightedRawRows</c>).
|
||||
/// </summary>
|
||||
public const int DefaultMaxWindowRows = 1_000_000;
|
||||
|
||||
/// <summary>
|
||||
/// The <c>TOP</c>/<c>LIMIT</c> a backend puts on its window query for a configured cap: <c>cap + 1</c> (so one
|
||||
/// row past the cap proves it was exceeded without reading on), or <see cref="int.MaxValue"/> when the cap is
|
||||
/// <= 0 (unbounded).
|
||||
/// </summary>
|
||||
/// <param name="maxWindowRows">The configured cap.</param>
|
||||
/// <returns>The row limit to put on the window query.</returns>
|
||||
public static int WindowFetchLimit(int maxWindowRows) =>
|
||||
maxWindowRows <= 0 || maxWindowRows == int.MaxValue ? int.MaxValue : maxWindowRows + 1;
|
||||
|
||||
/// <summary>Throws when <paramref name="rowsRead"/> has passed a positive <paramref name="maxWindowRows"/>.</summary>
|
||||
/// <param name="rowsRead">Window rows read so far.</param>
|
||||
/// <param name="maxWindowRows">The configured cap (<= 0 = unbounded).</param>
|
||||
/// <param name="tag">The tag, for the message.</param>
|
||||
/// <exception cref="HistoryReadLimitExceededException">The cap is exceeded.</exception>
|
||||
public static void CheckWindowRows(long rowsRead, int maxWindowRows, string tag)
|
||||
{
|
||||
if (maxWindowRows > 0 && rowsRead > maxWindowRows)
|
||||
throw new HistoryReadLimitExceededException(maxWindowRows,
|
||||
$"The time-weighted aggregate read of '{tag}' covers more than {maxWindowRows} raw samples " +
|
||||
"(ServerHistorian:MaxTimeWeightedRawRows); narrow the window or use a pushed-down aggregate such as Average.");
|
||||
}
|
||||
|
||||
private readonly HistoryAggregateType _aggregate;
|
||||
private readonly BucketPlan _plan;
|
||||
private readonly long _start;
|
||||
private readonly long _end;
|
||||
|
||||
// Simple-bounding (raw) signal, per bucket: ∫ over non-Bad time, non-Bad ticks, ticks inside an outage.
|
||||
private readonly double[] _rawArea;
|
||||
private readonly long[] _usableTicks;
|
||||
private readonly long[] _badTicks;
|
||||
// Interpolated-bounding (bridged) signal, per bucket.
|
||||
private readonly double[] _bridgedArea;
|
||||
private readonly long[] _bridgedTicks;
|
||||
// A bridged segment touching the bucket was non-Good: an Uncertain end, a Bad sample skipped, or extrapolated (#680).
|
||||
private readonly bool[] _bridgedNonGood;
|
||||
private readonly bool[] _invalid;
|
||||
// PercentGood / PercentBad (#680): stepped status regions, per bucket.
|
||||
private readonly long[] _goodRegionTicks;
|
||||
private readonly long[] _badRegionTicks;
|
||||
// Interpolative, per point (index = ascending time; see PointTicks).
|
||||
private readonly DataValueSnapshot?[] _points;
|
||||
|
||||
private StoredSample? _prevAny;
|
||||
private StoredSample? _prevUsable;
|
||||
private DateTime? _firstSampleUtc; // the earliest sample fed (any quality): where the data starts
|
||||
private long? _dataEndTicks; // the last sample fed, when usable and nothing at/after the end followed
|
||||
private long _nextPoint; // next point (ascending time) not yet classified
|
||||
private readonly List<PendingPoint> _pending = []; // points awaiting the next usable sample
|
||||
private bool _finished;
|
||||
|
||||
/// <summary>
|
||||
/// What the simple-bounding signal is at a pending bucket start: no sample yet; on a segment from a usable sample
|
||||
/// whose end is not known yet; or degraded (inside an outage, or held up to one).
|
||||
/// </summary>
|
||||
private enum PointQuality { NoPrior, OnSegment, Degraded }
|
||||
|
||||
private readonly record struct PendingPoint(long Bucket, long Ticks, PointQuality Quality);
|
||||
|
||||
/// <summary>Starts an evaluation of <paramref name="aggregate"/> over <paramref name="plan"/>.</summary>
|
||||
/// <param name="aggregate">One of the <see cref="AggregateFamily.TimeWeighted"/> aggregates.</param>
|
||||
/// <param name="plan">The bucket plan (forward window).</param>
|
||||
/// <exception cref="HistoryAggregateNotSupportedException">The aggregate is not time-weighted.</exception>
|
||||
public TimeWeightedAggregator(HistoryAggregateType aggregate, BucketPlan plan)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(plan);
|
||||
if (AggregateFamilies.FamilyOf(aggregate) != AggregateFamily.TimeWeighted)
|
||||
throw new HistoryAggregateNotSupportedException(aggregate);
|
||||
_aggregate = aggregate;
|
||||
_plan = plan;
|
||||
_start = plan.StartUtc.Ticks;
|
||||
_end = plan.EndUtc.Ticks;
|
||||
var n = (int)plan.BucketCount;
|
||||
_rawArea = new double[n];
|
||||
_usableTicks = new long[n];
|
||||
_badTicks = new long[n];
|
||||
_bridgedArea = new double[n];
|
||||
_bridgedTicks = new long[n];
|
||||
_bridgedNonGood = new bool[n];
|
||||
_invalid = new bool[n];
|
||||
_goodRegionTicks = new long[n];
|
||||
_badRegionTicks = new long[n];
|
||||
_points = new DataValueSnapshot?[n];
|
||||
}
|
||||
|
||||
/// <summary>Evaluates <paramref name="aggregate"/> over an already time-ordered sample sequence.</summary>
|
||||
/// <param name="aggregate">One of the <see cref="AggregateFamily.TimeWeighted"/> aggregates.</param>
|
||||
/// <param name="plan">The bucket plan.</param>
|
||||
/// <param name="samples">Bounds + window samples, ascending by time.</param>
|
||||
/// <param name="now">Server timestamp for the results.</param>
|
||||
/// <returns>One value per bucket, in the plan's order (descending for a reverse window).</returns>
|
||||
public static IReadOnlyList<DataValueSnapshot> Evaluate(
|
||||
HistoryAggregateType aggregate, BucketPlan plan, IEnumerable<StoredSample> samples, DateTime now)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(samples);
|
||||
var acc = new TimeWeightedAggregator(aggregate, plan);
|
||||
foreach (var s in samples) acc.Add(s);
|
||||
return acc.Finish(now);
|
||||
}
|
||||
|
||||
/// <summary>Feeds the next sample (ascending time; a sample at or before the previous one is ignored).</summary>
|
||||
/// <param name="s">The sample.</param>
|
||||
public void Add(StoredSample s)
|
||||
{
|
||||
if (_finished) throw new InvalidOperationException("The aggregation is already finished.");
|
||||
var t = s.TimeUtc.Ticks;
|
||||
if (_prevAny is { } last && t <= last.TimeUtc.Ticks) return;
|
||||
if (_firstSampleUtc is null)
|
||||
{
|
||||
_firstSampleUtc = s.TimeUtc;
|
||||
AddRegion(_badRegionTicks, _start, t); // PercentBad: the time before the first sample is Bad (no data)
|
||||
}
|
||||
|
||||
// Interpolative: classify every bucket start before this sample by the simple-bounding signal there.
|
||||
ClassifyPointsBefore(t);
|
||||
|
||||
// Simple-bounding segment [prevAny, s).
|
||||
if (_prevAny is { IsUsable: true } p)
|
||||
{
|
||||
// Toward a usable sample: sloped or stepped; toward a Bad one: held up to the outage.
|
||||
var sloped = s.IsUsable && IsSloped(p, s);
|
||||
AddRaw(p, t, sloped ? s.Value.Number : p.Value.Number);
|
||||
}
|
||||
else if (_prevAny is { } bad)
|
||||
{
|
||||
AddBad(bad, t);
|
||||
}
|
||||
if (_prevAny is { } r) AddStatusRegion(r, s, t);
|
||||
|
||||
if (!s.IsUsable) DegradePending(); // the segment the pending points sit on runs into an outage
|
||||
|
||||
if (s.IsUsable)
|
||||
{
|
||||
if (_prevUsable is { } pu)
|
||||
{
|
||||
var sloped = IsSloped(pu, s);
|
||||
// Part 13: Uncertain when a Bad value was skipped or an Uncertain value is used (the end sample only
|
||||
// contributes its value to a sloped segment).
|
||||
var nonGood = !pu.IsGood || (sloped && !s.IsGood) || _prevAny is { IsUsable: false };
|
||||
AddBridged(pu, t, sloped ? s.Value.Number : pu.Value.Number, nonGood);
|
||||
}
|
||||
ResolvePending(_prevUsable, s);
|
||||
_prevUsable = s;
|
||||
}
|
||||
|
||||
_prevAny = s;
|
||||
}
|
||||
|
||||
/// <summary>Completes the evaluation (extrapolating past the last sample) and assembles the per-bucket results.</summary>
|
||||
/// <param name="now">Server timestamp for the results.</param>
|
||||
/// <returns>One value per bucket, in the plan's order (descending for a reverse window).</returns>
|
||||
public IReadOnlyList<DataValueSnapshot> Finish(DateTime now)
|
||||
{
|
||||
if (!_finished)
|
||||
{
|
||||
_finished = true;
|
||||
ClassifyPointsBefore(long.MaxValue);
|
||||
if (_prevAny is { IsUsable: true } p && p.TimeUtc.Ticks < _end)
|
||||
{
|
||||
// No sample at or after the window end: the data ENDS here (Part 13 "After End of Data"). The simple-
|
||||
// bounding signal is not extrapolated (TimeAverage2/Total2 cover up to it; later intervals BadNoData), and
|
||||
// the status region it starts runs to the end of its own interval only (PercentGood/PercentBad).
|
||||
_dataEndTicks = p.TimeUtc.Ticks;
|
||||
if (p.TimeUtc.Ticks >= _start && _plan.BucketCount > 0)
|
||||
AddRegion(RegionTicks(p), p.TimeUtc.Ticks, BucketEnd(FirstBucketAt(p.TimeUtc.Ticks)));
|
||||
}
|
||||
else if (_prevAny is { } bad && bad.TimeUtc.Ticks < _end)
|
||||
{
|
||||
AddBad(bad, _end); // an outage marker last: the outage runs to the window end
|
||||
AddRegion(_badRegionTicks, bad.TimeUtc.Ticks, _end);
|
||||
}
|
||||
if (_prevUsable is { } pu && pu.TimeUtc.Ticks < _end)
|
||||
AddBridged(pu, _end, pu.Value.Number, nonGood: true); // stepped extrapolation ⇒ Uncertain
|
||||
ResolvePending(_prevUsable, next: null);
|
||||
}
|
||||
|
||||
var byBucket = new Dictionary<long, DataValueSnapshot>((int)_plan.BucketCount);
|
||||
for (long i = 0; i < _plan.BucketCount; i++)
|
||||
byBucket[i] = Result(i, now);
|
||||
return _plan.Assemble(byBucket, t => AggregateEvaluator.NoData(t, now));
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------------------------------ Results
|
||||
|
||||
private DataValueSnapshot Result(long i, DateTime now)
|
||||
{
|
||||
if (_aggregate == HistoryAggregateType.Interpolative)
|
||||
return _points[i] is { } point
|
||||
? point with { ServerTimestampUtc = now }
|
||||
: new DataValueSnapshot(null, HistorianStatusCodes.BadNoData, PointTime(i), now);
|
||||
|
||||
var stamp = _plan.BucketTimestamp(i);
|
||||
var lo = BucketStartTicks(i);
|
||||
var hi = BucketEnd(i);
|
||||
var duration = hi - lo;
|
||||
// Partial: the truncated interval (the latest forward, the earliest reverse — Part 13 §5.3.3.2 Case 3, which the
|
||||
// SDK does not implement), or the one the data starts inside IN THE READ DIRECTION (no bounding value on that
|
||||
// side — the SDK's m_startOfData rule, #677): forward, the earliest sample strictly inside; reverse, the latest
|
||||
// sample strictly inside (nothing after it — the SDK keys m_startOfData on the first value it is fed, #680).
|
||||
// Forward, the end-of-data interval (the latest sample inside it, nothing after) is Partial too — never on an
|
||||
// extrapolated TimeAverage/Total (the SDK's m_endOfData rule, #680). Never on a Bad result.
|
||||
var latestInside = _plan.Reversed && _prevAny is { } latest
|
||||
&& latest.TimeUtc.Ticks > lo && latest.TimeUtc.Ticks < hi;
|
||||
var partial = _plan.IsTruncated(i) || latestInside
|
||||
|| (!_plan.Reversed && _plan.DataStartsInside(i, _firstSampleUtc));
|
||||
var endOfData = !_plan.Reversed && _dataEndTicks is { } de && de >= lo && de < hi;
|
||||
DataValueSnapshot Calc(object? value, uint status, bool endPartial = false)
|
||||
{
|
||||
var code = HistorianInfoBits.WithCalculated(status);
|
||||
if ((partial || endPartial) && !StatusBits.IsBad(status)) code |= HistorianInfoBits.HistorianPartial;
|
||||
return new DataValueSnapshot(value, code, stamp, now);
|
||||
}
|
||||
|
||||
switch (_aggregate)
|
||||
{
|
||||
case HistoryAggregateType.TimeAverage or HistoryAggregateType.Total:
|
||||
{
|
||||
// Interpolated bounding: Uncertain when any part of the interval was bridged across a Bad value, used an
|
||||
// Uncertain value, was extrapolated, or is not covered at all (Part 13; the SDK's AverageAggregateCalculator).
|
||||
if (_invalid[i]) return Calc(null, HistorianStatusCodes.BadAggregateInvalidInputs);
|
||||
if (_bridgedTicks[i] == 0) return AggregateEvaluator.NoData(stamp, now);
|
||||
var status = _bridgedNonGood[i] || _bridgedTicks[i] < duration
|
||||
? HistorianStatusCodes.UncertainDataSubNormal
|
||||
: HistorianStatusCodes.Good;
|
||||
return _aggregate == HistoryAggregateType.TimeAverage
|
||||
? Calc(_bridgedArea[i] / _bridgedTicks[i], status)
|
||||
: Calc(_bridgedArea[i] / TicksPerSecond, status); // ∫ over the covered part only (Part 13 partial width)
|
||||
}
|
||||
case HistoryAggregateType.TimeAverage2 or HistoryAggregateType.Total2:
|
||||
{
|
||||
if (_invalid[i]) return Calc(null, HistorianStatusCodes.BadAggregateInvalidInputs);
|
||||
if (_dataEndTicks is { } dataEnd)
|
||||
{
|
||||
// After the end of the data: nothing to aggregate (Part 13 "After End of Data: Bad_NoData").
|
||||
if (lo > dataEnd) return AggregateEvaluator.NoData(stamp, now);
|
||||
if (_usableTicks[i] == 0 && dataEnd < hi && _prevAny is { } last)
|
||||
{
|
||||
// The data ends inside this interval and no non-Bad time precedes its last sample here (the data
|
||||
// ends on the interval's start; or the last sample is the first usable one after an outage — the
|
||||
// comms-loss recovery shape — or the tag's only sample): nothing covered, so the value at that
|
||||
// instant (Total2 0). Good only when the data ends exactly on the interval start with a Good
|
||||
// sample (nothing before it in the interval); otherwise the interval also holds Bad / no-data time
|
||||
// ⇒ Uncertain (the SDK answers the same, modulo its 1 ms artifact).
|
||||
if (!IsNumeric(last)) return Calc(null, HistorianStatusCodes.BadAggregateInvalidInputs);
|
||||
var st = lo == dataEnd && last.IsGood ? HistorianStatusCodes.Good : HistorianStatusCodes.UncertainDataSubNormal;
|
||||
return Calc(_aggregate == HistoryAggregateType.TimeAverage2 ? last.Value.Number : 0d, st, endOfData);
|
||||
}
|
||||
duration = Math.Min(hi, dataEnd) - lo;
|
||||
}
|
||||
if (_usableTicks[i] == 0)
|
||||
return _badTicks[i] > 0 ? Calc(null, HistorianStatusCodes.Bad) : AggregateEvaluator.NoData(stamp, now);
|
||||
var status = _usableTicks[i] == duration ? HistorianStatusCodes.Good : HistorianStatusCodes.UncertainDataSubNormal;
|
||||
return _aggregate == HistoryAggregateType.TimeAverage2
|
||||
? Calc(_rawArea[i] / _usableTicks[i], status, endOfData)
|
||||
: Calc(_rawArea[i] / TicksPerSecond, status, endOfData);
|
||||
}
|
||||
case HistoryAggregateType.PercentGood or HistoryAggregateType.PercentBad:
|
||||
{
|
||||
// No data in or around the interval (it ends before the first sample — the SDK's out-of-range test — or
|
||||
// starts after the end of the data) ⇒ BadNoData; otherwise the share of the interval's duration.
|
||||
if (_firstSampleUtc is not { } first || hi < first.Ticks || (_dataEndTicks is { } de2 && lo > de2))
|
||||
return AggregateEvaluator.NoData(stamp, now);
|
||||
var ticks = _aggregate == HistoryAggregateType.PercentGood ? _goodRegionTicks[i] : _badRegionTicks[i];
|
||||
return Calc(100d * ticks / duration, HistorianStatusCodes.Good, endOfData);
|
||||
}
|
||||
default:
|
||||
throw new HistoryAggregateNotSupportedException(_aggregate);
|
||||
}
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------------------------------ Segments
|
||||
|
||||
private static bool IsNumeric(StoredSample s) =>
|
||||
s.Value.Kind is HistorianValueKind.Double or HistorianValueKind.Int64 or HistorianValueKind.Boolean;
|
||||
|
||||
/// <summary>Sloped between two usable samples only when both are Double/Int64; anything Boolean is stepped.</summary>
|
||||
private static bool IsSloped(StoredSample a, StoredSample b) =>
|
||||
a.Value.Kind is HistorianValueKind.Double or HistorianValueKind.Int64
|
||||
&& b.Value.Kind is HistorianValueKind.Double or HistorianValueKind.Int64;
|
||||
|
||||
private long BucketStartTicks(long i) => _plan.LowerTicks(i);
|
||||
|
||||
private long BucketEnd(long i) => _plan.UpperTicks(i);
|
||||
|
||||
/// <summary>The ascending index of the bucket whose (half-open) span holds tick <paramref name="t"/> (t ≥ window start).</summary>
|
||||
private long FirstBucketAt(long t) => Math.Max(0, (t - _plan.OriginTicks) / _plan.IntervalTicks);
|
||||
|
||||
private void AddRaw(StoredSample from, long toTicks, double toValue) =>
|
||||
Distribute(from, toTicks, toValue, (i, area, ticks) =>
|
||||
{
|
||||
_rawArea[i] += area;
|
||||
_usableTicks[i] += ticks;
|
||||
});
|
||||
|
||||
/// <summary>Counts <c>[from.t, toTicks)</c> as outage (Bad) time; no value, never marks a bucket invalid.</summary>
|
||||
private void AddBad(StoredSample from, long toTicks)
|
||||
{
|
||||
var lo = Math.Max(from.TimeUtc.Ticks, _start);
|
||||
var hi = Math.Min(toTicks, _end);
|
||||
if (lo >= hi || _plan.BucketCount == 0) return;
|
||||
for (var i = FirstBucketAt(lo); i < _plan.BucketCount; i++)
|
||||
{
|
||||
var bs = BucketStartTicks(i);
|
||||
if (bs >= hi) break;
|
||||
var a = Math.Max(lo, bs);
|
||||
var b = Math.Min(hi, BucketEnd(i));
|
||||
if (a < b) _badTicks[i] += b - a;
|
||||
}
|
||||
}
|
||||
|
||||
private void AddBridged(StoredSample from, long toTicks, double toValue, bool nonGood) =>
|
||||
Distribute(from, toTicks, toValue, (i, area, ticks) =>
|
||||
{
|
||||
_bridgedArea[i] += area;
|
||||
_bridgedTicks[i] += ticks;
|
||||
if (nonGood) _bridgedNonGood[i] = true;
|
||||
});
|
||||
|
||||
// ------------------------------------------------------------------------------------------ PercentGood / PercentBad
|
||||
|
||||
/// <summary>The region array a stepped region of <paramref name="s"/>'s status counts toward (null: Uncertain, neither).</summary>
|
||||
private long[]? RegionTicks(StoredSample s) =>
|
||||
s.IsGood ? _goodRegionTicks : !s.IsUsable ? _badRegionTicks : null;
|
||||
|
||||
/// <summary>
|
||||
/// The status regions between sample <paramref name="p"/> and the next sample <paramref name="s"/> (at
|
||||
/// <paramref name="t"/>). Inside the interval <paramref name="p"/> lies in, the region has <paramref name="p"/>'s
|
||||
/// status (stepped, Part 13 §5.4.3.31-34). In a later interval the region starts at that interval's simple bounding
|
||||
/// value, whose status is the simple-bounding signal's: Bad after a Bad <paramref name="p"/>; Uncertain when the
|
||||
/// segment runs into an outage or a sloped one has a non-Good end; else <paramref name="p"/>'s (the SDK's
|
||||
/// <c>GetSimpleBound</c>).
|
||||
/// </summary>
|
||||
private void AddStatusRegion(StoredSample p, StoredSample s, long t)
|
||||
{
|
||||
var lo = Math.Max(p.TimeUtc.Ticks, _start);
|
||||
var hi = Math.Min(t, _end);
|
||||
if (lo >= hi || _plan.BucketCount == 0) return;
|
||||
long[]? bound;
|
||||
if (!p.IsUsable) bound = _badRegionTicks;
|
||||
else if (!s.IsUsable) bound = null; // held into an outage ⇒ Uncertain
|
||||
else if (IsSloped(p, s)) bound = p.IsGood && s.IsGood ? _goodRegionTicks : null;
|
||||
else bound = RegionTicks(p);
|
||||
var own = RegionTicks(p);
|
||||
for (var i = FirstBucketAt(lo); i < _plan.BucketCount; i++)
|
||||
{
|
||||
var bs = BucketStartTicks(i);
|
||||
if (bs >= hi) break;
|
||||
var a = Math.Max(lo, bs);
|
||||
var b = Math.Min(hi, BucketEnd(i));
|
||||
if (a >= b) continue;
|
||||
var target = p.TimeUtc.Ticks >= bs ? own : bound;
|
||||
if (target is not null) target[i] += b - a;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Adds <c>[from, to)</c>, clipped to the window, to <paramref name="target"/> per bucket.</summary>
|
||||
private void AddRegion(long[]? target, long from, long to)
|
||||
{
|
||||
if (target is null) return;
|
||||
var lo = Math.Max(from, _start);
|
||||
var hi = Math.Min(to, _end);
|
||||
if (lo >= hi || _plan.BucketCount == 0) return;
|
||||
for (var i = FirstBucketAt(lo); i < _plan.BucketCount; i++)
|
||||
{
|
||||
var bs = BucketStartTicks(i);
|
||||
if (bs >= hi) break;
|
||||
var a = Math.Max(lo, bs);
|
||||
var b = Math.Min(hi, BucketEnd(i));
|
||||
if (a < b) target[i] += b - a;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Splits the linear segment <c>[from.t, toTicks)</c> (value <c>from.v → toValue</c>) at the bucket edges, clipped to
|
||||
/// the window, and hands each bucket its trapezoid area (value·ticks) and duration. A non-numeric start sample
|
||||
/// marks every bucket it touches invalid instead.
|
||||
/// </summary>
|
||||
private void Distribute(StoredSample from, long toTicks, double toValue, Action<long, double, long> add)
|
||||
{
|
||||
var t0 = from.TimeUtc.Ticks;
|
||||
var lo = Math.Max(t0, _start);
|
||||
var hi = Math.Min(toTicks, _end);
|
||||
if (lo >= hi || _plan.BucketCount == 0) return;
|
||||
|
||||
var numeric = IsNumeric(from);
|
||||
var v0 = from.Value.Number;
|
||||
var span = (double)(toTicks - t0);
|
||||
double ValueAt(long x) => toValue == v0 ? v0 : v0 + (toValue - v0) * ((x - t0) / span);
|
||||
|
||||
for (var i = FirstBucketAt(lo); i < _plan.BucketCount; i++)
|
||||
{
|
||||
var bs = BucketStartTicks(i);
|
||||
if (bs >= hi) break;
|
||||
var a = Math.Max(lo, bs);
|
||||
var b = Math.Min(hi, BucketEnd(i));
|
||||
if (a >= b) continue;
|
||||
if (!numeric) { _invalid[i] = true; continue; }
|
||||
add(i, (ValueAt(a) + ValueAt(b)) / 2d * (b - a), b - a);
|
||||
}
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------------------------------ Interpolative
|
||||
|
||||
/// <summary>
|
||||
/// The time of Interpolative point <paramref name="j"/> (ascending time): the bucket's timestamp. Forward: the
|
||||
/// bucket starts <c>start + j·interval</c>. Reverse (Part 13 anchors a reverse read at its later time, the
|
||||
/// request's start): the later edges <c>End, End − interval, …</c> once <see cref="BucketPlan.Assemble"/> has
|
||||
/// reversed the order — so a reverse read 10:00 → 09:00 at 1 h yields the value at 10:00.
|
||||
/// </summary>
|
||||
private long PointTicks(long j) => _plan.BucketTimestamp(j).Ticks;
|
||||
|
||||
private DateTime PointTime(long j) => new(PointTicks(j), DateTimeKind.Utc);
|
||||
|
||||
/// <summary>Classifies every unclassified point strictly before <paramref name="beforeTicks"/>.</summary>
|
||||
private void ClassifyPointsBefore(long beforeTicks)
|
||||
{
|
||||
if (_aggregate != HistoryAggregateType.Interpolative) return;
|
||||
for (; _nextPoint < _plan.BucketCount; _nextPoint++)
|
||||
{
|
||||
var a = PointTicks(_nextPoint);
|
||||
if (a >= beforeTicks) break;
|
||||
|
||||
if (_prevAny is not { } p)
|
||||
{
|
||||
_pending.Add(new PendingPoint(_nextPoint, a, PointQuality.NoPrior));
|
||||
}
|
||||
else if (p.IsUsable && p.TimeUtc.Ticks == a)
|
||||
{
|
||||
// A usable sample exactly at the interval start: its own value and status.
|
||||
_points[_nextPoint] = IsNumeric(p)
|
||||
? new DataValueSnapshot(p.Value.Number, p.Status, PointTime(_nextPoint), default)
|
||||
: InvalidPoint(_nextPoint);
|
||||
}
|
||||
else if (!p.IsUsable)
|
||||
{
|
||||
_pending.Add(new PendingPoint(_nextPoint, a, PointQuality.Degraded));
|
||||
}
|
||||
else
|
||||
{
|
||||
// Inside a segment that starts at a usable sample; its quality is only known once the next sample
|
||||
// arrives (beforeTicks is that sample's time, or MaxValue at Finish ⇒ extrapolated).
|
||||
_pending.Add(new PendingPoint(_nextPoint, a, PointQuality.OnSegment));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Resolves the pending bucket starts against the bridging pair <c>(p, next)</c>.</summary>
|
||||
private void ResolvePending(StoredSample? p, StoredSample? next)
|
||||
{
|
||||
if (_pending.Count == 0) return;
|
||||
foreach (var pt in _pending)
|
||||
{
|
||||
var bucketStart = PointTime(pt.Bucket);
|
||||
if (p is not { } prev || pt.Quality == PointQuality.NoPrior)
|
||||
{
|
||||
_points[pt.Bucket] = new DataValueSnapshot(null, HistorianStatusCodes.BadNoData, bucketStart, default);
|
||||
continue;
|
||||
}
|
||||
if (!IsNumeric(prev) || next is { } nn && !IsNumeric(nn))
|
||||
{
|
||||
_points[pt.Bucket] = InvalidPoint(pt.Bucket);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Good only on a segment that runs straight from prev to next (no outage skipped, not extrapolated), and only
|
||||
// when the samples that define the value are Good: both ends when sloped, prev alone when stepped.
|
||||
double value;
|
||||
bool good;
|
||||
if (next is { } n && IsSloped(prev, n))
|
||||
{
|
||||
value = prev.Value.Number + (n.Value.Number - prev.Value.Number)
|
||||
* ((double)(pt.Ticks - prev.TimeUtc.Ticks) / (n.TimeUtc.Ticks - prev.TimeUtc.Ticks));
|
||||
good = pt.Quality == PointQuality.OnSegment && prev.IsGood && n.IsGood;
|
||||
}
|
||||
else
|
||||
{
|
||||
value = prev.Value.Number;
|
||||
good = pt.Quality == PointQuality.OnSegment && next is not null && prev.IsGood;
|
||||
}
|
||||
var status = good ? HistorianStatusCodes.Good : HistorianStatusCodes.UncertainDataSubNormal;
|
||||
_points[pt.Bucket] = new DataValueSnapshot(value, HistorianInfoBits.WithInterpolated(status), bucketStart, default);
|
||||
}
|
||||
_pending.Clear();
|
||||
}
|
||||
|
||||
/// <summary>Marks every pending on-segment point degraded: its segment ends at a Bad sample.</summary>
|
||||
private void DegradePending()
|
||||
{
|
||||
for (var k = 0; k < _pending.Count; k++)
|
||||
if (_pending[k].Quality == PointQuality.OnSegment)
|
||||
_pending[k] = _pending[k] with { Quality = PointQuality.Degraded };
|
||||
}
|
||||
|
||||
private DataValueSnapshot InvalidPoint(long bucket) =>
|
||||
new(null, HistorianStatusCodes.BadAggregateInvalidInputs, PointTime(bucket), default);
|
||||
}
|
||||
+29
@@ -0,0 +1,29 @@
|
||||
-- OtOpcUa SQL Server historian backend - schema version 2: OPC UA Part 11 modified-value history (Gitea #668).
|
||||
-- $(Schema) is substituted by SqlServerSchemaBootstrapper (identifier-validated) BEFORE the script is split on GO lines.
|
||||
-- Every statement is guarded so a re-run is a no-op.
|
||||
--
|
||||
-- One row per sample a HistoryUpdate inserted, replaced or deleted, carrying the PRIOR value (all NULL for an
|
||||
-- insert). Rows are written by the SAME statement that changes hist.Sample (OUTPUT ... INTO), so a failed update
|
||||
-- leaves no audit row and a committed one always has its row.
|
||||
-- WARNING: OUTPUT ... INTO forbids, on its target table: enabled triggers, either side of a FOREIGN KEY, and CHECK
|
||||
-- constraints. Do not add any of those to SampleModified, or every HistoryUpdate fails.
|
||||
IF OBJECT_ID(N'[$(Schema)].[SampleModified]', N'U') IS NULL
|
||||
CREATE TABLE [$(Schema)].[SampleModified] (
|
||||
TagId int NOT NULL,
|
||||
SourceTimeUtc datetime2(7) NOT NULL,
|
||||
ModifiedId bigint IDENTITY(1,1) NOT NULL,
|
||||
ModificationTimeUtc datetime2(7) NOT NULL,
|
||||
UpdateType tinyint NOT NULL,
|
||||
Principal nvarchar(200) NOT NULL,
|
||||
ApiKeyId nvarchar(200) NULL,
|
||||
ValueKind tinyint NULL,
|
||||
NumValue float NULL,
|
||||
IntValue bigint NULL,
|
||||
StrValue nvarchar(max) NULL,
|
||||
StatusCode int NULL,
|
||||
ServerTimeUtc datetime2(7) NULL,
|
||||
CONSTRAINT PK_SampleModified PRIMARY KEY CLUSTERED (TagId, SourceTimeUtc, ModifiedId))
|
||||
GO
|
||||
IF NOT EXISTS (SELECT 1 FROM [$(Schema)].[SchemaVersion] WHERE Version = 2)
|
||||
INSERT [$(Schema)].[SchemaVersion] (Version, AppliedUtc) VALUES (2, SYSUTCDATETIME())
|
||||
GO
|
||||
@@ -5,13 +5,18 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer;
|
||||
/// <summary>
|
||||
/// Push-down T-SQL for processed (aggregate) reads (spec §4 "Processed"), one query per
|
||||
/// <see cref="AggregateFamily"/> (<see cref="AggregateFamilies.FamilyOf"/> picks it). Every query takes
|
||||
/// <c>@id</c> (int TagId), <c>@start</c>/<c>@end</c> (<c>datetime2(7)</c>, the forward window
|
||||
/// <c>[start, end)</c>) and <c>@intervalTicks</c> (bigint bucket width in 100 ns ticks), and returns one row
|
||||
/// per <b>non-empty</b> bucket. The caller generates every interval and fills the empty ones.
|
||||
/// <c>@id</c> (int TagId), <c>@start</c>/<c>@end</c> (<c>datetime2(7)</c>, the earlier / later window edges —
|
||||
/// <see cref="BucketPlan.StartUtc"/>/<see cref="BucketPlan.EndUtc"/>), <c>@intervalTicks</c> (bigint bucket width in
|
||||
/// 100 ns ticks) and <c>@bucketAdjustTicks</c> (bigint, <see cref="BucketPlan.BucketIndexAdjustTicks"/>), and returns
|
||||
/// one row per <b>non-empty</b> bucket (ascending bucket index). The caller generates every interval and fills the
|
||||
/// empty ones.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>Bucketing.</b> <c>b = elapsedTicks / @intervalTicks</c>, where
|
||||
/// <b>Bucketing.</b> <c>b = (elapsedTicks + @bucketAdjustTicks) / @intervalTicks</c> (#677: the adjust is 0 for a
|
||||
/// forward read and re-anchors a reverse read's grid at its later time — <see cref="BucketPlan.IndexOf"/> is the C#
|
||||
/// statement of the same math). The window is <c>[@start, @end)</c> forward and <c>(@start, @end]</c> reverse (a
|
||||
/// reverse read includes its later time), selected by the <c>reverse</c> argument. <c>elapsedTicks</c> is
|
||||
/// <c>elapsedTicks = DATEDIFF_BIG(microsecond, @start, t) * 10 + tick-of-microsecond(t) − tick-of-microsecond(@start)</c>
|
||||
/// and <c>tick-of-microsecond(x) = (DATEPART(nanosecond, x) / 100) % 10</c>. The spec sketched
|
||||
/// <c>millisecond</c>, but <c>DATEDIFF</c> counts unit <i>boundaries crossed</i>, not elapsed time, so with a
|
||||
@@ -27,7 +32,8 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer;
|
||||
/// and bit 30): <i>usable</i> = severity Good or Uncertain (<c>StatusCode >= 0</c>) AND a stored value
|
||||
/// (<c>ValueKind <> 0</c>) — an <c>Empty</c> sample is an outage marker and counts as Bad whatever its
|
||||
/// code; <i>good</i> = usable with severity Good (<c>StatusCode < 0x40000000</c>); <i>bad</i> = not
|
||||
/// usable. Every value column below is computed over usable samples only.
|
||||
/// usable. Average/Minimum/Maximum/Range/*ActualTime value columns are computed over <i>good</i> samples only
|
||||
/// (#684); Start/End/Delta picks over usable samples (or every sample).
|
||||
/// </para>
|
||||
/// Common result columns (ordinals 0–5): <c>Bucket, TotalCount, UsableCount, GoodCount, BadCount,
|
||||
/// NonNumericCount</c> (usable String/DateTime samples). Family columns follow.
|
||||
@@ -41,64 +47,185 @@ internal static class SqlAggregateQueries
|
||||
private const string NonNumericKinds = "4, 5";
|
||||
|
||||
/// <summary>
|
||||
/// Average / Minimum / Maximum / Total / Count / Range / PercentGood / PercentBad.
|
||||
/// Family columns: <c>6 AvgValue, 7 MinValue, 8 MaxValue, 9 SumValue</c> (float, NULL when no usable
|
||||
/// numeric sample).
|
||||
/// Average / Minimum / Maximum / Count / Range (PercentGood / PercentBad and Total are time-weighted, not here).
|
||||
/// Family columns: <c>6 AvgValue, 7 MinValue, 8 MaxValue, 9 SumValue</c> over the <b>Good</b> numeric samples (float,
|
||||
/// NULL when none — #684: Part 13 and the SDK compute these over Good values only), then <c>10 UncMin, 11 UncMax</c>
|
||||
/// over the Uncertain ones (the Part 13 Minimum/Maximum/Range status rule, <see cref="AggregateEvaluator"/>), and
|
||||
/// <c>12 GoodNonNumericCount</c>.
|
||||
/// </summary>
|
||||
/// <param name="sample">Schema-qualified <c>Sample</c> table name.</param>
|
||||
/// <returns>The SQL text.</returns>
|
||||
public static string Simple(string sample) =>
|
||||
Classified(sample) +
|
||||
/// <param name="reverse">True for a reverse read (window <c>(@start, @end]</c>).</param>
|
||||
public static string Simple(string sample, bool reverse = false) =>
|
||||
Classified(sample, reverse) +
|
||||
"SELECT b, " + CommonCounts() + ", " +
|
||||
"AVG(CASE WHEN u = 1 THEN NumValue END) AS AvgValue, " +
|
||||
"MIN(CASE WHEN u = 1 THEN NumValue END) AS MinValue, " +
|
||||
"MAX(CASE WHEN u = 1 THEN NumValue END) AS MaxValue, " +
|
||||
"SUM(CASE WHEN u = 1 THEN NumValue END) AS SumValue " +
|
||||
"AVG(CASE WHEN g = 1 THEN NumValue END) AS AvgValue, " +
|
||||
"MIN(CASE WHEN g = 1 THEN NumValue END) AS MinValue, " +
|
||||
"MAX(CASE WHEN g = 1 THEN NumValue END) AS MaxValue, " +
|
||||
"SUM(CASE WHEN g = 1 THEN NumValue END) AS SumValue, " +
|
||||
UncertainExtremes() + " " +
|
||||
"FROM s GROUP BY b ORDER BY b;";
|
||||
|
||||
/// <summary>
|
||||
/// Start / End / Delta: the first and last usable sample of each bucket by <c>SourceTimeUtc</c>.
|
||||
/// Family columns: <c>6..10</c> first sample (<c>ValueKind, NumValue, IntValue, StrValue, SourceTimeUtc</c>),
|
||||
/// <c>11..15</c> last sample (same order); all NULL when the bucket has no usable sample.
|
||||
/// </summary>
|
||||
/// <param name="sample">Schema-qualified <c>Sample</c> table name.</param>
|
||||
/// <returns>The SQL text.</returns>
|
||||
public static string StartEnd(string sample) =>
|
||||
Classified(sample) +
|
||||
", st AS (SELECT b, " + CommonCounts() + " FROM s GROUP BY b)" +
|
||||
", f AS (SELECT b, ValueKind, NumValue, IntValue, StrValue, SourceTimeUtc, " +
|
||||
"ROW_NUMBER() OVER (PARTITION BY b ORDER BY SourceTimeUtc ASC) AS rf, " +
|
||||
"ROW_NUMBER() OVER (PARTITION BY b ORDER BY SourceTimeUtc DESC) AS rl FROM s WHERE u = 1) " +
|
||||
"SELECT st.*, " +
|
||||
"x.ValueKind, x.NumValue, x.IntValue, x.StrValue, x.SourceTimeUtc, " +
|
||||
"y.ValueKind, y.NumValue, y.IntValue, y.StrValue, y.SourceTimeUtc " +
|
||||
"FROM st LEFT JOIN f x ON x.b = st.b AND x.rf = 1 LEFT JOIN f y ON y.b = st.b AND y.rl = 1 ORDER BY st.b;";
|
||||
/// <summary>Ordinal of the StartEnd family's first-sample-of-any-quality columns (<see cref="StartEnd"/>).</summary>
|
||||
public const int FirstAnyColumn = FirstFamilyColumn + 10;
|
||||
|
||||
/// <summary>
|
||||
/// MinimumActualTime / MaximumActualTime: the usable sample with the smallest (largest) <c>NumValue</c>,
|
||||
/// ties broken by the earliest <c>SourceTimeUtc</c>. Family columns: <c>6 NumValue, 7 SourceTimeUtc</c>
|
||||
/// (NULL when the bucket has no usable numeric sample).
|
||||
/// Start / End / Delta: the first and last usable sample of each bucket by <c>SourceTimeUtc</c> (Delta's value), and
|
||||
/// the first and last sample of <b>any</b> quality (Start / End, and Delta's status — #680).
|
||||
/// Family columns: <c>6..10</c> first usable sample (<c>ValueKind, NumValue, IntValue, StrValue, SourceTimeUtc</c>),
|
||||
/// <c>11..15</c> last usable sample (same order) — all NULL when the bucket has no usable sample; then
|
||||
/// <c>16..21</c> the first sample of any quality (<c>SourceTimeUtc, ValueKind, NumValue, IntValue, StrValue,
|
||||
/// StatusCode</c>) and <c>22..27</c> the last (same order) — present for every returned (non-empty) bucket.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>Aggregate, then join back on the key (#683).</b> One <c>GROUP BY</c> over the window computes the counts and
|
||||
/// each bucket's four pick TIMES — <c>MIN</c>/<c>MAX(SourceTimeUtc)</c> over every sample (any quality) and over
|
||||
/// the usable ones — and each pick's columns are then fetched by a seek on the clustered key
|
||||
/// <c>(TagId, SourceTimeUtc)</c> (unique, so one row per pick; a NULL usable time joins nothing ⇒ NULL columns,
|
||||
/// as before). Output is identical to the previous windowed form (one <c>ROW_NUMBER</c> pass + conditional
|
||||
/// <c>MAX</c>, live-gate 8 F1) — <c>EXCEPT</c> both ways is empty on 6M rows, forward and reverse — without that
|
||||
/// form's four window sorts, which spilled: on 6M rows / 30 days, 30 d @ 1 h ~3.4 s vs 18–32 s, 7 d @ 1 d
|
||||
/// ~0.8 s vs 5.2–6.1 s, 1 d @ 1 min ~0.16 s vs 0.31–0.40 s (docs/Historian.md).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// ⚠️ The key seeks are separate reads of the table within one statement, so under READ COMMITTED a sample
|
||||
/// deleted (HistoryUpdate DeleteRaw/DeleteAtTime) between the aggregate and its seek leaves that pick NULL while
|
||||
/// the counts still include it. The reader maps a NULL pick to "absent" (<c>ReadStoredOrNull</c>, live-gate 8 F1)
|
||||
/// and the evaluator answers that bucket's Start/End/Delta <c>BadNoData</c> — transient, until the next read. A
|
||||
/// sample replaced in between is picked with its new value (same time) — except that a usable pick re-checks
|
||||
/// usability in its join (the exact <c>u</c> predicate), so a Replace that turned it Bad/Empty reads as absent
|
||||
/// (<c>BadNoData</c>), never as a Bad row's value under Delta. A sample inserted in between is in neither half
|
||||
/// (the pick times come from the aggregate).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Every pick is an exact-key seek, by hint (#683 live gate 9 F1).</b> Each pick join carries
|
||||
/// <c>WITH (FORCESEEK (PK_Sample (TagId, SourceTimeUtc)))</c>, so it can only be a nested-loops seek on BOTH key
|
||||
/// columns — one row per bucket per pick — whatever the statistics say. Without it the plan was the optimizer's
|
||||
/// choice, and both alternatives it found were pathological: keyed on the pick time alone, a hash/merge join whose
|
||||
/// inner input was the tag's whole history (review of 910befc8: 43,657 vs 361 logical reads); bounded to the window
|
||||
/// as well (4a3b1256), a nested-loops RANGE seek over the window with the pick time as a residual, once per bucket
|
||||
/// — O(buckets × window rows), chosen under understated statistics and then reused from the plan cache (gate 9:
|
||||
/// 1 d @ 1 min 20 s; here 47 s, 3.6M reads, vs 0.25 s). With the hint neither shape exists (a forced
|
||||
/// <c>HASH</c>/<c>MERGE JOIN</c> is refused, error 8622), so the window bound is no longer needed in the join.
|
||||
/// ⚠️ The hint names the clustered primary key <c>PK_Sample</c> created by <c>001_initial.sql</c>; a hand-built
|
||||
/// schema must keep that name.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
/// <param name="sample">Schema-qualified <c>Sample</c> table name.</param>
|
||||
/// <returns>The SQL text.</returns>
|
||||
/// <param name="reverse">True for a reverse read (window <c>(@start, @end]</c>).</param>
|
||||
public static string StartEnd(string sample, bool reverse = false)
|
||||
{
|
||||
// A pick's columns from its joined row, in the established order: usable picks (kind, num, int, str, time), any-
|
||||
// quality picks (time, kind, num, int, str, status).
|
||||
static string Usable(string a) => $"{a}.ValueKind, {a}.NumValue, {a}.IntValue, {a}.StrValue, {a}.SourceTimeUtc";
|
||||
static string Any(string a) => $"{a}.SourceTimeUtc, {a}.ValueKind, {a}.NumValue, {a}.IntValue, {a}.StrValue, {a}.StatusCode";
|
||||
// An exact-key seek per pick, forced (see remarks). A usable pick also re-checks usability (the exact `u`
|
||||
// predicate), so a sample a HistoryUpdate Replace turned Bad/Empty in between joins nothing ⇒ absent ⇒ BadNoData,
|
||||
// never a Bad row's value.
|
||||
string Join(string a, string time, bool usable) =>
|
||||
$"LEFT JOIN {sample} {a} WITH (FORCESEEK (PK_Sample (TagId, SourceTimeUtc))) ON {a}.TagId = @id AND {a}.SourceTimeUtc = st.{time}" +
|
||||
(usable ? $" AND {a}.StatusCode >= 0 AND {a}.ValueKind <> 0" : "") + " ";
|
||||
return Classified(sample, reverse) +
|
||||
", st AS (SELECT b, " + CommonCounts() + ", " +
|
||||
"MIN(CASE WHEN u = 1 THEN SourceTimeUtc END) AS fu, MAX(CASE WHEN u = 1 THEN SourceTimeUtc END) AS lu, " +
|
||||
"MIN(SourceTimeUtc) AS fa, MAX(SourceTimeUtc) AS la FROM s GROUP BY b) " +
|
||||
"SELECT st.b, st.TotalCount, st.UsableCount, st.GoodCount, st.BadCount, st.NonNumericCount, " +
|
||||
Usable("x") + ", " + Usable("y") + ", " + Any("p") + ", " + Any("q") + " FROM st " +
|
||||
Join("x", "fu", usable: true) + Join("y", "lu", usable: true) + Join("p", "fa", usable: false) + Join("q", "la", usable: false) + "ORDER BY st.b;";
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// MinimumActualTime / MaximumActualTime: the <b>Good</b> sample (#684) with the smallest (largest) <c>NumValue</c>,
|
||||
/// ties broken by the earliest <c>SourceTimeUtc</c>. Family columns: <c>6 NumValue, 7 SourceTimeUtc</c>
|
||||
/// (NULL when the bucket has no Good numeric sample), then <c>8 UncMin, 9 UncMax, 10 GoodNonNumericCount</c> as in <see cref="Simple"/>.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Deliberately still two references to <c>s</c> (counts + the ranked extreme), unlike <see cref="StartEnd"/>:
|
||||
/// every single-scan form measured (a <c>ROW_NUMBER</c> or windowed <c>MIN</c>/<c>MAX</c> over every row) was
|
||||
/// 1.2–2× slower on day-plus windows than ranking only the filtered usable rows. Live-gate 8 F1's inconsistency —
|
||||
/// a sample committed between the two scans, counted but not ranked — is instead absorbed by the reader: a
|
||||
/// NULL extreme in a bucket with usable samples evaluates to <c>BadNoData</c> (never a value-less Good).
|
||||
/// </remarks>
|
||||
/// <param name="sample">Schema-qualified <c>Sample</c> table name.</param>
|
||||
/// <param name="maximum">True for MaximumActualTime.</param>
|
||||
/// <returns>The SQL text.</returns>
|
||||
public static string ActualTime(string sample, bool maximum) =>
|
||||
Classified(sample) +
|
||||
", st AS (SELECT b, " + CommonCounts() + " FROM s GROUP BY b)" +
|
||||
/// <param name="reverse">True for a reverse read (window <c>(@start, @end]</c>).</param>
|
||||
public static string ActualTime(string sample, bool maximum, bool reverse = false) =>
|
||||
Classified(sample, reverse) +
|
||||
", st AS (SELECT b, " + CommonCounts() + ", " + UncertainExtremes() + " FROM s GROUP BY b)" +
|
||||
", r AS (SELECT b, NumValue, SourceTimeUtc, " +
|
||||
"ROW_NUMBER() OVER (PARTITION BY b ORDER BY NumValue " + (maximum ? "DESC" : "ASC") + ", SourceTimeUtc ASC) AS rn " +
|
||||
"FROM s WHERE u = 1 AND NumValue IS NOT NULL) " +
|
||||
"SELECT st.*, r.NumValue, r.SourceTimeUtc " +
|
||||
"FROM s WHERE g = 1 AND NumValue IS NOT NULL) " +
|
||||
"SELECT st.b, st.TotalCount, st.UsableCount, st.GoodCount, st.BadCount, st.NonNumericCount, " +
|
||||
"r.NumValue, r.SourceTimeUtc, st.UncMin, st.UncMax, st.GoodNonNumericCount " +
|
||||
"FROM st LEFT JOIN r ON r.b = st.b AND r.rn = 1 ORDER BY st.b;";
|
||||
|
||||
/// <summary>
|
||||
/// Time-weighted family (#671: TimeAverage / TimeAverage2 / Total / Total2 / Interpolative) — <b>not</b> pushed down.
|
||||
/// One batch of five result sets, each <c>SourceTimeUtc, ValueKind, NumValue, IntValue, StrValue, StatusCode</c>:
|
||||
/// (1) the latest usable sample before <c>@start</c>, (2) the latest sample of any quality before <c>@start</c>,
|
||||
/// (3) the samples in <c>[@start, @end)</c> ascending, at most <c>@maxRows</c> (the raw-row cap + 1), (4) the first sample of any quality at or after
|
||||
/// <c>@end</c>, (5) the first usable one at or after <c>@end</c>. Each bound is a <c>TOP 1</c> seek on the
|
||||
/// <c>(TagId, SourceTimeUtc)</c> key. The caller streams them, in that order, into
|
||||
/// <see cref="TimeWeightedAggregator"/> (shared with QuestDB, so both backends agree exactly).
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Trade-off: the window's rows travel to the server instead of one row per bucket. Part 13's per-segment
|
||||
/// interpolation, bucket-edge splitting and the two bounding-value definitions would otherwise have to be written
|
||||
/// twice (T-SQL and QuestDB SQL) and kept bit-identical; evaluating once in C# keeps the backends equal by
|
||||
/// construction. Memory stays O(buckets): the rows are streamed, never materialised.
|
||||
/// </remarks>
|
||||
/// <param name="sample">Schema-qualified <c>Sample</c> table name.</param>
|
||||
/// <returns>The SQL batch.</returns>
|
||||
public static string TimeWeighted(string sample)
|
||||
{
|
||||
const string cols = "SourceTimeUtc, ValueKind, NumValue, IntValue, StrValue, StatusCode";
|
||||
const string usable = " AND StatusCode >= 0 AND ValueKind <> 0";
|
||||
var key = $"FROM {sample} WHERE TagId = @id AND ";
|
||||
return
|
||||
$"SELECT TOP 1 {cols} {key}SourceTimeUtc < @start{usable} ORDER BY SourceTimeUtc DESC; " +
|
||||
$"SELECT TOP 1 {cols} {key}SourceTimeUtc < @start ORDER BY SourceTimeUtc DESC; " +
|
||||
$"SELECT TOP (@maxRows) {cols} {key}SourceTimeUtc >= @start AND SourceTimeUtc < @end ORDER BY SourceTimeUtc ASC; " +
|
||||
$"SELECT TOP 1 {cols} {key}SourceTimeUtc >= @end ORDER BY SourceTimeUtc ASC; " +
|
||||
$"SELECT TOP 1 {cols} {key}SourceTimeUtc >= @end{usable} ORDER BY SourceTimeUtc ASC;";
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The tag's data range as one row of two columns (each <c>NULL</c> when none): <c>0</c> the earliest stored sample
|
||||
/// of any quality at or before the window's later edge, <c>1</c> the latest at or after its earlier edge. Used by the
|
||||
/// start-of-data (#677) and end-of-data (#680) Partial rules and by Count's in-range empty interval (#680) — only when
|
||||
/// <see cref="AggregateEvaluator.NeedsDataRange"/> says the answer can change a result. Each is a <c>MIN</c> /
|
||||
/// <c>MAX</c> over the <c>(TagId, SourceTimeUtc)</c> key — a single seek — bounded by the window (a sample past the
|
||||
/// later edge cannot start the data inside it, nor one before the earlier edge end it there; a NULL then means "the
|
||||
/// data range does not reach the window").
|
||||
/// </summary>
|
||||
/// <param name="sample">Schema-qualified <c>Sample</c> table name.</param>
|
||||
/// <returns>The SQL text.</returns>
|
||||
public static string DataRange(string sample) =>
|
||||
$"SELECT (SELECT MIN(SourceTimeUtc) FROM {sample} WHERE TagId = @id AND SourceTimeUtc <= @end), " +
|
||||
$"(SELECT MAX(SourceTimeUtc) FROM {sample} WHERE TagId = @id AND SourceTimeUtc >= @start);";
|
||||
|
||||
/// <summary>The <c>s</c> CTE: the window's samples with bucket index and the usable/good flags.</summary>
|
||||
private static string Classified(string sample) =>
|
||||
private static string Classified(string sample, bool reverse) =>
|
||||
"WITH s AS (SELECT (DATEDIFF_BIG(microsecond, @start, SourceTimeUtc) * 10 " +
|
||||
"+ (DATEPART(nanosecond, SourceTimeUtc) / 100) % 10 - (DATEPART(nanosecond, @start) / 100) % 10) / @intervalTicks AS b, " +
|
||||
"SourceTimeUtc, ValueKind, NumValue, IntValue, StrValue, " +
|
||||
"+ (DATEPART(nanosecond, SourceTimeUtc) / 100) % 10 - (DATEPART(nanosecond, @start) / 100) % 10 " +
|
||||
"+ @bucketAdjustTicks) / @intervalTicks AS b, " +
|
||||
"SourceTimeUtc, ValueKind, NumValue, IntValue, StrValue, StatusCode, " +
|
||||
"CASE WHEN StatusCode >= 0 AND ValueKind <> 0 THEN 1 ELSE 0 END AS u, " +
|
||||
"CASE WHEN StatusCode >= 0 AND StatusCode < 1073741824 AND ValueKind <> 0 THEN 1 ELSE 0 END AS g " +
|
||||
$"FROM {sample} WHERE TagId = @id AND SourceTimeUtc >= @start AND SourceTimeUtc < @end) ";
|
||||
$"FROM {sample} WHERE TagId = @id AND " +
|
||||
(reverse ? "SourceTimeUtc > @start AND SourceTimeUtc <= @end) " : "SourceTimeUtc >= @start AND SourceTimeUtc < @end) ");
|
||||
|
||||
/// <summary>
|
||||
/// The minimum and maximum <b>Uncertain</b> (usable, not Good) numeric value of the bucket — <c>UncMin, UncMax</c>
|
||||
/// (#684). Not result values: Part 13 makes Minimum/Maximum/Range Uncertain when one lies beyond the Good extreme.
|
||||
/// Then <c>GoodNonNumericCount</c>, the Good String/DateTime samples (the Good-value aggregates' refusal check).
|
||||
/// </summary>
|
||||
private static string UncertainExtremes() =>
|
||||
"MIN(CASE WHEN u = 1 AND g = 0 THEN NumValue END) AS UncMin, MAX(CASE WHEN u = 1 AND g = 0 THEN NumValue END) AS UncMax, " +
|
||||
$"SUM(CASE WHEN g = 1 AND ValueKind IN ({NonNumericKinds}) THEN 1 ELSE 0 END) AS GoodNonNumericCount";
|
||||
|
||||
/// <summary>The five common count columns (after <c>b</c>).</summary>
|
||||
private static string CommonCounts() =>
|
||||
|
||||
@@ -106,7 +106,7 @@ internal sealed class SqlHistorianConnection
|
||||
/// <param name="reader">Reader positioned on a row.</param>
|
||||
/// <param name="ordinal">Column ordinal.</param>
|
||||
/// <returns>UTC timestamp.</returns>
|
||||
public static DateTime GetUtc(SqlDataReader reader, int ordinal) =>
|
||||
public static DateTime GetUtc(System.Data.Common.DbDataReader reader, int ordinal) =>
|
||||
DateTime.SpecifyKind(reader.GetDateTime(ordinal), DateTimeKind.Utc);
|
||||
|
||||
/// <summary>True for connection/timeout/deadlock/throttling errors worth retrying.</summary>
|
||||
|
||||
+254
-29
@@ -1,4 +1,5 @@
|
||||
using System.Data;
|
||||
using System.Data.Common;
|
||||
using System.Text;
|
||||
using Microsoft.Data.SqlClient;
|
||||
using Microsoft.Extensions.Logging;
|
||||
@@ -33,7 +34,7 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer;
|
||||
/// carries a value; an <c>Empty</c>-kind sample is an outage marker (comms-loss <c>BadNoCommunication</c>
|
||||
/// or a driver's Bad publish) and is Bad whatever its code. Raw returns every sample as stored, outage
|
||||
/// markers included. Aggregates exclude Bad samples from the math, report the interval Uncertain when
|
||||
/// any non-Good sample is present, and count them in PercentBad. AtTime never interpolates across a Bad
|
||||
/// any non-Good sample is present, and count their duration as Bad time in PercentBad. AtTime never interpolates across a Bad
|
||||
/// neighbour and reports a time inside an outage with the outage's Bad code (see <see cref="ReadAtTimeAsync"/>).
|
||||
/// <b>An Empty sample is always served Bad:</b> a row with no value whose stored status is not Bad (only
|
||||
/// possible for rows written before the writer forced Empty samples Bad) is read back as <c>BadNoData</c>,
|
||||
@@ -59,6 +60,7 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
|
||||
private readonly TimeProvider _time;
|
||||
|
||||
private readonly HistorianHealthCounters _health = new();
|
||||
private readonly int _maxTimeWeightedRawRows;
|
||||
|
||||
/// <summary>Creates the data source over the provider's shared connection, bootstrapper and tag catalog.</summary>
|
||||
/// <param name="conn">Connection helper.</param>
|
||||
@@ -66,10 +68,12 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
|
||||
/// <param name="catalog">Tag catalog (shared; reads use only its read-only lookup).</param>
|
||||
/// <param name="log">Optional logger.</param>
|
||||
/// <param name="timeProvider">Clock for "now" (AtTime beyond the last sample, server timestamps); system by default.</param>
|
||||
/// <param name="maxTimeWeightedRawRows">Raw in-window row cap of one time-weighted read (<= 0 = unbounded).</param>
|
||||
internal SqlServerHistorianDataSource(
|
||||
SqlHistorianConnection conn, SqlServerSchemaBootstrapper bootstrap, SqlServerTagCatalog catalog, ILogger? log = null,
|
||||
TimeProvider? timeProvider = null)
|
||||
TimeProvider? timeProvider = null, int maxTimeWeightedRawRows = TimeWeightedAggregator.DefaultMaxWindowRows)
|
||||
{
|
||||
_maxTimeWeightedRawRows = maxTimeWeightedRawRows;
|
||||
_conn = conn ?? throw new ArgumentNullException(nameof(conn));
|
||||
_bootstrap = bootstrap ?? throw new ArgumentNullException(nameof(bootstrap));
|
||||
_catalog = catalog ?? throw new ArgumentNullException(nameof(catalog));
|
||||
@@ -128,6 +132,105 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
|
||||
catch (Exception ex)
|
||||
{
|
||||
RecordFailure(ex);
|
||||
if (IsTransportFailure(ex)) throw new System.IO.IOException("Historian read transport is unavailable.", ex);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------------------------------ Modified (Part 11, #668)
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>
|
||||
/// Reads <c>SampleModified</c> (schema v2), the trail <see cref="SqlServerHistorianDataUpdater"/> writes in the
|
||||
/// same statement as each change. Windows follow <see cref="ReadRawAsync"/>; rows are ordered by
|
||||
/// <c>(SourceTimeUtc, ModifiedId)</c> (the clustered key after <c>TagId</c>, so a seek). <c>maxValues + 1</c> rows
|
||||
/// are fetched to report <see cref="HistoryModifiedReadResult.HasMore"/> exactly. An insert's audit row carries no
|
||||
/// prior value and is served <c>null</c> / <c>BadNoData</c>; its server timestamp is the modification time. An
|
||||
/// unknown tag is an empty result.
|
||||
/// </remarks>
|
||||
public async Task<HistoryModifiedReadResult> ReadModifiedAsync(
|
||||
string fullReference, DateTime startUtc, DateTime endUtc, uint maxValues, HistoryModifiedCursor? resumeAfter,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(fullReference);
|
||||
var bounded = maxValues > 0 && maxValues < int.MaxValue;
|
||||
try
|
||||
{
|
||||
// #674: a v1 schema has no SampleModified. NotSupportedException is the interface's "no trail" signal (the
|
||||
// node manager reports BadHistoryOperationUnsupported); it is not a backend failure and sends no query.
|
||||
await _bootstrap.EnsureAsync(cancellationToken).ConfigureAwait(false);
|
||||
if (!_bootstrap.SupportsModifiedTrail)
|
||||
throw new NotSupportedException(
|
||||
$"Historian schema [{_conn.Schema}] is below version {SqlServerSchemaBootstrapper.ModifiedTrailVersion}: no modified-value history.");
|
||||
var values = new List<HistoryModifiedValue>();
|
||||
var hasMore = false;
|
||||
var tagId = await ResolveAsync(fullReference, cancellationToken).ConfigureAwait(false);
|
||||
if (tagId is { } id)
|
||||
{
|
||||
var reverse = startUtc > endUtc;
|
||||
string where;
|
||||
if (startUtc == endUtc) where = "SourceTimeUtc = @lo";
|
||||
else if (!reverse) where = "SourceTimeUtc >= @lo AND SourceTimeUtc < @hi";
|
||||
else where = "SourceTimeUtc > @lo AND SourceTimeUtc <= @hi";
|
||||
if (resumeAfter is not null)
|
||||
where += reverse
|
||||
? " AND (SourceTimeUtc < @ct OR (SourceTimeUtc = @ct AND ModifiedId < @cid))"
|
||||
: " AND (SourceTimeUtc > @ct OR (SourceTimeUtc = @ct AND ModifiedId > @cid))";
|
||||
var order = reverse ? "DESC" : "ASC";
|
||||
var (lo, hi) = startUtc <= endUtc ? (startUtc, endUtc) : (endUtc, startUtc);
|
||||
|
||||
await using var conn = await _conn.OpenAsync(cancellationToken).ConfigureAwait(false);
|
||||
await using var cmd = _conn.CreateCommand(conn,
|
||||
$"SELECT {(bounded ? "TOP (@take) " : "")}SourceTimeUtc, ValueKind, NumValue, IntValue, StrValue, StatusCode, " +
|
||||
"ServerTimeUtc, ModifiedId, ModificationTimeUtc, UpdateType, Principal " +
|
||||
$"FROM {_conn.Q("SampleModified")} WHERE TagId = @id AND {where} " +
|
||||
$"ORDER BY SourceTimeUtc {order}, ModifiedId {order}");
|
||||
if (bounded) cmd.Parameters.Add(new SqlParameter("@take", SqlDbType.BigInt) { Value = (long)maxValues + 1 });
|
||||
cmd.Parameters.Add(new SqlParameter("@id", SqlDbType.Int) { Value = id });
|
||||
SqlHistorianConnection.AddDateTime2(cmd, "@lo", lo);
|
||||
SqlHistorianConnection.AddDateTime2(cmd, "@hi", hi);
|
||||
if (resumeAfter is not null)
|
||||
{
|
||||
SqlHistorianConnection.AddDateTime2(cmd, "@ct", resumeAfter.SourceTimeUtc);
|
||||
cmd.Parameters.Add(new SqlParameter("@cid", SqlDbType.BigInt) { Value = resumeAfter.ModifiedId });
|
||||
}
|
||||
|
||||
await using var reader = await cmd.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false);
|
||||
while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
if (bounded && values.Count == maxValues)
|
||||
{
|
||||
hasMore = true;
|
||||
break;
|
||||
}
|
||||
|
||||
var sourceTime = SqlHistorianConnection.GetUtc(reader, 0);
|
||||
var modifiedAt = SqlHistorianConnection.GetUtc(reader, 8);
|
||||
DataValueSnapshot prior;
|
||||
if (reader.IsDBNull(1))
|
||||
{
|
||||
// An insert: there was no prior value.
|
||||
prior = new DataValueSnapshot(null, HistorianStatusCodes.BadNoData, sourceTime, modifiedAt);
|
||||
}
|
||||
else
|
||||
{
|
||||
var s = ReadStored(reader, 0);
|
||||
prior = new DataValueSnapshot(s.Value.ToObject(), s.Status, s.TimeUtc,
|
||||
reader.IsDBNull(6) ? modifiedAt : SqlHistorianConnection.GetUtc(reader, 6));
|
||||
}
|
||||
|
||||
values.Add(new HistoryModifiedValue(
|
||||
prior, modifiedAt, (HistoryModificationType)reader.GetByte(9), reader.GetString(10), reader.GetInt64(7), sourceTime));
|
||||
}
|
||||
}
|
||||
|
||||
RecordSuccess();
|
||||
return new HistoryModifiedReadResult(values, hasMore);
|
||||
}
|
||||
catch (Exception ex) when (ex is not NotSupportedException)
|
||||
{
|
||||
RecordFailure(ex);
|
||||
if (IsTransportFailure(ex)) throw new System.IO.IOException("Historian read transport is unavailable.", ex);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
@@ -205,12 +308,30 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
|
||||
catch (Exception ex)
|
||||
{
|
||||
RecordFailure(ex);
|
||||
if (IsTransportFailure(ex)) throw new System.IO.IOException("Historian read transport is unavailable.", ex);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------------------------------ Events
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>Reverse raw reads are served natively (<c>(end, start]</c>, <c>ORDER BY … DESC</c>; #669/#686).</remarks>
|
||||
public bool SupportsReverseRaw => true;
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks><c>0</c>: a capped raw read is one bounded statement — no backend row ceiling (Gitea #694).</remarks>
|
||||
public int MaxRawValuesPerRead => 0;
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>Reverse events reads are served natively (<c>ORDER BY … DESC</c>, Gitea #675).</remarks>
|
||||
public bool SupportsReverseEvents => true;
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>The seventeen <see cref="AggregateFamilies.Served"/> — shared with the other SQL backend (#682).</remarks>
|
||||
public IReadOnlySet<HistoryAggregateType> SupportedAggregates => AggregateFamilies.Served;
|
||||
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>
|
||||
/// <c>[start, end)</c> by <c>EventTimeUtc</c> (a reversed window is swapped), ordered by
|
||||
@@ -220,8 +341,18 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
|
||||
/// of <c>"{EventTimeUtc.Ticks}|{EventId}"</c> of that first not-returned event (opaque to callers; only
|
||||
/// its presence is contractual).
|
||||
/// </remarks>
|
||||
public Task<HistoricalEventsResult> ReadEventsAsync(
|
||||
string? sourceName, DateTime startUtc, DateTime endUtc, int maxEvents, CancellationToken cancellationToken) =>
|
||||
ReadEventsAsync(sourceName, startUtc, endUtc, maxEvents, reverse: false, cancellationToken);
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>
|
||||
/// Forward is the five-argument read. Reverse (Gitea #675) is <c>(lo, hi]</c> ordered by
|
||||
/// <c>(EventTimeUtc, EventId)</c> DESCENDING — the clustered index scanned backwards — so <c>TOP</c> keeps the latest
|
||||
/// events; truncation and the continuation point work as forward.
|
||||
/// </remarks>
|
||||
public async Task<HistoricalEventsResult> ReadEventsAsync(
|
||||
string? sourceName, DateTime startUtc, DateTime endUtc, int maxEvents, CancellationToken cancellationToken)
|
||||
string? sourceName, DateTime startUtc, DateTime endUtc, int maxEvents, bool reverse, CancellationToken cancellationToken)
|
||||
{
|
||||
// No stored event can carry a source longer than the column (the alarm writer dead-letters it rather than
|
||||
// truncating), so an over-long filter matches nothing — never a truncated prefix of someone else's source.
|
||||
@@ -234,9 +365,10 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
|
||||
var bounded = maxEvents > 0 && maxEvents < int.MaxValue;
|
||||
var sql =
|
||||
$"SELECT {(bounded ? "TOP (@take) " : "")}EventId, SourceName, EventTimeUtc, ReceivedTimeUtc, Message, Severity " +
|
||||
$"FROM {_conn.Q("Event")} WHERE EventTimeUtc >= @lo AND EventTimeUtc < @hi" +
|
||||
$"FROM {_conn.Q("Event")} WHERE " +
|
||||
(reverse ? "EventTimeUtc > @lo AND EventTimeUtc <= @hi" : "EventTimeUtc >= @lo AND EventTimeUtc < @hi") +
|
||||
(sourceName is null ? "" : " AND SourceName = @source") +
|
||||
" ORDER BY EventTimeUtc ASC, EventId ASC";
|
||||
(reverse ? " ORDER BY EventTimeUtc DESC, EventId DESC" : " ORDER BY EventTimeUtc ASC, EventId ASC");
|
||||
|
||||
await using var conn = await _conn.OpenAsync(cancellationToken).ConfigureAwait(false);
|
||||
await using var cmd = _conn.CreateCommand(conn, sql);
|
||||
@@ -272,6 +404,7 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
|
||||
catch (Exception ex)
|
||||
{
|
||||
RecordFailure(ex);
|
||||
if (IsTransportFailure(ex)) throw new System.IO.IOException("Historian read transport is unavailable.", ex);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
@@ -284,24 +417,31 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
|
||||
/// One result per interval of <c>[start, end)</c>, bucket <c>i</c> = <c>[start + i·interval,
|
||||
/// start + (i+1)·interval)</c> (start-inclusive, end-exclusive to the tick; the last bucket is truncated at
|
||||
/// <c>end</c>). <paramref name="interval"/> <= 0 ⇒ one interval <c>[start, end)</c>. <c>start == end</c>
|
||||
/// ⇒ no intervals. <c>end < start</c> (reverse) ⇒ the window is swapped, computed forward, and returned
|
||||
/// in descending bucket order (buckets stay anchored at the earlier time).
|
||||
/// ⇒ no intervals. <c>end < start</c> (reverse, #677) ⇒ Part 13 / SDK anchoring: window <c>(end, start]</c>,
|
||||
/// buckets stepping back from <c>start</c> (the partial one is the earliest), each stamped with its later edge,
|
||||
/// returned latest first (<see cref="BucketPlan"/>).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Per interval: no samples ⇒ <c>BadNoData</c>. PercentGood / PercentBad ⇒ <c>100 × good|bad / all</c> over
|
||||
/// every sample, status Good. Otherwise no usable sample ⇒ <c>BadNoData</c>; a numeric aggregate with a usable
|
||||
/// String/DateTime sample ⇒ <c>BadAggregateInvalidInputs</c>; else the value, Good when every sample is Good,
|
||||
/// <c>UncertainDataSubNormal</c> when any is Bad or Uncertain. Every status — including <c>BadNoData</c> —
|
||||
/// carries the Calculated info bits. Timestamps are the bucket start, except Minimum/MaximumActualTime
|
||||
/// (the sample's time; ties ⇒ earliest).
|
||||
/// Per-sample aggregates, per interval: no samples ⇒ <c>BadNoData</c>. Otherwise no usable sample ⇒
|
||||
/// <c>BadNoData</c>; a numeric aggregate with a usable String/DateTime sample ⇒ <c>BadAggregateInvalidInputs</c>;
|
||||
/// else the value, Good when every sample is Good, <c>UncertainDataSubNormal</c> when any is Bad or Uncertain.
|
||||
/// Every calculated status — including <c>BadNoData</c> — carries the Calculated info bits. Timestamps are the
|
||||
/// bucket start (its later edge on a reverse read), except Minimum/MaximumActualTime (the sample's time; ties ⇒
|
||||
/// earliest) and Start/End. Every aggregate but Average sets the Partial bit on the bucket the tag's earliest
|
||||
/// sample lies strictly inside (forward, #677) or its latest sample lies in (#680; reverse: strictly inside).
|
||||
/// PercentGood / PercentBad are duration-based since #680 (time-weighted family, below). #680: Count counts Good samples (value-based status; 0/Good on an empty bucket inside the tag's
|
||||
/// data range), Start/End are the raw first/last sample of any quality, Delta is Uncertain only when the bucket's
|
||||
/// first or last sample is not usable (see <see cref="AggregateEvaluator"/>).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Values: Count is <see cref="int"/>; Start/End keep the sample's type; every other aggregate is
|
||||
/// <see cref="double"/> computed over <c>NumValue</c> (Boolean = 0/1). These are the simple per-sample
|
||||
/// definitions, not Part 13's interval-boundary (interpolated bound / time-weighted) semantics.
|
||||
/// definitions. The Part 13 time-weighted family (TimeAverage, TimeAverage2, Total, Total2, Interpolative, #671) is
|
||||
/// not pushed down: the window's rows plus four bounding rows are streamed into the shared
|
||||
/// <see cref="TimeWeightedAggregator"/>, which fills every interval (an empty one between bounds gets a value).
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
/// <exception cref="HistoryAggregateNotSupportedException">The aggregate is not one of the thirteen served.</exception>
|
||||
/// <exception cref="HistoryAggregateNotSupportedException">The aggregate is not one of the seventeen served.</exception>
|
||||
/// <exception cref="ArgumentOutOfRangeException">The window holds more than <see cref="MaxProcessedBuckets"/> intervals.</exception>
|
||||
public async Task<HistoryReadResult> ReadProcessedAsync(
|
||||
string fullReference, DateTime startUtc, DateTime endUtc, TimeSpan interval,
|
||||
@@ -314,49 +454,117 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
|
||||
try
|
||||
{
|
||||
var now = _time.GetUtcNow().UtcDateTime;
|
||||
var evaluated = new Dictionary<long, DataValueSnapshot>();
|
||||
var tagId = plan.BucketCount == 0 ? null : await ResolveAsync(fullReference, cancellationToken).ConfigureAwait(false);
|
||||
if (family == AggregateFamily.TimeWeighted)
|
||||
{
|
||||
var tw = new TimeWeightedAggregator(aggregate, plan);
|
||||
if (tagId is { } twId) await StreamTimeWeightedAsync(twId, fullReference, plan, tw, cancellationToken).ConfigureAwait(false);
|
||||
var twResult = tw.Finish(now);
|
||||
RecordSuccess();
|
||||
return new HistoryReadResult(twResult, ContinuationPoint: null);
|
||||
}
|
||||
var rows = new List<BucketRow>();
|
||||
DateTime? earliest = null, latest = null;
|
||||
if (tagId is { } id)
|
||||
{
|
||||
var sample = _conn.Q("Sample");
|
||||
var sql = family switch
|
||||
{
|
||||
AggregateFamily.Simple => SqlAggregateQueries.Simple(sample),
|
||||
AggregateFamily.StartEnd => SqlAggregateQueries.StartEnd(sample),
|
||||
_ => SqlAggregateQueries.ActualTime(sample, aggregate == HistoryAggregateType.MaximumActualTime),
|
||||
AggregateFamily.Simple => SqlAggregateQueries.Simple(sample, plan.Reversed),
|
||||
AggregateFamily.StartEnd => SqlAggregateQueries.StartEnd(sample, plan.Reversed),
|
||||
_ => SqlAggregateQueries.ActualTime(sample, aggregate == HistoryAggregateType.MaximumActualTime, plan.Reversed),
|
||||
};
|
||||
// FIRST result set when needed: the tag's data range (earliest + latest sample) — the start/end-of-data Partial
|
||||
// rules (#677/#680 — any per-sample aggregate but Average, either direction) and Count's in-range empty interval
|
||||
// (#680). Read before the bucket statement (#683): under READ COMMITTED the two statements can see different
|
||||
// rows, and a sample appended in between must reach the buckets rather than only the range — the opposite
|
||||
// order turned such a bucket into a Good Count of 0.
|
||||
var needsRange = AggregateEvaluator.NeedsDataRange(aggregate, plan);
|
||||
if (needsRange) sql = SqlAggregateQueries.DataRange(sample) + " " + sql;
|
||||
await using var conn = await _conn.OpenAsync(cancellationToken).ConfigureAwait(false);
|
||||
await using var cmd = _conn.CreateCommand(conn, sql);
|
||||
cmd.Parameters.Add(new SqlParameter("@id", SqlDbType.Int) { Value = id });
|
||||
SqlHistorianConnection.AddDateTime2(cmd, "@start", DateTime.SpecifyKind(plan.StartUtc, DateTimeKind.Utc));
|
||||
SqlHistorianConnection.AddDateTime2(cmd, "@end", DateTime.SpecifyKind(plan.EndUtc, DateTimeKind.Utc));
|
||||
cmd.Parameters.Add(new SqlParameter("@intervalTicks", SqlDbType.BigInt) { Value = plan.IntervalTicks });
|
||||
cmd.Parameters.Add(new SqlParameter("@bucketAdjustTicks", SqlDbType.BigInt) { Value = plan.BucketIndexAdjustTicks });
|
||||
await using var reader = await cmd.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false);
|
||||
while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false))
|
||||
if (needsRange)
|
||||
{
|
||||
var row = ReadBucket(reader, family);
|
||||
evaluated[row.Bucket] = AggregateEvaluator.Evaluate(aggregate, row, plan.BucketStart(row.Bucket), now);
|
||||
if (await reader.ReadAsync(cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
if (!reader.IsDBNull(0)) earliest = SqlHistorianConnection.GetUtc(reader, 0);
|
||||
if (!reader.IsDBNull(1)) latest = SqlHistorianConnection.GetUtc(reader, 1);
|
||||
}
|
||||
if (!await reader.NextResultAsync(cancellationToken).ConfigureAwait(false))
|
||||
throw new InvalidOperationException("The processed-read batch returned no bucket result set.");
|
||||
}
|
||||
while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false))
|
||||
rows.Add(ReadBucket(reader, family));
|
||||
}
|
||||
|
||||
var result = plan.Assemble(evaluated, bucketStart => AggregateEvaluator.NoData(bucketStart, now));
|
||||
var result = AggregateEvaluator.EvaluateAll(aggregate, plan, rows, earliest, latest, now);
|
||||
|
||||
RecordSuccess();
|
||||
return new HistoryReadResult(result, ContinuationPoint: null);
|
||||
}
|
||||
catch (HistoryReadLimitExceededException)
|
||||
{
|
||||
// A client-request limit (the raw-row cap), not a backend fault: not counted as a failure.
|
||||
throw;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
RecordFailure(ex);
|
||||
if (IsTransportFailure(ex)) throw new System.IO.IOException("Historian read transport is unavailable.", ex);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
private static BucketRow ReadBucket(SqlDataReader reader, AggregateFamily family)
|
||||
/// <summary>
|
||||
/// Streams the time-weighted batch (<see cref="SqlAggregateQueries.TimeWeighted"/>) into <paramref name="tw"/>: the
|
||||
/// two before-bounds, the window, the two after-bounds — ascending, with duplicates dropped by the aggregator.
|
||||
/// </summary>
|
||||
private async Task StreamTimeWeightedAsync(int id, string tag, BucketPlan plan, TimeWeightedAggregator tw, CancellationToken ct)
|
||||
{
|
||||
static double? D(SqlDataReader r, int i) => r.IsDBNull(i) ? null : r.GetDouble(i);
|
||||
static long? L(SqlDataReader r, int i) => r.IsDBNull(i) ? null : r.GetInt64(i);
|
||||
static string? S(SqlDataReader r, int i) => r.IsDBNull(i) ? null : r.GetString(i);
|
||||
static HistorianValue V(SqlDataReader r, int i) =>
|
||||
await using var conn = await _conn.OpenAsync(ct).ConfigureAwait(false);
|
||||
await using var cmd = _conn.CreateCommand(conn, SqlAggregateQueries.TimeWeighted(_conn.Q("Sample")));
|
||||
cmd.Parameters.Add(new SqlParameter("@id", SqlDbType.Int) { Value = id });
|
||||
SqlHistorianConnection.AddDateTime2(cmd, "@start", DateTime.SpecifyKind(plan.StartUtc, DateTimeKind.Utc));
|
||||
SqlHistorianConnection.AddDateTime2(cmd, "@end", DateTime.SpecifyKind(plan.EndUtc, DateTimeKind.Utc));
|
||||
cmd.Parameters.Add(new SqlParameter("@maxRows", SqlDbType.Int)
|
||||
{ Value = TimeWeightedAggregator.WindowFetchLimit(_maxTimeWeightedRawRows) });
|
||||
await using var reader = await cmd.ExecuteReaderAsync(ct).ConfigureAwait(false);
|
||||
var resultSet = 0;
|
||||
do
|
||||
{
|
||||
long windowRows = 0;
|
||||
while (await reader.ReadAsync(ct).ConfigureAwait(false))
|
||||
{
|
||||
// Result set 2 is the window (TOP (@maxRows) = cap + 1): one row past the cap fails the node.
|
||||
if (resultSet == 2) TimeWeightedAggregator.CheckWindowRows(++windowRows, _maxTimeWeightedRawRows, tag);
|
||||
tw.Add(ReadStored(reader, 0));
|
||||
}
|
||||
resultSet++;
|
||||
} while (await reader.NextResultAsync(ct).ConfigureAwait(false));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Maps one per-sample aggregate row (<see cref="SqlAggregateQueries"/> column layout) to a <see cref="BucketRow"/>.
|
||||
/// Every picked sample — first/last usable, first/last of any quality, the extreme — is optional: a NULL reads as
|
||||
/// "no such sample", never an exception (live-gate 8 F1: on QuestDB a row whose counts and picks disagreed threw
|
||||
/// out of <see cref="ReadStored"/> and failed the node's whole read; the evaluator answers such a bucket from what
|
||||
/// is present, or <c>BadNoData</c>).
|
||||
/// </summary>
|
||||
/// <param name="reader">Reader positioned on a bucket row.</param>
|
||||
/// <param name="family">The query's aggregate family.</param>
|
||||
/// <returns>The bucket statistics.</returns>
|
||||
internal static BucketRow ReadBucket(DbDataReader reader, AggregateFamily family)
|
||||
{
|
||||
static double? D(DbDataReader r, int i) => r.IsDBNull(i) ? null : r.GetDouble(i);
|
||||
static long? L(DbDataReader r, int i) => r.IsDBNull(i) ? null : r.GetInt64(i);
|
||||
static string? S(DbDataReader r, int i) => r.IsDBNull(i) ? null : r.GetString(i);
|
||||
static HistorianValue V(DbDataReader r, int i) =>
|
||||
r.IsDBNull(i) ? HistorianValue.Empty : SampleColumnMapper.FromColumns(r.GetByte(i), D(r, i + 1), L(r, i + 2), S(r, i + 3));
|
||||
|
||||
var row = new BucketRow
|
||||
@@ -376,14 +584,22 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
|
||||
row.Min = D(reader, f + 1);
|
||||
row.Max = D(reader, f + 2);
|
||||
row.Sum = D(reader, f + 3);
|
||||
row.UncertainMin = D(reader, f + 4);
|
||||
row.UncertainMax = D(reader, f + 5);
|
||||
row.GoodNonNumeric = reader.IsDBNull(f + 6) ? 0 : reader.GetInt32(f + 6);
|
||||
break;
|
||||
case AggregateFamily.StartEnd:
|
||||
row.First = V(reader, f);
|
||||
row.Last = V(reader, f + 5);
|
||||
row.FirstAny = ReadStoredOrNull(reader, SqlAggregateQueries.FirstAnyColumn);
|
||||
row.LastAny = ReadStoredOrNull(reader, SqlAggregateQueries.FirstAnyColumn + 6);
|
||||
break;
|
||||
default:
|
||||
row.Extreme = D(reader, f);
|
||||
row.ExtremeTime = reader.IsDBNull(f + 1) ? null : SqlHistorianConnection.GetUtc(reader, f + 1);
|
||||
row.UncertainMin = D(reader, f + 2);
|
||||
row.UncertainMax = D(reader, f + 3);
|
||||
row.GoodNonNumeric = reader.IsDBNull(f + 4) ? 0 : reader.GetInt32(f + 4);
|
||||
break;
|
||||
}
|
||||
return row;
|
||||
@@ -397,7 +613,7 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
|
||||
/// (<see cref="StoredSample.FromStored"/>) — defence in depth for rows written before the writer forced every
|
||||
/// Empty sample Bad.
|
||||
/// </summary>
|
||||
private static StoredSample ReadStored(SqlDataReader r, int o)
|
||||
private static StoredSample ReadStored(DbDataReader r, int o)
|
||||
{
|
||||
var value = SampleColumnMapper.FromColumns(
|
||||
r.GetByte(o + 1),
|
||||
@@ -408,6 +624,12 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
|
||||
return StoredSample.FromStored(SqlHistorianConnection.GetUtc(r, o), value, status);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// <see cref="ReadStored"/> for an optional picked sample: a NULL <c>SourceTimeUtc</c> means the row carries no such
|
||||
/// sample (live-gate 8 F1), so the result is null rather than an exception.
|
||||
/// </summary>
|
||||
private static StoredSample? ReadStoredOrNull(DbDataReader r, int o) => r.IsDBNull(o) ? null : ReadStored(r, o);
|
||||
|
||||
/// <summary>Read-only tag lookup (never creates a row); runs the schema bootstrap on a cache miss.</summary>
|
||||
private Task<int?> ResolveAsync(string name, CancellationToken ct) => _catalog.TryGetTagIdAsync(name, ct);
|
||||
|
||||
@@ -418,6 +640,9 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
|
||||
|
||||
private void RecordSuccess() => _health.RecordSuccess();
|
||||
|
||||
/// <summary>Classifies provider transport/timeouts, keeping SQL/query errors distinct (#697).</summary>
|
||||
private static bool IsTransportFailure(Exception ex) => ex is SqlException sql && sql.Number is -2 or 0 or 53 or 64 or 233 or 258 or 10053 or 10054 or 10060 or 10061 or 10065 or 11001 or 40197 or 40613;
|
||||
|
||||
private void RecordFailure(Exception ex)
|
||||
{
|
||||
// A cancelled read (deadline / client abort) is not a backend fault (the counters ignore it too).
|
||||
|
||||
+81
-9
@@ -22,27 +22,57 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer;
|
||||
/// is written.</para>
|
||||
/// <para><b>Unknown tag.</b> Insert/Update create it via <see cref="SqlServerTagCatalog"/>; Replace and the
|
||||
/// deletes report <c>BadNoEntryExists</c> (DeleteRaw: <c>BadNoData</c>) without creating anything.</para>
|
||||
/// <para><b>Modified-value history (Part 11, #668).</b> Every row the MERGE inserts or updates and every row a
|
||||
/// delete removes is copied into <c>SampleModified</c> by an <c>OUTPUT … INTO</c> on the SAME statement — the prior
|
||||
/// value from <c>deleted.*</c> (NULL for an insert), the caller's <see cref="HistoryUpdateContext"/>, the
|
||||
/// <see cref="HistoryModificationType"/> and <c>SYSUTCDATETIME()</c>. One statement means the sample change and its
|
||||
/// audit row commit or roll back together: a failed update leaves no audit row.</para>
|
||||
/// <para><b>Schema v1 (Gitea #674).</b> Without the trail (<see cref="SqlServerSchemaBootstrapper.SupportsModifiedTrail"/>
|
||||
/// false) a change could not be audited, so every call throws <see cref="NotSupportedException"/> — which the node
|
||||
/// manager reports as a node-level <c>BadHistoryOperationUnsupported</c>, on all three arms alike — and nothing is
|
||||
/// sent to the database beyond the one-time schema check.</para>
|
||||
/// </summary>
|
||||
internal sealed class SqlServerHistorianDataUpdater : IHistorianDataUpdater
|
||||
{
|
||||
private readonly SqlHistorianConnection _conn;
|
||||
private readonly SqlServerTagCatalog _tags;
|
||||
private readonly SqlServerSchemaBootstrapper _bootstrap;
|
||||
|
||||
/// <summary>Creates the updater.</summary>
|
||||
/// <param name="conn">Connection helper.</param>
|
||||
/// <param name="bootstrap">Schema bootstrapper — decides whether the modification trail exists (#674).</param>
|
||||
/// <param name="tags">Tag catalog (runs the schema bootstrap itself).</param>
|
||||
public SqlServerHistorianDataUpdater(SqlHistorianConnection conn, SqlServerTagCatalog tags)
|
||||
public SqlServerHistorianDataUpdater(SqlHistorianConnection conn, SqlServerSchemaBootstrapper bootstrap,
|
||||
SqlServerTagCatalog tags)
|
||||
{
|
||||
_conn = conn;
|
||||
_bootstrap = bootstrap;
|
||||
_tags = tags;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// #674: ensures the schema and refuses the call when it has no Part 11 trail (no SQL after the first success).
|
||||
/// </summary>
|
||||
/// <param name="ct">Cancellation.</param>
|
||||
/// <returns>A task completing when the trail is available.</returns>
|
||||
/// <exception cref="NotSupportedException">The schema is below <see cref="SqlServerSchemaBootstrapper.ModifiedTrailVersion"/>.</exception>
|
||||
private async Task RequireTrailAsync(CancellationToken ct)
|
||||
{
|
||||
await _bootstrap.EnsureAsync(ct).ConfigureAwait(false);
|
||||
if (!_bootstrap.SupportsModifiedTrail)
|
||||
throw new NotSupportedException(
|
||||
$"Historian schema [{_conn.Schema}] is below version {SqlServerSchemaBootstrapper.ModifiedTrailVersion}: " +
|
||||
"HistoryUpdate needs the Part 11 modification trail (apply 002_sample_modified.sql and restart).");
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public async Task<IReadOnlyList<uint>> UpdateDataAsync(string tag, HistoryUpdateMode mode,
|
||||
IReadOnlyList<HistorizationValue> values, CancellationToken ct)
|
||||
IReadOnlyList<HistorizationValue> values, HistoryUpdateContext context, CancellationToken ct)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(tag);
|
||||
ArgumentNullException.ThrowIfNull(values);
|
||||
ArgumentNullException.ThrowIfNull(context);
|
||||
await RequireTrailAsync(ct).ConfigureAwait(false);
|
||||
var plan = HistoryUpdatePlanner.Plan(tag, mode, values);
|
||||
if (plan.Winners.Count == 0) return HistoryUpdatePlanner.Complete(plan, mode, new Dictionary<long, UpsertOutcome>());
|
||||
|
||||
@@ -52,14 +82,17 @@ internal sealed class SqlServerHistorianDataUpdater : IHistorianDataUpdater
|
||||
|
||||
var outcome = tagId is null
|
||||
? new Dictionary<long, UpsertOutcome>()
|
||||
: await MergeAsync(tagId.Value, mode, plan.Winners.Select(w => w.Row).ToList(), ct).ConfigureAwait(false);
|
||||
: await MergeAsync(tagId.Value, mode, plan.Winners.Select(w => w.Row).ToList(), context, ct).ConfigureAwait(false);
|
||||
return HistoryUpdatePlanner.Complete(plan, mode, outcome);
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public async Task<uint> DeleteRawAsync(string tag, DateTime startUtc, DateTime endUtc, CancellationToken ct)
|
||||
public async Task<uint> DeleteRawAsync(string tag, DateTime startUtc, DateTime endUtc, HistoryUpdateContext context,
|
||||
CancellationToken ct)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(tag);
|
||||
ArgumentNullException.ThrowIfNull(context);
|
||||
await RequireTrailAsync(ct).ConfigureAwait(false);
|
||||
if (!HistorianTagNames.IsValid(tag)) return HistorianStatusCodes.BadInvalidArgument;
|
||||
var tagId = await _tags.TryGetTagIdAsync(tag, ct).ConfigureAwait(false);
|
||||
if (tagId is null) return HistoryUpdatePlanner.DeleteRawResult(tagKnown: false, 0);
|
||||
@@ -67,9 +100,11 @@ internal sealed class SqlServerHistorianDataUpdater : IHistorianDataUpdater
|
||||
await using var tx = (SqlTransaction)await conn.BeginTransactionAsync(ct).ConfigureAwait(false);
|
||||
int rows;
|
||||
await using (var cmd = _conn.CreateCommand(conn,
|
||||
$"DELETE FROM {_conn.Q("Sample")} WHERE TagId = @tagId AND SourceTimeUtc >= @s AND SourceTimeUtc < @e", tx))
|
||||
$"DELETE FROM {_conn.Q("Sample")} {AuditInto(HistoryModificationType.Delete)} " +
|
||||
"WHERE TagId = @tagId AND SourceTimeUtc >= @s AND SourceTimeUtc < @e", tx))
|
||||
{
|
||||
cmd.Parameters.Add(new SqlParameter("@tagId", System.Data.SqlDbType.Int) { Value = tagId.Value });
|
||||
AddAuditParameters(cmd, context);
|
||||
SqlHistorianConnection.AddDateTime2(cmd, "@s", startUtc);
|
||||
SqlHistorianConnection.AddDateTime2(cmd, "@e", endUtc);
|
||||
rows = await cmd.ExecuteNonQueryAsync(ct).ConfigureAwait(false);
|
||||
@@ -79,10 +114,13 @@ internal sealed class SqlServerHistorianDataUpdater : IHistorianDataUpdater
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public async Task<IReadOnlyList<uint>> DeleteAtTimeAsync(string tag, IReadOnlyList<DateTime> timesUtc, CancellationToken ct)
|
||||
public async Task<IReadOnlyList<uint>> DeleteAtTimeAsync(string tag, IReadOnlyList<DateTime> timesUtc,
|
||||
HistoryUpdateContext context, CancellationToken ct)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(tag);
|
||||
ArgumentNullException.ThrowIfNull(timesUtc);
|
||||
ArgumentNullException.ThrowIfNull(context);
|
||||
await RequireTrailAsync(ct).ConfigureAwait(false);
|
||||
if (!HistorianTagNames.IsValid(tag))
|
||||
{
|
||||
var refused = new uint[timesUtc.Count];
|
||||
@@ -100,9 +138,11 @@ internal sealed class SqlServerHistorianDataUpdater : IHistorianDataUpdater
|
||||
await using var conn = await _conn.OpenAsync(ct).ConfigureAwait(false);
|
||||
await using var tx = (SqlTransaction)await conn.BeginTransactionAsync(ct).ConfigureAwait(false);
|
||||
await using (var cmd = _conn.CreateCommand(conn,
|
||||
$"DELETE s OUTPUT deleted.SourceTimeUtc FROM {_conn.Q("Sample")} AS s JOIN @times AS t ON s.SourceTimeUtc = t.T WHERE s.TagId = @tagId", tx))
|
||||
$"DELETE s {AuditInto(HistoryModificationType.Delete)} OUTPUT deleted.SourceTimeUtc " +
|
||||
$"FROM {_conn.Q("Sample")} AS s JOIN @times AS t ON s.SourceTimeUtc = t.T WHERE s.TagId = @tagId", tx))
|
||||
{
|
||||
cmd.Parameters.Add(new SqlParameter("@tagId", System.Data.SqlDbType.Int) { Value = tagId.Value });
|
||||
AddAuditParameters(cmd, context);
|
||||
_conn.AddStructured(cmd, "@times", "TimeRow",
|
||||
SampleRowMapper.ToTimeRecords(distinct.Select(t => new DateTime(t, DateTimeKind.Utc))));
|
||||
await using var rd = await cmd.ExecuteReaderAsync(ct).ConfigureAwait(false);
|
||||
@@ -114,7 +154,7 @@ internal sealed class SqlServerHistorianDataUpdater : IHistorianDataUpdater
|
||||
}
|
||||
|
||||
private async Task<Dictionary<long, UpsertOutcome>> MergeAsync(int tagId, HistoryUpdateMode mode,
|
||||
List<HistorianSampleRow> rows, CancellationToken ct)
|
||||
List<HistorianSampleRow> rows, HistoryUpdateContext context, CancellationToken ct)
|
||||
{
|
||||
var sql = $"MERGE {_conn.Q("Sample")} WITH (HOLDLOCK) AS t USING @rows AS r ON t.TagId = @tagId AND t.SourceTimeUtc = r.SourceTimeUtc ";
|
||||
if (mode != HistoryUpdateMode.Insert)
|
||||
@@ -123,7 +163,9 @@ internal sealed class SqlServerHistorianDataUpdater : IHistorianDataUpdater
|
||||
if (mode != HistoryUpdateMode.Replace)
|
||||
sql += "WHEN NOT MATCHED THEN INSERT (TagId, SourceTimeUtc, ValueKind, NumValue, IntValue, StrValue, StatusCode, ServerTimeUtc) " +
|
||||
"VALUES (@tagId, r.SourceTimeUtc, r.ValueKind, r.NumValue, r.IntValue, r.StrValue, r.StatusCode, SYSUTCDATETIME()) ";
|
||||
sql += "OUTPUT $action, r.SourceTimeUtc;";
|
||||
// The MERGE's deleted.* is NULL for an inserted row, so one OUTPUT INTO records both an insert (no prior value)
|
||||
// and a replace (the prior value). TagId comes from inserted.* — present on both branches.
|
||||
sql += $"{AuditInto(ToModificationType(mode), "inserted")} OUTPUT $action, r.SourceTimeUtc;";
|
||||
|
||||
var outcome = new Dictionary<long, UpsertOutcome>();
|
||||
await using var conn = await _conn.OpenAsync(ct).ConfigureAwait(false);
|
||||
@@ -131,6 +173,7 @@ internal sealed class SqlServerHistorianDataUpdater : IHistorianDataUpdater
|
||||
await using (var cmd = _conn.CreateCommand(conn, sql, tx))
|
||||
{
|
||||
cmd.Parameters.Add(new SqlParameter("@tagId", System.Data.SqlDbType.Int) { Value = tagId });
|
||||
AddAuditParameters(cmd, context);
|
||||
_conn.AddStructured(cmd, "@rows", "SampleRow", SampleRowMapper.ToSampleRecords(rows));
|
||||
await using var rd = await cmd.ExecuteReaderAsync(ct).ConfigureAwait(false);
|
||||
while (await rd.ReadAsync(ct).ConfigureAwait(false))
|
||||
@@ -141,4 +184,33 @@ internal sealed class SqlServerHistorianDataUpdater : IHistorianDataUpdater
|
||||
await tx.CommitAsync(ct).ConfigureAwait(false);
|
||||
return outcome;
|
||||
}
|
||||
|
||||
/// <summary>The Part 11 modification type recorded for an update mode.</summary>
|
||||
/// <param name="mode">The update mode.</param>
|
||||
/// <returns>Insert ⇒ Insert, Replace ⇒ Replace, Update ⇒ Update.</returns>
|
||||
internal static HistoryModificationType ToModificationType(HistoryUpdateMode mode) => mode switch
|
||||
{
|
||||
HistoryUpdateMode.Insert => HistoryModificationType.Insert,
|
||||
HistoryUpdateMode.Replace => HistoryModificationType.Replace,
|
||||
_ => HistoryModificationType.Update,
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// The <c>OUTPUT … INTO SampleModified</c> clause copying each affected row's prior value (<c>deleted.*</c>) into
|
||||
/// the modification trail. <paramref name="keySource"/> names the pseudo-table that carries <c>TagId</c> /
|
||||
/// <c>SourceTimeUtc</c> for every affected row (<c>deleted</c> for a DELETE, <c>inserted</c> for the MERGE).
|
||||
/// </summary>
|
||||
private string AuditInto(HistoryModificationType type, string keySource = "deleted") =>
|
||||
$"OUTPUT {keySource}.TagId, {keySource}.SourceTimeUtc, SYSUTCDATETIME(), {(byte)type}, @principal, @apiKeyId, " +
|
||||
"deleted.ValueKind, deleted.NumValue, deleted.IntValue, deleted.StrValue, deleted.StatusCode, deleted.ServerTimeUtc " +
|
||||
$"INTO {_conn.Q("SampleModified")} (TagId, SourceTimeUtc, ModificationTimeUtc, UpdateType, Principal, ApiKeyId, " +
|
||||
"ValueKind, NumValue, IntValue, StrValue, StatusCode, ServerTimeUtc)";
|
||||
|
||||
private static void AddAuditParameters(SqlCommand cmd, HistoryUpdateContext context)
|
||||
{
|
||||
cmd.Parameters.Add(new SqlParameter("@principal", System.Data.SqlDbType.NVarChar, HistoryUpdateContext.MaxIdentityLength)
|
||||
{ Value = context.StoredPrincipal });
|
||||
cmd.Parameters.Add(new SqlParameter("@apiKeyId", System.Data.SqlDbType.NVarChar, HistoryUpdateContext.MaxIdentityLength)
|
||||
{ Value = (object?)context.StoredApiKeyId ?? DBNull.Value });
|
||||
}
|
||||
}
|
||||
|
||||
+25
-1
@@ -18,10 +18,28 @@ public sealed partial class SqlServerHistorianOptions
|
||||
|
||||
/// <summary>
|
||||
/// <c>true</c> creates the schema and applies missing scripts on first use; <c>false</c> only verifies
|
||||
/// that schema version 1 exists and fails every historian call with a clear error otherwise.
|
||||
/// that the base schema version (<c>SqlServerSchemaBootstrapper.RequiredVersion</c>, 1) exists and fails every
|
||||
/// historian call with a clear error otherwise. A verified v1 schema (no Part 11 modification trail, script 002
|
||||
/// not applied) is served with modified reads and HistoryUpdate refused <c>BadHistoryOperationUnsupported</c>
|
||||
/// and one Warning per boot (Gitea #674).
|
||||
/// </summary>
|
||||
public bool AutoCreateSchema { get; set; } = true;
|
||||
|
||||
/// <summary>
|
||||
/// Days of history to keep, measured on source time. <c>null</c> (default) keeps everything. When set,
|
||||
/// a driver node deletes older <c>hist.Sample</c>/<c>hist.Event</c> rows every <see cref="PurgeInterval"/>.
|
||||
/// </summary>
|
||||
public int? RetentionDays { get; set; }
|
||||
|
||||
/// <summary>How often the retention purge runs. Must be positive.</summary>
|
||||
public TimeSpan PurgeInterval { get; set; } = TimeSpan.FromHours(1);
|
||||
|
||||
/// <summary>
|
||||
/// Rows per <c>DELETE TOP(n)</c> batch. Default 4000 stays under SQL Server's 5000-lock escalation
|
||||
/// threshold so a purge never escalates to a table lock.
|
||||
/// </summary>
|
||||
public int PurgeBatchSize { get; set; } = 4000;
|
||||
|
||||
[GeneratedRegex("^[A-Za-z_][A-Za-z0-9_]{0,127}$")]
|
||||
private static partial Regex IdentifierRegex();
|
||||
|
||||
@@ -41,6 +59,12 @@ public sealed partial class SqlServerHistorianOptions
|
||||
errors.Add("ServerHistorian:SqlServer:Schema must be a SQL identifier matching ^[A-Za-z_][A-Za-z0-9_]{0,127}$.");
|
||||
if (CommandTimeout <= TimeSpan.Zero)
|
||||
errors.Add("ServerHistorian:SqlServer:CommandTimeout must be greater than zero.");
|
||||
if (RetentionDays is < 1)
|
||||
errors.Add("ServerHistorian:SqlServer:RetentionDays must be at least 1 when set (omit it to keep history forever).");
|
||||
if (PurgeInterval <= TimeSpan.Zero)
|
||||
errors.Add("ServerHistorian:SqlServer:PurgeInterval must be greater than zero.");
|
||||
if (PurgeBatchSize < 1)
|
||||
errors.Add("ServerHistorian:SqlServer:PurgeBatchSize must be at least 1.");
|
||||
return errors;
|
||||
}
|
||||
}
|
||||
|
||||
+18
-3
@@ -1,3 +1,4 @@
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common;
|
||||
using System.Collections.Frozen;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
@@ -29,11 +30,16 @@ public sealed class SqlServerHistorianProvider : IHistorianBackendProvider
|
||||
|
||||
/// <summary>Creates the provider over the <c>ServerHistorian:SqlServer</c> options (validated elsewhere).</summary>
|
||||
/// <param name="options">The <c>ServerHistorian:SqlServer</c> options.</param>
|
||||
public SqlServerHistorianProvider(SqlServerHistorianOptions options)
|
||||
/// <param name="maxTimeWeightedRawRows">The shared <c>ServerHistorian:MaxTimeWeightedRawRows</c> cap (<= 0 = unbounded).</param>
|
||||
public SqlServerHistorianProvider(SqlServerHistorianOptions options,
|
||||
int maxTimeWeightedRawRows = TimeWeightedAggregator.DefaultMaxWindowRows)
|
||||
{
|
||||
_options = options ?? throw new ArgumentNullException(nameof(options));
|
||||
_maxTimeWeightedRawRows = maxTimeWeightedRawRows;
|
||||
}
|
||||
|
||||
private readonly int _maxTimeWeightedRawRows;
|
||||
|
||||
/// <inheritdoc />
|
||||
public string Name => "SqlServer";
|
||||
|
||||
@@ -63,7 +69,7 @@ public sealed class SqlServerHistorianProvider : IHistorianBackendProvider
|
||||
{
|
||||
var s = Get(sp);
|
||||
return new SqlServerHistorianDataSource(s.Conn, s.Bootstrap, s.Catalog,
|
||||
Loggers(sp).CreateLogger<SqlServerHistorianDataSource>());
|
||||
Loggers(sp).CreateLogger<SqlServerHistorianDataSource>(), maxTimeWeightedRawRows: _maxTimeWeightedRawRows);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
@@ -93,7 +99,16 @@ public sealed class SqlServerHistorianProvider : IHistorianBackendProvider
|
||||
public IHistorianDataUpdater? CreateUpdater(IServiceProvider sp)
|
||||
{
|
||||
var s = Get(sp);
|
||||
return new SqlServerHistorianDataUpdater(s.Conn, s.Catalog);
|
||||
return new SqlServerHistorianDataUpdater(s.Conn, s.Bootstrap, s.Catalog);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public IHistorianRetention? CreateRetention(IServiceProvider sp)
|
||||
{
|
||||
if (_options.RetentionDays is not { } days) return null;
|
||||
var s = Get(sp);
|
||||
return new SqlServerHistorianRetention(s.Conn, s.Bootstrap, _options.PurgeInterval, days,
|
||||
_options.PurgeBatchSize, Loggers(sp).CreateLogger<SqlServerHistorianRetention>());
|
||||
}
|
||||
|
||||
private sealed record Shared(SqlHistorianConnection Conn, SqlServerSchemaBootstrapper Bootstrap, SqlServerTagCatalog Catalog);
|
||||
|
||||
+168
@@ -0,0 +1,168 @@
|
||||
using System.Data;
|
||||
using Microsoft.Data.SqlClient;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Historian;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer;
|
||||
|
||||
/// <summary>
|
||||
/// Deletes <c>hist.Sample</c> / <c>hist.SampleModified</c> / <c>hist.Event</c> rows older than the retention window, in small
|
||||
/// <c>DELETE TOP(n)</c> batches (under the lock-escalation threshold). Samples are purged per tag so each
|
||||
/// delete is a seek on the clustered key <c>(TagId, SourceTimeUtc)</c>; events seek on <c>CX_Event_Time</c>.
|
||||
/// A session <c>sp_getapplock</c> (timeout 0) serialises purgers that share one database; a loser skips.
|
||||
/// To purge another table later, add it to <see cref="PurgeAsync"/>. On a v1 schema (no Part 11 trail, Gitea #674)
|
||||
/// the <c>SampleModified</c> purge is skipped and absent from the results.
|
||||
/// </summary>
|
||||
internal sealed class SqlServerHistorianRetention : IHistorianRetention
|
||||
{
|
||||
private readonly SqlHistorianConnection _conn;
|
||||
private readonly SqlServerSchemaBootstrapper _bootstrap;
|
||||
private readonly int _days;
|
||||
private readonly int _batch;
|
||||
private readonly TimeProvider _time;
|
||||
private readonly ILogger _log;
|
||||
|
||||
public SqlServerHistorianRetention(SqlHistorianConnection conn, SqlServerSchemaBootstrapper bootstrap,
|
||||
TimeSpan interval, int retentionDays, int batchSize, ILogger? log = null, TimeProvider? time = null)
|
||||
{
|
||||
_conn = conn;
|
||||
_bootstrap = bootstrap;
|
||||
Interval = interval;
|
||||
_days = retentionDays;
|
||||
_batch = batchSize;
|
||||
_log = log ?? NullLogger.Instance;
|
||||
_time = time ?? TimeProvider.System;
|
||||
}
|
||||
|
||||
public TimeSpan Interval { get; }
|
||||
|
||||
/// <summary>Test hook invoked after every delete batch.</summary>
|
||||
internal Action? AfterBatch { get; set; }
|
||||
|
||||
/// <summary>Test seam for <see cref="SqlConnection.ClearPool"/>.</summary>
|
||||
internal Action<SqlConnection> ClearPool { get; set; } = SqlConnection.ClearPool;
|
||||
|
||||
/// <summary>Test seam: the release statement. Tests point it at a failing statement to exercise the pool-clear fallback.</summary>
|
||||
internal string ReleaseSql { get; set; } = "EXEC sp_releaseapplock @Resource = @res, @LockOwner = 'Session';";
|
||||
|
||||
public async Task<IReadOnlyList<RetentionPurgeResult>> PurgeAsync(CancellationToken ct)
|
||||
{
|
||||
await _bootstrap.EnsureAsync(ct).ConfigureAwait(false);
|
||||
var cutoff = _time.GetUtcNow().UtcDateTime.AddDays(-_days);
|
||||
|
||||
await using var conn = await _conn.OpenAsync(ct).ConfigureAwait(false);
|
||||
var acquired = false;
|
||||
try
|
||||
{
|
||||
try
|
||||
{
|
||||
acquired = await TryLockAsync(conn, ct).ConfigureAwait(false);
|
||||
}
|
||||
catch
|
||||
{
|
||||
// Ambiguous: the server may have granted the lock before the client saw the reply. We cannot
|
||||
// release what we are not sure we hold, so close the session physically instead.
|
||||
ClearPool(conn);
|
||||
throw;
|
||||
}
|
||||
|
||||
if (!acquired)
|
||||
{
|
||||
_log.LogInformation("Historian retention purge skipped: another purger holds the applock.");
|
||||
return [];
|
||||
}
|
||||
|
||||
// #674: a v1 schema has no SampleModified table — skip its purge rather than fail the whole pass.
|
||||
var trail = _bootstrap.SupportsModifiedTrail;
|
||||
long samples = 0, modified = 0;
|
||||
var tagIds = new List<int>();
|
||||
await using (var cmd = _conn.CreateCommand(conn, $"SELECT TagId FROM {_conn.Q("Tag")}"))
|
||||
await using (var r = await cmd.ExecuteReaderAsync(ct).ConfigureAwait(false))
|
||||
while (await r.ReadAsync(ct).ConfigureAwait(false)) tagIds.Add(r.GetInt32(0));
|
||||
|
||||
foreach (var id in tagIds)
|
||||
{
|
||||
samples += await DeleteLoopAsync(conn,
|
||||
$"DELETE TOP (@n) FROM {_conn.Q("Sample")} WHERE TagId = @id AND SourceTimeUtc < @cutoff",
|
||||
cutoff, id, ct).ConfigureAwait(false);
|
||||
// The Part 11 modification trail (#668) ages out on the SAME source-time cutoff as the samples it
|
||||
// describes; a purge never writes to it (no trigger, no OUTPUT INTO).
|
||||
if (trail)
|
||||
{
|
||||
modified += await DeleteLoopAsync(conn,
|
||||
$"DELETE TOP (@n) FROM {_conn.Q("SampleModified")} WHERE TagId = @id AND SourceTimeUtc < @cutoff",
|
||||
cutoff, id, ct).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
var events = await DeleteLoopAsync(conn,
|
||||
$"DELETE TOP (@n) FROM {_conn.Q("Event")} WHERE EventTimeUtc < @cutoff",
|
||||
cutoff, null, ct).ConfigureAwait(false);
|
||||
|
||||
return trail
|
||||
?
|
||||
[
|
||||
new RetentionPurgeResult("Sample", samples),
|
||||
new RetentionPurgeResult("SampleModified", modified),
|
||||
new RetentionPurgeResult("Event", events),
|
||||
]
|
||||
:
|
||||
[
|
||||
new RetentionPurgeResult("Sample", samples),
|
||||
new RetentionPurgeResult("Event", events),
|
||||
];
|
||||
}
|
||||
finally
|
||||
{
|
||||
// Only when WE hold it: releasing an applock we were refused fails, which would log a false Warning
|
||||
// and clear the shared pool on every skipped interval.
|
||||
if (acquired) await ReleaseAsync(conn).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<long> DeleteLoopAsync(SqlConnection conn, string sql, DateTime cutoff, int? tagId, CancellationToken ct)
|
||||
{
|
||||
long total = 0;
|
||||
while (true)
|
||||
{
|
||||
ct.ThrowIfCancellationRequested();
|
||||
await using var cmd = _conn.CreateCommand(conn, sql);
|
||||
cmd.Parameters.Add(new SqlParameter("@n", SqlDbType.Int) { Value = _batch });
|
||||
if (tagId is { } id) cmd.Parameters.Add(new SqlParameter("@id", SqlDbType.Int) { Value = id });
|
||||
SqlHistorianConnection.AddDateTime2(cmd, "@cutoff", cutoff);
|
||||
var n = await cmd.ExecuteNonQueryAsync(ct).ConfigureAwait(false);
|
||||
total += n;
|
||||
AfterBatch?.Invoke();
|
||||
if (n < _batch) return total;
|
||||
}
|
||||
}
|
||||
|
||||
private string LockResource => $"OtOpcUa.Historian.Retention.{_conn.Schema}";
|
||||
|
||||
private async Task<bool> TryLockAsync(SqlConnection conn, CancellationToken ct)
|
||||
{
|
||||
await using var cmd = _conn.CreateCommand(conn,
|
||||
"DECLARE @r int; EXEC @r = sp_getapplock @Resource = @res, @LockMode = 'Exclusive', @LockOwner = 'Session', @LockTimeout = 0; SELECT @r;");
|
||||
cmd.Parameters.Add(new SqlParameter("@res", SqlDbType.NVarChar, 255) { Value = LockResource });
|
||||
return Convert.ToInt32(await cmd.ExecuteScalarAsync(ct).ConfigureAwait(false)) >= 0;
|
||||
}
|
||||
|
||||
private async Task ReleaseAsync(SqlConnection conn)
|
||||
{
|
||||
try
|
||||
{
|
||||
await using var cmd = _conn.CreateCommand(conn, ReleaseSql);
|
||||
cmd.Parameters.Add(new SqlParameter("@res", SqlDbType.NVarChar, 255) { Value = LockResource });
|
||||
await cmd.ExecuteNonQueryAsync(CancellationToken.None).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
// A pooled connection keeps its server session (sp_reset_connection does not release a Session-owned
|
||||
// applock); clearing the pool makes the dispose physically close it, which releases the lock.
|
||||
_log.LogWarning("sp_releaseapplock failed ({Exception}); discarding the connection pool so the session " +
|
||||
"holding the retention applock is closed rather than pooled.", ex.GetType().Name);
|
||||
ClearPool(conn);
|
||||
}
|
||||
}
|
||||
}
|
||||
+34
-4
@@ -14,13 +14,29 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer;
|
||||
/// <see cref="BootstrapFailureCache.DefaultDuration"/> retries (<see cref="BootstrapFailureCache"/>). Within that window the last failure is re-thrown without touching
|
||||
/// SQL, so a missing schema (or a SQL outage) does not serialise every write, read and alarm event behind one
|
||||
/// semaphore and one round-trip each.
|
||||
/// With <c>AutoCreateSchema=false</c> it only verifies <c>MAX(Version) >= 1</c>.
|
||||
/// With <c>AutoCreateSchema=false</c> it only verifies <c>MAX(Version) >= </c><see cref="RequiredVersion"/>.
|
||||
/// <para><b>Schema v1 is accepted (Gitea #674).</b> A database at version 1 (no <c>SampleModified</c>) still serves
|
||||
/// live writes, raw / processed / at-time reads, alarm events and retention; only the Part 11 modification trail is
|
||||
/// unavailable — <see cref="SupportsModifiedTrail"/> is false, modified reads and HistoryUpdate report
|
||||
/// <c>BadHistoryOperationUnsupported</c> without touching SQL, and one Warning is logged per instance (= per boot: the
|
||||
/// provider shares one bootstrapper). Before #674 a v1 database refused EVERY historian call, continuous-historization
|
||||
/// drains included, until script 002 was applied by hand.</para>
|
||||
/// </summary>
|
||||
internal sealed partial class SqlServerSchemaBootstrapper
|
||||
{
|
||||
/// <summary>The schema version this build requires.</summary>
|
||||
/// <summary>
|
||||
/// The schema version this build requires to serve at all: 1 = <c>001_initial.sql</c>. Gitea #674 lowered it from
|
||||
/// 2 — a missing modification trail disables only the features that need it (<see cref="ModifiedTrailVersion"/>).
|
||||
/// </summary>
|
||||
public const int RequiredVersion = 1;
|
||||
|
||||
/// <summary>
|
||||
/// The schema version that carries the Part 11 modification trail: 2 = <c>002_sample_modified.sql</c> (Gitea #668).
|
||||
/// An auto-creating bootstrap always upgrades to it; with <c>AutoCreateSchema=false</c> the script must be applied by
|
||||
/// hand (and the node restarted — the verdict is taken once per instance).
|
||||
/// </summary>
|
||||
public const int ModifiedTrailVersion = 2;
|
||||
|
||||
private const string LockResource = "OtOpcUaHistorianSchema";
|
||||
private const int LockTimeoutMs = 30_000;
|
||||
|
||||
@@ -30,6 +46,13 @@ internal sealed partial class SqlServerSchemaBootstrapper
|
||||
private readonly SemaphoreSlim _gate = new(1, 1);
|
||||
private readonly BootstrapFailureCache _failure; // guarded by _gate
|
||||
private volatile bool _ready;
|
||||
private volatile bool _supportsModifiedTrail;
|
||||
|
||||
/// <summary>
|
||||
/// Whether the schema carries the Part 11 modification trail (version >= <see cref="ModifiedTrailVersion"/>).
|
||||
/// Meaningful only after <see cref="EnsureAsync"/> has succeeded (false before). Taken once per instance.
|
||||
/// </summary>
|
||||
public bool SupportsModifiedTrail => _supportsModifiedTrail;
|
||||
|
||||
/// <summary>Creates the bootstrapper.</summary>
|
||||
/// <param name="conn">Connection helper.</param>
|
||||
@@ -48,7 +71,7 @@ internal sealed partial class SqlServerSchemaBootstrapper
|
||||
/// <summary>Ensures the schema is ready; no-op after the first success.</summary>
|
||||
/// <param name="ct">Cancellation.</param>
|
||||
/// <returns>A task completing when the schema is usable.</returns>
|
||||
/// <exception cref="HistorianSchemaMissingException">AutoCreateSchema is false and version 1 is absent.</exception>
|
||||
/// <exception cref="HistorianSchemaMissingException">AutoCreateSchema is false and <see cref="RequiredVersion"/> is absent.</exception>
|
||||
public async Task EnsureAsync(CancellationToken ct)
|
||||
{
|
||||
if (_ready) return;
|
||||
@@ -94,7 +117,14 @@ internal sealed partial class SqlServerSchemaBootstrapper
|
||||
if (v < RequiredVersion)
|
||||
throw new HistorianSchemaMissingException(
|
||||
$"Historian schema [{_conn.Schema}] is missing version {RequiredVersion} (found {v}) and AutoCreateSchema is false. " +
|
||||
"Apply docs/historian/sqlserver/001_initial.sql or enable AutoCreateSchema.");
|
||||
"Apply the docs/historian/sqlserver/NNN_*.sql scripts above version " + v + " in order, or enable AutoCreateSchema.");
|
||||
_supportsModifiedTrail = v >= ModifiedTrailVersion;
|
||||
if (!_supportsModifiedTrail)
|
||||
_log.LogWarning("Historian schema [{Schema}] is at version {Version}, below {TrailVersion}, and AutoCreateSchema is false: " +
|
||||
"the Part 11 modification trail is unavailable, so modified-value history reads and HistoryUpdate report " +
|
||||
"BadHistoryOperationUnsupported and retention skips SampleModified. Live historization, reads, events and retention " +
|
||||
"are unaffected. Apply docs/historian/sqlserver/002_sample_modified.sql and restart the node to enable them.",
|
||||
_conn.Schema, v, ModifiedTrailVersion);
|
||||
}
|
||||
|
||||
private async Task ApplyAsync(SqlConnection conn, CancellationToken ct)
|
||||
|
||||
@@ -101,9 +101,6 @@ public sealed record OpcUaClientDriverOptions
|
||||
/// <summary>Client-side keep-alive interval.</summary>
|
||||
public TimeSpan KeepAliveInterval { get; init; } = TimeSpan.FromSeconds(5);
|
||||
|
||||
/// <summary>Initial reconnect delay after a session drop.</summary>
|
||||
public TimeSpan ReconnectPeriod { get; init; } = TimeSpan.FromSeconds(5);
|
||||
|
||||
/// <summary>
|
||||
/// When <c>true</c>, the driver accepts any self-signed / untrusted server certificate.
|
||||
/// Dev-only — must be <c>false</c> in production so MITM attacks against the opc.tcp
|
||||
|
||||
@@ -22,12 +22,10 @@ internal sealed class OpcUaClientDiagnostics
|
||||
public const string PublishError = "publish_error";
|
||||
/// <summary>A keep-alive came back Bad.</summary>
|
||||
public const string KeepAliveMiss = "keepalive_miss";
|
||||
/// <summary>The reconnect handler was armed.</summary>
|
||||
public const string ReconnectAttempt = "reconnect_attempt";
|
||||
/// <summary>A reconnect landed a new session.</summary>
|
||||
public const string ReconnectSuccess = "reconnect_success";
|
||||
/// <summary>A reconnect handler exhausted its retry window.</summary>
|
||||
public const string ReconnectExhausted = "reconnect_exhausted";
|
||||
/// <summary>A keep-alive came back Good after a bad one on the same session (an in-place recovery, #670).</summary>
|
||||
public const string KeepAliveRecovered = "keepalive_recovered";
|
||||
/// <summary>A late replayed notification (source time at or below the ref's high-water mark) was dropped inside the post-recovery replay window (#670 F2).</summary>
|
||||
public const string StaleNotificationDropped = "stale_notification_dropped";
|
||||
|
||||
private readonly string _driverInstanceId;
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,16 +1,101 @@
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Opc.Ua;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.OpcUaClient;
|
||||
|
||||
/// <summary>
|
||||
/// The one <see cref="ITelemetryContext" /> the driver hands the SDK's non-obsolete (telemetry-taking)
|
||||
/// overloads. An empty builder keeps SDK internals off any logging provider (mirrors Client.Shared's
|
||||
/// <c>ClientTelemetry</c>).
|
||||
/// The <see cref="ITelemetryContext" /> the driver hands the SDK's non-obsolete (telemetry-taking)
|
||||
/// overloads. <see cref="Context"/> has no logging provider (static helpers and tests);
|
||||
/// <see cref="ForLogger"/> routes the SDK's own Warning-and-above output (keep-alive, publish and
|
||||
/// channel errors) to a driver instance's logger — without it every SDK-side session error was
|
||||
/// invisible (Gitea #670).
|
||||
/// </summary>
|
||||
internal static class SdkTelemetry
|
||||
{
|
||||
private static readonly Lazy<ITelemetryContext> Instance = new(() => DefaultTelemetry.Create(_ => { }));
|
||||
|
||||
/// <summary>Gets the shared telemetry context.</summary>
|
||||
/// <summary>Gets the shared, provider-less telemetry context.</summary>
|
||||
public static ITelemetryContext Context => Instance.Value;
|
||||
|
||||
/// <summary>
|
||||
/// Creates a telemetry context whose SDK log output at Warning and above goes to <paramref name="logger"/>,
|
||||
/// rate-limited (a dead upstream makes the SDK warn on every keep-alive and publish for as long as the
|
||||
/// outage lasts). The context owns a <see cref="ILoggerFactory"/>: release it with <see cref="Release"/>
|
||||
/// when the driver is disposed — <see cref="DefaultTelemetry"/> is not itself disposable.
|
||||
/// </summary>
|
||||
/// <param name="logger">The driver instance's logger.</param>
|
||||
/// <returns>The telemetry context.</returns>
|
||||
public static ITelemetryContext ForLogger(ILogger logger) =>
|
||||
DefaultTelemetry.Create(builder => builder
|
||||
.SetMinimumLevel(LogLevel.Warning)
|
||||
.AddProvider(new ForwardingLoggerProvider(new RateLimitedLogger(logger, MaxEntriesPerWindow, Window))));
|
||||
|
||||
/// <summary>Disposes the logger factory a <see cref="ForLogger"/> context owns. Never the shared one.</summary>
|
||||
/// <param name="context">The context to release.</param>
|
||||
public static void Release(ITelemetryContext context)
|
||||
{
|
||||
if (ReferenceEquals(context, Context)) return;
|
||||
try { (context as TelemetryContextBase)?.LoggerFactory?.Dispose(); }
|
||||
catch { /* best-effort */ }
|
||||
}
|
||||
|
||||
/// <summary>SDK log entries forwarded per <see cref="Window"/> before the rest are dropped and counted.</summary>
|
||||
internal const int MaxEntriesPerWindow = 20;
|
||||
|
||||
/// <summary>The rate-limit window.</summary>
|
||||
internal static readonly TimeSpan Window = TimeSpan.FromMinutes(1);
|
||||
|
||||
/// <summary>Hands every SDK logger category the one (rate-limited) driver logger.</summary>
|
||||
private sealed class ForwardingLoggerProvider(ILogger target) : ILoggerProvider
|
||||
{
|
||||
public ILogger CreateLogger(string categoryName) => target;
|
||||
|
||||
public void Dispose() { }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Forwards at most <c>max</c> entries per window; the first entry of the next window carries a note of how
|
||||
/// many were suppressed.
|
||||
/// </summary>
|
||||
internal sealed class RateLimitedLogger(ILogger target, int max, TimeSpan window) : ILogger
|
||||
{
|
||||
private readonly object _lock = new();
|
||||
private DateTime _windowStart = DateTime.MinValue;
|
||||
private int _count;
|
||||
private int _suppressed;
|
||||
|
||||
public IDisposable? BeginScope<TState>(TState state) where TState : notnull => target.BeginScope(state);
|
||||
|
||||
public bool IsEnabled(LogLevel logLevel) => target.IsEnabled(logLevel);
|
||||
|
||||
public void Log<TState>(LogLevel logLevel, EventId eventId, TState state, Exception? exception,
|
||||
Func<TState, Exception?, string> formatter)
|
||||
{
|
||||
int suppressedNote;
|
||||
lock (_lock)
|
||||
{
|
||||
var now = DateTime.UtcNow;
|
||||
if (now - _windowStart >= window)
|
||||
{
|
||||
suppressedNote = _suppressed;
|
||||
_windowStart = now;
|
||||
_count = 0;
|
||||
_suppressed = 0;
|
||||
}
|
||||
else suppressedNote = 0;
|
||||
|
||||
if (_count >= max)
|
||||
{
|
||||
_suppressed++;
|
||||
return;
|
||||
}
|
||||
_count++;
|
||||
}
|
||||
|
||||
if (suppressedNote > 0)
|
||||
target.LogWarning("OPC UA SDK: {Suppressed} further log entries were suppressed in the last {Window}.",
|
||||
suppressedNote, window);
|
||||
target.Log(logLevel, eventId, state, exception, formatter);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+2
@@ -31,6 +31,8 @@
|
||||
<InternalsVisibleTo Include="ZB.MOM.WW.OtOpcUa.Driver.OpcUaClient.Tests"/>
|
||||
<!-- Gitea #656: the native-alarm re-statement end-to-end tests drive this driver into the real DriverHostActor. -->
|
||||
<InternalsVisibleTo Include="ZB.MOM.WW.OtOpcUa.Runtime.Tests"/>
|
||||
<!-- Gitea #670 review M1: the wire test deletes the driver's subscription server-side through its own session. -->
|
||||
<InternalsVisibleTo Include="ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests"/>
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
|
||||
-8
@@ -53,11 +53,6 @@
|
||||
<InputNumber @bind-Value="_form.OpcUa.KeepAliveIntervalSeconds" @bind-Value:after="EmitAsync" class="form-control form-control-sm" />
|
||||
<div class="form-text">Default 5 s.</div>
|
||||
</div>
|
||||
<div class="col-md-3">
|
||||
<label class="form-label">Reconnect period (s)</label>
|
||||
<InputNumber @bind-Value="_form.OpcUa.ReconnectPeriodSeconds" @bind-Value:after="EmitAsync" class="form-control form-control-sm" />
|
||||
<div class="form-text">Initial delay after session drop. Default 5 s.</div>
|
||||
</div>
|
||||
<div class="col-md-3">
|
||||
<label class="form-label">Max discovered nodes</label>
|
||||
<InputNumber @bind-Value="_form.OpcUa.MaxDiscoveredNodes" @bind-Value:after="EmitAsync" class="form-control form-control-sm" />
|
||||
@@ -330,7 +325,6 @@
|
||||
public int TimeoutSeconds { get; set; } = 10;
|
||||
public int SessionTimeoutSeconds { get; set; } = 120;
|
||||
public int KeepAliveIntervalSeconds { get; set; } = 5;
|
||||
public int ReconnectPeriodSeconds { get; set; } = 5;
|
||||
public int MaxDiscoveredNodes { get; set; } = 10_000;
|
||||
public int MaxBrowseDepth { get; set; } = 10;
|
||||
|
||||
@@ -370,7 +364,6 @@
|
||||
TimeoutSeconds = (int)r.Timeout.TotalSeconds,
|
||||
SessionTimeoutSeconds = (int)r.SessionTimeout.TotalSeconds,
|
||||
KeepAliveIntervalSeconds = (int)r.KeepAliveInterval.TotalSeconds,
|
||||
ReconnectPeriodSeconds = (int)r.ReconnectPeriod.TotalSeconds,
|
||||
MaxDiscoveredNodes = r.MaxDiscoveredNodes,
|
||||
MaxBrowseDepth = r.MaxBrowseDepth,
|
||||
SecurityMode = r.SecurityMode,
|
||||
@@ -404,7 +397,6 @@
|
||||
Timeout = TimeSpan.FromSeconds(TimeoutSeconds),
|
||||
SessionTimeout = TimeSpan.FromSeconds(SessionTimeoutSeconds),
|
||||
KeepAliveInterval = TimeSpan.FromSeconds(KeepAliveIntervalSeconds),
|
||||
ReconnectPeriod = TimeSpan.FromSeconds(ReconnectPeriodSeconds),
|
||||
MaxDiscoveredNodes = MaxDiscoveredNodes,
|
||||
MaxBrowseDepth = MaxBrowseDepth,
|
||||
SecurityMode = SecurityMode,
|
||||
|
||||
@@ -41,8 +41,8 @@ public static class HistorianBackendRegistry
|
||||
new Dictionary<string, Func<ServerHistorianOptions, IConfiguration, IHistorianBackendProvider>>(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
[ServerHistorianOptions.GatewayBackendName] = (o, _) => new GatewayHistorianProvider(o),
|
||||
[SqlServerBackendName] = (_, c) => new SqlServerHistorianProvider(BindSqlServer(c)),
|
||||
[QuestDbBackendName] = (_, c) => new QuestDbHistorianProvider(BindQuestDb(c)),
|
||||
[SqlServerBackendName] = (o, c) => new SqlServerHistorianProvider(BindSqlServer(c), o.MaxTimeWeightedRawRows),
|
||||
[QuestDbBackendName] = (o, c) => new QuestDbHistorianProvider(BindQuestDb(c), o.MaxTimeWeightedRawRows),
|
||||
};
|
||||
|
||||
/// <summary>The registered backend names, for validation messages.</summary>
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
using ZB.MOM.WW.OtOpcUa.Cluster;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Host.Configuration;
|
||||
|
||||
/// <summary>
|
||||
/// Names the per-role appsettings overlay (<c>appsettings.{suffix}.json</c>) for a node.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The suffix is built from the node's PLAIN roles only. <c>cluster-{ClusterId}</c> roles are mesh-membership
|
||||
/// tags (and carry a verbatim, operator-chosen ClusterId), not configuration profiles — folding one into the
|
||||
/// filename produced names like <c>appsettings.admin-cluster-MAIN-driver.json</c> that never exist, so on every
|
||||
/// node with a cluster role the shipped overlays silently never loaded.
|
||||
/// </remarks>
|
||||
internal static class RoleOverlay
|
||||
{
|
||||
/// <summary>
|
||||
/// The overlay filename suffix: the plain (non-cluster) roles, ordinal-sorted and joined with <c>-</c>, or
|
||||
/// <see langword="null"/> when the node has none.
|
||||
/// </summary>
|
||||
public static string? Suffix(IReadOnlyList<string> gossipedRoles)
|
||||
{
|
||||
var plain = gossipedRoles
|
||||
.Where(r => !string.IsNullOrWhiteSpace(r) && !RoleParser.IsClusterRole(r))
|
||||
.OrderBy(r => r, StringComparer.Ordinal)
|
||||
.ToList();
|
||||
return plain.Count == 0 ? null : string.Join('-', plain);
|
||||
}
|
||||
}
|
||||
@@ -13,6 +13,7 @@ using ZB.MOM.WW.OtOpcUa.OpcUaServer.Security;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Historian;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Redundancy;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.ScriptedAlarms;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Host.OpcUa;
|
||||
@@ -46,6 +47,7 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
|
||||
private readonly ILogger<OtOpcUaServerHostedService> _logger;
|
||||
private readonly IDriverPrimaryRoleSource? _primaryRoleSource;
|
||||
private readonly HistorianUpdaterHolder _historianUpdater;
|
||||
private readonly IHistoryUpdateRoleView? _historyUpdateRoleView;
|
||||
|
||||
private OpcUaApplicationHost? _appHost;
|
||||
private OtOpcUaSdkServer? _server;
|
||||
@@ -98,6 +100,9 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
|
||||
/// <param name="historianUpdater">The selected historian backend's optional HistoryUpdate capability.
|
||||
/// Optional: when unregistered it resolves to <see cref="HistorianUpdaterHolder.None"/> and HistoryUpdate is
|
||||
/// reported unsupported on every node.</param>
|
||||
/// <param name="historyUpdateRoleView">Gitea #674 — the HistoryUpdate verdict <c>DriverHostActor</c> publishes (the
|
||||
/// data-plane Primary verdict, closed until this node's first redundancy snapshot). Optional: registered on every
|
||||
/// driver-role node; an admin-only node has none and wires no HistoryUpdate Primary gate (allow).</param>
|
||||
public OtOpcUaServerHostedService(
|
||||
IOptions<OpcUaApplicationHostOptions> options,
|
||||
DeferredAddressSpaceSink deferredSink,
|
||||
@@ -113,7 +118,8 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
|
||||
TimeProvider time,
|
||||
ILoggerFactory loggerFactory,
|
||||
IDriverPrimaryRoleSource? primaryRoleSource = null,
|
||||
HistorianUpdaterHolder? historianUpdater = null)
|
||||
HistorianUpdaterHolder? historianUpdater = null,
|
||||
IHistoryUpdateRoleView? historyUpdateRoleView = null)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(aclOptions);
|
||||
ArgumentNullException.ThrowIfNull(time);
|
||||
@@ -137,6 +143,7 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
|
||||
_loggerFactory = loggerFactory;
|
||||
_primaryRoleSource = primaryRoleSource;
|
||||
_historianUpdater = historianUpdater ?? HistorianUpdaterHolder.None;
|
||||
_historyUpdateRoleView = historyUpdateRoleView;
|
||||
_logger = loggerFactory.CreateLogger<OtOpcUaServerHostedService>();
|
||||
}
|
||||
|
||||
@@ -310,6 +317,12 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
|
||||
// section is absent; NodeManager is non-null here (guarded above).
|
||||
_server.NodeManager.MaxTieClusterOverfetch = _serverHistorianOptions.MaxTieClusterOverfetch;
|
||||
|
||||
// Gitea #691: the ReturnBounds kill switch (default on).
|
||||
_server.NodeManager.HonorReturnBounds = HonorReturnBounds;
|
||||
|
||||
// Gitea #694: the opt-in HistoryReadEvents horizon (default off = exact to each read's own window).
|
||||
_server.NodeManager.EventReadHorizon = EventReadHorizon;
|
||||
|
||||
// Propagate the S3 HistoryRead concurrency bounds (arch-review 03/S3): bounded per-node fan-out
|
||||
// within a batch, the process-wide concurrent-batch limiter, and the per-request deadline. Defaults
|
||||
// (4 / 16 / 60 s) survive when the ServerHistorian section is absent.
|
||||
@@ -317,6 +330,14 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
|
||||
_server.NodeManager.MaxConcurrentHistoryReadBatches = _serverHistorianOptions.MaxConcurrentHistoryReadBatches;
|
||||
_server.NodeManager.HistoryReadDeadline = _serverHistorianOptions.HistoryReadDeadline;
|
||||
|
||||
// Gitea #674: HistoryUpdate is Primary-only (BadInvalidState elsewhere, and closed until this node's first
|
||||
// redundancy snapshot) and refuses a SourceTimestamp beyond now + HistoryUpdateMaxFutureSkew (BadOutOfRange),
|
||||
// judged against the host clock.
|
||||
var historyUpdate = BuildHistoryUpdatePosture();
|
||||
_server.NodeManager.HistoryUpdatePrimaryGate = historyUpdate.PrimaryGate;
|
||||
_server.NodeManager.HistoryUpdateMaxFutureSkew = historyUpdate.MaxFutureSkew;
|
||||
_server.NodeManager.Clock = historyUpdate.Clock;
|
||||
|
||||
// ServiceLevel publisher needs IServerInternal — only available after Start.
|
||||
if (_server.CurrentInstance is { } serverInternal)
|
||||
{
|
||||
@@ -332,6 +353,40 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
|
||||
_logger.LogInformation("OtOpcUaServerHostedService: SDK started, address-space + ServiceLevel sinks bound");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #691 — the <c>ServerHistorian:HonorReturnBounds</c> kill switch <c>StartAsync</c> applies to the node manager
|
||||
/// (default <c>true</c> when the section or key is absent). Exposed so the wiring is testable without booting the SDK.
|
||||
/// </summary>
|
||||
internal bool HonorReturnBounds => _serverHistorianOptions.HonorReturnBounds;
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #694 — the <c>ServerHistorian:EventReadHorizon</c> <c>StartAsync</c> applies to the node manager (default
|
||||
/// <see cref="TimeSpan.Zero"/> = none). Exposed so the wiring is testable without booting the SDK.
|
||||
/// </summary>
|
||||
internal TimeSpan EventReadHorizon => _serverHistorianOptions.EventReadHorizon;
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #674 — what <c>StartAsync</c> wires onto the node manager for HistoryUpdate: the Primary gate (read from
|
||||
/// the <see cref="IHistoryUpdateRoleView"/> on every call; null when none is registered, i.e. allow), the
|
||||
/// future-timestamp bound from <c>ServerHistorian:HistoryUpdateMaxFutureSkew</c>, and the host clock it is judged
|
||||
/// against. Separate from <c>StartAsync</c> so the wiring is testable without booting the SDK.
|
||||
/// </summary>
|
||||
/// <returns>The posture to apply.</returns>
|
||||
internal HistoryUpdatePosture BuildHistoryUpdatePosture()
|
||||
{
|
||||
Func<bool>? gate = _historyUpdateRoleView is { } view ? () => view.ShouldServeHistoryUpdate : null;
|
||||
if (gate is null)
|
||||
_logger.LogInformation(
|
||||
"OtOpcUaServerHostedService: no IHistoryUpdateRoleView registered (no redundancy manager on this node) — HistoryUpdate is not Primary-gated");
|
||||
return new HistoryUpdatePosture(gate, _serverHistorianOptions.HistoryUpdateMaxFutureSkew, _time);
|
||||
}
|
||||
|
||||
/// <summary>Gitea #674 — the HistoryUpdate settings <see cref="BuildHistoryUpdatePosture"/> produces.</summary>
|
||||
/// <param name="PrimaryGate">The node manager's HistoryUpdate Primary gate; null allows.</param>
|
||||
/// <param name="MaxFutureSkew">The future-timestamp bound; non-positive disables it.</param>
|
||||
/// <param name="Clock">The clock the bound is judged against.</param>
|
||||
internal sealed record HistoryUpdatePosture(Func<bool>? PrimaryGate, TimeSpan MaxFutureSkew, TimeProvider Clock);
|
||||
|
||||
/// <summary>
|
||||
/// Arms the clock-driven monitored-item revalidation tick (arch-review 03/S19).
|
||||
/// </summary>
|
||||
@@ -382,6 +437,7 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl
|
||||
// Clear the native-alarm-ack router so a late ack doesn't Tell a stopping DriverHostActor.
|
||||
_server?.SetNativeAlarmAckRouter(null);
|
||||
_server?.SetNativeAlarmAckPrimaryGate(null);
|
||||
_server?.SetHistoryUpdatePrimaryGate(null);
|
||||
// Restore the Null historian so a late HistoryRead doesn't hit a disposed read client.
|
||||
_server?.SetHistorianDataSource(null);
|
||||
_server?.SetHistorianDataUpdater(null);
|
||||
|
||||
@@ -90,8 +90,9 @@ builder.WebHost.UseStaticWebAssets();
|
||||
builder.Host.UseWindowsService(options => options.ServiceName = "OtOpcUaHost");
|
||||
|
||||
// Per-role appsettings overlay: appsettings.{role}.json (single role) or appsettings.admin-driver.json
|
||||
// (both). Optional — base appsettings.json carries enough to boot if these don't exist.
|
||||
var roleSuffix = roles.Count == 0 ? null : string.Join('-', roles.OrderBy(r => r, StringComparer.Ordinal));
|
||||
// (both). Optional — base appsettings.json carries enough to boot if these don't exist. Built from the
|
||||
// PLAIN roles only (#678.6): a cluster-{ClusterId} role in the name pointed at a file that never exists.
|
||||
var roleSuffix = RoleOverlay.Suffix(roles);
|
||||
if (roleSuffix is not null)
|
||||
{
|
||||
builder.Configuration.AddJsonFile($"appsettings.{roleSuffix}.json", optional: true, reloadOnChange: true);
|
||||
@@ -326,6 +327,14 @@ if (hasDriver)
|
||||
builder.Services.AddSingleton<IHistorianValueWriter>(sp => sp.GetRequiredService<IHistorianBackendProvider>().CreateValueWriter(sp));
|
||||
}
|
||||
|
||||
// Historian retention (#667). Only a backend that manages retention in-process returns a purger
|
||||
// (SqlServer with RetentionDays set); otherwise the service is inert. Primary-gated via IHistorizationRoleView.
|
||||
if (serverHistorianOptions.Enabled)
|
||||
builder.Services.AddHostedService(sp => new HistorianRetentionService(
|
||||
sp.GetRequiredService<IHistorianBackendProvider>().CreateRetention(sp),
|
||||
sp.GetService<ZB.MOM.WW.OtOpcUa.Runtime.Redundancy.IHistorizationRoleView>(),
|
||||
sp.GetService<ILogger<HistorianRetentionService>>()));
|
||||
|
||||
// Bind every cross-platform driver factory before AddAkka resolves IDriverFactory — replaces
|
||||
// the F7-default NullDriverFactory with a real DriverFactoryRegistryAdapter so DriverHostActor
|
||||
// can materialise real IDriver instances on deploy.
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
"MinimumLevel": {
|
||||
"Default": "Information",
|
||||
"Override": {
|
||||
"Opc.Ua": "Debug",
|
||||
"Opc.Ua": "Information",
|
||||
"Akka": "Information"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -35,7 +35,9 @@
|
||||
"AllowUntrustedServerCertificate": false,
|
||||
"CaCertificatePath": null,
|
||||
"CallTimeout": "00:00:30",
|
||||
"MaxTieClusterOverfetch": 65536
|
||||
"MaxTieClusterOverfetch": 65536,
|
||||
"EventReadHorizon": "00:00:00",
|
||||
"_EventReadHorizonComment": "Gitea #694, opt-in. 00:00:00 = no horizon: HistoryReadEvents is exact to each read's window (on the Gateway a reverse/end-only events read proves 'nothing older' with one read down to 1970 - seconds on a large Runtime.dbo.Events). A positive span (e.g. 365.00:00:00) never reads events older than now minus it, in either direction - LOSSY: older events become invisible to HistoryReadEvents."
|
||||
},
|
||||
"ContinuousHistorization": {
|
||||
"_comment": "Continuous historization of driver (non-Galaxy) tag values: a crash-safe FasterLog outbox + recorder draining to the ServerHistorian gateway's WriteLiveValues. Disabled => no recorder is spawned. Requires ServerHistorian to be configured; the gateway connection (endpoint/key/TLS) is sourced from the ServerHistorian section, not here.",
|
||||
@@ -55,7 +57,9 @@
|
||||
"MaxWriteAttempts": 5,
|
||||
"_MaxWriteAttemptsComment": "Gitea #538: consecutive failed drain passes one tag may accumulate — counted only across passes where OTHER tags wrote successfully — before its queued entries are abandoned so the FIFO head can advance.",
|
||||
"MinBackoffSeconds": 1,
|
||||
"MaxBackoffSeconds": 30
|
||||
"MaxBackoffSeconds": 30,
|
||||
"CaptureWhilePrimaryUnreachable": true,
|
||||
"_CaptureWhilePrimaryUnreachableComment": "Gitea #672: a standby stood down for its mesh Primary resumes capture as soon as that Primary is reported UNREACHABLE (not after auto-down) — shrinks a crash-failover capture gap to the failure-detector window, at the cost of short duplicate windows. false = pre-#672."
|
||||
},
|
||||
"AlarmHistorian": {
|
||||
"_comment": "Durable store-and-forward alarm sink. Buffers into the node's consolidated LocalDb (see the LocalDb section) so the queue replicates to the redundant pair peer, and drains alarm events to the ServerHistorian gateway's SendEvent path from whichever node holds the Primary role. The downstream connection (endpoint/key/TLS) is sourced from the ServerHistorian section.",
|
||||
|
||||
@@ -0,0 +1,91 @@
|
||||
using System.Reflection;
|
||||
using Opc.Ua;
|
||||
using Opc.Ua.Server;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.OpcUaServer;
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #682 — make <c>HistoryServerCapabilities/AggregateFunctions</c> (i=11201) list exactly the aggregates the
|
||||
/// wired historian backend serves for HistoryRead (<see cref="IHistorianDataSource.SupportedAggregates"/>), instead of
|
||||
/// the SDK's full registered set.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// ⚠️ <c>ServerCapabilities/AggregateFunctions</c> (i=2997) is deliberately NOT trimmed. Part 5 defines it as every
|
||||
/// aggregate the SERVER supports, and the server still supports all the SDK's registered aggregates for live
|
||||
/// data: a MonitoredItem <c>AggregateFilter</c> is validated and computed by the SDK's <c>AggregateManager</c>,
|
||||
/// not the historian. The SDK itself models the split — <c>DiagnosticsNodeManager.AddAggregateFunction</c> always
|
||||
/// adds to i=2997 and adds to i=11201 only when the aggregate is historical. i=11201 is the history surface, so it
|
||||
/// is the one that follows the backend.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Only the folder ⇄ aggregate <c>Organizes</c> reference pair is removed or restored (both directions); the
|
||||
/// calculator factories and the aggregate-function object nodes stay. Idempotent and reversible, so re-wiring a
|
||||
/// different source re-syncs the folder.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
internal static class AdvertisedAggregateFunctions
|
||||
{
|
||||
/// <summary>Every standard aggregate-function NodeId the SDK's nodeset knows (<c>ObjectIds.AggregateFunction_*</c>).</summary>
|
||||
private static readonly NodeId[] StandardAggregateIds = typeof(ObjectIds)
|
||||
.GetFields(BindingFlags.Public | BindingFlags.Static)
|
||||
.Where(f => f.FieldType == typeof(NodeId) && f.Name.StartsWith("AggregateFunction_", StringComparison.Ordinal))
|
||||
.Select(f => (NodeId)f.GetValue(null)!)
|
||||
.ToArray();
|
||||
|
||||
/// <summary>The trimmed folder — the HISTORY aggregate list only (see remarks for why not i=2997).</summary>
|
||||
internal static readonly NodeId Folder = ObjectIds.HistoryServerCapabilities_AggregateFunctions;
|
||||
|
||||
/// <summary>
|
||||
/// The set a source serves, treating a <c>null</c> <see cref="IHistorianDataSource.SupportedAggregates"/> (an
|
||||
/// out-of-tree source that returns null despite the non-nullable contract) as <see cref="HistoryAggregateTypes.All"/>,
|
||||
/// the interface default — the ONE null rule both the advertisement and HistoryRead-Processed use.
|
||||
/// </summary>
|
||||
/// <param name="source">The historian source.</param>
|
||||
/// <returns>Its served set, never null.</returns>
|
||||
public static IReadOnlySet<HistoryAggregateType> ServedBy(IHistorianDataSource source) =>
|
||||
source.SupportedAggregates ?? HistoryAggregateTypes.All;
|
||||
|
||||
/// <summary>
|
||||
/// Sync i=11201 to <paramref name="served"/>: an aggregate-function node is organized under it iff the server maps
|
||||
/// it (<see cref="AggregateFunctionMap"/>) AND <paramref name="served"/> contains it.
|
||||
/// </summary>
|
||||
/// <param name="server">The running server.</param>
|
||||
/// <param name="served">The backend's served aggregates.</param>
|
||||
/// <returns>The advertised NodeIds, or <c>null</c> when the diagnostics node manager or folder is not available.</returns>
|
||||
public static IReadOnlyList<NodeId>? Apply(IServerInternal server, IReadOnlySet<HistoryAggregateType> served)
|
||||
{
|
||||
if (server?.DiagnosticsNodeManager is not CustomNodeManager2 diagnostics) return null;
|
||||
|
||||
var advertised = new List<NodeId>();
|
||||
lock (diagnostics.Lock)
|
||||
{
|
||||
var folder = diagnostics.FindPredefinedNode(Folder, typeof(BaseObjectState));
|
||||
if (folder is null) return null;
|
||||
|
||||
foreach (var aggregateId in StandardAggregateIds)
|
||||
{
|
||||
var aggregateNode = diagnostics.FindPredefinedNode(aggregateId, null);
|
||||
if (aggregateNode is null) continue;
|
||||
|
||||
var advertise = AggregateFunctionMap.Map(aggregateId) is { } aggregate && served.Contains(aggregate);
|
||||
if (advertise) advertised.Add(aggregateId);
|
||||
|
||||
var linked = folder.ReferenceExists(ReferenceTypeIds.Organizes, false, aggregateId);
|
||||
if (advertise && !linked)
|
||||
{
|
||||
folder.AddReference(ReferenceTypeIds.Organizes, false, aggregateId);
|
||||
aggregateNode.AddReference(ReferenceTypeIds.Organizes, true, folder.NodeId);
|
||||
}
|
||||
else if (!advertise && linked)
|
||||
{
|
||||
folder.RemoveReference(ReferenceTypeIds.Organizes, false, aggregateId);
|
||||
aggregateNode.RemoveReference(ReferenceTypeIds.Organizes, true, folder.NodeId);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return advertised;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
using System.Collections.Frozen;
|
||||
using Opc.Ua;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.OpcUaServer;
|
||||
|
||||
/// <summary>
|
||||
/// The one table between OPC UA Part 13 standard aggregate-function NodeIds (<c>ObjectIds.AggregateFunction_*</c>)
|
||||
/// and <see cref="HistoryAggregateType"/>. HistoryRead-Processed maps a request's aggregate through it, and the
|
||||
/// advertised <c>AggregateFunctions</c> folders (#682) map a backend's served set back through it — one table, so a
|
||||
/// NodeId the server can dispatch is exactly a NodeId it can advertise.
|
||||
/// </summary>
|
||||
internal static class AggregateFunctionMap
|
||||
{
|
||||
/// <summary>NodeId ⇒ aggregate, for every aggregate the server can name to a backend.</summary>
|
||||
public static readonly FrozenDictionary<NodeId, HistoryAggregateType> ByNodeId = new Dictionary<NodeId, HistoryAggregateType>
|
||||
{
|
||||
[ObjectIds.AggregateFunction_Average] = HistoryAggregateType.Average,
|
||||
[ObjectIds.AggregateFunction_Minimum] = HistoryAggregateType.Minimum,
|
||||
[ObjectIds.AggregateFunction_Maximum] = HistoryAggregateType.Maximum,
|
||||
[ObjectIds.AggregateFunction_Total] = HistoryAggregateType.Total,
|
||||
[ObjectIds.AggregateFunction_Count] = HistoryAggregateType.Count,
|
||||
[ObjectIds.AggregateFunction_Interpolative] = HistoryAggregateType.Interpolative,
|
||||
[ObjectIds.AggregateFunction_TimeAverage] = HistoryAggregateType.TimeAverage,
|
||||
[ObjectIds.AggregateFunction_Start] = HistoryAggregateType.Start,
|
||||
[ObjectIds.AggregateFunction_End] = HistoryAggregateType.End,
|
||||
[ObjectIds.AggregateFunction_Delta] = HistoryAggregateType.Delta,
|
||||
[ObjectIds.AggregateFunction_Range] = HistoryAggregateType.Range,
|
||||
[ObjectIds.AggregateFunction_MinimumActualTime] = HistoryAggregateType.MinimumActualTime,
|
||||
[ObjectIds.AggregateFunction_MaximumActualTime] = HistoryAggregateType.MaximumActualTime,
|
||||
[ObjectIds.AggregateFunction_StartBound] = HistoryAggregateType.StartBound,
|
||||
[ObjectIds.AggregateFunction_EndBound] = HistoryAggregateType.EndBound,
|
||||
[ObjectIds.AggregateFunction_DeltaBounds] = HistoryAggregateType.DeltaBounds,
|
||||
[ObjectIds.AggregateFunction_DurationGood] = HistoryAggregateType.DurationGood,
|
||||
[ObjectIds.AggregateFunction_DurationBad] = HistoryAggregateType.DurationBad,
|
||||
[ObjectIds.AggregateFunction_PercentGood] = HistoryAggregateType.PercentGood,
|
||||
[ObjectIds.AggregateFunction_PercentBad] = HistoryAggregateType.PercentBad,
|
||||
[ObjectIds.AggregateFunction_WorstQuality] = HistoryAggregateType.WorstQuality,
|
||||
[ObjectIds.AggregateFunction_NumberOfTransitions] = HistoryAggregateType.NumberOfTransitions,
|
||||
[ObjectIds.AggregateFunction_AnnotationCount] = HistoryAggregateType.AnnotationCount,
|
||||
[ObjectIds.AggregateFunction_StandardDeviationPopulation] = HistoryAggregateType.StandardDeviationPopulation,
|
||||
[ObjectIds.AggregateFunction_VariancePopulation] = HistoryAggregateType.VariancePopulation,
|
||||
[ObjectIds.AggregateFunction_TimeAverage2] = HistoryAggregateType.TimeAverage2,
|
||||
[ObjectIds.AggregateFunction_Total2] = HistoryAggregateType.Total2,
|
||||
}.ToFrozenDictionary();
|
||||
|
||||
/// <summary>
|
||||
/// Map a request's aggregate-function NodeId to <see cref="HistoryAggregateType"/>; <c>null</c> for one the server
|
||||
/// cannot name to any backend (the caller answers <c>BadAggregateNotSupported</c>).
|
||||
/// </summary>
|
||||
/// <param name="aggregateNodeId">The per-node aggregate-function NodeId from the request.</param>
|
||||
/// <returns>The mapped aggregate, or <c>null</c>.</returns>
|
||||
public static HistoryAggregateType? Map(NodeId? aggregateNodeId) =>
|
||||
aggregateNodeId is not null && ByNodeId.TryGetValue(aggregateNodeId, out var aggregate) ? aggregate : null;
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
namespace ZB.MOM.WW.OtOpcUa.OpcUaServer;
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #694 item 2: the opt-in <c>ServerHistorian:EventReadHorizon</c> — HistoryReadEvents never reads events older than
|
||||
/// <c>now − horizon</c>. Off (<see cref="TimeSpan.Zero"/>, the default) a read is exact down to whatever its window says,
|
||||
/// which on the Gateway means a reverse / end-only read walked back with forward reads proves "nothing older" with one
|
||||
/// read down to the Unix epoch (#691) — ~3.8 s live on <c>Runtime.dbo.Events</c>, and an unfiltered one can hit the
|
||||
/// gateway's 30 s SQL timeout. A forward read whose StartTime is far in the past costs the same on that table. The
|
||||
/// horizon bounds both, at a stated price: events older than it are not returned, in either direction, on any backend.
|
||||
/// </summary>
|
||||
internal static class EventHorizon
|
||||
{
|
||||
/// <summary>
|
||||
/// Raises the EARLIER bound of a fresh events window to the horizon's floor (<c>now − horizon</c>, inclusive): the
|
||||
/// start of a forward <c>[start, end)</c> window, the exclusive lower end of a reverse <c>(end, start]</c> one.
|
||||
/// </summary>
|
||||
/// <param name="startUtc">The window start (the later bound of a reverse read).</param>
|
||||
/// <param name="endUtc">The window end (the exclusive earlier bound of a reverse read).</param>
|
||||
/// <param name="reverse">Whether the read is newest-first (<c>start > end</c>).</param>
|
||||
/// <param name="horizon">The horizon; non-positive = none (nothing changes).</param>
|
||||
/// <param name="nowUtc">Now.</param>
|
||||
/// <returns><c>false</c> when the whole window lies before the floor — the read is empty and must not be sent.</returns>
|
||||
public static bool TryClamp(ref DateTime startUtc, ref DateTime endUtc, bool reverse, TimeSpan horizon, DateTime nowUtc)
|
||||
{
|
||||
if (horizon <= TimeSpan.Zero) return true;
|
||||
var floor = nowUtc.Ticks - DateTime.MinValue.Ticks > horizon.Ticks
|
||||
? new DateTime(nowUtc.Ticks - horizon.Ticks, DateTimeKind.Utc)
|
||||
: new DateTime(0, DateTimeKind.Utc);
|
||||
if (floor.Ticks == 0) return true;
|
||||
|
||||
if (reverse)
|
||||
{
|
||||
if (startUtc < floor) return false; // (end, start] lies wholly before the floor
|
||||
var below = floor.AddTicks(-1); // exclusive lower end ⇒ the floor instant itself is read
|
||||
if (endUtc < below) endUtc = below;
|
||||
return true;
|
||||
}
|
||||
|
||||
if (endUtc <= floor) return false; // [start, end) lies wholly before the floor
|
||||
if (startUtc < floor) startUtc = floor;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,241 @@
|
||||
using Opc.Ua;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.OpcUaServer;
|
||||
|
||||
/// <summary>
|
||||
/// The end bound still owed to a HistoryRead-Raw read with <c>ReturnBounds = true</c> (Gitea #691). It rides the
|
||||
/// continuation point, because Part 11 returns the end bound with the LAST page only.
|
||||
/// </summary>
|
||||
/// <param name="AtUtc">The request's end time (the later bound of a forward read, the earlier of a reverse one).</param>
|
||||
/// <param name="Reverse">The read is newest-first (<c>StartTime > EndTime</c>, including an end-only read).</param>
|
||||
/// <param name="Inclusive">
|
||||
/// <c>false</c> only for <c>StartTime == EndTime</c>: the samples AT that instant are the data, so the end bound is the
|
||||
/// first sample strictly after it (Part 11 Table 1 footnote c). Otherwise the end bound is the sample at or beyond the
|
||||
/// end time, and the data window excludes the end time.
|
||||
/// </param>
|
||||
/// <param name="Open">The end time was not given (a one-sided read's sentinel): there is no end bound to look up.</param>
|
||||
/// <param name="DataExhausted">The data is drained and the end bound did not fit on the previous page.</param>
|
||||
/// <param name="LastReturnedUtc">The last value's time returned on an earlier page (stamps an open bound's placeholder).</param>
|
||||
/// <param name="Resolved">
|
||||
/// Gitea #692: the end bound an earlier page already looked up (a stored sample — never a placeholder), so the page that
|
||||
/// returns it does not read the backend again. <c>null</c> = not looked up yet.
|
||||
/// </param>
|
||||
internal sealed record HistoryEndBound(
|
||||
DateTime AtUtc, bool Reverse, bool Inclusive, bool Open, bool DataExhausted = false, DateTime? LastReturnedUtc = null,
|
||||
DataValueSnapshot? Resolved = null);
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #691: OPC UA Part 11 bounding values for HistoryRead-Raw (<c>ReadRawModifiedDetails.ReturnBounds</c>), composed
|
||||
/// in the server from every backend's existing raw read — the ONE authority, so SqlServer, QuestDB, Gateway and Null
|
||||
/// answer identically.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>What a bound is</b> (Part 11 §4.6, Table 1). Forward: the start bound is the stored sample AT or immediately
|
||||
/// BEFORE StartTime, the end bound the one AT or immediately AFTER EndTime. Reverse mirrors it: the start bound is at
|
||||
/// or after StartTime, the end bound at or before EndTime. A sample exactly on a bound IS the bound: on the start it
|
||||
/// is already the first sample of the data (the window includes StartTime), so nothing extra is added; on the end it is
|
||||
/// added (the window excludes EndTime). <c>StartTime == EndTime</c> reads forward, and its end bound is the first
|
||||
/// sample strictly after the instant. A bound is a stored sample of any quality — a Bad sample is returned as stored.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Missing bound.</b> <c>Bad_BoundNotFound</c>, a null value, stamped with the bound's time. A one-sided read's
|
||||
/// missing side (no EndTime / no StartTime) is never looked up; it is <c>Bad_BoundNotFound</c> stamped one second after
|
||||
/// (forward) or before (reverse) the previous returned value (Table 1 footnotes a and b).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Lookups.</b> "At or before T" is a newest-first read of <c>(0, T]</c> capped at one — native on a backend that
|
||||
/// declares <see cref="IHistorianDataSource.SupportsReverseRaw"/>, otherwise <see cref="ReverseRawReader"/>'s walk-back
|
||||
/// from bounded forward reads (the Gateway: NOT AVEVA's synthetic boundary row, which is stamped at the query start
|
||||
/// rather than at the prior sample's own time). "At or after T" is a forward read of <c>[T, MaxValue)</c> capped at one (on a
|
||||
/// forward-only backend ending at now + <see cref="ReverseRawReader.LookAhead"/> instead — #691 review);
|
||||
/// "strictly after T" starts one storage resolution later (lossless: nothing is stored in between). Inside a tie
|
||||
/// cluster the bound is the cluster's first sample in the read's direction — the forward read's first, the reverse
|
||||
/// read's first (the forward order's last).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Resolution.</b> "On the bound" compares at the backend's storage resolution, because a microsecond backend
|
||||
/// (QuestDB) truncates the window bounds it is sent: a sample at <c>⌊T⌋</c> is inside a window starting at <c>T</c>
|
||||
/// there, so it must not also be returned as the start bound.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
internal static class HistoryBounds
|
||||
{
|
||||
private static readonly TimeSpan OpenBoundStep = TimeSpan.FromSeconds(1);
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #692: the placeholders this server synthesised (<see cref="NotFound"/>), by reference. A placeholder is known by
|
||||
/// this marker, never by its shape: a stored row can legitimately carry <c>Bad_BoundNotFound</c> with a null value (a
|
||||
/// client may write one through HistoryUpdate), and that row is data.
|
||||
/// </summary>
|
||||
private static readonly System.Runtime.CompilerServices.ConditionalWeakTable<DataValueSnapshot, object> Placeholders = new();
|
||||
|
||||
private static readonly object PlaceholderMark = new();
|
||||
|
||||
/// <summary>The end bound owed by a fresh read over the normalised window.</summary>
|
||||
/// <param name="startUtc">The (normalised) start: the later time of a reverse read.</param>
|
||||
/// <param name="endUtc">The (normalised) end; a one-sided read's sentinel (tick 0 / MaxValue) is an open bound.</param>
|
||||
/// <returns>The pending end bound.</returns>
|
||||
public static HistoryEndBound For(DateTime startUtc, DateTime endUtc) =>
|
||||
new(endUtc, Reverse: startUtc > endUtc, Inclusive: startUtc != endUtc, Open: IsOpen(endUtc));
|
||||
|
||||
/// <summary>
|
||||
/// Whether a returned value is a <see cref="NotFound"/> placeholder this server synthesised rather than a stored sample
|
||||
/// — by an explicit marker (#692), so a stored row whose status happens to be <c>Bad_BoundNotFound</c> is not taken for one.
|
||||
/// </summary>
|
||||
/// <param name="value">The value.</param>
|
||||
/// <returns><c>true</c> only for an instance <see cref="NotFound"/> produced.</returns>
|
||||
public static bool IsPlaceholder(DataValueSnapshot value) => Placeholders.TryGetValue(value, out _);
|
||||
|
||||
/// <summary>The pending end bound to store behind a continuation point, remembering the page's last returned time.</summary>
|
||||
/// <param name="bound">The pending end bound (null = bounds not requested).</param>
|
||||
/// <param name="page">The page being returned.</param>
|
||||
/// <returns>The bound to carry, or null.</returns>
|
||||
public static HistoryEndBound? Carry(HistoryEndBound? bound, IReadOnlyList<DataValueSnapshot> page) =>
|
||||
bound is null || page.Count == 0 || page[^1].SourceTimestampUtc is not { } last
|
||||
? bound
|
||||
: bound with { LastReturnedUtc = last };
|
||||
|
||||
/// <summary>The <c>Bad_BoundNotFound</c> placeholder (null value) stamped at <paramref name="atUtc"/>.</summary>
|
||||
/// <param name="atUtc">The bound's time.</param>
|
||||
/// <returns>The placeholder.</returns>
|
||||
public static DataValueSnapshot NotFound(DateTime atUtc)
|
||||
{
|
||||
var placeholder = new DataValueSnapshot(null, StatusCodes.BadBoundNotFound, atUtc, atUtc);
|
||||
Placeholders.AddOrUpdate(placeholder, PlaceholderMark);
|
||||
return placeholder;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The start bound to return ahead of the data, or <c>null</c> when a stored sample sits on StartTime (that sample is
|
||||
/// the bound, and it is already the data's first sample).
|
||||
/// </summary>
|
||||
public static async Task<DataValueSnapshot?> StartBoundAsync(
|
||||
IHistorianDataSource source, string tagname, DateTime startUtc, bool reverse, int maxTieCluster, DateTime nowUtc,
|
||||
CancellationToken ct)
|
||||
{
|
||||
if (IsOpen(startUtc)) return NotFound(startUtc);
|
||||
var found = reverse
|
||||
? await EarliestAtOrAfterAsync(source, tagname, startUtc, nowUtc, ct).ConfigureAwait(false)
|
||||
: await LatestAtOrBeforeAsync(source, tagname, startUtc, maxTieCluster, nowUtc, ct).ConfigureAwait(false);
|
||||
if (found is null) return NotFound(startUtc);
|
||||
return SameInstant(Ticks(found), startUtc.Ticks, Resolution(source)) ? null : found;
|
||||
}
|
||||
|
||||
/// <summary>The end bound: the stored sample on or beyond the end time, else the placeholder.</summary>
|
||||
/// <param name="source">The backend.</param>
|
||||
/// <param name="tagname">Historian tag.</param>
|
||||
/// <param name="bound">The pending end bound.</param>
|
||||
/// <param name="previousReturnedUtc">The last value's time already returned (stamps an open bound's placeholder).</param>
|
||||
/// <param name="maxTieCluster">The walk-back's row budget (forward-only backends).</param>
|
||||
/// <param name="nowUtc">Now (the walk-back's look-ahead clamp).</param>
|
||||
/// <param name="ct">Cancellation.</param>
|
||||
public static async Task<DataValueSnapshot> EndBoundAsync(
|
||||
IHistorianDataSource source, string tagname, HistoryEndBound bound, DateTime previousReturnedUtc,
|
||||
int maxTieCluster, DateTime nowUtc, CancellationToken ct)
|
||||
{
|
||||
if (bound.Open)
|
||||
{
|
||||
var ticks = bound.Reverse
|
||||
? Math.Max(0L, previousReturnedUtc.Ticks - OpenBoundStep.Ticks)
|
||||
: Math.Min(DateTime.MaxValue.Ticks, previousReturnedUtc.Ticks + OpenBoundStep.Ticks);
|
||||
return NotFound(new DateTime(ticks, DateTimeKind.Utc));
|
||||
}
|
||||
|
||||
if (bound.Resolved is { } resolved) return resolved; // #692: an earlier page already looked it up
|
||||
|
||||
DataValueSnapshot? found;
|
||||
if (bound.Reverse)
|
||||
found = await LatestAtOrBeforeAsync(source, tagname, bound.AtUtc, maxTieCluster, nowUtc, ct).ConfigureAwait(false);
|
||||
else if (bound.Inclusive)
|
||||
found = await EarliestAtOrAfterAsync(source, tagname, bound.AtUtc, nowUtc, ct).ConfigureAwait(false);
|
||||
else
|
||||
{
|
||||
var resolution = Resolution(source);
|
||||
found = bound.AtUtc.Ticks > DateTime.MaxValue.Ticks - resolution
|
||||
? null
|
||||
: await EarliestAtOrAfterAsync(source, tagname, bound.AtUtc.AddTicks(resolution), nowUtc, ct).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
return found ?? NotFound(bound.AtUtc);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Whether a data sample lies inside the data window of a bounded read — a forward window excludes the end time (a
|
||||
/// sample there is the end bound), compared at the storage resolution. A backend whose forward end is inclusive, or a
|
||||
/// resumed read that steps onto the end time, would otherwise return the end bound twice.
|
||||
/// </summary>
|
||||
public static bool InDataWindow(DataValueSnapshot sample, HistoryEndBound bound, long resolution)
|
||||
{
|
||||
if (!bound.Inclusive || bound.Open) return true;
|
||||
var ts = Align(Ticks(sample), resolution);
|
||||
var end = Align(bound.AtUtc.Ticks, resolution);
|
||||
return bound.Reverse ? ts > end : ts < end;
|
||||
}
|
||||
|
||||
/// <summary>Whether a resumed read's next start is already past the data window (only the end bound remains).</summary>
|
||||
public static bool DataWindowDrained(DateTime nextStartUtc, HistoryEndBound bound, long resolution)
|
||||
{
|
||||
if (!bound.Inclusive || bound.Open) return false;
|
||||
var next = Align(nextStartUtc.Ticks, resolution);
|
||||
var end = Align(bound.AtUtc.Ticks, resolution);
|
||||
return bound.Reverse ? next <= end : next >= end;
|
||||
}
|
||||
|
||||
/// <summary>The latest stored sample at or before <paramref name="atUtc"/>, or null.</summary>
|
||||
internal static async Task<DataValueSnapshot?> LatestAtOrBeforeAsync(
|
||||
IHistorianDataSource source, string tagname, DateTime atUtc, int maxTieCluster, DateTime nowUtc, CancellationToken ct)
|
||||
{
|
||||
if (atUtc.Ticks == 0) return null;
|
||||
var floor = new DateTime(0, DateTimeKind.Utc);
|
||||
IReadOnlyList<DataValueSnapshot> rows = source.SupportsReverseRaw
|
||||
? (await source.ReadRawAsync(tagname, atUtc, floor, 1, ct).ConfigureAwait(false)).Samples
|
||||
: (await ReverseRawReader.ReadAsync(source, tagname, atUtc, floor, 1, maxTieCluster, nowUtc, 0, ct)
|
||||
.ConfigureAwait(false)).Samples;
|
||||
var limit = Align(atUtc.Ticks, Resolution(source));
|
||||
foreach (var row in rows)
|
||||
if (Align(Ticks(row), Resolution(source)) <= limit)
|
||||
return row;
|
||||
return null;
|
||||
}
|
||||
|
||||
/// <summary>The earliest stored sample at or after <paramref name="atUtc"/>, or null.</summary>
|
||||
/// <remarks>
|
||||
/// #691 review: on a backend that reads forward only (the Gateway) the read's end is clamped to now +
|
||||
/// <see cref="ReverseRawReader.LookAhead"/> — the same horizon the walk-back uses — and a time beyond it is "not found"
|
||||
/// without a read. The HistorianGateway only checks <c>start <= end</c>, so a <c>DateTime.MaxValue</c> end would
|
||||
/// reach AVEVA as a year-9999 query end (unverified: a rejection, or a fabricated row). The clamp is NOT applied to a
|
||||
/// backend that reads newest-first natively (SqlServer / QuestDB): they bind <c>MaxValue</c> safely (pinned by the
|
||||
/// <c>Extreme_bounds</c> conformance test) and can legitimately hold samples further ahead (HistoryUpdate's
|
||||
/// <c>HistoryUpdateMaxFutureSkew</c> is configurable, and driver-stamped live values carry device time) — clamping
|
||||
/// there would turn a real end bound into <c>Bad_BoundNotFound</c>.
|
||||
/// </remarks>
|
||||
internal static async Task<DataValueSnapshot?> EarliestAtOrAfterAsync(
|
||||
IHistorianDataSource source, string tagname, DateTime atUtc, DateTime nowUtc, CancellationToken ct)
|
||||
{
|
||||
var endTicks = DateTime.MaxValue.Ticks;
|
||||
if (!source.SupportsReverseRaw)
|
||||
endTicks = nowUtc.Ticks > DateTime.MaxValue.Ticks - ReverseRawReader.LookAhead.Ticks
|
||||
? DateTime.MaxValue.Ticks
|
||||
: nowUtc.Ticks + ReverseRawReader.LookAhead.Ticks;
|
||||
if (atUtc.Ticks >= endTicks) return null;
|
||||
var rows = (await source.ReadRawAsync(tagname, atUtc, new DateTime(endTicks, DateTimeKind.Utc), 1, ct)
|
||||
.ConfigureAwait(false)).Samples;
|
||||
var limit = Align(atUtc.Ticks, Resolution(source));
|
||||
foreach (var row in rows)
|
||||
if (Align(Ticks(row), Resolution(source)) >= limit)
|
||||
return row;
|
||||
return null;
|
||||
}
|
||||
|
||||
private static bool IsOpen(DateTime t) => t.Ticks == 0 || t.Ticks == DateTime.MaxValue.Ticks;
|
||||
|
||||
private static bool SameInstant(long a, long b, long resolution) => Align(a, resolution) == Align(b, resolution);
|
||||
|
||||
private static long Resolution(IHistorianDataSource source) => Math.Max(1L, source.TimestampResolutionTicks);
|
||||
|
||||
private static long Ticks(DataValueSnapshot s) => (s.SourceTimestampUtc ?? DateTime.MinValue).Ticks;
|
||||
|
||||
private static long Align(long ticks, long resolution) => ticks - ticks % resolution;
|
||||
}
|
||||
@@ -31,12 +31,78 @@ namespace ZB.MOM.WW.OtOpcUa.OpcUaServer;
|
||||
/// prior pages and must be dropped from the head of the next page (tie de-dup).
|
||||
/// </param>
|
||||
/// <param name="NumValuesPerNode">The client's per-page cap; re-applied to every resumed page.</param>
|
||||
/// <param name="Kind">
|
||||
/// Which read issued the cursor (Gitea #669 / #668). A cursor is only ever resumed by the read kind that issued it, so a
|
||||
/// raw point presented to an events or modified read (or any other mismatch) is a <c>BadContinuationPointInvalid</c>
|
||||
/// miss. <see cref="HistoryCursorKind.Events"/>: <paramref name="Tagname"/> carries the registered historian
|
||||
/// event-source name and the boundary fields are over <c>EventTimeUtc</c>.
|
||||
/// </param>
|
||||
/// <param name="LastModifiedId">
|
||||
/// <see cref="HistoryCursorKind.Modified"/> only: the <c>ModifiedId</c> of the last returned modification. With
|
||||
/// <paramref name="NextStartUtc"/> (its source time) it is the exclusive resume key; <paramref name="BoundarySkipCount"/>
|
||||
/// is unused because the key is unique.
|
||||
/// </param>
|
||||
/// <param name="WindowStartUtc">
|
||||
/// <see cref="HistoryCursorKind.Modified"/> only: the normalised window start, so a resumed page keeps the original
|
||||
/// window and direction (the modified read resumes by key, not by moving the window start).
|
||||
/// </param>
|
||||
/// <param name="Reverse">
|
||||
/// <see cref="HistoryCursorKind.Events"/> only (Gitea #675): the read is newest-first over
|
||||
/// <c>(EndUtc, NextStartUtc]</c>. A resumed events read takes its direction from here, never from the resume
|
||||
/// request's details. Raw keeps deriving it from the window (<c>NextStartUtc > EndUtc</c>).
|
||||
/// </param>
|
||||
/// <param name="SegmentWidthTicks">
|
||||
/// <see cref="HistoryCursorKind.Raw"/> reverse reads on a forward-only backend only (Gitea #686): the segment width the
|
||||
/// walk-back ended on, so the next page starts at the observed density instead of re-discovering it. 0 = none.
|
||||
/// </param>
|
||||
/// <param name="EmptyFromTicks">
|
||||
/// Walked reverse reads only (Gitea #691): with <paramref name="EmptyToTicks"/>, a stretch <c>(from, to)</c> the previous
|
||||
/// page's walk proved holds no row, so the next page steps over it rather than re-walking it (to the 1970 floor, ~14
|
||||
/// reads, at the end of a deep page-through). 0/0 = none. The cursor lives in the session's in-memory store (never
|
||||
/// serialized), so a new field with a "none" default is all the versioning it needs.
|
||||
/// </param>
|
||||
/// <param name="EmptyToTicks">The exclusive upper end of that stretch.</param>
|
||||
/// <param name="PendingEndBound">
|
||||
/// <see cref="HistoryCursorKind.Raw"/> with <c>ReturnBounds = true</c> only (Gitea #691): the end bound still owed —
|
||||
/// Part 11 returns it with the last page. <c>null</c> = bounds not requested.
|
||||
/// </param>
|
||||
/// <param name="WithinTieCluster">
|
||||
/// <see cref="HistoryCursorKind.Raw"/> only (Gitea #692): the previous page was a slice of an oversized tie cluster at
|
||||
/// <paramref name="NextStartUtc"/> that still has un-emitted ties — the next page over-fetches the cluster directly
|
||||
/// instead of first re-reading <c>N + skip + 1</c> rows that are known to be ties.
|
||||
/// </param>
|
||||
internal sealed record HistoryContinuationState(
|
||||
string Tagname,
|
||||
DateTime NextStartUtc,
|
||||
DateTime EndUtc,
|
||||
int BoundarySkipCount,
|
||||
uint NumValuesPerNode);
|
||||
uint NumValuesPerNode,
|
||||
HistoryCursorKind Kind = HistoryCursorKind.Raw,
|
||||
long LastModifiedId = 0,
|
||||
DateTime WindowStartUtc = default,
|
||||
bool Reverse = false,
|
||||
long SegmentWidthTicks = 0,
|
||||
long EmptyFromTicks = 0,
|
||||
long EmptyToTicks = 0,
|
||||
HistoryEndBound? PendingEndBound = null,
|
||||
bool WithinTieCluster = false)
|
||||
{
|
||||
/// <summary>The walk-back hint this cursor carries (#686/#689/#691).</summary>
|
||||
public ReverseWalkHint WalkHint => new(SegmentWidthTicks, EmptyFromTicks, EmptyToTicks);
|
||||
}
|
||||
|
||||
/// <summary>The HistoryRead kind that issued a <see cref="HistoryContinuationState"/>.</summary>
|
||||
internal enum HistoryCursorKind
|
||||
{
|
||||
/// <summary>HistoryReadRaw (values).</summary>
|
||||
Raw,
|
||||
|
||||
/// <summary>HistoryReadEvents (#669).</summary>
|
||||
Events,
|
||||
|
||||
/// <summary>HistoryReadRaw with <c>IsReadModified = true</c> — the Part 11 modification trail (#668).</summary>
|
||||
Modified,
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Pure server-side continuation-point paging decisions for the count-capped variable-history arms
|
||||
@@ -192,12 +258,11 @@ internal static class HistoryPaging
|
||||
/// <param name="cap">The client's per-page cap (<c>NumValuesPerNode</c>); must be > 0 to make progress.</param>
|
||||
/// <param name="boundaryT">The single SourceTimestamp every tie in the cluster shares.</param>
|
||||
/// <param name="endUtc">The (inclusive) upper bound of the read window; unchanged across pages.
|
||||
/// OpcUaServer-003, accepted by design: the backend contract
|
||||
/// (<c>IHistorianDataSource.ReadRawAsync</c>) treats <c>endUtc</c> as EXCLUSIVE, so the
|
||||
/// <c>next <= endUtc</c> test here is deliberately the conservative side — at worst one extra
|
||||
/// empty resume page (<c>[endUtc, endUtc)</c> → GoodNoData, no continuation point), never a
|
||||
/// dropped or duplicated sample. Tightening to <c><</c> would be correct only if the live
|
||||
/// backend's boundary is confirmed exclusive; it is not worth the opposite off-by-one risk.</param>
|
||||
/// The forward window is end-EXCLUSIVE (Part 11; the <c>IHistorianDataSource.ReadRawAsync</c> contract), so a drained
|
||||
/// cluster continues only while <c>next</c> is strictly before <paramref name="endUtc"/> truncated to
|
||||
/// <paramref name="resolutionTicks"/>. (OpcUaServer-003 had kept <c>next <= endUtc</c> as "the conservative side",
|
||||
/// but it was not: resuming AT EndTime makes the next read a <c>start == end</c> point read, which returns the samples
|
||||
/// at EndTime as data — #692 review F1; on a microsecond backend EndTime is truncated first.)</param>
|
||||
/// <param name="sliceStart">The index into the cluster the emitted slice starts at (clamped to the count).</param>
|
||||
/// <param name="sliceCount">How many ties the emitted slice contains (may be <c>0</c> on a stale-skip self-heal).</param>
|
||||
/// <param name="nextStartUtc">The next page's inclusive start: <paramref name="boundaryT"/> while the
|
||||
@@ -240,7 +305,9 @@ internal static class HistoryPaging
|
||||
else
|
||||
{
|
||||
var next = boundaryT.AddTicks(resolutionTicks);
|
||||
if (next <= endUtc) { nextStartUtc = next; nextSkip = 0; }
|
||||
// The forward window is end-EXCLUSIVE, at the backend's resolution: resuming AT EndTime would turn the next read
|
||||
// into a start == end point read, which returns the samples at EndTime as data (#692 review F1).
|
||||
if (next.Ticks < endUtc.Ticks - endUtc.Ticks % resolutionTicks) { nextStartUtc = next; nextSkip = 0; }
|
||||
else { nextStartUtc = null; nextSkip = 0; }
|
||||
}
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -83,6 +83,21 @@ public sealed class OtOpcUaSdkServer : StandardServer
|
||||
return true;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #674 — wire the redundancy verdict that decides whether this node serves an OPC UA HistoryUpdate
|
||||
/// (<see cref="OtOpcUaNodeManager.HistoryUpdatePrimaryGate"/>); on any other node every authorized node of a
|
||||
/// HistoryUpdate is refused <c>BadInvalidState</c>. Passing <c>null</c> clears it (allow). No-op (returns
|
||||
/// <c>false</c>) when the node manager has not been created yet (mirrors <see cref="SetNativeAlarmAckPrimaryGate"/>).
|
||||
/// </summary>
|
||||
/// <param name="isPrimary">A cheap, non-blocking read of the data-plane Primary verdict; null to allow.</param>
|
||||
/// <returns><c>true</c> when set on a live node manager; <c>false</c> when no node manager exists yet.</returns>
|
||||
public bool SetHistoryUpdatePrimaryGate(Func<bool>? isPrimary)
|
||||
{
|
||||
if (_otOpcUaNodeManager is null) return false;
|
||||
_otOpcUaNodeManager.HistoryUpdatePrimaryGate = isPrimary;
|
||||
return true;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Wire the reverse-path gateway for inbound operator writes to writable equipment-tag nodes onto the
|
||||
/// created <see cref="OtOpcUaNodeManager"/>. The host calls this after start with an
|
||||
@@ -185,6 +200,22 @@ public sealed class OtOpcUaSdkServer : StandardServer
|
||||
return true;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #682 — keep the SDK's full set of calculator factories (live-data <c>AggregateFilter</c> subscriptions
|
||||
/// need them, and <c>ServerCapabilities/AggregateFunctions</c> keeps listing all of them), then trim
|
||||
/// <c>HistoryServerCapabilities/AggregateFunctions</c> to the node manager's current historian source. At this
|
||||
/// point that is the Null default (no history aggregates); the Host's <see cref="SetHistorianDataSource"/>
|
||||
/// re-syncs to the configured backend's set right after start.
|
||||
/// </summary>
|
||||
/// <inheritdoc />
|
||||
protected override AggregateManager CreateAggregateManager(
|
||||
IServerInternal server, ApplicationConfiguration configuration)
|
||||
{
|
||||
var manager = base.CreateAggregateManager(server, configuration);
|
||||
_otOpcUaNodeManager?.SyncAdvertisedAggregates();
|
||||
return manager;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override MasterNodeManager CreateMasterNodeManager(
|
||||
IServerInternal server, ApplicationConfiguration configuration)
|
||||
|
||||
@@ -0,0 +1,97 @@
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.OpcUaServer;
|
||||
|
||||
/// <summary>How one HistoryRead-Raw page reads (<see cref="RawPageLimit.Plan"/>).</summary>
|
||||
/// <param name="Limit">The backend's <see cref="IHistorianDataSource.MaxRawValuesPerRead"/> the plan was made under (0 = none).</param>
|
||||
/// <param name="PageCap">The page: at most this many values are returned (0 = all values — only without a limit).</param>
|
||||
/// <param name="RequestCap">What the read asks for: page + the boundary ties it will trim + one lookahead row (0 = all).</param>
|
||||
/// <param name="NoRoom">The boundary ties alone fill a read — nothing can be read past them; serve the tie cluster.</param>
|
||||
internal readonly record struct RawPagePlan(int Limit, uint PageCap, uint RequestCap, bool NoRoom);
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #694: sizes a HistoryRead-Raw page to what the backend can return in one read
|
||||
/// (<see cref="IHistorianDataSource.MaxRawValuesPerRead"/> = L — a strict HistorianGateway's learned row ceiling, less
|
||||
/// its boundary-row allowance). With a limit the read — page P + boundary ties k + the #692 lookahead row — is at most L,
|
||||
/// so P = min(want, L − k − 1) (all values: P = L − k − 1); a full page carries a continuation point like any other, as
|
||||
/// Part 11 allows fewer values than <c>NumValuesPerNode</c> with one. Before #694 a cap above a strict gateway's ceiling
|
||||
/// (or an uncapped read of a window holding more) failed <c>BadTooManyOperations</c>.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>Never a silent truncation.</b> The backend still answers a read in full or refuses it; only the server's page is
|
||||
/// smaller, and a page is judged full against what was asked for. A read the backend cannot serve at all (a tie
|
||||
/// cluster larger than one read) stays <c>BadTooManyOperations</c>.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>One re-read, planned again.</b> A refused read (<see cref="HistoryReadLimitExceededException"/>) is re-planned and
|
||||
/// re-issued ONCE when (a) the backend learned a different limit while it ran — the first read after start-up teaches
|
||||
/// the ceiling — or (b) it was refused because rows before the window crowded the budget
|
||||
/// (<see cref="HistoryReadLimitExceededException.PreWindowRows"/> m): the page is then shrunk by m so that the
|
||||
/// backend's look-back fits beside it. Anything else is the answer.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
internal static class RawPageLimit
|
||||
{
|
||||
/// <summary>Plans one page.</summary>
|
||||
/// <param name="want">The page the client asked for (N, less a start bound; 0 = all values).</param>
|
||||
/// <param name="boundaryTies">Ties at the resume timestamp already returned, which the read re-reads and trims (k).</param>
|
||||
/// <param name="limit">The backend's limit L (<= 0 = none).</param>
|
||||
/// <param name="sizeLimit">The limit to size the page by (L, or L − m after a crowded refusal); defaults to <paramref name="limit"/>.</param>
|
||||
/// <returns>The plan.</returns>
|
||||
public static RawPagePlan Plan(uint want, int boundaryTies, int limit, int? sizeLimit = null)
|
||||
{
|
||||
var k = Math.Max(0, boundaryTies);
|
||||
if (limit <= 0)
|
||||
return new RawPagePlan(0, want, want == 0 ? 0u : (uint)Math.Min(uint.MaxValue, (long)want + k + 1L), false);
|
||||
|
||||
var room = (long)(sizeLimit ?? limit) - k - 1;
|
||||
if (room < 1) return new RawPagePlan(limit, 0, 0, true);
|
||||
var page = want == 0 ? room : Math.Min(want, room);
|
||||
return new RawPagePlan(limit, (uint)page, (uint)(page + k + 1), false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reads one page under the backend's current limit, re-planning and re-reading ONCE as described in the remarks.
|
||||
/// A <see cref="RawPagePlan.NoRoom"/> plan is returned without a read.
|
||||
/// </summary>
|
||||
/// <typeparam name="T">The page.</typeparam>
|
||||
/// <param name="source">The backend.</param>
|
||||
/// <param name="want">The page the client asked for (0 = all values).</param>
|
||||
/// <param name="boundaryTies">The boundary ties the read re-reads and trims.</param>
|
||||
/// <param name="read">One read with the given plan (it asks for <see cref="RawPagePlan.RequestCap"/>).</param>
|
||||
/// <returns>The page (null when <see cref="RawPagePlan.NoRoom"/>) and the plan it was read with.</returns>
|
||||
public static async Task<(T? Page, RawPagePlan Plan)> ReadAsync<T>(
|
||||
IHistorianDataSource source, uint want, int boundaryTies, Func<RawPagePlan, Task<T>> read)
|
||||
{
|
||||
var before = source.MaxRawValuesPerRead;
|
||||
var plan = Plan(want, boundaryTies, before);
|
||||
if (plan.NoRoom) return (default, plan);
|
||||
|
||||
HistoryReadLimitExceededException refused;
|
||||
try
|
||||
{
|
||||
return (await read(plan).ConfigureAwait(false), plan);
|
||||
}
|
||||
catch (HistoryReadLimitExceededException ex) when (source.MaxRawValuesPerRead is var now and > 0
|
||||
&& (now != before || ex.PreWindowRows > 0))
|
||||
{
|
||||
refused = ex; // re-read outside the catch: the second refusal is the answer
|
||||
}
|
||||
|
||||
var limit = source.MaxRawValuesPerRead;
|
||||
var shrink = Math.Max(0, refused.PreWindowRows);
|
||||
plan = Plan(want, boundaryTies, limit, (int)Math.Max(0L, (long)limit - shrink));
|
||||
// No room left once the crowding rows are allowed for: those rows are not (necessarily) the boundary ties, so this is
|
||||
// NOT "the cursor sits in a tie cluster" — the refusal is the answer (stepping the cursor through it a tick per page
|
||||
// would never end).
|
||||
// (A limit that changed under an un-crowded read leaves the boundary ties alone filling a read: that IS the tie
|
||||
// cluster at the cursor, served by the over-fetch.)
|
||||
if (plan.NoRoom)
|
||||
{
|
||||
if (shrink > 0) throw refused;
|
||||
return (default, plan);
|
||||
}
|
||||
return (await read(plan).ConfigureAwait(false), plan);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.OpcUaServer;
|
||||
|
||||
/// <summary>One walked-back reverse events page (<see cref="ReverseEventsReader"/>).</summary>
|
||||
/// <param name="Events">At most the requested count, newest first by <c>(EventTimeUtc, EventId)</c>.</param>
|
||||
/// <param name="Hint">What the next page of the same read resumes with.</param>
|
||||
internal readonly record struct ReverseEventsPage(List<HistoricalEvent> Events, ReverseWalkHint Hint);
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #691.4: serves a Part 11 reverse events read — <c>(lo, hi]</c> newest-first — on a backend that reads events
|
||||
/// forward only (<see cref="IHistorianDataSource.SupportsReverseEvents"/> = <c>false</c>: the Gateway) with the
|
||||
/// <see cref="ReverseWalk"/>. A segment read is cut short when the source flags truncation (its continuation point); ties
|
||||
/// are ordered by <see cref="EventIdTieOrder"/> because the gateway orders by event time only.
|
||||
/// </summary>
|
||||
internal static class ReverseEventsReader
|
||||
{
|
||||
/// <summary>Reads <c>(lo, hi]</c> newest-first from forward event reads.</summary>
|
||||
/// <param name="source">A backend that reads events forward.</param>
|
||||
/// <param name="sourceName">Event-source filter (null = all).</param>
|
||||
/// <param name="hi">The later (inclusive) bound.</param>
|
||||
/// <param name="lo">The earlier (exclusive) bound.</param>
|
||||
/// <param name="want">How many events; <c>0</c> = all in the window (one forward read).</param>
|
||||
/// <param name="maxTieCluster">The tie budget (<c>MaxTieClusterOverfetch</c>) — a ceiling only.</param>
|
||||
/// <param name="nowUtc">Now, for the look-ahead clamp.</param>
|
||||
/// <param name="hint">What the previous page carried (default = none).</param>
|
||||
/// <param name="ct">Cancellation.</param>
|
||||
/// <returns>The events, newest first, and the hint for the next page.</returns>
|
||||
/// <exception cref="HistoryReadLimitExceededException">One timestamp holds more than the tie budget.</exception>
|
||||
public static async Task<ReverseEventsPage> ReadAsync(
|
||||
IHistorianDataSource source, string? sourceName, DateTime hi, DateTime lo, uint want, int maxTieCluster,
|
||||
DateTime nowUtc, ReverseWalkHint hint, CancellationToken ct)
|
||||
{
|
||||
var budget = Math.Max(1, maxTieCluster);
|
||||
var (rows, next) = await ReverseWalk.WalkAsync<HistoricalEvent>(
|
||||
async (start, end, cap, token) =>
|
||||
{
|
||||
// cap 0 = the whole window: int.MaxValue is the events contract's "no cap" (a non-positive value would
|
||||
// mean the backend's DEFAULT cap).
|
||||
var r = await source.ReadEventsAsync(sourceName, start, end, cap == 0 ? int.MaxValue : cap, reverse: false, token)
|
||||
.ConfigureAwait(false);
|
||||
return new ForwardSegmentRead<HistoricalEvent>(r.Events, r.ContinuationPoint is not null);
|
||||
},
|
||||
ev => ev.EventTimeUtc.Ticks, EventIdTieOrder.Instance, hi, lo, want, budget, source.TimestampResolutionTicks,
|
||||
nowUtc, hint,
|
||||
at => new HistoryReadLimitExceededException(budget,
|
||||
$"A reverse events read of '{sourceName}' at {at:O} found more than {budget} events within one millisecond; the " +
|
||||
"backend reads forward only, so the latest of them cannot be selected. Increase " +
|
||||
"ServerHistorian:MaxTieClusterOverfetch."),
|
||||
ct)
|
||||
.ConfigureAwait(false);
|
||||
return new ReverseEventsPage(rows, next);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,136 @@
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.OpcUaServer;
|
||||
|
||||
/// <summary>One walked-back reverse raw page (<see cref="ReverseRawReader"/>).</summary>
|
||||
/// <param name="Samples">At most the requested cap, newest first.</param>
|
||||
/// <param name="WidthHintTicks">The segment width the walk ended on — the next page's starting width (0 = none).</param>
|
||||
internal readonly record struct ReverseRawPage(IReadOnlyList<DataValueSnapshot> Samples, long WidthHintTicks)
|
||||
{
|
||||
/// <summary>Everything the next page of the same read should resume with (its width is <see cref="WidthHintTicks"/>).</summary>
|
||||
public ReverseWalkHint Hint { get; init; } = new(WidthHintTicks);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #686: serves a Part 11 reverse raw read — <c>(lo, hi]</c> newest-first, capped — on a historian backend that can
|
||||
/// only read forward (<see cref="IHistorianDataSource.SupportsReverseRaw"/> = <c>false</c>: the Gateway, whose
|
||||
/// <c>ReadRaw</c> streams ascending and rejects <c>start > end</c>). The walk itself — segment sizing (#689), the
|
||||
/// truncation rules, edges, ties and bounds — is <see cref="ReverseWalk"/>; this adapter supplies the raw read (a read is
|
||||
/// cut short when it returned its full cap) and keeps the backend's tie order.
|
||||
/// </summary>
|
||||
internal static class ReverseRawReader
|
||||
{
|
||||
/// <summary>The first segment's width when no hint is carried.</summary>
|
||||
internal static readonly TimeSpan InitialSegment = ReverseWalk.InitialSegment;
|
||||
|
||||
/// <summary>The walk's exclusive lower limit.</summary>
|
||||
internal static readonly DateTime Floor = ReverseWalk.Floor;
|
||||
|
||||
/// <summary>How far past "now" the walk starts at most.</summary>
|
||||
internal static readonly TimeSpan LookAhead = ReverseWalk.LookAhead;
|
||||
|
||||
/// <summary>Reads <c>(lo, hi]</c> newest-first from forward reads.</summary>
|
||||
/// <param name="source">A backend that reads forward.</param>
|
||||
/// <param name="tagname">Historian tag.</param>
|
||||
/// <param name="hi">The later (inclusive) bound — the request's StartTime or a reverse resume point.</param>
|
||||
/// <param name="lo">The earlier (exclusive) bound.</param>
|
||||
/// <param name="maxValues">Cap; <c>0</c> = every sample in the window (one forward read).</param>
|
||||
/// <param name="maxTieCluster">The tie budget (<c>MaxTieClusterOverfetch</c>; values below 1 count as 1) — a ceiling only.</param>
|
||||
/// <param name="nowUtc">Now, for the look-ahead clamp.</param>
|
||||
/// <param name="widthHintTicks">Starting segment width (the previous page's <see cref="ReverseRawPage.WidthHintTicks"/>), 0 = none.</param>
|
||||
/// <param name="ct">Cancellation.</param>
|
||||
/// <returns>The page, newest first, and the width to start the next page with.</returns>
|
||||
/// <exception cref="HistoryReadLimitExceededException">One timestamp holds more than the tie budget.</exception>
|
||||
public static Task<ReverseRawPage> ReadAsync(
|
||||
IHistorianDataSource source, string tagname, DateTime hi, DateTime lo, uint maxValues, int maxTieCluster,
|
||||
DateTime nowUtc, long widthHintTicks, CancellationToken ct) =>
|
||||
ReadAsync(source, tagname, hi, lo, maxValues, maxTieCluster, nowUtc, new ReverseWalkHint(widthHintTicks), ct);
|
||||
|
||||
/// <summary>Reads <c>(lo, hi]</c> newest-first from forward reads, resuming with the previous page's full hint.</summary>
|
||||
/// <param name="source">A backend that reads forward.</param>
|
||||
/// <param name="tagname">Historian tag.</param>
|
||||
/// <param name="hi">The later (inclusive) bound.</param>
|
||||
/// <param name="lo">The earlier (exclusive) bound.</param>
|
||||
/// <param name="maxValues">Cap; <c>0</c> = every sample in the window (one forward read).</param>
|
||||
/// <param name="maxTieCluster">The tie budget — a ceiling only.</param>
|
||||
/// <param name="nowUtc">Now, for the look-ahead clamp.</param>
|
||||
/// <param name="hint">What the previous page carried (<see cref="ReverseRawPage.Hint"/>), default = none.</param>
|
||||
/// <param name="ct">Cancellation.</param>
|
||||
/// <returns>The page, newest first, and the hint for the next page.</returns>
|
||||
/// <exception cref="HistoryReadLimitExceededException">One timestamp holds more than the tie budget.</exception>
|
||||
/// <remarks>
|
||||
/// Gitea #694: every segment read fits the backend's <see cref="IHistorianDataSource.MaxRawValuesPerRead"/> (a strict
|
||||
/// gateway's learned ceiling) — the walk's tie budget is bounded by it so its segments are sized for it, and a
|
||||
/// crowded re-read asking more is cut to it (and counts as truncated, never as complete). A walk that TEACHES the
|
||||
/// backend its limit (the first read after start-up) and is refused for it is re-run once with segments sized for the
|
||||
/// new limit — unless <paramref name="retryOnLearnedLimit"/> is false: the HistoryRead page path passes false, so the
|
||||
/// refusal reaches <see cref="RawPageLimit"/>, which shrinks the PAGE to the limit as well (re-running the walk for a
|
||||
/// page of N far above the limit cost thousands of segment reads). The bound lookups (a page of one) keep the re-run.
|
||||
/// The page size itself is never cut here: a short answer would read as the last page. A segment refused because rows
|
||||
/// just before it crowded the backend's budget (<see cref="HistoryReadLimitExceededException.PreWindowRows"/>) is
|
||||
/// re-read once that many rows smaller and counts as truncated when full. An uncapped walk (one forward read of the
|
||||
/// whole window) is never cut: it is answered in full or refused.
|
||||
/// </remarks>
|
||||
/// <param name="retryOnLearnedLimit">Re-run the walk once when it taught the backend its limit (see the remarks).</param>
|
||||
public static async Task<ReverseRawPage> ReadAsync(
|
||||
IHistorianDataSource source, string tagname, DateTime hi, DateTime lo, uint maxValues, int maxTieCluster,
|
||||
DateTime nowUtc, ReverseWalkHint hint, CancellationToken ct, bool retryOnLearnedLimit = true)
|
||||
{
|
||||
var before = source.MaxRawValuesPerRead;
|
||||
try
|
||||
{
|
||||
return await WalkAsync(before).ConfigureAwait(false);
|
||||
}
|
||||
catch (HistoryReadLimitExceededException) when (retryOnLearnedLimit && maxValues > 0
|
||||
&& source.MaxRawValuesPerRead is var now and > 0 && now != before)
|
||||
{
|
||||
// fall through (outside the catch): the walk taught the backend its limit — re-run it sized for that limit once.
|
||||
// Not an uncapped walk: it is one read of the whole window, which the limit does not change.
|
||||
}
|
||||
|
||||
return await WalkAsync(source.MaxRawValuesPerRead).ConfigureAwait(false);
|
||||
|
||||
async Task<ReverseRawPage> WalkAsync(int limit)
|
||||
{
|
||||
// A segment asks for at most its budget + 2 rows (ReverseWalk), so the budget is bounded 2 under the limit.
|
||||
var budget = Math.Max(1, maxTieCluster);
|
||||
if (limit > 0) budget = Math.Min(budget, Math.Max(1, limit - 2));
|
||||
var (rows, next) = await ReverseWalk.WalkAsync(
|
||||
async (start, end, cap, token) =>
|
||||
{
|
||||
var asked = limit > 0 && cap > limit ? limit : cap; // cap 0 (the whole window) is never cut
|
||||
IReadOnlyList<DataValueSnapshot> samples;
|
||||
for (var attempt = 0; ; attempt++)
|
||||
{
|
||||
try
|
||||
{
|
||||
samples = (await source.ReadRawAsync(tagname, start, end, (uint)asked, token).ConfigureAwait(false)).Samples;
|
||||
break;
|
||||
}
|
||||
catch (HistoryReadLimitExceededException ex) when (asked > 0 && ex.PreWindowRows > 0 && attempt < 2)
|
||||
{
|
||||
// #695: the discovery request may be smaller than the pre-window cluster. Size against
|
||||
// the ceiling, not that request; a second pass can discover more crowding rows.
|
||||
var ceilingRoom = source.MaxRawValuesPerRead is var learned and > 0 ? learned : asked;
|
||||
var smaller = Math.Min(asked - 1, ceilingRoom - ex.PreWindowRows - 1);
|
||||
if (smaller < 1) throw;
|
||||
asked = smaller;
|
||||
}
|
||||
}
|
||||
return new ForwardSegmentRead<DataValueSnapshot>(samples, asked > 0 && samples.Count >= asked);
|
||||
},
|
||||
Ticks, tieOrder: null, hi, lo, maxValues, budget, source.TimestampResolutionTicks, nowUtc, hint,
|
||||
at => new HistoryReadLimitExceededException(budget,
|
||||
$"A reverse raw read of '{tagname}' at {at:O} found more than {budget} samples within one millisecond; the " +
|
||||
"backend reads forward only, so the latest of them cannot be selected. Increase " +
|
||||
(budget < Math.Max(1, maxTieCluster)
|
||||
? "the backend's row ceiling (HistorianGateway Historian:ReadLimits:MaxValuesCeiling)."
|
||||
: "ServerHistorian:MaxTieClusterOverfetch.")),
|
||||
ct, useDiscoveryBudgetForGaps: limit > 0)
|
||||
.ConfigureAwait(false);
|
||||
return new ReverseRawPage(rows, next.WidthTicks) { Hint = next };
|
||||
}
|
||||
}
|
||||
|
||||
private static long Ticks(DataValueSnapshot s) => (s.SourceTimestampUtc ?? DateTime.MinValue).Ticks;
|
||||
}
|
||||
@@ -0,0 +1,454 @@
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.OpcUaServer;
|
||||
|
||||
/// <summary>What one walked-back page tells the next page of the same read (carried in the continuation point).</summary>
|
||||
/// <param name="WidthTicks">The segment width the walk ended on, aimed at the page's density — the next page's first width (0 = none).</param>
|
||||
/// <param name="EmptyFromTicks">
|
||||
/// With <paramref name="EmptyToTicks"/>: a stretch <c>(EmptyFrom, EmptyTo)</c> (both exclusive) a complete segment read
|
||||
/// proved holds no row (#691) — typically from the oldest row the page saw down to the walk's floor. A resumed page
|
||||
/// steps straight over it instead of re-walking it. Both 0 = none.
|
||||
/// </param>
|
||||
/// <param name="EmptyToTicks">The exclusive upper end of the empty stretch.</param>
|
||||
internal readonly record struct ReverseWalkHint(long WidthTicks, long EmptyFromTicks = 0, long EmptyToTicks = 0)
|
||||
{
|
||||
/// <summary>Whether an empty stretch is carried.</summary>
|
||||
public bool HasEmptyStretch => EmptyToTicks > EmptyFromTicks && EmptyToTicks > 0;
|
||||
}
|
||||
|
||||
/// <summary>One forward read of a segment: the rows (ascending by time) and whether a cap cut the read short.</summary>
|
||||
/// <typeparam name="T">Row type.</typeparam>
|
||||
/// <param name="Rows">Rows in ascending time order (ties in any order).</param>
|
||||
/// <param name="Truncated">A cap stopped the read — more rows may follow the last one.</param>
|
||||
internal readonly record struct ForwardSegmentRead<T>(IReadOnlyList<T> Rows, bool Truncated);
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #686/#689: the walk-back that serves a Part 11 reverse read — <c>(lo, hi]</c> newest-first, capped — from a
|
||||
/// backend that can only read FORWARD. Shared by raw values (<see cref="ReverseRawReader"/>) and events; the row type
|
||||
/// only has to expose a timestamp.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>Uncapped</b> (<c>want = 0</c>): ONE forward read of the whole window, filtered and reversed in memory.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Capped: walk back in segments.</b> Starting at <c>hi</c>, read one segment <c>(segLo, segHi]</c> FORWARD, keep
|
||||
/// it newest-first, and step to the next older segment until the cap is met or <c>lo</c> is reached. The first
|
||||
/// segment is <see cref="InitialSegment"/> wide, or the width the previous page of the same read ended on (carried in
|
||||
/// the continuation point). After each read the next width is re-aimed from the density just observed at about half
|
||||
/// the PAGE budget <c>2N + 16</c> (≈ N + 8 rows), growing at most ×4 per step (#689 — it used to aim at half the tie
|
||||
/// budget, ~32k rows, whatever N was) — so a live tag's "latest N" is usually one read of about N rows, and sparse
|
||||
/// history is found in a few dozen tiny reads.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Read sizes.</b> A segment read asks for its budget + 2 rows (room for a row an AVEVA read fabricates at its query
|
||||
/// start, and one to see past the segment). The budget is the page budget, except a <b>discovery</b> segment — the
|
||||
/// first of a page with no carried width, or one after an empty segment (whose ×4 growth may overshoot into dense
|
||||
/// data) — which may read up to <see cref="DiscoveryRows"/>. The tie budget <c>M = max(1, MaxTieClusterOverfetch)</c>
|
||||
/// is only a CEILING: a single timestamp holding more rows than the segment budget escalates to <c>M</c>.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Never answer from a truncated read.</b> Rows stream ascending, so a read is complete for its segment when it was
|
||||
/// not cut short OR its last row is already past <c>segHi</c>. An incomplete read is discarded: when rows BEFORE the
|
||||
/// segment (a neighbouring tie cluster inside the alignment slack, a fabricated row) crowded it out it is re-read once
|
||||
/// with room for them (at most twice the budget + 4); otherwise the segment is narrowed — to the density the truncated
|
||||
/// read showed, and never again below the newest row it saw (<c>known</c>: rows exist there, so walking past it before
|
||||
/// finding what lies above it would re-read the same dense stretch over and over; the gap above it is bisected
|
||||
/// instead). Only more than <c>M</c> rows within one millisecond — a segment no wider than the 1 ms request
|
||||
/// granularity that still does not fit at the tie budget — is refused (per millisecond, not per timestamp: narrower
|
||||
/// requests are the same request, and the Gateway stores milliseconds).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Proving the rest empty (#691).</b> Walking down to the floor through empty history used to take one read per ×4
|
||||
/// growth step (~14 reads from a minute to the epoch) — on every page that ran out of rows. After two consecutive empty
|
||||
/// segments the walk asks once per page for the WHOLE remainder <c>(lo, segHi]</c> at the page budget: a complete
|
||||
/// answer ends the walk in one read; a truncated one (deep history) is discarded and the walk carries on as before. And
|
||||
/// the stretch a page's last complete segment proved empty below its oldest row rides the continuation point
|
||||
/// (<see cref="ReverseWalkHint.EmptyFromTicks"/>), so the next page steps over it — the rows of a page are often the
|
||||
/// last ones before a stretch the previous page already read.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Edges.</b> A segment is requested as <c>[⌊segLo⌋, ⌈segHi + 1 tick⌉]</c>, both rounded outward to the millisecond
|
||||
/// (the gateway's / AVEVA's resolution), and filtered to <c>(segLo, segHi]</c> on the rows' own timestamps — exact
|
||||
/// whether the backend's forward end is exclusive or inclusive and whether it truncates its bounds; a row fabricated at
|
||||
/// the (millisecond) query start can never land inside the segment (#687). Adjacent segments share no timestamp.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Ties</b> come back in the backend's forward order reversed, or — with a tie order given (events, whose gateway
|
||||
/// order within a timestamp is arbitrary) — sorted by it and reversed, so a resumed page's boundary-tie skip lines up.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Bounds.</b> The walk never asks for data after now + <see cref="LookAhead"/> nor before <see cref="Floor"/> (the
|
||||
/// Unix epoch — AVEVA holds nothing earlier). A walked-back read returns only rows strictly after the epoch and no
|
||||
/// later than now + one day.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
internal static class ReverseWalk
|
||||
{
|
||||
/// <summary>The first segment's width when no hint is carried.</summary>
|
||||
internal static readonly TimeSpan InitialSegment = TimeSpan.FromMinutes(1);
|
||||
|
||||
/// <summary>The walk's exclusive lower limit.</summary>
|
||||
internal static readonly DateTime Floor = DateTime.UnixEpoch;
|
||||
|
||||
/// <summary>How far past "now" the walk starts at most.</summary>
|
||||
internal static readonly TimeSpan LookAhead = TimeSpan.FromDays(1);
|
||||
|
||||
/// <summary>Slack added to a page's row budget (#689): room for density variation between segments.</summary>
|
||||
internal const int PageSlack = 16;
|
||||
|
||||
/// <summary>The most rows a discovery segment (see the remarks) may read before it counts as truncated (#689).</summary>
|
||||
internal const int DiscoveryRows = 1024;
|
||||
|
||||
/// <summary>Request bounds are rounded outward to this (the gateway / AVEVA timestamp resolution).</summary>
|
||||
private const long Alignment = TimeSpan.TicksPerMillisecond;
|
||||
|
||||
/// <summary>The rows a segment is sized for: <c>2N + </c><see cref="PageSlack"/>, never above the tie budget.</summary>
|
||||
/// <param name="want">The page cap N (> 0).</param>
|
||||
/// <param name="tieBudget">The tie budget M (>= 1).</param>
|
||||
/// <returns>The page budget.</returns>
|
||||
internal static int PageBudget(uint want, int tieBudget) => (int)Math.Clamp(2L * want + PageSlack, 1L, Math.Max(1, tieBudget));
|
||||
|
||||
/// <summary>Reads <c>(lo, hi]</c> newest-first from forward reads.</summary>
|
||||
/// <typeparam name="T">Row type.</typeparam>
|
||||
/// <param name="read">One forward read <c>[start, end]</c> asking for at most <c>cap</c> rows (<c>0</c> = all).</param>
|
||||
/// <param name="ticksOf">A row's timestamp, in ticks.</param>
|
||||
/// <param name="tieOrder">Order within one timestamp (null = the backend's forward order).</param>
|
||||
/// <param name="hi">The later (inclusive) bound.</param>
|
||||
/// <param name="lo">The earlier (exclusive) bound.</param>
|
||||
/// <param name="want">Cap; <c>0</c> = every row in the window (one forward read).</param>
|
||||
/// <param name="tieBudget">The tie budget M (values below 1 count as 1).</param>
|
||||
/// <param name="resolutionTicks">The backend's timestamp resolution (>= 1).</param>
|
||||
/// <param name="nowUtc">Now, for the look-ahead clamp.</param>
|
||||
/// <param name="hint">What the previous page of the same read carried (default = none).</param>
|
||||
/// <param name="tooManyAtOneTimestamp">Builds the exception for a timestamp holding more than M rows.</param>
|
||||
/// <param name="ct">Cancellation.</param>
|
||||
/// <param name="useDiscoveryBudgetForGaps">Use ceiling-sized discovery and retain truncated remainder rows (#696, strict raw backends only).</param>
|
||||
/// <returns>At most <paramref name="want"/> rows, newest first, and the hint for the next page.</returns>
|
||||
public static async Task<(List<T> Rows, ReverseWalkHint Next)> WalkAsync<T>(
|
||||
Func<DateTime, DateTime, int, CancellationToken, Task<ForwardSegmentRead<T>>> read,
|
||||
Func<T, long> ticksOf,
|
||||
IComparer<T>? tieOrder,
|
||||
DateTime hi, DateTime lo, uint want, int tieBudget, long resolutionTicks, DateTime nowUtc,
|
||||
ReverseWalkHint hint, Func<DateTime, Exception> tooManyAtOneTimestamp, CancellationToken ct, bool useDiscoveryBudgetForGaps = false)
|
||||
{
|
||||
var resolution = Math.Max(1L, resolutionTicks);
|
||||
var granularity = Math.Max(Alignment, resolution);
|
||||
var budget = Math.Max(1, tieBudget);
|
||||
var result = new List<T>();
|
||||
|
||||
var horizon = nowUtc.Ticks > DateTime.MaxValue.Ticks - LookAhead.Ticks ? DateTime.MaxValue.Ticks : nowUtc.Ticks + LookAhead.Ticks;
|
||||
var hiTicks = Math.Min(hi.Ticks, horizon);
|
||||
var stopTicks = Math.Max(lo.Ticks, Floor.Ticks);
|
||||
if (hiTicks <= stopTicks) return (result, hint);
|
||||
|
||||
if (want == 0)
|
||||
{
|
||||
// One forward read of the whole window, reversed — the uncapped forward read's own cost and bound.
|
||||
var all = (await read(Utc(AlignDown(stopTicks, granularity)), Utc(AlignUp(hiTicks + 1, granularity)), 0, ct)
|
||||
.ConfigureAwait(false)).Rows;
|
||||
AppendNewestFirst(all, ticksOf, tieOrder, stopTicks, hiTicks, result, int.MaxValue);
|
||||
return (result, hint);
|
||||
}
|
||||
|
||||
var maxWidth = DateTime.MaxValue.Ticks;
|
||||
var discovering = hint.WidthTicks <= 0;
|
||||
var width = discovering ? Math.Max(resolution, InitialSegment.Ticks) : Math.Clamp(hint.WidthTicks, resolution, maxWidth);
|
||||
var pageBudget = PageBudget(want, budget);
|
||||
var discoveryBudget = Math.Min(budget, Math.Max(pageBudget, DiscoveryRows));
|
||||
|
||||
// `known` > 0: a truncated read saw rows up to this timestamp (below segHi), and nothing is known of (known, segHi].
|
||||
// Segments stay above it until that stretch is read; then `knownWidth` (the density the truncated read showed)
|
||||
// sizes the segment that ends on it. Searching the gap: first a top-anchored segment at that density (data running
|
||||
// right up to segHi); if it is empty, the data stops somewhere above `known`, so GALLOP up from it — read
|
||||
// (known + lowerBy, segHi] with lowerBy growing ×4, `known` following what each truncated read saw — until a read
|
||||
// comes back empty (the top is now bracketed below its segLo), then BISECT that bracket. Any complete read holding
|
||||
// rows ends the search. A burst that ended long before segHi is found in a few reads instead of bisecting the whole
|
||||
// gap down to the millisecond, and a long dense stretch is crossed geometrically, not a budget at a time (#691.4
|
||||
// review). `noGallop` falls back to plain top-anchored halving if a galloping read is crowded by rows before it.
|
||||
var known = 0L;
|
||||
var knownWidth = 0L;
|
||||
var lowerBy = 0L;
|
||||
var bisecting = false;
|
||||
var noGallop = false;
|
||||
var segHi = hiTicks;
|
||||
|
||||
// #691: the remainder probe (once per page, after two empty segments) and the empty stretches.
|
||||
var emptyRun = 0;
|
||||
var probed = false;
|
||||
var carried = hint.HasEmptyStretch ? (hint.EmptyFromTicks, hint.EmptyToTicks) : (0L, 0L);
|
||||
(long From, long To) proven = (0L, 0L);
|
||||
|
||||
while (segHi > stopTicks && result.Count < want)
|
||||
{
|
||||
if (carried.Item2 > carried.Item1 && segHi > carried.Item1 && segHi < carried.Item2)
|
||||
{
|
||||
segHi = carried.Item1; // (from, segHi] lies inside a stretch a previous page proved empty
|
||||
continue;
|
||||
}
|
||||
|
||||
var probe = !probed && known == 0 && emptyRun >= 2 && segHi - stopTicks > width;
|
||||
var w = probe ? segHi - stopTicks : width;
|
||||
var lowered = false;
|
||||
var bisected = false;
|
||||
if (known > 0)
|
||||
{
|
||||
var gap = segHi - known;
|
||||
lowered = lowerBy > 0 && lowerBy < gap;
|
||||
bisected = !lowered && (bisecting || lowerBy > 0) && gap > 2 * Math.Max(granularity, knownWidth);
|
||||
// Never narrower than the request granularity: requests are rounded outward to it, so a narrower segment
|
||||
// re-sends the same request — and a width left at one tick (a tie cluster's) stepped segHi down a tick per
|
||||
// read, ~10,000 identical reads (#691.4 re-review). Keep the Min: a truncated read crowded by rows before
|
||||
// the segment must still shrink the segment.
|
||||
w = lowered ? gap - lowerBy
|
||||
: bisected ? gap / 2
|
||||
: Math.Min(Math.Max(w, granularity), gap <= 2 * granularity ? gap : gap / 2);
|
||||
}
|
||||
var segLo = segHi - stopTicks <= w ? stopTicks : segHi - w;
|
||||
var start = Utc(AlignDown(segLo, granularity));
|
||||
var end = Utc(AlignUp(segHi + 1, granularity));
|
||||
var segBudget = (discovering && !probe) || (useDiscoveryBudgetForGaps && (probe || known > 0))
|
||||
? discoveryBudget : pageBudget;
|
||||
var cap = segBudget + 2;
|
||||
IReadOnlyList<T> rows;
|
||||
var narrowed = false;
|
||||
|
||||
while (true)
|
||||
{
|
||||
ct.ThrowIfCancellationRequested();
|
||||
var segment = await read(start, end, cap, ct).ConfigureAwait(false);
|
||||
rows = segment.Rows;
|
||||
if (!segment.Truncated || rows.Count == 0 || ticksOf(rows[^1]) > segHi) break; // complete for (segLo, segHi]
|
||||
|
||||
if (probe)
|
||||
{
|
||||
// #696: strict-ceiling discovery retains the rows/density instead of discarding the remainder probe.
|
||||
probed = true;
|
||||
if (!useDiscoveryBudgetForGaps)
|
||||
{
|
||||
narrowed = true;
|
||||
break;
|
||||
}
|
||||
probe = false;
|
||||
}
|
||||
|
||||
var leading = 0;
|
||||
var inner = 0;
|
||||
var firstInner = 0L;
|
||||
var lastInner = 0L;
|
||||
foreach (var r in rows)
|
||||
{
|
||||
var ts = ticksOf(r);
|
||||
if (ts <= segLo) leading++;
|
||||
else if (ts <= segHi)
|
||||
{
|
||||
if (inner++ == 0) firstInner = ts;
|
||||
lastInner = ts;
|
||||
}
|
||||
}
|
||||
|
||||
// Crowded out by rows before the segment, with the segment itself possibly within budget: re-read once
|
||||
// with room for them.
|
||||
var roomier = leading + segBudget + 2;
|
||||
if (leading > 0 && inner <= segBudget && roomier > cap && roomier <= 2 * segBudget + 4)
|
||||
{
|
||||
cap = roomier;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (segHi - segLo <= granularity)
|
||||
{
|
||||
// A segment no wider than the request granularity (1 ms — narrower requests are the same request) still
|
||||
// over its budget: a tie cluster, or a burst inside one millisecond. Escalate to the tie budget M before
|
||||
// refusing — up to M rows within one millisecond are served (#686); more are refused, per millisecond
|
||||
// (#691.4 re-review — exact for the millisecond-resolution Gateway).
|
||||
if (segBudget < budget)
|
||||
{
|
||||
segBudget = budget;
|
||||
cap = segBudget + 2;
|
||||
continue;
|
||||
}
|
||||
|
||||
throw tooManyAtOneTimestamp(Utc(segHi));
|
||||
}
|
||||
|
||||
// Narrow — the segment shrinks every time, so this terminates.
|
||||
if (inner > 0 && lastInner == segHi)
|
||||
{
|
||||
width = resolution; // a tie cluster at the top of the segment: read that one timestamp
|
||||
lowerBy = 0;
|
||||
bisecting = false;
|
||||
}
|
||||
else if (inner > 0 && (lowered || bisected))
|
||||
{
|
||||
// Data reaches past this segment's bottom: move `known` up to what this read saw; a gallop goes further.
|
||||
known = lastInner;
|
||||
if (lowered) lowerBy = lowerBy > maxWidth / 4 ? maxWidth : lowerBy * 4;
|
||||
}
|
||||
else if (inner > 0)
|
||||
{
|
||||
// Rows exist at lastInner; what lies above it is unknown. Aim at about half the page budget at the
|
||||
// density the truncated rows showed (their span, not the segment's — it may open on an empty stretch).
|
||||
var aimed = inner > 1 && lastInner > firstInner
|
||||
? (long)Math.Min((double)(lastInner - firstInner) * ((useDiscoveryBudgetForGaps ? segBudget : pageBudget) / 2.0) / (inner - 1), maxWidth)
|
||||
: resolution;
|
||||
known = lastInner;
|
||||
knownWidth = Math.Max(resolution, aimed);
|
||||
width = Math.Max(resolution, Math.Min(aimed, segHi - segLo - 1));
|
||||
lowerBy = 0;
|
||||
bisecting = false;
|
||||
}
|
||||
else
|
||||
{
|
||||
// Only rows before the segment: shrink it — top-anchored from here on (a gallop would re-send the same
|
||||
// crowded read).
|
||||
width = Math.Max(resolution, Math.Min(width, segHi - segLo) / 4);
|
||||
if (lowered || bisected) noGallop = true;
|
||||
lowerBy = 0;
|
||||
bisecting = false;
|
||||
}
|
||||
|
||||
discovering = false;
|
||||
narrowed = true;
|
||||
break;
|
||||
}
|
||||
|
||||
if (narrowed) continue; // re-read the same segHi, narrower
|
||||
|
||||
var count = AppendNewestFirst(rows, ticksOf, tieOrder, segLo, segHi, result, (int)Math.Min(int.MaxValue, want),
|
||||
out var oldest);
|
||||
// This complete read proves (segLo, oldest) empty — or the whole segment when it held nothing.
|
||||
proven = count > 0 ? (segLo, oldest) : (segLo, segHi + 1);
|
||||
if (probe)
|
||||
{
|
||||
probed = true;
|
||||
segHi = segLo; // the whole remainder is read
|
||||
continue;
|
||||
}
|
||||
|
||||
emptyRun = count == 0 ? emptyRun + 1 : 0;
|
||||
var grown = width > maxWidth / 4 ? maxWidth : width * 4;
|
||||
if (count == 0)
|
||||
{
|
||||
discovering = true; // ×4 growth may overshoot into dense data
|
||||
if (known > 0)
|
||||
{
|
||||
// An empty top-anchored segment: gallop up from `known`. An empty galloping read: the top is bracketed
|
||||
// in (known, segLo] — bisect it (an empty bisecting read keeps bisecting).
|
||||
if (lowered || bisected)
|
||||
{
|
||||
lowerBy = 0;
|
||||
bisecting = true;
|
||||
}
|
||||
else if (noGallop)
|
||||
{
|
||||
width = maxWidth; // halve the gap, top-anchored
|
||||
}
|
||||
else
|
||||
{
|
||||
lowerBy = Math.Max(knownWidth, granularity);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
width = grown;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
// Re-aim at about half the page budget from the density just seen.
|
||||
width = (long)Math.Clamp((double)(segHi - segLo) * Math.Max(1.0, pageBudget / 2.0) / count, resolution, grown);
|
||||
discovering = false;
|
||||
known = 0;
|
||||
lowerBy = 0;
|
||||
bisecting = false;
|
||||
noGallop = false;
|
||||
}
|
||||
|
||||
segHi = segLo;
|
||||
if (known > 0 && segHi <= known)
|
||||
{
|
||||
// Everything above the newest row the truncated read saw is read: the next segment ends on it, at its density.
|
||||
known = 0;
|
||||
lowerBy = 0;
|
||||
bisecting = false;
|
||||
noGallop = false;
|
||||
width = knownWidth;
|
||||
discovering = false;
|
||||
}
|
||||
}
|
||||
|
||||
// Carry the stretch the last complete read proved empty, joined to the one carried in when they touch; when they
|
||||
// do not, keep the one reaching the floor (it saves a whole walk), else the newer.
|
||||
var next = carried;
|
||||
if (proven.To > proven.From)
|
||||
{
|
||||
if (carried.Item2 > carried.Item1 && proven.From < carried.Item2 && carried.Item1 < proven.To)
|
||||
next = (Math.Min(proven.From, carried.Item1), Math.Max(proven.To, carried.Item2));
|
||||
else if (!(carried.Item2 > carried.Item1 && carried.Item1 <= stopTicks) || proven.From <= stopTicks)
|
||||
next = proven;
|
||||
}
|
||||
|
||||
return (result, new ReverseWalkHint(width, next.Item1, next.Item2));
|
||||
}
|
||||
|
||||
/// <summary>Appends the rows in <c>(segLo, segHi]</c> newest-first (until <paramref name="limit"/>) and counts them.</summary>
|
||||
private static int AppendNewestFirst<T>(
|
||||
IReadOnlyList<T> rows, Func<T, long> ticksOf, IComparer<T>? tieOrder, long segLo, long segHi, List<T> result, int limit) =>
|
||||
AppendNewestFirst(rows, ticksOf, tieOrder, segLo, segHi, result, limit, out _);
|
||||
|
||||
/// <summary>As above, also returning the oldest in-range timestamp (meaningless when none).</summary>
|
||||
private static int AppendNewestFirst<T>(
|
||||
IReadOnlyList<T> rows, Func<T, long> ticksOf, IComparer<T>? tieOrder, long segLo, long segHi, List<T> result, int limit,
|
||||
out long oldest)
|
||||
{
|
||||
oldest = long.MaxValue;
|
||||
IReadOnlyList<T> ordered = rows;
|
||||
if (tieOrder is not null)
|
||||
{
|
||||
var copy = rows.ToList();
|
||||
copy.Sort((a, b) =>
|
||||
{
|
||||
var c = ticksOf(a).CompareTo(ticksOf(b));
|
||||
return c != 0 ? c : tieOrder.Compare(a, b);
|
||||
});
|
||||
ordered = copy;
|
||||
}
|
||||
|
||||
var count = 0;
|
||||
for (var i = ordered.Count - 1; i >= 0; i--)
|
||||
{
|
||||
var ts = ticksOf(ordered[i]);
|
||||
if (ts <= segLo || ts > segHi) continue;
|
||||
count++;
|
||||
if (ts < oldest) oldest = ts;
|
||||
if (result.Count < limit) result.Add(ordered[i]);
|
||||
}
|
||||
|
||||
return count;
|
||||
}
|
||||
|
||||
private static DateTime Utc(long ticks) => new(ticks, DateTimeKind.Utc);
|
||||
|
||||
private static long AlignDown(long ticks, long step) => ticks - ticks % step;
|
||||
|
||||
private static long AlignUp(long ticks, long step)
|
||||
{
|
||||
var down = AlignDown(ticks, step);
|
||||
if (down == ticks) return ticks;
|
||||
return down > DateTime.MaxValue.Ticks - step ? DateTime.MaxValue.Ticks : down + step;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Orders events within one timestamp by <see cref="HistoricalEvent.EventId"/> (ordinal) — the tie order a walked-back
|
||||
/// events read (#691.4) and its oversized-cluster over-fetch share, because the Gateway orders by event time only.
|
||||
/// </summary>
|
||||
internal sealed class EventIdTieOrder : IComparer<HistoricalEvent>
|
||||
{
|
||||
/// <summary>The shared instance.</summary>
|
||||
public static readonly EventIdTieOrder Instance = new();
|
||||
|
||||
/// <inheritdoc />
|
||||
public int Compare(HistoricalEvent? x, HistoricalEvent? y) => string.CompareOrdinal(x?.EventId, y?.EventId);
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
using Opc.Ua;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.OpcUaServer;
|
||||
|
||||
/// <summary>
|
||||
/// The Part 13 <see cref="AggregateConfiguration"/> processed reads are evaluated under (#677), and the rule for a
|
||||
/// client-supplied one in <c>ReadProcessedDetails</c>.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// The aggregate evaluators (<c>AggregateEvaluator</c> / <c>TimeWeightedAggregator</c> in
|
||||
/// <c>Driver.Historian.Sql.Common</c>) are written against fixed settings, not parameterised: Uncertain time counts as
|
||||
/// Good (<see cref="TreatUncertainAsBad"/> = false) in the time-weighted family; the per-sample Average / Minimum /
|
||||
/// Maximum / Range / *ActualTime / Count are defined by Part 13 over Good values whatever this setting (#684), as the
|
||||
/// SDK computes them under it; an interval is Good only when all of
|
||||
/// it is (<see cref="PercentDataGood"/> = 100) and Bad only when none of it is non-Bad
|
||||
/// (<see cref="PercentDataBad"/> = 100), otherwise <c>UncertainDataSubNormal</c>; the value past the last sample
|
||||
/// is held (stepped, <see cref="UseSlopedExtrapolation"/> = false). These are also the SDK's own server defaults
|
||||
/// (<c>AggregateManager.GetDefaultConfiguration</c>), which a test pins.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// A request asking for <c>UseServerCapabilitiesDefaults</c>, or spelling out exactly these values, is served;
|
||||
/// any other configuration is refused per node with <c>BadAggregateConfigurationRejected</c> rather than silently
|
||||
/// answered under settings the client did not ask for (it used to be ignored).
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public static class ServerAggregateConfiguration
|
||||
{
|
||||
/// <summary>Uncertain data is not treated as Bad.</summary>
|
||||
public const bool TreatUncertainAsBad = false;
|
||||
|
||||
/// <summary>An interval is Bad only when 100 % of it is Bad.</summary>
|
||||
public const byte PercentDataBad = 100;
|
||||
|
||||
/// <summary>An interval is Good only when 100 % of it is Good.</summary>
|
||||
public const byte PercentDataGood = 100;
|
||||
|
||||
/// <summary>The value past the last sample is held (stepped), not sloped.</summary>
|
||||
public const bool UseSlopedExtrapolation = false;
|
||||
|
||||
/// <summary>The server defaults as an <see cref="AggregateConfiguration"/> (explicit values, not "use defaults").</summary>
|
||||
public static AggregateConfiguration Defaults => new()
|
||||
{
|
||||
UseServerCapabilitiesDefaults = false,
|
||||
TreatUncertainAsBad = TreatUncertainAsBad,
|
||||
PercentDataBad = PercentDataBad,
|
||||
PercentDataGood = PercentDataGood,
|
||||
UseSlopedExtrapolation = UseSlopedExtrapolation,
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// True when a processed read with <paramref name="requested"/> can be served: none supplied,
|
||||
/// <c>UseServerCapabilitiesDefaults</c>, or every setting equal to the server's.
|
||||
/// </summary>
|
||||
/// <param name="requested">The request's configuration (may be null).</param>
|
||||
/// <returns>Whether to serve; false ⇒ <c>BadAggregateConfigurationRejected</c>.</returns>
|
||||
public static bool IsServable(AggregateConfiguration? requested) =>
|
||||
requested is null
|
||||
|| requested.UseServerCapabilitiesDefaults
|
||||
|| (requested.TreatUncertainAsBad == TreatUncertainAsBad
|
||||
&& requested.PercentDataBad == PercentDataBad
|
||||
&& requested.PercentDataGood == PercentDataGood
|
||||
&& requested.UseSlopedExtrapolation == UseSlopedExtrapolation);
|
||||
}
|
||||
@@ -35,6 +35,8 @@
|
||||
<!-- Gitea #562: the over-the-wire event-revalidation test asserts the server-side fan-out, because a
|
||||
deleted item resurrected onto a notifier is invisible to the client that deleted it. -->
|
||||
<InternalsVisibleTo Include="ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests"/>
|
||||
<!-- Gitea #686: the reverse-raw walk-back is driven over the real Gateway source + its fake gateway client. -->
|
||||
<InternalsVisibleTo Include="ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests"/>
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
|
||||
@@ -452,6 +452,13 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
|
||||
/// </summary>
|
||||
private readonly IHistorizationRoleView? _historizationRoleView;
|
||||
|
||||
/// <summary>Gitea #674 review: where the HistoryUpdate verdict is published. Null on nodes that wire no redundancy manager.</summary>
|
||||
private readonly IHistoryUpdateRoleView? _historyUpdateRoleView;
|
||||
|
||||
/// <summary>Gitea #674 review: whether any redundancy snapshot has reached this host — the HistoryUpdate gate stays
|
||||
/// closed until one has (see <see cref="PrimaryGatePolicy.ShouldServeHistoryUpdate"/>).</summary>
|
||||
private bool _redundancySnapshotReceived;
|
||||
|
||||
/// <summary>
|
||||
/// Tests whether a node named in a redundancy snapshot serves this node's address space — i.e.
|
||||
/// is a per-cluster mesh partner recording the very same historized refs. Null (the default)
|
||||
@@ -461,6 +468,30 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
|
||||
/// </summary>
|
||||
private readonly Func<CommonsNodeId, bool>? _sharesHistorizationScopeWithLocal;
|
||||
|
||||
/// <summary>Gitea #672: whether an UNREACHABLE recording Primary still stands this node's historization
|
||||
/// capture down (<c>false</c>) or not (<c>true</c>, the default). See <see cref="ComputeHistorizationVerdict"/>.</summary>
|
||||
private readonly bool _captureWhilePrimaryUnreachable;
|
||||
|
||||
/// <summary>Gitea #672: the last redundancy snapshot's nodes, so a reachability change can re-derive the
|
||||
/// historization capture verdict without waiting for the next snapshot — which, when the recording Primary
|
||||
/// crashed, may not come until auto-down (the redundancy singleton usually lived on that Primary).</summary>
|
||||
private IReadOnlyList<NodeRedundancyState>? _lastRedundancySnapshotNodes;
|
||||
|
||||
/// <summary>Gitea #672: peers the cluster failure detector currently reports unreachable (never this node).
|
||||
/// Fed by <see cref="Akka.Cluster.ClusterEvent.ReachabilityEvent"/>; cleared for an address that comes Up again.</summary>
|
||||
private readonly HashSet<CommonsNodeId> _unreachableNodes = new();
|
||||
|
||||
/// <summary>Gitea #672: the unreachable recording Primary this node is currently capturing in place of, or
|
||||
/// null — drives the once-per-transition Information log.</summary>
|
||||
private CommonsNodeId? _capturingForUnreachablePrimary;
|
||||
|
||||
/// <summary>The last historization capture verdict published (null before the first), so the reachability
|
||||
/// path republishes only a CHANGED verdict.</summary>
|
||||
private bool? _lastPublishedHistorizationVerdict;
|
||||
|
||||
/// <summary>Whether this actor holds its own (#672) cluster reachability subscription.</summary>
|
||||
private bool _subscribedToReachabilityEvents;
|
||||
|
||||
/// <summary>Gitea #601 — how long an Applied ack waits for the address-space outcome (see
|
||||
/// <see cref="DefaultApplyOutcomeTimeout"/>).</summary>
|
||||
private readonly TimeSpan _applyOutcomeTimeout;
|
||||
@@ -664,7 +695,9 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
|
||||
TimeSpan? reprovisionBackoff = null,
|
||||
TimeSpan? commsRecoveryAuditDelay = null,
|
||||
Func<CommonsNodeId, bool>? sharesAlarmQueueWithLocal = null,
|
||||
TimeSpan? applyOutcomeTimeout = null) =>
|
||||
TimeSpan? applyOutcomeTimeout = null,
|
||||
bool captureWhilePrimaryUnreachable = true,
|
||||
IHistoryUpdateRoleView? historyUpdateRoleView = null) =>
|
||||
// WARNING: this forwarding list is POSITIONAL, and Props.Create compiles it into an
|
||||
// expression tree. Six IActorRef? parameters and several interface-typed ones mean a
|
||||
// mis-ordered argument is usually type-compatible and therefore compiles clean, then binds
|
||||
@@ -677,7 +710,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
|
||||
deploymentArtifactCache, redundancyRoleView, replicationPeerHost, fetchAndCacheMode, artifactFetcher,
|
||||
alarmStateStore, telemetryHub, primaryRoleSource, historizationRoleView,
|
||||
sharesHistorizationScopeWithLocal, applyWedgeTimeout, reprovisionBackoff, commsRecoveryAuditDelay,
|
||||
sharesAlarmQueueWithLocal, applyOutcomeTimeout));
|
||||
sharesAlarmQueueWithLocal, applyOutcomeTimeout, captureWhilePrimaryUnreachable, historyUpdateRoleView));
|
||||
|
||||
/// <summary>Initializes a new DriverHostActor with the specified dependencies.</summary>
|
||||
/// <param name="dbFactory">Database context factory for configuration database access.</param>
|
||||
@@ -745,6 +778,15 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
|
||||
/// PeerAddress), whose partner is its per-cluster mesh partner. Ignored when
|
||||
/// <paramref name="replicationPeerHost"/> is set. Null ⇒ no such peer is known and this node never stands its
|
||||
/// drain down.</param>
|
||||
/// <param name="captureWhilePrimaryUnreachable">Gitea #672 (<c>ContinuousHistorization:CaptureWhilePrimaryUnreachable</c>,
|
||||
/// default <c>true</c>): when a mesh-partner Primary this node would stand its historization capture down for is
|
||||
/// reported UNREACHABLE by the cluster failure detector, stop deferring to it and capture — rather than waiting
|
||||
/// for auto-down + a fresh redundancy snapshot. Affects ONLY the historization capture verdict; the alarm drain
|
||||
/// and the Primary data-plane verdict are unchanged. <c>false</c> restores the pre-#672 behaviour.</param>
|
||||
/// <param name="historyUpdateRoleView">Gitea #674 review: where the OPC UA HistoryUpdate verdict
|
||||
/// (<see cref="PrimaryGatePolicy.ShouldServeHistoryUpdate"/> — the data-plane verdict, but closed until the first
|
||||
/// redundancy snapshot) is published for the node manager's HistoryUpdate Primary gate. Null on nodes that wire no
|
||||
/// redundancy manager.</param>
|
||||
public DriverHostActor(
|
||||
IDbContextFactory<OtOpcUaConfigDbContext>? dbFactory,
|
||||
CommonsNodeId localNode,
|
||||
@@ -776,8 +818,12 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
|
||||
TimeSpan? reprovisionBackoff = null,
|
||||
TimeSpan? commsRecoveryAuditDelay = null,
|
||||
Func<CommonsNodeId, bool>? sharesAlarmQueueWithLocal = null,
|
||||
TimeSpan? applyOutcomeTimeout = null)
|
||||
TimeSpan? applyOutcomeTimeout = null,
|
||||
bool captureWhilePrimaryUnreachable = true,
|
||||
IHistoryUpdateRoleView? historyUpdateRoleView = null)
|
||||
{
|
||||
_captureWhilePrimaryUnreachable = captureWhilePrimaryUnreachable;
|
||||
_historyUpdateRoleView = historyUpdateRoleView;
|
||||
_applyOutcomeTimeout = applyOutcomeTimeout ?? DefaultApplyOutcomeTimeout;
|
||||
_sharesAlarmQueueWithLocal = sharesAlarmQueueWithLocal;
|
||||
_applyWedgeTimeout = applyWedgeTimeout ?? DefaultApplyWedgeTimeout;
|
||||
@@ -847,6 +893,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
|
||||
// ApplyScriptedAlarms through the same PushDesiredSubscriptions pass.
|
||||
SpawnScriptedAlarmHost();
|
||||
StartPrimaryVerdictPublishing();
|
||||
StartReachabilityTracking();
|
||||
Bootstrap();
|
||||
}
|
||||
|
||||
@@ -876,6 +923,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
|
||||
/// </summary>
|
||||
private void StartPrimaryVerdictPublishing()
|
||||
{
|
||||
PublishHistoryUpdateVerdict();
|
||||
if (_primaryRoleSource is null) return;
|
||||
PublishPrimaryVerdict();
|
||||
try
|
||||
@@ -894,8 +942,18 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
|
||||
/// <summary>A cluster member changed status: re-publish the Primary verdict now and once more shortly after.
|
||||
/// The deferred pass covers the cluster read-view (what <see cref="DriverMemberCount"/> reads) updating on its
|
||||
/// own subscriber AFTER this event was delivered here — the two are not ordered.</summary>
|
||||
private void OnDriverMembershipChanged(Akka.Cluster.ClusterEvent.IMemberEvent _)
|
||||
private void OnDriverMembershipChanged(Akka.Cluster.ClusterEvent.IMemberEvent evt)
|
||||
{
|
||||
// Gitea #672: a member that comes Up at an address is reachable — the usual case being a restarted
|
||||
// node's new incarnation. A REMOVED member deliberately stays in the set: it never answers again, and
|
||||
// dropping it would re-derive the verdict from a cached snapshot that may still name it Primary,
|
||||
// closing the gate on a dead node until the next snapshot (the very hole #672 closes).
|
||||
if (evt is Akka.Cluster.ClusterEvent.MemberUp up
|
||||
&& _unreachableNodes.Remove(ToNodeId(up.Member.Address)))
|
||||
{
|
||||
RepublishHistorizationVerdictIfChanged();
|
||||
}
|
||||
|
||||
PublishPrimaryVerdictIfChanged();
|
||||
// One keyed timer: a burst of member events collapses into a single deferred pass.
|
||||
Timers.StartSingleTimer("republish-primary-verdict", RepublishPrimaryVerdict.Instance, MembershipRepublishDelay);
|
||||
@@ -905,6 +963,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
|
||||
/// The snapshot path publishes on every snapshot, as the other verdicts do.</summary>
|
||||
private void PublishPrimaryVerdict()
|
||||
{
|
||||
PublishHistoryUpdateVerdict();
|
||||
if (_primaryRoleSource is null) return;
|
||||
var verdict = ShouldServiceAsPrimary();
|
||||
_lastPublishedPrimaryVerdict = verdict;
|
||||
@@ -915,14 +974,74 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
|
||||
/// role information, so republishing an unchanged verdict on each would be noise.</summary>
|
||||
private void PublishPrimaryVerdictIfChanged()
|
||||
{
|
||||
PublishHistoryUpdateVerdict();
|
||||
if (_primaryRoleSource is null || ShouldServiceAsPrimary() == _lastPublishedPrimaryVerdict) return;
|
||||
PublishPrimaryVerdict();
|
||||
}
|
||||
|
||||
/// <summary>Gitea #674 review: publishes <see cref="PrimaryGatePolicy.ShouldServeHistoryUpdate"/> (no-op when no view
|
||||
/// is wired). Rides every Primary-verdict publish point — host start, every snapshot, membership changes — so it is
|
||||
/// closed from host start and opens only once a snapshot has arrived and the data-plane verdict allows.</summary>
|
||||
private void PublishHistoryUpdateVerdict() =>
|
||||
_historyUpdateRoleView?.PublishHistoryUpdateRole(
|
||||
PrimaryGatePolicy.ShouldServeHistoryUpdate(_redundancySnapshotReceived, _localRole, DriverMemberCount()));
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #672: follow cluster reachability so the historization capture gate stops deferring to a recording
|
||||
/// Primary the failure detector already considers dead. Its OWN subscription, deliberately not folded into
|
||||
/// <see cref="StartPrimaryVerdictPublishing"/>, which early-returns on nodes with no primary-role source.
|
||||
/// <see cref="Akka.Cluster.ClusterEvent.MemberUp"/> rides along so a re-joined incarnation leaves the set.
|
||||
/// </summary>
|
||||
private void StartReachabilityTracking()
|
||||
{
|
||||
if (_historizationRoleView is null || !_captureWhilePrimaryUnreachable) return;
|
||||
try
|
||||
{
|
||||
Akka.Cluster.Cluster.Get(Context.System).Subscribe(
|
||||
Self, Akka.Cluster.ClusterEvent.SubscriptionInitialStateMode.InitialStateAsEvents,
|
||||
typeof(Akka.Cluster.ClusterEvent.ReachabilityEvent),
|
||||
typeof(Akka.Cluster.ClusterEvent.MemberUp));
|
||||
_subscribedToReachabilityEvents = true;
|
||||
}
|
||||
catch (Exception)
|
||||
{
|
||||
// Non-cluster ActorRefProvider — no failure detector to follow; the snapshot path alone decides.
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Gitea #672: a peer became unreachable or reachable again. Records it and re-derives ONLY the
|
||||
/// historization capture verdict from the cached snapshot — the alarm drain and the Primary data-plane verdict
|
||||
/// are deliberately untouched (a false allow there is a dual claimant, not a duplicate sample).</summary>
|
||||
private void OnReachabilityChanged(Akka.Cluster.ClusterEvent.ReachabilityEvent evt)
|
||||
{
|
||||
var node = ToNodeId(evt.Member.Address);
|
||||
if (node == _localNode) return; // this node's own reachability as seen by others is not a peer question
|
||||
var changed = evt is Akka.Cluster.ClusterEvent.UnreachableMember
|
||||
? _unreachableNodes.Add(node)
|
||||
: _unreachableNodes.Remove(node);
|
||||
if (changed) RepublishHistorizationVerdictIfChanged();
|
||||
}
|
||||
|
||||
/// <summary>Re-derives the historization capture verdict from the cached snapshot and publishes it only when it
|
||||
/// differs from the last one published. No-op before the first snapshot.</summary>
|
||||
private void RepublishHistorizationVerdictIfChanged()
|
||||
{
|
||||
if (_historizationRoleView is null || _lastRedundancySnapshotNodes is null) return;
|
||||
var verdict = ComputeHistorizationVerdict(_lastRedundancySnapshotNodes);
|
||||
if (verdict == _lastPublishedHistorizationVerdict) return;
|
||||
_lastPublishedHistorizationVerdict = verdict;
|
||||
_historizationRoleView.PublishHistorizationRole(verdict);
|
||||
}
|
||||
|
||||
/// <summary>Same host:port projection as <c>ClusterRoleInfo</c> / <c>RedundancyStateActor.ToNodeId</c>, so a
|
||||
/// reachability event's member compares equal to the NodeId a redundancy snapshot names.</summary>
|
||||
private static CommonsNodeId ToNodeId(Akka.Actor.Address address) =>
|
||||
CommonsNodeId.Parse($"{address.Host ?? string.Empty}:{address.Port ?? 0}");
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override void PostStop()
|
||||
{
|
||||
if (_subscribedToMemberEvents)
|
||||
if (_subscribedToMemberEvents || _subscribedToReachabilityEvents)
|
||||
{
|
||||
try { Akka.Cluster.Cluster.Get(Context.System).Unsubscribe(Self); }
|
||||
catch (Exception) { /* system shutting down */ }
|
||||
@@ -1349,6 +1468,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
|
||||
Receive<RouteNativeAlarmAck>(HandleRouteNativeAlarmAck);
|
||||
Receive<RedundancyStateChanged>(OnRedundancyStateChanged);
|
||||
Receive<Akka.Cluster.ClusterEvent.IMemberEvent>(OnDriverMembershipChanged);
|
||||
Receive<Akka.Cluster.ClusterEvent.ReachabilityEvent>(OnReachabilityChanged);
|
||||
Receive<RepublishPrimaryVerdict>(_ => PublishPrimaryVerdictIfChanged());
|
||||
Receive<Akka.Cluster.ClusterEvent.CurrentClusterState>(_ => PublishPrimaryVerdictIfChanged());
|
||||
// The redundancy-state topic also carries OpcUaProbeResult (OpcUaPublishActor peer-probes).
|
||||
@@ -1410,6 +1530,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
|
||||
Receive<RouteNativeAlarmAck>(HandleRouteNativeAlarmAck);
|
||||
Receive<RedundancyStateChanged>(OnRedundancyStateChanged);
|
||||
Receive<Akka.Cluster.ClusterEvent.IMemberEvent>(OnDriverMembershipChanged);
|
||||
Receive<Akka.Cluster.ClusterEvent.ReachabilityEvent>(OnReachabilityChanged);
|
||||
Receive<RepublishPrimaryVerdict>(_ => PublishPrimaryVerdictIfChanged());
|
||||
Receive<Akka.Cluster.ClusterEvent.CurrentClusterState>(_ => PublishPrimaryVerdictIfChanged());
|
||||
// The redundancy-state topic also carries OpcUaProbeResult (OpcUaPublishActor peer-probes).
|
||||
@@ -2173,6 +2294,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
|
||||
/// <c>ScriptedAlarmHostActor.OnRedundancyStateChanged</c> / <c>OpcUaPublishActor</c>.</summary>
|
||||
private void OnRedundancyStateChanged(RedundancyStateChanged msg)
|
||||
{
|
||||
_redundancySnapshotReceived = true;
|
||||
var local = msg.Nodes.FirstOrDefault(n => n.NodeId == _localNode);
|
||||
if (local is not null)
|
||||
{
|
||||
@@ -2208,20 +2330,15 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
|
||||
PrimaryGatePolicy.ShouldDrainAlarmHistory(_localRole, queueSharingPeerIsPrimary: peerIsPrimary));
|
||||
|
||||
// Gitea #537: the continuous-historization CAPTURE verdict. Same role rule as the drain gate
|
||||
// above, but a different peer question — historization is not pair-local, it is fleet-wide by
|
||||
// historian tag name, so the only node worth standing down for is one recording the SAME refs:
|
||||
// a mesh partner serving this node's address space. With no way to establish that (a legacy
|
||||
// node carrying no cluster-scoped role, or a test harness), the predicate is null and this
|
||||
// node keeps capturing. Duplicate samples are recoverable; a hole in the trend is not.
|
||||
var historizationPeerIsPrimary =
|
||||
_sharesHistorizationScopeWithLocal is { } sharesScope
|
||||
&& msg.Nodes.Any(n => n.Role == RedundancyRole.Primary
|
||||
&& n.NodeId != _localNode
|
||||
&& sharesScope(n.NodeId));
|
||||
|
||||
_historizationRoleView?.PublishHistorizationRole(
|
||||
PrimaryGatePolicy.ShouldRecordHistorizedValues(
|
||||
_localRole, addressSpaceSharingPeerIsPrimary: historizationPeerIsPrimary));
|
||||
// above, but a different peer question — see ComputeHistorizationVerdict. Cached (#672) so a
|
||||
// reachability change can re-derive it before the next snapshot arrives.
|
||||
_lastRedundancySnapshotNodes = msg.Nodes;
|
||||
if (_historizationRoleView is not null)
|
||||
{
|
||||
var historizationVerdict = ComputeHistorizationVerdict(msg.Nodes);
|
||||
_lastPublishedHistorizationVerdict = historizationVerdict;
|
||||
_historizationRoleView.PublishHistorizationRole(historizationVerdict);
|
||||
}
|
||||
|
||||
// The DATA-PLANE verdict, deliberately — the same one HandleRouteNodeWrite gates on, not the
|
||||
// drain verdict published immediately above. A driver reading this claims an exclusive identity
|
||||
@@ -2230,6 +2347,66 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
|
||||
PublishPrimaryVerdict();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The continuous-historization capture verdict for a snapshot (Gitea #537, #672).
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>Historization is not pair-local, it is fleet-wide by historian tag name, so the only node worth
|
||||
/// standing down for is one recording the SAME refs: a mesh partner serving this node's address space. With
|
||||
/// no way to establish that (a legacy node carrying no cluster-scoped role, or a test harness), the
|
||||
/// predicate is null and this node keeps capturing. Duplicate samples are recoverable; a hole in the trend
|
||||
/// is not.</para>
|
||||
/// <para><b>#672 — an UNREACHABLE recording Primary does not count.</b> A crashed (SIGKILLed) Primary stays a
|
||||
/// cluster member until auto-down, and the redundancy singleton usually died with it, so the cached snapshot
|
||||
/// keeps naming it Primary for the whole failure-detector + auto-down window (~25 s) while nobody captures.
|
||||
/// Excluding unreachable peers shrinks that hole to the failure-detector window alone. The cost — a short
|
||||
/// duplicate window, and both nodes capturing for the duration of a genuine partition — is the same
|
||||
/// asymmetric trade #537 already makes. Gated by <c>ContinuousHistorization:CaptureWhilePrimaryUnreachable</c>.</para>
|
||||
/// </remarks>
|
||||
private bool ComputeHistorizationVerdict(IReadOnlyList<NodeRedundancyState> nodes)
|
||||
{
|
||||
var peerIsPrimary = false;
|
||||
CommonsNodeId? unreachablePrimary = null;
|
||||
if (_sharesHistorizationScopeWithLocal is { } sharesScope)
|
||||
{
|
||||
foreach (var n in nodes)
|
||||
{
|
||||
if (n.Role != RedundancyRole.Primary || n.NodeId == _localNode || !sharesScope(n.NodeId)) continue;
|
||||
if (_captureWhilePrimaryUnreachable && _unreachableNodes.Contains(n.NodeId))
|
||||
{
|
||||
unreachablePrimary ??= n.NodeId;
|
||||
continue;
|
||||
}
|
||||
|
||||
peerIsPrimary = true;
|
||||
}
|
||||
}
|
||||
|
||||
var verdict = PrimaryGatePolicy.ShouldRecordHistorizedValues(
|
||||
_localRole, addressSpaceSharingPeerIsPrimary: peerIsPrimary);
|
||||
|
||||
// Log only when the unreachable exclusion is what OPENED the gate — i.e. the verdict would have been
|
||||
// "stand down" had that Primary been reachable.
|
||||
var overriding = unreachablePrimary is not null && verdict
|
||||
&& !PrimaryGatePolicy.ShouldRecordHistorizedValues(_localRole, addressSpaceSharingPeerIsPrimary: true);
|
||||
if (overriding && _capturingForUnreachablePrimary is null)
|
||||
{
|
||||
_capturingForUnreachablePrimary = unreachablePrimary;
|
||||
_log.Info(
|
||||
"DriverHost {Node}: capturing historized values while recording Primary {Primary} is unreachable (Gitea #672)",
|
||||
_localNode, unreachablePrimary);
|
||||
}
|
||||
else if (!overriding && _capturingForUnreachablePrimary is { } previous)
|
||||
{
|
||||
_capturingForUnreachablePrimary = null;
|
||||
_log.Info(
|
||||
"DriverHost {Node}: no longer capturing in place of unreachable recording Primary {Primary}; capture verdict now {Verdict}",
|
||||
_localNode, previous, verdict ? "capture" : "stand down");
|
||||
}
|
||||
|
||||
return verdict;
|
||||
}
|
||||
|
||||
private void Stale()
|
||||
{
|
||||
Receive<DispatchDeployment>(_ =>
|
||||
@@ -2251,6 +2428,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
|
||||
Receive<ReconnectDriver>(HandleReconnectDriver);
|
||||
Receive<RedundancyStateChanged>(OnRedundancyStateChanged);
|
||||
Receive<Akka.Cluster.ClusterEvent.IMemberEvent>(OnDriverMembershipChanged);
|
||||
Receive<Akka.Cluster.ClusterEvent.ReachabilityEvent>(OnReachabilityChanged);
|
||||
Receive<RepublishPrimaryVerdict>(_ => PublishPrimaryVerdictIfChanged());
|
||||
Receive<Akka.Cluster.ClusterEvent.CurrentClusterState>(_ => PublishPrimaryVerdictIfChanged());
|
||||
// The redundancy-state topic also carries OpcUaProbeResult (OpcUaPublishActor peer-probes).
|
||||
|
||||
@@ -153,6 +153,11 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
|
||||
/// behaviour for the same reason as <see cref="RediscoveryRaised"/> — a probe tick can land while the
|
||||
/// driver is between connects.</summary>
|
||||
private sealed record HostStatusRaised;
|
||||
/// <summary>Self-sent when the wrapped driver raises <see cref="IDriverConnectionLossSource.ConnectionLost"/> (#676)
|
||||
/// — it KNOWS its connection is gone for good. Marshals the event off the driver's thread. Acted on only in
|
||||
/// Connected (it becomes a disconnect at once, instead of after the #527 watchdog's silence window); every other
|
||||
/// behaviour is already reconnecting or connecting and ignores it.</summary>
|
||||
private sealed record ConnectionLossRaised(DriverConnectionLossEventArgs Args);
|
||||
|
||||
public sealed class RetryConnect
|
||||
{
|
||||
@@ -233,6 +238,7 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
|
||||
private EventHandler<AlarmEventArgs>? _alarmEventHandler;
|
||||
private EventHandler<RediscoveryEventArgs>? _rediscoveryHandler;
|
||||
private EventHandler<HostStatusChangedEventArgs>? _hostStatusHandler;
|
||||
private EventHandler<DriverConnectionLossEventArgs>? _connectionLossHandler;
|
||||
|
||||
/// <summary>When the driver last raised <see cref="IRediscoverable.OnRediscoveryNeeded"/>, and the reason
|
||||
/// it gave. Null until the first raise. Carried on every subsequent health snapshot so the AdminUI can
|
||||
@@ -400,6 +406,7 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
|
||||
// has no connection affinity, and a driver can observe a remote change while disconnected.
|
||||
AttachRediscoverySource();
|
||||
AttachHostStatusSource();
|
||||
AttachConnectionLossSource();
|
||||
PublishHealthSnapshot();
|
||||
Timers.StartPeriodicTimer("health-poll", HealthPollTick.Instance, _healthPollInterval);
|
||||
}
|
||||
@@ -427,6 +434,7 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
|
||||
// Stubbed drivers have no upstream alarm system — swallow the ack (it's fire-and-forget, no reply).
|
||||
Receive<RouteAlarmAck>(_ => { /* stubbed drivers have no alarm backend */ });
|
||||
Receive<DisconnectObserved>(_ => { /* stubbed drivers don't disconnect */ });
|
||||
Receive<ConnectionLossRaised>(IgnoreConnectionLoss);
|
||||
Receive<ForceReconnect>(_ => { /* stubbed drivers don't reconnect */ });
|
||||
Receive<SetDesiredSubscriptions>(StoreDesiredSubscriptions);
|
||||
// Stubbed drivers never enter Connected, so they never kick discovery; swallow defensively in case a
|
||||
@@ -468,6 +476,7 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
|
||||
});
|
||||
Receive<SetDesiredSubscriptions>(StoreDesiredSubscriptions);
|
||||
Receive<ForceReconnect>(_ => { /* already connecting — no-op */ });
|
||||
Receive<ConnectionLossRaised>(IgnoreConnectionLoss);
|
||||
// ResubscribeDesired self-Tells Subscribe; HandleSubscribeAsync replies SubscriptionEstablished to the
|
||||
// sender, which on the self-resubscribe path is Self. Swallow it (trace only) so it doesn't dead-letter.
|
||||
Receive<SubscriptionEstablished>(msg =>
|
||||
@@ -500,15 +509,11 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
|
||||
// reconnects but never manages a read is measured from THIS connect, not from a stale earlier one.
|
||||
_connectedSinceUtc = DateTime.UtcNow;
|
||||
|
||||
Receive<DisconnectObserved>(msg =>
|
||||
{
|
||||
_log.Warning("DriverInstance {Id}: disconnect observed ({Reason}); reconnecting",
|
||||
_driverInstanceId, msg.Reason);
|
||||
DetachSubscription();
|
||||
RecordFault();
|
||||
Become(Reconnecting);
|
||||
PublishHealthSnapshot();
|
||||
});
|
||||
Receive<DisconnectObserved>(msg => HandleDisconnectObserved(msg.Reason));
|
||||
// #676 — the driver KNOWS its connection is gone (e.g. OpcUaClient: the upstream server halted or forgot the
|
||||
// session). Same handling as the watchdog's disconnect, minus the ~90 s of silence it would need to prove it.
|
||||
// Handled inline rather than re-told as DisconnectObserved (Reconnecting also swallows a stray DisconnectObserved).
|
||||
Receive<ConnectionLossRaised>(msg => HandleDisconnectObserved($"driver reported connection lost: {msg.Args.Reason}"));
|
||||
Receive<ForceReconnect>(_ =>
|
||||
{
|
||||
_log.Info("DriverInstance {Id}: ForceReconnect requested by admin; re-entering Reconnecting", _driverInstanceId);
|
||||
@@ -579,8 +584,8 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
|
||||
/// came back — live-confirmed against a stopped Modbus fixture, which needed a node restart. Both
|
||||
/// comms-loss quality paths (#477 native conditions, #481 scripted-alarm inputs) hang off
|
||||
/// <see cref="ConnectivityChanged"/>, so both were wired to a signal that never fired.
|
||||
/// <c>OpcUaClientDriver</c>'s own reconnect-exhausted branch already assumed this mechanism existed
|
||||
/// ("the Core never learns it must offer an operator reinitialize") — it did not.
|
||||
/// Since Gitea #670 this is also <c>OpcUaClientDriver</c>'s only recovery path for a lost upstream
|
||||
/// session (its in-driver SDK reconnect handler was deleted).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Why the trip condition is a conjunction, and NOT simply "non-Healthy for N polls".</b>
|
||||
@@ -764,6 +769,10 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
|
||||
Receive<InitializeFailed>(_ => { /* keep retrying via timer */ });
|
||||
Receive<SetDesiredSubscriptions>(StoreDesiredSubscriptions);
|
||||
Receive<ForceReconnect>(_ => { /* already reconnecting — no-op */ });
|
||||
Receive<ConnectionLossRaised>(IgnoreConnectionLoss);
|
||||
// A watchdog DisconnectObserved self-sent on the Connected tick can still be queued when a ConnectionLossRaised
|
||||
// (or another trip) has already moved us here — already reconnecting, so it is a no-op, not an Unhandled.
|
||||
Receive<DisconnectObserved>(_ => { /* already reconnecting — no-op */ });
|
||||
// ResubscribeDesired self-Tells Subscribe; HandleSubscribeAsync replies SubscriptionEstablished to the
|
||||
// sender, which on the self-resubscribe path is Self. Swallow it (trace only) so it doesn't dead-letter.
|
||||
Receive<SubscriptionEstablished>(msg =>
|
||||
@@ -1101,6 +1110,50 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
|
||||
_rediscoveryHandler = null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Subscribe the driver's <see cref="IDriverConnectionLossSource.ConnectionLost"/> (#676), if it is one,
|
||||
/// marshalling each raise to the actor thread — actor state is never touched on the driver's thread. Attached
|
||||
/// once in <c>PreStart</c> and detached in <c>PostStop</c>, like <see cref="AttachRediscoverySource"/>: the
|
||||
/// <see cref="IDriver"/> instance can outlive this actor, so a per-connect attach would leak a handler (holding
|
||||
/// a dead <c>Self</c>) per respawn.
|
||||
/// </summary>
|
||||
private void AttachConnectionLossSource()
|
||||
{
|
||||
if (_driver is not IDriverConnectionLossSource src || _connectionLossHandler is not null) return;
|
||||
var self = Self;
|
||||
_connectionLossHandler = (_, e) => self.Tell(new ConnectionLossRaised(e));
|
||||
src.ConnectionLost += _connectionLossHandler;
|
||||
}
|
||||
|
||||
/// <summary>Symmetric teardown, called from PostStop — same leak argument as <see cref="DetachRediscoverySource"/>.</summary>
|
||||
private void DetachConnectionLossSource()
|
||||
{
|
||||
if (_driver is IDriverConnectionLossSource src && _connectionLossHandler is not null)
|
||||
src.ConnectionLost -= _connectionLossHandler;
|
||||
_connectionLossHandler = null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A connection loss reported while NOT Connected (#676): a (re)initialise is already pending or running, so
|
||||
/// there is nothing to tear down — and acting on it could tear down the connection that attempt is building.
|
||||
/// </summary>
|
||||
private void IgnoreConnectionLoss(ConnectionLossRaised msg) =>
|
||||
_log.Debug("DriverInstance {Id}: driver reported connection lost ({Reason}) while not connected — ignored",
|
||||
_driverInstanceId, msg.Args.Reason);
|
||||
|
||||
/// <summary>
|
||||
/// Connected → Reconnecting on an observed disconnect — from the #527 watchdog or a driver's definitive
|
||||
/// <see cref="IDriverConnectionLossSource.ConnectionLost"/> (#676).
|
||||
/// </summary>
|
||||
private void HandleDisconnectObserved(string reason)
|
||||
{
|
||||
_log.Warning("DriverInstance {Id}: disconnect observed ({Reason}); reconnecting", _driverInstanceId, reason);
|
||||
DetachSubscription();
|
||||
RecordFault();
|
||||
Become(Reconnecting);
|
||||
PublishHealthSnapshot();
|
||||
}
|
||||
|
||||
/// <summary>Subscribe the driver's <see cref="IHostConnectivityProbe.OnHostStatusChanged"/> (if it is
|
||||
/// one), marshaling each transition to the actor thread. Idempotent; mirrors
|
||||
/// <see cref="AttachRediscoverySource"/>, including the PreStart-not-per-connect placement — a probe
|
||||
@@ -1497,6 +1550,7 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
|
||||
// same driver object), so a missing unsubscribe accumulates a handler per respawn holding a dead Self.
|
||||
DetachRediscoverySource();
|
||||
DetachHostStatusSource();
|
||||
DetachConnectionLossSource();
|
||||
try { _driver.ShutdownAsync(CancellationToken.None).GetAwaiter().GetResult(); }
|
||||
catch (Exception ex) { _log.Warning(ex, "DriverInstance {Id}: ShutdownAsync threw on PostStop", _driverInstanceId); }
|
||||
OtOpcUaTelemetry.DriverInstanceLifecycle.Add(1,
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user