Compare commits
18 Commits
8ebee2ac40
...
70fb1f1fda
| Author | SHA1 | Date | |
|---|---|---|---|
| 70fb1f1fda | |||
| ce3369ebca | |||
| fe644d7eda | |||
| 4a3b12564a | |||
| 910befc84c | |||
| cc0cc8cd89 | |||
| 549d2d68b3 | |||
| 197246c3c1 | |||
| 9d5eef658f | |||
| 5f225f0b44 | |||
| 78f604e0cd | |||
| dbdf29c8fc | |||
| c0d6bacd9c | |||
| a11ab0dc08 | |||
| 113f1eaf7c | |||
| c019d0fc2b | |||
| ede8610804 | |||
| 6ea3ba44da |
@@ -1350,9 +1350,10 @@ historian tagname, which is the tag's driver `FullName`.
|
||||
### Pluggable backends, SQL Server and HistoryUpdate (2026-10)
|
||||
|
||||
- **Selection:** `ServerHistorian:Backend` (`Gateway` default | `SqlServer` | `QuestDb`; unknown fails host start). `HistorianBackendRegistry` → `IHistorianBackendProvider` builds all five seams; the gateway keys are only required for `Gateway`. `SqlServer` uses `ServerHistorian:SqlServer:{ConnectionString (required), Schema=hist, CommandTimeout=30s, AutoCreateSchema=true}`.
|
||||
- **SQL Server** (`Driver.Historian.SqlServer`): `hist.Sample` PK `(TagId, SourceTimeUtc)` — writes are insert-if-absent, so outbox redelivery is idempotent. Stores every value kind with its **real status**; per-sample aggregates (10, simple definitions) are pushed down to T-SQL with bucket math in **100 ns ticks**; the Part 13 time-weighted seven (TimeAverage/TimeAverage2/Total/Total2/Interpolative, #671 — ⚠️ `Total` is now the time-weighted integral, no longer `SUM` — plus the **duration-based** PercentGood/PercentBad, #680) stream window + 4 bound rows into the shared `TimeWeightedAggregator` (both SQL backends, identical results; reverse `Interpolative` anchored at the later time; window capped by `ServerHistorian:MaxTimeWeightedRawRows` ⇒ `BadTooManyOperations`, percentages included). ⚠️ **#680 status rules (client-visible):** TimeAverage/Total are Uncertain if any part of the interval uses an Uncertain value, is bridged across a Bad sample, is extrapolated past the last sample (⇒ a live tag's latest interval is Uncertain) or is not covered; TimeAverage2/Total2 keep the SDK time-based rule (Uncertain time counts as Good) but never extrapolate past the last sample — that interval is Partial, later ones `BadNoData` (a report-by-exception tag with nothing after its last sample reads `BadNoData` until its next sample arrives). `docs/Historian.md` §"Time-weighted aggregates". ⚠️ **#680 per-sample (client-visible):** `Count` counts **Good** samples only (value-based status; an all-Bad interval is generic Bad, value null) and an empty interval **inside the tag's data range** is `0`/Good (outside it still `BadNoData`); `Start`/`End` return the **raw** first/last sample of any quality — its own timestamp and status, value null when Bad; `Delta` is Uncertain only when the interval's first or last sample is unusable; and every per-sample aggregate but `Average` sets **end-of-data Partial** on the interval holding the last sample (forward incl. its earlier edge, reverse strictly inside), never on a Bad result. The data-range lookup is a seek on SQL Server but a partition scan on QuestDB (unindexed `tag` SYMBOL). `docs/Historian.md` §"Aggregates". Docker-dev recipe, schema and aggregate semantics: `docs/Historian.md` §"SQL Server backend". Integration tests need `HISTORIAN_SQL_TEST_CONNECTION`. **Retention (#667):** opt-in `SqlServer:RetentionDays` (+ `PurgeInterval` 1h, `PurgeBatchSize` 4000) -> `HistorianRetentionService` purges old `hist.Sample` (per tag) / `hist.Event` in `DELETE TOP(n)` batches, Primary-gated on `IHistorizationRoleView`, one purger per DB via `sp_getapplock`; counter `otopcua.historian.retention.purged{table}`. Optional `IHistorianBackendProvider.CreateRetention` (default null). `docs/Historian.md` §Retention.
|
||||
- **SQL Server** (`Driver.Historian.SqlServer`): `hist.Sample` PK `(TagId, SourceTimeUtc)` — writes are insert-if-absent, so outbox redelivery is idempotent. Stores every value kind with its **real status**; per-sample aggregates (10, simple definitions) are pushed down to T-SQL with bucket math in **100 ns ticks**; the Part 13 time-weighted seven (TimeAverage/TimeAverage2/Total/Total2/Interpolative, #671 — ⚠️ `Total` is now the time-weighted integral, no longer `SUM` — plus the **duration-based** PercentGood/PercentBad, #680) stream window + 4 bound rows into the shared `TimeWeightedAggregator` (both SQL backends, identical results; reverse `Interpolative` anchored at the later time; window capped by `ServerHistorian:MaxTimeWeightedRawRows` ⇒ `BadTooManyOperations`, percentages included). ⚠️ **#680 status rules (client-visible):** TimeAverage/Total are Uncertain if any part of the interval uses an Uncertain value, is bridged across a Bad sample, is extrapolated past the last sample (⇒ a live tag's latest interval is Uncertain) or is not covered; TimeAverage2/Total2 keep the SDK time-based rule (Uncertain time counts as Good) but never extrapolate past the last sample — that interval is Partial, later ones `BadNoData` (a report-by-exception tag with nothing after its last sample reads `BadNoData` until its next sample arrives). `docs/Historian.md` §"Time-weighted aggregates". ⚠️ **#680 per-sample (client-visible):** `Count` counts **Good** samples only (value-based status; an all-Bad interval is generic Bad, value null) and an empty interval **inside the tag's data range** is `0`/Good (outside it still `BadNoData`); `Start`/`End` return the **raw** first/last sample of any quality — its own timestamp and status, value null when Bad; `Delta` is Uncertain only when the interval's first or last sample is unusable; ⚠️ #684: `Average`/`Minimum`/`Maximum`/`Range`/`*ActualTime` use **Good samples only** (was Good + Uncertain; an Uncertain-only interval is `BadNoData`), Min/Max-family status Uncertain on a Bad sample, an Uncertain value beyond the Good extreme (Part 13 — the SDK ignores that) or an Uncertain non-numeric sample, and only a **Good** String/DateTime sample refuses them (`BadAggregateInvalidInputs`; Delta still refuses any usable one); and every per-sample aggregate but `Average` sets **end-of-data Partial** on the interval holding the last sample (forward incl. its earlier edge, reverse strictly inside), never on a Bad result. The data-range lookup is a key seek on SQL Server and a `tag` symbol-index seek on QuestDB (#683 — `{p}sample.tag` is `SYMBOL INDEX`; the bootstrapper ADDs it to a pre-#683 table, idempotent), read BEFORE the bucket statement so a racing append can never yield a Good `Count` of 0 (a concurrent HistoryUpdate delete is the mirror case and still can, transiently). `docs/Historian.md` §"Aggregates". Docker-dev recipe, schema and aggregate semantics: `docs/Historian.md` §"SQL Server backend". Integration tests need `HISTORIAN_SQL_TEST_CONNECTION`. **Retention (#667):** opt-in `SqlServer:RetentionDays` (+ `PurgeInterval` 1h, `PurgeBatchSize` 4000) -> `HistorianRetentionService` purges old `hist.Sample` (per tag) / `hist.Event` in `DELETE TOP(n)` batches, Primary-gated on `IHistorizationRoleView`, one purger per DB via `sp_getapplock`; counter `otopcua.historian.retention.purged{table}`. Optional `IHistorianBackendProvider.CreateRetention` (default null). `docs/Historian.md` §Retention.
|
||||
- **QuestDB** (`Driver.Historian.QuestDb`, `ServerHistorian:QuestDb:*`, Npgsql over PG wire; docker-dev overlay `docker-compose.questdb.yml`): same kinds/status/aggregates/HistoryUpdate as SqlServer, soft delete by tombstone (INSERT only — never `UPDATE`/`DELETE`; every read filters `NOT deleted`). ⚠️ **Timestamps are µs** (`QuestDbConnection.Round`/`BindTimestamp`; `IHistorianDataSource.TimestampResolutionTicks` = 10 steps HistoryRead paging), **pre-1970 cannot be stored** (writer drops, HistoryUpdate `BadOutOfRange`), every write waits for WAL visibility, TTL is enforced on write commits only, set Npgsql `Maximum Pool Size` < QuestDB's 64. `docs/Historian.md` §"QuestDB backend".
|
||||
- ⚠️ **Reverse processed reads are Part 13/SDK-anchored (#677, client-visible):** every aggregate on SqlServer/QuestDB now anchors a reverse read at its later time — window `(end, start]`, partial interval earliest, each stamped with its later edge (was: earlier-anchored, stamped earlier edge); on forward reads Partial also marks the interval the data starts inside (not Average — the SDK's rule, forward-only); a non-default client `AggregateConfiguration` ⇒ `BadAggregateConfigurationRejected`; Gateway provisioning sends `IntegralDivisor = 1`. SDK-parity tests: `SdkAggregateParityTests`. `docs/Historian.md` §Aggregates.
|
||||
- **Advertised aggregates (#682):** `IHistorianDataSource.SupportedAggregates` (Gateway 4 via `GatewayHistorianDataSource.ServedAggregates`, SqlServer/QuestDB 17 via `AggregateFamilies.Served`, Null **none**; a null set ⇒ all) is the one authority for `HistoryServerCapabilities/AggregateFunctions` (i=11201 — only its folder references are trimmed) and the up-front `BadAggregateNotSupported` in HistoryRead-Processed. ⚠️ `ServerCapabilities/AggregateFunctions` (i=2997) is deliberately NOT trimmed: it keeps the SDK's 37 because live `AggregateFilter` subscriptions still compute them all. ⚠️ Historian disabled ⇒ Processed is now `BadAggregateNotSupported`, not `GoodNoData`. `docs/Historian.md` §Advertised aggregates.
|
||||
- **Real status fix:** the recorder used to stamp every sample Good (`192`); it now carries the driver's status. ⚠️ On the **Gateway** backend this changes nothing stored: the gateway discards quality (C-002), so AVEVA still stores Good.
|
||||
- **Comms loss is historized as an outage marker** (`Empty` + `BadNoCommunication`, one per tag per outage) on SqlServer; dropped (`DroppedUnsupportedKind`) on Gateway. Outbox records are **format v2** — drain or delete `ContinuousHistorization:OutboxPath` before downgrading a node.
|
||||
- **HistoryUpdate** (SqlServer + QuestDb; Gateway ⇒ `BadHistoryOperationUnsupported`): Insert/Replace/Update/DeleteRaw/DeleteAtTime, **Primary-only** since #674 (off-Primary ⇒ `BadInvalidState`, after authorization AND after planning — an unservable request (Remove / `IsDeleteModified` / not historized) stays `BadHistoryOperationUnsupported` off-Primary; the verdict is `IHistoryUpdateRoleView` = the data-plane verdict but **closed until this node's first redundancy snapshot**, so a pair cannot both accept in the boot window; admin-only nodes register no view ⇒ no gate; reads ungated), items with a `SourceTimestamp` past now + `ServerHistorian:HistoryUpdateMaxFutureSkew` (default 1 day, ≤0 disables) ⇒ `BadOutOfRange` in the node manager (every backend), gated in both ACL modes — flag off needs the `HistoryUpdate` role from a **Cluster-scoped** grant (+ key re-check), flag on uses `DataPlaneAuthorizer`'s HistoryUpdate operation; `WriteOperate` never implies it. The `HistoryWrite` AccessLevel bit is set only when an updater is wired at materialise time. `docs/Historian.md` §HistoryUpdate, `docs/security.md`. ⚠️ **SqlServer schema v1 is accepted (#674)** with `AutoCreateSchema=false`: writes/reads/events/retention work; modified reads + HistoryUpdate ⇒ node-level `BadHistoryOperationUnsupported` on every arm (the updater throws `NotSupportedException`, nothing sent; the `HistoryWrite` bit is still advertised — not knowable at materialise time), retention skips `SampleModified`, one Warning per boot (`SqlServerSchemaBootstrapper.SupportsModifiedTrail`; `RequiredVersion` = 1, `ModifiedTrailVersion` = 2). QuestDB: keep `cairo.ttl.use.wall.clock=true`; a QuestDB trail INSERT retried after a client timeout duplicates (no DEDUP key).
|
||||
@@ -1362,8 +1363,8 @@ historian tagname, which is the tag's driver `FullName`.
|
||||
|
||||
The server dispatches all OPC UA HistoryRead to the registered `IHistorianDataSource` — the
|
||||
`GatewayHistorianDataSource` read client when enabled, else the `NullHistorianDataSource` default
|
||||
(historized nodes return `GoodNoData`, never an error). Supported variants: Raw, Processed
|
||||
(backend-dependent: Gateway serves TimeAverage/Minimum/Maximum/Total, with Average/Count `BadAggregateNotSupported` since AVEVA has no mean or sample count, #679; SqlServer/QuestDb serve seventeen), and AtTime over historized variable nodes; Events over
|
||||
(historized nodes return `GoodNoData` for Raw/AtTime — Processed is `BadAggregateNotSupported` since #682). Supported variants: Raw, Processed
|
||||
(backend-dependent: SqlServer/QuestDb serve seventeen; Gateway twelve = `GatewayHistorianDataSource.ServedAggregates` — ⚠️ #681, client-visible: the ten per-sample ones (Average, Count, Min/Max, Range, Start/End/Delta, Min/MaxActualTime) are evaluated from RAW gateway reads by the shared `Sql.Common` evaluator — identical to the SQL backends, Calculated bits, interval-start stamps, #677 reverse; Min/Max are no longer AVEVA modes, the read moves every raw sample of the window (a month of 1 Hz ⇒ `BadTooManyOperations` at the default `MaxTimeWeightedRawRows` cap), and every gateway stream is consumed to completion (an abandoned one makes the gateway evict its shared historian session) — plus TimeAverage/Total via AVEVA modes (now interval-start stamped — pending the live leg — + Calculated); the other time-weighted five stay `BadAggregateNotSupported` because the gateway has no prior-sample read for Part 13 bounds; `docs/Historian.md` §Aggregate set), and AtTime over historized variable nodes; Events over
|
||||
alarm-owning equipment-folder event-notifier nodes. Reads are ungated (served from any redundancy node);
|
||||
authorization uses the standard `AccessLevels.HistoryRead` bit set at materialization.
|
||||
|
||||
@@ -1381,7 +1382,7 @@ authorization uses the standard `AccessLevels.HistoryRead` bit set at materializ
|
||||
| `MaxTieClusterOverfetch` | `65536` | Bounded over-fetch the HistoryRead-Raw paging uses to page within an oversized same-timestamp tie cluster (retained from the prior backend) |
|
||||
| `HistoryReadBatchParallelism` | `4` | Max nodes served concurrently within one HistoryRead batch (03/S3); `⌈N/P⌉ × CallTimeout` worst case instead of sequential |
|
||||
| `MaxConcurrentHistoryReadBatches` | `16` | Process-wide concurrent HistoryRead batch cap (03/S3); flood degrades to `BadTooManyOperations`. Total in-flight reads capped at 16 × 4 = 64 |
|
||||
| `MaxTimeWeightedRawRows` | `1000000` | `SqlServer`/`QuestDb` only (#671): raw in-window rows one time-weighted aggregate read (incl. `Total` and, since #680, PercentGood/PercentBad) may stream into the C# evaluator; window query capped at this + 1, past it ⇒ `BadTooManyOperations` for that node. ≤ 0 = unbounded |
|
||||
| `MaxTimeWeightedRawRows` | `1000000` | `SqlServer`/`QuestDb` (#671): raw in-window rows one time-weighted aggregate read (incl. `Total` and, since #680, PercentGood/PercentBad) may stream into the C# evaluator — and, since #681, every `Gateway` per-sample read (raw-evaluated); window query capped at this + 1, past it ⇒ `BadTooManyOperations` for that node. ≤ 0 = unbounded |
|
||||
| `HistoryReadDeadline` | `00:01:00` | Per-request HistoryRead wall-clock deadline (03/S3); a node still in flight surfaces `BadTimeout`. Non-positive = unbounded |
|
||||
|
||||
### Alarm-history path (`AlarmHistorian` section)
|
||||
|
||||
+2
-2
@@ -210,8 +210,8 @@ otopcua-cli historyread -u opc.tcp://localhost:4840/OtOpcUa \
|
||||
-n "ns=1;s=TestMachine_001.TestHistoryValue" \
|
||||
--start "2026-03-25T00:00:00Z" --end "2026-03-30T00:00:00Z"
|
||||
|
||||
# Aggregate: 1-hour average (SqlServer/QuestDb; on the Gateway historian backend Average is
|
||||
# BadAggregateNotSupported — use --aggregate TimeAverage there, #679)
|
||||
# Aggregate: 1-hour average (every backend; on Gateway it is computed from raw samples since #681 —
|
||||
# capped by ServerHistorian:MaxTimeWeightedRawRows, so read long dense windows in slices)
|
||||
otopcua-cli historyread -u opc.tcp://localhost:4840/OtOpcUa \
|
||||
-n "ns=1;s=TestMachine_001.TestHistoryValue" \
|
||||
--start "2026-03-25T00:00:00Z" --end "2026-03-30T00:00:00Z" \
|
||||
|
||||
+261
-34
@@ -77,7 +77,7 @@ consumes the historian. The existing enable gates (`ServerHistorian:Enabled`, `A
|
||||
| Value kinds recorded | `Double`, `Int64`, `Boolean` (numeric analog only) | all: `Double`, `Int64`, `Boolean`, `String`, `DateTime`, plus `Empty` (a Bad sample with no value) | same as `SqlServer` |
|
||||
| Status code stored | Sent as DA quality but **discarded** by the gateway (C-002); AVEVA stores every sample Good | Stored bit-for-bit (`StatusCode` int column) | Stored bit-for-bit (unsigned, in a `LONG`) |
|
||||
| Comms-loss / driver-Bad publishes | Dropped, counted in `DroppedUnsupportedKind` | Stored as an `Empty` outage-marker sample carrying the Bad code (see [below](#typed-values-real-status-and-outage-markers)) | Stored as an `Empty` outage-marker sample, as on `SqlServer` |
|
||||
| Processed aggregates | TimeAverage, Minimum, Maximum, Total (Average and Count: `BadAggregateNotSupported`, #679) | The seventeen in the [aggregate table](#aggregates-processed-reads) (incl. Part 13 time-weighted, #671) | The same seventeen, identical results |
|
||||
| Processed aggregates | Twelve: the ten per-sample ones (Average, Count, Minimum, Maximum, Range, Start, End, Delta, Min/MaxActualTime) evaluated from raw reads with the SQL backends' evaluator — identical results (#681) — plus TimeAverage and Total from AVEVA's native modes | The seventeen in the [aggregate table](#aggregates-processed-reads) (incl. Part 13 time-weighted, #671) | The same seventeen, identical results |
|
||||
| HistoryUpdate | No (`BadHistoryOperationUnsupported`) | Yes (Insert / Replace / Update / DeleteRaw / DeleteAtTime) | Yes — soft delete (tombstones), see [below](#historyupdate-soft-delete) |
|
||||
| Modified-value history (`IsReadModified`, #668) | No (`BadHistoryOperationUnsupported`) | Yes — `SampleModified`, atomic with the change | Yes — `{prefix}sample_modified`, written before the change (orphan-on-crash) |
|
||||
| Alarm history | gateway `SendEvent` | `Event` table (insert-if-absent) | `{prefix}event` table (dedup upsert on `(ts, event_id)`) |
|
||||
@@ -95,7 +95,9 @@ HistoryRead is served from any node (the database is shared). HistoryUpdate is *
|
||||
|
||||
The `ServerHistorian` section in `appsettings.json` controls the historian read path.
|
||||
`Enabled` defaults to `false`; when disabled, the server registers `NullHistorianDataSource`
|
||||
and all HistoryRead calls on historized nodes return `GoodNoData` (empty, not an error).
|
||||
and Raw / AtTime / Events HistoryRead calls on historized nodes return `GoodNoData` (empty, not an error); a
|
||||
Processed read is `BadAggregateNotSupported` and no history aggregate functions are advertised
|
||||
([#682](#advertised-aggregates-aggregatefunctions-682)).
|
||||
|
||||
```json
|
||||
{
|
||||
@@ -144,7 +146,7 @@ and all HistoryRead calls on historized nodes return `GoodNoData` (empty, not an
|
||||
| `MaxTieClusterOverfetch` | int | `65536` | (Node-manager paging; all backends — a `SqlServer` PK rules out tie clusters, so it is moot there.) Maximum samples the server will fetch in one shot to page through a tie cluster (multiple samples sharing one `SourceTimestamp`). A cluster larger than this ceiling fails `BadHistoryOperationUnsupported`. Raise to handle abnormally large tie clusters; the default covers all normal-data cases. |
|
||||
| `HistoryReadBatchParallelism` | int | `4` | Max historized nodes served **concurrently** within one HistoryRead batch (arch-review 03/S3). The SDK's `HistoryRead*` override surface is synchronous, so each arm bridges once at its boundary; this turns per-node serving from sequential (`N × CallTimeout` worst case) into `⌈N / P⌉ × CallTimeout`. Also caps the gateway-load multiplication a single client can command. `≤ 1` serves sequentially. |
|
||||
| `MaxConcurrentHistoryReadBatches` | int | `16` | Process-wide limit on HistoryRead (and HistoryUpdate) batches served **concurrently** (arch-review 03/S3). A flood of history clients degrades to `BadTooManyOperations` rather than exhausting the SDK request-thread pool. With `HistoryReadBatchParallelism` this caps total in-flight gateway reads at `MaxConcurrentHistoryReadBatches × HistoryReadBatchParallelism` (default 16 × 4 = 64). A batch that cannot get a slot within `min(HistoryReadDeadline, 5 s)` fails every handle `BadTooManyOperations`. |
|
||||
| `MaxTimeWeightedRawRows` | int | `1000000` | `SqlServer` / `QuestDb` only (#671). The most raw in-window samples one [time-weighted aggregate](#time-weighted-aggregates-part-13-671) read (TimeAverage, TimeAverage2, **Total**, Total2, Interpolative, and — since #680 — **PercentGood/PercentBad**) may stream into the server-side evaluator — that family is **not** pushed down to SQL. The window query is capped at this + 1 rows; one row past the cap fails **that node** with `BadTooManyOperations` (not counted as a backend fault) instead of holding a pooled connection until `HistoryReadDeadline` and returning `BadTimeout`. Bound rows outside the window do not count; per-sample aggregates are not capped. 1 000 000 ≈ 11.5 days of a 1 Hz tag. Non-positive = unbounded (a `Validate()` warning). |
|
||||
| `MaxTimeWeightedRawRows` | int | `1000000` | `SqlServer` / `QuestDb` (#671), and the `Gateway` per-sample aggregates (#681 — every per-sample read there streams raw samples, so the cap applies to the window's raw rows; see [Aggregate set](#aggregate-set-and-badaggregatenotsupported-285)). The most raw in-window samples one [time-weighted aggregate](#time-weighted-aggregates-part-13-671) read (TimeAverage, TimeAverage2, **Total**, Total2, Interpolative, and — since #680 — **PercentGood/PercentBad**) may stream into the server-side evaluator — that family is **not** pushed down to SQL. The window query is capped at this + 1 rows; one row past the cap fails **that node** with `BadTooManyOperations` (not counted as a backend fault) instead of holding a pooled connection until `HistoryReadDeadline` and returning `BadTimeout`. Bound rows outside the window do not count; per-sample aggregates are not capped on the SQL backends (pushed down). 1 000 000 ≈ 11.5 days of a 1 Hz tag. Non-positive = unbounded (a `Validate()` warning). |
|
||||
| `HistoryReadDeadline` | TimeSpan | `00:01:00` | Per-request wall-clock deadline for a HistoryRead (and HistoryUpdate) batch (arch-review 03/S3). A node still in flight at the deadline surfaces `BadTimeout`, so one request can never hold an SDK request thread longer than this regardless of node count. Non-positive = unbounded (a `Validate()` warning). |
|
||||
| `HistoryUpdateMaxFutureSkew` | TimeSpan | `1.00:00:00` | #674. How far past the server clock a HistoryUpdate (Insert / Replace / Update) item's `SourceTimestamp` may lie; an item beyond `now + skew` is refused **`BadOutOfRange`** by the node manager before any backend sees it (every backend). Guards against a far-future typo (a 2062 timestamp). The live recorder is not bounded. Non-positive disables the bound (no warning — a deliberate choice). See [HistoryUpdate](#historyupdate). |
|
||||
|
||||
@@ -563,7 +565,7 @@ node without `isHistorized` has no HistoryRead access bit and answers
|
||||
|---|---|
|
||||
| Historized node, historian configured and reachable, results found | `Good` |
|
||||
| Historized node, historian configured, time range empty | `GoodNoData` |
|
||||
| Historized node, historian NOT configured (`Enabled=false` / Null source) | `GoodNoData` (empty) |
|
||||
| Historized node, historian NOT configured (`Enabled=false` / Null source) | `GoodNoData` (empty) for Raw / AtTime; Processed ⇒ `BadAggregateNotSupported` (#682) |
|
||||
| Non-historized node | `BadHistoryOperationUnsupported` |
|
||||
| Backend timeout or exception | `BadHistoryOperationUnsupported` per node; other nodes in the same batch are unaffected |
|
||||
| `Backend=SqlServer`, `AutoCreateSchema=false`, schema missing/behind | every read/write fails with a clear error naming the missing schema version (health `LastError`); reads answer `BadHistoryOperationUnsupported` per node; no host crash |
|
||||
@@ -745,33 +747,144 @@ otopcua-cli historyread -u opc.tcp://localhost:4840 -U apikey -P otua_<keyId>_<s
|
||||
The OPC UA `Total` aggregate is **supported** over the HistorianGateway backend. The gateway exposes a
|
||||
native **`Integral`** retrieval mode, so `Total` maps straight to it (`HistoryAggregateType.Total →
|
||||
RetrievalMode.Integral`) — no client-side scaling. (This replaces the retired Wonderware path, which had no
|
||||
`Total` column and derived it client-side as time-weighted `Average × interval-seconds`.) Bucket status codes and timestamps are preserved unchanged; empty / null
|
||||
buckets surface as `BadNoData`.
|
||||
`Total` column and derived it client-side as time-weighted `Average × interval-seconds`.) Values pass through unchanged; a Good
|
||||
bucket is `Good | Calculated` (`0x00000401`) and an empty / null bucket `BadNoData | Calculated` (#681), stamped with its interval
|
||||
start (see below).
|
||||
|
||||
**Integral unit (#677).** AVEVA's integral is `∫ value dt` (seconds) ÷ the tag's `IntegralDivisor`. The proto3 `double` default for
|
||||
`HistorianTagDefinition.IntegralDivisor` is `0`, so `GatewayTagProvisioner` now sends **`IntegralDivisor = 1`** explicitly on every
|
||||
`EnsureTags` — the value·seconds unit the SQL backends' `Total` uses. A tag provisioned earlier keeps its divisor until its next
|
||||
`EnsureTags` (an upsert). The live magnitude check against AVEVA (a constant 1 integrated over 1 s reads 1) is still open — it needs the
|
||||
VPN. Reverse processed reads on the `Gateway` backend are answered by AVEVA and are not re-anchored by #677.
|
||||
VPN. Reverse processed reads of the AVEVA-native `TimeAverage`/`Total` are not re-anchored by #677 (the gateway refuses `end < start`);
|
||||
the raw-evaluated per-sample aggregates (#681) do follow #677. Since #681 every native bucket is stamped with its interval **start**
|
||||
(Part 13; it was the interval end) and carries the Calculated info bits (`Good` ⇒ `0x00000401`, no data ⇒ `BadNoData | 0x401`).
|
||||
⚠️ **The interval-start stamp is pending the live leg:** it reads the aggregate row's `start_time`, which histsdk decodes from a row
|
||||
field (`tail + 43`) its own reverse-engineering notes still mark unverified (HistorianGateway
|
||||
`docs/plans/2026-07-13-archreview2-D-histsdk-residue-design.md`); if that field proves wrong on live AVEVA the stamp is wrong with it.
|
||||
|
||||
### Aggregate set and `BadAggregateNotSupported` (#285)
|
||||
|
||||
> ⚠️ **#679 (client-visible): on the `Gateway` backend, `Average` and `Count` are now `BadAggregateNotSupported`.** AVEVA's 15
|
||||
> retrieval modes contain no arithmetic mean and no sample count. `Average` used to be mapped to `TimeWeightedAverage` (a different
|
||||
> statistic from the OPC UA mean the SQL backends return); `Count` to `Counter`, which is a rollover-counter increase, not a sample
|
||||
> count. Request **`TimeAverage`** for the time-weighted number (now mapped to `TimeWeightedAverage`). There is no Gateway sample
|
||||
> count. `TimeAverage` magnitude and timestamping against live AVEVA are unverified (needs the VPN).
|
||||
> ⚠️ **#681 (client-visible): on the `Gateway` backend the per-sample aggregates are computed from RAW reads.** `Average`, `Count`
|
||||
> (both `BadAggregateNotSupported` since #679 — AVEVA has no arithmetic mean and no sample count), `Minimum` and `Maximum` (were
|
||||
> AVEVA's `MinimumWithTime` / `MaximumWithTime`), and the newly served `Range`, `Start`, `End`, `Delta`, `MinimumActualTime`,
|
||||
> `MaximumActualTime` stream the window's raw samples through the gateway and are evaluated by the SQL backends' shared
|
||||
> `Sql.Common` evaluator (`BucketPlan` + `AggregateEvaluator`), so Gateway answers exactly what `SqlServer`/`QuestDb` answer for the
|
||||
> same samples — values, status (Calculated / Uncertain / Partial bits, #680 Count/Start/End/Delta semantics), interval-START
|
||||
> timestamps, and reverse reads anchored at their later time (#677). What a client sees change: `Minimum`/`Maximum` values and
|
||||
> statuses may differ from AVEVA's own modes; every per-sample result now carries the Calculated info bits; a per-sample read is
|
||||
> bounded by **`ServerHistorian:MaxTimeWeightedRawRows`** (in-window raw rows; past it ⇒ `BadTooManyOperations` for that node — read
|
||||
> a long window over a dense tag in shorter windows). `TimeAverage` and `Total` stay AVEVA-native (`TimeWeightedAverage` /
|
||||
> `Integral`), now stamped with the interval start (was its end) and carrying the Calculated bit; `TimeAverage2`, `Total2`,
|
||||
> `Interpolative`, `PercentGood` and `PercentBad` remain `BadAggregateNotSupported` on Gateway. The served set is the single
|
||||
> static `GatewayHistorianDataSource.ServedAggregates`.
|
||||
|
||||
**How the Gateway raw evaluation works (#681).** Forward `ReadRaw` reads only (the gateway streams oldest-first and refuses
|
||||
`end < start`; `BucketPlan` assembles a reverse result newest-first), O(buckets) memory, and **every gateway stream is consumed to
|
||||
completion** — breaking out of one cancels the call, and the gateway currently classifies that cancellation as a connection fault
|
||||
and evicts its shared historian session (see the HistorianGateway issue text in the #681 record).
|
||||
|
||||
- **Window read** `[start − 1 ms, end]`. An AVEVA `Full` raw read emits an *initial-value* row stamped exactly at the query start
|
||||
(histsdk evidence: `Quality = 133`, `OpcQuality = 192`) — the value in effect there, not a stored sample. Starting 1 ms early puts
|
||||
that row outside every bucket. A pre-window row only proves "the tag has earlier data" (start-of-data Partial, Count's in-range
|
||||
zero), and only when its **OPC quality is not Bad** — a "no earlier data" NULL row still carries a value on the wire (histsdk sets
|
||||
`NumericValue = 0` on every numeric row and `StringValue = ""` on a string one) but with a Bad OPC quality (captured as 1, no-data; histsdk string-read-layout.md).
|
||||
- **Earlier-data probe.** When the data range matters (every per-sample aggregate but `Average`) and no non-Bad pre-window row
|
||||
arrived, a 2-row read `[1970-01-01, start)` asks whether the tag has data before the window. Normally this fires only for a tag
|
||||
whose history really begins inside the window.
|
||||
- **Look-ahead.** The end-of-data rules need to know whether data exists after the window: a sample exactly at the window end
|
||||
(forward) comes from the window read; otherwise a 2-row read `[end, now + 1 day]`. In both small reads a row stamped exactly at
|
||||
the query start (a boundary row) or the query end (a possible terminal row) is not a sample, whatever its value.
|
||||
- **Bound.** In-window rows are capped by `MaxTimeWeightedRawRows`; the gateway is asked for `cap + 18` rows. Past the cap the rest
|
||||
of the (bounded) stream is drained, then the node fails `BadTooManyOperations`; a stream that delivers the whole budget cannot be
|
||||
proven complete and fails the same way. The gateway's own refusal of that budget (`InvalidArgument` "max_values exceeds the
|
||||
server ceiling" — `Historian:ReadLimits:MaxValuesCeiling`, default 1,000,000, with `AllowStreamingBeyondCeiling=false`; the
|
||||
default `1,000,000 + 18` exceeds it) is also `BadTooManyOperations`, not a backend fault — lower `MaxTimeWeightedRawRows` to
|
||||
`ceiling − 18` on such a gateway. A caller cancellation (client cancel, `HistoryReadDeadline`) is not counted as a backend fault.
|
||||
- **Bucket statistics** are `RawBucketAccumulator`, the C# statement of the SQL backends' per-sample columns — incl. #684:
|
||||
Average / Minimum / Maximum / Range / *ActualTime over **Good** samples only, the Uncertain extremes feeding only the Min/Max
|
||||
status rule, and an Uncertain String/DateTime sample not refusing the Good-value aggregates. A change to those SQL columns
|
||||
must be repeated there.
|
||||
- **Status** is the raw read's quality mapping. C-002: the gateway discards quality on write, so AVEVA history is normally all Good.
|
||||
⚠️ **Known pre-existing limitation:** `SampleMapper` falls back to AVEVA's own `Quality` field when `OpcQuality` is 0 — but 0 is
|
||||
also the OPC DA Bad value, so a Bad row whose AVEVA `Quality` is ≥ 64 (e.g. 133) maps Uncertain/Good. Raw reads have always done
|
||||
this; since #681 such a row also feeds the per-sample aggregates (as a usable sample). The earlier-data evidence rule reads
|
||||
`OpcQuality` alone and is not affected.
|
||||
|
||||
**Cost (client-visible).** A per-sample read now moves every raw sample of the window over the gateway instead of one row per
|
||||
interval — notably `Minimum`/`Maximum`, which were native AVEVA modes before #681. Latency and gateway session time grow with the
|
||||
window's sample count, and the cap makes a long window over a dense tag fail: a month of a 1 Hz tag is ~2.6 M samples ⇒
|
||||
`BadTooManyOperations` at the default 1,000,000 cap (≈ 11.5 days of 1 Hz). Read such windows in shorter slices, or use
|
||||
`TimeAverage` (still native).
|
||||
|
||||
**Why the time-weighted family is not raw-evaluated.** Part 13 time-weighted aggregates need the real *bounding* samples — the time,
|
||||
value and quality of the latest sample before the window (sloped interpolation across the window start, an outage straddling it). The
|
||||
gateway has no "previous stored sample" read: a raw read streams forward from its start, and AVEVA replaces whatever lies before
|
||||
with the initial-value row stamped at the query start, losing the original time. So `TimeAverage`/`Total` keep AVEVA's native modes
|
||||
(exact to AVEVA's own definition, not provably Part 13), and the other five time-weighted members stay unsupported rather than
|
||||
approximated.
|
||||
|
||||
**Live leg (open — needs the VPN to `wonder-sql-vd03`):** (1) that AVEVA's raw read emits the initial-value row with a non-Bad OPC
|
||||
quality when earlier data exists and a Bad (NULL) one when none does; (2) that the window read emits no terminal row at its query
|
||||
end (the window read relies on there being none; the small reads ignore one); (3) that a 2-row raw read starting at `1970-01-01`
|
||||
and one ending at `now + 1 day` are accepted and cheap; (4) the gateway's ceiling rejection text still matches
|
||||
"max_values exceeds the server ceiling"; (5) `Minimum`/`Maximum` against live data vs. the old AVEVA modes; (6)
|
||||
`TimeAverage`/`Total` magnitude and the interval-start stamp (`tail + 43`, above).
|
||||
|
||||
`HistoryAggregateType` carries the wider Part 13 set (Interpolative, TimeAverage, Start, End, Delta, Range,
|
||||
Min/MaxActualTime, Start/EndBound, DeltaBounds, DurationGood/Bad, PercentGood/Bad, WorstQuality,
|
||||
NumberOfTransitions, AnnotationCount, StandardDeviationPopulation, VariancePopulation); `OtOpcUaNodeManager.MapAggregate`
|
||||
maps each standard aggregate NodeId (`TimeAverage2` and `Total2` were appended for #671). The **`SqlServer`** and
|
||||
NumberOfTransitions, AnnotationCount, StandardDeviationPopulation, VariancePopulation); `AggregateFunctionMap`
|
||||
(`OpcUaServer`) maps each standard aggregate NodeId (`TimeAverage2` and `Total2` were appended for #671). The **`SqlServer`** and
|
||||
**`QuestDb`** backends serve the seventeen listed in the [aggregate table](#aggregates-processed-reads); everything else
|
||||
(e.g. `StartBound`, `DurationGood`, `StandardDeviationPopulation`) is `BadAggregateNotSupported` for that node. The **HistorianGateway** backend only has retrieval modes for the four
|
||||
(TimeAverage, Minimum, Maximum, Total): for any other member `AggregateModeMapper` throws
|
||||
`HistoryAggregateNotSupportedException`, which the HistoryRead path turns into `BadAggregateNotSupported` for that
|
||||
node only (not a generic failure, and not counted as a backend fault). The OPC UA Client driver forwards every
|
||||
member upstream. A NodeId outside the mapped set (e.g. `StandardDeviationSample`) is still `BadAggregateNotSupported`.
|
||||
(e.g. `StartBound`, `DurationGood`, `StandardDeviationPopulation`) is `BadAggregateNotSupported` for that node. The **HistorianGateway** backend serves twelve (#681 —
|
||||
`GatewayHistorianDataSource.ServedAggregates`): the ten per-sample aggregates from raw reads, plus TimeAverage and Total
|
||||
through AVEVA retrieval modes (`AggregateModeMapper`); for any other member it throws
|
||||
`HistoryAggregateNotSupportedException` without calling the gateway (and since #682 the server refuses it up front), which the HistoryRead path turns into
|
||||
`BadAggregateNotSupported` for that node only (not a generic failure, and not counted as a backend fault). A NodeId outside the mapped set (e.g. `StandardDeviationSample`) is still `BadAggregateNotSupported`.
|
||||
|
||||
### Advertised aggregates (`AggregateFunctions`, #682)
|
||||
|
||||
Each backend declares the aggregates it serves as `IHistorianDataSource.SupportedAggregates` — **the single authority** for
|
||||
two things:
|
||||
|
||||
- **What the server advertises as HISTORY aggregates.** `HistoryServerCapabilities/AggregateFunctions` (`i=11201`) lists
|
||||
exactly that set. Before #682 it listed the SDK's full 37 regardless of backend.
|
||||
- **What HistoryRead-Processed refuses up front.** An aggregate outside the set is `BadAggregateNotSupported` for that node
|
||||
without a backend call (the backend still throws `HistoryAggregateNotSupportedException` for one as defence in depth —
|
||||
derived from the same table, never a second list). A source whose `SupportedAggregates` is `null` (out-of-tree, against
|
||||
the contract) is treated as the interface default — **all** — by both consumers.
|
||||
|
||||
⚠️ **`Server/ServerCapabilities/AggregateFunctions` (`i=2997`) is deliberately NOT trimmed** — it keeps the SDK's full 37 in
|
||||
every configuration, including with no historian. Part 5 defines it as every aggregate the *server* supports, and the server
|
||||
still computes all of them for live data: a MonitoredItem `AggregateFilter` is validated and computed by the SDK's
|
||||
`AggregateManager`, not the historian. The SDK models the same split (`DiagnosticsNodeManager.AddAggregateFunction` always adds
|
||||
to i=2997 and to i=11201 only for historical aggregates). The two are separate folder nodes organizing the same
|
||||
aggregate-function objects. A client choosing a history aggregate should read **i=11201**.
|
||||
|
||||
| `ServerHistorian:Backend` | i=11201 (history) | i=2997 (server) | Declared by |
|
||||
|---|---|---|---|
|
||||
| `Gateway` | 12 — the ten per-sample aggregates (raw-evaluated, #681) + TimeAverage, Total (native AVEVA modes) | all 37 | `GatewayHistorianDataSource.ServedAggregates` (the per-sample families of `AggregateFamilies` ∪ `AggregateModeMapper.NativeAggregates`, derived from its mapping switch) |
|
||||
| `SqlServer`, `QuestDb` | 17 — the [aggregate table](#aggregates-processed-reads) | all 37 | `AggregateFamilies.Served` (Sql.Common; every member with a family) |
|
||||
| historian disabled (Null source) | **none** | all 37 | `NullHistorianDataSource` |
|
||||
|
||||
⚠️ **Client-visible: with no historian configured a processed HistoryRead is now `BadAggregateNotSupported`, not
|
||||
`GoodNoData`.** An aggregate is a statistic the server cannot compute without a history, and the empty i=11201 now says so;
|
||||
Raw / AtTime / Events reads on the Null source still return the empty `GoodNoData` result.
|
||||
|
||||
How it is wired: `OtOpcUaSdkServer` keeps the SDK's `CreateAggregateManager` (all 37 calculator factories) and then trims
|
||||
only the i=11201 ⇄ aggregate `Organizes` references (both directions) to the node manager's current source — the Null
|
||||
default at boot; the Host's `SetHistorianDataSource` (the backend resolved once from `ServerHistorian:Backend`) re-syncs it
|
||||
right after start, and every later assignment re-syncs both ways. ⚠️ **The calculator factories are deliberately NOT
|
||||
removed**: live aggregate subscriptions keep working for every SDK aggregate (pinned over the wire by
|
||||
`AdvertisedAggregateFunctionsWireTests`). The aggregate-function object nodes themselves stay readable. A boot log line
|
||||
names the advertised history set.
|
||||
|
||||
⚠️ **Boot window.** `SetHistorianDataSource` runs right after the SDK server has started (`OtOpcUaServerHostedService.StartAsync`),
|
||||
so for the few milliseconds between the endpoint opening and that call i=11201 is empty and a processed HistoryRead is
|
||||
refused `BadAggregateNotSupported` — the Null default's answer. A client that browses i=11201 in that window sees an empty
|
||||
list; re-browse after connect if it matters.
|
||||
|
||||
Not changed: the `HistoryServerCapabilities` booleans (`AccessHistoryDataCapability`, `InsertDataCapability`, …) are still
|
||||
whatever the SDK sets — #682 touches only the i=11201 folder (follow-up candidate).
|
||||
|
||||
### Known limitations
|
||||
|
||||
@@ -951,7 +1064,31 @@ point (see [Continuation-point paging (Events)](#continuation-point-paging-event
|
||||
> so a "live" tag's latest interval is now flagged.
|
||||
>
|
||||
> A client that read `Count` as "usable samples", keyed `Start`/`End` on the interval start, or treated `Count = BadNoData` as "no
|
||||
> samples" must adjust. Not changed: `Average`/`Minimum`/`Maximum` still use Good + Uncertain samples (the SDK uses Good only).
|
||||
> samples" must adjust. (`Average`/`Minimum`/`Maximum` were left on Good + Uncertain samples here; #684, below, moved them to Good only.)
|
||||
|
||||
> ⚠️ **#684 (client-visible): `Average`, `Minimum`, `Maximum`, `Range` and `Minimum`/`MaximumActualTime` use GOOD samples only** on
|
||||
> the `SqlServer` and `QuestDb` backends (they used every *usable* sample — Good **and** Uncertain). Part 13 defines each over "Good
|
||||
> raw values" (§5.4.3.5, .10–.14), independent of `TreatUncertainAsBad` (only `Minimum2`/`Maximum2`, not served, consult it), and the
|
||||
> SDK's `AverageAggregateCalculator`/`MinMaxAggregateCalculator` skip Uncertain values under the server's own `TreatUncertainAsBad = false`.
|
||||
> - An Uncertain sample is **never part of the value** (an interval `Good 6, Good 8, Uncertain 4` now averages 7, was 6; its Minimum is
|
||||
> 6, was 4), and an interval holding **only** Uncertain (and Bad) samples is now **`BadNoData`** (it used to return their value,
|
||||
> Uncertain).
|
||||
> - **Status:** `Average` is value-based (Good only when every sample is Good, else `UncertainDataSubNormal` — unchanged, the SDK's
|
||||
> rule). `Minimum`/`Maximum`/`Range`/`*ActualTime` follow their Part 13 tables: `UncertainDataSubNormal` when a **Bad** sample
|
||||
> exists, or an **Uncertain value lies beyond the reported Good extreme** (below the minimum for Minimum/MinimumActualTime, above the
|
||||
> maximum for Maximum/MaximumActualTime, either for Range; strictly — equal does not count); otherwise **Good**. So an Uncertain
|
||||
> sample inside the Good range no longer makes Minimum/Maximum Uncertain (it used to).
|
||||
> - The SDK applies only the Bad half of that rule (its `ComputeMinMax` collects the Uncertain extremes and never reads them) and answers
|
||||
> Good for an Uncertain value beyond the extreme; ours follows Part 13 there — the one remaining per-sample Uncertain difference, pinned
|
||||
> by `SdkAggregateParityTests`.
|
||||
> - **Non-numeric samples:** these aggregates now answer `BadAggregateInvalidInputs` only for a **Good** `String`/`DateTime` sample
|
||||
> in the interval. An **Uncertain** one is skipped like any Uncertain value — the interval is served from its Good numeric values
|
||||
> with `UncertainDataSubNormal` (the SDK's Average skips it; its Min/Max flags the value it cannot cast). `Delta` reads usable
|
||||
> samples and still refuses any usable non-numeric one.
|
||||
> - `Count` (Good only, #680), `Start`/`End`/`Delta` (usable / any-quality picks) and the time-weighted family (Uncertain time counts as
|
||||
> Good time under `TreatUncertainAsBad = false`, as Part 13 §5.4.3.2 says) are unchanged. The advertised `TreatUncertainAsBad = false`
|
||||
> stays correct: it governs the time-weighted and status calculations; the per-sample aggregates above are defined over Good values
|
||||
> whatever its setting.
|
||||
|
||||
**Bucketing.** Forward: one result per interval of `[start, end)`; bucket *i* = `[start + i·interval, start + (i+1)·interval)`,
|
||||
start-inclusive / end-exclusive to the tick, the last bucket truncated at `end`, each stamped with its start. Reverse (`end < start`,
|
||||
@@ -987,9 +1124,27 @@ Data continuing past the window sets nothing. Two deliberate limits, kept: the p
|
||||
**Data-range queries.** The start/end-of-data rules and `Count`'s in-range empty interval need the tag's **data range**: its earliest
|
||||
stored sample (any quality) at or before the window's later edge and its latest at or after the earlier edge. On **SQL Server** these
|
||||
are one extra result set of two `MIN`/`MAX` subqueries, each a single seek on the `(TagId, SourceTimeUtc)` key. On **QuestDB** they are
|
||||
two `LIMIT 1` statements, but `tag` is an **unindexed `SYMBOL`**, so each is a **partition scan**, not a seek: measured 1–3 ms for a
|
||||
live tag, and for a stopped tag read over an old window 6–8 ms warm / ~32 ms cold on a 52 M-row table — the cost grows with the table
|
||||
(bounded by the TTL), not with the window. They run only when the answer can change a result
|
||||
two `LIMIT 1` statements that **seek the `tag` symbol index** (#683) partition by partition. Before #683 `tag` was unindexed and
|
||||
each was a partition scan whose cost grew with the table, not the window — worst for a stopped tag read over an old window (`LatestSample`
|
||||
scans every later partition) and a late-starting tag (`EarliestSample` scans from the table start). Measured on 53.6 M rows / 120 tags /
|
||||
61 daily partitions (QuestDB 10.0.1, HTTP round-trip included, median of 7):
|
||||
|
||||
| Data-range lookup | unindexed warm / cold | indexed warm / cold |
|
||||
|---|---|---|
|
||||
| live tag (either lookup) | 1.5–3 ms / 2–14 ms | 1.1–1.4 ms / 2.5–3.5 ms |
|
||||
| stopped tag, window inside its data (`LatestSample`) | 11 ms / 11 ms | 1.5 ms / 3.5 ms |
|
||||
| stopped tag, window after it stopped (`LatestSample`, no row) | 7 ms / 9 ms | 1.0 ms / 1.9 ms |
|
||||
| late-starting tag (`EarliestSample` from table start) | 7–7.5 ms / 7–10 ms | 1.2 ms / 2–3 ms |
|
||||
|
||||
With the index an absent side still visits every partition on that side, but as one index-key lookup each rather than a column scan, so
|
||||
it no longer grows with the rows per partition (the tag count); it still grows with the partition count (bounded by the TTL). Alternatives
|
||||
measured and rejected: bounded existence probes ("any sample before the window start / after its end", `LIMIT 1` from the window edge)
|
||||
are cheap only when data *exists* on that side — an absent side is still a full scan, the very case the issue is about; `LATEST ON ts
|
||||
PARTITION BY tag` was the slowest unindexed form (19–32 ms) and no faster than the plain seek indexed; and a per-tag first/last-write side
|
||||
table cannot be kept correct INSERT-only — a TTL partition drop or a HistoryUpdate delete moves the first/last sample with no write to hook.
|
||||
They are read **before** the bucket statement (a separate table reader), so a sample appended in between reaches the buckets but not the
|
||||
range — at worst a stale end-of-data Partial or a `BadNoData` where a `Count` of 0 belongs, never a Good `0` for an interval that has
|
||||
samples (the result of the opposite order). SQL Server reads its range result set first for the same reason. They run only when the answer can change a result
|
||||
(`AggregateEvaluator.NeedsDataRange`): a per-sample read of any aggregate other than `Average`, in either direction (`Count`
|
||||
included); a per-sample `Average` read skips them.
|
||||
|
||||
@@ -998,18 +1153,19 @@ included); a per-sample `Average` read skips them.
|
||||
|
||||
| Aggregate | Computed over | Value timestamp |
|
||||
|---|---|---|
|
||||
| `Average`, `Minimum`, `Maximum` | `AVG` / `MIN` / `MAX` of `NumValue` over usable samples | bucket start (reverse read: its later edge, as every row marked "bucket start" here) |
|
||||
| `Average`, `Minimum`, `Maximum` | `AVG` / `MIN` / `MAX` of `NumValue` over **Good** samples (#684) | bucket start (reverse read: its later edge, as every row marked "bucket start" here) |
|
||||
| `Count` | number of **Good** samples (`Int32`; #680) | bucket start |
|
||||
| `Range` | `Maximum − Minimum` | bucket start |
|
||||
| `Range` | `Maximum − Minimum` (Good samples) | bucket start |
|
||||
| `Start`, `End` | the **raw** first / last sample of any quality (its type, its own status; value null when Bad — #680) | **the sample's own time** |
|
||||
| `Delta` | last usable − first usable value (numeric samples) | bucket start |
|
||||
| `MinimumActualTime`, `MaximumActualTime` | the usable sample with the smallest / largest `NumValue`; ties ⇒ earliest | **the sample's own time** |
|
||||
| `MinimumActualTime`, `MaximumActualTime` | the **Good** sample (#684) with the smallest / largest `NumValue`; ties ⇒ earliest | **the sample's own time** |
|
||||
|
||||
Status and edge rules:
|
||||
|
||||
- An **empty interval** (no samples) ⇒ `BadNoData` — except **`Count`**, which is `0`, Good, when the interval lies inside the tag's data range `[first sample, last sample]` (both edges inclusive, either direction; #680). **Every value aggregate over an interval whose samples are all Bad ⇒ `BadNoData`**; `Count` there is the generic **Bad** (value null).
|
||||
- Value aggregates are `Good` when every sample in the bucket is Good and `UncertainDataSubNormal` when the bucket also holds any Bad **or Uncertain** sample. `Count`'s status is value-based (all Good ⇒ Good, all Bad ⇒ Bad, else Uncertain); `Start`/`End` carry the raw sample's own status; `Delta` is Good unless the interval's first or last sample is not usable (then Uncertain) — #680.
|
||||
- `Count`, `Start`, `End` accept any value kind; the other aggregates (and `Delta`) on a bucket that holds a usable `String`/`DateTime` sample ⇒ `BadAggregateInvalidInputs` for that interval.
|
||||
- An interval with samples but **no Good numeric value** (only Uncertain / Bad) ⇒ `BadNoData` for `Average`/`Minimum`/`Maximum`/`Range`/`*ActualTime` (#684).
|
||||
- `Average` is `Good` when every sample in the bucket is Good, else `UncertainDataSubNormal` (value-based, the SDK's rule). `Minimum`/`Maximum`/`Range`/`*ActualTime` are `UncertainDataSubNormal` when the bucket holds a Bad sample or an Uncertain value beyond the reported Good extreme (Part 13 Tables 57–62; strictly beyond), else `Good` (#684). `Count`'s status is value-based (all Good ⇒ Good, all Bad ⇒ Bad, else Uncertain); `Start`/`End` carry the raw sample's own status; `Delta` is Good unless the interval's first or last sample is not usable (then Uncertain) — #680.
|
||||
- `Count`, `Start`, `End` accept any value kind. `Average`/`Minimum`/`Maximum`/`Range`/`*ActualTime` on a bucket that holds a **Good** `String`/`DateTime` sample ⇒ `BadAggregateInvalidInputs` for that interval; an Uncertain one is skipped and makes `Minimum`/`Maximum`/`Range`/`*ActualTime` `UncertainDataSubNormal` (#684). `Delta` is refused by any usable non-numeric sample.
|
||||
- `PercentGood` / `PercentBad` are **duration-based since #680** — see [Time-weighted aggregates](#time-weighted-aggregates-part-13-671).
|
||||
- Everything except `Count` and `Start`/`End` is a `double`; `Int64` samples aggregate as doubles and `Boolean` as 0/1.
|
||||
|
||||
@@ -1153,7 +1309,8 @@ Start, End, Delta and Minimum/MaximumActualTime; the time-weighted one TimeAvera
|
||||
PercentGood/Bad (#680), including datasets whose data ends inside the window and with an interval wholly inside an outage; reverse
|
||||
reads over data that starts inside the window are compared too. Known differences that remain (tracked as
|
||||
#680) — documented and masked in the tests, deliberately not changed here because forward results were to stay as they are — are:
|
||||
the SDK's per-sample Average/Minimum/Maximum use Good samples only, ours Good + Uncertain; the SDK sets Partial on Bad per-sample
|
||||
an Uncertain value beyond the Good extreme makes Minimum/Maximum/Range/*ActualTime Uncertain here (Part 13), Good in the SDK
|
||||
(#684 aligned the values — Good samples only — and every other Uncertain case); the SDK sets Partial on Bad per-sample
|
||||
results too, ours never (#680 closed the Start/End, Delta, Count and end-of-data Partial differences). Time-weighted differences (#680 closed
|
||||
the TimeAverage/Total status, `Total` coverage, end-of-data and PercentGood/Bad ones), each deliberate: ours flags a truncated interval
|
||||
Partial (Part 13 §5.3.3.2 Case 3), the SDK does not; ours carries the Calculated bit on a `BadNoData`; `TimeAverage2`/`Total2` over an
|
||||
@@ -1180,6 +1337,12 @@ bounding values back to the source integer type, so its `Interpolative` returns
|
||||
`TimeAverage`/`TimeAverage2`/`Total`/`Total2` differ from ours by the truncated bound values (~0.002–0.02 on the gate's data). Ours
|
||||
always computes and returns `Double` (as stated above for every aggregate but Count and Start/End), which is the exact Part 13
|
||||
value; the parity datasets are all `Double`, so this one is not masked there but recorded here.
|
||||
Two more, found by live gate 9 (both kept deliberately): **Minimum/MaximumActualTime carry the Calculated bit on every interval**
|
||||
(`0x…401`, `0x…405` with Partial) where the SDK returns the picked raw sample's own status (`0x…000`, `0x…404`) — the same value and
|
||||
timestamp, and the same treatment as the Min/Max-at-interval-start case above; masked in `SdkAggregateParityTests` (class remarks).
|
||||
And an interval holding a **Good String (or DateTime) sample** is `BadAggregateInvalidInputs` for Average/Minimum/Maximum/Range/
|
||||
*ActualTime here (the documented #684 refusal rule — a Good non-numeric value is not a number to aggregate), while the SDK skips the
|
||||
string and serves the numeric samples (e.g. Average 4 Good); an Uncertain non-numeric sample refuses nothing on either side.
|
||||
|
||||
**Pushdown trade-off.** Unlike the per-sample aggregates, the time-weighted family is **not** pushed down to SQL: the backend streams
|
||||
the window's rows plus the four bounds (one batch of five result sets on SQL Server; a bounds `UNION ALL` + a window select on QuestDB)
|
||||
@@ -1313,11 +1476,34 @@ makes HistoryUpdate Replace work; the writer's own insert-if-absent semantics ar
|
||||
|
||||
| Table | Key | Partitioning |
|
||||
|---|---|---|
|
||||
| `{p}sample` (`ts, tag, kind, num_value, int_value, str_value, status, deleted, server_ts`) | `(ts, tag)` | `PARTITION BY DAY` (TTL when configured) |
|
||||
| `{p}sample` (`ts, tag, kind, num_value, int_value, str_value, status, deleted, server_ts`; **`tag` is a `SYMBOL INDEX`**, #683) | `(ts, tag)` | `PARTITION BY DAY` (TTL when configured) |
|
||||
| `{p}event` (one-for-one with `hist.Event`) | `(ts, event_id)` | `PARTITION BY DAY` (TTL when configured) |
|
||||
| `{p}tag` (`ts, name, data_type, eng_unit, description, updated_utc`) | `(ts, name)` with `ts` fixed at `1970-01-01` — an upsert by name | `PARTITION BY YEAR`, never expires |
|
||||
| `{p}sample_modified` (`ts` = the sample's source time, `tag, modified_id, modification_ts, update_type, principal, api_key_id, kind, num_value, int_value, str_value, status, server_ts`) (#668) | none — append-only, no DEDUP; `modified_id` (process-monotonic ≈ UTC ticks) orders a timestamp's trail | `PARTITION BY DAY` (TTL when configured). A NULL `status` marks "no prior value" (QuestDB `BYTE` has no NULL, so `kind` is 0) |
|
||||
|
||||
**The `tag` index (#683).** Every read filters on one tag; without a symbol index QuestDB answers `tag = x` by scanning the tag
|
||||
column of every partition in the time range, which made the processed-read [data-range queries](#aggregates-processed-reads) of a stopped or late-starting
|
||||
tag scan every partition on one side of the window. A table created before #683 has no index: with `AutoCreateSchema=true` the
|
||||
bootstrapper adds it on first use (`ALTER TABLE {p}sample ALTER COLUMN tag ADD INDEX`, a WAL transaction; idempotent — re-sending it on an
|
||||
indexed column is harmless, so a pair booting together is safe — and a boot that finds the column indexed sends nothing). Building it is a
|
||||
one-off WAL apply, measured **~3 s for 68 M rows / 63 daily partitions**, during which writes queue and reads keep scanning. With
|
||||
`AutoCreateSchema=false` a missing index only logs a Warning naming that statement — reads stay correct, just slower. With
|
||||
`AutoCreateSchema=true` an ALTER refused up front (e.g. no DDL rights) is likewise only a Warning — the bootstrap still succeeds and the
|
||||
next boot retries; a failure at **WAL-apply** time is not observed by the bootstrapper (QuestDB would suspend the table, visible in
|
||||
`wal_tables()`) — the same blind spot as the TTL ALTER. Cost: ~12 % more disk (0.5 GB on a 4.2 GB 53 M-row table); no measurable change
|
||||
in live-append, O3 back-fill or bulk-load apply time beyond run-to-run noise (a first bulk load into an empty table measured ~40 %
|
||||
slower). Out-of-order writes (back-fill, HistoryUpdate, outbox redelivery) rewrite the index files of every partition they touch along
|
||||
with the column files, so their apply cost carries the index too. On a 120-tag table the per-sample / raw / time-weighted window queries
|
||||
were unchanged or slightly faster (the tag filter becomes an index scan), but **a tag holding a large share of the table can be ~20 %
|
||||
slower on long windows**: the index scan is single-threaded where the unindexed filter is a multi-threaded JIT scan (a review
|
||||
measurement: 4 tags / 30 M rows, 245–294 ms indexed vs 195–218 ms unindexed; with 1000 tags, 9 ms vs 13 ms — the index wins as the
|
||||
tag's share of each partition shrinks).
|
||||
|
||||
**Deliberately not indexed:** `{p}sample_modified.tag` — read only by `IsReadModified` HistoryRead over a window, an append-only trail a
|
||||
tiny fraction of the sample table's size, with no "nearest row outside the window" lookup to make it scan partitions; and
|
||||
`{p}event.source` — event reads are window-bounded (`ts` range, optionally `source = x`) and never seek beyond the window, and the event table is far
|
||||
smaller than the sample table. Neither has the unbounded-scan shape #683 fixes, so neither pays the index's disk and O3 cost.
|
||||
|
||||
`status` holds the uint OPC UA status code **unsigned** in a `LONG` (so "Bad" is `status >= 2147483648`, not `< 0`); `int_value` also holds
|
||||
Booleans as 0/1; `kind` is the `HistorianValueKind` in a `BYTE`. Tag names are limited to 400 characters (longer is a per-tag failure,
|
||||
never truncated). `{p}tag` is the **only** "is this tag known" authority (`QuestDbTagCatalog`): reads never create a tag, Insert/Update and the
|
||||
@@ -1413,8 +1599,8 @@ Every query filters `NOT deleted`; every value goes through `StoredSample`, so t
|
||||
later time, later-edge-inclusive, partial bucket earliest — #677; interval 0 ⇒ one bucket, 100,000-bucket cap). On a per-sample read of any
|
||||
aggregate but Average, in either direction, two `LIMIT 1` statements fetch the tag's data range — its earliest live sample at or before the
|
||||
window's later edge and its latest at or after its earlier edge — for the start/end-of-data Partial bits and Count's in-range empty interval
|
||||
(#677/#680). `tag` is an unindexed SYMBOL, so these are partition scans (a few ms on a live tag; tens of ms cold for a stopped tag on a
|
||||
large table — see *Data-range queries* above). Start/End/Delta also return each bucket's first and last sample of any quality (`row_number()` over every row).
|
||||
(#677/#680), read before the bucket statement. They seek the `tag` symbol index (#683; ~1–1.5 ms warm, 2–4 ms cold on a 53 M-row table
|
||||
whatever the tag's history — see *Data-range queries* above). Start/End/Delta also return each bucket's first and last sample of any quality (`row_number()` over every row).
|
||||
Min/max times use `row_number()` windows with the earliest-timestamp tie-break.
|
||||
⚠️ **Live-gate 8 F1 — counts and picks must come from one table version.** QuestDB opens a separate table reader per reference to a
|
||||
CTE, so the earlier Start/End/Delta form (counts CTE + self-joins) could count a row committed between two references without
|
||||
@@ -1422,8 +1608,49 @@ Every query filters `NOT deleted`; every value goes through `StoredSample`, so t
|
||||
query now reads the table **once**: one `GROUP BY` with `first()`/`last()` for the first/last sample of any quality and
|
||||
`arg_min`/`arg_max` keyed on the usable rows' time for the first/last usable one (~15 % faster than before; the usable picks carry
|
||||
no `str_value` — `arg_min` has no VARCHAR form, and only Delta reads them, never with a usable string in the bucket). On `SqlServer`
|
||||
(READ COMMITTED gives no cross-reference guarantee either, though it was never observed there) Start/End/Delta was reshaped the same
|
||||
way with a windowed pass + conditional `MAX` — 1.4–3× *faster* than the old five-reference form on a 1 M-row tag. Min/MaxActualTime
|
||||
(READ COMMITTED gives no cross-reference guarantee either, though it was never observed there) Start/End/Delta was first reshaped
|
||||
into a windowed pass + conditional `MAX`; **#683 replaced that** — its four `ROW_NUMBER` sorts spilled and it measured *slower* than
|
||||
the five-reference form on long windows. It is now **aggregate, then join back on the key**: one `GROUP BY` computes the counts and each
|
||||
bucket's four pick *times* (`MIN`/`MAX(SourceTimeUtc)`, any quality and usable), and each pick's columns are fetched by a seek on the
|
||||
clustered key `(TagId, SourceTimeUtc)` — **forced** with `WITH (FORCESEEK (PK_Sample (TagId, SourceTimeUtc)))`, so every pick is a
|
||||
nested-loops seek on both key columns, one row per bucket per pick, whatever the statistics say. The optimizer's own choices were
|
||||
both pathological, and plans are cached and reused across reads and tags: keyed on the pick time alone, a hash/merge join whose inner
|
||||
input was the tag's **whole history** (#683 review: 43,657 logical reads vs 361 for a 1 d @ 1 h read of a tag with 3 M rows outside the
|
||||
window); bounded to the window as well, a nested-loops **range** seek over the window with the pick time as a residual, once per
|
||||
bucket — O(buckets × window rows) — compiled whenever the statistics understate the table (a fresh or freshly bulk-loaded
|
||||
historian, ascending-key estimates for recent windows) and then reused for a wide read (**live gate 9 F1**: 1 d @ 1 min 20 s). With
|
||||
the hint neither shape exists: a forced `HASH`/`MERGE JOIN` is refused (error 8622). ⚠️ The hint names the primary key `PK_Sample`
|
||||
created by `001_initial.sql`; a hand-built schema must keep that name.
|
||||
|
||||
Output is identical to every earlier form (`EXCEPT` both ways empty, forward and reverse, including a tag of mixed Double/Int64/
|
||||
Boolean/String/DateTime/outage samples). Measured on 6 M rows / 30 days for one tag (a 1 d or 7 d window leaves up to 5.8 M rows of
|
||||
that tag outside it) plus an 86 k-row mixed-kinds tag; elapsed ms / session logical reads, each variant's plan compiled on its first
|
||||
read and reused by the rest:
|
||||
|
||||
| Window | before #683 (windowed) | #683 as merged, accurate stats | #683 as merged, understated stats ‡ | **now (FORCESEEK)**, accurate | **now**, understated ‡ |
|
||||
|---|---|---|---|---|---|
|
||||
| 1 d @ 1 min (1440 buckets) | 329 / 6.7 k | 121 / 14.9 k | **47,223 / 3.6 M** | 160 / 25.0 k | 253 / 25.6 k |
|
||||
| 1 d @ 1 min, reverse | 371 / 6.6 k | 129 / 14.9 k | **46,806 / 3.6 M** | 167 / 25.0 k | 246 / 25.6 k |
|
||||
| 1 d @ 1 h (24 buckets) | 578 / 5.4 k | 79 / 5.5 k | 984 / 63 k | 117 / 1.8 k | 206 / 3.6 k |
|
||||
| 7 d @ 1 min (10,080 buckets) | 3,426 / 103 k | 1,019 / 92 k | (hours — not run) | 1,472 / 180 k | 1,819 / 179 k |
|
||||
| 30 d @ 1 h (720 buckets) † | 35,339 / 322 k | 1,487 / 191 k | (hours — not run) | 5,720 / 73 k | 6,921 / 101 k |
|
||||
| mixed kinds 1 d @ 1 min | 190 / 5.9 k | 92 / 12.0 k | **20,164 / 1.66 M** | 107 / 24.4 k | 137 / 24.0 k |
|
||||
| mixed kinds 1 d @ 7 min, reverse | 199 / 1.5 k | 71 / 3.9 k | 2,977 / 238 k | 59 / 4.2 k | 93 / 4.7 k |
|
||||
|
||||
‡ `UPDATE STATISTICS … WITH ROWCOUNT = 34, PAGECOUNT = 1`, then the plan compiled on a 1 h @ 1 min read and reused — gate 9's
|
||||
recipe. † The 30 d row is dominated by the window `GROUP BY`, not the picks: compiled on the 30 d read itself, the merged query and
|
||||
the hinted one both take ~6.9 s (serial, a small memory grant, spills); the merged query's 1.5 s above is a plan compiled on the
|
||||
1 d read and reused. Alternatives measured and rejected: the same seek written as `OUTER APPLY (SELECT TOP (1) … WITH (FORCESEEK
|
||||
…))` (identical plans and numbers — the join form is shorter); `OPTION (RECOMPILE)` on the merged query (compiles under the bad
|
||||
statistics on every call: still 20.4 s / 1.66 M reads on the mixed-kinds tag, and a compile per read). `StartEndJoinBackTests`
|
||||
pins it — one sample a second over a day with 300 k rows of history either side: under understated statistics 18,645 reads
|
||||
(the merged query: **1,645,845 reads, 23 s**, so the test failed against it), with accurate statistics 22,616, and forced
|
||||
`HASH`/`MERGE` joins refused with 8622. (The dev SQL container is slow in absolute terms — the ratios are the point.)
|
||||
|
||||
⚠️ The seeks are separate reads of the table within one statement, so a sample **deleted** by HistoryUpdate between the aggregate
|
||||
and its seek leaves that pick NULL while the counts include it: the null-tolerant reader below answers that bucket's Start/End/Delta
|
||||
`BadNoData`, transiently. A usable pick's join also re-checks usability (the exact `u` predicate), so a HistoryUpdate **Replace**
|
||||
that turned the picked sample Bad/Empty in between reads the same way — never a Bad row's value under Delta. Min/MaxActualTime
|
||||
keep their counts + filtered-rank form on both backends: every single-scan variant measured 1.2–2.4× slower, and QuestDB's
|
||||
`arg_max` cannot express the earliest-time tie-break. So, on both backends, a row whose picks are absent anyway maps to "no such
|
||||
sample" — `BadNoData` for that interval, never an exception and never a value-less Good. `LiveWriteReadRaceStressTests` (QuestDB
|
||||
|
||||
@@ -55,6 +55,18 @@ public enum HistoryAggregateType
|
||||
Total2,
|
||||
}
|
||||
|
||||
/// <summary>Shared <see cref="HistoryAggregateType"/> sets for <see cref="IHistorianDataSource.SupportedAggregates"/> (#682).</summary>
|
||||
public static class HistoryAggregateTypes
|
||||
{
|
||||
/// <summary>Every defined <see cref="HistoryAggregateType"/> — the interface default.</summary>
|
||||
public static readonly IReadOnlySet<HistoryAggregateType> All =
|
||||
System.Collections.Frozen.FrozenSet.ToFrozenSet(Enum.GetValues<HistoryAggregateType>());
|
||||
|
||||
/// <summary>No aggregates — a source with no history to aggregate (the Null historian).</summary>
|
||||
public static readonly IReadOnlySet<HistoryAggregateType> None =
|
||||
System.Collections.Frozen.FrozenSet<HistoryAggregateType>.Empty;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Thrown by a history backend when it cannot serve the requested <see cref="HistoryAggregateType"/>.
|
||||
/// The Server HistoryRead path translates it to <c>BadAggregateNotSupported</c> for that node only.
|
||||
|
||||
@@ -174,6 +174,25 @@ public interface IHistorianDataSource : IDisposable
|
||||
/// </remarks>
|
||||
long TimestampResolutionTicks => 1;
|
||||
|
||||
/// <summary>
|
||||
/// The Part 13 aggregates this backend serves for a processed HistoryRead (Gitea #682) — the ONE authority for
|
||||
/// both what the server advertises under <c>HistoryServerCapabilities/AggregateFunctions</c> (i=11201) and what
|
||||
/// its HistoryRead-Processed answers <c>BadAggregateNotSupported</c> up front without calling the backend.
|
||||
/// (<c>ServerCapabilities/AggregateFunctions</c>, i=2997, is not affected: it lists every aggregate the server
|
||||
/// computes, including for live <c>AggregateFilter</c> subscriptions.)
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The default is <see cref="HistoryAggregateTypes.All"/> — "every aggregate the server can name; a member I cannot
|
||||
/// serve throws <see cref="HistoryAggregateNotSupportedException"/> at read time", which is exactly the pre-#682
|
||||
/// behaviour, so a test fake or an out-of-tree source keeps working unchanged. It is a concrete set rather than a
|
||||
/// <c>null</c> "all" sentinel so every consumer handles one shape. Every PRODUCTION backend declares its set itself
|
||||
/// (a reflection guard pins that): a wrapper that forgot to forward this would otherwise advertise every aggregate.
|
||||
/// The set is read once when the source is wired (the backend is fixed at boot by <c>ServerHistorian:Backend</c>)
|
||||
/// and again on every processed read, so it must be cheap and stable — return a static frozen set. A source that
|
||||
/// returns <c>null</c> anyway is treated as the default (all) by both consumers.
|
||||
/// </remarks>
|
||||
IReadOnlySet<HistoryAggregateType> SupportedAggregates => HistoryAggregateTypes.All;
|
||||
|
||||
/// <summary>
|
||||
/// Read modified-value history — OPC UA HistoryReadRaw with <c>IsReadModified = true</c> (Part 11, Gitea #668):
|
||||
/// every modification a HistoryUpdate applied to the tag inside the window, each carrying the PRIOR value.
|
||||
|
||||
@@ -74,6 +74,15 @@ public sealed class NullHistorianDataSource : IHistorianDataSource
|
||||
? throw new NotSupportedException("The null historian cannot read events newest-first (reverse).")
|
||||
: Task.FromResult(EmptyEvents);
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>
|
||||
/// EMPTY (Gitea #682): with no historian configured the server advertises no aggregate functions, and a processed
|
||||
/// HistoryRead answers <c>BadAggregateNotSupported</c> per node rather than <c>GoodNoData</c> — an aggregate is a
|
||||
/// statistic the server cannot compute without a history, and advertising the SDK's full list on a server with no
|
||||
/// historian misled capability discovery. Raw / AtTime / Events reads still return the empty <c>GoodNoData</c> result.
|
||||
/// </remarks>
|
||||
public IReadOnlySet<HistoryAggregateType> SupportedAggregates => HistoryAggregateTypes.None;
|
||||
|
||||
/// <inheritdoc />
|
||||
public HistorianHealthSnapshot GetHealthSnapshot() => new(
|
||||
TotalQueries: 0,
|
||||
|
||||
+117
-3
@@ -1,7 +1,10 @@
|
||||
using System.Collections.Frozen;
|
||||
using Grpc.Core;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using ZB.MOM.WW.HistorianGateway.Contracts.Grpc;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Mapping;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway;
|
||||
|
||||
@@ -23,8 +26,26 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway;
|
||||
/// </remarks>
|
||||
public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDisposable
|
||||
{
|
||||
/// <summary>
|
||||
/// The processed (aggregate) functions the Gateway backend serves — the single source of truth for this backend's
|
||||
/// aggregate set (#681): the per-sample aggregates evaluated from raw reads (every member of the
|
||||
/// <see cref="AggregateFamily.Simple"/> / <see cref="AggregateFamily.StartEnd"/> /
|
||||
/// <see cref="AggregateFamily.ActualTime"/> families of <see cref="AggregateFamilies"/>: Average, Count, Minimum,
|
||||
/// Maximum, Range, Start, End, Delta, MinimumActualTime, MaximumActualTime) plus the two answered natively by AVEVA
|
||||
/// (TimeAverage, Total — <see cref="AggregateModeMapper"/>). Any other aggregate throws
|
||||
/// <see cref="HistoryAggregateNotSupportedException"/> without calling the gateway. It is what
|
||||
/// <see cref="SupportedAggregates"/> returns, so the advertised <c>AggregateFunctions</c> folders and the server's
|
||||
/// up-front <c>BadAggregateNotSupported</c> read it too (#682).
|
||||
/// </summary>
|
||||
public static IReadOnlySet<HistoryAggregateType> ServedAggregates { get; } =
|
||||
Enum.GetValues<HistoryAggregateType>()
|
||||
.Where(a => RawAggregateReader.Serves(a) || AggregateModeMapper.NativeAggregates.Contains(a))
|
||||
.ToFrozenSet();
|
||||
|
||||
private readonly IHistorianGatewayClient _client;
|
||||
private readonly ILogger<GatewayHistorianDataSource> _logger;
|
||||
private readonly int _maxRawRows;
|
||||
private readonly TimeProvider _time;
|
||||
|
||||
private readonly object _healthLock = new();
|
||||
private DateTime? _lastSuccessUtc;
|
||||
@@ -38,12 +59,20 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
/// <summary>Creates a gateway-backed historian data source.</summary>
|
||||
/// <param name="client">The gateway client seam used for all reads.</param>
|
||||
/// <param name="logger">Diagnostic logger; failures are recorded without leaking tag/host detail.</param>
|
||||
public GatewayHistorianDataSource(IHistorianGatewayClient client, ILogger<GatewayHistorianDataSource> logger)
|
||||
/// <param name="maxRawRows">
|
||||
/// In-window raw-row cap of one raw-evaluated processed read (<c>ServerHistorian:MaxTimeWeightedRawRows</c>;
|
||||
/// <= 0 = unbounded). Past it the node fails <c>BadTooManyOperations</c>.
|
||||
/// </param>
|
||||
/// <param name="timeProvider">Clock for result server timestamps and the raw look-ahead horizon; null = system.</param>
|
||||
public GatewayHistorianDataSource(IHistorianGatewayClient client, ILogger<GatewayHistorianDataSource> logger,
|
||||
int maxRawRows = TimeWeightedAggregator.DefaultMaxWindowRows, TimeProvider? timeProvider = null)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(client);
|
||||
ArgumentNullException.ThrowIfNull(logger);
|
||||
_client = client;
|
||||
_logger = logger;
|
||||
_maxRawRows = maxRawRows;
|
||||
_time = timeProvider ?? TimeProvider.System;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
@@ -75,17 +104,38 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>Per-sample aggregates (#681)</b> — Average, Count, Minimum, Maximum, Range, Start, End, Delta,
|
||||
/// Minimum/MaximumActualTime — are evaluated from a raw read with the SQL backends' shared evaluator
|
||||
/// (<see cref="RawAggregateReader"/>): identical values, status, Calculated/Partial bits and timestamps to
|
||||
/// SqlServer/QuestDB for the same samples, forward and reverse (#677). The in-window raw rows are capped by
|
||||
/// <c>ServerHistorian:MaxTimeWeightedRawRows</c> (<see cref="HistoryReadLimitExceededException"/>, not counted as
|
||||
/// a backend failure).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>TimeAverage / Total</b> stay native AVEVA retrieval modes (<see cref="AggregateModeMapper"/>), stamped with
|
||||
/// the interval start and the Calculated bit. Everything else is <see cref="HistoryAggregateNotSupportedException"/>
|
||||
/// (see <see cref="ServedAggregates"/>).
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
/// <exception cref="ArgumentOutOfRangeException">A raw-evaluated read whose window holds more than
|
||||
/// <see cref="BucketPlan.MaxBuckets"/> intervals.</exception>
|
||||
public async Task<HistoryReadResult> ReadProcessedAsync(
|
||||
string fullReference, DateTime startUtc, DateTime endUtc, TimeSpan interval,
|
||||
HistoryAggregateType aggregate, CancellationToken cancellationToken)
|
||||
{
|
||||
if (RawAggregateReader.Serves(aggregate))
|
||||
return await ReadProcessedFromRawAsync(fullReference, startUtc, endUtc, interval, aggregate, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
// Mapped OUTSIDE the failure-recording try: an aggregate the gateway has no mode for is a client
|
||||
// request the server answers BadAggregateNotSupported, not a backend fault to count (#285).
|
||||
var mode = AggregateModeMapper.ToRetrievalMode(aggregate);
|
||||
try
|
||||
{
|
||||
// Total is a native gateway retrieval mode (Integral) — no client-side scaling. Average and Count
|
||||
// have no AVEVA mode and were rejected by the mapper above (#679).
|
||||
// TimeAverage / Total are native gateway retrieval modes (TimeWeightedAverage / Integral) — no
|
||||
// client-side scaling.
|
||||
var buckets = new List<HistorianAggregateSample>();
|
||||
await foreach (var bucket in _client
|
||||
.ReadAggregateAsync(fullReference, startUtc, endUtc, mode, interval, cancellationToken)
|
||||
@@ -98,6 +148,10 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
RecordOutcome(success: true, error: null);
|
||||
return new HistoryReadResult(snapshots, ContinuationPoint: null);
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
throw; // caller-requested, not a backend fault
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
RecordReadFailure(ex);
|
||||
@@ -105,6 +159,62 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Serves a per-sample aggregate from a raw read (#681); see <see cref="RawAggregateReader"/>.</summary>
|
||||
private async Task<HistoryReadResult> ReadProcessedFromRawAsync(
|
||||
string fullReference, DateTime startUtc, DateTime endUtc, TimeSpan interval,
|
||||
HistoryAggregateType aggregate, CancellationToken cancellationToken)
|
||||
{
|
||||
// Planned OUTSIDE the failure-recording try (as on the SQL backends): too many intervals is a client request
|
||||
// limit, not a backend fault.
|
||||
var plan = BucketPlan.Create(startUtc, endUtc, interval);
|
||||
try
|
||||
{
|
||||
var now = _time.GetUtcNow().UtcDateTime;
|
||||
var values = await RawAggregateReader
|
||||
.ReadAsync(_client, fullReference, aggregate, plan, _maxRawRows, now, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
RecordOutcome(success: true, error: null);
|
||||
return new HistoryReadResult(values, ContinuationPoint: null);
|
||||
}
|
||||
catch (HistoryReadLimitExceededException)
|
||||
{
|
||||
// A client-request limit (the raw-row cap), not a backend fault: not counted as a failure.
|
||||
throw;
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
// The caller (client cancel / HistoryReadDeadline) asked for this — not a backend fault.
|
||||
throw;
|
||||
}
|
||||
catch (Exception ex) when (IsMaxValuesCeilingRejection(ex))
|
||||
{
|
||||
// The gateway refused the row budget (Historian:ReadLimits:MaxValuesCeiling with
|
||||
// AllowStreamingBeyondCeiling=false): the same client-request limit as the cap, not a backend fault.
|
||||
throw new HistoryReadLimitExceededException(_maxRawRows,
|
||||
$"The HistorianGateway refused the raw row budget of the processed read of '{fullReference}' (its " +
|
||||
"Historian:ReadLimits:MaxValuesCeiling); lower ServerHistorian:MaxTimeWeightedRawRows below that ceiling " +
|
||||
"or narrow the window.");
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
RecordReadFailure(ex);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// True when <paramref name="ex"/> (or an inner exception — the client package wraps the <see cref="RpcException"/>)
|
||||
/// is the gateway's <c>InvalidArgument</c> "max_values exceeds the server ceiling" rejection.
|
||||
/// </summary>
|
||||
internal static bool IsMaxValuesCeilingRejection(Exception ex)
|
||||
{
|
||||
for (var e = ex; e is not null; e = e.InnerException)
|
||||
if (e is RpcException { StatusCode: StatusCode.InvalidArgument } rpc
|
||||
&& rpc.Status.Detail.Contains("max_values exceeds the server ceiling", StringComparison.Ordinal))
|
||||
return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<HistoryReadResult> ReadAtTimeAsync(
|
||||
string fullReference, IReadOnlyList<DateTime> timestampsUtc, CancellationToken cancellationToken)
|
||||
@@ -132,6 +242,10 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
/// </remarks>
|
||||
public bool SupportsReverseEvents => false;
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks><see cref="ServedAggregates"/>; every other member answers <c>BadAggregateNotSupported</c> and is not advertised.</remarks>
|
||||
public IReadOnlySet<HistoryAggregateType> SupportedAggregates => ServedAggregates;
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>
|
||||
/// A reverse (newest-first) read is refused with <see cref="NotSupportedException"/> — the server reports
|
||||
|
||||
+3
-1
@@ -38,7 +38,9 @@ public static class GatewayHistorian
|
||||
|
||||
return new GatewayHistorianDataSource(
|
||||
HistorianGatewayClientAdapter.Create(options, loggerFactory),
|
||||
logger);
|
||||
logger,
|
||||
// #681: the raw-evaluated per-sample aggregates share the SQL backends' raw-row cap.
|
||||
maxRawRows: options.MaxTimeWeightedRawRows);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
||||
+35
-16
@@ -1,3 +1,4 @@
|
||||
using System.Collections.Frozen;
|
||||
using ZB.MOM.WW.HistorianGateway.Contracts.Grpc;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
|
||||
@@ -5,39 +6,57 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Mapping;
|
||||
|
||||
/// <summary>
|
||||
/// Maps the driver-agnostic <see cref="HistoryAggregateType"/> (OPC UA Part 13 aggregate) onto the
|
||||
/// gateway's native <see cref="RetrievalMode"/>.
|
||||
/// gateway's native <see cref="RetrievalMode"/> — for the aggregates still answered by AVEVA itself.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// Served: <see cref="HistoryAggregateType.TimeAverage"/> → <see cref="RetrievalMode.TimeWeightedAverage"/>,
|
||||
/// Minimum → <see cref="RetrievalMode.MinimumWithTime"/>, Maximum → <see cref="RetrievalMode.MaximumWithTime"/>,
|
||||
/// Total → <see cref="RetrievalMode.Integral"/>.
|
||||
/// Native (AVEVA-computed): <see cref="HistoryAggregateType.TimeAverage"/> → <see cref="RetrievalMode.TimeWeightedAverage"/>
|
||||
/// and <see cref="HistoryAggregateType.Total"/> → <see cref="RetrievalMode.Integral"/>. They stay native because an exact
|
||||
/// Part 13 time-weighted evaluation needs the real bounding samples on both sides of the window (time, value and
|
||||
/// quality of the latest sample before it), and the gateway has no "previous stored sample" read — a raw read streams
|
||||
/// forward from its start and AVEVA replaces whatever lies before with an initial-value row stamped at the query start,
|
||||
/// losing the original time a sloped segment across the window start needs (#681).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Deliberately NOT served (#679): <see cref="HistoryAggregateType.Average"/> (OPC UA Average is the arithmetic mean;
|
||||
/// AVEVA's modes include no arithmetic mean — only the time-weighted one, which is TimeAverage) and
|
||||
/// <see cref="HistoryAggregateType.Count"/> (AVEVA <c>Counter</c> is a rollover-counter increase, not a sample count).
|
||||
/// Both surface as <c>BadAggregateNotSupported</c> rather than returning a different statistic under the requested name.
|
||||
/// Since #681 the per-sample aggregates (Average, Count, Minimum, Maximum, Range, Start, End, Delta,
|
||||
/// Minimum/MaximumActualTime) are NOT native: <see cref="GatewayHistorianDataSource"/> evaluates them from raw reads
|
||||
/// with the SQL backends' shared evaluator (Minimum/Maximum used to map to AVEVA's <c>MinimumWithTime</c> /
|
||||
/// <c>MaximumWithTime</c>). Every other member — incl. TimeAverage2, Total2, Interpolative, PercentGood, PercentBad — is
|
||||
/// <see cref="HistoryAggregateNotSupportedException"/> here.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
internal static class AggregateModeMapper
|
||||
{
|
||||
/// <summary>
|
||||
/// The aggregates answered by a native AVEVA retrieval mode — every defined member <see cref="TryMap"/> maps, derived
|
||||
/// from it so the dispatch and the declared set can never disagree. Part of
|
||||
/// <see cref="GatewayHistorianDataSource.ServedAggregates"/> (#681/#682).
|
||||
/// </summary>
|
||||
public static readonly FrozenSet<HistoryAggregateType> NativeAggregates =
|
||||
Enum.GetValues<HistoryAggregateType>().Where(a => TryMap(a) is not null).ToFrozenSet();
|
||||
|
||||
/// <summary>Maps an aggregate function to the gateway retrieval mode.</summary>
|
||||
/// <param name="aggregate">The driver-agnostic aggregate function.</param>
|
||||
/// <returns>The matching gateway <see cref="RetrievalMode"/>.</returns>
|
||||
/// <exception cref="HistoryAggregateNotSupportedException">A defined member the gateway has no retrieval mode for.</exception>
|
||||
/// <exception cref="ArgumentOutOfRangeException">An undefined enum value.</exception>
|
||||
public static RetrievalMode ToRetrievalMode(HistoryAggregateType aggregate) => aggregate switch
|
||||
public static RetrievalMode ToRetrievalMode(HistoryAggregateType aggregate)
|
||||
{
|
||||
HistoryAggregateType.TimeAverage => RetrievalMode.TimeWeightedAverage,
|
||||
HistoryAggregateType.Minimum => RetrievalMode.MinimumWithTime,
|
||||
HistoryAggregateType.Maximum => RetrievalMode.MaximumWithTime,
|
||||
HistoryAggregateType.Total => RetrievalMode.Integral,
|
||||
if (TryMap(aggregate) is { } mode) return mode;
|
||||
|
||||
// Part 13 aggregates the gateway has no retrieval mode for (#285) — an UNDEFINED enum value is a
|
||||
// programming error, but a defined-yet-unserved member is a normal client request the server
|
||||
// answers with BadAggregateNotSupported.
|
||||
_ when Enum.IsDefined(aggregate) => throw new HistoryAggregateNotSupportedException(aggregate),
|
||||
_ => throw new ArgumentOutOfRangeException(
|
||||
nameof(aggregate), aggregate, "Unmapped HistoryAggregateType — add a RetrievalMode mapping."),
|
||||
if (Enum.IsDefined(aggregate)) throw new HistoryAggregateNotSupportedException(aggregate);
|
||||
throw new ArgumentOutOfRangeException(
|
||||
nameof(aggregate), aggregate, "Unmapped HistoryAggregateType — add a RetrievalMode mapping.");
|
||||
}
|
||||
|
||||
/// <summary>The single mapping table: the retrieval mode for a natively served aggregate, or null.</summary>
|
||||
private static RetrievalMode? TryMap(HistoryAggregateType aggregate) => aggregate switch
|
||||
{
|
||||
HistoryAggregateType.TimeAverage => RetrievalMode.TimeWeightedAverage,
|
||||
HistoryAggregateType.Total => RetrievalMode.Integral,
|
||||
_ => null,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
using ZB.MOM.WW.HistorianGateway.Contracts.Grpc;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Mapping;
|
||||
|
||||
@@ -30,16 +31,23 @@ internal static class SampleMapper
|
||||
else
|
||||
value = null;
|
||||
|
||||
return new DataValueSnapshot(
|
||||
Value: value,
|
||||
StatusCode: ToStatus(sample),
|
||||
SourceTimestampUtc: sample.Timestamp?.ToDateTime(), // Utc kind
|
||||
ServerTimestampUtc: DateTime.UtcNow);
|
||||
}
|
||||
|
||||
/// <summary>The OPC UA status of a raw sample (shared by raw reads and the #681 raw-evaluated aggregates).</summary>
|
||||
/// <param name="sample">The gateway raw sample.</param>
|
||||
/// <returns>The mapped status code.</returns>
|
||||
public static uint ToStatus(HistorianSample sample)
|
||||
{
|
||||
// Prefer the OPC DA quality byte (opc_quality); the gateway populates it directly from the
|
||||
// SDK's OpcQuality, so it is the authoritative byte for GatewayQualityMapper. Fall back to
|
||||
// the historian quality field only when opc_quality is unset (0).
|
||||
byte qualityByte = sample.OpcQuality != 0 ? (byte)sample.OpcQuality : (byte)sample.Quality;
|
||||
|
||||
return new DataValueSnapshot(
|
||||
Value: value,
|
||||
StatusCode: GatewayQualityMapper.Map(qualityByte),
|
||||
SourceTimestampUtc: sample.Timestamp?.ToDateTime(), // Utc kind
|
||||
ServerTimestampUtc: DateTime.UtcNow);
|
||||
return GatewayQualityMapper.Map(qualityByte);
|
||||
}
|
||||
|
||||
/// <summary>Maps a batch of raw samples to value snapshots, in order.</summary>
|
||||
@@ -57,20 +65,28 @@ internal static class SampleMapper
|
||||
/// <param name="aggregate">The gateway aggregate sample.</param>
|
||||
/// <returns>The driver-agnostic snapshot.</returns>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// Unlike the legacy Wonderware DTO (a nullable <c>double?</c>), the gateway proto carries a
|
||||
/// non-optional <c>double value</c>, so an unavailable (no-data) bucket cannot be signalled by a
|
||||
/// null value. Instead it is signalled by a non-Good <c>opc_quality</c>: a Good bucket
|
||||
/// (<c>opc_quality >= 192</c>) yields its value with a Good status, anything else maps to
|
||||
/// <c>BadNoData</c> with a null value — preserving the Wonderware aggregate contract (binary
|
||||
/// Good-with-value / BadNoData-null).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// #681: both statuses carry the Calculated info bits (<c>0x401</c>) like every other backend's processed value, and
|
||||
/// the bucket is stamped with its interval START (Part 13; it was the interval end), falling back to the end only when
|
||||
/// the start is absent. Pending the live leg: histsdk decodes <c>start_time</c> from a row field (<c>tail + 43</c>) its
|
||||
/// reverse-engineering notes still mark unverified.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public static DataValueSnapshot ToAggregateSnapshot(HistorianAggregateSample aggregate)
|
||||
{
|
||||
bool hasData = aggregate.OpcQuality >= GoodQualityFloor;
|
||||
return new DataValueSnapshot(
|
||||
Value: hasData ? aggregate.Value : null, // boxes as System.Double when present
|
||||
StatusCode: hasData ? StatusGood : StatusBadNoData,
|
||||
SourceTimestampUtc: (aggregate.EndTime ?? aggregate.StartTime)?.ToDateTime(), // bucket timestamp
|
||||
StatusCode: HistorianInfoBits.WithCalculated(hasData ? StatusGood : StatusBadNoData),
|
||||
SourceTimestampUtc: (aggregate.StartTime ?? aggregate.EndTime)?.ToDateTime(), // bucket timestamp = interval start
|
||||
ServerTimestampUtc: DateTime.UtcNow);
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,312 @@
|
||||
using ZB.MOM.WW.HistorianGateway.Contracts.Grpc;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Mapping;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway;
|
||||
|
||||
/// <summary>
|
||||
/// #681: serves the per-sample processed aggregates (the <see cref="AggregateFamily.Simple"/>,
|
||||
/// <see cref="AggregateFamily.StartEnd"/> and <see cref="AggregateFamily.ActualTime"/> families) on the Gateway backend by
|
||||
/// streaming the window's RAW samples through the gateway and evaluating them with the shared
|
||||
/// <c>Driver.Historian.Sql.Common</c> code — <see cref="BucketPlan"/> for the bucket math (forward and reverse, #677) and
|
||||
/// <see cref="AggregateEvaluator.EvaluateAll"/> for values, status and the Partial/Calculated bits — so the Gateway answers
|
||||
/// exactly what SqlServer and QuestDB answer for the same stored samples.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>Forward raw reads, each consumed to completion.</b> The gateway's <c>ReadRaw</c> streams oldest-first over an
|
||||
/// inclusive <c>[start, end]</c> and refuses <c>end < start</c>, so a reverse read streams the same ascending range
|
||||
/// and <see cref="BucketPlan"/> assembles the result newest-first. The window stream is classified sample by sample
|
||||
/// with <see cref="BucketPlan.IndexOf"/> (<c>[start, end)</c> forward, <c>(start, end]</c> reverse) — memory is
|
||||
/// O(buckets), never O(samples). No gateway stream is ever abandoned mid-way: a client-side break cancels the call,
|
||||
/// and the gateway currently classifies that cancellation as a connection fault and evicts its shared historian
|
||||
/// session (HistorianGateway <c>HistorianRpcFaults.IsConnectionFault</c>: any <c>RpcException</c> ⇒ true).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>The AVEVA start-boundary row.</b> An AVEVA <c>Full</c> raw read emits an initial-value row stamped exactly at
|
||||
/// the query start (histsdk reverse-engineering: <c>Quality = 133</c>, <c>OpcQuality = 192</c>) carrying the value
|
||||
/// in effect there, not a stored sample. The window stream is therefore requested from <see cref="BoundaryMargin"/>
|
||||
/// (1 ms) BEFORE the window, so that row lands outside every bucket and can never be counted as a sample. A row
|
||||
/// before the window counts only as evidence that the tag has earlier data, and only when its OPC quality is not
|
||||
/// Bad — a "no earlier data" NULL row still arrives WITH a value (histsdk sets <c>NumericValue = 0</c> on every
|
||||
/// numeric row, <c>StringValue = ""</c> on a string one) but with a Bad OPC quality (captured: 1, no-data).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>The data range.</b> The start/end-of-data Partial rules and Count's in-range empty interval need the tag's
|
||||
/// earliest sample at or before the window's later edge and latest at or after its earlier edge
|
||||
/// (<see cref="AggregateEvaluator.NeedsDataRange"/>; a plain <c>Average</c> needs neither). Earlier data is a non-Bad
|
||||
/// pre-window row, or — when none arrived — a <see cref="ProbeRows"/>-row read <c>[</c><see cref="ProbeFloor"/><c>,
|
||||
/// start)</c>. Later data is a sample exactly at the window end (forward) from the window read, or else a
|
||||
/// <see cref="ProbeRows"/>-row read <c>[end, now + </c><see cref="LookAheadHorizon"/><c>]</c> (the historian stores
|
||||
/// nothing further in the future). In both small reads a row stamped exactly at the query start (a boundary row) or
|
||||
/// at the query end (a possible terminal row) is not a stored sample, whatever its value.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Bounded.</b> The in-window rows are capped by <c>ServerHistorian:MaxTimeWeightedRawRows</c> (the same knob the
|
||||
/// SQL backends' time-weighted reads use): past it the node fails with
|
||||
/// <see cref="HistoryReadLimitExceededException"/> (⇒ <c>BadTooManyOperations</c>) once the stream — bounded by the
|
||||
/// requested row budget — has been drained. The gateway is asked for at most <c>cap + 2 + </c>
|
||||
/// <see cref="PreWindowAllowance"/> rows; a stream that delivers all of them cannot be proven complete and fails the
|
||||
/// same way rather than answering from a truncated window.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Values and status.</b> A sample's value is its numeric value (Double), else its string value, else Empty; its
|
||||
/// status is <see cref="SampleMapper.ToStatus"/> — the same quality mapping a raw HistoryRead serves (C-002: the
|
||||
/// gateway stores no quality, so AVEVA history is normally all Good). An Empty sample is clamped Bad
|
||||
/// (<see cref="StoredSample.FromStored"/>), exactly as on the SQL backends.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
internal static class RawAggregateReader
|
||||
{
|
||||
/// <summary>How far before the window the raw stream starts, so the AVEVA initial-value row lands outside it.</summary>
|
||||
internal static readonly TimeSpan BoundaryMargin = TimeSpan.FromMilliseconds(1);
|
||||
|
||||
/// <summary>How far past "now" the stream may look for a sample after the window (end-of-data rules).</summary>
|
||||
internal static readonly TimeSpan LookAheadHorizon = TimeSpan.FromDays(1);
|
||||
|
||||
/// <summary>Rows the stream may spend before the window (the boundary row plus samples inside the margin).</summary>
|
||||
internal const int PreWindowAllowance = 16;
|
||||
|
||||
/// <summary>Start of the "is there data before the window?" probe (the Unix epoch — before any AVEVA history).</summary>
|
||||
internal static readonly DateTime ProbeFloor = DateTime.UnixEpoch;
|
||||
|
||||
/// <summary>Rows the earlier-data probe asks for (a possible boundary row plus the first stored sample).</summary>
|
||||
internal const int ProbeRows = 2;
|
||||
|
||||
/// <summary>True for the aggregates this reader serves: the per-sample families of <see cref="AggregateFamilies"/>.</summary>
|
||||
/// <param name="aggregate">Requested aggregate.</param>
|
||||
/// <returns>Whether the aggregate is evaluated from raw samples.</returns>
|
||||
public static bool Serves(HistoryAggregateType aggregate) =>
|
||||
AggregateFamilies.FamilyOf(aggregate) is AggregateFamily.Simple or AggregateFamily.StartEnd or AggregateFamily.ActualTime;
|
||||
|
||||
/// <summary>The gateway <c>max_values</c> for a configured window cap (<see cref="int.MaxValue"/> when unbounded).</summary>
|
||||
/// <param name="maxWindowRows">The configured cap (<= 0 = unbounded).</param>
|
||||
/// <returns>The row limit to request.</returns>
|
||||
public static int MaxValuesFor(int maxWindowRows) =>
|
||||
maxWindowRows <= 0 ? int.MaxValue : (int)Math.Min(int.MaxValue, (long)maxWindowRows + 2 + PreWindowAllowance);
|
||||
|
||||
/// <summary>Maps one gateway raw sample to a stored sample (value + mapped status, Empty clamped Bad).</summary>
|
||||
/// <param name="sample">The gateway sample; its timestamp must be set.</param>
|
||||
/// <returns>The stored sample.</returns>
|
||||
public static StoredSample ToStored(HistorianSample sample)
|
||||
{
|
||||
var value = sample.HasNumericValue ? HistorianValue.FromDouble(sample.NumericValue)
|
||||
: sample.HasStringValue ? HistorianValue.FromString(sample.StringValue)
|
||||
: HistorianValue.Empty;
|
||||
return StoredSample.FromStored(sample.Timestamp.ToDateTime(), value, SampleMapper.ToStatus(sample));
|
||||
}
|
||||
|
||||
/// <summary>Reads and evaluates one per-sample processed read.</summary>
|
||||
/// <param name="client">The gateway client.</param>
|
||||
/// <param name="tag">Historian tag name.</param>
|
||||
/// <param name="aggregate">A per-sample aggregate (<see cref="Serves"/>).</param>
|
||||
/// <param name="plan">The bucket plan.</param>
|
||||
/// <param name="maxWindowRows">The in-window raw-row cap (<= 0 = unbounded).</param>
|
||||
/// <param name="now">Server timestamp, and the anchor of the look-ahead horizon.</param>
|
||||
/// <param name="ct">Cancellation token.</param>
|
||||
/// <returns>One value per interval, in the plan's order.</returns>
|
||||
/// <exception cref="HistoryReadLimitExceededException">The window holds more than <paramref name="maxWindowRows"/> rows,
|
||||
/// or the stream could not be read to completion within the requested row budget.</exception>
|
||||
public static async Task<IReadOnlyList<DataValueSnapshot>> ReadAsync(IHistorianGatewayClient client, string tag,
|
||||
HistoryAggregateType aggregate, BucketPlan plan, int maxWindowRows, DateTime now, CancellationToken ct)
|
||||
{
|
||||
if (plan.BucketCount == 0) return [];
|
||||
var needsRange = AggregateEvaluator.NeedsDataRange(aggregate, plan);
|
||||
var requestStart = plan.StartUtc.Ticks > BoundaryMargin.Ticks ? plan.StartUtc - BoundaryMargin : plan.StartUtc;
|
||||
var maxValues = MaxValuesFor(maxWindowRows);
|
||||
|
||||
var buckets = new RawBucketAccumulator(aggregate == HistoryAggregateType.MaximumActualTime);
|
||||
DateTime? earliest = null, latest = null;
|
||||
long windowRows = 0, received = 0;
|
||||
var overCap = false;
|
||||
// Consumed to COMPLETION (never broken out of): abandoning a gateway stream cancels the call server-side, and the
|
||||
// gateway currently treats that cancellation as a connection fault and evicts its shared historian session. Past
|
||||
// the cap the rest is drained (bounded by maxValues) without being evaluated.
|
||||
await foreach (var raw in client.ReadRawAsync(tag, requestStart, plan.EndUtc, maxValues, ct).ConfigureAwait(false))
|
||||
{
|
||||
received++;
|
||||
if (overCap || raw.Timestamp is null) continue;
|
||||
var s = ToStored(raw);
|
||||
var t = s.TimeUtc;
|
||||
if (plan.IndexOf(t) is { } bucket)
|
||||
{
|
||||
if (maxWindowRows > 0 && ++windowRows > maxWindowRows)
|
||||
{
|
||||
overCap = true;
|
||||
continue;
|
||||
}
|
||||
buckets.Add(bucket, s);
|
||||
}
|
||||
// The data range as the SQL backends define it: earliest stored sample at or before the window's later edge,
|
||||
// latest at or after its earlier edge. A row before the window (the AVEVA boundary row, or a stored sample in
|
||||
// the margin) proves earlier data only when its OPC quality is not Bad: AVEVA's boundary row for a tag with no
|
||||
// earlier data is a NULL row, which arrives WITH a value (histsdk sets NumericValue = 0 / StringValue "") and
|
||||
// a Bad OPC quality (captured: 1, no-data).
|
||||
if (t >= plan.StartUtc || IsNotBadOnTheWire(raw))
|
||||
{
|
||||
if (t <= plan.EndUtc && (earliest is null || t < earliest)) earliest = t;
|
||||
if (t >= plan.StartUtc && (latest is null || t > latest)) latest = t;
|
||||
}
|
||||
}
|
||||
if (overCap || (maxWindowRows > 0 && received >= maxValues))
|
||||
throw LimitExceeded(tag, maxWindowRows);
|
||||
|
||||
if (needsRange)
|
||||
{
|
||||
// Earlier data: no non-Bad pre-window row ⇒ ask explicitly (normally only when the tag's history really starts
|
||||
// inside the window).
|
||||
var earlierSeen = earliest is { } e && (e < plan.StartUtc || (plan.Reversed && e == plan.StartUtc));
|
||||
if (!earlierSeen && plan.StartUtc.AddTicks(-1) > ProbeFloor)
|
||||
if (await FirstSampleAsync(client, tag, ProbeFloor, plan.StartUtc.AddTicks(-1), ct).ConfigureAwait(false) is { } before)
|
||||
earliest = before;
|
||||
|
||||
// Later data: the main read covers a sample exactly at the window end (forward: that is already later data);
|
||||
// otherwise a small separate read from the end to now + horizon.
|
||||
var laterSeen = latest is { } l && (plan.Reversed ? l > plan.EndUtc : l >= plan.EndUtc);
|
||||
var horizon = DateTime.MaxValue - now > LookAheadHorizon
|
||||
? DateTime.SpecifyKind(now + LookAheadHorizon, DateTimeKind.Utc)
|
||||
: DateTime.MaxValue;
|
||||
if (!laterSeen && horizon > plan.EndUtc)
|
||||
if (await FirstSampleAsync(client, tag, plan.EndUtc, horizon, ct).ConfigureAwait(false) is { } after)
|
||||
latest = after;
|
||||
}
|
||||
|
||||
return AggregateEvaluator.EvaluateAll(aggregate, plan, buckets.Rows(), earliest, latest, now);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The time of the first stored sample (any quality) strictly between <paramref name="startUtc"/> and
|
||||
/// <paramref name="endUtc"/>, read as a <see cref="ProbeRows"/>-row raw read consumed to completion. A row stamped
|
||||
/// exactly at the query start is the historian's boundary row (the value in effect there, or a NULL row when there is
|
||||
/// none) and a row exactly at the query end may be a terminal row — neither is a stored sample, whatever its value.
|
||||
/// </summary>
|
||||
private static async Task<DateTime?> FirstSampleAsync(IHistorianGatewayClient client, string tag, DateTime startUtc,
|
||||
DateTime endUtc, CancellationToken ct)
|
||||
{
|
||||
DateTime? found = null;
|
||||
await foreach (var raw in client.ReadRawAsync(tag, startUtc, endUtc, ProbeRows, ct).ConfigureAwait(false))
|
||||
{
|
||||
if (found is not null || raw.Timestamp is null) continue; // drain, never break (see ReadAsync)
|
||||
var t = raw.Timestamp.ToDateTime();
|
||||
if (t > startUtc && t < endUtc) found = t;
|
||||
}
|
||||
return found;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The row's OPC DA quality alone (<c>opc_quality</c> ≥ 64: Good or Uncertain) — deliberately without
|
||||
/// <see cref="SampleMapper.ToStatus"/>'s fallback to the AVEVA <c>quality</c> field when <c>opc_quality</c> is 0, which
|
||||
/// is also the OPC DA Bad value a NULL row carries.
|
||||
/// </summary>
|
||||
private static bool IsNotBadOnTheWire(HistorianSample raw) => (byte)raw.OpcQuality >= 64; // low byte, as SampleMapper.ToStatus reads it
|
||||
|
||||
private static HistoryReadLimitExceededException LimitExceeded(string tag, int maxWindowRows) =>
|
||||
new(maxWindowRows,
|
||||
$"The processed read of '{tag}' covers more than {maxWindowRows} raw samples " +
|
||||
"(ServerHistorian:MaxTimeWeightedRawRows); the Gateway backend evaluates per-sample aggregates from raw reads — " +
|
||||
"narrow the window.");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Builds the per-bucket statistics (<see cref="BucketRow"/>) the SQL backends' aggregate queries produce, from a stream
|
||||
/// of raw samples — the C# statement of <c>SqlAggregateQueries</c>' columns: counts by class (usable = non-Bad with a
|
||||
/// value; good = usable with Good severity; NonNumeric = usable String/DateTime; GoodNonNumeric = good String/DateTime),
|
||||
/// Avg/Min/Max/Sum over the GOOD numeric values and UncertainMin/UncertainMax over the usable-but-not-Good numeric values
|
||||
/// (#684), first/last usable value by time, the extreme GOOD numeric value with its time (ties ⇒ earliest), and the
|
||||
/// first/last sample of any quality.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Keep in step with the SQL backends' per-sample value columns: a change to which samples feed a statistic there
|
||||
/// (e.g. Uncertain handling) must be repeated here, or the Gateway backend silently diverges.
|
||||
/// </remarks>
|
||||
internal sealed class RawBucketAccumulator
|
||||
{
|
||||
private readonly bool _maximum;
|
||||
private readonly Dictionary<long, State> _buckets = new();
|
||||
|
||||
/// <summary>Creates an accumulator.</summary>
|
||||
/// <param name="maximum">Extreme = the maximum (MaximumActualTime); otherwise the minimum.</param>
|
||||
public RawBucketAccumulator(bool maximum) => _maximum = maximum;
|
||||
|
||||
private sealed class State
|
||||
{
|
||||
public int Total, Usable, Good, NonNumeric, GoodNonNumeric, GoodNumeric, UncertainNumeric;
|
||||
public double Sum, Min = double.PositiveInfinity, Max = double.NegativeInfinity;
|
||||
public double UncertainMin = double.PositiveInfinity, UncertainMax = double.NegativeInfinity;
|
||||
public StoredSample? First, Last, FirstAny, LastAny, Extreme;
|
||||
}
|
||||
|
||||
/// <summary>Adds one in-window sample to its bucket. Order-independent (picks compare by time).</summary>
|
||||
/// <param name="bucket">Ascending bucket index (<see cref="BucketPlan.IndexOf"/>).</param>
|
||||
/// <param name="s">The sample.</param>
|
||||
public void Add(long bucket, StoredSample s)
|
||||
{
|
||||
if (!_buckets.TryGetValue(bucket, out var st)) _buckets[bucket] = st = new State();
|
||||
st.Total++;
|
||||
if (st.FirstAny is not { } fa || s.TimeUtc < fa.TimeUtc) st.FirstAny = s;
|
||||
if (st.LastAny is not { } la || s.TimeUtc >= la.TimeUtc) st.LastAny = s;
|
||||
if (!s.IsUsable) return;
|
||||
st.Usable++;
|
||||
if (s.IsGood) st.Good++;
|
||||
if (st.First is not { } f || s.TimeUtc < f.TimeUtc) st.First = s;
|
||||
if (st.Last is not { } l || s.TimeUtc >= l.TimeUtc) st.Last = s;
|
||||
if (!s.Value.TryGetDouble(out var v))
|
||||
{
|
||||
st.NonNumeric++;
|
||||
if (s.IsGood) st.GoodNonNumeric++;
|
||||
return;
|
||||
}
|
||||
if (!s.IsGood)
|
||||
{
|
||||
// #684: an Uncertain value is not part of the value set; it only feeds the Min/Max status rule.
|
||||
st.UncertainNumeric++;
|
||||
if (v < st.UncertainMin) st.UncertainMin = v;
|
||||
if (v > st.UncertainMax) st.UncertainMax = v;
|
||||
return;
|
||||
}
|
||||
st.GoodNumeric++;
|
||||
st.Sum += v;
|
||||
if (v < st.Min) st.Min = v;
|
||||
if (v > st.Max) st.Max = v;
|
||||
if (st.Extreme is not { } e) st.Extreme = s;
|
||||
else
|
||||
{
|
||||
var ev = e.Value.Number;
|
||||
var better = _maximum ? v > ev : v < ev;
|
||||
if (better || (v == ev && s.TimeUtc < e.TimeUtc)) st.Extreme = s;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>The non-empty buckets' rows, ascending by bucket index.</summary>
|
||||
/// <returns>The rows.</returns>
|
||||
public IEnumerable<BucketRow> Rows() =>
|
||||
_buckets.OrderBy(kv => kv.Key).Select(kv =>
|
||||
{
|
||||
var st = kv.Value;
|
||||
var hasNumeric = st.GoodNumeric > 0;
|
||||
var hasUncertain = st.UncertainNumeric > 0;
|
||||
return new BucketRow
|
||||
{
|
||||
Bucket = kv.Key,
|
||||
Total = st.Total,
|
||||
Usable = st.Usable,
|
||||
Good = st.Good,
|
||||
Bad = st.Total - st.Usable,
|
||||
NonNumeric = st.NonNumeric,
|
||||
GoodNonNumeric = st.GoodNonNumeric,
|
||||
Avg = hasNumeric ? st.Sum / st.GoodNumeric : null,
|
||||
Min = hasNumeric ? st.Min : null,
|
||||
Max = hasNumeric ? st.Max : null,
|
||||
Sum = hasNumeric ? st.Sum : null,
|
||||
UncertainMin = hasUncertain ? st.UncertainMin : null,
|
||||
UncertainMax = hasUncertain ? st.UncertainMax : null,
|
||||
First = st.First?.Value ?? HistorianValue.Empty,
|
||||
Last = st.Last?.Value ?? HistorianValue.Empty,
|
||||
Extreme = st.Extreme?.Value.Number,
|
||||
ExtremeTime = st.Extreme?.TimeUtc,
|
||||
FirstAny = st.FirstAny,
|
||||
LastAny = st.LastAny,
|
||||
};
|
||||
});
|
||||
}
|
||||
+5
@@ -11,6 +11,11 @@
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\..\Core\ZB.MOM.WW.OtOpcUa.Core.Abstractions\ZB.MOM.WW.OtOpcUa.Core.Abstractions.csproj"/>
|
||||
<ProjectReference Include="..\..\Core\ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian\ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian.csproj"/>
|
||||
<!-- #681: the per-sample processed aggregates are evaluated from raw reads with the SAME database-independent code the
|
||||
SQL backends use (BucketPlan, BucketRow, AggregateEvaluator), so Gateway matches SqlServer/QuestDB exactly.
|
||||
Sql.Common references only Core.Abstractions + Core.AlarmHistorian (both already referenced here) and no
|
||||
driver, so this adds no cycle. -->
|
||||
<ProjectReference Include="..\ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common\ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common.csproj"/>
|
||||
<!-- Runtime owns ServerHistorianOptions (the bound ServerHistorian config). The read-cutover
|
||||
factory (GatewayHistorian.CreateDataSource / HistorianGatewayClientAdapter.Create) maps
|
||||
those options onto the package client, so the driver references it. Runtime references no
|
||||
|
||||
@@ -58,17 +58,20 @@ internal static class QuestDbAggregateQueries
|
||||
|
||||
/// <summary>
|
||||
/// Average / Minimum / Maximum / Count / Range (PercentGood / PercentBad and Total are time-weighted, not here).
|
||||
/// Family columns: <c>6 avg_value, 7 min_value, 8 max_value, 9 sum_value</c> (DOUBLE, NULL when no usable sample).
|
||||
/// Family columns: <c>6 avg_value, 7 min_value, 8 max_value, 9 sum_value</c> over the <b>Good</b> numeric samples
|
||||
/// (DOUBLE, NULL when none — #684, Part 13 / the SDK), then <c>10 unc_min, 11 unc_max</c> over the Uncertain ones (the
|
||||
/// Part 13 Minimum/Maximum/Range status rule) and <c>12 good_non_numeric_count</c>.
|
||||
/// </summary>
|
||||
/// <param name="sample">Prefixed sample table name.</param>
|
||||
/// <returns>The SQL text.</returns>
|
||||
/// <param name="reverse">True for a reverse read (window <c>($4, $5]</c>).</param>
|
||||
public static string Simple(string sample, bool reverse = false) =>
|
||||
"SELECT b, " + CommonCounts() + ", " +
|
||||
"avg(CASE WHEN u = 1 THEN num_value END) AS avg_value, " +
|
||||
"min(CASE WHEN u = 1 THEN num_value END) AS min_value, " +
|
||||
"max(CASE WHEN u = 1 THEN num_value END) AS max_value, " +
|
||||
"sum(CASE WHEN u = 1 THEN num_value END) AS sum_value " +
|
||||
"avg(CASE WHEN g = 1 THEN num_value END) AS avg_value, " +
|
||||
"min(CASE WHEN g = 1 THEN num_value END) AS min_value, " +
|
||||
"max(CASE WHEN g = 1 THEN num_value END) AS max_value, " +
|
||||
"sum(CASE WHEN g = 1 THEN num_value END) AS sum_value, " +
|
||||
UncertainExtremes() + " " +
|
||||
"FROM (" + Classified(sample, reverse) + ") GROUP BY b ORDER BY b";
|
||||
|
||||
/// <summary>Ordinal of the StartEnd family's first-sample-of-any-quality columns (<see cref="StartEnd"/>).</summary>
|
||||
@@ -126,9 +129,9 @@ internal static class QuestDbAggregateQueries
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// MinimumActualTime / MaximumActualTime: the usable sample with the smallest (largest) <c>num_value</c>, ties
|
||||
/// broken by the earliest <c>ts</c>. Family columns: <c>6 num_value, 7 ts</c> (NULL when the bucket has no usable
|
||||
/// numeric sample).
|
||||
/// MinimumActualTime / MaximumActualTime: the <b>Good</b> sample (#684) with the smallest (largest) <c>num_value</c>,
|
||||
/// ties broken by the earliest <c>ts</c>. Family columns: <c>6 num_value, 7 ts</c> (NULL when the bucket has no Good
|
||||
/// numeric sample), then <c>8 unc_min, 9 unc_max, 10 good_non_numeric_count</c> as in <see cref="Simple"/>.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Deliberately still two references to the classified rows (counts + the ranked extreme), unlike
|
||||
@@ -143,11 +146,12 @@ internal static class QuestDbAggregateQueries
|
||||
/// <param name="reverse">True for a reverse read (window <c>($4, $5]</c>).</param>
|
||||
public static string ActualTime(string sample, bool maximum, bool reverse = false) =>
|
||||
"WITH s AS (" + Classified(sample, reverse) + "), " +
|
||||
"st AS (SELECT b, " + CommonCounts() + " FROM s GROUP BY b), " +
|
||||
"st AS (SELECT b, " + CommonCounts() + ", " + UncertainExtremes() + " FROM s GROUP BY b), " +
|
||||
"r AS (SELECT b, num_value, ts, " +
|
||||
"row_number() OVER (PARTITION BY b ORDER BY num_value " + (maximum ? "DESC" : "ASC") + ", ts ASC) AS rn " +
|
||||
"FROM s WHERE u = 1 AND num_value IS NOT NULL) " +
|
||||
"SELECT st.*, r.num_value, r.ts " +
|
||||
"FROM s WHERE g = 1 AND num_value IS NOT NULL) " +
|
||||
"SELECT st.b, st.total_count, st.usable_count, st.good_count, st.bad_count, st.non_numeric_count, " +
|
||||
"r.num_value, r.ts, st.unc_min, st.unc_max, st.good_non_numeric_count " +
|
||||
"FROM st LEFT JOIN (SELECT * FROM r WHERE rn = 1) r ON r.b = st.b ORDER BY st.b";
|
||||
|
||||
/// <summary>
|
||||
@@ -217,6 +221,15 @@ internal static class QuestDbAggregateQueries
|
||||
$"CASE WHEN status < {GoodLimit} AND kind <> {EmptyKind} THEN 1 ELSE 0 END AS g " +
|
||||
$"FROM {sample} WHERE tag = $3 AND " + (reverse ? "ts > $4 AND ts <= $5" : "ts >= $4 AND ts < $5") + " AND NOT deleted";
|
||||
|
||||
/// <summary>
|
||||
/// The minimum and maximum <b>Uncertain</b> (usable, not Good) numeric value of the bucket — <c>unc_min, unc_max</c>
|
||||
/// (#684). Not result values: Part 13 makes Minimum/Maximum/Range Uncertain when one lies beyond the Good extreme.
|
||||
/// Then <c>good_non_numeric_count</c>, the Good String/DateTime samples (the Good-value aggregates' refusal check).
|
||||
/// </summary>
|
||||
private static string UncertainExtremes() =>
|
||||
"min(CASE WHEN u = 1 AND g = 0 THEN num_value END) AS unc_min, max(CASE WHEN u = 1 AND g = 0 THEN num_value END) AS unc_max, " +
|
||||
$"sum(CASE WHEN g = 1 AND kind IN ({NonNumericKinds}) THEN 1 ELSE 0 END) AS good_non_numeric_count";
|
||||
|
||||
/// <summary>The five common count columns (after <c>b</c>).</summary>
|
||||
private static string CommonCounts() =>
|
||||
"count() AS total_count, sum(u) AS usable_count, sum(g) AS good_count, count() - sum(u) AS bad_count, " +
|
||||
|
||||
+27
-11
@@ -359,6 +359,10 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource
|
||||
/// <remarks>Reverse events reads are served natively (<c>ORDER BY … DESC</c>, Gitea #675).</remarks>
|
||||
public bool SupportsReverseEvents => true;
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>The seventeen <see cref="AggregateFamilies.Served"/> — shared with the other SQL backend (#682).</remarks>
|
||||
public IReadOnlySet<HistoryAggregateType> SupportedAggregates => AggregateFamilies.Served;
|
||||
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>
|
||||
@@ -446,7 +450,9 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource
|
||||
/// the SDK: window <c>(end, start]</c>, buckets stepping back from <c>start</c>, the partial one earliest, each stamped
|
||||
/// with its later edge, returned latest first. One statement per node returns the <see cref="BucketRow"/> statistics
|
||||
/// of each non-empty bucket (bucket index <c>(cast(ts AS LONG)*10 − BucketBaseUnixTicks) / IntervalTicks</c>, never
|
||||
/// <c>SAMPLE BY</c>); a second fetches the tag's earliest sample for the start-of-data Partial bit; and
|
||||
/// <c>SAMPLE BY</c>); before it, when <see cref="AggregateEvaluator.NeedsDataRange"/>, two tag-index seeks fetch the
|
||||
/// tag's data range (earliest / latest live sample) for the start/end-of-data Partial bits and Count's in-range
|
||||
/// empty interval; and
|
||||
/// <see cref="AggregateEvaluator.EvaluateAll"/> produces the values and fills the empty buckets with <c>BadNoData</c>. The time-weighted family (#671) instead streams the window plus four bounding
|
||||
/// rows into <see cref="TimeWeightedAggregator"/>, exactly as the SQL Server backend does.
|
||||
/// </remarks>
|
||||
@@ -484,6 +490,20 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource
|
||||
_ => QuestDbAggregateQueries.ActualTime(sample, aggregate == HistoryAggregateType.MaximumActualTime, plan.Reversed),
|
||||
};
|
||||
await using var conn = await _conn.OpenAsync(cancellationToken).ConfigureAwait(false);
|
||||
// The tag's data range (earliest + latest live sample): the start/end-of-data Partial rules (#677/#680 —
|
||||
// any per-sample aggregate but Average, either direction) and Count's in-range empty interval (#680); skipped
|
||||
// otherwise. Index seeks on tag (#683). Read BEFORE the bucket statement, which is a separate table reader:
|
||||
// a sample appended in between (the common race — a live tag read up to now) then reaches the buckets but
|
||||
// not the range — at worst a stale end-of-data Partial or a BadNoData where a 0 belongs — instead of a
|
||||
// range covering a bucket the rows do not show, which reported a Good Count of 0 for an interval that has
|
||||
// samples. (A concurrent HistoryUpdate delete is the mirror case and can still do that, transiently.)
|
||||
if (AggregateEvaluator.NeedsDataRange(aggregate, plan))
|
||||
{
|
||||
earliest = await ReadBoundAsync(conn, QuestDbAggregateQueries.EarliestSample(sample), fullReference,
|
||||
plan.EndUtc, cancellationToken).ConfigureAwait(false);
|
||||
latest = await ReadBoundAsync(conn, QuestDbAggregateQueries.LatestSample(sample), fullReference,
|
||||
plan.StartUtc, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
await using (var cmd = _conn.CreateCommand(conn, sql))
|
||||
{
|
||||
QuestDbConnection.BindLong(cmd, plan.BucketBaseUnixTicks);
|
||||
@@ -495,16 +515,6 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource
|
||||
while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false))
|
||||
rows.Add(ReadBucket(reader, family));
|
||||
}
|
||||
// The tag's data range (earliest + latest live sample): the start/end-of-data Partial rules (#677/#680 —
|
||||
// any per-sample aggregate but Average, either direction) and Count's in-range empty interval (#680); skipped
|
||||
// otherwise.
|
||||
if (AggregateEvaluator.NeedsDataRange(aggregate, plan))
|
||||
{
|
||||
earliest = await ReadBoundAsync(conn, QuestDbAggregateQueries.EarliestSample(sample), fullReference,
|
||||
plan.EndUtc, cancellationToken).ConfigureAwait(false);
|
||||
latest = await ReadBoundAsync(conn, QuestDbAggregateQueries.LatestSample(sample), fullReference,
|
||||
plan.StartUtc, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
var result = AggregateEvaluator.EvaluateAll(aggregate, plan, rows, earliest, latest, now);
|
||||
@@ -613,6 +623,9 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource
|
||||
row.Min = D(reader, f + 1);
|
||||
row.Max = D(reader, f + 2);
|
||||
row.Sum = D(reader, f + 3);
|
||||
row.UncertainMin = D(reader, f + 4);
|
||||
row.UncertainMax = D(reader, f + 5);
|
||||
row.GoodNonNumeric = N(reader, f + 6);
|
||||
break;
|
||||
case AggregateFamily.StartEnd:
|
||||
row.First = V(reader, f);
|
||||
@@ -623,6 +636,9 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource
|
||||
default:
|
||||
row.Extreme = D(reader, f);
|
||||
row.ExtremeTime = reader.IsDBNull(f + 1) ? null : QuestDbConnection.ReadUtc(reader, f + 1);
|
||||
row.UncertainMin = D(reader, f + 2);
|
||||
row.UncertainMax = D(reader, f + 3);
|
||||
row.GoodNonNumeric = N(reader, f + 4);
|
||||
break;
|
||||
}
|
||||
return row;
|
||||
|
||||
+49
-1
@@ -1,3 +1,4 @@
|
||||
using System.Globalization;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Npgsql;
|
||||
@@ -15,10 +16,17 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb;
|
||||
/// <item><b>TTL (RC3):</b> <c>tables()</c> normalises units (7 days reads back as <c>1 WEEK</c>) so values are compared
|
||||
/// in hours, and an ALTER becomes visible only after the WAL applies it, so the bootstrapper never re-reads to verify
|
||||
/// its own ALTER. A boot whose TTL already matches issues no ALTER.</item>
|
||||
/// <item><b>Tag index (#683):</b> a sample table created before the <c>tag</c> symbol index existed gains it with
|
||||
/// <see cref="QuestDbTables.AddTagIndex"/> (idempotent; a boot that finds it indexed issues no ALTER, and like the TTL
|
||||
/// the ALTER is a WAL transaction the bootstrapper does not re-read). Building it is a one-off WAL apply — ~3 s for
|
||||
/// 68M rows / 63 daily partitions measured — during which writes queue and reads keep scanning. A refused ALTER is
|
||||
/// logged as a Warning, never fails the bootstrap (the index is optional); a failure at WAL-apply time would suspend
|
||||
/// the table unseen, the same blind spot as the TTL ALTER.</item>
|
||||
/// <item><b>Failure:</b> not latched. Within <see cref="BootstrapFailureCache.DefaultDuration"/> the last failure is
|
||||
/// re-thrown without touching QuestDB.</item>
|
||||
/// <item><c>AutoCreateSchema=false</c> only verifies the tables exist (no TTL changes) and throws
|
||||
/// <see cref="HistorianSchemaMissingException"/> otherwise.</item>
|
||||
/// <see cref="HistorianSchemaMissingException"/> otherwise; a missing tag index is only logged (a Warning naming
|
||||
/// the ALTER to run), since reads stay correct without it.</item>
|
||||
/// </list>
|
||||
/// </summary>
|
||||
internal sealed class QuestDbSchemaBootstrapper
|
||||
@@ -63,10 +71,12 @@ internal sealed class QuestDbSchemaBootstrapper
|
||||
{
|
||||
await CreateTablesAsync(conn, ct).ConfigureAwait(false);
|
||||
await ReconcileTtlAsync(conn, ct).ConfigureAwait(false);
|
||||
await EnsureTagIndexAsync(conn, apply: true, ct).ConfigureAwait(false);
|
||||
}
|
||||
else
|
||||
{
|
||||
await VerifyAsync(conn, ct).ConfigureAwait(false);
|
||||
await EnsureTagIndexAsync(conn, apply: false, ct).ConfigureAwait(false);
|
||||
}
|
||||
_ready = true;
|
||||
_failure.Clear();
|
||||
@@ -117,6 +127,44 @@ internal sealed class QuestDbSchemaBootstrapper
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// #683: indexes the sample table's <c>tag</c> column when it is not (a table created before the index existed).
|
||||
/// <paramref name="apply"/> false (AutoCreateSchema off) only logs the statement to run.
|
||||
/// </summary>
|
||||
private async Task EnsureTagIndexAsync(NpgsqlConnection conn, bool apply, CancellationToken ct)
|
||||
{
|
||||
var table = _conn.SampleTable;
|
||||
await using (var cmd = _conn.CreateCommand(conn, QuestDbTables.TagIndexedQuery(table)))
|
||||
{
|
||||
var indexed = await cmd.ExecuteScalarAsync(ct).ConfigureAwait(false);
|
||||
if (indexed is null or DBNull || Convert.ToBoolean(indexed, CultureInfo.InvariantCulture)) return;
|
||||
}
|
||||
if (!apply)
|
||||
{
|
||||
_log.LogWarning("QuestDB historian table {Table} has no index on its tag column, so the data-range lookups of " +
|
||||
"processed reads scan partitions instead of seeking (#683). AutoCreateSchema is false; run: {Statement}",
|
||||
table, QuestDbTables.AddTagIndex(table));
|
||||
return;
|
||||
}
|
||||
// A WAL transaction: table_columns() keeps reporting false until it applies, so do NOT re-read to verify.
|
||||
// The index is optional (reads are correct without it), so a refused ALTER (e.g. no DDL rights) must not fail the
|
||||
// bootstrap: it is logged and retried on the next boot. A failure at WAL-APPLY time is not seen here (QuestDB would
|
||||
// suspend the table) — the same blind spot as the TTL ALTER.
|
||||
try
|
||||
{
|
||||
await using var alter = _conn.CreateCommand(conn, QuestDbTables.AddTagIndex(table));
|
||||
await alter.ExecuteNonQueryAsync(ct).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex) when (ex is not OperationCanceledException || !ct.IsCancellationRequested)
|
||||
{
|
||||
_log.LogWarning("Could not add the tag index to QuestDB table {Table} ({Exception}: {Message}); data-range lookups " +
|
||||
"of processed reads scan partitions instead of seeking (#683). Run: {Statement}",
|
||||
table, ex.GetType().Name, ex.Message, QuestDbTables.AddTagIndex(table));
|
||||
return;
|
||||
}
|
||||
_log.LogInformation("Added the tag index to QuestDB table {Table} (#683).", table);
|
||||
}
|
||||
|
||||
private async Task ReconcileTtlAsync(NpgsqlConnection conn, CancellationToken ct)
|
||||
{
|
||||
var wantHours = _options.RetentionDays is { } d ? d * 24L : 0L;
|
||||
|
||||
@@ -39,9 +39,15 @@ internal static class QuestDbTables
|
||||
/// </summary>
|
||||
public static string[] Expiring(string prefix) => [Sample(prefix), Event(prefix), SampleModified(prefix)];
|
||||
|
||||
/// <summary><c>CREATE TABLE IF NOT EXISTS</c> for the sample table.</summary>
|
||||
/// <summary>
|
||||
/// <c>CREATE TABLE IF NOT EXISTS</c> for the sample table. <c>tag</c> carries a symbol (bitmap) index (#683): every
|
||||
/// read filters on one tag, and without the index QuestDB answers a <c>tag = x</c> seek by scanning the tag column of
|
||||
/// every partition in the time range — the data-range lookups (<see cref="QuestDbAggregateQueries.EarliestSample"/> /
|
||||
/// <c>LatestSample</c>) of a stopped or late-starting tag then scan every partition on one side of the window.
|
||||
/// A table created before #683 gains the index through <see cref="AddTagIndex"/>.
|
||||
/// </summary>
|
||||
public static string CreateSample(string prefix, int? retentionDays) =>
|
||||
$"CREATE TABLE IF NOT EXISTS {Sample(prefix)} (ts TIMESTAMP, tag SYMBOL, kind BYTE, num_value DOUBLE, " +
|
||||
$"CREATE TABLE IF NOT EXISTS {Sample(prefix)} (ts TIMESTAMP, tag SYMBOL INDEX, kind BYTE, num_value DOUBLE, " +
|
||||
"int_value LONG, str_value VARCHAR, status LONG, deleted BOOLEAN, server_ts TIMESTAMP) " +
|
||||
$"TIMESTAMP(ts) PARTITION BY DAY{Ttl(retentionDays)} WAL DEDUP UPSERT KEYS(ts, tag)";
|
||||
|
||||
@@ -74,6 +80,21 @@ internal static class QuestDbTables
|
||||
[CreateSample(prefix, retentionDays), CreateEvent(prefix, retentionDays), CreateTag(prefix),
|
||||
CreateSampleModified(prefix, retentionDays)];
|
||||
|
||||
/// <summary>
|
||||
/// <c>ALTER TABLE t ALTER COLUMN tag ADD INDEX</c> — the #683 migration of a sample table created without the tag
|
||||
/// index. A WAL transaction (visible once applied); re-issuing it on an indexed column is a harmless no-op (the table
|
||||
/// is not suspended), so two nodes booting together may both send it.
|
||||
/// </summary>
|
||||
public static string AddTagIndex(string table) => $"ALTER TABLE {table} ALTER COLUMN tag ADD INDEX";
|
||||
|
||||
/// <summary>
|
||||
/// Whether <paramref name="table"/>'s <c>tag</c> column is indexed: one row, one BOOLEAN column, no row when the
|
||||
/// table or column is missing. The name is inlined (<c>table_columns()</c> takes a constant); it is always a
|
||||
/// prefix-validated name built here.
|
||||
/// </summary>
|
||||
public static string TagIndexedQuery(string table) =>
|
||||
$"SELECT indexed FROM table_columns('{table}') WHERE \"column\" = 'tag'";
|
||||
|
||||
/// <summary><c>ALTER TABLE t SET TTL n DAYS</c>; <c>null</c> removes the TTL (<c>SET TTL 0 DAYS</c>).</summary>
|
||||
public static string AlterTtl(string table, int? retentionDays) =>
|
||||
$"ALTER TABLE {table} SET TTL {(retentionDays ?? 0).ToString(CultureInfo.InvariantCulture)} DAYS";
|
||||
|
||||
@@ -9,7 +9,11 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common;
|
||||
/// <para>
|
||||
/// Per interval: no samples ⇒ <c>BadNoData</c> (Count: 0 inside the data range). Otherwise no usable sample ⇒
|
||||
/// <c>BadNoData</c>; a numeric aggregate with a usable String/DateTime sample ⇒ <c>BadAggregateInvalidInputs</c>;
|
||||
/// else the value, Good when every sample is Good, <c>UncertainDataSubNormal</c> when any is Bad or Uncertain.
|
||||
/// else the value. #684: Average / Minimum / Maximum / Range / *ActualTime are computed over <b>Good</b> values only
|
||||
/// (Part 13 and the SDK — an Uncertain sample is never part of the value; an interval with no Good value is
|
||||
/// <c>BadNoData</c>). Status: Average value-based (Good only when every sample is Good, else
|
||||
/// <c>UncertainDataSubNormal</c>); Minimum / Maximum / Range / *ActualTime per their Part 13 tables — Uncertain when a
|
||||
/// Bad sample exists or an Uncertain value lies beyond the reported extreme (<see cref="MinMaxStatus"/>).
|
||||
/// Every calculated status — including <c>BadNoData</c> — carries the Calculated info bits. Timestamps are the
|
||||
/// bucket's <see cref="BucketPlan.BucketTimestamp"/> (its start forward, its later edge reverse), except
|
||||
/// Minimum/MaximumActualTime (the sample's time; ties ⇒ earliest) and Start/End. Every aggregate but
|
||||
@@ -70,15 +74,30 @@ public static class AggregateEvaluator
|
||||
return Calc(r.Good, r.Good == r.Total ? HistorianStatusCodes.Good : HistorianStatusCodes.UncertainDataSubNormal);
|
||||
}
|
||||
if (r.Usable == 0) return NoData(bucketStart, now);
|
||||
if (r.NonNumeric > 0) return Calc(null, HistorianStatusCodes.BadAggregateInvalidInputs);
|
||||
// #684 review: Delta reads usable picks, so any usable String/DateTime sample refuses it. The Good-value aggregates
|
||||
// refuse only a GOOD non-numeric sample — an Uncertain one is not part of their value set (it only degrades the
|
||||
// status, as the SDK's skipped/uncastable value does).
|
||||
if ((aggregate == HistoryAggregateType.Delta ? r.NonNumeric : r.GoodNonNumericOrAll) > 0)
|
||||
return Calc(null, HistorianStatusCodes.BadAggregateInvalidInputs);
|
||||
|
||||
var status = r.Good == r.Total ? HistorianStatusCodes.Good : HistorianStatusCodes.UncertainDataSubNormal;
|
||||
switch (aggregate)
|
||||
{
|
||||
case HistoryAggregateType.Average: return Calc(r.Avg, status);
|
||||
case HistoryAggregateType.Minimum: return Calc(r.Min, status);
|
||||
case HistoryAggregateType.Maximum: return Calc(r.Max, status);
|
||||
case HistoryAggregateType.Range: return Calc(r.Max - r.Min, status);
|
||||
// #684 (Part 13 §5.4.3.5; SDK ComputeAverage): the mean of the GOOD values — an interval with none (only
|
||||
// Uncertain / Bad) is BadNoData; value-based status: Good only when every sample is Good, else Uncertain.
|
||||
case HistoryAggregateType.Average:
|
||||
if (r.Avg is null) return NoData(bucketStart, now);
|
||||
return Calc(r.Avg, r.Good == r.Total ? HistorianStatusCodes.Good : HistorianStatusCodes.UncertainDataSubNormal);
|
||||
// #684 (Part 13 §5.4.3.10/11/14; SDK ComputeMinMax): the extreme(s) of the GOOD values; Uncertain when a Bad
|
||||
// sample exists or an Uncertain value lies beyond the reported extreme (see MinMaxStatus).
|
||||
case HistoryAggregateType.Minimum:
|
||||
if (r.Min is null) return NoData(bucketStart, now);
|
||||
return Calc(r.Min, MinMaxStatus(r, r.Min, r.Max, low: true, high: false));
|
||||
case HistoryAggregateType.Maximum:
|
||||
if (r.Max is null) return NoData(bucketStart, now);
|
||||
return Calc(r.Max, MinMaxStatus(r, r.Min, r.Max, low: false, high: true));
|
||||
case HistoryAggregateType.Range:
|
||||
if (r.Min is null || r.Max is null) return NoData(bucketStart, now);
|
||||
return Calc(r.Max - r.Min, MinMaxStatus(r, r.Min, r.Max, low: true, high: true));
|
||||
case HistoryAggregateType.Delta:
|
||||
// #680 (SDK ComputeDelta): last usable − first usable; Uncertain only when the interval's first or last
|
||||
// sample was skipped (not usable) — a Bad sample in the middle does not degrade it.
|
||||
@@ -96,12 +115,32 @@ public static class AggregateEvaluator
|
||||
// Live-gate 8 F1: usable numeric samples counted but no extreme picked (an inconsistent row) is no data,
|
||||
// never a value-less Good.
|
||||
if (r.Extreme is null || r.ExtremeTime is null) return NoData(bucketStart, now);
|
||||
return Calc(r.Extreme, status, r.ExtremeTime);
|
||||
var min = aggregate == HistoryAggregateType.MinimumActualTime;
|
||||
return Calc(r.Extreme, MinMaxStatus(r, r.Extreme, r.Extreme, low: min, high: !min), r.ExtremeTime);
|
||||
default:
|
||||
throw new HistoryAggregateNotSupportedException(aggregate);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// #684 — the Part 13 status of Minimum / Maximum / Range / *ActualTime (Tables 57–62): <c>Uncertain_DataSubNormal</c>
|
||||
/// when a Bad sample exists in the interval, or when an Uncertain value is less than the minimum Good value
|
||||
/// (<paramref name="low"/>) / greater than the maximum Good value (<paramref name="high"/>); else Good. An Uncertain
|
||||
/// value inside the Good range leaves it Good. An Uncertain String/DateTime sample (no order against the Good values)
|
||||
/// also makes it Uncertain — the SDK's <c>ComputeMinMax</c> flags a value it cannot cast exactly that way.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The SDK's <c>ComputeMinMax</c> (1.5.378) applies only the Bad half: it collects the Uncertain extremes and never
|
||||
/// reads them, so it answers Good where Part 13 says Uncertain. Ours follows the specification there — the more
|
||||
/// conservative of the two, and never a status better than the inputs warrant. The comparison is strict, so an
|
||||
/// Uncertain value EQUAL to the Good extreme does not degrade it.
|
||||
/// </remarks>
|
||||
private static uint MinMaxStatus(BucketRow r, double? goodMin, double? goodMax, bool low, bool high) =>
|
||||
r.Bad > 0 || r.NonNumeric > r.GoodNonNumericOrAll
|
||||
|| (low && r.UncertainMin < goodMin) || (high && r.UncertainMax > goodMax)
|
||||
? HistorianStatusCodes.UncertainDataSubNormal
|
||||
: HistorianStatusCodes.Good;
|
||||
|
||||
/// <summary>
|
||||
/// True for the per-sample aggregates the SDK sets the start-of-data Partial bit on (every calculator but
|
||||
/// <c>AverageAggregateCalculator</c>'s plain <c>Average</c>, which turns <c>SetPartialBit</c> off).
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
using System.Collections.Frozen;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common;
|
||||
@@ -40,4 +41,12 @@ public static class AggregateFamilies
|
||||
or HistoryAggregateType.PercentGood or HistoryAggregateType.PercentBad => AggregateFamily.TimeWeighted,
|
||||
_ => null,
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// The seventeen aggregates the SQL backends serve — every member <see cref="FamilyOf"/> assigns a family, derived
|
||||
/// from it so the set and the dispatch can never disagree. Both <c>SqlServerHistorianDataSource</c> and
|
||||
/// <c>QuestDbHistorianDataSource</c> return it as <see cref="IHistorianDataSource.SupportedAggregates"/> (#682).
|
||||
/// </summary>
|
||||
public static readonly IReadOnlySet<HistoryAggregateType> Served =
|
||||
Enum.GetValues<HistoryAggregateType>().Where(a => FamilyOf(a) is not null).ToFrozenSet();
|
||||
}
|
||||
|
||||
@@ -9,7 +9,11 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common;
|
||||
/// <remarks>
|
||||
/// Sample classes: <i>usable</i> = severity Good or Uncertain AND a stored value (an Empty sample is an outage marker
|
||||
/// and counts as Bad whatever its code); <i>good</i> = usable with severity Good; <i>bad</i> = not usable.
|
||||
/// Every value field is computed over usable samples only, except <see cref="FirstAny"/> / <see cref="LastAny"/>.
|
||||
/// <see cref="Avg"/> / <see cref="Min"/> / <see cref="Max"/> / <see cref="Sum"/> / <see cref="Extreme"/> are computed over
|
||||
/// <b>Good</b> numeric samples only (#684 — Part 13 defines Average / Minimum / Maximum / Range / *ActualTime over Good
|
||||
/// raw values, as the SDK's calculators do); <see cref="UncertainMin"/> / <see cref="UncertainMax"/> over the Uncertain
|
||||
/// ones (the Part 13 Minimum/Maximum/Range status rule); <see cref="First"/> / <see cref="Last"/> over usable samples;
|
||||
/// <see cref="FirstAny"/> / <see cref="LastAny"/> over every sample.
|
||||
/// </remarks>
|
||||
public sealed class BucketRow
|
||||
{
|
||||
@@ -28,19 +32,30 @@ public sealed class BucketRow
|
||||
/// <summary>Bad (not usable) samples.</summary>
|
||||
public int Bad { get; set; }
|
||||
|
||||
/// <summary>Usable String / DateTime samples.</summary>
|
||||
/// <summary>Usable String / DateTime samples (Delta's refusal check).</summary>
|
||||
public int NonNumeric { get; set; }
|
||||
|
||||
/// <summary>Average of the usable numeric values (null when none).</summary>
|
||||
/// <summary>
|
||||
/// <b>Good</b> String / DateTime samples — the refusal check of the Good-value aggregates (Average / Minimum / Maximum /
|
||||
/// Range / *ActualTime, #684 review): an Uncertain non-numeric sample is not part of their value set. Filled by the
|
||||
/// Simple and ActualTime queries; null (a row that does not carry it) falls back to <see cref="NonNumeric"/>, the
|
||||
/// pre-#684 (stricter) check.
|
||||
/// </summary>
|
||||
public int? GoodNonNumeric { get; set; }
|
||||
|
||||
/// <summary><see cref="GoodNonNumeric"/>, or <see cref="NonNumeric"/> when the row does not carry it.</summary>
|
||||
internal int GoodNonNumericOrAll => GoodNonNumeric ?? NonNumeric;
|
||||
|
||||
/// <summary>Average of the Good numeric values (null when none — #684).</summary>
|
||||
public double? Avg { get; set; }
|
||||
|
||||
/// <summary>Minimum of the usable numeric values (null when none).</summary>
|
||||
/// <summary>Minimum of the Good numeric values (null when none — #684).</summary>
|
||||
public double? Min { get; set; }
|
||||
|
||||
/// <summary>Maximum of the usable numeric values (null when none).</summary>
|
||||
/// <summary>Maximum of the Good numeric values (null when none — #684).</summary>
|
||||
public double? Max { get; set; }
|
||||
|
||||
/// <summary>Sum of the usable numeric values (null when none).</summary>
|
||||
/// <summary>Sum of the Good numeric values (null when none — #684).</summary>
|
||||
public double? Sum { get; set; }
|
||||
|
||||
/// <summary>First usable sample's value by time (Empty when none).</summary>
|
||||
@@ -49,12 +64,25 @@ public sealed class BucketRow
|
||||
/// <summary>Last usable sample's value by time (Empty when none).</summary>
|
||||
public HistorianValue Last { get; set; }
|
||||
|
||||
/// <summary>Extreme (min or max) usable numeric value (null when none).</summary>
|
||||
/// <summary>Extreme (min or max) Good numeric value (null when none — #684).</summary>
|
||||
public double? Extreme { get; set; }
|
||||
|
||||
/// <summary>Time of <see cref="Extreme"/> (ties ⇒ earliest; null when none).</summary>
|
||||
public DateTime? ExtremeTime { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Minimum of the bucket's <b>Uncertain</b> (usable, not Good) numeric values (null when none). Not a result value:
|
||||
/// Part 13 makes Minimum / MinimumActualTime / Range <c>Uncertain_DataSubNormal</c> when an Uncertain value is less
|
||||
/// than the minimum Good value (#684).
|
||||
/// </summary>
|
||||
public double? UncertainMin { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Maximum of the bucket's Uncertain numeric values (null when none): Maximum / MaximumActualTime / Range are
|
||||
/// Uncertain when one is greater than the maximum Good value (#684).
|
||||
/// </summary>
|
||||
public double? UncertainMax { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// The bucket's first sample by time of <b>any</b> quality (null when the bucket is empty): Start returns it raw, and
|
||||
/// Delta is Uncertain when it is not usable (#680, Part 13 Tables 73/74 — the SDK's <c>StartEndAggregateCalculator</c>).
|
||||
|
||||
@@ -32,7 +32,8 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer;
|
||||
/// and bit 30): <i>usable</i> = severity Good or Uncertain (<c>StatusCode >= 0</c>) AND a stored value
|
||||
/// (<c>ValueKind <> 0</c>) — an <c>Empty</c> sample is an outage marker and counts as Bad whatever its
|
||||
/// code; <i>good</i> = usable with severity Good (<c>StatusCode < 0x40000000</c>); <i>bad</i> = not
|
||||
/// usable. Every value column below is computed over usable samples only.
|
||||
/// usable. Average/Minimum/Maximum/Range/*ActualTime value columns are computed over <i>good</i> samples only
|
||||
/// (#684); Start/End/Delta picks over usable samples (or every sample).
|
||||
/// </para>
|
||||
/// Common result columns (ordinals 0–5): <c>Bucket, TotalCount, UsableCount, GoodCount, BadCount,
|
||||
/// NonNumericCount</c> (usable String/DateTime samples). Family columns follow.
|
||||
@@ -47,8 +48,10 @@ internal static class SqlAggregateQueries
|
||||
|
||||
/// <summary>
|
||||
/// Average / Minimum / Maximum / Count / Range (PercentGood / PercentBad and Total are time-weighted, not here).
|
||||
/// Family columns: <c>6 AvgValue, 7 MinValue, 8 MaxValue, 9 SumValue</c> (float, NULL when no usable
|
||||
/// numeric sample).
|
||||
/// Family columns: <c>6 AvgValue, 7 MinValue, 8 MaxValue, 9 SumValue</c> over the <b>Good</b> numeric samples (float,
|
||||
/// NULL when none — #684: Part 13 and the SDK compute these over Good values only), then <c>10 UncMin, 11 UncMax</c>
|
||||
/// over the Uncertain ones (the Part 13 Minimum/Maximum/Range status rule, <see cref="AggregateEvaluator"/>), and
|
||||
/// <c>12 GoodNonNumericCount</c>.
|
||||
/// </summary>
|
||||
/// <param name="sample">Schema-qualified <c>Sample</c> table name.</param>
|
||||
/// <returns>The SQL text.</returns>
|
||||
@@ -56,10 +59,11 @@ internal static class SqlAggregateQueries
|
||||
public static string Simple(string sample, bool reverse = false) =>
|
||||
Classified(sample, reverse) +
|
||||
"SELECT b, " + CommonCounts() + ", " +
|
||||
"AVG(CASE WHEN u = 1 THEN NumValue END) AS AvgValue, " +
|
||||
"MIN(CASE WHEN u = 1 THEN NumValue END) AS MinValue, " +
|
||||
"MAX(CASE WHEN u = 1 THEN NumValue END) AS MaxValue, " +
|
||||
"SUM(CASE WHEN u = 1 THEN NumValue END) AS SumValue " +
|
||||
"AVG(CASE WHEN g = 1 THEN NumValue END) AS AvgValue, " +
|
||||
"MIN(CASE WHEN g = 1 THEN NumValue END) AS MinValue, " +
|
||||
"MAX(CASE WHEN g = 1 THEN NumValue END) AS MaxValue, " +
|
||||
"SUM(CASE WHEN g = 1 THEN NumValue END) AS SumValue, " +
|
||||
UncertainExtremes() + " " +
|
||||
"FROM s GROUP BY b ORDER BY b;";
|
||||
|
||||
/// <summary>Ordinal of the StartEnd family's first-sample-of-any-quality columns (<see cref="StartEnd"/>).</summary>
|
||||
@@ -74,42 +78,68 @@ internal static class SqlAggregateQueries
|
||||
/// StatusCode</c>) and <c>22..27</c> the last (same order) — present for every returned (non-empty) bucket.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <b>One scan (live-gate 8 F1).</b> The counts and all four picked samples come from one reference to <c>s</c>:
|
||||
/// a windowed pass ranks each row within its bucket (<c>rfa</c>/<c>rla</c> over every sample, <c>rfu</c>/<c>rlu</c>
|
||||
/// within its usable / not-usable partition) and the outer <c>GROUP BY</c> both counts and lifts each picked row's
|
||||
/// columns with a conditional <c>MAX</c> (one row per bucket matches each pick). The previous form referenced the
|
||||
/// CTE five times; SQL Server expands each reference into its own scan, and under READ COMMITTED nothing makes those
|
||||
/// scans see the same rows, so a sample committed between two of them could be counted but not picked (QuestDB hit
|
||||
/// exactly that). It is also fewer scans of the key range.
|
||||
/// <para>
|
||||
/// <b>Aggregate, then join back on the key (#683).</b> One <c>GROUP BY</c> over the window computes the counts and
|
||||
/// each bucket's four pick TIMES — <c>MIN</c>/<c>MAX(SourceTimeUtc)</c> over every sample (any quality) and over
|
||||
/// the usable ones — and each pick's columns are then fetched by a seek on the clustered key
|
||||
/// <c>(TagId, SourceTimeUtc)</c> (unique, so one row per pick; a NULL usable time joins nothing ⇒ NULL columns,
|
||||
/// as before). Output is identical to the previous windowed form (one <c>ROW_NUMBER</c> pass + conditional
|
||||
/// <c>MAX</c>, live-gate 8 F1) — <c>EXCEPT</c> both ways is empty on 6M rows, forward and reverse — without that
|
||||
/// form's four window sorts, which spilled: on 6M rows / 30 days, 30 d @ 1 h ~3.4 s vs 18–32 s, 7 d @ 1 d
|
||||
/// ~0.8 s vs 5.2–6.1 s, 1 d @ 1 min ~0.16 s vs 0.31–0.40 s (docs/Historian.md).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// ⚠️ The key seeks are separate reads of the table within one statement, so under READ COMMITTED a sample
|
||||
/// deleted (HistoryUpdate DeleteRaw/DeleteAtTime) between the aggregate and its seek leaves that pick NULL while
|
||||
/// the counts still include it. The reader maps a NULL pick to "absent" (<c>ReadStoredOrNull</c>, live-gate 8 F1)
|
||||
/// and the evaluator answers that bucket's Start/End/Delta <c>BadNoData</c> — transient, until the next read. A
|
||||
/// sample replaced in between is picked with its new value (same time) — except that a usable pick re-checks
|
||||
/// usability in its join (the exact <c>u</c> predicate), so a Replace that turned it Bad/Empty reads as absent
|
||||
/// (<c>BadNoData</c>), never as a Bad row's value under Delta. A sample inserted in between is in neither half
|
||||
/// (the pick times come from the aggregate).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Every pick is an exact-key seek, by hint (#683 live gate 9 F1).</b> Each pick join carries
|
||||
/// <c>WITH (FORCESEEK (PK_Sample (TagId, SourceTimeUtc)))</c>, so it can only be a nested-loops seek on BOTH key
|
||||
/// columns — one row per bucket per pick — whatever the statistics say. Without it the plan was the optimizer's
|
||||
/// choice, and both alternatives it found were pathological: keyed on the pick time alone, a hash/merge join whose
|
||||
/// inner input was the tag's whole history (review of 910befc8: 43,657 vs 361 logical reads); bounded to the window
|
||||
/// as well (4a3b1256), a nested-loops RANGE seek over the window with the pick time as a residual, once per bucket
|
||||
/// — O(buckets × window rows), chosen under understated statistics and then reused from the plan cache (gate 9:
|
||||
/// 1 d @ 1 min 20 s; here 47 s, 3.6M reads, vs 0.25 s). With the hint neither shape exists (a forced
|
||||
/// <c>HASH</c>/<c>MERGE JOIN</c> is refused, error 8622), so the window bound is no longer needed in the join.
|
||||
/// ⚠️ The hint names the clustered primary key <c>PK_Sample</c> created by <c>001_initial.sql</c>; a hand-built
|
||||
/// schema must keep that name.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
/// <param name="sample">Schema-qualified <c>Sample</c> table name.</param>
|
||||
/// <returns>The SQL text.</returns>
|
||||
/// <param name="reverse">True for a reverse read (window <c>(@start, @end]</c>).</param>
|
||||
public static string StartEnd(string sample, bool reverse = false)
|
||||
{
|
||||
static string Pick(string when, bool any) =>
|
||||
(any ? $"MAX(CASE WHEN {when} THEN SourceTimeUtc END), " : "") +
|
||||
$"MAX(CASE WHEN {when} THEN ValueKind END), MAX(CASE WHEN {when} THEN NumValue END), " +
|
||||
$"MAX(CASE WHEN {when} THEN IntValue END), MAX(CASE WHEN {when} THEN StrValue END), " +
|
||||
(any ? $"MAX(CASE WHEN {when} THEN StatusCode END)" : $"MAX(CASE WHEN {when} THEN SourceTimeUtc END)");
|
||||
// A pick's columns from its joined row, in the established order: usable picks (kind, num, int, str, time), any-
|
||||
// quality picks (time, kind, num, int, str, status).
|
||||
static string Usable(string a) => $"{a}.ValueKind, {a}.NumValue, {a}.IntValue, {a}.StrValue, {a}.SourceTimeUtc";
|
||||
static string Any(string a) => $"{a}.SourceTimeUtc, {a}.ValueKind, {a}.NumValue, {a}.IntValue, {a}.StrValue, {a}.StatusCode";
|
||||
// An exact-key seek per pick, forced (see remarks). A usable pick also re-checks usability (the exact `u`
|
||||
// predicate), so a sample a HistoryUpdate Replace turned Bad/Empty in between joins nothing ⇒ absent ⇒ BadNoData,
|
||||
// never a Bad row's value.
|
||||
string Join(string a, string time, bool usable) =>
|
||||
$"LEFT JOIN {sample} {a} WITH (FORCESEEK (PK_Sample (TagId, SourceTimeUtc))) ON {a}.TagId = @id AND {a}.SourceTimeUtc = st.{time}" +
|
||||
(usable ? $" AND {a}.StatusCode >= 0 AND {a}.ValueKind <> 0" : "") + " ";
|
||||
return Classified(sample, reverse) +
|
||||
", w AS (SELECT b, SourceTimeUtc, ValueKind, NumValue, IntValue, StrValue, StatusCode, u, g, " +
|
||||
"ROW_NUMBER() OVER (PARTITION BY b ORDER BY SourceTimeUtc ASC) AS rfa, " +
|
||||
"ROW_NUMBER() OVER (PARTITION BY b ORDER BY SourceTimeUtc DESC) AS rla, " +
|
||||
"ROW_NUMBER() OVER (PARTITION BY b, u ORDER BY SourceTimeUtc ASC) AS rfu, " +
|
||||
"ROW_NUMBER() OVER (PARTITION BY b, u ORDER BY SourceTimeUtc DESC) AS rlu FROM s) " +
|
||||
"SELECT b, " + CommonCounts() + ", " +
|
||||
Pick("u = 1 AND rfu = 1", any: false) + ", " +
|
||||
Pick("u = 1 AND rlu = 1", any: false) + ", " +
|
||||
Pick("rfa = 1", any: true) + ", " +
|
||||
Pick("rla = 1", any: true) + " " +
|
||||
"FROM w GROUP BY b ORDER BY b;";
|
||||
", st AS (SELECT b, " + CommonCounts() + ", " +
|
||||
"MIN(CASE WHEN u = 1 THEN SourceTimeUtc END) AS fu, MAX(CASE WHEN u = 1 THEN SourceTimeUtc END) AS lu, " +
|
||||
"MIN(SourceTimeUtc) AS fa, MAX(SourceTimeUtc) AS la FROM s GROUP BY b) " +
|
||||
"SELECT st.b, st.TotalCount, st.UsableCount, st.GoodCount, st.BadCount, st.NonNumericCount, " +
|
||||
Usable("x") + ", " + Usable("y") + ", " + Any("p") + ", " + Any("q") + " FROM st " +
|
||||
Join("x", "fu", usable: true) + Join("y", "lu", usable: true) + Join("p", "fa", usable: false) + Join("q", "la", usable: false) + "ORDER BY st.b;";
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// MinimumActualTime / MaximumActualTime: the usable sample with the smallest (largest) <c>NumValue</c>,
|
||||
/// MinimumActualTime / MaximumActualTime: the <b>Good</b> sample (#684) with the smallest (largest) <c>NumValue</c>,
|
||||
/// ties broken by the earliest <c>SourceTimeUtc</c>. Family columns: <c>6 NumValue, 7 SourceTimeUtc</c>
|
||||
/// (NULL when the bucket has no usable numeric sample).
|
||||
/// (NULL when the bucket has no Good numeric sample), then <c>8 UncMin, 9 UncMax, 10 GoodNonNumericCount</c> as in <see cref="Simple"/>.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Deliberately still two references to <c>s</c> (counts + the ranked extreme), unlike <see cref="StartEnd"/>:
|
||||
@@ -124,11 +154,12 @@ internal static class SqlAggregateQueries
|
||||
/// <param name="reverse">True for a reverse read (window <c>(@start, @end]</c>).</param>
|
||||
public static string ActualTime(string sample, bool maximum, bool reverse = false) =>
|
||||
Classified(sample, reverse) +
|
||||
", st AS (SELECT b, " + CommonCounts() + " FROM s GROUP BY b)" +
|
||||
", st AS (SELECT b, " + CommonCounts() + ", " + UncertainExtremes() + " FROM s GROUP BY b)" +
|
||||
", r AS (SELECT b, NumValue, SourceTimeUtc, " +
|
||||
"ROW_NUMBER() OVER (PARTITION BY b ORDER BY NumValue " + (maximum ? "DESC" : "ASC") + ", SourceTimeUtc ASC) AS rn " +
|
||||
"FROM s WHERE u = 1 AND NumValue IS NOT NULL) " +
|
||||
"SELECT st.*, r.NumValue, r.SourceTimeUtc " +
|
||||
"FROM s WHERE g = 1 AND NumValue IS NOT NULL) " +
|
||||
"SELECT st.b, st.TotalCount, st.UsableCount, st.GoodCount, st.BadCount, st.NonNumericCount, " +
|
||||
"r.NumValue, r.SourceTimeUtc, st.UncMin, st.UncMax, st.GoodNonNumericCount " +
|
||||
"FROM st LEFT JOIN r ON r.b = st.b AND r.rn = 1 ORDER BY st.b;";
|
||||
|
||||
/// <summary>
|
||||
@@ -187,6 +218,15 @@ internal static class SqlAggregateQueries
|
||||
$"FROM {sample} WHERE TagId = @id AND " +
|
||||
(reverse ? "SourceTimeUtc > @start AND SourceTimeUtc <= @end) " : "SourceTimeUtc >= @start AND SourceTimeUtc < @end) ");
|
||||
|
||||
/// <summary>
|
||||
/// The minimum and maximum <b>Uncertain</b> (usable, not Good) numeric value of the bucket — <c>UncMin, UncMax</c>
|
||||
/// (#684). Not result values: Part 13 makes Minimum/Maximum/Range Uncertain when one lies beyond the Good extreme.
|
||||
/// Then <c>GoodNonNumericCount</c>, the Good String/DateTime samples (the Good-value aggregates' refusal check).
|
||||
/// </summary>
|
||||
private static string UncertainExtremes() =>
|
||||
"MIN(CASE WHEN u = 1 AND g = 0 THEN NumValue END) AS UncMin, MAX(CASE WHEN u = 1 AND g = 0 THEN NumValue END) AS UncMax, " +
|
||||
$"SUM(CASE WHEN g = 1 AND ValueKind IN ({NonNumericKinds}) THEN 1 ELSE 0 END) AS GoodNonNumericCount";
|
||||
|
||||
/// <summary>The five common count columns (after <c>b</c>).</summary>
|
||||
private static string CommonCounts() =>
|
||||
"COUNT(*) AS TotalCount, SUM(u) AS UsableCount, SUM(g) AS GoodCount, COUNT(*) - SUM(u) AS BadCount, " +
|
||||
|
||||
+26
-9
@@ -316,6 +316,10 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
|
||||
/// <remarks>Reverse events reads are served natively (<c>ORDER BY … DESC</c>, Gitea #675).</remarks>
|
||||
public bool SupportsReverseEvents => true;
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>The seventeen <see cref="AggregateFamilies.Served"/> — shared with the other SQL backend (#682).</remarks>
|
||||
public IReadOnlySet<HistoryAggregateType> SupportedAggregates => AggregateFamilies.Served;
|
||||
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>
|
||||
@@ -458,10 +462,13 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
|
||||
AggregateFamily.StartEnd => SqlAggregateQueries.StartEnd(sample, plan.Reversed),
|
||||
_ => SqlAggregateQueries.ActualTime(sample, aggregate == HistoryAggregateType.MaximumActualTime, plan.Reversed),
|
||||
};
|
||||
// Second result set: the tag's data range (earliest + latest sample) — the start/end-of-data Partial rules
|
||||
// (#677/#680 — any per-sample aggregate but Average, either direction) and Count's in-range empty interval (#680).
|
||||
// FIRST result set when needed: the tag's data range (earliest + latest sample) — the start/end-of-data Partial
|
||||
// rules (#677/#680 — any per-sample aggregate but Average, either direction) and Count's in-range empty interval
|
||||
// (#680). Read before the bucket statement (#683): under READ COMMITTED the two statements can see different
|
||||
// rows, and a sample appended in between must reach the buckets rather than only the range — the opposite
|
||||
// order turned such a bucket into a Good Count of 0.
|
||||
var needsRange = AggregateEvaluator.NeedsDataRange(aggregate, plan);
|
||||
if (needsRange) sql += " " + SqlAggregateQueries.DataRange(sample);
|
||||
if (needsRange) sql = SqlAggregateQueries.DataRange(sample) + " " + sql;
|
||||
await using var conn = await _conn.OpenAsync(cancellationToken).ConfigureAwait(false);
|
||||
await using var cmd = _conn.CreateCommand(conn, sql);
|
||||
cmd.Parameters.Add(new SqlParameter("@id", SqlDbType.Int) { Value = id });
|
||||
@@ -470,14 +477,18 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
|
||||
cmd.Parameters.Add(new SqlParameter("@intervalTicks", SqlDbType.BigInt) { Value = plan.IntervalTicks });
|
||||
cmd.Parameters.Add(new SqlParameter("@bucketAdjustTicks", SqlDbType.BigInt) { Value = plan.BucketIndexAdjustTicks });
|
||||
await using var reader = await cmd.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false);
|
||||
if (needsRange)
|
||||
{
|
||||
if (await reader.ReadAsync(cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
if (!reader.IsDBNull(0)) earliest = SqlHistorianConnection.GetUtc(reader, 0);
|
||||
if (!reader.IsDBNull(1)) latest = SqlHistorianConnection.GetUtc(reader, 1);
|
||||
}
|
||||
if (!await reader.NextResultAsync(cancellationToken).ConfigureAwait(false))
|
||||
throw new InvalidOperationException("The processed-read batch returned no bucket result set.");
|
||||
}
|
||||
while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false))
|
||||
rows.Add(ReadBucket(reader, family));
|
||||
if (needsRange && await reader.NextResultAsync(cancellationToken).ConfigureAwait(false)
|
||||
&& await reader.ReadAsync(cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
if (!reader.IsDBNull(0)) earliest = SqlHistorianConnection.GetUtc(reader, 0);
|
||||
if (!reader.IsDBNull(1)) latest = SqlHistorianConnection.GetUtc(reader, 1);
|
||||
}
|
||||
}
|
||||
|
||||
var result = AggregateEvaluator.EvaluateAll(aggregate, plan, rows, earliest, latest, now);
|
||||
@@ -560,6 +571,9 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
|
||||
row.Min = D(reader, f + 1);
|
||||
row.Max = D(reader, f + 2);
|
||||
row.Sum = D(reader, f + 3);
|
||||
row.UncertainMin = D(reader, f + 4);
|
||||
row.UncertainMax = D(reader, f + 5);
|
||||
row.GoodNonNumeric = reader.IsDBNull(f + 6) ? 0 : reader.GetInt32(f + 6);
|
||||
break;
|
||||
case AggregateFamily.StartEnd:
|
||||
row.First = V(reader, f);
|
||||
@@ -570,6 +584,9 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
|
||||
default:
|
||||
row.Extreme = D(reader, f);
|
||||
row.ExtremeTime = reader.IsDBNull(f + 1) ? null : SqlHistorianConnection.GetUtc(reader, f + 1);
|
||||
row.UncertainMin = D(reader, f + 2);
|
||||
row.UncertainMax = D(reader, f + 3);
|
||||
row.GoodNonNumeric = reader.IsDBNull(f + 4) ? 0 : reader.GetInt32(f + 4);
|
||||
break;
|
||||
}
|
||||
return row;
|
||||
|
||||
@@ -0,0 +1,91 @@
|
||||
using System.Reflection;
|
||||
using Opc.Ua;
|
||||
using Opc.Ua.Server;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.OpcUaServer;
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #682 — make <c>HistoryServerCapabilities/AggregateFunctions</c> (i=11201) list exactly the aggregates the
|
||||
/// wired historian backend serves for HistoryRead (<see cref="IHistorianDataSource.SupportedAggregates"/>), instead of
|
||||
/// the SDK's full registered set.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// ⚠️ <c>ServerCapabilities/AggregateFunctions</c> (i=2997) is deliberately NOT trimmed. Part 5 defines it as every
|
||||
/// aggregate the SERVER supports, and the server still supports all the SDK's registered aggregates for live
|
||||
/// data: a MonitoredItem <c>AggregateFilter</c> is validated and computed by the SDK's <c>AggregateManager</c>,
|
||||
/// not the historian. The SDK itself models the split — <c>DiagnosticsNodeManager.AddAggregateFunction</c> always
|
||||
/// adds to i=2997 and adds to i=11201 only when the aggregate is historical. i=11201 is the history surface, so it
|
||||
/// is the one that follows the backend.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Only the folder ⇄ aggregate <c>Organizes</c> reference pair is removed or restored (both directions); the
|
||||
/// calculator factories and the aggregate-function object nodes stay. Idempotent and reversible, so re-wiring a
|
||||
/// different source re-syncs the folder.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
internal static class AdvertisedAggregateFunctions
|
||||
{
|
||||
/// <summary>Every standard aggregate-function NodeId the SDK's nodeset knows (<c>ObjectIds.AggregateFunction_*</c>).</summary>
|
||||
private static readonly NodeId[] StandardAggregateIds = typeof(ObjectIds)
|
||||
.GetFields(BindingFlags.Public | BindingFlags.Static)
|
||||
.Where(f => f.FieldType == typeof(NodeId) && f.Name.StartsWith("AggregateFunction_", StringComparison.Ordinal))
|
||||
.Select(f => (NodeId)f.GetValue(null)!)
|
||||
.ToArray();
|
||||
|
||||
/// <summary>The trimmed folder — the HISTORY aggregate list only (see remarks for why not i=2997).</summary>
|
||||
internal static readonly NodeId Folder = ObjectIds.HistoryServerCapabilities_AggregateFunctions;
|
||||
|
||||
/// <summary>
|
||||
/// The set a source serves, treating a <c>null</c> <see cref="IHistorianDataSource.SupportedAggregates"/> (an
|
||||
/// out-of-tree source that returns null despite the non-nullable contract) as <see cref="HistoryAggregateTypes.All"/>,
|
||||
/// the interface default — the ONE null rule both the advertisement and HistoryRead-Processed use.
|
||||
/// </summary>
|
||||
/// <param name="source">The historian source.</param>
|
||||
/// <returns>Its served set, never null.</returns>
|
||||
public static IReadOnlySet<HistoryAggregateType> ServedBy(IHistorianDataSource source) =>
|
||||
source.SupportedAggregates ?? HistoryAggregateTypes.All;
|
||||
|
||||
/// <summary>
|
||||
/// Sync i=11201 to <paramref name="served"/>: an aggregate-function node is organized under it iff the server maps
|
||||
/// it (<see cref="AggregateFunctionMap"/>) AND <paramref name="served"/> contains it.
|
||||
/// </summary>
|
||||
/// <param name="server">The running server.</param>
|
||||
/// <param name="served">The backend's served aggregates.</param>
|
||||
/// <returns>The advertised NodeIds, or <c>null</c> when the diagnostics node manager or folder is not available.</returns>
|
||||
public static IReadOnlyList<NodeId>? Apply(IServerInternal server, IReadOnlySet<HistoryAggregateType> served)
|
||||
{
|
||||
if (server?.DiagnosticsNodeManager is not CustomNodeManager2 diagnostics) return null;
|
||||
|
||||
var advertised = new List<NodeId>();
|
||||
lock (diagnostics.Lock)
|
||||
{
|
||||
var folder = diagnostics.FindPredefinedNode(Folder, typeof(BaseObjectState));
|
||||
if (folder is null) return null;
|
||||
|
||||
foreach (var aggregateId in StandardAggregateIds)
|
||||
{
|
||||
var aggregateNode = diagnostics.FindPredefinedNode(aggregateId, null);
|
||||
if (aggregateNode is null) continue;
|
||||
|
||||
var advertise = AggregateFunctionMap.Map(aggregateId) is { } aggregate && served.Contains(aggregate);
|
||||
if (advertise) advertised.Add(aggregateId);
|
||||
|
||||
var linked = folder.ReferenceExists(ReferenceTypeIds.Organizes, false, aggregateId);
|
||||
if (advertise && !linked)
|
||||
{
|
||||
folder.AddReference(ReferenceTypeIds.Organizes, false, aggregateId);
|
||||
aggregateNode.AddReference(ReferenceTypeIds.Organizes, true, folder.NodeId);
|
||||
}
|
||||
else if (!advertise && linked)
|
||||
{
|
||||
folder.RemoveReference(ReferenceTypeIds.Organizes, false, aggregateId);
|
||||
aggregateNode.RemoveReference(ReferenceTypeIds.Organizes, true, folder.NodeId);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return advertised;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
using System.Collections.Frozen;
|
||||
using Opc.Ua;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.OpcUaServer;
|
||||
|
||||
/// <summary>
|
||||
/// The one table between OPC UA Part 13 standard aggregate-function NodeIds (<c>ObjectIds.AggregateFunction_*</c>)
|
||||
/// and <see cref="HistoryAggregateType"/>. HistoryRead-Processed maps a request's aggregate through it, and the
|
||||
/// advertised <c>AggregateFunctions</c> folders (#682) map a backend's served set back through it — one table, so a
|
||||
/// NodeId the server can dispatch is exactly a NodeId it can advertise.
|
||||
/// </summary>
|
||||
internal static class AggregateFunctionMap
|
||||
{
|
||||
/// <summary>NodeId ⇒ aggregate, for every aggregate the server can name to a backend.</summary>
|
||||
public static readonly FrozenDictionary<NodeId, HistoryAggregateType> ByNodeId = new Dictionary<NodeId, HistoryAggregateType>
|
||||
{
|
||||
[ObjectIds.AggregateFunction_Average] = HistoryAggregateType.Average,
|
||||
[ObjectIds.AggregateFunction_Minimum] = HistoryAggregateType.Minimum,
|
||||
[ObjectIds.AggregateFunction_Maximum] = HistoryAggregateType.Maximum,
|
||||
[ObjectIds.AggregateFunction_Total] = HistoryAggregateType.Total,
|
||||
[ObjectIds.AggregateFunction_Count] = HistoryAggregateType.Count,
|
||||
[ObjectIds.AggregateFunction_Interpolative] = HistoryAggregateType.Interpolative,
|
||||
[ObjectIds.AggregateFunction_TimeAverage] = HistoryAggregateType.TimeAverage,
|
||||
[ObjectIds.AggregateFunction_Start] = HistoryAggregateType.Start,
|
||||
[ObjectIds.AggregateFunction_End] = HistoryAggregateType.End,
|
||||
[ObjectIds.AggregateFunction_Delta] = HistoryAggregateType.Delta,
|
||||
[ObjectIds.AggregateFunction_Range] = HistoryAggregateType.Range,
|
||||
[ObjectIds.AggregateFunction_MinimumActualTime] = HistoryAggregateType.MinimumActualTime,
|
||||
[ObjectIds.AggregateFunction_MaximumActualTime] = HistoryAggregateType.MaximumActualTime,
|
||||
[ObjectIds.AggregateFunction_StartBound] = HistoryAggregateType.StartBound,
|
||||
[ObjectIds.AggregateFunction_EndBound] = HistoryAggregateType.EndBound,
|
||||
[ObjectIds.AggregateFunction_DeltaBounds] = HistoryAggregateType.DeltaBounds,
|
||||
[ObjectIds.AggregateFunction_DurationGood] = HistoryAggregateType.DurationGood,
|
||||
[ObjectIds.AggregateFunction_DurationBad] = HistoryAggregateType.DurationBad,
|
||||
[ObjectIds.AggregateFunction_PercentGood] = HistoryAggregateType.PercentGood,
|
||||
[ObjectIds.AggregateFunction_PercentBad] = HistoryAggregateType.PercentBad,
|
||||
[ObjectIds.AggregateFunction_WorstQuality] = HistoryAggregateType.WorstQuality,
|
||||
[ObjectIds.AggregateFunction_NumberOfTransitions] = HistoryAggregateType.NumberOfTransitions,
|
||||
[ObjectIds.AggregateFunction_AnnotationCount] = HistoryAggregateType.AnnotationCount,
|
||||
[ObjectIds.AggregateFunction_StandardDeviationPopulation] = HistoryAggregateType.StandardDeviationPopulation,
|
||||
[ObjectIds.AggregateFunction_VariancePopulation] = HistoryAggregateType.VariancePopulation,
|
||||
[ObjectIds.AggregateFunction_TimeAverage2] = HistoryAggregateType.TimeAverage2,
|
||||
[ObjectIds.AggregateFunction_Total2] = HistoryAggregateType.Total2,
|
||||
}.ToFrozenDictionary();
|
||||
|
||||
/// <summary>
|
||||
/// Map a request's aggregate-function NodeId to <see cref="HistoryAggregateType"/>; <c>null</c> for one the server
|
||||
/// cannot name to any backend (the caller answers <c>BadAggregateNotSupported</c>).
|
||||
/// </summary>
|
||||
/// <param name="aggregateNodeId">The per-node aggregate-function NodeId from the request.</param>
|
||||
/// <returns>The mapped aggregate, or <c>null</c>.</returns>
|
||||
public static HistoryAggregateType? Map(NodeId? aggregateNodeId) =>
|
||||
aggregateNodeId is not null && ByNodeId.TryGetValue(aggregateNodeId, out var aggregate) ? aggregate : null;
|
||||
}
|
||||
@@ -344,11 +344,49 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2
|
||||
/// <see cref="NodeWriteGateway"/>, the HistoryRead override does NOT run under the
|
||||
/// node-manager <c>Lock</c>, so the override may block-bridge to this (async) source.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Gitea #682: assigning a source also re-syncs <c>HistoryServerCapabilities/AggregateFunctions</c> to its
|
||||
/// <see cref="IHistorianDataSource.SupportedAggregates"/> (see <see cref="SyncAdvertisedAggregates"/>).
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public IHistorianDataSource HistorianDataSource
|
||||
{
|
||||
get => _historianDataSource;
|
||||
set => _historianDataSource = value ?? NullHistorianDataSource.Instance;
|
||||
set
|
||||
{
|
||||
_historianDataSource = value ?? NullHistorianDataSource.Instance;
|
||||
SyncAdvertisedAggregates();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #682 — trim <c>HistoryServerCapabilities/AggregateFunctions</c> (i=11201) to the current source's
|
||||
/// <see cref="IHistorianDataSource.SupportedAggregates"/> (null ⇒ all, <see cref="AdvertisedAggregateFunctions.ServedBy"/>).
|
||||
/// <c>ServerCapabilities/AggregateFunctions</c> (i=2997) keeps the SDK's full set — the server still computes every
|
||||
/// one of them for live <c>AggregateFilter</c> subscriptions. Called on every <see cref="HistorianDataSource"/>
|
||||
/// assignment and by <see cref="OtOpcUaSdkServer"/> right after the SDK registers its aggregate factories, so the
|
||||
/// Null default is applied from boot. Only the folder references change — the AggregateManager's calculator
|
||||
/// factories (live <c>AggregateFilter</c> subscriptions) are untouched. Never throws: a failure leaves the previous
|
||||
/// advertisement in place and is logged, because HistoryRead's own up-front check still enforces the set.
|
||||
/// </summary>
|
||||
internal void SyncAdvertisedAggregates()
|
||||
{
|
||||
try
|
||||
{
|
||||
var source = _historianDataSource;
|
||||
var advertised = AdvertisedAggregateFunctions.Apply(Server, AdvertisedAggregateFunctions.ServedBy(source));
|
||||
if (advertised is not null)
|
||||
{
|
||||
_logger.LogInformation(
|
||||
"OtOpcUaNodeManager: advertising {Count} history aggregate function(s) for historian {Source}: {Aggregates}",
|
||||
advertised.Count, source.GetType().Name,
|
||||
string.Join(", ", advertised.Select(id => AggregateFunctionMap.Map(id)?.ToString() ?? id.ToString())));
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "OtOpcUaNodeManager: failed to sync the advertised aggregate functions");
|
||||
}
|
||||
}
|
||||
|
||||
private volatile IHistorianDataUpdater? _historianDataUpdater;
|
||||
@@ -5054,6 +5092,7 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2
|
||||
// #677: a client AggregateConfiguration other than the server defaults is refused per node, never silently ignored.
|
||||
var configurationServable = ServerAggregateConfiguration.IsServable(details.AggregateConfiguration);
|
||||
nodesToProcess = AuthorizeHistoryRead(context, nodesToProcess, results, errors);
|
||||
var served = AdvertisedAggregateFunctions.ServedBy(HistorianDataSource);
|
||||
WithHistoryReadBatchLimiter(nodesToProcess, results, errors, () =>
|
||||
{
|
||||
using var deadline = CreateDeadline();
|
||||
@@ -5064,8 +5103,11 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2
|
||||
// AggregateType is a per-node parallel collection (same length as nodesToRead, enforced by
|
||||
// the base dispatcher). handle.Index is the node's position in that collection.
|
||||
var aggregateNodeId = details.AggregateType[handle.Index];
|
||||
var aggregate = MapAggregate(aggregateNodeId);
|
||||
if (aggregate is null)
|
||||
// #682: one authority — the source's SupportedAggregates (also what HistoryServerCapabilities/
|
||||
// AggregateFunctions advertises; null ⇒ all, the same rule). An aggregate the server cannot name, or the
|
||||
// backend does not serve, is refused up front without a backend call.
|
||||
var aggregate = AggregateFunctionMap.Map(aggregateNodeId);
|
||||
if (aggregate is null || !served.Contains(aggregate.Value))
|
||||
{
|
||||
errors[handle.Index] = StatusCodes.BadAggregateNotSupported;
|
||||
continue;
|
||||
@@ -6478,45 +6520,6 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2
|
||||
return new HistoryData { DataValues = values };
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Map an OPC UA Part 13 standard-aggregate function NodeId to our
|
||||
/// <see cref="HistoryAggregateType"/>. Returns <c>null</c> for any aggregate we don't serve so
|
||||
/// the caller can surface <c>BadAggregateNotSupported</c>.
|
||||
/// </summary>
|
||||
/// <param name="aggregateNodeId">The per-node aggregate-function NodeId from the request.</param>
|
||||
/// <returns>The mapped aggregate, or <c>null</c> when unsupported.</returns>
|
||||
private static HistoryAggregateType? MapAggregate(NodeId aggregateNodeId)
|
||||
{
|
||||
if (aggregateNodeId == ObjectIds.AggregateFunction_Average) return HistoryAggregateType.Average;
|
||||
if (aggregateNodeId == ObjectIds.AggregateFunction_Minimum) return HistoryAggregateType.Minimum;
|
||||
if (aggregateNodeId == ObjectIds.AggregateFunction_Maximum) return HistoryAggregateType.Maximum;
|
||||
if (aggregateNodeId == ObjectIds.AggregateFunction_Total) return HistoryAggregateType.Total;
|
||||
if (aggregateNodeId == ObjectIds.AggregateFunction_Count) return HistoryAggregateType.Count;
|
||||
if (aggregateNodeId == ObjectIds.AggregateFunction_Interpolative) return HistoryAggregateType.Interpolative;
|
||||
if (aggregateNodeId == ObjectIds.AggregateFunction_TimeAverage) return HistoryAggregateType.TimeAverage;
|
||||
if (aggregateNodeId == ObjectIds.AggregateFunction_Start) return HistoryAggregateType.Start;
|
||||
if (aggregateNodeId == ObjectIds.AggregateFunction_End) return HistoryAggregateType.End;
|
||||
if (aggregateNodeId == ObjectIds.AggregateFunction_Delta) return HistoryAggregateType.Delta;
|
||||
if (aggregateNodeId == ObjectIds.AggregateFunction_Range) return HistoryAggregateType.Range;
|
||||
if (aggregateNodeId == ObjectIds.AggregateFunction_MinimumActualTime) return HistoryAggregateType.MinimumActualTime;
|
||||
if (aggregateNodeId == ObjectIds.AggregateFunction_MaximumActualTime) return HistoryAggregateType.MaximumActualTime;
|
||||
if (aggregateNodeId == ObjectIds.AggregateFunction_StartBound) return HistoryAggregateType.StartBound;
|
||||
if (aggregateNodeId == ObjectIds.AggregateFunction_EndBound) return HistoryAggregateType.EndBound;
|
||||
if (aggregateNodeId == ObjectIds.AggregateFunction_DeltaBounds) return HistoryAggregateType.DeltaBounds;
|
||||
if (aggregateNodeId == ObjectIds.AggregateFunction_DurationGood) return HistoryAggregateType.DurationGood;
|
||||
if (aggregateNodeId == ObjectIds.AggregateFunction_DurationBad) return HistoryAggregateType.DurationBad;
|
||||
if (aggregateNodeId == ObjectIds.AggregateFunction_PercentGood) return HistoryAggregateType.PercentGood;
|
||||
if (aggregateNodeId == ObjectIds.AggregateFunction_PercentBad) return HistoryAggregateType.PercentBad;
|
||||
if (aggregateNodeId == ObjectIds.AggregateFunction_WorstQuality) return HistoryAggregateType.WorstQuality;
|
||||
if (aggregateNodeId == ObjectIds.AggregateFunction_NumberOfTransitions) return HistoryAggregateType.NumberOfTransitions;
|
||||
if (aggregateNodeId == ObjectIds.AggregateFunction_AnnotationCount) return HistoryAggregateType.AnnotationCount;
|
||||
if (aggregateNodeId == ObjectIds.AggregateFunction_StandardDeviationPopulation) return HistoryAggregateType.StandardDeviationPopulation;
|
||||
if (aggregateNodeId == ObjectIds.AggregateFunction_VariancePopulation) return HistoryAggregateType.VariancePopulation;
|
||||
if (aggregateNodeId == ObjectIds.AggregateFunction_TimeAverage2) return HistoryAggregateType.TimeAverage2;
|
||||
if (aggregateNodeId == ObjectIds.AggregateFunction_Total2) return HistoryAggregateType.Total2;
|
||||
return null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Project the historian source's <see cref="HistorianRead"/> (Core.Abstractions DTO) into an
|
||||
/// SDK <see cref="HistoryData"/> — one <see cref="DataValue"/> per <see cref="DataValueSnapshot"/>,
|
||||
|
||||
@@ -200,6 +200,22 @@ public sealed class OtOpcUaSdkServer : StandardServer
|
||||
return true;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #682 — keep the SDK's full set of calculator factories (live-data <c>AggregateFilter</c> subscriptions
|
||||
/// need them, and <c>ServerCapabilities/AggregateFunctions</c> keeps listing all of them), then trim
|
||||
/// <c>HistoryServerCapabilities/AggregateFunctions</c> to the node manager's current historian source. At this
|
||||
/// point that is the Null default (no history aggregates); the Host's <see cref="SetHistorianDataSource"/>
|
||||
/// re-syncs to the configured backend's set right after start.
|
||||
/// </summary>
|
||||
/// <inheritdoc />
|
||||
protected override AggregateManager CreateAggregateManager(
|
||||
IServerInternal server, ApplicationConfiguration configuration)
|
||||
{
|
||||
var manager = base.CreateAggregateManager(server, configuration);
|
||||
_otOpcUaNodeManager?.SyncAdvertisedAggregates();
|
||||
return manager;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override MasterNodeManager CreateMasterNodeManager(
|
||||
IServerInternal server, ApplicationConfiguration configuration)
|
||||
|
||||
@@ -9,8 +9,10 @@ namespace ZB.MOM.WW.OtOpcUa.OpcUaServer;
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// The aggregate evaluators (<c>AggregateEvaluator</c> / <c>TimeWeightedAggregator</c> in
|
||||
/// <c>Driver.Historian.Sql.Common</c>) are written against fixed settings, not parameterised: Uncertain samples and
|
||||
/// Uncertain time count as Good (<see cref="TreatUncertainAsBad"/> = false); an interval is Good only when all of
|
||||
/// <c>Driver.Historian.Sql.Common</c>) are written against fixed settings, not parameterised: Uncertain time counts as
|
||||
/// Good (<see cref="TreatUncertainAsBad"/> = false) in the time-weighted family; the per-sample Average / Minimum /
|
||||
/// Maximum / Range / *ActualTime / Count are defined by Part 13 over Good values whatever this setting (#684), as the
|
||||
/// SDK computes them under it; an interval is Good only when all of
|
||||
/// it is (<see cref="PercentDataGood"/> = 100) and Bad only when none of it is non-Bad
|
||||
/// (<see cref="PercentDataBad"/> = 100), otherwise <c>UncertainDataSubNormal</c>; the value past the last sample
|
||||
/// is held (stepped, <see cref="UseSlopedExtrapolation"/> = false). These are also the SDK's own server defaults
|
||||
|
||||
+15
@@ -63,4 +63,19 @@ public sealed class HistorianDataSourceReverseEventsGuardTests
|
||||
$"{type.FullName} must declare SupportsReverseEvents itself, not inherit the interface default");
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Every_implementation_declares_its_supported_aggregates_itself()
|
||||
{
|
||||
// Gitea #682: the interface default is "every aggregate" — a production backend (or a future wrapper) that
|
||||
// inherited it would advertise all of them under AggregateFunctions and forward reads it cannot serve.
|
||||
foreach (var type in ProductionImplementations())
|
||||
{
|
||||
var map = type.GetInterfaceMap(typeof(IHistorianDataSource));
|
||||
var getter = map.TargetMethods[Array.FindIndex(map.InterfaceMethods,
|
||||
m => m.Name == "get_" + nameof(IHistorianDataSource.SupportedAggregates))];
|
||||
getter.DeclaringType.ShouldBe(type,
|
||||
$"{type.FullName} must declare SupportedAggregates itself, not inherit the interface default");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -351,12 +351,46 @@ public abstract class ProcessedConformance : ConformanceBase
|
||||
}
|
||||
|
||||
/// <summary>Dense: every 2.5 s from −10 to +100, values cycling, one Bad sample at +22.5 (no empty interval).</summary>
|
||||
private Task SeedDenseAsync() => B.SeedAsync("P", DenseRows());
|
||||
private Task SeedDenseAsync(bool uncertain = false) => B.SeedAsync(uncertain ? "PU" : "P", DenseRows(uncertain));
|
||||
|
||||
private static (DateTime T, HistorianValue V, uint Status)[] DenseRows() => Enumerable.Range(0, 45)
|
||||
.Select(i => (At(-10 + i * 2.5), HistorianValue.FromDouble(10 + (i * 7 % 13)), i == 13 ? Bad : Good))
|
||||
/// <summary>
|
||||
/// #684 <paramref name="uncertain"/>: every fifth sample (other than the Bad one) Uncertain at 16 — inside every
|
||||
/// interval's Good range (the cycling values alternate around 16), so the min/max family matches the SDK unmasked
|
||||
/// (SdkAggregateParityTests.DenseUncertain is the same data).
|
||||
/// </summary>
|
||||
private static (DateTime T, HistorianValue V, uint Status)[] DenseRows(bool uncertain = false) => Enumerable.Range(0, 45)
|
||||
.Select(i => uncertain && i % 5 == 0 && i != 13
|
||||
? (At(-10 + i * 2.5), HistorianValue.FromDouble(16), Uncertain)
|
||||
: (At(-10 + i * 2.5), HistorianValue.FromDouble(10 + (i * 7 % 13)), i == 13 ? Bad : Good))
|
||||
.ToArray();
|
||||
|
||||
[Theory]
|
||||
[InlineData(0, 80, 20)]
|
||||
[InlineData(80, 0, 20)]
|
||||
[InlineData(77, 3, 20)]
|
||||
[InlineData(3, 77, 20)]
|
||||
public async Task Per_sample_processed_reads_over_uncertain_samples_match_the_SDK(double from, double to, double intervalS)
|
||||
{
|
||||
// #684: the Good-only value columns through the real SQL against the SDK's calculators, over Dense with Uncertain
|
||||
// samples — values, timestamps, code bits and Partial, no mask.
|
||||
await SeedDenseAsync(uncertain: true);
|
||||
var rows = DenseRows(uncertain: true).Select(r => (B.Round(r.T), r.V, r.Status)).ToArray();
|
||||
var src = B.CreateDataSource();
|
||||
var interval = TimeSpan.FromSeconds(intervalS);
|
||||
var (start, end) = (B.Round(At(from)), B.Round(At(to)));
|
||||
foreach (var agg in new[]
|
||||
{
|
||||
HistoryAggregateType.Average, HistoryAggregateType.Minimum, HistoryAggregateType.Maximum,
|
||||
HistoryAggregateType.Count, HistoryAggregateType.Range, HistoryAggregateType.MinimumActualTime,
|
||||
HistoryAggregateType.MaximumActualTime,
|
||||
})
|
||||
{
|
||||
var ours = (await src.ReadProcessedAsync("PU", At(from), At(to), interval, agg, Ct)).Samples.ToList();
|
||||
SdkAggregateOracle.ShouldMatch(ours, SdkAggregateOracle.Run(agg, start, end, interval, rows),
|
||||
$"{agg} uncertain {from}→{to}/{intervalS}s", compareCalculatedInterpolatedBits: false);
|
||||
}
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(0, 80, 20)]
|
||||
[InlineData(80, 0, 20)]
|
||||
@@ -476,14 +510,142 @@ public abstract class ProcessedConformance : ConformanceBase
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Uncertain_samples_are_used_but_make_the_interval_uncertain()
|
||||
public async Task Uncertain_samples_are_left_out_of_the_value_and_degrade_the_status_per_Part_13()
|
||||
{
|
||||
// #684 (Part 13 §5.4.3.5/10-14; the SDK's calculators): Average / Minimum / Maximum / Range / *ActualTime use GOOD
|
||||
// values only. One interval [0,20): Good 6 @+2, Good 8 @+3, Uncertain 4 @+1 (below the Good minimum) and
|
||||
// Uncertain 7 @+4 (inside the Good range); a Good sample before (-5) and after (+30) the window keep the data range
|
||||
// around it (no Partial). Pre-#684 the backends averaged 6.25 and answered Minimum 4.
|
||||
await B.SeedAsync("U",
|
||||
(At(-5), HistorianValue.FromDouble(100), Good),
|
||||
(At(1), HistorianValue.FromDouble(4), Uncertain),
|
||||
(At(2), HistorianValue.FromDouble(6), Good));
|
||||
var s = (await B.CreateDataSource().ReadProcessedAsync("U", At(0), At(20), Twenty, HistoryAggregateType.Average, Ct)).Samples.Single();
|
||||
s.Value.ShouldBe(5d);
|
||||
s.StatusCode.ShouldBe(UncertainCalc);
|
||||
(At(2), HistorianValue.FromDouble(6), Good),
|
||||
(At(3), HistorianValue.FromDouble(8), Good),
|
||||
(At(4), HistorianValue.FromDouble(7), Uncertain),
|
||||
(At(30), HistorianValue.FromDouble(100), Good));
|
||||
var src = B.CreateDataSource();
|
||||
async Task<DataValueSnapshot> One(HistoryAggregateType a) =>
|
||||
(await src.ReadProcessedAsync("U", At(0), At(20), Twenty, a, Ct)).Samples.Single();
|
||||
|
||||
// Average: value-based status — not every sample is Good ⇒ Uncertain.
|
||||
var avg = await One(HistoryAggregateType.Average);
|
||||
avg.Value.ShouldBe(7d);
|
||||
avg.StatusCode.ShouldBe(UncertainCalc);
|
||||
// Minimum: Uncertain 4 is below the Good minimum 6 ⇒ Uncertain (Part 13 Table 57), value still 6.
|
||||
var min = await One(HistoryAggregateType.Minimum);
|
||||
min.Value.ShouldBe(6d);
|
||||
min.StatusCode.ShouldBe(UncertainCalc);
|
||||
// Maximum: no Uncertain value above 8 and no Bad sample ⇒ Good.
|
||||
var max = await One(HistoryAggregateType.Maximum);
|
||||
max.Value.ShouldBe(8d);
|
||||
max.StatusCode.ShouldBe(GoodCalc);
|
||||
var range = await One(HistoryAggregateType.Range);
|
||||
range.Value.ShouldBe(2d);
|
||||
range.StatusCode.ShouldBe(UncertainCalc);
|
||||
var minAt = await One(HistoryAggregateType.MinimumActualTime);
|
||||
minAt.Value.ShouldBe(6d);
|
||||
minAt.SourceTimestampUtc.ShouldBe(B.Round(At(2)));
|
||||
minAt.StatusCode.ShouldBe(UncertainCalc);
|
||||
var maxAt = await One(HistoryAggregateType.MaximumActualTime);
|
||||
maxAt.Value.ShouldBe(8d);
|
||||
maxAt.SourceTimestampUtc.ShouldBe(B.Round(At(3)));
|
||||
maxAt.StatusCode.ShouldBe(GoodCalc);
|
||||
// Count (#680, unchanged): the two Good samples, Uncertain.
|
||||
(await One(HistoryAggregateType.Count)).Value.ShouldBe(2);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task An_uncertain_value_above_the_good_maximum_makes_the_maximum_family_uncertain()
|
||||
{
|
||||
// #684 review: pins the Uncertain-MAXIMUM column (UncMax / unc_max) through the real SQL — Good 6 @+2, Good 8 @+3,
|
||||
// Uncertain 9 @+4 (above the Good maximum, nothing below the minimum). Part 13 Tables 58/60/62 ⇒ Maximum,
|
||||
// MaximumActualTime and Range Uncertain (values still from the Good samples); Minimum Good.
|
||||
await B.SeedAsync("UA",
|
||||
(At(-5), HistorianValue.FromDouble(7), Good),
|
||||
(At(2), HistorianValue.FromDouble(6), Good),
|
||||
(At(3), HistorianValue.FromDouble(8), Good),
|
||||
(At(4), HistorianValue.FromDouble(9), Uncertain),
|
||||
(At(30), HistorianValue.FromDouble(7), Good));
|
||||
var src = B.CreateDataSource();
|
||||
async Task<DataValueSnapshot> One(HistoryAggregateType a) =>
|
||||
(await src.ReadProcessedAsync("UA", At(0), At(20), Twenty, a, Ct)).Samples.Single();
|
||||
|
||||
var max = await One(HistoryAggregateType.Maximum);
|
||||
max.Value.ShouldBe(8d);
|
||||
max.StatusCode.ShouldBe(UncertainCalc);
|
||||
var maxAt = await One(HistoryAggregateType.MaximumActualTime);
|
||||
maxAt.Value.ShouldBe(8d);
|
||||
maxAt.SourceTimestampUtc.ShouldBe(B.Round(At(3)));
|
||||
maxAt.StatusCode.ShouldBe(UncertainCalc);
|
||||
var range = await One(HistoryAggregateType.Range);
|
||||
range.Value.ShouldBe(2d);
|
||||
range.StatusCode.ShouldBe(UncertainCalc);
|
||||
var min = await One(HistoryAggregateType.Minimum);
|
||||
min.Value.ShouldBe(6d);
|
||||
min.StatusCode.ShouldBe(GoodCalc);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task An_uncertain_string_sample_does_not_refuse_the_good_value_aggregates()
|
||||
{
|
||||
// #684 review: the Good-value aggregates refuse only a GOOD String/DateTime sample (the good-non-numeric count the
|
||||
// SQL now returns); an Uncertain one is skipped with an Uncertain status — the SDK's behaviour. Delta keeps its
|
||||
// usable-sample refusal.
|
||||
await B.SeedAsync("US",
|
||||
(At(-5), HistorianValue.FromDouble(7), Good),
|
||||
(At(2), HistorianValue.FromDouble(6), Good),
|
||||
(At(3), HistorianValue.FromString("x"), Uncertain),
|
||||
(At(4), HistorianValue.FromDouble(8), Good),
|
||||
(At(30), HistorianValue.FromDouble(7), Good));
|
||||
var src = B.CreateDataSource();
|
||||
async Task<DataValueSnapshot> One(HistoryAggregateType a) =>
|
||||
(await src.ReadProcessedAsync("US", At(0), At(20), Twenty, a, Ct)).Samples.Single();
|
||||
|
||||
foreach (var (agg, value) in new[]
|
||||
{
|
||||
(HistoryAggregateType.Average, 7d), (HistoryAggregateType.Minimum, 6d), (HistoryAggregateType.Maximum, 8d),
|
||||
(HistoryAggregateType.Range, 2d), (HistoryAggregateType.MinimumActualTime, 6d),
|
||||
(HistoryAggregateType.MaximumActualTime, 8d),
|
||||
})
|
||||
{
|
||||
var v = await One(agg);
|
||||
v.Value.ShouldBe(value, agg.ToString());
|
||||
v.StatusCode.ShouldBe(UncertainCalc, agg.ToString());
|
||||
}
|
||||
(await One(HistoryAggregateType.Delta)).StatusCode.ShouldBe(InvalidInputsCalc);
|
||||
|
||||
// A GOOD string in the same shape still refuses them.
|
||||
await B.SeedAsync("GS",
|
||||
(At(-5), HistorianValue.FromDouble(7), Good),
|
||||
(At(2), HistorianValue.FromDouble(6), Good),
|
||||
(At(3), HistorianValue.FromString("x"), Good),
|
||||
(At(30), HistorianValue.FromDouble(7), Good));
|
||||
(await src.ReadProcessedAsync("GS", At(0), At(20), Twenty, HistoryAggregateType.Maximum, Ct)).Samples.Single()
|
||||
.StatusCode.ShouldBe(InvalidInputsCalc);
|
||||
(await src.ReadProcessedAsync("GS", At(0), At(20), Twenty, HistoryAggregateType.MinimumActualTime, Ct)).Samples
|
||||
.Single().StatusCode.ShouldBe(InvalidInputsCalc);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task An_interval_of_only_uncertain_samples_is_BadNoData_for_the_good_value_aggregates()
|
||||
{
|
||||
// #684: Part 13 defines these over Good values; the SDK returns no-data when an interval has none.
|
||||
await B.SeedAsync("UO",
|
||||
(At(-5), HistorianValue.FromDouble(1), Good),
|
||||
(At(1), HistorianValue.FromDouble(4), Uncertain),
|
||||
(At(2), HistorianValue.FromDouble(6), Uncertain),
|
||||
(At(30), HistorianValue.FromDouble(1), Good));
|
||||
var src = B.CreateDataSource();
|
||||
foreach (var agg in new[]
|
||||
{
|
||||
HistoryAggregateType.Average, HistoryAggregateType.Minimum, HistoryAggregateType.Maximum,
|
||||
HistoryAggregateType.Range, HistoryAggregateType.MinimumActualTime, HistoryAggregateType.MaximumActualTime,
|
||||
})
|
||||
{
|
||||
var v = (await src.ReadProcessedAsync("UO", At(0), At(20), Twenty, agg, Ct)).Samples.Single();
|
||||
v.Value.ShouldBeNull(agg.ToString());
|
||||
v.StatusCode.ShouldBe(NoDataCalc, agg.ToString());
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
|
||||
+24
-1
@@ -19,6 +19,17 @@ public sealed class FakeHistorianGatewayClient : IHistorianGatewayClient
|
||||
public DateTime LastReadRawEndUtc;
|
||||
public int LastReadRawMaxValues;
|
||||
public int ReadRawCallCount;
|
||||
/// <summary>
|
||||
/// When set, ReadRaw serves <c>RawSource(start, end, maxValues)</c> instead of <see cref="RawSamples"/> (a
|
||||
/// store-like fake honouring the window, #681).
|
||||
/// </summary>
|
||||
public Func<DateTime, DateTime, int, IEnumerable<HistorianSample>>? RawSource;
|
||||
/// <summary>Raw samples actually pulled by the consumer across every ReadRaw stream.</summary>
|
||||
public int RawYieldedCount;
|
||||
/// <summary>ReadRaw streams the consumer enumerated to the END (never broke out of — #681 review).</summary>
|
||||
public int RawStreamsCompleted;
|
||||
/// <summary>Every ReadRaw call's (start, end, maxValues), in order.</summary>
|
||||
public readonly List<(DateTime Start, DateTime End, int MaxValues)> RawCalls = new();
|
||||
|
||||
// ---- ReadAggregate -------------------------------------------------------------------------
|
||||
public IReadOnlyList<HistorianAggregateSample> AggregateSamples = Array.Empty<HistorianAggregateSample>();
|
||||
@@ -90,7 +101,9 @@ public sealed class FakeHistorianGatewayClient : IHistorianGatewayClient
|
||||
LastReadRawEndUtc = endUtc;
|
||||
LastReadRawMaxValues = maxValues;
|
||||
ReadRawCallCount++;
|
||||
return ToAsyncStream(RawSamples, ThrowOnRead, ct);
|
||||
RawCalls.Add((startUtc, endUtc, maxValues));
|
||||
var items = RawSource is null ? RawSamples : RawSource(startUtc, endUtc, maxValues).ToList();
|
||||
return CountingStream(ToAsyncStream(items, ThrowOnRead, ct));
|
||||
}
|
||||
|
||||
public IAsyncEnumerable<HistorianAggregateSample> ReadAggregateAsync(
|
||||
@@ -193,6 +206,16 @@ public sealed class FakeHistorianGatewayClient : IHistorianGatewayClient
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
|
||||
private async IAsyncEnumerable<HistorianSample> CountingStream(IAsyncEnumerable<HistorianSample> inner)
|
||||
{
|
||||
await foreach (var item in inner.ConfigureAwait(false))
|
||||
{
|
||||
RawYieldedCount++;
|
||||
yield return item;
|
||||
}
|
||||
RawStreamsCompleted++;
|
||||
}
|
||||
|
||||
private static async IAsyncEnumerable<T> ToAsyncStream<T>(
|
||||
IReadOnlyList<T> items,
|
||||
Exception? error,
|
||||
|
||||
+37
-3
@@ -32,11 +32,29 @@ public sealed class GatewayHistorianDataSourceTests
|
||||
{
|
||||
var fake = new FakeHistorianGatewayClient();
|
||||
var ds = new GatewayHistorianDataSource(fake, NullLogger<GatewayHistorianDataSource>.Instance);
|
||||
await ds.ReadProcessedAsync("T", default, default, TimeSpan.FromSeconds(60), HistoryAggregateType.Minimum, TestContext.Current.CancellationToken);
|
||||
Assert.Equal(RetrievalMode.MinimumWithTime, fake.LastAggregateMode);
|
||||
await ds.ReadProcessedAsync("T", default, default, TimeSpan.FromSeconds(60), HistoryAggregateType.Total, TestContext.Current.CancellationToken);
|
||||
Assert.Equal(RetrievalMode.Integral, fake.LastAggregateMode);
|
||||
Assert.Equal(TimeSpan.FromSeconds(60), fake.LastAggregateInterval);
|
||||
}
|
||||
|
||||
[Fact] // #681: the native buckets are stamped with the interval START and carry the Calculated bits
|
||||
public async Task ReadProcessed_native_buckets_are_stamped_at_the_interval_start_with_the_Calculated_bit()
|
||||
{
|
||||
var start = new DateTime(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc);
|
||||
var fake = new FakeHistorianGatewayClient
|
||||
{
|
||||
AggregateSamples = new[]
|
||||
{
|
||||
new HistorianAggregateSample { Value = 7.0, OpcQuality = 192, StartTime = Timestamp.FromDateTime(start), EndTime = Timestamp.FromDateTime(start.AddMinutes(1)) },
|
||||
new HistorianAggregateSample { OpcQuality = 0, StartTime = Timestamp.FromDateTime(start.AddMinutes(1)), EndTime = Timestamp.FromDateTime(start.AddMinutes(2)) },
|
||||
},
|
||||
};
|
||||
var ds = new GatewayHistorianDataSource(fake, NullLogger<GatewayHistorianDataSource>.Instance);
|
||||
var r = await ds.ReadProcessedAsync("T", start, start.AddMinutes(2), TimeSpan.FromMinutes(1), HistoryAggregateType.TimeAverage, TestContext.Current.CancellationToken);
|
||||
Assert.Equal(new DateTime?[] { start, start.AddMinutes(1) }, r.Samples.Select(x => x.SourceTimestampUtc).ToArray());
|
||||
Assert.Equal(new[] { 0x00000401u, 0x809B0401u }, r.Samples.Select(x => x.StatusCode).ToArray());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ReadProcessed_TimeAverage_requests_TimeWeightedAverage()
|
||||
{
|
||||
@@ -46,9 +64,24 @@ public sealed class GatewayHistorianDataSourceTests
|
||||
Assert.Equal(RetrievalMode.TimeWeightedAverage, fake.LastAggregateMode);
|
||||
}
|
||||
|
||||
[Theory] // #679: no AVEVA equivalent — not supported, gateway never called, no backend failure recorded
|
||||
[Theory] // #681 (was #679 BadAggregateNotSupported): served from a raw read, never a native retrieval mode
|
||||
[InlineData(HistoryAggregateType.Average)]
|
||||
[InlineData(HistoryAggregateType.Count)]
|
||||
[InlineData(HistoryAggregateType.Minimum)]
|
||||
[InlineData(HistoryAggregateType.Maximum)]
|
||||
public async Task ReadProcessed_per_sample_aggregate_reads_raw_not_a_retrieval_mode(HistoryAggregateType aggregate)
|
||||
{
|
||||
var fake = new FakeHistorianGatewayClient();
|
||||
var ds = new GatewayHistorianDataSource(fake, NullLogger<GatewayHistorianDataSource>.Instance);
|
||||
var t0 = new DateTime(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc);
|
||||
await ds.ReadProcessedAsync("T", t0, t0.AddMinutes(1), TimeSpan.FromSeconds(60), aggregate, TestContext.Current.CancellationToken);
|
||||
Assert.Equal(0, fake.ReadAggregateCallCount);
|
||||
Assert.True(fake.ReadRawCallCount >= 1); // the window, plus the earlier-data probe when the range matters
|
||||
}
|
||||
|
||||
[Theory] // no AVEVA mode and not a per-sample aggregate: not supported, gateway never called, no failure recorded
|
||||
[InlineData(HistoryAggregateType.TimeAverage2)]
|
||||
[InlineData(HistoryAggregateType.PercentGood)]
|
||||
public async Task ReadProcessed_unsupported_aggregate_throws_without_calling_gateway_or_recording_failure(HistoryAggregateType aggregate)
|
||||
{
|
||||
var fake = new FakeHistorianGatewayClient();
|
||||
@@ -56,6 +89,7 @@ public sealed class GatewayHistorianDataSourceTests
|
||||
await Assert.ThrowsAsync<HistoryAggregateNotSupportedException>(() => ds.ReadProcessedAsync(
|
||||
"T", default, default, TimeSpan.FromSeconds(60), aggregate, TestContext.Current.CancellationToken));
|
||||
Assert.Equal(0, fake.ReadAggregateCallCount);
|
||||
Assert.Equal(0, fake.ReadRawCallCount);
|
||||
var h = ds.GetHealthSnapshot();
|
||||
Assert.Equal(0, h.TotalFailures);
|
||||
Assert.Equal(0, h.TotalQueries);
|
||||
|
||||
+568
@@ -0,0 +1,568 @@
|
||||
using Google.Protobuf.WellKnownTypes;
|
||||
using Grpc.Core;
|
||||
using ZB.MOM.WW.HistorianGateway.Client;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.HistorianGateway.Contracts.Grpc;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// #681: the Gateway backend serves the per-sample aggregates from raw reads with the SQL backends' shared evaluator.
|
||||
/// The parity oracle is what SqlServer/QuestDB answer for the same stored samples: per-bucket statistics built
|
||||
/// independently in memory (LINQ, <see cref="BucketPlan.IndexOf"/>) — the C# statement of the backends' SQL that
|
||||
/// <c>Sql.Common.Tests.SdkAggregateParityTests</c> pins against the OPC UA SDK's own Part 13 calculators — evaluated by
|
||||
/// <see cref="AggregateEvaluator.EvaluateAll"/> with the data range the SQL backends query (earliest sample ≤ the
|
||||
/// window's later edge, latest ≥ its earlier edge, over EVERY stored sample). The fake gateway serves raw reads from
|
||||
/// the same stored samples, optionally emitting AVEVA's initial-value row stamped at the query start.
|
||||
/// </summary>
|
||||
public sealed class GatewayRawAggregateTests
|
||||
{
|
||||
private static readonly CancellationToken Ct = CancellationToken.None;
|
||||
|
||||
/// <summary>2026-10-09 08:00:00.1234567 UTC — not millisecond-aligned (the SQL fixtures' T0).</summary>
|
||||
private static readonly DateTime T0 = new DateTime(2026, 10, 9, 8, 0, 0, DateTimeKind.Utc).AddTicks(1_234_567);
|
||||
|
||||
private static readonly DateTime Now = At(3600);
|
||||
|
||||
private const uint Good = 0x00000000u;
|
||||
private const uint Bad = 0x80000000u;
|
||||
private const uint Uncertain = 0x40000000u;
|
||||
private const uint GoodCalc = 0x00000401u;
|
||||
private const uint UncertainCalc = 0x40A40401u;
|
||||
private const uint NoDataCalc = 0x809B0401u;
|
||||
private const uint InvalidInputsCalc = 0x80D60401u;
|
||||
private const uint Partial = 0x4u;
|
||||
|
||||
private static DateTime At(double seconds) => T0.AddTicks((long)(seconds * TimeSpan.TicksPerSecond));
|
||||
|
||||
private static StoredSample S(double seconds, HistorianValue v, uint status) => new(At(seconds), v, status);
|
||||
|
||||
private static StoredSample D(double seconds, double v, uint status = Good) => S(seconds, HistorianValue.FromDouble(v), status);
|
||||
|
||||
// ---- datasets (statuses restricted to codes that round-trip the OPC DA quality byte: Good, Uncertain, Bad) ----------
|
||||
|
||||
/// <summary>Tag A of the conformance suite: +0=10, +10=20, +20=Bad(30), +30=40, +40=50.</summary>
|
||||
private static readonly StoredSample[] A = [D(0, 10), D(10, 20), D(20, 30, Bad), D(30, 40), D(40, 50)];
|
||||
|
||||
/// <summary>Every 2.5 s from −10 to +100, values cycling, one Bad sample at +22.5.</summary>
|
||||
private static readonly StoredSample[] Dense = Enumerable.Range(0, 45)
|
||||
.Select(i => (t: -10 + i * 2.5, i))
|
||||
.Select(x => D(x.t, 10 + (x.i * 7 % 13), x.t == 22.5 ? Bad : Good))
|
||||
.ToArray();
|
||||
|
||||
private static readonly StoredSample[] Gappy =
|
||||
[D(1, 10), D(2, 11, Uncertain), D(25, 12, Bad), D(45, 13), D(70, 1)];
|
||||
|
||||
private static readonly StoredSample[] Bracketed =
|
||||
[
|
||||
D(-5, 1), D(1, 10, Bad), D(3, 11), D(5, 12, Bad), D(7, 13, Uncertain), D(9, 15), D(12, 20),
|
||||
S(14, HistorianValue.Empty, Bad), D(16, 21), S(19, HistorianValue.Empty, Bad), D(40, 1),
|
||||
];
|
||||
|
||||
private static readonly StoredSample[] EarlyEnd = [D(-5, 1), D(3, 2), D(25, 3)];
|
||||
private static readonly StoredSample[] EndOnEdge = [D(-5, 1), D(3, 2), D(20, 3)];
|
||||
|
||||
private static readonly StoredSample[] LateStart = Enumerable.Range(0, 20)
|
||||
.Select(i => D(7 + i * 5, i % 2 == 0 ? 20 : 40)).ToArray();
|
||||
|
||||
private static readonly StoredSample[] OutageEnds = [D(-5, 1), D(3, 2), S(25, HistorianValue.Empty, Bad)];
|
||||
|
||||
/// <summary>A usable String sample among numbers (numeric aggregates ⇒ BadAggregateInvalidInputs in its bucket).</summary>
|
||||
private static readonly StoredSample[] Mixed =
|
||||
[D(-5, 1), D(2, 4), S(12, HistorianValue.FromString("x"), Good), D(15, 6), D(33, 9, Uncertain)];
|
||||
|
||||
/// <summary>Equal extremes in one bucket (ActualTime ties ⇒ earliest).</summary>
|
||||
private static readonly StoredSample[] Ties = [D(1, 5), D(3, 9), D(4, 1), D(6, 9), D(8, 1), D(25, 2)];
|
||||
|
||||
/// <summary>#684 (Sql.Common's DenseUncertain): Dense with every 5th sample an in-range Uncertain 16.</summary>
|
||||
private static readonly StoredSample[] DenseUncertain = Enumerable.Range(0, 45)
|
||||
.Select(i => (t: -10 + i * 2.5, i))
|
||||
.Select(x => x.i % 5 == 0 && x.t != 22.5
|
||||
? D(x.t, 16, Uncertain)
|
||||
: D(x.t, 10 + (x.i * 7 % 13), x.t == 22.5 ? Bad : Good))
|
||||
.ToArray();
|
||||
|
||||
/// <summary>#684 (Sql.Common's UncertainBeyond): Uncertain values below and above the Good extremes in one interval.</summary>
|
||||
private static readonly StoredSample[] UncertainBeyond = [D(-5, 20), D(2, 10), D(4, 5, Uncertain), D(6, 99, Uncertain), D(8, 30), D(25, 20)];
|
||||
|
||||
/// <summary>#684: an Uncertain string among Good numbers (does not refuse the Good-value aggregates) and a Good one.</summary>
|
||||
private static readonly StoredSample[] UncertainText =
|
||||
[
|
||||
D(-5, 1), D(2, 4), S(4, HistorianValue.FromString("u"), Uncertain), D(6, 8), S(14, HistorianValue.FromString("g"), Good),
|
||||
D(16, 3),
|
||||
];
|
||||
|
||||
private static readonly (string Name, StoredSample[] Data)[] Datasets =
|
||||
[
|
||||
("A", A), ("Dense", Dense), ("Gappy", Gappy), ("Bracketed", Bracketed), ("EarlyEnd", EarlyEnd),
|
||||
("EndOnEdge", EndOnEdge), ("LateStart", LateStart), ("OutageEnds", OutageEnds), ("Mixed", Mixed), ("Ties", Ties),
|
||||
("DenseUncertain", DenseUncertain), ("UncertainBeyond", UncertainBeyond), ("UncertainText", UncertainText),
|
||||
];
|
||||
|
||||
private static readonly (double From, double To, double Interval)[] Windows =
|
||||
[
|
||||
(0, 80, 20), (80, 0, 20), (0, 50, 20), (50, 0, 20), (5, 80, 15), (80, 5, 15), (3, 77, 20), (77, 3, 20),
|
||||
(0, 60, 0), (60, 0, 0), (0, 60, 10), (60, 0, 10), (-10, 1, 5), (1, -10, 5), (70, 85, 5), (85, 70, 5), (0, 1, 1),
|
||||
(0, 40, 10), (40, 0, 10), (0, 40, 25), (40, 0, 25),
|
||||
];
|
||||
|
||||
private static readonly HistoryAggregateType[] PerSample =
|
||||
[
|
||||
HistoryAggregateType.Average, HistoryAggregateType.Minimum, HistoryAggregateType.Maximum,
|
||||
HistoryAggregateType.Count, HistoryAggregateType.Range, HistoryAggregateType.Start, HistoryAggregateType.End,
|
||||
HistoryAggregateType.Delta, HistoryAggregateType.MinimumActualTime, HistoryAggregateType.MaximumActualTime,
|
||||
];
|
||||
|
||||
// ---- the fake store ----------------------------------------------------------------------------------------------
|
||||
|
||||
private static HistorianSample ToWire(StoredSample s)
|
||||
{
|
||||
var w = new HistorianSample
|
||||
{
|
||||
Tag = "T",
|
||||
Timestamp = Timestamp.FromDateTime(s.TimeUtc),
|
||||
OpcQuality = s.Status switch { Good => 192u, Uncertain => 64u, _ => 0u },
|
||||
};
|
||||
// The REAL wire shape (histsdk): every numeric row carries NumericValue — a NULL / no-data row arrives as 0 with
|
||||
// OPC quality 0 — and a string row always carries StringValue.
|
||||
if (s.Value.Kind == HistorianValueKind.String) w.StringValue = s.Value.Text;
|
||||
else w.NumericValue = s.Value.TryGetDouble(out var d) ? d : 0d;
|
||||
return w;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A gateway over a stored sample set: ReadRaw returns the samples in <c>[start, end]</c> ascending, capped at
|
||||
/// <c>maxValues</c>. With <paramref name="avevaBoundary"/> it first emits what an AVEVA <c>Full</c> read does at
|
||||
/// the query start when nothing is stored exactly there: a row stamped AT the query start carrying the value in
|
||||
/// effect (the last stored sample before it, with its quality; <c>Quality = 133</c>) — or, with no earlier data, a
|
||||
/// NULL row in the real wire shape: <c>NumericValue = 0</c> present, <c>OpcQuality = 0</c>. With
|
||||
/// <paramref name="terminalRow"/> the small (2-row) reads also end with a NULL row stamped at the query END (a
|
||||
/// hypothetical shape the probe / look-ahead must ignore; the window read relies on there being none — live leg).
|
||||
/// </summary>
|
||||
private static FakeHistorianGatewayClient Store(IReadOnlyList<StoredSample> data, bool avevaBoundary = false,
|
||||
bool terminalRow = false)
|
||||
{
|
||||
var sorted = data.OrderBy(s => s.TimeUtc).ToList();
|
||||
return new FakeHistorianGatewayClient
|
||||
{
|
||||
RawSource = (start, end, max) =>
|
||||
{
|
||||
var rows = new List<HistorianSample>();
|
||||
if (avevaBoundary && sorted.All(s => s.TimeUtc != start))
|
||||
{
|
||||
var prior = sorted.LastOrDefault(s => s.TimeUtc < start);
|
||||
var boundary = prior.TimeUtc == default
|
||||
? new HistorianSample { Tag = "T", NumericValue = 0, OpcQuality = 0, Quality = 1 }
|
||||
: ToWire(prior);
|
||||
boundary.Timestamp = Timestamp.FromDateTime(start);
|
||||
if (prior.TimeUtc != default) boundary.Quality = 133;
|
||||
rows.Add(boundary);
|
||||
}
|
||||
rows.AddRange(sorted.Where(s => s.TimeUtc >= start && s.TimeUtc <= end).Select(ToWire));
|
||||
if (terminalRow && max <= 2 && end > start && sorted.All(s => s.TimeUtc != end))
|
||||
rows.Add(new HistorianSample
|
||||
{ Tag = "T", Timestamp = Timestamp.FromDateTime(end), NumericValue = 0, OpcQuality = 0, Quality = 1 });
|
||||
return rows.Take(max);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
private sealed class FixedClock(DateTime now) : TimeProvider
|
||||
{
|
||||
public override DateTimeOffset GetUtcNow() => new(now);
|
||||
}
|
||||
|
||||
private static GatewayHistorianDataSource Source(FakeHistorianGatewayClient client, int maxRawRows = 1_000_000) =>
|
||||
new(client, NullLogger<GatewayHistorianDataSource>.Instance, maxRawRows, new FixedClock(Now));
|
||||
|
||||
// ---- the oracle: what the SQL backends answer --------------------------------------------------------------------
|
||||
|
||||
/// <summary>The SQL backends' answer for <paramref name="data"/> (statuses as the gateway quality byte carries them).</summary>
|
||||
private static IReadOnlyList<DataValueSnapshot> SqlBackendAnswer(HistoryAggregateType agg, BucketPlan plan,
|
||||
IReadOnlyList<StoredSample> data)
|
||||
{
|
||||
// Same clamp the backends apply (an Empty sample is Bad).
|
||||
var stored = data.Select(s => StoredSample.FromStored(s.TimeUtc, s.Value, s.Status)).ToList();
|
||||
var earliest = stored.Where(s => s.TimeUtc <= plan.EndUtc).Select(s => (DateTime?)s.TimeUtc).Min();
|
||||
var latest = stored.Where(s => s.TimeUtc >= plan.StartUtc).Select(s => (DateTime?)s.TimeUtc).Max();
|
||||
return AggregateEvaluator.EvaluateAll(agg, plan, InMemoryRows(plan, stored, agg == HistoryAggregateType.MaximumActualTime),
|
||||
earliest, latest, Now);
|
||||
}
|
||||
|
||||
/// <summary>Independent LINQ statement of the per-bucket statistics (copied from Sql.Common.Tests, #684 semantics).</summary>
|
||||
private static IEnumerable<BucketRow> InMemoryRows(BucketPlan plan, IEnumerable<StoredSample> samples, bool maximum)
|
||||
{
|
||||
foreach (var g in samples.Select(s => (s, b: plan.IndexOf(s.TimeUtc))).Where(x => x.b is not null)
|
||||
.GroupBy(x => x.b!.Value).OrderBy(g => g.Key))
|
||||
{
|
||||
var all = g.Select(x => x.s).OrderBy(s => s.TimeUtc).ToList();
|
||||
var usable = all.Where(s => s.IsUsable).ToList();
|
||||
var numeric = usable.Where(s => s.Value.Kind is HistorianValueKind.Double or HistorianValueKind.Int64
|
||||
or HistorianValueKind.Boolean).ToList();
|
||||
var good = numeric.Where(s => s.IsGood).ToList();
|
||||
var uncertain = numeric.Where(s => !s.IsGood).ToList();
|
||||
var extreme = good.OrderBy(s => maximum ? -s.Value.Number : s.Value.Number).ThenBy(s => s.TimeUtc)
|
||||
.Cast<StoredSample?>().FirstOrDefault();
|
||||
yield return new BucketRow
|
||||
{
|
||||
Bucket = g.Key,
|
||||
Total = all.Count,
|
||||
Usable = usable.Count,
|
||||
Good = all.Count(s => s.IsGood),
|
||||
Bad = all.Count - usable.Count,
|
||||
NonNumeric = usable.Count - numeric.Count,
|
||||
GoodNonNumeric = usable.Count(s => s.IsGood) - numeric.Count(s => s.IsGood),
|
||||
Avg = good.Count == 0 ? null : good.Average(s => s.Value.Number),
|
||||
Min = good.Count == 0 ? null : good.Min(s => s.Value.Number),
|
||||
Max = good.Count == 0 ? null : good.Max(s => s.Value.Number),
|
||||
Sum = good.Count == 0 ? null : good.Sum(s => s.Value.Number),
|
||||
UncertainMin = uncertain.Count == 0 ? null : uncertain.Min(s => s.Value.Number),
|
||||
UncertainMax = uncertain.Count == 0 ? null : uncertain.Max(s => s.Value.Number),
|
||||
First = usable.Count == 0 ? HistorianValue.Empty : usable[0].Value,
|
||||
Last = usable.Count == 0 ? HistorianValue.Empty : usable[^1].Value,
|
||||
Extreme = extreme?.Value.Number,
|
||||
ExtremeTime = extreme?.TimeUtc,
|
||||
FirstAny = all[0],
|
||||
LastAny = all[^1],
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
private static string Describe(IReadOnlyList<DataValueSnapshot> v) =>
|
||||
string.Join(" | ", v.Select(x => $"{x.Value ?? "null"} 0x{x.StatusCode:X8} {(x.SourceTimestampUtc!.Value - T0).TotalSeconds}s"));
|
||||
|
||||
// ---- parity ------------------------------------------------------------------------------------------------------
|
||||
|
||||
[Theory]
|
||||
[InlineData(false)]
|
||||
[InlineData(true)]
|
||||
public async Task Every_per_sample_aggregate_matches_the_SQL_backends_forward_and_reverse(bool avevaBoundary)
|
||||
{
|
||||
var failures = new List<string>();
|
||||
var compared = 0;
|
||||
foreach (var agg in PerSample)
|
||||
foreach (var (name, data) in Datasets)
|
||||
foreach (var (from, to, interval) in Windows)
|
||||
{
|
||||
var plan = BucketPlan.Create(At(from), At(to), TimeSpan.FromSeconds(interval));
|
||||
var expected = SqlBackendAnswer(agg, plan, data);
|
||||
var fake = Store(data, avevaBoundary);
|
||||
var actual = (await Source(fake)
|
||||
.ReadProcessedAsync("T", At(from), At(to), TimeSpan.FromSeconds(interval), agg, Ct)).Samples;
|
||||
compared++;
|
||||
if (fake.RawStreamsCompleted != fake.ReadRawCallCount)
|
||||
failures.Add($"{agg} {name} {from}→{to}/{interval}s: a raw stream was abandoned mid-way");
|
||||
var same = actual.Count == expected.Count && actual.Zip(expected).All(p =>
|
||||
Equals(p.First.Value, p.Second.Value) && p.First.StatusCode == p.Second.StatusCode
|
||||
&& p.First.SourceTimestampUtc == p.Second.SourceTimestampUtc);
|
||||
if (!same)
|
||||
failures.Add($"{agg} {name} {from}→{to}/{interval}s\n ours {Describe(actual)}\n sql {Describe(expected)}");
|
||||
}
|
||||
compared.ShouldBe(PerSample.Length * Datasets.Length * Windows.Length);
|
||||
failures.ShouldBeEmpty(string.Join("\n", failures));
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(HistoryAggregateType.Average, 15d, 40d, 50d)]
|
||||
[InlineData(HistoryAggregateType.Minimum, 10d, 40d, 50d)]
|
||||
[InlineData(HistoryAggregateType.Maximum, 20d, 40d, 50d)]
|
||||
[InlineData(HistoryAggregateType.Delta, 10d, 0d, 0d)]
|
||||
[InlineData(HistoryAggregateType.Range, 10d, 0d, 0d)]
|
||||
public async Task Numeric_aggregates_match_the_conformance_suites_hand_computed_literals(HistoryAggregateType agg,
|
||||
double b0, double b1, double b2)
|
||||
{
|
||||
// The literals of ProcessedConformance.Numeric_aggregates_over_the_fixture (SqlServer + QuestDB), over tag A.
|
||||
var s = (await Source(Store(A, avevaBoundary: true))
|
||||
.ReadProcessedAsync("A", At(0), At(80), TimeSpan.FromSeconds(20), agg, Ct)).Samples;
|
||||
s.Select(x => x.Value).ShouldBe([b0, b1, b2, null]);
|
||||
var b2Status = agg == HistoryAggregateType.Average ? GoodCalc : GoodCalc | Partial;
|
||||
s.Select(x => x.StatusCode).ShouldBe([GoodCalc, UncertainCalc, b2Status, NoDataCalc]);
|
||||
s.Select(x => x.SourceTimestampUtc).ShouldBe([At(0), At(20), At(40), At(60)]); // interval STARTS (Part 13)
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Count_is_a_Good_sample_count_as_int_and_zero_inside_the_data_range()
|
||||
{
|
||||
// #679 made Count BadAggregateNotSupported on Gateway; #681 serves it with the SQL backends' #680 semantics.
|
||||
var s = (await Source(Store(Gappy)).ReadProcessedAsync("T", At(0), At(60), TimeSpan.FromSeconds(10),
|
||||
HistoryAggregateType.Count, Ct)).Samples;
|
||||
// [0,10): +1 G, +2 U ⇒ 1, Uncertain (data starts inside ⇒ Partial); [10,20) empty inside the range ⇒ 0 Good;
|
||||
// [20,30): +25 Bad only ⇒ generic Bad; [30,40) 0; [40,50) +45 ⇒ 1; [50,60) 0 (+70 is later data).
|
||||
s.Select(x => x.Value).ShouldBe([1, 0, null, 0, 1, 0]);
|
||||
s.Select(x => x.StatusCode).ShouldBe([UncertainCalc | Partial, GoodCalc, 0x80000401u, GoodCalc, GoodCalc, GoodCalc]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_usable_string_sample_makes_a_numeric_aggregate_BadAggregateInvalidInputs()
|
||||
{
|
||||
var s = (await Source(Store(Mixed)).ReadProcessedAsync("T", At(0), At(40), TimeSpan.FromSeconds(10),
|
||||
HistoryAggregateType.Average, Ct)).Samples;
|
||||
s[1].StatusCode.ShouldBe(InvalidInputsCalc);
|
||||
s[1].Value.ShouldBeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ActualTime_carries_the_extreme_samples_own_time_ties_earliest()
|
||||
{
|
||||
var max = (await Source(Store(Ties)).ReadProcessedAsync("T", At(0), At(10), TimeSpan.Zero,
|
||||
HistoryAggregateType.MaximumActualTime, Ct)).Samples.Single();
|
||||
max.Value.ShouldBe(9d);
|
||||
max.SourceTimestampUtc.ShouldBe(At(3));
|
||||
var min = (await Source(Store(Ties)).ReadProcessedAsync("T", At(0), At(10), TimeSpan.Zero,
|
||||
HistoryAggregateType.MinimumActualTime, Ct)).Samples.Single();
|
||||
min.Value.ShouldBe(1d);
|
||||
min.SourceTimestampUtc.ShouldBe(At(4));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_reverse_read_is_anchored_at_its_later_time_and_returned_latest_first()
|
||||
{
|
||||
// #677: 50 → 0 at 20 s ⇒ (30,50], (10,30], (0,10] stamped 50, 30, 10.
|
||||
var s = (await Source(Store(A)).ReadProcessedAsync("A", At(50), At(0), TimeSpan.FromSeconds(20),
|
||||
HistoryAggregateType.Maximum, Ct)).Samples;
|
||||
s.Select(x => x.SourceTimestampUtc).ShouldBe([At(50), At(30), At(10)]);
|
||||
s.Select(x => x.Value).ShouldBe([50d, 40d, 20d]);
|
||||
}
|
||||
|
||||
// ---- the raw request ---------------------------------------------------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public async Task The_raw_stream_starts_one_millisecond_before_the_window_so_the_AVEVA_boundary_row_is_never_a_sample()
|
||||
{
|
||||
// Data before the window, nothing inside [0,10) but +5: the boundary row (value 1, stamped at the query start) must
|
||||
// not be counted. Count over [0,10) is 1, not 2.
|
||||
var fake = Store([D(-5, 1), D(5, 2)], avevaBoundary: true);
|
||||
var s = (await Source(fake).ReadProcessedAsync("T", At(0), At(10), TimeSpan.Zero, HistoryAggregateType.Count, Ct))
|
||||
.Samples.Single();
|
||||
s.Value.ShouldBe(1);
|
||||
fake.RawCalls[0].Start.ShouldBe(At(0) - TimeSpan.FromMilliseconds(1));
|
||||
fake.RawCalls.ShouldNotContain(c => c.Start == DateTime.UnixEpoch); // the Good boundary row proved earlier data: no probe
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Without_boundary_evidence_a_probe_asks_for_data_before_the_window()
|
||||
{
|
||||
// No boundary row (a historian that emits none): the second call probes [epoch, start − 1 tick] for 2 rows, and the
|
||||
// sample at −5 it finds keeps the first interval free of the start-of-data Partial bit.
|
||||
var fake = Store([D(-5, 1), D(5, 2), D(15, 3)]);
|
||||
var s = (await Source(fake).ReadProcessedAsync("T", At(0), At(20), TimeSpan.FromSeconds(10),
|
||||
HistoryAggregateType.Minimum, Ct)).Samples;
|
||||
fake.RawCalls[1].ShouldBe((DateTime.UnixEpoch, At(0).AddTicks(-1), 2));
|
||||
(s[0].StatusCode & Partial).ShouldBe(0u);
|
||||
|
||||
// Data that really starts inside the first interval: the probe finds nothing, the interval is Partial.
|
||||
var late = Store([D(5, 2), D(15, 3)], avevaBoundary: true);
|
||||
var l = (await Source(late).ReadProcessedAsync("T", At(0), At(20), TimeSpan.FromSeconds(10),
|
||||
HistoryAggregateType.Minimum, Ct)).Samples;
|
||||
late.RawCalls[1].Start.ShouldBe(DateTime.UnixEpoch); // the NULL boundary row (value 0, OPC quality 0) is no evidence
|
||||
(l[0].StatusCode & Partial).ShouldBe(Partial);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task The_raw_request_looks_ahead_past_the_window_only_when_the_data_range_matters()
|
||||
{
|
||||
var count = Store(A);
|
||||
await Source(count).ReadProcessedAsync("A", At(0), At(80), TimeSpan.FromSeconds(20), HistoryAggregateType.Count, Ct);
|
||||
// The window read ends AT the window end; probe + look-ahead are separate 2-row reads, all consumed to completion.
|
||||
count.RawCalls.ShouldBe([
|
||||
(At(0) - TimeSpan.FromMilliseconds(1), At(80), 1_000_000 + 2 + 16),
|
||||
(DateTime.UnixEpoch, At(0).AddTicks(-1), 2),
|
||||
(At(80), Now + TimeSpan.FromDays(1), 2),
|
||||
]);
|
||||
count.RawStreamsCompleted.ShouldBe(3);
|
||||
|
||||
var average = Store(A);
|
||||
await Source(average).ReadProcessedAsync("A", At(0), At(80), TimeSpan.FromSeconds(20), HistoryAggregateType.Average, Ct);
|
||||
average.RawCalls.Single().End.ShouldBe(At(80)); // Average needs no data range: no look-ahead, no probe
|
||||
average.ReadAggregateCallCount.ShouldBe(0);
|
||||
|
||||
var unbounded = Store(A);
|
||||
await Source(unbounded, maxRawRows: 0).ReadProcessedAsync("A", At(0), At(80), TimeSpan.FromSeconds(20),
|
||||
HistoryAggregateType.Count, Ct);
|
||||
unbounded.RawCalls[0].MaxValues.ShouldBe(int.MaxValue);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task No_gateway_stream_is_ever_abandoned_and_later_data_comes_from_a_two_row_look_ahead()
|
||||
{
|
||||
// #681 review: breaking out of a gateway stream cancels the call, which the gateway treats as a connection fault
|
||||
// (it evicts its shared historian session). 1000 samples after the window: the window read never sees them, the
|
||||
// 2-row look-ahead reads two, and every stream runs to its end.
|
||||
var data = Enumerable.Range(0, 1000).Select(i => D(100 + i, i)).Prepend(D(-5, 0)).Prepend(D(5, 1)).ToArray();
|
||||
var fake = Store(data, avevaBoundary: true);
|
||||
var s = (await Source(fake).ReadProcessedAsync("T", At(0), At(50), TimeSpan.FromSeconds(10),
|
||||
HistoryAggregateType.Minimum, Ct)).Samples;
|
||||
fake.RawStreamsCompleted.ShouldBe(fake.ReadRawCallCount);
|
||||
fake.RawCalls.Last().ShouldBe((At(50), Now + TimeSpan.FromDays(1), 2));
|
||||
fake.RawYieldedCount.ShouldBe(2 + 2); // window read: boundary + +5; look-ahead: boundary at +50 + first sample
|
||||
(s[0].StatusCode & Partial).ShouldBe(0u); // data continues after the window: no end-of-data Partial
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Rows_stamped_at_a_small_reads_query_start_or_end_are_not_samples()
|
||||
{
|
||||
// Data ends at +15: the look-ahead's NULL rows at its own start (+20, boundary) and end (now + 1 day, terminal)
|
||||
// must not count as later data, so (10,20] / [10,20) holds the end of the data ⇒ Partial.
|
||||
var fake = Store([D(-5, 1), D(5, 2), D(15, 3)], avevaBoundary: true, terminalRow: true);
|
||||
var s = (await Source(fake).ReadProcessedAsync("T", At(0), At(20), TimeSpan.FromSeconds(10),
|
||||
HistoryAggregateType.Maximum, Ct)).Samples;
|
||||
(s[1].StatusCode & Partial).ShouldBe(Partial);
|
||||
fake.RawCalls.Last().Start.ShouldBe(At(20));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_window_over_the_raw_row_cap_fails_the_node_without_counting_a_backend_failure()
|
||||
{
|
||||
var data = Enumerable.Range(0, 10).Select(i => D(i, i)).ToArray(); // 10 rows in [0,10)
|
||||
var ds = Source(Store(data), maxRawRows: 9);
|
||||
var ex = await Should.ThrowAsync<HistoryReadLimitExceededException>(() =>
|
||||
ds.ReadProcessedAsync("T", At(0), At(10), TimeSpan.Zero, HistoryAggregateType.Average, Ct));
|
||||
ex.Limit.ShouldBe(9);
|
||||
ds.GetHealthSnapshot().TotalFailures.ShouldBe(0);
|
||||
|
||||
// Exactly at the cap is served.
|
||||
var ok = await Source(Store(data), maxRawRows: 10)
|
||||
.ReadProcessedAsync("T", At(0), At(10), TimeSpan.Zero, HistoryAggregateType.Average, Ct);
|
||||
ok.Samples.Single().Value.ShouldBe(4.5);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_stream_that_exhausts_its_row_budget_before_the_window_is_never_answered_from_a_truncated_window()
|
||||
{
|
||||
// 40 rows inside the 1 ms margin before the window eat the whole budget (cap 5 ⇒ 23 rows): the window could not be
|
||||
// read, so the node fails rather than reporting BadNoData.
|
||||
var margin = Enumerable.Range(0, 40).Select(i => new StoredSample(At(0).AddTicks(-1 - i), HistorianValue.FromDouble(i), Good));
|
||||
var data = margin.Append(D(5, 1)).ToArray();
|
||||
await Should.ThrowAsync<HistoryReadLimitExceededException>(() => Source(Store(data), maxRawRows: 5)
|
||||
.ReadProcessedAsync("T", At(0), At(10), TimeSpan.Zero, HistoryAggregateType.Count, Ct));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Past_the_cap_the_window_stream_is_drained_not_abandoned()
|
||||
{
|
||||
var data = Enumerable.Range(0, 10).Select(i => D(i, i)).ToArray();
|
||||
var fake = Store(data);
|
||||
await Should.ThrowAsync<HistoryReadLimitExceededException>(() => Source(fake, maxRawRows: 3)
|
||||
.ReadProcessedAsync("T", At(0), At(10), TimeSpan.Zero, HistoryAggregateType.Average, Ct));
|
||||
fake.RawStreamsCompleted.ShouldBe(1);
|
||||
fake.RawYieldedCount.ShouldBe(10);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task The_gateways_max_values_ceiling_rejection_is_a_request_limit_not_a_backend_fault()
|
||||
{
|
||||
// Historian:ReadLimits:MaxValuesCeiling (1,000,000) with AllowStreamingBeyondCeiling=false refuses the default
|
||||
// cap + 18 budget with InvalidArgument; the client package wraps it in HistorianGatewayException.
|
||||
var rpc = new RpcException(new Status(StatusCode.InvalidArgument,
|
||||
"max_values exceeds the server ceiling of 1000000; read large ranges by time-slicing ..."));
|
||||
var fake = new FakeHistorianGatewayClient { ThrowOnRead = new HistorianGatewayException(rpc.Status.Detail, rpc) };
|
||||
var ds = Source(fake);
|
||||
await Should.ThrowAsync<HistoryReadLimitExceededException>(() =>
|
||||
ds.ReadProcessedAsync("T", At(0), At(10), TimeSpan.Zero, HistoryAggregateType.Count, Ct));
|
||||
ds.GetHealthSnapshot().TotalFailures.ShouldBe(0);
|
||||
|
||||
// Any other InvalidArgument is still a backend fault.
|
||||
var other = new RpcException(new Status(StatusCode.InvalidArgument, "start_time must be less than or equal to end_time."));
|
||||
var ds2 = Source(new FakeHistorianGatewayClient { ThrowOnRead = new HistorianGatewayException(other.Status.Detail, other) });
|
||||
await Should.ThrowAsync<HistorianGatewayException>(() =>
|
||||
ds2.ReadProcessedAsync("T", At(0), At(10), TimeSpan.Zero, HistoryAggregateType.Count, Ct));
|
||||
ds2.GetHealthSnapshot().TotalFailures.ShouldBe(1);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_caller_cancellation_is_not_counted_as_a_backend_failure()
|
||||
{
|
||||
using var cts = new CancellationTokenSource();
|
||||
await cts.CancelAsync();
|
||||
var ds = Source(new FakeHistorianGatewayClient { ThrowOnRead = new OperationCanceledException(cts.Token) });
|
||||
await Should.ThrowAsync<OperationCanceledException>(() =>
|
||||
ds.ReadProcessedAsync("T", At(0), At(10), TimeSpan.Zero, HistoryAggregateType.Count, cts.Token));
|
||||
ds.GetHealthSnapshot().TotalFailures.ShouldBe(0);
|
||||
|
||||
// A cancellation the caller did NOT ask for (e.g. the gateway cancelled the call) is still a backend fault.
|
||||
var ds2 = Source(new FakeHistorianGatewayClient { ThrowOnRead = new OperationCanceledException() });
|
||||
await Should.ThrowAsync<OperationCanceledException>(() =>
|
||||
ds2.ReadProcessedAsync("T", At(0), At(10), TimeSpan.Zero, HistoryAggregateType.Count, Ct));
|
||||
ds2.GetHealthSnapshot().TotalFailures.ShouldBe(1);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_gateway_fault_on_the_raw_read_is_recorded_and_rethrown()
|
||||
{
|
||||
var fake = new FakeHistorianGatewayClient { ThrowOnRead = new InvalidOperationException("down") };
|
||||
var ds = Source(fake);
|
||||
await Should.ThrowAsync<InvalidOperationException>(() =>
|
||||
ds.ReadProcessedAsync("T", At(0), At(10), TimeSpan.Zero, HistoryAggregateType.Count, Ct));
|
||||
ds.GetHealthSnapshot().TotalFailures.ShouldBe(1);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task An_empty_window_calls_nothing_and_returns_no_intervals()
|
||||
{
|
||||
var fake = Store(A);
|
||||
var r = await Source(fake).ReadProcessedAsync("A", At(10), At(10), TimeSpan.Zero, HistoryAggregateType.Count, Ct);
|
||||
r.Samples.ShouldBeEmpty();
|
||||
fake.ReadRawCallCount.ShouldBe(0);
|
||||
}
|
||||
|
||||
// ---- the served set ----------------------------------------------------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public void The_served_set_is_the_per_sample_families_plus_the_two_native_modes()
|
||||
{
|
||||
GatewayHistorianDataSource.ServedAggregates.OrderBy(a => a).ShouldBe(
|
||||
PerSample.Append(HistoryAggregateType.TimeAverage).Append(HistoryAggregateType.Total).OrderBy(a => a));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Every_member_outside_the_served_set_is_refused_without_calling_the_gateway_and_every_member_inside_is_served()
|
||||
{
|
||||
foreach (var agg in System.Enum.GetValues<HistoryAggregateType>())
|
||||
{
|
||||
var fake = Store(A);
|
||||
var ds = Source(fake);
|
||||
if (GatewayHistorianDataSource.ServedAggregates.Contains(agg))
|
||||
{
|
||||
await ds.ReadProcessedAsync("A", At(0), At(40), TimeSpan.FromSeconds(20), agg, Ct);
|
||||
(fake.ReadRawCallCount + fake.ReadAggregateCallCount).ShouldBeGreaterThan(0, agg.ToString());
|
||||
}
|
||||
else
|
||||
{
|
||||
await Should.ThrowAsync<HistoryAggregateNotSupportedException>(() =>
|
||||
ds.ReadProcessedAsync("A", At(0), At(40), TimeSpan.FromSeconds(20), agg, Ct));
|
||||
(fake.ReadRawCallCount + fake.ReadAggregateCallCount).ShouldBe(0, agg.ToString());
|
||||
ds.GetHealthSnapshot().TotalQueries.ShouldBe(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---- the accumulator against the independent oracle, sample order shuffled ---------------------------------------
|
||||
|
||||
[Fact]
|
||||
public void The_bucket_statistics_do_not_depend_on_arrival_order()
|
||||
{
|
||||
var plan = BucketPlan.Create(At(-10), At(100), TimeSpan.FromSeconds(10));
|
||||
foreach (var maximum in new[] { false, true })
|
||||
{
|
||||
var acc = new RawBucketAccumulator(maximum);
|
||||
foreach (var s in DenseUncertain.Reverse())
|
||||
if (plan.IndexOf(s.TimeUtc) is { } b) acc.Add(b, s);
|
||||
var expected = InMemoryRows(plan, DenseUncertain, maximum).ToList();
|
||||
var actual = acc.Rows().ToList();
|
||||
actual.Count.ShouldBe(expected.Count);
|
||||
foreach (var (a, e) in actual.Zip(expected))
|
||||
{
|
||||
(a.Bucket, a.Total, a.Usable, a.Good, a.Bad, a.NonNumeric).ShouldBe((e.Bucket, e.Total, e.Usable, e.Good, e.Bad, e.NonNumeric));
|
||||
(a.Min, a.Max, a.Extreme, a.ExtremeTime).ShouldBe((e.Min, e.Max, e.Extreme, e.ExtremeTime));
|
||||
(a.UncertainMin, a.UncertainMax, a.GoodNonNumeric).ShouldBe((e.UncertainMin, e.UncertainMax, e.GoodNonNumeric));
|
||||
(a.Avg is null).ShouldBe(e.Avg is null);
|
||||
if (a.Avg is { } avg) avg.ShouldBe(e.Avg!.Value, 1e-9);
|
||||
(a.First, a.Last, a.FirstAny, a.LastAny).ShouldBe((e.First, e.Last, e.FirstAny, e.LastAny));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
+27
-9
@@ -9,8 +9,6 @@ public sealed class AggregateModeMapperTests
|
||||
{
|
||||
[Theory]
|
||||
[InlineData(HistoryAggregateType.TimeAverage, RetrievalMode.TimeWeightedAverage)]
|
||||
[InlineData(HistoryAggregateType.Minimum, RetrievalMode.MinimumWithTime)]
|
||||
[InlineData(HistoryAggregateType.Maximum, RetrievalMode.MaximumWithTime)]
|
||||
[InlineData(HistoryAggregateType.Total, RetrievalMode.Integral)]
|
||||
public void Maps_each_aggregate(HistoryAggregateType a, RetrievalMode expected)
|
||||
=> Assert.Equal(expected, AggregateModeMapper.ToRetrievalMode(a));
|
||||
@@ -18,11 +16,8 @@ public sealed class AggregateModeMapperTests
|
||||
[Fact] // matrix guard: every member is either mapped or explicitly reported as not supported
|
||||
public void Every_aggregate_member_is_mapped_or_reports_not_supported()
|
||||
{
|
||||
var original = new[]
|
||||
{
|
||||
HistoryAggregateType.Minimum, HistoryAggregateType.Maximum,
|
||||
HistoryAggregateType.Total, HistoryAggregateType.TimeAverage,
|
||||
};
|
||||
var original = new[] { HistoryAggregateType.Total, HistoryAggregateType.TimeAverage };
|
||||
Assert.Equal(original.OrderBy(a => a), AggregateModeMapper.NativeAggregates.OrderBy(a => a));
|
||||
foreach (var a in Enum.GetValues<HistoryAggregateType>())
|
||||
{
|
||||
if (original.Contains(a))
|
||||
@@ -32,12 +27,35 @@ public sealed class AggregateModeMapperTests
|
||||
}
|
||||
}
|
||||
|
||||
[Theory] // #679: AVEVA has no arithmetic mean and no sample count (Counter is a rollover increase)
|
||||
[Theory] // #679: AVEVA has no arithmetic mean and no sample count; #681: the per-sample aggregates are raw-evaluated, not native
|
||||
[InlineData(HistoryAggregateType.Average)]
|
||||
[InlineData(HistoryAggregateType.Count)]
|
||||
public void Average_and_Count_are_not_supported(HistoryAggregateType a)
|
||||
[InlineData(HistoryAggregateType.Minimum)]
|
||||
[InlineData(HistoryAggregateType.Maximum)]
|
||||
public void Per_sample_aggregates_have_no_native_mode(HistoryAggregateType a)
|
||||
=> Assert.Throws<HistoryAggregateNotSupportedException>(() => AggregateModeMapper.ToRetrievalMode(a));
|
||||
|
||||
[Fact] // #682/#681: the served (= advertised) set is the native members plus the raw-evaluated per-sample families,
|
||||
// and the data source returns exactly that static
|
||||
public void Served_is_exactly_the_mapped_members_and_the_data_source_declares_it()
|
||||
{
|
||||
foreach (var a in Enum.GetValues<HistoryAggregateType>())
|
||||
{
|
||||
var native = AggregateModeMapper.NativeAggregates.Contains(a);
|
||||
var perSample = ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common.AggregateFamilies.FamilyOf(a) is
|
||||
ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common.AggregateFamily.Simple
|
||||
or ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common.AggregateFamily.StartEnd
|
||||
or ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common.AggregateFamily.ActualTime;
|
||||
Assert.Equal(native || perSample, GatewayHistorianDataSource.ServedAggregates.Contains(a));
|
||||
if (!native) Assert.Throws<HistoryAggregateNotSupportedException>(() => AggregateModeMapper.ToRetrievalMode(a));
|
||||
}
|
||||
Assert.Equal(12, GatewayHistorianDataSource.ServedAggregates.Count);
|
||||
|
||||
var source = new GatewayHistorianDataSource(
|
||||
new FakeHistorianGatewayClient(), Microsoft.Extensions.Logging.Abstractions.NullLogger<GatewayHistorianDataSource>.Instance);
|
||||
Assert.Same(GatewayHistorianDataSource.ServedAggregates, source.SupportedAggregates);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Undefined_value_is_a_programming_error_not_a_not_supported()
|
||||
=> Assert.Throws<ArgumentOutOfRangeException>(() => AggregateModeMapper.ToRetrievalMode((HistoryAggregateType)9999));
|
||||
|
||||
+12
-2
@@ -37,7 +37,7 @@ public sealed class SampleMapperTests
|
||||
{
|
||||
var a = new HistorianAggregateSample { Tag = "T", /* Value unset, no Good quality */ EndTime = Ts(2026, 1, 1, 0, 0, 0) };
|
||||
var snap = SampleMapper.ToAggregateSnapshot(a);
|
||||
Assert.Equal(0x809B0000u, snap.StatusCode); // BadNoData
|
||||
Assert.Equal(0x809B0401u, snap.StatusCode); // BadNoData + Calculated (#681)
|
||||
Assert.Null(snap.Value);
|
||||
}
|
||||
|
||||
@@ -46,10 +46,20 @@ public sealed class SampleMapperTests
|
||||
{
|
||||
var a = new HistorianAggregateSample { Tag = "T", Value = 42.0, OpcQuality = 192, EndTime = Ts(2026, 1, 1, 0, 0, 0) };
|
||||
var snap = SampleMapper.ToAggregateSnapshot(a);
|
||||
Assert.Equal(0x00000000u, snap.StatusCode); // Good
|
||||
Assert.Equal(0x00000401u, snap.StatusCode); // Good + Calculated (#681)
|
||||
Assert.Equal(42.0, Assert.IsType<double>(snap.Value));
|
||||
}
|
||||
|
||||
[Fact] // #681: Part 13 stamps a processed value with its interval START (it used to be the end)
|
||||
public void Aggregate_bucket_is_stamped_with_its_interval_start()
|
||||
{
|
||||
var a = new HistorianAggregateSample
|
||||
{
|
||||
Tag = "T", Value = 1.0, OpcQuality = 192, StartTime = Ts(2026, 1, 1, 0, 0, 0), EndTime = Ts(2026, 1, 1, 0, 1, 0),
|
||||
};
|
||||
Assert.Equal(new DateTime(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc), SampleMapper.ToAggregateSnapshot(a).SourceTimestampUtc);
|
||||
}
|
||||
|
||||
// Ts(...) builds a Google.Protobuf.WellKnownTypes.Timestamp from UTC parts.
|
||||
private static Timestamp Ts(int y, int mo, int d, int h, int mi, int s)
|
||||
=> Timestamp.FromDateTime(new DateTime(y, mo, d, h, mi, s, DateTimeKind.Utc));
|
||||
|
||||
+34
@@ -62,6 +62,40 @@ internal sealed class QuestDbConformanceBackend : IConformanceBackend
|
||||
return new QuestDbConformanceBackend(fx, prefix, o, conn, boot);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A backend whose sample table has the pre-#683 shape — <c>tag</c> NOT indexed — as a deployment created before
|
||||
/// the index existed has it. Its bootstrapper runs with <c>AutoCreateSchema=false</c>, so it leaves the table as is.
|
||||
/// </summary>
|
||||
/// <param name="fx">The shared fixture.</param>
|
||||
/// <returns>The backend.</returns>
|
||||
public static async Task<QuestDbConformanceBackend> CreatePre683Async(QuestDbFixture fx)
|
||||
{
|
||||
fx.SkipIfUnavailable();
|
||||
var prefix = fx.UniquePrefix();
|
||||
var o = fx.Options(prefix, autoCreate: false);
|
||||
var conn = new QuestDbConnection(o);
|
||||
await using (var c = await conn.OpenAsync(CancellationToken.None))
|
||||
{
|
||||
foreach (var ddl in QuestDbTables.CreateAll(prefix, null))
|
||||
{
|
||||
await using var cmd = conn.CreateCommand(c, ddl.Replace("tag SYMBOL INDEX", "tag SYMBOL", StringComparison.Ordinal));
|
||||
await cmd.ExecuteNonQueryAsync();
|
||||
}
|
||||
}
|
||||
var boot = new QuestDbSchemaBootstrapper(conn, o);
|
||||
await boot.EnsureAsync(CancellationToken.None);
|
||||
return new QuestDbConformanceBackend(fx, prefix, o, conn, boot);
|
||||
}
|
||||
|
||||
/// <summary>Whether this instance's sample table has its <c>tag</c> column indexed (#683).</summary>
|
||||
/// <returns>The <c>table_columns()</c> flag.</returns>
|
||||
public async Task<bool> TagIndexedAsync()
|
||||
{
|
||||
await using var c = await Conn.OpenAsync(CancellationToken.None);
|
||||
await using var cmd = Conn.CreateCommand(c, QuestDbTables.TagIndexedQuery(Conn.SampleTable));
|
||||
return Convert.ToBoolean(await cmd.ExecuteScalarAsync());
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public long ResolutionTicks => 10;
|
||||
|
||||
|
||||
+197
@@ -0,0 +1,197 @@
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Historian.Conformance;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb.IntegrationTests;
|
||||
|
||||
/// <summary>
|
||||
/// #683: the sample table's <c>tag</c> symbol index — created with new tables, added by the bootstrapper to a table
|
||||
/// from before it existed — and the data-range lookups it serves (the start/end-of-data Partial bits and Count's
|
||||
/// in-range empty interval), which must answer exactly as they did from the unindexed scan: tombstones excluded, an
|
||||
/// outage marker counted as a sample.
|
||||
/// </summary>
|
||||
[Collection(QuestDbCollection.Name)]
|
||||
public sealed class TagIndexTests(QuestDbFixture fx)
|
||||
{
|
||||
private static readonly CancellationToken Ct = CancellationToken.None;
|
||||
private const uint GoodCalc = 0x00000401u;
|
||||
private const uint NoDataCalc = 0x809B0401u;
|
||||
|
||||
private static DateTime At(double s) => ConformanceData.At(s);
|
||||
|
||||
/// <summary>
|
||||
/// Tag <c>R</c>: an outage marker at +0 (its earliest sample), 1 at +100, 2 at +200 (its latest live sample), a row
|
||||
/// at +300 replaced by a tombstone and a tombstone alone at +400 — neither may extend the data range.
|
||||
/// </summary>
|
||||
private static async Task SeedAsync(QuestDbConformanceBackend b)
|
||||
{
|
||||
await b.SeedAsync("R",
|
||||
(At(0), HistorianValue.Empty, ConformanceData.BadNoCommunication),
|
||||
(At(100), HistorianValue.FromDouble(1), ConformanceData.Good),
|
||||
(At(200), HistorianValue.FromDouble(2), ConformanceData.Good),
|
||||
(At(300), HistorianValue.FromDouble(3), ConformanceData.Good));
|
||||
await b.InsertRowsAsync("R",
|
||||
[
|
||||
(At(300), HistorianValue.Empty, HistorianStatusCodes.BadNoData, true),
|
||||
(At(400), HistorianValue.Empty, HistorianStatusCodes.BadNoData, true),
|
||||
]);
|
||||
}
|
||||
|
||||
private static readonly HistoryAggregateType[] Aggregates =
|
||||
[
|
||||
HistoryAggregateType.Count, HistoryAggregateType.Minimum, HistoryAggregateType.Maximum, HistoryAggregateType.Average,
|
||||
HistoryAggregateType.Range, HistoryAggregateType.Start, HistoryAggregateType.End, HistoryAggregateType.Delta,
|
||||
HistoryAggregateType.MinimumActualTime, HistoryAggregateType.MaximumActualTime,
|
||||
];
|
||||
|
||||
private static readonly (double From, double To)[] Windows = [(-100, 100), (50, 450), (150, 250), (250, 500)];
|
||||
|
||||
/// <summary>Every per-sample aggregate over every window, forward and reverse, as comparable tuples.</summary>
|
||||
private static async Task<List<string>> ReadMatrixAsync(QuestDbConformanceBackend b)
|
||||
{
|
||||
var src = b.CreateDataSource();
|
||||
var all = new List<string>();
|
||||
foreach (var a in Aggregates)
|
||||
foreach (var (from, to) in Windows)
|
||||
foreach (var reverse in new[] { false, true })
|
||||
{
|
||||
var r = reverse
|
||||
? await src.ReadProcessedAsync("R", At(to), At(from), TimeSpan.FromSeconds(50), a, Ct)
|
||||
: await src.ReadProcessedAsync("R", At(from), At(to), TimeSpan.FromSeconds(50), a, Ct);
|
||||
all.AddRange(r.Samples.Select(s =>
|
||||
$"{a} [{from},{to}) rev={reverse} {s.SourceTimestampUtc:O} {s.Value} 0x{s.StatusCode:X8}"));
|
||||
}
|
||||
return all;
|
||||
}
|
||||
|
||||
private static async Task AwaitIndexedAsync(QuestDbConformanceBackend b)
|
||||
{
|
||||
var deadline = DateTime.UtcNow.AddSeconds(15);
|
||||
while (!await b.TagIndexedAsync())
|
||||
{
|
||||
if (DateTime.UtcNow > deadline) (await b.TagIndexedAsync()).ShouldBeTrue("the ADD INDEX was never applied");
|
||||
await Task.Delay(50); // the ALTER is a WAL transaction: visible once applied
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_new_sample_table_is_created_with_the_tag_index()
|
||||
{
|
||||
await using var b = await QuestDbConformanceBackend.CreateConcreteAsync(fx);
|
||||
(await b.TagIndexedAsync()).ShouldBeTrue();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_refused_ADD_INDEX_does_not_fail_the_bootstrap()
|
||||
{
|
||||
fx.SkipIfUnavailable();
|
||||
// A sample table whose tag column cannot be indexed (VARCHAR, not SYMBOL): the ALTER is refused up front, which
|
||||
// stands in for any refusal (e.g. no DDL rights). The index is optional, so the bootstrap must still succeed.
|
||||
var p = fx.UniquePrefix();
|
||||
var o = fx.Options(p);
|
||||
var conn = new QuestDbConnection(o);
|
||||
await using (var c = await conn.OpenAsync(Ct))
|
||||
{
|
||||
var ddl = QuestDbTables.CreateSample(p, null).Replace("tag SYMBOL INDEX", "tag VARCHAR", StringComparison.Ordinal);
|
||||
await using var cmd = conn.CreateCommand(c, ddl);
|
||||
await cmd.ExecuteNonQueryAsync(Ct);
|
||||
await using var alter = conn.CreateCommand(c, QuestDbTables.AddTagIndex(conn.SampleTable));
|
||||
await Should.ThrowAsync<Exception>(() => alter.ExecuteNonQueryAsync(Ct)); // precondition: really refused
|
||||
}
|
||||
var log = new ListLogger();
|
||||
await new QuestDbSchemaBootstrapper(conn, o, log).EnsureAsync(Ct);
|
||||
log.Warnings.ShouldContain(m => m.Contains("Could not add the tag index", StringComparison.Ordinal)
|
||||
&& m.Contains(QuestDbTables.AddTagIndex(conn.SampleTable), StringComparison.Ordinal));
|
||||
}
|
||||
|
||||
/// <summary>Captures the rendered Warning messages.</summary>
|
||||
private sealed class ListLogger : Microsoft.Extensions.Logging.ILogger
|
||||
{
|
||||
private readonly System.Collections.Concurrent.ConcurrentQueue<string> _warnings = new();
|
||||
public IReadOnlyCollection<string> Warnings => _warnings;
|
||||
public IDisposable? BeginScope<TState>(TState state) where TState : notnull => null;
|
||||
public bool IsEnabled(Microsoft.Extensions.Logging.LogLevel logLevel) => true;
|
||||
public void Log<TState>(Microsoft.Extensions.Logging.LogLevel logLevel, Microsoft.Extensions.Logging.EventId eventId,
|
||||
TState state, Exception? exception, Func<TState, Exception?, string> formatter)
|
||||
{
|
||||
if (logLevel == Microsoft.Extensions.Logging.LogLevel.Warning) _warnings.Enqueue(formatter(state, exception));
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task The_data_range_seeks_the_tag_index()
|
||||
{
|
||||
await using var b = await QuestDbConformanceBackend.CreateConcreteAsync(fx);
|
||||
foreach (var sql in new[]
|
||||
{
|
||||
QuestDbAggregateQueries.EarliestSample(b.Conn.SampleTable),
|
||||
QuestDbAggregateQueries.LatestSample(b.Conn.SampleTable),
|
||||
})
|
||||
{
|
||||
await using var c = await b.Conn.OpenAsync(Ct);
|
||||
await using var cmd = b.Conn.CreateCommand(c, "EXPLAIN " + sql);
|
||||
QuestDbConnection.BindText(cmd, "R");
|
||||
QuestDbConnection.BindTimestamp(cmd, At(0));
|
||||
var plan = new List<string>();
|
||||
await using var r = await cmd.ExecuteReaderAsync(Ct);
|
||||
while (await r.ReadAsync(Ct)) plan.Add(r.GetString(0));
|
||||
string.Join("\n", plan).ShouldContain("Index", Case.Sensitive, "a bound-parameter tag seek must use the symbol index");
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task The_data_range_excludes_tombstones_and_counts_outage_markers()
|
||||
{
|
||||
await using var b = await QuestDbConformanceBackend.CreateConcreteAsync(fx);
|
||||
await SeedAsync(b);
|
||||
var src = b.CreateDataSource();
|
||||
|
||||
// [+50, +450) by 50 s: data range [+0 outage marker, +200]; the +300/+400 tombstones must not extend it.
|
||||
var count = (await src.ReadProcessedAsync("R", At(50), At(450), TimeSpan.FromSeconds(50), HistoryAggregateType.Count, Ct)).Samples;
|
||||
count.Select(s => s.Value is null ? (long?)null : Convert.ToInt64(s.Value)).ShouldBe([0L, 1L, 0L, 1L, null, null, null, null]);
|
||||
count.Select(s => s.StatusCode).ShouldBe(
|
||||
[
|
||||
GoodCalc, GoodCalc, GoodCalc,
|
||||
GoodCalc | HistorianInfoBits.HistorianPartial, // the data ends inside [+200, +250)
|
||||
NoDataCalc, NoDataCalc, NoDataCalc, NoDataCalc,
|
||||
]);
|
||||
|
||||
// [-100, +100): the outage marker at +0 starts the data range, so [-50, 0] reaches it (edges inclusive) ⇒ 0, Good.
|
||||
var early = (await src.ReadProcessedAsync("R", At(-100), At(100), TimeSpan.FromSeconds(50), HistoryAggregateType.Count, Ct)).Samples;
|
||||
early[0].StatusCode.ShouldBe(NoDataCalc);
|
||||
Convert.ToInt64(early[1].Value).ShouldBe(0L);
|
||||
early[1].StatusCode.ShouldBe(GoodCalc);
|
||||
Convert.ToInt64(early[3].Value).ShouldBe(0L);
|
||||
early[3].StatusCode.ShouldBe(GoodCalc);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_pre_683_table_gains_the_index_on_boot_and_every_processed_read_answers_the_same()
|
||||
{
|
||||
await using var b = await QuestDbConformanceBackend.CreatePre683Async(fx);
|
||||
(await b.TagIndexedAsync()).ShouldBeFalse();
|
||||
await SeedAsync(b);
|
||||
var unindexed = await ReadMatrixAsync(b);
|
||||
|
||||
// AutoCreateSchema=false never alters (it only logs the statement to run).
|
||||
await new QuestDbSchemaBootstrapper(b.Conn, b.Options).EnsureAsync(Ct);
|
||||
await Task.Delay(300);
|
||||
(await b.TagIndexedAsync()).ShouldBeFalse();
|
||||
|
||||
// AutoCreateSchema=true migrates; a second, concurrent-style re-send on the same column is harmless.
|
||||
var migrate = fx.Options(b.Options.TablePrefix);
|
||||
await new QuestDbSchemaBootstrapper(new QuestDbConnection(migrate), migrate).EnsureAsync(Ct);
|
||||
await new QuestDbSchemaBootstrapper(new QuestDbConnection(migrate), migrate).EnsureAsync(Ct);
|
||||
await AwaitIndexedAsync(b);
|
||||
await b.Wal.WaitAsync(b.Conn.SampleTable, Ct);
|
||||
|
||||
var indexed = await ReadMatrixAsync(b);
|
||||
indexed.ShouldBe(unindexed);
|
||||
|
||||
// Still writable and readable after the migration (the table was not suspended by the double ALTER).
|
||||
await b.SeedAsync("R", (At(500), HistorianValue.FromDouble(5), ConformanceData.Good));
|
||||
(await b.ReadBackAsync("R")).Count.ShouldBe(4);
|
||||
}
|
||||
}
|
||||
+53
-1
@@ -107,10 +107,62 @@ public sealed class BucketRowMappingTests
|
||||
foreach (var c in new[] { "total", "usable", "good", "bad", "nonnum" }) t.Columns.Add(c, typeof(long));
|
||||
t.Columns.Add("num_value", typeof(double));
|
||||
t.Columns.Add("ts", typeof(DateTime));
|
||||
t.Rows.Add(0L, 1L, 1L, 1L, 0L, 0L, DBNull.Value, DBNull.Value);
|
||||
t.Columns.Add("unc_min", typeof(double));
|
||||
t.Columns.Add("unc_max", typeof(double));
|
||||
t.Columns.Add("goodnonnum", typeof(long));
|
||||
t.Rows.Add(0L, 1L, 1L, 1L, 0L, 0L, DBNull.Value, DBNull.Value, 4d, DBNull.Value, 0L);
|
||||
|
||||
var b = Map(t, AggregateFamily.ActualTime);
|
||||
b.Extreme.ShouldBeNull();
|
||||
AggregateEvaluator.Evaluate(HistoryAggregateType.MaximumActualTime, b, T, T).StatusCode.ShouldBe(NoDataCalc);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_simple_row_maps_the_good_value_columns_and_the_uncertain_extremes()
|
||||
{
|
||||
// #684: AvgValue/MinValue/MaxValue/SumValue are over Good samples; UncMin/UncMax (ordinals 10, 11) feed the Part 13
|
||||
// Minimum/Maximum/Range status rule.
|
||||
var t = new DataTable();
|
||||
t.Columns.Add("b", typeof(long));
|
||||
foreach (var c in new[] { "total", "usable", "good", "bad", "nonnum" }) t.Columns.Add(c, typeof(long));
|
||||
foreach (var c in new[] { "avg", "min", "max", "sum", "uncmin", "uncmax" }) t.Columns.Add(c, typeof(double));
|
||||
t.Columns.Add("goodnonnum", typeof(long));
|
||||
t.Rows.Add(0L, 4L, 4L, 2L, 0L, 1L, 20d, 10d, 30d, 40d, 5d, 25d, 0L);
|
||||
|
||||
var b = Map(t, AggregateFamily.Simple);
|
||||
b.Avg.ShouldBe(20d);
|
||||
b.Min.ShouldBe(10d);
|
||||
b.Max.ShouldBe(30d);
|
||||
b.Sum.ShouldBe(40d);
|
||||
b.UncertainMin.ShouldBe(5d);
|
||||
b.UncertainMax.ShouldBe(25d);
|
||||
b.NonNumeric.ShouldBe(1);
|
||||
b.GoodNonNumeric.ShouldBe(0);
|
||||
// Uncertain 5 is below the Good minimum ⇒ Minimum Uncertain. The one non-numeric sample is Uncertain (ordinal 12 says
|
||||
// no Good one), so Average is served — not refused — and Maximum is Uncertain for it (25 alone is inside the range).
|
||||
AggregateEvaluator.Evaluate(HistoryAggregateType.Minimum, b, T, T).StatusCode.ShouldBe(0x40A40401u);
|
||||
AggregateEvaluator.Evaluate(HistoryAggregateType.Average, b, T, T).Value.ShouldBe(20d);
|
||||
AggregateEvaluator.Evaluate(HistoryAggregateType.Maximum, b, T, T).StatusCode.ShouldBe(0x40A40401u);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void An_actual_time_row_maps_the_uncertain_extremes_after_the_picked_sample()
|
||||
{
|
||||
var t = new DataTable();
|
||||
t.Columns.Add("b", typeof(long));
|
||||
foreach (var c in new[] { "total", "usable", "good", "bad", "nonnum" }) t.Columns.Add(c, typeof(long));
|
||||
t.Columns.Add("num_value", typeof(double));
|
||||
t.Columns.Add("ts", typeof(DateTime));
|
||||
t.Columns.Add("unc_min", typeof(double));
|
||||
t.Columns.Add("unc_max", typeof(double));
|
||||
t.Columns.Add("goodnonnum", typeof(long));
|
||||
t.Rows.Add(0L, 3L, 3L, 2L, 0L, 0L, 10d, T, 5d, 25d, 0L);
|
||||
|
||||
var b = Map(t, AggregateFamily.ActualTime);
|
||||
b.Extreme.ShouldBe(10d);
|
||||
b.ExtremeTime.ShouldBe(T);
|
||||
b.UncertainMin.ShouldBe(5d);
|
||||
b.UncertainMax.ShouldBe(25d);
|
||||
AggregateEvaluator.Evaluate(HistoryAggregateType.MinimumActualTime, b, T, T).StatusCode.ShouldBe(0x40A40401u);
|
||||
}
|
||||
}
|
||||
|
||||
+11
-2
@@ -8,17 +8,26 @@ public sealed class QuestDbTablesTests
|
||||
[Fact]
|
||||
public void Sample_ddl_without_ttl_is_pinned() =>
|
||||
QuestDbTables.CreateSample("hist_", null).ShouldBe(
|
||||
"CREATE TABLE IF NOT EXISTS hist_sample (ts TIMESTAMP, tag SYMBOL, kind BYTE, num_value DOUBLE, " +
|
||||
"CREATE TABLE IF NOT EXISTS hist_sample (ts TIMESTAMP, tag SYMBOL INDEX, kind BYTE, num_value DOUBLE, " +
|
||||
"int_value LONG, str_value VARCHAR, status LONG, deleted BOOLEAN, server_ts TIMESTAMP) " +
|
||||
"TIMESTAMP(ts) PARTITION BY DAY WAL DEDUP UPSERT KEYS(ts, tag)");
|
||||
|
||||
[Fact]
|
||||
public void Sample_ddl_with_ttl_puts_ttl_between_partition_and_wal() =>
|
||||
QuestDbTables.CreateSample("hist_", 7).ShouldBe(
|
||||
"CREATE TABLE IF NOT EXISTS hist_sample (ts TIMESTAMP, tag SYMBOL, kind BYTE, num_value DOUBLE, " +
|
||||
"CREATE TABLE IF NOT EXISTS hist_sample (ts TIMESTAMP, tag SYMBOL INDEX, kind BYTE, num_value DOUBLE, " +
|
||||
"int_value LONG, str_value VARCHAR, status LONG, deleted BOOLEAN, server_ts TIMESTAMP) " +
|
||||
"TIMESTAMP(ts) PARTITION BY DAY TTL 7 DAYS WAL DEDUP UPSERT KEYS(ts, tag)");
|
||||
|
||||
[Fact]
|
||||
public void Tag_index_migration_statements_are_pinned()
|
||||
{
|
||||
// #683: the migration for a sample table created before the tag index existed, and its probe.
|
||||
QuestDbTables.AddTagIndex("hist_sample").ShouldBe("ALTER TABLE hist_sample ALTER COLUMN tag ADD INDEX");
|
||||
QuestDbTables.TagIndexedQuery("hist_sample").ShouldBe(
|
||||
"SELECT indexed FROM table_columns('hist_sample') WHERE \"column\" = 'tag'");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Event_ddl_is_pinned()
|
||||
{
|
||||
|
||||
+129
-7
@@ -248,14 +248,136 @@ public sealed class AggregateEvaluatorTests
|
||||
ex.Aggregate.ShouldBe(HistoryAggregateType.TimeAverage);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void An_uncertain_sample_in_a_bucket_degrades_the_aggregate_to_uncertain()
|
||||
/// <summary>
|
||||
/// #684: a bucket of Good 10 and Good 30 plus the given Uncertain sample(s) (no Bad). The value columns are the Good
|
||||
/// values only, which is what the backends' SQL now computes (the good flag, not the usable one).
|
||||
/// </summary>
|
||||
private static BucketRow GoodTenThirtyPlusUncertain(params double[] uncertain) => new()
|
||||
{
|
||||
// Two usable samples, one Good and one Uncertain, no Bad: the value is computed, the status is Uncertain.
|
||||
var row = new BucketRow { Total = 2, Usable = 2, Good = 1, Bad = 0, Avg = 15, Min = 10, Max = 20, Sum = 30 };
|
||||
var avg = AggregateEvaluator.Evaluate(HistoryAggregateType.Average, row, At(0), Now);
|
||||
avg.Value.ShouldBe(15d);
|
||||
avg.StatusCode.ShouldBe(UncertainCalc);
|
||||
Total = 2 + uncertain.Length, Usable = 2 + uncertain.Length, Good = 2, Bad = 0,
|
||||
Avg = 20, Min = 10, Max = 30, Sum = 40,
|
||||
UncertainMin = uncertain.Length == 0 ? null : uncertain.Min(),
|
||||
UncertainMax = uncertain.Length == 0 ? null : uncertain.Max(),
|
||||
};
|
||||
|
||||
private static DataValueSnapshot Eval(HistoryAggregateType agg, BucketRow row)
|
||||
{
|
||||
if (agg is HistoryAggregateType.MinimumActualTime or HistoryAggregateType.MaximumActualTime)
|
||||
{
|
||||
row.Extreme = agg == HistoryAggregateType.MinimumActualTime ? row.Min : row.Max;
|
||||
row.ExtremeTime = At(5);
|
||||
}
|
||||
return AggregateEvaluator.Evaluate(agg, row, At(0), Now);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
// Uncertain 20 sits INSIDE the Good range [10, 30]: Part 13 (and the SDK's ComputeMinMax) leave Min/Max/Range Good;
|
||||
// Average is value-based (Good only when every sample is Good, the SDK's GetValueBasedStatusCode) => Uncertain.
|
||||
[InlineData(HistoryAggregateType.Average, new[] { 20d }, 20d, false)]
|
||||
[InlineData(HistoryAggregateType.Minimum, new[] { 20d }, 10d, true)]
|
||||
[InlineData(HistoryAggregateType.Maximum, new[] { 20d }, 30d, true)]
|
||||
[InlineData(HistoryAggregateType.Range, new[] { 20d }, 20d, true)]
|
||||
[InlineData(HistoryAggregateType.MinimumActualTime, new[] { 20d }, 10d, true)]
|
||||
[InlineData(HistoryAggregateType.MaximumActualTime, new[] { 20d }, 30d, true)]
|
||||
// Equal to an extreme: the Part 13 comparison is strict ("less than" / "greater than") => still Good.
|
||||
[InlineData(HistoryAggregateType.Minimum, new[] { 10d }, 10d, true)]
|
||||
[InlineData(HistoryAggregateType.Maximum, new[] { 30d }, 30d, true)]
|
||||
[InlineData(HistoryAggregateType.Range, new[] { 10d, 30d }, 20d, true)]
|
||||
// Below the Good minimum: Minimum / MinimumActualTime / Range Uncertain, Maximum untouched; the value is still the
|
||||
// Good extreme, never the Uncertain one (pre-#684 Minimum answered 5).
|
||||
[InlineData(HistoryAggregateType.Minimum, new[] { 5d }, 10d, false)]
|
||||
[InlineData(HistoryAggregateType.MinimumActualTime, new[] { 5d }, 10d, false)]
|
||||
[InlineData(HistoryAggregateType.Maximum, new[] { 5d }, 30d, true)]
|
||||
[InlineData(HistoryAggregateType.MaximumActualTime, new[] { 5d }, 30d, true)]
|
||||
[InlineData(HistoryAggregateType.Range, new[] { 5d }, 20d, false)]
|
||||
[InlineData(HistoryAggregateType.Average, new[] { 5d }, 20d, false)]
|
||||
// Above the Good maximum: the mirror.
|
||||
[InlineData(HistoryAggregateType.Maximum, new[] { 99d }, 30d, false)]
|
||||
[InlineData(HistoryAggregateType.MaximumActualTime, new[] { 99d }, 30d, false)]
|
||||
[InlineData(HistoryAggregateType.Minimum, new[] { 99d }, 10d, true)]
|
||||
[InlineData(HistoryAggregateType.MinimumActualTime, new[] { 99d }, 10d, true)]
|
||||
[InlineData(HistoryAggregateType.Range, new[] { 99d }, 20d, false)]
|
||||
public void Uncertain_samples_are_not_part_of_the_value_and_degrade_the_status_per_Part_13(HistoryAggregateType agg,
|
||||
double[] uncertain, double expected, bool good)
|
||||
{
|
||||
var v = Eval(agg, GoodTenThirtyPlusUncertain(uncertain));
|
||||
v.Value.ShouldBe(expected);
|
||||
v.StatusCode.ShouldBe(good ? GoodCalc : UncertainCalc);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(HistoryAggregateType.Average)]
|
||||
[InlineData(HistoryAggregateType.Minimum)]
|
||||
[InlineData(HistoryAggregateType.Maximum)]
|
||||
[InlineData(HistoryAggregateType.Range)]
|
||||
[InlineData(HistoryAggregateType.MinimumActualTime)]
|
||||
[InlineData(HistoryAggregateType.MaximumActualTime)]
|
||||
public void Only_a_good_non_numeric_sample_refuses_a_good_value_aggregate(HistoryAggregateType agg)
|
||||
{
|
||||
// #684 review: an UNCERTAIN String/DateTime sample is outside the Good-value set — served (Uncertain), not refused.
|
||||
var uncertainString = GoodTenThirtyPlusUncertain();
|
||||
uncertainString.Total = uncertainString.Usable = 3;
|
||||
uncertainString.NonNumeric = 1;
|
||||
uncertainString.GoodNonNumeric = 0;
|
||||
var v = Eval(agg, uncertainString);
|
||||
v.Value.ShouldNotBeNull();
|
||||
v.StatusCode.ShouldBe(UncertainCalc);
|
||||
// A GOOD one still refuses the interval (the value set would hold a value with no numeric meaning).
|
||||
var goodString = GoodTenThirtyPlusUncertain();
|
||||
goodString.Total = goodString.Usable = goodString.Good = 3;
|
||||
goodString.NonNumeric = 1;
|
||||
goodString.GoodNonNumeric = 1;
|
||||
Eval(agg, goodString).StatusCode.ShouldBe(InvalidInputsCalc);
|
||||
// A row that does not carry the Good count (null) keeps the stricter pre-#684 check: any usable one refuses.
|
||||
uncertainString.GoodNonNumeric = null;
|
||||
Eval(agg, uncertainString).StatusCode.ShouldBe(InvalidInputsCalc);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Delta_still_refuses_any_usable_non_numeric_sample()
|
||||
{
|
||||
// Delta reads the first/last USABLE samples (#680), so an Uncertain String can be its input: unchanged by #684.
|
||||
var row = new BucketRow
|
||||
{
|
||||
Total = 2, Usable = 2, Good = 1, NonNumeric = 1, GoodNonNumeric = 0,
|
||||
First = HistorianValue.FromString("x"), Last = HistorianValue.FromDouble(1), FirstAny = A[0], LastAny = A[1],
|
||||
};
|
||||
AggregateEvaluator.Evaluate(HistoryAggregateType.Delta, row, At(0), Now).StatusCode.ShouldBe(InvalidInputsCalc);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(HistoryAggregateType.Minimum)]
|
||||
[InlineData(HistoryAggregateType.Maximum)]
|
||||
[InlineData(HistoryAggregateType.Range)]
|
||||
[InlineData(HistoryAggregateType.MinimumActualTime)]
|
||||
[InlineData(HistoryAggregateType.MaximumActualTime)]
|
||||
public void A_bad_sample_makes_the_min_max_family_uncertain_even_with_no_uncertain_value(HistoryAggregateType agg)
|
||||
{
|
||||
var row = GoodTenThirtyPlusUncertain();
|
||||
row.Total = 3;
|
||||
row.Bad = 1;
|
||||
Eval(agg, row).StatusCode.ShouldBe(UncertainCalc);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(HistoryAggregateType.Average)]
|
||||
[InlineData(HistoryAggregateType.Minimum)]
|
||||
[InlineData(HistoryAggregateType.Maximum)]
|
||||
[InlineData(HistoryAggregateType.Range)]
|
||||
[InlineData(HistoryAggregateType.MinimumActualTime)]
|
||||
[InlineData(HistoryAggregateType.MaximumActualTime)]
|
||||
public void An_interval_holding_only_uncertain_values_is_no_data(HistoryAggregateType agg)
|
||||
{
|
||||
// #684 (Part 13: these aggregates are defined over Good values; the SDK returns GetNoDataValue when none is Good).
|
||||
// The SQL's Good-only value columns are NULL; the Uncertain extremes are not a substitute.
|
||||
var row = new BucketRow { Total = 2, Usable = 2, Good = 0, UncertainMin = 4, UncertainMax = 6 };
|
||||
var v = AggregateEvaluator.Evaluate(agg, row, At(0), Now);
|
||||
v.Value.ShouldBeNull();
|
||||
v.StatusCode.ShouldBe(NoDataCalc);
|
||||
// Count is unchanged by #684: 0 Good samples, value-based Uncertain.
|
||||
var count = AggregateEvaluator.Evaluate(HistoryAggregateType.Count, row, At(0), Now);
|
||||
count.Value.ShouldBe(0);
|
||||
count.StatusCode.ShouldBe(UncertainCalc);
|
||||
}
|
||||
|
||||
// ---- Live-gate 8 F1: a row whose counts and picks disagree (a backend that read them from two table versions).
|
||||
|
||||
+9
@@ -19,4 +19,13 @@ public sealed class ProcessedReadTests
|
||||
HistoryAggregateType.PercentGood, HistoryAggregateType.PercentBad, HistoryAggregateType.TimeAverage2,
|
||||
HistoryAggregateType.Total2]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Served_is_exactly_the_members_with_a_family()
|
||||
{
|
||||
// #682: the advertised set and the dispatch are one definition — a member is served iff it has a family.
|
||||
AggregateFamilies.Served.Count.ShouldBe(17);
|
||||
foreach (var a in Enum.GetValues<HistoryAggregateType>())
|
||||
AggregateFamilies.Served.Contains(a).ShouldBe(AggregateFamilies.FamilyOf(a) is not null, a.ToString());
|
||||
}
|
||||
}
|
||||
|
||||
+162
-11
@@ -16,9 +16,14 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common.Tests;
|
||||
/// <remarks>
|
||||
/// Deliberate, documented divergences the datasets avoid (they are not anchoring questions):
|
||||
/// <list type="bullet">
|
||||
/// <item><b>Uncertain</b> samples in a per-sample aggregate: the SDK's Average/Min/Max use Good samples only,
|
||||
/// ours every usable (Good + Uncertain) sample (TreatUncertainAsBad = false applied consistently). Count matches
|
||||
/// the SDK since #680 (Good samples only);</item>
|
||||
/// <item><b>an Uncertain value beyond the Good extreme</b> in Minimum / Maximum / Range / *ActualTime: Part 13
|
||||
/// (Tables 57-62) makes the result <c>Uncertain_DataSubNormal</c>, ours follows it; the SDK's <c>ComputeMinMax</c>
|
||||
/// collects the Uncertain extremes and never reads them, so it answers Good. The value is the Good extreme in both
|
||||
/// (#684). <see cref="DenseUncertain"/> keeps its Uncertain values inside every interval's Good range so the
|
||||
/// per-sample comparison needs no mask; the divergence itself is pinned by
|
||||
/// <see cref="An_uncertain_value_beyond_the_good_extreme_is_Uncertain_per_Part_13_where_the_SDK_says_Good"/>.
|
||||
/// (Uncertain samples otherwise match since #684: Average / Min / Max / Range / *ActualTime / Count all use Good
|
||||
/// samples only, as the SDK does under the server's TreatUncertainAsBad = false);</item>
|
||||
/// <item>the per-sample <b>Calculated / MultipleValues</b> info bits (the SDK's MinMax sets Calculated only when the
|
||||
/// extreme is not at the interval start, and flags ties) — those comparisons check code bits + Partial only;</item>
|
||||
/// <item>a <b>truncated</b> interval: ours sets Partial on a non-Bad time-weighted result (Part 13 §5.3.3.2 Case 3
|
||||
@@ -66,6 +71,20 @@ public sealed class SdkAggregateParityTests
|
||||
.Select(x => S(x.t, HistorianValue.FromDouble(10 + (x.i * 7 % 13)), x.t == 22.5 ? Bad : Good))
|
||||
.ToArray();
|
||||
|
||||
/// <summary>
|
||||
/// #684: <see cref="Dense"/> with every fifth sample (from +2.5) Uncertain at 16 and the one Bad at +22.5. The cycling
|
||||
/// Good values alternate around 16 (every adjacent pair has one at or below it and one at or above it), so an
|
||||
/// Uncertain 16 lies inside the Good range of every interval of every window (each holds 5+ samples): Average /
|
||||
/// Minimum / Maximum / Range / *ActualTime must leave it out of the value (Good only — the SDK, Part 13), and the
|
||||
/// min/max family's status matches the SDK's without a mask.
|
||||
/// </summary>
|
||||
private static readonly StoredSample[] DenseUncertain = Enumerable.Range(0, 45)
|
||||
.Select(i => (t: -10 + i * 2.5, i))
|
||||
.Select(x => x.i % 5 == 0 && x.t != 22.5
|
||||
? S(x.t, HistorianValue.FromDouble(16), Uncertain)
|
||||
: S(x.t, HistorianValue.FromDouble(10 + (x.i * 7 % 13)), x.t == 22.5 ? Bad : Good))
|
||||
.ToArray();
|
||||
|
||||
/// <summary>Tag A with Good bounds on both sides, plus an Uncertain sample and an outage marker further on.</summary>
|
||||
private static readonly StoredSample[] Bounded =
|
||||
[
|
||||
@@ -302,11 +321,12 @@ public sealed class SdkAggregateParityTests
|
||||
{
|
||||
var failures = new List<string>();
|
||||
foreach (var agg in PerSample)
|
||||
foreach (var (name, data) in new[] { ("Dense", Dense), ("DenseUncertain", DenseUncertain) })
|
||||
{
|
||||
try
|
||||
{
|
||||
SdkAggregateOracle.ShouldMatch(Ours(agg, from, to, intervalS, Dense), Sdk(agg, from, to, intervalS, Dense),
|
||||
$"{agg} Dense {from}→{to}/{intervalS}s", compareCalculatedInterpolatedBits: false,
|
||||
SdkAggregateOracle.ShouldMatch(Ours(agg, from, to, intervalS, data), Sdk(agg, from, to, intervalS, data),
|
||||
$"{agg} {name} {from}→{to}/{intervalS}s", compareCalculatedInterpolatedBits: false,
|
||||
compareValues: true);
|
||||
}
|
||||
catch (Exception ex)
|
||||
@@ -317,6 +337,131 @@ public sealed class SdkAggregateParityTests
|
||||
failures.ShouldBeEmpty(string.Join("\n", failures));
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[MemberData(nameof(Windows))]
|
||||
public void The_uncertain_dataset_is_not_vacuous(double from, double to, double intervalS)
|
||||
{
|
||||
// #684: DenseUncertain must discriminate the pre-#684 behaviour in every window, or its parity proves nothing.
|
||||
// (1) VALUES: Average over the usable (Good + Uncertain) samples differs from the SDK's Good-only Average in some
|
||||
// interval. (2) STATUS: Minimum / Maximum hold an Uncertain sample in some interval the SDK still calls Good (16 sits
|
||||
// inside the Good range) — the pre-#684 "any non-Good sample ⇒ Uncertain" rule would answer Uncertain there.
|
||||
// (Minimum / Maximum VALUES do not discriminate here: an in-range 16 never is the extreme — by design, so the
|
||||
// min/max family needs no mask; the value half is pinned by UncertainBeyond and AggregateEvaluatorTests.)
|
||||
var plan = BucketPlan.Create(At(from), At(to), TimeSpan.FromSeconds(intervalS));
|
||||
// The one-interval windows hold the Bad +22.5 sample, so the SDK is Uncertain there for a reason the old rule shares:
|
||||
// the status half is checked on the multi-interval windows only (the value half below covers every window).
|
||||
foreach (var agg in plan.BucketCount > 1 ? new[] { HistoryAggregateType.Minimum, HistoryAggregateType.Maximum } : [])
|
||||
{
|
||||
var sdkMinMax = Sdk(agg, from, to, intervalS, DenseUncertain);
|
||||
var rows = InMemoryRows(plan, DenseUncertain).ToList();
|
||||
var holdsUncertain = (plan.Reversed ? Enumerable.Reverse(rows) : rows).Select(r => r.Usable > r.Good).ToList();
|
||||
holdsUncertain.Zip(sdkMinMax).Any(p => p.First && (p.Second.StatusCode & 0xFFFF0000u) == 0)
|
||||
.ShouldBeTrue($"{agg}: some interval holds an Uncertain sample yet is Good in the SDK");
|
||||
}
|
||||
var usableAverage = InMemoryRows(plan, DenseUncertain.Select(x => x.IsUsable && !x.IsGood
|
||||
? x with { Status = Good } : x))
|
||||
.Select(r => r.Avg).ToList();
|
||||
var sdk = Sdk(HistoryAggregateType.Average, from, to, intervalS, DenseUncertain).Select(r => r.Value).ToList();
|
||||
var ours = plan.Reversed ? Enumerable.Reverse(usableAverage).ToList() : usableAverage;
|
||||
ours.Zip(sdk).Any(p => p.Second is double d && p.First is double u && Math.Abs(d - u) > 1e-9)
|
||||
.ShouldBeTrue("an Uncertain sample must change some interval's Average when it is (wrongly) included");
|
||||
}
|
||||
|
||||
/// <summary>#684: Good 10 / 30 with an Uncertain value BELOW the Good minimum (+4) and one ABOVE the maximum (+6).</summary>
|
||||
private static readonly StoredSample[] UncertainBeyond =
|
||||
[
|
||||
S(-5, HistorianValue.FromDouble(20), Good),
|
||||
S(2, HistorianValue.FromDouble(10), Good),
|
||||
S(4, HistorianValue.FromDouble(5), Uncertain),
|
||||
S(6, HistorianValue.FromDouble(99), Uncertain),
|
||||
S(8, HistorianValue.FromDouble(30), Good),
|
||||
S(25, HistorianValue.FromDouble(20), Good),
|
||||
];
|
||||
|
||||
[Theory]
|
||||
[InlineData(HistoryAggregateType.Minimum, 0, 20)]
|
||||
[InlineData(HistoryAggregateType.Minimum, 20, 0)]
|
||||
[InlineData(HistoryAggregateType.Maximum, 0, 20)]
|
||||
[InlineData(HistoryAggregateType.Maximum, 20, 0)]
|
||||
[InlineData(HistoryAggregateType.Range, 0, 20)]
|
||||
[InlineData(HistoryAggregateType.Range, 20, 0)]
|
||||
[InlineData(HistoryAggregateType.MinimumActualTime, 0, 20)]
|
||||
[InlineData(HistoryAggregateType.MaximumActualTime, 20, 0)]
|
||||
public void An_uncertain_value_beyond_the_good_extreme_is_Uncertain_per_Part_13_where_the_SDK_says_Good(
|
||||
HistoryAggregateType agg, double from, double to)
|
||||
{
|
||||
// #684, documented divergence (class remarks). Part 13 Tables 57-62: "If an Uncertain value is less than the
|
||||
// minimum Good value [greater than the maximum] the Status is Uncertain_DataSubNormal". The SDK's ComputeMinMax
|
||||
// tracks the Uncertain extremes and never reads them. Everything else — the value (the GOOD extreme, never 5 or
|
||||
// 99), the timestamp, the Partial bit — matches the SDK; only the code bits differ, and only as Part 13 says.
|
||||
var ours = Ours(agg, from, to, 0, UncertainBeyond).Single();
|
||||
var sdk = Sdk(agg, from, to, 0, UncertainBeyond).Single();
|
||||
(sdk.StatusCode & StatusBits.CodeBitsMask).ShouldBe(HistorianStatusCodes.Good, "the SDK ignores the Uncertain extreme");
|
||||
(ours.StatusCode & StatusBits.CodeBitsMask).ShouldBe(HistorianStatusCodes.UncertainDataSubNormal);
|
||||
SdkAggregateOracle.ShouldMatch([ours with { StatusCode = ours.StatusCode & ~StatusBits.CodeBitsMask }], [sdk],
|
||||
$"{agg} UncertainBeyond {from}→{to}", compareCalculatedInterpolatedBits: false);
|
||||
ours.Value.ShouldBe(agg switch
|
||||
{
|
||||
HistoryAggregateType.Minimum or HistoryAggregateType.MinimumActualTime => 10d,
|
||||
HistoryAggregateType.Maximum or HistoryAggregateType.MaximumActualTime => 30d,
|
||||
_ => 20d,
|
||||
});
|
||||
}
|
||||
|
||||
/// <summary>#684 review: Good numeric samples with an Uncertain STRING among them (+4).</summary>
|
||||
private static readonly StoredSample[] UncertainString =
|
||||
[
|
||||
S(-5, HistorianValue.FromDouble(20), Good),
|
||||
S(2, HistorianValue.FromDouble(10), Good),
|
||||
S(4, HistorianValue.FromString("x"), Uncertain),
|
||||
S(8, HistorianValue.FromDouble(30), Good),
|
||||
S(25, HistorianValue.FromDouble(20), Good),
|
||||
];
|
||||
|
||||
[Theory]
|
||||
[InlineData(0, 20)]
|
||||
[InlineData(20, 0)]
|
||||
public void An_uncertain_non_numeric_sample_is_skipped_by_the_good_value_aggregates_like_the_SDK(double from, double to)
|
||||
{
|
||||
// #684 review: an Uncertain String is not part of the Good-value set, so it no longer refuses the interval
|
||||
// (BadAggregateInvalidInputs); the SDK's Average skips it (value-based status: Uncertain) and its ComputeMinMax flags
|
||||
// the value it cannot cast (Uncertain). Exact match, every bit but Calculated.
|
||||
var failures = new List<string>();
|
||||
foreach (var agg in new[]
|
||||
{
|
||||
HistoryAggregateType.Average, HistoryAggregateType.Minimum, HistoryAggregateType.Maximum,
|
||||
HistoryAggregateType.Range, HistoryAggregateType.MinimumActualTime, HistoryAggregateType.MaximumActualTime,
|
||||
})
|
||||
{
|
||||
try
|
||||
{
|
||||
var ours = Ours(agg, from, to, 0, UncertainString);
|
||||
SdkAggregateOracle.ShouldMatch(ours, Sdk(agg, from, to, 0, UncertainString), $"{agg} UncertainString {from}→{to}",
|
||||
compareCalculatedInterpolatedBits: false);
|
||||
(ours.Single().StatusCode & StatusBits.CodeBitsMask).ShouldBe(HistorianStatusCodes.UncertainDataSubNormal, agg.ToString());
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
failures.Add(ex.Message);
|
||||
}
|
||||
}
|
||||
failures.ShouldBeEmpty(string.Join("\n", failures));
|
||||
// Delta keeps its usable picks and refusal (#680, unchanged): no Good-only reading applies to it.
|
||||
Ours(HistoryAggregateType.Delta, from, to, 0, UncertainString).Single().StatusCode
|
||||
.ShouldBe(HistorianInfoBits.WithCalculated(HistorianStatusCodes.BadAggregateInvalidInputs));
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(0, 20)]
|
||||
[InlineData(20, 0)]
|
||||
public void Average_over_uncertain_values_beyond_the_good_range_matches_the_SDK_exactly(double from, double to)
|
||||
{
|
||||
// #684: Average has no extreme rule — Good values only, value-based status (Uncertain: not every sample is Good).
|
||||
SdkAggregateOracle.ShouldMatch(Ours(HistoryAggregateType.Average, from, to, 0, UncertainBeyond),
|
||||
Sdk(HistoryAggregateType.Average, from, to, 0, UncertainBeyond), $"Average UncertainBeyond {from}→{to}");
|
||||
Ours(HistoryAggregateType.Average, from, to, 0, UncertainBeyond).Single().Value.ShouldBe(20d);
|
||||
}
|
||||
|
||||
private static readonly HistoryAggregateType[] PerSample =
|
||||
[
|
||||
HistoryAggregateType.Average, HistoryAggregateType.Minimum, HistoryAggregateType.Maximum,
|
||||
@@ -544,8 +689,11 @@ public sealed class SdkAggregateParityTests
|
||||
var usable = all.Where(s => s.IsUsable).ToList();
|
||||
var numeric = usable.Where(s => s.Value.Kind is HistorianValueKind.Double or HistorianValueKind.Int64
|
||||
or HistorianValueKind.Boolean).ToList();
|
||||
// ActualTime: the extreme usable value, ties ⇒ earliest (the SQL's ROW_NUMBER order).
|
||||
var extreme = numeric.OrderBy(s => maximum ? -s.Value.Number : s.Value.Number).ThenBy(s => s.TimeUtc)
|
||||
// #684: the value columns are over Good numeric samples; the Uncertain extremes feed the Min/Max status rule.
|
||||
var good = numeric.Where(s => s.IsGood).ToList();
|
||||
var uncertain = numeric.Where(s => !s.IsGood).ToList();
|
||||
// ActualTime: the extreme Good value, ties ⇒ earliest (the SQL's ROW_NUMBER order).
|
||||
var extreme = good.OrderBy(s => maximum ? -s.Value.Number : s.Value.Number).ThenBy(s => s.TimeUtc)
|
||||
.Cast<StoredSample?>().FirstOrDefault();
|
||||
yield return new BucketRow
|
||||
{
|
||||
@@ -555,10 +703,13 @@ public sealed class SdkAggregateParityTests
|
||||
Good = all.Count(s => s.IsGood),
|
||||
Bad = all.Count - usable.Count,
|
||||
NonNumeric = usable.Count - numeric.Count,
|
||||
Avg = numeric.Count == 0 ? null : numeric.Average(s => s.Value.Number),
|
||||
Min = numeric.Count == 0 ? null : numeric.Min(s => s.Value.Number),
|
||||
Max = numeric.Count == 0 ? null : numeric.Max(s => s.Value.Number),
|
||||
Sum = numeric.Count == 0 ? null : numeric.Sum(s => s.Value.Number),
|
||||
GoodNonNumeric = usable.Count(s => s.IsGood) - numeric.Count(s => s.IsGood),
|
||||
Avg = good.Count == 0 ? null : good.Average(s => s.Value.Number),
|
||||
Min = good.Count == 0 ? null : good.Min(s => s.Value.Number),
|
||||
Max = good.Count == 0 ? null : good.Max(s => s.Value.Number),
|
||||
Sum = good.Count == 0 ? null : good.Sum(s => s.Value.Number),
|
||||
UncertainMin = uncertain.Count == 0 ? null : uncertain.Min(s => s.Value.Number),
|
||||
UncertainMax = uncertain.Count == 0 ? null : uncertain.Max(s => s.Value.Number),
|
||||
First = usable.Count == 0 ? HistorianValue.Empty : usable[0].Value,
|
||||
Last = usable.Count == 0 ? HistorianValue.Empty : usable[^1].Value,
|
||||
Extreme = extreme?.Value.Number,
|
||||
|
||||
+150
@@ -0,0 +1,150 @@
|
||||
using System.Data;
|
||||
using Microsoft.Data.SqlClient;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer.IntegrationTests;
|
||||
|
||||
/// <summary>
|
||||
/// #683: <see cref="SqlAggregateQueries.StartEnd"/> aggregates the pick times, then fetches each pick by a key join.
|
||||
/// Every pick must be an exact-key seek whatever plan the optimizer would like, because both alternatives it finds are
|
||||
/// pathological and plans are cached and reused across reads/tags:
|
||||
/// <list type="bullet">
|
||||
/// <item>keyed on the pick time alone, a hash/merge/adaptive join reads the tag's whole history (review of 910befc8);</item>
|
||||
/// <item>bounded to the window, a nested-loops RANGE seek over the window per bucket — O(buckets × window rows) — chosen
|
||||
/// when the statistics understate the table and then reused for a wide read (live gate 9 F1: 1 d @ 1 min 20 s).</item>
|
||||
/// </list>
|
||||
/// The tests use a tag with one sample a second over a day plus long history either side, and bound the session's
|
||||
/// logical reads (<c>sys.dm_exec_sessions.logical_reads</c> before/after — STATISTICS IO messages do not reach
|
||||
/// SqlClient's InfoMessage here).
|
||||
/// </summary>
|
||||
[Collection(SqlHistorianCollection.Name)]
|
||||
public sealed class StartEndJoinBackTests(SqlHistorianFixture fx)
|
||||
{
|
||||
private static readonly CancellationToken Ct = CancellationToken.None;
|
||||
private static DateTime At(double s) => ReadTestDb.At(s);
|
||||
private const int WindowSeconds = 86_400;
|
||||
private const int HistoryRowsEachSide = 150_000;
|
||||
|
||||
/// <summary>
|
||||
/// Tag <c>H</c>: one sample a second at <c>+i.25 s</c> over [+0, +1 d) of mixed quality (Good, Bad, Uncertain, an
|
||||
/// outage marker, a String), plus <see cref="HistoryRowsEachSide"/> rows a second before +0 and after +1 d.
|
||||
/// </summary>
|
||||
private static async Task<(ReadTestDb Db, int Id)> SeedAsync(SqlHistorianFixture fx)
|
||||
{
|
||||
var db = await ReadTestDb.CreateAsync(fx);
|
||||
var id = await db.SeedAsync("H", (At(-1e6), HistorianValue.FromDouble(0), ReadTestDb.Good));
|
||||
string T(DateTime t) => $"CAST('{t:yyyy-MM-dd HH:mm:ss.fffffff}' AS datetime2(7))";
|
||||
var cols = "(TagId, SourceTimeUtc, ValueKind, NumValue, IntValue, StrValue, StatusCode, ServerTimeUtc)";
|
||||
await db.ScalarAsync(
|
||||
$"WITH n AS (SELECT TOP ({Math.Max(WindowSeconds, HistoryRowsEachSide)}) ROW_NUMBER() OVER (ORDER BY (SELECT NULL)) - 1 AS x " +
|
||||
"FROM sys.all_objects a CROSS JOIN sys.all_objects b) " +
|
||||
$"INSERT {db.Conn.Q("Sample")} {cols} " +
|
||||
$"SELECT {id}, DATEADD(millisecond, 250, DATEADD(second, x, {T(At(0))})), " +
|
||||
"CASE x % 7 WHEN 5 THEN 0 WHEN 6 THEN 4 ELSE 3 END, CASE WHEN x % 7 IN (5, 6) THEN NULL ELSE x END, NULL, " +
|
||||
"CASE WHEN x % 7 = 6 THEN N's' + CAST(x AS nvarchar(10)) END, " +
|
||||
$"CASE x % 7 WHEN 3 THEN {unchecked((int)ReadTestDb.Bad)} WHEN 4 THEN {unchecked((int)ReadTestDb.Uncertain)} " +
|
||||
$"WHEN 5 THEN {unchecked((int)ReadTestDb.BadNoCommunication)} ELSE 0 END, SYSUTCDATETIME() FROM n WHERE x < {WindowSeconds} " +
|
||||
$"UNION ALL SELECT {id}, DATEADD(second, -1 - x, {T(At(0))}), 3, x, NULL, NULL, 0, SYSUTCDATETIME() FROM n WHERE x < {HistoryRowsEachSide} " +
|
||||
$"UNION ALL SELECT {id}, DATEADD(second, 1 + x, {T(At(WindowSeconds))}), 3, x, NULL, NULL, 0, SYSUTCDATETIME() FROM n WHERE x < {HistoryRowsEachSide}; " +
|
||||
"SELECT 0");
|
||||
await db.ScalarAsync($"UPDATE STATISTICS {db.Conn.Q("Sample")} WITH FULLSCAN; SELECT 0");
|
||||
return (db, id);
|
||||
}
|
||||
|
||||
/// <summary>Runs the StartEnd SQL (optional query hint) and returns its rows as strings plus the session's logical reads.</summary>
|
||||
private static async Task<(List<string> Rows, long LogicalReads)> RunAsync(ReadTestDb db, int id, BucketPlan plan, string? hint = null)
|
||||
{
|
||||
var sql = SqlAggregateQueries.StartEnd(db.Conn.Q("Sample"), plan.Reversed);
|
||||
if (hint is not null) sql = sql.TrimEnd().TrimEnd(';') + $" OPTION ({hint});";
|
||||
await using var c = await db.Conn.OpenAsync(Ct);
|
||||
async Task<long> SessionReadsAsync()
|
||||
{
|
||||
await using var probe = db.Conn.CreateCommand(c, "SELECT logical_reads FROM sys.dm_exec_sessions WHERE session_id = @@SPID");
|
||||
return Convert.ToInt64(await probe.ExecuteScalarAsync(Ct));
|
||||
}
|
||||
var before = await SessionReadsAsync();
|
||||
await using var cmd = db.Conn.CreateCommand(c, sql);
|
||||
cmd.CommandTimeout = 300;
|
||||
cmd.Parameters.Add(new SqlParameter("@id", SqlDbType.Int) { Value = id });
|
||||
SqlHistorianConnection.AddDateTime2(cmd, "@start", plan.StartUtc);
|
||||
SqlHistorianConnection.AddDateTime2(cmd, "@end", plan.EndUtc);
|
||||
cmd.Parameters.Add(new SqlParameter("@intervalTicks", SqlDbType.BigInt) { Value = plan.IntervalTicks });
|
||||
cmd.Parameters.Add(new SqlParameter("@bucketAdjustTicks", SqlDbType.BigInt) { Value = plan.BucketIndexAdjustTicks });
|
||||
var rows = new List<string>();
|
||||
await using (var r = await cmd.ExecuteReaderAsync(Ct))
|
||||
while (await r.ReadAsync(Ct))
|
||||
rows.Add(string.Join("|", Enumerable.Range(0, r.FieldCount).Select(i => r.IsDBNull(i) ? "NULL" : r.GetValue(i) is DateTime d ? d.Ticks.ToString(System.Globalization.CultureInfo.InvariantCulture) : Convert.ToString(r.GetValue(i), System.Globalization.CultureInfo.InvariantCulture))));
|
||||
return (rows, await SessionReadsAsync() - before);
|
||||
}
|
||||
|
||||
private static BucketPlan Day(bool reverse, TimeSpan interval) =>
|
||||
reverse ? BucketPlan.Create(At(WindowSeconds), At(0), interval) : BucketPlan.Create(At(0), At(WindowSeconds), interval);
|
||||
|
||||
/// <summary>
|
||||
/// Forward: bucket <c>b</c> = [b·60, b·60 + 60) holds +b·60.25 … +b·60+59.25. Reverse: (b·60, b·60 + 60] holds the
|
||||
/// same samples (none sits on an edge).
|
||||
/// </summary>
|
||||
private static void AssertPicks(List<string> rows)
|
||||
{
|
||||
rows.Count.ShouldBe(WindowSeconds / 60);
|
||||
for (var b = 0; b < rows.Count; b++)
|
||||
{
|
||||
var f = rows[b].Split('|');
|
||||
f[0].ShouldBe(b.ToString(System.Globalization.CultureInfo.InvariantCulture));
|
||||
f[1].ShouldBe("60"); // TotalCount
|
||||
// any-quality first / last times (ordinals 16, 22)
|
||||
long.Parse(f[16], System.Globalization.CultureInfo.InvariantCulture).ShouldBe(At(b * 60 + 0.25).Ticks);
|
||||
long.Parse(f[22], System.Globalization.CultureInfo.InvariantCulture).ShouldBe(At(b * 60 + 59.25).Ticks);
|
||||
}
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(false)]
|
||||
[InlineData(true)]
|
||||
public async Task A_plan_compiled_under_understated_statistics_still_seeks_each_pick_by_its_exact_key(bool reverse)
|
||||
{
|
||||
fx.SkipIfUnavailable();
|
||||
var (db, id) = await SeedAsync(fx);
|
||||
// Live gate 9 F1's recipe: the optimizer believes the table is tiny, compiles on a small read, and the cached plan is
|
||||
// then reused for a wide many-bucket read. (The schema is unique to this test, so the plan is compiled here.)
|
||||
await db.ScalarAsync($"UPDATE STATISTICS {db.Conn.Q("Sample")} WITH ROWCOUNT = 34, PAGECOUNT = 1; SELECT 0");
|
||||
var small = reverse ? BucketPlan.Create(At(3600), At(0), TimeSpan.FromMinutes(1)) : BucketPlan.Create(At(0), At(3600), TimeSpan.FromMinutes(1));
|
||||
await RunAsync(db, id, small);
|
||||
|
||||
var (rows, reads) = await RunAsync(db, id, Day(reverse, TimeSpan.FromMinutes(1)));
|
||||
|
||||
AssertPicks(rows);
|
||||
reads.ShouldBeGreaterThan(0L, "the session read counter must move (positive control)");
|
||||
// Measured: 18,645 reads (one window scan + 4 key seeks per bucket); the previous query (4a3b1256, bounded joins,
|
||||
// no hint) took the window-range-seek-per-bucket plan here: 1,645,845 reads, 23 s — this test failed against it.
|
||||
reads.ShouldBeLessThan(200_000, "every pick must be an exact-key seek, never a per-bucket window range seek");
|
||||
(await RunAsync(db, id, Day(reverse, TimeSpan.FromMinutes(1)), "RECOMPILE")).Rows.ShouldBe(rows);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task With_accurate_statistics_the_long_history_outside_the_window_is_never_read()
|
||||
{
|
||||
fx.SkipIfUnavailable();
|
||||
var (db, id) = await SeedAsync(fx);
|
||||
var (rows, reads) = await RunAsync(db, id, Day(false, TimeSpan.FromMinutes(1)));
|
||||
AssertPicks(rows);
|
||||
reads.ShouldBeGreaterThan(0L);
|
||||
reads.ShouldBeLessThan(200_000, "the 300k history rows outside the window must not be read");
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("HASH JOIN")]
|
||||
[InlineData("MERGE JOIN")]
|
||||
public async Task No_hash_or_merge_pick_join_exists_to_fall_back_to(string hint)
|
||||
{
|
||||
// The FORCESEEK hint makes the exact-key nested-loops seek the ONLY legal pick join: forcing a hash or merge join
|
||||
// (the whole-history shape of 910befc8) is refused outright (8622) rather than silently chosen.
|
||||
fx.SkipIfUnavailable();
|
||||
var (db, id) = await SeedAsync(fx);
|
||||
var ex = await Should.ThrowAsync<SqlException>(() => RunAsync(db, id, Day(false, TimeSpan.FromHours(1)), hint));
|
||||
ex.Number.ShouldBe(8622);
|
||||
}
|
||||
}
|
||||
+72
-1
@@ -76,6 +76,25 @@ public sealed class BucketRowMappingTests
|
||||
AggregateEvaluator.Evaluate(agg, b, T, T).StatusCode.ShouldBe(NoDataCalc);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_pick_nulled_by_a_delete_racing_the_key_join_is_no_data_for_that_end_only()
|
||||
{
|
||||
// #683 join-back: the bucket's first sample (usable, so both first picks) was deleted between the GROUP BY and its
|
||||
// key seek — the counts still include it, its pick columns are NULL; the last picks joined normally.
|
||||
var t = StartEndTable();
|
||||
var row = Counts(t, 0, 2);
|
||||
row["k1"] = (byte)1; row["n1"] = 20d; row["t1"] = T.AddSeconds(1);
|
||||
row["at1"] = T.AddSeconds(1); row["ak1"] = (byte)1; row["an1"] = 20d; row["st1"] = 0;
|
||||
t.Rows.Add(row);
|
||||
|
||||
var b = Map(t, AggregateFamily.StartEnd);
|
||||
b.FirstAny.ShouldBeNull();
|
||||
b.First.Kind.ShouldBe(HistorianValueKind.Empty);
|
||||
AggregateEvaluator.Evaluate(HistoryAggregateType.Start, b, T, T).StatusCode.ShouldBe(NoDataCalc);
|
||||
AggregateEvaluator.Evaluate(HistoryAggregateType.Delta, b, T, T).StatusCode.ShouldBe(NoDataCalc);
|
||||
AggregateEvaluator.Evaluate(HistoryAggregateType.End, b, T, T).Value.ShouldBe(20d);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_consistent_row_still_maps_every_pick()
|
||||
{
|
||||
@@ -103,10 +122,62 @@ public sealed class BucketRowMappingTests
|
||||
foreach (var c in new[] { "total", "usable", "good", "bad", "nonnum" }) t.Columns.Add(c, typeof(int));
|
||||
t.Columns.Add("NumValue", typeof(double));
|
||||
t.Columns.Add("SourceTimeUtc", typeof(DateTime));
|
||||
t.Rows.Add(0L, 1, 1, 1, 0, 0, DBNull.Value, DBNull.Value);
|
||||
t.Columns.Add("UncMin", typeof(double));
|
||||
t.Columns.Add("UncMax", typeof(double));
|
||||
t.Columns.Add("goodnonnum", typeof(int));
|
||||
t.Rows.Add(0L, 1, 1, 1, 0, 0, DBNull.Value, DBNull.Value, 4d, DBNull.Value, 0);
|
||||
|
||||
var b = Map(t, AggregateFamily.ActualTime);
|
||||
b.Extreme.ShouldBeNull();
|
||||
AggregateEvaluator.Evaluate(HistoryAggregateType.MaximumActualTime, b, T, T).StatusCode.ShouldBe(NoDataCalc);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_simple_row_maps_the_good_value_columns_and_the_uncertain_extremes()
|
||||
{
|
||||
// #684: AvgValue/MinValue/MaxValue/SumValue are over Good samples; UncMin/UncMax (ordinals 10, 11) feed the Part 13
|
||||
// Minimum/Maximum/Range status rule.
|
||||
var t = new DataTable();
|
||||
t.Columns.Add("b", typeof(long));
|
||||
foreach (var c in new[] { "total", "usable", "good", "bad", "nonnum" }) t.Columns.Add(c, typeof(int));
|
||||
foreach (var c in new[] { "avg", "min", "max", "sum", "uncmin", "uncmax" }) t.Columns.Add(c, typeof(double));
|
||||
t.Columns.Add("goodnonnum", typeof(int));
|
||||
t.Rows.Add(0L, 4, 4, 2, 0, 1, 20d, 10d, 30d, 40d, 5d, 25d, 0);
|
||||
|
||||
var b = Map(t, AggregateFamily.Simple);
|
||||
b.Avg.ShouldBe(20d);
|
||||
b.Min.ShouldBe(10d);
|
||||
b.Max.ShouldBe(30d);
|
||||
b.Sum.ShouldBe(40d);
|
||||
b.UncertainMin.ShouldBe(5d);
|
||||
b.UncertainMax.ShouldBe(25d);
|
||||
b.NonNumeric.ShouldBe(1);
|
||||
b.GoodNonNumeric.ShouldBe(0);
|
||||
// Uncertain 5 is below the Good minimum ⇒ Minimum Uncertain. The one non-numeric sample is Uncertain (ordinal 12 says
|
||||
// no Good one), so Average is served — not refused — and Maximum is Uncertain for it (25 alone is inside the range).
|
||||
AggregateEvaluator.Evaluate(HistoryAggregateType.Minimum, b, T, T).StatusCode.ShouldBe(0x40A40401u);
|
||||
AggregateEvaluator.Evaluate(HistoryAggregateType.Average, b, T, T).Value.ShouldBe(20d);
|
||||
AggregateEvaluator.Evaluate(HistoryAggregateType.Maximum, b, T, T).StatusCode.ShouldBe(0x40A40401u);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void An_actual_time_row_maps_the_uncertain_extremes_after_the_picked_sample()
|
||||
{
|
||||
var t = new DataTable();
|
||||
t.Columns.Add("b", typeof(long));
|
||||
foreach (var c in new[] { "total", "usable", "good", "bad", "nonnum" }) t.Columns.Add(c, typeof(int));
|
||||
t.Columns.Add("NumValue", typeof(double));
|
||||
t.Columns.Add("SourceTimeUtc", typeof(DateTime));
|
||||
t.Columns.Add("UncMin", typeof(double));
|
||||
t.Columns.Add("UncMax", typeof(double));
|
||||
t.Columns.Add("goodnonnum", typeof(int));
|
||||
t.Rows.Add(0L, 3, 3, 2, 0, 0, 10d, T, 5d, 25d, 0);
|
||||
|
||||
var b = Map(t, AggregateFamily.ActualTime);
|
||||
b.Extreme.ShouldBe(10d);
|
||||
b.ExtremeTime.ShouldBe(T);
|
||||
b.UncertainMin.ShouldBe(5d);
|
||||
b.UncertainMax.ShouldBe(25d);
|
||||
AggregateEvaluator.Evaluate(HistoryAggregateType.MinimumActualTime, b, T, T).StatusCode.ShouldBe(0x40A40401u);
|
||||
}
|
||||
}
|
||||
|
||||
+35
@@ -129,4 +129,39 @@ public sealed class HistorianBackendRegistryTests
|
||||
o.ConnectionString.ShouldContain("Port=18812");
|
||||
o.TablePrefix.ShouldBe("rig_");
|
||||
}
|
||||
|
||||
/// <summary>The seventeen aggregates the SQL backends serve, spelled out (not derived) so a drift in either the
|
||||
/// family table or a backend's declaration fails here (#682).</summary>
|
||||
private static readonly HistoryAggregateType[] SqlServed =
|
||||
[
|
||||
HistoryAggregateType.Average, HistoryAggregateType.Minimum, HistoryAggregateType.Maximum, HistoryAggregateType.Count,
|
||||
HistoryAggregateType.Range, HistoryAggregateType.Start, HistoryAggregateType.End, HistoryAggregateType.Delta,
|
||||
HistoryAggregateType.MinimumActualTime, HistoryAggregateType.MaximumActualTime, HistoryAggregateType.TimeAverage,
|
||||
HistoryAggregateType.TimeAverage2, HistoryAggregateType.Total, HistoryAggregateType.Total2,
|
||||
HistoryAggregateType.Interpolative, HistoryAggregateType.PercentGood, HistoryAggregateType.PercentBad,
|
||||
];
|
||||
|
||||
[Fact]
|
||||
public void Each_backend_data_source_declares_the_aggregates_it_serves()
|
||||
{
|
||||
// #682: what the server advertises under AggregateFunctions comes from these — resolved exactly as the Host does.
|
||||
var sp = new ServiceCollection().BuildServiceProvider();
|
||||
IHistorianDataSource Source(ServerHistorianOptions o, IConfiguration c) =>
|
||||
HistorianBackendRegistry.Resolve(o, c).CreateDataSource(sp);
|
||||
|
||||
// #681: the ten per-sample aggregates (raw-evaluated) + the two AVEVA-native ones.
|
||||
Source(Gateway(), Cfg).SupportedAggregates.ShouldBe(
|
||||
[
|
||||
HistoryAggregateType.Average, HistoryAggregateType.Count, HistoryAggregateType.Minimum,
|
||||
HistoryAggregateType.Maximum, HistoryAggregateType.Range, HistoryAggregateType.Start, HistoryAggregateType.End,
|
||||
HistoryAggregateType.Delta, HistoryAggregateType.MinimumActualTime, HistoryAggregateType.MaximumActualTime,
|
||||
HistoryAggregateType.TimeAverage, HistoryAggregateType.Total,
|
||||
],
|
||||
ignoreOrder: true);
|
||||
Source(new ServerHistorianOptions { Backend = "SqlServer", Enabled = true }, SqlCfg())
|
||||
.SupportedAggregates.ShouldBe(SqlServed, ignoreOrder: true);
|
||||
Source(new ServerHistorianOptions { Backend = "QuestDb", Enabled = true }, QuestDbCfg())
|
||||
.SupportedAggregates.ShouldBe(SqlServed, ignoreOrder: true);
|
||||
NullHistorianDataSource.Instance.SupportedAggregates.ShouldBeEmpty();
|
||||
}
|
||||
}
|
||||
|
||||
+221
@@ -0,0 +1,221 @@
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Opc.Ua;
|
||||
using Opc.Ua.Client;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Host.Configuration;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Historian;
|
||||
using static ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests.ServerEventRig;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests;
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #682 over a real SDK client session: <c>HistoryServerCapabilities/AggregateFunctions</c> (i=11201) lists
|
||||
/// exactly the aggregates the wired historian backend serves — the data sources are built through
|
||||
/// <see cref="HistorianBackendRegistry"/> exactly as the Host builds them — while
|
||||
/// <c>ServerCapabilities/AggregateFunctions</c> (i=2997) keeps the SDK's full registered set in every configuration,
|
||||
/// because a live-data <c>AggregateFilter</c> subscription keeps working for every one of them.
|
||||
/// </summary>
|
||||
public sealed class AdvertisedAggregateFunctionsWireTests
|
||||
{
|
||||
// #681: the ten per-sample aggregates (raw-evaluated) + TimeAverage / Total (AVEVA-native).
|
||||
private static readonly string[] GatewayNames =
|
||||
[
|
||||
"Average", "Count", "Delta", "End", "Maximum", "MaximumActualTime", "Minimum", "MinimumActualTime", "Range", "Start",
|
||||
"TimeAverage", "Total",
|
||||
];
|
||||
|
||||
private static readonly string[] SqlNames =
|
||||
[
|
||||
"Average", "Count", "Delta", "End", "Interpolative", "Maximum", "MaximumActualTime", "Minimum", "MinimumActualTime",
|
||||
"PercentBad", "PercentGood", "Range", "Start", "TimeAverage", "TimeAverage2", "Total", "Total2",
|
||||
];
|
||||
|
||||
/// <summary>The SDK's 37 registered aggregates (<c>StandardServer.CreateAggregateManager</c>) — i=2997, always.</summary>
|
||||
private static readonly string[] AllSdkNames =
|
||||
[
|
||||
"AnnotationCount", "Average", "Count", "Delta", "DeltaBounds", "DurationBad", "DurationGood", "DurationInStateNonZero",
|
||||
"DurationInStateZero", "End", "EndBound", "Interpolative", "Maximum", "Maximum2", "MaximumActualTime",
|
||||
"MaximumActualTime2", "Minimum", "Minimum2", "MinimumActualTime", "MinimumActualTime2", "NumberOfTransitions",
|
||||
"PercentBad", "PercentGood", "Range", "Range2", "StandardDeviationPopulation", "StandardDeviationSample", "Start",
|
||||
"StartBound", "TimeAverage", "TimeAverage2", "Total", "Total2", "VariancePopulation", "VarianceSample",
|
||||
"WorstQuality", "WorstQuality2",
|
||||
];
|
||||
|
||||
private static IHistorianDataSource Backend(string backend)
|
||||
{
|
||||
var config = new ConfigurationBuilder().AddInMemoryCollection(new Dictionary<string, string?>
|
||||
{
|
||||
["ServerHistorian:SqlServer:ConnectionString"] = "Server=localhost;User Id=sa;Password=x;TrustServerCertificate=True",
|
||||
["ServerHistorian:QuestDb:ConnectionString"] = "Host=localhost;Port=18812;Username=admin;Password=quest;Database=qdb",
|
||||
}).Build();
|
||||
var options = new ServerHistorianOptions
|
||||
{
|
||||
Backend = backend, Enabled = true, Endpoint = "http://localhost:5222", ApiKey = "histgw_id_secret", UseTls = false,
|
||||
};
|
||||
// Constructing a data source opens no connection; nothing here reaches a database or a gateway.
|
||||
return HistorianBackendRegistry.Resolve(options, config).CreateDataSource(new ServiceCollection().BuildServiceProvider());
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("Gateway")]
|
||||
[InlineData("SqlServer")]
|
||||
[InlineData("QuestDb")]
|
||||
public async Task History_folder_advertises_the_backend_set_and_the_server_folder_keeps_every_aggregate(string backend)
|
||||
{
|
||||
var ct = TestContext.Current.CancellationToken;
|
||||
var rig = await StartAsync($"AggAdv{backend}", encrypted: false, ct);
|
||||
try
|
||||
{
|
||||
rig.Nm.HistorianDataSource = Backend(backend);
|
||||
rig.Enforce(false, [Key()]);
|
||||
using var session = await rig.ConnectAsync(anonymous: true, ct);
|
||||
|
||||
var expected = backend == "Gateway" ? GatewayNames : SqlNames;
|
||||
(await AdvertisedAsync(session, ObjectIds.HistoryServerCapabilities_AggregateFunctions, ct)).ShouldBe(expected);
|
||||
(await AdvertisedAsync(session, ObjectIds.Server_ServerCapabilities_AggregateFunctions, ct)).ShouldBe(AllSdkNames);
|
||||
}
|
||||
finally { await rig.DisposeAsync(); }
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task With_no_historian_wired_the_history_folder_is_empty_from_boot()
|
||||
{
|
||||
// The Null default is applied by the CreateAggregateManager override, before any source is assigned.
|
||||
var ct = TestContext.Current.CancellationToken;
|
||||
var rig = await StartAsync("AggAdvNull", encrypted: false, ct);
|
||||
try
|
||||
{
|
||||
rig.Enforce(false, [Key()]);
|
||||
using var session = await rig.ConnectAsync(anonymous: true, ct);
|
||||
|
||||
(await AdvertisedAsync(session, ObjectIds.HistoryServerCapabilities_AggregateFunctions, ct)).ShouldBeEmpty();
|
||||
(await AdvertisedAsync(session, ObjectIds.Server_ServerCapabilities_AggregateFunctions, ct)).ShouldBe(AllSdkNames);
|
||||
}
|
||||
finally { await rig.DisposeAsync(); }
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Rewiring_the_source_resyncs_both_directions_of_the_references()
|
||||
{
|
||||
var ct = TestContext.Current.CancellationToken;
|
||||
var rig = await StartAsync("AggAdvResync", encrypted: false, ct);
|
||||
try
|
||||
{
|
||||
rig.Enforce(false, [Key()]);
|
||||
using var session = await rig.ConnectAsync(anonymous: true, ct);
|
||||
|
||||
rig.Nm.HistorianDataSource = Backend("SqlServer");
|
||||
(await AdvertisedAsync(session, ObjectIds.HistoryServerCapabilities_AggregateFunctions, ct)).ShouldBe(SqlNames);
|
||||
(await ParentsAsync(session, ObjectIds.AggregateFunction_TimeAverage2, ct)).ShouldBe(
|
||||
[ObjectIds.Server_ServerCapabilities_AggregateFunctions, ObjectIds.HistoryServerCapabilities_AggregateFunctions],
|
||||
ignoreOrder: true);
|
||||
|
||||
// Gateway has no TimeAverage2 (#681: no prior-sample read for Part 13 bounds): dropped from the history folder
|
||||
// (inverse reference too), kept in the server folder.
|
||||
rig.Nm.HistorianDataSource = Backend("Gateway");
|
||||
(await AdvertisedAsync(session, ObjectIds.HistoryServerCapabilities_AggregateFunctions, ct)).ShouldBe(GatewayNames);
|
||||
(await AdvertisedAsync(session, ObjectIds.Server_ServerCapabilities_AggregateFunctions, ct)).ShouldBe(AllSdkNames);
|
||||
(await ParentsAsync(session, ObjectIds.AggregateFunction_TimeAverage2, ct)).ShouldBe(
|
||||
[ObjectIds.Server_ServerCapabilities_AggregateFunctions]);
|
||||
|
||||
// And back: removed references are restored.
|
||||
rig.Nm.HistorianDataSource = Backend("QuestDb");
|
||||
(await AdvertisedAsync(session, ObjectIds.HistoryServerCapabilities_AggregateFunctions, ct)).ShouldBe(SqlNames);
|
||||
}
|
||||
finally { await rig.DisposeAsync(); }
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("StandardDeviationSample")] // served by no historian backend, never in the history folder
|
||||
[InlineData("TimeAverage2")] // not served by the Gateway backend wired below (#681)
|
||||
public async Task A_live_AggregateFilter_subscription_for_an_unadvertised_aggregate_still_works(string aggregateName)
|
||||
{
|
||||
// The trim touches only the advertised folders, not the AggregateManager's calculator factories, which is what
|
||||
// validates and computes a live-data AggregateFilter. Removing a factory turns this create into
|
||||
// BadAggregateNotSupported.
|
||||
var ct = TestContext.Current.CancellationToken;
|
||||
var rig = await StartAsync($"AggAdvLive{aggregateName}", encrypted: false, ct);
|
||||
try
|
||||
{
|
||||
rig.Nm.HistorianDataSource = Backend("Gateway");
|
||||
rig.Enforce(false, [Key()]);
|
||||
var id = rig.EnsureRawVariable("agg", "Double", writable: false);
|
||||
using var session = await rig.ConnectAsync(anonymous: true, ct);
|
||||
var aggregateId = (NodeId)typeof(ObjectIds).GetField("AggregateFunction_" + aggregateName)!.GetValue(null)!;
|
||||
|
||||
(await AdvertisedAsync(session, ObjectIds.HistoryServerCapabilities_AggregateFunctions, ct))
|
||||
.ShouldNotContain(aggregateName);
|
||||
(await AdvertisedAsync(session, ObjectIds.Server_ServerCapabilities_AggregateFunctions, ct))
|
||||
.ShouldContain(aggregateName); // i=2997 tells the truth: the live aggregate IS supported
|
||||
|
||||
var subscription = new Subscription(session.DefaultSubscription) { PublishingInterval = 100 };
|
||||
session.AddSubscription(subscription);
|
||||
await subscription.CreateAsync(ct);
|
||||
var item = new MonitoredItem(subscription.DefaultItem)
|
||||
{
|
||||
StartNodeId = ExpandedNodeId.ToNodeId(ExpandedNodeId.Parse(RawNsu(id)), session.NamespaceUris),
|
||||
AttributeId = Attributes.Value,
|
||||
SamplingInterval = 100,
|
||||
QueueSize = 10,
|
||||
Filter = new AggregateFilter
|
||||
{
|
||||
AggregateType = aggregateId,
|
||||
ProcessingInterval = 1000,
|
||||
StartTime = DateTime.UtcNow,
|
||||
},
|
||||
};
|
||||
var notifications = 0;
|
||||
item.Notification += (mi, _) => Interlocked.Add(ref notifications, mi.DequeueValues().Count);
|
||||
subscription.AddItem(item);
|
||||
await subscription.ApplyChangesAsync(ct);
|
||||
|
||||
item.Status.Error.ShouldBeNull($"AggregateFilter({aggregateName}) must be accepted on live data");
|
||||
// And it is computed: the SDK calculator emits one value per processing interval.
|
||||
var deadline = DateTime.UtcNow.AddSeconds(15);
|
||||
while (Volatile.Read(ref notifications) == 0 && DateTime.UtcNow < deadline) await Task.Delay(100, ct);
|
||||
Volatile.Read(ref notifications).ShouldBeGreaterThan(0, "the live aggregate must produce interval values");
|
||||
}
|
||||
finally { await rig.DisposeAsync(); }
|
||||
}
|
||||
|
||||
/// <summary>Forward Organizes children of an AggregateFunctions folder, by BrowseName, sorted.</summary>
|
||||
private static async Task<List<string>> AdvertisedAsync(ISession session, NodeId folder, CancellationToken ct)
|
||||
{
|
||||
var refs = await BrowseAsync(session, folder, BrowseDirection.Forward, ct);
|
||||
return refs.Select(r => r.BrowseName.Name).OrderBy(n => n, StringComparer.Ordinal).ToList();
|
||||
}
|
||||
|
||||
/// <summary>The folders an aggregate-function node is organized under (inverse Organizes).</summary>
|
||||
private static async Task<List<NodeId>> ParentsAsync(ISession session, NodeId aggregate, CancellationToken ct)
|
||||
{
|
||||
var refs = await BrowseAsync(session, aggregate, BrowseDirection.Inverse, ct);
|
||||
return refs.Select(r => ExpandedNodeId.ToNodeId(r.NodeId, session.NamespaceUris)).ToList();
|
||||
}
|
||||
|
||||
private static async Task<ReferenceDescriptionCollection> BrowseAsync(
|
||||
ISession session, NodeId node, BrowseDirection direction, CancellationToken ct)
|
||||
{
|
||||
var response = await session.BrowseAsync(
|
||||
requestHeader: null,
|
||||
view: null,
|
||||
requestedMaxReferencesPerNode: 0,
|
||||
nodesToBrowse:
|
||||
[
|
||||
new BrowseDescription
|
||||
{
|
||||
NodeId = node,
|
||||
BrowseDirection = direction,
|
||||
ReferenceTypeId = ReferenceTypeIds.Organizes,
|
||||
IncludeSubtypes = false,
|
||||
NodeClassMask = 0,
|
||||
ResultMask = (uint)BrowseResultMask.All,
|
||||
},
|
||||
],
|
||||
ct: ct);
|
||||
StatusCode.IsGood(response.Results[0].StatusCode).ShouldBeTrue($"browse {node}: {response.Results[0].StatusCode}");
|
||||
return response.Results[0].References;
|
||||
}
|
||||
}
|
||||
@@ -278,6 +278,96 @@ public sealed class NodeManagerHistoryReadTests : IDisposable
|
||||
await host.DisposeAsync();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// #682: an aggregate the server maps but the wired backend does not declare in
|
||||
/// <see cref="IHistorianDataSource.SupportedAggregates"/> is refused <c>BadAggregateNotSupported</c> UP FRONT — the
|
||||
/// backend is never called (the same set is what the AggregateFunctions folders advertise). The Null default
|
||||
/// declares none, so a processed read with no historian is refused too.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task Processed_aggregate_outside_the_backend_set_is_refused_without_a_backend_call()
|
||||
{
|
||||
var (host, server) = await BootAsync();
|
||||
var nm = server.NodeManager!;
|
||||
var fake = new RecordingHistorianDataSource
|
||||
{
|
||||
ProcessedResult = Empty(),
|
||||
Supported = new HashSet<HistoryAggregateType> { HistoryAggregateType.TimeAverage },
|
||||
};
|
||||
nm.HistorianDataSource = fake;
|
||||
|
||||
nm.EnsureVariable("eq-1/gw", parentFolderNodeId: null, displayName: "Gw", dataType: "Float",
|
||||
writable: false, historianTagname: "WW.Gw", realm: AddressSpaceRealm.Uns);
|
||||
var nodeId = nm.TryGetVariable("eq-1/gw")!.NodeId;
|
||||
|
||||
ReadProcessedDetails Details(NodeId aggregate) => new()
|
||||
{
|
||||
StartTime = DateTime.UtcNow.AddHours(-1),
|
||||
EndTime = DateTime.UtcNow,
|
||||
ProcessingInterval = 10_000.0,
|
||||
AggregateType = new NodeIdCollection { aggregate },
|
||||
};
|
||||
|
||||
// Average is mapped by the server but not in this backend's set ⇒ refused, no call.
|
||||
InvokeHistoryRead(server, nm, Details(ObjectIds.AggregateFunction_Average), nodeId).Item2[0].StatusCode.Code
|
||||
.ShouldBe(StatusCodes.BadAggregateNotSupported);
|
||||
fake.LastCall.ShouldBeNull();
|
||||
|
||||
// Positive control: a member of the set reaches the backend.
|
||||
InvokeHistoryRead(server, nm, Details(ObjectIds.AggregateFunction_TimeAverage), nodeId);
|
||||
fake.LastCall.ShouldBe("Processed");
|
||||
fake.LastAggregate.ShouldBe(HistoryAggregateType.TimeAverage);
|
||||
|
||||
// No historian: the Null default serves no aggregate.
|
||||
nm.HistorianDataSource = NullHistorianDataSource.Instance;
|
||||
InvokeHistoryRead(server, nm, Details(ObjectIds.AggregateFunction_TimeAverage), nodeId).Item2[0].StatusCode.Code
|
||||
.ShouldBe(StatusCodes.BadAggregateNotSupported);
|
||||
|
||||
await host.DisposeAsync();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// #682 review: an out-of-tree source returning a NULL <see cref="IHistorianDataSource.SupportedAggregates"/> (despite
|
||||
/// the non-nullable contract) is treated as the interface default — ALL — by BOTH consumers: the processed read
|
||||
/// dispatches (no NullReferenceException failing the whole request) and HistoryServerCapabilities/AggregateFunctions
|
||||
/// advertises every mapped aggregate.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task A_null_supported_set_means_all_for_both_the_read_and_the_advertisement()
|
||||
{
|
||||
var (host, server) = await BootAsync();
|
||||
var nm = server.NodeManager!;
|
||||
var fake = new RecordingHistorianDataSource { ProcessedResult = Empty(), Supported = null };
|
||||
nm.HistorianDataSource = fake;
|
||||
|
||||
nm.EnsureVariable("eq-1/nul", parentFolderNodeId: null, displayName: "Nul", dataType: "Float",
|
||||
writable: false, historianTagname: "WW.Nul", realm: AddressSpaceRealm.Uns);
|
||||
var nodeId = nm.TryGetVariable("eq-1/nul")!.NodeId;
|
||||
|
||||
var (_, errors) = InvokeHistoryRead(server, nm, new ReadProcessedDetails
|
||||
{
|
||||
StartTime = DateTime.UtcNow.AddHours(-1),
|
||||
EndTime = DateTime.UtcNow,
|
||||
ProcessingInterval = 10_000.0,
|
||||
AggregateType = new NodeIdCollection { ObjectIds.AggregateFunction_StandardDeviationPopulation },
|
||||
}, nodeId);
|
||||
StatusCode.IsBad(errors[0].StatusCode).ShouldBeFalse($"got {errors[0].StatusCode}");
|
||||
fake.LastCall.ShouldBe("Processed");
|
||||
fake.LastAggregate.ShouldBe(HistoryAggregateType.StandardDeviationPopulation);
|
||||
|
||||
var diagnostics = (CustomNodeManager2)nm.Server.DiagnosticsNodeManager;
|
||||
var folder = diagnostics.FindPredefinedNode(ObjectIds.HistoryServerCapabilities_AggregateFunctions, null)!;
|
||||
AggregateFunctionMap.ByNodeId.Keys.Count(id => folder.ReferenceExists(ReferenceTypeIds.Organizes, false, id))
|
||||
.ShouldBe(AggregateFunctionMap.ByNodeId.Count);
|
||||
|
||||
// Positive control on the same folder: a real (non-null) set trims it.
|
||||
nm.HistorianDataSource = NullHistorianDataSource.Instance;
|
||||
AggregateFunctionMap.ByNodeId.Keys.Count(id => folder.ReferenceExists(ReferenceTypeIds.Organizes, false, id))
|
||||
.ShouldBe(0);
|
||||
|
||||
await host.DisposeAsync();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// #677: a client <c>AggregateConfiguration</c> that differs from the server defaults is refused per node with
|
||||
/// <c>BadAggregateConfigurationRejected</c> (it used to be silently ignored); the source is never invoked.
|
||||
@@ -626,6 +716,8 @@ public sealed class NodeManagerHistoryReadTests : IDisposable
|
||||
public bool ThrowOnRead { get; init; }
|
||||
public bool AggregateNotSupported { get; init; }
|
||||
public bool LimitExceeded { get; init; }
|
||||
public IReadOnlySet<HistoryAggregateType>? Supported { get; init; } = HistoryAggregateTypes.All;
|
||||
public IReadOnlySet<HistoryAggregateType> SupportedAggregates => Supported!;
|
||||
public HistorianRead RawResult { get; set; } = new(Array.Empty<DataValueSnapshot>(), null);
|
||||
public HistorianRead ProcessedResult { get; set; } = new(Array.Empty<DataValueSnapshot>(), null);
|
||||
public HistorianRead AtTimeResult { get; set; } = new(Array.Empty<DataValueSnapshot>(), null);
|
||||
|
||||
Reference in New Issue
Block a user