Compare commits
12 Commits
6d5c1c768b
...
8ebee2ac40
| Author | SHA1 | Date | |
|---|---|---|---|
| 8ebee2ac40 | |||
| d266ff83f4 | |||
| 2ceed561ad | |||
| 11f792538e | |||
| b5e3e5108c | |||
| b8344bb151 | |||
| dbfe2fc4de | |||
| f6a2628656 | |||
| 0d206a0884 | |||
| b61975f7d7 | |||
| 19a4c093be | |||
| fb5ec80fe6 |
@@ -1350,7 +1350,7 @@ historian tagname, which is the tag's driver `FullName`.
|
||||
### Pluggable backends, SQL Server and HistoryUpdate (2026-10)
|
||||
|
||||
- **Selection:** `ServerHistorian:Backend` (`Gateway` default | `SqlServer` | `QuestDb`; unknown fails host start). `HistorianBackendRegistry` → `IHistorianBackendProvider` builds all five seams; the gateway keys are only required for `Gateway`. `SqlServer` uses `ServerHistorian:SqlServer:{ConnectionString (required), Schema=hist, CommandTimeout=30s, AutoCreateSchema=true}`.
|
||||
- **SQL Server** (`Driver.Historian.SqlServer`): `hist.Sample` PK `(TagId, SourceTimeUtc)` — writes are insert-if-absent, so outbox redelivery is idempotent. Stores every value kind with its **real status**; per-sample aggregates (12, simple definitions) are pushed down to T-SQL with bucket math in **100 ns ticks**; the Part 13 time-weighted five (TimeAverage/TimeAverage2/Total/Total2/Interpolative, #671 — ⚠️ `Total` is now the time-weighted integral, no longer `SUM`) stream window + 4 bound rows into the shared `TimeWeightedAggregator` (both SQL backends, identical results; client `AggregateConfiguration` ignored; status = the SDK `AggregateCalculator` time-based rule, Uncertain time counts as Good; reverse `Interpolative` anchored at the later time; window capped by `ServerHistorian:MaxTimeWeightedRawRows` ⇒ `BadTooManyOperations`). Docker-dev recipe, schema and aggregate semantics: `docs/Historian.md` §"SQL Server backend". Integration tests need `HISTORIAN_SQL_TEST_CONNECTION`. **Retention (#667):** opt-in `SqlServer:RetentionDays` (+ `PurgeInterval` 1h, `PurgeBatchSize` 4000) -> `HistorianRetentionService` purges old `hist.Sample` (per tag) / `hist.Event` in `DELETE TOP(n)` batches, Primary-gated on `IHistorizationRoleView`, one purger per DB via `sp_getapplock`; counter `otopcua.historian.retention.purged{table}`. Optional `IHistorianBackendProvider.CreateRetention` (default null). `docs/Historian.md` §Retention.
|
||||
- **SQL Server** (`Driver.Historian.SqlServer`): `hist.Sample` PK `(TagId, SourceTimeUtc)` — writes are insert-if-absent, so outbox redelivery is idempotent. Stores every value kind with its **real status**; per-sample aggregates (10, simple definitions) are pushed down to T-SQL with bucket math in **100 ns ticks**; the Part 13 time-weighted seven (TimeAverage/TimeAverage2/Total/Total2/Interpolative, #671 — ⚠️ `Total` is now the time-weighted integral, no longer `SUM` — plus the **duration-based** PercentGood/PercentBad, #680) stream window + 4 bound rows into the shared `TimeWeightedAggregator` (both SQL backends, identical results; reverse `Interpolative` anchored at the later time; window capped by `ServerHistorian:MaxTimeWeightedRawRows` ⇒ `BadTooManyOperations`, percentages included). ⚠️ **#680 status rules (client-visible):** TimeAverage/Total are Uncertain if any part of the interval uses an Uncertain value, is bridged across a Bad sample, is extrapolated past the last sample (⇒ a live tag's latest interval is Uncertain) or is not covered; TimeAverage2/Total2 keep the SDK time-based rule (Uncertain time counts as Good) but never extrapolate past the last sample — that interval is Partial, later ones `BadNoData` (a report-by-exception tag with nothing after its last sample reads `BadNoData` until its next sample arrives). `docs/Historian.md` §"Time-weighted aggregates". ⚠️ **#680 per-sample (client-visible):** `Count` counts **Good** samples only (value-based status; an all-Bad interval is generic Bad, value null) and an empty interval **inside the tag's data range** is `0`/Good (outside it still `BadNoData`); `Start`/`End` return the **raw** first/last sample of any quality — its own timestamp and status, value null when Bad; `Delta` is Uncertain only when the interval's first or last sample is unusable; and every per-sample aggregate but `Average` sets **end-of-data Partial** on the interval holding the last sample (forward incl. its earlier edge, reverse strictly inside), never on a Bad result. The data-range lookup is a seek on SQL Server but a partition scan on QuestDB (unindexed `tag` SYMBOL). `docs/Historian.md` §"Aggregates". Docker-dev recipe, schema and aggregate semantics: `docs/Historian.md` §"SQL Server backend". Integration tests need `HISTORIAN_SQL_TEST_CONNECTION`. **Retention (#667):** opt-in `SqlServer:RetentionDays` (+ `PurgeInterval` 1h, `PurgeBatchSize` 4000) -> `HistorianRetentionService` purges old `hist.Sample` (per tag) / `hist.Event` in `DELETE TOP(n)` batches, Primary-gated on `IHistorizationRoleView`, one purger per DB via `sp_getapplock`; counter `otopcua.historian.retention.purged{table}`. Optional `IHistorianBackendProvider.CreateRetention` (default null). `docs/Historian.md` §Retention.
|
||||
- **QuestDB** (`Driver.Historian.QuestDb`, `ServerHistorian:QuestDb:*`, Npgsql over PG wire; docker-dev overlay `docker-compose.questdb.yml`): same kinds/status/aggregates/HistoryUpdate as SqlServer, soft delete by tombstone (INSERT only — never `UPDATE`/`DELETE`; every read filters `NOT deleted`). ⚠️ **Timestamps are µs** (`QuestDbConnection.Round`/`BindTimestamp`; `IHistorianDataSource.TimestampResolutionTicks` = 10 steps HistoryRead paging), **pre-1970 cannot be stored** (writer drops, HistoryUpdate `BadOutOfRange`), every write waits for WAL visibility, TTL is enforced on write commits only, set Npgsql `Maximum Pool Size` < QuestDB's 64. `docs/Historian.md` §"QuestDB backend".
|
||||
- ⚠️ **Reverse processed reads are Part 13/SDK-anchored (#677, client-visible):** every aggregate on SqlServer/QuestDB now anchors a reverse read at its later time — window `(end, start]`, partial interval earliest, each stamped with its later edge (was: earlier-anchored, stamped earlier edge); on forward reads Partial also marks the interval the data starts inside (not Average — the SDK's rule, forward-only); a non-default client `AggregateConfiguration` ⇒ `BadAggregateConfigurationRejected`; Gateway provisioning sends `IntegralDivisor = 1`. SDK-parity tests: `SdkAggregateParityTests`. `docs/Historian.md` §Aggregates.
|
||||
- **Real status fix:** the recorder used to stamp every sample Good (`192`); it now carries the driver's status. ⚠️ On the **Gateway** backend this changes nothing stored: the gateway discards quality (C-002), so AVEVA still stores Good.
|
||||
@@ -1363,7 +1363,7 @@ historian tagname, which is the tag's driver `FullName`.
|
||||
The server dispatches all OPC UA HistoryRead to the registered `IHistorianDataSource` — the
|
||||
`GatewayHistorianDataSource` read client when enabled, else the `NullHistorianDataSource` default
|
||||
(historized nodes return `GoodNoData`, never an error). Supported variants: Raw, Processed
|
||||
(Average/Minimum/Maximum/Total/Count aggregates), and AtTime over historized variable nodes; Events over
|
||||
(backend-dependent: Gateway serves TimeAverage/Minimum/Maximum/Total, with Average/Count `BadAggregateNotSupported` since AVEVA has no mean or sample count, #679; SqlServer/QuestDb serve seventeen), and AtTime over historized variable nodes; Events over
|
||||
alarm-owning equipment-folder event-notifier nodes. Reads are ungated (served from any redundancy node);
|
||||
authorization uses the standard `AccessLevels.HistoryRead` bit set at materialization.
|
||||
|
||||
@@ -1381,7 +1381,7 @@ authorization uses the standard `AccessLevels.HistoryRead` bit set at materializ
|
||||
| `MaxTieClusterOverfetch` | `65536` | Bounded over-fetch the HistoryRead-Raw paging uses to page within an oversized same-timestamp tie cluster (retained from the prior backend) |
|
||||
| `HistoryReadBatchParallelism` | `4` | Max nodes served concurrently within one HistoryRead batch (03/S3); `⌈N/P⌉ × CallTimeout` worst case instead of sequential |
|
||||
| `MaxConcurrentHistoryReadBatches` | `16` | Process-wide concurrent HistoryRead batch cap (03/S3); flood degrades to `BadTooManyOperations`. Total in-flight reads capped at 16 × 4 = 64 |
|
||||
| `MaxTimeWeightedRawRows` | `1000000` | `SqlServer`/`QuestDb` only (#671): raw in-window rows one time-weighted aggregate read (incl. `Total`) may stream into the C# evaluator; window query capped at this + 1, past it ⇒ `BadTooManyOperations` for that node. ≤ 0 = unbounded |
|
||||
| `MaxTimeWeightedRawRows` | `1000000` | `SqlServer`/`QuestDb` only (#671): raw in-window rows one time-weighted aggregate read (incl. `Total` and, since #680, PercentGood/PercentBad) may stream into the C# evaluator; window query capped at this + 1, past it ⇒ `BadTooManyOperations` for that node. ≤ 0 = unbounded |
|
||||
| `HistoryReadDeadline` | `00:01:00` | Per-request HistoryRead wall-clock deadline (03/S3); a node still in flight surfaces `BadTimeout`. Non-positive = unbounded |
|
||||
|
||||
### Alarm-history path (`AlarmHistorian` section)
|
||||
|
||||
+2
-1
@@ -210,7 +210,8 @@ otopcua-cli historyread -u opc.tcp://localhost:4840/OtOpcUa \
|
||||
-n "ns=1;s=TestMachine_001.TestHistoryValue" \
|
||||
--start "2026-03-25T00:00:00Z" --end "2026-03-30T00:00:00Z"
|
||||
|
||||
# Aggregate: 1-hour average
|
||||
# Aggregate: 1-hour average (SqlServer/QuestDb; on the Gateway historian backend Average is
|
||||
# BadAggregateNotSupported — use --aggregate TimeAverage there, #679)
|
||||
otopcua-cli historyread -u opc.tcp://localhost:4840/OtOpcUa \
|
||||
-n "ns=1;s=TestMachine_001.TestHistoryValue" \
|
||||
--start "2026-03-25T00:00:00Z" --end "2026-03-30T00:00:00Z" \
|
||||
|
||||
+199
-62
@@ -77,7 +77,7 @@ consumes the historian. The existing enable gates (`ServerHistorian:Enabled`, `A
|
||||
| Value kinds recorded | `Double`, `Int64`, `Boolean` (numeric analog only) | all: `Double`, `Int64`, `Boolean`, `String`, `DateTime`, plus `Empty` (a Bad sample with no value) | same as `SqlServer` |
|
||||
| Status code stored | Sent as DA quality but **discarded** by the gateway (C-002); AVEVA stores every sample Good | Stored bit-for-bit (`StatusCode` int column) | Stored bit-for-bit (unsigned, in a `LONG`) |
|
||||
| Comms-loss / driver-Bad publishes | Dropped, counted in `DroppedUnsupportedKind` | Stored as an `Empty` outage-marker sample carrying the Bad code (see [below](#typed-values-real-status-and-outage-markers)) | Stored as an `Empty` outage-marker sample, as on `SqlServer` |
|
||||
| Processed aggregates | Average, Minimum, Maximum, Total, Count | The seventeen in the [aggregate table](#aggregates-processed-reads) (incl. Part 13 time-weighted, #671) | The same seventeen, identical results |
|
||||
| Processed aggregates | TimeAverage, Minimum, Maximum, Total (Average and Count: `BadAggregateNotSupported`, #679) | The seventeen in the [aggregate table](#aggregates-processed-reads) (incl. Part 13 time-weighted, #671) | The same seventeen, identical results |
|
||||
| HistoryUpdate | No (`BadHistoryOperationUnsupported`) | Yes (Insert / Replace / Update / DeleteRaw / DeleteAtTime) | Yes — soft delete (tombstones), see [below](#historyupdate-soft-delete) |
|
||||
| Modified-value history (`IsReadModified`, #668) | No (`BadHistoryOperationUnsupported`) | Yes — `SampleModified`, atomic with the change | Yes — `{prefix}sample_modified`, written before the change (orphan-on-crash) |
|
||||
| Alarm history | gateway `SendEvent` | `Event` table (insert-if-absent) | `{prefix}event` table (dedup upsert on `(ts, event_id)`) |
|
||||
@@ -144,7 +144,7 @@ and all HistoryRead calls on historized nodes return `GoodNoData` (empty, not an
|
||||
| `MaxTieClusterOverfetch` | int | `65536` | (Node-manager paging; all backends — a `SqlServer` PK rules out tie clusters, so it is moot there.) Maximum samples the server will fetch in one shot to page through a tie cluster (multiple samples sharing one `SourceTimestamp`). A cluster larger than this ceiling fails `BadHistoryOperationUnsupported`. Raise to handle abnormally large tie clusters; the default covers all normal-data cases. |
|
||||
| `HistoryReadBatchParallelism` | int | `4` | Max historized nodes served **concurrently** within one HistoryRead batch (arch-review 03/S3). The SDK's `HistoryRead*` override surface is synchronous, so each arm bridges once at its boundary; this turns per-node serving from sequential (`N × CallTimeout` worst case) into `⌈N / P⌉ × CallTimeout`. Also caps the gateway-load multiplication a single client can command. `≤ 1` serves sequentially. |
|
||||
| `MaxConcurrentHistoryReadBatches` | int | `16` | Process-wide limit on HistoryRead (and HistoryUpdate) batches served **concurrently** (arch-review 03/S3). A flood of history clients degrades to `BadTooManyOperations` rather than exhausting the SDK request-thread pool. With `HistoryReadBatchParallelism` this caps total in-flight gateway reads at `MaxConcurrentHistoryReadBatches × HistoryReadBatchParallelism` (default 16 × 4 = 64). A batch that cannot get a slot within `min(HistoryReadDeadline, 5 s)` fails every handle `BadTooManyOperations`. |
|
||||
| `MaxTimeWeightedRawRows` | int | `1000000` | `SqlServer` / `QuestDb` only (#671). The most raw in-window samples one [time-weighted aggregate](#time-weighted-aggregates-part-13-671) read (TimeAverage, TimeAverage2, **Total**, Total2, Interpolative) may stream into the server-side evaluator — that family is **not** pushed down to SQL. The window query is capped at this + 1 rows; one row past the cap fails **that node** with `BadTooManyOperations` (not counted as a backend fault) instead of holding a pooled connection until `HistoryReadDeadline` and returning `BadTimeout`. Bound rows outside the window do not count; per-sample aggregates are not capped. 1 000 000 ≈ 11.5 days of a 1 Hz tag. Non-positive = unbounded (a `Validate()` warning). |
|
||||
| `MaxTimeWeightedRawRows` | int | `1000000` | `SqlServer` / `QuestDb` only (#671). The most raw in-window samples one [time-weighted aggregate](#time-weighted-aggregates-part-13-671) read (TimeAverage, TimeAverage2, **Total**, Total2, Interpolative, and — since #680 — **PercentGood/PercentBad**) may stream into the server-side evaluator — that family is **not** pushed down to SQL. The window query is capped at this + 1 rows; one row past the cap fails **that node** with `BadTooManyOperations` (not counted as a backend fault) instead of holding a pooled connection until `HistoryReadDeadline` and returning `BadTimeout`. Bound rows outside the window do not count; per-sample aggregates are not capped. 1 000 000 ≈ 11.5 days of a 1 Hz tag. Non-positive = unbounded (a `Validate()` warning). |
|
||||
| `HistoryReadDeadline` | TimeSpan | `00:01:00` | Per-request wall-clock deadline for a HistoryRead (and HistoryUpdate) batch (arch-review 03/S3). A node still in flight at the deadline surfaces `BadTimeout`, so one request can never hold an SDK request thread longer than this regardless of node count. Non-positive = unbounded (a `Validate()` warning). |
|
||||
| `HistoryUpdateMaxFutureSkew` | TimeSpan | `1.00:00:00` | #674. How far past the server clock a HistoryUpdate (Insert / Replace / Update) item's `SourceTimestamp` may lie; an item beyond `now + skew` is refused **`BadOutOfRange`** by the node manager before any backend sees it (every backend). Guards against a far-future typo (a 2062 timestamp). The live recorder is not bounded. Non-positive disables the bound (no warning — a deliberate choice). See [HistoryUpdate](#historyupdate). |
|
||||
|
||||
@@ -745,8 +745,7 @@ otopcua-cli historyread -u opc.tcp://localhost:4840 -U apikey -P otua_<keyId>_<s
|
||||
The OPC UA `Total` aggregate is **supported** over the HistorianGateway backend. The gateway exposes a
|
||||
native **`Integral`** retrieval mode, so `Total` maps straight to it (`HistoryAggregateType.Total →
|
||||
RetrievalMode.Integral`) — no client-side scaling. (This replaces the retired Wonderware path, which had no
|
||||
`Total` column and derived it client-side as time-weighted `Average × interval-seconds`.) `Count` is
|
||||
likewise a native gateway mode. Bucket status codes and timestamps are preserved unchanged; empty / null
|
||||
`Total` column and derived it client-side as time-weighted `Average × interval-seconds`.) Bucket status codes and timestamps are preserved unchanged; empty / null
|
||||
buckets surface as `BadNoData`.
|
||||
|
||||
**Integral unit (#677).** AVEVA's integral is `∫ value dt` (seconds) ÷ the tag's `IntegralDivisor`. The proto3 `double` default for
|
||||
@@ -757,13 +756,19 @@ VPN. Reverse processed reads on the `Gateway` backend are answered by AVEVA and
|
||||
|
||||
### Aggregate set and `BadAggregateNotSupported` (#285)
|
||||
|
||||
> ⚠️ **#679 (client-visible): on the `Gateway` backend, `Average` and `Count` are now `BadAggregateNotSupported`.** AVEVA's 15
|
||||
> retrieval modes contain no arithmetic mean and no sample count. `Average` used to be mapped to `TimeWeightedAverage` (a different
|
||||
> statistic from the OPC UA mean the SQL backends return); `Count` to `Counter`, which is a rollover-counter increase, not a sample
|
||||
> count. Request **`TimeAverage`** for the time-weighted number (now mapped to `TimeWeightedAverage`). There is no Gateway sample
|
||||
> count. `TimeAverage` magnitude and timestamping against live AVEVA are unverified (needs the VPN).
|
||||
|
||||
`HistoryAggregateType` carries the wider Part 13 set (Interpolative, TimeAverage, Start, End, Delta, Range,
|
||||
Min/MaxActualTime, Start/EndBound, DeltaBounds, DurationGood/Bad, PercentGood/Bad, WorstQuality,
|
||||
NumberOfTransitions, AnnotationCount, StandardDeviationPopulation, VariancePopulation); `OtOpcUaNodeManager.MapAggregate`
|
||||
maps each standard aggregate NodeId (`TimeAverage2` and `Total2` were appended for #671). The **`SqlServer`** and
|
||||
**`QuestDb`** backends serve the seventeen listed in the [aggregate table](#aggregates-processed-reads); everything else
|
||||
(e.g. `StartBound`, `DurationGood`, `StandardDeviationPopulation`) is `BadAggregateNotSupported` for that node. The **HistorianGateway** backend only has retrieval modes for the original
|
||||
five (Average, Minimum, Maximum, Total, Count): for any other member `AggregateModeMapper` throws
|
||||
(e.g. `StartBound`, `DurationGood`, `StandardDeviationPopulation`) is `BadAggregateNotSupported` for that node. The **HistorianGateway** backend only has retrieval modes for the four
|
||||
(TimeAverage, Minimum, Maximum, Total): for any other member `AggregateModeMapper` throws
|
||||
`HistoryAggregateNotSupportedException`, which the HistoryRead path turns into `BadAggregateNotSupported` for that
|
||||
node only (not a generic failure, and not counted as a backend fault). The OPC UA Client driver forwards every
|
||||
member upstream. A NodeId outside the mapped set (e.g. `StandardDeviationSample`) is still `BadAggregateNotSupported`.
|
||||
@@ -903,10 +908,10 @@ point (see [Continuation-point paging (Events)](#continuation-point-paging-event
|
||||
|
||||
### Aggregates (Processed reads)
|
||||
|
||||
> **Two kinds.** The *per-sample* aggregates (Average, Minimum, Maximum, Count, Range, Start/End/Delta, Min/MaxActualTime,
|
||||
> PercentGood/Bad) are simple per-sample definitions — no time-weighting, no interval-boundary values — pushed down to SQL. The
|
||||
> *time-weighted* aggregates (TimeAverage, TimeAverage2, Total, Total2, Interpolative; #671) follow OPC UA Part 13 with bounding values
|
||||
> — see [Time-weighted aggregates](#time-weighted-aggregates-part-13-671). Any `HistoryAggregateType` not in the tables throws
|
||||
> **Two kinds.** The *per-sample* aggregates (Average, Minimum, Maximum, Count, Range, Start/End/Delta, Min/MaxActualTime) are
|
||||
> simple per-sample definitions — no time-weighting, no interval-boundary values — pushed down to SQL. The *time-weighted* aggregates
|
||||
> (TimeAverage, TimeAverage2, Total, Total2, Interpolative; #671 — and the duration-based PercentGood/PercentBad since #680) follow
|
||||
> OPC UA Part 13 with bounding values — see [Time-weighted aggregates](#time-weighted-aggregates-part-13-671). Any `HistoryAggregateType` not in the tables throws
|
||||
> `HistoryAggregateNotSupportedException` ⇒ `BadAggregateNotSupported` for that node (e.g. `StartBound`, `DurationGood`,
|
||||
> `StandardDeviationPopulation`).
|
||||
>
|
||||
@@ -930,6 +935,24 @@ point (see [Continuation-point paging (Events)](#continuation-point-paging-event
|
||||
> from the new (forward-only) start-of-data Partial bit below. A client that paired reverse results with forward ones by timestamp must shift by one
|
||||
> interval.
|
||||
|
||||
> ⚠️ **#680 (client-visible): `Count`, `Start`, `End`, `Delta` and the end-of-data Partial bit now follow Part 13 / the SDK** on the
|
||||
> `SqlServer` and `QuestDb` backends:
|
||||
> - **`Count`** counts **Good** samples only (it used to count Good + Uncertain); its status is value-based — Good when every sample in
|
||||
> the interval is Good, the generic **Bad** (`0x80000401` — generic Bad + our Calculated bit; the SDK returns a bare `0x80000000` — value null) when every one is Bad (it used to be `BadNoData`), else
|
||||
> Uncertain. An **empty interval inside the tag's data range** `[first sample, last sample]` is now **`0`, Good** (it used to be
|
||||
> `BadNoData`); an empty interval wholly before the first or after the last sample stays `BadNoData`. Both directions.
|
||||
> - **`Start` / `End`** return the interval's **raw** first / last sample of **any** quality — its own value (null when Bad), its own
|
||||
> stored status (Raw: no Calculated bit) and **its own timestamp** (they used to return the first/last *usable* value stamped with
|
||||
> the interval start under the per-sample status rule). A reverse read returns the same samples: `Start` is still the earliest.
|
||||
> - **`Delta`** (last usable − first usable) is Good unless the interval's first or last sample is not usable (then Uncertain); a Bad
|
||||
> sample in the middle no longer degrades it, and neither does an Uncertain sample.
|
||||
> - **End-of-data Partial:** the interval holding the tag's **last** sample carries the Partial bit for every per-sample aggregate but
|
||||
> `Average` — forward with the interval's earlier edge included, reverse only when the sample lies strictly inside (the SDK's rules) —
|
||||
> so a "live" tag's latest interval is now flagged.
|
||||
>
|
||||
> A client that read `Count` as "usable samples", keyed `Start`/`End` on the interval start, or treated `Count = BadNoData` as "no
|
||||
> samples" must adjust. Not changed: `Average`/`Minimum`/`Maximum` still use Good + Uncertain samples (the SDK uses Good only).
|
||||
|
||||
**Bucketing.** Forward: one result per interval of `[start, end)`; bucket *i* = `[start + i·interval, start + (i+1)·interval)`,
|
||||
start-inclusive / end-exclusive to the tick, the last bucket truncated at `end`, each stamped with its start. Reverse (`end < start`,
|
||||
#677): the window is `(end, start]`; result *k* = `(start − (k+1)·interval, start − k·interval]`, clipped at `end` (the **earliest**
|
||||
@@ -946,39 +969,84 @@ returned status carries the historian *Calculated* info bit, `BadNoData` include
|
||||
result for the interval the tag's data **starts inside** — its earliest stored sample (any quality) lies strictly between the interval's
|
||||
edges, so there is no bounding value before it and the data does not cover the whole interval. This follows the SDK's `m_startOfData`
|
||||
rule, which the SDK turns off for `Average` and in effect applies to forward reads only (a reverse read is fed latest-first, so its
|
||||
"start of data" is the latest sample, past the window). A reverse read therefore sets no start-of-data Partial — matching the SDK. Only a
|
||||
forward per-sample read of an aggregate other than `Average` fetches the tag's earliest sample (one seek, bounded by the window end);
|
||||
every other read skips that query. An earliest sample exactly on an edge, or before the interval, sets nothing. The time-weighted
|
||||
"start of data" is the latest sample, past the window). A reverse read therefore sets no start-of-data Partial on the interval holding
|
||||
the *earliest* sample — matching the SDK. An earliest sample exactly on an edge, or before the interval, sets nothing. The time-weighted
|
||||
aggregates also keep setting Partial on the truncated interval (below).
|
||||
|
||||
**End-of-data Partial (#680).** Every aggregate except `Average` (and `Interpolative`) also sets Partial on a non-Bad result for the
|
||||
interval the tag's data **ends inside** — nothing stored after it:
|
||||
- **forward:** the latest stored sample (any quality) lies at or after the interval's earlier edge and before its later one (Part 13
|
||||
§5.3.3.2; the SDK's `m_endOfData` rule — the earlier edge is *included*);
|
||||
- **reverse:** the latest sample lies *strictly* inside the interval (the SDK's reverse read keys its `m_startOfData` rule on the first
|
||||
value it is fed — the latest — so a latest sample exactly on an interval edge sets nothing). The time-weighted family does the same
|
||||
(see [Time-weighted aggregates](#time-weighted-aggregates-part-13-671)).
|
||||
|
||||
Data continuing past the window sets nothing. Two deliberate limits, kept: the per-sample aggregates **never** set Partial on a merely
|
||||
*truncated* interval (only the time-weighted ones do), and **a Bad result never carries Partial** (the SDK sets it on Bad results too).
|
||||
|
||||
**Data-range queries.** The start/end-of-data rules and `Count`'s in-range empty interval need the tag's **data range**: its earliest
|
||||
stored sample (any quality) at or before the window's later edge and its latest at or after the earlier edge. On **SQL Server** these
|
||||
are one extra result set of two `MIN`/`MAX` subqueries, each a single seek on the `(TagId, SourceTimeUtc)` key. On **QuestDB** they are
|
||||
two `LIMIT 1` statements, but `tag` is an **unindexed `SYMBOL`**, so each is a **partition scan**, not a seek: measured 1–3 ms for a
|
||||
live tag, and for a stopped tag read over an old window 6–8 ms warm / ~32 ms cold on a 52 M-row table — the cost grows with the table
|
||||
(bounded by the TTL), not with the window. They run only when the answer can change a result
|
||||
(`AggregateEvaluator.NeedsDataRange`): a per-sample read of any aggregate other than `Average`, in either direction (`Count`
|
||||
included); a per-sample `Average` read skips them.
|
||||
|
||||
**Sample classes.** *Usable* = severity Good or Uncertain **and** a stored value (not `Empty`). *Good* = usable with severity Good.
|
||||
*Bad* = not usable.
|
||||
|
||||
| Aggregate | Computed over | Value timestamp |
|
||||
|---|---|---|
|
||||
| `Average`, `Minimum`, `Maximum` | `AVG` / `MIN` / `MAX` of `NumValue` over usable samples | bucket start (reverse read: its later edge, as every row marked "bucket start" here) |
|
||||
| `Count` | number of usable samples (`Int32`) | bucket start |
|
||||
| `Count` | number of **Good** samples (`Int32`; #680) | bucket start |
|
||||
| `Range` | `Maximum − Minimum` | bucket start |
|
||||
| `Start`, `End` | first / last usable sample by time (keeps the sample's type) | bucket start |
|
||||
| `Delta` | `End − Start` (numeric samples) | bucket start |
|
||||
| `Start`, `End` | the **raw** first / last sample of any quality (its type, its own status; value null when Bad — #680) | **the sample's own time** |
|
||||
| `Delta` | last usable − first usable value (numeric samples) | bucket start |
|
||||
| `MinimumActualTime`, `MaximumActualTime` | the usable sample with the smallest / largest `NumValue`; ties ⇒ earliest | **the sample's own time** |
|
||||
| `PercentGood` | `100 × (Good samples) / (all samples)` | bucket start |
|
||||
| `PercentBad` | `100 × (Bad samples) / (all samples)` | bucket start |
|
||||
|
||||
Status and edge rules:
|
||||
|
||||
- An **empty interval** (no samples) ⇒ `BadNoData`. **`Count` — and every value aggregate — over an interval whose samples are all Bad ⇒ `BadNoData`**, not `0`.
|
||||
- Value aggregates are `Good` when every sample in the bucket is Good and `UncertainDataSubNormal` when the bucket also holds any Bad **or Uncertain** sample.
|
||||
- `Count`, `Start`, `End`, `PercentGood`, `PercentBad` accept any value kind; the other aggregates (and `Delta`) on a bucket that holds a usable `String`/`DateTime` sample ⇒ `BadAggregateInvalidInputs` for that interval.
|
||||
- **`PercentGood` / `PercentBad` and Uncertain samples:** Uncertain samples are counted in the denominator but are **neither Good nor Bad**, so `PercentGood + PercentBad < 100` whenever a bucket holds Uncertain samples. Both report status `Good` for any non-empty bucket.
|
||||
- An **empty interval** (no samples) ⇒ `BadNoData` — except **`Count`**, which is `0`, Good, when the interval lies inside the tag's data range `[first sample, last sample]` (both edges inclusive, either direction; #680). **Every value aggregate over an interval whose samples are all Bad ⇒ `BadNoData`**; `Count` there is the generic **Bad** (value null).
|
||||
- Value aggregates are `Good` when every sample in the bucket is Good and `UncertainDataSubNormal` when the bucket also holds any Bad **or Uncertain** sample. `Count`'s status is value-based (all Good ⇒ Good, all Bad ⇒ Bad, else Uncertain); `Start`/`End` carry the raw sample's own status; `Delta` is Good unless the interval's first or last sample is not usable (then Uncertain) — #680.
|
||||
- `Count`, `Start`, `End` accept any value kind; the other aggregates (and `Delta`) on a bucket that holds a usable `String`/`DateTime` sample ⇒ `BadAggregateInvalidInputs` for that interval.
|
||||
- `PercentGood` / `PercentBad` are **duration-based since #680** — see [Time-weighted aggregates](#time-weighted-aggregates-part-13-671).
|
||||
- Everything except `Count` and `Start`/`End` is a `double`; `Int64` samples aggregate as doubles and `Boolean` as 0/1.
|
||||
|
||||
### Time-weighted aggregates (Part 13, #671)
|
||||
|
||||
`TimeAverage`, `TimeAverage2`, `Total`, `Total2` and `Interpolative`, evaluated by the shared
|
||||
`Driver.Historian.Sql.Common/TimeWeightedAggregator.cs` for **both** SQL backends (so `SqlServer` and `QuestDb` agree exactly).
|
||||
**This whole family — `Total` included — is evaluated in C#, not pushed down to SQL** (see *Pushdown trade-off* and the raw-row cap
|
||||
below). Bucketing (edges, truncated interval, reverse anchoring, the 100 000-interval cap) is the same as above.
|
||||
`TimeAverage`, `TimeAverage2`, `Total`, `Total2`, `Interpolative` and (since #680) `PercentGood` / `PercentBad`, evaluated by the
|
||||
shared `Driver.Historian.Sql.Common/TimeWeightedAggregator.cs` for **both** SQL backends (so `SqlServer` and `QuestDb` agree exactly).
|
||||
**This whole family — `Total` and the percentages included — is evaluated in C#, not pushed down to SQL** (see *Pushdown trade-off*
|
||||
and the raw-row cap below). Bucketing (edges, truncated interval, reverse anchoring, the 100 000-interval cap) is the same as above.
|
||||
|
||||
> ⚠️ **#680 (client-visible): time-weighted results now follow Part 13 more closely** (`SqlServer` / `QuestDb`; the SDK's own
|
||||
> calculators agree except where noted under *SDK parity*):
|
||||
> - **`TimeAverage` / `Total` status** is `UncertainDataSubNormal` whenever the interval uses an Uncertain value, a segment bridged
|
||||
> across a Bad value, the stepped extrapolation past the last sample, or does not cover the interval — no longer the time-based
|
||||
> rule. **The latest interval of a live tag now reads Uncertain, not Good**, and so does a fully-covered interval bridged across an
|
||||
> outage.
|
||||
> - **`TimeAverage` / `Total` over an interval wholly inside an outage** (data on both sides) now return the bridged value,
|
||||
> Uncertain — they used to return generic `Bad`, value null.
|
||||
> - **`Total` over an interval the data starts inside** integrates only the covered part (it used to be `TimeAverage × interval`).
|
||||
> - **`TimeAverage2` / `Total2` no longer extrapolate past the end of the data** (the last sample, when nothing at or after the window
|
||||
> end exists and it is usable): the interval holding it covers only up to it and is **Partial**; later intervals are **`BadNoData`**
|
||||
> (they used to hold the last value, Good). A one-interval read over the whole history therefore averages the stored data only.
|
||||
> - **`PercentGood` / `PercentBad` are duration-based** (they counted samples), moved to this family — so they are now subject to
|
||||
> `ServerHistorian:MaxTimeWeightedRawRows` (`BadTooManyOperations` past the cap) — and an interval with no sample of its own between
|
||||
> two samples now has a value instead of `BadNoData`.
|
||||
> - **Partial:** set on the forward end-of-data interval (`TimeAverage2`/`Total2`/percentages) and, on a reverse read, on the interval
|
||||
> holding the latest sample when nothing follows it (all of this family but `Interpolative`).
|
||||
> - ⚠️ **Report-by-exception tags (a slowly changing or deadband-filtered value) are the visible case.** A tag whose last stored
|
||||
> sample lies *before* the window and that has nothing after it has, by Part 13, **ended its data**: every interval of
|
||||
> `TimeAverage2` / `Total2` / `PercentGood` / `PercentBad` is **`BadNoData`**, and every interval of `TimeAverage` / `Total` is the
|
||||
> held value, **Uncertain**. When the next sample arrives, a re-read of the same window returns values / `Good` for those intervals
|
||||
> (the held value is then interpolated between real samples, not extrapolated). So a dashboard, a ScadaBridge consumer or any
|
||||
> client that caches processed results for "closed" intervals can see an interval go `BadNoData` → value, or Uncertain → Good, on a
|
||||
> later read — re-read recent intervals, or use `TimeAverage`/`Total` (which always carry a value) for a report-by-exception tag. The
|
||||
> SDK's own calculators behave the same way. Likewise the interval holding a tag's latest sample is Partial and, until a later
|
||||
> sample arrives, `TimeAverage2`/`Total2` there cover only up to it — the comms-loss recovery shape (a single Good sample after an
|
||||
> outage marker, nothing after) answers that sample's value (`Total2` 0), Uncertain + Partial, never Bad/null.
|
||||
|
||||
**Server defaults, fixed:** `TreatUncertainAsBad = false`, `PercentDataBad = PercentDataGood = 100`, `UseSlopedExtrapolation = false`
|
||||
(`ServerAggregateConfiguration` in the OPC UA server project — the SDK's own `AggregateManager` defaults, which a test pins). **A
|
||||
@@ -996,40 +1064,73 @@ still gets a value.
|
||||
|
||||
**The signal.** Between a usable sample `p` and the next sample `s`: both `Double`/`Int64` ⇒ **sloped** (linear), segment quality the
|
||||
worse of the two; either `Boolean` ⇒ **stepped** (`p` held), quality `p`'s; `s` Bad ⇒ `p` held up to the outage (never interpolated into
|
||||
it — the AtTime rule), Uncertain; no later sample at all ⇒ `p` held to the window end (stepped extrapolation), Uncertain. From a Bad
|
||||
sample (incl. an `Empty` outage marker) to the next sample the signal is **Bad** (outage); before the first sample there is **no
|
||||
data**. That is the
|
||||
*simple-bounding* signal. The *interpolated-bounding* signal skips Bad samples: it joins consecutive usable samples (sloped / stepped)
|
||||
straight across an outage.
|
||||
it — the AtTime rule), Uncertain. From a Bad sample (incl. an `Empty` outage marker) to the next sample the signal is **Bad**
|
||||
(outage) — an outage marker with nothing after it runs to the window end; before the first sample there is **no data**; after a
|
||||
*usable* last sample with nothing at or after the window end the data has **ended** (#680, Part 13 "After End of Data": no
|
||||
extrapolation). That is the *simple-bounding* signal. The *interpolated-bounding* signal skips Bad samples: it joins consecutive usable
|
||||
samples (sloped / stepped) straight across an outage, and holds the last usable value to the window end (stepped extrapolation).
|
||||
|
||||
| Aggregate | Value | Bad intervals |
|
||||
|---|---|---|
|
||||
| `TimeAverage` | ∫ interpolated-bounding signal ÷ its defined duration | interpolated across (Part 13) |
|
||||
| `Total` | `TimeAverage × interval seconds` (value·s; the truncated last bucket uses its own length) | interpolated across |
|
||||
| `Total` | ∫ interpolated-bounding signal over its defined part (value·s) — `TimeAverage × interval seconds` on a covered interval; only the covered part when the data starts inside (#680) | interpolated across |
|
||||
| `TimeAverage2` | ∫ simple-bounding signal over non-Bad time ÷ non-Bad duration | excluded |
|
||||
| `Total2` | ∫ simple-bounding signal over non-Bad time (value·s) | excluded |
|
||||
| `Interpolative` | interpolated-bounding value **at the interval start** (reverse read: see below) | interpolated across |
|
||||
| `PercentGood` / `PercentBad` | `100 ×` Good / Bad time ÷ interval duration (#680, Part 13 §5.4.3.31-34) — see below | counted as Bad time |
|
||||
|
||||
**Status, the four time-weighted values** — aligned with the OPC Foundation SDK's `AggregateCalculator.GetTimeBasedStatusCode`
|
||||
(inspected in `Opc.Ua.Server` 1.5.378.106) under the fixed defaults. With `TreatUncertainAsBad = false`, **Uncertain time counts as
|
||||
Good time**: Uncertain samples, a value held up to an outage and the stepped extrapolation past the last sample all count toward
|
||||
PercentDataGood.
|
||||
**Status, `TimeAverage` / `Total`** (interpolated bounding, #680 — Part 13 "Uncertain if Bad values are skipped or Uncertain values
|
||||
are used", and the SDK's `AverageAggregateCalculator`):
|
||||
|
||||
- Every instant of the interval non-Bad ⇒ `Good`.
|
||||
- No defined value anywhere in the interval (before the first usable sample) ⇒ `BadNoData`.
|
||||
- Any part of the interval uses an Uncertain value (either end of a sloped segment, the start of a stepped one), lies on a segment
|
||||
bridged across a Bad sample, is the stepped extrapolation past the last usable sample, or is not covered (the data starts inside it)
|
||||
⇒ `UncertainDataSubNormal`. A segment bridged across an outage marks **every** interval it touches, not just the one holding the Bad
|
||||
sample.
|
||||
- Otherwise ⇒ `Good`.
|
||||
|
||||
**Status, `TimeAverage2` / `Total2`** — the SDK's `AggregateCalculator.GetTimeBasedStatusCode` (inspected in `Opc.Ua.Server`
|
||||
1.5.378.106) under the fixed defaults. With `TreatUncertainAsBad = false`, **Uncertain time counts as Good time** (Uncertain samples, a
|
||||
value held up to an outage). The duration is the interval's, cut at the end of the data:
|
||||
|
||||
- After the end of the data ⇒ `BadNoData` (Part 13 "After End of Data"); the interval the data ends inside covers only up to the last
|
||||
sample and is **Partial** (forward). When it has no non-Bad time before the last sample — the data ends exactly on its start, or the
|
||||
last sample is the first usable one after an outage (comms-loss recovery), or the tag's only sample — there is nothing to cover and
|
||||
the value is the last sample's (`Total2` 0): `Good` only when the data ends exactly on the interval start with a Good sample,
|
||||
otherwise `UncertainDataSubNormal` (the SDK answers the same, modulo its 1 ms artifact).
|
||||
- Every instant of the (covered) interval non-Bad ⇒ `Good`.
|
||||
- Some non-Bad time but not all (an outage, or time before the first sample) ⇒ `UncertainDataSubNormal`.
|
||||
- No non-Bad time but some outage time ⇒ generic **`Bad`** (`0x80000000`) — the SDK's all-Bad result, value null.
|
||||
- No non-Bad time but some outage time (an interval wholly inside an outage) ⇒ generic **`Bad`** (`0x80000000`), value null — Part 13's
|
||||
PercentDataBad rule. (The SDK answers `BadNoData` there; see *SDK parity*.)
|
||||
- No data at all ⇒ `BadNoData`.
|
||||
- A usable `String`/`DateTime` sample feeding the interval ⇒ `BadAggregateInvalidInputs`.
|
||||
|
||||
All carry the Calculated bit; a non-Bad result for the **truncated interval** (shorter than the processing interval — the latest one
|
||||
forward, the earliest one reverse) or for the interval the tag's **data starts inside** (#677, see *Partial bit* above) also carries the
|
||||
**Partial** bit (`0x4`, `AggregateBits.Partial`). Timestamp = interval start (reverse: its later edge); value `double` (Boolean 0/1).
|
||||
For all four, a usable `String`/`DateTime` sample feeding the interval ⇒ `BadAggregateInvalidInputs`.
|
||||
|
||||
> **A live tag's latest interval.** Past the tag's last stored sample the value is held (stepped extrapolation, Uncertain). Under
|
||||
> the SDK rule that Uncertain time is Good time, so the latest interval of a live tag normally reads **`Good`** — not Uncertain —
|
||||
> with the last value held to the interval end. It reads Uncertain only if the interval also contains outage time or starts
|
||||
> before the tag's first sample. Do not take a Good latest interval as proof that data arrived up to its end. If it was truncated
|
||||
> by the request's end time it also carries **Partial**.
|
||||
**`PercentGood` / `PercentBad`** (#680, Part 13 §5.4.3.31-34 — they used to count samples). The interval is cut into **stepped
|
||||
regions**, each with the status of the sample that starts it: Good regions count toward `PercentGood`, Bad ones (incl. outage markers)
|
||||
toward `PercentBad`, Uncertain ones toward neither (so the two sum to < 100 when Uncertain data is present). The region that starts an
|
||||
interval before its first own sample takes the status of the *simple bounding value* there (the SDK's `GetSimpleBound`): Bad after a
|
||||
Bad sample, Uncertain on a segment that runs into an outage or slopes to/from an Uncertain sample, else the earlier sample's. Time
|
||||
before the tag's first sample is **Bad**. When the data ends (a usable last sample, nothing at or after the window end) its region runs
|
||||
to the **end of its own interval** (Part 13 — the SDK gives it 1 ms) and later intervals are `BadNoData`; an outage marker with nothing
|
||||
after it is Bad to the window end. An interval that ends before the first sample ⇒ `BadNoData`. Any value kind is accepted; the status
|
||||
is `Good` (plus Calculated / Partial). Value = `100 × region time ÷ interval duration`.
|
||||
|
||||
All carry the Calculated bit. A non-Bad result also carries the **Partial** bit (`0x4`, `AggregateBits.Partial`) — never on a Bad
|
||||
result — for: the **truncated interval** (shorter than the processing interval — the latest one forward, the earliest one reverse;
|
||||
Part 13 §5.3.3.2 Case 3 requires it — the SDK does not set it, a deliberate difference); the interval the tag's **data starts inside**
|
||||
(#677, forward, see *Partial bit* above); on a **reverse** read, the interval the **latest** sample lies strictly inside when nothing
|
||||
follows it (where a reverse read's data starts — the SDK's `m_startOfData`, keyed on the first value it is fed, #680); and, forward,
|
||||
the **end-of-data** interval of `TimeAverage2` / `Total2` / the percentages (the SDK's `m_endOfData` rule, which it skips for an
|
||||
extrapolated `TimeAverage` / `Total`). Timestamp = interval start (reverse: its later edge); value `double` (Boolean 0/1).
|
||||
|
||||
> **A live tag's latest interval (#680 — changed, client-visible).** Past the tag's last stored sample `TimeAverage` / `Total` hold the
|
||||
> last value to the interval end (stepped extrapolation), and that interval now reads **`UncertainDataSubNormal`** — the extrapolated
|
||||
> part is not data. (It used to read `Good`, because the time-based rule counted the extrapolated time as Good time.) `TimeAverage2` /
|
||||
> `Total2` do not extrapolate: they average / integrate only up to the last sample and set **Partial**; any later interval is
|
||||
> `BadNoData`. A `Good` latest interval therefore now does mean the data covered it — for these aggregates, the latest interval of a
|
||||
> tag that has not yet received a sample at or after the request's end time is never `Good` (`TimeAverage`/`Total`) or always
|
||||
> `Partial` (`TimeAverage2`/`Total2`, forward). If it was truncated by the request's end time it also carries **Partial**.
|
||||
|
||||
**Status, `Interpolative`** (point-based, not the PercentDataGood rule):
|
||||
- A usable sample exactly at the point ⇒ that value with **its own stored status** (no historian bit).
|
||||
@@ -1046,19 +1147,39 @@ value at **10:00**, not 09:00, and no point ever falls before the window. The in
|
||||
**SDK parity (#677).** `SdkAggregateParityTests` (unit, `Sql.Common.Tests`) and
|
||||
`ProcessedConformance.Processed_reads_match_the_SDK_aggregate_calculators` (both SQL backends, through the real SQL) run the pinned SDK's
|
||||
own calculators (`AverageAggregateCalculator`, `MinMaxAggregateCalculator`, `CountAggregateCalculator`, the base `AggregateCalculator`
|
||||
for Interpolative) over the same samples, forward and reverse, and compare every interval's timestamp, value, status code and Partial
|
||||
bit. The per-sample comparison covers Average, Minimum, Maximum, Count, Range, Start, End, Delta, Minimum/MaximumActualTime and
|
||||
PercentGood/Bad; reverse reads over data that starts inside the window are compared too. Known differences that remain (tracked as
|
||||
for Interpolative, `StatusAggregateCalculator` for PercentGood/Bad) over the same samples, forward and reverse, and compare every
|
||||
interval's timestamp, value, status code and Partial bit. The per-sample comparison covers Average, Minimum, Maximum, Count, Range,
|
||||
Start, End, Delta and Minimum/MaximumActualTime; the time-weighted one TimeAverage, TimeAverage2, Total, Total2, Interpolative and
|
||||
PercentGood/Bad (#680), including datasets whose data ends inside the window and with an interval wholly inside an outage; reverse
|
||||
reads over data that starts inside the window are compared too. Known differences that remain (tracked as
|
||||
#680) — documented and masked in the tests, deliberately not changed here because forward results were to stay as they are — are:
|
||||
`Start`/`End` (the SDK returns the raw first/last sample, of any quality, with its own timestamp and stored status; ours returns the
|
||||
first/last usable value stamped with the interval, under the per-sample status rule — the values agree wherever the SDK's is not Bad);
|
||||
`Delta` status (the SDK's stays Good with a Bad sample in the interval, ours is Uncertain); `PercentGood`/`PercentBad` (ours count
|
||||
samples, the SDK measures duration — different definitions, so only timestamp and status are compared); the SDK reports `TimeAverage`/`Total` Uncertain whenever any region of the interval is non-Good, where ours applies the
|
||||
time-based rule (a fully-covered interval bridged across a Bad sample is Good here); `Total` over an interval the data starts inside is
|
||||
`TimeAverage × interval` here but the integral of the covered part in the SDK; ours flags a truncated interval Partial, the SDK does
|
||||
not; the SDK's forward end-of-data Partial rule is not implemented; the SDK's per-sample Count answers 0/Good for an empty interval
|
||||
inside the data and its per-sample aggregates use Good samples only; and on a reverse `Interpolative` read the SDK marks a sample exactly
|
||||
at a point Interpolated (Uncertain if a Bad sample follows) where ours — like the SDK forward — returns the raw sample.
|
||||
the SDK's per-sample Average/Minimum/Maximum use Good samples only, ours Good + Uncertain; the SDK sets Partial on Bad per-sample
|
||||
results too, ours never (#680 closed the Start/End, Delta, Count and end-of-data Partial differences). Time-weighted differences (#680 closed
|
||||
the TimeAverage/Total status, `Total` coverage, end-of-data and PercentGood/Bad ones), each deliberate: ours flags a truncated interval
|
||||
Partial (Part 13 §5.3.3.2 Case 3), the SDK does not; ours carries the Calculated bit on a `BadNoData`; `TimeAverage2`/`Total2` over an
|
||||
interval wholly inside an outage are generic `Bad` here (Part 13's PercentDataBad rule), `BadNoData` in the SDK; in the end-of-data
|
||||
interval the SDK gives the region after the last sample 1 ms (so `Total2` there differs by ~1 ms × value and the percentages by a lot —
|
||||
ours follow Part 13: `TimeAverage2`/`Total2` stop at the last sample, the percentages' region runs to the interval end); and on a reverse
|
||||
read the SDK's backward slice files a sample exactly on an interval's later edge as its early bound — so a reverse `Interpolative` at
|
||||
such a point comes back Interpolated (Uncertain if a Bad sample follows) and a reverse `TimeAverage`/`Total` interval ending on it takes
|
||||
the status of the segment beyond it — where ours, like the SDK forward, returns the raw sample and keeps the interval's own segments
|
||||
(an SDK defect: its forward and reverse answers differ). Three more, kept deliberately (#680 review): an **outage marker as the last
|
||||
sample** (nothing after it) runs Bad to the window end here — `TimeAverage2`/`Total2` generic `Bad` and the percentages 100 % Bad
|
||||
after it, no end-of-data Partial on the interval holding it, and `TimeAverage`/`Total` Uncertain over the span the last usable value is
|
||||
held up to the marker (no Good value follows ⇒ extrapolated) — where the SDK treats the data as ended at the marker (`BadNoData` after
|
||||
it, Partial on that interval, the held span Good): the tag is in an outage, not out of data. **Booleans** are stepped here (a
|
||||
non-analog value is held; the simple bound next to an Uncertain sample takes the earlier sample's status), while the SDK interpolates
|
||||
them as numbers — e.g. an interval that is `false` for its first half and `true` for its second averages 0.5 here, 1 in the SDK, and a
|
||||
PercentGood region ahead of an Uncertain Boolean is Good here, Uncertain in the SDK. Each is pinned by `TimeWeightedAggregatorTests` and
|
||||
by a parity dataset (`OutageLast`, `BoolSwitch`) whose masks are documented in `SdkAggregateParityTests`.
|
||||
Two more, found by the #680 live gate (both cosmetic, both kept deliberately): when an interval's **Minimum/Maximum sample sits exactly
|
||||
on the interval's start**, the SDK reports that result as a raw value (no Calculated/InfoType bits, e.g. `0x00000000` / `0x40A40000`)
|
||||
while ours keeps the Calculated bit (`0x00000401` / `0x40A40401`) — value and timestamp are identical, and ours is consistent with
|
||||
every other interval of the same read. And on an **integer-typed tag** (e.g. a `UInt32`) the SDK casts interpolated and sloped
|
||||
bounding values back to the source integer type, so its `Interpolative` returns `758` where ours returns `758.0459…`, and its
|
||||
`TimeAverage`/`TimeAverage2`/`Total`/`Total2` differ from ours by the truncated bound values (~0.002–0.02 on the gate's data). Ours
|
||||
always computes and returns `Double` (as stated above for every aggregate but Count and Start/End), which is the exact Part 13
|
||||
value; the parity datasets are all `Double`, so this one is not masked there but recorded here.
|
||||
|
||||
**Pushdown trade-off.** Unlike the per-sample aggregates, the time-weighted family is **not** pushed down to SQL: the backend streams
|
||||
the window's rows plus the four bounds (one batch of five result sets on SQL Server; a bounds `UNION ALL` + a window select on QuestDB)
|
||||
@@ -1072,7 +1193,8 @@ judged the larger risk. For very long, densely sampled windows prefer the pushed
|
||||
SQL backends; `TOP (@maxRows)` / `LIMIT $4`). When that extra row arrives the read stops and the node fails with
|
||||
**`BadTooManyOperations`** — for that node only; it is not counted in the backend's health failures. Without the cap a one-year
|
||||
`Total` on a 1 Hz tag (≈31 M rows) would hold a pooled connection until `HistoryReadDeadline` and then return `BadTimeout`. The four
|
||||
bound rows do not count toward the cap, and the per-sample aggregates are never capped. Raise the cap, or read in shorter windows.
|
||||
bound rows do not count toward the cap, and the per-sample aggregates are never capped. Since #680 `PercentGood`/`PercentBad` are in
|
||||
this family and **are** capped (they used to be pushed down). Raise the cap, or read in shorter windows.
|
||||
|
||||
### Health
|
||||
|
||||
@@ -1288,9 +1410,24 @@ Every query filters `NOT deleted`; every value goes through `StoredSample`, so t
|
||||
reverse read (#675) is `(lo, hi]` ordered `(ts, event_id)` descending.
|
||||
- **Processed:** one query per node; bucket index `(cast(ts AS LONG) * 10 - bucketBaseUnixTicks) / intervalTicks` (never `SAMPLE BY`, whose alignment
|
||||
differs), so bucket edges are identical to `SqlServer` (forward start-inclusive / end-exclusive with the partial bucket last; reverse anchored at the
|
||||
later time, later-edge-inclusive, partial bucket earliest — #677; interval 0 ⇒ one bucket, 100,000-bucket cap). On a forward read of any aggregate
|
||||
but Average, a second `LIMIT 1` statement (capped at the window end) fetches the tag's earliest live sample for the start-of-data Partial bit.
|
||||
later time, later-edge-inclusive, partial bucket earliest — #677; interval 0 ⇒ one bucket, 100,000-bucket cap). On a per-sample read of any
|
||||
aggregate but Average, in either direction, two `LIMIT 1` statements fetch the tag's data range — its earliest live sample at or before the
|
||||
window's later edge and its latest at or after its earlier edge — for the start/end-of-data Partial bits and Count's in-range empty interval
|
||||
(#677/#680). `tag` is an unindexed SYMBOL, so these are partition scans (a few ms on a live tag; tens of ms cold for a stopped tag on a
|
||||
large table — see *Data-range queries* above). Start/End/Delta also return each bucket's first and last sample of any quality (`row_number()` over every row).
|
||||
Min/max times use `row_number()` windows with the earliest-timestamp tie-break.
|
||||
⚠️ **Live-gate 8 F1 — counts and picks must come from one table version.** QuestDB opens a separate table reader per reference to a
|
||||
CTE, so the earlier Start/End/Delta form (counts CTE + self-joins) could count a row committed between two references without
|
||||
finding it — a NULL first-of-any-quality time that threw and failed the node's read (~1 in 90k reads of a tag being written). That
|
||||
query now reads the table **once**: one `GROUP BY` with `first()`/`last()` for the first/last sample of any quality and
|
||||
`arg_min`/`arg_max` keyed on the usable rows' time for the first/last usable one (~15 % faster than before; the usable picks carry
|
||||
no `str_value` — `arg_min` has no VARCHAR form, and only Delta reads them, never with a usable string in the bucket). On `SqlServer`
|
||||
(READ COMMITTED gives no cross-reference guarantee either, though it was never observed there) Start/End/Delta was reshaped the same
|
||||
way with a windowed pass + conditional `MAX` — 1.4–3× *faster* than the old five-reference form on a 1 M-row tag. Min/MaxActualTime
|
||||
keep their counts + filtered-rank form on both backends: every single-scan variant measured 1.2–2.4× slower, and QuestDB's
|
||||
`arg_max` cannot express the earliest-time tie-break. So, on both backends, a row whose picks are absent anyway maps to "no such
|
||||
sample" — `BadNoData` for that interval, never an exception and never a value-less Good. `LiveWriteReadRaceStressTests` (QuestDB
|
||||
integration, opt-in via `HISTORIAN_QUESTDB_STRESS_SECONDS`) interleaves a single-row writer with tight-loop reads to pin it.
|
||||
|
||||
### Retention (TTL)
|
||||
|
||||
|
||||
+2
-2
@@ -84,8 +84,8 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
var mode = AggregateModeMapper.ToRetrievalMode(aggregate);
|
||||
try
|
||||
{
|
||||
// Total/Count are now native gateway retrieval modes — no client-side scaling
|
||||
// (unlike the Wonderware path that derived Total as Average × interval-seconds).
|
||||
// Total is a native gateway retrieval mode (Integral) — no client-side scaling. Average and Count
|
||||
// have no AVEVA mode and were rejected by the mapper above (#679).
|
||||
var buckets = new List<HistorianAggregateSample>();
|
||||
await foreach (var bucket in _client
|
||||
.ReadAggregateAsync(fullReference, startUtc, endUtc, mode, interval, cancellationToken)
|
||||
|
||||
+8
-9
@@ -9,15 +9,15 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Mapping;
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// Average/Minimum/Maximum line up with the legacy Wonderware client's aggregate mapping. The two
|
||||
/// remaining members are now served by <b>native</b> gateway retrieval modes:
|
||||
/// <see cref="HistoryAggregateType.Total"/> → <see cref="RetrievalMode.Integral"/> and
|
||||
/// <see cref="HistoryAggregateType.Count"/> → <see cref="RetrievalMode.Counter"/>.
|
||||
/// Served: <see cref="HistoryAggregateType.TimeAverage"/> → <see cref="RetrievalMode.TimeWeightedAverage"/>,
|
||||
/// Minimum → <see cref="RetrievalMode.MinimumWithTime"/>, Maximum → <see cref="RetrievalMode.MaximumWithTime"/>,
|
||||
/// Total → <see cref="RetrievalMode.Integral"/>.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// This replaces the Wonderware-era client-side workarounds (Total derived as Average × interval,
|
||||
/// Count approximated from a value count): no client-side scaling is performed any more, so the
|
||||
/// gateway path is a strict improvement.
|
||||
/// Deliberately NOT served (#679): <see cref="HistoryAggregateType.Average"/> (OPC UA Average is the arithmetic mean;
|
||||
/// AVEVA's modes include no arithmetic mean — only the time-weighted one, which is TimeAverage) and
|
||||
/// <see cref="HistoryAggregateType.Count"/> (AVEVA <c>Counter</c> is a rollover-counter increase, not a sample count).
|
||||
/// Both surface as <c>BadAggregateNotSupported</c> rather than returning a different statistic under the requested name.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
internal static class AggregateModeMapper
|
||||
@@ -29,11 +29,10 @@ internal static class AggregateModeMapper
|
||||
/// <exception cref="ArgumentOutOfRangeException">An undefined enum value.</exception>
|
||||
public static RetrievalMode ToRetrievalMode(HistoryAggregateType aggregate) => aggregate switch
|
||||
{
|
||||
HistoryAggregateType.Average => RetrievalMode.TimeWeightedAverage,
|
||||
HistoryAggregateType.TimeAverage => RetrievalMode.TimeWeightedAverage,
|
||||
HistoryAggregateType.Minimum => RetrievalMode.MinimumWithTime,
|
||||
HistoryAggregateType.Maximum => RetrievalMode.MaximumWithTime,
|
||||
HistoryAggregateType.Total => RetrievalMode.Integral,
|
||||
HistoryAggregateType.Count => RetrievalMode.Counter,
|
||||
// Part 13 aggregates the gateway has no retrieval mode for (#285) — an UNDEFINED enum value is a
|
||||
// programming error, but a defined-yet-unserved member is a normal client request the server
|
||||
// answers with BadAggregateNotSupported.
|
||||
|
||||
@@ -57,7 +57,7 @@ internal static class QuestDbAggregateQueries
|
||||
private static readonly string EmptyKind = ((byte)HistorianValueKind.Empty).ToString(CultureInfo.InvariantCulture);
|
||||
|
||||
/// <summary>
|
||||
/// Average / Minimum / Maximum / Total / Count / Range / PercentGood / PercentBad.
|
||||
/// Average / Minimum / Maximum / Count / Range (PercentGood / PercentBad and Total are time-weighted, not here).
|
||||
/// Family columns: <c>6 avg_value, 7 min_value, 8 max_value, 9 sum_value</c> (DOUBLE, NULL when no usable sample).
|
||||
/// </summary>
|
||||
/// <param name="sample">Prefixed sample table name.</param>
|
||||
@@ -71,32 +71,72 @@ internal static class QuestDbAggregateQueries
|
||||
"sum(CASE WHEN u = 1 THEN num_value END) AS sum_value " +
|
||||
"FROM (" + Classified(sample, reverse) + ") GROUP BY b ORDER BY b";
|
||||
|
||||
/// <summary>Ordinal of the StartEnd family's first-sample-of-any-quality columns (<see cref="StartEnd"/>).</summary>
|
||||
public const int FirstAnyColumn = FirstFamilyColumn + 10;
|
||||
|
||||
/// <summary>
|
||||
/// Start / End / Delta: the first and last usable sample of each bucket by <c>ts</c>.
|
||||
/// Family columns: <c>6..10</c> first sample (<c>kind, num_value, int_value, str_value, ts</c>), <c>11..15</c> last
|
||||
/// sample (same order). A bucket without a usable sample has NULL <c>ts</c> but <c>kind = 0</c> (BYTE has no NULL),
|
||||
/// so the reader keys absence on the NULL time.
|
||||
/// Start / End / Delta: the first and last usable sample of each bucket by <c>ts</c> (Delta's value), and the first
|
||||
/// and last sample of <b>any</b> quality (Start / End, and Delta's status — #680).
|
||||
/// Family columns: <c>6..10</c> first usable sample (<c>kind, num_value, int_value, str_value, ts</c>), <c>11..15</c>
|
||||
/// last usable sample (same order) — a bucket without a usable sample has NULL <c>ts</c> but <c>kind = 0</c> (BYTE
|
||||
/// has no NULL), so the reader keys absence on the NULL time; then <c>16..21</c> the first sample of any quality
|
||||
/// (<c>ts, kind, num_value, int_value, str_value, status</c>) and <c>22..27</c> the last (same order), present for
|
||||
/// every returned (non-empty) bucket.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>One scan, one GROUP BY (live-gate 8 F1).</b> The counts and all four picked samples come from a single read
|
||||
/// of the sample table. The first/last sample of any quality are QuestDB's <c>first()</c>/<c>last()</c> (the
|
||||
/// first/last row of the group in designated-timestamp order — null values included, so every column comes from
|
||||
/// the same row); the first/last <i>usable</i> sample are <c>arg_min</c>/<c>arg_max</c> keyed on <c>uts</c>, the
|
||||
/// row's time when usable and NULL otherwise (a NULL key is skipped, a NULL value is returned as is, and
|
||||
/// <c>(ts, tag)</c> is unique — DEDUP — so there are no ties). <c>arg_min</c> has no VARCHAR form, so the usable
|
||||
/// picks carry no <c>str_value</c>: only Delta reads them, and only when the bucket has no usable non-numeric
|
||||
/// sample.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// The previous form referenced its CTE four times (counts + three self-joins); QuestDB opens a separate table
|
||||
/// reader per reference, so a row committed between two of them produced a bucket whose counts included a sample
|
||||
/// its first/last join could not find (a NULL first-of-any-quality time that threw) — ~1 in 90k reads of a tag
|
||||
/// being written. One reference means one reader at one table version. A windowed (<c>row_number()</c>)
|
||||
/// single-scan form was also correct but ~2× slower; this one is not slower than the old query.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// ⚠️ <c>first()</c>/<c>last()</c> following timestamp order is QuestDB 10.0.1 IMPLEMENTATION behaviour (rows reach
|
||||
/// the GROUP BY in row-id order, and the parallel merge keeps row id), not a documented guarantee. It was verified
|
||||
/// on 20M rows with O3 inserts, single-threaded and parallel. Adding a JOIN, <c>ORDER BY</c> or <c>UNION</c> inside
|
||||
/// <c>Classified</c>, or upgrading QuestDB, can silently break Start/End — re-verify <c>first(ts) = min(ts)</c>.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
/// <param name="sample">Prefixed sample table name.</param>
|
||||
/// <returns>The SQL text.</returns>
|
||||
/// <param name="reverse">True for a reverse read (window <c>($4, $5]</c>).</param>
|
||||
public static string StartEnd(string sample, bool reverse = false) =>
|
||||
"WITH s AS (" + Classified(sample, reverse) + "), " +
|
||||
"st AS (SELECT b, " + CommonCounts() + " FROM s GROUP BY b), " +
|
||||
"f AS (SELECT b, kind, num_value, int_value, str_value, ts, " +
|
||||
"row_number() OVER (PARTITION BY b ORDER BY ts ASC) AS rf, " +
|
||||
"row_number() OVER (PARTITION BY b ORDER BY ts DESC) AS rl FROM s WHERE u = 1) " +
|
||||
"SELECT st.*, " +
|
||||
"x.kind, x.num_value, x.int_value, x.str_value, x.ts, " +
|
||||
"y.kind, y.num_value, y.int_value, y.str_value, y.ts " +
|
||||
"FROM st LEFT JOIN (SELECT * FROM f WHERE rf = 1) x ON x.b = st.b " +
|
||||
"LEFT JOIN (SELECT * FROM f WHERE rl = 1) y ON y.b = st.b ORDER BY st.b";
|
||||
public static string StartEnd(string sample, bool reverse = false)
|
||||
{
|
||||
// The usable picks: kind, num_value, int_value, str_value (not available — see remarks), ts.
|
||||
static string Usable(string f, string g) =>
|
||||
$"cast({f}(kind, uts) AS SHORT), {f}(num_value, uts), {f}(int_value, uts), cast(NULL AS VARCHAR), {g}(uts)";
|
||||
// The any-quality picks: ts, kind, num_value, int_value, str_value, status — all from the group's first / last row.
|
||||
static string Any(string f) =>
|
||||
$"{f}(ts), cast({f}(kind) AS SHORT), {f}(num_value), {f}(int_value), {f}(str_value), {f}(status)";
|
||||
return "SELECT b, " + CommonCounts() + ", " +
|
||||
Usable("arg_min", "min") + ", " + Usable("arg_max", "max") + ", " + Any("first") + ", " + Any("last") + " " +
|
||||
"FROM (SELECT b, ts, kind, num_value, int_value, str_value, status, u, g, CASE WHEN u = 1 THEN ts END AS uts " +
|
||||
"FROM (" + Classified(sample, reverse) + ")) GROUP BY b ORDER BY b";
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// MinimumActualTime / MaximumActualTime: the usable sample with the smallest (largest) <c>num_value</c>, ties
|
||||
/// broken by the earliest <c>ts</c>. Family columns: <c>6 num_value, 7 ts</c> (NULL when the bucket has no usable
|
||||
/// numeric sample).
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Deliberately still two references to the classified rows (counts + the ranked extreme), unlike
|
||||
/// <see cref="StartEnd"/>: the single-scan <c>row_number()</c> form measured ~2× slower, and QuestDB's
|
||||
/// <c>arg_max</c>/<c>arg_min</c> cannot express the earliest-time tie-break. Live-gate 8 F1's inconsistency (a row
|
||||
/// committed between the two readers — counted but not ranked) is absorbed by the reader and evaluator instead: a
|
||||
/// NULL extreme in a bucket with usable samples evaluates to <c>BadNoData</c>, never a value-less Good.
|
||||
/// </remarks>
|
||||
/// <param name="sample">Prefixed sample table name.</param>
|
||||
/// <param name="maximum">True for MaximumActualTime.</param>
|
||||
/// <returns>The SQL text.</returns>
|
||||
@@ -150,18 +190,29 @@ internal static class QuestDbAggregateQueries
|
||||
internal const string BucketExpression = "(cast(ts AS LONG) * 10 - $1) / $2";
|
||||
|
||||
/// <summary>
|
||||
/// The tag's earliest live sample (any quality) before the window end: the start-of-data Partial rule (#677; forward
|
||||
/// reads only, so the forward window's exclusive end bounds the scan). Parameters <c>$1</c> tag, <c>$2</c> the window
|
||||
/// end. Column: <c>ts</c>; no row when the tag has none before the end.
|
||||
/// The tag's earliest live sample (any quality) at or before the window's later edge — half of the tag's data range
|
||||
/// (the start-of-data Partial rule, #677, and Count's in-range empty interval, #680; see
|
||||
/// <see cref="AggregateEvaluator.NeedsDataRange"/>). Parameters <c>$1</c> tag, <c>$2</c> the later edge. Column:
|
||||
/// <c>ts</c>; no row when the tag has none — the data range then does not reach the window.
|
||||
/// </summary>
|
||||
/// <param name="sample">Prefixed sample table name.</param>
|
||||
/// <returns>The SQL text.</returns>
|
||||
public static string EarliestSample(string sample) =>
|
||||
$"SELECT ts FROM {sample} WHERE tag = $1 AND ts < $2 AND NOT deleted ORDER BY ts ASC LIMIT 1";
|
||||
$"SELECT ts FROM {sample} WHERE tag = $1 AND ts <= $2 AND NOT deleted ORDER BY ts ASC LIMIT 1";
|
||||
|
||||
/// <summary>
|
||||
/// The tag's latest live sample (any quality) at or after the window's earlier edge — the other half of the data
|
||||
/// range (the end-of-data Partial rule and Count's in-range empty interval, #680). Parameters <c>$1</c> tag,
|
||||
/// <c>$2</c> the earlier edge. Column: <c>ts</c>; no row when the tag has none.
|
||||
/// </summary>
|
||||
/// <param name="sample">Prefixed sample table name.</param>
|
||||
/// <returns>The SQL text.</returns>
|
||||
public static string LatestSample(string sample) =>
|
||||
$"SELECT ts FROM {sample} WHERE tag = $1 AND ts >= $2 AND NOT deleted ORDER BY ts DESC LIMIT 1";
|
||||
|
||||
/// <summary>The classified subquery <c>s</c>: the window's live samples with bucket index and the usable/good flags.</summary>
|
||||
private static string Classified(string sample, bool reverse) =>
|
||||
"SELECT " + BucketExpression + " AS b, ts, kind, num_value, int_value, str_value, " +
|
||||
"SELECT " + BucketExpression + " AS b, ts, kind, num_value, int_value, str_value, status, " +
|
||||
$"CASE WHEN status < {NotBadLimit} AND kind <> {EmptyKind} THEN 1 ELSE 0 END AS u, " +
|
||||
$"CASE WHEN status < {GoodLimit} AND kind <> {EmptyKind} THEN 1 ELSE 0 END AS g " +
|
||||
$"FROM {sample} WHERE tag = $3 AND " + (reverse ? "ts > $4 AND ts <= $5" : "ts >= $4 AND ts < $5") + " AND NOT deleted";
|
||||
|
||||
@@ -201,14 +201,14 @@ internal sealed class QuestDbConnection : IAsyncDisposable, IDisposable
|
||||
/// <param name="r">Reader positioned on a row.</param>
|
||||
/// <param name="i">Column ordinal.</param>
|
||||
/// <returns>The timestamp with <c>Kind=Utc</c>.</returns>
|
||||
public static DateTime ReadUtc(NpgsqlDataReader r, int i) =>
|
||||
public static DateTime ReadUtc(System.Data.Common.DbDataReader r, int i) =>
|
||||
DateTime.SpecifyKind(r.GetDateTime(i), DateTimeKind.Utc);
|
||||
|
||||
/// <summary>Reads a <c>BYTE</c> column (QuestDB returns <c>smallint</c>); NULL (never stored) reads as 0 (Empty).</summary>
|
||||
/// <param name="r">Reader positioned on a row.</param>
|
||||
/// <param name="i">Column ordinal.</param>
|
||||
/// <returns>The byte value.</returns>
|
||||
public static byte ReadKind(NpgsqlDataReader r, int i) => r.IsDBNull(i) ? (byte)0 : (byte)r.GetInt16(i);
|
||||
public static byte ReadKind(System.Data.Common.DbDataReader r, int i) => r.IsDBNull(i) ? (byte)0 : (byte)r.GetInt16(i);
|
||||
|
||||
// ---- error classification --------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
+44
-16
@@ -1,3 +1,4 @@
|
||||
using System.Data.Common;
|
||||
using System.Globalization;
|
||||
using System.Text;
|
||||
using Microsoft.Extensions.Logging;
|
||||
@@ -472,7 +473,7 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource
|
||||
return new HistoryReadResult(twResult, ContinuationPoint: null);
|
||||
}
|
||||
var rows = new List<BucketRow>();
|
||||
DateTime? earliest = null;
|
||||
DateTime? earliest = null, latest = null;
|
||||
if (plan.BucketCount > 0 && await ResolveAsync(fullReference, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
var sample = _conn.SampleTable;
|
||||
@@ -494,20 +495,19 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource
|
||||
while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false))
|
||||
rows.Add(ReadBucket(reader, family));
|
||||
}
|
||||
// The tag's earliest live sample before the window end: the start-of-data Partial rule (#677) — forward
|
||||
// reads of an aggregate other than Average only; skipped otherwise.
|
||||
if (AggregateEvaluator.NeedsEarliestSample(aggregate, plan))
|
||||
// The tag's data range (earliest + latest live sample): the start/end-of-data Partial rules (#677/#680 —
|
||||
// any per-sample aggregate but Average, either direction) and Count's in-range empty interval (#680); skipped
|
||||
// otherwise.
|
||||
if (AggregateEvaluator.NeedsDataRange(aggregate, plan))
|
||||
{
|
||||
await using var cmd = _conn.CreateCommand(conn, QuestDbAggregateQueries.EarliestSample(sample));
|
||||
QuestDbConnection.BindText(cmd, fullReference);
|
||||
QuestDbConnection.BindTimestamp(cmd, plan.EndUtc);
|
||||
await using var reader = await cmd.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false);
|
||||
if (await reader.ReadAsync(cancellationToken).ConfigureAwait(false) && !reader.IsDBNull(0))
|
||||
earliest = QuestDbConnection.ReadUtc(reader, 0);
|
||||
earliest = await ReadBoundAsync(conn, QuestDbAggregateQueries.EarliestSample(sample), fullReference,
|
||||
plan.EndUtc, cancellationToken).ConfigureAwait(false);
|
||||
latest = await ReadBoundAsync(conn, QuestDbAggregateQueries.LatestSample(sample), fullReference,
|
||||
plan.StartUtc, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
var result = AggregateEvaluator.EvaluateAll(aggregate, plan, rows, earliest, now);
|
||||
var result = AggregateEvaluator.EvaluateAll(aggregate, plan, rows, earliest, latest, now);
|
||||
|
||||
RecordSuccess();
|
||||
return new HistoryReadResult(result, ContinuationPoint: null);
|
||||
@@ -565,12 +565,32 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource
|
||||
if (bounds[3] is { } usableAfter) tw.Add(usableAfter);
|
||||
}
|
||||
|
||||
private static BucketRow ReadBucket(NpgsqlDataReader reader, AggregateFamily family)
|
||||
/// <summary>Runs a one-column <c>ts</c> seek (<see cref="QuestDbAggregateQueries.EarliestSample"/> / <c>LatestSample</c>).</summary>
|
||||
private async Task<DateTime?> ReadBoundAsync(NpgsqlConnection conn, string sql, string tag, DateTime edge, CancellationToken ct)
|
||||
{
|
||||
static double? D(NpgsqlDataReader r, int i) => r.IsDBNull(i) ? null : r.GetDouble(i);
|
||||
static int N(NpgsqlDataReader r, int i) => r.IsDBNull(i) ? 0 : Convert.ToInt32(r.GetValue(i), CultureInfo.InvariantCulture);
|
||||
await using var cmd = _conn.CreateCommand(conn, sql);
|
||||
QuestDbConnection.BindText(cmd, tag);
|
||||
QuestDbConnection.BindTimestamp(cmd, edge);
|
||||
await using var reader = await cmd.ExecuteReaderAsync(ct).ConfigureAwait(false);
|
||||
return await reader.ReadAsync(ct).ConfigureAwait(false) && !reader.IsDBNull(0) ? QuestDbConnection.ReadUtc(reader, 0) : null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Maps one per-sample aggregate row (<see cref="QuestDbAggregateQueries"/> column layout) to a
|
||||
/// <see cref="BucketRow"/>. Every picked sample — first/last usable, first/last of any quality, the extreme — is
|
||||
/// optional: a NULL time reads as "no such sample", never an exception (live-gate 8 F1: a row whose counts and
|
||||
/// picks disagreed used to throw out of <see cref="ReadStored"/> and fail the node's whole read; the evaluator
|
||||
/// answers such a bucket from what is present, or <c>BadNoData</c>).
|
||||
/// </summary>
|
||||
/// <param name="reader">Reader positioned on a bucket row.</param>
|
||||
/// <param name="family">The query's aggregate family.</param>
|
||||
/// <returns>The bucket statistics.</returns>
|
||||
internal static BucketRow ReadBucket(DbDataReader reader, AggregateFamily family)
|
||||
{
|
||||
static double? D(DbDataReader r, int i) => r.IsDBNull(i) ? null : r.GetDouble(i);
|
||||
static int N(DbDataReader r, int i) => r.IsDBNull(i) ? 0 : Convert.ToInt32(r.GetValue(i), CultureInfo.InvariantCulture);
|
||||
// A bucket without a usable sample has NULL times but kind 0 (BYTE has no NULL): absence is keyed on the time.
|
||||
static HistorianValue V(NpgsqlDataReader r, int i) =>
|
||||
static HistorianValue V(DbDataReader r, int i) =>
|
||||
r.IsDBNull(i + 4)
|
||||
? HistorianValue.Empty
|
||||
: ToValue(QuestDbConnection.ReadKind(r, i), D(r, i + 1), r.IsDBNull(i + 2) ? null : r.GetInt64(i + 2),
|
||||
@@ -597,6 +617,8 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource
|
||||
case AggregateFamily.StartEnd:
|
||||
row.First = V(reader, f);
|
||||
row.Last = V(reader, f + 5);
|
||||
row.FirstAny = ReadStoredOrNull(reader, QuestDbAggregateQueries.FirstAnyColumn);
|
||||
row.LastAny = ReadStoredOrNull(reader, QuestDbAggregateQueries.FirstAnyColumn + 6);
|
||||
break;
|
||||
default:
|
||||
row.Extreme = D(reader, f);
|
||||
@@ -612,7 +634,7 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource
|
||||
/// Reads <c>ts, kind, num_value, int_value, str_value, status</c> starting at <paramref name="o"/> and applies the
|
||||
/// shared quality clamp (<see cref="StoredSample.FromStored"/>).
|
||||
/// </summary>
|
||||
private static StoredSample ReadStored(NpgsqlDataReader r, int o)
|
||||
private static StoredSample ReadStored(DbDataReader r, int o)
|
||||
{
|
||||
var value = ToValue(
|
||||
QuestDbConnection.ReadKind(r, o + 1),
|
||||
@@ -624,6 +646,12 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource
|
||||
return StoredSample.FromStored(QuestDbConnection.ReadUtc(r, o), value, status);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// <see cref="ReadStored"/> for an optional picked sample: a NULL <c>ts</c> means the row carries no such sample
|
||||
/// (live-gate 8 F1), so the result is null rather than an <see cref="InvalidCastException"/>.
|
||||
/// </summary>
|
||||
private static StoredSample? ReadStoredOrNull(DbDataReader r, int o) => r.IsDBNull(o) ? null : ReadStored(r, o);
|
||||
|
||||
/// <summary>
|
||||
/// Rebuilds a value from its stored columns via the shared <see cref="SampleColumnMapper.FromColumns"/>, after the
|
||||
/// RC2 repair: QuestDB reads a <c>LONG</c> holding <see cref="long.MinValue"/> back as NULL, so an Int64 row with
|
||||
|
||||
@@ -7,14 +7,28 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common;
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// Per interval: no samples ⇒ <c>BadNoData</c>. PercentGood / PercentBad ⇒ <c>100 × good|bad / all</c> over
|
||||
/// every sample, status Good. Otherwise no usable sample ⇒ <c>BadNoData</c>; a numeric aggregate with a usable
|
||||
/// String/DateTime sample ⇒ <c>BadAggregateInvalidInputs</c>; else the value, Good when every sample is Good,
|
||||
/// <c>UncertainDataSubNormal</c> when any is Bad or Uncertain. Every status — including <c>BadNoData</c> —
|
||||
/// carries the Calculated info bits. Timestamps are the bucket's <see cref="BucketPlan.BucketTimestamp"/> (its
|
||||
/// start forward, its later edge reverse), except Minimum/MaximumActualTime (the sample's time; ties ⇒ earliest).
|
||||
/// Every aggregate but <c>Average</c> also carries the Partial bit on the bucket the data starts inside
|
||||
/// (<see cref="EvaluateAll"/>).
|
||||
/// Per interval: no samples ⇒ <c>BadNoData</c> (Count: 0 inside the data range). Otherwise no usable sample ⇒
|
||||
/// <c>BadNoData</c>; a numeric aggregate with a usable String/DateTime sample ⇒ <c>BadAggregateInvalidInputs</c>;
|
||||
/// else the value, Good when every sample is Good, <c>UncertainDataSubNormal</c> when any is Bad or Uncertain.
|
||||
/// Every calculated status — including <c>BadNoData</c> — carries the Calculated info bits. Timestamps are the
|
||||
/// bucket's <see cref="BucketPlan.BucketTimestamp"/> (its start forward, its later edge reverse), except
|
||||
/// Minimum/MaximumActualTime (the sample's time; ties ⇒ earliest) and Start/End. Every aggregate but
|
||||
/// <c>Average</c> also carries the Partial bit on the bucket the data starts (forward) or ends inside
|
||||
/// (<see cref="EvaluateAll"/>). PercentGood / PercentBad are duration-based since #680 and evaluated by
|
||||
/// <see cref="TimeWeightedAggregator"/>, not here.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// #680 (Part 13 / the SDK's calculators): <b>Count</b> counts Good samples only, value-based status (all Good ⇒
|
||||
/// Good, all Bad ⇒ the generic Bad, else Uncertain), and is 0/Good on an empty bucket inside the tag's data range;
|
||||
/// <b>Start / End</b> are the raw first / last sample of any quality (own value — null when Bad —, own status
|
||||
/// without the Calculated bit, own timestamp); <b>Delta</b> is Uncertain only when the bucket's first or last
|
||||
/// sample is not usable.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Inconsistent rows</b> (live-gate 8 F1): a row whose counts say a bucket holds samples while a picked sample
|
||||
/// (first/last of any quality, first/last usable, the extreme) is absent is answered from what is present —
|
||||
/// Start/End/Delta/ActualTime without the sample they need are <c>BadNoData</c> — never an exception. The backends'
|
||||
/// queries read counts and picks in one scan so this should not arise; the evaluator does not rely on it.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Values: Count is <see cref="int"/>; Start/End keep the sample's type; every other aggregate is
|
||||
@@ -38,34 +52,50 @@ public static class AggregateEvaluator
|
||||
DataValueSnapshot Calc(object? value, uint status, DateTime? t = null) =>
|
||||
new(value, HistorianInfoBits.WithCalculated(status), t ?? bucketStart, now);
|
||||
|
||||
// Refuse a non-per-sample aggregate up front, whatever the bucket holds (an all-Bad bucket would otherwise answer
|
||||
// BadNoData before the switch's default is reached).
|
||||
if (AggregateFamilies.FamilyOf(aggregate) is not (AggregateFamily.Simple or AggregateFamily.StartEnd or AggregateFamily.ActualTime))
|
||||
throw new HistoryAggregateNotSupportedException(aggregate);
|
||||
if (r.Total == 0) return NoData(bucketStart, now);
|
||||
switch (aggregate)
|
||||
{
|
||||
case HistoryAggregateType.PercentGood: return Calc(100d * r.Good / r.Total, HistorianStatusCodes.Good);
|
||||
case HistoryAggregateType.PercentBad: return Calc(100d * r.Bad / r.Total, HistorianStatusCodes.Good);
|
||||
// #680 (Part 13 Tables 73/74; SDK ComputeStartEnd): the raw first / last sample of ANY quality — its own value
|
||||
// (null when Bad), its own status (Raw: no Calculated bit) and its own timestamp.
|
||||
case HistoryAggregateType.Start: return Raw(r.FirstAny, bucketStart, now);
|
||||
case HistoryAggregateType.End: return Raw(r.LastAny, bucketStart, now);
|
||||
// #680 (Part 13 §5.4.3.21; SDK ComputeCount): Good samples only; value-based status — Good when every sample
|
||||
// is Good, the generic Bad when every one is Bad (value null), else Uncertain.
|
||||
case HistoryAggregateType.Count:
|
||||
if (r.Bad == r.Total) return Calc(null, HistorianStatusCodes.Bad);
|
||||
return Calc(r.Good, r.Good == r.Total ? HistorianStatusCodes.Good : HistorianStatusCodes.UncertainDataSubNormal);
|
||||
}
|
||||
if (r.Usable == 0) return NoData(bucketStart, now);
|
||||
|
||||
var anyText = aggregate is HistoryAggregateType.Count or HistoryAggregateType.Start or HistoryAggregateType.End;
|
||||
if (!anyText && r.NonNumeric > 0)
|
||||
return Calc(null, HistorianStatusCodes.BadAggregateInvalidInputs);
|
||||
if (r.NonNumeric > 0) return Calc(null, HistorianStatusCodes.BadAggregateInvalidInputs);
|
||||
|
||||
var status = r.Good == r.Total ? HistorianStatusCodes.Good : HistorianStatusCodes.UncertainDataSubNormal;
|
||||
switch (aggregate)
|
||||
{
|
||||
case HistoryAggregateType.Count: return Calc(r.Usable, status);
|
||||
case HistoryAggregateType.Average: return Calc(r.Avg, status);
|
||||
case HistoryAggregateType.Minimum: return Calc(r.Min, status);
|
||||
case HistoryAggregateType.Maximum: return Calc(r.Max, status);
|
||||
case HistoryAggregateType.Range: return Calc(r.Max - r.Min, status);
|
||||
case HistoryAggregateType.Start: return Calc(r.First.ToObject(), status);
|
||||
case HistoryAggregateType.End: return Calc(r.Last.ToObject(), status);
|
||||
case HistoryAggregateType.Delta:
|
||||
return r.First.TryGetDouble(out var a) && r.Last.TryGetDouble(out var b)
|
||||
? Calc(b - a, status)
|
||||
: Calc(null, HistorianStatusCodes.BadAggregateInvalidInputs);
|
||||
// #680 (SDK ComputeDelta): last usable − first usable; Uncertain only when the interval's first or last
|
||||
// sample was skipped (not usable) — a Bad sample in the middle does not degrade it.
|
||||
// Live-gate 8 F1: a row whose counts promise a usable sample its picks do not carry (a backend that read
|
||||
// the counts and the picks from two table versions) has no consistent first/last — no data, not a throw
|
||||
// and not a wrong-input verdict.
|
||||
if (r.First.Kind == HistorianValueKind.Empty || r.Last.Kind == HistorianValueKind.Empty)
|
||||
return NoData(bucketStart, now);
|
||||
if (!r.First.TryGetDouble(out var a) || !r.Last.TryGetDouble(out var b))
|
||||
return Calc(null, HistorianStatusCodes.BadAggregateInvalidInputs);
|
||||
var bounded = r.FirstAny is { IsUsable: true } && r.LastAny is { IsUsable: true };
|
||||
return Calc(b - a, bounded ? HistorianStatusCodes.Good : HistorianStatusCodes.UncertainDataSubNormal);
|
||||
case HistoryAggregateType.MinimumActualTime:
|
||||
case HistoryAggregateType.MaximumActualTime:
|
||||
// Live-gate 8 F1: usable numeric samples counted but no extreme picked (an inconsistent row) is no data,
|
||||
// never a value-less Good.
|
||||
if (r.Extreme is null || r.ExtremeTime is null) return NoData(bucketStart, now);
|
||||
return Calc(r.Extreme, status, r.ExtremeTime);
|
||||
default:
|
||||
throw new HistoryAggregateNotSupportedException(aggregate);
|
||||
@@ -81,49 +111,74 @@ public static class AggregateEvaluator
|
||||
public static bool SetsStartOfDataPartial(HistoryAggregateType aggregate) => aggregate != HistoryAggregateType.Average;
|
||||
|
||||
/// <summary>
|
||||
/// True when a per-sample read must fetch the tag's earliest sample: the start-of-data Partial rule applies only to a
|
||||
/// <b>forward</b> read (as in the SDK, whose reverse read keys <c>m_startOfData</c> on the first value it is fed — the
|
||||
/// latest) of an aggregate other than <c>Average</c>. Otherwise the backend skips that query.
|
||||
/// True when a per-sample read must fetch the tag's data range — its earliest and latest stored samples of any
|
||||
/// quality — because the answer can change a result: the start/end-of-data Partial rules for an aggregate other
|
||||
/// than <c>Average</c> (either direction: the end-of-data rule applies to reverse reads too), and Count (an empty
|
||||
/// interval inside the range is 0, outside it <c>BadNoData</c>, #680). Only a per-sample <c>Average</c> read skips
|
||||
/// both queries.
|
||||
/// </summary>
|
||||
/// <param name="aggregate">Requested aggregate.</param>
|
||||
/// <param name="plan">The bucket plan.</param>
|
||||
/// <returns>Whether the earliest sample is needed.</returns>
|
||||
public static bool NeedsEarliestSample(HistoryAggregateType aggregate, BucketPlan plan)
|
||||
/// <returns>Whether the earliest and latest samples are needed.</returns>
|
||||
public static bool NeedsDataRange(HistoryAggregateType aggregate, BucketPlan plan)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(plan);
|
||||
return !plan.Reversed && plan.BucketCount > 0 && SetsStartOfDataPartial(aggregate);
|
||||
return plan.BucketCount > 0 && (aggregate == HistoryAggregateType.Count || SetsStartOfDataPartial(aggregate));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Evaluates every non-empty bucket row and assembles one result per interval in the plan's order (descending for a
|
||||
/// reverse read): each value stamped with <see cref="BucketPlan.BucketTimestamp"/> (the earlier edge forward, the
|
||||
/// later edge reverse); empty buckets <c>BadNoData</c>; and — on a forward read, for every aggregate but
|
||||
/// <c>Average</c> — the Partial bit on a non-Bad result of the bucket the tag's data starts inside (#677, the SDK's
|
||||
/// <c>m_startOfData</c> rule; the SDK applies it to forward reads only, <see cref="NeedsEarliestSample"/>).
|
||||
/// later edge reverse) — Start/End/ActualTime with their sample's own time; empty buckets <c>BadNoData</c>, except a
|
||||
/// Count inside the tag's data range (0, Good — #680); and, for every aggregate but <c>Average</c>, the Partial bit
|
||||
/// on a non-Bad result of the bucket the tag's data starts strictly inside (forward only — #677, the SDK's
|
||||
/// <c>m_startOfData</c> rule) or ends inside (#680 — <see cref="BucketPlan.DataEndsInside"/>: forward the SDK's
|
||||
/// <c>m_endOfData</c>, earlier edge included; reverse strictly inside, the SDK's <c>m_startOfData</c> keyed on the
|
||||
/// first value a reverse read is fed — the latest). A Bad result never carries Partial.
|
||||
/// </summary>
|
||||
/// <param name="aggregate">A per-sample (Simple / StartEnd / ActualTime) aggregate.</param>
|
||||
/// <param name="plan">The bucket plan.</param>
|
||||
/// <param name="rows">The backend's non-empty bucket rows (ascending bucket index).</param>
|
||||
/// <param name="earliestSampleUtc">The tag's earliest stored sample of any quality, or null when none.</param>
|
||||
/// <param name="latestSampleUtc">The tag's latest stored sample of any quality, or null when none.</param>
|
||||
/// <param name="now">Server timestamp.</param>
|
||||
/// <returns><see cref="BucketPlan.BucketCount"/> results.</returns>
|
||||
public static IReadOnlyList<DataValueSnapshot> EvaluateAll(HistoryAggregateType aggregate, BucketPlan plan,
|
||||
IEnumerable<BucketRow> rows, DateTime? earliestSampleUtc, DateTime now)
|
||||
IEnumerable<BucketRow> rows, DateTime? earliestSampleUtc, DateTime? latestSampleUtc, DateTime now)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(plan);
|
||||
ArgumentNullException.ThrowIfNull(rows);
|
||||
var evaluated = new Dictionary<long, DataValueSnapshot>();
|
||||
var partialRule = NeedsEarliestSample(aggregate, plan);
|
||||
var startRule = !plan.Reversed && SetsStartOfDataPartial(aggregate);
|
||||
var endRule = SetsStartOfDataPartial(aggregate);
|
||||
foreach (var row in rows)
|
||||
{
|
||||
var v = Evaluate(aggregate, row, plan.BucketTimestamp(row.Bucket), now);
|
||||
if (partialRule && !StatusBits.IsBad(v.StatusCode) && plan.DataStartsInside(row.Bucket, earliestSampleUtc))
|
||||
v = v with { StatusCode = v.StatusCode | HistorianInfoBits.HistorianPartial };
|
||||
if (!StatusBits.IsBad(v.StatusCode)
|
||||
&& ((startRule && plan.DataStartsInside(row.Bucket, earliestSampleUtc))
|
||||
|| (endRule && plan.DataEndsInside(row.Bucket, latestSampleUtc))))
|
||||
// A raw Start/End sample gains the DataValue InfoType with its first aggregate bit (as the SDK's
|
||||
// SetAggregateBits does); a calculated status already carries it.
|
||||
v = v with { StatusCode = v.StatusCode | HistorianInfoBits.InfoTypeDataValue | HistorianInfoBits.HistorianPartial };
|
||||
evaluated[row.Bucket] = v;
|
||||
}
|
||||
if (aggregate == HistoryAggregateType.Count)
|
||||
{
|
||||
// #680 (Part 13 §5.4.3.21; SDK GetValues): an empty interval inside [earliest, latest] is a Count of 0, Good.
|
||||
for (long i = 0; i < plan.BucketCount; i++)
|
||||
if (!evaluated.ContainsKey(i) && plan.WithinDataRange(i, earliestSampleUtc, latestSampleUtc))
|
||||
evaluated[i] = new DataValueSnapshot(0, HistorianInfoBits.WithCalculated(HistorianStatusCodes.Good),
|
||||
plan.BucketTimestamp(i), now);
|
||||
}
|
||||
return plan.Assemble(evaluated, t => NoData(t, now));
|
||||
}
|
||||
|
||||
/// <summary>A raw stored sample as an aggregate result (Start / End): value null when Bad, its own status and time.</summary>
|
||||
private static DataValueSnapshot Raw(StoredSample? sample, DateTime bucketStart, DateTime now) =>
|
||||
sample is not { } s
|
||||
? NoData(bucketStart, now)
|
||||
: new(StatusBits.IsBad(s.Status) ? null : s.Value.ToObject(), s.Status, s.TimeUtc, now);
|
||||
|
||||
/// <summary>The result for a bucket with no samples (or no usable sample): <c>BadNoData</c> + Calculated.</summary>
|
||||
/// <param name="t">The bucket's start time.</param>
|
||||
/// <param name="now">Server timestamp.</param>
|
||||
|
||||
@@ -5,7 +5,7 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common;
|
||||
/// <summary>The query shape that serves an aggregate. One query per family.</summary>
|
||||
public enum AggregateFamily
|
||||
{
|
||||
/// <summary>Average / Minimum / Maximum / Count / Range / PercentGood / PercentBad: one GROUP BY.</summary>
|
||||
/// <summary>Average / Minimum / Maximum / Count / Range: one GROUP BY.</summary>
|
||||
Simple,
|
||||
|
||||
/// <summary>Start / End / Delta: first + last usable sample per bucket.</summary>
|
||||
@@ -15,7 +15,8 @@ public enum AggregateFamily
|
||||
ActualTime,
|
||||
|
||||
/// <summary>
|
||||
/// TimeAverage / TimeAverage2 / Total / Total2 / Interpolative: Part 13 time-weighted, with bounding values.
|
||||
/// TimeAverage / TimeAverage2 / Total / Total2 / Interpolative, and the duration-based PercentGood / PercentBad
|
||||
/// (#680): Part 13 time-weighted, with bounding values.
|
||||
/// The backend streams the window's samples plus the bounding rows and <see cref="TimeWeightedAggregator"/>
|
||||
/// evaluates them (identically for every backend).
|
||||
/// </summary>
|
||||
@@ -31,12 +32,12 @@ public static class AggregateFamilies
|
||||
public static AggregateFamily? FamilyOf(HistoryAggregateType aggregate) => aggregate switch
|
||||
{
|
||||
HistoryAggregateType.Average or HistoryAggregateType.Minimum or HistoryAggregateType.Maximum
|
||||
or HistoryAggregateType.Count or HistoryAggregateType.Range
|
||||
or HistoryAggregateType.PercentGood or HistoryAggregateType.PercentBad => AggregateFamily.Simple,
|
||||
or HistoryAggregateType.Count or HistoryAggregateType.Range => AggregateFamily.Simple,
|
||||
HistoryAggregateType.Start or HistoryAggregateType.End or HistoryAggregateType.Delta => AggregateFamily.StartEnd,
|
||||
HistoryAggregateType.MinimumActualTime or HistoryAggregateType.MaximumActualTime => AggregateFamily.ActualTime,
|
||||
HistoryAggregateType.TimeAverage or HistoryAggregateType.TimeAverage2 or HistoryAggregateType.Total
|
||||
or HistoryAggregateType.Total2 or HistoryAggregateType.Interpolative => AggregateFamily.TimeWeighted,
|
||||
or HistoryAggregateType.Total2 or HistoryAggregateType.Interpolative
|
||||
or HistoryAggregateType.PercentGood or HistoryAggregateType.PercentBad => AggregateFamily.TimeWeighted,
|
||||
_ => null,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -155,6 +155,33 @@ public sealed class BucketPlan
|
||||
public bool DataStartsInside(long index, DateTime? earliestSampleUtc) =>
|
||||
earliestSampleUtc is { } e && e.Ticks > LowerTicks(index) && e.Ticks < UpperTicks(index);
|
||||
|
||||
/// <summary>
|
||||
/// True when the latest stored sample of the tag lies in bucket <paramref name="index"/>: the data ends part-way
|
||||
/// through the interval, so Part 13 §5.3.3.2 / the SDK set the Partial bit (#680). Forward the earlier edge is
|
||||
/// included (the SDK's <c>m_endOfData >= startTime && m_endOfData < endTime</c>); reverse the sample
|
||||
/// must lie strictly inside (the SDK's reverse read keys <c>m_startOfData</c> — a strict test — on the first value it
|
||||
/// is fed, the latest).
|
||||
/// </summary>
|
||||
/// <param name="index">Zero-based ascending bucket index.</param>
|
||||
/// <param name="latestSampleUtc">The tag's latest stored sample (any quality), or null when it has none.</param>
|
||||
/// <returns>Whether the data ends inside the bucket.</returns>
|
||||
public bool DataEndsInside(long index, DateTime? latestSampleUtc) =>
|
||||
latestSampleUtc is { } l
|
||||
&& (Reversed ? l.Ticks > LowerTicks(index) : l.Ticks >= LowerTicks(index)) && l.Ticks < UpperTicks(index);
|
||||
|
||||
/// <summary>
|
||||
/// True when bucket <paramref name="index"/> overlaps the tag's data range <c>[earliest, latest]</c> — neither
|
||||
/// wholly before the first sample nor wholly after the last (the SDK's <c>GetValues</c> null test:
|
||||
/// <c>start > last || end < first</c> ⇒ no data, both edges inclusive in either direction). An empty bucket
|
||||
/// inside the range is a Count of 0, not <c>BadNoData</c> (#680, Part 13 §5.4.3.21).
|
||||
/// </summary>
|
||||
/// <param name="index">Zero-based ascending bucket index.</param>
|
||||
/// <param name="earliestSampleUtc">The tag's earliest stored sample (any quality), or null.</param>
|
||||
/// <param name="latestSampleUtc">The tag's latest stored sample (any quality), or null.</param>
|
||||
/// <returns>Whether the bucket is within the data range.</returns>
|
||||
public bool WithinDataRange(long index, DateTime? earliestSampleUtc, DateTime? latestSampleUtc) =>
|
||||
earliestSampleUtc is { } e && latestSampleUtc is { } l && LowerTicks(index) <= l.Ticks && UpperTicks(index) >= e.Ticks;
|
||||
|
||||
/// <summary>One snapshot per bucket in result order; missing buckets get <paramref name="empty"/>; descending if reversed.</summary>
|
||||
/// <param name="byBucket">Evaluated snapshots of the non-empty buckets, by ascending bucket index.</param>
|
||||
/// <param name="empty">Builds the snapshot of a bucket with no row, from its <see cref="BucketTimestamp"/>.</param>
|
||||
|
||||
@@ -9,7 +9,7 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common;
|
||||
/// <remarks>
|
||||
/// Sample classes: <i>usable</i> = severity Good or Uncertain AND a stored value (an Empty sample is an outage marker
|
||||
/// and counts as Bad whatever its code); <i>good</i> = usable with severity Good; <i>bad</i> = not usable.
|
||||
/// Every value field is computed over usable samples only.
|
||||
/// Every value field is computed over usable samples only, except <see cref="FirstAny"/> / <see cref="LastAny"/>.
|
||||
/// </remarks>
|
||||
public sealed class BucketRow
|
||||
{
|
||||
@@ -54,4 +54,13 @@ public sealed class BucketRow
|
||||
|
||||
/// <summary>Time of <see cref="Extreme"/> (ties ⇒ earliest; null when none).</summary>
|
||||
public DateTime? ExtremeTime { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// The bucket's first sample by time of <b>any</b> quality (null when the bucket is empty): Start returns it raw, and
|
||||
/// Delta is Uncertain when it is not usable (#680, Part 13 Tables 73/74 — the SDK's <c>StartEndAggregateCalculator</c>).
|
||||
/// </summary>
|
||||
public StoredSample? FirstAny { get; set; }
|
||||
|
||||
/// <summary>The bucket's last sample by time of any quality (null when the bucket is empty): End, Delta (#680).</summary>
|
||||
public StoredSample? LastAny { get; set; }
|
||||
}
|
||||
|
||||
+204
-42
@@ -4,8 +4,8 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common;
|
||||
|
||||
/// <summary>
|
||||
/// The OPC UA Part 13 time-weighted aggregates (<see cref="AggregateFamily.TimeWeighted"/>): <c>TimeAverage</c>,
|
||||
/// <c>TimeAverage2</c>, <c>Total</c>, <c>Total2</c> and <c>Interpolative</c>, evaluated over a time-ordered stream of
|
||||
/// stored samples. Both SQL backends fetch the same rows and feed them here, so the two produce identical results
|
||||
/// <c>TimeAverage2</c>, <c>Total</c>, <c>Total2</c>, <c>Interpolative</c> and the duration-based <c>PercentGood</c> /
|
||||
/// <c>PercentBad</c> (#680), evaluated over a time-ordered stream of stored samples. Both SQL backends fetch the same rows and feed them here, so the two produce identical results
|
||||
/// by construction.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
@@ -24,10 +24,12 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common;
|
||||
/// <item><c>s</c> usable, both Double/Int64 ⇒ <b>sloped</b> (linear <c>p → s</c>); quality = worse of the two.</item>
|
||||
/// <item><c>s</c> usable, either Boolean ⇒ <b>stepped</b> (<c>p</c> held); quality = <c>p</c>'s.</item>
|
||||
/// <item><c>s</c> Bad ⇒ <c>p</c> held up to the outage (never interpolated into it, as AtTime does); Uncertain.</item>
|
||||
/// <item>no later sample at all ⇒ <c>p</c> held to the window end (stepped extrapolation); Uncertain.</item>
|
||||
/// <item>no later sample at all (nothing at or after the window end) ⇒ the data has ENDED at <c>p</c>: no
|
||||
/// extrapolation (Part 13 "After End of Data", #680).</item>
|
||||
/// </list>
|
||||
/// From a Bad sample (incl. an <c>Empty</c> outage marker) to the next sample, and before the first sample, the
|
||||
/// signal is Bad (no data). This is the <i>simple bounding</i> signal.
|
||||
/// From a Bad sample (incl. an <c>Empty</c> outage marker) to the next sample the signal is Bad (outage) — to the
|
||||
/// window end when nothing follows it; before the first sample there is no data. This is the <i>simple bounding</i>
|
||||
/// signal.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// The <i>interpolated bounding</i> signal (Part 13 "interpolated bounding values") skips Bad samples: it joins
|
||||
@@ -36,32 +38,58 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common;
|
||||
/// </para>
|
||||
/// <list type="table">
|
||||
/// <item><term>TimeAverage</term><description>∫ interpolated-bounding signal ÷ its defined duration in the interval.</description></item>
|
||||
/// <item><term>Total</term><description><c>TimeAverage × interval seconds</c> (value·s) — Part 13 §5.4.3.8.</description></item>
|
||||
/// <item><term>Total</term><description>∫ interpolated-bounding signal over its defined part, in value·s — Part 13
|
||||
/// §5.4.3.8 (= <c>TimeAverage × interval seconds</c> on a covered interval; only the covered part when the data
|
||||
/// starts inside it, #680).</description></item>
|
||||
/// <item><term>TimeAverage2</term><description>∫ simple-bounding signal over its non-Bad parts ÷ non-Bad duration (Bad intervals excluded).</description></item>
|
||||
/// <item><term>Total2</term><description>∫ simple-bounding signal over its non-Bad parts, in value·s (= TimeAverage2 × non-Bad seconds).</description></item>
|
||||
/// <item><term>PercentGood / PercentBad</term><description><c>100 ×</c> Good / Bad region time ÷ interval duration
|
||||
/// (Part 13 §5.4.3.31-34): stepped regions, each with the status of the sample that starts it; the region before an
|
||||
/// interval's first own sample has the simple bounding value's status (Bad after a Bad sample; Uncertain on a
|
||||
/// segment running into an outage or sloping to/from an Uncertain sample; else the earlier sample's — the SDK's
|
||||
/// <c>GetSimpleBound</c>); time before the first sample is Bad; at the end of the data the last region runs to the
|
||||
/// end of its own interval (Part 13; the SDK gives it 1 ms) and later intervals are <c>BadNoData</c>; an interval
|
||||
/// ending before the first sample ⇒ <c>BadNoData</c>. Status Good; any value kind.</description></item>
|
||||
/// <item><term>Interpolative</term><description>The interpolated-bounding signal's value at each interval start —
|
||||
/// for a reverse read, at <c>End − i·interval</c> (Part 13 anchors a reverse read at its later time), emitted
|
||||
/// latest first.</description></item>
|
||||
/// </list>
|
||||
/// <para>
|
||||
/// <b>Status (time-weighted four)</b> — the SDK <c>AggregateCalculator.GetTimeBasedStatusCode</c> rule with
|
||||
/// TreatUncertainAsBad = false, so Uncertain time (Uncertain samples, a value held up to an outage, stepped
|
||||
/// extrapolation past the last sample) counts as Good time: every instant of the interval non-Bad ⇒
|
||||
/// <c>Good</c>; no non-Bad time but some Bad (outage) time ⇒ <c>Bad</c>; no data at all ⇒ <c>BadNoData</c>;
|
||||
/// otherwise (PercentDataGood = PercentDataBad = 100) <c>UncertainDataSubNormal</c>. Time before the first sample
|
||||
/// counts as Bad (no data) time. A usable String/DateTime sample contributing to the interval ⇒
|
||||
/// <c>BadAggregateInvalidInputs</c>. Every status carries the Calculated bit, plus the Partial bit on a non-Bad
|
||||
/// result of the truncated interval or of the interval the data starts inside (the first sample fed lies strictly
|
||||
/// within it — forward reads only, as in the SDK, #677); the timestamp is the interval's <see cref="BucketPlan.BucketTimestamp"/> (its start forward, its
|
||||
/// later edge reverse); the value is a
|
||||
/// <see cref="double"/> (Boolean = 0/1, Int64 as double).
|
||||
/// <b>Status (TimeAverage / Total, #680)</b> — Part 13 ("Uncertain if Bad values are skipped or Uncertain values are
|
||||
/// used") and the SDK's <c>AverageAggregateCalculator</c>: no defined value in the interval ⇒ <c>BadNoData</c>; any
|
||||
/// part of it on a bridged segment with an Uncertain end (or Uncertain start when stepped), bridged across a Bad
|
||||
/// sample, extrapolated past the last usable sample, or not covered at all ⇒ <c>UncertainDataSubNormal</c> (so the
|
||||
/// latest interval of a live tag is Uncertain, and an interval wholly inside an outage has the bridged value,
|
||||
/// Uncertain); else <c>Good</c>.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Status (TimeAverage2 / Total2)</b> — the SDK <c>AggregateCalculator.GetTimeBasedStatusCode</c> rule with
|
||||
/// TreatUncertainAsBad = false, so Uncertain time (Uncertain samples, a value held up to an outage) counts as Good
|
||||
/// time, over the interval cut at the end of the data: after the end of the data ⇒ <c>BadNoData</c>; the data
|
||||
/// ending exactly on the interval start ⇒ the last sample's value (Total2 0) with its status; every covered instant
|
||||
/// non-Bad ⇒ <c>Good</c>; no non-Bad time but some Bad (outage) time ⇒ <c>Bad</c> (Part 13's PercentDataBad rule;
|
||||
/// the SDK answers <c>BadNoData</c>); no data at all ⇒ <c>BadNoData</c>; otherwise (PercentDataGood = PercentDataBad
|
||||
/// = 100) <c>UncertainDataSubNormal</c>. Time before the first sample counts as Bad (no data) time.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// For the four, a usable String/DateTime sample contributing to the interval ⇒ <c>BadAggregateInvalidInputs</c>.
|
||||
/// Every status of this family but Interpolative's carries the Calculated bit, plus — on a non-Bad result only — the
|
||||
/// Partial bit on: the truncated interval (Part 13 §5.3.3.2 Case 3; the SDK does not set it — deliberate); the
|
||||
/// interval the data starts inside in the read direction (forward: the first sample fed lies strictly within it,
|
||||
/// #677; reverse: the latest sample fed lies strictly within it, nothing after — the SDK's <c>m_startOfData</c>,
|
||||
/// keyed on the first value it is fed, #680); and, forward, the end-of-data interval of TimeAverage2 / Total2 /
|
||||
/// PercentGood / PercentBad (the SDK's <c>m_endOfData</c>, which it skips for an extrapolated TimeAverage / Total).
|
||||
/// The timestamp is the interval's <see cref="BucketPlan.BucketTimestamp"/> (its start forward, its later edge
|
||||
/// reverse); the value is a <see cref="double"/> (Boolean = 0/1, Int64 as double).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Status (Interpolative).</b> A usable sample exactly at the point ⇒ its value with its own status
|
||||
/// (raw, no historian bit). Otherwise no usable sample at or before ⇒ <c>BadNoData</c>; else the interpolated
|
||||
/// value with the Interpolated bit and the status of the simple-bounding signal at that instant: <c>Good</c> on
|
||||
/// a Good segment, <c>UncertainDataSubNormal</c> on an Uncertain one, inside an outage (interpolated across it)
|
||||
/// or past the last sample (extrapolated).
|
||||
/// or past the last sample (extrapolated). A reverse read returns the raw sample at a point exactly on a sample
|
||||
/// too — the SDK's reverse answer (Interpolated, from the segment beyond) differs from its own forward one; that
|
||||
/// asymmetry is the SDK's defect, deliberately not copied.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public sealed class TimeWeightedAggregator
|
||||
@@ -108,13 +136,19 @@ public sealed class TimeWeightedAggregator
|
||||
// Interpolated-bounding (bridged) signal, per bucket.
|
||||
private readonly double[] _bridgedArea;
|
||||
private readonly long[] _bridgedTicks;
|
||||
// A bridged segment touching the bucket was non-Good: an Uncertain end, a Bad sample skipped, or extrapolated (#680).
|
||||
private readonly bool[] _bridgedNonGood;
|
||||
private readonly bool[] _invalid;
|
||||
// PercentGood / PercentBad (#680): stepped status regions, per bucket.
|
||||
private readonly long[] _goodRegionTicks;
|
||||
private readonly long[] _badRegionTicks;
|
||||
// Interpolative, per point (index = ascending time; see PointTicks).
|
||||
private readonly DataValueSnapshot?[] _points;
|
||||
|
||||
private StoredSample? _prevAny;
|
||||
private StoredSample? _prevUsable;
|
||||
private DateTime? _firstSampleUtc; // the earliest sample fed (any quality): where the data starts
|
||||
private long? _dataEndTicks; // the last sample fed, when usable and nothing at/after the end followed
|
||||
private long _nextPoint; // next point (ascending time) not yet classified
|
||||
private readonly List<PendingPoint> _pending = []; // points awaiting the next usable sample
|
||||
private bool _finished;
|
||||
@@ -146,7 +180,10 @@ public sealed class TimeWeightedAggregator
|
||||
_badTicks = new long[n];
|
||||
_bridgedArea = new double[n];
|
||||
_bridgedTicks = new long[n];
|
||||
_bridgedNonGood = new bool[n];
|
||||
_invalid = new bool[n];
|
||||
_goodRegionTicks = new long[n];
|
||||
_badRegionTicks = new long[n];
|
||||
_points = new DataValueSnapshot?[n];
|
||||
}
|
||||
|
||||
@@ -172,7 +209,11 @@ public sealed class TimeWeightedAggregator
|
||||
if (_finished) throw new InvalidOperationException("The aggregation is already finished.");
|
||||
var t = s.TimeUtc.Ticks;
|
||||
if (_prevAny is { } last && t <= last.TimeUtc.Ticks) return;
|
||||
_firstSampleUtc ??= s.TimeUtc;
|
||||
if (_firstSampleUtc is null)
|
||||
{
|
||||
_firstSampleUtc = s.TimeUtc;
|
||||
AddRegion(_badRegionTicks, _start, t); // PercentBad: the time before the first sample is Bad (no data)
|
||||
}
|
||||
|
||||
// Interpolative: classify every bucket start before this sample by the simple-bounding signal there.
|
||||
ClassifyPointsBefore(t);
|
||||
@@ -188,6 +229,7 @@ public sealed class TimeWeightedAggregator
|
||||
{
|
||||
AddBad(bad, t);
|
||||
}
|
||||
if (_prevAny is { } r) AddStatusRegion(r, s, t);
|
||||
|
||||
if (!s.IsUsable) DegradePending(); // the segment the pending points sit on runs into an outage
|
||||
|
||||
@@ -196,7 +238,10 @@ public sealed class TimeWeightedAggregator
|
||||
if (_prevUsable is { } pu)
|
||||
{
|
||||
var sloped = IsSloped(pu, s);
|
||||
AddBridged(pu, t, sloped ? s.Value.Number : pu.Value.Number);
|
||||
// Part 13: Uncertain when a Bad value was skipped or an Uncertain value is used (the end sample only
|
||||
// contributes its value to a sloped segment).
|
||||
var nonGood = !pu.IsGood || (sloped && !s.IsGood) || _prevAny is { IsUsable: false };
|
||||
AddBridged(pu, t, sloped ? s.Value.Number : pu.Value.Number, nonGood);
|
||||
}
|
||||
ResolvePending(_prevUsable, s);
|
||||
_prevUsable = s;
|
||||
@@ -215,11 +260,21 @@ public sealed class TimeWeightedAggregator
|
||||
_finished = true;
|
||||
ClassifyPointsBefore(long.MaxValue);
|
||||
if (_prevAny is { IsUsable: true } p && p.TimeUtc.Ticks < _end)
|
||||
AddRaw(p, _end, p.Value.Number); // stepped extrapolation
|
||||
{
|
||||
// No sample at or after the window end: the data ENDS here (Part 13 "After End of Data"). The simple-
|
||||
// bounding signal is not extrapolated (TimeAverage2/Total2 cover up to it; later intervals BadNoData), and
|
||||
// the status region it starts runs to the end of its own interval only (PercentGood/PercentBad).
|
||||
_dataEndTicks = p.TimeUtc.Ticks;
|
||||
if (p.TimeUtc.Ticks >= _start && _plan.BucketCount > 0)
|
||||
AddRegion(RegionTicks(p), p.TimeUtc.Ticks, BucketEnd(FirstBucketAt(p.TimeUtc.Ticks)));
|
||||
}
|
||||
else if (_prevAny is { } bad && bad.TimeUtc.Ticks < _end)
|
||||
AddBad(bad, _end); // the outage runs to the window end
|
||||
{
|
||||
AddBad(bad, _end); // an outage marker last: the outage runs to the window end
|
||||
AddRegion(_badRegionTicks, bad.TimeUtc.Ticks, _end);
|
||||
}
|
||||
if (_prevUsable is { } pu && pu.TimeUtc.Ticks < _end)
|
||||
AddBridged(pu, _end, pu.Value.Number);
|
||||
AddBridged(pu, _end, pu.Value.Number, nonGood: true); // stepped extrapolation ⇒ Uncertain
|
||||
ResolvePending(_prevUsable, next: null);
|
||||
}
|
||||
|
||||
@@ -239,30 +294,82 @@ public sealed class TimeWeightedAggregator
|
||||
: new DataValueSnapshot(null, HistorianStatusCodes.BadNoData, PointTime(i), now);
|
||||
|
||||
var stamp = _plan.BucketTimestamp(i);
|
||||
var duration = BucketEnd(i) - BucketStartTicks(i);
|
||||
// Partial: the truncated interval (the latest forward, the earliest reverse), or the one the data starts inside
|
||||
// (no bounding value before it — the SDK's m_startOfData rule, #677), which the SDK applies to forward reads only.
|
||||
var partial = _plan.IsTruncated(i) || (!_plan.Reversed && _plan.DataStartsInside(i, _firstSampleUtc));
|
||||
DataValueSnapshot Calc(object? value, uint status)
|
||||
var lo = BucketStartTicks(i);
|
||||
var hi = BucketEnd(i);
|
||||
var duration = hi - lo;
|
||||
// Partial: the truncated interval (the latest forward, the earliest reverse — Part 13 §5.3.3.2 Case 3, which the
|
||||
// SDK does not implement), or the one the data starts inside IN THE READ DIRECTION (no bounding value on that
|
||||
// side — the SDK's m_startOfData rule, #677): forward, the earliest sample strictly inside; reverse, the latest
|
||||
// sample strictly inside (nothing after it — the SDK keys m_startOfData on the first value it is fed, #680).
|
||||
// Forward, the end-of-data interval (the latest sample inside it, nothing after) is Partial too — never on an
|
||||
// extrapolated TimeAverage/Total (the SDK's m_endOfData rule, #680). Never on a Bad result.
|
||||
var latestInside = _plan.Reversed && _prevAny is { } latest
|
||||
&& latest.TimeUtc.Ticks > lo && latest.TimeUtc.Ticks < hi;
|
||||
var partial = _plan.IsTruncated(i) || latestInside
|
||||
|| (!_plan.Reversed && _plan.DataStartsInside(i, _firstSampleUtc));
|
||||
var endOfData = !_plan.Reversed && _dataEndTicks is { } de && de >= lo && de < hi;
|
||||
DataValueSnapshot Calc(object? value, uint status, bool endPartial = false)
|
||||
{
|
||||
var code = HistorianInfoBits.WithCalculated(status);
|
||||
if (partial && !StatusBits.IsBad(status)) code |= HistorianInfoBits.HistorianPartial;
|
||||
if ((partial || endPartial) && !StatusBits.IsBad(status)) code |= HistorianInfoBits.HistorianPartial;
|
||||
return new DataValueSnapshot(value, code, stamp, now);
|
||||
}
|
||||
|
||||
if (_invalid[i]) return Calc(null, HistorianStatusCodes.BadAggregateInvalidInputs);
|
||||
if (_usableTicks[i] == 0)
|
||||
return _badTicks[i] > 0 ? Calc(null, HistorianStatusCodes.Bad) : AggregateEvaluator.NoData(stamp, now);
|
||||
|
||||
var status = _usableTicks[i] == duration ? HistorianStatusCodes.Good : HistorianStatusCodes.UncertainDataSubNormal;
|
||||
return _aggregate switch
|
||||
switch (_aggregate)
|
||||
{
|
||||
HistoryAggregateType.TimeAverage => Calc(_bridgedArea[i] / _bridgedTicks[i], status),
|
||||
HistoryAggregateType.Total => Calc(_bridgedArea[i] / _bridgedTicks[i] * (duration / TicksPerSecond), status),
|
||||
HistoryAggregateType.TimeAverage2 => Calc(_rawArea[i] / _usableTicks[i], status),
|
||||
HistoryAggregateType.Total2 => Calc(_rawArea[i] / TicksPerSecond, status),
|
||||
_ => throw new HistoryAggregateNotSupportedException(_aggregate),
|
||||
};
|
||||
case HistoryAggregateType.TimeAverage or HistoryAggregateType.Total:
|
||||
{
|
||||
// Interpolated bounding: Uncertain when any part of the interval was bridged across a Bad value, used an
|
||||
// Uncertain value, was extrapolated, or is not covered at all (Part 13; the SDK's AverageAggregateCalculator).
|
||||
if (_invalid[i]) return Calc(null, HistorianStatusCodes.BadAggregateInvalidInputs);
|
||||
if (_bridgedTicks[i] == 0) return AggregateEvaluator.NoData(stamp, now);
|
||||
var status = _bridgedNonGood[i] || _bridgedTicks[i] < duration
|
||||
? HistorianStatusCodes.UncertainDataSubNormal
|
||||
: HistorianStatusCodes.Good;
|
||||
return _aggregate == HistoryAggregateType.TimeAverage
|
||||
? Calc(_bridgedArea[i] / _bridgedTicks[i], status)
|
||||
: Calc(_bridgedArea[i] / TicksPerSecond, status); // ∫ over the covered part only (Part 13 partial width)
|
||||
}
|
||||
case HistoryAggregateType.TimeAverage2 or HistoryAggregateType.Total2:
|
||||
{
|
||||
if (_invalid[i]) return Calc(null, HistorianStatusCodes.BadAggregateInvalidInputs);
|
||||
if (_dataEndTicks is { } dataEnd)
|
||||
{
|
||||
// After the end of the data: nothing to aggregate (Part 13 "After End of Data: Bad_NoData").
|
||||
if (lo > dataEnd) return AggregateEvaluator.NoData(stamp, now);
|
||||
if (_usableTicks[i] == 0 && dataEnd < hi && _prevAny is { } last)
|
||||
{
|
||||
// The data ends inside this interval and no non-Bad time precedes its last sample here (the data
|
||||
// ends on the interval's start; or the last sample is the first usable one after an outage — the
|
||||
// comms-loss recovery shape — or the tag's only sample): nothing covered, so the value at that
|
||||
// instant (Total2 0). Good only when the data ends exactly on the interval start with a Good
|
||||
// sample (nothing before it in the interval); otherwise the interval also holds Bad / no-data time
|
||||
// ⇒ Uncertain (the SDK answers the same, modulo its 1 ms artifact).
|
||||
if (!IsNumeric(last)) return Calc(null, HistorianStatusCodes.BadAggregateInvalidInputs);
|
||||
var st = lo == dataEnd && last.IsGood ? HistorianStatusCodes.Good : HistorianStatusCodes.UncertainDataSubNormal;
|
||||
return Calc(_aggregate == HistoryAggregateType.TimeAverage2 ? last.Value.Number : 0d, st, endOfData);
|
||||
}
|
||||
duration = Math.Min(hi, dataEnd) - lo;
|
||||
}
|
||||
if (_usableTicks[i] == 0)
|
||||
return _badTicks[i] > 0 ? Calc(null, HistorianStatusCodes.Bad) : AggregateEvaluator.NoData(stamp, now);
|
||||
var status = _usableTicks[i] == duration ? HistorianStatusCodes.Good : HistorianStatusCodes.UncertainDataSubNormal;
|
||||
return _aggregate == HistoryAggregateType.TimeAverage2
|
||||
? Calc(_rawArea[i] / _usableTicks[i], status, endOfData)
|
||||
: Calc(_rawArea[i] / TicksPerSecond, status, endOfData);
|
||||
}
|
||||
case HistoryAggregateType.PercentGood or HistoryAggregateType.PercentBad:
|
||||
{
|
||||
// No data in or around the interval (it ends before the first sample — the SDK's out-of-range test — or
|
||||
// starts after the end of the data) ⇒ BadNoData; otherwise the share of the interval's duration.
|
||||
if (_firstSampleUtc is not { } first || hi < first.Ticks || (_dataEndTicks is { } de2 && lo > de2))
|
||||
return AggregateEvaluator.NoData(stamp, now);
|
||||
var ticks = _aggregate == HistoryAggregateType.PercentGood ? _goodRegionTicks[i] : _badRegionTicks[i];
|
||||
return Calc(100d * ticks / duration, HistorianStatusCodes.Good, endOfData);
|
||||
}
|
||||
default:
|
||||
throw new HistoryAggregateNotSupportedException(_aggregate);
|
||||
}
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------------------------------ Segments
|
||||
@@ -305,13 +412,68 @@ public sealed class TimeWeightedAggregator
|
||||
}
|
||||
}
|
||||
|
||||
private void AddBridged(StoredSample from, long toTicks, double toValue) =>
|
||||
private void AddBridged(StoredSample from, long toTicks, double toValue, bool nonGood) =>
|
||||
Distribute(from, toTicks, toValue, (i, area, ticks) =>
|
||||
{
|
||||
_bridgedArea[i] += area;
|
||||
_bridgedTicks[i] += ticks;
|
||||
if (nonGood) _bridgedNonGood[i] = true;
|
||||
});
|
||||
|
||||
// ------------------------------------------------------------------------------------------ PercentGood / PercentBad
|
||||
|
||||
/// <summary>The region array a stepped region of <paramref name="s"/>'s status counts toward (null: Uncertain, neither).</summary>
|
||||
private long[]? RegionTicks(StoredSample s) =>
|
||||
s.IsGood ? _goodRegionTicks : !s.IsUsable ? _badRegionTicks : null;
|
||||
|
||||
/// <summary>
|
||||
/// The status regions between sample <paramref name="p"/> and the next sample <paramref name="s"/> (at
|
||||
/// <paramref name="t"/>). Inside the interval <paramref name="p"/> lies in, the region has <paramref name="p"/>'s
|
||||
/// status (stepped, Part 13 §5.4.3.31-34). In a later interval the region starts at that interval's simple bounding
|
||||
/// value, whose status is the simple-bounding signal's: Bad after a Bad <paramref name="p"/>; Uncertain when the
|
||||
/// segment runs into an outage or a sloped one has a non-Good end; else <paramref name="p"/>'s (the SDK's
|
||||
/// <c>GetSimpleBound</c>).
|
||||
/// </summary>
|
||||
private void AddStatusRegion(StoredSample p, StoredSample s, long t)
|
||||
{
|
||||
var lo = Math.Max(p.TimeUtc.Ticks, _start);
|
||||
var hi = Math.Min(t, _end);
|
||||
if (lo >= hi || _plan.BucketCount == 0) return;
|
||||
long[]? bound;
|
||||
if (!p.IsUsable) bound = _badRegionTicks;
|
||||
else if (!s.IsUsable) bound = null; // held into an outage ⇒ Uncertain
|
||||
else if (IsSloped(p, s)) bound = p.IsGood && s.IsGood ? _goodRegionTicks : null;
|
||||
else bound = RegionTicks(p);
|
||||
var own = RegionTicks(p);
|
||||
for (var i = FirstBucketAt(lo); i < _plan.BucketCount; i++)
|
||||
{
|
||||
var bs = BucketStartTicks(i);
|
||||
if (bs >= hi) break;
|
||||
var a = Math.Max(lo, bs);
|
||||
var b = Math.Min(hi, BucketEnd(i));
|
||||
if (a >= b) continue;
|
||||
var target = p.TimeUtc.Ticks >= bs ? own : bound;
|
||||
if (target is not null) target[i] += b - a;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Adds <c>[from, to)</c>, clipped to the window, to <paramref name="target"/> per bucket.</summary>
|
||||
private void AddRegion(long[]? target, long from, long to)
|
||||
{
|
||||
if (target is null) return;
|
||||
var lo = Math.Max(from, _start);
|
||||
var hi = Math.Min(to, _end);
|
||||
if (lo >= hi || _plan.BucketCount == 0) return;
|
||||
for (var i = FirstBucketAt(lo); i < _plan.BucketCount; i++)
|
||||
{
|
||||
var bs = BucketStartTicks(i);
|
||||
if (bs >= hi) break;
|
||||
var a = Math.Max(lo, bs);
|
||||
var b = Math.Min(hi, BucketEnd(i));
|
||||
if (a < b) target[i] += b - a;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Splits the linear segment <c>[from.t, toTicks)</c> (value <c>from.v → toValue</c>) at the bucket edges, clipped to
|
||||
/// the window, and hands each bucket its trapezoid area (value·ticks) and duration. A non-numeric start sample
|
||||
|
||||
@@ -46,7 +46,7 @@ internal static class SqlAggregateQueries
|
||||
private const string NonNumericKinds = "4, 5";
|
||||
|
||||
/// <summary>
|
||||
/// Average / Minimum / Maximum / Total / Count / Range / PercentGood / PercentBad.
|
||||
/// Average / Minimum / Maximum / Count / Range (PercentGood / PercentBad and Total are time-weighted, not here).
|
||||
/// Family columns: <c>6 AvgValue, 7 MinValue, 8 MaxValue, 9 SumValue</c> (float, NULL when no usable
|
||||
/// numeric sample).
|
||||
/// </summary>
|
||||
@@ -62,30 +62,62 @@ internal static class SqlAggregateQueries
|
||||
"SUM(CASE WHEN u = 1 THEN NumValue END) AS SumValue " +
|
||||
"FROM s GROUP BY b ORDER BY b;";
|
||||
|
||||
/// <summary>Ordinal of the StartEnd family's first-sample-of-any-quality columns (<see cref="StartEnd"/>).</summary>
|
||||
public const int FirstAnyColumn = FirstFamilyColumn + 10;
|
||||
|
||||
/// <summary>
|
||||
/// Start / End / Delta: the first and last usable sample of each bucket by <c>SourceTimeUtc</c>.
|
||||
/// Family columns: <c>6..10</c> first sample (<c>ValueKind, NumValue, IntValue, StrValue, SourceTimeUtc</c>),
|
||||
/// <c>11..15</c> last sample (same order); all NULL when the bucket has no usable sample.
|
||||
/// Start / End / Delta: the first and last usable sample of each bucket by <c>SourceTimeUtc</c> (Delta's value), and
|
||||
/// the first and last sample of <b>any</b> quality (Start / End, and Delta's status — #680).
|
||||
/// Family columns: <c>6..10</c> first usable sample (<c>ValueKind, NumValue, IntValue, StrValue, SourceTimeUtc</c>),
|
||||
/// <c>11..15</c> last usable sample (same order) — all NULL when the bucket has no usable sample; then
|
||||
/// <c>16..21</c> the first sample of any quality (<c>SourceTimeUtc, ValueKind, NumValue, IntValue, StrValue,
|
||||
/// StatusCode</c>) and <c>22..27</c> the last (same order) — present for every returned (non-empty) bucket.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <b>One scan (live-gate 8 F1).</b> The counts and all four picked samples come from one reference to <c>s</c>:
|
||||
/// a windowed pass ranks each row within its bucket (<c>rfa</c>/<c>rla</c> over every sample, <c>rfu</c>/<c>rlu</c>
|
||||
/// within its usable / not-usable partition) and the outer <c>GROUP BY</c> both counts and lifts each picked row's
|
||||
/// columns with a conditional <c>MAX</c> (one row per bucket matches each pick). The previous form referenced the
|
||||
/// CTE five times; SQL Server expands each reference into its own scan, and under READ COMMITTED nothing makes those
|
||||
/// scans see the same rows, so a sample committed between two of them could be counted but not picked (QuestDB hit
|
||||
/// exactly that). It is also fewer scans of the key range.
|
||||
/// </remarks>
|
||||
/// <param name="sample">Schema-qualified <c>Sample</c> table name.</param>
|
||||
/// <returns>The SQL text.</returns>
|
||||
/// <param name="reverse">True for a reverse read (window <c>(@start, @end]</c>).</param>
|
||||
public static string StartEnd(string sample, bool reverse = false) =>
|
||||
Classified(sample, reverse) +
|
||||
", st AS (SELECT b, " + CommonCounts() + " FROM s GROUP BY b)" +
|
||||
", f AS (SELECT b, ValueKind, NumValue, IntValue, StrValue, SourceTimeUtc, " +
|
||||
"ROW_NUMBER() OVER (PARTITION BY b ORDER BY SourceTimeUtc ASC) AS rf, " +
|
||||
"ROW_NUMBER() OVER (PARTITION BY b ORDER BY SourceTimeUtc DESC) AS rl FROM s WHERE u = 1) " +
|
||||
"SELECT st.*, " +
|
||||
"x.ValueKind, x.NumValue, x.IntValue, x.StrValue, x.SourceTimeUtc, " +
|
||||
"y.ValueKind, y.NumValue, y.IntValue, y.StrValue, y.SourceTimeUtc " +
|
||||
"FROM st LEFT JOIN f x ON x.b = st.b AND x.rf = 1 LEFT JOIN f y ON y.b = st.b AND y.rl = 1 ORDER BY st.b;";
|
||||
public static string StartEnd(string sample, bool reverse = false)
|
||||
{
|
||||
static string Pick(string when, bool any) =>
|
||||
(any ? $"MAX(CASE WHEN {when} THEN SourceTimeUtc END), " : "") +
|
||||
$"MAX(CASE WHEN {when} THEN ValueKind END), MAX(CASE WHEN {when} THEN NumValue END), " +
|
||||
$"MAX(CASE WHEN {when} THEN IntValue END), MAX(CASE WHEN {when} THEN StrValue END), " +
|
||||
(any ? $"MAX(CASE WHEN {when} THEN StatusCode END)" : $"MAX(CASE WHEN {when} THEN SourceTimeUtc END)");
|
||||
return Classified(sample, reverse) +
|
||||
", w AS (SELECT b, SourceTimeUtc, ValueKind, NumValue, IntValue, StrValue, StatusCode, u, g, " +
|
||||
"ROW_NUMBER() OVER (PARTITION BY b ORDER BY SourceTimeUtc ASC) AS rfa, " +
|
||||
"ROW_NUMBER() OVER (PARTITION BY b ORDER BY SourceTimeUtc DESC) AS rla, " +
|
||||
"ROW_NUMBER() OVER (PARTITION BY b, u ORDER BY SourceTimeUtc ASC) AS rfu, " +
|
||||
"ROW_NUMBER() OVER (PARTITION BY b, u ORDER BY SourceTimeUtc DESC) AS rlu FROM s) " +
|
||||
"SELECT b, " + CommonCounts() + ", " +
|
||||
Pick("u = 1 AND rfu = 1", any: false) + ", " +
|
||||
Pick("u = 1 AND rlu = 1", any: false) + ", " +
|
||||
Pick("rfa = 1", any: true) + ", " +
|
||||
Pick("rla = 1", any: true) + " " +
|
||||
"FROM w GROUP BY b ORDER BY b;";
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// MinimumActualTime / MaximumActualTime: the usable sample with the smallest (largest) <c>NumValue</c>,
|
||||
/// ties broken by the earliest <c>SourceTimeUtc</c>. Family columns: <c>6 NumValue, 7 SourceTimeUtc</c>
|
||||
/// (NULL when the bucket has no usable numeric sample).
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Deliberately still two references to <c>s</c> (counts + the ranked extreme), unlike <see cref="StartEnd"/>:
|
||||
/// every single-scan form measured (a <c>ROW_NUMBER</c> or windowed <c>MIN</c>/<c>MAX</c> over every row) was
|
||||
/// 1.2–2× slower on day-plus windows than ranking only the filtered usable rows. Live-gate 8 F1's inconsistency —
|
||||
/// a sample committed between the two scans, counted but not ranked — is instead absorbed by the reader: a
|
||||
/// NULL extreme in a bucket with usable samples evaluates to <c>BadNoData</c> (never a value-less Good).
|
||||
/// </remarks>
|
||||
/// <param name="sample">Schema-qualified <c>Sample</c> table name.</param>
|
||||
/// <param name="maximum">True for MaximumActualTime.</param>
|
||||
/// <returns>The SQL text.</returns>
|
||||
@@ -130,21 +162,26 @@ internal static class SqlAggregateQueries
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The earliest stored sample of the tag (any quality), one row (<c>NULL</c> when none): the start-of-data Partial
|
||||
/// rule (#677; forward reads only). A <c>MIN</c> over the <c>(TagId, SourceTimeUtc)</c> key — a single seek — bounded
|
||||
/// by the window end (a sample at or past it cannot lie inside an interval).
|
||||
/// The tag's data range as one row of two columns (each <c>NULL</c> when none): <c>0</c> the earliest stored sample
|
||||
/// of any quality at or before the window's later edge, <c>1</c> the latest at or after its earlier edge. Used by the
|
||||
/// start-of-data (#677) and end-of-data (#680) Partial rules and by Count's in-range empty interval (#680) — only when
|
||||
/// <see cref="AggregateEvaluator.NeedsDataRange"/> says the answer can change a result. Each is a <c>MIN</c> /
|
||||
/// <c>MAX</c> over the <c>(TagId, SourceTimeUtc)</c> key — a single seek — bounded by the window (a sample past the
|
||||
/// later edge cannot start the data inside it, nor one before the earlier edge end it there; a NULL then means "the
|
||||
/// data range does not reach the window").
|
||||
/// </summary>
|
||||
/// <param name="sample">Schema-qualified <c>Sample</c> table name.</param>
|
||||
/// <returns>The SQL text.</returns>
|
||||
public static string EarliestSample(string sample) =>
|
||||
$"SELECT MIN(SourceTimeUtc) FROM {sample} WHERE TagId = @id AND SourceTimeUtc < @end;";
|
||||
public static string DataRange(string sample) =>
|
||||
$"SELECT (SELECT MIN(SourceTimeUtc) FROM {sample} WHERE TagId = @id AND SourceTimeUtc <= @end), " +
|
||||
$"(SELECT MAX(SourceTimeUtc) FROM {sample} WHERE TagId = @id AND SourceTimeUtc >= @start);";
|
||||
|
||||
/// <summary>The <c>s</c> CTE: the window's samples with bucket index and the usable/good flags.</summary>
|
||||
private static string Classified(string sample, bool reverse) =>
|
||||
"WITH s AS (SELECT (DATEDIFF_BIG(microsecond, @start, SourceTimeUtc) * 10 " +
|
||||
"+ (DATEPART(nanosecond, SourceTimeUtc) / 100) % 10 - (DATEPART(nanosecond, @start) / 100) % 10 " +
|
||||
"+ @bucketAdjustTicks) / @intervalTicks AS b, " +
|
||||
"SourceTimeUtc, ValueKind, NumValue, IntValue, StrValue, " +
|
||||
"SourceTimeUtc, ValueKind, NumValue, IntValue, StrValue, StatusCode, " +
|
||||
"CASE WHEN StatusCode >= 0 AND ValueKind <> 0 THEN 1 ELSE 0 END AS u, " +
|
||||
"CASE WHEN StatusCode >= 0 AND StatusCode < 1073741824 AND ValueKind <> 0 THEN 1 ELSE 0 END AS g " +
|
||||
$"FROM {sample} WHERE TagId = @id AND " +
|
||||
|
||||
@@ -106,7 +106,7 @@ internal sealed class SqlHistorianConnection
|
||||
/// <param name="reader">Reader positioned on a row.</param>
|
||||
/// <param name="ordinal">Column ordinal.</param>
|
||||
/// <returns>UTC timestamp.</returns>
|
||||
public static DateTime GetUtc(SqlDataReader reader, int ordinal) =>
|
||||
public static DateTime GetUtc(System.Data.Common.DbDataReader reader, int ordinal) =>
|
||||
DateTime.SpecifyKind(reader.GetDateTime(ordinal), DateTimeKind.Utc);
|
||||
|
||||
/// <summary>True for connection/timeout/deadlock/throttling errors worth retrying.</summary>
|
||||
|
||||
+48
-23
@@ -1,4 +1,5 @@
|
||||
using System.Data;
|
||||
using System.Data.Common;
|
||||
using System.Text;
|
||||
using Microsoft.Data.SqlClient;
|
||||
using Microsoft.Extensions.Logging;
|
||||
@@ -33,7 +34,7 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer;
|
||||
/// carries a value; an <c>Empty</c>-kind sample is an outage marker (comms-loss <c>BadNoCommunication</c>
|
||||
/// or a driver's Bad publish) and is Bad whatever its code. Raw returns every sample as stored, outage
|
||||
/// markers included. Aggregates exclude Bad samples from the math, report the interval Uncertain when
|
||||
/// any non-Good sample is present, and count them in PercentBad. AtTime never interpolates across a Bad
|
||||
/// any non-Good sample is present, and count their duration as Bad time in PercentBad. AtTime never interpolates across a Bad
|
||||
/// neighbour and reports a time inside an outage with the outage's Bad code (see <see cref="ReadAtTimeAsync"/>).
|
||||
/// <b>An Empty sample is always served Bad:</b> a row with no value whose stored status is not Bad (only
|
||||
/// possible for rows written before the writer forced Empty samples Bad) is read back as <c>BadNoData</c>,
|
||||
@@ -405,13 +406,16 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
|
||||
/// returned latest first (<see cref="BucketPlan"/>).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Per interval: no samples ⇒ <c>BadNoData</c>. PercentGood / PercentBad ⇒ <c>100 × good|bad / all</c> over
|
||||
/// every sample, status Good. Otherwise no usable sample ⇒ <c>BadNoData</c>; a numeric aggregate with a usable
|
||||
/// String/DateTime sample ⇒ <c>BadAggregateInvalidInputs</c>; else the value, Good when every sample is Good,
|
||||
/// <c>UncertainDataSubNormal</c> when any is Bad or Uncertain. Every status — including <c>BadNoData</c> —
|
||||
/// carries the Calculated info bits. Timestamps are the bucket start (its later edge on a reverse read), except
|
||||
/// Minimum/MaximumActualTime (the sample's time; ties ⇒ earliest). Every aggregate but Average sets the Partial
|
||||
/// bit on the bucket the tag's earliest sample lies strictly inside (#677).
|
||||
/// Per-sample aggregates, per interval: no samples ⇒ <c>BadNoData</c>. Otherwise no usable sample ⇒
|
||||
/// <c>BadNoData</c>; a numeric aggregate with a usable String/DateTime sample ⇒ <c>BadAggregateInvalidInputs</c>;
|
||||
/// else the value, Good when every sample is Good, <c>UncertainDataSubNormal</c> when any is Bad or Uncertain.
|
||||
/// Every calculated status — including <c>BadNoData</c> — carries the Calculated info bits. Timestamps are the
|
||||
/// bucket start (its later edge on a reverse read), except Minimum/MaximumActualTime (the sample's time; ties ⇒
|
||||
/// earliest) and Start/End. Every aggregate but Average sets the Partial bit on the bucket the tag's earliest
|
||||
/// sample lies strictly inside (forward, #677) or its latest sample lies in (#680; reverse: strictly inside).
|
||||
/// PercentGood / PercentBad are duration-based since #680 (time-weighted family, below). #680: Count counts Good samples (value-based status; 0/Good on an empty bucket inside the tag's
|
||||
/// data range), Start/End are the raw first/last sample of any quality, Delta is Uncertain only when the bucket's
|
||||
/// first or last sample is not usable (see <see cref="AggregateEvaluator"/>).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Values: Count is <see cref="int"/>; Start/End keep the sample's type; every other aggregate is
|
||||
@@ -444,7 +448,7 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
|
||||
return new HistoryReadResult(twResult, ContinuationPoint: null);
|
||||
}
|
||||
var rows = new List<BucketRow>();
|
||||
DateTime? earliest = null;
|
||||
DateTime? earliest = null, latest = null;
|
||||
if (tagId is { } id)
|
||||
{
|
||||
var sample = _conn.Q("Sample");
|
||||
@@ -454,10 +458,10 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
|
||||
AggregateFamily.StartEnd => SqlAggregateQueries.StartEnd(sample, plan.Reversed),
|
||||
_ => SqlAggregateQueries.ActualTime(sample, aggregate == HistoryAggregateType.MaximumActualTime, plan.Reversed),
|
||||
};
|
||||
// Second result set: the tag's earliest sample (the start-of-data Partial rule, #677) — forward reads of an
|
||||
// aggregate other than Average only.
|
||||
var needsEarliest = AggregateEvaluator.NeedsEarliestSample(aggregate, plan);
|
||||
if (needsEarliest) sql += " " + SqlAggregateQueries.EarliestSample(sample);
|
||||
// Second result set: the tag's data range (earliest + latest sample) — the start/end-of-data Partial rules
|
||||
// (#677/#680 — any per-sample aggregate but Average, either direction) and Count's in-range empty interval (#680).
|
||||
var needsRange = AggregateEvaluator.NeedsDataRange(aggregate, plan);
|
||||
if (needsRange) sql += " " + SqlAggregateQueries.DataRange(sample);
|
||||
await using var conn = await _conn.OpenAsync(cancellationToken).ConfigureAwait(false);
|
||||
await using var cmd = _conn.CreateCommand(conn, sql);
|
||||
cmd.Parameters.Add(new SqlParameter("@id", SqlDbType.Int) { Value = id });
|
||||
@@ -468,12 +472,15 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
|
||||
await using var reader = await cmd.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false);
|
||||
while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false))
|
||||
rows.Add(ReadBucket(reader, family));
|
||||
if (needsEarliest && await reader.NextResultAsync(cancellationToken).ConfigureAwait(false)
|
||||
&& await reader.ReadAsync(cancellationToken).ConfigureAwait(false) && !reader.IsDBNull(0))
|
||||
earliest = DateTime.SpecifyKind(reader.GetDateTime(0), DateTimeKind.Utc);
|
||||
if (needsRange && await reader.NextResultAsync(cancellationToken).ConfigureAwait(false)
|
||||
&& await reader.ReadAsync(cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
if (!reader.IsDBNull(0)) earliest = SqlHistorianConnection.GetUtc(reader, 0);
|
||||
if (!reader.IsDBNull(1)) latest = SqlHistorianConnection.GetUtc(reader, 1);
|
||||
}
|
||||
}
|
||||
|
||||
var result = AggregateEvaluator.EvaluateAll(aggregate, plan, rows, earliest, now);
|
||||
var result = AggregateEvaluator.EvaluateAll(aggregate, plan, rows, earliest, latest, now);
|
||||
|
||||
RecordSuccess();
|
||||
return new HistoryReadResult(result, ContinuationPoint: null);
|
||||
@@ -518,12 +525,22 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
|
||||
} while (await reader.NextResultAsync(ct).ConfigureAwait(false));
|
||||
}
|
||||
|
||||
private static BucketRow ReadBucket(SqlDataReader reader, AggregateFamily family)
|
||||
/// <summary>
|
||||
/// Maps one per-sample aggregate row (<see cref="SqlAggregateQueries"/> column layout) to a <see cref="BucketRow"/>.
|
||||
/// Every picked sample — first/last usable, first/last of any quality, the extreme — is optional: a NULL reads as
|
||||
/// "no such sample", never an exception (live-gate 8 F1: on QuestDB a row whose counts and picks disagreed threw
|
||||
/// out of <see cref="ReadStored"/> and failed the node's whole read; the evaluator answers such a bucket from what
|
||||
/// is present, or <c>BadNoData</c>).
|
||||
/// </summary>
|
||||
/// <param name="reader">Reader positioned on a bucket row.</param>
|
||||
/// <param name="family">The query's aggregate family.</param>
|
||||
/// <returns>The bucket statistics.</returns>
|
||||
internal static BucketRow ReadBucket(DbDataReader reader, AggregateFamily family)
|
||||
{
|
||||
static double? D(SqlDataReader r, int i) => r.IsDBNull(i) ? null : r.GetDouble(i);
|
||||
static long? L(SqlDataReader r, int i) => r.IsDBNull(i) ? null : r.GetInt64(i);
|
||||
static string? S(SqlDataReader r, int i) => r.IsDBNull(i) ? null : r.GetString(i);
|
||||
static HistorianValue V(SqlDataReader r, int i) =>
|
||||
static double? D(DbDataReader r, int i) => r.IsDBNull(i) ? null : r.GetDouble(i);
|
||||
static long? L(DbDataReader r, int i) => r.IsDBNull(i) ? null : r.GetInt64(i);
|
||||
static string? S(DbDataReader r, int i) => r.IsDBNull(i) ? null : r.GetString(i);
|
||||
static HistorianValue V(DbDataReader r, int i) =>
|
||||
r.IsDBNull(i) ? HistorianValue.Empty : SampleColumnMapper.FromColumns(r.GetByte(i), D(r, i + 1), L(r, i + 2), S(r, i + 3));
|
||||
|
||||
var row = new BucketRow
|
||||
@@ -547,6 +564,8 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
|
||||
case AggregateFamily.StartEnd:
|
||||
row.First = V(reader, f);
|
||||
row.Last = V(reader, f + 5);
|
||||
row.FirstAny = ReadStoredOrNull(reader, SqlAggregateQueries.FirstAnyColumn);
|
||||
row.LastAny = ReadStoredOrNull(reader, SqlAggregateQueries.FirstAnyColumn + 6);
|
||||
break;
|
||||
default:
|
||||
row.Extreme = D(reader, f);
|
||||
@@ -564,7 +583,7 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
|
||||
/// (<see cref="StoredSample.FromStored"/>) — defence in depth for rows written before the writer forced every
|
||||
/// Empty sample Bad.
|
||||
/// </summary>
|
||||
private static StoredSample ReadStored(SqlDataReader r, int o)
|
||||
private static StoredSample ReadStored(DbDataReader r, int o)
|
||||
{
|
||||
var value = SampleColumnMapper.FromColumns(
|
||||
r.GetByte(o + 1),
|
||||
@@ -575,6 +594,12 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
|
||||
return StoredSample.FromStored(SqlHistorianConnection.GetUtc(r, o), value, status);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// <see cref="ReadStored"/> for an optional picked sample: a NULL <c>SourceTimeUtc</c> means the row carries no such
|
||||
/// sample (live-gate 8 F1), so the result is null rather than an exception.
|
||||
/// </summary>
|
||||
private static StoredSample? ReadStoredOrNull(DbDataReader r, int o) => r.IsDBNull(o) ? null : ReadStored(r, o);
|
||||
|
||||
/// <summary>Read-only tag lookup (never creates a row); runs the schema bootstrap on a cache miss.</summary>
|
||||
private Task<int?> ResolveAsync(string name, CancellationToken ct) => _catalog.TryGetTagIdAsync(name, ct);
|
||||
|
||||
|
||||
@@ -21,6 +21,7 @@ public abstract class ProcessedConformance : ConformanceBase
|
||||
private const uint UncertainCalc = 0x40A40401u; // UncertainDataSubNormal | 0x401
|
||||
private const uint NoDataCalc = 0x809B0401u; // BadNoData | 0x401
|
||||
private const uint InvalidInputsCalc = 0x80D60401u; // BadAggregateInvalidInputs | 0x401
|
||||
private const uint Partial = 0x4u; // the Partial aggregate bit (#677 start-of-data, #680 end-of-data)
|
||||
|
||||
private async Task<HistoryReadResult> ReadA(HistoryAggregateType agg) =>
|
||||
await B.CreateDataSource().ReadProcessedAsync("A", At(0), At(80), Twenty, agg, Ct);
|
||||
@@ -40,8 +41,6 @@ public abstract class ProcessedConformance : ConformanceBase
|
||||
[InlineData(HistoryAggregateType.Average, 15d, 40d, 50d)]
|
||||
[InlineData(HistoryAggregateType.Minimum, 10d, 40d, 50d)]
|
||||
[InlineData(HistoryAggregateType.Maximum, 20d, 40d, 50d)]
|
||||
[InlineData(HistoryAggregateType.Start, 10d, 40d, 50d)]
|
||||
[InlineData(HistoryAggregateType.End, 20d, 40d, 50d)]
|
||||
[InlineData(HistoryAggregateType.Delta, 10d, 0d, 0d)]
|
||||
[InlineData(HistoryAggregateType.Range, 10d, 0d, 0d)]
|
||||
public async Task Numeric_aggregates_over_the_fixture(HistoryAggregateType agg, double b0, double b1, double b2)
|
||||
@@ -51,30 +50,81 @@ public abstract class ProcessedConformance : ConformanceBase
|
||||
|
||||
s.Count.ShouldBe(4);
|
||||
s.Select(x => x.Value).ShouldBe([b0, b1, b2, null]);
|
||||
// B1 holds a Bad sample alongside a usable one ⇒ Uncertain; the empty B3 ⇒ BadNoData.
|
||||
s.Select(x => x.StatusCode).ShouldBe([GoodCalc, UncertainCalc, GoodCalc, NoDataCalc]);
|
||||
// B1 holds a Bad sample alongside a usable one ⇒ Uncertain (Delta: its first sample is the Bad one); the empty B3,
|
||||
// after the last sample, ⇒ BadNoData. #680: B2 holds the last sample (+40) ⇒ end-of-data Partial on a forward
|
||||
// read, for every aggregate but Average.
|
||||
var b2Status = agg == HistoryAggregateType.Average ? GoodCalc : GoodCalc | Partial;
|
||||
s.Select(x => x.StatusCode).ShouldBe([GoodCalc, UncertainCalc, b2Status, NoDataCalc]);
|
||||
s.Select(x => x.SourceTimestampUtc).ShouldBe([B.Round(At(0)), B.Round(At(20)), B.Round(At(40)), B.Round(At(60))]); // bucket starts
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Count_counts_usable_samples_as_int()
|
||||
public async Task Start_and_End_return_the_raw_first_and_last_sample_of_any_quality()
|
||||
{
|
||||
// #680 (Part 13 Tables 73/74; SDK ComputeStartEnd): the interval's raw first / last sample — its own value (null
|
||||
// when Bad), its own status (Raw: no Calculated bit) and its own timestamp. B1's Start is the Bad sample at +20.
|
||||
await ConformanceData.SeedAAsync(B);
|
||||
var start = (await ReadA(HistoryAggregateType.Start)).Samples;
|
||||
start.Select(x => x.Value).ShouldBe([10d, null, 50d, null]);
|
||||
start.Select(x => x.StatusCode).ShouldBe([Good, Bad, Good | 0x400u | Partial, NoDataCalc]);
|
||||
start.Select(x => x.SourceTimestampUtc).ShouldBe([B.Round(At(0)), B.Round(At(20)), B.Round(At(40)), B.Round(At(60))]);
|
||||
var end = (await ReadA(HistoryAggregateType.End)).Samples;
|
||||
end.Select(x => x.Value).ShouldBe([20d, 40d, 50d, null]);
|
||||
end.Select(x => x.StatusCode).ShouldBe([Good, Good, Good | 0x400u | Partial, NoDataCalc]);
|
||||
end.Select(x => x.SourceTimestampUtc).ShouldBe([B.Round(At(10)), B.Round(At(30)), B.Round(At(40)), B.Round(At(60))]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Count_counts_good_samples_as_int()
|
||||
{
|
||||
// #680 (Part 13 §5.4.3.21): Good samples only, value-based status; end-of-data Partial on B2.
|
||||
await ConformanceData.SeedAAsync(B);
|
||||
var s = (await ReadA(HistoryAggregateType.Count)).Samples;
|
||||
s.Select(x => x.Value).ShouldBe([(object?)2, 1, 1, null]);
|
||||
s.Select(x => x.StatusCode).ShouldBe([GoodCalc, UncertainCalc, GoodCalc, NoDataCalc]);
|
||||
s.Select(x => x.StatusCode).ShouldBe([GoodCalc, UncertainCalc, GoodCalc | Partial, NoDataCalc]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Count_is_zero_on_an_empty_interval_inside_the_data_range_in_both_directions()
|
||||
{
|
||||
// #680: +1 G, +2 U, +25 Bad, +45 G, +70 G. Forward [0,60) at 10 s: [10,20), [30,40) and [50,60) are empty but
|
||||
// inside [first, last] ⇒ 0, Good; [20,30) holds only a Bad sample ⇒ generic Bad. Reverse (60 → 0) likewise, and a
|
||||
// window wholly after the data ⇒ BadNoData — through the real SQL (the data range is two pushed-down seeks).
|
||||
await B.SeedAsync("CZ",
|
||||
(At(1), HistorianValue.FromDouble(10), Good),
|
||||
(At(2), HistorianValue.FromDouble(10), Uncertain),
|
||||
(At(25), HistorianValue.FromDouble(10), Bad),
|
||||
(At(45), HistorianValue.FromDouble(10), Good),
|
||||
(At(70), HistorianValue.FromDouble(1), Good));
|
||||
var src = B.CreateDataSource();
|
||||
var ten = TimeSpan.FromSeconds(10);
|
||||
var fwd = (await src.ReadProcessedAsync("CZ", At(0), At(60), ten, HistoryAggregateType.Count, Ct)).Samples;
|
||||
fwd.Select(x => x.Value).ShouldBe([(object?)1, 0, null, 0, 1, 0]);
|
||||
fwd.Select(x => x.StatusCode).ShouldBe(
|
||||
[UncertainCalc | Partial, GoodCalc, BadCalc, GoodCalc, GoodCalc, GoodCalc]);
|
||||
var rev = (await src.ReadProcessedAsync("CZ", At(60), At(0), ten, HistoryAggregateType.Count, Ct)).Samples;
|
||||
rev.Select(x => x.Value).ShouldBe([(object?)0, 1, 0, null, 0, 1]);
|
||||
rev.Select(x => x.StatusCode).ShouldBe([GoodCalc, GoodCalc, GoodCalc, BadCalc, GoodCalc, UncertainCalc]);
|
||||
(await src.ReadProcessedAsync("CZ", At(80), At(100), ten, HistoryAggregateType.Count, Ct)).Samples
|
||||
.ShouldAllBe(x => x.StatusCode == NoDataCalc);
|
||||
(await src.ReadProcessedAsync("CZ", At(-20), At(0), ten, HistoryAggregateType.Count, Ct)).Samples
|
||||
.ShouldAllBe(x => x.StatusCode == NoDataCalc);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(HistoryAggregateType.PercentGood, 100d, 50d, 100d)]
|
||||
[InlineData(HistoryAggregateType.PercentBad, 0d, 50d, 0d)]
|
||||
public async Task Percent_aggregates_count_every_sample(HistoryAggregateType agg, double b0, double b1, double b2)
|
||||
public async Task Percent_aggregates_are_shares_of_the_interval_duration(HistoryAggregateType agg, double b0, double b1,
|
||||
double b2)
|
||||
{
|
||||
// #680 (Part 13 §5.4.3.31-34): stepped regions, each with the status of the sample starting it. B1: [20,30) Bad,
|
||||
// [30,40) Good. B2 holds the end of the data (+40, nothing after) ⇒ its region runs to the interval end, Partial;
|
||||
// B3 is after the end of the data ⇒ BadNoData.
|
||||
await ConformanceData.SeedAAsync(B);
|
||||
var s = (await ReadA(agg)).Samples;
|
||||
s.Select(x => x.Value).ShouldBe([b0, b1, b2, null]);
|
||||
// A percentage is a statement about quality, so it is not itself degraded by the Bad input.
|
||||
s.Select(x => x.StatusCode).ShouldBe([GoodCalc, GoodCalc, GoodCalc, NoDataCalc]);
|
||||
s.Select(x => x.StatusCode).ShouldBe([GoodCalc, GoodCalc, GoodCalc | 0x4u, NoDataCalc]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -84,7 +134,8 @@ public abstract class ProcessedConformance : ConformanceBase
|
||||
var min = (await ReadA(HistoryAggregateType.MinimumActualTime)).Samples;
|
||||
min.Select(x => x.Value).ShouldBe([10d, 40d, 50d, null]);
|
||||
min.Select(x => x.SourceTimestampUtc).ShouldBe([B.Round(At(0)), B.Round(At(30)), B.Round(At(40)), B.Round(At(60))]);
|
||||
min.Select(x => x.StatusCode).ShouldBe([GoodCalc, UncertainCalc, GoodCalc, NoDataCalc]);
|
||||
// #680: B2 holds the last sample (+40) ⇒ end-of-data Partial.
|
||||
min.Select(x => x.StatusCode).ShouldBe([GoodCalc, UncertainCalc, GoodCalc | Partial, NoDataCalc]);
|
||||
var max = (await ReadA(HistoryAggregateType.MaximumActualTime)).Samples;
|
||||
max.Select(x => x.Value).ShouldBe([20d, 40d, 50d, null]);
|
||||
max.Select(x => x.SourceTimestampUtc).ShouldBe([B.Round(At(10)), B.Round(At(30)), B.Round(At(40)), B.Round(At(60))]);
|
||||
@@ -169,9 +220,10 @@ public abstract class ProcessedConformance : ConformanceBase
|
||||
// An interval wider than the window (here TimeSpan.MaxValue) is the same single interval, no overflow.
|
||||
(await src.ReadProcessedAsync("A", start, end, TimeSpan.MaxValue, HistoryAggregateType.Count, Ct))
|
||||
.Samples.Single().Value.ShouldBe(4);
|
||||
// The time-weighted path over the whole range: ≈ the extrapolated 50 that dominates eight millennia.
|
||||
// The time-weighted path over the whole range. TimeAverage2 is not extrapolated past the end of the data (+40, #680):
|
||||
// (150 + 200 + 450) value·s over the 30 non-Bad seconds = 80/3.
|
||||
var tw = (await src.ReadProcessedAsync("A", start, end, TimeSpan.Zero, HistoryAggregateType.TimeAverage2, Ct)).Samples.Single();
|
||||
((double)tw.Value!).ShouldBe(50d, 1e-6);
|
||||
((double)tw.Value!).ShouldBe(80d / 3d, 1e-6);
|
||||
// The data starts (+0) strictly inside the one interval ⇒ Partial as well (#677, the SDK's m_startOfData rule).
|
||||
tw.StatusCode.ShouldBe(UncertainCalc | 0x4u);
|
||||
src.GetHealthSnapshot().TotalFailures.ShouldBe(0);
|
||||
@@ -267,8 +319,9 @@ public abstract class ProcessedConformance : ConformanceBase
|
||||
(At(5), HistorianValue.FromDouble(10), Good),
|
||||
(At(15), HistorianValue.FromDouble(30), Good),
|
||||
(At(25), HistorianValue.FromDouble(20), Good));
|
||||
// ... but LS2's data ENDS (+25) inside [20,40) ⇒ Partial there (#680, the SDK's m_endOfData rule).
|
||||
(await src.ReadProcessedAsync("LS2", At(0), At(40), Twenty, HistoryAggregateType.Count, Ct)).Samples
|
||||
.ShouldAllBe(x => (x.StatusCode & 0x4u) == 0);
|
||||
.Select(x => x.StatusCode & Partial).ShouldBe([0u, Partial]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -293,9 +346,7 @@ public abstract class ProcessedConformance : ConformanceBase
|
||||
ours.ShouldAllBe(x => (x.StatusCode & 0x4u) == 0, agg.ToString());
|
||||
var tw = AggregateFamilies.FamilyOf(agg) == AggregateFamily.TimeWeighted;
|
||||
SdkAggregateOracle.ShouldMatch(ours, SdkAggregateOracle.Run(agg, B.Round(At(40)), B.Round(At(0)), ten, sdkRows),
|
||||
$"{agg} reverse late start", compareCalculatedInterpolatedBits: tw,
|
||||
// pre-existing, #680: the SDK's Delta is Good with a Bad sample present; ours Uncertain (none here).
|
||||
compareCodeBits: agg != HistoryAggregateType.Delta);
|
||||
$"{agg} reverse late start", compareCalculatedInterpolatedBits: tw);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -333,7 +384,6 @@ public abstract class ProcessedConformance : ConformanceBase
|
||||
})
|
||||
{
|
||||
var ours = (await src.ReadProcessedAsync("P", At(from), At(to), interval, agg, Ct)).Samples.ToList();
|
||||
var startOrEnd = agg is HistoryAggregateType.Start or HistoryAggregateType.End;
|
||||
var tw = AggregateFamilies.FamilyOf(agg) == AggregateFamily.TimeWeighted;
|
||||
if (tw && agg != HistoryAggregateType.Interpolative)
|
||||
{
|
||||
@@ -347,15 +397,27 @@ public abstract class ProcessedConformance : ConformanceBase
|
||||
}
|
||||
}
|
||||
var reverseInterpolative = agg == HistoryAggregateType.Interpolative && from > to;
|
||||
SdkAggregateOracle.ShouldMatch(ours, SdkAggregateOracle.Run(agg, start, end, interval, rows),
|
||||
var sdk = SdkAggregateOracle.Run(agg, start, end, interval, rows);
|
||||
if (from > to && agg is HistoryAggregateType.TimeAverage or HistoryAggregateType.Total)
|
||||
{
|
||||
// Documented SDK divergence (#680, see SdkAggregateParityTests.MaskDocumented): on a reverse read the SDK files
|
||||
// a sample exactly on an interval's later edge as its early bound, so a TimeAverage/Total interval ending on a
|
||||
// sample takes the status of the segment beyond it (Dense: +20 next to the Bad +22.5). Only that interval,
|
||||
// only Good (ours) vs Uncertain (SDK), is aligned.
|
||||
var plan = BucketPlan.Create(start, end, interval);
|
||||
for (long i = 0; i < plan.BucketCount; i++)
|
||||
{
|
||||
var k = (int)(plan.BucketCount - 1 - i);
|
||||
if (k < sdk.Count && rows.Any(r => r.Item1.Ticks == plan.UpperTicks(i))
|
||||
&& (ours[k].StatusCode & 0xFFFF0000u) == 0
|
||||
&& (sdk[k].StatusCode & 0xFFFF0000u) == 0x40A40000u)
|
||||
ours[k] = ours[k] with { StatusCode = ours[k].StatusCode | 0x40A40000u };
|
||||
}
|
||||
}
|
||||
SdkAggregateOracle.ShouldMatch(ours, sdk,
|
||||
$"{agg} {from}→{to}/{intervalS}s",
|
||||
compareCalculatedInterpolatedBits: tw && !reverseInterpolative,
|
||||
// pre-existing, #680: TimeAverage/Total status rule; Start/End raw-sample timestamp + status; Delta status;
|
||||
// PercentGood/Bad per-sample vs per-duration values (see SdkAggregateParityTests).
|
||||
compareCodeBits: agg is not (HistoryAggregateType.TimeAverage or HistoryAggregateType.Total
|
||||
or HistoryAggregateType.Delta) && !reverseInterpolative && !startOrEnd,
|
||||
compareTimestamps: !startOrEnd,
|
||||
compareValues: agg is not (HistoryAggregateType.PercentGood or HistoryAggregateType.PercentBad));
|
||||
compareCodeBits: !reverseInterpolative);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -383,7 +445,8 @@ public abstract class ProcessedConformance : ConformanceBase
|
||||
avg.StatusCode.ShouldBe(UncertainCalc);
|
||||
(await One(HistoryAggregateType.Count)).Value.ShouldBe(2);
|
||||
(await One(HistoryAggregateType.Start)).Value.ShouldBe(10d);
|
||||
((double)(await One(HistoryAggregateType.PercentBad)).Value!).ShouldBe(33.333333333333336, 1e-9); // 100 × 1/3
|
||||
// #680: duration-based — [0,5) Good, [5,10) Bad, [10,20) Good (no end bound: the last region runs to the interval end).
|
||||
((double)(await One(HistoryAggregateType.PercentBad)).Value!).ShouldBe(25d, 1e-9);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -397,7 +460,15 @@ public abstract class ProcessedConformance : ConformanceBase
|
||||
(await src.ReadProcessedAsync("Z", At(0), At(20), Twenty, a, Ct)).Samples.Single();
|
||||
|
||||
(await One(HistoryAggregateType.Average)).StatusCode.ShouldBe(NoDataCalc);
|
||||
(await One(HistoryAggregateType.Count)).StatusCode.ShouldBe(NoDataCalc);
|
||||
// #680 (SDK value-based status): an interval holding only Bad samples counts as the generic Bad, value null.
|
||||
var count = await One(HistoryAggregateType.Count);
|
||||
count.StatusCode.ShouldBe(BadCalc);
|
||||
count.Value.ShouldBeNull();
|
||||
// Start is the raw first sample: the outage marker itself, with its own status and time.
|
||||
var start = await One(HistoryAggregateType.Start);
|
||||
start.StatusCode.ShouldBe(BadNoCommunication);
|
||||
start.SourceTimestampUtc.ShouldBe(B.Round(At(1)));
|
||||
start.Value.ShouldBeNull();
|
||||
var pb = await One(HistoryAggregateType.PercentBad);
|
||||
pb.Value.ShouldBe(100d);
|
||||
// The tag's data starts (+1) inside the interval ⇒ Partial on the non-Bad percentage (#677).
|
||||
@@ -611,6 +682,9 @@ public abstract class ProcessedConformance : ConformanceBase
|
||||
var ex = await Should.ThrowAsync<HistoryReadLimitExceededException>(
|
||||
() => under.ReadProcessedAsync("A", At(0), At(80), Twenty, HistoryAggregateType.Total, Ct));
|
||||
ex.Limit.ShouldBe(4);
|
||||
// #680: PercentGood/PercentBad are duration-based, served by the same window stream ⇒ capped too.
|
||||
(await Should.ThrowAsync<HistoryReadLimitExceededException>(
|
||||
() => under.ReadProcessedAsync("A", At(0), At(80), Twenty, HistoryAggregateType.PercentGood, Ct))).Limit.ShouldBe(4);
|
||||
// Pushed-down (per-sample) aggregates are not capped; the bound rows outside the window do not count.
|
||||
(await under.ReadProcessedAsync("A", At(0), At(80), Twenty, HistoryAggregateType.Average, Ct)).Samples.Count.ShouldBe(4);
|
||||
(await B.CreateDataSource(3).ReadProcessedAsync("W", At(0), At(80), Twenty, HistoryAggregateType.TimeAverage, Ct))
|
||||
|
||||
+24
@@ -37,6 +37,30 @@ public sealed class GatewayHistorianDataSourceTests
|
||||
Assert.Equal(TimeSpan.FromSeconds(60), fake.LastAggregateInterval);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ReadProcessed_TimeAverage_requests_TimeWeightedAverage()
|
||||
{
|
||||
var fake = new FakeHistorianGatewayClient();
|
||||
var ds = new GatewayHistorianDataSource(fake, NullLogger<GatewayHistorianDataSource>.Instance);
|
||||
await ds.ReadProcessedAsync("T", default, default, TimeSpan.FromSeconds(60), HistoryAggregateType.TimeAverage, TestContext.Current.CancellationToken);
|
||||
Assert.Equal(RetrievalMode.TimeWeightedAverage, fake.LastAggregateMode);
|
||||
}
|
||||
|
||||
[Theory] // #679: no AVEVA equivalent — not supported, gateway never called, no backend failure recorded
|
||||
[InlineData(HistoryAggregateType.Average)]
|
||||
[InlineData(HistoryAggregateType.Count)]
|
||||
public async Task ReadProcessed_unsupported_aggregate_throws_without_calling_gateway_or_recording_failure(HistoryAggregateType aggregate)
|
||||
{
|
||||
var fake = new FakeHistorianGatewayClient();
|
||||
var ds = new GatewayHistorianDataSource(fake, NullLogger<GatewayHistorianDataSource>.Instance);
|
||||
await Assert.ThrowsAsync<HistoryAggregateNotSupportedException>(() => ds.ReadProcessedAsync(
|
||||
"T", default, default, TimeSpan.FromSeconds(60), aggregate, TestContext.Current.CancellationToken));
|
||||
Assert.Equal(0, fake.ReadAggregateCallCount);
|
||||
var h = ds.GetHealthSnapshot();
|
||||
Assert.Equal(0, h.TotalFailures);
|
||||
Assert.Equal(0, h.TotalQueries);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ReadAtTime_aligns_one_snapshot_per_timestamp_with_gaps_Bad()
|
||||
{
|
||||
|
||||
+9
-4
@@ -8,11 +8,10 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests.Mapping;
|
||||
public sealed class AggregateModeMapperTests
|
||||
{
|
||||
[Theory]
|
||||
[InlineData(HistoryAggregateType.Average, RetrievalMode.TimeWeightedAverage)]
|
||||
[InlineData(HistoryAggregateType.TimeAverage, RetrievalMode.TimeWeightedAverage)]
|
||||
[InlineData(HistoryAggregateType.Minimum, RetrievalMode.MinimumWithTime)]
|
||||
[InlineData(HistoryAggregateType.Maximum, RetrievalMode.MaximumWithTime)]
|
||||
[InlineData(HistoryAggregateType.Total, RetrievalMode.Integral)]
|
||||
[InlineData(HistoryAggregateType.Count, RetrievalMode.Counter)]
|
||||
public void Maps_each_aggregate(HistoryAggregateType a, RetrievalMode expected)
|
||||
=> Assert.Equal(expected, AggregateModeMapper.ToRetrievalMode(a));
|
||||
|
||||
@@ -21,8 +20,8 @@ public sealed class AggregateModeMapperTests
|
||||
{
|
||||
var original = new[]
|
||||
{
|
||||
HistoryAggregateType.Average, HistoryAggregateType.Minimum, HistoryAggregateType.Maximum,
|
||||
HistoryAggregateType.Total, HistoryAggregateType.Count,
|
||||
HistoryAggregateType.Minimum, HistoryAggregateType.Maximum,
|
||||
HistoryAggregateType.Total, HistoryAggregateType.TimeAverage,
|
||||
};
|
||||
foreach (var a in Enum.GetValues<HistoryAggregateType>())
|
||||
{
|
||||
@@ -33,6 +32,12 @@ public sealed class AggregateModeMapperTests
|
||||
}
|
||||
}
|
||||
|
||||
[Theory] // #679: AVEVA has no arithmetic mean and no sample count (Counter is a rollover increase)
|
||||
[InlineData(HistoryAggregateType.Average)]
|
||||
[InlineData(HistoryAggregateType.Count)]
|
||||
public void Average_and_Count_are_not_supported(HistoryAggregateType a)
|
||||
=> Assert.Throws<HistoryAggregateNotSupportedException>(() => AggregateModeMapper.ToRetrievalMode(a));
|
||||
|
||||
[Fact]
|
||||
public void Undefined_value_is_a_programming_error_not_a_not_supported()
|
||||
=> Assert.Throws<ArgumentOutOfRangeException>(() => AggregateModeMapper.ToRetrievalMode((HistoryAggregateType)9999));
|
||||
|
||||
+98
@@ -0,0 +1,98 @@
|
||||
using System.Collections.Concurrent;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb.IntegrationTests;
|
||||
|
||||
/// <summary>
|
||||
/// Live-gate 8 F1 (#680 follow-up): per-sample processed reads of a tag that is being written. The StartEnd and
|
||||
/// ActualTime queries reference their classified rows more than once; QuestDB evaluates each reference with its own
|
||||
/// table reader, so a row committed between two of them produced a bucket whose counts included a sample its
|
||||
/// first/last (or extreme) join could not find — a NULL first-of-any-quality time threw <c>InvalidCastException</c>
|
||||
/// and failed the node's whole read (~1 in 90k reads on the rig), and ActualTime returned a value-less Good. StartEnd
|
||||
/// now reads the table once; ActualTime keeps its form and an inconsistent bucket evaluates to <c>BadNoData</c>. This
|
||||
/// test interleaves a continuous single-row writer with tight-loop reads and asserts that no read throws and no
|
||||
/// non-Bad result is value-less.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Opt-in: runs only when <c>HISTORIAN_QUESTDB_STRESS_SECONDS</c> is set (and the fixture's connection is), so it never
|
||||
/// slows the default suite. Category <c>SlowUnit</c> as well, for filter-based exclusion.
|
||||
/// </remarks>
|
||||
[Collection(QuestDbCollection.Name)]
|
||||
[Trait("Category", "SlowUnit")]
|
||||
public sealed class LiveWriteReadRaceStressTests(QuestDbFixture fx)
|
||||
{
|
||||
/// <summary>Seconds to run the race loop for; absent ⇒ the test skips.</summary>
|
||||
public const string SecondsEnvVar = "HISTORIAN_QUESTDB_STRESS_SECONDS";
|
||||
|
||||
private static readonly HistoryAggregateType[] Aggregates =
|
||||
[
|
||||
HistoryAggregateType.Start, HistoryAggregateType.End, HistoryAggregateType.Delta,
|
||||
HistoryAggregateType.MinimumActualTime, HistoryAggregateType.MaximumActualTime,
|
||||
];
|
||||
|
||||
[Fact]
|
||||
public async Task Per_sample_reads_of_a_tag_being_written_never_fault_and_never_return_a_valueless_non_bad_result()
|
||||
{
|
||||
if (!int.TryParse(Environment.GetEnvironmentVariable(SecondsEnvVar), out var seconds) || seconds <= 0)
|
||||
Assert.Skip($"{SecondsEnvVar} is not set; the live write/read race stress test is opt-in.");
|
||||
await using var b = await QuestDbConformanceBackend.CreateConcreteAsync(fx);
|
||||
const string tag = "Race";
|
||||
await b.NewCatalog().EnsureAsync(tag, 0, CancellationToken.None);
|
||||
var src = b.CreateDataSource();
|
||||
|
||||
using var stop = new CancellationTokenSource(TimeSpan.FromSeconds(seconds));
|
||||
long written = 0, reads = 0;
|
||||
var faults = new ConcurrentBag<string>();
|
||||
|
||||
var writer = Task.Run(async () =>
|
||||
{
|
||||
var i = 0;
|
||||
while (!stop.IsCancellationRequested)
|
||||
{
|
||||
// One committed row per statement (one WAL transaction each), every 7th Bad, never waiting for visibility.
|
||||
var status = ++i % 7 == 0 ? HistorianStatusCodes.Bad : 0u;
|
||||
await b.InsertRowsNoWaitAsync(tag, [(DateTime.UtcNow, HistorianValue.FromDouble(i), status, false)]);
|
||||
Interlocked.Increment(ref written);
|
||||
}
|
||||
});
|
||||
|
||||
var readers = Enumerable.Range(0, 4).Select(_ => Task.Run(async () =>
|
||||
{
|
||||
while (!stop.IsCancellationRequested)
|
||||
{
|
||||
var now = DateTime.UtcNow;
|
||||
var start = new DateTime(now.Ticks / TimeSpan.TicksPerSecond / 10 * 10 * TimeSpan.TicksPerSecond, DateTimeKind.Utc)
|
||||
.AddSeconds(-30);
|
||||
var end = start.AddSeconds(70);
|
||||
foreach (var agg in Aggregates)
|
||||
foreach (var reverse in new[] { false, true })
|
||||
{
|
||||
try
|
||||
{
|
||||
var r = await src.ReadProcessedAsync(tag, reverse ? end : start, reverse ? start : end,
|
||||
TimeSpan.FromSeconds(10), agg, CancellationToken.None);
|
||||
foreach (var v in r.Samples)
|
||||
if (!StatusBits.IsBad(v.StatusCode) && v.Value is null)
|
||||
faults.Add($"{agg} {(reverse ? "rev" : "fwd")}: non-Bad 0x{v.StatusCode:X8} with no value at {v.SourceTimestampUtc:O}");
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
faults.Add($"{agg} {(reverse ? "rev" : "fwd")}: {ex.GetType().Name}: {ex.Message}");
|
||||
}
|
||||
Interlocked.Increment(ref reads);
|
||||
}
|
||||
}
|
||||
})).ToArray();
|
||||
|
||||
await Task.WhenAll(readers.Append(writer));
|
||||
var summary = $"race stress: {seconds}s, {Interlocked.Read(ref written)} rows written, {Interlocked.Read(ref reads)} reads, {faults.Count} faults";
|
||||
var output = TestContext.Current.TestOutputHelper;
|
||||
output?.WriteLine(summary);
|
||||
foreach (var f in faults.Take(20)) output?.WriteLine("fault: " + f);
|
||||
written.ShouldBeGreaterThan(0);
|
||||
faults.ShouldBeEmpty(summary);
|
||||
}
|
||||
}
|
||||
+13
-1
@@ -107,6 +107,19 @@ internal sealed class QuestDbConformanceBackend : IConformanceBackend
|
||||
/// <param name="tag">The tag.</param>
|
||||
/// <param name="rows">Rows: time (already rounded by the caller or not — it is bound through BindTimestamp), value, status, deleted.</param>
|
||||
public async Task InsertRowsAsync(string tag, (DateTime T, HistorianValue V, uint Status, bool Deleted)[] rows)
|
||||
{
|
||||
if (rows.Length == 0) return;
|
||||
await InsertRowsNoWaitAsync(tag, rows);
|
||||
await Wal.WaitAsync(Conn.SampleTable, CancellationToken.None);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// <see cref="InsertRowsAsync"/> without the WAL visibility wait: the rows become visible whenever QuestDB applies
|
||||
/// them (the live write/read race stress test commits one row per call and never waits).
|
||||
/// </summary>
|
||||
/// <param name="tag">The tag.</param>
|
||||
/// <param name="rows">Rows: time, value, status, deleted.</param>
|
||||
public async Task InsertRowsNoWaitAsync(string tag, (DateTime T, HistorianValue V, uint Status, bool Deleted)[] rows)
|
||||
{
|
||||
if (rows.Length == 0) return;
|
||||
await using (var c = await Conn.OpenAsync(CancellationToken.None))
|
||||
@@ -140,7 +153,6 @@ internal sealed class QuestDbConformanceBackend : IConformanceBackend
|
||||
await cmd.ExecuteNonQueryAsync();
|
||||
}
|
||||
}
|
||||
await Wal.WaitAsync(Conn.SampleTable, CancellationToken.None);
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
|
||||
+9
-2
@@ -82,8 +82,15 @@ public sealed class ReadTests(QuestDbFixture fx) : IAsyncLifetime
|
||||
foreach (var agg in new[] { HistoryAggregateType.Average, HistoryAggregateType.Start, HistoryAggregateType.MaximumActualTime })
|
||||
{
|
||||
var s = (await src.ReadProcessedAsync("A", start, end, TimeSpan.Zero, agg, Ct)).Samples.Single();
|
||||
// The data starts (+0) strictly inside the one interval ⇒ Partial on every aggregate but Average (#677).
|
||||
s.StatusCode.ShouldBe(agg == HistoryAggregateType.Average ? UncertainCalc : UncertainCalc | 0x4u, agg.ToString());
|
||||
// The data starts (+0) strictly inside the one interval ⇒ Partial on every aggregate but Average (#677). Start
|
||||
// is the raw first sample (#680): its own Good status, no Calculated bit, plus InfoType + Partial.
|
||||
var expected = agg switch
|
||||
{
|
||||
HistoryAggregateType.Average => UncertainCalc,
|
||||
HistoryAggregateType.Start => 0x00000404u,
|
||||
_ => UncertainCalc | 0x4u,
|
||||
};
|
||||
s.StatusCode.ShouldBe(expected, agg.ToString());
|
||||
}
|
||||
(await src.ReadProcessedAsync("A", start, end, TimeSpan.Zero, HistoryAggregateType.Average, Ct)).Samples.Single().Value.ShouldBe(30d);
|
||||
(await src.ReadProcessedAsync("A", start, end, TimeSpan.Zero, HistoryAggregateType.Start, Ct)).Samples.Single().Value.ShouldBe(10d);
|
||||
|
||||
+116
@@ -0,0 +1,116 @@
|
||||
using System.Data;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Live-gate 8 F1: the per-sample aggregate row mapper (<see cref="QuestDbHistorianDataSource.ReadBucket"/>) over a
|
||||
/// hand-built row in the <see cref="QuestDbAggregateQueries.StartEnd"/> column layout. A row whose counts include
|
||||
/// samples but whose first/last-of-any-quality picks are NULL used to throw <c>InvalidCastException: Column 'ts' is
|
||||
/// null</c> and fail the node's whole read; it now maps to absent picks the evaluator answers as <c>BadNoData</c>.
|
||||
/// </summary>
|
||||
public sealed class BucketRowMappingTests
|
||||
{
|
||||
private const uint NoDataCalc = 0x809B0401u;
|
||||
private static readonly DateTime T = new(2026, 10, 11, 12, 0, 0, DateTimeKind.Utc);
|
||||
|
||||
/// <summary>The StartEnd layout: b + 5 counts, two usable picks (kind, num, int, str, ts), two any picks (ts, kind, num, int, str, status).</summary>
|
||||
private static DataTable StartEndTable()
|
||||
{
|
||||
var t = new DataTable();
|
||||
t.Columns.Add("b", typeof(long));
|
||||
foreach (var c in new[] { "total", "usable", "good", "bad", "nonnum" }) t.Columns.Add(c, typeof(long));
|
||||
for (var p = 0; p < 2; p++)
|
||||
{
|
||||
t.Columns.Add($"k{p}", typeof(short));
|
||||
t.Columns.Add($"n{p}", typeof(double));
|
||||
t.Columns.Add($"i{p}", typeof(long));
|
||||
t.Columns.Add($"s{p}", typeof(string));
|
||||
t.Columns.Add($"t{p}", typeof(DateTime));
|
||||
}
|
||||
for (var p = 0; p < 2; p++)
|
||||
{
|
||||
t.Columns.Add($"at{p}", typeof(DateTime));
|
||||
t.Columns.Add($"ak{p}", typeof(short));
|
||||
t.Columns.Add($"an{p}", typeof(double));
|
||||
t.Columns.Add($"ai{p}", typeof(long));
|
||||
t.Columns.Add($"as{p}", typeof(string));
|
||||
t.Columns.Add($"st{p}", typeof(long));
|
||||
}
|
||||
return t;
|
||||
}
|
||||
|
||||
private static DataRow Counts(DataTable t, long bucket, long total)
|
||||
{
|
||||
var row = t.NewRow();
|
||||
row[0] = bucket;
|
||||
row[1] = total; row[2] = total; row[3] = total; row[4] = 0L; row[5] = 0L;
|
||||
return row;
|
||||
}
|
||||
|
||||
private static BucketRow Map(DataTable t, AggregateFamily family)
|
||||
{
|
||||
using var r = t.CreateDataReader();
|
||||
r.Read().ShouldBeTrue();
|
||||
return QuestDbHistorianDataSource.ReadBucket(r, family);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_row_with_counts_but_null_first_and_last_of_any_quality_maps_to_absent_picks_without_throwing()
|
||||
{
|
||||
var t = StartEndTable();
|
||||
var row = Counts(t, 3, 2);
|
||||
// A non-matching BYTE pick reads back as kind 0 (BYTE has no NULL) with every other column NULL.
|
||||
row["k0"] = (short)0;
|
||||
row["k1"] = (short)0;
|
||||
t.Rows.Add(row);
|
||||
|
||||
var b = Map(t, AggregateFamily.StartEnd);
|
||||
b.Bucket.ShouldBe(3);
|
||||
b.Total.ShouldBe(2);
|
||||
b.FirstAny.ShouldBeNull();
|
||||
b.LastAny.ShouldBeNull();
|
||||
b.First.Kind.ShouldBe(HistorianValueKind.Empty);
|
||||
b.Last.Kind.ShouldBe(HistorianValueKind.Empty);
|
||||
|
||||
foreach (var agg in new[] { HistoryAggregateType.Start, HistoryAggregateType.End, HistoryAggregateType.Delta })
|
||||
AggregateEvaluator.Evaluate(agg, b, T, T).StatusCode.ShouldBe(NoDataCalc);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_consistent_row_still_maps_every_pick()
|
||||
{
|
||||
var t = StartEndTable();
|
||||
var row = Counts(t, 0, 2);
|
||||
row["k0"] = (short)1; row["n0"] = 10d; row["t0"] = T;
|
||||
row["k1"] = (short)1; row["n1"] = 20d; row["t1"] = T.AddSeconds(1);
|
||||
row["at0"] = T; row["ak0"] = (short)1; row["an0"] = 10d; row["st0"] = 0L;
|
||||
row["at1"] = T.AddSeconds(1); row["ak1"] = (short)1; row["an1"] = 20d; row["st1"] = 0L;
|
||||
t.Rows.Add(row);
|
||||
|
||||
var b = Map(t, AggregateFamily.StartEnd);
|
||||
b.First.ShouldBe(HistorianValue.FromDouble(10));
|
||||
b.Last.ShouldBe(HistorianValue.FromDouble(20));
|
||||
b.FirstAny.ShouldNotBeNull().TimeUtc.ShouldBe(T);
|
||||
b.LastAny.ShouldNotBeNull().Value.ShouldBe(HistorianValue.FromDouble(20));
|
||||
AggregateEvaluator.Evaluate(HistoryAggregateType.Delta, b, T, T).Value.ShouldBe(10d);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void An_actual_time_row_with_counts_but_no_extreme_is_no_data()
|
||||
{
|
||||
var t = new DataTable();
|
||||
t.Columns.Add("b", typeof(long));
|
||||
foreach (var c in new[] { "total", "usable", "good", "bad", "nonnum" }) t.Columns.Add(c, typeof(long));
|
||||
t.Columns.Add("num_value", typeof(double));
|
||||
t.Columns.Add("ts", typeof(DateTime));
|
||||
t.Rows.Add(0L, 1L, 1L, 1L, 0L, 0L, DBNull.Value, DBNull.Value);
|
||||
|
||||
var b = Map(t, AggregateFamily.ActualTime);
|
||||
b.Extreme.ShouldBeNull();
|
||||
AggregateEvaluator.Evaluate(HistoryAggregateType.MaximumActualTime, b, T, T).StatusCode.ShouldBe(NoDataCalc);
|
||||
}
|
||||
}
|
||||
+197
-18
@@ -21,31 +21,36 @@ public sealed class AggregateEvaluatorTests
|
||||
Bucket = 0, Total = 2, Usable = 2, Good = 2, Bad = 0, NonNumeric = 0, Avg = 15, Min = 10, Max = 20, Sum = 30,
|
||||
First = HistorianValue.FromDouble(10), Last = HistorianValue.FromDouble(20),
|
||||
Extreme = maximum ? 20 : 10, ExtremeTime = maximum ? At(10) : At(0),
|
||||
FirstAny = A[0], LastAny = A[1],
|
||||
},
|
||||
[1] = new BucketRow
|
||||
{
|
||||
Bucket = 1, Total = 2, Usable = 1, Good = 1, Bad = 1, NonNumeric = 0, Avg = 40, Min = 40, Max = 40, Sum = 40,
|
||||
First = HistorianValue.FromDouble(40), Last = HistorianValue.FromDouble(40), Extreme = 40, ExtremeTime = At(30),
|
||||
FirstAny = A[2], LastAny = A[3],
|
||||
},
|
||||
[2] = new BucketRow
|
||||
{
|
||||
Bucket = 2, Total = 1, Usable = 1, Good = 1, Bad = 0, NonNumeric = 0, Avg = 50, Min = 50, Max = 50, Sum = 50,
|
||||
First = HistorianValue.FromDouble(50), Last = HistorianValue.FromDouble(50), Extreme = 50, ExtremeTime = At(40),
|
||||
FirstAny = A[4], LastAny = A[4],
|
||||
},
|
||||
};
|
||||
|
||||
private const uint Partial = 0x4u;
|
||||
private const uint BadCalc = 0x80000401u; // generic Bad | Calculated
|
||||
|
||||
/// <summary>Tag A's data runs +0 … +40, so the forward read [0,80) holds the end of the data in B2 [40,60).</summary>
|
||||
private static IReadOnlyList<DataValueSnapshot> ReadA(HistoryAggregateType agg, bool maximum = false)
|
||||
{
|
||||
var plan = BucketPlan.Create(At(0), At(80), TimeSpan.FromSeconds(20));
|
||||
return AggregateEvaluator.EvaluateAll(agg, plan, RowsA(maximum).Values, At(0), Now);
|
||||
return AggregateEvaluator.EvaluateAll(agg, plan, RowsA(maximum).Values, At(0), At(40), Now);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(HistoryAggregateType.Average, 15d, 40d, 50d)]
|
||||
[InlineData(HistoryAggregateType.Minimum, 10d, 40d, 50d)]
|
||||
[InlineData(HistoryAggregateType.Maximum, 20d, 40d, 50d)]
|
||||
[InlineData(HistoryAggregateType.Start, 10d, 40d, 50d)]
|
||||
[InlineData(HistoryAggregateType.End, 20d, 40d, 50d)]
|
||||
[InlineData(HistoryAggregateType.Delta, 10d, 0d, 0d)]
|
||||
[InlineData(HistoryAggregateType.Range, 10d, 0d, 0d)]
|
||||
public void Numeric_aggregates_over_hand_built_bucket_rows(HistoryAggregateType agg, double b0, double b1, double b2)
|
||||
@@ -53,26 +58,148 @@ public sealed class AggregateEvaluatorTests
|
||||
var s = ReadA(agg);
|
||||
s.Count.ShouldBe(4);
|
||||
s.Select(x => x.Value).ShouldBe([b0, b1, b2, null]);
|
||||
s.Select(x => x.StatusCode).ShouldBe([GoodCalc, UncertainCalc, GoodCalc, NoDataCalc]);
|
||||
// #680: B2 holds the last sample (+40) ⇒ end-of-data Partial on a forward read, for every aggregate but Average.
|
||||
// B1 opens on a Bad sample ⇒ Uncertain (for Delta: the interval's first sample was skipped).
|
||||
var b2Status = agg == HistoryAggregateType.Average ? GoodCalc : GoodCalc | Partial;
|
||||
s.Select(x => x.StatusCode).ShouldBe([GoodCalc, UncertainCalc, b2Status, NoDataCalc]);
|
||||
s.Select(x => x.SourceTimestampUtc).ShouldBe([At(0), At(20), At(40), At(60)]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Count_counts_usable_samples_as_int()
|
||||
public void Start_and_End_return_the_raw_first_and_last_sample_of_any_quality()
|
||||
{
|
||||
var s = ReadA(HistoryAggregateType.Count);
|
||||
s.Select(x => x.Value).ShouldBe([(object?)2, 1, 1, null]);
|
||||
s.Select(x => x.StatusCode).ShouldBe([GoodCalc, UncertainCalc, GoodCalc, NoDataCalc]);
|
||||
// #680 (Part 13 Tables 73/74; SDK ComputeStartEnd): the raw sample — its own value (null when Bad), its own status
|
||||
// (Raw, no Calculated bit) and its own timestamp. B1's Start is the Bad sample at +20.
|
||||
var start = ReadA(HistoryAggregateType.Start);
|
||||
start.Select(x => x.Value).ShouldBe([10d, null, 50d, null]);
|
||||
start.Select(x => x.StatusCode).ShouldBe([Good, Bad, Good | 0x400u | Partial, NoDataCalc]);
|
||||
start.Select(x => x.SourceTimestampUtc).ShouldBe([At(0), At(20), At(40), At(60)]);
|
||||
var end = ReadA(HistoryAggregateType.End);
|
||||
end.Select(x => x.Value).ShouldBe([20d, 40d, 50d, null]);
|
||||
end.Select(x => x.StatusCode).ShouldBe([Good, Good, Good | 0x400u | Partial, NoDataCalc]);
|
||||
end.Select(x => x.SourceTimestampUtc).ShouldBe([At(10), At(30), At(40), At(60)]);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(HistoryAggregateType.PercentGood, 100d, 50d, 100d)]
|
||||
[InlineData(HistoryAggregateType.PercentBad, 0d, 50d, 0d)]
|
||||
public void Percent_aggregates_count_every_sample_and_stay_good(HistoryAggregateType agg, double b0, double b1, double b2)
|
||||
[Fact]
|
||||
public void Start_and_End_follow_the_sample_order_in_a_reverse_read()
|
||||
{
|
||||
var s = ReadA(agg);
|
||||
s.Select(x => x.Value).ShouldBe([b0, b1, b2, null]);
|
||||
s.Select(x => x.StatusCode).ShouldBe([GoodCalc, GoodCalc, GoodCalc, NoDataCalc]);
|
||||
// Reverse (80 → 0): (20,40] = {+30, +40}, (0,20] = {+10, +20 Bad}. Start is still the EARLIEST sample of each
|
||||
// interval and End the latest (the SDK's GetValues list is ascending in both directions).
|
||||
var plan = BucketPlan.Create(At(80), At(0), TimeSpan.FromSeconds(20));
|
||||
var rows = new[]
|
||||
{
|
||||
new BucketRow { Bucket = 1, Total = 2, Usable = 2, Good = 2, FirstAny = A[3], LastAny = A[4] },
|
||||
new BucketRow { Bucket = 0, Total = 2, Usable = 1, Good = 1, Bad = 1, FirstAny = A[1], LastAny = A[2] },
|
||||
};
|
||||
var start = AggregateEvaluator.EvaluateAll(HistoryAggregateType.Start, plan, rows, At(0), At(40), Now);
|
||||
start.Select(x => x.SourceTimestampUtc).ShouldBe([At(80), At(60), At(30), At(10)]);
|
||||
start.Select(x => x.Value).ShouldBe([null, null, 40d, 20d]);
|
||||
var end = AggregateEvaluator.EvaluateAll(HistoryAggregateType.End, plan, rows, At(0), At(40), Now);
|
||||
end.Select(x => x.SourceTimestampUtc).ShouldBe([At(80), At(60), At(40), At(20)]);
|
||||
end.Select(x => x.Value).ShouldBe([null, null, 50d, null]);
|
||||
end.Select(x => x.StatusCode).ShouldBe([NoDataCalc, NoDataCalc, Good, Bad]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Delta_is_good_when_a_bad_sample_sits_mid_interval_and_uncertain_when_one_bounds_it()
|
||||
{
|
||||
// SDK ComputeDelta: Uncertain only when the first or last sample of the interval is skipped (not usable).
|
||||
var mid = new BucketRow
|
||||
{
|
||||
Total = 3, Usable = 2, Good = 2, Bad = 1, First = HistorianValue.FromDouble(1), Last = HistorianValue.FromDouble(4),
|
||||
FirstAny = S(0, HistorianValue.FromDouble(1), Good), LastAny = S(9, HistorianValue.FromDouble(4), Good),
|
||||
};
|
||||
var d = AggregateEvaluator.Evaluate(HistoryAggregateType.Delta, mid, At(0), Now);
|
||||
d.Value.ShouldBe(3d);
|
||||
d.StatusCode.ShouldBe(GoodCalc);
|
||||
mid.LastAny = S(9, HistorianValue.Empty, BadNoCommunication);
|
||||
AggregateEvaluator.Evaluate(HistoryAggregateType.Delta, mid, At(0), Now).StatusCode.ShouldBe(UncertainCalc);
|
||||
// An Uncertain bound is used (TreatUncertainAsBad = false), so it does not degrade Delta.
|
||||
mid.LastAny = S(9, HistorianValue.FromDouble(4), Uncertain);
|
||||
AggregateEvaluator.Evaluate(HistoryAggregateType.Delta, mid, At(0), Now).StatusCode.ShouldBe(GoodCalc);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Count_counts_good_samples_as_int()
|
||||
{
|
||||
// #680 (Part 13 §5.4.3.21; SDK ComputeCount): Good samples only; value-based status; end-of-data Partial on B2.
|
||||
var s = ReadA(HistoryAggregateType.Count);
|
||||
s.Select(x => x.Value).ShouldBe([(object?)2, 1, 1, null]);
|
||||
s.Select(x => x.StatusCode).ShouldBe([GoodCalc, UncertainCalc, GoodCalc | Partial, NoDataCalc]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Count_excludes_uncertain_samples_and_an_all_bad_interval_is_generic_bad()
|
||||
{
|
||||
var unc = new BucketRow { Total = 3, Usable = 3, Good = 1, Bad = 0 };
|
||||
var c = AggregateEvaluator.Evaluate(HistoryAggregateType.Count, unc, At(0), Now);
|
||||
c.Value.ShouldBe(1);
|
||||
c.StatusCode.ShouldBe(UncertainCalc);
|
||||
var bad = AggregateEvaluator.Evaluate(HistoryAggregateType.Count, new BucketRow { Total = 2, Bad = 2 }, At(0), Now);
|
||||
bad.Value.ShouldBeNull();
|
||||
bad.StatusCode.ShouldBe(BadCalc);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void An_empty_interval_inside_the_data_range_counts_zero_and_outside_it_is_no_data()
|
||||
{
|
||||
// #680: forward [0,60) at 10 s over data +1 … +45: [10,20) and [30,40) hold no sample but lie inside
|
||||
// [earliest, latest] ⇒ 0, Good; [50,60) is after the last sample ⇒ BadNoData.
|
||||
var plan = BucketPlan.Create(At(0), At(60), TimeSpan.FromSeconds(10));
|
||||
var rows = new[]
|
||||
{
|
||||
new BucketRow { Bucket = 0, Total = 1, Usable = 1, Good = 1 },
|
||||
new BucketRow { Bucket = 2, Total = 1, Usable = 1, Good = 1 },
|
||||
new BucketRow { Bucket = 4, Total = 1, Usable = 1, Good = 1 },
|
||||
};
|
||||
var s = AggregateEvaluator.EvaluateAll(HistoryAggregateType.Count, plan, rows, At(1), At(45), Now);
|
||||
s.Select(x => x.Value).ShouldBe([(object?)1, 0, 1, 0, 1, null]);
|
||||
s.Select(x => x.StatusCode).ShouldBe(
|
||||
[GoodCalc | Partial, GoodCalc, GoodCalc, GoodCalc, GoodCalc | Partial, NoDataCalc]);
|
||||
s.Select(x => x.SourceTimestampUtc).ShouldBe([At(0), At(10), At(20), At(30), At(40), At(50)]);
|
||||
|
||||
// Reverse (60 → 0): (40,50] is inside the range (its earlier edge +40 < +45); (50,60] is after it. The last sample
|
||||
// (+45) lies strictly inside (40,50] ⇒ Partial there; the first sample's interval is not flagged in reverse.
|
||||
var rev = BucketPlan.Create(At(60), At(0), TimeSpan.FromSeconds(10));
|
||||
var r = AggregateEvaluator.EvaluateAll(HistoryAggregateType.Count, rev, rows, At(1), At(45), Now);
|
||||
r.Select(x => x.Value).ShouldBe([null, (object?)1, 0, 1, 0, 1]);
|
||||
r.Select(x => x.StatusCode).ShouldBe([NoDataCalc, GoodCalc | Partial, GoodCalc, GoodCalc, GoodCalc, GoodCalc]);
|
||||
|
||||
// Other per-sample aggregates keep BadNoData on an empty interval.
|
||||
AggregateEvaluator.EvaluateAll(HistoryAggregateType.Minimum, plan, rows, At(1), At(45), Now)[1].StatusCode
|
||||
.ShouldBe(NoDataCalc);
|
||||
// No data range known (an unknown tag / no sample near the window) ⇒ every empty interval is BadNoData.
|
||||
AggregateEvaluator.EvaluateAll(HistoryAggregateType.Count, plan, [], null, null, Now)
|
||||
.ShouldAllBe(x => x.StatusCode == NoDataCalc);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void End_of_data_Partial_includes_the_interval_start_forward_and_is_strict_in_reverse()
|
||||
{
|
||||
var row = new BucketRow { Bucket = 2, Total = 1, Usable = 1, Good = 1, Min = 3, Avg = 3 };
|
||||
var fwd = BucketPlan.Create(At(0), At(40), TimeSpan.FromSeconds(10));
|
||||
// Latest sample exactly on the interval's earlier edge (+20) ⇒ Partial (SDK m_endOfData >= startTime).
|
||||
AggregateEvaluator.EvaluateAll(HistoryAggregateType.Minimum, fwd, [row], At(-5), At(20), Now)[2].StatusCode
|
||||
.ShouldBe(GoodCalc | Partial);
|
||||
// Data past the window end ⇒ no end-of-data Partial.
|
||||
AggregateEvaluator.EvaluateAll(HistoryAggregateType.Minimum, fwd, [row], At(-5), At(100), Now)[2].StatusCode
|
||||
.ShouldBe(GoodCalc);
|
||||
// Average never sets it.
|
||||
AggregateEvaluator.EvaluateAll(HistoryAggregateType.Average, fwd, [row], At(-5), At(25), Now)[2].StatusCode
|
||||
.ShouldBe(GoodCalc);
|
||||
// A Bad result never carries Partial (ours; the SDK sets it on a Bad result too).
|
||||
var bad = new BucketRow { Bucket = 2, Total = 1, Bad = 1 };
|
||||
AggregateEvaluator.EvaluateAll(HistoryAggregateType.Minimum, fwd, [bad], At(-5), At(25), Now)[2].StatusCode
|
||||
.ShouldBe(NoDataCalc);
|
||||
// Reverse (40 → 0; bucket 2 = (20,30], result index 1): the latest sample strictly inside ⇒ Partial (the SDK's
|
||||
// reverse m_startOfData); on the interval's later edge (+30) or its excluded earlier edge (+20) ⇒ nothing.
|
||||
var rev = BucketPlan.Create(At(40), At(0), TimeSpan.FromSeconds(10));
|
||||
AggregateEvaluator.EvaluateAll(HistoryAggregateType.Minimum, rev, [row], At(-5), At(25), Now)
|
||||
.Select(x => x.StatusCode & Partial).ShouldBe([0u, Partial, 0u, 0u]);
|
||||
AggregateEvaluator.EvaluateAll(HistoryAggregateType.Minimum, rev, [row], At(-5), At(30), Now)
|
||||
.ShouldAllBe(x => (x.StatusCode & Partial) == 0);
|
||||
AggregateEvaluator.EvaluateAll(HistoryAggregateType.Average, rev, [row], At(-5), At(25), Now)
|
||||
.ShouldAllBe(x => (x.StatusCode & Partial) == 0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -81,7 +208,7 @@ public sealed class AggregateEvaluatorTests
|
||||
var min = ReadA(HistoryAggregateType.MinimumActualTime);
|
||||
min.Select(x => x.Value).ShouldBe([10d, 40d, 50d, null]);
|
||||
min.Select(x => x.SourceTimestampUtc).ShouldBe([At(0), At(30), At(40), At(60)]);
|
||||
min.Select(x => x.StatusCode).ShouldBe([GoodCalc, UncertainCalc, GoodCalc, NoDataCalc]);
|
||||
min.Select(x => x.StatusCode).ShouldBe([GoodCalc, UncertainCalc, GoodCalc | Partial, NoDataCalc]);
|
||||
var max = ReadA(HistoryAggregateType.MaximumActualTime, maximum: true);
|
||||
max.Select(x => x.Value).ShouldBe([20d, 40d, 50d, null]);
|
||||
max.Select(x => x.SourceTimestampUtc).ShouldBe([At(10), At(30), At(40), At(60)]);
|
||||
@@ -100,13 +227,17 @@ public sealed class AggregateEvaluatorTests
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_bucket_with_only_bad_samples_is_no_data_except_for_the_percentages()
|
||||
public void A_bucket_with_only_bad_samples_is_no_data_except_for_count()
|
||||
{
|
||||
var row = new BucketRow { Total = 2, Usable = 0, Good = 0, Bad = 2 };
|
||||
var avg = AggregateEvaluator.Evaluate(HistoryAggregateType.Average, row, At(0), Now);
|
||||
avg.Value.ShouldBeNull();
|
||||
avg.StatusCode.ShouldBe(NoDataCalc);
|
||||
AggregateEvaluator.Evaluate(HistoryAggregateType.PercentBad, row, At(0), Now).Value.ShouldBe(100d);
|
||||
// PercentGood / PercentBad are duration-based (time-weighted family) since #680: not served here.
|
||||
Should.Throw<HistoryAggregateNotSupportedException>(() =>
|
||||
AggregateEvaluator.Evaluate(HistoryAggregateType.PercentBad, row, At(0), Now));
|
||||
// #680: Count of an all-Bad interval is the generic Bad (the SDK's value-based status), not BadNoData.
|
||||
AggregateEvaluator.Evaluate(HistoryAggregateType.Count, row, At(0), Now).StatusCode.ShouldBe(BadCalc);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -126,4 +257,52 @@ public sealed class AggregateEvaluatorTests
|
||||
avg.Value.ShouldBe(15d);
|
||||
avg.StatusCode.ShouldBe(UncertainCalc);
|
||||
}
|
||||
|
||||
// ---- Live-gate 8 F1: a row whose counts and picks disagree (a backend that read them from two table versions).
|
||||
|
||||
/// <summary>Counts say B0 holds two Good samples, but none of the picked samples came back.</summary>
|
||||
private static BucketRow InconsistentRow() => new()
|
||||
{
|
||||
Bucket = 0, Total = 2, Usable = 2, Good = 2, Bad = 0, NonNumeric = 0, Avg = 15, Min = 10, Max = 20, Sum = 30,
|
||||
First = HistorianValue.Empty, Last = HistorianValue.Empty, Extreme = null, ExtremeTime = null,
|
||||
FirstAny = null, LastAny = null,
|
||||
};
|
||||
|
||||
[Theory]
|
||||
[InlineData(HistoryAggregateType.Start)]
|
||||
[InlineData(HistoryAggregateType.End)]
|
||||
[InlineData(HistoryAggregateType.Delta)]
|
||||
[InlineData(HistoryAggregateType.MinimumActualTime)]
|
||||
[InlineData(HistoryAggregateType.MaximumActualTime)]
|
||||
public void A_bucket_whose_picked_samples_are_missing_is_no_data_not_a_throw_or_a_valueless_good(HistoryAggregateType agg)
|
||||
{
|
||||
var v = AggregateEvaluator.Evaluate(agg, InconsistentRow(), At(0), Now);
|
||||
v.Value.ShouldBeNull();
|
||||
v.StatusCode.ShouldBe(NoDataCalc);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_delta_whose_first_usable_pick_alone_is_missing_is_no_data_not_invalid_inputs()
|
||||
{
|
||||
var row = InconsistentRow();
|
||||
row.Last = HistorianValue.FromDouble(20);
|
||||
row.FirstAny = A[0];
|
||||
row.LastAny = A[1];
|
||||
AggregateEvaluator.Evaluate(HistoryAggregateType.Delta, row, At(0), Now).StatusCode.ShouldBe(NoDataCalc);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void An_inconsistent_bucket_does_not_fail_the_whole_read_and_its_neighbours_keep_their_values()
|
||||
{
|
||||
var rows = RowsA();
|
||||
rows[0] = InconsistentRow();
|
||||
var plan = BucketPlan.Create(At(0), At(80), TimeSpan.FromSeconds(20));
|
||||
foreach (var agg in new[] { HistoryAggregateType.Start, HistoryAggregateType.Delta, HistoryAggregateType.MaximumActualTime })
|
||||
{
|
||||
var s = AggregateEvaluator.EvaluateAll(agg, plan, rows.Values, At(0), At(40), Now);
|
||||
s.Count.ShouldBe(4);
|
||||
s[0].StatusCode.ShouldBe(NoDataCalc);
|
||||
s[2].Value.ShouldNotBeNull();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+42
-6
@@ -133,15 +133,51 @@ public sealed class BucketPlanTests
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void The_earliest_sample_is_needed_only_for_forward_reads_of_aggregates_other_than_Average()
|
||||
public void Data_ends_inside_from_the_interval_start_inclusive_to_its_end_exclusive()
|
||||
{
|
||||
// Review F2: backends skip the earliest-sample query when its answer cannot change a result.
|
||||
var p = BucketPlan.Create(At(0), At(40), TimeSpan.FromSeconds(20));
|
||||
p.DataEndsInside(0, At(5)).ShouldBeTrue();
|
||||
p.DataEndsInside(1, At(20)).ShouldBeTrue(); // on the earlier edge: the SDK's m_endOfData >= startTime
|
||||
p.DataEndsInside(0, At(20)).ShouldBeFalse();
|
||||
p.DataEndsInside(1, At(40)).ShouldBeFalse();
|
||||
p.DataEndsInside(0, null).ShouldBeFalse();
|
||||
// Reverse (40 → 0): (0,20], (20,40] — strictly inside only (the SDK's reverse m_startOfData test).
|
||||
var r = BucketPlan.Create(At(40), At(0), TimeSpan.FromSeconds(20));
|
||||
r.DataEndsInside(1, At(30)).ShouldBeTrue();
|
||||
r.DataEndsInside(1, At(40)).ShouldBeFalse();
|
||||
r.DataEndsInside(1, At(20)).ShouldBeFalse();
|
||||
r.DataEndsInside(0, At(20)).ShouldBeFalse();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void The_data_range_includes_both_of_its_edges()
|
||||
{
|
||||
var p = BucketPlan.Create(At(0), At(60), TimeSpan.FromSeconds(20));
|
||||
p.WithinDataRange(0, At(10), At(30)).ShouldBeTrue();
|
||||
p.WithinDataRange(1, At(10), At(30)).ShouldBeTrue();
|
||||
p.WithinDataRange(2, At(10), At(30)).ShouldBeFalse(); // [40,60) is after the last sample
|
||||
p.WithinDataRange(2, At(10), At(40)).ShouldBeTrue(); // last sample on the earlier edge
|
||||
p.WithinDataRange(0, At(20), At(30)).ShouldBeTrue(); // first sample on the later edge
|
||||
p.WithinDataRange(0, At(21), At(30)).ShouldBeFalse();
|
||||
p.WithinDataRange(0, null, At(30)).ShouldBeFalse();
|
||||
p.WithinDataRange(0, At(10), null).ShouldBeFalse();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void The_data_range_is_needed_for_every_per_sample_aggregate_but_Average()
|
||||
{
|
||||
// Review F2 (#677) / #680: backends skip the earliest/latest-sample queries when their answer cannot change a
|
||||
// result — the start/end-of-data Partial rules are not Average's (end-of-data applies in reverse too); Count's in-range empty interval
|
||||
// (0, not BadNoData) applies in both directions.
|
||||
var fwd = BucketPlan.Create(At(0), At(40), TimeSpan.FromSeconds(10));
|
||||
var rev = BucketPlan.Create(At(40), At(0), TimeSpan.FromSeconds(10));
|
||||
AggregateEvaluator.NeedsEarliestSample(HistoryAggregateType.Count, fwd).ShouldBeTrue();
|
||||
AggregateEvaluator.NeedsEarliestSample(HistoryAggregateType.Average, fwd).ShouldBeFalse();
|
||||
AggregateEvaluator.NeedsEarliestSample(HistoryAggregateType.Count, rev).ShouldBeFalse();
|
||||
AggregateEvaluator.NeedsEarliestSample(HistoryAggregateType.Count, BucketPlan.Create(At(0), At(0), TimeSpan.Zero))
|
||||
AggregateEvaluator.NeedsDataRange(HistoryAggregateType.Count, fwd).ShouldBeTrue();
|
||||
AggregateEvaluator.NeedsDataRange(HistoryAggregateType.Minimum, fwd).ShouldBeTrue();
|
||||
AggregateEvaluator.NeedsDataRange(HistoryAggregateType.Average, fwd).ShouldBeFalse();
|
||||
AggregateEvaluator.NeedsDataRange(HistoryAggregateType.Count, rev).ShouldBeTrue();
|
||||
AggregateEvaluator.NeedsDataRange(HistoryAggregateType.Minimum, rev).ShouldBeTrue(); // reverse end-of-data
|
||||
AggregateEvaluator.NeedsDataRange(HistoryAggregateType.Average, rev).ShouldBeFalse();
|
||||
AggregateEvaluator.NeedsDataRange(HistoryAggregateType.Count, BucketPlan.Create(At(0), At(0), TimeSpan.Zero))
|
||||
.ShouldBeFalse();
|
||||
}
|
||||
|
||||
|
||||
+4
-2
@@ -11,10 +11,12 @@ public sealed class ProcessedReadTests
|
||||
public void Every_enum_member_is_either_served_or_refused()
|
||||
{
|
||||
// Seventeen served (#671 added TimeAverage, TimeAverage2, Interpolative and Total2, and moved Total to the
|
||||
// time-weighted family); anything appended to the enum later falls to the refusal (total default).
|
||||
// time-weighted family; #680 moved PercentGood/PercentBad there too — they are duration-based); anything appended to
|
||||
// the enum later falls to the refusal (total default).
|
||||
Enum.GetValues<HistoryAggregateType>().Count(a => AggregateFamilies.FamilyOf(a) is not null).ShouldBe(17);
|
||||
Enum.GetValues<HistoryAggregateType>().Where(a => AggregateFamilies.FamilyOf(a) == AggregateFamily.TimeWeighted)
|
||||
.ShouldBe([HistoryAggregateType.Total, HistoryAggregateType.Interpolative, HistoryAggregateType.TimeAverage,
|
||||
HistoryAggregateType.TimeAverage2, HistoryAggregateType.Total2]);
|
||||
HistoryAggregateType.PercentGood, HistoryAggregateType.PercentBad, HistoryAggregateType.TimeAverage2,
|
||||
HistoryAggregateType.Total2]);
|
||||
}
|
||||
}
|
||||
|
||||
+363
-51
@@ -16,31 +16,44 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common.Tests;
|
||||
/// <remarks>
|
||||
/// Deliberate, documented divergences the datasets avoid (they are not anchoring questions):
|
||||
/// <list type="bullet">
|
||||
/// <item>an <b>empty</b> interval inside the data range: the SDK's Count answers 0/Good, ours <c>BadNoData</c>;</item>
|
||||
/// <item><b>Uncertain</b> samples in a per-sample aggregate: the SDK's Average/Count/Min/Max use Good samples only,
|
||||
/// ours every usable (Good + Uncertain) sample (TreatUncertainAsBad = false applied consistently);</item>
|
||||
/// <item><b>Uncertain</b> samples in a per-sample aggregate: the SDK's Average/Min/Max use Good samples only,
|
||||
/// ours every usable (Good + Uncertain) sample (TreatUncertainAsBad = false applied consistently). Count matches
|
||||
/// the SDK since #680 (Good samples only);</item>
|
||||
/// <item>the per-sample <b>Calculated / MultipleValues</b> info bits (the SDK's MinMax sets Calculated only when the
|
||||
/// extreme is not at the interval start, and flags ties) — those comparisons check code bits + Partial only;</item>
|
||||
/// <item>a <b>truncated</b> interval: ours sets Partial on a non-Bad time-weighted result (the SDK does not), so the
|
||||
/// non-multiple windows mask that one bucket's Partial bit;</item>
|
||||
/// <item>the SDK's <b>end-of-data</b> Partial rule (forward, the interval holding the last sample when nothing follows
|
||||
/// and nothing was extrapolated) is not implemented: every dataset carries data past the window end.</item>
|
||||
/// </list>
|
||||
/// Pre-existing forward divergences this parity test surfaced (#671 behaviour, unchanged here — forward results were
|
||||
/// to stay as they are; recorded for a follow-up rather than fixed in place):
|
||||
/// <list type="bullet">
|
||||
/// <item><b>TimeAverage / Total status:</b> the SDK (interpolated bounds) reports <c>UncertainDataSubNormal</c> when
|
||||
/// <i>any</i> region of the interval is non-Good (e.g. a segment bridged across a Bad sample); ours applies the
|
||||
/// time-based rule (Uncertain time counts as Good) to these two as well, so a bridged-but-fully-covered interval
|
||||
/// is Good. Values and timestamps agree; the status code bits are not compared for these two.</item>
|
||||
/// <item><b>Total over an interval the data starts inside:</b> the SDK integrates only the defined part
|
||||
/// (value·s over the covered seconds); ours is <c>TimeAverage × interval seconds</c>. The Partial bit and timestamp
|
||||
/// agree; that value is not compared.</item>
|
||||
/// <item>a <b>truncated</b> interval: ours sets Partial on a non-Bad time-weighted result (Part 13 §5.3.3.2 Case 3
|
||||
/// requires it; the SDK does not), so the non-multiple windows mask that one bucket's Partial bit;</item>
|
||||
/// <item>a time-weighted <c>BadNoData</c> carries the Calculated bit in ours (every processed status does), not in the
|
||||
/// SDK's — masked by <see cref="MaskDocumented"/>;</item>
|
||||
/// <item><b>TimeAverage2 / Total2 over an interval wholly inside an outage:</b> ours generic <c>Bad</c> (data, but no
|
||||
/// non-Bad time — Part 13's PercentData rule), the SDK <c>BadNoData</c> — masked by <see cref="MaskDocumented"/>;</item>
|
||||
/// <item><b>the end-of-data interval</b> (the last sample inside it, nothing after) of TimeAverage2 / Total2 /
|
||||
/// PercentGood / PercentBad: the SDK gives the region after the last sample a 1 ms duration (an artifact of its
|
||||
/// <c>GetRegionsInValueSet</c>); Part 13 ends TimeAverage2/Total2 at the last sample and runs the PercentGood/Bad
|
||||
/// region to the interval end, which ours does (#680; pinned by <c>TimeWeightedAggregatorTests</c>). That
|
||||
/// interval's value is not compared; its status, Partial bit and timestamp are.</item>
|
||||
/// <item><b>an outage marker as the last sample</b> (<see cref="OutageLast"/>): ours runs the outage to the window
|
||||
/// end — TimeAverage2/Total2 <c>Bad</c>, the percentages 100 % Bad, no end-of-data Partial on the interval holding
|
||||
/// it; the SDK treats the data as ended (<c>BadNoData</c> after it, Partial on that interval); and TimeAverage/Total
|
||||
/// over the span the last usable value is held up to the marker are Uncertain here (no Good value follows ⇒
|
||||
/// extrapolated, Part 13), Good in the SDK — masked by
|
||||
/// <see cref="MaskDocumented"/>;</item>
|
||||
/// <item><b>Booleans</b> (<see cref="BoolSwitch"/>): ours steps them (Part 13: a non-analog value is held, and the
|
||||
/// bound next to an Uncertain sample takes the earlier sample's status); the SDK interpolates them as numbers — its
|
||||
/// values (e.g. 1 vs our 0.5 over an interval switching half-way) and the Good/Uncertain choice are not compared,
|
||||
/// Bad vs non-Bad, timestamps and Partial are;</item>
|
||||
/// <item><b>TimeAverage / Total, reverse, a sample exactly on an interval's later edge:</b> the SDK's backward
|
||||
/// <c>UpdateSlice</c> files it as the early bound (below), so the interval takes the status of the segment beyond
|
||||
/// it (Uncertain when that segment skips a Bad sample); ours keeps the interval's own segments — masked by
|
||||
/// <see cref="MaskDocumented"/>;</item>
|
||||
/// <item>a <b>Bad</b> per-sample result never carries Partial (the SDK sets start/end-of-data Partial on Bad results
|
||||
/// too) — the datasets keep the data's start and end out of all-Bad intervals;</item>
|
||||
/// <item><b>Interpolative, reverse, a sample exactly at the point:</b> the SDK's backward <c>UpdateSlice</c> files
|
||||
/// a sample AT the point as the early bound (<c>>= 0</c>), so it interpolates from it to the next Good sample —
|
||||
/// same value, but the Interpolated bit, and Uncertain when a Bad sample sits between. Ours returns the raw
|
||||
/// sample with its own status in both directions (as the SDK itself does forward). Reverse Interpolative
|
||||
/// therefore compares timestamps and values (the anchoring question) but not status.</item>
|
||||
/// therefore compares timestamps and values (the anchoring question) but not status. (The SDK's asymmetry —
|
||||
/// forward it returns the raw sample — is its defect; ours is the same in both directions.)</item>
|
||||
/// </list>
|
||||
/// </remarks>
|
||||
public sealed class SdkAggregateParityTests
|
||||
@@ -64,6 +77,74 @@ public sealed class SdkAggregateParityTests
|
||||
S(100, HistorianValue.FromDouble(60), Good),
|
||||
];
|
||||
|
||||
/// <summary>
|
||||
/// #680: data that ENDS inside every window (nothing after +52.5): every 2.5 s from −10, values cycling, a Bad
|
||||
/// sample at +22.5 and an Uncertain one at +40 — the end-of-data rules (TimeAverage2/Total2 stop at the last
|
||||
/// sample, Partial, then BadNoData; TimeAverage/Total extrapolate, Uncertain).
|
||||
/// </summary>
|
||||
private static readonly StoredSample[] EndsInside = Enumerable.Range(0, 26)
|
||||
.Select(i => (t: -10 + i * 2.5, i))
|
||||
.Select(x => S(x.t, HistorianValue.FromDouble(10 + (x.i * 7 % 13)),
|
||||
x.t == 22.5 ? Bad : x.t == 40 ? Uncertain : Good))
|
||||
.ToArray();
|
||||
|
||||
/// <summary>
|
||||
/// #680: an outage +18 … +45 that wholly contains an interval of every window ([20,40), [20,35), [23,43), (20,40]):
|
||||
/// TimeAverage/Total bridge it (Uncertain); TimeAverage2/Total2 have no non-Bad time there.
|
||||
/// </summary>
|
||||
private static readonly StoredSample[] OutageInside =
|
||||
[
|
||||
S(-10, HistorianValue.FromDouble(5), Good),
|
||||
S(0, HistorianValue.FromDouble(10), Good),
|
||||
S(15, HistorianValue.FromDouble(20), Good),
|
||||
S(18, HistorianValue.Empty, BadNoCommunication),
|
||||
S(45, HistorianValue.FromDouble(30), Good),
|
||||
S(60, HistorianValue.FromDouble(40), Good),
|
||||
S(100, HistorianValue.FromDouble(40), Good),
|
||||
];
|
||||
|
||||
/// <summary>
|
||||
/// #680 review: the comms-loss recovery shape — a Good sample, an outage marker, then ONE recovery sample and nothing
|
||||
/// after (+27.5, inside an interval of every window, never on an edge): the interval holding the recovery sample has
|
||||
/// no non-Bad time before it, so TimeAverage2/Total2 answer the sample's value (Total2 0), Uncertain + Partial.
|
||||
/// </summary>
|
||||
private static readonly StoredSample[] RecoveryEnds =
|
||||
[
|
||||
S(-10, HistorianValue.FromDouble(10), Good),
|
||||
S(5, HistorianValue.Empty, BadNoCommunication),
|
||||
S(27.5, HistorianValue.FromDouble(20), Good),
|
||||
];
|
||||
|
||||
/// <summary>
|
||||
/// #680 review: an outage marker as the LAST sample (+31, nothing after) — ours: the outage runs to the window end
|
||||
/// (TimeAverage2/Total2 Bad, the percentages 100 % Bad); the SDK: the data ended (BadNoData, Partial on the interval
|
||||
/// holding the marker). Also TimeAverage/Total over the span the last usable value is held up to the marker: ours
|
||||
/// Uncertain (Part 13 — no Good value after it, so extrapolated), the SDK Good. Documented, deliberate differences.
|
||||
/// </summary>
|
||||
private static readonly StoredSample[] OutageLast =
|
||||
[
|
||||
S(-10, HistorianValue.FromDouble(10), Good),
|
||||
S(25, HistorianValue.FromDouble(20), Good),
|
||||
S(31, HistorianValue.Empty, BadNoCommunication),
|
||||
];
|
||||
|
||||
/// <summary>
|
||||
/// #680 review: Boolean (stepped) data with an Uncertain and a Bad sample. Ours steps Booleans (Part 13: a
|
||||
/// non-analog value is held); the SDK interpolates them as numbers and bounds them sloped — so only timestamps,
|
||||
/// status and Partial are compared for the values it computes that way (see <see cref="MaskDocumented"/>).
|
||||
/// </summary>
|
||||
private static readonly StoredSample[] BoolSwitch =
|
||||
[
|
||||
S(-10, HistorianValue.FromBoolean(true), Good),
|
||||
S(10, HistorianValue.FromBoolean(false), Good),
|
||||
S(27.5, HistorianValue.FromBoolean(true), Uncertain),
|
||||
S(32, HistorianValue.FromBoolean(false), Good),
|
||||
S(47.5, HistorianValue.FromBoolean(true), Good),
|
||||
S(62, HistorianValue.FromBoolean(true), Bad),
|
||||
S(71, HistorianValue.FromBoolean(false), Good),
|
||||
S(100, HistorianValue.FromBoolean(true), Good),
|
||||
];
|
||||
|
||||
/// <summary>Data that starts inside the window (no sample before +7) and runs past its end.</summary>
|
||||
private static readonly StoredSample[] LateStart = Enumerable.Range(0, 20)
|
||||
.Select(i => S(7 + i * 5, HistorianValue.FromDouble(i % 2 == 0 ? 20 : 40), Good))
|
||||
@@ -85,7 +166,7 @@ public sealed class SdkAggregateParityTests
|
||||
if (AggregateFamilies.FamilyOf(agg) == AggregateFamily.TimeWeighted)
|
||||
return TimeWeightedAggregator.Evaluate(agg, plan, data, Now);
|
||||
return AggregateEvaluator.EvaluateAll(agg, plan, InMemoryRows(plan, data, agg == HistoryAggregateType.MaximumActualTime),
|
||||
data.Min(s => s.TimeUtc), Now);
|
||||
data.Min(s => s.TimeUtc), data.Max(s => s.TimeUtc), Now);
|
||||
}
|
||||
|
||||
private static IReadOnlyList<SdkAggregateOracle.Result> Sdk(HistoryAggregateType agg, double from, double to,
|
||||
@@ -110,21 +191,109 @@ public sealed class SdkAggregateParityTests
|
||||
return list;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Applies the documented divergences listed on the class (other than truncation): strips the Calculated bit from our
|
||||
/// <c>BadNoData</c>; maps our generic-Bad TimeAverage2/Total2 outage interval to the SDK's <c>BadNoData</c>; takes the
|
||||
/// SDK's value for the end-of-data interval of TimeAverage2/Total2/PercentGood/PercentBad (its 1 ms artifact).
|
||||
/// </summary>
|
||||
private static IReadOnlyList<DataValueSnapshot> MaskDocumented(HistoryAggregateType agg, IReadOnlyList<DataValueSnapshot> ours,
|
||||
IReadOnlyList<SdkAggregateOracle.Result> sdk, double from, double to, double intervalS, StoredSample[] data)
|
||||
{
|
||||
var plan = BucketPlan.Create(At(from), At(to), TimeSpan.FromSeconds(intervalS));
|
||||
var list = ours.ToList();
|
||||
var last = data.Max(x => x.TimeUtc).Ticks;
|
||||
var endsInWindow = last < Math.Max(At(from).Ticks, At(to).Ticks);
|
||||
var dataEnds = endsInWindow && data.Single(x => x.TimeUtc.Ticks == last).IsUsable;
|
||||
var outageEnds = endsInWindow && !dataEnds;
|
||||
var boolean = data.Any(x => x.Value.Kind == HistorianValueKind.Boolean);
|
||||
var percent = agg is HistoryAggregateType.PercentGood or HistoryAggregateType.PercentBad;
|
||||
for (long i = 0; i < plan.BucketCount; i++)
|
||||
{
|
||||
var k = (int)(plan.Reversed ? plan.BucketCount - 1 - i : i);
|
||||
if (k >= sdk.Count) continue; // the count mismatch is reported by ShouldMatch
|
||||
var code = list[k].StatusCode & StatusBits.CodeBitsMask;
|
||||
if (code == HistorianStatusCodes.BadNoData)
|
||||
list[k] = list[k] with { StatusCode = list[k].StatusCode & ~StatusBits.HistorianBitsMask };
|
||||
var twoVariant = agg is HistoryAggregateType.TimeAverage2 or HistoryAggregateType.Total2;
|
||||
if (twoVariant && code == HistorianStatusCodes.Bad && (sdk[k].StatusCode & StatusBits.CodeBitsMask) == HistorianStatusCodes.BadNoData)
|
||||
list[k] = list[k] with { StatusCode = HistorianStatusCodes.BadNoData };
|
||||
// Reverse TimeAverage/Total: the SDK's backward UpdateSlice files a sample exactly on the interval's later edge
|
||||
// as the early bound, so the interval inherits the status of the segment beyond it (the root cause of the
|
||||
// reverse Interpolative divergence). Ours keeps the interval's own segments.
|
||||
if (plan.Reversed && agg is HistoryAggregateType.TimeAverage or HistoryAggregateType.Total
|
||||
&& data.Any(x => x.TimeUtc.Ticks == plan.UpperTicks(i))
|
||||
&& code == HistorianStatusCodes.Good
|
||||
&& (sdk[k].StatusCode & StatusBits.CodeBitsMask) == HistorianStatusCodes.UncertainDataSubNormal)
|
||||
list[k] = list[k] with { StatusCode = (list[k].StatusCode & ~StatusBits.CodeBitsMask) | HistorianStatusCodes.UncertainDataSubNormal };
|
||||
if (dataEnds && (twoVariant || agg is HistoryAggregateType.PercentGood or HistoryAggregateType.PercentBad)
|
||||
&& last >= plan.LowerTicks(i) && last < plan.UpperTicks(i) && sdk[k].Value is not null)
|
||||
list[k] = list[k] with { Value = sdk[k].Value };
|
||||
// An outage marker as the last sample: ours runs the outage to the window end (no end-of-data Partial on the
|
||||
// interval holding it; Bad / 100 % Bad after it); the SDK treats the data as ended there.
|
||||
if (outageEnds && (twoVariant || percent))
|
||||
{
|
||||
if (last >= plan.LowerTicks(i) && last < plan.UpperTicks(i))
|
||||
list[k] = list[k] with
|
||||
{
|
||||
StatusCode = (list[k].StatusCode & ~HistorianInfoBits.HistorianPartial)
|
||||
| (sdk[k].StatusCode & HistorianInfoBits.HistorianPartial),
|
||||
};
|
||||
else if (plan.LowerTicks(i) > last && (sdk[k].StatusCode & StatusBits.CodeBitsMask) == HistorianStatusCodes.BadNoData)
|
||||
list[k] = list[k] with { Value = null, StatusCode = HistorianStatusCodes.BadNoData };
|
||||
}
|
||||
// ...and TimeAverage/Total hold the last usable value up to that marker: Part 13 calls that extrapolation (no
|
||||
// Good value after it) ⇒ Uncertain; the SDK, which sees a later (Bad) sample, reports the held span Good.
|
||||
var lastUsable = data.Where(x => x.IsUsable).Max(x => x.TimeUtc).Ticks;
|
||||
if (outageEnds && agg is HistoryAggregateType.TimeAverage or HistoryAggregateType.Total
|
||||
&& plan.LowerTicks(i) < last && plan.UpperTicks(i) > lastUsable
|
||||
&& (list[k].StatusCode & StatusBits.CodeBitsMask) == HistorianStatusCodes.UncertainDataSubNormal
|
||||
&& (sdk[k].StatusCode & StatusBits.CodeBitsMask) == HistorianStatusCodes.Good)
|
||||
list[k] = list[k] with { StatusCode = list[k].StatusCode & ~StatusBits.CodeBitsMask };
|
||||
// Booleans: the SDK slopes them (values, and the status of a bound next to an Uncertain sample); ours steps.
|
||||
if (boolean && sdk[k].Value is not null && list[k].Value is not null)
|
||||
list[k] = list[k] with { Value = sdk[k].Value };
|
||||
if (boolean && (StatusBits.IsGood(list[k].StatusCode) || StatusBits.IsGood(sdk[k].StatusCode))
|
||||
&& !StatusBits.IsBad(list[k].StatusCode) && !StatusBits.IsBad(sdk[k].StatusCode))
|
||||
list[k] = list[k] with
|
||||
{
|
||||
StatusCode = (list[k].StatusCode & ~StatusBits.CodeBitsMask) | (sdk[k].StatusCode & StatusBits.CodeBitsMask),
|
||||
};
|
||||
}
|
||||
return list;
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[MemberData(nameof(Windows))]
|
||||
public void Time_weighted_aggregates_match_the_SDK(double from, double to, double intervalS)
|
||||
{
|
||||
var failures = new List<string>();
|
||||
foreach (var agg in new[]
|
||||
{
|
||||
HistoryAggregateType.TimeAverage, HistoryAggregateType.TimeAverage2, HistoryAggregateType.Total,
|
||||
HistoryAggregateType.Total2, HistoryAggregateType.Interpolative,
|
||||
HistoryAggregateType.Total2, HistoryAggregateType.Interpolative, HistoryAggregateType.PercentGood,
|
||||
HistoryAggregateType.PercentBad,
|
||||
})
|
||||
foreach (var (name, data) in new[] { ("Bounded", Bounded), ("Dense", Dense) })
|
||||
SdkAggregateOracle.ShouldMatch(MaskTruncation(Ours(agg, from, to, intervalS, data), from, to, intervalS),
|
||||
Sdk(agg, from, to, intervalS, data), $"{agg} {name} {from}→{to}/{intervalS}s",
|
||||
compareCalculatedInterpolatedBits: !(agg == HistoryAggregateType.Interpolative && from > to),
|
||||
compareCodeBits: agg is not (HistoryAggregateType.TimeAverage or HistoryAggregateType.Total)
|
||||
&& !(agg == HistoryAggregateType.Interpolative && from > to));
|
||||
foreach (var (name, data) in new[]
|
||||
{
|
||||
("Bounded", Bounded), ("Dense", Dense), ("EndsInside", EndsInside), ("OutageInside", OutageInside),
|
||||
("RecoveryEnds", RecoveryEnds), ("OutageLast", OutageLast), ("BoolSwitch", BoolSwitch),
|
||||
})
|
||||
{
|
||||
try
|
||||
{
|
||||
var sdk = Sdk(agg, from, to, intervalS, data);
|
||||
var ours = MaskDocumented(agg, MaskTruncation(Ours(agg, from, to, intervalS, data), from, to, intervalS),
|
||||
sdk, from, to, intervalS, data);
|
||||
SdkAggregateOracle.ShouldMatch(ours, sdk, $"{agg} {name} {from}→{to}/{intervalS}s",
|
||||
compareCalculatedInterpolatedBits: !(agg == HistoryAggregateType.Interpolative && from > to),
|
||||
compareCodeBits: !(agg == HistoryAggregateType.Interpolative && from > to));
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
failures.Add(ex.Message);
|
||||
}
|
||||
}
|
||||
failures.ShouldBeEmpty(string.Join("\n", failures));
|
||||
}
|
||||
|
||||
[Theory]
|
||||
@@ -138,13 +307,7 @@ public sealed class SdkAggregateParityTests
|
||||
{
|
||||
SdkAggregateOracle.ShouldMatch(Ours(agg, from, to, intervalS, Dense), Sdk(agg, from, to, intervalS, Dense),
|
||||
$"{agg} Dense {from}→{to}/{intervalS}s", compareCalculatedInterpolatedBits: false,
|
||||
compareValues: !PercentAggregate(agg),
|
||||
// pre-existing, #680: the SDK's Start/End return the raw first/last sample (any quality) with its own
|
||||
// timestamp and stored status; ours return the first/last USABLE value stamped with the interval, under the
|
||||
// per-sample status rule. The SDK's Delta is Good even when a Bad sample sits in the interval; ours is
|
||||
// Uncertain. Values are compared wherever the SDK's answer is not Bad.
|
||||
compareTimestamps: !StartOrEnd(agg),
|
||||
compareCodeBits: !StartOrEnd(agg) && agg != HistoryAggregateType.Delta);
|
||||
compareValues: true);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
@@ -159,19 +322,8 @@ public sealed class SdkAggregateParityTests
|
||||
HistoryAggregateType.Average, HistoryAggregateType.Minimum, HistoryAggregateType.Maximum,
|
||||
HistoryAggregateType.Count, HistoryAggregateType.Range, HistoryAggregateType.Start, HistoryAggregateType.End,
|
||||
HistoryAggregateType.Delta, HistoryAggregateType.MinimumActualTime, HistoryAggregateType.MaximumActualTime,
|
||||
HistoryAggregateType.PercentGood, HistoryAggregateType.PercentBad,
|
||||
];
|
||||
|
||||
/// <summary>
|
||||
/// pre-existing, #680: ours is the per-SAMPLE percentage (100 × Good samples / all samples); the SDK's
|
||||
/// <c>StatusAggregateCalculator</c> is the per-DURATION one (Good time / interval time). Different definitions, so the
|
||||
/// values are not compared; timestamp, status and Partial still are.
|
||||
/// </summary>
|
||||
private static bool StartOrEnd(HistoryAggregateType agg) =>
|
||||
agg is HistoryAggregateType.Start or HistoryAggregateType.End;
|
||||
|
||||
private static bool PercentAggregate(HistoryAggregateType agg) =>
|
||||
agg is HistoryAggregateType.PercentGood or HistoryAggregateType.PercentBad;
|
||||
|
||||
[Theory]
|
||||
[InlineData(HistoryAggregateType.TimeAverage)]
|
||||
@@ -194,11 +346,7 @@ public sealed class SdkAggregateParityTests
|
||||
var ours = Ours(agg, 40, 0, 10, LateStart);
|
||||
SdkAggregateOracle.ShouldMatch(ours, Sdk(agg, 40, 0, 10, LateStart), $"{agg} reverse LateStart",
|
||||
compareCalculatedInterpolatedBits: tw,
|
||||
// pre-existing, #680: Total over the interval the data starts inside (TimeAverage × interval vs the SDK's
|
||||
// integral of the covered part); Start's raw-sample timestamp/status (see Per_sample_aggregates_match_the_SDK).
|
||||
compareValues: agg != HistoryAggregateType.Total,
|
||||
compareTimestamps: !StartOrEnd(agg),
|
||||
compareCodeBits: !StartOrEnd(agg) && agg != HistoryAggregateType.Delta);
|
||||
compareCodeBits: true);
|
||||
ours.ShouldAllBe(x => (x.StatusCode & HistorianInfoBits.HistorianPartial) == 0);
|
||||
}
|
||||
|
||||
@@ -218,12 +366,174 @@ public sealed class SdkAggregateParityTests
|
||||
var tw = AggregateFamilies.FamilyOf(agg) == AggregateFamily.TimeWeighted;
|
||||
var ours = Ours(agg, 0, 40, 10, LateStart);
|
||||
SdkAggregateOracle.ShouldMatch(ours, Sdk(agg, 0, 40, 10, LateStart), $"{agg} LateStart",
|
||||
compareCalculatedInterpolatedBits: tw, compareValues: agg != HistoryAggregateType.Total);
|
||||
compareCalculatedInterpolatedBits: tw);
|
||||
(ours[0].StatusCode & HistorianInfoBits.HistorianPartial).ShouldBe(
|
||||
agg == HistoryAggregateType.Average ? 0u : HistorianInfoBits.HistorianPartial);
|
||||
ours.Skip(1).ShouldAllBe(x => (x.StatusCode & HistorianInfoBits.HistorianPartial) == 0);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// #680: sparse data with Good, Uncertain and Bad samples, empty intervals inside the data range, and intervals before
|
||||
/// and after it: +1 G, +2 U, +25 Bad, +45 G, +70 G.
|
||||
/// </summary>
|
||||
private static readonly StoredSample[] Gappy =
|
||||
[
|
||||
S(1, HistorianValue.FromDouble(10), Good),
|
||||
S(2, HistorianValue.FromDouble(11), Uncertain),
|
||||
S(25, HistorianValue.FromDouble(12), Bad),
|
||||
S(45, HistorianValue.FromDouble(13), Good),
|
||||
S(70, HistorianValue.FromDouble(1), Good),
|
||||
];
|
||||
|
||||
/// <summary>#680: Bad samples leading, trailing and mid-interval, plus an Uncertain one (Start / End / Delta).</summary>
|
||||
private static readonly StoredSample[] Bracketed =
|
||||
[
|
||||
S(-5, HistorianValue.FromDouble(1), Good),
|
||||
S(1, HistorianValue.FromDouble(10), Bad),
|
||||
S(3, HistorianValue.FromDouble(11), Good),
|
||||
S(5, HistorianValue.FromDouble(12), Bad),
|
||||
S(7, HistorianValue.FromDouble(13), Uncertain),
|
||||
S(9, HistorianValue.FromDouble(15), Good),
|
||||
S(12, HistorianValue.FromDouble(20), Good),
|
||||
S(14, HistorianValue.Empty, BadNoCommunication),
|
||||
S(16, HistorianValue.FromDouble(21), Good),
|
||||
S(19, HistorianValue.Empty, BadNoCommunication),
|
||||
S(40, HistorianValue.FromDouble(1), Good),
|
||||
];
|
||||
|
||||
public static TheoryData<double, double, double> GappyWindows => new()
|
||||
{
|
||||
{ 0, 60, 10 }, { 60, 0, 10 }, // empty intervals inside the data range; +70 is past the window
|
||||
{ -10, 1, 5 }, { 1, -10, 5 }, // intervals before the first sample; one ending ON it (forward)
|
||||
{ 70, 85, 5 }, { 85, 70, 5 }, // intervals after the last sample; one starting ON it (reverse)
|
||||
{ 0, 1, 1 }, // one interval ending exactly on the first sample: empty but in range
|
||||
};
|
||||
|
||||
[Theory]
|
||||
[MemberData(nameof(GappyWindows))]
|
||||
public void Count_Start_End_and_Delta_match_the_SDK_across_the_data_range(double from, double to, double intervalS)
|
||||
{
|
||||
// #680: Count over Good samples only with the value-based status, 0 (not BadNoData) for an empty interval inside
|
||||
// [first, last], BadNoData outside it; Start/End the raw first/last sample; Delta's status from its bounding samples.
|
||||
var failures = new List<string>();
|
||||
foreach (var agg in new[]
|
||||
{
|
||||
HistoryAggregateType.Count, HistoryAggregateType.Start, HistoryAggregateType.End, HistoryAggregateType.Delta,
|
||||
})
|
||||
foreach (var (name, data) in new[] { ("Gappy", Gappy), ("Bracketed", Bracketed) })
|
||||
{
|
||||
try
|
||||
{
|
||||
SdkAggregateOracle.ShouldMatch(Ours(agg, from, to, intervalS, data), Sdk(agg, from, to, intervalS, data),
|
||||
$"{agg} {name} {from}→{to}/{intervalS}s", compareCalculatedInterpolatedBits: false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
failures.Add(ex.Message);
|
||||
}
|
||||
}
|
||||
failures.ShouldBeEmpty(string.Join("\n", failures));
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(0, 20, 10)]
|
||||
[InlineData(20, 0, 10)]
|
||||
[InlineData(0, 20, 0)]
|
||||
[InlineData(20, 0, 0)]
|
||||
public void Start_End_and_Delta_over_bad_bounding_samples_match_the_SDK_exactly(double from, double to, double intervalS)
|
||||
{
|
||||
// #680: with the raw sample's own status (incl. a Bad one and its null value) Start/End must match the SDK to the
|
||||
// bit; Delta's Calculated bit as well.
|
||||
foreach (var agg in new[] { HistoryAggregateType.Start, HistoryAggregateType.End, HistoryAggregateType.Delta })
|
||||
SdkAggregateOracle.ShouldMatch(Ours(agg, from, to, intervalS, Bracketed), Sdk(agg, from, to, intervalS, Bracketed),
|
||||
$"{agg} Bracketed {from}→{to}/{intervalS}s");
|
||||
}
|
||||
|
||||
/// <summary>#680: data ending inside the window (−5, +3, +25) and ending exactly on an interval edge (+20).</summary>
|
||||
private static readonly StoredSample[] EarlyEnd =
|
||||
[S(-5, HistorianValue.FromDouble(1), Good), S(3, HistorianValue.FromDouble(2), Good), S(25, HistorianValue.FromDouble(3), Good)];
|
||||
|
||||
private static readonly StoredSample[] EndOnEdge =
|
||||
[S(-5, HistorianValue.FromDouble(1), Good), S(3, HistorianValue.FromDouble(2), Good), S(20, HistorianValue.FromDouble(3), Good)];
|
||||
|
||||
[Theory]
|
||||
[InlineData(0, 40, 10)]
|
||||
[InlineData(0, 40, 25)]
|
||||
[InlineData(0, 40, 0)]
|
||||
[InlineData(40, 0, 10)]
|
||||
[InlineData(40, 0, 25)]
|
||||
[InlineData(40, 0, 0)]
|
||||
public void The_interval_the_data_ends_inside_matches_the_SDK_Partial_bit(double from, double to, double intervalS)
|
||||
{
|
||||
// #680 (Part 13 §5.3.3.2 Case 2): the interval holding the last sample is Partial for every per-sample aggregate but
|
||||
// Average — forward with its earlier edge included (SDK m_endOfData); reverse only strictly inside (the SDK's
|
||||
// reverse read keys m_startOfData on the first value it is fed, the latest), so EndOnEdge sets nothing in reverse.
|
||||
var failures = new List<string>();
|
||||
foreach (var agg in PerSample)
|
||||
foreach (var (name, data) in new[] { ("EarlyEnd", EarlyEnd), ("EndOnEdge", EndOnEdge) })
|
||||
{
|
||||
try
|
||||
{
|
||||
SdkAggregateOracle.ShouldMatch(Ours(agg, from, to, intervalS, data), Sdk(agg, from, to, intervalS, data),
|
||||
$"{agg} {name} {from}→{to}/{intervalS}s", compareCalculatedInterpolatedBits: false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
failures.Add(ex.Message);
|
||||
}
|
||||
}
|
||||
failures.ShouldBeEmpty(string.Join("\n", failures));
|
||||
// Not vacuous: the forward EarlyEnd read does flag [20,30) (and only it).
|
||||
Ours(HistoryAggregateType.Minimum, 0, 40, 10, EarlyEnd).Select(x => x.StatusCode & HistorianInfoBits.HistorianPartial)
|
||||
.ShouldBe([0u, 0u, HistorianInfoBits.HistorianPartial, 0u]);
|
||||
// ... and the reverse one (40 → 0: (30,40], (20,30], (10,20], (0,10]) flags (20,30], which holds +25 strictly.
|
||||
Ours(HistoryAggregateType.Minimum, 40, 0, 10, EarlyEnd).Select(x => x.StatusCode & HistorianInfoBits.HistorianPartial)
|
||||
.ShouldBe([0u, HistorianInfoBits.HistorianPartial, 0u, 0u]);
|
||||
}
|
||||
|
||||
/// <summary>#680 review: the data ends on an OUTAGE MARKER (+25) — the interval holding it is all-Bad.</summary>
|
||||
private static readonly StoredSample[] OutageEndsData =
|
||||
[
|
||||
S(-5, HistorianValue.FromDouble(1), Good), S(3, HistorianValue.FromDouble(2), Good),
|
||||
S(25, HistorianValue.Empty, BadNoCommunication),
|
||||
];
|
||||
|
||||
[Theory]
|
||||
[InlineData(0, 40, 10)]
|
||||
[InlineData(40, 0, 10)]
|
||||
public void An_outage_marker_as_the_last_sample_sets_no_end_of_data_Partial_on_its_Bad_interval(double from, double to,
|
||||
double intervalS)
|
||||
{
|
||||
// Documented divergence (remarks): the SDK sets the end-of-data Partial bit on that interval's Bad result too; ours
|
||||
// never puts Partial on a Bad result. Masked by clearing Partial on the SDK's Bad results only — every other bit,
|
||||
// timestamp and value (and every non-Bad interval's Partial) is still compared.
|
||||
var failures = new List<string>();
|
||||
var sawSdkPartialOnBad = false;
|
||||
foreach (var agg in PerSample)
|
||||
{
|
||||
var ours = Ours(agg, from, to, intervalS, OutageEndsData);
|
||||
var sdk = Sdk(agg, from, to, intervalS, OutageEndsData);
|
||||
sawSdkPartialOnBad |= sdk.Any(r => (r.StatusCode & 0x80000000u) != 0 && (r.StatusCode & 0x4u) != 0);
|
||||
var masked = sdk.Select(r => (r.StatusCode & 0x80000000u) != 0 ? r with { StatusCode = r.StatusCode & ~0x4u } : r)
|
||||
.ToList();
|
||||
try
|
||||
{
|
||||
SdkAggregateOracle.ShouldMatch(ours, masked, $"{agg} OutageEndsData {from}→{to}/{intervalS}s",
|
||||
compareCalculatedInterpolatedBits: false);
|
||||
// The interval holding the marker ((20,30] / [20,30)) is Bad and carries no Partial.
|
||||
var k = from < to ? 2 : 1;
|
||||
((ours[k].StatusCode & 0x80000000u) != 0).ShouldBeTrue($"{agg}: the outage interval is Bad");
|
||||
(ours[k].StatusCode & HistorianInfoBits.HistorianPartial).ShouldBe(0u, $"{agg}: no Partial on a Bad result");
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
failures.Add(ex.Message);
|
||||
}
|
||||
}
|
||||
failures.ShouldBeEmpty(string.Join("\n", failures));
|
||||
sawSdkPartialOnBad.ShouldBeTrue("the mask is not vacuous: the SDK does flag the Bad interval Partial");
|
||||
}
|
||||
|
||||
/// <summary>The per-bucket statistics the backends' SQL computes, built in memory with <see cref="BucketPlan.IndexOf"/>.</summary>
|
||||
internal static IEnumerable<BucketRow> InMemoryRows(BucketPlan plan, IEnumerable<StoredSample> samples, bool maximum = false)
|
||||
{
|
||||
@@ -253,6 +563,8 @@ public sealed class SdkAggregateParityTests
|
||||
Last = usable.Count == 0 ? HistorianValue.Empty : usable[^1].Value,
|
||||
Extreme = extreme?.Value.Number,
|
||||
ExtremeTime = extreme?.TimeUtc,
|
||||
FirstAny = all[0],
|
||||
LastAny = all[^1],
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
+285
-30
@@ -17,6 +17,7 @@ public sealed class TimeWeightedAggregatorTests
|
||||
private const uint InvalidInputs = 0x80D60000u;
|
||||
private const uint BadCalc = 0x80000401u; // generic Bad | Calculated (an all-outage interval)
|
||||
private const uint GoodCalcPartial = 0x00000405u; // Good | Calculated | Partial (truncated last interval)
|
||||
private const uint UncertainCalcPartial = 0x40A40405u; // UncertainDataSubNormal | Calculated | Partial
|
||||
|
||||
private static IReadOnlyList<DataValueSnapshot> Run(HistoryAggregateType agg, double from, double to, double intervalS,
|
||||
params StoredSample[] samples) =>
|
||||
@@ -33,43 +34,286 @@ public sealed class TimeWeightedAggregatorTests
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- tag A: +0=10 G, +10=20 G, +20=Bad, +30=40 G, +40=50 G
|
||||
// Window [0,80), 20 s buckets. Raw: [0,10) 10→20 Good (150); [10,20) 20 held into the outage, Uncertain (200);
|
||||
// [20,30) Bad; [30,40) 40→50 Good (450); [40,80) 50 extrapolated, Uncertain (2000).
|
||||
// Bridged: [0,10) 150; [10,30) 20→40 = 250 in B0 + 350 in B1; [30,40) 450; [40,80) 50 held.
|
||||
// Window [0,80), 20 s buckets; nothing after +40 (no end bound: the data ENDS at +40). Raw: [0,10) 10→20 Good (150);
|
||||
// [10,20) 20 held into the outage, Uncertain (200); [20,30) Bad; [30,40) 40→50 Good (450); after +40 nothing (#680:
|
||||
// Part 13 "After End of Data" — the simple-bounding signal is not extrapolated).
|
||||
// Bridged: [0,10) 150; [10,30) 20→40 = 250 in B0 + 350 in B1; [30,40) 450; [40,80) 50 held (extrapolated).
|
||||
|
||||
[Fact]
|
||||
public void TimeAverage_bridges_the_outage_on_tag_A()
|
||||
{
|
||||
// B0 (150+250)/20 = 20; B1 (350+450)/20 = 40; B2, B3 = 50. Status (SDK rule, Uncertain time counts as Good):
|
||||
// B0 is non-Bad throughout (held-into-outage time is Uncertain) ⇒ Good; B1 has the Bad [20,30) ⇒ Uncertain;
|
||||
// B2, B3 are extrapolated (Uncertain) throughout ⇒ Good.
|
||||
// B0 (150+250)/20 = 20; B1 (350+450)/20 = 40; B2, B3 = 50. Status (#680, Part 13 + the SDK's interpolated-bounds
|
||||
// rule): Uncertain when a Bad value was skipped or the value was extrapolated — B0 and B1 hold the segment bridged
|
||||
// across the Bad +20; B2 and B3 are extrapolated past the last sample.
|
||||
var s = Run(HistoryAggregateType.TimeAverage, 0, 80, 20, A);
|
||||
Values(s, 20, 40, 50, 50);
|
||||
s.Select(x => x.StatusCode).ShouldBe([GoodCalc, UncertainCalc, GoodCalc, GoodCalc]);
|
||||
s.Select(x => x.StatusCode).ShouldBe([UncertainCalc, UncertainCalc, UncertainCalc, UncertainCalc]);
|
||||
s.Select(x => x.SourceTimestampUtc).ShouldBe([At(0), At(20), At(40), At(60)]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Total_is_TimeAverage_times_interval_seconds_on_tag_A()
|
||||
{
|
||||
// 20×20, 40×20, 50×20, 50×20.
|
||||
// Fully covered intervals: 20×20, 40×20, 50×20, 50×20 (the integral of the interpolated-bounding signal).
|
||||
Values(Run(HistoryAggregateType.Total, 0, 80, 20, A), 400, 800, 1000, 1000);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void TimeAverage2_excludes_the_bad_interval_on_tag_A()
|
||||
public void TimeAverage2_excludes_the_bad_interval_and_stops_at_the_end_of_data_on_tag_A()
|
||||
{
|
||||
// B0 350/20 = 17.5; B1 450/10 = 45 (the Bad [20,30) excluded); B2, B3 = 50.
|
||||
// B0 350/20 = 17.5; B1 450/10 = 45 (the Bad [20,30) excluded). B2 [40,60) holds the end of the data (+40, on its
|
||||
// start): no covered time, so the value is the last sample's (50), Good, Partial (#680; the SDK answers the same).
|
||||
// B3 is after the end of the data ⇒ BadNoData (Part 13 "After End of Data").
|
||||
var s = Run(HistoryAggregateType.TimeAverage2, 0, 80, 20, A);
|
||||
Values(s, 17.5, 45, 50, 50);
|
||||
s.Select(x => x.StatusCode).ShouldBe([GoodCalc, UncertainCalc, GoodCalc, GoodCalc]);
|
||||
Values(s, 17.5, 45, 50, null);
|
||||
s.Select(x => x.StatusCode).ShouldBe([GoodCalc, UncertainCalc, GoodCalcPartial, NoDataCalc]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Total2_integrates_only_the_non_bad_data_on_tag_A()
|
||||
{
|
||||
// B0 150+200 = 350; B1 450; B2, B3 1000.
|
||||
Values(Run(HistoryAggregateType.Total2, 0, 80, 20, A), 350, 450, 1000, 1000);
|
||||
// B0 150+200 = 350; B1 450; B2 nothing covered after the last sample ⇒ 0, Partial; B3 BadNoData.
|
||||
var s = Run(HistoryAggregateType.Total2, 0, 80, 20, A);
|
||||
Values(s, 350, 450, 0, null);
|
||||
s.Select(x => x.StatusCode).ShouldBe([GoodCalc, UncertainCalc, GoodCalcPartial, NoDataCalc]);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- #680: end of data, live tags, outages, Total coverage
|
||||
|
||||
/// <summary>−5 = 1, +3 = 2, +25 = 3 (all Good), nothing later: the data ends at +25.</summary>
|
||||
private static readonly StoredSample[] EndsInside =
|
||||
[
|
||||
S(-5, HistorianValue.FromDouble(1), Good), S(3, HistorianValue.FromDouble(2), Good),
|
||||
S(25, HistorianValue.FromDouble(3), Good),
|
||||
];
|
||||
|
||||
[Fact]
|
||||
public void TimeAverage2_and_Total2_stop_at_the_end_of_data_with_Partial_then_BadNoData()
|
||||
{
|
||||
// Window [0,40), 10 s. Segment +3..+25 slopes 2→3 (1/22 per s). [20,30): covered only [20,25): 2+17/22 → 3, area
|
||||
// 5 × (2.772727… + 3)/2 = 14.431818…; TimeAverage2 = 2.886363…, Good + Partial (the SDK: 14.4348 for Total2 —
|
||||
// its last region carries a 1 ms artifact). [30,40) is after the end of the data ⇒ BadNoData.
|
||||
var t2 = Run(HistoryAggregateType.Total2, 0, 40, 10, EndsInside);
|
||||
((double)t2[2].Value!).ShouldBe(5 * ((2 + 17d / 22) + 3) / 2, 1e-9);
|
||||
t2[2].StatusCode.ShouldBe(GoodCalcPartial);
|
||||
t2[3].StatusCode.ShouldBe(NoDataCalc);
|
||||
t2[3].Value.ShouldBeNull();
|
||||
var ta2 = Run(HistoryAggregateType.TimeAverage2, 0, 40, 10, EndsInside);
|
||||
((double)ta2[2].Value!).ShouldBe(((2 + 17d / 22) + 3) / 2, 1e-9);
|
||||
ta2.Select(x => x.StatusCode).ShouldBe([GoodCalc, GoodCalc, GoodCalcPartial, NoDataCalc]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void TimeAverage_and_Total_extrapolate_past_the_end_of_data_as_Uncertain_without_Partial()
|
||||
{
|
||||
// The interpolated-bounding signal holds the last value to the window end (stepped extrapolation) — Part 13 and
|
||||
// the SDK make that Uncertain; the SDK sets no end-of-data Partial on an extrapolated result.
|
||||
// [20,30): [20,25) 2.772727…→3 + [25,30) 3 ⇒ (14.431818… + 15) / 10; [30,40): 3.
|
||||
var ta = Run(HistoryAggregateType.TimeAverage, 0, 40, 10, EndsInside);
|
||||
((double)ta[2].Value!).ShouldBe((5 * ((2 + 17d / 22) + 3) / 2 + 15) / 10, 1e-9);
|
||||
((double)ta[3].Value!).ShouldBe(3d);
|
||||
ta.Select(x => x.StatusCode).ShouldBe([GoodCalc, GoodCalc, UncertainCalc, UncertainCalc]);
|
||||
Values(Run(HistoryAggregateType.Total, 30, 40, 0, EndsInside), 30);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void The_latest_interval_of_a_live_tag_is_Uncertain()
|
||||
{
|
||||
// A live tag: −10 = 10, +5 = 20, +12 = 30, nothing after yet. [10,20) is extrapolated from +12 ⇒ Uncertain
|
||||
// (#680 — it used to read Good). [0,10) is bridged Good→Good with no skip ⇒ Good. Values as the SDK:
|
||||
// [0,10) 20.952380…, [10,20) 29.714285….
|
||||
StoredSample[] live =
|
||||
[
|
||||
S(-10, HistorianValue.FromDouble(10), Good), S(5, HistorianValue.FromDouble(20), Good),
|
||||
S(12, HistorianValue.FromDouble(30), Good),
|
||||
];
|
||||
var s = Run(HistoryAggregateType.TimeAverage, 0, 20, 10, live);
|
||||
Values(s, 20.952380952380953, 29.71428571428571);
|
||||
s.Select(x => x.StatusCode).ShouldBe([GoodCalc, UncertainCalc]);
|
||||
Run(HistoryAggregateType.Total, 0, 20, 10, live)[1].StatusCode.ShouldBe(UncertainCalc);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void An_uncertain_sample_makes_TimeAverage_and_Total_uncertain()
|
||||
{
|
||||
// Part 13: "Uncertain if ... Uncertain values are used". −10 = 10 G, +5 = 20 U, +15 = 30 G, +40 = 30 G; [0,20):
|
||||
// the value 24.583333… (SDK), Uncertain.
|
||||
StoredSample[] unc =
|
||||
[
|
||||
S(-10, HistorianValue.FromDouble(10), Good), S(5, HistorianValue.FromDouble(20), Uncertain),
|
||||
S(15, HistorianValue.FromDouble(30), Good), S(40, HistorianValue.FromDouble(30), Good),
|
||||
];
|
||||
var ta = Run(HistoryAggregateType.TimeAverage, 0, 20, 0, unc).Single();
|
||||
((double)ta.Value!).ShouldBe(24.583333333333332, 1e-9);
|
||||
ta.StatusCode.ShouldBe(UncertainCalc);
|
||||
Run(HistoryAggregateType.Total, 0, 20, 0, unc).Single().StatusCode.ShouldBe(UncertainCalc);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_segment_bridged_across_a_bad_sample_is_uncertain_in_every_interval_it_touches()
|
||||
{
|
||||
// −10 = 10 G, +10 Bad, +15 = 20 G, +40 = 20 G: the bridged segment −10..+15 skips the Bad +10, so [0,10) — which
|
||||
// holds no Bad sample itself — is Uncertain too (the SDK agrees).
|
||||
StoredSample[] edge =
|
||||
[
|
||||
S(-10, HistorianValue.FromDouble(10), Good), S(10, HistorianValue.FromDouble(30), Bad),
|
||||
S(15, HistorianValue.FromDouble(20), Good), S(40, HistorianValue.FromDouble(20), Good),
|
||||
];
|
||||
var s = Run(HistoryAggregateType.TimeAverage, 0, 20, 10, edge);
|
||||
Values(s, 16, 19.5);
|
||||
s.Select(x => x.StatusCode).ShouldBe([UncertainCalc, UncertainCalc]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Total_over_an_interval_the_data_starts_inside_integrates_only_the_covered_part()
|
||||
{
|
||||
// +5 = 10, +15 = 10, +40 = 10, nothing before: [0,20) is covered from +5 only ⇒ Total = 10 × 15 s = 150 (Part 13
|
||||
// partial-interval width; the SDK agrees). TimeAverage is still 10. Uncertain (no data for [0,5)) + Partial.
|
||||
StoredSample[] late =
|
||||
[
|
||||
S(5, HistorianValue.FromDouble(10), Good), S(15, HistorianValue.FromDouble(10), Good),
|
||||
S(40, HistorianValue.FromDouble(10), Good),
|
||||
];
|
||||
var total = Run(HistoryAggregateType.Total, 0, 20, 0, late).Single();
|
||||
((double)total.Value!).ShouldBe(150d);
|
||||
total.StatusCode.ShouldBe(UncertainCalcPartial);
|
||||
((double)Run(HistoryAggregateType.TimeAverage, 0, 20, 0, late).Single().Value!).ShouldBe(10d);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_recovery_sample_after_an_outage_that_ends_the_data_is_its_own_value_Uncertain_and_Partial()
|
||||
{
|
||||
// #680 review (the comms-loss recovery shape): −10 = 10 G, +5 outage, +25 = 20 G (the recovery), nothing after.
|
||||
// [0,20): 10 held into the outage over [0,5) ⇒ TimeAverage2 10 (Total2 50), Uncertain. [20,40) holds the end of the
|
||||
// data but no non-Bad time before it ⇒ the sample's value 20 (Total2 0), Uncertain + Partial — never Bad/null (the
|
||||
// SDK answers the same). [40,60) is after the end of the data ⇒ BadNoData.
|
||||
StoredSample[] s =
|
||||
[
|
||||
S(-10, HistorianValue.FromDouble(10), Good), S(5, HistorianValue.Empty, BadNoCommunication),
|
||||
S(25, HistorianValue.FromDouble(20), Good),
|
||||
];
|
||||
var ta2 = Run(HistoryAggregateType.TimeAverage2, 0, 60, 20, s);
|
||||
Values(ta2, 10, 20, null);
|
||||
ta2.Select(x => x.StatusCode).ShouldBe([UncertainCalc, UncertainCalcPartial, NoDataCalc]);
|
||||
var t2 = Run(HistoryAggregateType.Total2, 0, 60, 20, s);
|
||||
Values(t2, 50, 0, null);
|
||||
t2.Select(x => x.StatusCode).ShouldBe([UncertainCalc, UncertainCalcPartial, NoDataCalc]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_single_sample_is_its_own_value_Uncertain_and_Partial()
|
||||
{
|
||||
// #680 review: the tag's only sample, +25 = 20. One interval [0,60) ⇒ 20 (Total2 0), Uncertain + Partial (time
|
||||
// before the data is Bad). At 20 s: [0,20) before the data ⇒ BadNoData; [20,40) the sample ⇒ 20, Uncertain +
|
||||
// Partial; [40,60) after the end of the data ⇒ BadNoData.
|
||||
StoredSample[] one = [S(25, HistorianValue.FromDouble(20), Good)];
|
||||
var whole = Run(HistoryAggregateType.TimeAverage2, 0, 60, 0, one).Single();
|
||||
((double)whole.Value!).ShouldBe(20d);
|
||||
whole.StatusCode.ShouldBe(UncertainCalcPartial);
|
||||
var total = Run(HistoryAggregateType.Total2, 0, 60, 0, one).Single();
|
||||
((double)total.Value!).ShouldBe(0d);
|
||||
total.StatusCode.ShouldBe(UncertainCalcPartial);
|
||||
var split = Run(HistoryAggregateType.TimeAverage2, 0, 60, 20, one);
|
||||
Values(split, null, 20, null);
|
||||
split.Select(x => x.StatusCode).ShouldBe([NoDataCalc, UncertainCalcPartial, NoDataCalc]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void An_outage_marker_as_the_last_sample_runs_Bad_to_the_window_end()
|
||||
{
|
||||
// #680 review, a deliberate difference from the SDK (which treats the data as ended at the marker): −10 = 10 G,
|
||||
// +25 = 20 G, +30 outage, nothing after. TimeAverage2 [20,40): [20,25) 18.5714…→20 (96.43) + [25,30) 20 held into
|
||||
// the outage (100) over 10 s = 19.642857…, Uncertain, NOT Partial (the data did not end — the tag is in an
|
||||
// outage); [40,60) inside the still-open outage ⇒ generic Bad. PercentBad [40,60) = 100, Good. TimeAverage holds
|
||||
// 20 to the window end — extrapolated (no Good value follows) ⇒ Uncertain.
|
||||
StoredSample[] s =
|
||||
[
|
||||
S(-10, HistorianValue.FromDouble(10), Good), S(25, HistorianValue.FromDouble(20), Good),
|
||||
S(30, HistorianValue.Empty, BadNoCommunication),
|
||||
];
|
||||
var ta2 = Run(HistoryAggregateType.TimeAverage2, 0, 60, 20, s);
|
||||
((double)ta2[1].Value!).ShouldBe(19.642857142857142, 1e-9);
|
||||
ta2.Select(x => x.StatusCode).Skip(1).ShouldBe([UncertainCalc, BadCalc]);
|
||||
var pb = Run(HistoryAggregateType.PercentBad, 0, 60, 20, s);
|
||||
Values(pb, 0, 50, 100);
|
||||
pb.ShouldAllBe(x => x.StatusCode == GoodCalc);
|
||||
Run(HistoryAggregateType.TimeAverage, 0, 60, 20, s).Select(x => x.StatusCode).Skip(1)
|
||||
.ShouldBe([UncertainCalc, UncertainCalc]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Boolean_percent_regions_take_the_stepped_bound_status()
|
||||
{
|
||||
// #680 review: −10 = true G, +15 = true U, +30 = false G, +50 = false G. [0,20): the bound at +0 is stepped from the
|
||||
// Good −10 (a Boolean is held, not sloped toward the Uncertain +15) ⇒ [0,15) Good, [15,20) Uncertain ⇒ PercentGood
|
||||
// 75. (The SDK slopes Booleans, so it calls the bound Uncertain and answers 0.) [20,40): [20,30) Uncertain (bound
|
||||
// stepped from the Uncertain +15), [30,40) Good ⇒ 50.
|
||||
StoredSample[] s =
|
||||
[
|
||||
S(-10, HistorianValue.FromBoolean(true), Good), S(15, HistorianValue.FromBoolean(true), Uncertain),
|
||||
S(30, HistorianValue.FromBoolean(false), Good), S(50, HistorianValue.FromBoolean(false), Good),
|
||||
];
|
||||
Values(Run(HistoryAggregateType.PercentGood, 0, 40, 20, s), 75, 50);
|
||||
// TimeAverage over the same Booleans is stepped: [20,40) = 1 over [20,30), 0 over [30,40) ⇒ 0.5 (the SDK: 1).
|
||||
Values(Run(HistoryAggregateType.TimeAverage, 0, 40, 20, s), 1, 0.5);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- #680: PercentGood / PercentBad are duration-based
|
||||
|
||||
[Fact]
|
||||
public void Percent_aggregates_are_durations_on_tag_A()
|
||||
{
|
||||
// Stepped regions, each with the status of the sample that starts it. B0 [0,20) Good throughout; B1 [20,30) Bad
|
||||
// (the Bad +20), [30,40) Good ⇒ 50 / 50; B2 holds the end of the data (+40, Good) ⇒ its region runs to the interval
|
||||
// end (Part 13), Partial (forward end-of-data, as the SDK); B3 is after the end of the data ⇒ BadNoData.
|
||||
var good = Run(HistoryAggregateType.PercentGood, 0, 80, 20, A);
|
||||
Values(good, 100, 50, 100, null);
|
||||
good.Select(x => x.StatusCode).ShouldBe([GoodCalc, GoodCalc, GoodCalcPartial, NoDataCalc]);
|
||||
var bad = Run(HistoryAggregateType.PercentBad, 0, 80, 20, A);
|
||||
Values(bad, 0, 50, 0, null);
|
||||
bad.Select(x => x.StatusCode).ShouldBe([GoodCalc, GoodCalc, GoodCalcPartial, NoDataCalc]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void With_no_end_bound_the_last_region_runs_to_the_interval_end()
|
||||
{
|
||||
// 0 = 10 G, +5 outage, +10 = 30 G, nothing after; [0,20) one interval: [0,5) Good, [5,10) Bad, [10,20) Good
|
||||
// (Part 13: the last region extends to the interval end — the SDK instead gives it 1 ms, ≈ 49.995 % Bad).
|
||||
StoredSample[] s =
|
||||
[
|
||||
S(0, HistorianValue.FromDouble(10), Good), S(5, HistorianValue.Empty, BadNoCommunication),
|
||||
S(10, HistorianValue.FromDouble(30), Good),
|
||||
];
|
||||
var bad = Run(HistoryAggregateType.PercentBad, 0, 20, 0, s).Single();
|
||||
((double)bad.Value!).ShouldBe(25d, 1e-9);
|
||||
bad.StatusCode.ShouldBe(GoodCalcPartial);
|
||||
((double)Run(HistoryAggregateType.PercentGood, 0, 20, 0, s).Single().Value!).ShouldBe(75d, 1e-9);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Percent_regions_use_the_simple_bound_at_the_interval_start_and_count_time_before_the_data_as_Bad()
|
||||
{
|
||||
// −3 = 10 G, +4 outage, +6 = 30 U, +12 = 3 G, +30 = 3 G; [0,20), 10 s. The simple bound at +0 lies on a segment
|
||||
// that runs into the outage ⇒ Uncertain for [0,4) (the SDK's simple bounding value); [4,6) Bad; [6,10) Uncertain.
|
||||
// B0: Good 0 %, Bad 20 %. B1: [10,12) Uncertain (bound between the Uncertain +6 and +12), [12,20) Good ⇒ 80 / 0.
|
||||
StoredSample[] s =
|
||||
[
|
||||
S(-3, HistorianValue.FromDouble(10), Good), S(4, HistorianValue.Empty, BadNoCommunication),
|
||||
S(6, HistorianValue.FromDouble(30), Uncertain), S(12, HistorianValue.FromDouble(3), Good),
|
||||
S(30, HistorianValue.FromDouble(3), Good),
|
||||
];
|
||||
Values(Run(HistoryAggregateType.PercentGood, 0, 20, 10, s), 0, 80);
|
||||
Values(Run(HistoryAggregateType.PercentBad, 0, 20, 10, s), 20, 0);
|
||||
// The data starts at +10, exactly on the end of [0,10): all of [0,10) is before the data ⇒ 100 % Bad (the SDK
|
||||
// agrees); data starting at +5 ⇒ [0,5) Bad, Partial.
|
||||
StoredSample[] late = [S(10, HistorianValue.FromDouble(1), Good), S(30, HistorianValue.FromDouble(2), Good)];
|
||||
Values(Run(HistoryAggregateType.PercentBad, 0, 20, 10, late), 100, 0);
|
||||
StoredSample[] mid = [S(5, HistorianValue.FromDouble(1), Good), S(30, HistorianValue.FromDouble(2), Good)];
|
||||
var m = Run(HistoryAggregateType.PercentBad, 0, 20, 0, mid).Single();
|
||||
((double)m.Value!).ShouldBe(25d, 1e-9);
|
||||
m.StatusCode.ShouldBe(GoodCalcPartial);
|
||||
// Nothing at all ⇒ BadNoData.
|
||||
Run(HistoryAggregateType.PercentGood, 0, 20, 10).ShouldAllBe(x => x.StatusCode == NoDataCalc);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -146,12 +390,13 @@ public sealed class TimeWeightedAggregatorTests
|
||||
[Fact]
|
||||
public void Past_the_last_sample_the_value_is_extrapolated_stepped()
|
||||
{
|
||||
// -5 = 7 and nothing later: held at 7 over [0,10). The extrapolated time is Uncertain, which the SDK rule
|
||||
// (TreatUncertainAsBad = false) counts as Good time ⇒ the interval is Good; the Interpolative point is Uncertain.
|
||||
// -5 = 7 and nothing later: TimeAverage holds 7 over [0,10), Uncertain (extrapolated, #680). TimeAverage2 does not
|
||||
// extrapolate: [0,10) is after the end of the data ⇒ BadNoData. The Interpolative point is Uncertain.
|
||||
StoredSample[] one = [S(-5, HistorianValue.FromDouble(7), Good)];
|
||||
var s = Run(HistoryAggregateType.TimeAverage2, 0, 10, 0, one).Single();
|
||||
var s = Run(HistoryAggregateType.TimeAverage, 0, 10, 0, one).Single();
|
||||
((double)s.Value!).ShouldBe(7d);
|
||||
s.StatusCode.ShouldBe(GoodCalc);
|
||||
s.StatusCode.ShouldBe(UncertainCalc);
|
||||
Run(HistoryAggregateType.TimeAverage2, 0, 10, 0, one).Single().StatusCode.ShouldBe(NoDataCalc);
|
||||
var i = Run(HistoryAggregateType.Interpolative, 0, 10, 0, one).Single();
|
||||
((double)i.Value!).ShouldBe(7d);
|
||||
i.StatusCode.ShouldBe(UncertainInterpolated);
|
||||
@@ -208,13 +453,17 @@ public sealed class TimeWeightedAggregatorTests
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void An_interval_entirely_inside_an_outage_is_Bad_but_Interpolative_bridges_it()
|
||||
public void An_interval_entirely_inside_an_outage_bridges_for_TimeAverage_but_is_Bad_for_TimeAverage2()
|
||||
{
|
||||
// Window [10,15): raw is Bad throughout ⇒ data but no non-Bad time ⇒ generic Bad (the SDK rule; BadNoData is
|
||||
// reserved for no data at all). Interpolative at +10 = 10 + 20·(10/15).
|
||||
// Window [10,15): the interpolated-bounding signal (10→30 over [0,15)) bridges it — 23.333… → 30, mean 26.666…,
|
||||
// Uncertain (a Bad value skipped; #680, Part 13 + the SDK). Total = 26.666… × 5 s. The simple-bounding signal is
|
||||
// Bad throughout ⇒ TimeAverage2/Total2 generic Bad (Part 13's PercentData rule; the SDK answers BadNoData).
|
||||
// Interpolative at +10 = 10 + 20·(10/15).
|
||||
var ta = Run(HistoryAggregateType.TimeAverage, 10, 15, 0, Outage).Single();
|
||||
ta.StatusCode.ShouldBe(BadCalc);
|
||||
ta.Value.ShouldBeNull();
|
||||
((double)ta.Value!).ShouldBe((10d + 20d * 10d / 15d + 30d) / 2, 1e-9);
|
||||
ta.StatusCode.ShouldBe(UncertainCalc);
|
||||
((double)Run(HistoryAggregateType.Total, 10, 15, 0, Outage).Single().Value!)
|
||||
.ShouldBe((10d + 20d * 10d / 15d + 30d) / 2 * 5, 1e-9);
|
||||
Run(HistoryAggregateType.TimeAverage2, 10, 15, 0, Outage).Single().StatusCode.ShouldBe(BadCalc);
|
||||
Run(HistoryAggregateType.Total2, 10, 15, 0, Outage).Single().StatusCode.ShouldBe(BadCalc);
|
||||
var i = Run(HistoryAggregateType.Interpolative, 10, 15, 0, Outage).Single();
|
||||
@@ -324,11 +573,14 @@ public sealed class TimeWeightedAggregatorTests
|
||||
[Fact]
|
||||
public void A_reverse_window_is_returned_latest_first_stamped_with_later_edges()
|
||||
{
|
||||
// #677: the four whole intervals are the same spans as forward, but each is stamped with its later edge.
|
||||
// #677: the four whole intervals are the same spans as forward, but each is stamped with its later edge. The data
|
||||
// ends at +40 (#680): the latest interval is after it ⇒ BadNoData; the next holds it on its earlier edge ⇒ 50, with
|
||||
// no end-of-data Partial on a reverse read (the SDK answers the same).
|
||||
var s = TimeWeightedAggregator.Evaluate(HistoryAggregateType.TimeAverage2,
|
||||
BucketPlan.Create(At(80), At(0), TimeSpan.FromSeconds(20)), A, Now);
|
||||
s.Select(x => x.SourceTimestampUtc).ShouldBe([At(80), At(60), At(40), At(20)]);
|
||||
Values(s, 50, 50, 45, 17.5);
|
||||
Values(s, null, 50, 45, 17.5);
|
||||
s.Select(x => x.StatusCode).ShouldBe([NoDataCalc, GoodCalc, UncertainCalc, GoodCalc]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -336,13 +588,15 @@ public sealed class TimeWeightedAggregatorTests
|
||||
{
|
||||
// Tag A, At(50) → At(0), 20 s ⇒ (30,50], (10,30], (0,10]. Raw signal: [0,10) 10→20 Good (150); [10,20) 20 held
|
||||
// into the outage, Uncertain (200); [20,30) Bad; [30,40) 40→50 Good (450); [40,50) 50 extrapolated (500).
|
||||
// (30,50]: 950/20 = 47.5 Good; (10,30]: 200/10 = 20, the Bad 10 s ⇒ Uncertain; (0,10]: 150/10 = 15, truncated ⇒
|
||||
// (30,50]: the data ends at +40 (#680: no extrapolation) ⇒ 450/10 = 45, Good; the latest sample lies strictly
|
||||
// inside it, where the reverse read's data starts ⇒ Partial (the SDK's m_startOfData keyed on the first value it
|
||||
// is fed); (10,30]: 200/10 = 20, the Bad 10 s ⇒ Uncertain; (0,10]: 150/10 = 15, truncated ⇒
|
||||
// Partial. The old layout would have answered [0,20), [20,40), [40,50) stamped 0/20/40, partial at the LATEST end.
|
||||
var s = TimeWeightedAggregator.Evaluate(HistoryAggregateType.TimeAverage2,
|
||||
BucketPlan.Create(At(50), At(0), TimeSpan.FromSeconds(20)), A, Now);
|
||||
s.Select(x => x.SourceTimestampUtc).ShouldBe([At(50), At(30), At(10)]);
|
||||
Values(s, 47.5, 20, 15);
|
||||
s.Select(x => x.StatusCode).ShouldBe([GoodCalc, UncertainCalc, GoodCalcPartial]);
|
||||
Values(s, 45, 20, 15);
|
||||
s.Select(x => x.StatusCode).ShouldBe([GoodCalcPartial, UncertainCalc, GoodCalcPartial]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -368,8 +622,9 @@ public sealed class TimeWeightedAggregatorTests
|
||||
// Window [0,50), 20 s: B2 = [40,50) is 10 s of the extrapolated 50 ⇒ TimeAverage 50, Total 500.
|
||||
var s = Run(HistoryAggregateType.Total, 0, 50, 20, A);
|
||||
Values(s, 400, 800, 500);
|
||||
// Only the truncated interval carries the Partial bit.
|
||||
s.Select(x => x.StatusCode).ShouldBe([GoodCalc, UncertainCalc, GoodCalcPartial]);
|
||||
// Only the truncated interval carries the Partial bit (Part 13 §5.3.3.2 Case 3 — the SDK does not set it; ours
|
||||
// does, deliberately). Every interval is Uncertain (bridged across the Bad +20, or extrapolated).
|
||||
s.Select(x => x.StatusCode).ShouldBe([UncertainCalc, UncertainCalc, UncertainCalcPartial]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
|
||||
+112
@@ -0,0 +1,112 @@
|
||||
using System.Data;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Live-gate 8 F1 (defence in depth on SQL Server, where it was not observed): the per-sample aggregate row mapper
|
||||
/// (<see cref="SqlServerHistorianDataSource.ReadBucket"/>) over a hand-built row in the
|
||||
/// <see cref="SqlAggregateQueries.StartEnd"/> column layout. A row whose counts include samples but whose
|
||||
/// first/last-of-any-quality picks are NULL used to throw out of <c>ReadStored</c> and fail the node's whole read; it now
|
||||
/// maps to absent picks the evaluator answers as <c>BadNoData</c>.
|
||||
/// </summary>
|
||||
public sealed class BucketRowMappingTests
|
||||
{
|
||||
private const uint NoDataCalc = 0x809B0401u;
|
||||
private static readonly DateTime T = new(2026, 10, 11, 12, 0, 0, DateTimeKind.Utc);
|
||||
|
||||
/// <summary>b + 5 counts, two usable picks (kind, num, int, str, ts), two any picks (ts, kind, num, int, str, status).</summary>
|
||||
private static DataTable StartEndTable()
|
||||
{
|
||||
var t = new DataTable();
|
||||
t.Columns.Add("b", typeof(long));
|
||||
foreach (var c in new[] { "total", "usable", "good", "bad", "nonnum" }) t.Columns.Add(c, typeof(int));
|
||||
for (var p = 0; p < 2; p++)
|
||||
{
|
||||
t.Columns.Add($"k{p}", typeof(byte));
|
||||
t.Columns.Add($"n{p}", typeof(double));
|
||||
t.Columns.Add($"i{p}", typeof(long));
|
||||
t.Columns.Add($"s{p}", typeof(string));
|
||||
t.Columns.Add($"t{p}", typeof(DateTime));
|
||||
}
|
||||
for (var p = 0; p < 2; p++)
|
||||
{
|
||||
t.Columns.Add($"at{p}", typeof(DateTime));
|
||||
t.Columns.Add($"ak{p}", typeof(byte));
|
||||
t.Columns.Add($"an{p}", typeof(double));
|
||||
t.Columns.Add($"ai{p}", typeof(long));
|
||||
t.Columns.Add($"as{p}", typeof(string));
|
||||
t.Columns.Add($"st{p}", typeof(int));
|
||||
}
|
||||
return t;
|
||||
}
|
||||
|
||||
private static DataRow Counts(DataTable t, long bucket, int total)
|
||||
{
|
||||
var row = t.NewRow();
|
||||
row[0] = bucket;
|
||||
row[1] = total; row[2] = total; row[3] = total; row[4] = 0; row[5] = 0;
|
||||
return row;
|
||||
}
|
||||
|
||||
private static BucketRow Map(DataTable t, AggregateFamily family)
|
||||
{
|
||||
using var r = t.CreateDataReader();
|
||||
r.Read().ShouldBeTrue();
|
||||
return SqlServerHistorianDataSource.ReadBucket(r, family);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_row_with_counts_but_null_first_and_last_of_any_quality_maps_to_absent_picks_without_throwing()
|
||||
{
|
||||
var t = StartEndTable();
|
||||
t.Rows.Add(Counts(t, 3, 2));
|
||||
|
||||
var b = Map(t, AggregateFamily.StartEnd);
|
||||
b.Bucket.ShouldBe(3);
|
||||
b.Total.ShouldBe(2);
|
||||
b.FirstAny.ShouldBeNull();
|
||||
b.LastAny.ShouldBeNull();
|
||||
b.First.Kind.ShouldBe(HistorianValueKind.Empty);
|
||||
|
||||
foreach (var agg in new[] { HistoryAggregateType.Start, HistoryAggregateType.End, HistoryAggregateType.Delta })
|
||||
AggregateEvaluator.Evaluate(agg, b, T, T).StatusCode.ShouldBe(NoDataCalc);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_consistent_row_still_maps_every_pick()
|
||||
{
|
||||
var t = StartEndTable();
|
||||
var row = Counts(t, 0, 2);
|
||||
row["k0"] = (byte)1; row["n0"] = 10d; row["t0"] = T;
|
||||
row["k1"] = (byte)1; row["n1"] = 20d; row["t1"] = T.AddSeconds(1);
|
||||
row["at0"] = T; row["ak0"] = (byte)1; row["an0"] = 10d; row["st0"] = 0;
|
||||
row["at1"] = T.AddSeconds(1); row["ak1"] = (byte)1; row["an1"] = 20d; row["st1"] = 0;
|
||||
t.Rows.Add(row);
|
||||
|
||||
var b = Map(t, AggregateFamily.StartEnd);
|
||||
b.First.ShouldBe(HistorianValue.FromDouble(10));
|
||||
b.Last.ShouldBe(HistorianValue.FromDouble(20));
|
||||
b.FirstAny.ShouldNotBeNull().TimeUtc.ShouldBe(T);
|
||||
b.LastAny.ShouldNotBeNull().Value.ShouldBe(HistorianValue.FromDouble(20));
|
||||
AggregateEvaluator.Evaluate(HistoryAggregateType.Delta, b, T, T).Value.ShouldBe(10d);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void An_actual_time_row_with_counts_but_no_extreme_is_no_data()
|
||||
{
|
||||
var t = new DataTable();
|
||||
t.Columns.Add("b", typeof(long));
|
||||
foreach (var c in new[] { "total", "usable", "good", "bad", "nonnum" }) t.Columns.Add(c, typeof(int));
|
||||
t.Columns.Add("NumValue", typeof(double));
|
||||
t.Columns.Add("SourceTimeUtc", typeof(DateTime));
|
||||
t.Rows.Add(0L, 1, 1, 1, 0, 0, DBNull.Value, DBNull.Value);
|
||||
|
||||
var b = Map(t, AggregateFamily.ActualTime);
|
||||
b.Extreme.ShouldBeNull();
|
||||
AggregateEvaluator.Evaluate(HistoryAggregateType.MaximumActualTime, b, T, T).StatusCode.ShouldBe(NoDataCalc);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user