Merge feat/820-duration-good-bad

This commit is contained in:
Joseph Doherty
2026-11-11 01:53:34 -05:00
24 changed files with 450 additions and 82 deletions
+3 -3
View File
@@ -32,11 +32,11 @@ historian tagname, which is the tag's driver `FullName`.
### Pluggable backends, SQL Server and HistoryUpdate (2026-10)
- **Selection:** `ServerHistorian:Backend` (`Gateway` default | `SqlServer` | `QuestDb`; unknown fails host start). `HistorianBackendRegistry` → `IHistorianBackendProvider` builds all five seams; the gateway keys are only required for `Gateway`. `SqlServer` uses `ServerHistorian:SqlServer:{ConnectionString (required), Schema=hist, CommandTimeout=30s, AutoCreateSchema=true}`.
- **SQL Server** (`Driver.Historian.SqlServer`): `hist.Sample` PK `(TagId, SourceTimeUtc)` — writes are insert-if-absent, so outbox redelivery is idempotent. Stores every value kind with its **real status**; per-sample aggregates (10, simple definitions) are pushed down to T-SQL with bucket math in **100 ns ticks**; the Part 13 time-weighted seven (TimeAverage/TimeAverage2/Total/Total2/Interpolative, #671 — ⚠️ `Total` is now the time-weighted integral, no longer `SUM` — plus the **duration-based** PercentGood/PercentBad, #680) stream window + 4 bound rows into the shared `TimeWeightedAggregator` (both SQL backends, identical results; reverse `Interpolative` anchored at the later time; window capped by `ServerHistorian:MaxTimeWeightedRawRows` ⇒ `BadTooManyOperations`, percentages included). ⚠️ **#680 status rules (client-visible):** TimeAverage/Total are Uncertain if any part of the interval uses an Uncertain value, is bridged across a Bad sample, is extrapolated past the last sample (⇒ a live tag's latest interval is Uncertain) or is not covered; TimeAverage2/Total2 keep the SDK time-based rule (Uncertain time counts as Good) but never extrapolate past the last sample — that interval is Partial, later ones `BadNoData` (a report-by-exception tag with nothing after its last sample reads `BadNoData` until its next sample arrives). `docs/Historian.md` §"Time-weighted aggregates". ⚠️ **#680 per-sample (client-visible):** `Count` counts **Good** samples only (value-based status; an all-Bad interval is generic Bad, value null) and an empty interval **inside the tag's data range** is `0`/Good (outside it still `BadNoData`); `Start`/`End` return the **raw** first/last sample of any quality — its own timestamp and status, value null when Bad; `Delta` is Uncertain only when the interval's first or last sample is unusable; ⚠️ #684: `Average`/`Minimum`/`Maximum`/`Range`/`*ActualTime` use **Good samples only** (was Good + Uncertain; an Uncertain-only interval is `BadNoData`), Min/Max-family status Uncertain on a Bad sample, an Uncertain value beyond the Good extreme (Part 13 — the SDK ignores that) or an Uncertain non-numeric sample, and only a **Good** String/DateTime sample refuses them (`BadAggregateInvalidInputs`; Delta still refuses any usable one); and every per-sample aggregate but `Average` sets **end-of-data Partial** on the interval holding the last sample (forward incl. its earlier edge, reverse strictly inside), never on a Bad result. The data-range lookup is a key seek on SQL Server and a `tag` symbol-index seek on QuestDB (#683 — `{p}sample.tag` is `SYMBOL INDEX`; the bootstrapper ADDs it to a pre-#683 table, idempotent), read BEFORE the bucket statement so a racing append can never yield a Good `Count` of 0 (a concurrent HistoryUpdate delete is the mirror case and still can, transiently). `docs/Historian.md` §"Aggregates". Docker-dev recipe, schema and aggregate semantics: `docs/Historian.md` §"SQL Server backend". Integration tests need `HISTORIAN_SQL_TEST_CONNECTION`. **Retention (#667):** opt-in `SqlServer:RetentionDays` (+ `PurgeInterval` 1h, `PurgeBatchSize` 4000) -> `HistorianRetentionService` purges old `hist.Sample` (per tag) / `hist.Event` in `DELETE TOP(n)` batches, Primary-gated on `IHistorizationRoleView`, one purger per DB via `sp_getapplock`; counter `otopcua.historian.retention.purged{table}`. Optional `IHistorianBackendProvider.CreateRetention` (default null). `docs/Historian.md` §Retention.
- **SQL Server** (`Driver.Historian.SqlServer`): `hist.Sample` PK `(TagId, SourceTimeUtc)` — writes are insert-if-absent, so outbox redelivery is idempotent. Stores every value kind with its **real status**; per-sample aggregates (10, simple definitions) are pushed down to T-SQL with bucket math in **100 ns ticks**; the Part 13 time-weighted seven (TimeAverage/TimeAverage2/Total/Total2/Interpolative, #671 — ⚠️ `Total` is now the time-weighted integral, no longer `SUM` — plus the **duration-based** PercentGood/PercentBad, #680, and DurationGood/DurationBad, #820 — ms, Uncertain time in neither; the live AggregateFilter still counts Uncertain as Good, #809) stream window + 4 bound rows into the shared `TimeWeightedAggregator` (both SQL backends, identical results; reverse `Interpolative` anchored at the later time; window capped by `ServerHistorian:MaxTimeWeightedRawRows` ⇒ `BadTooManyOperations`, percentages included). ⚠️ **#680 status rules (client-visible):** TimeAverage/Total are Uncertain if any part of the interval uses an Uncertain value, is bridged across a Bad sample, is extrapolated past the last sample (⇒ a live tag's latest interval is Uncertain) or is not covered; TimeAverage2/Total2 keep the SDK time-based rule (Uncertain time counts as Good) but never extrapolate past the last sample — that interval is Partial, later ones `BadNoData` (a report-by-exception tag with nothing after its last sample reads `BadNoData` until its next sample arrives). `docs/Historian.md` §"Time-weighted aggregates". ⚠️ **#680 per-sample (client-visible):** `Count` counts **Good** samples only (value-based status; an all-Bad interval is generic Bad, value null) and an empty interval **inside the tag's data range** is `0`/Good (outside it still `BadNoData`); `Start`/`End` return the **raw** first/last sample of any quality — its own timestamp and status, value null when Bad; `Delta` is Uncertain only when the interval's first or last sample is unusable; ⚠️ #684: `Average`/`Minimum`/`Maximum`/`Range`/`*ActualTime` use **Good samples only** (was Good + Uncertain; an Uncertain-only interval is `BadNoData`), Min/Max-family status Uncertain on a Bad sample, an Uncertain value beyond the Good extreme (Part 13 — the SDK ignores that) or an Uncertain non-numeric sample, and only a **Good** String/DateTime sample refuses them (`BadAggregateInvalidInputs`; Delta still refuses any usable one); and every per-sample aggregate but `Average` sets **end-of-data Partial** on the interval holding the last sample (forward incl. its earlier edge, reverse strictly inside), never on a Bad result. The data-range lookup is a key seek on SQL Server and a `tag` symbol-index seek on QuestDB (#683 — `{p}sample.tag` is `SYMBOL INDEX`; the bootstrapper ADDs it to a pre-#683 table, idempotent), read BEFORE the bucket statement so a racing append can never yield a Good `Count` of 0 (a concurrent HistoryUpdate delete is the mirror case and still can, transiently). `docs/Historian.md` §"Aggregates". Docker-dev recipe, schema and aggregate semantics: `docs/Historian.md` §"SQL Server backend". Integration tests need `HISTORIAN_SQL_TEST_CONNECTION` (shared host `10.100.0.35,14330`, `sa`/`OtOpcUaDev_2026!` — not docker-dev's `OtOpcUa!Dev123`; QuestDB: `HISTORIAN_QUESTDB_TEST_CONNECTION` against the shared `/opt/otopcua-questdb` fixture `10.100.0.35:8812`, #822, `infra/README.md` §2a). **Retention (#667):** opt-in `SqlServer:RetentionDays` (+ `PurgeInterval` 1h, `PurgeBatchSize` 4000) -> `HistorianRetentionService` purges old `hist.Sample` (per tag) / `hist.Event` in `DELETE TOP(n)` batches, Primary-gated on `IHistorizationRoleView`, one purger per DB via `sp_getapplock`; counter `otopcua.historian.retention.purged{table}`. Optional `IHistorianBackendProvider.CreateRetention` (default null). `docs/Historian.md` §Retention.
- **QuestDB** (`Driver.Historian.QuestDb`, `ServerHistorian:QuestDb:*`, Npgsql over PG wire; docker-dev overlay `docker-compose.questdb.yml`): same kinds/status/aggregates/HistoryUpdate as SqlServer, soft delete by tombstone (INSERT only — never `UPDATE`/`DELETE`; every read filters `NOT deleted`). ⚠️ **Timestamps are µs** (`QuestDbConnection.Round`/`BindTimestamp`; `IHistorianDataSource.TimestampResolutionTicks` = 10 steps HistoryRead paging), **pre-1970 cannot be stored** (writer drops, HistoryUpdate `BadOutOfRange`), every write waits for WAL visibility, TTL is enforced on write commits only, set Npgsql `Maximum Pool Size` < QuestDB's 64. `docs/Historian.md` §"QuestDB backend".
- ⚠️ **Reverse processed reads are Part 13/SDK-anchored (#677, client-visible):** every aggregate on SqlServer/QuestDB now anchors a reverse read at its later time — window `(end, start]`, partial interval earliest, each stamped with its later edge (was: earlier-anchored, stamped earlier edge); on forward reads Partial also marks the interval the data starts inside (not Average — the SDK's rule, forward-only); a non-default client `AggregateConfiguration` ⇒ `BadAggregateConfigurationRejected`; Gateway provisioning sends `IntegralDivisor = 1`. SDK-parity tests: `SdkAggregateParityTests`. `docs/Historian.md` §Aggregates.
- ⚠️ **Reverse / end-only RAW reads work on Gateway (#686, client-visible):** `IHistorianDataSource.SupportsReverseRaw` (SqlServer/QuestDB/Null `true`, Gateway `false`, interface default `false`, guard-tested) — the node manager sends `start > end` only to a backend that declares it; otherwise `ReverseRawReader` serves it from FORWARD reads: uncapped ⇒ ONE forward read reversed in memory; capped ⇒ a bounded walk back in segments (ms-aligned request bounds, width aimed from observed density at ≈ N + 8 rows and carried page to page in the continuation point — #689: it used to aim at half `MaxTieClusterOverfetch`, ~32k rows per page on a dense tag; a read asks for `2N + 18` rows (a discovery segment ≤ 1,026), `MaxTieClusterOverfetch` is only the tie-cluster ceiling — a full read is re-read, never answered truncated; only more than max(1, `MaxTieClusterOverfetch`) samples within ONE MILLISECOND ⇒ `BadTooManyOperations` (per ms, not per timestamp — requests are ms-rounded, and letting a segment narrow below 1 ms stepped a page through one ms a tick per read, ~10k reads); walks no later than now + 1 day and never before the Unix epoch; #691: after two empty segments it asks once for the whole remainder, and the stretch a page proved empty rides the continuation point — the last pages of a deep page-through no longer each walk ~14 reads to 1970). The raw/events tie-path oversized-cluster refusal is now `BadTooManyOperations` too (was `BadHistoryOperationUnsupported`). Uncapped Gateway raw reads (forward too) used to send `max_values = 0`, which the client/gateway reject — now unbounded (`int.MaxValue`; a ceiling refusal ⇒ `BadTooManyOperations`), and a cancelled read is no longer a backend failure. Since #669 such reads failed the node on Gateway and counted as a backend fault (the gateway rejects start > end — `GatewayHistorianDataSource.ReadRawAsync` now refuses one unsent); before #669 an end-only Gateway read returned the OLDEST N ascending — now the latest N at/before EndTime, newest first. Events follow since #691.4: a reverse/end-only capped events read on Gateway (`SupportsReverseEvents` false; Null now reports `true` and answers empty) is walked back the same way (ties sorted by EventId — the gateway orders by time only; one extra event found ⇒ continuation point) — it used to be refused `BadHistoryOperationUnsupported` (explicit reverse) or answered with the OLDEST N ascending (end-only); needs the gateway's `RuntimeDb:EventReadMaxRows` 0/unlimited (it truncates silently). Uncapped Gateway events reads (forward too) used to throw "Array dimensions exceeded" (result list pre-sized to `int.MaxValue`) — fixed. Above a truncated read the walk gallops up from the newest row seen, then bisects (never below 1 ms): a burst that ended 100 days ago ≈ 18 reads, a dense tag that stopped days ago ≈ 26. `docs/Historian.md` §"Open-ended and reverse raw reads".
- **Advertised aggregates (#682):** `IHistorianDataSource.SupportedAggregates` (Gateway 4 via `GatewayHistorianDataSource.ServedAggregates`, SqlServer/QuestDB 17 via `AggregateFamilies.Served`, Null **none**; a null set ⇒ all) is the one authority for `HistoryServerCapabilities/AggregateFunctions` (i=11201 — only its folder references are trimmed) and the up-front `BadAggregateNotSupported` in HistoryRead-Processed. ⚠️ `ServerCapabilities/AggregateFunctions` (i=2997) is deliberately NOT trimmed: it keeps the SDK's 37 because live `AggregateFilter` subscriptions still compute them all. ⚠️ Historian disabled ⇒ Processed is now `BadAggregateNotSupported`, not `GoodNoData`. `docs/Historian.md` §Advertised aggregates.
- **Advertised aggregates (#682):** `IHistorianDataSource.SupportedAggregates` (Gateway 4 via `GatewayHistorianDataSource.ServedAggregates`, SqlServer/QuestDB 19 via `AggregateFamilies.Served` — DurationGood/DurationBad added by #820, the PercentGood/PercentBad region times in ms, Null **none**; a null set ⇒ all) is the one authority for `HistoryServerCapabilities/AggregateFunctions` (i=11201 — only its folder references are trimmed) and the up-front `BadAggregateNotSupported` in HistoryRead-Processed. ⚠️ `ServerCapabilities/AggregateFunctions` (i=2997) is deliberately NOT trimmed: it keeps the SDK's 37 because live `AggregateFilter` subscriptions still compute them all. ⚠️ Historian disabled ⇒ Processed is now `BadAggregateNotSupported`, not `GoodNoData`. `docs/Historian.md` §Advertised aggregates.
- **Raw `ReturnBounds` is honoured on every backend (#691, client-visible):** Part 11 bounding values (§4.6 Table 1 is the test oracle) are composed in the server by `HistoryBounds` from each backend's raw reads — "at or before" = capped reverse read (native on SQL backends, the #686 `ReverseRawReader` walk-back on Gateway, never AVEVA's synthetic boundary row), "at or after" = capped forward read. Bounds are stored samples of any quality, count toward `NumValuesPerNode`, start bound on the first page / end bound on the last (pending end bound rides the continuation point); missing ⇒ `Bad_BoundNotFound` (null, at the bound time; a one-sided read's open side ±1 s from the previous value). ⚠️ **Breaking for SDK-default clients:** the .NET SDK's `new ReadRawModifiedDetails()` defaults BOTH `ReturnBounds = true` AND `IsReadModified = true` — a client that cleared only `IsReadModified` now gets bounds, and one that set neither is a modified read with bounds ⇒ `Bad_InvalidArgument` (§6.5.3.3); set both explicitly (Client.Shared does; tests building details must too). No data AND no bound found ⇒ `GoodNoData`, no values (the pre-bounds answer). On Gateway the at-or-after lookup ends at now + 1 day (never a year-9999 end to AVEVA). Kill switch `ServerHistorian:HonorReturnBounds` (default `true`; `false` = pre-#691). `docs/Historian.md` §Bounding values.
- ⚠️ **Raw pages are full (#692, client-visible, every backend):** each raw page asks the backend for `N + skip + 1` rows (the boundary ties it will trim + one lookahead row) — a resumed page used to ask only N, trim the ties and come back short; at N=1 that was an EMPTY page + continuation point that then fell into the `MaxTieClusterOverfetch + 1` over-fetch (live: 8 values = 17 pages / 25 reads / 524k rows ⇒ now 8/8/≤24). No lookahead row ⇒ last page, no CP (an exact multiple no longer ends on an empty page). Exception: an oversized tie cluster — its last slice is `min(N, remaining)` with a CP, and stepping past one at the end of data can give one empty final page. The over-fetch fires only when every row past the trimmed ties sits on the boundary timestamp; a cursor inside such a cluster is marked `WithinTieCluster` (next page goes straight to the over-fetch). `SliceTieCluster` resumes only strictly before EndTime (truncated to the backend resolution) — it used to resume AT the exclusive EndTime and return samples there. Bound placeholders carry an explicit marker (`HistoryBounds.IsPlaceholder` no longer matches a stored `Bad_BoundNotFound`/null row); the N=1 bounds probe carries its walk hint + resolved end bound into the cursor. Reviewer fuzz harnesses (paged vs unpaged over random tie series) are the right test for any paging change — repo tests passed while master mismatched 1-in-160.
- ⚠️ **Strict-ceiling paging (#694):** `IHistorianDataSource.MaxRawValuesPerRead` = L = learned gateway ceiling − 16 (0 on SQL backends/Null, guard-tested; 0 until a refusal names the ceiling). `RawPageLimit.Plan(want, k, L)` sizes every raw page so page + boundary ties + lookahead ≤ L (room = L − k − 1, signed; first page L−1, resumed L−k−1; uncapped `N = 0` reads page too) — a capped read above the ceiling used to be `BadTooManyOperations`, now a ceiling-sized page + CP. One re-plan when the read taught a new limit or was refused for crowding (`HistoryReadLimitExceededException.PreWindowRows`, set only above the 16-row allowance; the page shrinks by it). No room on a resumed page ⇒ the tie-cluster over-fetch, bounded min(M, L−1); the slice size is never 0. The page path passes `retryOnLearnedLimit: false` to `ReverseRawReader` (the walk re-running at the old N cost 200–3,000 reads cold); bounds lookups keep it. ⚠️ A Gateway window starting at/before the Unix epoch counts AVEVA's synthetic row as pre-window — it used to set `reachedEnd`, so a ceiling-truncated read came back Good with no CP (forward-from-1970 = first page only; end-only past the data = the OLDEST rows as "latest"; pre-existing). The #687 crowding re-read now doubles its room up to 8 re-reads, never past a known ceiling, and its room is capped at `MaxTieClusterOverfetch` (passed to the Gateway source by the factory; past it ⇒ `BadTooManyOperations` with `PreWindowRows`). A raised ceiling is not re-probed by raw paging (pages stay small until another read re-probes or restart).
@@ -50,7 +50,7 @@ historian tagname, which is the tag's driver `FullName`.
The server dispatches all OPC UA HistoryRead to the registered `IHistorianDataSource` — the
`GatewayHistorianDataSource` read client when enabled, else the `NullHistorianDataSource` default
(historized nodes return `GoodNoData` for Raw/AtTime — Processed is `BadAggregateNotSupported` since #682). Supported variants: Raw (⚠️ #687, client-visible: Gateway raw reads no longer return AVEVA's synthetic start-boundary row — asked from 1 ms early (epoch-floored) with `NumValuesPerNode + 16` rows (unclamped; 0 ⇒ `int.MaxValue`) and filtered by timestamp — so an empty window is `GoodNoData`, not a fabricated value at StartTime; the forward end is now exclusive (a point read inclusive); pre-window rows that crowd the budget trigger ONE re-read with room for them, else `BadTooManyOperations`; `ReturnBounds` is ignored on every backend; ⚠️ #690: a Gateway Bad row — a stored NULL, which histsdk sends as value 0 — now reads value **null** + Bad on Raw and AtTime, as on the SQL backends; ⚠️ #691.3: a strict gateway (`AllowStreamingBeyondCeiling=false`) that refuses a raw budget is re-asked ONCE at the ceiling its refusal names — raw and per-sample reads alike; the ceiling is then remembered per data source (`GatewayRowCeiling`: refusals count as gateway errors), re-learned from a refusal when lowered, and re-probed once at the full budget when a ceiling-bounded read comes back full (raised / streaming) — so an uncapped (`NumValuesPerNode = 0`) read of a small window is served; a ceiling-bounded stream that is not provably complete is `BadTooManyOperations`, never a truncated answer), Processed
(backend-dependent: SqlServer/QuestDb serve seventeen; Gateway twelve = `GatewayHistorianDataSource.ServedAggregates` — ⚠️ #681, client-visible: the ten per-sample ones (Average, Count, Min/Max, Range, Start/End/Delta, Min/MaxActualTime) are evaluated from RAW gateway reads by the shared `Sql.Common` evaluator — identical to the SQL backends, Calculated bits, interval-start stamps, #677 reverse; Min/Max are no longer AVEVA modes, the read moves every raw sample of the window (a month of 1 Hz ⇒ `BadTooManyOperations` at the default `MaxTimeWeightedRawRows` cap), and every gateway stream is consumed to completion (an abandoned one makes the gateway evict its shared historian session) — plus TimeAverage/Total via AVEVA modes (interval-start stamped + Calculated; ⚠️ #685: AVEVA's extra pre-window row is dropped ⇒ one row per interval in `[Start, End)`, and an Uncertain (opc 64) interval keeps its value as `UncertainDataSubNormal` — AVEVA's TWA excludes NULL-gap time where the SQL backends bridge it; per-interval values still need gateway fix historiangw#27; ⚠️ #691.2: native reads use the SQL backends' grid in both directions via forward requests only — a REVERSE read (refused by the gateway before) is anchored at its later time (#677), and the clipped interval (earliest reverse, LATEST forward — which used to be averaged past EndTime) is its own exact-span read marked Partial unless Bad; an interval wider than the window is sent as the window, interval 0 as one interval; more than 100,000 intervals is refused up front (`BadHistoryOperationUnsupported`, as on the SQL backends — forward used to be served up to the gateway's 1,000,000); until historiangw#27 lands both directions return Good-status wrong values); the other time-weighted five stay `BadAggregateNotSupported` because the gateway has no prior-sample read for Part 13 bounds; `docs/Historian.md` §Aggregate set), and AtTime over historized variable nodes (⚠️ #693, client-visible: Gateway AtTime is evaluated from RAW reads by the shared `Sql.Common` `AtTimeEvaluator` — exact stored value at a stored time, null + Bad at/after a stored NULL, Uncertain hold before a Bad neighbour, linear + Interpolated bit, `BadNoData` before the first sample — never AVEVA's own interpolation (which answered 107 Good at a NULL, null+Bad at a Good sample after it, 99.99999999944 for 100, no Interpolated bit); `GatewayAtTimeReader`: one ≤1,024-row window read for the batch (jumps to far-apart times use 64-row reads) + a walk-back for the first time's prev (gallop ×4, whole-remainder read after two empties, then real BISECTION seeded by any full read's last row — a dense tag stopped a year ago ≈ 25–31 reads; the first cut regrew segments after a full read and gave up at 65) + an 18-row probe for the last time's next — 10 dense times = 1 read (exact) / 3 (interpolated), 10 hourly = 12; read cap 64 + 16 per distinct time (a runaway guard: a time in a gap of a bursty tag needs ≈ 6 reads; 4/time refused a 1 h/day tag asked daily ×30) and a strict ceiling < 4 rows ⇒ `BadTooManyOperations` with a message naming the limit; DateTime hints stepped, non-finite/out-of-range values under an integer hint unusable; all request bounds whole ms, rows counted only strictly inside a request; the node manager passes the node's scalar type (`ReadAtTimeAsync(…, TypeCode? valueType, …)` default-interface overload, ignored by SqlServer/QuestDB/Null) and the Gateway converts raw rows to it before evaluating — Boolean stepped, integers interpolated + rounded half-away and returned as the node's integer type, floats Double; no hint ⇒ Double; ⚠️ the shared evaluator used to box interpolated Int64 as Double (`cond ? (long)x : double` widens) — fixed for SQL backends too, and Shouldly `ShouldBe` hides numeric-type bugs (assert `ShouldBeOfType`); kill switch `ServerHistorian:GatewayAtTimeFromRaw` (default `true`); `docs/Historian.md` §Gateway AtTime); Events over
(backend-dependent: SqlServer/QuestDb serve nineteen; Gateway twelve = `GatewayHistorianDataSource.ServedAggregates` — ⚠️ #681, client-visible: the ten per-sample ones (Average, Count, Min/Max, Range, Start/End/Delta, Min/MaxActualTime) are evaluated from RAW gateway reads by the shared `Sql.Common` evaluator — identical to the SQL backends, Calculated bits, interval-start stamps, #677 reverse; Min/Max are no longer AVEVA modes, the read moves every raw sample of the window (a month of 1 Hz ⇒ `BadTooManyOperations` at the default `MaxTimeWeightedRawRows` cap), and every gateway stream is consumed to completion (an abandoned one makes the gateway evict its shared historian session) — plus TimeAverage/Total via AVEVA modes (interval-start stamped + Calculated; ⚠️ #685: AVEVA's extra pre-window row is dropped ⇒ one row per interval in `[Start, End)`, and an Uncertain (opc 64) interval keeps its value as `UncertainDataSubNormal` — AVEVA's TWA excludes NULL-gap time where the SQL backends bridge it; per-interval values still need gateway fix historiangw#27; ⚠️ #691.2: native reads use the SQL backends' grid in both directions via forward requests only — a REVERSE read (refused by the gateway before) is anchored at its later time (#677), and the clipped interval (earliest reverse, LATEST forward — which used to be averaged past EndTime) is its own exact-span read marked Partial unless Bad; an interval wider than the window is sent as the window, interval 0 as one interval; more than 100,000 intervals is refused up front (`BadHistoryOperationUnsupported`, as on the SQL backends — forward used to be served up to the gateway's 1,000,000); until historiangw#27 lands both directions return Good-status wrong values); the other time-weighted five stay `BadAggregateNotSupported` because the gateway has no prior-sample read for Part 13 bounds; `docs/Historian.md` §Aggregate set), and AtTime over historized variable nodes (⚠️ #693, client-visible: Gateway AtTime is evaluated from RAW reads by the shared `Sql.Common` `AtTimeEvaluator` — exact stored value at a stored time, null + Bad at/after a stored NULL, Uncertain hold before a Bad neighbour, linear + Interpolated bit, `BadNoData` before the first sample — never AVEVA's own interpolation (which answered 107 Good at a NULL, null+Bad at a Good sample after it, 99.99999999944 for 100, no Interpolated bit); `GatewayAtTimeReader`: one ≤1,024-row window read for the batch (jumps to far-apart times use 64-row reads) + a walk-back for the first time's prev (gallop ×4, whole-remainder read after two empties, then real BISECTION seeded by any full read's last row — a dense tag stopped a year ago ≈ 25–31 reads; the first cut regrew segments after a full read and gave up at 65) + an 18-row probe for the last time's next — 10 dense times = 1 read (exact) / 3 (interpolated), 10 hourly = 12; read cap 64 + 16 per distinct time (a runaway guard: a time in a gap of a bursty tag needs ≈ 6 reads; 4/time refused a 1 h/day tag asked daily ×30) and a strict ceiling < 4 rows ⇒ `BadTooManyOperations` with a message naming the limit; DateTime hints stepped, non-finite/out-of-range values under an integer hint unusable; all request bounds whole ms, rows counted only strictly inside a request; the node manager passes the node's scalar type (`ReadAtTimeAsync(…, TypeCode? valueType, …)` default-interface overload, ignored by SqlServer/QuestDB/Null) and the Gateway converts raw rows to it before evaluating — Boolean stepped, integers interpolated + rounded half-away and returned as the node's integer type, floats Double; no hint ⇒ Double; ⚠️ the shared evaluator used to box interpolated Int64 as Double (`cond ? (long)x : double` widens) — fixed for SQL backends too, and Shouldly `ShouldBe` hides numeric-type bugs (assert `ShouldBeOfType`); kill switch `ServerHistorian:GatewayAtTimeFromRaw` (default `true`); `docs/Historian.md` §Gateway AtTime); Events over
alarm-owning equipment-folder event-notifier nodes. Reads are ungated (served from any redundancy node);
authorization uses the standard `AccessLevels.HistoryRead` bit set at materialization.
+3
View File
@@ -317,6 +317,9 @@ lmxopcua-fix sync modbus # rsync this repo's tests/.../Docker/
- AB CIP: `10.100.0.35:44818` (`AB_SERVER_ENDPOINT`)
- S7: `10.100.0.35:1102` (`S7_SIM_ENDPOINT`)
- OPC UA reference (opc-plc): `opc.tcp://10.100.0.35:50000` (`OPCUA_SIM_ENDPOINT`)
- QuestDB historian fixture (#822, stack `/opt/otopcua-questdb`): PG wire `10.100.0.35:8812` —
`HISTORIAN_QUESTDB_TEST_CONNECTION='Host=10.100.0.35;Port=8812;Username=admin;Password=quest;Database=qdb;ServerCompatibilityMode=NoTypeLoading'`;
the SQL Server historian suite uses the central SQL above (`HISTORIAN_SQL_TEST_CONNECTION`, `infra/README.md` §2a).
- MQTT (Mosquitto, **auth on both listeners — no anonymous fallback**): `10.100.0.35:8883` TLS+auth
(`MQTT_FIXTURE_ENDPOINT`) · `10.100.0.35:1883` plaintext-but-authenticated (`MQTT_FIXTURE_PLAIN_ENDPOINT`).
Also needs `MQTT_FIXTURE_USERNAME` / `MQTT_FIXTURE_PASSWORD`, and `MQTT_FIXTURE_CA_CERT` to pin the
+45 -18
View File
@@ -108,7 +108,7 @@ consumes the historian. The existing enable gates (`ServerHistorian:Enabled`, `A
| Value kinds recorded | `Double`, `Int64`, `Boolean` (numeric analog only) | all: `Double`, `Int64`, `Boolean`, `String`, `DateTime`, plus `Empty` (a Bad sample with no value) | same as `SqlServer` |
| Status code stored | Sent as DA quality but **discarded** by the gateway (C-002); AVEVA stores every sample Good | Stored bit-for-bit (`StatusCode` int column) | Stored bit-for-bit (unsigned, in a `LONG`) |
| Comms-loss / driver-Bad publishes | Dropped, counted in `DroppedUnsupportedKind` | Stored as an `Empty` outage-marker sample carrying the Bad code (see [below](#typed-values-real-status-and-outage-markers)) | Stored as an `Empty` outage-marker sample, as on `SqlServer` |
| Processed aggregates | **Ten** by default: the per-sample ones (Average, Count, Minimum, Maximum, Range, Start, End, Delta, Min/MaxActualTime) evaluated from raw reads with the SQL backends' evaluator — identical results (#681). TimeAverage and Total from AVEVA's native modes **only with `ServerHistorian:Gateway:EnableNativeTimeWeighted=true`** (default off: Good-status wrong values until historiangw#27). TimeAverage2 / Total2 / Interpolative / PercentGood / PercentBad: never | The seventeen in the [aggregate table](#aggregates-processed-reads) (incl. Part 13 time-weighted, #671) | The same seventeen, identical results |
| Processed aggregates | **Ten** by default: the per-sample ones (Average, Count, Minimum, Maximum, Range, Start, End, Delta, Min/MaxActualTime) evaluated from raw reads with the SQL backends' evaluator — identical results (#681). TimeAverage and Total from AVEVA's native modes **only with `ServerHistorian:Gateway:EnableNativeTimeWeighted=true`** (default off: Good-status wrong values until historiangw#27). TimeAverage2 / Total2 / Interpolative / PercentGood / PercentBad / DurationGood / DurationBad: never | The nineteen in the [aggregate table](#aggregates-processed-reads) (incl. Part 13 time-weighted, #671, and DurationGood/DurationBad, #820) | The same nineteen, identical results |
| Reverse / end-only Raw and Events reads | Emulated: walked back from bounded forward reads by the server (`ReverseRawReader` / `ReverseEventsReader`) — correct, but many round trips on a sparse tag | Native newest-first | Native newest-first |
| Capped Events read | The cap is applied client-side (the gateway RPC has no max-events field); the stream past it is drained up to `Gateway:EventDrainBudget` rows | `TOP` | `LIMIT` |
| Retention | AVEVA Historian's own | `SqlServer:RetentionDays` purge (opt-in) | `QuestDb:RetentionDays` TTL (opt-in) |
@@ -192,7 +192,7 @@ Processed read is `BadAggregateNotSupported` and no history aggregate functions
| `EventReadHorizon` | TimeSpan | `00:00:00` | (#694, all backends, **opt-in, lossy**.) `00:00:00` = no horizon: every HistoryReadEvents read is exact to its own window. A positive span raises the earlier bound of every fresh events window to `now − EventReadHorizon` (the floor instant itself included), in **both** directions — a window wholly older is `GoodNoData` and is not sent; a page-through keeps the first page's floor. It exists for the Gateway, where a walked reverse / end-only read proves "nothing older" down to the Unix epoch (~3.8 s live; an unfiltered one can hit the gateway's 30 s SQL timeout) and a forward read from a far-past `StartTime` scans the same range. ⚠️ While set, events older than the horizon are **invisible** to HistoryReadEvents. Raw/Processed/AtTime are unaffected. Negative ⇒ treated as none (a `Validate()` warning). See [Continuation-point paging (Events)](#continuation-point-paging-events-669). |
| `HistoryReadBatchParallelism` | int | `4` | Max historized nodes served **concurrently** within one HistoryRead batch (arch-review 03/S3). The SDK's `HistoryRead*` override surface is synchronous, so each arm bridges once at its boundary; this turns per-node serving from sequential (`N × CallTimeout` worst case) into `⌈N / P⌉ × CallTimeout`. Also caps the gateway-load multiplication a single client can command. `≤ 1` serves sequentially. |
| `MaxConcurrentHistoryReadBatches` | int | `16` | Process-wide limit on HistoryRead (and HistoryUpdate) batches served **concurrently** (arch-review 03/S3). A flood of history clients degrades to `BadTooManyOperations` rather than exhausting the SDK request-thread pool. With `HistoryReadBatchParallelism` this caps total in-flight gateway reads at `MaxConcurrentHistoryReadBatches × HistoryReadBatchParallelism` (default 16 × 4 = 64). A batch that cannot get a slot within `min(HistoryReadDeadline, 5 s)` fails every handle `BadTooManyOperations`. |
| `MaxTimeWeightedRawRows` | int | `1000000` | `SqlServer` / `QuestDb` (#671), and the `Gateway` per-sample aggregates (#681 — every per-sample read there streams raw samples, so the cap applies to the window's raw rows; see [Aggregate set](#aggregate-set-and-badaggregatenotsupported-285)). The most raw in-window samples one [time-weighted aggregate](#time-weighted-aggregates-part-13-671) read (TimeAverage, TimeAverage2, **Total**, Total2, Interpolative, and — since #680 — **PercentGood/PercentBad**) may stream into the server-side evaluator — that family is **not** pushed down to SQL. The window query is capped at this + 1 rows; one row past the cap fails **that node** with `BadTooManyOperations` (not counted as a backend fault) instead of holding a pooled connection until `HistoryReadDeadline` and returning `BadTimeout`. Bound rows outside the window do not count; per-sample aggregates are not capped on the SQL backends (pushed down). 1 000 000 ≈ 11.5 days of a 1 Hz tag. Non-positive = unbounded (a `Validate()` warning). `Gateway` (#688): the row budget it asks the gateway for (`cap + 18`) is clamped at the gateway's default `MaxValuesCeiling` (1,000,000) while the cap is at or under it, so the default works against a gateway with `AllowStreamingBeyondCeiling=false`; against a gateway with a *lower* ceiling the refused budget is retried once at the ceiling the gateway names (#691.3), so no change is needed (`ceiling − 18` only saves that round-trip). |
| `MaxTimeWeightedRawRows` | int | `1000000` | `SqlServer` / `QuestDb` (#671), and the `Gateway` per-sample aggregates (#681 — every per-sample read there streams raw samples, so the cap applies to the window's raw rows; see [Aggregate set](#aggregate-set-and-badaggregatenotsupported-285)). The most raw in-window samples one [time-weighted aggregate](#time-weighted-aggregates-part-13-671) read (TimeAverage, TimeAverage2, **Total**, Total2, Interpolative, and — since #680 — **PercentGood/PercentBad**, since #820 **DurationGood/DurationBad**) may stream into the server-side evaluator — that family is **not** pushed down to SQL. The window query is capped at this + 1 rows; one row past the cap fails **that node** with `BadTooManyOperations` (not counted as a backend fault) instead of holding a pooled connection until `HistoryReadDeadline` and returning `BadTimeout`. Bound rows outside the window do not count; per-sample aggregates are not capped on the SQL backends (pushed down). 1 000 000 ≈ 11.5 days of a 1 Hz tag. Non-positive = unbounded (a `Validate()` warning). `Gateway` (#688): the row budget it asks the gateway for (`cap + 18`) is clamped at the gateway's default `MaxValuesCeiling` (1,000,000) while the cap is at or under it, so the default works against a gateway with `AllowStreamingBeyondCeiling=false`; against a gateway with a *lower* ceiling the refused budget is retried once at the ceiling the gateway names (#691.3), so no change is needed (`ceiling − 18` only saves that round-trip). |
| `HistoryReadDeadline` | TimeSpan | `00:01:00` | Per-request wall-clock deadline for a HistoryRead (and HistoryUpdate) batch (arch-review 03/S3). A node still in flight at the deadline surfaces `BadTimeout`, so one request can never hold an SDK request thread longer than this regardless of node count. Non-positive = unbounded (a `Validate()` warning). |
| `HistoryUpdateMaxFutureSkew` | TimeSpan | `1.00:00:00` | #674. How far past the server clock a HistoryUpdate (Insert / Replace / Update) item's `SourceTimestamp` may lie; an item beyond `now + skew` is refused **`BadOutOfRange`** by the node manager before any backend sees it (every backend). Guards against a far-future typo (a 2062 timestamp). The live recorder is not bounded. Non-positive disables the bound (no warning — a deliberate choice). See [HistoryUpdate](#historyupdate). |
@@ -608,7 +608,7 @@ The server supports all four OPC UA HistoryRead variants:
| Variant | Node type | CLI `--aggregate` |
|---|---|---|
| **Raw** | Historized variable | (omit `--aggregate`) |
| **Processed** | Historized variable | `Average`, `Minimum`, `Maximum`, `Count`, `Start`, `End`, `StandardDeviation` (the CLI's own list; `SqlServer` / `QuestDb` serve seventeen aggregates — see the [aggregate table](#aggregates-processed-reads)) |
| **Processed** | Historized variable | `Average`, `Minimum`, `Maximum`, `Count`, `Start`, `End`, `StandardDeviation` (the CLI's own list; `SqlServer` / `QuestDb` serve nineteen aggregates — see the [aggregate table](#aggregates-processed-reads)) |
| **AtTime** | Historized variable | n/a (client supplies exact timestamps) |
| **Events** | Equipment folder (event notifier) | `--events` (+ `--fields`, `--of-type`; #776) |
| **Modified** (Raw with `IsReadModified = true`, #668) | Historized variable | `--modified` (see [Modified-value history](#modified-value-history-part-11)) |
@@ -1312,7 +1312,7 @@ field (`tail + 43`) its own reverse-engineering notes still mark unverified (His
> `Integral`) — ⚠️ **and are refused `BadAggregateNotSupported` unless `ServerHistorian:Gateway:EnableNativeTimeWeighted` is
> `true`** (default `false`, until historiangw#27 is live-verified; the default set is the ten raw-evaluated aggregates,
> `GatewayHistorianDataSource.RawEvaluatedAggregates`) — now stamped with the interval start (was its end) and carrying the Calculated bit; `TimeAverage2`, `Total2`,
> `Interpolative`, `PercentGood` and `PercentBad` remain `BadAggregateNotSupported` on Gateway. The most the backend can serve is
> `Interpolative`, `PercentGood`, `PercentBad`, `DurationGood` and `DurationBad` remain `BadAggregateNotSupported` on Gateway. The most the backend can serve is
> `GatewayHistorianDataSource.ServedAggregates` (12); an instance serves and advertises `SupportedAggregates` — those 12 with the
> flag on, the 10 raw-evaluated ones without it.
@@ -1378,8 +1378,8 @@ and one ending at `now + 1 day` are accepted and cheap; (4) the gateway's ceilin
Min/MaxActualTime, Start/EndBound, DeltaBounds, DurationGood/Bad, PercentGood/Bad, WorstQuality,
NumberOfTransitions, AnnotationCount, StandardDeviationPopulation, VariancePopulation); `AggregateFunctionMap`
(`OpcUaServer`) maps each standard aggregate NodeId (`TimeAverage2` and `Total2` were appended for #671). The **`SqlServer`** and
**`QuestDb`** backends serve the seventeen listed in the [aggregate table](#aggregates-processed-reads); everything else
(e.g. `StartBound`, `DurationGood`, `StandardDeviationPopulation`) is `BadAggregateNotSupported` for that node. The **HistorianGateway** backend serves twelve (#681 —
**`QuestDb`** backends serve the nineteen listed in the [aggregate table](#aggregates-processed-reads); everything else
(e.g. `StartBound`, `WorstQuality`, `StandardDeviationPopulation`) is `BadAggregateNotSupported` for that node. The **HistorianGateway** backend serves twelve (#681 —
`GatewayHistorianDataSource.ServedAggregates`): the ten per-sample aggregates from raw reads, plus TimeAverage and Total
through AVEVA retrieval modes (`AggregateModeMapper`); for any other member it throws
`HistoryAggregateNotSupportedException` without calling the gateway (and since #682 the server refuses it up front), which the HistoryRead path turns into
@@ -1407,7 +1407,7 @@ aggregate-function objects. A client choosing a history aggregate should read **
| `ServerHistorian:Backend` | i=11201 (history) | i=2997 (server) | Declared by |
|---|---|---|---|
| `Gateway` | 12 — the ten per-sample aggregates (raw-evaluated, #681) + TimeAverage, Total (native AVEVA modes) | all 37 | `GatewayHistorianDataSource.ServedAggregates` (the per-sample families of `AggregateFamilies` ∪ `AggregateModeMapper.NativeAggregates`, derived from its mapping switch) |
| `SqlServer`, `QuestDb` | 17 — the [aggregate table](#aggregates-processed-reads) | all 37 | `AggregateFamilies.Served` (Sql.Common; every member with a family) |
| `SqlServer`, `QuestDb` | 19 — the [aggregate table](#aggregates-processed-reads) (DurationGood/DurationBad added by #820) | all 37 | `AggregateFamilies.Served` (Sql.Common; every member with a family) |
| historian disabled (Null source) | **none** | all 37 | `NullHistorianDataSource` |
⚠️ **Client-visible: with no historian configured a processed HistoryRead is now `BadAggregateNotSupported`, not
@@ -1573,9 +1573,10 @@ point (see [Continuation-point paging (Events)](#continuation-point-paging-event
> **Two kinds.** The *per-sample* aggregates (Average, Minimum, Maximum, Count, Range, Start/End/Delta, Min/MaxActualTime) are
> simple per-sample definitions — no time-weighting, no interval-boundary values — pushed down to SQL. The *time-weighted* aggregates
> (TimeAverage, TimeAverage2, Total, Total2, Interpolative; #671 — and the duration-based PercentGood/PercentBad since #680) follow
> (TimeAverage, TimeAverage2, Total, Total2, Interpolative; #671 — and the duration-based PercentGood/PercentBad since #680,
> DurationGood/DurationBad since #820) follow
> OPC UA Part 13 with bounding values — see [Time-weighted aggregates](#time-weighted-aggregates-part-13-671). Any `HistoryAggregateType` not in the tables throws
> `HistoryAggregateNotSupportedException` ⇒ `BadAggregateNotSupported` for that node (e.g. `StartBound`, `DurationGood`,
> `HistoryAggregateNotSupportedException` ⇒ `BadAggregateNotSupported` for that node (e.g. `StartBound`, `WorstQuality`,
> `StandardDeviationPopulation`).
>
> ⚠️ **Release note / migration (#671): `Total` changed meaning on the `SqlServer` and `QuestDb` backends.** It was the plain `SUM` of
@@ -1716,12 +1717,12 @@ Status and edge rules:
- An interval with samples but **no Good numeric value** (only Uncertain / Bad) ⇒ `BadNoData` for `Average`/`Minimum`/`Maximum`/`Range`/`*ActualTime` (#684).
- `Average` is `Good` when every sample in the bucket is Good, else `UncertainDataSubNormal` (value-based, the SDK's rule). `Minimum`/`Maximum`/`Range`/`*ActualTime` are `UncertainDataSubNormal` when the bucket holds a Bad sample or an Uncertain value beyond the reported Good extreme (Part 13 Tables 57–62; strictly beyond), else `Good` (#684). `Count`'s status is value-based (all Good ⇒ Good, all Bad ⇒ Bad, else Uncertain); `Start`/`End` carry the raw sample's own status; `Delta` is Good unless the interval's first or last sample is not usable (then Uncertain) — #680.
- `Count`, `Start`, `End` accept any value kind. `Average`/`Minimum`/`Maximum`/`Range`/`*ActualTime` on a bucket that holds a **Good** `String`/`DateTime` sample ⇒ `BadAggregateInvalidInputs` for that interval; an Uncertain one is skipped and makes `Minimum`/`Maximum`/`Range`/`*ActualTime` `UncertainDataSubNormal` (#684). `Delta` is refused by any usable non-numeric sample.
- `PercentGood` / `PercentBad` are **duration-based since #680** — see [Time-weighted aggregates](#time-weighted-aggregates-part-13-671).
- `PercentGood` / `PercentBad` are **duration-based since #680**, and `DurationGood` / `DurationBad` (#820) are the same region times — see [Time-weighted aggregates](#time-weighted-aggregates-part-13-671).
- Everything except `Count` and `Start`/`End` is a `double`; `Int64` samples aggregate as doubles and `Boolean` as 0/1.
### Time-weighted aggregates (Part 13, #671)
`TimeAverage`, `TimeAverage2`, `Total`, `Total2`, `Interpolative` and (since #680) `PercentGood` / `PercentBad`, evaluated by the
`TimeAverage`, `TimeAverage2`, `Total`, `Total2`, `Interpolative`, (since #680) `PercentGood` / `PercentBad` and (since #820) `DurationGood` / `DurationBad`, evaluated by the
shared `Driver.Historian.Sql.Common/TimeWeightedAggregator.cs` for **both** SQL backends (so `SqlServer` and `QuestDb` agree exactly).
**This whole family — `Total` and the percentages included — is evaluated in C#, not pushed down to SQL** (see *Pushdown trade-off*
and the raw-row cap below). Bucketing (edges, truncated interval, reverse anchoring, the 100 000-interval cap) is the same as above.
@@ -1784,6 +1785,7 @@ samples (sloped / stepped) straight across an outage, and holds the last usable
| `Total2` | ∫ simple-bounding signal over non-Bad time (value·s) | excluded |
| `Interpolative` | interpolated-bounding value **at the interval start** (reverse read: see below) | interpolated across |
| `PercentGood` / `PercentBad` | `100 ×` Good / Bad time ÷ interval duration (#680, Part 13 §5.4.3.31-34) — see below | counted as Bad time |
| `DurationGood` / `DurationBad` | Good / Bad time of the interval in **milliseconds** (an OPC UA `Duration`; #820, Part 13 §5.4.3.31-32) — the same regions, see below | counted as Bad time |
**Status, `TimeAverage` / `Total`** (interpolated bounding, #680 — Part 13 "Uncertain if Bad values are skipped or Uncertain values
are used", and the SDK's `AverageAggregateCalculator`):
@@ -1822,6 +1824,14 @@ to the **end of its own interval** (Part 13 — the SDK gives it 1 ms) and later
after it is Bad to the window end. An interval that ends before the first sample ⇒ `BadNoData`. Any value kind is accepted; the status
is `Good` (plus Calculated / Partial). Value = `100 × region time ÷ interval duration`.
**`DurationGood` / `DurationBad`** (#820) are the same Good / Bad region times as `PercentGood` / `PercentBad` — every rule above
(regions, simple bounds, time before the first sample = Bad, end of data, `BadNoData`, status, Partial) applies unchanged — reported as
an OPC UA `Duration`, i.e. a `double` in **milliseconds** (the SDK's unit): `DurationGood = PercentGood × interval ms ÷ 100`. Uncertain
time counts toward neither (Mantis 0006166). Before #820 both were `BadAggregateNotSupported` on the SQL backends. ⚠️ A **live**
`AggregateFilter` subscription computes DurationGood/PercentGood with the frozen 1.5.378.182 calculators, which count Uncertain time as
Good — they disagree with HistoryRead whenever Uncertain data is present (#809, owner decision pending). The Gateway backend still
refuses both (`BadAggregateNotSupported`, not advertised in i=11201).
All carry the Calculated bit. A non-Bad result also carries the **Partial** bit (`0x4`, `AggregateBits.Partial`) — never on a Bad
result — for: the **truncated interval** (shorter than the processing interval — the latest one forward, the earliest one reverse;
Part 13 §5.3.3.2 Case 3 requires it — the SDK does not set it, a deliberate difference); the interval the tag's **data starts inside**
@@ -1856,7 +1866,7 @@ value at **10:00**, not 09:00, and no point ever falls before the window. The in
forward and reverse, and compare every
interval's timestamp, value, status code and Partial bit. The per-sample comparison covers Average, Minimum, Maximum, Count, Range,
Start, End, Delta and Minimum/MaximumActualTime; the time-weighted one TimeAverage, TimeAverage2, Total, Total2, Interpolative and
PercentGood/Bad (#680), including datasets whose data ends inside the window and with an interval wholly inside an outage; reverse
PercentGood/Bad (#680) and DurationGood/Bad (#820), including datasets whose data ends inside the window and with an interval wholly inside an outage; reverse
reads over data that starts inside the window are compared too. Known differences that remain (tracked as
#680) — documented and masked in the tests, deliberately not changed here because forward results were to stay as they are — are:
an Uncertain value beyond the Good extreme makes Minimum/Maximum/Range/*ActualTime Uncertain here (Part 13), Good in the SDK
@@ -1895,9 +1905,14 @@ And an interval holding a **Good String (or DateTime) sample** is `BadAggregateI
string and serves the numeric samples (e.g. Average 4 Good); an Uncertain non-numeric sample refuses nothing on either side.
One more, found by the #810 Uncertain conformance data (kept deliberately): over the interval the data **starts inside** when the
**first sample is Uncertain**, the SDK's `TimeAverage`/`Total` integrate the uncovered span before it as a ramp from 0 (QP: `Total`
188.96 for 163.96 over the covered 15 s); ours integrates only the covered part (Part 13, as with a Good first sample, where the two
agree). Pinned by `QualityMixConformance.Time_weighted_values_over_partial_data_cover_only_the_data` on both SQL backends. The same
suite feeds PercentGood/PercentBad (and Count, Average, TimeAverage(2), Total(2), Start/End/Delta, Interpolative) Good/Uncertain/Bad
188.96 for 163.96 over the covered 15 s, and `TimeAverage` 9.448 = 188.96 ÷ 20 s for 10.931 = 163.96 ÷ 15 s); ours integrates only
the covered part (Part 13, as with a Good first sample, where all agree). The same answer comes from the frozen calculators the server
registers for **live** `AggregateFilter` items and from SDK 2.0.1's own stock calculators, in both read directions — pinned by
`SdkAggregateParityTests.An_Uncertain_first_sample_in_a_late_start_interval_makes_the_SDK_integrate_the_time_before_the_data`
(ours / frozen / stock 2.0.1, with a Good-first control) and `OpcUaServer.IntegrationTests/LiveAggregateLateStartFactsTests` (through
the booted server's runtime `AggregateManager`), and on both SQL backends by `QualityMixConformance.Time_weighted_values_over_partial_data_cover_only_the_data`
(forward) and `…_in_a_reverse_read` (#821). Whether live subscriptions should follow is the open #809/#821 owner decision. The same
suite feeds PercentGood/PercentBad, DurationGood/DurationBad (and Count, Average, TimeAverage(2), Total(2), Start/End/Delta, Interpolative) Good/Uncertain/Bad
mixes, all-Uncertain intervals (0 % Good **and** 0 % Bad, Mantis 0006166) and partial-data intervals, hand-computed and against the
#796 oracle — the baseline calculator's Uncertain-as-Good PercentGood fails it.
@@ -1991,10 +2006,15 @@ round-trip, redelivery, raw / AtTime / aggregate / event reads, every update res
run creates a throwaway `OtOpcUaHistorian_T_<guid>` database and drops it on dispose.
```bash
export HISTORIAN_SQL_TEST_CONNECTION='Server=localhost,14330;User Id=sa;Password=OtOpcUa!Dev123;TrustServerCertificate=True;Encrypt=False'
# The shared always-on dev SQL Server on the Docker host (otopcua-mssql; the committed dev sa credential, see
# docs/v2/dev-environment.md — not a secret):
export HISTORIAN_SQL_TEST_CONNECTION='Server=10.100.0.35,14330;User Id=sa;Password=OtOpcUaDev_2026!;TrustServerCertificate=True;Encrypt=False'
dotnet test tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer.IntegrationTests
```
The local docker-dev rig's `sql` service also publishes `localhost,14330`, but with **its own** committed password
(`OtOpcUa!Dev123`, `docker-dev/docker-compose.yml`) — the two are different instances; do not mix the host and the password (#822).
---
## QuestDB backend
@@ -2006,7 +2026,7 @@ the spike that pinned every QuestDB-specific form: [plans/2026-10-10-questdb-spi
`questdb/questdb:10.0.1`.
QuestDB serves every value kind (`Double`, `Int64`, `Boolean`, `String`, `DateTime`, `Empty`), stores the real status, records comms-loss
outage markers, serves the seventeen aggregates (time-weighted ones through the same shared evaluator, so results are identical) and supports **HistoryUpdate** (soft delete), exactly like `SqlServer`. The same conformance
outage markers, serves the nineteen aggregates (time-weighted ones through the same shared evaluator, so results are identical) and supports **HistoryUpdate** (soft delete), exactly like `SqlServer`. The same conformance
suite (`tests/Drivers/Historian.Conformance`) runs against both backends; the differences below are the only ones.
### Configuration (`ServerHistorian:QuestDb`)
@@ -2276,11 +2296,18 @@ tombstones and redelivery, the visibility wait, updater, reads, paging, and the
unique throwaway prefix (`t_<guid>_`) and drops its tables afterwards. The QuestDB server allows 64 connections, so the fixture prunes idle pool connections.
```bash
# The always-on shared fixture on the Docker host (#822; stack /opt/otopcua-questdb, infra/README.md §2a):
export HISTORIAN_QUESTDB_TEST_CONNECTION='Host=10.100.0.35;Port=8812;Username=admin;Password=quest;Database=qdb;ServerCompatibilityMode=NoTypeLoading'
dotnet test tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb.IntegrationTests
# …or a local one from the docker-dev overlay (same image, PG wire on localhost:18812):
docker compose -f docker-dev/docker-compose.yml -f docker-dev/docker-compose.questdb.yml up -d --no-deps questdb
export HISTORIAN_QUESTDB_TEST_CONNECTION='Host=localhost;Port=18812;Username=admin;Password=quest;Database=qdb;ServerCompatibilityMode=NoTypeLoading'
dotnet test tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb.IntegrationTests
```
`admin` / `quest` is QuestDB's stock default — a throwaway fixture credential, not a secret. The fixture's compose file is
`tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb.IntegrationTests/Docker/docker-compose.yml` (label `project=lmxopcua`).
---
## HistoryUpdate
+27 -1
View File
@@ -37,6 +37,7 @@ ssh dohertj2@10.100.0.35 'docker ps --filter label=project=lmxopcua --format "{{
| Stack dir | Purpose |
|---|---|
| `/opt/otopcua-mssql` | central SQL (always-on) |
| `/opt/otopcua-questdb` | QuestDB historian-backend test fixture (always-on, #822) — see §2a |
| `/opt/otopcua-modbus` · `/opt/otopcua-abcip` · `/opt/otopcua-s7` · `/opt/otopcua-opcuaclient` · `/opt/otopcua-opcuaclient-nat` · `/opt/otopcua-mqtt` · `/opt/otopcua-mtconnect` · `/opt/otopcua-focas` | driver fixtures |
| `~/otopcua-ablegacy` | driver fixture (user-owned) |
| `~/otopcua-harness` | Host.IntegrationTests real-mode deps (SQL + GLAuth) — see §4 |
@@ -55,6 +56,30 @@ ssh dohertj2@10.100.0.35 'docker ps --filter label=project=lmxopcua --format "{{
Nuclear reset: `ssh dohertj2@10.100.0.35 'cd /opt/otopcua-mssql && docker compose down -v && docker compose up -d'`.
## 2a. QuestDB historian fixture (always-on, #822)
| | |
|---|---|
| Container | `otopcua-questdb` (`questdb/questdb:10.0.1`, label `project=lmxopcua`), stack `/opt/otopcua-questdb` |
| Source of truth | `tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb.IntegrationTests/Docker/docker-compose.yml` |
| Endpoint | PG wire `10.100.0.35:8812` · web console `http://10.100.0.35:19000` (host `:9000` is taken) |
| Creds (dev) | QuestDB stock default `admin` / `quest` (throwaway fixture credential) |
| Env var | `HISTORIAN_QUESTDB_TEST_CONNECTION='Host=10.100.0.35;Port=8812;Username=admin;Password=quest;Database=qdb;ServerCompatibilityMode=NoTypeLoading'` |
| Used by | `Driver.Historian.QuestDb.IntegrationTests` (incl. the shared `Historian.Conformance` suite); absent env var ⇒ every test skips |
Deploy / update (the `/opt` dir was created with `sudo mkdir` + `chown dohertj2`):
```bash
rsync -av tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb.IntegrationTests/Docker/ \
dohertj2@10.100.0.35:/opt/otopcua-questdb/
ssh dohertj2@10.100.0.35 'cd /opt/otopcua-questdb && docker compose up -d'
```
Tests isolate by a unique table prefix per class and drop their tables on dispose, so the instance is safe to share.
Nuclear reset: `ssh dohertj2@10.100.0.35 'cd /opt/otopcua-questdb && docker compose down -v && docker compose up -d'`.
The SQL Server historian suite uses §2's instance: `HISTORIAN_SQL_TEST_CONNECTION='Server=10.100.0.35,14330;User Id=sa;Password=OtOpcUaDev_2026!;TrustServerCertificate=True;Encrypt=False'`
(each run creates and drops its own `OtOpcUaHistorian_T_<guid>` database).
## 3. Driver integration fixtures
Each driver's integration suite TCP-probes its fixture at collection init and **skips cleanly**
@@ -77,6 +102,7 @@ when it isn't running. Bring one up, `dotnet test`, tear down. Repo compose live
**Endpoint overrides** point a suite at a real PLC instead of the sim:
`MODBUS_SIM_ENDPOINT` · `AB_SERVER_ENDPOINT` · `S7_SIM_ENDPOINT` · `OPCUA_SIM_ENDPOINT` · `OPCUA_NAT_ENDPOINT` (opt-in, no default) ·
`MQTT_FIXTURE_ENDPOINT` / `MQTT_FIXTURE_PLAIN_ENDPOINT` · `MTCONNECT_AGENT_ENDPOINT` · `OTOPCUA_FOCAS_SIM_ENDPOINT`.
The historian backends are gated on full connection strings instead (`HISTORIAN_SQL_TEST_CONNECTION` · `HISTORIAN_QUESTDB_TEST_CONNECTION`, §2a).
> **`otopcua-focas-sim` on the host is NOT ours (#651).** It is ignitionoee's mock (created by that
> project's `provision-oee-focas.sh` for `ignition-site`; no published port, different defaults:
@@ -92,7 +118,7 @@ when it isn't running. Bring one up, `dotnet test`, tear down. Repo compose live
> CA + server cert/key, gitignored) and `MQTT_FIXTURE_USERNAME`/`MQTT_FIXTURE_PASSWORD` are exported
> at `docker compose up`. The live suite additionally wants `MQTT_FIXTURE_CA_CERT` pointing at a local
> copy of the CA: `scp dohertj2@10.100.0.35:/opt/otopcua-mqtt/secrets/ca.crt /tmp/mqtt-fixture-ca.crt`.
> It and the OpcUaClient NAT fixture are the only ones whose compose files carry the `project: lmxopcua` label (see CLAUDE.md).
> Not every fixture's compose file carries the `project: lmxopcua` label yet (MQTT, Modbus, FOCAS, MTConnect, Sql, the OpcUaClient NAT stack and the QuestDB historian fixture §2a do); enumerate by container name when in doubt.
> **The OpcUaClient NAT fixture exists for what it ADVERTISES (#806/#813).** `--ph` sets a private `.invalid`
> host (RFC 6761: never resolves, so no DNS search domain can make it reachable) and the published port 50010 differs
@@ -48,7 +48,7 @@ public class HistoryReadCommand : CommandBase
/// <summary>
/// Gets the optional aggregate name used when the operator wants processed history instead of raw samples.
/// </summary>
[CommandOption("aggregate", Description = "Aggregate function: Average, Minimum, Maximum, Count, Start, End, TimeAverage, TimeAverage2, Total, Total2, Interpolative, Range, Delta, MinimumActualTime, MaximumActualTime, PercentGood, PercentBad")]
[CommandOption("aggregate", Description = "Aggregate function: Average, Minimum, Maximum, Count, Start, End, TimeAverage, TimeAverage2, Total, Total2, Interpolative, Range, Delta, MinimumActualTime, MaximumActualTime, PercentGood, PercentBad, DurationGood, DurationBad")]
public string? Aggregate { get; init; }
/// <summary>
@@ -34,6 +34,8 @@ public static class AggregateTypeMapper
AggregateType.MaximumActualTime => ObjectIds.AggregateFunction_MaximumActualTime,
AggregateType.PercentGood => ObjectIds.AggregateFunction_PercentGood,
AggregateType.PercentBad => ObjectIds.AggregateFunction_PercentBad,
AggregateType.DurationGood => ObjectIds.AggregateFunction_DurationGood,
AggregateType.DurationBad => ObjectIds.AggregateFunction_DurationBad,
_ => throw new ArgumentOutOfRangeException(nameof(aggregate), aggregate, "Unknown AggregateType value.")
};
}
@@ -1,7 +1,7 @@
namespace ZB.MOM.WW.OtOpcUa.Client.Shared.Models;
/// <summary>
/// Aggregate functions for processed history reads — exactly the seventeen the server's SQL historian backends
/// Aggregate functions for processed history reads — exactly the nineteen the server's SQL historian backends
/// serve (<c>AggregateFamilies.Served</c>; a parity test pins it). The Gateway backend serves a subset and refuses
/// the rest with <c>BadAggregateNotSupported</c>.
/// </summary>
@@ -60,5 +60,11 @@ public enum AggregateType
PercentGood = 16,
/// <summary>Percentage of the interval with Bad data (Part 13 PercentBad).</summary>
PercentBad = 17
PercentBad = 17,
/// <summary>Time of the interval with Good data, in milliseconds (Part 13 DurationGood).</summary>
DurationGood = 18,
/// <summary>Time of the interval with Bad data, in milliseconds (Part 13 DurationBad).</summary>
DurationBad = 19
}
@@ -57,7 +57,7 @@ internal static class QuestDbAggregateQueries
private static readonly string EmptyKind = ((byte)HistorianValueKind.Empty).ToString(CultureInfo.InvariantCulture);
/// <summary>
/// Average / Minimum / Maximum / Count / Range (PercentGood / PercentBad and Total are time-weighted, not here).
/// Average / Minimum / Maximum / Count / Range (DurationGood / DurationBad / PercentGood / PercentBad and Total are time-weighted, not here).
/// Family columns: <c>6 avg_value, 7 min_value, 8 max_value, 9 sum_value</c> over the <b>Good</b> numeric samples
/// (DOUBLE, NULL when none — Part 13 / the SDK), then <c>10 unc_min, 11 unc_max</c> over the Uncertain ones (the
/// Part 13 Minimum/Maximum/Range status rule) and <c>12 good_non_numeric_count</c>.
@@ -434,7 +434,7 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource
/// <inheritdoc />
public bool SupportsReverseEvents => true;
// The seventeen AggregateFamilies.Served - shared with the other SQL backend.
// The nineteen AggregateFamilies.Served - shared with the other SQL backend.
/// <inheritdoc />
public IReadOnlySet<HistoryAggregateType> SupportedAggregates => AggregateFamilies.Served;
@@ -16,7 +16,8 @@ public enum AggregateFamily
ActualTime,
/// <summary>
/// TimeAverage / TimeAverage2 / Total / Total2 / Interpolative, and the duration-based PercentGood / PercentBad:
/// TimeAverage / TimeAverage2 / Total / Total2 / Interpolative, and the duration-based DurationGood / DurationBad /
/// PercentGood / PercentBad:
/// Part 13 time-weighted, with bounding values.
/// The backend streams the window's samples plus the bounding rows and <see cref="TimeWeightedAggregator"/>
/// evaluates them (identically for every backend).
@@ -38,12 +39,13 @@ public static class AggregateFamilies
HistoryAggregateType.MinimumActualTime or HistoryAggregateType.MaximumActualTime => AggregateFamily.ActualTime,
HistoryAggregateType.TimeAverage or HistoryAggregateType.TimeAverage2 or HistoryAggregateType.Total
or HistoryAggregateType.Total2 or HistoryAggregateType.Interpolative
or HistoryAggregateType.PercentGood or HistoryAggregateType.PercentBad => AggregateFamily.TimeWeighted,
or HistoryAggregateType.PercentGood or HistoryAggregateType.PercentBad
or HistoryAggregateType.DurationGood or HistoryAggregateType.DurationBad => AggregateFamily.TimeWeighted,
_ => null,
};
/// <summary>
/// The seventeen aggregates the SQL backends serve — every member <see cref="FamilyOf"/> assigns a family, derived
/// The nineteen aggregates the SQL backends serve — every member <see cref="FamilyOf"/> assigns a family, derived
/// from it so the set and the dispatch can never disagree. Both <c>SqlServerHistorianDataSource</c> and
/// <c>QuestDbHistorianDataSource</c> return it as <see cref="IHistorianDataSource.SupportedAggregates"/>.
/// </summary>
@@ -4,8 +4,8 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common;
/// <summary>
/// The OPC UA Part 13 time-weighted aggregates (<see cref="AggregateFamily.TimeWeighted"/>): <c>TimeAverage</c>,
/// <c>TimeAverage2</c>, <c>Total</c>, <c>Total2</c>, <c>Interpolative</c> and the duration-based <c>PercentGood</c> /
/// <c>PercentBad</c>, evaluated over a time-ordered stream of stored samples. Both SQL backends fetch the same rows and feed them here, so the two produce identical results
/// <c>TimeAverage2</c>, <c>Total</c>, <c>Total2</c>, <c>Interpolative</c> and the duration-based <c>DurationGood</c> /
/// <c>DurationBad</c> / <c>PercentGood</c> / <c>PercentBad</c>, evaluated over a time-ordered stream of stored samples. Both SQL backends fetch the same rows and feed them here, so the two produce identical results
/// by construction.
/// </summary>
/// <remarks>
@@ -50,6 +50,9 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common;
/// <c>GetSimpleBound</c>); time before the first sample is Bad; at the end of the data the last region runs to the
/// end of its own interval (Part 13; the SDK gives it 1 ms) and later intervals are <c>BadNoData</c>; an interval
/// ending before the first sample ⇒ <c>BadNoData</c>. Status Good; any value kind.</description></item>
/// <item><term>DurationGood / DurationBad</term><description>The same Good / Bad region time, in milliseconds (an
/// OPC UA <c>Duration</c>, as the SDK reports it) — PercentGood / PercentBad × interval ÷ 100, with the same
/// <c>BadNoData</c>, status and Partial rules (#820).</description></item>
/// <item><term>Interpolative</term><description>The interpolated-bounding signal's value at each interval start —
/// for a reverse read, at <c>End − i·interval</c> (Part 13 anchors a reverse read at its later time), emitted
/// latest first.</description></item>
@@ -78,7 +81,7 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common;
/// interval the data starts inside in the read direction (forward: the first sample fed lies strictly within it;
/// reverse: the latest sample fed lies strictly within it, nothing after — the SDK's <c>m_startOfData</c>,
/// keyed on the first value it is fed); and, forward, the end-of-data interval of TimeAverage2 / Total2 /
/// PercentGood / PercentBad (the SDK's <c>m_endOfData</c>, which it skips for an extrapolated TimeAverage / Total).
/// PercentGood / PercentBad / DurationGood / DurationBad (the SDK's <c>m_endOfData</c>, which it skips for an extrapolated TimeAverage / Total).
/// The timestamp is the interval's <see cref="BucketPlan.BucketTimestamp"/> (its start forward, its later edge
/// reverse); the value is a <see cref="double"/> (Boolean = 0/1, Int64 as double).
/// </para>
@@ -358,14 +361,22 @@ public sealed class TimeWeightedAggregator
? Calc(_rawArea[i] / _usableTicks[i], status, endOfData)
: Calc(_rawArea[i] / TicksPerSecond, status, endOfData);
}
case HistoryAggregateType.PercentGood or HistoryAggregateType.PercentBad:
case HistoryAggregateType.PercentGood or HistoryAggregateType.PercentBad
or HistoryAggregateType.DurationGood or HistoryAggregateType.DurationBad:
{
// No data in or around the interval (it ends before the first sample — the SDK's out-of-range test — or
// starts after the end of the data) ⇒ BadNoData; otherwise the share of the interval's duration.
// starts after the end of the data) ⇒ BadNoData; otherwise the region time (Duration*, in milliseconds —
// an OPC UA Duration) or its share of the interval's duration (Percent*). Uncertain regions count toward
// neither (Part 13 §5.4.3.31-34, Mantis 0006166).
if (_firstSampleUtc is not { } first || hi < first.Ticks || (_dataEndTicks is { } de2 && lo > de2))
return AggregateEvaluator.NoData(stamp, now);
var ticks = _aggregate == HistoryAggregateType.PercentGood ? _goodRegionTicks[i] : _badRegionTicks[i];
return Calc(100d * ticks / duration, HistorianStatusCodes.Good, endOfData);
var ticks = _aggregate is HistoryAggregateType.PercentGood or HistoryAggregateType.DurationGood
? _goodRegionTicks[i]
: _badRegionTicks[i];
var value = _aggregate is HistoryAggregateType.DurationGood or HistoryAggregateType.DurationBad
? (double)ticks / TimeSpan.TicksPerMillisecond
: 100d * ticks / duration;
return Calc(value, HistorianStatusCodes.Good, endOfData);
}
default:
throw new HistoryAggregateNotSupportedException(_aggregate);
@@ -47,7 +47,7 @@ internal static class SqlAggregateQueries
private const string NonNumericKinds = "4, 5";
/// <summary>
/// Average / Minimum / Maximum / Count / Range (PercentGood / PercentBad and Total are time-weighted, not here).
/// Average / Minimum / Maximum / Count / Range (DurationGood / DurationBad / PercentGood / PercentBad and Total are time-weighted, not here).
/// Family columns: <c>6 AvgValue, 7 MinValue, 8 MaxValue, 9 SumValue</c> over the <b>Good</b> numeric samples (float,
/// NULL when none — Part 13 and the SDK compute these over Good values only), then <c>10 UncMin, 11 UncMax</c>
/// over the Uncertain ones (the Part 13 Minimum/Maximum/Range status rule, <see cref="AggregateEvaluator"/>), and
@@ -359,7 +359,7 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
/// <inheritdoc />
public bool SupportsReverseEvents => true;
// <remarks>The seventeen <see cref="AggregateFamilies.Served"/> — shared with the other SQL backend.</remarks>
// <remarks>The nineteen <see cref="AggregateFamilies.Served"/> — shared with the other SQL backend.</remarks>
/// <inheritdoc />
public IReadOnlySet<HistoryAggregateType> SupportedAggregates => AggregateFamilies.Served;
@@ -463,7 +463,7 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
// bucket start (its later edge on a reverse read), except Minimum/MaximumActualTime (the sample's time; ties ⇒
// earliest) and Start/End. Every aggregate but Average sets the Partial bit on the bucket the tag's earliest
// sample lies strictly inside (forward) or its latest sample lies in (reverse: strictly inside).
// PercentGood / PercentBad are duration-based (time-weighted family, below). Count counts Good samples (value-based status; 0/Good on an empty bucket inside the tag's
// DurationGood / DurationBad / PercentGood / PercentBad are duration-based (time-weighted family, below). Count counts Good samples (value-based status; 0/Good on an empty bucket inside the tag's
// data range), Start/End are the raw first/last sample of any quality, Delta is Uncertain only when the bucket's
// first or last sample is not usable (see <see cref="AggregateEvaluator"/>).
// </para>
@@ -475,7 +475,7 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
// <see cref="TimeWeightedAggregator"/>, which fills every interval (an empty one between bounds gets a value).
// </para>
// </remarks>
// <exception cref="HistoryAggregateNotSupportedException">The aggregate is not one of the seventeen served.</exception>
// <exception cref="HistoryAggregateNotSupportedException">The aggregate is not one of the nineteen served.</exception>
// <exception cref="ArgumentOutOfRangeException">The window holds more than <see cref="MaxProcessedBuckets"/> intervals.</exception>
/// <inheritdoc />
public async Task<HistoryReadResult> ReadProcessedAsync(
@@ -132,10 +132,12 @@ public class ModelConstructionTests
((int)ConnectionState.Reconnecting).ShouldBe(3);
}
/// <summary>AggregateType holds the seventeen served aggregates (the parity test pins which).</summary>
/// <summary>AggregateType holds the nineteen served aggregates (the parity test pins which).</summary>
[Fact]
public void AggregateType_Enum_HasExpectedValues()
{
Enum.GetValues<AggregateType>().Length.ShouldBe(17);
Enum.GetValues<AggregateType>().Length.ShouldBe(19);
((int)AggregateType.DurationGood).ShouldBe(18);
((int)AggregateType.DurationBad).ShouldBe(19);
}
}
@@ -713,8 +713,6 @@ public abstract class ProcessedConformance : ConformanceBase
[InlineData(HistoryAggregateType.StartBound)]
[InlineData(HistoryAggregateType.EndBound)]
[InlineData(HistoryAggregateType.DeltaBounds)]
[InlineData(HistoryAggregateType.DurationGood)]
[InlineData(HistoryAggregateType.DurationBad)]
[InlineData(HistoryAggregateType.WorstQuality)]
[InlineData(HistoryAggregateType.NumberOfTransitions)]
[InlineData(HistoryAggregateType.AnnotationCount)]
@@ -11,7 +11,8 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Conformance;
/// Uncertain time as Good (#796) as well as against the Part 13 oracle. Every expectation here is derived by hand from
/// Part 13 (§5.4.3.31-34: DurationGood / DurationBad are the Good / Bad region time of the interval, PercentGood /
/// PercentBad that time ÷ the interval; Uncertain regions count toward neither — Mantis 0006166 — and the time before
/// the first sample is Bad, no data), and the oracle itself is checked against the same hand values.
/// the first sample is Bad, no data), and the oracle itself is checked against the same hand values. Since #820 the SQL
/// backends serve DurationGood / DurationBad (the same region times, in ms) and this suite asserts them directly.
/// </summary>
public abstract class QualityMixConformance : ConformanceBase
{
@@ -155,15 +156,52 @@ public abstract class QualityMixConformance : ConformanceBase
}
[Fact]
public async Task DurationGood_and_DurationBad_are_refused_rather_than_answered_from_the_baseline_rule()
public async Task DurationGood_and_DurationBad_over_a_Good_Uncertain_Bad_mix_are_the_Part_13_region_times()
{
// Part 13 DurationGood / DurationBad are not served by the SQL backends (#682 advertises 17 aggregates); they
// must stay refused up front — never answered by a calculator that counts Uncertain time as Good.
// #820: served on both SQL backends from the same region totals as PercentGood / PercentBad. Part 13
// §5.4.3.31/32: the Good / Bad region time of the interval as an OPC UA Duration (milliseconds); Uncertain regions
// count toward neither (Mantis 0006166) — never the 1.5.378.182 baseline rule that counts Uncertain time as Good.
await B.SeedAsync("Q", Q);
var src = B.CreateDataSource();
foreach (var agg in new[] { HistoryAggregateType.DurationGood, HistoryAggregateType.DurationBad })
(await Should.ThrowAsync<HistoryAggregateNotSupportedException>(
() => src.ReadProcessedAsync("Q", At(0), At(80), Ten, agg, Ct))).Aggregate.ShouldBe(agg);
src.SupportedAggregates.ShouldContain(HistoryAggregateType.DurationGood);
src.SupportedAggregates.ShouldContain(HistoryAggregateType.DurationBad);
var good = (await src.ReadProcessedAsync("Q", At(0), At(80), Ten, HistoryAggregateType.DurationGood, Ct)).Samples;
var bad = (await src.ReadProcessedAsync("Q", At(0), At(80), Ten, HistoryAggregateType.DurationBad, Ct)).Samples;
ShouldBeClose(Values(good), QDurationGood.Select(d => d * 1000).ToArray(), "DurationGood");
ShouldBeClose(Values(bad), QDurationBad.Select(d => d * 1000).ToArray(), "DurationBad");
good.ShouldAllBe(x => x.Value is double);
good.Select(x => x.StatusCode).ShouldAllBe(c => c == GoodCalc);
bad.Select(x => x.StatusCode).ShouldAllBe(c => c == GoodCalc);
good.Select(x => x.SourceTimestampUtc).ShouldBe(Enumerable.Range(0, 8).Select(i => (DateTime?)B.Round(At(i * 10))));
// Uncertain time is neither Good nor Bad: only I4 (no Uncertain region) adds up to the whole 10 s.
Values(good).Zip(Values(bad), (g, b) => Math.Round(g + b, 6)).ShouldBe([7000d, 3000d, 0d, 0d, 10000d, 4000d, 3000d, 5000d]);
// DurationGood / DurationBad = PercentGood / PercentBad × interval ÷ 100, interval by interval.
var pGood = (await src.ReadProcessedAsync("Q", At(0), At(80), Ten, HistoryAggregateType.PercentGood, Ct)).Samples;
ShouldBeClose(Values(good), Values(pGood).Select(p => p * Ten.TotalMilliseconds / 100).ToArray(), "DurationGood vs PercentGood");
// The baseline (the live AggregateFilter calculator, #809) answers Q's DurationGood with the Uncertain time added.
var baseline = SdkAggregateOracle.RunSdkAsIs(HistoryAggregateType.DurationGood, At(0), At(80), Ten,
Q.Select(r => (r.T, r.V, r.Status)));
ShouldBeClose(baseline.Select(r => Convert.ToDouble(r.Value)).ToArray(),
QBaselinePercentGood.Select(p => p * 100).ToArray(), "baseline DurationGood (#809)");
}
[Fact]
public async Task DurationGood_and_DurationBad_over_partial_data_count_no_data_as_Bad_and_Uncertain_as_neither()
{
// #820 over QP: [0,20) 5 s no data (Bad), Uncertain 7 s, Good 8 s; [20,40) all Uncertain; outside the data BadNoData.
await B.SeedAsync("QP", QP);
var src = B.CreateDataSource();
var good = (await src.ReadProcessedAsync("QP", At(-20), At(60), Twenty, HistoryAggregateType.DurationGood, Ct)).Samples;
var bad = (await src.ReadProcessedAsync("QP", At(-20), At(60), Twenty, HistoryAggregateType.DurationBad, Ct)).Samples;
good.Select(x => x.Value).ShouldBe([null, 8000d, 0d, null]);
bad.Select(x => x.Value).ShouldBe([null, 5000d, 0d, null]);
var statuses = new[] { NoDataCalc, GoodCalc | Partial, GoodCalc | Partial, NoDataCalc };
good.Select(x => x.StatusCode).ShouldBe(statuses);
bad.Select(x => x.StatusCode).ShouldBe(statuses);
}
[Theory]
@@ -182,7 +220,8 @@ public abstract class QualityMixConformance : ConformanceBase
await B.SeedAsync("Q", Q);
await AssertMatchesOracle("Q", Rounded(Q), from, to, intervalS,
[
HistoryAggregateType.PercentGood, HistoryAggregateType.PercentBad, HistoryAggregateType.Count,
HistoryAggregateType.PercentGood, HistoryAggregateType.PercentBad, HistoryAggregateType.DurationGood,
HistoryAggregateType.DurationBad, HistoryAggregateType.Count,
HistoryAggregateType.Average, HistoryAggregateType.TimeAverage, HistoryAggregateType.TimeAverage2,
HistoryAggregateType.Total, HistoryAggregateType.Total2, HistoryAggregateType.Start, HistoryAggregateType.End,
HistoryAggregateType.Delta, HistoryAggregateType.Interpolative,
@@ -192,15 +231,18 @@ public abstract class QualityMixConformance : ConformanceBase
[Theory]
[InlineData(-20, 60, 20)]
[InlineData(-20, 60, 10)]
[InlineData(60, -20, 20)]
[InlineData(60, -20, 10)]
public async Task Processed_reads_over_partial_data_match_the_Part_13_oracle(double from, double to, double intervalS)
{
// QP's start-of-data and end-of-data intervals against the oracle. Left out, each a documented or pinned divergence:
// QP's start-of-data and end-of-data intervals against the oracle, forward and reverse (#821). Left out, each a documented or pinned divergence:
// TimeAverage2 / Total2 (the SDK's 1 ms end-of-data region), the min/max family (the SDK sets Partial on a Bad
// result), and TimeAverage / Total (pinned by Time_weighted_values_over_partial_data_cover_only_the_data below).
await B.SeedAsync("QP", QP);
await AssertMatchesOracle("QP", Rounded(QP), from, to, intervalS,
[
HistoryAggregateType.PercentGood, HistoryAggregateType.PercentBad, HistoryAggregateType.Count,
HistoryAggregateType.PercentGood, HistoryAggregateType.PercentBad, HistoryAggregateType.DurationGood,
HistoryAggregateType.DurationBad, HistoryAggregateType.Count,
HistoryAggregateType.Average, HistoryAggregateType.Start, HistoryAggregateType.End, HistoryAggregateType.Delta,
HistoryAggregateType.Interpolative,
]);
@@ -229,6 +271,28 @@ public abstract class QualityMixConformance : ConformanceBase
Convert.ToDouble(sdk.Value).ShouldBe(total + 25, 1e-6, "baseline divergence — if this moves, compare QP's TimeAverage/Total to the oracle");
}
[Fact]
public async Task Time_weighted_values_over_partial_data_cover_only_the_data_in_a_reverse_read()
{
// #821: the reverse-order variant of the late-start case above — (0,20] read 20 → 0. The same Part 13 integral and
// average over the covered [5,20); Uncertain; NOT Partial (a reverse read keys start-of-data on the latest sample,
// which is past the window — the SDK's rule, SdkAggregateParityTests); stamped with the later edge (#677). The
// baseline's reverse answer carries the same +25 ramp divergence.
await B.SeedAsync("QP", QP);
var src = B.CreateDataSource();
const double total = 73.5 + 88 + 32d / 13;
var tot = (await src.ReadProcessedAsync("QP", At(20), At(0), Twenty, HistoryAggregateType.Total, Ct)).Samples.Single();
((double)tot.Value!).ShouldBe(total, 1e-6);
tot.StatusCode.ShouldBe(UncertainCalc);
tot.SourceTimestampUtc.ShouldBe(B.Round(At(20)));
var avg = (await src.ReadProcessedAsync("QP", At(20), At(0), Twenty, HistoryAggregateType.TimeAverage, Ct)).Samples.Single();
((double)avg.Value!).ShouldBe(total / 15, 1e-6);
avg.StatusCode.ShouldBe(UncertainCalc);
var sdk = SdkAggregateOracle.Run(HistoryAggregateType.Total, B.Round(At(20)), B.Round(At(0)), Twenty, Rounded(QP)).Single();
Convert.ToDouble(sdk.Value).ShouldBe(total + 25, 1e-6, "baseline divergence (reverse) — #821");
}
private async Task AssertMatchesOracle(string tag, (DateTime, HistorianValue, uint)[] rows, double from, double to,
double intervalS, HistoryAggregateType[] aggregates)
{
@@ -18,16 +18,16 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Conformance;
/// <remarks>
/// <para>Shared (source-linked) by <c>Sql.Common.Tests</c> and both SQL backends' integration suites.</para>
/// <para>
/// <b>PercentGood is the one aggregate not taken as-is (#796).</b> The 1.5.378.182 baseline changed
/// <b>PercentGood (and, since #820, DurationGood) are the aggregates not taken as-is (#796).</b> The 1.5.378.182 baseline changed
/// <c>StatusAggregateCalculator.ComputeDurationGoodBad</c> (1.5.378.106 and earlier counted Good regions only):
/// with TreatUncertainAsBad = false it counts Uncertain regions as <b>Good</b> time (as Bad when true). Part 13
/// keeps Uncertain out of both (§5.4.3.31 DurationGood counts Good regions; §5.4.3.33 PercentGood =
/// DurationGood ÷ interval; Mantis 0006166, closed "no change required": an all-Uncertain interval is
/// PercentGood 0 <i>and</i> PercentBad 0), and so does ours. <see cref="Run"/> therefore answers PercentGood
/// through <see cref="Part13PercentGoodCalculator"/>: the SAME baseline calculator, region machinery (simple
/// (and DurationGood) through <see cref="Part13GoodCalculator"/>: the SAME baseline calculator, region machinery (simple
/// bounds, end-of-data 1 ms region, status and Partial bits) and server-default configuration, counting only
/// Good regions — exactly what the pre-.182 oracle computed. PercentBad still counts Bad regions only (unchanged
/// by .182) and is compared as-is. <see cref="RunSdkAsIs"/> runs the baseline untouched, for the test that pins
/// Good regions — exactly what the pre-.182 oracle computed. PercentBad / DurationBad still count Bad regions only
/// (unchanged by .182) and are compared as-is. <see cref="RunSdkAsIs"/> runs the baseline untouched, for the test that pins
/// the divergence.
/// </para>
/// <para>
@@ -55,7 +55,7 @@ public static class SdkAggregateOracle
/// <summary>
/// Runs the Part 13 oracle for <paramref name="aggregate"/> over <paramref name="samples"/>: the baseline calculator,
/// except PercentGood, which counts Good regions only (see the remarks, #796).
/// except PercentGood / DurationGood, which count Good regions only (see the remarks, #796/#820).
/// </summary>
/// <param name="aggregate">The aggregate.</param>
/// <param name="start">Request start (later than <paramref name="end"/> for a reverse read).</param>
@@ -65,9 +65,10 @@ public static class SdkAggregateOracle
/// <returns>The oracle's processed values, in the order it returns them.</returns>
public static IReadOnlyList<Result> Run(HistoryAggregateType aggregate, DateTime start, DateTime end, TimeSpan interval,
IEnumerable<(DateTime Time, HistorianValue Value, uint Status)> samples) =>
aggregate == HistoryAggregateType.PercentGood
? RunCore(new LegacyCalculatorAdapter(new Part13PercentGoodCalculator(ObjectIds.AggregateFunction_PercentGood,
start, end, interval.TotalMilliseconds, false, DefaultConfiguration, Telemetry)), start, end, samples)
aggregate is HistoryAggregateType.PercentGood or HistoryAggregateType.DurationGood
? RunCore(new LegacyCalculatorAdapter(new Part13GoodCalculator(AggregateId(aggregate),
start, end, interval.TotalMilliseconds, false, DefaultConfiguration, Telemetry,
usePercent: aggregate == HistoryAggregateType.PercentGood)), start, end, samples)
: RunSdkAsIs(aggregate, start, end, interval, samples);
/// <summary>
@@ -155,12 +156,14 @@ public static class SdkAggregateOracle
}
/// <summary>
/// The baseline <see cref="BaselineStatusAggregateCalculator"/> with PercentGood counting <b>Good</b> regions only — Part 13
/// §5.4.3.31/§5.4.3.33, and the 1.5.378.106 rule. Everything else (bounds, regions, Partial, status) is the
/// baseline's own code, run under the same configuration as PercentBad.
/// The baseline <see cref="BaselineStatusAggregateCalculator"/> with PercentGood / DurationGood counting <b>Good</b>
/// regions only — Part 13 §5.4.3.31/§5.4.3.33, and the 1.5.378.106 rule. Everything else (bounds, regions, Partial,
/// status, the millisecond Duration unit) is the baseline's own code, run under the same configuration as
/// PercentBad / DurationBad.
/// </summary>
private sealed class Part13PercentGoodCalculator(NodeId aggregateId, DateTime startTime, DateTime endTime,
double processingInterval, bool stepped, AggregateConfiguration configuration, ITelemetryContext telemetry)
private sealed class Part13GoodCalculator(NodeId aggregateId, DateTime startTime, DateTime endTime,
double processingInterval, bool stepped, AggregateConfiguration configuration, ITelemetryContext telemetry,
bool usePercent)
: BaselineStatusAggregateCalculator(aggregateId, startTime, endTime, processingInterval, stepped, configuration, telemetry)
{
protected override LegacyDataValue ComputeValue(TimeSlice slice)
@@ -178,7 +181,7 @@ public static class SdkAggregateOracle
var value = new LegacyDataValue
{
WrappedValue = ZB.MOM.WW.OtOpcUa.OpcUaServer.SdkValueBoundary.FromClr(good / total * 100),
WrappedValue = ZB.MOM.WW.OtOpcUa.OpcUaServer.SdkValueBoundary.FromClr(usePercent ? good / total * 100 : good),
SourceTimestamp = GetTimestamp(slice),
ServerTimestamp = GetTimestamp(slice),
};
@@ -223,6 +226,8 @@ public static class SdkAggregateOracle
HistoryAggregateType.Count => ObjectIds.AggregateFunction_Count,
HistoryAggregateType.PercentGood => ObjectIds.AggregateFunction_PercentGood,
HistoryAggregateType.PercentBad => ObjectIds.AggregateFunction_PercentBad,
HistoryAggregateType.DurationGood => ObjectIds.AggregateFunction_DurationGood,
HistoryAggregateType.DurationBad => ObjectIds.AggregateFunction_DurationBad,
HistoryAggregateType.Start => ObjectIds.AggregateFunction_Start,
HistoryAggregateType.End => ObjectIds.AggregateFunction_End,
HistoryAggregateType.Delta => ObjectIds.AggregateFunction_Delta,
@@ -0,0 +1,41 @@
# QuestDB historian integration-test fixture (#822) — the QuestDB backend's shared, always-available instance.
#
# Deployed on the shared Docker host (see infra/README.md §1) as /opt/otopcua-questdb:
#
# rsync -av tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb.IntegrationTests/Docker/ \
# dohertj2@10.100.0.35:/opt/otopcua-questdb/
# ssh dohertj2@10.100.0.35 'cd /opt/otopcua-questdb && docker compose up -d'
#
# Endpoint (HISTORIAN_QUESTDB_TEST_CONNECTION):
# Host=10.100.0.35;Port=8812;Username=admin;Password=quest;Database=qdb;ServerCompatibilityMode=NoTypeLoading
# Web console: http://10.100.0.35:19000 (host :9000 is taken on the Docker host).
#
# admin/quest is QuestDB's stock default — a throwaway dev fixture credential, not a secret. The tests isolate by a
# unique table prefix per test class and drop their tables on dispose, so the instance can be shared by concurrent runs.
#
# Image pinned as in docker-dev/docker-compose.questdb.yml (questdb/questdb:10.0.1, the #666 T0 spike's version; ≥ 8.2.3
# for table TTL). cairo.ttl.use.wall.clock stays at its default (true) — the historian relies on it (#674.1).
services:
questdb:
image: questdb/questdb:10.0.1
container_name: otopcua-questdb
restart: unless-stopped
# Every lmxopcua fixture container carries this so the shared Docker host stays
# discoverable with `docker ps --filter label=project=lmxopcua`.
labels:
project: lmxopcua
ports:
- "8812:8812" # PG wire (the historian backend + tests)
- "19000:9000" # web console / REST
volumes:
- otopcua-questdb-fixture-data:/var/lib/questdb
healthcheck:
test: ["CMD-SHELL", "curl -fs http://localhost:9003/status || exit 1"]
interval: 10s
timeout: 5s
retries: 12
volumes:
otopcua-questdb-fixture-data:
name: otopcua-questdb-fixture-data
labels:
project: lmxopcua
@@ -10,21 +10,23 @@ public sealed class ProcessedReadTests
[Fact]
public void Every_enum_member_is_either_served_or_refused()
{
// Seventeen served (#671 added TimeAverage, TimeAverage2, Interpolative and Total2, and moved Total to the
// time-weighted family; #680 moved PercentGood/PercentBad there too — they are duration-based); anything appended to
// the enum later falls to the refusal (total default).
Enum.GetValues<HistoryAggregateType>().Count(a => AggregateFamilies.FamilyOf(a) is not null).ShouldBe(17);
// Nineteen served (#671 added TimeAverage, TimeAverage2, Interpolative and Total2, and moved Total to the
// time-weighted family; #680 moved PercentGood/PercentBad there too — they are duration-based; #820 added
// DurationGood/DurationBad, the same region time); anything appended to the enum later falls to the refusal
// (total default).
Enum.GetValues<HistoryAggregateType>().Count(a => AggregateFamilies.FamilyOf(a) is not null).ShouldBe(19);
Enum.GetValues<HistoryAggregateType>().Where(a => AggregateFamilies.FamilyOf(a) == AggregateFamily.TimeWeighted)
.ShouldBe([HistoryAggregateType.Total, HistoryAggregateType.Interpolative, HistoryAggregateType.TimeAverage,
HistoryAggregateType.PercentGood, HistoryAggregateType.PercentBad, HistoryAggregateType.TimeAverage2,
HistoryAggregateType.Total2]);
HistoryAggregateType.DurationGood, HistoryAggregateType.DurationBad, HistoryAggregateType.PercentGood,
HistoryAggregateType.PercentBad, HistoryAggregateType.TimeAverage2, HistoryAggregateType.Total2],
ignoreOrder: true);
}
[Fact]
public void Served_is_exactly_the_members_with_a_family()
{
// #682: the advertised set and the dispatch are one definition — a member is served iff it has a family.
AggregateFamilies.Served.Count.ShouldBe(17);
AggregateFamilies.Served.Count.ShouldBe(19);
foreach (var a in Enum.GetValues<HistoryAggregateType>())
AggregateFamilies.Served.Contains(a).ShouldBe(AggregateFamilies.FamilyOf(a) is not null, a.ToString());
}
@@ -213,7 +213,8 @@ public sealed class SdkAggregateParityTests
/// <summary>
/// Applies the documented divergences listed on the class (other than truncation): strips the Calculated bit from our
/// <c>BadNoData</c>; maps our generic-Bad TimeAverage2/Total2 outage interval to the SDK's <c>BadNoData</c>; takes the
/// SDK's value for the end-of-data interval of TimeAverage2/Total2/PercentGood/PercentBad (its 1 ms artifact).
/// SDK's value for the end-of-data interval of TimeAverage2/Total2/PercentGood/PercentBad/DurationGood/DurationBad
/// (its 1 ms artifact).
/// </summary>
private static IReadOnlyList<DataValueSnapshot> MaskDocumented(HistoryAggregateType agg, IReadOnlyList<DataValueSnapshot> ours,
IReadOnlyList<SdkAggregateOracle.Result> sdk, double from, double to, double intervalS, StoredSample[] data)
@@ -225,7 +226,8 @@ public sealed class SdkAggregateParityTests
var dataEnds = endsInWindow && data.Single(x => x.TimeUtc.Ticks == last).IsUsable;
var outageEnds = endsInWindow && !dataEnds;
var boolean = data.Any(x => x.Value.Kind == HistorianValueKind.Boolean);
var percent = agg is HistoryAggregateType.PercentGood or HistoryAggregateType.PercentBad;
var percent = agg is HistoryAggregateType.PercentGood or HistoryAggregateType.PercentBad
or HistoryAggregateType.DurationGood or HistoryAggregateType.DurationBad;
for (long i = 0; i < plan.BucketCount; i++)
{
var k = (int)(plan.Reversed ? plan.BucketCount - 1 - i : i);
@@ -244,7 +246,7 @@ public sealed class SdkAggregateParityTests
&& code == HistorianStatusCodes.Good
&& (sdk[k].StatusCode & StatusBits.CodeBitsMask) == HistorianStatusCodes.UncertainDataSubNormal)
list[k] = list[k] with { StatusCode = (list[k].StatusCode & ~StatusBits.CodeBitsMask) | HistorianStatusCodes.UncertainDataSubNormal };
if (dataEnds && (twoVariant || agg is HistoryAggregateType.PercentGood or HistoryAggregateType.PercentBad)
if (dataEnds && (twoVariant || percent)
&& last >= plan.LowerTicks(i) && last < plan.UpperTicks(i) && sdk[k].Value is not null)
list[k] = list[k] with { Value = sdk[k].Value };
// An outage marker as the last sample: ours runs the outage to the window end (no end-of-data Partial on the
@@ -290,7 +292,7 @@ public sealed class SdkAggregateParityTests
{
HistoryAggregateType.TimeAverage, HistoryAggregateType.TimeAverage2, HistoryAggregateType.Total,
HistoryAggregateType.Total2, HistoryAggregateType.Interpolative, HistoryAggregateType.PercentGood,
HistoryAggregateType.PercentBad,
HistoryAggregateType.PercentBad, HistoryAggregateType.DurationGood, HistoryAggregateType.DurationBad,
})
foreach (var (name, data) in new[]
{
@@ -473,6 +475,98 @@ public sealed class SdkAggregateParityTests
(100 - Pct(AsIs(HistoryAggregateType.PercentBad, 0, 20, data, cfg))).ShouldBe(50d, 1e-9);
}
/// <summary>
/// #821: data that starts inside [0, 20) at +5 with an <b>Uncertain</b> first sample — the #810 QualityMix tag QP:
/// Uncertain 10 at +5, Good 11 at +12, Uncertain 12 at +25, nothing after.
/// </summary>
private static readonly StoredSample[] UncertainFirstLateStart =
[
S(5, HistorianValue.FromDouble(10), Uncertain),
S(12, HistorianValue.FromDouble(11), Good),
S(25, HistorianValue.FromDouble(12), Uncertain),
];
/// <summary>The <see cref="UncertainFirstLateStart"/> control: the same data with a <b>Good</b> first sample.</summary>
private static readonly StoredSample[] GoodFirstLateStart =
[S(5, HistorianValue.FromDouble(10), Good), .. UncertainFirstLateStart[1..]];
/// <summary>
/// Runs SDK 2.0.1's OWN stock calculator (<see cref="Opc.Ua.Server.Aggregators"/>, not the frozen 1.5.378.182 copy
/// the server registers) under the server-default configuration — for the #809/#821 facts only.
/// </summary>
private static List<(double? Value, uint Status)> Stock201(HistoryAggregateType agg, double from, double to,
StoredSample[] data)
{
var id = agg switch
{
HistoryAggregateType.Total => Opc.Ua.ObjectIds.AggregateFunction_Total,
HistoryAggregateType.TimeAverage => Opc.Ua.ObjectIds.AggregateFunction_TimeAverage,
HistoryAggregateType.DurationGood => Opc.Ua.ObjectIds.AggregateFunction_DurationGood,
HistoryAggregateType.PercentGood => Opc.Ua.ObjectIds.AggregateFunction_PercentGood,
_ => throw new ArgumentOutOfRangeException(nameof(agg)),
};
var calc = Opc.Ua.Server.Aggregators.CreateStandardCalculator(id, At(from), At(to),
(At(to) - At(from)).Duration().TotalMilliseconds, false, SdkAggregateOracle.DefaultConfiguration,
Opc.Ua.DefaultTelemetry.Create(_ => { }))!;
var ordered = data.OrderBy(s => s.TimeUtc).ToList();
if (to < from) ordered.Reverse();
foreach (var s in ordered)
calc.QueueRawValue(new Opc.Ua.DataValue(Opc.Ua.Variant.From(s.Value.Number), s.Status, s.TimeUtc, s.TimeUtc));
var results = new List<(double?, uint)>();
while (calc.TryGetProcessedValue(true, out var dv))
results.Add((dv.WrappedValue.TryGetValue(out double d) ? d : null, dv.StatusCode.Code));
return results;
}
[Theory]
[InlineData(0, 20)]
[InlineData(20, 0)]
public void An_Uncertain_first_sample_in_a_late_start_interval_makes_the_SDK_integrate_the_time_before_the_data(
double from, double to)
{
// #821, pinned divergence (pending the #809/#821 owner decision — this test records, it does not choose).
// Part 13 §5.4.3.3/§5.4.3.8: the interpolated-bounding signal is defined from the first sample only — 10 → 11 over
// [5,12) (73.5) and 11 → 11 + 8/13 over [12,20) (4·(22 + 8/13)) ⇒ Total 163.96…, TimeAverage = Total ÷ 15 s.
const double part13Total = 73.5 + 88 + 32d / 13;
Pct(Ours(HistoryAggregateType.Total, from, to, 0, UncertainFirstLateStart)).ShouldBe(part13Total, 1e-9);
Pct(Ours(HistoryAggregateType.TimeAverage, from, to, 0, UncertainFirstLateStart)).ShouldBe(part13Total / 15, 1e-9);
// The frozen 1.5.378.182 calculators — exactly what the server registers for live AggregateFilter items
// (CompatibilityAggregateFactory; Sdk2AggregateCompatibilityTests proves the runtime manager serves the same code)
// — integrate the uncovered [0,5) as a ramp 0 → 10 (+25) and divide TimeAverage by the WHOLE 20 s interval.
Pct(AsIs(HistoryAggregateType.Total, from, to, UncertainFirstLateStart)).ShouldBe(part13Total + 25, 1e-9);
Pct(AsIs(HistoryAggregateType.TimeAverage, from, to, UncertainFirstLateStart)).ShouldBe((part13Total + 25) / 20, 1e-9);
// SDK 2.0.1's own stock calculators answer the same: the upgrade did not change this.
Stock201(HistoryAggregateType.Total, from, to, UncertainFirstLateStart).Single().Value!.Value
.ShouldBe(part13Total + 25, 1e-9);
Stock201(HistoryAggregateType.TimeAverage, from, to, UncertainFirstLateStart).Single().Value!.Value
.ShouldBe((part13Total + 25) / 20, 1e-9);
// Status agrees everywhere (Uncertain — an Uncertain value is used and the interval is not fully covered).
(AsIs(HistoryAggregateType.Total, from, to, UncertainFirstLateStart).Single().StatusCode & StatusBits.CodeBitsMask)
.ShouldBe(HistorianStatusCodes.UncertainDataSubNormal);
(Ours(HistoryAggregateType.Total, from, to, 0, UncertainFirstLateStart).Single().StatusCode & StatusBits.CodeBitsMask)
.ShouldBe(HistorianStatusCodes.UncertainDataSubNormal);
// Control: the SAME data with a Good first sample — all three agree on the Part 13 answer, so the trigger is the
// Uncertain first sample, not the late start.
foreach (var agg in new[] { HistoryAggregateType.Total, HistoryAggregateType.TimeAverage })
{
var expected = agg == HistoryAggregateType.Total ? part13Total : part13Total / 15;
Pct(Ours(agg, from, to, 0, GoodFirstLateStart)).ShouldBe(expected, 1e-9);
Pct(AsIs(agg, from, to, GoodFirstLateStart)).ShouldBe(expected, 1e-9);
Stock201(agg, from, to, GoodFirstLateStart).Single().Value!.Value.ShouldBe(expected, 1e-9);
}
// #809 on the same input: DurationGood / PercentGood — Part 13 (ours) and stock SDK 2.0.1 count Good regions only
// ([12,20) = 8 s); the frozen live calculators also count the Uncertain [5,12) as Good (15 s).
Pct(Ours(HistoryAggregateType.DurationGood, from, to, 0, UncertainFirstLateStart)).ShouldBe(8_000d, 1e-9);
Stock201(HistoryAggregateType.DurationGood, from, to, UncertainFirstLateStart).Single().Value!.Value.ShouldBe(8_000d, 1e-9);
Pct(AsIs(HistoryAggregateType.DurationGood, from, to, UncertainFirstLateStart)).ShouldBe(15_000d, 1e-9);
Pct(Ours(HistoryAggregateType.PercentGood, from, to, 0, UncertainFirstLateStart)).ShouldBe(40d, 1e-9);
Stock201(HistoryAggregateType.PercentGood, from, to, UncertainFirstLateStart).Single().Value!.Value.ShouldBe(40d, 1e-9);
Pct(AsIs(HistoryAggregateType.PercentGood, from, to, UncertainFirstLateStart)).ShouldBe(75d, 1e-9);
}
/// <summary>#684: Good 10 / 30 with an Uncertain value BELOW the Good minimum (+4) and one ABOVE the maximum (+6).</summary>
private static readonly StoredSample[] UncertainBeyond =
[
@@ -274,6 +274,33 @@ public sealed class TimeWeightedAggregatorTests
bad.Select(x => x.StatusCode).ShouldBe([GoodCalc, GoodCalc, GoodCalcPartial, NoDataCalc]);
}
[Fact]
public void Duration_aggregates_are_the_percent_regions_in_milliseconds()
{
// #820: DurationGood / DurationBad are the same Good / Bad region time as PercentGood / PercentBad, as an OPC UA
// Duration (ms, the SDK's unit), with the same statuses and Partial bits — tag A over [0,80) at 20 s.
var good = Run(HistoryAggregateType.DurationGood, 0, 80, 20, A);
Values(good, 20_000, 10_000, 20_000, null);
good.Select(x => x.StatusCode).ShouldBe([GoodCalc, GoodCalc, GoodCalcPartial, NoDataCalc]);
good.Where(x => x.Value is not null).ShouldAllBe(x => x.Value is double);
var bad = Run(HistoryAggregateType.DurationBad, 0, 80, 20, A);
Values(bad, 0, 10_000, 0, null);
bad.Select(x => x.StatusCode).ShouldBe([GoodCalc, GoodCalc, GoodCalcPartial, NoDataCalc]);
// Uncertain time counts toward neither (Mantis 0006166): −3 G, +4 outage, +6 U, +12 G, +30 G over [0,20) at 10 s ⇒
// B0 [0,4) Uncertain (bound into the outage), [4,6) Bad, [6,10) Uncertain; B1 [10,12) Uncertain, [12,20) Good.
StoredSample[] s =
[
S(-3, HistorianValue.FromDouble(10), Good), S(4, HistorianValue.Empty, BadNoCommunication),
S(6, HistorianValue.FromDouble(30), Uncertain), S(12, HistorianValue.FromDouble(3), Good),
S(30, HistorianValue.FromDouble(3), Good),
];
Values(Run(HistoryAggregateType.DurationGood, 0, 20, 10, s), 0, 8_000);
Values(Run(HistoryAggregateType.DurationBad, 0, 20, 10, s), 2_000, 0);
// A truncated interval measures its own (shorter) span: [10,15) holds [10,12) Uncertain + [12,15) Good.
Values(Run(HistoryAggregateType.DurationGood, 0, 15, 10, s), 0, 3_000);
}
[Fact]
public void With_no_end_bound_the_last_region_runs_to_the_interval_end()
{
@@ -130,7 +130,7 @@ public sealed class HistorianBackendRegistryTests
o.TablePrefix.ShouldBe("rig_");
}
/// <summary>The seventeen aggregates the SQL backends serve, spelled out (not derived) so a drift in either the
/// <summary>The nineteen aggregates the SQL backends serve, spelled out (not derived) so a drift in either the
/// family table or a backend's declaration fails here (#682).</summary>
private static readonly HistoryAggregateType[] SqlServed =
[
@@ -139,6 +139,7 @@ public sealed class HistorianBackendRegistryTests
HistoryAggregateType.MinimumActualTime, HistoryAggregateType.MaximumActualTime, HistoryAggregateType.TimeAverage,
HistoryAggregateType.TimeAverage2, HistoryAggregateType.Total, HistoryAggregateType.Total2,
HistoryAggregateType.Interpolative, HistoryAggregateType.PercentGood, HistoryAggregateType.PercentBad,
HistoryAggregateType.DurationGood, HistoryAggregateType.DurationBad,
];
[Fact]
@@ -27,9 +27,11 @@ public sealed class AdvertisedAggregateFunctionsWireTests
"Average", "Count", "Delta", "End", "Maximum", "MaximumActualTime", "Minimum", "MinimumActualTime", "Range", "Start",
];
// #820 added DurationBad / DurationGood (the PercentBad / PercentGood region times, in ms).
private static readonly string[] SqlNames =
[
"Average", "Count", "Delta", "End", "Interpolative", "Maximum", "MaximumActualTime", "Minimum", "MinimumActualTime",
"Average", "Count", "Delta", "DurationBad", "DurationGood", "End", "Interpolative", "Maximum", "MaximumActualTime",
"Minimum", "MinimumActualTime",
"PercentBad", "PercentGood", "Range", "Start", "TimeAverage", "TimeAverage2", "Total", "Total2",
];
@@ -0,0 +1,55 @@
using Opc.Ua;
using Shouldly;
using Xunit;
namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests;
/// <summary>
/// #821 / #809 facts, through the BOOTED server's runtime <c>AggregateManager</c> — the calculators a live
/// AggregateFilter monitored item gets, under the server's own default configuration. This pins current behaviour for
/// the owner decision; it does not choose. The HistoryRead (Part 13) answers and SDK 2.0.1's stock answers for the
/// same input are pinned beside it in <c>SdkAggregateParityTests</c>.
/// </summary>
public sealed class LiveAggregateLateStartFactsTests
{
private static readonly DateTime T0 = new(2026, 11, 11, 8, 0, 0, DateTimeKind.Utc);
/// <summary>The #810 QualityMix tag QP: Uncertain 10 at +5, Good 11 at +12, Uncertain 12 at +25, nothing after.</summary>
private static readonly (double Seconds, double Value, uint Status)[] Qp =
[
(5, 10, 0x40000000u), (12, 11, 0x00000000u), (25, 12, 0x40000000u), // Uncertain, Good, Uncertain
];
[Fact]
public async Task Live_AggregateFilter_calculators_on_an_Uncertain_first_late_start_interval()
{
await using var rig = await ServerEventRig.StartAsync("AggregateLateStart", encrypted: false,
TestContext.Current.CancellationToken);
var manager = rig.Server.CurrentInstance.AggregateManager;
var config = manager.GetDefaultConfiguration(NodeId.Null);
config.TreatUncertainAsBad.ShouldBeFalse("the server pins TreatUncertainAsBad = false (#782)");
double One(NodeId aggregate)
{
var calc = manager.CreateCalculator(aggregate, T0, T0.AddSeconds(20), 20_000, false, config);
calc.ShouldNotBeNull();
foreach (var (s, v, st) in Qp)
calc.QueueRawValue(new DataValue(Variant.From(v), st, T0.AddSeconds(s), T0.AddSeconds(s)));
calc.TryGetProcessedValue(true, out var dv).ShouldBeTrue();
dv.WrappedValue.TryGetValue(out double d).ShouldBeTrue();
return d;
}
// Part 13 (HistoryRead on SqlServer/QuestDB): Total = 73.5 + 88 + 32/13 = 163.96…, TimeAverage = Total ÷ 15 s.
// The live calculators integrate the uncovered [0,5) as a ramp from 0 (+25) and divide by the whole 20 s.
const double part13Total = 73.5 + 88 + 32d / 13;
One(ObjectIds.AggregateFunction_Total).ShouldBe(part13Total + 25, 1e-9); // 188.96…
One(ObjectIds.AggregateFunction_TimeAverage).ShouldBe((part13Total + 25) / 20, 1e-9); // 9.448… (Part 13: 10.93…)
// #809: the Uncertain [5,12) counts as Good live (HistoryRead: 8 000 ms / 40 %).
One(ObjectIds.AggregateFunction_DurationGood).ShouldBe(15_000d, 1e-9);
One(ObjectIds.AggregateFunction_PercentGood).ShouldBe(75d, 1e-9);
// DurationBad / PercentBad agree with HistoryRead: the 5 s before the data is Bad.
One(ObjectIds.AggregateFunction_DurationBad).ShouldBe(5_000d, 1e-9);
One(ObjectIds.AggregateFunction_PercentBad).ShouldBe(25d, 1e-9);
}
}