diff --git a/CLAUDE.md b/CLAUDE.md index 2cd77cb4..669bcb1d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1350,7 +1350,7 @@ historian tagname, which is the tag's driver `FullName`. ### Pluggable backends, SQL Server and HistoryUpdate (2026-10) - **Selection:** `ServerHistorian:Backend` (`Gateway` default | `SqlServer` | `QuestDb`; unknown fails host start). `HistorianBackendRegistry` → `IHistorianBackendProvider` builds all five seams; the gateway keys are only required for `Gateway`. `SqlServer` uses `ServerHistorian:SqlServer:{ConnectionString (required), Schema=hist, CommandTimeout=30s, AutoCreateSchema=true}`. -- **SQL Server** (`Driver.Historian.SqlServer`): `hist.Sample` PK `(TagId, SourceTimeUtc)` — writes are insert-if-absent, so outbox redelivery is idempotent. Stores every value kind with its **real status**; per-sample aggregates (10, simple definitions) are pushed down to T-SQL with bucket math in **100 ns ticks**; the Part 13 time-weighted seven (TimeAverage/TimeAverage2/Total/Total2/Interpolative, #671 — ⚠️ `Total` is now the time-weighted integral, no longer `SUM` — plus the **duration-based** PercentGood/PercentBad, #680) stream window + 4 bound rows into the shared `TimeWeightedAggregator` (both SQL backends, identical results; reverse `Interpolative` anchored at the later time; window capped by `ServerHistorian:MaxTimeWeightedRawRows` ⇒ `BadTooManyOperations`, percentages included). ⚠️ **#680 status rules (client-visible):** TimeAverage/Total are Uncertain if any part of the interval uses an Uncertain value, is bridged across a Bad sample, is extrapolated past the last sample (⇒ a live tag's latest interval is Uncertain) or is not covered; TimeAverage2/Total2 keep the SDK time-based rule (Uncertain time counts as Good) but never extrapolate past the last sample — that interval is Partial, later ones `BadNoData` (a report-by-exception tag with nothing after its last sample reads `BadNoData` until its next sample arrives). `docs/Historian.md` §"Time-weighted aggregates". ⚠️ **#680 per-sample (client-visible):** `Count` counts **Good** samples only (value-based status; an all-Bad interval is generic Bad, value null) and an empty interval **inside the tag's data range** is `0`/Good (outside it still `BadNoData`); `Start`/`End` return the **raw** first/last sample of any quality — its own timestamp and status, value null when Bad; `Delta` is Uncertain only when the interval's first or last sample is unusable; and every per-sample aggregate but `Average` sets **end-of-data Partial** on the interval holding the last sample (forward incl. its earlier edge, reverse strictly inside), never on a Bad result. The data-range lookup is a seek on SQL Server but a partition scan on QuestDB (unindexed `tag` SYMBOL). `docs/Historian.md` §"Aggregates". Docker-dev recipe, schema and aggregate semantics: `docs/Historian.md` §"SQL Server backend". Integration tests need `HISTORIAN_SQL_TEST_CONNECTION`. **Retention (#667):** opt-in `SqlServer:RetentionDays` (+ `PurgeInterval` 1h, `PurgeBatchSize` 4000) -> `HistorianRetentionService` purges old `hist.Sample` (per tag) / `hist.Event` in `DELETE TOP(n)` batches, Primary-gated on `IHistorizationRoleView`, one purger per DB via `sp_getapplock`; counter `otopcua.historian.retention.purged{table}`. Optional `IHistorianBackendProvider.CreateRetention` (default null). `docs/Historian.md` §Retention. +- **SQL Server** (`Driver.Historian.SqlServer`): `hist.Sample` PK `(TagId, SourceTimeUtc)` — writes are insert-if-absent, so outbox redelivery is idempotent. Stores every value kind with its **real status**; per-sample aggregates (10, simple definitions) are pushed down to T-SQL with bucket math in **100 ns ticks**; the Part 13 time-weighted seven (TimeAverage/TimeAverage2/Total/Total2/Interpolative, #671 — ⚠️ `Total` is now the time-weighted integral, no longer `SUM` — plus the **duration-based** PercentGood/PercentBad, #680) stream window + 4 bound rows into the shared `TimeWeightedAggregator` (both SQL backends, identical results; reverse `Interpolative` anchored at the later time; window capped by `ServerHistorian:MaxTimeWeightedRawRows` ⇒ `BadTooManyOperations`, percentages included). ⚠️ **#680 status rules (client-visible):** TimeAverage/Total are Uncertain if any part of the interval uses an Uncertain value, is bridged across a Bad sample, is extrapolated past the last sample (⇒ a live tag's latest interval is Uncertain) or is not covered; TimeAverage2/Total2 keep the SDK time-based rule (Uncertain time counts as Good) but never extrapolate past the last sample — that interval is Partial, later ones `BadNoData` (a report-by-exception tag with nothing after its last sample reads `BadNoData` until its next sample arrives). `docs/Historian.md` §"Time-weighted aggregates". ⚠️ **#680 per-sample (client-visible):** `Count` counts **Good** samples only (value-based status; an all-Bad interval is generic Bad, value null) and an empty interval **inside the tag's data range** is `0`/Good (outside it still `BadNoData`); `Start`/`End` return the **raw** first/last sample of any quality — its own timestamp and status, value null when Bad; `Delta` is Uncertain only when the interval's first or last sample is unusable; ⚠️ #684: `Average`/`Minimum`/`Maximum`/`Range`/`*ActualTime` use **Good samples only** (was Good + Uncertain; an Uncertain-only interval is `BadNoData`), Min/Max-family status Uncertain on a Bad sample, an Uncertain value beyond the Good extreme (Part 13 — the SDK ignores that) or an Uncertain non-numeric sample, and only a **Good** String/DateTime sample refuses them (`BadAggregateInvalidInputs`; Delta still refuses any usable one); and every per-sample aggregate but `Average` sets **end-of-data Partial** on the interval holding the last sample (forward incl. its earlier edge, reverse strictly inside), never on a Bad result. The data-range lookup is a seek on SQL Server but a partition scan on QuestDB (unindexed `tag` SYMBOL). `docs/Historian.md` §"Aggregates". Docker-dev recipe, schema and aggregate semantics: `docs/Historian.md` §"SQL Server backend". Integration tests need `HISTORIAN_SQL_TEST_CONNECTION`. **Retention (#667):** opt-in `SqlServer:RetentionDays` (+ `PurgeInterval` 1h, `PurgeBatchSize` 4000) -> `HistorianRetentionService` purges old `hist.Sample` (per tag) / `hist.Event` in `DELETE TOP(n)` batches, Primary-gated on `IHistorizationRoleView`, one purger per DB via `sp_getapplock`; counter `otopcua.historian.retention.purged{table}`. Optional `IHistorianBackendProvider.CreateRetention` (default null). `docs/Historian.md` §Retention. - **QuestDB** (`Driver.Historian.QuestDb`, `ServerHistorian:QuestDb:*`, Npgsql over PG wire; docker-dev overlay `docker-compose.questdb.yml`): same kinds/status/aggregates/HistoryUpdate as SqlServer, soft delete by tombstone (INSERT only — never `UPDATE`/`DELETE`; every read filters `NOT deleted`). ⚠️ **Timestamps are µs** (`QuestDbConnection.Round`/`BindTimestamp`; `IHistorianDataSource.TimestampResolutionTicks` = 10 steps HistoryRead paging), **pre-1970 cannot be stored** (writer drops, HistoryUpdate `BadOutOfRange`), every write waits for WAL visibility, TTL is enforced on write commits only, set Npgsql `Maximum Pool Size` < QuestDB's 64. `docs/Historian.md` §"QuestDB backend". - ⚠️ **Reverse processed reads are Part 13/SDK-anchored (#677, client-visible):** every aggregate on SqlServer/QuestDB now anchors a reverse read at its later time — window `(end, start]`, partial interval earliest, each stamped with its later edge (was: earlier-anchored, stamped earlier edge); on forward reads Partial also marks the interval the data starts inside (not Average — the SDK's rule, forward-only); a non-default client `AggregateConfiguration` ⇒ `BadAggregateConfigurationRejected`; Gateway provisioning sends `IntegralDivisor = 1`. SDK-parity tests: `SdkAggregateParityTests`. `docs/Historian.md` §Aggregates. - **Advertised aggregates (#682):** `IHistorianDataSource.SupportedAggregates` (Gateway 4 via `GatewayHistorianDataSource.ServedAggregates`, SqlServer/QuestDB 17 via `AggregateFamilies.Served`, Null **none**; a null set ⇒ all) is the one authority for `HistoryServerCapabilities/AggregateFunctions` (i=11201 — only its folder references are trimmed) and the up-front `BadAggregateNotSupported` in HistoryRead-Processed. ⚠️ `ServerCapabilities/AggregateFunctions` (i=2997) is deliberately NOT trimmed: it keeps the SDK's 37 because live `AggregateFilter` subscriptions still compute them all. ⚠️ Historian disabled ⇒ Processed is now `BadAggregateNotSupported`, not `GoodNoData`. `docs/Historian.md` §Advertised aggregates. diff --git a/docs/Historian.md b/docs/Historian.md index 6b8af02c..0fd60e08 100644 --- a/docs/Historian.md +++ b/docs/Historian.md @@ -996,7 +996,31 @@ point (see [Continuation-point paging (Events)](#continuation-point-paging-event > so a "live" tag's latest interval is now flagged. > > A client that read `Count` as "usable samples", keyed `Start`/`End` on the interval start, or treated `Count = BadNoData` as "no -> samples" must adjust. Not changed: `Average`/`Minimum`/`Maximum` still use Good + Uncertain samples (the SDK uses Good only). +> samples" must adjust. (`Average`/`Minimum`/`Maximum` were left on Good + Uncertain samples here; #684, below, moved them to Good only.) + +> ⚠️ **#684 (client-visible): `Average`, `Minimum`, `Maximum`, `Range` and `Minimum`/`MaximumActualTime` use GOOD samples only** on +> the `SqlServer` and `QuestDb` backends (they used every *usable* sample — Good **and** Uncertain). Part 13 defines each over "Good +> raw values" (§5.4.3.5, .10–.14), independent of `TreatUncertainAsBad` (only `Minimum2`/`Maximum2`, not served, consult it), and the +> SDK's `AverageAggregateCalculator`/`MinMaxAggregateCalculator` skip Uncertain values under the server's own `TreatUncertainAsBad = false`. +> - An Uncertain sample is **never part of the value** (an interval `Good 6, Good 8, Uncertain 4` now averages 7, was 6; its Minimum is +> 6, was 4), and an interval holding **only** Uncertain (and Bad) samples is now **`BadNoData`** (it used to return their value, +> Uncertain). +> - **Status:** `Average` is value-based (Good only when every sample is Good, else `UncertainDataSubNormal` — unchanged, the SDK's +> rule). `Minimum`/`Maximum`/`Range`/`*ActualTime` follow their Part 13 tables: `UncertainDataSubNormal` when a **Bad** sample +> exists, or an **Uncertain value lies beyond the reported Good extreme** (below the minimum for Minimum/MinimumActualTime, above the +> maximum for Maximum/MaximumActualTime, either for Range; strictly — equal does not count); otherwise **Good**. So an Uncertain +> sample inside the Good range no longer makes Minimum/Maximum Uncertain (it used to). +> - The SDK applies only the Bad half of that rule (its `ComputeMinMax` collects the Uncertain extremes and never reads them) and answers +> Good for an Uncertain value beyond the extreme; ours follows Part 13 there — the one remaining per-sample Uncertain difference, pinned +> by `SdkAggregateParityTests`. +> - **Non-numeric samples:** these aggregates now answer `BadAggregateInvalidInputs` only for a **Good** `String`/`DateTime` sample +> in the interval. An **Uncertain** one is skipped like any Uncertain value — the interval is served from its Good numeric values +> with `UncertainDataSubNormal` (the SDK's Average skips it; its Min/Max flags the value it cannot cast). `Delta` reads usable +> samples and still refuses any usable non-numeric one. +> - `Count` (Good only, #680), `Start`/`End`/`Delta` (usable / any-quality picks) and the time-weighted family (Uncertain time counts as +> Good time under `TreatUncertainAsBad = false`, as Part 13 §5.4.3.2 says) are unchanged. The advertised `TreatUncertainAsBad = false` +> stays correct: it governs the time-weighted and status calculations; the per-sample aggregates above are defined over Good values +> whatever its setting. **Bucketing.** Forward: one result per interval of `[start, end)`; bucket *i* = `[start + i·interval, start + (i+1)·interval)`, start-inclusive / end-exclusive to the tick, the last bucket truncated at `end`, each stamped with its start. Reverse (`end < start`, @@ -1043,18 +1067,19 @@ included); a per-sample `Average` read skips them. | Aggregate | Computed over | Value timestamp | |---|---|---| -| `Average`, `Minimum`, `Maximum` | `AVG` / `MIN` / `MAX` of `NumValue` over usable samples | bucket start (reverse read: its later edge, as every row marked "bucket start" here) | +| `Average`, `Minimum`, `Maximum` | `AVG` / `MIN` / `MAX` of `NumValue` over **Good** samples (#684) | bucket start (reverse read: its later edge, as every row marked "bucket start" here) | | `Count` | number of **Good** samples (`Int32`; #680) | bucket start | -| `Range` | `Maximum − Minimum` | bucket start | +| `Range` | `Maximum − Minimum` (Good samples) | bucket start | | `Start`, `End` | the **raw** first / last sample of any quality (its type, its own status; value null when Bad — #680) | **the sample's own time** | | `Delta` | last usable − first usable value (numeric samples) | bucket start | -| `MinimumActualTime`, `MaximumActualTime` | the usable sample with the smallest / largest `NumValue`; ties ⇒ earliest | **the sample's own time** | +| `MinimumActualTime`, `MaximumActualTime` | the **Good** sample (#684) with the smallest / largest `NumValue`; ties ⇒ earliest | **the sample's own time** | Status and edge rules: - An **empty interval** (no samples) ⇒ `BadNoData` — except **`Count`**, which is `0`, Good, when the interval lies inside the tag's data range `[first sample, last sample]` (both edges inclusive, either direction; #680). **Every value aggregate over an interval whose samples are all Bad ⇒ `BadNoData`**; `Count` there is the generic **Bad** (value null). -- Value aggregates are `Good` when every sample in the bucket is Good and `UncertainDataSubNormal` when the bucket also holds any Bad **or Uncertain** sample. `Count`'s status is value-based (all Good ⇒ Good, all Bad ⇒ Bad, else Uncertain); `Start`/`End` carry the raw sample's own status; `Delta` is Good unless the interval's first or last sample is not usable (then Uncertain) — #680. -- `Count`, `Start`, `End` accept any value kind; the other aggregates (and `Delta`) on a bucket that holds a usable `String`/`DateTime` sample ⇒ `BadAggregateInvalidInputs` for that interval. +- An interval with samples but **no Good numeric value** (only Uncertain / Bad) ⇒ `BadNoData` for `Average`/`Minimum`/`Maximum`/`Range`/`*ActualTime` (#684). +- `Average` is `Good` when every sample in the bucket is Good, else `UncertainDataSubNormal` (value-based, the SDK's rule). `Minimum`/`Maximum`/`Range`/`*ActualTime` are `UncertainDataSubNormal` when the bucket holds a Bad sample or an Uncertain value beyond the reported Good extreme (Part 13 Tables 57–62; strictly beyond), else `Good` (#684). `Count`'s status is value-based (all Good ⇒ Good, all Bad ⇒ Bad, else Uncertain); `Start`/`End` carry the raw sample's own status; `Delta` is Good unless the interval's first or last sample is not usable (then Uncertain) — #680. +- `Count`, `Start`, `End` accept any value kind. `Average`/`Minimum`/`Maximum`/`Range`/`*ActualTime` on a bucket that holds a **Good** `String`/`DateTime` sample ⇒ `BadAggregateInvalidInputs` for that interval; an Uncertain one is skipped and makes `Minimum`/`Maximum`/`Range`/`*ActualTime` `UncertainDataSubNormal` (#684). `Delta` is refused by any usable non-numeric sample. - `PercentGood` / `PercentBad` are **duration-based since #680** — see [Time-weighted aggregates](#time-weighted-aggregates-part-13-671). - Everything except `Count` and `Start`/`End` is a `double`; `Int64` samples aggregate as doubles and `Boolean` as 0/1. @@ -1198,7 +1223,8 @@ Start, End, Delta and Minimum/MaximumActualTime; the time-weighted one TimeAvera PercentGood/Bad (#680), including datasets whose data ends inside the window and with an interval wholly inside an outage; reverse reads over data that starts inside the window are compared too. Known differences that remain (tracked as #680) — documented and masked in the tests, deliberately not changed here because forward results were to stay as they are — are: -the SDK's per-sample Average/Minimum/Maximum use Good samples only, ours Good + Uncertain; the SDK sets Partial on Bad per-sample +an Uncertain value beyond the Good extreme makes Minimum/Maximum/Range/*ActualTime Uncertain here (Part 13), Good in the SDK +(#684 aligned the values — Good samples only — and every other Uncertain case); the SDK sets Partial on Bad per-sample results too, ours never (#680 closed the Start/End, Delta, Count and end-of-data Partial differences). Time-weighted differences (#680 closed the TimeAverage/Total status, `Total` coverage, end-of-data and PercentGood/Bad ones), each deliberate: ours flags a truncated interval Partial (Part 13 §5.3.3.2 Case 3), the SDK does not; ours carries the Calculated bit on a `BadNoData`; `TimeAverage2`/`Total2` over an diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb/QuestDbAggregateQueries.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb/QuestDbAggregateQueries.cs index 7bff0c19..a7e6f2c9 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb/QuestDbAggregateQueries.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb/QuestDbAggregateQueries.cs @@ -58,17 +58,20 @@ internal static class QuestDbAggregateQueries /// /// Average / Minimum / Maximum / Count / Range (PercentGood / PercentBad and Total are time-weighted, not here). - /// Family columns: 6 avg_value, 7 min_value, 8 max_value, 9 sum_value (DOUBLE, NULL when no usable sample). + /// Family columns: 6 avg_value, 7 min_value, 8 max_value, 9 sum_value over the Good numeric samples + /// (DOUBLE, NULL when none — #684, Part 13 / the SDK), then 10 unc_min, 11 unc_max over the Uncertain ones (the + /// Part 13 Minimum/Maximum/Range status rule) and 12 good_non_numeric_count. /// /// Prefixed sample table name. /// The SQL text. /// True for a reverse read (window ($4, $5]). public static string Simple(string sample, bool reverse = false) => "SELECT b, " + CommonCounts() + ", " + - "avg(CASE WHEN u = 1 THEN num_value END) AS avg_value, " + - "min(CASE WHEN u = 1 THEN num_value END) AS min_value, " + - "max(CASE WHEN u = 1 THEN num_value END) AS max_value, " + - "sum(CASE WHEN u = 1 THEN num_value END) AS sum_value " + + "avg(CASE WHEN g = 1 THEN num_value END) AS avg_value, " + + "min(CASE WHEN g = 1 THEN num_value END) AS min_value, " + + "max(CASE WHEN g = 1 THEN num_value END) AS max_value, " + + "sum(CASE WHEN g = 1 THEN num_value END) AS sum_value, " + + UncertainExtremes() + " " + "FROM (" + Classified(sample, reverse) + ") GROUP BY b ORDER BY b"; /// Ordinal of the StartEnd family's first-sample-of-any-quality columns (). @@ -126,9 +129,9 @@ internal static class QuestDbAggregateQueries } /// - /// MinimumActualTime / MaximumActualTime: the usable sample with the smallest (largest) num_value, ties - /// broken by the earliest ts. Family columns: 6 num_value, 7 ts (NULL when the bucket has no usable - /// numeric sample). + /// MinimumActualTime / MaximumActualTime: the Good sample (#684) with the smallest (largest) num_value, + /// ties broken by the earliest ts. Family columns: 6 num_value, 7 ts (NULL when the bucket has no Good + /// numeric sample), then 8 unc_min, 9 unc_max, 10 good_non_numeric_count as in . /// /// /// Deliberately still two references to the classified rows (counts + the ranked extreme), unlike @@ -143,11 +146,12 @@ internal static class QuestDbAggregateQueries /// True for a reverse read (window ($4, $5]). public static string ActualTime(string sample, bool maximum, bool reverse = false) => "WITH s AS (" + Classified(sample, reverse) + "), " + - "st AS (SELECT b, " + CommonCounts() + " FROM s GROUP BY b), " + + "st AS (SELECT b, " + CommonCounts() + ", " + UncertainExtremes() + " FROM s GROUP BY b), " + "r AS (SELECT b, num_value, ts, " + "row_number() OVER (PARTITION BY b ORDER BY num_value " + (maximum ? "DESC" : "ASC") + ", ts ASC) AS rn " + - "FROM s WHERE u = 1 AND num_value IS NOT NULL) " + - "SELECT st.*, r.num_value, r.ts " + + "FROM s WHERE g = 1 AND num_value IS NOT NULL) " + + "SELECT st.b, st.total_count, st.usable_count, st.good_count, st.bad_count, st.non_numeric_count, " + + "r.num_value, r.ts, st.unc_min, st.unc_max, st.good_non_numeric_count " + "FROM st LEFT JOIN (SELECT * FROM r WHERE rn = 1) r ON r.b = st.b ORDER BY st.b"; /// @@ -217,6 +221,15 @@ internal static class QuestDbAggregateQueries $"CASE WHEN status < {GoodLimit} AND kind <> {EmptyKind} THEN 1 ELSE 0 END AS g " + $"FROM {sample} WHERE tag = $3 AND " + (reverse ? "ts > $4 AND ts <= $5" : "ts >= $4 AND ts < $5") + " AND NOT deleted"; + /// + /// The minimum and maximum Uncertain (usable, not Good) numeric value of the bucket — unc_min, unc_max + /// (#684). Not result values: Part 13 makes Minimum/Maximum/Range Uncertain when one lies beyond the Good extreme. + /// Then good_non_numeric_count, the Good String/DateTime samples (the Good-value aggregates' refusal check). + /// + private static string UncertainExtremes() => + "min(CASE WHEN u = 1 AND g = 0 THEN num_value END) AS unc_min, max(CASE WHEN u = 1 AND g = 0 THEN num_value END) AS unc_max, " + + $"sum(CASE WHEN g = 1 AND kind IN ({NonNumericKinds}) THEN 1 ELSE 0 END) AS good_non_numeric_count"; + /// The five common count columns (after b). private static string CommonCounts() => "count() AS total_count, sum(u) AS usable_count, sum(g) AS good_count, count() - sum(u) AS bad_count, " + diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb/QuestDbHistorianDataSource.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb/QuestDbHistorianDataSource.cs index f192a2b5..53f134a3 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb/QuestDbHistorianDataSource.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb/QuestDbHistorianDataSource.cs @@ -617,6 +617,9 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource row.Min = D(reader, f + 1); row.Max = D(reader, f + 2); row.Sum = D(reader, f + 3); + row.UncertainMin = D(reader, f + 4); + row.UncertainMax = D(reader, f + 5); + row.GoodNonNumeric = N(reader, f + 6); break; case AggregateFamily.StartEnd: row.First = V(reader, f); @@ -627,6 +630,9 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource default: row.Extreme = D(reader, f); row.ExtremeTime = reader.IsDBNull(f + 1) ? null : QuestDbConnection.ReadUtc(reader, f + 1); + row.UncertainMin = D(reader, f + 2); + row.UncertainMax = D(reader, f + 3); + row.GoodNonNumeric = N(reader, f + 4); break; } return row; diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common/AggregateEvaluator.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common/AggregateEvaluator.cs index a3f49514..97b7baa4 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common/AggregateEvaluator.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common/AggregateEvaluator.cs @@ -9,7 +9,11 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common; /// /// Per interval: no samples ⇒ BadNoData (Count: 0 inside the data range). Otherwise no usable sample ⇒ /// BadNoData; a numeric aggregate with a usable String/DateTime sample ⇒ BadAggregateInvalidInputs; -/// else the value, Good when every sample is Good, UncertainDataSubNormal when any is Bad or Uncertain. +/// else the value. #684: Average / Minimum / Maximum / Range / *ActualTime are computed over Good values only +/// (Part 13 and the SDK — an Uncertain sample is never part of the value; an interval with no Good value is +/// BadNoData). Status: Average value-based (Good only when every sample is Good, else +/// UncertainDataSubNormal); Minimum / Maximum / Range / *ActualTime per their Part 13 tables — Uncertain when a +/// Bad sample exists or an Uncertain value lies beyond the reported extreme (). /// Every calculated status — including BadNoData — carries the Calculated info bits. Timestamps are the /// bucket's (its start forward, its later edge reverse), except /// Minimum/MaximumActualTime (the sample's time; ties ⇒ earliest) and Start/End. Every aggregate but @@ -70,15 +74,30 @@ public static class AggregateEvaluator return Calc(r.Good, r.Good == r.Total ? HistorianStatusCodes.Good : HistorianStatusCodes.UncertainDataSubNormal); } if (r.Usable == 0) return NoData(bucketStart, now); - if (r.NonNumeric > 0) return Calc(null, HistorianStatusCodes.BadAggregateInvalidInputs); + // #684 review: Delta reads usable picks, so any usable String/DateTime sample refuses it. The Good-value aggregates + // refuse only a GOOD non-numeric sample — an Uncertain one is not part of their value set (it only degrades the + // status, as the SDK's skipped/uncastable value does). + if ((aggregate == HistoryAggregateType.Delta ? r.NonNumeric : r.GoodNonNumericOrAll) > 0) + return Calc(null, HistorianStatusCodes.BadAggregateInvalidInputs); - var status = r.Good == r.Total ? HistorianStatusCodes.Good : HistorianStatusCodes.UncertainDataSubNormal; switch (aggregate) { - case HistoryAggregateType.Average: return Calc(r.Avg, status); - case HistoryAggregateType.Minimum: return Calc(r.Min, status); - case HistoryAggregateType.Maximum: return Calc(r.Max, status); - case HistoryAggregateType.Range: return Calc(r.Max - r.Min, status); + // #684 (Part 13 §5.4.3.5; SDK ComputeAverage): the mean of the GOOD values — an interval with none (only + // Uncertain / Bad) is BadNoData; value-based status: Good only when every sample is Good, else Uncertain. + case HistoryAggregateType.Average: + if (r.Avg is null) return NoData(bucketStart, now); + return Calc(r.Avg, r.Good == r.Total ? HistorianStatusCodes.Good : HistorianStatusCodes.UncertainDataSubNormal); + // #684 (Part 13 §5.4.3.10/11/14; SDK ComputeMinMax): the extreme(s) of the GOOD values; Uncertain when a Bad + // sample exists or an Uncertain value lies beyond the reported extreme (see MinMaxStatus). + case HistoryAggregateType.Minimum: + if (r.Min is null) return NoData(bucketStart, now); + return Calc(r.Min, MinMaxStatus(r, r.Min, r.Max, low: true, high: false)); + case HistoryAggregateType.Maximum: + if (r.Max is null) return NoData(bucketStart, now); + return Calc(r.Max, MinMaxStatus(r, r.Min, r.Max, low: false, high: true)); + case HistoryAggregateType.Range: + if (r.Min is null || r.Max is null) return NoData(bucketStart, now); + return Calc(r.Max - r.Min, MinMaxStatus(r, r.Min, r.Max, low: true, high: true)); case HistoryAggregateType.Delta: // #680 (SDK ComputeDelta): last usable − first usable; Uncertain only when the interval's first or last // sample was skipped (not usable) — a Bad sample in the middle does not degrade it. @@ -96,12 +115,32 @@ public static class AggregateEvaluator // Live-gate 8 F1: usable numeric samples counted but no extreme picked (an inconsistent row) is no data, // never a value-less Good. if (r.Extreme is null || r.ExtremeTime is null) return NoData(bucketStart, now); - return Calc(r.Extreme, status, r.ExtremeTime); + var min = aggregate == HistoryAggregateType.MinimumActualTime; + return Calc(r.Extreme, MinMaxStatus(r, r.Extreme, r.Extreme, low: min, high: !min), r.ExtremeTime); default: throw new HistoryAggregateNotSupportedException(aggregate); } } + /// + /// #684 — the Part 13 status of Minimum / Maximum / Range / *ActualTime (Tables 57–62): Uncertain_DataSubNormal + /// when a Bad sample exists in the interval, or when an Uncertain value is less than the minimum Good value + /// () / greater than the maximum Good value (); else Good. An Uncertain + /// value inside the Good range leaves it Good. An Uncertain String/DateTime sample (no order against the Good values) + /// also makes it Uncertain — the SDK's ComputeMinMax flags a value it cannot cast exactly that way. + /// + /// + /// The SDK's ComputeMinMax (1.5.378) applies only the Bad half: it collects the Uncertain extremes and never + /// reads them, so it answers Good where Part 13 says Uncertain. Ours follows the specification there — the more + /// conservative of the two, and never a status better than the inputs warrant. The comparison is strict, so an + /// Uncertain value EQUAL to the Good extreme does not degrade it. + /// + private static uint MinMaxStatus(BucketRow r, double? goodMin, double? goodMax, bool low, bool high) => + r.Bad > 0 || r.NonNumeric > r.GoodNonNumericOrAll + || (low && r.UncertainMin < goodMin) || (high && r.UncertainMax > goodMax) + ? HistorianStatusCodes.UncertainDataSubNormal + : HistorianStatusCodes.Good; + /// /// True for the per-sample aggregates the SDK sets the start-of-data Partial bit on (every calculator but /// AverageAggregateCalculator's plain Average, which turns SetPartialBit off). diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common/BucketRow.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common/BucketRow.cs index 4d0c4c9e..7a08e086 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common/BucketRow.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common/BucketRow.cs @@ -9,7 +9,11 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common; /// /// Sample classes: usable = severity Good or Uncertain AND a stored value (an Empty sample is an outage marker /// and counts as Bad whatever its code); good = usable with severity Good; bad = not usable. -/// Every value field is computed over usable samples only, except / . +/// / / / / are computed over +/// Good numeric samples only (#684 — Part 13 defines Average / Minimum / Maximum / Range / *ActualTime over Good +/// raw values, as the SDK's calculators do); / over the Uncertain +/// ones (the Part 13 Minimum/Maximum/Range status rule); / over usable samples; +/// / over every sample. /// public sealed class BucketRow { @@ -28,19 +32,30 @@ public sealed class BucketRow /// Bad (not usable) samples. public int Bad { get; set; } - /// Usable String / DateTime samples. + /// Usable String / DateTime samples (Delta's refusal check). public int NonNumeric { get; set; } - /// Average of the usable numeric values (null when none). + /// + /// Good String / DateTime samples — the refusal check of the Good-value aggregates (Average / Minimum / Maximum / + /// Range / *ActualTime, #684 review): an Uncertain non-numeric sample is not part of their value set. Filled by the + /// Simple and ActualTime queries; null (a row that does not carry it) falls back to , the + /// pre-#684 (stricter) check. + /// + public int? GoodNonNumeric { get; set; } + + /// , or when the row does not carry it. + internal int GoodNonNumericOrAll => GoodNonNumeric ?? NonNumeric; + + /// Average of the Good numeric values (null when none — #684). public double? Avg { get; set; } - /// Minimum of the usable numeric values (null when none). + /// Minimum of the Good numeric values (null when none — #684). public double? Min { get; set; } - /// Maximum of the usable numeric values (null when none). + /// Maximum of the Good numeric values (null when none — #684). public double? Max { get; set; } - /// Sum of the usable numeric values (null when none). + /// Sum of the Good numeric values (null when none — #684). public double? Sum { get; set; } /// First usable sample's value by time (Empty when none). @@ -49,12 +64,25 @@ public sealed class BucketRow /// Last usable sample's value by time (Empty when none). public HistorianValue Last { get; set; } - /// Extreme (min or max) usable numeric value (null when none). + /// Extreme (min or max) Good numeric value (null when none — #684). public double? Extreme { get; set; } /// Time of (ties ⇒ earliest; null when none). public DateTime? ExtremeTime { get; set; } + /// + /// Minimum of the bucket's Uncertain (usable, not Good) numeric values (null when none). Not a result value: + /// Part 13 makes Minimum / MinimumActualTime / Range Uncertain_DataSubNormal when an Uncertain value is less + /// than the minimum Good value (#684). + /// + public double? UncertainMin { get; set; } + + /// + /// Maximum of the bucket's Uncertain numeric values (null when none): Maximum / MaximumActualTime / Range are + /// Uncertain when one is greater than the maximum Good value (#684). + /// + public double? UncertainMax { get; set; } + /// /// The bucket's first sample by time of any quality (null when the bucket is empty): Start returns it raw, and /// Delta is Uncertain when it is not usable (#680, Part 13 Tables 73/74 — the SDK's StartEndAggregateCalculator). diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/SqlAggregateQueries.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/SqlAggregateQueries.cs index 8b45164a..d57120e5 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/SqlAggregateQueries.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/SqlAggregateQueries.cs @@ -32,7 +32,8 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer; /// and bit 30): usable = severity Good or Uncertain (StatusCode >= 0) AND a stored value /// (ValueKind <> 0) — an Empty sample is an outage marker and counts as Bad whatever its /// code; good = usable with severity Good (StatusCode < 0x40000000); bad = not -/// usable. Every value column below is computed over usable samples only. +/// usable. Average/Minimum/Maximum/Range/*ActualTime value columns are computed over good samples only +/// (#684); Start/End/Delta picks over usable samples (or every sample). /// /// Common result columns (ordinals 0–5): Bucket, TotalCount, UsableCount, GoodCount, BadCount, /// NonNumericCount (usable String/DateTime samples). Family columns follow. @@ -47,8 +48,10 @@ internal static class SqlAggregateQueries /// /// Average / Minimum / Maximum / Count / Range (PercentGood / PercentBad and Total are time-weighted, not here). - /// Family columns: 6 AvgValue, 7 MinValue, 8 MaxValue, 9 SumValue (float, NULL when no usable - /// numeric sample). + /// Family columns: 6 AvgValue, 7 MinValue, 8 MaxValue, 9 SumValue over the Good numeric samples (float, + /// NULL when none — #684: Part 13 and the SDK compute these over Good values only), then 10 UncMin, 11 UncMax + /// over the Uncertain ones (the Part 13 Minimum/Maximum/Range status rule, ), and + /// 12 GoodNonNumericCount. /// /// Schema-qualified Sample table name. /// The SQL text. @@ -56,10 +59,11 @@ internal static class SqlAggregateQueries public static string Simple(string sample, bool reverse = false) => Classified(sample, reverse) + "SELECT b, " + CommonCounts() + ", " + - "AVG(CASE WHEN u = 1 THEN NumValue END) AS AvgValue, " + - "MIN(CASE WHEN u = 1 THEN NumValue END) AS MinValue, " + - "MAX(CASE WHEN u = 1 THEN NumValue END) AS MaxValue, " + - "SUM(CASE WHEN u = 1 THEN NumValue END) AS SumValue " + + "AVG(CASE WHEN g = 1 THEN NumValue END) AS AvgValue, " + + "MIN(CASE WHEN g = 1 THEN NumValue END) AS MinValue, " + + "MAX(CASE WHEN g = 1 THEN NumValue END) AS MaxValue, " + + "SUM(CASE WHEN g = 1 THEN NumValue END) AS SumValue, " + + UncertainExtremes() + " " + "FROM s GROUP BY b ORDER BY b;"; /// Ordinal of the StartEnd family's first-sample-of-any-quality columns (). @@ -107,9 +111,9 @@ internal static class SqlAggregateQueries } /// - /// MinimumActualTime / MaximumActualTime: the usable sample with the smallest (largest) NumValue, + /// MinimumActualTime / MaximumActualTime: the Good sample (#684) with the smallest (largest) NumValue, /// ties broken by the earliest SourceTimeUtc. Family columns: 6 NumValue, 7 SourceTimeUtc - /// (NULL when the bucket has no usable numeric sample). + /// (NULL when the bucket has no Good numeric sample), then 8 UncMin, 9 UncMax, 10 GoodNonNumericCount as in . /// /// /// Deliberately still two references to s (counts + the ranked extreme), unlike : @@ -124,11 +128,12 @@ internal static class SqlAggregateQueries /// True for a reverse read (window (@start, @end]). public static string ActualTime(string sample, bool maximum, bool reverse = false) => Classified(sample, reverse) + - ", st AS (SELECT b, " + CommonCounts() + " FROM s GROUP BY b)" + + ", st AS (SELECT b, " + CommonCounts() + ", " + UncertainExtremes() + " FROM s GROUP BY b)" + ", r AS (SELECT b, NumValue, SourceTimeUtc, " + "ROW_NUMBER() OVER (PARTITION BY b ORDER BY NumValue " + (maximum ? "DESC" : "ASC") + ", SourceTimeUtc ASC) AS rn " + - "FROM s WHERE u = 1 AND NumValue IS NOT NULL) " + - "SELECT st.*, r.NumValue, r.SourceTimeUtc " + + "FROM s WHERE g = 1 AND NumValue IS NOT NULL) " + + "SELECT st.b, st.TotalCount, st.UsableCount, st.GoodCount, st.BadCount, st.NonNumericCount, " + + "r.NumValue, r.SourceTimeUtc, st.UncMin, st.UncMax, st.GoodNonNumericCount " + "FROM st LEFT JOIN r ON r.b = st.b AND r.rn = 1 ORDER BY st.b;"; /// @@ -187,6 +192,15 @@ internal static class SqlAggregateQueries $"FROM {sample} WHERE TagId = @id AND " + (reverse ? "SourceTimeUtc > @start AND SourceTimeUtc <= @end) " : "SourceTimeUtc >= @start AND SourceTimeUtc < @end) "); + /// + /// The minimum and maximum Uncertain (usable, not Good) numeric value of the bucket — UncMin, UncMax + /// (#684). Not result values: Part 13 makes Minimum/Maximum/Range Uncertain when one lies beyond the Good extreme. + /// Then GoodNonNumericCount, the Good String/DateTime samples (the Good-value aggregates' refusal check). + /// + private static string UncertainExtremes() => + "MIN(CASE WHEN u = 1 AND g = 0 THEN NumValue END) AS UncMin, MAX(CASE WHEN u = 1 AND g = 0 THEN NumValue END) AS UncMax, " + + $"SUM(CASE WHEN g = 1 AND ValueKind IN ({NonNumericKinds}) THEN 1 ELSE 0 END) AS GoodNonNumericCount"; + /// The five common count columns (after b). private static string CommonCounts() => "COUNT(*) AS TotalCount, SUM(u) AS UsableCount, SUM(g) AS GoodCount, COUNT(*) - SUM(u) AS BadCount, " + diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/SqlServerHistorianDataSource.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/SqlServerHistorianDataSource.cs index e1e458b0..c47f8b20 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/SqlServerHistorianDataSource.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/SqlServerHistorianDataSource.cs @@ -564,6 +564,9 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource row.Min = D(reader, f + 1); row.Max = D(reader, f + 2); row.Sum = D(reader, f + 3); + row.UncertainMin = D(reader, f + 4); + row.UncertainMax = D(reader, f + 5); + row.GoodNonNumeric = reader.IsDBNull(f + 6) ? 0 : reader.GetInt32(f + 6); break; case AggregateFamily.StartEnd: row.First = V(reader, f); @@ -574,6 +577,9 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource default: row.Extreme = D(reader, f); row.ExtremeTime = reader.IsDBNull(f + 1) ? null : SqlHistorianConnection.GetUtc(reader, f + 1); + row.UncertainMin = D(reader, f + 2); + row.UncertainMax = D(reader, f + 3); + row.GoodNonNumeric = reader.IsDBNull(f + 4) ? 0 : reader.GetInt32(f + 4); break; } return row; diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/ServerAggregateConfiguration.cs b/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/ServerAggregateConfiguration.cs index 32187303..9f3fc83d 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/ServerAggregateConfiguration.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/ServerAggregateConfiguration.cs @@ -9,8 +9,10 @@ namespace ZB.MOM.WW.OtOpcUa.OpcUaServer; /// /// /// The aggregate evaluators (AggregateEvaluator / TimeWeightedAggregator in -/// Driver.Historian.Sql.Common) are written against fixed settings, not parameterised: Uncertain samples and -/// Uncertain time count as Good ( = false); an interval is Good only when all of +/// Driver.Historian.Sql.Common) are written against fixed settings, not parameterised: Uncertain time counts as +/// Good ( = false) in the time-weighted family; the per-sample Average / Minimum / +/// Maximum / Range / *ActualTime / Count are defined by Part 13 over Good values whatever this setting (#684), as the +/// SDK computes them under it; an interval is Good only when all of /// it is ( = 100) and Bad only when none of it is non-Bad /// ( = 100), otherwise UncertainDataSubNormal; the value past the last sample /// is held (stepped, = false). These are also the SDK's own server defaults diff --git a/tests/Drivers/Historian.Conformance/ProcessedConformance.cs b/tests/Drivers/Historian.Conformance/ProcessedConformance.cs index 0a06031b..5f51434a 100644 --- a/tests/Drivers/Historian.Conformance/ProcessedConformance.cs +++ b/tests/Drivers/Historian.Conformance/ProcessedConformance.cs @@ -351,12 +351,46 @@ public abstract class ProcessedConformance : ConformanceBase } /// Dense: every 2.5 s from −10 to +100, values cycling, one Bad sample at +22.5 (no empty interval). - private Task SeedDenseAsync() => B.SeedAsync("P", DenseRows()); + private Task SeedDenseAsync(bool uncertain = false) => B.SeedAsync(uncertain ? "PU" : "P", DenseRows(uncertain)); - private static (DateTime T, HistorianValue V, uint Status)[] DenseRows() => Enumerable.Range(0, 45) - .Select(i => (At(-10 + i * 2.5), HistorianValue.FromDouble(10 + (i * 7 % 13)), i == 13 ? Bad : Good)) + /// + /// #684 : every fifth sample (other than the Bad one) Uncertain at 16 — inside every + /// interval's Good range (the cycling values alternate around 16), so the min/max family matches the SDK unmasked + /// (SdkAggregateParityTests.DenseUncertain is the same data). + /// + private static (DateTime T, HistorianValue V, uint Status)[] DenseRows(bool uncertain = false) => Enumerable.Range(0, 45) + .Select(i => uncertain && i % 5 == 0 && i != 13 + ? (At(-10 + i * 2.5), HistorianValue.FromDouble(16), Uncertain) + : (At(-10 + i * 2.5), HistorianValue.FromDouble(10 + (i * 7 % 13)), i == 13 ? Bad : Good)) .ToArray(); + [Theory] + [InlineData(0, 80, 20)] + [InlineData(80, 0, 20)] + [InlineData(77, 3, 20)] + [InlineData(3, 77, 20)] + public async Task Per_sample_processed_reads_over_uncertain_samples_match_the_SDK(double from, double to, double intervalS) + { + // #684: the Good-only value columns through the real SQL against the SDK's calculators, over Dense with Uncertain + // samples — values, timestamps, code bits and Partial, no mask. + await SeedDenseAsync(uncertain: true); + var rows = DenseRows(uncertain: true).Select(r => (B.Round(r.T), r.V, r.Status)).ToArray(); + var src = B.CreateDataSource(); + var interval = TimeSpan.FromSeconds(intervalS); + var (start, end) = (B.Round(At(from)), B.Round(At(to))); + foreach (var agg in new[] + { + HistoryAggregateType.Average, HistoryAggregateType.Minimum, HistoryAggregateType.Maximum, + HistoryAggregateType.Count, HistoryAggregateType.Range, HistoryAggregateType.MinimumActualTime, + HistoryAggregateType.MaximumActualTime, + }) + { + var ours = (await src.ReadProcessedAsync("PU", At(from), At(to), interval, agg, Ct)).Samples.ToList(); + SdkAggregateOracle.ShouldMatch(ours, SdkAggregateOracle.Run(agg, start, end, interval, rows), + $"{agg} uncertain {from}→{to}/{intervalS}s", compareCalculatedInterpolatedBits: false); + } + } + [Theory] [InlineData(0, 80, 20)] [InlineData(80, 0, 20)] @@ -476,14 +510,142 @@ public abstract class ProcessedConformance : ConformanceBase } [Fact] - public async Task Uncertain_samples_are_used_but_make_the_interval_uncertain() + public async Task Uncertain_samples_are_left_out_of_the_value_and_degrade_the_status_per_Part_13() { + // #684 (Part 13 §5.4.3.5/10-14; the SDK's calculators): Average / Minimum / Maximum / Range / *ActualTime use GOOD + // values only. One interval [0,20): Good 6 @+2, Good 8 @+3, Uncertain 4 @+1 (below the Good minimum) and + // Uncertain 7 @+4 (inside the Good range); a Good sample before (-5) and after (+30) the window keep the data range + // around it (no Partial). Pre-#684 the backends averaged 6.25 and answered Minimum 4. await B.SeedAsync("U", + (At(-5), HistorianValue.FromDouble(100), Good), (At(1), HistorianValue.FromDouble(4), Uncertain), - (At(2), HistorianValue.FromDouble(6), Good)); - var s = (await B.CreateDataSource().ReadProcessedAsync("U", At(0), At(20), Twenty, HistoryAggregateType.Average, Ct)).Samples.Single(); - s.Value.ShouldBe(5d); - s.StatusCode.ShouldBe(UncertainCalc); + (At(2), HistorianValue.FromDouble(6), Good), + (At(3), HistorianValue.FromDouble(8), Good), + (At(4), HistorianValue.FromDouble(7), Uncertain), + (At(30), HistorianValue.FromDouble(100), Good)); + var src = B.CreateDataSource(); + async Task One(HistoryAggregateType a) => + (await src.ReadProcessedAsync("U", At(0), At(20), Twenty, a, Ct)).Samples.Single(); + + // Average: value-based status — not every sample is Good ⇒ Uncertain. + var avg = await One(HistoryAggregateType.Average); + avg.Value.ShouldBe(7d); + avg.StatusCode.ShouldBe(UncertainCalc); + // Minimum: Uncertain 4 is below the Good minimum 6 ⇒ Uncertain (Part 13 Table 57), value still 6. + var min = await One(HistoryAggregateType.Minimum); + min.Value.ShouldBe(6d); + min.StatusCode.ShouldBe(UncertainCalc); + // Maximum: no Uncertain value above 8 and no Bad sample ⇒ Good. + var max = await One(HistoryAggregateType.Maximum); + max.Value.ShouldBe(8d); + max.StatusCode.ShouldBe(GoodCalc); + var range = await One(HistoryAggregateType.Range); + range.Value.ShouldBe(2d); + range.StatusCode.ShouldBe(UncertainCalc); + var minAt = await One(HistoryAggregateType.MinimumActualTime); + minAt.Value.ShouldBe(6d); + minAt.SourceTimestampUtc.ShouldBe(B.Round(At(2))); + minAt.StatusCode.ShouldBe(UncertainCalc); + var maxAt = await One(HistoryAggregateType.MaximumActualTime); + maxAt.Value.ShouldBe(8d); + maxAt.SourceTimestampUtc.ShouldBe(B.Round(At(3))); + maxAt.StatusCode.ShouldBe(GoodCalc); + // Count (#680, unchanged): the two Good samples, Uncertain. + (await One(HistoryAggregateType.Count)).Value.ShouldBe(2); + } + + [Fact] + public async Task An_uncertain_value_above_the_good_maximum_makes_the_maximum_family_uncertain() + { + // #684 review: pins the Uncertain-MAXIMUM column (UncMax / unc_max) through the real SQL — Good 6 @+2, Good 8 @+3, + // Uncertain 9 @+4 (above the Good maximum, nothing below the minimum). Part 13 Tables 58/60/62 ⇒ Maximum, + // MaximumActualTime and Range Uncertain (values still from the Good samples); Minimum Good. + await B.SeedAsync("UA", + (At(-5), HistorianValue.FromDouble(7), Good), + (At(2), HistorianValue.FromDouble(6), Good), + (At(3), HistorianValue.FromDouble(8), Good), + (At(4), HistorianValue.FromDouble(9), Uncertain), + (At(30), HistorianValue.FromDouble(7), Good)); + var src = B.CreateDataSource(); + async Task One(HistoryAggregateType a) => + (await src.ReadProcessedAsync("UA", At(0), At(20), Twenty, a, Ct)).Samples.Single(); + + var max = await One(HistoryAggregateType.Maximum); + max.Value.ShouldBe(8d); + max.StatusCode.ShouldBe(UncertainCalc); + var maxAt = await One(HistoryAggregateType.MaximumActualTime); + maxAt.Value.ShouldBe(8d); + maxAt.SourceTimestampUtc.ShouldBe(B.Round(At(3))); + maxAt.StatusCode.ShouldBe(UncertainCalc); + var range = await One(HistoryAggregateType.Range); + range.Value.ShouldBe(2d); + range.StatusCode.ShouldBe(UncertainCalc); + var min = await One(HistoryAggregateType.Minimum); + min.Value.ShouldBe(6d); + min.StatusCode.ShouldBe(GoodCalc); + } + + [Fact] + public async Task An_uncertain_string_sample_does_not_refuse_the_good_value_aggregates() + { + // #684 review: the Good-value aggregates refuse only a GOOD String/DateTime sample (the good-non-numeric count the + // SQL now returns); an Uncertain one is skipped with an Uncertain status — the SDK's behaviour. Delta keeps its + // usable-sample refusal. + await B.SeedAsync("US", + (At(-5), HistorianValue.FromDouble(7), Good), + (At(2), HistorianValue.FromDouble(6), Good), + (At(3), HistorianValue.FromString("x"), Uncertain), + (At(4), HistorianValue.FromDouble(8), Good), + (At(30), HistorianValue.FromDouble(7), Good)); + var src = B.CreateDataSource(); + async Task One(HistoryAggregateType a) => + (await src.ReadProcessedAsync("US", At(0), At(20), Twenty, a, Ct)).Samples.Single(); + + foreach (var (agg, value) in new[] + { + (HistoryAggregateType.Average, 7d), (HistoryAggregateType.Minimum, 6d), (HistoryAggregateType.Maximum, 8d), + (HistoryAggregateType.Range, 2d), (HistoryAggregateType.MinimumActualTime, 6d), + (HistoryAggregateType.MaximumActualTime, 8d), + }) + { + var v = await One(agg); + v.Value.ShouldBe(value, agg.ToString()); + v.StatusCode.ShouldBe(UncertainCalc, agg.ToString()); + } + (await One(HistoryAggregateType.Delta)).StatusCode.ShouldBe(InvalidInputsCalc); + + // A GOOD string in the same shape still refuses them. + await B.SeedAsync("GS", + (At(-5), HistorianValue.FromDouble(7), Good), + (At(2), HistorianValue.FromDouble(6), Good), + (At(3), HistorianValue.FromString("x"), Good), + (At(30), HistorianValue.FromDouble(7), Good)); + (await src.ReadProcessedAsync("GS", At(0), At(20), Twenty, HistoryAggregateType.Maximum, Ct)).Samples.Single() + .StatusCode.ShouldBe(InvalidInputsCalc); + (await src.ReadProcessedAsync("GS", At(0), At(20), Twenty, HistoryAggregateType.MinimumActualTime, Ct)).Samples + .Single().StatusCode.ShouldBe(InvalidInputsCalc); + } + + [Fact] + public async Task An_interval_of_only_uncertain_samples_is_BadNoData_for_the_good_value_aggregates() + { + // #684: Part 13 defines these over Good values; the SDK returns no-data when an interval has none. + await B.SeedAsync("UO", + (At(-5), HistorianValue.FromDouble(1), Good), + (At(1), HistorianValue.FromDouble(4), Uncertain), + (At(2), HistorianValue.FromDouble(6), Uncertain), + (At(30), HistorianValue.FromDouble(1), Good)); + var src = B.CreateDataSource(); + foreach (var agg in new[] + { + HistoryAggregateType.Average, HistoryAggregateType.Minimum, HistoryAggregateType.Maximum, + HistoryAggregateType.Range, HistoryAggregateType.MinimumActualTime, HistoryAggregateType.MaximumActualTime, + }) + { + var v = (await src.ReadProcessedAsync("UO", At(0), At(20), Twenty, agg, Ct)).Samples.Single(); + v.Value.ShouldBeNull(agg.ToString()); + v.StatusCode.ShouldBe(NoDataCalc, agg.ToString()); + } } [Fact] diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb.Tests/BucketRowMappingTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb.Tests/BucketRowMappingTests.cs index b029702f..d6fc98c6 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb.Tests/BucketRowMappingTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb.Tests/BucketRowMappingTests.cs @@ -107,10 +107,62 @@ public sealed class BucketRowMappingTests foreach (var c in new[] { "total", "usable", "good", "bad", "nonnum" }) t.Columns.Add(c, typeof(long)); t.Columns.Add("num_value", typeof(double)); t.Columns.Add("ts", typeof(DateTime)); - t.Rows.Add(0L, 1L, 1L, 1L, 0L, 0L, DBNull.Value, DBNull.Value); + t.Columns.Add("unc_min", typeof(double)); + t.Columns.Add("unc_max", typeof(double)); + t.Columns.Add("goodnonnum", typeof(long)); + t.Rows.Add(0L, 1L, 1L, 1L, 0L, 0L, DBNull.Value, DBNull.Value, 4d, DBNull.Value, 0L); var b = Map(t, AggregateFamily.ActualTime); b.Extreme.ShouldBeNull(); AggregateEvaluator.Evaluate(HistoryAggregateType.MaximumActualTime, b, T, T).StatusCode.ShouldBe(NoDataCalc); } + + [Fact] + public void A_simple_row_maps_the_good_value_columns_and_the_uncertain_extremes() + { + // #684: AvgValue/MinValue/MaxValue/SumValue are over Good samples; UncMin/UncMax (ordinals 10, 11) feed the Part 13 + // Minimum/Maximum/Range status rule. + var t = new DataTable(); + t.Columns.Add("b", typeof(long)); + foreach (var c in new[] { "total", "usable", "good", "bad", "nonnum" }) t.Columns.Add(c, typeof(long)); + foreach (var c in new[] { "avg", "min", "max", "sum", "uncmin", "uncmax" }) t.Columns.Add(c, typeof(double)); + t.Columns.Add("goodnonnum", typeof(long)); + t.Rows.Add(0L, 4L, 4L, 2L, 0L, 1L, 20d, 10d, 30d, 40d, 5d, 25d, 0L); + + var b = Map(t, AggregateFamily.Simple); + b.Avg.ShouldBe(20d); + b.Min.ShouldBe(10d); + b.Max.ShouldBe(30d); + b.Sum.ShouldBe(40d); + b.UncertainMin.ShouldBe(5d); + b.UncertainMax.ShouldBe(25d); + b.NonNumeric.ShouldBe(1); + b.GoodNonNumeric.ShouldBe(0); + // Uncertain 5 is below the Good minimum ⇒ Minimum Uncertain. The one non-numeric sample is Uncertain (ordinal 12 says + // no Good one), so Average is served — not refused — and Maximum is Uncertain for it (25 alone is inside the range). + AggregateEvaluator.Evaluate(HistoryAggregateType.Minimum, b, T, T).StatusCode.ShouldBe(0x40A40401u); + AggregateEvaluator.Evaluate(HistoryAggregateType.Average, b, T, T).Value.ShouldBe(20d); + AggregateEvaluator.Evaluate(HistoryAggregateType.Maximum, b, T, T).StatusCode.ShouldBe(0x40A40401u); + } + + [Fact] + public void An_actual_time_row_maps_the_uncertain_extremes_after_the_picked_sample() + { + var t = new DataTable(); + t.Columns.Add("b", typeof(long)); + foreach (var c in new[] { "total", "usable", "good", "bad", "nonnum" }) t.Columns.Add(c, typeof(long)); + t.Columns.Add("num_value", typeof(double)); + t.Columns.Add("ts", typeof(DateTime)); + t.Columns.Add("unc_min", typeof(double)); + t.Columns.Add("unc_max", typeof(double)); + t.Columns.Add("goodnonnum", typeof(long)); + t.Rows.Add(0L, 3L, 3L, 2L, 0L, 0L, 10d, T, 5d, 25d, 0L); + + var b = Map(t, AggregateFamily.ActualTime); + b.Extreme.ShouldBe(10d); + b.ExtremeTime.ShouldBe(T); + b.UncertainMin.ShouldBe(5d); + b.UncertainMax.ShouldBe(25d); + AggregateEvaluator.Evaluate(HistoryAggregateType.MinimumActualTime, b, T, T).StatusCode.ShouldBe(0x40A40401u); + } } diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common.Tests/AggregateEvaluatorTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common.Tests/AggregateEvaluatorTests.cs index 3c681fc7..5fdc4f5f 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common.Tests/AggregateEvaluatorTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common.Tests/AggregateEvaluatorTests.cs @@ -248,14 +248,136 @@ public sealed class AggregateEvaluatorTests ex.Aggregate.ShouldBe(HistoryAggregateType.TimeAverage); } - [Fact] - public void An_uncertain_sample_in_a_bucket_degrades_the_aggregate_to_uncertain() + /// + /// #684: a bucket of Good 10 and Good 30 plus the given Uncertain sample(s) (no Bad). The value columns are the Good + /// values only, which is what the backends' SQL now computes (the good flag, not the usable one). + /// + private static BucketRow GoodTenThirtyPlusUncertain(params double[] uncertain) => new() { - // Two usable samples, one Good and one Uncertain, no Bad: the value is computed, the status is Uncertain. - var row = new BucketRow { Total = 2, Usable = 2, Good = 1, Bad = 0, Avg = 15, Min = 10, Max = 20, Sum = 30 }; - var avg = AggregateEvaluator.Evaluate(HistoryAggregateType.Average, row, At(0), Now); - avg.Value.ShouldBe(15d); - avg.StatusCode.ShouldBe(UncertainCalc); + Total = 2 + uncertain.Length, Usable = 2 + uncertain.Length, Good = 2, Bad = 0, + Avg = 20, Min = 10, Max = 30, Sum = 40, + UncertainMin = uncertain.Length == 0 ? null : uncertain.Min(), + UncertainMax = uncertain.Length == 0 ? null : uncertain.Max(), + }; + + private static DataValueSnapshot Eval(HistoryAggregateType agg, BucketRow row) + { + if (agg is HistoryAggregateType.MinimumActualTime or HistoryAggregateType.MaximumActualTime) + { + row.Extreme = agg == HistoryAggregateType.MinimumActualTime ? row.Min : row.Max; + row.ExtremeTime = At(5); + } + return AggregateEvaluator.Evaluate(agg, row, At(0), Now); + } + + [Theory] + // Uncertain 20 sits INSIDE the Good range [10, 30]: Part 13 (and the SDK's ComputeMinMax) leave Min/Max/Range Good; + // Average is value-based (Good only when every sample is Good, the SDK's GetValueBasedStatusCode) => Uncertain. + [InlineData(HistoryAggregateType.Average, new[] { 20d }, 20d, false)] + [InlineData(HistoryAggregateType.Minimum, new[] { 20d }, 10d, true)] + [InlineData(HistoryAggregateType.Maximum, new[] { 20d }, 30d, true)] + [InlineData(HistoryAggregateType.Range, new[] { 20d }, 20d, true)] + [InlineData(HistoryAggregateType.MinimumActualTime, new[] { 20d }, 10d, true)] + [InlineData(HistoryAggregateType.MaximumActualTime, new[] { 20d }, 30d, true)] + // Equal to an extreme: the Part 13 comparison is strict ("less than" / "greater than") => still Good. + [InlineData(HistoryAggregateType.Minimum, new[] { 10d }, 10d, true)] + [InlineData(HistoryAggregateType.Maximum, new[] { 30d }, 30d, true)] + [InlineData(HistoryAggregateType.Range, new[] { 10d, 30d }, 20d, true)] + // Below the Good minimum: Minimum / MinimumActualTime / Range Uncertain, Maximum untouched; the value is still the + // Good extreme, never the Uncertain one (pre-#684 Minimum answered 5). + [InlineData(HistoryAggregateType.Minimum, new[] { 5d }, 10d, false)] + [InlineData(HistoryAggregateType.MinimumActualTime, new[] { 5d }, 10d, false)] + [InlineData(HistoryAggregateType.Maximum, new[] { 5d }, 30d, true)] + [InlineData(HistoryAggregateType.MaximumActualTime, new[] { 5d }, 30d, true)] + [InlineData(HistoryAggregateType.Range, new[] { 5d }, 20d, false)] + [InlineData(HistoryAggregateType.Average, new[] { 5d }, 20d, false)] + // Above the Good maximum: the mirror. + [InlineData(HistoryAggregateType.Maximum, new[] { 99d }, 30d, false)] + [InlineData(HistoryAggregateType.MaximumActualTime, new[] { 99d }, 30d, false)] + [InlineData(HistoryAggregateType.Minimum, new[] { 99d }, 10d, true)] + [InlineData(HistoryAggregateType.MinimumActualTime, new[] { 99d }, 10d, true)] + [InlineData(HistoryAggregateType.Range, new[] { 99d }, 20d, false)] + public void Uncertain_samples_are_not_part_of_the_value_and_degrade_the_status_per_Part_13(HistoryAggregateType agg, + double[] uncertain, double expected, bool good) + { + var v = Eval(agg, GoodTenThirtyPlusUncertain(uncertain)); + v.Value.ShouldBe(expected); + v.StatusCode.ShouldBe(good ? GoodCalc : UncertainCalc); + } + + [Theory] + [InlineData(HistoryAggregateType.Average)] + [InlineData(HistoryAggregateType.Minimum)] + [InlineData(HistoryAggregateType.Maximum)] + [InlineData(HistoryAggregateType.Range)] + [InlineData(HistoryAggregateType.MinimumActualTime)] + [InlineData(HistoryAggregateType.MaximumActualTime)] + public void Only_a_good_non_numeric_sample_refuses_a_good_value_aggregate(HistoryAggregateType agg) + { + // #684 review: an UNCERTAIN String/DateTime sample is outside the Good-value set — served (Uncertain), not refused. + var uncertainString = GoodTenThirtyPlusUncertain(); + uncertainString.Total = uncertainString.Usable = 3; + uncertainString.NonNumeric = 1; + uncertainString.GoodNonNumeric = 0; + var v = Eval(agg, uncertainString); + v.Value.ShouldNotBeNull(); + v.StatusCode.ShouldBe(UncertainCalc); + // A GOOD one still refuses the interval (the value set would hold a value with no numeric meaning). + var goodString = GoodTenThirtyPlusUncertain(); + goodString.Total = goodString.Usable = goodString.Good = 3; + goodString.NonNumeric = 1; + goodString.GoodNonNumeric = 1; + Eval(agg, goodString).StatusCode.ShouldBe(InvalidInputsCalc); + // A row that does not carry the Good count (null) keeps the stricter pre-#684 check: any usable one refuses. + uncertainString.GoodNonNumeric = null; + Eval(agg, uncertainString).StatusCode.ShouldBe(InvalidInputsCalc); + } + + [Fact] + public void Delta_still_refuses_any_usable_non_numeric_sample() + { + // Delta reads the first/last USABLE samples (#680), so an Uncertain String can be its input: unchanged by #684. + var row = new BucketRow + { + Total = 2, Usable = 2, Good = 1, NonNumeric = 1, GoodNonNumeric = 0, + First = HistorianValue.FromString("x"), Last = HistorianValue.FromDouble(1), FirstAny = A[0], LastAny = A[1], + }; + AggregateEvaluator.Evaluate(HistoryAggregateType.Delta, row, At(0), Now).StatusCode.ShouldBe(InvalidInputsCalc); + } + + [Theory] + [InlineData(HistoryAggregateType.Minimum)] + [InlineData(HistoryAggregateType.Maximum)] + [InlineData(HistoryAggregateType.Range)] + [InlineData(HistoryAggregateType.MinimumActualTime)] + [InlineData(HistoryAggregateType.MaximumActualTime)] + public void A_bad_sample_makes_the_min_max_family_uncertain_even_with_no_uncertain_value(HistoryAggregateType agg) + { + var row = GoodTenThirtyPlusUncertain(); + row.Total = 3; + row.Bad = 1; + Eval(agg, row).StatusCode.ShouldBe(UncertainCalc); + } + + [Theory] + [InlineData(HistoryAggregateType.Average)] + [InlineData(HistoryAggregateType.Minimum)] + [InlineData(HistoryAggregateType.Maximum)] + [InlineData(HistoryAggregateType.Range)] + [InlineData(HistoryAggregateType.MinimumActualTime)] + [InlineData(HistoryAggregateType.MaximumActualTime)] + public void An_interval_holding_only_uncertain_values_is_no_data(HistoryAggregateType agg) + { + // #684 (Part 13: these aggregates are defined over Good values; the SDK returns GetNoDataValue when none is Good). + // The SQL's Good-only value columns are NULL; the Uncertain extremes are not a substitute. + var row = new BucketRow { Total = 2, Usable = 2, Good = 0, UncertainMin = 4, UncertainMax = 6 }; + var v = AggregateEvaluator.Evaluate(agg, row, At(0), Now); + v.Value.ShouldBeNull(); + v.StatusCode.ShouldBe(NoDataCalc); + // Count is unchanged by #684: 0 Good samples, value-based Uncertain. + var count = AggregateEvaluator.Evaluate(HistoryAggregateType.Count, row, At(0), Now); + count.Value.ShouldBe(0); + count.StatusCode.ShouldBe(UncertainCalc); } // ---- Live-gate 8 F1: a row whose counts and picks disagree (a backend that read them from two table versions). diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common.Tests/SdkAggregateParityTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common.Tests/SdkAggregateParityTests.cs index eb45c4ed..598d6adb 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common.Tests/SdkAggregateParityTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common.Tests/SdkAggregateParityTests.cs @@ -16,9 +16,14 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common.Tests; /// /// Deliberate, documented divergences the datasets avoid (they are not anchoring questions): /// -/// Uncertain samples in a per-sample aggregate: the SDK's Average/Min/Max use Good samples only, -/// ours every usable (Good + Uncertain) sample (TreatUncertainAsBad = false applied consistently). Count matches -/// the SDK since #680 (Good samples only); +/// an Uncertain value beyond the Good extreme in Minimum / Maximum / Range / *ActualTime: Part 13 +/// (Tables 57-62) makes the result Uncertain_DataSubNormal, ours follows it; the SDK's ComputeMinMax +/// collects the Uncertain extremes and never reads them, so it answers Good. The value is the Good extreme in both +/// (#684). keeps its Uncertain values inside every interval's Good range so the +/// per-sample comparison needs no mask; the divergence itself is pinned by +/// . +/// (Uncertain samples otherwise match since #684: Average / Min / Max / Range / *ActualTime / Count all use Good +/// samples only, as the SDK does under the server's TreatUncertainAsBad = false); /// the per-sample Calculated / MultipleValues info bits (the SDK's MinMax sets Calculated only when the /// extreme is not at the interval start, and flags ties) — those comparisons check code bits + Partial only; /// a truncated interval: ours sets Partial on a non-Bad time-weighted result (Part 13 §5.3.3.2 Case 3 @@ -66,6 +71,20 @@ public sealed class SdkAggregateParityTests .Select(x => S(x.t, HistorianValue.FromDouble(10 + (x.i * 7 % 13)), x.t == 22.5 ? Bad : Good)) .ToArray(); + /// + /// #684: with every fifth sample (from +2.5) Uncertain at 16 and the one Bad at +22.5. The cycling + /// Good values alternate around 16 (every adjacent pair has one at or below it and one at or above it), so an + /// Uncertain 16 lies inside the Good range of every interval of every window (each holds 5+ samples): Average / + /// Minimum / Maximum / Range / *ActualTime must leave it out of the value (Good only — the SDK, Part 13), and the + /// min/max family's status matches the SDK's without a mask. + /// + private static readonly StoredSample[] DenseUncertain = Enumerable.Range(0, 45) + .Select(i => (t: -10 + i * 2.5, i)) + .Select(x => x.i % 5 == 0 && x.t != 22.5 + ? S(x.t, HistorianValue.FromDouble(16), Uncertain) + : S(x.t, HistorianValue.FromDouble(10 + (x.i * 7 % 13)), x.t == 22.5 ? Bad : Good)) + .ToArray(); + /// Tag A with Good bounds on both sides, plus an Uncertain sample and an outage marker further on. private static readonly StoredSample[] Bounded = [ @@ -302,11 +321,12 @@ public sealed class SdkAggregateParityTests { var failures = new List(); foreach (var agg in PerSample) + foreach (var (name, data) in new[] { ("Dense", Dense), ("DenseUncertain", DenseUncertain) }) { try { - SdkAggregateOracle.ShouldMatch(Ours(agg, from, to, intervalS, Dense), Sdk(agg, from, to, intervalS, Dense), - $"{agg} Dense {from}→{to}/{intervalS}s", compareCalculatedInterpolatedBits: false, + SdkAggregateOracle.ShouldMatch(Ours(agg, from, to, intervalS, data), Sdk(agg, from, to, intervalS, data), + $"{agg} {name} {from}→{to}/{intervalS}s", compareCalculatedInterpolatedBits: false, compareValues: true); } catch (Exception ex) @@ -317,6 +337,131 @@ public sealed class SdkAggregateParityTests failures.ShouldBeEmpty(string.Join("\n", failures)); } + [Theory] + [MemberData(nameof(Windows))] + public void The_uncertain_dataset_is_not_vacuous(double from, double to, double intervalS) + { + // #684: DenseUncertain must discriminate the pre-#684 behaviour in every window, or its parity proves nothing. + // (1) VALUES: Average over the usable (Good + Uncertain) samples differs from the SDK's Good-only Average in some + // interval. (2) STATUS: Minimum / Maximum hold an Uncertain sample in some interval the SDK still calls Good (16 sits + // inside the Good range) — the pre-#684 "any non-Good sample ⇒ Uncertain" rule would answer Uncertain there. + // (Minimum / Maximum VALUES do not discriminate here: an in-range 16 never is the extreme — by design, so the + // min/max family needs no mask; the value half is pinned by UncertainBeyond and AggregateEvaluatorTests.) + var plan = BucketPlan.Create(At(from), At(to), TimeSpan.FromSeconds(intervalS)); + // The one-interval windows hold the Bad +22.5 sample, so the SDK is Uncertain there for a reason the old rule shares: + // the status half is checked on the multi-interval windows only (the value half below covers every window). + foreach (var agg in plan.BucketCount > 1 ? new[] { HistoryAggregateType.Minimum, HistoryAggregateType.Maximum } : []) + { + var sdkMinMax = Sdk(agg, from, to, intervalS, DenseUncertain); + var rows = InMemoryRows(plan, DenseUncertain).ToList(); + var holdsUncertain = (plan.Reversed ? Enumerable.Reverse(rows) : rows).Select(r => r.Usable > r.Good).ToList(); + holdsUncertain.Zip(sdkMinMax).Any(p => p.First && (p.Second.StatusCode & 0xFFFF0000u) == 0) + .ShouldBeTrue($"{agg}: some interval holds an Uncertain sample yet is Good in the SDK"); + } + var usableAverage = InMemoryRows(plan, DenseUncertain.Select(x => x.IsUsable && !x.IsGood + ? x with { Status = Good } : x)) + .Select(r => r.Avg).ToList(); + var sdk = Sdk(HistoryAggregateType.Average, from, to, intervalS, DenseUncertain).Select(r => r.Value).ToList(); + var ours = plan.Reversed ? Enumerable.Reverse(usableAverage).ToList() : usableAverage; + ours.Zip(sdk).Any(p => p.Second is double d && p.First is double u && Math.Abs(d - u) > 1e-9) + .ShouldBeTrue("an Uncertain sample must change some interval's Average when it is (wrongly) included"); + } + + /// #684: Good 10 / 30 with an Uncertain value BELOW the Good minimum (+4) and one ABOVE the maximum (+6). + private static readonly StoredSample[] UncertainBeyond = + [ + S(-5, HistorianValue.FromDouble(20), Good), + S(2, HistorianValue.FromDouble(10), Good), + S(4, HistorianValue.FromDouble(5), Uncertain), + S(6, HistorianValue.FromDouble(99), Uncertain), + S(8, HistorianValue.FromDouble(30), Good), + S(25, HistorianValue.FromDouble(20), Good), + ]; + + [Theory] + [InlineData(HistoryAggregateType.Minimum, 0, 20)] + [InlineData(HistoryAggregateType.Minimum, 20, 0)] + [InlineData(HistoryAggregateType.Maximum, 0, 20)] + [InlineData(HistoryAggregateType.Maximum, 20, 0)] + [InlineData(HistoryAggregateType.Range, 0, 20)] + [InlineData(HistoryAggregateType.Range, 20, 0)] + [InlineData(HistoryAggregateType.MinimumActualTime, 0, 20)] + [InlineData(HistoryAggregateType.MaximumActualTime, 20, 0)] + public void An_uncertain_value_beyond_the_good_extreme_is_Uncertain_per_Part_13_where_the_SDK_says_Good( + HistoryAggregateType agg, double from, double to) + { + // #684, documented divergence (class remarks). Part 13 Tables 57-62: "If an Uncertain value is less than the + // minimum Good value [greater than the maximum] the Status is Uncertain_DataSubNormal". The SDK's ComputeMinMax + // tracks the Uncertain extremes and never reads them. Everything else — the value (the GOOD extreme, never 5 or + // 99), the timestamp, the Partial bit — matches the SDK; only the code bits differ, and only as Part 13 says. + var ours = Ours(agg, from, to, 0, UncertainBeyond).Single(); + var sdk = Sdk(agg, from, to, 0, UncertainBeyond).Single(); + (sdk.StatusCode & StatusBits.CodeBitsMask).ShouldBe(HistorianStatusCodes.Good, "the SDK ignores the Uncertain extreme"); + (ours.StatusCode & StatusBits.CodeBitsMask).ShouldBe(HistorianStatusCodes.UncertainDataSubNormal); + SdkAggregateOracle.ShouldMatch([ours with { StatusCode = ours.StatusCode & ~StatusBits.CodeBitsMask }], [sdk], + $"{agg} UncertainBeyond {from}→{to}", compareCalculatedInterpolatedBits: false); + ours.Value.ShouldBe(agg switch + { + HistoryAggregateType.Minimum or HistoryAggregateType.MinimumActualTime => 10d, + HistoryAggregateType.Maximum or HistoryAggregateType.MaximumActualTime => 30d, + _ => 20d, + }); + } + + /// #684 review: Good numeric samples with an Uncertain STRING among them (+4). + private static readonly StoredSample[] UncertainString = + [ + S(-5, HistorianValue.FromDouble(20), Good), + S(2, HistorianValue.FromDouble(10), Good), + S(4, HistorianValue.FromString("x"), Uncertain), + S(8, HistorianValue.FromDouble(30), Good), + S(25, HistorianValue.FromDouble(20), Good), + ]; + + [Theory] + [InlineData(0, 20)] + [InlineData(20, 0)] + public void An_uncertain_non_numeric_sample_is_skipped_by_the_good_value_aggregates_like_the_SDK(double from, double to) + { + // #684 review: an Uncertain String is not part of the Good-value set, so it no longer refuses the interval + // (BadAggregateInvalidInputs); the SDK's Average skips it (value-based status: Uncertain) and its ComputeMinMax flags + // the value it cannot cast (Uncertain). Exact match, every bit but Calculated. + var failures = new List(); + foreach (var agg in new[] + { + HistoryAggregateType.Average, HistoryAggregateType.Minimum, HistoryAggregateType.Maximum, + HistoryAggregateType.Range, HistoryAggregateType.MinimumActualTime, HistoryAggregateType.MaximumActualTime, + }) + { + try + { + var ours = Ours(agg, from, to, 0, UncertainString); + SdkAggregateOracle.ShouldMatch(ours, Sdk(agg, from, to, 0, UncertainString), $"{agg} UncertainString {from}→{to}", + compareCalculatedInterpolatedBits: false); + (ours.Single().StatusCode & StatusBits.CodeBitsMask).ShouldBe(HistorianStatusCodes.UncertainDataSubNormal, agg.ToString()); + } + catch (Exception ex) + { + failures.Add(ex.Message); + } + } + failures.ShouldBeEmpty(string.Join("\n", failures)); + // Delta keeps its usable picks and refusal (#680, unchanged): no Good-only reading applies to it. + Ours(HistoryAggregateType.Delta, from, to, 0, UncertainString).Single().StatusCode + .ShouldBe(HistorianInfoBits.WithCalculated(HistorianStatusCodes.BadAggregateInvalidInputs)); + } + + [Theory] + [InlineData(0, 20)] + [InlineData(20, 0)] + public void Average_over_uncertain_values_beyond_the_good_range_matches_the_SDK_exactly(double from, double to) + { + // #684: Average has no extreme rule — Good values only, value-based status (Uncertain: not every sample is Good). + SdkAggregateOracle.ShouldMatch(Ours(HistoryAggregateType.Average, from, to, 0, UncertainBeyond), + Sdk(HistoryAggregateType.Average, from, to, 0, UncertainBeyond), $"Average UncertainBeyond {from}→{to}"); + Ours(HistoryAggregateType.Average, from, to, 0, UncertainBeyond).Single().Value.ShouldBe(20d); + } + private static readonly HistoryAggregateType[] PerSample = [ HistoryAggregateType.Average, HistoryAggregateType.Minimum, HistoryAggregateType.Maximum, @@ -544,8 +689,11 @@ public sealed class SdkAggregateParityTests var usable = all.Where(s => s.IsUsable).ToList(); var numeric = usable.Where(s => s.Value.Kind is HistorianValueKind.Double or HistorianValueKind.Int64 or HistorianValueKind.Boolean).ToList(); - // ActualTime: the extreme usable value, ties ⇒ earliest (the SQL's ROW_NUMBER order). - var extreme = numeric.OrderBy(s => maximum ? -s.Value.Number : s.Value.Number).ThenBy(s => s.TimeUtc) + // #684: the value columns are over Good numeric samples; the Uncertain extremes feed the Min/Max status rule. + var good = numeric.Where(s => s.IsGood).ToList(); + var uncertain = numeric.Where(s => !s.IsGood).ToList(); + // ActualTime: the extreme Good value, ties ⇒ earliest (the SQL's ROW_NUMBER order). + var extreme = good.OrderBy(s => maximum ? -s.Value.Number : s.Value.Number).ThenBy(s => s.TimeUtc) .Cast().FirstOrDefault(); yield return new BucketRow { @@ -555,10 +703,13 @@ public sealed class SdkAggregateParityTests Good = all.Count(s => s.IsGood), Bad = all.Count - usable.Count, NonNumeric = usable.Count - numeric.Count, - Avg = numeric.Count == 0 ? null : numeric.Average(s => s.Value.Number), - Min = numeric.Count == 0 ? null : numeric.Min(s => s.Value.Number), - Max = numeric.Count == 0 ? null : numeric.Max(s => s.Value.Number), - Sum = numeric.Count == 0 ? null : numeric.Sum(s => s.Value.Number), + GoodNonNumeric = usable.Count(s => s.IsGood) - numeric.Count(s => s.IsGood), + Avg = good.Count == 0 ? null : good.Average(s => s.Value.Number), + Min = good.Count == 0 ? null : good.Min(s => s.Value.Number), + Max = good.Count == 0 ? null : good.Max(s => s.Value.Number), + Sum = good.Count == 0 ? null : good.Sum(s => s.Value.Number), + UncertainMin = uncertain.Count == 0 ? null : uncertain.Min(s => s.Value.Number), + UncertainMax = uncertain.Count == 0 ? null : uncertain.Max(s => s.Value.Number), First = usable.Count == 0 ? HistorianValue.Empty : usable[0].Value, Last = usable.Count == 0 ? HistorianValue.Empty : usable[^1].Value, Extreme = extreme?.Value.Number, diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer.Tests/BucketRowMappingTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer.Tests/BucketRowMappingTests.cs index 9e211022..24f40663 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer.Tests/BucketRowMappingTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer.Tests/BucketRowMappingTests.cs @@ -103,10 +103,62 @@ public sealed class BucketRowMappingTests foreach (var c in new[] { "total", "usable", "good", "bad", "nonnum" }) t.Columns.Add(c, typeof(int)); t.Columns.Add("NumValue", typeof(double)); t.Columns.Add("SourceTimeUtc", typeof(DateTime)); - t.Rows.Add(0L, 1, 1, 1, 0, 0, DBNull.Value, DBNull.Value); + t.Columns.Add("UncMin", typeof(double)); + t.Columns.Add("UncMax", typeof(double)); + t.Columns.Add("goodnonnum", typeof(int)); + t.Rows.Add(0L, 1, 1, 1, 0, 0, DBNull.Value, DBNull.Value, 4d, DBNull.Value, 0); var b = Map(t, AggregateFamily.ActualTime); b.Extreme.ShouldBeNull(); AggregateEvaluator.Evaluate(HistoryAggregateType.MaximumActualTime, b, T, T).StatusCode.ShouldBe(NoDataCalc); } + + [Fact] + public void A_simple_row_maps_the_good_value_columns_and_the_uncertain_extremes() + { + // #684: AvgValue/MinValue/MaxValue/SumValue are over Good samples; UncMin/UncMax (ordinals 10, 11) feed the Part 13 + // Minimum/Maximum/Range status rule. + var t = new DataTable(); + t.Columns.Add("b", typeof(long)); + foreach (var c in new[] { "total", "usable", "good", "bad", "nonnum" }) t.Columns.Add(c, typeof(int)); + foreach (var c in new[] { "avg", "min", "max", "sum", "uncmin", "uncmax" }) t.Columns.Add(c, typeof(double)); + t.Columns.Add("goodnonnum", typeof(int)); + t.Rows.Add(0L, 4, 4, 2, 0, 1, 20d, 10d, 30d, 40d, 5d, 25d, 0); + + var b = Map(t, AggregateFamily.Simple); + b.Avg.ShouldBe(20d); + b.Min.ShouldBe(10d); + b.Max.ShouldBe(30d); + b.Sum.ShouldBe(40d); + b.UncertainMin.ShouldBe(5d); + b.UncertainMax.ShouldBe(25d); + b.NonNumeric.ShouldBe(1); + b.GoodNonNumeric.ShouldBe(0); + // Uncertain 5 is below the Good minimum ⇒ Minimum Uncertain. The one non-numeric sample is Uncertain (ordinal 12 says + // no Good one), so Average is served — not refused — and Maximum is Uncertain for it (25 alone is inside the range). + AggregateEvaluator.Evaluate(HistoryAggregateType.Minimum, b, T, T).StatusCode.ShouldBe(0x40A40401u); + AggregateEvaluator.Evaluate(HistoryAggregateType.Average, b, T, T).Value.ShouldBe(20d); + AggregateEvaluator.Evaluate(HistoryAggregateType.Maximum, b, T, T).StatusCode.ShouldBe(0x40A40401u); + } + + [Fact] + public void An_actual_time_row_maps_the_uncertain_extremes_after_the_picked_sample() + { + var t = new DataTable(); + t.Columns.Add("b", typeof(long)); + foreach (var c in new[] { "total", "usable", "good", "bad", "nonnum" }) t.Columns.Add(c, typeof(int)); + t.Columns.Add("NumValue", typeof(double)); + t.Columns.Add("SourceTimeUtc", typeof(DateTime)); + t.Columns.Add("UncMin", typeof(double)); + t.Columns.Add("UncMax", typeof(double)); + t.Columns.Add("goodnonnum", typeof(int)); + t.Rows.Add(0L, 3, 3, 2, 0, 0, 10d, T, 5d, 25d, 0); + + var b = Map(t, AggregateFamily.ActualTime); + b.Extreme.ShouldBe(10d); + b.ExtremeTime.ShouldBe(T); + b.UncertainMin.ShouldBe(5d); + b.UncertainMax.ShouldBe(25d); + AggregateEvaluator.Evaluate(HistoryAggregateType.MinimumActualTime, b, T, T).StatusCode.ShouldBe(0x40A40401u); + } }