diff --git a/CLAUDE.md b/CLAUDE.md index e0d26f66..669bcb1d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1350,7 +1350,7 @@ historian tagname, which is the tag's driver `FullName`. ### Pluggable backends, SQL Server and HistoryUpdate (2026-10) - **Selection:** `ServerHistorian:Backend` (`Gateway` default | `SqlServer` | `QuestDb`; unknown fails host start). `HistorianBackendRegistry` → `IHistorianBackendProvider` builds all five seams; the gateway keys are only required for `Gateway`. `SqlServer` uses `ServerHistorian:SqlServer:{ConnectionString (required), Schema=hist, CommandTimeout=30s, AutoCreateSchema=true}`. -- **SQL Server** (`Driver.Historian.SqlServer`): `hist.Sample` PK `(TagId, SourceTimeUtc)` — writes are insert-if-absent, so outbox redelivery is idempotent. Stores every value kind with its **real status**; per-sample aggregates (10, simple definitions) are pushed down to T-SQL with bucket math in **100 ns ticks**; the Part 13 time-weighted seven (TimeAverage/TimeAverage2/Total/Total2/Interpolative, #671 — ⚠️ `Total` is now the time-weighted integral, no longer `SUM` — plus the **duration-based** PercentGood/PercentBad, #680) stream window + 4 bound rows into the shared `TimeWeightedAggregator` (both SQL backends, identical results; reverse `Interpolative` anchored at the later time; window capped by `ServerHistorian:MaxTimeWeightedRawRows` ⇒ `BadTooManyOperations`, percentages included). ⚠️ **#680 status rules (client-visible):** TimeAverage/Total are Uncertain if any part of the interval uses an Uncertain value, is bridged across a Bad sample, is extrapolated past the last sample (⇒ a live tag's latest interval is Uncertain) or is not covered; TimeAverage2/Total2 keep the SDK time-based rule (Uncertain time counts as Good) but never extrapolate past the last sample — that interval is Partial, later ones `BadNoData` (a report-by-exception tag with nothing after its last sample reads `BadNoData` until its next sample arrives). `docs/Historian.md` §"Time-weighted aggregates". ⚠️ **#680 per-sample (client-visible):** `Count` counts **Good** samples only (value-based status; an all-Bad interval is generic Bad, value null) and an empty interval **inside the tag's data range** is `0`/Good (outside it still `BadNoData`); `Start`/`End` return the **raw** first/last sample of any quality — its own timestamp and status, value null when Bad; `Delta` is Uncertain only when the interval's first or last sample is unusable; ⚠️ #684: `Average`/`Minimum`/`Maximum`/`Range`/`*ActualTime` use **Good samples only** (was Good + Uncertain; an Uncertain-only interval is `BadNoData`), Min/Max-family status Uncertain on a Bad sample or an Uncertain value beyond the Good extreme (Part 13 — the SDK ignores the latter); and every per-sample aggregate but `Average` sets **end-of-data Partial** on the interval holding the last sample (forward incl. its earlier edge, reverse strictly inside), never on a Bad result. The data-range lookup is a seek on SQL Server but a partition scan on QuestDB (unindexed `tag` SYMBOL). `docs/Historian.md` §"Aggregates". Docker-dev recipe, schema and aggregate semantics: `docs/Historian.md` §"SQL Server backend". Integration tests need `HISTORIAN_SQL_TEST_CONNECTION`. **Retention (#667):** opt-in `SqlServer:RetentionDays` (+ `PurgeInterval` 1h, `PurgeBatchSize` 4000) -> `HistorianRetentionService` purges old `hist.Sample` (per tag) / `hist.Event` in `DELETE TOP(n)` batches, Primary-gated on `IHistorizationRoleView`, one purger per DB via `sp_getapplock`; counter `otopcua.historian.retention.purged{table}`. Optional `IHistorianBackendProvider.CreateRetention` (default null). `docs/Historian.md` §Retention. +- **SQL Server** (`Driver.Historian.SqlServer`): `hist.Sample` PK `(TagId, SourceTimeUtc)` — writes are insert-if-absent, so outbox redelivery is idempotent. Stores every value kind with its **real status**; per-sample aggregates (10, simple definitions) are pushed down to T-SQL with bucket math in **100 ns ticks**; the Part 13 time-weighted seven (TimeAverage/TimeAverage2/Total/Total2/Interpolative, #671 — ⚠️ `Total` is now the time-weighted integral, no longer `SUM` — plus the **duration-based** PercentGood/PercentBad, #680) stream window + 4 bound rows into the shared `TimeWeightedAggregator` (both SQL backends, identical results; reverse `Interpolative` anchored at the later time; window capped by `ServerHistorian:MaxTimeWeightedRawRows` ⇒ `BadTooManyOperations`, percentages included). ⚠️ **#680 status rules (client-visible):** TimeAverage/Total are Uncertain if any part of the interval uses an Uncertain value, is bridged across a Bad sample, is extrapolated past the last sample (⇒ a live tag's latest interval is Uncertain) or is not covered; TimeAverage2/Total2 keep the SDK time-based rule (Uncertain time counts as Good) but never extrapolate past the last sample — that interval is Partial, later ones `BadNoData` (a report-by-exception tag with nothing after its last sample reads `BadNoData` until its next sample arrives). `docs/Historian.md` §"Time-weighted aggregates". ⚠️ **#680 per-sample (client-visible):** `Count` counts **Good** samples only (value-based status; an all-Bad interval is generic Bad, value null) and an empty interval **inside the tag's data range** is `0`/Good (outside it still `BadNoData`); `Start`/`End` return the **raw** first/last sample of any quality — its own timestamp and status, value null when Bad; `Delta` is Uncertain only when the interval's first or last sample is unusable; ⚠️ #684: `Average`/`Minimum`/`Maximum`/`Range`/`*ActualTime` use **Good samples only** (was Good + Uncertain; an Uncertain-only interval is `BadNoData`), Min/Max-family status Uncertain on a Bad sample, an Uncertain value beyond the Good extreme (Part 13 — the SDK ignores that) or an Uncertain non-numeric sample, and only a **Good** String/DateTime sample refuses them (`BadAggregateInvalidInputs`; Delta still refuses any usable one); and every per-sample aggregate but `Average` sets **end-of-data Partial** on the interval holding the last sample (forward incl. its earlier edge, reverse strictly inside), never on a Bad result. The data-range lookup is a seek on SQL Server but a partition scan on QuestDB (unindexed `tag` SYMBOL). `docs/Historian.md` §"Aggregates". Docker-dev recipe, schema and aggregate semantics: `docs/Historian.md` §"SQL Server backend". Integration tests need `HISTORIAN_SQL_TEST_CONNECTION`. **Retention (#667):** opt-in `SqlServer:RetentionDays` (+ `PurgeInterval` 1h, `PurgeBatchSize` 4000) -> `HistorianRetentionService` purges old `hist.Sample` (per tag) / `hist.Event` in `DELETE TOP(n)` batches, Primary-gated on `IHistorizationRoleView`, one purger per DB via `sp_getapplock`; counter `otopcua.historian.retention.purged{table}`. Optional `IHistorianBackendProvider.CreateRetention` (default null). `docs/Historian.md` §Retention. - **QuestDB** (`Driver.Historian.QuestDb`, `ServerHistorian:QuestDb:*`, Npgsql over PG wire; docker-dev overlay `docker-compose.questdb.yml`): same kinds/status/aggregates/HistoryUpdate as SqlServer, soft delete by tombstone (INSERT only — never `UPDATE`/`DELETE`; every read filters `NOT deleted`). ⚠️ **Timestamps are µs** (`QuestDbConnection.Round`/`BindTimestamp`; `IHistorianDataSource.TimestampResolutionTicks` = 10 steps HistoryRead paging), **pre-1970 cannot be stored** (writer drops, HistoryUpdate `BadOutOfRange`), every write waits for WAL visibility, TTL is enforced on write commits only, set Npgsql `Maximum Pool Size` < QuestDB's 64. `docs/Historian.md` §"QuestDB backend". - ⚠️ **Reverse processed reads are Part 13/SDK-anchored (#677, client-visible):** every aggregate on SqlServer/QuestDB now anchors a reverse read at its later time — window `(end, start]`, partial interval earliest, each stamped with its later edge (was: earlier-anchored, stamped earlier edge); on forward reads Partial also marks the interval the data starts inside (not Average — the SDK's rule, forward-only); a non-default client `AggregateConfiguration` ⇒ `BadAggregateConfigurationRejected`; Gateway provisioning sends `IntegralDivisor = 1`. SDK-parity tests: `SdkAggregateParityTests`. `docs/Historian.md` §Aggregates. - **Advertised aggregates (#682):** `IHistorianDataSource.SupportedAggregates` (Gateway 4 via `GatewayHistorianDataSource.ServedAggregates`, SqlServer/QuestDB 17 via `AggregateFamilies.Served`, Null **none**; a null set ⇒ all) is the one authority for `HistoryServerCapabilities/AggregateFunctions` (i=11201 — only its folder references are trimmed) and the up-front `BadAggregateNotSupported` in HistoryRead-Processed. ⚠️ `ServerCapabilities/AggregateFunctions` (i=2997) is deliberately NOT trimmed: it keeps the SDK's 37 because live `AggregateFilter` subscriptions still compute them all. ⚠️ Historian disabled ⇒ Processed is now `BadAggregateNotSupported`, not `GoodNoData`. `docs/Historian.md` §Advertised aggregates. diff --git a/docs/Historian.md b/docs/Historian.md index e2fe68ac..0fd60e08 100644 --- a/docs/Historian.md +++ b/docs/Historian.md @@ -1013,6 +1013,10 @@ point (see [Continuation-point paging (Events)](#continuation-point-paging-event > - The SDK applies only the Bad half of that rule (its `ComputeMinMax` collects the Uncertain extremes and never reads them) and answers > Good for an Uncertain value beyond the extreme; ours follows Part 13 there — the one remaining per-sample Uncertain difference, pinned > by `SdkAggregateParityTests`. +> - **Non-numeric samples:** these aggregates now answer `BadAggregateInvalidInputs` only for a **Good** `String`/`DateTime` sample +> in the interval. An **Uncertain** one is skipped like any Uncertain value — the interval is served from its Good numeric values +> with `UncertainDataSubNormal` (the SDK's Average skips it; its Min/Max flags the value it cannot cast). `Delta` reads usable +> samples and still refuses any usable non-numeric one. > - `Count` (Good only, #680), `Start`/`End`/`Delta` (usable / any-quality picks) and the time-weighted family (Uncertain time counts as > Good time under `TreatUncertainAsBad = false`, as Part 13 §5.4.3.2 says) are unchanged. The advertised `TreatUncertainAsBad = false` > stays correct: it governs the time-weighted and status calculations; the per-sample aggregates above are defined over Good values @@ -1075,7 +1079,7 @@ Status and edge rules: - An **empty interval** (no samples) ⇒ `BadNoData` — except **`Count`**, which is `0`, Good, when the interval lies inside the tag's data range `[first sample, last sample]` (both edges inclusive, either direction; #680). **Every value aggregate over an interval whose samples are all Bad ⇒ `BadNoData`**; `Count` there is the generic **Bad** (value null). - An interval with samples but **no Good numeric value** (only Uncertain / Bad) ⇒ `BadNoData` for `Average`/`Minimum`/`Maximum`/`Range`/`*ActualTime` (#684). - `Average` is `Good` when every sample in the bucket is Good, else `UncertainDataSubNormal` (value-based, the SDK's rule). `Minimum`/`Maximum`/`Range`/`*ActualTime` are `UncertainDataSubNormal` when the bucket holds a Bad sample or an Uncertain value beyond the reported Good extreme (Part 13 Tables 57–62; strictly beyond), else `Good` (#684). `Count`'s status is value-based (all Good ⇒ Good, all Bad ⇒ Bad, else Uncertain); `Start`/`End` carry the raw sample's own status; `Delta` is Good unless the interval's first or last sample is not usable (then Uncertain) — #680. -- `Count`, `Start`, `End` accept any value kind; the other aggregates (and `Delta`) on a bucket that holds a usable `String`/`DateTime` sample ⇒ `BadAggregateInvalidInputs` for that interval. +- `Count`, `Start`, `End` accept any value kind. `Average`/`Minimum`/`Maximum`/`Range`/`*ActualTime` on a bucket that holds a **Good** `String`/`DateTime` sample ⇒ `BadAggregateInvalidInputs` for that interval; an Uncertain one is skipped and makes `Minimum`/`Maximum`/`Range`/`*ActualTime` `UncertainDataSubNormal` (#684). `Delta` is refused by any usable non-numeric sample. - `PercentGood` / `PercentBad` are **duration-based since #680** — see [Time-weighted aggregates](#time-weighted-aggregates-part-13-671). - Everything except `Count` and `Start`/`End` is a `double`; `Int64` samples aggregate as doubles and `Boolean` as 0/1. diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb/QuestDbAggregateQueries.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb/QuestDbAggregateQueries.cs index ef33b30d..a7e6f2c9 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb/QuestDbAggregateQueries.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb/QuestDbAggregateQueries.cs @@ -60,7 +60,7 @@ internal static class QuestDbAggregateQueries /// Average / Minimum / Maximum / Count / Range (PercentGood / PercentBad and Total are time-weighted, not here). /// Family columns: 6 avg_value, 7 min_value, 8 max_value, 9 sum_value over the Good numeric samples /// (DOUBLE, NULL when none — #684, Part 13 / the SDK), then 10 unc_min, 11 unc_max over the Uncertain ones (the - /// Part 13 Minimum/Maximum/Range status rule). + /// Part 13 Minimum/Maximum/Range status rule) and 12 good_non_numeric_count. /// /// Prefixed sample table name. /// The SQL text. @@ -131,7 +131,7 @@ internal static class QuestDbAggregateQueries /// /// MinimumActualTime / MaximumActualTime: the Good sample (#684) with the smallest (largest) num_value, /// ties broken by the earliest ts. Family columns: 6 num_value, 7 ts (NULL when the bucket has no Good - /// numeric sample), then 8 unc_min, 9 unc_max as in . + /// numeric sample), then 8 unc_min, 9 unc_max, 10 good_non_numeric_count as in . /// /// /// Deliberately still two references to the classified rows (counts + the ranked extreme), unlike @@ -151,7 +151,7 @@ internal static class QuestDbAggregateQueries "row_number() OVER (PARTITION BY b ORDER BY num_value " + (maximum ? "DESC" : "ASC") + ", ts ASC) AS rn " + "FROM s WHERE g = 1 AND num_value IS NOT NULL) " + "SELECT st.b, st.total_count, st.usable_count, st.good_count, st.bad_count, st.non_numeric_count, " + - "r.num_value, r.ts, st.unc_min, st.unc_max " + + "r.num_value, r.ts, st.unc_min, st.unc_max, st.good_non_numeric_count " + "FROM st LEFT JOIN (SELECT * FROM r WHERE rn = 1) r ON r.b = st.b ORDER BY st.b"; /// @@ -224,9 +224,11 @@ internal static class QuestDbAggregateQueries /// /// The minimum and maximum Uncertain (usable, not Good) numeric value of the bucket — unc_min, unc_max /// (#684). Not result values: Part 13 makes Minimum/Maximum/Range Uncertain when one lies beyond the Good extreme. + /// Then good_non_numeric_count, the Good String/DateTime samples (the Good-value aggregates' refusal check). /// private static string UncertainExtremes() => - "min(CASE WHEN u = 1 AND g = 0 THEN num_value END) AS unc_min, max(CASE WHEN u = 1 AND g = 0 THEN num_value END) AS unc_max"; + "min(CASE WHEN u = 1 AND g = 0 THEN num_value END) AS unc_min, max(CASE WHEN u = 1 AND g = 0 THEN num_value END) AS unc_max, " + + $"sum(CASE WHEN g = 1 AND kind IN ({NonNumericKinds}) THEN 1 ELSE 0 END) AS good_non_numeric_count"; /// The five common count columns (after b). private static string CommonCounts() => diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb/QuestDbHistorianDataSource.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb/QuestDbHistorianDataSource.cs index f3f3e944..53f134a3 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb/QuestDbHistorianDataSource.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb/QuestDbHistorianDataSource.cs @@ -619,6 +619,7 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource row.Sum = D(reader, f + 3); row.UncertainMin = D(reader, f + 4); row.UncertainMax = D(reader, f + 5); + row.GoodNonNumeric = N(reader, f + 6); break; case AggregateFamily.StartEnd: row.First = V(reader, f); @@ -631,6 +632,7 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource row.ExtremeTime = reader.IsDBNull(f + 1) ? null : QuestDbConnection.ReadUtc(reader, f + 1); row.UncertainMin = D(reader, f + 2); row.UncertainMax = D(reader, f + 3); + row.GoodNonNumeric = N(reader, f + 4); break; } return row; diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common/AggregateEvaluator.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common/AggregateEvaluator.cs index f651d772..97b7baa4 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common/AggregateEvaluator.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common/AggregateEvaluator.cs @@ -74,7 +74,11 @@ public static class AggregateEvaluator return Calc(r.Good, r.Good == r.Total ? HistorianStatusCodes.Good : HistorianStatusCodes.UncertainDataSubNormal); } if (r.Usable == 0) return NoData(bucketStart, now); - if (r.NonNumeric > 0) return Calc(null, HistorianStatusCodes.BadAggregateInvalidInputs); + // #684 review: Delta reads usable picks, so any usable String/DateTime sample refuses it. The Good-value aggregates + // refuse only a GOOD non-numeric sample — an Uncertain one is not part of their value set (it only degrades the + // status, as the SDK's skipped/uncastable value does). + if ((aggregate == HistoryAggregateType.Delta ? r.NonNumeric : r.GoodNonNumericOrAll) > 0) + return Calc(null, HistorianStatusCodes.BadAggregateInvalidInputs); switch (aggregate) { @@ -122,7 +126,8 @@ public static class AggregateEvaluator /// #684 — the Part 13 status of Minimum / Maximum / Range / *ActualTime (Tables 57–62): Uncertain_DataSubNormal /// when a Bad sample exists in the interval, or when an Uncertain value is less than the minimum Good value /// () / greater than the maximum Good value (); else Good. An Uncertain - /// value inside the Good range leaves it Good. + /// value inside the Good range leaves it Good. An Uncertain String/DateTime sample (no order against the Good values) + /// also makes it Uncertain — the SDK's ComputeMinMax flags a value it cannot cast exactly that way. /// /// /// The SDK's ComputeMinMax (1.5.378) applies only the Bad half: it collects the Uncertain extremes and never @@ -131,7 +136,8 @@ public static class AggregateEvaluator /// Uncertain value EQUAL to the Good extreme does not degrade it. /// private static uint MinMaxStatus(BucketRow r, double? goodMin, double? goodMax, bool low, bool high) => - r.Bad > 0 || (low && r.UncertainMin < goodMin) || (high && r.UncertainMax > goodMax) + r.Bad > 0 || r.NonNumeric > r.GoodNonNumericOrAll + || (low && r.UncertainMin < goodMin) || (high && r.UncertainMax > goodMax) ? HistorianStatusCodes.UncertainDataSubNormal : HistorianStatusCodes.Good; diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common/BucketRow.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common/BucketRow.cs index bc241cbd..7a08e086 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common/BucketRow.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common/BucketRow.cs @@ -32,9 +32,20 @@ public sealed class BucketRow /// Bad (not usable) samples. public int Bad { get; set; } - /// Usable String / DateTime samples. + /// Usable String / DateTime samples (Delta's refusal check). public int NonNumeric { get; set; } + /// + /// Good String / DateTime samples — the refusal check of the Good-value aggregates (Average / Minimum / Maximum / + /// Range / *ActualTime, #684 review): an Uncertain non-numeric sample is not part of their value set. Filled by the + /// Simple and ActualTime queries; null (a row that does not carry it) falls back to , the + /// pre-#684 (stricter) check. + /// + public int? GoodNonNumeric { get; set; } + + /// , or when the row does not carry it. + internal int GoodNonNumericOrAll => GoodNonNumeric ?? NonNumeric; + /// Average of the Good numeric values (null when none — #684). public double? Avg { get; set; } diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/SqlAggregateQueries.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/SqlAggregateQueries.cs index d6fa974d..d57120e5 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/SqlAggregateQueries.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/SqlAggregateQueries.cs @@ -50,7 +50,8 @@ internal static class SqlAggregateQueries /// Average / Minimum / Maximum / Count / Range (PercentGood / PercentBad and Total are time-weighted, not here). /// Family columns: 6 AvgValue, 7 MinValue, 8 MaxValue, 9 SumValue over the Good numeric samples (float, /// NULL when none — #684: Part 13 and the SDK compute these over Good values only), then 10 UncMin, 11 UncMax - /// over the Uncertain ones (the Part 13 Minimum/Maximum/Range status rule, ). + /// over the Uncertain ones (the Part 13 Minimum/Maximum/Range status rule, ), and + /// 12 GoodNonNumericCount. /// /// Schema-qualified Sample table name. /// The SQL text. @@ -112,7 +113,7 @@ internal static class SqlAggregateQueries /// /// MinimumActualTime / MaximumActualTime: the Good sample (#684) with the smallest (largest) NumValue, /// ties broken by the earliest SourceTimeUtc. Family columns: 6 NumValue, 7 SourceTimeUtc - /// (NULL when the bucket has no Good numeric sample), then 8 UncMin, 9 UncMax as in . + /// (NULL when the bucket has no Good numeric sample), then 8 UncMin, 9 UncMax, 10 GoodNonNumericCount as in . /// /// /// Deliberately still two references to s (counts + the ranked extreme), unlike : @@ -132,7 +133,7 @@ internal static class SqlAggregateQueries "ROW_NUMBER() OVER (PARTITION BY b ORDER BY NumValue " + (maximum ? "DESC" : "ASC") + ", SourceTimeUtc ASC) AS rn " + "FROM s WHERE g = 1 AND NumValue IS NOT NULL) " + "SELECT st.b, st.TotalCount, st.UsableCount, st.GoodCount, st.BadCount, st.NonNumericCount, " + - "r.NumValue, r.SourceTimeUtc, st.UncMin, st.UncMax " + + "r.NumValue, r.SourceTimeUtc, st.UncMin, st.UncMax, st.GoodNonNumericCount " + "FROM st LEFT JOIN r ON r.b = st.b AND r.rn = 1 ORDER BY st.b;"; /// @@ -194,9 +195,11 @@ internal static class SqlAggregateQueries /// /// The minimum and maximum Uncertain (usable, not Good) numeric value of the bucket — UncMin, UncMax /// (#684). Not result values: Part 13 makes Minimum/Maximum/Range Uncertain when one lies beyond the Good extreme. + /// Then GoodNonNumericCount, the Good String/DateTime samples (the Good-value aggregates' refusal check). /// private static string UncertainExtremes() => - "MIN(CASE WHEN u = 1 AND g = 0 THEN NumValue END) AS UncMin, MAX(CASE WHEN u = 1 AND g = 0 THEN NumValue END) AS UncMax"; + "MIN(CASE WHEN u = 1 AND g = 0 THEN NumValue END) AS UncMin, MAX(CASE WHEN u = 1 AND g = 0 THEN NumValue END) AS UncMax, " + + $"SUM(CASE WHEN g = 1 AND ValueKind IN ({NonNumericKinds}) THEN 1 ELSE 0 END) AS GoodNonNumericCount"; /// The five common count columns (after b). private static string CommonCounts() => diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/SqlServerHistorianDataSource.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/SqlServerHistorianDataSource.cs index fdc22e47..c47f8b20 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/SqlServerHistorianDataSource.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/SqlServerHistorianDataSource.cs @@ -566,6 +566,7 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource row.Sum = D(reader, f + 3); row.UncertainMin = D(reader, f + 4); row.UncertainMax = D(reader, f + 5); + row.GoodNonNumeric = reader.IsDBNull(f + 6) ? 0 : reader.GetInt32(f + 6); break; case AggregateFamily.StartEnd: row.First = V(reader, f); @@ -578,6 +579,7 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource row.ExtremeTime = reader.IsDBNull(f + 1) ? null : SqlHistorianConnection.GetUtc(reader, f + 1); row.UncertainMin = D(reader, f + 2); row.UncertainMax = D(reader, f + 3); + row.GoodNonNumeric = reader.IsDBNull(f + 4) ? 0 : reader.GetInt32(f + 4); break; } return row; diff --git a/tests/Drivers/Historian.Conformance/ProcessedConformance.cs b/tests/Drivers/Historian.Conformance/ProcessedConformance.cs index 71cf231f..5f51434a 100644 --- a/tests/Drivers/Historian.Conformance/ProcessedConformance.cs +++ b/tests/Drivers/Historian.Conformance/ProcessedConformance.cs @@ -554,6 +554,78 @@ public abstract class ProcessedConformance : ConformanceBase (await One(HistoryAggregateType.Count)).Value.ShouldBe(2); } + [Fact] + public async Task An_uncertain_value_above_the_good_maximum_makes_the_maximum_family_uncertain() + { + // #684 review: pins the Uncertain-MAXIMUM column (UncMax / unc_max) through the real SQL — Good 6 @+2, Good 8 @+3, + // Uncertain 9 @+4 (above the Good maximum, nothing below the minimum). Part 13 Tables 58/60/62 ⇒ Maximum, + // MaximumActualTime and Range Uncertain (values still from the Good samples); Minimum Good. + await B.SeedAsync("UA", + (At(-5), HistorianValue.FromDouble(7), Good), + (At(2), HistorianValue.FromDouble(6), Good), + (At(3), HistorianValue.FromDouble(8), Good), + (At(4), HistorianValue.FromDouble(9), Uncertain), + (At(30), HistorianValue.FromDouble(7), Good)); + var src = B.CreateDataSource(); + async Task One(HistoryAggregateType a) => + (await src.ReadProcessedAsync("UA", At(0), At(20), Twenty, a, Ct)).Samples.Single(); + + var max = await One(HistoryAggregateType.Maximum); + max.Value.ShouldBe(8d); + max.StatusCode.ShouldBe(UncertainCalc); + var maxAt = await One(HistoryAggregateType.MaximumActualTime); + maxAt.Value.ShouldBe(8d); + maxAt.SourceTimestampUtc.ShouldBe(B.Round(At(3))); + maxAt.StatusCode.ShouldBe(UncertainCalc); + var range = await One(HistoryAggregateType.Range); + range.Value.ShouldBe(2d); + range.StatusCode.ShouldBe(UncertainCalc); + var min = await One(HistoryAggregateType.Minimum); + min.Value.ShouldBe(6d); + min.StatusCode.ShouldBe(GoodCalc); + } + + [Fact] + public async Task An_uncertain_string_sample_does_not_refuse_the_good_value_aggregates() + { + // #684 review: the Good-value aggregates refuse only a GOOD String/DateTime sample (the good-non-numeric count the + // SQL now returns); an Uncertain one is skipped with an Uncertain status — the SDK's behaviour. Delta keeps its + // usable-sample refusal. + await B.SeedAsync("US", + (At(-5), HistorianValue.FromDouble(7), Good), + (At(2), HistorianValue.FromDouble(6), Good), + (At(3), HistorianValue.FromString("x"), Uncertain), + (At(4), HistorianValue.FromDouble(8), Good), + (At(30), HistorianValue.FromDouble(7), Good)); + var src = B.CreateDataSource(); + async Task One(HistoryAggregateType a) => + (await src.ReadProcessedAsync("US", At(0), At(20), Twenty, a, Ct)).Samples.Single(); + + foreach (var (agg, value) in new[] + { + (HistoryAggregateType.Average, 7d), (HistoryAggregateType.Minimum, 6d), (HistoryAggregateType.Maximum, 8d), + (HistoryAggregateType.Range, 2d), (HistoryAggregateType.MinimumActualTime, 6d), + (HistoryAggregateType.MaximumActualTime, 8d), + }) + { + var v = await One(agg); + v.Value.ShouldBe(value, agg.ToString()); + v.StatusCode.ShouldBe(UncertainCalc, agg.ToString()); + } + (await One(HistoryAggregateType.Delta)).StatusCode.ShouldBe(InvalidInputsCalc); + + // A GOOD string in the same shape still refuses them. + await B.SeedAsync("GS", + (At(-5), HistorianValue.FromDouble(7), Good), + (At(2), HistorianValue.FromDouble(6), Good), + (At(3), HistorianValue.FromString("x"), Good), + (At(30), HistorianValue.FromDouble(7), Good)); + (await src.ReadProcessedAsync("GS", At(0), At(20), Twenty, HistoryAggregateType.Maximum, Ct)).Samples.Single() + .StatusCode.ShouldBe(InvalidInputsCalc); + (await src.ReadProcessedAsync("GS", At(0), At(20), Twenty, HistoryAggregateType.MinimumActualTime, Ct)).Samples + .Single().StatusCode.ShouldBe(InvalidInputsCalc); + } + [Fact] public async Task An_interval_of_only_uncertain_samples_is_BadNoData_for_the_good_value_aggregates() { diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb.Tests/BucketRowMappingTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb.Tests/BucketRowMappingTests.cs index 889b853e..d6fc98c6 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb.Tests/BucketRowMappingTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb.Tests/BucketRowMappingTests.cs @@ -109,7 +109,8 @@ public sealed class BucketRowMappingTests t.Columns.Add("ts", typeof(DateTime)); t.Columns.Add("unc_min", typeof(double)); t.Columns.Add("unc_max", typeof(double)); - t.Rows.Add(0L, 1L, 1L, 1L, 0L, 0L, DBNull.Value, DBNull.Value, 4d, DBNull.Value); + t.Columns.Add("goodnonnum", typeof(long)); + t.Rows.Add(0L, 1L, 1L, 1L, 0L, 0L, DBNull.Value, DBNull.Value, 4d, DBNull.Value, 0L); var b = Map(t, AggregateFamily.ActualTime); b.Extreme.ShouldBeNull(); @@ -125,7 +126,8 @@ public sealed class BucketRowMappingTests t.Columns.Add("b", typeof(long)); foreach (var c in new[] { "total", "usable", "good", "bad", "nonnum" }) t.Columns.Add(c, typeof(long)); foreach (var c in new[] { "avg", "min", "max", "sum", "uncmin", "uncmax" }) t.Columns.Add(c, typeof(double)); - t.Rows.Add(0L, 3L, 3L, 2L, 0L, 0L, 20d, 10d, 30d, 40d, 5d, 25d); + t.Columns.Add("goodnonnum", typeof(long)); + t.Rows.Add(0L, 4L, 4L, 2L, 0L, 1L, 20d, 10d, 30d, 40d, 5d, 25d, 0L); var b = Map(t, AggregateFamily.Simple); b.Avg.ShouldBe(20d); @@ -134,9 +136,13 @@ public sealed class BucketRowMappingTests b.Sum.ShouldBe(40d); b.UncertainMin.ShouldBe(5d); b.UncertainMax.ShouldBe(25d); - // Uncertain 5 is below the Good minimum ⇒ Minimum Uncertain; 25 is inside the Good range ⇒ Maximum Good. + b.NonNumeric.ShouldBe(1); + b.GoodNonNumeric.ShouldBe(0); + // Uncertain 5 is below the Good minimum ⇒ Minimum Uncertain. The one non-numeric sample is Uncertain (ordinal 12 says + // no Good one), so Average is served — not refused — and Maximum is Uncertain for it (25 alone is inside the range). AggregateEvaluator.Evaluate(HistoryAggregateType.Minimum, b, T, T).StatusCode.ShouldBe(0x40A40401u); - AggregateEvaluator.Evaluate(HistoryAggregateType.Maximum, b, T, T).StatusCode.ShouldBe(0x00000401u); + AggregateEvaluator.Evaluate(HistoryAggregateType.Average, b, T, T).Value.ShouldBe(20d); + AggregateEvaluator.Evaluate(HistoryAggregateType.Maximum, b, T, T).StatusCode.ShouldBe(0x40A40401u); } [Fact] @@ -149,7 +155,8 @@ public sealed class BucketRowMappingTests t.Columns.Add("ts", typeof(DateTime)); t.Columns.Add("unc_min", typeof(double)); t.Columns.Add("unc_max", typeof(double)); - t.Rows.Add(0L, 3L, 3L, 2L, 0L, 0L, 10d, T, 5d, 25d); + t.Columns.Add("goodnonnum", typeof(long)); + t.Rows.Add(0L, 3L, 3L, 2L, 0L, 0L, 10d, T, 5d, 25d, 0L); var b = Map(t, AggregateFamily.ActualTime); b.Extreme.ShouldBe(10d); diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common.Tests/AggregateEvaluatorTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common.Tests/AggregateEvaluatorTests.cs index 44bc049f..5fdc4f5f 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common.Tests/AggregateEvaluatorTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common.Tests/AggregateEvaluatorTests.cs @@ -305,6 +305,46 @@ public sealed class AggregateEvaluatorTests v.StatusCode.ShouldBe(good ? GoodCalc : UncertainCalc); } + [Theory] + [InlineData(HistoryAggregateType.Average)] + [InlineData(HistoryAggregateType.Minimum)] + [InlineData(HistoryAggregateType.Maximum)] + [InlineData(HistoryAggregateType.Range)] + [InlineData(HistoryAggregateType.MinimumActualTime)] + [InlineData(HistoryAggregateType.MaximumActualTime)] + public void Only_a_good_non_numeric_sample_refuses_a_good_value_aggregate(HistoryAggregateType agg) + { + // #684 review: an UNCERTAIN String/DateTime sample is outside the Good-value set — served (Uncertain), not refused. + var uncertainString = GoodTenThirtyPlusUncertain(); + uncertainString.Total = uncertainString.Usable = 3; + uncertainString.NonNumeric = 1; + uncertainString.GoodNonNumeric = 0; + var v = Eval(agg, uncertainString); + v.Value.ShouldNotBeNull(); + v.StatusCode.ShouldBe(UncertainCalc); + // A GOOD one still refuses the interval (the value set would hold a value with no numeric meaning). + var goodString = GoodTenThirtyPlusUncertain(); + goodString.Total = goodString.Usable = goodString.Good = 3; + goodString.NonNumeric = 1; + goodString.GoodNonNumeric = 1; + Eval(agg, goodString).StatusCode.ShouldBe(InvalidInputsCalc); + // A row that does not carry the Good count (null) keeps the stricter pre-#684 check: any usable one refuses. + uncertainString.GoodNonNumeric = null; + Eval(agg, uncertainString).StatusCode.ShouldBe(InvalidInputsCalc); + } + + [Fact] + public void Delta_still_refuses_any_usable_non_numeric_sample() + { + // Delta reads the first/last USABLE samples (#680), so an Uncertain String can be its input: unchanged by #684. + var row = new BucketRow + { + Total = 2, Usable = 2, Good = 1, NonNumeric = 1, GoodNonNumeric = 0, + First = HistorianValue.FromString("x"), Last = HistorianValue.FromDouble(1), FirstAny = A[0], LastAny = A[1], + }; + AggregateEvaluator.Evaluate(HistoryAggregateType.Delta, row, At(0), Now).StatusCode.ShouldBe(InvalidInputsCalc); + } + [Theory] [InlineData(HistoryAggregateType.Minimum)] [InlineData(HistoryAggregateType.Maximum)] diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common.Tests/SdkAggregateParityTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common.Tests/SdkAggregateParityTests.cs index 63622b5f..598d6adb 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common.Tests/SdkAggregateParityTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common.Tests/SdkAggregateParityTests.cs @@ -341,10 +341,23 @@ public sealed class SdkAggregateParityTests [MemberData(nameof(Windows))] public void The_uncertain_dataset_is_not_vacuous(double from, double to, double intervalS) { - // #684: DenseUncertain must actually move the per-sample values the pre-#684 (Good + Uncertain) columns computed — - // otherwise its parity proves nothing. Average over the usable samples differs from the SDK's in some interval of - // every window. + // #684: DenseUncertain must discriminate the pre-#684 behaviour in every window, or its parity proves nothing. + // (1) VALUES: Average over the usable (Good + Uncertain) samples differs from the SDK's Good-only Average in some + // interval. (2) STATUS: Minimum / Maximum hold an Uncertain sample in some interval the SDK still calls Good (16 sits + // inside the Good range) — the pre-#684 "any non-Good sample ⇒ Uncertain" rule would answer Uncertain there. + // (Minimum / Maximum VALUES do not discriminate here: an in-range 16 never is the extreme — by design, so the + // min/max family needs no mask; the value half is pinned by UncertainBeyond and AggregateEvaluatorTests.) var plan = BucketPlan.Create(At(from), At(to), TimeSpan.FromSeconds(intervalS)); + // The one-interval windows hold the Bad +22.5 sample, so the SDK is Uncertain there for a reason the old rule shares: + // the status half is checked on the multi-interval windows only (the value half below covers every window). + foreach (var agg in plan.BucketCount > 1 ? new[] { HistoryAggregateType.Minimum, HistoryAggregateType.Maximum } : []) + { + var sdkMinMax = Sdk(agg, from, to, intervalS, DenseUncertain); + var rows = InMemoryRows(plan, DenseUncertain).ToList(); + var holdsUncertain = (plan.Reversed ? Enumerable.Reverse(rows) : rows).Select(r => r.Usable > r.Good).ToList(); + holdsUncertain.Zip(sdkMinMax).Any(p => p.First && (p.Second.StatusCode & 0xFFFF0000u) == 0) + .ShouldBeTrue($"{agg}: some interval holds an Uncertain sample yet is Good in the SDK"); + } var usableAverage = InMemoryRows(plan, DenseUncertain.Select(x => x.IsUsable && !x.IsGood ? x with { Status = Good } : x)) .Select(r => r.Avg).ToList(); @@ -395,6 +408,49 @@ public sealed class SdkAggregateParityTests }); } + /// #684 review: Good numeric samples with an Uncertain STRING among them (+4). + private static readonly StoredSample[] UncertainString = + [ + S(-5, HistorianValue.FromDouble(20), Good), + S(2, HistorianValue.FromDouble(10), Good), + S(4, HistorianValue.FromString("x"), Uncertain), + S(8, HistorianValue.FromDouble(30), Good), + S(25, HistorianValue.FromDouble(20), Good), + ]; + + [Theory] + [InlineData(0, 20)] + [InlineData(20, 0)] + public void An_uncertain_non_numeric_sample_is_skipped_by_the_good_value_aggregates_like_the_SDK(double from, double to) + { + // #684 review: an Uncertain String is not part of the Good-value set, so it no longer refuses the interval + // (BadAggregateInvalidInputs); the SDK's Average skips it (value-based status: Uncertain) and its ComputeMinMax flags + // the value it cannot cast (Uncertain). Exact match, every bit but Calculated. + var failures = new List(); + foreach (var agg in new[] + { + HistoryAggregateType.Average, HistoryAggregateType.Minimum, HistoryAggregateType.Maximum, + HistoryAggregateType.Range, HistoryAggregateType.MinimumActualTime, HistoryAggregateType.MaximumActualTime, + }) + { + try + { + var ours = Ours(agg, from, to, 0, UncertainString); + SdkAggregateOracle.ShouldMatch(ours, Sdk(agg, from, to, 0, UncertainString), $"{agg} UncertainString {from}→{to}", + compareCalculatedInterpolatedBits: false); + (ours.Single().StatusCode & StatusBits.CodeBitsMask).ShouldBe(HistorianStatusCodes.UncertainDataSubNormal, agg.ToString()); + } + catch (Exception ex) + { + failures.Add(ex.Message); + } + } + failures.ShouldBeEmpty(string.Join("\n", failures)); + // Delta keeps its usable picks and refusal (#680, unchanged): no Good-only reading applies to it. + Ours(HistoryAggregateType.Delta, from, to, 0, UncertainString).Single().StatusCode + .ShouldBe(HistorianInfoBits.WithCalculated(HistorianStatusCodes.BadAggregateInvalidInputs)); + } + [Theory] [InlineData(0, 20)] [InlineData(20, 0)] @@ -647,6 +703,7 @@ public sealed class SdkAggregateParityTests Good = all.Count(s => s.IsGood), Bad = all.Count - usable.Count, NonNumeric = usable.Count - numeric.Count, + GoodNonNumeric = usable.Count(s => s.IsGood) - numeric.Count(s => s.IsGood), Avg = good.Count == 0 ? null : good.Average(s => s.Value.Number), Min = good.Count == 0 ? null : good.Min(s => s.Value.Number), Max = good.Count == 0 ? null : good.Max(s => s.Value.Number), diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer.Tests/BucketRowMappingTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer.Tests/BucketRowMappingTests.cs index ee6af373..24f40663 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer.Tests/BucketRowMappingTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer.Tests/BucketRowMappingTests.cs @@ -105,7 +105,8 @@ public sealed class BucketRowMappingTests t.Columns.Add("SourceTimeUtc", typeof(DateTime)); t.Columns.Add("UncMin", typeof(double)); t.Columns.Add("UncMax", typeof(double)); - t.Rows.Add(0L, 1, 1, 1, 0, 0, DBNull.Value, DBNull.Value, 4d, DBNull.Value); + t.Columns.Add("goodnonnum", typeof(int)); + t.Rows.Add(0L, 1, 1, 1, 0, 0, DBNull.Value, DBNull.Value, 4d, DBNull.Value, 0); var b = Map(t, AggregateFamily.ActualTime); b.Extreme.ShouldBeNull(); @@ -121,7 +122,8 @@ public sealed class BucketRowMappingTests t.Columns.Add("b", typeof(long)); foreach (var c in new[] { "total", "usable", "good", "bad", "nonnum" }) t.Columns.Add(c, typeof(int)); foreach (var c in new[] { "avg", "min", "max", "sum", "uncmin", "uncmax" }) t.Columns.Add(c, typeof(double)); - t.Rows.Add(0L, 3, 3, 2, 0, 0, 20d, 10d, 30d, 40d, 5d, 25d); + t.Columns.Add("goodnonnum", typeof(int)); + t.Rows.Add(0L, 4, 4, 2, 0, 1, 20d, 10d, 30d, 40d, 5d, 25d, 0); var b = Map(t, AggregateFamily.Simple); b.Avg.ShouldBe(20d); @@ -130,9 +132,13 @@ public sealed class BucketRowMappingTests b.Sum.ShouldBe(40d); b.UncertainMin.ShouldBe(5d); b.UncertainMax.ShouldBe(25d); - // Uncertain 5 is below the Good minimum ⇒ Minimum Uncertain; 25 is inside the Good range ⇒ Maximum Good. + b.NonNumeric.ShouldBe(1); + b.GoodNonNumeric.ShouldBe(0); + // Uncertain 5 is below the Good minimum ⇒ Minimum Uncertain. The one non-numeric sample is Uncertain (ordinal 12 says + // no Good one), so Average is served — not refused — and Maximum is Uncertain for it (25 alone is inside the range). AggregateEvaluator.Evaluate(HistoryAggregateType.Minimum, b, T, T).StatusCode.ShouldBe(0x40A40401u); - AggregateEvaluator.Evaluate(HistoryAggregateType.Maximum, b, T, T).StatusCode.ShouldBe(0x00000401u); + AggregateEvaluator.Evaluate(HistoryAggregateType.Average, b, T, T).Value.ShouldBe(20d); + AggregateEvaluator.Evaluate(HistoryAggregateType.Maximum, b, T, T).StatusCode.ShouldBe(0x40A40401u); } [Fact] @@ -145,7 +151,8 @@ public sealed class BucketRowMappingTests t.Columns.Add("SourceTimeUtc", typeof(DateTime)); t.Columns.Add("UncMin", typeof(double)); t.Columns.Add("UncMax", typeof(double)); - t.Rows.Add(0L, 3, 3, 2, 0, 0, 10d, T, 5d, 25d); + t.Columns.Add("goodnonnum", typeof(int)); + t.Rows.Add(0L, 3, 3, 2, 0, 0, 10d, T, 5d, 25d, 0); var b = Map(t, AggregateFamily.ActualTime); b.Extreme.ShouldBe(10d);