From dbfe2fc4de8fc8dcc7758dc36bf05397374bab95 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Sun, 11 Oct 2026 15:52:49 -0400 Subject: [PATCH] =?UTF-8?q?fix(historian):=20#680=20lane=20B=20=E2=80=94?= =?UTF-8?q?=20per-sample=20Count/Start/End/Delta=20per=20Part=2013=20+=20f?= =?UTF-8?q?orward=20end-of-data=20Partial?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Count: Good samples only, value-based status (all Bad => generic Bad), 0/Good on an empty interval inside the tag's data range [first, last] (both directions). Start/End: the raw first/last sample of any quality (own value, status, timestamp). Delta: Uncertain only when the interval's first or last sample is not usable. Forward end-of-data Partial (SDK m_endOfData, earlier edge inclusive) for every per-sample aggregate but Average (PercentGood/Bad left to lane A). Backends push down first/last-any-quality columns and fetch the tag's data range (earliest <= window end, latest >= window start) only when NeedsDataRange. SDK parity masks for Start/End/Delta/Count removed; new parity datasets. --- docs/Historian.md | 62 ++++++-- .../QuestDbAggregateQueries.cs | 48 ++++-- .../QuestDbHistorianDataSource.cs | 33 ++-- .../AggregateEvaluator.cs | 108 +++++++++---- .../BucketPlan.cs | 24 +++ .../BucketRow.cs | 11 +- .../SqlAggregateQueries.cs | 41 +++-- .../SqlServerHistorianDataSource.cs | 30 ++-- .../ProcessedConformance.cs | 83 ++++++++-- .../ReadTests.cs | 11 +- .../AggregateEvaluatorTests.cs | 148 +++++++++++++++++- .../BucketPlanTests.cs | 41 ++++- .../SdkAggregateParityTests.cs | 140 +++++++++++++++-- 13 files changed, 644 insertions(+), 136 deletions(-) diff --git a/docs/Historian.md b/docs/Historian.md index 9d2a5d2a..6d3bfa6f 100644 --- a/docs/Historian.md +++ b/docs/Historian.md @@ -935,6 +935,24 @@ point (see [Continuation-point paging (Events)](#continuation-point-paging-event > from the new (forward-only) start-of-data Partial bit below. A client that paired reverse results with forward ones by timestamp must shift by one > interval. +> ⚠️ **#680 (client-visible): `Count`, `Start`, `End`, `Delta` and the end-of-data Partial bit now follow Part 13 / the SDK** on the +> `SqlServer` and `QuestDb` backends: +> - **`Count`** counts **Good** samples only (it used to count Good + Uncertain); its status is value-based — Good when every sample in +> the interval is Good, the generic **Bad** (`0x80000000`, value null) when every one is Bad (it used to be `BadNoData`), else +> Uncertain. An **empty interval inside the tag's data range** `[first sample, last sample]` is now **`0`, Good** (it used to be +> `BadNoData`); an empty interval wholly before the first or after the last sample stays `BadNoData`. Both directions. +> - **`Start` / `End`** return the interval's **raw** first / last sample of **any** quality — its own value (null when Bad), its own +> stored status (Raw: no Calculated bit) and **its own timestamp** (they used to return the first/last *usable* value stamped with +> the interval start under the per-sample status rule). A reverse read returns the same samples: `Start` is still the earliest. +> - **`Delta`** (last usable − first usable) is Good unless the interval's first or last sample is not usable (then Uncertain); a Bad +> sample in the middle no longer degrades it, and neither does an Uncertain sample. +> - **End-of-data Partial:** the interval holding the tag's **last** sample carries the Partial bit for every per-sample aggregate but +> `Average` — forward with the interval's earlier edge included, reverse only when the sample lies strictly inside (the SDK's rules) — +> so a "live" tag's latest interval is now flagged. +> +> A client that read `Count` as "usable samples", keyed `Start`/`End` on the interval start, or treated `Count = BadNoData` as "no +> samples" must adjust. Not changed: `Average`/`Minimum`/`Maximum` still use Good + Uncertain samples (the SDK uses Good only). + **Bucketing.** Forward: one result per interval of `[start, end)`; bucket *i* = `[start + i·interval, start + (i+1)·interval)`, start-inclusive / end-exclusive to the tick, the last bucket truncated at `end`, each stamped with its start. Reverse (`end < start`, #677): the window is `(end, start]`; result *k* = `(start − (k+1)·interval, start − k·interval]`, clipped at `end` (the **earliest** @@ -951,29 +969,43 @@ returned status carries the historian *Calculated* info bit, `BadNoData` include result for the interval the tag's data **starts inside** — its earliest stored sample (any quality) lies strictly between the interval's edges, so there is no bounding value before it and the data does not cover the whole interval. This follows the SDK's `m_startOfData` rule, which the SDK turns off for `Average` and in effect applies to forward reads only (a reverse read is fed latest-first, so its -"start of data" is the latest sample, past the window). A reverse read therefore sets no start-of-data Partial — matching the SDK -(the time-weighted family does set it, on a reverse read, on the interval holding the *latest* sample when nothing follows it — #680, -see [Time-weighted aggregates](#time-weighted-aggregates-part-13-671)). Only a -forward per-sample read of an aggregate other than `Average` fetches the tag's earliest sample (one seek, bounded by the window end); -every other read skips that query. An earliest sample exactly on an edge, or before the interval, sets nothing. The time-weighted +"start of data" is the latest sample, past the window). A reverse read therefore sets no start-of-data Partial on the interval holding +the *earliest* sample — matching the SDK. An earliest sample exactly on an edge, or before the interval, sets nothing. The time-weighted aggregates also keep setting Partial on the truncated interval (below). +**End-of-data Partial (#680).** Every aggregate except `Average` (and `Interpolative`) also sets Partial on a non-Bad result for the +interval the tag's data **ends inside** — nothing stored after it: +- **forward:** the latest stored sample (any quality) lies at or after the interval's earlier edge and before its later one (Part 13 + §5.3.3.2; the SDK's `m_endOfData` rule — the earlier edge is *included*); +- **reverse:** the latest sample lies *strictly* inside the interval (the SDK's reverse read keys its `m_startOfData` rule on the first + value it is fed — the latest — so a latest sample exactly on an interval edge sets nothing). The time-weighted family does the same + (see [Time-weighted aggregates](#time-weighted-aggregates-part-13-671)). + +Data continuing past the window sets nothing. Two deliberate limits, kept: the per-sample aggregates **never** set Partial on a merely +*truncated* interval (only the time-weighted ones do), and **a Bad result never carries Partial** (the SDK sets it on Bad results too). + +**Data-range queries.** The start/end-of-data rules and `Count`'s in-range empty interval need the tag's **data range**: its earliest +stored sample (any quality) at or before the window's later edge and its latest at or after the earlier edge — two single-key seeks +(SQL Server: one extra result set of two `MIN`/`MAX` subqueries; QuestDB: two `LIMIT 1` statements). They run only when the answer can +change a result (`AggregateEvaluator.NeedsDataRange`): a per-sample read of an aggregate other than `Average` (either direction), or +`Count`; a per-sample `Average` read skips them. + **Sample classes.** *Usable* = severity Good or Uncertain **and** a stored value (not `Empty`). *Good* = usable with severity Good. *Bad* = not usable. | Aggregate | Computed over | Value timestamp | |---|---|---| | `Average`, `Minimum`, `Maximum` | `AVG` / `MIN` / `MAX` of `NumValue` over usable samples | bucket start (reverse read: its later edge, as every row marked "bucket start" here) | -| `Count` | number of usable samples (`Int32`) | bucket start | +| `Count` | number of **Good** samples (`Int32`; #680) | bucket start | | `Range` | `Maximum − Minimum` | bucket start | -| `Start`, `End` | first / last usable sample by time (keeps the sample's type) | bucket start | -| `Delta` | `End − Start` (numeric samples) | bucket start | +| `Start`, `End` | the **raw** first / last sample of any quality (its type, its own status; value null when Bad — #680) | **the sample's own time** | +| `Delta` | last usable − first usable value (numeric samples) | bucket start | | `MinimumActualTime`, `MaximumActualTime` | the usable sample with the smallest / largest `NumValue`; ties ⇒ earliest | **the sample's own time** | Status and edge rules: -- An **empty interval** (no samples) ⇒ `BadNoData`. **`Count` — and every value aggregate — over an interval whose samples are all Bad ⇒ `BadNoData`**, not `0`. -- Value aggregates are `Good` when every sample in the bucket is Good and `UncertainDataSubNormal` when the bucket also holds any Bad **or Uncertain** sample. +- An **empty interval** (no samples) ⇒ `BadNoData` — except **`Count`**, which is `0`, Good, when the interval lies inside the tag's data range `[first sample, last sample]` (both edges inclusive, either direction; #680). **Every value aggregate over an interval whose samples are all Bad ⇒ `BadNoData`**; `Count` there is the generic **Bad** (value null). +- Value aggregates are `Good` when every sample in the bucket is Good and `UncertainDataSubNormal` when the bucket also holds any Bad **or Uncertain** sample. `Count`'s status is value-based (all Good ⇒ Good, all Bad ⇒ Bad, else Uncertain); `Start`/`End` carry the raw sample's own status; `Delta` is Good unless the interval's first or last sample is not usable (then Uncertain) — #680. - `Count`, `Start`, `End` accept any value kind; the other aggregates (and `Delta`) on a bucket that holds a usable `String`/`DateTime` sample ⇒ `BadAggregateInvalidInputs` for that interval. - `PercentGood` / `PercentBad` are **duration-based since #680** — see [Time-weighted aggregates](#time-weighted-aggregates-part-13-671). - Everything except `Count` and `Start`/`End` is a `double`; `Int64` samples aggregate as doubles and `Boolean` as 0/1. @@ -1118,10 +1150,8 @@ Start, End, Delta and Minimum/MaximumActualTime; the time-weighted one TimeAvera PercentGood/Bad (#680), including datasets whose data ends inside the window and with an interval wholly inside an outage; reverse reads over data that starts inside the window are compared too. Known differences that remain (tracked as #680) — documented and masked in the tests, deliberately not changed here because forward results were to stay as they are — are: -`Start`/`End` (the SDK returns the raw first/last sample, of any quality, with its own timestamp and stored status; ours returns the -first/last usable value stamped with the interval, under the per-sample status rule — the values agree wherever the SDK's is not Bad); -`Delta` status (the SDK's stays Good with a Bad sample in the interval, ours is Uncertain); the SDK's per-sample Count answers -0/Good for an empty interval inside the data and its per-sample aggregates use Good samples only. Time-weighted differences (#680 closed +the SDK's per-sample Average/Minimum/Maximum use Good samples only, ours Good + Uncertain; the SDK sets Partial on Bad per-sample +results too, ours never (#680 closed the Start/End, Delta, Count and end-of-data Partial differences). Time-weighted differences (#680 closed the TimeAverage/Total status, `Total` coverage, end-of-data and PercentGood/Bad ones), each deliberate: ours flags a truncated interval Partial (Part 13 §5.3.3.2 Case 3), the SDK does not; ours carries the Calculated bit on a `BadNoData`; `TimeAverage2`/`Total2` over an interval wholly inside an outage are generic `Bad` here (Part 13's PercentDataBad rule), `BadNoData` in the SDK; in the end-of-data @@ -1370,7 +1400,9 @@ Every query filters `NOT deleted`; every value goes through `StoredSample`, so t - **Processed:** one query per node; bucket index `(cast(ts AS LONG) * 10 - bucketBaseUnixTicks) / intervalTicks` (never `SAMPLE BY`, whose alignment differs), so bucket edges are identical to `SqlServer` (forward start-inclusive / end-exclusive with the partial bucket last; reverse anchored at the later time, later-edge-inclusive, partial bucket earliest — #677; interval 0 ⇒ one bucket, 100,000-bucket cap). On a forward read of any aggregate - but Average, a second `LIMIT 1` statement (capped at the window end) fetches the tag's earliest live sample for the start-of-data Partial bit. + but Average, and for Count in either direction, two `LIMIT 1` statements fetch the tag's data range — its earliest live sample at or before the + window's later edge and its latest at or after its earlier edge — for the start/end-of-data Partial bits and Count's in-range empty interval + (#677/#680). Start/End/Delta also return each bucket's first and last sample of any quality (`row_number()` over every row). Min/max times use `row_number()` windows with the earliest-timestamp tie-break. ### Retention (TTL) diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb/QuestDbAggregateQueries.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb/QuestDbAggregateQueries.cs index e94efdf7..fb30fef3 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb/QuestDbAggregateQueries.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb/QuestDbAggregateQueries.cs @@ -71,11 +71,17 @@ internal static class QuestDbAggregateQueries "sum(CASE WHEN u = 1 THEN num_value END) AS sum_value " + "FROM (" + Classified(sample, reverse) + ") GROUP BY b ORDER BY b"; + /// Ordinal of the StartEnd family's first-sample-of-any-quality columns (). + public const int FirstAnyColumn = FirstFamilyColumn + 10; + /// - /// Start / End / Delta: the first and last usable sample of each bucket by ts. - /// Family columns: 6..10 first sample (kind, num_value, int_value, str_value, ts), 11..15 last - /// sample (same order). A bucket without a usable sample has NULL ts but kind = 0 (BYTE has no NULL), - /// so the reader keys absence on the NULL time. + /// Start / End / Delta: the first and last usable sample of each bucket by ts (Delta's value), and the first + /// and last sample of any quality (Start / End, and Delta's status — #680). + /// Family columns: 6..10 first usable sample (kind, num_value, int_value, str_value, ts), 11..15 + /// last usable sample (same order) — a bucket without a usable sample has NULL ts but kind = 0 (BYTE + /// has no NULL), so the reader keys absence on the NULL time; then 16..21 the first sample of any quality + /// (ts, kind, num_value, int_value, str_value, status) and 22..27 the last (same order), present for + /// every returned (non-empty) bucket. /// /// Prefixed sample table name. /// The SQL text. @@ -85,12 +91,19 @@ internal static class QuestDbAggregateQueries "st AS (SELECT b, " + CommonCounts() + " FROM s GROUP BY b), " + "f AS (SELECT b, kind, num_value, int_value, str_value, ts, " + "row_number() OVER (PARTITION BY b ORDER BY ts ASC) AS rf, " + - "row_number() OVER (PARTITION BY b ORDER BY ts DESC) AS rl FROM s WHERE u = 1) " + + "row_number() OVER (PARTITION BY b ORDER BY ts DESC) AS rl FROM s WHERE u = 1), " + + "a AS (SELECT b, ts, kind, num_value, int_value, str_value, status, " + + "row_number() OVER (PARTITION BY b ORDER BY ts ASC) AS rf, " + + "row_number() OVER (PARTITION BY b ORDER BY ts DESC) AS rl FROM s) " + "SELECT st.*, " + "x.kind, x.num_value, x.int_value, x.str_value, x.ts, " + - "y.kind, y.num_value, y.int_value, y.str_value, y.ts " + + "y.kind, y.num_value, y.int_value, y.str_value, y.ts, " + + "p.ts, p.kind, p.num_value, p.int_value, p.str_value, p.status, " + + "q.ts, q.kind, q.num_value, q.int_value, q.str_value, q.status " + "FROM st LEFT JOIN (SELECT * FROM f WHERE rf = 1) x ON x.b = st.b " + - "LEFT JOIN (SELECT * FROM f WHERE rl = 1) y ON y.b = st.b ORDER BY st.b"; + "LEFT JOIN (SELECT * FROM f WHERE rl = 1) y ON y.b = st.b " + + "LEFT JOIN (SELECT * FROM a WHERE rf = 1) p ON p.b = st.b " + + "LEFT JOIN (SELECT * FROM a WHERE rl = 1) q ON q.b = st.b ORDER BY st.b"; /// /// MinimumActualTime / MaximumActualTime: the usable sample with the smallest (largest) num_value, ties @@ -150,18 +163,29 @@ internal static class QuestDbAggregateQueries internal const string BucketExpression = "(cast(ts AS LONG) * 10 - $1) / $2"; /// - /// The tag's earliest live sample (any quality) before the window end: the start-of-data Partial rule (#677; forward - /// reads only, so the forward window's exclusive end bounds the scan). Parameters $1 tag, $2 the window - /// end. Column: ts; no row when the tag has none before the end. + /// The tag's earliest live sample (any quality) at or before the window's later edge — half of the tag's data range + /// (the start-of-data Partial rule, #677, and Count's in-range empty interval, #680; see + /// ). Parameters $1 tag, $2 the later edge. Column: + /// ts; no row when the tag has none — the data range then does not reach the window. /// /// Prefixed sample table name. /// The SQL text. public static string EarliestSample(string sample) => - $"SELECT ts FROM {sample} WHERE tag = $1 AND ts < $2 AND NOT deleted ORDER BY ts ASC LIMIT 1"; + $"SELECT ts FROM {sample} WHERE tag = $1 AND ts <= $2 AND NOT deleted ORDER BY ts ASC LIMIT 1"; + + /// + /// The tag's latest live sample (any quality) at or after the window's earlier edge — the other half of the data + /// range (the end-of-data Partial rule and Count's in-range empty interval, #680). Parameters $1 tag, + /// $2 the earlier edge. Column: ts; no row when the tag has none. + /// + /// Prefixed sample table name. + /// The SQL text. + public static string LatestSample(string sample) => + $"SELECT ts FROM {sample} WHERE tag = $1 AND ts >= $2 AND NOT deleted ORDER BY ts DESC LIMIT 1"; /// The classified subquery s: the window's live samples with bucket index and the usable/good flags. private static string Classified(string sample, bool reverse) => - "SELECT " + BucketExpression + " AS b, ts, kind, num_value, int_value, str_value, " + + "SELECT " + BucketExpression + " AS b, ts, kind, num_value, int_value, str_value, status, " + $"CASE WHEN status < {NotBadLimit} AND kind <> {EmptyKind} THEN 1 ELSE 0 END AS u, " + $"CASE WHEN status < {GoodLimit} AND kind <> {EmptyKind} THEN 1 ELSE 0 END AS g " + $"FROM {sample} WHERE tag = $3 AND " + (reverse ? "ts > $4 AND ts <= $5" : "ts >= $4 AND ts < $5") + " AND NOT deleted"; diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb/QuestDbHistorianDataSource.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb/QuestDbHistorianDataSource.cs index adca1562..8cf13bc1 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb/QuestDbHistorianDataSource.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb/QuestDbHistorianDataSource.cs @@ -472,7 +472,7 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource return new HistoryReadResult(twResult, ContinuationPoint: null); } var rows = new List(); - DateTime? earliest = null; + DateTime? earliest = null, latest = null; if (plan.BucketCount > 0 && await ResolveAsync(fullReference, cancellationToken).ConfigureAwait(false)) { var sample = _conn.SampleTable; @@ -494,20 +494,19 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) rows.Add(ReadBucket(reader, family)); } - // The tag's earliest live sample before the window end: the start-of-data Partial rule (#677) — forward - // reads of an aggregate other than Average only; skipped otherwise. - if (AggregateEvaluator.NeedsEarliestSample(aggregate, plan)) + // The tag's data range (earliest + latest live sample): the start/end-of-data Partial rules (#677/#680 — + // forward reads of an aggregate other than Average) and Count's in-range empty interval (#680); skipped + // otherwise. + if (AggregateEvaluator.NeedsDataRange(aggregate, plan)) { - await using var cmd = _conn.CreateCommand(conn, QuestDbAggregateQueries.EarliestSample(sample)); - QuestDbConnection.BindText(cmd, fullReference); - QuestDbConnection.BindTimestamp(cmd, plan.EndUtc); - await using var reader = await cmd.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); - if (await reader.ReadAsync(cancellationToken).ConfigureAwait(false) && !reader.IsDBNull(0)) - earliest = QuestDbConnection.ReadUtc(reader, 0); + earliest = await ReadBoundAsync(conn, QuestDbAggregateQueries.EarliestSample(sample), fullReference, + plan.EndUtc, cancellationToken).ConfigureAwait(false); + latest = await ReadBoundAsync(conn, QuestDbAggregateQueries.LatestSample(sample), fullReference, + plan.StartUtc, cancellationToken).ConfigureAwait(false); } } - var result = AggregateEvaluator.EvaluateAll(aggregate, plan, rows, earliest, now); + var result = AggregateEvaluator.EvaluateAll(aggregate, plan, rows, earliest, latest, now); RecordSuccess(); return new HistoryReadResult(result, ContinuationPoint: null); @@ -565,6 +564,16 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource if (bounds[3] is { } usableAfter) tw.Add(usableAfter); } + /// Runs a one-column ts seek ( / LatestSample). + private async Task ReadBoundAsync(NpgsqlConnection conn, string sql, string tag, DateTime edge, CancellationToken ct) + { + await using var cmd = _conn.CreateCommand(conn, sql); + QuestDbConnection.BindText(cmd, tag); + QuestDbConnection.BindTimestamp(cmd, edge); + await using var reader = await cmd.ExecuteReaderAsync(ct).ConfigureAwait(false); + return await reader.ReadAsync(ct).ConfigureAwait(false) && !reader.IsDBNull(0) ? QuestDbConnection.ReadUtc(reader, 0) : null; + } + private static BucketRow ReadBucket(NpgsqlDataReader reader, AggregateFamily family) { static double? D(NpgsqlDataReader r, int i) => r.IsDBNull(i) ? null : r.GetDouble(i); @@ -597,6 +606,8 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource case AggregateFamily.StartEnd: row.First = V(reader, f); row.Last = V(reader, f + 5); + row.FirstAny = ReadStored(reader, QuestDbAggregateQueries.FirstAnyColumn); + row.LastAny = ReadStored(reader, QuestDbAggregateQueries.FirstAnyColumn + 6); break; default: row.Extreme = D(reader, f); diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common/AggregateEvaluator.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common/AggregateEvaluator.cs index 1cac4cb7..fb62c0fc 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common/AggregateEvaluator.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common/AggregateEvaluator.cs @@ -10,11 +10,18 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common; /// Per interval: no samples ⇒ BadNoData. PercentGood / PercentBad ⇒ 100 × good|bad / all over /// every sample, status Good. Otherwise no usable sample ⇒ BadNoData; a numeric aggregate with a usable /// String/DateTime sample ⇒ BadAggregateInvalidInputs; else the value, Good when every sample is Good, -/// UncertainDataSubNormal when any is Bad or Uncertain. Every status — including BadNoData — -/// carries the Calculated info bits. Timestamps are the bucket's (its -/// start forward, its later edge reverse), except Minimum/MaximumActualTime (the sample's time; ties ⇒ earliest). -/// Every aggregate but Average also carries the Partial bit on the bucket the data starts inside -/// (). +/// UncertainDataSubNormal when any is Bad or Uncertain. Every calculated status — including +/// BadNoData — carries the Calculated info bits. Timestamps are the bucket's +/// (its start forward, its later edge reverse), except +/// Minimum/MaximumActualTime (the sample's time; ties ⇒ earliest) and Start/End. Every aggregate but Average +/// also carries the Partial bit on a forward read's bucket the data starts or ends inside (). +/// +/// +/// #680 (Part 13 / the SDK's calculators): Count counts Good samples only, value-based status (all Good ⇒ +/// Good, all Bad ⇒ the generic Bad, else Uncertain), and is 0/Good on an empty bucket inside the tag's data range; +/// Start / End are the raw first / last sample of any quality (own value — null when Bad —, own status +/// without the Calculated bit, own timestamp); Delta is Uncertain only when the bucket's first or last +/// sample is not usable. /// /// /// Values: Count is ; Start/End keep the sample's type; every other aggregate is @@ -44,26 +51,35 @@ public static class AggregateEvaluator case HistoryAggregateType.PercentGood: return Calc(100d * r.Good / r.Total, HistorianStatusCodes.Good); case HistoryAggregateType.PercentBad: return Calc(100d * r.Bad / r.Total, HistorianStatusCodes.Good); } + switch (aggregate) + { + // #680 (Part 13 Tables 73/74; SDK ComputeStartEnd): the raw first / last sample of ANY quality — its own value + // (null when Bad), its own status (Raw: no Calculated bit) and its own timestamp. + case HistoryAggregateType.Start: return Raw(r.FirstAny, bucketStart, now); + case HistoryAggregateType.End: return Raw(r.LastAny, bucketStart, now); + // #680 (Part 13 §5.4.3.21; SDK ComputeCount): Good samples only; value-based status — Good when every sample + // is Good, the generic Bad when every one is Bad (value null), else Uncertain. + case HistoryAggregateType.Count: + if (r.Bad == r.Total) return Calc(null, HistorianStatusCodes.Bad); + return Calc(r.Good, r.Good == r.Total ? HistorianStatusCodes.Good : HistorianStatusCodes.UncertainDataSubNormal); + } if (r.Usable == 0) return NoData(bucketStart, now); - - var anyText = aggregate is HistoryAggregateType.Count or HistoryAggregateType.Start or HistoryAggregateType.End; - if (!anyText && r.NonNumeric > 0) - return Calc(null, HistorianStatusCodes.BadAggregateInvalidInputs); + if (r.NonNumeric > 0) return Calc(null, HistorianStatusCodes.BadAggregateInvalidInputs); var status = r.Good == r.Total ? HistorianStatusCodes.Good : HistorianStatusCodes.UncertainDataSubNormal; switch (aggregate) { - case HistoryAggregateType.Count: return Calc(r.Usable, status); case HistoryAggregateType.Average: return Calc(r.Avg, status); case HistoryAggregateType.Minimum: return Calc(r.Min, status); case HistoryAggregateType.Maximum: return Calc(r.Max, status); case HistoryAggregateType.Range: return Calc(r.Max - r.Min, status); - case HistoryAggregateType.Start: return Calc(r.First.ToObject(), status); - case HistoryAggregateType.End: return Calc(r.Last.ToObject(), status); case HistoryAggregateType.Delta: - return r.First.TryGetDouble(out var a) && r.Last.TryGetDouble(out var b) - ? Calc(b - a, status) - : Calc(null, HistorianStatusCodes.BadAggregateInvalidInputs); + // #680 (SDK ComputeDelta): last usable − first usable; Uncertain only when the interval's first or last + // sample was skipped (not usable) — a Bad sample in the middle does not degrade it. + if (!r.First.TryGetDouble(out var a) || !r.Last.TryGetDouble(out var b)) + return Calc(null, HistorianStatusCodes.BadAggregateInvalidInputs); + var bounded = r.FirstAny is { IsUsable: true } && r.LastAny is { IsUsable: true }; + return Calc(b - a, bounded ? HistorianStatusCodes.Good : HistorianStatusCodes.UncertainDataSubNormal); case HistoryAggregateType.MinimumActualTime: case HistoryAggregateType.MaximumActualTime: return Calc(r.Extreme, status, r.ExtremeTime); @@ -81,49 +97,85 @@ public static class AggregateEvaluator public static bool SetsStartOfDataPartial(HistoryAggregateType aggregate) => aggregate != HistoryAggregateType.Average; /// - /// True when a per-sample read must fetch the tag's earliest sample: the start-of-data Partial rule applies only to a - /// forward read (as in the SDK, whose reverse read keys m_startOfData on the first value it is fed — the - /// latest) of an aggregate other than Average. Otherwise the backend skips that query. + /// True for the per-sample aggregates the forward end-of-data Partial rule (#680) applies to: the same set as + /// less PercentGood / PercentBad, which #680 moves to the time-weighted family + /// (their per-duration definition) — not changed here. + /// + /// Requested aggregate. + /// Whether the end-of-data Partial rule applies. + public static bool SetsEndOfDataPartial(HistoryAggregateType aggregate) => + SetsStartOfDataPartial(aggregate) + && aggregate is not (HistoryAggregateType.PercentGood or HistoryAggregateType.PercentBad); + + /// + /// True when a per-sample read must fetch the tag's data range — its earliest and latest stored samples of any + /// quality — because the answer can change a result: the start- and end-of-data Partial rules on a forward + /// read of an aggregate other than Average (as in the SDK, which applies both to forward reads only), and + /// Count in either direction (an empty interval inside the range is 0, outside it BadNoData, #680). + /// Otherwise the backend skips both queries. /// /// Requested aggregate. /// The bucket plan. - /// Whether the earliest sample is needed. - public static bool NeedsEarliestSample(HistoryAggregateType aggregate, BucketPlan plan) + /// Whether the earliest and latest samples are needed. + public static bool NeedsDataRange(HistoryAggregateType aggregate, BucketPlan plan) { ArgumentNullException.ThrowIfNull(plan); - return !plan.Reversed && plan.BucketCount > 0 && SetsStartOfDataPartial(aggregate); + return plan.BucketCount > 0 + && (aggregate == HistoryAggregateType.Count || (!plan.Reversed && SetsStartOfDataPartial(aggregate))); } /// /// Evaluates every non-empty bucket row and assembles one result per interval in the plan's order (descending for a /// reverse read): each value stamped with (the earlier edge forward, the - /// later edge reverse); empty buckets BadNoData; and — on a forward read, for every aggregate but - /// Average — the Partial bit on a non-Bad result of the bucket the tag's data starts inside (#677, the SDK's - /// m_startOfData rule; the SDK applies it to forward reads only, ). + /// later edge reverse) — Start/End/ActualTime with their sample's own time; empty buckets BadNoData, except a + /// Count inside the tag's data range (0, Good — #680); and, on a forward read, for every aggregate but + /// Average, the Partial bit on a non-Bad result of the bucket the tag's data starts strictly inside (#677, the + /// SDK's m_startOfData rule) or ends inside (#680, m_endOfData, the bucket's earlier edge included). + /// A Bad result never carries Partial, and a reverse read none at all (). /// /// A per-sample (Simple / StartEnd / ActualTime) aggregate. /// The bucket plan. /// The backend's non-empty bucket rows (ascending bucket index). /// The tag's earliest stored sample of any quality, or null when none. + /// The tag's latest stored sample of any quality, or null when none. /// Server timestamp. /// results. public static IReadOnlyList EvaluateAll(HistoryAggregateType aggregate, BucketPlan plan, - IEnumerable rows, DateTime? earliestSampleUtc, DateTime now) + IEnumerable rows, DateTime? earliestSampleUtc, DateTime? latestSampleUtc, DateTime now) { ArgumentNullException.ThrowIfNull(plan); ArgumentNullException.ThrowIfNull(rows); var evaluated = new Dictionary(); - var partialRule = NeedsEarliestSample(aggregate, plan); + var startRule = !plan.Reversed && SetsStartOfDataPartial(aggregate); + var endRule = !plan.Reversed && SetsEndOfDataPartial(aggregate); foreach (var row in rows) { var v = Evaluate(aggregate, row, plan.BucketTimestamp(row.Bucket), now); - if (partialRule && !StatusBits.IsBad(v.StatusCode) && plan.DataStartsInside(row.Bucket, earliestSampleUtc)) - v = v with { StatusCode = v.StatusCode | HistorianInfoBits.HistorianPartial }; + if (!StatusBits.IsBad(v.StatusCode) + && ((startRule && plan.DataStartsInside(row.Bucket, earliestSampleUtc)) + || (endRule && plan.DataEndsInside(row.Bucket, latestSampleUtc)))) + // A raw Start/End sample gains the DataValue InfoType with its first aggregate bit (as the SDK's + // SetAggregateBits does); a calculated status already carries it. + v = v with { StatusCode = v.StatusCode | HistorianInfoBits.InfoTypeDataValue | HistorianInfoBits.HistorianPartial }; evaluated[row.Bucket] = v; } + if (aggregate == HistoryAggregateType.Count) + { + // #680 (Part 13 §5.4.3.21; SDK GetValues): an empty interval inside [earliest, latest] is a Count of 0, Good. + for (long i = 0; i < plan.BucketCount; i++) + if (!evaluated.ContainsKey(i) && plan.WithinDataRange(i, earliestSampleUtc, latestSampleUtc)) + evaluated[i] = new DataValueSnapshot(0, HistorianInfoBits.WithCalculated(HistorianStatusCodes.Good), + plan.BucketTimestamp(i), now); + } return plan.Assemble(evaluated, t => NoData(t, now)); } + /// A raw stored sample as an aggregate result (Start / End): value null when Bad, its own status and time. + private static DataValueSnapshot Raw(StoredSample? sample, DateTime bucketStart, DateTime now) => + sample is not { } s + ? NoData(bucketStart, now) + : new(StatusBits.IsBad(s.Status) ? null : s.Value.ToObject(), s.Status, s.TimeUtc, now); + /// The result for a bucket with no samples (or no usable sample): BadNoData + Calculated. /// The bucket's start time. /// Server timestamp. diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common/BucketPlan.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common/BucketPlan.cs index fc46d236..e2decf89 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common/BucketPlan.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common/BucketPlan.cs @@ -155,6 +155,30 @@ public sealed class BucketPlan public bool DataStartsInside(long index, DateTime? earliestSampleUtc) => earliestSampleUtc is { } e && e.Ticks > LowerTicks(index) && e.Ticks < UpperTicks(index); + /// + /// True when the latest stored sample of the tag lies in bucket (its earlier edge + /// included): the data ends part-way through the interval, so Part 13 §5.3.3.2 / the SDK set the Partial bit on a + /// forward read (#680, the SDK's m_endOfData >= startTime && m_endOfData < endTime). + /// + /// Zero-based ascending bucket index. + /// The tag's latest stored sample (any quality), or null when it has none. + /// Whether the data ends inside the bucket. + public bool DataEndsInside(long index, DateTime? latestSampleUtc) => + latestSampleUtc is { } l && l.Ticks >= LowerTicks(index) && l.Ticks < UpperTicks(index); + + /// + /// True when bucket overlaps the tag's data range [earliest, latest] — neither + /// wholly before the first sample nor wholly after the last (the SDK's GetValues null test: + /// start > last || end < first ⇒ no data, both edges inclusive in either direction). An empty bucket + /// inside the range is a Count of 0, not BadNoData (#680, Part 13 §5.4.3.21). + /// + /// Zero-based ascending bucket index. + /// The tag's earliest stored sample (any quality), or null. + /// The tag's latest stored sample (any quality), or null. + /// Whether the bucket is within the data range. + public bool WithinDataRange(long index, DateTime? earliestSampleUtc, DateTime? latestSampleUtc) => + earliestSampleUtc is { } e && latestSampleUtc is { } l && LowerTicks(index) <= l.Ticks && UpperTicks(index) >= e.Ticks; + /// One snapshot per bucket in result order; missing buckets get ; descending if reversed. /// Evaluated snapshots of the non-empty buckets, by ascending bucket index. /// Builds the snapshot of a bucket with no row, from its . diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common/BucketRow.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common/BucketRow.cs index 2f178a39..4d0c4c9e 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common/BucketRow.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common/BucketRow.cs @@ -9,7 +9,7 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common; /// /// Sample classes: usable = severity Good or Uncertain AND a stored value (an Empty sample is an outage marker /// and counts as Bad whatever its code); good = usable with severity Good; bad = not usable. -/// Every value field is computed over usable samples only. +/// Every value field is computed over usable samples only, except / . /// public sealed class BucketRow { @@ -54,4 +54,13 @@ public sealed class BucketRow /// Time of (ties ⇒ earliest; null when none). public DateTime? ExtremeTime { get; set; } + + /// + /// The bucket's first sample by time of any quality (null when the bucket is empty): Start returns it raw, and + /// Delta is Uncertain when it is not usable (#680, Part 13 Tables 73/74 — the SDK's StartEndAggregateCalculator). + /// + public StoredSample? FirstAny { get; set; } + + /// The bucket's last sample by time of any quality (null when the bucket is empty): End, Delta (#680). + public StoredSample? LastAny { get; set; } } diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/SqlAggregateQueries.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/SqlAggregateQueries.cs index 57117811..a4cf3aaf 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/SqlAggregateQueries.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/SqlAggregateQueries.cs @@ -62,10 +62,16 @@ internal static class SqlAggregateQueries "SUM(CASE WHEN u = 1 THEN NumValue END) AS SumValue " + "FROM s GROUP BY b ORDER BY b;"; + /// Ordinal of the StartEnd family's first-sample-of-any-quality columns (). + public const int FirstAnyColumn = FirstFamilyColumn + 10; + /// - /// Start / End / Delta: the first and last usable sample of each bucket by SourceTimeUtc. - /// Family columns: 6..10 first sample (ValueKind, NumValue, IntValue, StrValue, SourceTimeUtc), - /// 11..15 last sample (same order); all NULL when the bucket has no usable sample. + /// Start / End / Delta: the first and last usable sample of each bucket by SourceTimeUtc (Delta's value), and + /// the first and last sample of any quality (Start / End, and Delta's status — #680). + /// Family columns: 6..10 first usable sample (ValueKind, NumValue, IntValue, StrValue, SourceTimeUtc), + /// 11..15 last usable sample (same order) — all NULL when the bucket has no usable sample; then + /// 16..21 the first sample of any quality (SourceTimeUtc, ValueKind, NumValue, IntValue, StrValue, + /// StatusCode) and 22..27 the last (same order) — never NULL for a returned (non-empty) bucket. /// /// Schema-qualified Sample table name. /// The SQL text. @@ -75,11 +81,17 @@ internal static class SqlAggregateQueries ", st AS (SELECT b, " + CommonCounts() + " FROM s GROUP BY b)" + ", f AS (SELECT b, ValueKind, NumValue, IntValue, StrValue, SourceTimeUtc, " + "ROW_NUMBER() OVER (PARTITION BY b ORDER BY SourceTimeUtc ASC) AS rf, " + - "ROW_NUMBER() OVER (PARTITION BY b ORDER BY SourceTimeUtc DESC) AS rl FROM s WHERE u = 1) " + + "ROW_NUMBER() OVER (PARTITION BY b ORDER BY SourceTimeUtc DESC) AS rl FROM s WHERE u = 1)" + + ", a AS (SELECT b, SourceTimeUtc, ValueKind, NumValue, IntValue, StrValue, StatusCode, " + + "ROW_NUMBER() OVER (PARTITION BY b ORDER BY SourceTimeUtc ASC) AS rf, " + + "ROW_NUMBER() OVER (PARTITION BY b ORDER BY SourceTimeUtc DESC) AS rl FROM s) " + "SELECT st.*, " + "x.ValueKind, x.NumValue, x.IntValue, x.StrValue, x.SourceTimeUtc, " + - "y.ValueKind, y.NumValue, y.IntValue, y.StrValue, y.SourceTimeUtc " + - "FROM st LEFT JOIN f x ON x.b = st.b AND x.rf = 1 LEFT JOIN f y ON y.b = st.b AND y.rl = 1 ORDER BY st.b;"; + "y.ValueKind, y.NumValue, y.IntValue, y.StrValue, y.SourceTimeUtc, " + + "p.SourceTimeUtc, p.ValueKind, p.NumValue, p.IntValue, p.StrValue, p.StatusCode, " + + "q.SourceTimeUtc, q.ValueKind, q.NumValue, q.IntValue, q.StrValue, q.StatusCode " + + "FROM st LEFT JOIN f x ON x.b = st.b AND x.rf = 1 LEFT JOIN f y ON y.b = st.b AND y.rl = 1 " + + "JOIN a p ON p.b = st.b AND p.rf = 1 JOIN a q ON q.b = st.b AND q.rl = 1 ORDER BY st.b;"; /// /// MinimumActualTime / MaximumActualTime: the usable sample with the smallest (largest) NumValue, @@ -130,21 +142,26 @@ internal static class SqlAggregateQueries } /// - /// The earliest stored sample of the tag (any quality), one row (NULL when none): the start-of-data Partial - /// rule (#677; forward reads only). A MIN over the (TagId, SourceTimeUtc) key — a single seek — bounded - /// by the window end (a sample at or past it cannot lie inside an interval). + /// The tag's data range as one row of two columns (each NULL when none): 0 the earliest stored sample + /// of any quality at or before the window's later edge, 1 the latest at or after its earlier edge. Used by the + /// start-of-data (#677) and end-of-data (#680) Partial rules and by Count's in-range empty interval (#680) — only when + /// says the answer can change a result. Each is a MIN / + /// MAX over the (TagId, SourceTimeUtc) key — a single seek — bounded by the window (a sample past the + /// later edge cannot start the data inside it, nor one before the earlier edge end it there; a NULL then means "the + /// data range does not reach the window"). /// /// Schema-qualified Sample table name. /// The SQL text. - public static string EarliestSample(string sample) => - $"SELECT MIN(SourceTimeUtc) FROM {sample} WHERE TagId = @id AND SourceTimeUtc < @end;"; + public static string DataRange(string sample) => + $"SELECT (SELECT MIN(SourceTimeUtc) FROM {sample} WHERE TagId = @id AND SourceTimeUtc <= @end), " + + $"(SELECT MAX(SourceTimeUtc) FROM {sample} WHERE TagId = @id AND SourceTimeUtc >= @start);"; /// The s CTE: the window's samples with bucket index and the usable/good flags. private static string Classified(string sample, bool reverse) => "WITH s AS (SELECT (DATEDIFF_BIG(microsecond, @start, SourceTimeUtc) * 10 " + "+ (DATEPART(nanosecond, SourceTimeUtc) / 100) % 10 - (DATEPART(nanosecond, @start) / 100) % 10 " + "+ @bucketAdjustTicks) / @intervalTicks AS b, " + - "SourceTimeUtc, ValueKind, NumValue, IntValue, StrValue, " + + "SourceTimeUtc, ValueKind, NumValue, IntValue, StrValue, StatusCode, " + "CASE WHEN StatusCode >= 0 AND ValueKind <> 0 THEN 1 ELSE 0 END AS u, " + "CASE WHEN StatusCode >= 0 AND StatusCode < 1073741824 AND ValueKind <> 0 THEN 1 ELSE 0 END AS g " + $"FROM {sample} WHERE TagId = @id AND " + diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/SqlServerHistorianDataSource.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/SqlServerHistorianDataSource.cs index 11ddb4c3..ebb362f6 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/SqlServerHistorianDataSource.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/SqlServerHistorianDataSource.cs @@ -410,8 +410,11 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource /// String/DateTime sample ⇒ BadAggregateInvalidInputs; else the value, Good when every sample is Good, /// UncertainDataSubNormal when any is Bad or Uncertain. Every status — including BadNoData — /// carries the Calculated info bits. Timestamps are the bucket start (its later edge on a reverse read), except - /// Minimum/MaximumActualTime (the sample's time; ties ⇒ earliest). Every aggregate but Average sets the Partial - /// bit on the bucket the tag's earliest sample lies strictly inside (#677). + /// Minimum/MaximumActualTime (the sample's time; ties ⇒ earliest). On a forward read every aggregate but Average + /// sets the Partial bit on the bucket the tag's earliest sample lies strictly inside (#677) or its latest sample + /// lies in (#680). #680: Count counts Good samples (value-based status; 0/Good on an empty bucket inside the tag's + /// data range), Start/End are the raw first/last sample of any quality, Delta is Uncertain only when the bucket's + /// first or last sample is not usable (see ). /// /// /// Values: Count is ; Start/End keep the sample's type; every other aggregate is @@ -444,7 +447,7 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource return new HistoryReadResult(twResult, ContinuationPoint: null); } var rows = new List(); - DateTime? earliest = null; + DateTime? earliest = null, latest = null; if (tagId is { } id) { var sample = _conn.Q("Sample"); @@ -454,10 +457,10 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource AggregateFamily.StartEnd => SqlAggregateQueries.StartEnd(sample, plan.Reversed), _ => SqlAggregateQueries.ActualTime(sample, aggregate == HistoryAggregateType.MaximumActualTime, plan.Reversed), }; - // Second result set: the tag's earliest sample (the start-of-data Partial rule, #677) — forward reads of an - // aggregate other than Average only. - var needsEarliest = AggregateEvaluator.NeedsEarliestSample(aggregate, plan); - if (needsEarliest) sql += " " + SqlAggregateQueries.EarliestSample(sample); + // Second result set: the tag's data range (earliest + latest sample) — the start/end-of-data Partial rules + // (#677/#680, forward reads of an aggregate other than Average) and Count's in-range empty interval (#680). + var needsRange = AggregateEvaluator.NeedsDataRange(aggregate, plan); + if (needsRange) sql += " " + SqlAggregateQueries.DataRange(sample); await using var conn = await _conn.OpenAsync(cancellationToken).ConfigureAwait(false); await using var cmd = _conn.CreateCommand(conn, sql); cmd.Parameters.Add(new SqlParameter("@id", SqlDbType.Int) { Value = id }); @@ -468,12 +471,15 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource await using var reader = await cmd.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) rows.Add(ReadBucket(reader, family)); - if (needsEarliest && await reader.NextResultAsync(cancellationToken).ConfigureAwait(false) - && await reader.ReadAsync(cancellationToken).ConfigureAwait(false) && !reader.IsDBNull(0)) - earliest = DateTime.SpecifyKind(reader.GetDateTime(0), DateTimeKind.Utc); + if (needsRange && await reader.NextResultAsync(cancellationToken).ConfigureAwait(false) + && await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) + { + if (!reader.IsDBNull(0)) earliest = SqlHistorianConnection.GetUtc(reader, 0); + if (!reader.IsDBNull(1)) latest = SqlHistorianConnection.GetUtc(reader, 1); + } } - var result = AggregateEvaluator.EvaluateAll(aggregate, plan, rows, earliest, now); + var result = AggregateEvaluator.EvaluateAll(aggregate, plan, rows, earliest, latest, now); RecordSuccess(); return new HistoryReadResult(result, ContinuationPoint: null); @@ -547,6 +553,8 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource case AggregateFamily.StartEnd: row.First = V(reader, f); row.Last = V(reader, f + 5); + row.FirstAny = ReadStored(reader, SqlAggregateQueries.FirstAnyColumn); + row.LastAny = ReadStored(reader, SqlAggregateQueries.FirstAnyColumn + 6); break; default: row.Extreme = D(reader, f); diff --git a/tests/Drivers/Historian.Conformance/ProcessedConformance.cs b/tests/Drivers/Historian.Conformance/ProcessedConformance.cs index 8958b19f..0a06031b 100644 --- a/tests/Drivers/Historian.Conformance/ProcessedConformance.cs +++ b/tests/Drivers/Historian.Conformance/ProcessedConformance.cs @@ -21,6 +21,7 @@ public abstract class ProcessedConformance : ConformanceBase private const uint UncertainCalc = 0x40A40401u; // UncertainDataSubNormal | 0x401 private const uint NoDataCalc = 0x809B0401u; // BadNoData | 0x401 private const uint InvalidInputsCalc = 0x80D60401u; // BadAggregateInvalidInputs | 0x401 + private const uint Partial = 0x4u; // the Partial aggregate bit (#677 start-of-data, #680 end-of-data) private async Task ReadA(HistoryAggregateType agg) => await B.CreateDataSource().ReadProcessedAsync("A", At(0), At(80), Twenty, agg, Ct); @@ -40,8 +41,6 @@ public abstract class ProcessedConformance : ConformanceBase [InlineData(HistoryAggregateType.Average, 15d, 40d, 50d)] [InlineData(HistoryAggregateType.Minimum, 10d, 40d, 50d)] [InlineData(HistoryAggregateType.Maximum, 20d, 40d, 50d)] - [InlineData(HistoryAggregateType.Start, 10d, 40d, 50d)] - [InlineData(HistoryAggregateType.End, 20d, 40d, 50d)] [InlineData(HistoryAggregateType.Delta, 10d, 0d, 0d)] [InlineData(HistoryAggregateType.Range, 10d, 0d, 0d)] public async Task Numeric_aggregates_over_the_fixture(HistoryAggregateType agg, double b0, double b1, double b2) @@ -51,18 +50,65 @@ public abstract class ProcessedConformance : ConformanceBase s.Count.ShouldBe(4); s.Select(x => x.Value).ShouldBe([b0, b1, b2, null]); - // B1 holds a Bad sample alongside a usable one ⇒ Uncertain; the empty B3 ⇒ BadNoData. - s.Select(x => x.StatusCode).ShouldBe([GoodCalc, UncertainCalc, GoodCalc, NoDataCalc]); + // B1 holds a Bad sample alongside a usable one ⇒ Uncertain (Delta: its first sample is the Bad one); the empty B3, + // after the last sample, ⇒ BadNoData. #680: B2 holds the last sample (+40) ⇒ end-of-data Partial on a forward + // read, for every aggregate but Average. + var b2Status = agg == HistoryAggregateType.Average ? GoodCalc : GoodCalc | Partial; + s.Select(x => x.StatusCode).ShouldBe([GoodCalc, UncertainCalc, b2Status, NoDataCalc]); s.Select(x => x.SourceTimestampUtc).ShouldBe([B.Round(At(0)), B.Round(At(20)), B.Round(At(40)), B.Round(At(60))]); // bucket starts } [Fact] - public async Task Count_counts_usable_samples_as_int() + public async Task Start_and_End_return_the_raw_first_and_last_sample_of_any_quality() { + // #680 (Part 13 Tables 73/74; SDK ComputeStartEnd): the interval's raw first / last sample — its own value (null + // when Bad), its own status (Raw: no Calculated bit) and its own timestamp. B1's Start is the Bad sample at +20. + await ConformanceData.SeedAAsync(B); + var start = (await ReadA(HistoryAggregateType.Start)).Samples; + start.Select(x => x.Value).ShouldBe([10d, null, 50d, null]); + start.Select(x => x.StatusCode).ShouldBe([Good, Bad, Good | 0x400u | Partial, NoDataCalc]); + start.Select(x => x.SourceTimestampUtc).ShouldBe([B.Round(At(0)), B.Round(At(20)), B.Round(At(40)), B.Round(At(60))]); + var end = (await ReadA(HistoryAggregateType.End)).Samples; + end.Select(x => x.Value).ShouldBe([20d, 40d, 50d, null]); + end.Select(x => x.StatusCode).ShouldBe([Good, Good, Good | 0x400u | Partial, NoDataCalc]); + end.Select(x => x.SourceTimestampUtc).ShouldBe([B.Round(At(10)), B.Round(At(30)), B.Round(At(40)), B.Round(At(60))]); + } + + [Fact] + public async Task Count_counts_good_samples_as_int() + { + // #680 (Part 13 §5.4.3.21): Good samples only, value-based status; end-of-data Partial on B2. await ConformanceData.SeedAAsync(B); var s = (await ReadA(HistoryAggregateType.Count)).Samples; s.Select(x => x.Value).ShouldBe([(object?)2, 1, 1, null]); - s.Select(x => x.StatusCode).ShouldBe([GoodCalc, UncertainCalc, GoodCalc, NoDataCalc]); + s.Select(x => x.StatusCode).ShouldBe([GoodCalc, UncertainCalc, GoodCalc | Partial, NoDataCalc]); + } + + [Fact] + public async Task Count_is_zero_on_an_empty_interval_inside_the_data_range_in_both_directions() + { + // #680: +1 G, +2 U, +25 Bad, +45 G, +70 G. Forward [0,60) at 10 s: [10,20), [30,40) and [50,60) are empty but + // inside [first, last] ⇒ 0, Good; [20,30) holds only a Bad sample ⇒ generic Bad. Reverse (60 → 0) likewise, and a + // window wholly after the data ⇒ BadNoData — through the real SQL (the data range is two pushed-down seeks). + await B.SeedAsync("CZ", + (At(1), HistorianValue.FromDouble(10), Good), + (At(2), HistorianValue.FromDouble(10), Uncertain), + (At(25), HistorianValue.FromDouble(10), Bad), + (At(45), HistorianValue.FromDouble(10), Good), + (At(70), HistorianValue.FromDouble(1), Good)); + var src = B.CreateDataSource(); + var ten = TimeSpan.FromSeconds(10); + var fwd = (await src.ReadProcessedAsync("CZ", At(0), At(60), ten, HistoryAggregateType.Count, Ct)).Samples; + fwd.Select(x => x.Value).ShouldBe([(object?)1, 0, null, 0, 1, 0]); + fwd.Select(x => x.StatusCode).ShouldBe( + [UncertainCalc | Partial, GoodCalc, BadCalc, GoodCalc, GoodCalc, GoodCalc]); + var rev = (await src.ReadProcessedAsync("CZ", At(60), At(0), ten, HistoryAggregateType.Count, Ct)).Samples; + rev.Select(x => x.Value).ShouldBe([(object?)0, 1, 0, null, 0, 1]); + rev.Select(x => x.StatusCode).ShouldBe([GoodCalc, GoodCalc, GoodCalc, BadCalc, GoodCalc, UncertainCalc]); + (await src.ReadProcessedAsync("CZ", At(80), At(100), ten, HistoryAggregateType.Count, Ct)).Samples + .ShouldAllBe(x => x.StatusCode == NoDataCalc); + (await src.ReadProcessedAsync("CZ", At(-20), At(0), ten, HistoryAggregateType.Count, Ct)).Samples + .ShouldAllBe(x => x.StatusCode == NoDataCalc); } [Theory] @@ -88,7 +134,8 @@ public abstract class ProcessedConformance : ConformanceBase var min = (await ReadA(HistoryAggregateType.MinimumActualTime)).Samples; min.Select(x => x.Value).ShouldBe([10d, 40d, 50d, null]); min.Select(x => x.SourceTimestampUtc).ShouldBe([B.Round(At(0)), B.Round(At(30)), B.Round(At(40)), B.Round(At(60))]); - min.Select(x => x.StatusCode).ShouldBe([GoodCalc, UncertainCalc, GoodCalc, NoDataCalc]); + // #680: B2 holds the last sample (+40) ⇒ end-of-data Partial. + min.Select(x => x.StatusCode).ShouldBe([GoodCalc, UncertainCalc, GoodCalc | Partial, NoDataCalc]); var max = (await ReadA(HistoryAggregateType.MaximumActualTime)).Samples; max.Select(x => x.Value).ShouldBe([20d, 40d, 50d, null]); max.Select(x => x.SourceTimestampUtc).ShouldBe([B.Round(At(10)), B.Round(At(30)), B.Round(At(40)), B.Round(At(60))]); @@ -272,8 +319,9 @@ public abstract class ProcessedConformance : ConformanceBase (At(5), HistorianValue.FromDouble(10), Good), (At(15), HistorianValue.FromDouble(30), Good), (At(25), HistorianValue.FromDouble(20), Good)); + // ... but LS2's data ENDS (+25) inside [20,40) ⇒ Partial there (#680, the SDK's m_endOfData rule). (await src.ReadProcessedAsync("LS2", At(0), At(40), Twenty, HistoryAggregateType.Count, Ct)).Samples - .ShouldAllBe(x => (x.StatusCode & 0x4u) == 0); + .Select(x => x.StatusCode & Partial).ShouldBe([0u, Partial]); } [Fact] @@ -298,9 +346,7 @@ public abstract class ProcessedConformance : ConformanceBase ours.ShouldAllBe(x => (x.StatusCode & 0x4u) == 0, agg.ToString()); var tw = AggregateFamilies.FamilyOf(agg) == AggregateFamily.TimeWeighted; SdkAggregateOracle.ShouldMatch(ours, SdkAggregateOracle.Run(agg, B.Round(At(40)), B.Round(At(0)), ten, sdkRows), - $"{agg} reverse late start", compareCalculatedInterpolatedBits: tw, - // pre-existing, #680: the SDK's Delta is Good with a Bad sample present; ours Uncertain (none here). - compareCodeBits: agg != HistoryAggregateType.Delta); + $"{agg} reverse late start", compareCalculatedInterpolatedBits: tw); } } @@ -338,7 +384,6 @@ public abstract class ProcessedConformance : ConformanceBase }) { var ours = (await src.ReadProcessedAsync("P", At(from), At(to), interval, agg, Ct)).Samples.ToList(); - var startOrEnd = agg is HistoryAggregateType.Start or HistoryAggregateType.End; var tw = AggregateFamilies.FamilyOf(agg) == AggregateFamily.TimeWeighted; if (tw && agg != HistoryAggregateType.Interpolative) { @@ -372,9 +417,7 @@ public abstract class ProcessedConformance : ConformanceBase SdkAggregateOracle.ShouldMatch(ours, sdk, $"{agg} {from}→{to}/{intervalS}s", compareCalculatedInterpolatedBits: tw && !reverseInterpolative, - // pre-existing, #680: Start/End raw-sample timestamp + status; Delta status (see SdkAggregateParityTests). - compareCodeBits: agg != HistoryAggregateType.Delta && !reverseInterpolative && !startOrEnd, - compareTimestamps: !startOrEnd); + compareCodeBits: !reverseInterpolative); } } @@ -417,7 +460,15 @@ public abstract class ProcessedConformance : ConformanceBase (await src.ReadProcessedAsync("Z", At(0), At(20), Twenty, a, Ct)).Samples.Single(); (await One(HistoryAggregateType.Average)).StatusCode.ShouldBe(NoDataCalc); - (await One(HistoryAggregateType.Count)).StatusCode.ShouldBe(NoDataCalc); + // #680 (SDK value-based status): an interval holding only Bad samples counts as the generic Bad, value null. + var count = await One(HistoryAggregateType.Count); + count.StatusCode.ShouldBe(BadCalc); + count.Value.ShouldBeNull(); + // Start is the raw first sample: the outage marker itself, with its own status and time. + var start = await One(HistoryAggregateType.Start); + start.StatusCode.ShouldBe(BadNoCommunication); + start.SourceTimestampUtc.ShouldBe(B.Round(At(1))); + start.Value.ShouldBeNull(); var pb = await One(HistoryAggregateType.PercentBad); pb.Value.ShouldBe(100d); // The tag's data starts (+1) inside the interval ⇒ Partial on the non-Bad percentage (#677). diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb.IntegrationTests/ReadTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb.IntegrationTests/ReadTests.cs index 745343ab..9d6668f8 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb.IntegrationTests/ReadTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb.IntegrationTests/ReadTests.cs @@ -82,8 +82,15 @@ public sealed class ReadTests(QuestDbFixture fx) : IAsyncLifetime foreach (var agg in new[] { HistoryAggregateType.Average, HistoryAggregateType.Start, HistoryAggregateType.MaximumActualTime }) { var s = (await src.ReadProcessedAsync("A", start, end, TimeSpan.Zero, agg, Ct)).Samples.Single(); - // The data starts (+0) strictly inside the one interval ⇒ Partial on every aggregate but Average (#677). - s.StatusCode.ShouldBe(agg == HistoryAggregateType.Average ? UncertainCalc : UncertainCalc | 0x4u, agg.ToString()); + // The data starts (+0) strictly inside the one interval ⇒ Partial on every aggregate but Average (#677). Start + // is the raw first sample (#680): its own Good status, no Calculated bit, plus InfoType + Partial. + var expected = agg switch + { + HistoryAggregateType.Average => UncertainCalc, + HistoryAggregateType.Start => 0x00000404u, + _ => UncertainCalc | 0x4u, + }; + s.StatusCode.ShouldBe(expected, agg.ToString()); } (await src.ReadProcessedAsync("A", start, end, TimeSpan.Zero, HistoryAggregateType.Average, Ct)).Samples.Single().Value.ShouldBe(30d); (await src.ReadProcessedAsync("A", start, end, TimeSpan.Zero, HistoryAggregateType.Start, Ct)).Samples.Single().Value.ShouldBe(10d); diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common.Tests/AggregateEvaluatorTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common.Tests/AggregateEvaluatorTests.cs index aa516941..138e70bb 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common.Tests/AggregateEvaluatorTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common.Tests/AggregateEvaluatorTests.cs @@ -21,31 +21,36 @@ public sealed class AggregateEvaluatorTests Bucket = 0, Total = 2, Usable = 2, Good = 2, Bad = 0, NonNumeric = 0, Avg = 15, Min = 10, Max = 20, Sum = 30, First = HistorianValue.FromDouble(10), Last = HistorianValue.FromDouble(20), Extreme = maximum ? 20 : 10, ExtremeTime = maximum ? At(10) : At(0), + FirstAny = A[0], LastAny = A[1], }, [1] = new BucketRow { Bucket = 1, Total = 2, Usable = 1, Good = 1, Bad = 1, NonNumeric = 0, Avg = 40, Min = 40, Max = 40, Sum = 40, First = HistorianValue.FromDouble(40), Last = HistorianValue.FromDouble(40), Extreme = 40, ExtremeTime = At(30), + FirstAny = A[2], LastAny = A[3], }, [2] = new BucketRow { Bucket = 2, Total = 1, Usable = 1, Good = 1, Bad = 0, NonNumeric = 0, Avg = 50, Min = 50, Max = 50, Sum = 50, First = HistorianValue.FromDouble(50), Last = HistorianValue.FromDouble(50), Extreme = 50, ExtremeTime = At(40), + FirstAny = A[4], LastAny = A[4], }, }; + private const uint Partial = 0x4u; + private const uint BadCalc = 0x80000401u; // generic Bad | Calculated + + /// Tag A's data runs +0 … +40, so the forward read [0,80) holds the end of the data in B2 [40,60). private static IReadOnlyList ReadA(HistoryAggregateType agg, bool maximum = false) { var plan = BucketPlan.Create(At(0), At(80), TimeSpan.FromSeconds(20)); - return AggregateEvaluator.EvaluateAll(agg, plan, RowsA(maximum).Values, At(0), Now); + return AggregateEvaluator.EvaluateAll(agg, plan, RowsA(maximum).Values, At(0), At(40), Now); } [Theory] [InlineData(HistoryAggregateType.Average, 15d, 40d, 50d)] [InlineData(HistoryAggregateType.Minimum, 10d, 40d, 50d)] [InlineData(HistoryAggregateType.Maximum, 20d, 40d, 50d)] - [InlineData(HistoryAggregateType.Start, 10d, 40d, 50d)] - [InlineData(HistoryAggregateType.End, 20d, 40d, 50d)] [InlineData(HistoryAggregateType.Delta, 10d, 0d, 0d)] [InlineData(HistoryAggregateType.Range, 10d, 0d, 0d)] public void Numeric_aggregates_over_hand_built_bucket_rows(HistoryAggregateType agg, double b0, double b1, double b2) @@ -53,16 +58,143 @@ public sealed class AggregateEvaluatorTests var s = ReadA(agg); s.Count.ShouldBe(4); s.Select(x => x.Value).ShouldBe([b0, b1, b2, null]); - s.Select(x => x.StatusCode).ShouldBe([GoodCalc, UncertainCalc, GoodCalc, NoDataCalc]); + // #680: B2 holds the last sample (+40) ⇒ end-of-data Partial on a forward read, for every aggregate but Average. + // B1 opens on a Bad sample ⇒ Uncertain (for Delta: the interval's first sample was skipped). + var b2Status = agg == HistoryAggregateType.Average ? GoodCalc : GoodCalc | Partial; + s.Select(x => x.StatusCode).ShouldBe([GoodCalc, UncertainCalc, b2Status, NoDataCalc]); s.Select(x => x.SourceTimestampUtc).ShouldBe([At(0), At(20), At(40), At(60)]); } [Fact] - public void Count_counts_usable_samples_as_int() + public void Start_and_End_return_the_raw_first_and_last_sample_of_any_quality() { + // #680 (Part 13 Tables 73/74; SDK ComputeStartEnd): the raw sample — its own value (null when Bad), its own status + // (Raw, no Calculated bit) and its own timestamp. B1's Start is the Bad sample at +20. + var start = ReadA(HistoryAggregateType.Start); + start.Select(x => x.Value).ShouldBe([10d, null, 50d, null]); + start.Select(x => x.StatusCode).ShouldBe([Good, Bad, Good | 0x400u | Partial, NoDataCalc]); + start.Select(x => x.SourceTimestampUtc).ShouldBe([At(0), At(20), At(40), At(60)]); + var end = ReadA(HistoryAggregateType.End); + end.Select(x => x.Value).ShouldBe([20d, 40d, 50d, null]); + end.Select(x => x.StatusCode).ShouldBe([Good, Good, Good | 0x400u | Partial, NoDataCalc]); + end.Select(x => x.SourceTimestampUtc).ShouldBe([At(10), At(30), At(40), At(60)]); + } + + [Fact] + public void Start_and_End_follow_the_sample_order_in_a_reverse_read() + { + // Reverse (80 → 0): (20,40] = {+30, +40}, (0,20] = {+10, +20 Bad}. Start is still the EARLIEST sample of each + // interval and End the latest (the SDK's GetValues list is ascending in both directions). + var plan = BucketPlan.Create(At(80), At(0), TimeSpan.FromSeconds(20)); + var rows = new[] + { + new BucketRow { Bucket = 1, Total = 2, Usable = 2, Good = 2, FirstAny = A[3], LastAny = A[4] }, + new BucketRow { Bucket = 0, Total = 2, Usable = 1, Good = 1, Bad = 1, FirstAny = A[1], LastAny = A[2] }, + }; + var start = AggregateEvaluator.EvaluateAll(HistoryAggregateType.Start, plan, rows, At(0), At(40), Now); + start.Select(x => x.SourceTimestampUtc).ShouldBe([At(80), At(60), At(30), At(10)]); + start.Select(x => x.Value).ShouldBe([null, null, 40d, 20d]); + var end = AggregateEvaluator.EvaluateAll(HistoryAggregateType.End, plan, rows, At(0), At(40), Now); + end.Select(x => x.SourceTimestampUtc).ShouldBe([At(80), At(60), At(40), At(20)]); + end.Select(x => x.Value).ShouldBe([null, null, 50d, null]); + end.Select(x => x.StatusCode).ShouldBe([NoDataCalc, NoDataCalc, Good, Bad]); + } + + [Fact] + public void Delta_is_good_when_a_bad_sample_sits_mid_interval_and_uncertain_when_one_bounds_it() + { + // SDK ComputeDelta: Uncertain only when the first or last sample of the interval is skipped (not usable). + var mid = new BucketRow + { + Total = 3, Usable = 2, Good = 2, Bad = 1, First = HistorianValue.FromDouble(1), Last = HistorianValue.FromDouble(4), + FirstAny = S(0, HistorianValue.FromDouble(1), Good), LastAny = S(9, HistorianValue.FromDouble(4), Good), + }; + var d = AggregateEvaluator.Evaluate(HistoryAggregateType.Delta, mid, At(0), Now); + d.Value.ShouldBe(3d); + d.StatusCode.ShouldBe(GoodCalc); + mid.LastAny = S(9, HistorianValue.Empty, BadNoCommunication); + AggregateEvaluator.Evaluate(HistoryAggregateType.Delta, mid, At(0), Now).StatusCode.ShouldBe(UncertainCalc); + // An Uncertain bound is used (TreatUncertainAsBad = false), so it does not degrade Delta. + mid.LastAny = S(9, HistorianValue.FromDouble(4), Uncertain); + AggregateEvaluator.Evaluate(HistoryAggregateType.Delta, mid, At(0), Now).StatusCode.ShouldBe(GoodCalc); + } + + [Fact] + public void Count_counts_good_samples_as_int() + { + // #680 (Part 13 §5.4.3.21; SDK ComputeCount): Good samples only; value-based status; end-of-data Partial on B2. var s = ReadA(HistoryAggregateType.Count); s.Select(x => x.Value).ShouldBe([(object?)2, 1, 1, null]); - s.Select(x => x.StatusCode).ShouldBe([GoodCalc, UncertainCalc, GoodCalc, NoDataCalc]); + s.Select(x => x.StatusCode).ShouldBe([GoodCalc, UncertainCalc, GoodCalc | Partial, NoDataCalc]); + } + + [Fact] + public void Count_excludes_uncertain_samples_and_an_all_bad_interval_is_generic_bad() + { + var unc = new BucketRow { Total = 3, Usable = 3, Good = 1, Bad = 0 }; + var c = AggregateEvaluator.Evaluate(HistoryAggregateType.Count, unc, At(0), Now); + c.Value.ShouldBe(1); + c.StatusCode.ShouldBe(UncertainCalc); + var bad = AggregateEvaluator.Evaluate(HistoryAggregateType.Count, new BucketRow { Total = 2, Bad = 2 }, At(0), Now); + bad.Value.ShouldBeNull(); + bad.StatusCode.ShouldBe(BadCalc); + } + + [Fact] + public void An_empty_interval_inside_the_data_range_counts_zero_and_outside_it_is_no_data() + { + // #680: forward [0,60) at 10 s over data +1 … +45: [10,20) and [30,40) hold no sample but lie inside + // [earliest, latest] ⇒ 0, Good; [50,60) is after the last sample ⇒ BadNoData. + var plan = BucketPlan.Create(At(0), At(60), TimeSpan.FromSeconds(10)); + var rows = new[] + { + new BucketRow { Bucket = 0, Total = 1, Usable = 1, Good = 1 }, + new BucketRow { Bucket = 2, Total = 1, Usable = 1, Good = 1 }, + new BucketRow { Bucket = 4, Total = 1, Usable = 1, Good = 1 }, + }; + var s = AggregateEvaluator.EvaluateAll(HistoryAggregateType.Count, plan, rows, At(1), At(45), Now); + s.Select(x => x.Value).ShouldBe([(object?)1, 0, 1, 0, 1, null]); + s.Select(x => x.StatusCode).ShouldBe( + [GoodCalc | Partial, GoodCalc, GoodCalc, GoodCalc, GoodCalc | Partial, NoDataCalc]); + s.Select(x => x.SourceTimestampUtc).ShouldBe([At(0), At(10), At(20), At(30), At(40), At(50)]); + + // Reverse (60 → 0): (40,50] is inside the range (its earlier edge +40 < +45); (50,60] is after it. A reverse read + // carries no Partial bits. + var rev = BucketPlan.Create(At(60), At(0), TimeSpan.FromSeconds(10)); + var r = AggregateEvaluator.EvaluateAll(HistoryAggregateType.Count, rev, rows, At(1), At(45), Now); + r.Select(x => x.Value).ShouldBe([null, (object?)1, 0, 1, 0, 1]); + r.Select(x => x.StatusCode).ShouldBe([NoDataCalc, GoodCalc, GoodCalc, GoodCalc, GoodCalc, GoodCalc]); + + // Other per-sample aggregates keep BadNoData on an empty interval. + AggregateEvaluator.EvaluateAll(HistoryAggregateType.Minimum, plan, rows, At(1), At(45), Now)[1].StatusCode + .ShouldBe(NoDataCalc); + // No data range known (an unknown tag / no sample near the window) ⇒ every empty interval is BadNoData. + AggregateEvaluator.EvaluateAll(HistoryAggregateType.Count, plan, [], null, null, Now) + .ShouldAllBe(x => x.StatusCode == NoDataCalc); + } + + [Fact] + public void End_of_data_Partial_is_forward_only_and_inclusive_of_the_interval_start() + { + var row = new BucketRow { Bucket = 2, Total = 1, Usable = 1, Good = 1, Min = 3, Avg = 3 }; + var fwd = BucketPlan.Create(At(0), At(40), TimeSpan.FromSeconds(10)); + // Latest sample exactly on the interval's earlier edge (+20) ⇒ Partial (SDK m_endOfData >= startTime). + AggregateEvaluator.EvaluateAll(HistoryAggregateType.Minimum, fwd, [row], At(-5), At(20), Now)[2].StatusCode + .ShouldBe(GoodCalc | Partial); + // Data past the window end ⇒ no end-of-data Partial. + AggregateEvaluator.EvaluateAll(HistoryAggregateType.Minimum, fwd, [row], At(-5), At(100), Now)[2].StatusCode + .ShouldBe(GoodCalc); + // Average never sets it. + AggregateEvaluator.EvaluateAll(HistoryAggregateType.Average, fwd, [row], At(-5), At(25), Now)[2].StatusCode + .ShouldBe(GoodCalc); + // A Bad result never carries Partial (ours; the SDK sets it on a Bad result too). + var bad = new BucketRow { Bucket = 2, Total = 1, Bad = 1 }; + AggregateEvaluator.EvaluateAll(HistoryAggregateType.Minimum, fwd, [bad], At(-5), At(25), Now)[2].StatusCode + .ShouldBe(NoDataCalc); + // Reverse: not applied. + var rev = BucketPlan.Create(At(40), At(0), TimeSpan.FromSeconds(10)); + AggregateEvaluator.EvaluateAll(HistoryAggregateType.Minimum, rev, [row], At(-5), At(25), Now) + .ShouldAllBe(x => (x.StatusCode & Partial) == 0); } [Theory] @@ -81,7 +213,7 @@ public sealed class AggregateEvaluatorTests var min = ReadA(HistoryAggregateType.MinimumActualTime); min.Select(x => x.Value).ShouldBe([10d, 40d, 50d, null]); min.Select(x => x.SourceTimestampUtc).ShouldBe([At(0), At(30), At(40), At(60)]); - min.Select(x => x.StatusCode).ShouldBe([GoodCalc, UncertainCalc, GoodCalc, NoDataCalc]); + min.Select(x => x.StatusCode).ShouldBe([GoodCalc, UncertainCalc, GoodCalc | Partial, NoDataCalc]); var max = ReadA(HistoryAggregateType.MaximumActualTime, maximum: true); max.Select(x => x.Value).ShouldBe([20d, 40d, 50d, null]); max.Select(x => x.SourceTimestampUtc).ShouldBe([At(10), At(30), At(40), At(60)]); @@ -107,6 +239,8 @@ public sealed class AggregateEvaluatorTests avg.Value.ShouldBeNull(); avg.StatusCode.ShouldBe(NoDataCalc); AggregateEvaluator.Evaluate(HistoryAggregateType.PercentBad, row, At(0), Now).Value.ShouldBe(100d); + // #680: Count of an all-Bad interval is the generic Bad (the SDK's value-based status), not BadNoData. + AggregateEvaluator.Evaluate(HistoryAggregateType.Count, row, At(0), Now).StatusCode.ShouldBe(BadCalc); } [Fact] diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common.Tests/BucketPlanTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common.Tests/BucketPlanTests.cs index 972bb7c2..d32efdea 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common.Tests/BucketPlanTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common.Tests/BucketPlanTests.cs @@ -133,15 +133,44 @@ public sealed class BucketPlanTests } [Fact] - public void The_earliest_sample_is_needed_only_for_forward_reads_of_aggregates_other_than_Average() + public void Data_ends_inside_from_the_interval_start_inclusive_to_its_end_exclusive() { - // Review F2: backends skip the earliest-sample query when its answer cannot change a result. + var p = BucketPlan.Create(At(0), At(40), TimeSpan.FromSeconds(20)); + p.DataEndsInside(0, At(5)).ShouldBeTrue(); + p.DataEndsInside(1, At(20)).ShouldBeTrue(); // on the earlier edge: the SDK's m_endOfData >= startTime + p.DataEndsInside(0, At(20)).ShouldBeFalse(); + p.DataEndsInside(1, At(40)).ShouldBeFalse(); + p.DataEndsInside(0, null).ShouldBeFalse(); + } + + [Fact] + public void The_data_range_includes_both_of_its_edges() + { + var p = BucketPlan.Create(At(0), At(60), TimeSpan.FromSeconds(20)); + p.WithinDataRange(0, At(10), At(30)).ShouldBeTrue(); + p.WithinDataRange(1, At(10), At(30)).ShouldBeTrue(); + p.WithinDataRange(2, At(10), At(30)).ShouldBeFalse(); // [40,60) is after the last sample + p.WithinDataRange(2, At(10), At(40)).ShouldBeTrue(); // last sample on the earlier edge + p.WithinDataRange(0, At(20), At(30)).ShouldBeTrue(); // first sample on the later edge + p.WithinDataRange(0, At(21), At(30)).ShouldBeFalse(); + p.WithinDataRange(0, null, At(30)).ShouldBeFalse(); + p.WithinDataRange(0, At(10), null).ShouldBeFalse(); + } + + [Fact] + public void The_data_range_is_needed_for_forward_reads_of_aggregates_other_than_Average_and_for_Count() + { + // Review F2 (#677) / #680: backends skip the earliest/latest-sample queries when their answer cannot change a + // result — the start/end-of-data Partial rules are forward-only and not Average's; Count's in-range empty interval + // (0, not BadNoData) applies in both directions. var fwd = BucketPlan.Create(At(0), At(40), TimeSpan.FromSeconds(10)); var rev = BucketPlan.Create(At(40), At(0), TimeSpan.FromSeconds(10)); - AggregateEvaluator.NeedsEarliestSample(HistoryAggregateType.Count, fwd).ShouldBeTrue(); - AggregateEvaluator.NeedsEarliestSample(HistoryAggregateType.Average, fwd).ShouldBeFalse(); - AggregateEvaluator.NeedsEarliestSample(HistoryAggregateType.Count, rev).ShouldBeFalse(); - AggregateEvaluator.NeedsEarliestSample(HistoryAggregateType.Count, BucketPlan.Create(At(0), At(0), TimeSpan.Zero)) + AggregateEvaluator.NeedsDataRange(HistoryAggregateType.Count, fwd).ShouldBeTrue(); + AggregateEvaluator.NeedsDataRange(HistoryAggregateType.Minimum, fwd).ShouldBeTrue(); + AggregateEvaluator.NeedsDataRange(HistoryAggregateType.Average, fwd).ShouldBeFalse(); + AggregateEvaluator.NeedsDataRange(HistoryAggregateType.Count, rev).ShouldBeTrue(); + AggregateEvaluator.NeedsDataRange(HistoryAggregateType.Minimum, rev).ShouldBeFalse(); + AggregateEvaluator.NeedsDataRange(HistoryAggregateType.Count, BucketPlan.Create(At(0), At(0), TimeSpan.Zero)) .ShouldBeFalse(); } diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common.Tests/SdkAggregateParityTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common.Tests/SdkAggregateParityTests.cs index f7d03091..9c6c00fa 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common.Tests/SdkAggregateParityTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common.Tests/SdkAggregateParityTests.cs @@ -16,9 +16,9 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common.Tests; /// /// Deliberate, documented divergences the datasets avoid (they are not anchoring questions): /// -/// an empty interval inside the data range: the SDK's Count answers 0/Good, ours BadNoData; -/// Uncertain samples in a per-sample aggregate: the SDK's Average/Count/Min/Max use Good samples only, -/// ours every usable (Good + Uncertain) sample (TreatUncertainAsBad = false applied consistently); +/// Uncertain samples in a per-sample aggregate: the SDK's Average/Min/Max use Good samples only, +/// ours every usable (Good + Uncertain) sample (TreatUncertainAsBad = false applied consistently). Count matches +/// the SDK since #680 (Good samples only); /// the per-sample Calculated / MultipleValues info bits (the SDK's MinMax sets Calculated only when the /// extreme is not at the interval start, and flags ties) — those comparisons check code bits + Partial only; /// a truncated interval: ours sets Partial on a non-Bad time-weighted result (Part 13 §5.3.3.2 Case 3 @@ -46,6 +46,8 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common.Tests; /// UpdateSlice files it as the early bound (below), so the interval takes the status of the segment beyond /// it (Uncertain when that segment skips a Bad sample); ours keeps the interval's own segments — masked by /// ; +/// a Bad per-sample result never carries Partial (the SDK sets start/end-of-data Partial on Bad results +/// too) — the datasets keep the data's start and end out of all-Bad intervals; /// Interpolative, reverse, a sample exactly at the point: the SDK's backward UpdateSlice files /// a sample AT the point as the early bound (>= 0), so it interpolates from it to the next Good sample — /// same value, but the Interpolated bit, and Uncertain when a Bad sample sits between. Ours returns the raw @@ -164,7 +166,7 @@ public sealed class SdkAggregateParityTests if (AggregateFamilies.FamilyOf(agg) == AggregateFamily.TimeWeighted) return TimeWeightedAggregator.Evaluate(agg, plan, data, Now); return AggregateEvaluator.EvaluateAll(agg, plan, InMemoryRows(plan, data, agg == HistoryAggregateType.MaximumActualTime), - data.Min(s => s.TimeUtc), Now); + data.Min(s => s.TimeUtc), data.Max(s => s.TimeUtc), Now); } private static IReadOnlyList Sdk(HistoryAggregateType agg, double from, double to, @@ -305,12 +307,7 @@ public sealed class SdkAggregateParityTests { SdkAggregateOracle.ShouldMatch(Ours(agg, from, to, intervalS, Dense), Sdk(agg, from, to, intervalS, Dense), $"{agg} Dense {from}→{to}/{intervalS}s", compareCalculatedInterpolatedBits: false, - // pre-existing, #680: the SDK's Start/End return the raw first/last sample (any quality) with its own - // timestamp and stored status; ours return the first/last USABLE value stamped with the interval, under the - // per-sample status rule. The SDK's Delta is Good even when a Bad sample sits in the interval; ours is - // Uncertain. Values are compared wherever the SDK's answer is not Bad. - compareTimestamps: !StartOrEnd(agg), - compareCodeBits: !StartOrEnd(agg) && agg != HistoryAggregateType.Delta); + compareValues: true); } catch (Exception ex) { @@ -327,8 +324,6 @@ public sealed class SdkAggregateParityTests HistoryAggregateType.Delta, HistoryAggregateType.MinimumActualTime, HistoryAggregateType.MaximumActualTime, ]; - private static bool StartOrEnd(HistoryAggregateType agg) => - agg is HistoryAggregateType.Start or HistoryAggregateType.End; [Theory] [InlineData(HistoryAggregateType.TimeAverage)] @@ -351,9 +346,7 @@ public sealed class SdkAggregateParityTests var ours = Ours(agg, 40, 0, 10, LateStart); SdkAggregateOracle.ShouldMatch(ours, Sdk(agg, 40, 0, 10, LateStart), $"{agg} reverse LateStart", compareCalculatedInterpolatedBits: tw, - // pre-existing, #680: Start's raw-sample timestamp/status (see Per_sample_aggregates_match_the_SDK). - compareTimestamps: !StartOrEnd(agg), - compareCodeBits: !StartOrEnd(agg) && agg != HistoryAggregateType.Delta); + compareCodeBits: true); ours.ShouldAllBe(x => (x.StatusCode & HistorianInfoBits.HistorianPartial) == 0); } @@ -379,6 +372,121 @@ public sealed class SdkAggregateParityTests ours.Skip(1).ShouldAllBe(x => (x.StatusCode & HistorianInfoBits.HistorianPartial) == 0); } + /// + /// #680: sparse data with Good, Uncertain and Bad samples, empty intervals inside the data range, and intervals before + /// and after it: +1 G, +2 U, +25 Bad, +45 G, +70 G. + /// + private static readonly StoredSample[] Gappy = + [ + S(1, HistorianValue.FromDouble(10), Good), + S(2, HistorianValue.FromDouble(11), Uncertain), + S(25, HistorianValue.FromDouble(12), Bad), + S(45, HistorianValue.FromDouble(13), Good), + S(70, HistorianValue.FromDouble(1), Good), + ]; + + /// #680: Bad samples leading, trailing and mid-interval, plus an Uncertain one (Start / End / Delta). + private static readonly StoredSample[] Bracketed = + [ + S(-5, HistorianValue.FromDouble(1), Good), + S(1, HistorianValue.FromDouble(10), Bad), + S(3, HistorianValue.FromDouble(11), Good), + S(5, HistorianValue.FromDouble(12), Bad), + S(7, HistorianValue.FromDouble(13), Uncertain), + S(9, HistorianValue.FromDouble(15), Good), + S(12, HistorianValue.FromDouble(20), Good), + S(14, HistorianValue.Empty, BadNoCommunication), + S(16, HistorianValue.FromDouble(21), Good), + S(19, HistorianValue.Empty, BadNoCommunication), + S(40, HistorianValue.FromDouble(1), Good), + ]; + + public static TheoryData GappyWindows => new() + { + { 0, 60, 10 }, { 60, 0, 10 }, // empty intervals inside the data range; +70 is past the window + { -10, 1, 5 }, { 1, -10, 5 }, // intervals before the first sample; one ending ON it (forward) + { 70, 85, 5 }, { 85, 70, 5 }, // intervals after the last sample; one starting ON it (reverse) + { 0, 1, 1 }, // one interval ending exactly on the first sample: empty but in range + }; + + [Theory] + [MemberData(nameof(GappyWindows))] + public void Count_Start_End_and_Delta_match_the_SDK_across_the_data_range(double from, double to, double intervalS) + { + // #680: Count over Good samples only with the value-based status, 0 (not BadNoData) for an empty interval inside + // [first, last], BadNoData outside it; Start/End the raw first/last sample; Delta's status from its bounding samples. + var failures = new List(); + foreach (var agg in new[] + { + HistoryAggregateType.Count, HistoryAggregateType.Start, HistoryAggregateType.End, HistoryAggregateType.Delta, + }) + foreach (var (name, data) in new[] { ("Gappy", Gappy), ("Bracketed", Bracketed) }) + { + try + { + SdkAggregateOracle.ShouldMatch(Ours(agg, from, to, intervalS, data), Sdk(agg, from, to, intervalS, data), + $"{agg} {name} {from}→{to}/{intervalS}s", compareCalculatedInterpolatedBits: false); + } + catch (Exception ex) + { + failures.Add(ex.Message); + } + } + failures.ShouldBeEmpty(string.Join("\n", failures)); + } + + [Theory] + [InlineData(0, 20, 10)] + [InlineData(20, 0, 10)] + [InlineData(0, 20, 0)] + [InlineData(20, 0, 0)] + public void Start_End_and_Delta_over_bad_bounding_samples_match_the_SDK_exactly(double from, double to, double intervalS) + { + // #680: with the raw sample's own status (incl. a Bad one and its null value) Start/End must match the SDK to the + // bit; Delta's Calculated bit as well. + foreach (var agg in new[] { HistoryAggregateType.Start, HistoryAggregateType.End, HistoryAggregateType.Delta }) + SdkAggregateOracle.ShouldMatch(Ours(agg, from, to, intervalS, Bracketed), Sdk(agg, from, to, intervalS, Bracketed), + $"{agg} Bracketed {from}→{to}/{intervalS}s"); + } + + /// #680: data ending inside the window (−5, +3, +25) and ending exactly on an interval edge (+20). + private static readonly StoredSample[] EarlyEnd = + [S(-5, HistorianValue.FromDouble(1), Good), S(3, HistorianValue.FromDouble(2), Good), S(25, HistorianValue.FromDouble(3), Good)]; + + private static readonly StoredSample[] EndOnEdge = + [S(-5, HistorianValue.FromDouble(1), Good), S(3, HistorianValue.FromDouble(2), Good), S(20, HistorianValue.FromDouble(3), Good)]; + + [Theory] + [InlineData(0, 40, 10)] + [InlineData(0, 40, 25)] + [InlineData(0, 40, 0)] + [InlineData(40, 0, 10)] + public void The_interval_the_data_ends_inside_matches_the_SDK_Partial_bit(double from, double to, double intervalS) + { + // #680 (Part 13 §5.3.3.2 Case 2; SDK m_endOfData): forward, the interval holding the last sample (its earlier edge + // included) is Partial for every per-sample aggregate but Average. Reverse 40 → 0 over EndOnEdge: the SDK's + // reverse rule (strictly inside, see the remarks) does not fire on an edge, and ours sets nothing. + var failures = new List(); + var datasets = from > to ? new[] { ("EndOnEdge", EndOnEdge) } : new[] { ("EarlyEnd", EarlyEnd), ("EndOnEdge", EndOnEdge) }; + foreach (var agg in PerSample) + foreach (var (name, data) in datasets) + { + try + { + SdkAggregateOracle.ShouldMatch(Ours(agg, from, to, intervalS, data), Sdk(agg, from, to, intervalS, data), + $"{agg} {name} {from}→{to}/{intervalS}s", compareCalculatedInterpolatedBits: false); + } + catch (Exception ex) + { + failures.Add(ex.Message); + } + } + failures.ShouldBeEmpty(string.Join("\n", failures)); + // Not vacuous: the forward EarlyEnd read does flag [20,30) (and only it). + Ours(HistoryAggregateType.Minimum, 0, 40, 10, EarlyEnd).Select(x => x.StatusCode & HistorianInfoBits.HistorianPartial) + .ShouldBe([0u, 0u, HistorianInfoBits.HistorianPartial, 0u]); + } + /// The per-bucket statistics the backends' SQL computes, built in memory with . internal static IEnumerable InMemoryRows(BucketPlan plan, IEnumerable samples, bool maximum = false) { @@ -408,6 +516,8 @@ public sealed class SdkAggregateParityTests Last = usable.Count == 0 ? HistorianValue.Empty : usable[^1].Value, Extreme = extreme?.Value.Number, ExtremeTime = extreme?.TimeUtc, + FirstAny = all[0], + LastAny = all[^1], }; } }