From b5e3e5108ca839e27eca913b4e65a5f48cd23f20 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Sun, 11 Oct 2026 16:39:39 -0400 Subject: [PATCH] =?UTF-8?q?docs(historian):=20#680=20lane=20B=20review=20?= =?UTF-8?q?=E2=80=94=20QuestDB=20data-range=20lookups=20are=20partition=20?= =?UTF-8?q?scans,=20both-direction=20gate,=20all-Bad=20Count=200x80000401,?= =?UTF-8?q?=20outage-last=20parity=20test,=20CLAUDE.md=20per-sample=20bull?= =?UTF-8?q?et?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CLAUDE.md | 2 +- docs/Historian.md | 20 +++++---- .../QuestDbHistorianDataSource.cs | 2 +- .../SqlServerHistorianDataSource.cs | 2 +- .../SdkAggregateParityTests.cs | 43 +++++++++++++++++++ 5 files changed, 58 insertions(+), 11 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 57b72912..0d6f9d5b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1350,7 +1350,7 @@ historian tagname, which is the tag's driver `FullName`. ### Pluggable backends, SQL Server and HistoryUpdate (2026-10) - **Selection:** `ServerHistorian:Backend` (`Gateway` default | `SqlServer` | `QuestDb`; unknown fails host start). `HistorianBackendRegistry` → `IHistorianBackendProvider` builds all five seams; the gateway keys are only required for `Gateway`. `SqlServer` uses `ServerHistorian:SqlServer:{ConnectionString (required), Schema=hist, CommandTimeout=30s, AutoCreateSchema=true}`. -- **SQL Server** (`Driver.Historian.SqlServer`): `hist.Sample` PK `(TagId, SourceTimeUtc)` — writes are insert-if-absent, so outbox redelivery is idempotent. Stores every value kind with its **real status**; per-sample aggregates (10, simple definitions) are pushed down to T-SQL with bucket math in **100 ns ticks**; the Part 13 time-weighted seven (TimeAverage/TimeAverage2/Total/Total2/Interpolative, #671 — ⚠️ `Total` is now the time-weighted integral, no longer `SUM` — plus the **duration-based** PercentGood/PercentBad, #680) stream window + 4 bound rows into the shared `TimeWeightedAggregator` (both SQL backends, identical results; reverse `Interpolative` anchored at the later time; window capped by `ServerHistorian:MaxTimeWeightedRawRows` ⇒ `BadTooManyOperations`, percentages included). ⚠️ **#680 status rules (client-visible):** TimeAverage/Total are Uncertain if any part of the interval uses an Uncertain value, is bridged across a Bad sample, is extrapolated past the last sample (⇒ a live tag's latest interval is Uncertain) or is not covered; TimeAverage2/Total2 keep the SDK time-based rule (Uncertain time counts as Good) but never extrapolate past the last sample — that interval is Partial, later ones `BadNoData` (a report-by-exception tag with nothing after its last sample reads `BadNoData` until its next sample arrives). `docs/Historian.md` §"Time-weighted aggregates". Docker-dev recipe, schema and aggregate semantics: `docs/Historian.md` §"SQL Server backend". Integration tests need `HISTORIAN_SQL_TEST_CONNECTION`. **Retention (#667):** opt-in `SqlServer:RetentionDays` (+ `PurgeInterval` 1h, `PurgeBatchSize` 4000) -> `HistorianRetentionService` purges old `hist.Sample` (per tag) / `hist.Event` in `DELETE TOP(n)` batches, Primary-gated on `IHistorizationRoleView`, one purger per DB via `sp_getapplock`; counter `otopcua.historian.retention.purged{table}`. Optional `IHistorianBackendProvider.CreateRetention` (default null). `docs/Historian.md` §Retention. +- **SQL Server** (`Driver.Historian.SqlServer`): `hist.Sample` PK `(TagId, SourceTimeUtc)` — writes are insert-if-absent, so outbox redelivery is idempotent. Stores every value kind with its **real status**; per-sample aggregates (10, simple definitions) are pushed down to T-SQL with bucket math in **100 ns ticks**; the Part 13 time-weighted seven (TimeAverage/TimeAverage2/Total/Total2/Interpolative, #671 — ⚠️ `Total` is now the time-weighted integral, no longer `SUM` — plus the **duration-based** PercentGood/PercentBad, #680) stream window + 4 bound rows into the shared `TimeWeightedAggregator` (both SQL backends, identical results; reverse `Interpolative` anchored at the later time; window capped by `ServerHistorian:MaxTimeWeightedRawRows` ⇒ `BadTooManyOperations`, percentages included). ⚠️ **#680 status rules (client-visible):** TimeAverage/Total are Uncertain if any part of the interval uses an Uncertain value, is bridged across a Bad sample, is extrapolated past the last sample (⇒ a live tag's latest interval is Uncertain) or is not covered; TimeAverage2/Total2 keep the SDK time-based rule (Uncertain time counts as Good) but never extrapolate past the last sample — that interval is Partial, later ones `BadNoData` (a report-by-exception tag with nothing after its last sample reads `BadNoData` until its next sample arrives). `docs/Historian.md` §"Time-weighted aggregates". ⚠️ **#680 per-sample (client-visible):** `Count` counts **Good** samples only (value-based status; an all-Bad interval is generic Bad, value null) and an empty interval **inside the tag's data range** is `0`/Good (outside it still `BadNoData`); `Start`/`End` return the **raw** first/last sample of any quality — its own timestamp and status, value null when Bad; `Delta` is Uncertain only when the interval's first or last sample is unusable; and every per-sample aggregate but `Average` sets **end-of-data Partial** on the interval holding the last sample (forward incl. its earlier edge, reverse strictly inside), never on a Bad result. The data-range lookup is a seek on SQL Server but a partition scan on QuestDB (unindexed `tag` SYMBOL). `docs/Historian.md` §"Aggregates". Docker-dev recipe, schema and aggregate semantics: `docs/Historian.md` §"SQL Server backend". Integration tests need `HISTORIAN_SQL_TEST_CONNECTION`. **Retention (#667):** opt-in `SqlServer:RetentionDays` (+ `PurgeInterval` 1h, `PurgeBatchSize` 4000) -> `HistorianRetentionService` purges old `hist.Sample` (per tag) / `hist.Event` in `DELETE TOP(n)` batches, Primary-gated on `IHistorizationRoleView`, one purger per DB via `sp_getapplock`; counter `otopcua.historian.retention.purged{table}`. Optional `IHistorianBackendProvider.CreateRetention` (default null). `docs/Historian.md` §Retention. - **QuestDB** (`Driver.Historian.QuestDb`, `ServerHistorian:QuestDb:*`, Npgsql over PG wire; docker-dev overlay `docker-compose.questdb.yml`): same kinds/status/aggregates/HistoryUpdate as SqlServer, soft delete by tombstone (INSERT only — never `UPDATE`/`DELETE`; every read filters `NOT deleted`). ⚠️ **Timestamps are µs** (`QuestDbConnection.Round`/`BindTimestamp`; `IHistorianDataSource.TimestampResolutionTicks` = 10 steps HistoryRead paging), **pre-1970 cannot be stored** (writer drops, HistoryUpdate `BadOutOfRange`), every write waits for WAL visibility, TTL is enforced on write commits only, set Npgsql `Maximum Pool Size` < QuestDB's 64. `docs/Historian.md` §"QuestDB backend". - ⚠️ **Reverse processed reads are Part 13/SDK-anchored (#677, client-visible):** every aggregate on SqlServer/QuestDB now anchors a reverse read at its later time — window `(end, start]`, partial interval earliest, each stamped with its later edge (was: earlier-anchored, stamped earlier edge); on forward reads Partial also marks the interval the data starts inside (not Average — the SDK's rule, forward-only); a non-default client `AggregateConfiguration` ⇒ `BadAggregateConfigurationRejected`; Gateway provisioning sends `IntegralDivisor = 1`. SDK-parity tests: `SdkAggregateParityTests`. `docs/Historian.md` §Aggregates. - **Real status fix:** the recorder used to stamp every sample Good (`192`); it now carries the driver's status. ⚠️ On the **Gateway** backend this changes nothing stored: the gateway discards quality (C-002), so AVEVA still stores Good. diff --git a/docs/Historian.md b/docs/Historian.md index 6d3bfa6f..c8b00913 100644 --- a/docs/Historian.md +++ b/docs/Historian.md @@ -938,7 +938,7 @@ point (see [Continuation-point paging (Events)](#continuation-point-paging-event > ⚠️ **#680 (client-visible): `Count`, `Start`, `End`, `Delta` and the end-of-data Partial bit now follow Part 13 / the SDK** on the > `SqlServer` and `QuestDb` backends: > - **`Count`** counts **Good** samples only (it used to count Good + Uncertain); its status is value-based — Good when every sample in -> the interval is Good, the generic **Bad** (`0x80000000`, value null) when every one is Bad (it used to be `BadNoData`), else +> the interval is Good, the generic **Bad** (`0x80000401` — generic Bad + our Calculated bit; the SDK returns a bare `0x80000000` — value null) when every one is Bad (it used to be `BadNoData`), else > Uncertain. An **empty interval inside the tag's data range** `[first sample, last sample]` is now **`0`, Good** (it used to be > `BadNoData`); an empty interval wholly before the first or after the last sample stays `BadNoData`. Both directions. > - **`Start` / `End`** return the interval's **raw** first / last sample of **any** quality — its own value (null when Bad), its own @@ -985,10 +985,13 @@ Data continuing past the window sets nothing. Two deliberate limits, kept: the p *truncated* interval (only the time-weighted ones do), and **a Bad result never carries Partial** (the SDK sets it on Bad results too). **Data-range queries.** The start/end-of-data rules and `Count`'s in-range empty interval need the tag's **data range**: its earliest -stored sample (any quality) at or before the window's later edge and its latest at or after the earlier edge — two single-key seeks -(SQL Server: one extra result set of two `MIN`/`MAX` subqueries; QuestDB: two `LIMIT 1` statements). They run only when the answer can -change a result (`AggregateEvaluator.NeedsDataRange`): a per-sample read of an aggregate other than `Average` (either direction), or -`Count`; a per-sample `Average` read skips them. +stored sample (any quality) at or before the window's later edge and its latest at or after the earlier edge. On **SQL Server** these +are one extra result set of two `MIN`/`MAX` subqueries, each a single seek on the `(TagId, SourceTimeUtc)` key. On **QuestDB** they are +two `LIMIT 1` statements, but `tag` is an **unindexed `SYMBOL`**, so each is a **partition scan**, not a seek: measured 1–3 ms for a +live tag, and for a stopped tag read over an old window 6–8 ms warm / ~32 ms cold on a 52 M-row table — the cost grows with the table +(bounded by the TTL), not with the window. They run only when the answer can change a result +(`AggregateEvaluator.NeedsDataRange`): a per-sample read of any aggregate other than `Average`, in either direction (`Count` +included); a per-sample `Average` read skips them. **Sample classes.** *Usable* = severity Good or Uncertain **and** a stored value (not `Empty`). *Good* = usable with severity Good. *Bad* = not usable. @@ -1399,10 +1402,11 @@ Every query filters `NOT deleted`; every value goes through `StoredSample`, so t reverse read (#675) is `(lo, hi]` ordered `(ts, event_id)` descending. - **Processed:** one query per node; bucket index `(cast(ts AS LONG) * 10 - bucketBaseUnixTicks) / intervalTicks` (never `SAMPLE BY`, whose alignment differs), so bucket edges are identical to `SqlServer` (forward start-inclusive / end-exclusive with the partial bucket last; reverse anchored at the - later time, later-edge-inclusive, partial bucket earliest — #677; interval 0 ⇒ one bucket, 100,000-bucket cap). On a forward read of any aggregate - but Average, and for Count in either direction, two `LIMIT 1` statements fetch the tag's data range — its earliest live sample at or before the + later time, later-edge-inclusive, partial bucket earliest — #677; interval 0 ⇒ one bucket, 100,000-bucket cap). On a per-sample read of any + aggregate but Average, in either direction, two `LIMIT 1` statements fetch the tag's data range — its earliest live sample at or before the window's later edge and its latest at or after its earlier edge — for the start/end-of-data Partial bits and Count's in-range empty interval - (#677/#680). Start/End/Delta also return each bucket's first and last sample of any quality (`row_number()` over every row). + (#677/#680). `tag` is an unindexed SYMBOL, so these are partition scans (a few ms on a live tag; tens of ms cold for a stopped tag on a + large table — see *Data-range queries* above). Start/End/Delta also return each bucket's first and last sample of any quality (`row_number()` over every row). Min/max times use `row_number()` windows with the earliest-timestamp tie-break. ### Retention (TTL) diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb/QuestDbHistorianDataSource.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb/QuestDbHistorianDataSource.cs index 8cf13bc1..638c1611 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb/QuestDbHistorianDataSource.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb/QuestDbHistorianDataSource.cs @@ -495,7 +495,7 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource rows.Add(ReadBucket(reader, family)); } // The tag's data range (earliest + latest live sample): the start/end-of-data Partial rules (#677/#680 — - // forward reads of an aggregate other than Average) and Count's in-range empty interval (#680); skipped + // any per-sample aggregate but Average, either direction) and Count's in-range empty interval (#680); skipped // otherwise. if (AggregateEvaluator.NeedsDataRange(aggregate, plan)) { diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/SqlServerHistorianDataSource.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/SqlServerHistorianDataSource.cs index e090a8b4..0cda638e 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/SqlServerHistorianDataSource.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/SqlServerHistorianDataSource.cs @@ -458,7 +458,7 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource _ => SqlAggregateQueries.ActualTime(sample, aggregate == HistoryAggregateType.MaximumActualTime, plan.Reversed), }; // Second result set: the tag's data range (earliest + latest sample) — the start/end-of-data Partial rules - // (#677/#680, forward reads of an aggregate other than Average) and Count's in-range empty interval (#680). + // (#677/#680 — any per-sample aggregate but Average, either direction) and Count's in-range empty interval (#680). var needsRange = AggregateEvaluator.NeedsDataRange(aggregate, plan); if (needsRange) sql += " " + SqlAggregateQueries.DataRange(sample); await using var conn = await _conn.OpenAsync(cancellationToken).ConfigureAwait(false); diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common.Tests/SdkAggregateParityTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common.Tests/SdkAggregateParityTests.cs index a2b6a6d6..eb45c4ed 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common.Tests/SdkAggregateParityTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common.Tests/SdkAggregateParityTests.cs @@ -491,6 +491,49 @@ public sealed class SdkAggregateParityTests .ShouldBe([0u, HistorianInfoBits.HistorianPartial, 0u, 0u]); } + /// #680 review: the data ends on an OUTAGE MARKER (+25) — the interval holding it is all-Bad. + private static readonly StoredSample[] OutageEndsData = + [ + S(-5, HistorianValue.FromDouble(1), Good), S(3, HistorianValue.FromDouble(2), Good), + S(25, HistorianValue.Empty, BadNoCommunication), + ]; + + [Theory] + [InlineData(0, 40, 10)] + [InlineData(40, 0, 10)] + public void An_outage_marker_as_the_last_sample_sets_no_end_of_data_Partial_on_its_Bad_interval(double from, double to, + double intervalS) + { + // Documented divergence (remarks): the SDK sets the end-of-data Partial bit on that interval's Bad result too; ours + // never puts Partial on a Bad result. Masked by clearing Partial on the SDK's Bad results only — every other bit, + // timestamp and value (and every non-Bad interval's Partial) is still compared. + var failures = new List(); + var sawSdkPartialOnBad = false; + foreach (var agg in PerSample) + { + var ours = Ours(agg, from, to, intervalS, OutageEndsData); + var sdk = Sdk(agg, from, to, intervalS, OutageEndsData); + sawSdkPartialOnBad |= sdk.Any(r => (r.StatusCode & 0x80000000u) != 0 && (r.StatusCode & 0x4u) != 0); + var masked = sdk.Select(r => (r.StatusCode & 0x80000000u) != 0 ? r with { StatusCode = r.StatusCode & ~0x4u } : r) + .ToList(); + try + { + SdkAggregateOracle.ShouldMatch(ours, masked, $"{agg} OutageEndsData {from}→{to}/{intervalS}s", + compareCalculatedInterpolatedBits: false); + // The interval holding the marker ((20,30] / [20,30)) is Bad and carries no Partial. + var k = from < to ? 2 : 1; + ((ours[k].StatusCode & 0x80000000u) != 0).ShouldBeTrue($"{agg}: the outage interval is Bad"); + (ours[k].StatusCode & HistorianInfoBits.HistorianPartial).ShouldBe(0u, $"{agg}: no Partial on a Bad result"); + } + catch (Exception ex) + { + failures.Add(ex.Message); + } + } + failures.ShouldBeEmpty(string.Join("\n", failures)); + sawSdkPartialOnBad.ShouldBeTrue("the mask is not vacuous: the SDK does flag the Bad interval Partial"); + } + /// The per-bucket statistics the backends' SQL computes, built in memory with . internal static IEnumerable InMemoryRows(BucketPlan plan, IEnumerable samples, bool maximum = false) {