From 56bb4855bb88049ea5a7706ebb1fbe7fb66494fa Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Sun, 11 Oct 2026 09:10:26 -0400 Subject: [PATCH] =?UTF-8?q?fix(historian):=20#671=20review=20=E2=80=94=20r?= =?UTF-8?q?aw-row=20cap,=20reverse=20Interpolative,=20SDK=20status=20rule,?= =?UTF-8?q?=20Partial=20bit?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit M1: the time-weighted family (incl. Total, formerly a pushed-down SUM) streams raw window rows into the C# evaluator. New shared knob ServerHistorian:MaxTimeWeightedRawRows (default 1,000,000; <=0 unbounded with a Validate warning) flows through HistorianBackendRegistry into both providers. The window query is capped at cap+1 (SQL Server TOP (@maxRows), QuestDB LIMIT $4); one row past the cap throws the new Core.Abstractions HistoryReadLimitExceededException, which the node manager maps to BadTooManyOperations for that node. Not counted as a backend health failure; bound rows are not counted; per-sample aggregates are not capped. M2: reverse Interpolative is anchored at the later time (Part 13): points at End - i*interval, emitted latest first. 10:00->09:00 @1h yields 10:00. L1: Partial bit (0x4, = Opc.Ua.AggregateBits.Partial) on non-Bad results of a truncated last interval (time-weighted family; per-sample family is a follow-up). L2/L3: aligned with the SDK AggregateCalculator.GetTimeBasedStatusCode (decompiled from Opc.Ua.Server 1.5.378.106): TreatUncertainAsBad=false counts Uncertain time (Uncertain samples, held-into-outage, extrapolation) as Good time; data but no non-Bad time => generic Bad (0x80000000); BadNoData only when there is no data at all. Consequence: a live tag's latest interval reads Good (documented). L5: release note no longer claims Total matches the Gateway Integral exactly (AVEVA's divisor is unverified); it notes Total is now evaluated in C# and subject to the cap. Tests: unit (reverse Interpolative x3, Partial, Bad vs BadNoData incl. an open-ended outage, cap helpers, SDK bit oracles); conformance (reverse Interpolative, raw-row cap at/over/0, bounds not counted, health unaffected); node manager BadTooManyOperations; options default + warning. Each new behaviour falsified by mutation (aggregator x9, data-source cap checks on both backends against the real DBs, node-manager catch). --- CLAUDE.md | 3 +- docs/Historian.md | 65 ++++++-- .../Historian/HistoryReadContracts.cs | 16 ++ .../QuestDbAggregateQueries.cs | 5 +- .../QuestDbHistorianDataSource.cs | 17 ++- .../QuestDbHistorianProvider.cs | 10 +- .../HistorianInfoBits.cs | 6 + .../HistorianStatusCodes.cs | 3 + .../TimeWeightedAggregator.cs | 140 +++++++++++++----- .../SqlAggregateQueries.cs | 4 +- .../SqlServerHistorianDataSource.cs | 23 ++- .../SqlServerHistorianProvider.cs | 10 +- .../Configuration/HistorianBackendRegistry.cs | 4 +- .../OtOpcUaNodeManager.cs | 8 + .../Historian/ServerHistorianOptions.cs | 12 ++ .../IConformanceBackend.cs | 4 + .../ProcessedConformance.cs | 45 +++++- .../QuestDbConformanceBackend.cs | 4 + .../StatusCodeOracleTests.cs | 8 + .../TimeWeightedAggregatorTests.cs | 117 +++++++++++++-- .../ReadTestDb.cs | 5 +- .../SqlServerConformanceBackend.cs | 3 + .../NodeManagerHistoryReadTests.cs | 24 +++ .../Historian/ServerHistorianOptionsTests.cs | 10 ++ 24 files changed, 460 insertions(+), 86 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 105699bb..fc790e63 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1341,7 +1341,7 @@ historian tagname, which is the tag's driver `FullName`. ### Pluggable backends, SQL Server and HistoryUpdate (2026-10) - **Selection:** `ServerHistorian:Backend` (`Gateway` default | `SqlServer` | `QuestDb`; unknown fails host start). `HistorianBackendRegistry` → `IHistorianBackendProvider` builds all five seams; the gateway keys are only required for `Gateway`. `SqlServer` uses `ServerHistorian:SqlServer:{ConnectionString (required), Schema=hist, CommandTimeout=30s, AutoCreateSchema=true}`. -- **SQL Server** (`Driver.Historian.SqlServer`): `hist.Sample` PK `(TagId, SourceTimeUtc)` — writes are insert-if-absent, so outbox redelivery is idempotent. Stores every value kind with its **real status**; per-sample aggregates (12, simple definitions) are pushed down to T-SQL with bucket math in **100 ns ticks**; the Part 13 time-weighted five (TimeAverage/TimeAverage2/Total/Total2/Interpolative, #671 — ⚠️ `Total` is now the time-weighted integral, no longer `SUM`) stream window + 4 bound rows into the shared `TimeWeightedAggregator` (both SQL backends, identical results; client `AggregateConfiguration` ignored). Docker-dev recipe, schema and aggregate semantics: `docs/Historian.md` §"SQL Server backend". Integration tests need `HISTORIAN_SQL_TEST_CONNECTION`. +- **SQL Server** (`Driver.Historian.SqlServer`): `hist.Sample` PK `(TagId, SourceTimeUtc)` — writes are insert-if-absent, so outbox redelivery is idempotent. Stores every value kind with its **real status**; per-sample aggregates (12, simple definitions) are pushed down to T-SQL with bucket math in **100 ns ticks**; the Part 13 time-weighted five (TimeAverage/TimeAverage2/Total/Total2/Interpolative, #671 — ⚠️ `Total` is now the time-weighted integral, no longer `SUM`) stream window + 4 bound rows into the shared `TimeWeightedAggregator` (both SQL backends, identical results; client `AggregateConfiguration` ignored; status = the SDK `AggregateCalculator` time-based rule, Uncertain time counts as Good; reverse `Interpolative` anchored at the later time; window capped by `ServerHistorian:MaxTimeWeightedRawRows` ⇒ `BadTooManyOperations`). Docker-dev recipe, schema and aggregate semantics: `docs/Historian.md` §"SQL Server backend". Integration tests need `HISTORIAN_SQL_TEST_CONNECTION`. - **QuestDB** (`Driver.Historian.QuestDb`, `ServerHistorian:QuestDb:*`, Npgsql over PG wire; docker-dev overlay `docker-compose.questdb.yml`): same kinds/status/aggregates/HistoryUpdate as SqlServer, soft delete by tombstone (INSERT only — never `UPDATE`/`DELETE`; every read filters `NOT deleted`). ⚠️ **Timestamps are µs** (`QuestDbConnection.Round`/`BindTimestamp`; `IHistorianDataSource.TimestampResolutionTicks` = 10 steps HistoryRead paging), **pre-1970 cannot be stored** (writer drops, HistoryUpdate `BadOutOfRange`), every write waits for WAL visibility, TTL is enforced on write commits only, set Npgsql `Maximum Pool Size` < QuestDB's 64. `docs/Historian.md` §"QuestDB backend". - **Real status fix:** the recorder used to stamp every sample Good (`192`); it now carries the driver's status. ⚠️ On the **Gateway** backend this changes nothing stored: the gateway discards quality (C-002), so AVEVA still stores Good. - **Comms loss is historized as an outage marker** (`Empty` + `BadNoCommunication`, one per tag per outage) on SqlServer; dropped (`DroppedUnsupportedKind`) on Gateway. Outbox records are **format v2** — drain or delete `ContinuousHistorization:OutboxPath` before downgrading a node. @@ -1370,6 +1370,7 @@ authorization uses the standard `AccessLevels.HistoryRead` bit set at materializ | `MaxTieClusterOverfetch` | `65536` | Bounded over-fetch the HistoryRead-Raw paging uses to page within an oversized same-timestamp tie cluster (retained from the prior backend) | | `HistoryReadBatchParallelism` | `4` | Max nodes served concurrently within one HistoryRead batch (03/S3); `⌈N/P⌉ × CallTimeout` worst case instead of sequential | | `MaxConcurrentHistoryReadBatches` | `16` | Process-wide concurrent HistoryRead batch cap (03/S3); flood degrades to `BadTooManyOperations`. Total in-flight reads capped at 16 × 4 = 64 | +| `MaxTimeWeightedRawRows` | `1000000` | `SqlServer`/`QuestDb` only (#671): raw in-window rows one time-weighted aggregate read (incl. `Total`) may stream into the C# evaluator; window query capped at this + 1, past it ⇒ `BadTooManyOperations` for that node. ≤ 0 = unbounded | | `HistoryReadDeadline` | `00:01:00` | Per-request HistoryRead wall-clock deadline (03/S3); a node still in flight surfaces `BadTimeout`. Non-positive = unbounded | ### Alarm-history path (`AlarmHistorian` section) diff --git a/docs/Historian.md b/docs/Historian.md index 0a718d05..1de330bd 100644 --- a/docs/Historian.md +++ b/docs/Historian.md @@ -110,6 +110,7 @@ and all HistoryRead calls on historized nodes return `GoodNoData` (empty, not an "HistoryReadBatchParallelism": 4, "MaxConcurrentHistoryReadBatches": 16, "HistoryReadDeadline": "00:01:00", + "MaxTimeWeightedRawRows": 1000000, "SqlServer": { "ConnectionString": "", "Schema": "hist", @@ -141,6 +142,7 @@ and all HistoryRead calls on historized nodes return `GoodNoData` (empty, not an | `MaxTieClusterOverfetch` | int | `65536` | (Node-manager paging; all backends — a `SqlServer` PK rules out tie clusters, so it is moot there.) Maximum samples the server will fetch in one shot to page through a tie cluster (multiple samples sharing one `SourceTimestamp`). A cluster larger than this ceiling fails `BadHistoryOperationUnsupported`. Raise to handle abnormally large tie clusters; the default covers all normal-data cases. | | `HistoryReadBatchParallelism` | int | `4` | Max historized nodes served **concurrently** within one HistoryRead batch (arch-review 03/S3). The SDK's `HistoryRead*` override surface is synchronous, so each arm bridges once at its boundary; this turns per-node serving from sequential (`N × CallTimeout` worst case) into `⌈N / P⌉ × CallTimeout`. Also caps the gateway-load multiplication a single client can command. `≤ 1` serves sequentially. | | `MaxConcurrentHistoryReadBatches` | int | `16` | Process-wide limit on HistoryRead (and HistoryUpdate) batches served **concurrently** (arch-review 03/S3). A flood of history clients degrades to `BadTooManyOperations` rather than exhausting the SDK request-thread pool. With `HistoryReadBatchParallelism` this caps total in-flight gateway reads at `MaxConcurrentHistoryReadBatches × HistoryReadBatchParallelism` (default 16 × 4 = 64). A batch that cannot get a slot within `min(HistoryReadDeadline, 5 s)` fails every handle `BadTooManyOperations`. | +| `MaxTimeWeightedRawRows` | int | `1000000` | `SqlServer` / `QuestDb` only (#671). The most raw in-window samples one [time-weighted aggregate](#time-weighted-aggregates-part-13-671) read (TimeAverage, TimeAverage2, **Total**, Total2, Interpolative) may stream into the server-side evaluator — that family is **not** pushed down to SQL. The window query is capped at this + 1 rows; one row past the cap fails **that node** with `BadTooManyOperations` (not counted as a backend fault) instead of holding a pooled connection until `HistoryReadDeadline` and returning `BadTimeout`. Bound rows outside the window do not count; per-sample aggregates are not capped. 1 000 000 ≈ 11.5 days of a 1 Hz tag. Non-positive = unbounded (a `Validate()` warning). | | `HistoryReadDeadline` | TimeSpan | `00:01:00` | Per-request wall-clock deadline for a HistoryRead (and HistoryUpdate) batch (arch-review 03/S3). A node still in flight at the deadline surfaces `BadTimeout`, so one request can never hold an SDK request thread longer than this regardless of node count. Non-positive = unbounded (a `Validate()` warning). | **`ServerHistorian:SqlServer`** (read only when `Backend=SqlServer`; bound by `HistorianBackendRegistry.BindSqlServer`): @@ -738,10 +740,13 @@ continuation points, so a client-capped read returns only the first page. > `StandardDeviationPopulation`). > > ⚠️ **Release note / migration (#671): `Total` changed meaning on the `SqlServer` and `QuestDb` backends.** It was the plain `SUM` of -> the usable samples in the interval; it is now the Part 13 time-weighted integral, `TimeAverage × interval seconds` (value·seconds), -> matching what the `Gateway` backend's native `Integral` mode always returned. A client that relied on the old sum gets a different -> number for the same data (e.g. two samples 10 and 20 ten seconds apart no longer total 30). There is no switch back; `Count` × -> `Average` reproduces the old sum. +> the usable samples in the interval, pushed down to SQL; it is now the Part 13 time-weighted integral, `TimeAverage × interval seconds` +> (value·seconds), **evaluated in C# from the raw rows** (not pushed down) and therefore subject to `ServerHistorian:MaxTimeWeightedRawRows` +> — a long `Total` over a dense tag that used to succeed may now fail `BadTooManyOperations`; read it in shorter windows. This brings +> `Total` closer to the `Gateway` backend, whose `Total` maps to AVEVA's native time-weighted `Integral` retrieval mode — but AVEVA's +> integral time unit/divisor has not been verified, so the two backends may differ by a constant factor for the same data. A client +> that relied on the old sum gets a different number for the same data (e.g. two samples 10 and 20 ten seconds apart no longer total +> 30). There is no switch back; `Count` × `Average` reproduces the old sum. **Bucketing.** One result per interval of `[start, end)`; bucket *i* = `[start + i·interval, start + (i+1)·interval)`, start-inclusive / end-exclusive to the tick, the last bucket truncated at `end`. `interval <= 0` ⇒ a single interval `[start, end)`; `start == end` ⇒ no @@ -778,7 +783,8 @@ Status and edge rules: `TimeAverage`, `TimeAverage2`, `Total`, `Total2` and `Interpolative`, evaluated by the shared `Driver.Historian.Sql.Common/TimeWeightedAggregator.cs` for **both** SQL backends (so `SqlServer` and `QuestDb` agree exactly). -Bucketing (edges, truncated last bucket, reverse windows, the 100 000-interval cap) is the same as above. +**This whole family — `Total` included — is evaluated in C#, not pushed down to SQL** (see *Pushdown trade-off* and the raw-row cap +below). Bucketing (edges, truncated last bucket, the 100 000-interval cap) is the same as above; reverse windows are covered below. **Server defaults, fixed:** `TreatUncertainAsBad = false`, `PercentDataBad = PercentDataGood = 100`, `UseSlopedExtrapolation = false`. **A client's `AggregateConfiguration` in the HistoryRead request is ignored** (the node manager does not forward it); every read uses @@ -792,7 +798,8 @@ still gets a value. **The signal.** Between a usable sample `p` and the next sample `s`: both `Double`/`Int64` ⇒ **sloped** (linear), segment quality the worse of the two; either `Boolean` ⇒ **stepped** (`p` held), quality `p`'s; `s` Bad ⇒ `p` held up to the outage (never interpolated into it — the AtTime rule), Uncertain; no later sample at all ⇒ `p` held to the window end (stepped extrapolation), Uncertain. From a Bad -sample (incl. an `Empty` outage marker) to the next sample, and before the first sample, the signal is **Bad**. That is the +sample (incl. an `Empty` outage marker) to the next sample the signal is **Bad** (outage); before the first sample there is **no +data**. That is the *simple-bounding* signal. The *interpolated-bounding* signal skips Bad samples: it joins consecutive usable samples (sloped / stepped) straight across an outage. @@ -802,15 +809,41 @@ straight across an outage. | `Total` | `TimeAverage × interval seconds` (value·s; the truncated last bucket uses its own length) | interpolated across | | `TimeAverage2` | ∫ simple-bounding signal over non-Bad time ÷ non-Bad duration | excluded | | `Total2` | ∫ simple-bounding signal over non-Bad time (value·s) | excluded | -| `Interpolative` | interpolated-bounding value **at the interval start** | interpolated across | +| `Interpolative` | interpolated-bounding value **at the interval start** (reverse read: see below) | interpolated across | -Status, the four time-weighted values: no non-Bad time in the interval (including an interval entirely inside an outage) ⇒ -`BadNoData`; the whole interval Good ⇒ `Good`; otherwise `UncertainDataSubNormal`. A usable `String`/`DateTime` sample feeding the -interval ⇒ `BadAggregateInvalidInputs`. All carry the Calculated bit; timestamp = interval start; value `double` (Boolean 0/1). +**Status, the four time-weighted values** — aligned with the OPC Foundation SDK's `AggregateCalculator.GetTimeBasedStatusCode` +(inspected in `Opc.Ua.Server` 1.5.378.106) under the fixed defaults. With `TreatUncertainAsBad = false`, **Uncertain time counts as +Good time**: Uncertain samples, a value held up to an outage and the stepped extrapolation past the last sample all count toward +PercentDataGood. -Status, `Interpolative`: a usable sample exactly at the interval start ⇒ that value with **its own stored status** (no historian bit); -no usable sample at or before ⇒ plain `BadNoData`; otherwise the value with the **Interpolated** bit, `Good` on a segment whose -defining samples are Good, `UncertainDataSubNormal` inside or up to an outage (bridged) and past the last sample (extrapolated). +- Every instant of the interval non-Bad ⇒ `Good`. +- Some non-Bad time but not all (an outage, or time before the first sample) ⇒ `UncertainDataSubNormal`. +- No non-Bad time but some outage time ⇒ generic **`Bad`** (`0x80000000`) — the SDK's all-Bad result, value null. +- No data at all ⇒ `BadNoData`. +- A usable `String`/`DateTime` sample feeding the interval ⇒ `BadAggregateInvalidInputs`. + +All carry the Calculated bit; a non-Bad result for the **truncated last interval** (shorter than the processing interval) also carries +the **Partial** bit (`0x4`, `AggregateBits.Partial`). Timestamp = interval start; value `double` (Boolean 0/1). (The per-sample +aggregates do not set Partial — a follow-up.) + +> **A live tag's latest interval.** Past the tag's last stored sample the value is held (stepped extrapolation, Uncertain). Under +> the SDK rule that Uncertain time is Good time, so the latest interval of a live tag normally reads **`Good`** — not Uncertain — +> with the last value held to the interval end. It reads Uncertain only if the interval also contains outage time or starts +> before the tag's first sample. Do not take a Good latest interval as proof that data arrived up to its end. If it was truncated +> by the request's end time it also carries **Partial**. + +**Status, `Interpolative`** (point-based, not the PercentDataGood rule): +- A usable sample exactly at the point ⇒ that value with **its own stored status** (no historian bit). +- No usable sample at or before the point ⇒ plain `BadNoData`. +- Otherwise the value with the **Interpolated** bit: + - `Good` on a segment whose defining samples are Good; + - `UncertainDataSubNormal` inside or up to an outage (bridged) and past the last sample (extrapolated). + +**Reverse reads (`end < start`).** Part 13 anchors a reverse read at the request's start, which is the later time. `Interpolative` +follows that: its points are `start, start − interval, …` (in the forward window's terms `End − i·interval`), latest first. A +reverse read 10:00 → 09:00 at 1 h returns the value at **10:00**, not 09:00, and no point ever falls before the window. The interval +aggregates (TimeAverage, Total, …) keep the existing backend-wide reverse rule: computed forward with intervals anchored at the +earlier time, returned in descending order. Re-anchoring those at the later time for every aggregate is a follow-up. **Pushdown trade-off.** Unlike the per-sample aggregates, the time-weighted family is **not** pushed down to SQL: the backend streams the window's rows plus the four bounds (one batch of five result sets on SQL Server; a bounds `UNION ALL` + a window select on QuestDB) @@ -820,6 +853,12 @@ interpolation, edge splitting and two bounding definitions written twice, in T-S judged the larger risk. For very long, densely sampled windows prefer the pushed-down `Average` (or a coarser read) over `TimeAverage`. +**Raw-row cap.** The window query is limited to `ServerHistorian:MaxTimeWeightedRawRows` + 1 rows (default 1 000 000, shared by both +SQL backends; `TOP (@maxRows)` / `LIMIT $4`). When that extra row arrives the read stops and the node fails with +**`BadTooManyOperations`** — for that node only; it is not counted in the backend's health failures. Without the cap a one-year +`Total` on a 1 Hz tag (≈31 M rows) would hold a pooled connection until `HistoryReadDeadline` and then return `BadTimeout`. The four +bound rows do not count toward the cap, and the per-sample aggregates are never capped. Raise the cap, or read in shorter windows. + ### Health The read client reports `GetHealthSnapshot()` (query/failure counters, consecutive failures, `LastError` — e.g. a missing schema under diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Abstractions/Historian/HistoryReadContracts.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Abstractions/Historian/HistoryReadContracts.cs index aa602e09..0be71792 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Abstractions/Historian/HistoryReadContracts.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Abstractions/Historian/HistoryReadContracts.cs @@ -71,6 +71,22 @@ public sealed class HistoryAggregateNotSupportedException : NotSupportedExceptio => Aggregate = aggregate; } +/// +/// Thrown by a history backend when a read would exceed a configured size limit (e.g. the raw-row cap of the SQL +/// backends' time-weighted aggregates, #671). The Server HistoryRead path translates it to +/// BadTooManyOperations for that node only; it is a client-request limit, not a backend fault. +/// +public sealed class HistoryReadLimitExceededException : InvalidOperationException +{ + /// The configured limit that was exceeded. + public long Limit { get; } + + /// Creates the exception. + /// The configured limit. + /// Operator-facing description. + public HistoryReadLimitExceededException(long limit, string message) : base(message) => Limit = limit; +} + /// /// One row returned by — a historical /// alarm/event record, not the OPC UA live-event stream. Fields match the minimum set the diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb/QuestDbAggregateQueries.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb/QuestDbAggregateQueries.cs index 1b094cf0..85684bd9 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb/QuestDbAggregateQueries.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb/QuestDbAggregateQueries.cs @@ -125,7 +125,8 @@ internal static class QuestDbAggregateQueries } /// - /// Time-weighted family (#671), window statement: every live sample in [$2, $3) for tag $1, ascending. + /// Time-weighted family (#671), window statement: the live samples in [$2, $3) for tag $1, ascending, + /// at most $4 (the raw-row cap + 1). /// Columns: ts, kind, num_value, int_value, str_value, status, server_ts. The caller feeds the before-bounds, /// these rows and the after-bounds, in that order, into — the same evaluator /// the SQL Server backend uses, so the two agree exactly. Trade-off: the window's rows travel to the server rather @@ -135,7 +136,7 @@ internal static class QuestDbAggregateQueries /// Prefixed sample table name. /// The SQL text. public static string TimeWeightedWindow(string sample) => - $"SELECT {SampleColumns} FROM {sample} WHERE tag = $1 AND ts >= $2 AND ts < $3 AND NOT deleted ORDER BY ts ASC"; + $"SELECT {SampleColumns} FROM {sample} WHERE tag = $1 AND ts >= $2 AND ts < $3 AND NOT deleted ORDER BY ts ASC LIMIT $4"; /// The row columns the time-weighted statements return (after k for the bounds). private const string SampleColumns = "ts, kind, num_value, int_value, str_value, status, server_ts"; diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb/QuestDbHistorianDataSource.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb/QuestDbHistorianDataSource.cs index b987fe55..28042318 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb/QuestDbHistorianDataSource.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb/QuestDbHistorianDataSource.cs @@ -64,6 +64,7 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource private readonly ILogger _log; private readonly TimeProvider _time; private readonly HistorianHealthCounters _health = new(); + private readonly int _maxTimeWeightedRawRows; /// Creates the data source over the provider's shared connection, bootstrapper and tag catalog. /// Connection helper. @@ -71,10 +72,12 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource /// Tag catalog (reads use only its read-only ). /// Optional logger. /// Clock for "now" (AtTime beyond the last sample, server timestamps); system by default. + /// Raw in-window row cap of one time-weighted read (<= 0 = unbounded). internal QuestDbHistorianDataSource( QuestDbConnection conn, QuestDbSchemaBootstrapper bootstrap, QuestDbTagCatalog catalog, ILogger? log = null, - TimeProvider? timeProvider = null) + TimeProvider? timeProvider = null, int maxTimeWeightedRawRows = TimeWeightedAggregator.DefaultMaxWindowRows) { + _maxTimeWeightedRawRows = maxTimeWeightedRawRows; _conn = conn ?? throw new ArgumentNullException(nameof(conn)); _bootstrap = bootstrap ?? throw new ArgumentNullException(nameof(bootstrap)); _catalog = catalog ?? throw new ArgumentNullException(nameof(catalog)); @@ -387,6 +390,11 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource RecordSuccess(); return new HistoryReadResult(result, ContinuationPoint: null); } + catch (HistoryReadLimitExceededException) + { + // A client-request limit (the raw-row cap), not a backend fault: not counted as a failure. + throw; + } catch (Exception ex) { RecordFailure(ex); @@ -420,9 +428,16 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource QuestDbConnection.BindText(cmd, tag); QuestDbConnection.BindTimestamp(cmd, plan.StartUtc); QuestDbConnection.BindTimestamp(cmd, plan.EndUtc); + cmd.Parameters.Add(new NpgsqlParameter + { NpgsqlDbType = NpgsqlDbType.Integer, Value = TimeWeightedAggregator.WindowFetchLimit(_maxTimeWeightedRawRows) }); await using var reader = await cmd.ExecuteReaderAsync(ct).ConfigureAwait(false); + long windowRows = 0; while (await reader.ReadAsync(ct).ConfigureAwait(false)) + { + // LIMIT $4 = cap + 1: one row past the cap fails the node without reading on. + TimeWeightedAggregator.CheckWindowRows(++windowRows, _maxTimeWeightedRawRows, tag); tw.Add(ReadStored(reader, 0)); + } } if (bounds[2] is { } anyAfter) tw.Add(anyAfter); if (bounds[3] is { } usableAfter) tw.Add(usableAfter); diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb/QuestDbHistorianProvider.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb/QuestDbHistorianProvider.cs index f56d2205..19230a6b 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb/QuestDbHistorianProvider.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb/QuestDbHistorianProvider.cs @@ -1,3 +1,4 @@ +using ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common; using System.Collections.Frozen; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; @@ -33,11 +34,16 @@ public sealed class QuestDbHistorianProvider : IHistorianBackendProvider, IAsync /// Creates the provider over the ServerHistorian:QuestDb options (validated elsewhere). /// The ServerHistorian:QuestDb options. - public QuestDbHistorianProvider(QuestDbHistorianOptions options) + /// The shared ServerHistorian:MaxTimeWeightedRawRows cap (<= 0 = unbounded). + public QuestDbHistorianProvider(QuestDbHistorianOptions options, + int maxTimeWeightedRawRows = TimeWeightedAggregator.DefaultMaxWindowRows) { _options = options ?? throw new ArgumentNullException(nameof(options)); + _maxTimeWeightedRawRows = maxTimeWeightedRawRows; } + private readonly int _maxTimeWeightedRawRows; + /// public string Name => "QuestDb"; @@ -69,7 +75,7 @@ public sealed class QuestDbHistorianProvider : IHistorianBackendProvider, IAsync { var s = Get(sp); return new QuestDbHistorianDataSource(s.Conn, s.Bootstrap, s.Catalog, - Loggers(sp).CreateLogger()); + Loggers(sp).CreateLogger(), maxTimeWeightedRawRows: _maxTimeWeightedRawRows); } /// diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common/HistorianInfoBits.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common/HistorianInfoBits.cs index 1a067546..3e6b9d6d 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common/HistorianInfoBits.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common/HistorianInfoBits.cs @@ -12,6 +12,12 @@ public static class HistorianInfoBits /// The value was interpolated. public const uint HistorianInterpolated = 0x2u; + /// + /// Partial (Part 4 §7.39 / Opc.Ua.AggregateBits.Partial): the calculated value covers an interval shorter + /// than the processing interval (the truncated last interval of a processed read). + /// + public const uint HistorianPartial = 0x4u; + /// /// Marks as a Calculated data value. Any historian bits the status already carries /// (e.g. a stored Interpolated sample) are cleared first, so the result is never both. diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common/HistorianStatusCodes.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common/HistorianStatusCodes.cs index e8896f66..9fa08bcf 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common/HistorianStatusCodes.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common/HistorianStatusCodes.cs @@ -10,6 +10,9 @@ public static class HistorianStatusCodes public const uint GoodEntryInserted = 0x00A20000u; public const uint GoodEntryReplaced = 0x00A30000u; public const uint BadNoData = 0x809B0000u; + + /// Generic Bad: a time-weighted interval holding data but no non-Bad time (the SDK aggregate rule). + public const uint Bad = 0x80000000u; public const uint BadEntryExists = 0x809F0000u; public const uint BadNoEntryExists = 0x80A00000u; public const uint BadInvalidTimestamp = 0x80230000u; diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common/TimeWeightedAggregator.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common/TimeWeightedAggregator.cs index 1c66ca69..a4737fa8 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common/TimeWeightedAggregator.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common/TimeWeightedAggregator.cs @@ -39,17 +39,23 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common; /// TotalTimeAverage × interval seconds (value·s) — Part 13 §5.4.3.8. /// TimeAverage2∫ simple-bounding signal over its non-Bad parts ÷ non-Bad duration (Bad intervals excluded). /// Total2∫ simple-bounding signal over its non-Bad parts, in value·s (= TimeAverage2 × non-Bad seconds). -/// InterpolativeThe interpolated-bounding signal's value at each interval start. +/// InterpolativeThe interpolated-bounding signal's value at each interval start — +/// for a reverse read, at End − i·interval (Part 13 anchors a reverse read at its later time), emitted +/// latest first. /// /// -/// Status (time-weighted four). No non-Bad time in the interval ⇒ BadNoData; the whole interval -/// Good ⇒ Good; otherwise UncertainDataSubNormal (PercentDataGood = PercentDataBad = 100). A -/// usable String/DateTime sample contributing to the interval ⇒ BadAggregateInvalidInputs. Every status -/// carries the Calculated bit; the timestamp is the interval start; the value is a -/// (Boolean = 0/1, Int64 as double). +/// Status (time-weighted four) — the SDK AggregateCalculator.GetTimeBasedStatusCode rule with +/// TreatUncertainAsBad = false, so Uncertain time (Uncertain samples, a value held up to an outage, stepped +/// extrapolation past the last sample) counts as Good time: every instant of the interval non-Bad ⇒ +/// Good; no non-Bad time but some Bad (outage) time ⇒ Bad; no data at all ⇒ BadNoData; +/// otherwise (PercentDataGood = PercentDataBad = 100) UncertainDataSubNormal. Time before the first sample +/// counts as Bad (no data) time. A usable String/DateTime sample contributing to the interval ⇒ +/// BadAggregateInvalidInputs. Every status carries the Calculated bit, plus the Partial bit on a non-Bad +/// result of a truncated last interval; the timestamp is the interval start; the value is a +/// (Boolean = 0/1, Int64 as double). /// /// -/// Status (Interpolative). A usable sample exactly at the interval start ⇒ its value with its own status +/// Status (Interpolative). A usable sample exactly at the point ⇒ its value with its own status /// (raw, no historian bit). Otherwise no usable sample at or before ⇒ BadNoData; else the interpolated /// value with the Interpolated bit and the status of the simple-bounding signal at that instant: Good on /// a Good segment, UncertainDataSubNormal on an Uncertain one, inside an outage (interpolated across it) @@ -60,26 +66,54 @@ public sealed class TimeWeightedAggregator { private const double TicksPerSecond = TimeSpan.TicksPerSecond; + /// + /// Default cap on the raw in-window rows one time-weighted read may stream (ServerHistorian:MaxTimeWeightedRawRows). + /// + public const int DefaultMaxWindowRows = 1_000_000; + + /// + /// The TOP/LIMIT a backend puts on its window query for a configured cap: cap + 1 (so one + /// row past the cap proves it was exceeded without reading on), or when the cap is + /// <= 0 (unbounded). + /// + /// The configured cap. + /// The row limit to put on the window query. + public static int WindowFetchLimit(int maxWindowRows) => + maxWindowRows <= 0 || maxWindowRows == int.MaxValue ? int.MaxValue : maxWindowRows + 1; + + /// Throws when has passed a positive . + /// Window rows read so far. + /// The configured cap (<= 0 = unbounded). + /// The tag, for the message. + /// The cap is exceeded. + public static void CheckWindowRows(long rowsRead, int maxWindowRows, string tag) + { + if (maxWindowRows > 0 && rowsRead > maxWindowRows) + throw new HistoryReadLimitExceededException(maxWindowRows, + $"The time-weighted aggregate read of '{tag}' covers more than {maxWindowRows} raw samples " + + "(ServerHistorian:MaxTimeWeightedRawRows); narrow the window or use a pushed-down aggregate such as Average."); + } + private readonly HistoryAggregateType _aggregate; private readonly BucketPlan _plan; private readonly long _start; private readonly long _end; - // Simple-bounding (raw) signal, per bucket: ∫ over non-Bad time, non-Bad ticks, Good ticks. + // Simple-bounding (raw) signal, per bucket: ∫ over non-Bad time, non-Bad ticks, ticks inside an outage. private readonly double[] _rawArea; private readonly long[] _usableTicks; - private readonly long[] _goodTicks; + private readonly long[] _badTicks; // Interpolated-bounding (bridged) signal, per bucket. private readonly double[] _bridgedArea; private readonly long[] _bridgedTicks; private readonly bool[] _invalid; - // Interpolative, per bucket start. + // Interpolative, per point (index = ascending time; see PointTicks). private readonly DataValueSnapshot?[] _points; private StoredSample? _prevAny; private StoredSample? _prevUsable; - private long _nextPoint; // next bucket start not yet classified - private readonly List _pending = []; // bucket starts awaiting the next usable sample + private long _nextPoint; // next point (ascending time) not yet classified + private readonly List _pending = []; // points awaiting the next usable sample private bool _finished; /// @@ -106,7 +140,7 @@ public sealed class TimeWeightedAggregator var n = (int)plan.BucketCount; _rawArea = new double[n]; _usableTicks = new long[n]; - _goodTicks = new long[n]; + _badTicks = new long[n]; _bridgedArea = new double[n]; _bridgedTicks = new long[n]; _invalid = new bool[n]; @@ -142,15 +176,13 @@ public sealed class TimeWeightedAggregator // Simple-bounding segment [prevAny, s). if (_prevAny is { IsUsable: true } p) { - if (s.IsUsable) - { - var sloped = IsSloped(p, s); - AddRaw(p, t, sloped ? s.Value.Number : p.Value.Number, sloped ? p.IsGood && s.IsGood : p.IsGood); - } - else - { - AddRaw(p, t, p.Value.Number, good: false); // held up to the outage - } + // Toward a usable sample: sloped or stepped; toward a Bad one: held up to the outage. + var sloped = s.IsUsable && IsSloped(p, s); + AddRaw(p, t, sloped ? s.Value.Number : p.Value.Number); + } + else if (_prevAny is { } bad) + { + AddBad(bad, t); } if (!s.IsUsable) DegradePending(); // the segment the pending points sit on runs into an outage @@ -179,7 +211,9 @@ public sealed class TimeWeightedAggregator _finished = true; ClassifyPointsBefore(long.MaxValue); if (_prevAny is { IsUsable: true } p && p.TimeUtc.Ticks < _end) - AddRaw(p, _end, p.Value.Number, good: false); // stepped extrapolation + AddRaw(p, _end, p.Value.Number); // stepped extrapolation + else if (_prevAny is { } bad && bad.TimeUtc.Ticks < _end) + AddBad(bad, _end); // the outage runs to the window end if (_prevUsable is { } pu && pu.TimeUtc.Ticks < _end) AddBridged(pu, _end, pu.Value.Number); ResolvePending(_prevUsable, next: null); @@ -195,20 +229,26 @@ public sealed class TimeWeightedAggregator private DataValueSnapshot Result(long i, DateTime now) { - var bucketStart = _plan.BucketStart(i); if (_aggregate == HistoryAggregateType.Interpolative) return _points[i] is { } point ? point with { ServerTimestampUtc = now } - : new DataValueSnapshot(null, HistorianStatusCodes.BadNoData, bucketStart, now); + : new DataValueSnapshot(null, HistorianStatusCodes.BadNoData, PointTime(i), now); - DataValueSnapshot Calc(object? value, uint status) => - new(value, HistorianInfoBits.WithCalculated(status), bucketStart, now); + var bucketStart = _plan.BucketStart(i); + var duration = BucketEnd(i) - BucketStartTicks(i); + var partial = duration < _plan.IntervalTicks; // only the truncated last interval + DataValueSnapshot Calc(object? value, uint status) + { + var code = HistorianInfoBits.WithCalculated(status); + if (partial && !StatusBits.IsBad(status)) code |= HistorianInfoBits.HistorianPartial; + return new DataValueSnapshot(value, code, bucketStart, now); + } if (_invalid[i]) return Calc(null, HistorianStatusCodes.BadAggregateInvalidInputs); - if (_usableTicks[i] == 0) return AggregateEvaluator.NoData(bucketStart, now); + if (_usableTicks[i] == 0) + return _badTicks[i] > 0 ? Calc(null, HistorianStatusCodes.Bad) : AggregateEvaluator.NoData(bucketStart, now); - var duration = BucketEnd(i) - BucketStartTicks(i); - var status = _goodTicks[i] == duration ? HistorianStatusCodes.Good : HistorianStatusCodes.UncertainDataSubNormal; + var status = _usableTicks[i] == duration ? HistorianStatusCodes.Good : HistorianStatusCodes.UncertainDataSubNormal; return _aggregate switch { HistoryAggregateType.TimeAverage => Calc(_bridgedArea[i] / _bridgedTicks[i], status), @@ -233,14 +273,29 @@ public sealed class TimeWeightedAggregator private long BucketEnd(long i) => Math.Min(BucketStartTicks(i) + _plan.IntervalTicks, _end); - private void AddRaw(StoredSample from, long toTicks, double toValue, bool good) => + private void AddRaw(StoredSample from, long toTicks, double toValue) => Distribute(from, toTicks, toValue, (i, area, ticks) => { _rawArea[i] += area; _usableTicks[i] += ticks; - if (good) _goodTicks[i] += ticks; }); + /// Counts [from.t, toTicks) as outage (Bad) time; no value, never marks a bucket invalid. + private void AddBad(StoredSample from, long toTicks) + { + var lo = Math.Max(from.TimeUtc.Ticks, _start); + var hi = Math.Min(toTicks, _end); + if (lo >= hi || _plan.BucketCount == 0) return; + for (var i = (lo - _start) / _plan.IntervalTicks; i < _plan.BucketCount; i++) + { + var bs = BucketStartTicks(i); + if (bs >= hi) break; + var a = Math.Max(lo, bs); + var b = Math.Min(hi, BucketEnd(i)); + if (a < b) _badTicks[i] += b - a; + } + } + private void AddBridged(StoredSample from, long toTicks, double toValue) => Distribute(from, toTicks, toValue, (i, area, ticks) => { @@ -279,13 +334,24 @@ public sealed class TimeWeightedAggregator // ------------------------------------------------------------------------------------------ Interpolative - /// Classifies every unclassified bucket start strictly before . + /// + /// The time of Interpolative point (ascending time). Forward: the bucket starts + /// start + j·interval. Reverse (Part 13 anchors a reverse read at its later time, the request's start): + /// End − (n−1−j)·interval, i.e. End, End − interval, … once has + /// reversed the order — so a reverse read 10:00 → 09:00 at 1 h yields the value at 10:00. + /// + private long PointTicks(long j) => + _plan.Reversed ? _end - (_plan.BucketCount - 1 - j) * _plan.IntervalTicks : BucketStartTicks(j); + + private DateTime PointTime(long j) => new(PointTicks(j), DateTimeKind.Utc); + + /// Classifies every unclassified point strictly before . private void ClassifyPointsBefore(long beforeTicks) { if (_aggregate != HistoryAggregateType.Interpolative) return; for (; _nextPoint < _plan.BucketCount; _nextPoint++) { - var a = BucketStartTicks(_nextPoint); + var a = PointTicks(_nextPoint); if (a >= beforeTicks) break; if (_prevAny is not { } p) @@ -296,7 +362,7 @@ public sealed class TimeWeightedAggregator { // A usable sample exactly at the interval start: its own value and status. _points[_nextPoint] = IsNumeric(p) - ? new DataValueSnapshot(p.Value.Number, p.Status, _plan.BucketStart(_nextPoint), default) + ? new DataValueSnapshot(p.Value.Number, p.Status, PointTime(_nextPoint), default) : InvalidPoint(_nextPoint); } else if (!p.IsUsable) @@ -318,7 +384,7 @@ public sealed class TimeWeightedAggregator if (_pending.Count == 0) return; foreach (var pt in _pending) { - var bucketStart = _plan.BucketStart(pt.Bucket); + var bucketStart = PointTime(pt.Bucket); if (p is not { } prev || pt.Quality == PointQuality.NoPrior) { _points[pt.Bucket] = new DataValueSnapshot(null, HistorianStatusCodes.BadNoData, bucketStart, default); @@ -360,5 +426,5 @@ public sealed class TimeWeightedAggregator } private DataValueSnapshot InvalidPoint(long bucket) => - new(null, HistorianStatusCodes.BadAggregateInvalidInputs, _plan.BucketStart(bucket), default); + new(null, HistorianStatusCodes.BadAggregateInvalidInputs, PointTime(bucket), default); } diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/SqlAggregateQueries.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/SqlAggregateQueries.cs index 874c8479..4beb6b15 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/SqlAggregateQueries.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/SqlAggregateQueries.cs @@ -95,7 +95,7 @@ internal static class SqlAggregateQueries /// Time-weighted family (#671: TimeAverage / TimeAverage2 / Total / Total2 / Interpolative) — not pushed down. /// One batch of five result sets, each SourceTimeUtc, ValueKind, NumValue, IntValue, StrValue, StatusCode: /// (1) the latest usable sample before @start, (2) the latest sample of any quality before @start, - /// (3) every sample in [@start, @end) ascending, (4) the first sample of any quality at or after + /// (3) the samples in [@start, @end) ascending, at most @maxRows (the raw-row cap + 1), (4) the first sample of any quality at or after /// @end, (5) the first usable one at or after @end. Each bound is a TOP 1 seek on the /// (TagId, SourceTimeUtc) key. The caller streams them, in that order, into /// (shared with QuestDB, so both backends agree exactly). @@ -116,7 +116,7 @@ internal static class SqlAggregateQueries return $"SELECT TOP 1 {cols} {key}SourceTimeUtc < @start{usable} ORDER BY SourceTimeUtc DESC; " + $"SELECT TOP 1 {cols} {key}SourceTimeUtc < @start ORDER BY SourceTimeUtc DESC; " + - $"SELECT {cols} {key}SourceTimeUtc >= @start AND SourceTimeUtc < @end ORDER BY SourceTimeUtc ASC; " + + $"SELECT TOP (@maxRows) {cols} {key}SourceTimeUtc >= @start AND SourceTimeUtc < @end ORDER BY SourceTimeUtc ASC; " + $"SELECT TOP 1 {cols} {key}SourceTimeUtc >= @end ORDER BY SourceTimeUtc ASC; " + $"SELECT TOP 1 {cols} {key}SourceTimeUtc >= @end{usable} ORDER BY SourceTimeUtc ASC;"; } diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/SqlServerHistorianDataSource.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/SqlServerHistorianDataSource.cs index 71ade926..0dbb7f05 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/SqlServerHistorianDataSource.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/SqlServerHistorianDataSource.cs @@ -59,6 +59,7 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource private readonly TimeProvider _time; private readonly HistorianHealthCounters _health = new(); + private readonly int _maxTimeWeightedRawRows; /// Creates the data source over the provider's shared connection, bootstrapper and tag catalog. /// Connection helper. @@ -66,10 +67,12 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource /// Tag catalog (shared; reads use only its read-only lookup). /// Optional logger. /// Clock for "now" (AtTime beyond the last sample, server timestamps); system by default. + /// Raw in-window row cap of one time-weighted read (<= 0 = unbounded). internal SqlServerHistorianDataSource( SqlHistorianConnection conn, SqlServerSchemaBootstrapper bootstrap, SqlServerTagCatalog catalog, ILogger? log = null, - TimeProvider? timeProvider = null) + TimeProvider? timeProvider = null, int maxTimeWeightedRawRows = TimeWeightedAggregator.DefaultMaxWindowRows) { + _maxTimeWeightedRawRows = maxTimeWeightedRawRows; _conn = conn ?? throw new ArgumentNullException(nameof(conn)); _bootstrap = bootstrap ?? throw new ArgumentNullException(nameof(bootstrap)); _catalog = catalog ?? throw new ArgumentNullException(nameof(catalog)); @@ -321,7 +324,7 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource if (family == AggregateFamily.TimeWeighted) { var tw = new TimeWeightedAggregator(aggregate, plan); - if (tagId is { } twId) await StreamTimeWeightedAsync(twId, plan, tw, cancellationToken).ConfigureAwait(false); + if (tagId is { } twId) await StreamTimeWeightedAsync(twId, fullReference, plan, tw, cancellationToken).ConfigureAwait(false); var twResult = tw.Finish(now); RecordSuccess(); return new HistoryReadResult(twResult, ContinuationPoint: null); @@ -354,6 +357,11 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource RecordSuccess(); return new HistoryReadResult(result, ContinuationPoint: null); } + catch (HistoryReadLimitExceededException) + { + // A client-request limit (the raw-row cap), not a backend fault: not counted as a failure. + throw; + } catch (Exception ex) { RecordFailure(ex); @@ -365,18 +373,27 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource /// Streams the time-weighted batch () into : the /// two before-bounds, the window, the two after-bounds — ascending, with duplicates dropped by the aggregator. /// - private async Task StreamTimeWeightedAsync(int id, BucketPlan plan, TimeWeightedAggregator tw, CancellationToken ct) + private async Task StreamTimeWeightedAsync(int id, string tag, BucketPlan plan, TimeWeightedAggregator tw, CancellationToken ct) { await using var conn = await _conn.OpenAsync(ct).ConfigureAwait(false); await using var cmd = _conn.CreateCommand(conn, SqlAggregateQueries.TimeWeighted(_conn.Q("Sample"))); cmd.Parameters.Add(new SqlParameter("@id", SqlDbType.Int) { Value = id }); SqlHistorianConnection.AddDateTime2(cmd, "@start", DateTime.SpecifyKind(plan.StartUtc, DateTimeKind.Utc)); SqlHistorianConnection.AddDateTime2(cmd, "@end", DateTime.SpecifyKind(plan.EndUtc, DateTimeKind.Utc)); + cmd.Parameters.Add(new SqlParameter("@maxRows", SqlDbType.Int) + { Value = TimeWeightedAggregator.WindowFetchLimit(_maxTimeWeightedRawRows) }); await using var reader = await cmd.ExecuteReaderAsync(ct).ConfigureAwait(false); + var resultSet = 0; do { + long windowRows = 0; while (await reader.ReadAsync(ct).ConfigureAwait(false)) + { + // Result set 2 is the window (TOP (@maxRows) = cap + 1): one row past the cap fails the node. + if (resultSet == 2) TimeWeightedAggregator.CheckWindowRows(++windowRows, _maxTimeWeightedRawRows, tag); tw.Add(ReadStored(reader, 0)); + } + resultSet++; } while (await reader.NextResultAsync(ct).ConfigureAwait(false)); } diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/SqlServerHistorianProvider.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/SqlServerHistorianProvider.cs index 8f98031b..5b48471d 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/SqlServerHistorianProvider.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/SqlServerHistorianProvider.cs @@ -1,3 +1,4 @@ +using ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common; using System.Collections.Frozen; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; @@ -29,11 +30,16 @@ public sealed class SqlServerHistorianProvider : IHistorianBackendProvider /// Creates the provider over the ServerHistorian:SqlServer options (validated elsewhere). /// The ServerHistorian:SqlServer options. - public SqlServerHistorianProvider(SqlServerHistorianOptions options) + /// The shared ServerHistorian:MaxTimeWeightedRawRows cap (<= 0 = unbounded). + public SqlServerHistorianProvider(SqlServerHistorianOptions options, + int maxTimeWeightedRawRows = TimeWeightedAggregator.DefaultMaxWindowRows) { _options = options ?? throw new ArgumentNullException(nameof(options)); + _maxTimeWeightedRawRows = maxTimeWeightedRawRows; } + private readonly int _maxTimeWeightedRawRows; + /// public string Name => "SqlServer"; @@ -63,7 +69,7 @@ public sealed class SqlServerHistorianProvider : IHistorianBackendProvider { var s = Get(sp); return new SqlServerHistorianDataSource(s.Conn, s.Bootstrap, s.Catalog, - Loggers(sp).CreateLogger()); + Loggers(sp).CreateLogger(), maxTimeWeightedRawRows: _maxTimeWeightedRawRows); } /// diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/HistorianBackendRegistry.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/HistorianBackendRegistry.cs index a14e5bd6..52056ba9 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/HistorianBackendRegistry.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/HistorianBackendRegistry.cs @@ -41,8 +41,8 @@ public static class HistorianBackendRegistry new Dictionary>(StringComparer.OrdinalIgnoreCase) { [ServerHistorianOptions.GatewayBackendName] = (o, _) => new GatewayHistorianProvider(o), - [SqlServerBackendName] = (_, c) => new SqlServerHistorianProvider(BindSqlServer(c)), - [QuestDbBackendName] = (_, c) => new QuestDbHistorianProvider(BindQuestDb(c)), + [SqlServerBackendName] = (o, c) => new SqlServerHistorianProvider(BindSqlServer(c), o.MaxTimeWeightedRawRows), + [QuestDbBackendName] = (o, c) => new QuestDbHistorianProvider(BindQuestDb(c), o.MaxTimeWeightedRawRows), }; /// The registered backend names, for validation messages. diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/OtOpcUaNodeManager.cs b/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/OtOpcUaNodeManager.cs index b2c2d16d..35d6541e 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/OtOpcUaNodeManager.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/OtOpcUaNodeManager.cs @@ -5779,6 +5779,14 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2 errors[handle.Index] = StatusCodes.BadAggregateNotSupported; results[handle.Index] = new SdkHistoryReadResult { StatusCode = StatusCodes.BadAggregateNotSupported }; } + catch (HistoryReadLimitExceededException ex) + { + // The read would exceed a backend size limit (#671: the time-weighted raw-row cap) — refuse THIS node + // explicitly instead of a generic fault or a deadline-elapsed BadTimeout. + _logger.LogWarning("OtOpcUaNodeManager: HistoryRead limit exceeded for node {NodeId}: {Message}", handle.NodeId, ex.Message); + errors[handle.Index] = StatusCodes.BadTooManyOperations; + results[handle.Index] = new SdkHistoryReadResult { StatusCode = StatusCodes.BadTooManyOperations }; + } catch (Exception ex) { // One node's backend failure (throw / timeout) must not throw out of the batch — surface a diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Historian/ServerHistorianOptions.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Historian/ServerHistorianOptions.cs index c9db04a0..fc6cc012 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Historian/ServerHistorianOptions.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Historian/ServerHistorianOptions.cs @@ -112,6 +112,16 @@ public sealed class ServerHistorianOptions /// public TimeSpan HistoryReadDeadline { get; init; } = TimeSpan.FromSeconds(60); + /// + /// SqlServer / QuestDb backends only (#671): the most raw in-window samples one Part 13 + /// time-weighted read (TimeAverage, TimeAverage2, Total, Total2, Interpolative) may stream into the server-side + /// evaluator — that family is not pushed down to SQL. A read past the cap stops at cap + 1 rows and fails THAT + /// node with BadTooManyOperations instead of holding a pooled connection until + /// . Non-positive = unbounded (a warning). Default + /// 1 000 000 (≈ 11.5 days of a 1 Hz tag). + /// + public int MaxTimeWeightedRawRows { get; init; } = 1_000_000; + /// Returns operator-facing misconfiguration warnings for an Enabled historian /// (empty when disabled or correctly configured). Pure — the registration logs each entry. /// @@ -154,6 +164,8 @@ public sealed class ServerHistorianOptions warnings.Add($"ServerHistorian:HistoryReadBatchParallelism is {HistoryReadBatchParallelism} — must be > 0; HistoryRead falls back to serving each node sequentially."); if (MaxConcurrentHistoryReadBatches <= 0) warnings.Add($"ServerHistorian:MaxConcurrentHistoryReadBatches is {MaxConcurrentHistoryReadBatches} — must be > 0; the process-wide HistoryRead batch limiter is disabled."); + if (MaxTimeWeightedRawRows <= 0) + warnings.Add($"ServerHistorian:MaxTimeWeightedRawRows is {MaxTimeWeightedRawRows} — must be > 0; time-weighted aggregate reads on the SqlServer/QuestDb backends stream an unbounded number of raw samples."); if (HistoryReadDeadline <= TimeSpan.Zero) warnings.Add($"ServerHistorian:HistoryReadDeadline is {HistoryReadDeadline} — must be > 0; HistoryRead requests run unbounded (no per-request deadline)."); return warnings; diff --git a/tests/Drivers/Historian.Conformance/IConformanceBackend.cs b/tests/Drivers/Historian.Conformance/IConformanceBackend.cs index 55077224..7e817533 100644 --- a/tests/Drivers/Historian.Conformance/IConformanceBackend.cs +++ b/tests/Drivers/Historian.Conformance/IConformanceBackend.cs @@ -22,6 +22,10 @@ public interface IConformanceBackend : IAsyncDisposable /// Its "now"; null means the system clock. IHistorianDataSource CreateDataSource(TimeProvider? clock = null); + /// A fresh data source whose time-weighted raw-row cap (ServerHistorian:MaxTimeWeightedRawRows) is . + /// The cap (<= 0 = unbounded). + IHistorianDataSource CreateDataSource(int maxTimeWeightedRawRows); + /// A fresh live-value writer over this backend's store. /// The clock used to stamp values that carry no timestamp; null means the system clock. IHistorianValueWriter CreateValueWriter(Func? utcNow = null); diff --git a/tests/Drivers/Historian.Conformance/ProcessedConformance.cs b/tests/Drivers/Historian.Conformance/ProcessedConformance.cs index 8f30596e..9ce58c18 100644 --- a/tests/Drivers/Historian.Conformance/ProcessedConformance.cs +++ b/tests/Drivers/Historian.Conformance/ProcessedConformance.cs @@ -346,6 +346,7 @@ public abstract class ProcessedConformance : ConformanceBase private const uint GoodInterp = 0x00000402u; // Good | InfoType 0x400 | Interpolated 0x2 private const uint UncertainInterp = 0x40A40402u; // UncertainDataSubNormal | 0x402 private const uint NoData = 0x809B0000u; + private const uint BadCalc = 0x80000401u; // generic Bad | Calculated (an all-outage interval) /// /// Sparse tag W: −30 = 0 G (bound before the window), +20 = 40 G, +30 = outage marker, +50 = 20 G, +100 = 20 G @@ -405,12 +406,13 @@ public abstract class ProcessedConformance : ConformanceBase public async Task TimeAverage2_over_a_sparse_tag_with_narrow_intervals() { await SeedWAsync(); - // 10 s intervals (narrower than the gaps): [0,10) 24→32 = 28 G; [10,20) 32→40 = 36 G; [20,30) 40 held, Uncertain; - // [30,40) and [40,50) inside the outage ⇒ BadNoData; [50,60) … [70,80) the flat 20 up to the bound after, Good. + // 10 s intervals (narrower than the gaps): [0,10) 24→32 = 28; [10,20) 32→40 = 36; [20,30) 40 held up to the + // outage (Uncertain time, which counts as Good: TreatUncertainAsBad = false); [30,40) and [40,50) inside the outage + // ⇒ generic Bad (data, but none non-Bad); [50,60) … [70,80) the flat 20 up to the bound after, Good. var s = await ReadW(HistoryAggregateType.TimeAverage2, 10); Values(s, 28, 36, 40, null, null, 20, 20, 20); s.Select(x => x.StatusCode).ShouldBe( - [GoodCalc, GoodCalc, UncertainCalc, NoDataCalc, NoDataCalc, GoodCalc, GoodCalc, GoodCalc]); + [GoodCalc, GoodCalc, GoodCalc, BadCalc, BadCalc, GoodCalc, GoodCalc, GoodCalc]); // Total2 of the first interval: 28 × 10 s. ((double)(await ReadW(HistoryAggregateType.Total2, 10))[0].Value!).ShouldBe(280d, 1e-9); } @@ -427,6 +429,43 @@ public abstract class ProcessedConformance : ConformanceBase [GoodInterp, GoodInterp, GoodRaw, UncertainInterp, UncertainInterp, GoodRaw, GoodInterp, GoodInterp]); } + [Fact] + public async Task A_reverse_Interpolative_read_is_anchored_at_its_later_time() + { + await SeedWAsync(); + // At(80) → At(0), 40 s: points At(80) (on the Good 20→20 segment to the bound at +100 ⇒ 20) and At(40) (inside + // the outage ⇒ bridged 80/3, Uncertain), latest first. A forward read would answer At(0) and At(40) instead. + var s = (await B.CreateDataSource().ReadProcessedAsync("W", At(80), At(0), TimeSpan.FromSeconds(40), + HistoryAggregateType.Interpolative, Ct)).Samples; + s.Select(x => x.SourceTimestampUtc).ShouldBe([B.Round(At(80)), B.Round(At(40))]); + Values(s, 20, 80d / 3d); + s.Select(x => x.StatusCode).ShouldBe([GoodInterp, UncertainInterp]); + } + + [Fact] + public async Task The_time_weighted_raw_row_cap_fails_the_read_past_the_cap_only() + { + // Tag A holds five samples in [0,80); W holds three in [0,80) plus two bound rows outside it. + await ConformanceData.SeedAAsync(B); + await SeedWAsync(); + var atCap = B.CreateDataSource(5); + (await atCap.ReadProcessedAsync("A", At(0), At(80), Twenty, HistoryAggregateType.Total, Ct)).Samples.Count.ShouldBe(4); + + var under = B.CreateDataSource(4); + var ex = await Should.ThrowAsync( + () => under.ReadProcessedAsync("A", At(0), At(80), Twenty, HistoryAggregateType.Total, Ct)); + ex.Limit.ShouldBe(4); + // Pushed-down (per-sample) aggregates are not capped; the bound rows outside the window do not count. + (await under.ReadProcessedAsync("A", At(0), At(80), Twenty, HistoryAggregateType.Average, Ct)).Samples.Count.ShouldBe(4); + (await B.CreateDataSource(3).ReadProcessedAsync("W", At(0), At(80), Twenty, HistoryAggregateType.TimeAverage, Ct)) + .Samples.Count.ShouldBe(4); + // A client-request limit, not a backend fault. + under.GetHealthSnapshot().TotalFailures.ShouldBe(0); + // Non-positive = unbounded. + (await B.CreateDataSource(0).ReadProcessedAsync("A", At(0), At(80), Twenty, HistoryAggregateType.Total, Ct)) + .Samples.Count.ShouldBe(4); + } + [Fact] public async Task Outages_straddling_both_window_edges_use_the_four_bound_seeks() { diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb.IntegrationTests/QuestDbConformanceBackend.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb.IntegrationTests/QuestDbConformanceBackend.cs index bf863363..1f3be1b3 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb.IntegrationTests/QuestDbConformanceBackend.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb.IntegrationTests/QuestDbConformanceBackend.cs @@ -75,6 +75,10 @@ internal sealed class QuestDbConformanceBackend : IConformanceBackend public IHistorianDataSource CreateDataSource(TimeProvider? clock = null) => new QuestDbHistorianDataSource(Conn, Boot, NewCatalog(), timeProvider: clock); + /// + public IHistorianDataSource CreateDataSource(int maxTimeWeightedRawRows) => + new QuestDbHistorianDataSource(Conn, Boot, NewCatalog(), maxTimeWeightedRawRows: maxTimeWeightedRawRows); + /// /// The T4 live-value writer over a fresh catalog (fresh positive cache) and this instance's WAL wait. public IHistorianValueWriter CreateValueWriter(Func? utcNow = null) => diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common.Tests/StatusCodeOracleTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common.Tests/StatusCodeOracleTests.cs index aa50f312..a672a621 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common.Tests/StatusCodeOracleTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common.Tests/StatusCodeOracleTests.cs @@ -39,6 +39,14 @@ public sealed class StatusCodeOracleTests HistorianInfoBits.WithCalculated(interpolatedGood).ShouldBe(0x400u | HistorianInfoBits.HistorianCalculated); } + [Fact] + public void Partial_bit_matches_the_SDK() => + ((uint)Opc.Ua.AggregateBits.Partial).ShouldBe(HistorianInfoBits.HistorianPartial); + + [Fact] + public void Generic_bad_matches_the_SDK() => + HistorianStatusCodes.Bad.ShouldBe(Opc.Ua.StatusCodes.Bad); + [Fact] public void Info_type_is_data_value() => HistorianInfoBits.InfoTypeDataValue.ShouldBe(0x400u); diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common.Tests/TimeWeightedAggregatorTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common.Tests/TimeWeightedAggregatorTests.cs index 4ee56877..e93d14b2 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common.Tests/TimeWeightedAggregatorTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common.Tests/TimeWeightedAggregatorTests.cs @@ -15,6 +15,8 @@ public sealed class TimeWeightedAggregatorTests private const uint GoodRaw = 0x00000000u; private const uint NoData = 0x809B0000u; private const uint InvalidInputs = 0x80D60000u; + private const uint BadCalc = 0x80000401u; // generic Bad | Calculated (an all-outage interval) + private const uint GoodCalcPartial = 0x00000405u; // Good | Calculated | Partial (truncated last interval) private static IReadOnlyList Run(HistoryAggregateType agg, double from, double to, double intervalS, params StoredSample[] samples) => @@ -38,10 +40,12 @@ public sealed class TimeWeightedAggregatorTests [Fact] public void TimeAverage_bridges_the_outage_on_tag_A() { - // B0 (150+250)/20 = 20; B1 (350+450)/20 = 40; B2, B3 = 50. Only B1's [30,40) and B0's [0,10) are Good ⇒ all Uncertain. + // B0 (150+250)/20 = 20; B1 (350+450)/20 = 40; B2, B3 = 50. Status (SDK rule, Uncertain time counts as Good): + // B0 is non-Bad throughout (held-into-outage time is Uncertain) ⇒ Good; B1 has the Bad [20,30) ⇒ Uncertain; + // B2, B3 are extrapolated (Uncertain) throughout ⇒ Good. var s = Run(HistoryAggregateType.TimeAverage, 0, 80, 20, A); Values(s, 20, 40, 50, 50); - s.Select(x => x.StatusCode).ShouldBe([UncertainCalc, UncertainCalc, UncertainCalc, UncertainCalc]); + s.Select(x => x.StatusCode).ShouldBe([GoodCalc, UncertainCalc, GoodCalc, GoodCalc]); s.Select(x => x.SourceTimestampUtc).ShouldBe([At(0), At(20), At(40), At(60)]); } @@ -58,7 +62,7 @@ public sealed class TimeWeightedAggregatorTests // B0 350/20 = 17.5; B1 450/10 = 45 (the Bad [20,30) excluded); B2, B3 = 50. var s = Run(HistoryAggregateType.TimeAverage2, 0, 80, 20, A); Values(s, 17.5, 45, 50, 50); - s.Select(x => x.StatusCode).ShouldBe([UncertainCalc, UncertainCalc, UncertainCalc, UncertainCalc]); + s.Select(x => x.StatusCode).ShouldBe([GoodCalc, UncertainCalc, GoodCalc, GoodCalc]); } [Fact] @@ -140,13 +144,14 @@ public sealed class TimeWeightedAggregatorTests } [Fact] - public void Past_the_last_sample_the_value_is_extrapolated_stepped_and_uncertain() + public void Past_the_last_sample_the_value_is_extrapolated_stepped() { - // -5 = 7 and nothing later: held at 7 over [0,10). + // -5 = 7 and nothing later: held at 7 over [0,10). The extrapolated time is Uncertain, which the SDK rule + // (TreatUncertainAsBad = false) counts as Good time ⇒ the interval is Good; the Interpolative point is Uncertain. StoredSample[] one = [S(-5, HistorianValue.FromDouble(7), Good)]; var s = Run(HistoryAggregateType.TimeAverage2, 0, 10, 0, one).Single(); ((double)s.Value!).ShouldBe(7d); - s.StatusCode.ShouldBe(UncertainCalc); + s.StatusCode.ShouldBe(GoodCalc); var i = Run(HistoryAggregateType.Interpolative, 0, 10, 0, one).Single(); ((double)i.Value!).ShouldBe(7d); i.StatusCode.ShouldBe(UncertainInterpolated); @@ -203,11 +208,15 @@ public sealed class TimeWeightedAggregatorTests } [Fact] - public void An_interval_entirely_inside_an_outage_is_BadNoData_but_Interpolative_bridges_it() + public void An_interval_entirely_inside_an_outage_is_Bad_but_Interpolative_bridges_it() { - // Window [10,15): raw is Bad throughout ⇒ no non-Bad time ⇒ BadNoData. Interpolative at +10 = 10 + 20·(10/15). - Run(HistoryAggregateType.TimeAverage, 10, 15, 0, Outage).Single().StatusCode.ShouldBe(NoDataCalc); - Run(HistoryAggregateType.TimeAverage2, 10, 15, 0, Outage).Single().StatusCode.ShouldBe(NoDataCalc); + // Window [10,15): raw is Bad throughout ⇒ data but no non-Bad time ⇒ generic Bad (the SDK rule; BadNoData is + // reserved for no data at all). Interpolative at +10 = 10 + 20·(10/15). + var ta = Run(HistoryAggregateType.TimeAverage, 10, 15, 0, Outage).Single(); + ta.StatusCode.ShouldBe(BadCalc); + ta.Value.ShouldBeNull(); + Run(HistoryAggregateType.TimeAverage2, 10, 15, 0, Outage).Single().StatusCode.ShouldBe(BadCalc); + Run(HistoryAggregateType.Total2, 10, 15, 0, Outage).Single().StatusCode.ShouldBe(BadCalc); var i = Run(HistoryAggregateType.Interpolative, 10, 15, 0, Outage).Single(); ((double)i.Value!).ShouldBe(10d + 20d * 10d / 15d, 1e-9); i.StatusCode.ShouldBe(UncertainInterpolated); @@ -269,10 +278,10 @@ public sealed class TimeWeightedAggregatorTests } [Fact] - public void An_uncertain_sample_makes_its_segment_uncertain() + public void Uncertain_time_counts_as_good_time() { - // 0 = 4 Uncertain, 10 = 6 Good, 20 = 6 Good: [0,10) sloped 4→6, quality Uncertain (worse end) ⇒ mean 5; - // [10,20) 6 Good. Window [0,20), 10 s. + // TreatUncertainAsBad = false (the SDK's time-based status rule): Uncertain time counts toward PercentDataGood. + // 0 = 4 Uncertain, 10 = 6 Good, 20 = 6 Good: [0,10) sloped 4→6 ⇒ mean 5; [10,20) 6. Window [0,20), 10 s. StoredSample[] s = [ S(0, HistorianValue.FromDouble(4), Uncertain), S(10, HistorianValue.FromDouble(6), Good), @@ -280,7 +289,7 @@ public sealed class TimeWeightedAggregatorTests ]; var r = Run(HistoryAggregateType.TimeAverage2, 0, 20, 10, s); Values(r, 5, 6); - r.Select(x => x.StatusCode).ShouldBe([UncertainCalc, GoodCalc]); + r.Select(x => x.StatusCode).ShouldBe([GoodCalc, GoodCalc]); } [Fact] @@ -322,10 +331,86 @@ public sealed class TimeWeightedAggregatorTests } [Fact] - public void A_truncated_last_bucket_uses_its_own_duration_for_Total() + public void A_truncated_last_bucket_uses_its_own_duration_for_Total_and_is_Partial() { // Window [0,50), 20 s: B2 = [40,50) is 10 s of the extrapolated 50 ⇒ TimeAverage 50, Total 500. - Values(Run(HistoryAggregateType.Total, 0, 50, 20, A), 400, 800, 500); + var s = Run(HistoryAggregateType.Total, 0, 50, 20, A); + Values(s, 400, 800, 500); + // Only the truncated interval carries the Partial bit. + s.Select(x => x.StatusCode).ShouldBe([GoodCalc, UncertainCalc, GoodCalcPartial]); + } + + [Fact] + public void An_outage_with_no_later_sample_is_Bad_to_the_window_end_not_no_data() + { + // 0 = 10 G, then an outage marker at +5 and nothing after: [10,20) lies wholly inside the still-open outage. + StoredSample[] open = [S(0, HistorianValue.FromDouble(10), Good), S(5, HistorianValue.Empty, BadNoCommunication)]; + Run(HistoryAggregateType.TimeAverage2, 10, 20, 0, open).Single().StatusCode.ShouldBe(BadCalc); + } + + [Fact] + public void A_bad_or_no_data_truncated_interval_carries_no_partial_bit() + { + // Window [0,15), 10 s: B1 = [10,15) is inside the outage ⇒ Bad without Partial. + Run(HistoryAggregateType.TimeAverage2, 0, 15, 10, Outage)[1].StatusCode.ShouldBe(BadCalc); + Run(HistoryAggregateType.TimeAverage2, 0, 15, 10)[1].StatusCode.ShouldBe(NoDataCalc); + } + + // ---------------------------------------------------------------- reverse Interpolative (Part 13: anchored at the later time) + + [Fact] + public void A_reverse_Interpolative_read_is_anchored_at_its_later_time() + { + // Request At(20) → At(0), 20 s: one point, at At(20) (not At(0)). Bounded: segment 10..30 is 20 flat, Good. + // A forward read of the same window would give the value at At(0) = 10. + var s = TimeWeightedAggregator.Evaluate(HistoryAggregateType.Interpolative, + BucketPlan.Create(At(20), At(0), TimeSpan.FromSeconds(20)), Bounded, Now); + s.Single().SourceTimestampUtc.ShouldBe(At(20)); + ((double)s.Single().Value!).ShouldBe(20d); + s.Single().StatusCode.ShouldBe(GoodInterpolated); + } + + [Fact] + public void A_reverse_Interpolative_read_steps_back_from_the_later_time_latest_first() + { + // Tag A, At(80) → At(0), 20 s: points 80, 60, 40, 20 (latest first). 80 and 60 are past the last sample ⇒ 50 + // extrapolated, Uncertain; 40 exact Good 50; 20 sits on the Bad sample ⇒ bridged 30, Uncertain. + var s = TimeWeightedAggregator.Evaluate(HistoryAggregateType.Interpolative, + BucketPlan.Create(At(80), At(0), TimeSpan.FromSeconds(20)), A, Now); + s.Select(x => x.SourceTimestampUtc).ShouldBe([At(80), At(60), At(40), At(20)]); + Values(s, 50, 50, 50, 30); + s.Select(x => x.StatusCode).ShouldBe([UncertainInterpolated, UncertainInterpolated, GoodRaw, UncertainInterpolated]); + } + + [Fact] + public void A_reverse_Interpolative_read_with_a_partial_interval_ends_inside_the_window() + { + // At(25) → At(0), 10 s: three intervals ⇒ points 25, 15, 5 (never before the window start). + // Bounded: -10..10 slope 1/s from 0 ⇒ +5 = 15; 10..30 flat 20 ⇒ +15 = 20, +25 = 20. + var s = TimeWeightedAggregator.Evaluate(HistoryAggregateType.Interpolative, + BucketPlan.Create(At(25), At(0), TimeSpan.FromSeconds(10)), Bounded, Now); + s.Select(x => x.SourceTimestampUtc).ShouldBe([At(25), At(15), At(5)]); + Values(s, 20, 20, 15); + } + + // ---------------------------------------------------------------- raw-row cap helpers + + [Theory] + [InlineData(1_000_000, 1_000_001)] + [InlineData(1, 2)] + [InlineData(0, int.MaxValue)] + [InlineData(-5, int.MaxValue)] + [InlineData(int.MaxValue, int.MaxValue)] + public void The_window_fetch_limit_is_one_past_the_cap(int cap, int expected) => + TimeWeightedAggregator.WindowFetchLimit(cap).ShouldBe(expected); + + [Fact] + public void The_window_row_check_fails_only_past_a_positive_cap() + { + TimeWeightedAggregator.CheckWindowRows(4, 4, "T"); + TimeWeightedAggregator.CheckWindowRows(5_000_000, 0, "T"); + var ex = Should.Throw(() => TimeWeightedAggregator.CheckWindowRows(5, 4, "T")); + ex.Limit.ShouldBe(4); } [Fact] diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer.IntegrationTests/ReadTestDb.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer.IntegrationTests/ReadTestDb.cs index 8af5e084..0fd95144 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer.IntegrationTests/ReadTestDb.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer.IntegrationTests/ReadTestDb.cs @@ -44,8 +44,9 @@ internal sealed class ReadTestDb } /// A fresh data source (fresh catalog cache) over this schema; is its "now". - public SqlServerHistorianDataSource Source(TimeProvider? clock = null) => - new(Conn, Boot, new SqlServerTagCatalog(Conn, Boot), timeProvider: clock); + public SqlServerHistorianDataSource Source(TimeProvider? clock = null, + int maxTimeWeightedRawRows = TimeWeightedAggregator.DefaultMaxWindowRows) => + new(Conn, Boot, new SqlServerTagCatalog(Conn, Boot), timeProvider: clock, maxTimeWeightedRawRows: maxTimeWeightedRawRows); /// Inserts a tag row and the given samples. ServerTimeUtc = source time + 1 s (so tests can tell them apart). public async Task SeedAsync(string tag, params (DateTime T, HistorianValue V, uint Status)[] samples) diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer.IntegrationTests/SqlServerConformanceBackend.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer.IntegrationTests/SqlServerConformanceBackend.cs index e4c27a8c..79b71ce4 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer.IntegrationTests/SqlServerConformanceBackend.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer.IntegrationTests/SqlServerConformanceBackend.cs @@ -25,6 +25,9 @@ internal sealed class SqlServerConformanceBackend : IConformanceBackend /// public IHistorianDataSource CreateDataSource(TimeProvider? clock = null) => _db.Source(clock); + /// + public IHistorianDataSource CreateDataSource(int maxTimeWeightedRawRows) => _db.Source(null, maxTimeWeightedRawRows); + /// public IHistorianValueWriter CreateValueWriter(Func? utcNow = null) => new SqlServerHistorianValueWriter(_db.Conn, _db.Boot, new SqlServerTagCatalog(_db.Conn, _db.Boot), utcNow: utcNow); diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.Tests/NodeManagerHistoryReadTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.Tests/NodeManagerHistoryReadTests.cs index 9d3394d7..f9eebd88 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.Tests/NodeManagerHistoryReadTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.Tests/NodeManagerHistoryReadTests.cs @@ -342,6 +342,28 @@ public sealed class NodeManagerHistoryReadTests : IDisposable await host.DisposeAsync(); } + /// #671: a read past a backend size limit (the time-weighted raw-row cap) answers BadTooManyOperations + /// for that node (not the generic BadHistoryOperationUnsupported, nor a deadline BadTimeout). + [Fact] + public async Task Processed_limit_exceeded_yields_BadTooManyOperations() + { + var (host, server) = await BootAsync(); + var nm = server.NodeManager!; + nm.HistorianDataSource = new RecordingHistorianDataSource { LimitExceeded = true }; + nm.EnsureVariable("eq-1/cap", parentFolderNodeId: null, displayName: "C", dataType: "Float", + writable: false, historianTagname: "WW.Cap", realm: AddressSpaceRealm.Uns); + var nodeId = nm.TryGetVariable("eq-1/cap")!.NodeId; + + var (_, errors) = InvokeHistoryRead(server, nm, new ReadProcessedDetails + { + StartTime = DateTime.UtcNow.AddHours(-1), EndTime = DateTime.UtcNow, ProcessingInterval = 10_000.0, + AggregateType = new NodeIdCollection { ObjectIds.AggregateFunction_Total }, + }, nodeId); + + errors[0].StatusCode.Code.ShouldBe(StatusCodes.BadTooManyOperations); + await host.DisposeAsync(); + } + /// AtTime read: the requested timestamps reach ReadAtTimeAsync in order. [Fact] public async Task AtTime_dispatches_requested_timestamps() @@ -505,6 +527,7 @@ public sealed class NodeManagerHistoryReadTests : IDisposable { public bool ThrowOnRead { get; init; } public bool AggregateNotSupported { get; init; } + public bool LimitExceeded { get; init; } public HistorianRead RawResult { get; set; } = new(Array.Empty(), null); public HistorianRead ProcessedResult { get; set; } = new(Array.Empty(), null); public HistorianRead AtTimeResult { get; set; } = new(Array.Empty(), null); @@ -537,6 +560,7 @@ public sealed class NodeManagerHistoryReadTests : IDisposable { if (ThrowOnRead) throw new InvalidOperationException("backend boom"); if (AggregateNotSupported) throw new HistoryAggregateNotSupportedException(aggregate); + if (LimitExceeded) throw new HistoryReadLimitExceededException(4, "too many raw rows"); LastCall = "Processed"; LastTagname = fullReference; LastStart = startUtc; diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Historian/ServerHistorianOptionsTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Historian/ServerHistorianOptionsTests.cs index 458070cd..4f38cc8a 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Historian/ServerHistorianOptionsTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Historian/ServerHistorianOptionsTests.cs @@ -89,6 +89,16 @@ public sealed class ServerHistorianOptionsTests Assert.Contains(w, m => m.Contains("HistoryReadDeadline")); } + [Fact] + public void MaxTimeWeightedRawRows_defaults_to_one_million_and_warns_when_nonpositive() + { + Assert.Equal(1_000_000, new ServerHistorianOptions().MaxTimeWeightedRawRows); + var w = new ServerHistorianOptions + { Enabled = true, Endpoint = "https://h:5222", ApiKey = "histgw_x_y", MaxTimeWeightedRawRows = 0 } + .Validate(); + Assert.Contains(w, m => m.Contains("MaxTimeWeightedRawRows")); + } + [Fact] public void Disabled_with_nonpositive_HistoryRead_knobs_is_silent() {