From 39552e15b94d6d9eca3750f82b2440a9ce69f33b Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Mon, 12 Oct 2026 16:08:54 -0400 Subject: [PATCH] fix(historian): #690 Gateway raw/AtTime Bad rows carry no value (a stored NULL was value 0) --- CLAUDE.md | 2 +- docs/Historian.md | 7 +++++- .../Mapping/SampleMapper.cs | 14 +++++++++-- .../GatewayHistorianDataSourceTests.cs | 15 +++++++++++ .../GatewayRawBoundaryRowTests.cs | 9 +++++++ .../Mapping/SampleMapperTests.cs | 25 +++++++++++++++++++ 6 files changed, 68 insertions(+), 4 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 9672b8ce..18a614fe 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1364,7 +1364,7 @@ historian tagname, which is the tag's driver `FullName`. The server dispatches all OPC UA HistoryRead to the registered `IHistorianDataSource` — the `GatewayHistorianDataSource` read client when enabled, else the `NullHistorianDataSource` default -(historized nodes return `GoodNoData` for Raw/AtTime — Processed is `BadAggregateNotSupported` since #682). Supported variants: Raw (⚠️ #687, client-visible: Gateway raw reads no longer return AVEVA's synthetic start-boundary row — asked from 1 ms early (epoch-floored) with `NumValuesPerNode + 16` rows (unclamped; 0 ⇒ `int.MaxValue`) and filtered by timestamp — so an empty window is `GoodNoData`, not a fabricated value at StartTime; the forward end is now exclusive (a point read inclusive); pre-window rows that crowd the budget trigger ONE re-read with room for them, else `BadTooManyOperations`; `ReturnBounds` is ignored on every backend), Processed +(historized nodes return `GoodNoData` for Raw/AtTime — Processed is `BadAggregateNotSupported` since #682). Supported variants: Raw (⚠️ #687, client-visible: Gateway raw reads no longer return AVEVA's synthetic start-boundary row — asked from 1 ms early (epoch-floored) with `NumValuesPerNode + 16` rows (unclamped; 0 ⇒ `int.MaxValue`) and filtered by timestamp — so an empty window is `GoodNoData`, not a fabricated value at StartTime; the forward end is now exclusive (a point read inclusive); pre-window rows that crowd the budget trigger ONE re-read with room for them, else `BadTooManyOperations`; `ReturnBounds` is ignored on every backend; ⚠️ #690: a Gateway Bad row — a stored NULL, which histsdk sends as value 0 — now reads value **null** + Bad on Raw and AtTime, as on the SQL backends), Processed (backend-dependent: SqlServer/QuestDb serve seventeen; Gateway twelve = `GatewayHistorianDataSource.ServedAggregates` — ⚠️ #681, client-visible: the ten per-sample ones (Average, Count, Min/Max, Range, Start/End/Delta, Min/MaxActualTime) are evaluated from RAW gateway reads by the shared `Sql.Common` evaluator — identical to the SQL backends, Calculated bits, interval-start stamps, #677 reverse; Min/Max are no longer AVEVA modes, the read moves every raw sample of the window (a month of 1 Hz ⇒ `BadTooManyOperations` at the default `MaxTimeWeightedRawRows` cap), and every gateway stream is consumed to completion (an abandoned one makes the gateway evict its shared historian session) — plus TimeAverage/Total via AVEVA modes (interval-start stamped + Calculated; ⚠️ #685: AVEVA's extra pre-window row is dropped ⇒ one row per interval in `[Start, End)`, and an Uncertain (opc 64) interval keeps its value as `UncertainDataSubNormal` — AVEVA's TWA excludes NULL-gap time where the SQL backends bridge it; per-interval values still need gateway fix historiangw#27); the other time-weighted five stay `BadAggregateNotSupported` because the gateway has no prior-sample read for Part 13 bounds; `docs/Historian.md` §Aggregate set), and AtTime over historized variable nodes; Events over alarm-owning equipment-folder event-notifier nodes. Reads are ungated (served from any redundancy node); authorization uses the standard `AccessLevels.HistoryRead` bit set at materialization. diff --git a/docs/Historian.md b/docs/Historian.md index 39a894af..f15f4a0e 100644 --- a/docs/Historian.md +++ b/docs/Historian.md @@ -686,7 +686,12 @@ data. the gateway is asked from **1 ms before** the window (never before the Unix epoch, the start of the historian's time encoding), so the boundary row lands before it, and only rows `RawAggregateReader.IsWindowSample` are returned. The decision is by **timestamp**, not by value or quality — a stored NULL sample looks exactly like the no-data boundary row on the wire (value 0, - Bad), and a stored NULL inside the window is still returned (Bad), as it is a stored sample. A row stamped **at or before the + Bad), and a stored NULL inside the window is still returned (Bad), as it is a stored sample. ⚠️ **Since #690 it is returned + with a null value** (client-visible): histsdk puts a value on every row, so the NULL used to come back as value `0` + Bad (`""` + on a string tag), a fabricated 0 for any client that reads the value without the status. `SampleMapper.ToSnapshot` drops the + value of every row whose mapped status is **Bad** (AVEVA stores a Bad-quality sample as NULL, so a Bad row never carries a real + value) — raw reads and AtTime alike, matching SqlServer/QuestDB (an Empty sample reads null + Bad). Uncertain rows keep their + value; the raw-evaluated aggregates were already unaffected (a Bad sample is never usable, and Start/End null a Bad value). A row stamped **at or before the epoch** is never a stored sample (AVEVA stores nothing there — the #686 walk-back's floor too): a window starting at or before the epoch gets no margin, so AVEVA's no-data row lands exactly at its start; raw reads and the per-sample aggregates both drop it. - **The forward end is exclusive** (`[StartTime, EndTime)`, the `IHistorianDataSource` contract and what the SQL backends return; diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway/Mapping/SampleMapper.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway/Mapping/SampleMapper.cs index 1e090edd..925db864 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway/Mapping/SampleMapper.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway/Mapping/SampleMapper.cs @@ -23,11 +23,21 @@ internal static class SampleMapper /// Maps a single raw sample to a value snapshot. /// The gateway raw sample. /// The driver-agnostic snapshot. + /// + /// #690: a Bad row carries no value. histsdk puts a value on EVERY row (NumericValue = 0 on a numeric row, + /// StringValue = "" on a string one), so a stored NULL arrives as 0 / "" with a Bad OPC quality + /// (live: q=1 opc=0) — that 0 is not a value, and AVEVA stores a Bad-quality sample as NULL, so a Bad row never + /// carries a real one. The value is therefore dropped whenever the mapped status is Bad, matching the SqlServer/QuestDB + /// backends (an Empty sample is null + Bad) and the aggregate mapper's no-data rows. Uncertain rows keep their value. + /// public static DataValueSnapshot ToSnapshot(HistorianSample sample) { + var status = ToStatus(sample); // proto3 explicit presence: prefer the numeric value, else the string value, else null. object? value; - if (sample.HasNumericValue) + if (StatusBits.IsBad(status)) + value = null; + else if (sample.HasNumericValue) value = sample.NumericValue; // boxes as System.Double else if (sample.HasStringValue) value = sample.StringValue; @@ -36,7 +46,7 @@ internal static class SampleMapper return new DataValueSnapshot( Value: value, - StatusCode: ToStatus(sample), + StatusCode: status, SourceTimestampUtc: sample.Timestamp?.ToDateTime(), // Utc kind ServerTimestampUtc: DateTime.UtcNow); } diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests/GatewayHistorianDataSourceTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests/GatewayHistorianDataSourceTests.cs index 8149ed0d..8ed951c2 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests/GatewayHistorianDataSourceTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests/GatewayHistorianDataSourceTests.cs @@ -111,6 +111,21 @@ public sealed class GatewayHistorianDataSourceTests Assert.Equal(0x80000000u, r.Samples[1].StatusCode); // gap → Bad at requested ts } + [Fact] // #690: an at-time NULL (histsdk: value 0 + Bad) is null + Bad, like the SQL backends' AtTime of a Bad sample + public async Task ReadAtTime_of_a_NULL_row_carries_no_value() + { + var fake = new FakeHistorianGatewayClient(); + var t0 = new DateTime(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc); + fake.AtTimeSamples = new[] + { + new HistorianSample { NumericValue = 0, OpcQuality = 0, Quality = 1, Timestamp = Timestamp.FromDateTime(t0) }, + }; + var ds = new GatewayHistorianDataSource(fake, NullLogger.Instance); + var r = await ds.ReadAtTimeAsync("T", new[] { t0 }, TestContext.Current.CancellationToken); + Assert.Null(r.Samples[0].Value); + Assert.Equal(0x80000000u, r.Samples[0].StatusCode); + } + [Fact] public async Task Empty_window_is_not_a_fault() { diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests/GatewayRawBoundaryRowTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests/GatewayRawBoundaryRowTests.cs index 2aec396e..b0b318de 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests/GatewayRawBoundaryRowTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests/GatewayRawBoundaryRowTests.cs @@ -109,6 +109,15 @@ public sealed class GatewayRawBoundaryRowTests r.Samples[0].StatusCode.ShouldBe(0x80000000u); } + [Fact] // #690: the wire's value 0 on a NULL row is not a value — SqlServer/QuestDB return null + Bad for the same sample + public async Task A_stored_NULL_sample_carries_no_value_not_the_wires_zero() + { + var r = await Source(Gate()).ReadRawAsync("T", At(30), At(55), 0, Ct); + r.Samples[0].SourceTimestampUtc.ShouldBe(At(40)); + r.Samples[0].Value.ShouldBeNull(); + r.Samples[1].Value.ShouldBe(5d); // a Good neighbour keeps its value + } + [Fact] public async Task A_stored_sample_exactly_at_the_window_start_is_kept() { diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests/Mapping/SampleMapperTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests/Mapping/SampleMapperTests.cs index 44d2692b..1092650b 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests/Mapping/SampleMapperTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests/Mapping/SampleMapperTests.cs @@ -32,6 +32,31 @@ public sealed class SampleMapperTests Assert.Equal(0x80000000u, SampleMapper.ToSnapshot(s).StatusCode); } + [Theory] // #690: histsdk puts a value on every row, so a NULL row arrives as 0 / "" with a Bad quality + [InlineData(0u, 1u)] // the live NULL row: opc_quality 0, AVEVA quality 1 + [InlineData(24u, 0u)] // any Bad OPC quality (out of service) + public void A_Bad_numeric_row_carries_no_value(uint opc, uint quality) + { + var s = new HistorianSample { Tag = "T", NumericValue = 0, OpcQuality = opc, Quality = quality, Timestamp = Ts(2026, 1, 1, 0, 0, 0) }; + var snap = SampleMapper.ToSnapshot(s); + Assert.True(snap.StatusCode >= 0x80000000u); + Assert.Null(snap.Value); + } + + [Fact] // #690: a string tag's NULL row arrives as "" + Bad + public void A_Bad_string_row_carries_no_value() + { + var s = new HistorianSample { Tag = "T", StringValue = "", OpcQuality = 0, Quality = 1, Timestamp = Ts(2026, 1, 1, 0, 0, 0) }; + Assert.Null(SampleMapper.ToSnapshot(s).Value); + } + + [Fact] // #690: an Uncertain row keeps its value (only Bad means "no value") + public void An_Uncertain_row_keeps_its_value() + { + var s = new HistorianSample { Tag = "T", NumericValue = 3, OpcQuality = 64, Timestamp = Ts(2026, 1, 1, 0, 0, 0) }; + Assert.Equal(3d, SampleMapper.ToSnapshot(s).Value); + } + [Fact] public void Aggregate_null_value_is_BadNoData() {