Merge #821 live TimeAverage/Total late-start with Uncertain first sample matches HistoryRead (Part 13 partial interval)
Claude-Session: https://claude.ai/code/session_01GT67AD99zQQqpLCVm3p7xk
This commit is contained in:
@@ -32,7 +32,7 @@ historian tagname, which is the tag's driver `FullName`.
|
||||
### Pluggable backends, SQL Server and HistoryUpdate (2026-10)
|
||||
|
||||
- **Selection:** `ServerHistorian:Backend` (`Gateway` default | `SqlServer` | `QuestDb`; unknown fails host start). `HistorianBackendRegistry` → `IHistorianBackendProvider` builds all five seams; the gateway keys are only required for `Gateway`. `SqlServer` uses `ServerHistorian:SqlServer:{ConnectionString (required), Schema=hist, CommandTimeout=30s, AutoCreateSchema=true}`.
|
||||
- **SQL Server** (`Driver.Historian.SqlServer`): `hist.Sample` PK `(TagId, SourceTimeUtc)` — writes are insert-if-absent, so outbox redelivery is idempotent. Stores every value kind with its **real status**; per-sample aggregates (10, simple definitions) are pushed down to T-SQL with bucket math in **100 ns ticks**; the Part 13 time-weighted seven (TimeAverage/TimeAverage2/Total/Total2/Interpolative, #671 — ⚠️ `Total` is now the time-weighted integral, no longer `SUM` — plus the **duration-based** PercentGood/PercentBad, #680, and DurationGood/DurationBad, #820 — ms, Uncertain time in neither; since #809 the live AggregateFilter agrees — `GoodOnlyStatusAggregateCalculator` overrides only those two of the frozen calculators) stream window + 4 bound rows into the shared `TimeWeightedAggregator` (both SQL backends, identical results; reverse `Interpolative` anchored at the later time; window capped by `ServerHistorian:MaxTimeWeightedRawRows` ⇒ `BadTooManyOperations`, percentages included). ⚠️ **#680 status rules (client-visible):** TimeAverage/Total are Uncertain if any part of the interval uses an Uncertain value, is bridged across a Bad sample, is extrapolated past the last sample (⇒ a live tag's latest interval is Uncertain) or is not covered; TimeAverage2/Total2 keep the SDK time-based rule (Uncertain time counts as Good) but never extrapolate past the last sample — that interval is Partial, later ones `BadNoData` (a report-by-exception tag with nothing after its last sample reads `BadNoData` until its next sample arrives). `docs/Historian.md` §"Time-weighted aggregates". ⚠️ **#680 per-sample (client-visible):** `Count` counts **Good** samples only (value-based status; an all-Bad interval is generic Bad, value null) and an empty interval **inside the tag's data range** is `0`/Good (outside it still `BadNoData`); `Start`/`End` return the **raw** first/last sample of any quality — its own timestamp and status, value null when Bad; `Delta` is Uncertain only when the interval's first or last sample is unusable; ⚠️ #684: `Average`/`Minimum`/`Maximum`/`Range`/`*ActualTime` use **Good samples only** (was Good + Uncertain; an Uncertain-only interval is `BadNoData`), Min/Max-family status Uncertain on a Bad sample, an Uncertain value beyond the Good extreme (Part 13 — the SDK ignores that) or an Uncertain non-numeric sample, and only a **Good** String/DateTime sample refuses them (`BadAggregateInvalidInputs`; Delta still refuses any usable one); and every per-sample aggregate but `Average` sets **end-of-data Partial** on the interval holding the last sample (forward incl. its earlier edge, reverse strictly inside), never on a Bad result. The data-range lookup is a key seek on SQL Server and a `tag` symbol-index seek on QuestDB (#683 — `{p}sample.tag` is `SYMBOL INDEX`; the bootstrapper ADDs it to a pre-#683 table, idempotent), read BEFORE the bucket statement so a racing append can never yield a Good `Count` of 0 (a concurrent HistoryUpdate delete is the mirror case and still can, transiently). `docs/Historian.md` §"Aggregates". Docker-dev recipe, schema and aggregate semantics: `docs/Historian.md` §"SQL Server backend". Integration tests need `HISTORIAN_SQL_TEST_CONNECTION` (shared host `10.100.0.35,14330`, `sa`/`OtOpcUaDev_2026!` — not docker-dev's `OtOpcUa!Dev123`; QuestDB: `HISTORIAN_QUESTDB_TEST_CONNECTION` against the shared `/opt/otopcua-questdb` fixture `10.100.0.35:8812`, #822, `infra/README.md` §2a). **Retention (#667):** opt-in `SqlServer:RetentionDays` (+ `PurgeInterval` 1h, `PurgeBatchSize` 4000) -> `HistorianRetentionService` purges old `hist.Sample` (per tag) / `hist.Event` in `DELETE TOP(n)` batches, Primary-gated on `IHistorizationRoleView`, one purger per DB via `sp_getapplock`; counter `otopcua.historian.retention.purged{table}`. Optional `IHistorianBackendProvider.CreateRetention` (default null). `docs/Historian.md` §Retention.
|
||||
- **SQL Server** (`Driver.Historian.SqlServer`): `hist.Sample` PK `(TagId, SourceTimeUtc)` — writes are insert-if-absent, so outbox redelivery is idempotent. Stores every value kind with its **real status**; per-sample aggregates (10, simple definitions) are pushed down to T-SQL with bucket math in **100 ns ticks**; the Part 13 time-weighted seven (TimeAverage/TimeAverage2/Total/Total2/Interpolative, #671 — ⚠️ `Total` is now the time-weighted integral, no longer `SUM` — plus the **duration-based** PercentGood/PercentBad, #680, and DurationGood/DurationBad, #820 — ms, Uncertain time in neither; since #809 the live AggregateFilter agrees — `GoodOnlyStatusAggregateCalculator` overrides only those two of the frozen calculators; since #821 live TimeAverage/Total also leave the span before an Uncertain first sample out when the data starts inside an interval (no start bound — Part 13 partial interval), via `CoveredTimeAverageAggregateCalculator`) stream window + 4 bound rows into the shared `TimeWeightedAggregator` (both SQL backends, identical results; reverse `Interpolative` anchored at the later time; window capped by `ServerHistorian:MaxTimeWeightedRawRows` ⇒ `BadTooManyOperations`, percentages included). ⚠️ **#680 status rules (client-visible):** TimeAverage/Total are Uncertain if any part of the interval uses an Uncertain value, is bridged across a Bad sample, is extrapolated past the last sample (⇒ a live tag's latest interval is Uncertain) or is not covered; TimeAverage2/Total2 keep the SDK time-based rule (Uncertain time counts as Good) but never extrapolate past the last sample — that interval is Partial, later ones `BadNoData` (a report-by-exception tag with nothing after its last sample reads `BadNoData` until its next sample arrives). `docs/Historian.md` §"Time-weighted aggregates". ⚠️ **#680 per-sample (client-visible):** `Count` counts **Good** samples only (value-based status; an all-Bad interval is generic Bad, value null) and an empty interval **inside the tag's data range** is `0`/Good (outside it still `BadNoData`); `Start`/`End` return the **raw** first/last sample of any quality — its own timestamp and status, value null when Bad; `Delta` is Uncertain only when the interval's first or last sample is unusable; ⚠️ #684: `Average`/`Minimum`/`Maximum`/`Range`/`*ActualTime` use **Good samples only** (was Good + Uncertain; an Uncertain-only interval is `BadNoData`), Min/Max-family status Uncertain on a Bad sample, an Uncertain value beyond the Good extreme (Part 13 — the SDK ignores that) or an Uncertain non-numeric sample, and only a **Good** String/DateTime sample refuses them (`BadAggregateInvalidInputs`; Delta still refuses any usable one); and every per-sample aggregate but `Average` sets **end-of-data Partial** on the interval holding the last sample (forward incl. its earlier edge, reverse strictly inside), never on a Bad result. The data-range lookup is a key seek on SQL Server and a `tag` symbol-index seek on QuestDB (#683 — `{p}sample.tag` is `SYMBOL INDEX`; the bootstrapper ADDs it to a pre-#683 table, idempotent), read BEFORE the bucket statement so a racing append can never yield a Good `Count` of 0 (a concurrent HistoryUpdate delete is the mirror case and still can, transiently). `docs/Historian.md` §"Aggregates". Docker-dev recipe, schema and aggregate semantics: `docs/Historian.md` §"SQL Server backend". Integration tests need `HISTORIAN_SQL_TEST_CONNECTION` (shared host `10.100.0.35,14330`, `sa`/`OtOpcUaDev_2026!` — not docker-dev's `OtOpcUa!Dev123`; QuestDB: `HISTORIAN_QUESTDB_TEST_CONNECTION` against the shared `/opt/otopcua-questdb` fixture `10.100.0.35:8812`, #822, `infra/README.md` §2a). **Retention (#667):** opt-in `SqlServer:RetentionDays` (+ `PurgeInterval` 1h, `PurgeBatchSize` 4000) -> `HistorianRetentionService` purges old `hist.Sample` (per tag) / `hist.Event` in `DELETE TOP(n)` batches, Primary-gated on `IHistorizationRoleView`, one purger per DB via `sp_getapplock`; counter `otopcua.historian.retention.purged{table}`. Optional `IHistorianBackendProvider.CreateRetention` (default null). `docs/Historian.md` §Retention.
|
||||
- **QuestDB** (`Driver.Historian.QuestDb`, `ServerHistorian:QuestDb:*`, Npgsql over PG wire; docker-dev overlay `docker-compose.questdb.yml`): same kinds/status/aggregates/HistoryUpdate as SqlServer, soft delete by tombstone (INSERT only — never `UPDATE`/`DELETE`; every read filters `NOT deleted`). ⚠️ **Timestamps are µs** (`QuestDbConnection.Round`/`BindTimestamp`; `IHistorianDataSource.TimestampResolutionTicks` = 10 steps HistoryRead paging), **pre-1970 cannot be stored** (writer drops, HistoryUpdate `BadOutOfRange`), every write waits for WAL visibility, TTL is enforced on write commits only, set Npgsql `Maximum Pool Size` < QuestDB's 64. `docs/Historian.md` §"QuestDB backend".
|
||||
- ⚠️ **Reverse processed reads are Part 13/SDK-anchored (#677, client-visible):** every aggregate on SqlServer/QuestDB now anchors a reverse read at its later time — window `(end, start]`, partial interval earliest, each stamped with its later edge (was: earlier-anchored, stamped earlier edge); on forward reads Partial also marks the interval the data starts inside (not Average — the SDK's rule, forward-only); a non-default client `AggregateConfiguration` ⇒ `BadAggregateConfigurationRejected`; Gateway provisioning sends `IntegralDivisor = 1`. SDK-parity tests: `SdkAggregateParityTests`. `docs/Historian.md` §Aggregates.
|
||||
- ⚠️ **Reverse / end-only RAW reads work on Gateway (#686, client-visible):** `IHistorianDataSource.SupportsReverseRaw` (SqlServer/QuestDB/Null `true`, Gateway `false`, interface default `false`, guard-tested) — the node manager sends `start > end` only to a backend that declares it; otherwise `ReverseRawReader` serves it from FORWARD reads: uncapped ⇒ ONE forward read reversed in memory; capped ⇒ a bounded walk back in segments (ms-aligned request bounds, width aimed from observed density at ≈ N + 8 rows and carried page to page in the continuation point — #689: it used to aim at half `MaxTieClusterOverfetch`, ~32k rows per page on a dense tag; a read asks for `2N + 18` rows (a discovery segment ≤ 1,026), `MaxTieClusterOverfetch` is only the tie-cluster ceiling — a full read is re-read, never answered truncated; only more than max(1, `MaxTieClusterOverfetch`) samples within ONE MILLISECOND ⇒ `BadTooManyOperations` (per ms, not per timestamp — requests are ms-rounded, and letting a segment narrow below 1 ms stepped a page through one ms a tick per read, ~10k reads); walks no later than now + 1 day and never before the Unix epoch; #691: after two empty segments it asks once for the whole remainder, and the stretch a page proved empty rides the continuation point — the last pages of a deep page-through no longer each walk ~14 reads to 1970). The raw/events tie-path oversized-cluster refusal is now `BadTooManyOperations` too (was `BadHistoryOperationUnsupported`). Uncapped Gateway raw reads (forward too) used to send `max_values = 0`, which the client/gateway reject — now unbounded (`int.MaxValue`; a ceiling refusal ⇒ `BadTooManyOperations`), and a cancelled read is no longer a backend failure. Since #669 such reads failed the node on Gateway and counted as a backend fault (the gateway rejects start > end — `GatewayHistorianDataSource.ReadRawAsync` now refuses one unsent); before #669 an end-only Gateway read returned the OLDEST N ascending — now the latest N at/before EndTime, newest first. Events follow since #691.4: a reverse/end-only capped events read on Gateway (`SupportsReverseEvents` false; Null now reports `true` and answers empty) is walked back the same way (ties sorted by EventId — the gateway orders by time only; one extra event found ⇒ continuation point) — it used to be refused `BadHistoryOperationUnsupported` (explicit reverse) or answered with the OLDEST N ascending (end-only); needs the gateway's `RuntimeDb:EventReadMaxRows` 0/unlimited (it truncates silently). Uncapped Gateway events reads (forward too) used to throw "Array dimensions exceeded" (result list pre-sized to `int.MaxValue`) — fixed. Above a truncated read the walk gallops up from the newest row seen, then bisects (never below 1 ms): a burst that ended 100 days ago ≈ 18 reads, a dense tag that stopped days ago ≈ 26. `docs/Historian.md` §"Open-ended and reverse raw reads".
|
||||
|
||||
+13
-10
@@ -1906,16 +1906,19 @@ timestamp, and the same treatment as the Min/Max-at-interval-start case above; m
|
||||
And an interval holding a **Good String (or DateTime) sample** is `BadAggregateInvalidInputs` for Average/Minimum/Maximum/Range/
|
||||
*ActualTime here (the documented #684 refusal rule — a Good non-numeric value is not a number to aggregate), while the SDK skips the
|
||||
string and serves the numeric samples (e.g. Average 4 Good); an Uncertain non-numeric sample refuses nothing on either side.
|
||||
One more, found by the #810 Uncertain conformance data (kept deliberately): over the interval the data **starts inside** when the
|
||||
**first sample is Uncertain**, the SDK's `TimeAverage`/`Total` integrate the uncovered span before it as a ramp from 0 (QP: `Total`
|
||||
188.96 for 163.96 over the covered 15 s, and `TimeAverage` 9.448 = 188.96 ÷ 20 s for 10.931 = 163.96 ÷ 15 s); ours integrates only
|
||||
the covered part (Part 13, as with a Good first sample, where all agree). The same answer comes from the frozen calculators the server
|
||||
registers for **live** `AggregateFilter` items and from SDK 2.0.1's own stock calculators, in both read directions — pinned by
|
||||
`SdkAggregateParityTests.An_Uncertain_first_sample_in_a_late_start_interval_makes_the_SDK_integrate_the_time_before_the_data`
|
||||
(ours / frozen / stock 2.0.1, with a Good-first control) and `OpcUaServer.IntegrationTests/LiveAggregateLateStartFactsTests` (through
|
||||
the booted server's runtime `AggregateManager`), and on both SQL backends by `QualityMixConformance.Time_weighted_values_over_partial_data_cover_only_the_data`
|
||||
(forward) and `…_in_a_reverse_read` (#821). Whether live subscriptions should follow is the open #821 owner decision (#809, the
|
||||
PercentGood/DurationGood half, is decided: live follows Part 13). The same
|
||||
One more, found by the #810 Uncertain conformance data: over the interval the data **starts inside** when the **first sample is
|
||||
Uncertain**, the SDK's `TimeAverage`/`Total` integrate the uncovered span before it as a ramp from 0 (QP: `Total` 188.96 for 163.96
|
||||
over the covered 15 s, and `TimeAverage` 9.448 = 188.96 ÷ 20 s for 10.931 = 163.96 ÷ 15 s); ours integrates only the covered part
|
||||
(Part 13 "No Start Bound: Calculate Partial Interval", as with a Good first sample, where all agree). SDK 2.0.1's stock calculators
|
||||
answer the same as the frozen 1.5.378.182 ones, in both read directions. ⚠️ **Since #821 (owner-approved 2026-11-11, client-visible)
|
||||
the LIVE `AggregateFilter` calculators follow Part 13 here too** — `CoveredTimeAverageAggregateCalculator` overrides only
|
||||
TimeAverage/Total of the frozen copies (only the value of such an interval changes; status, timestamps and bits are the baseline's).
|
||||
Pinned by `SdkAggregateParityTests.An_Uncertain_first_sample_in_a_late_start_interval_makes_the_SDK_integrate_the_time_before_the_data`
|
||||
(ours / live / untouched baseline / stock 2.0.1, with a Good-first control), `…Live_AggregateFilter_TimeAverage_and_Total_equal_HistoryRead_over_an_Uncertain_first_late_start`
|
||||
(forward + reverse, no masks), `OpcUaServer.IntegrationTests/LiveAggregateLateStartFactsTests` (through the booted server's runtime
|
||||
`AggregateManager`), `Sdk2AggregateCompatibilityTests.Issue821_*`, and on both SQL backends by `QualityMixConformance` (QP's
|
||||
TimeAverage/Total against the live oracle, and `Time_weighted_values_over_partial_data_cover_only_the_data` forward / `…_in_a_reverse_read`).
|
||||
(#809, the PercentGood/DurationGood half, is likewise decided: live follows Part 13.) The same
|
||||
suite feeds PercentGood/PercentBad, DurationGood/DurationBad (and Count, Average, TimeAverage(2), Total(2), Start/End/Delta, Interpolative) Good/Uncertain/Bad
|
||||
mixes, all-Uncertain intervals (0 % Good **and** 0 % Bad, Mantis 0006166) and partial-data intervals, hand-computed and against the
|
||||
#796 oracle — the baseline calculator's Uncertain-as-Good PercentGood fails it.
|
||||
|
||||
+7
-3
@@ -8,14 +8,18 @@ internal static class CompatibilityAggregateFactory
|
||||
{
|
||||
/// <remarks>
|
||||
/// PercentGood / DurationGood are served by the application-owned <see cref="GoodOnlyStatusAggregateCalculator"/>
|
||||
/// (#809: Uncertain time is not Good, as in Part 13 and HistoryRead); every other id is the unchanged baseline.
|
||||
/// (#809: Uncertain time is not Good, as in Part 13 and HistoryRead), TimeAverage / Total by
|
||||
/// <see cref="CoveredTimeAverageAggregateCalculator"/> (#821: no start bound ⇒ only the covered part of the interval,
|
||||
/// even when the first value is Uncertain); every other id is the unchanged baseline.
|
||||
/// </remarks>
|
||||
internal static IAggregateCalculator Create(NodeId id, DateTimeUtc start, DateTimeUtc end, double interval,
|
||||
bool stepped, AggregateConfiguration configuration, ITelemetryContext telemetry) =>
|
||||
new Adapter(GoodOnlyStatusAggregateCalculator.Overrides(id)
|
||||
? new GoodOnlyStatusAggregateCalculator(id, (DateTime)start, (DateTime)end, interval, stepped, configuration, telemetry)
|
||||
: Aggregators.CreateStandardCalculator(id, (DateTime)start, (DateTime)end, interval, stepped, configuration, telemetry)
|
||||
?? throw new ArgumentOutOfRangeException(nameof(id), "Unsupported baseline aggregate."));
|
||||
: CoveredTimeAverageAggregateCalculator.Overrides(id)
|
||||
? new CoveredTimeAverageAggregateCalculator(id, (DateTime)start, (DateTime)end, interval, stepped, configuration, telemetry)
|
||||
: Aggregators.CreateStandardCalculator(id, (DateTime)start, (DateTime)end, interval, stepped, configuration, telemetry)
|
||||
?? throw new ArgumentOutOfRangeException(nameof(id), "Unsupported baseline aggregate."));
|
||||
|
||||
/// <summary>Replaces exactly the existing standard factories; no aggregate IDs are added or removed.</summary>
|
||||
internal static async ValueTask RegisterAsync(AggregateManager manager, CancellationToken ct)
|
||||
|
||||
+115
@@ -0,0 +1,115 @@
|
||||
using Opc.Ua;
|
||||
using DataValue = ZB.MOM.WW.OtOpcUa.OpcUaServer.Compatibility.Aggregates.LegacyDataValue;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.Compatibility.Aggregates;
|
||||
|
||||
/// <summary>
|
||||
/// Application-owned override (#821) of the baseline <see cref="AverageAggregateCalculator"/> for <c>TimeAverage</c>
|
||||
/// and <c>Total</c> only: the time before the first usable value (no start bound) is never integrated, whatever the
|
||||
/// quality of that first value. Part 13 §5.4.3.6/§5.4.3.8 "No Start Bound: Calculate Partial Interval" — the rule the
|
||||
/// HistoryRead evaluator (<c>TimeWeightedAggregator</c>, SqlServer + QuestDB) already follows, and the one the baseline
|
||||
/// itself follows when the first value is Good.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// With no usable value before the interval, the baseline's interpolated start bound is <c>BadNoData</c> (value 0),
|
||||
/// so the first region <c>[start, first value)</c> is Bad and excluded. But <c>GetRegionsInValueSet</c>
|
||||
/// (ignoreBadData, sloped) marks the region before an Uncertain value <c>UncertainDataSubNormal</c> without the
|
||||
/// not-Bad guard every other branch has — so an <b>Uncertain</b> first value turned the Bad region non-Bad and the
|
||||
/// uncovered span was integrated as a ramp from 0 (QP: Total 188.96 for Part 13's 163.96, TimeAverage 9.448 for
|
||||
/// 10.931). SDK 2.0.1's stock calculator does the same.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// This class leaves the copied file untouched and restores exactly that one overwritten status: a region that
|
||||
/// starts at a Bad point (only the interpolated start bound can — the region machinery skips Bad raw values)
|
||||
/// keeps that point's Bad status. Every other region, the values, the result status (Uncertain either way: the
|
||||
/// region is non-Good), timestamp, value type and Calculated / Partial bits are the baseline's.
|
||||
/// <c>TimeAverage2</c> / <c>Total2</c> (simple bounds, no ignoreBadData) and stepped variables never reach that
|
||||
/// branch and fall through to the baseline unchanged.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
internal sealed class CoveredTimeAverageAggregateCalculator : AverageAggregateCalculator
|
||||
{
|
||||
/// <summary>True for the two aggregate ids this class overrides.</summary>
|
||||
internal static bool Overrides(NodeId aggregateId) =>
|
||||
aggregateId == ObjectIds.AggregateFunction_TimeAverage || aggregateId == ObjectIds.AggregateFunction_Total;
|
||||
|
||||
/// <summary>Initializes the calculator; same contract as the baseline constructor.</summary>
|
||||
public CoveredTimeAverageAggregateCalculator(
|
||||
NodeId aggregateId,
|
||||
DateTime startTime,
|
||||
DateTime endTime,
|
||||
double processingInterval,
|
||||
bool stepped,
|
||||
AggregateConfiguration configuration,
|
||||
ITelemetryContext? telemetry)
|
||||
: base(aggregateId, startTime, endTime, processingInterval, stepped, configuration, telemetry)
|
||||
{
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override DataValue ComputeValue(TimeSlice slice)
|
||||
{
|
||||
if (AggregateId == ObjectIds.AggregateFunction_TimeAverage) return ComputeCovered(slice, total: false);
|
||||
if (AggregateId == ObjectIds.AggregateFunction_Total) return ComputeCovered(slice, total: true);
|
||||
return base.ComputeValue(slice);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The baseline <c>ComputeTimeAverage(slice, useSimpleBounds: false, valueType)</c> with a region that starts at a
|
||||
/// Bad point kept Bad.
|
||||
/// </summary>
|
||||
private DataValue ComputeCovered(TimeSlice slice, bool total)
|
||||
{
|
||||
List<DataValue>? values = GetValuesWithInterpolatedBounds(slice);
|
||||
if (values == null || values.Count == 0)
|
||||
{
|
||||
return GetNoDataValue(slice);
|
||||
}
|
||||
|
||||
List<SubRegion> regions = GetRegionsInValueSet(values, true, Stepped);
|
||||
|
||||
double area = 0;
|
||||
double totalDuration = 0;
|
||||
bool nonGoodRegionsExists = false;
|
||||
|
||||
foreach (var region in regions)
|
||||
{
|
||||
// #821: the one change — undo the baseline's Bad → Uncertain overwrite of a region starting at a Bad point.
|
||||
if (StatusCode.IsBad(region.DataPoint.StatusCode) && StatusCode.IsNotBad(region.StatusCode))
|
||||
{
|
||||
region.StatusCode = region.DataPoint.StatusCode;
|
||||
}
|
||||
|
||||
double duration = region.Duration / 1000.0;
|
||||
|
||||
if (StatusCode.IsNotBad(region.StatusCode))
|
||||
{
|
||||
area += (region.StartValue + region.EndValue) * duration / 2;
|
||||
totalDuration += duration;
|
||||
}
|
||||
|
||||
if (StatusCode.IsNotGood(region.StatusCode))
|
||||
{
|
||||
nonGoodRegionsExists = true;
|
||||
}
|
||||
}
|
||||
|
||||
if (totalDuration == 0)
|
||||
{
|
||||
return GetNoDataValue(slice);
|
||||
}
|
||||
|
||||
double result = total ? area : area / totalDuration;
|
||||
|
||||
var value = new DataValue
|
||||
{
|
||||
WrappedValue = SdkValueBoundary.FromClr(result),
|
||||
SourceTimestamp = GetTimestamp(slice),
|
||||
ServerTimestamp = GetTimestamp(slice),
|
||||
StatusCode = nonGoodRegionsExists ? StatusCodes.UncertainDataSubNormal : StatusCodes.Good
|
||||
};
|
||||
value.StatusCode = value.StatusCode.WithAggregateBits(AggregateBits.Calculated);
|
||||
return value;
|
||||
}
|
||||
}
|
||||
@@ -28,3 +28,12 @@ The baseline `StatusAggregateCalculator.ComputeDurationGoodBad` counts Uncertain
|
||||
- `GoodOnlyStatusAggregateCalculator.cs` (application code, not a baseline copy) subclasses the baseline `StatusAggregateCalculator` and replaces only the PercentGood / DurationGood computation: the same slice values, simple bounds, stepped regions, empty-slice `BadNoData`, timestamps, value type and Calculated bit, with a region counted as Good only when its status is Good — whatever `TreatUncertainAsBad` says (when true, Uncertain is Bad, so it was never Good). `CompatibilityAggregateFactory.Create` routes exactly those two IDs to it; the other 35 IDs, including DurationBad / PercentBad / WorstQuality, are the baseline unchanged.
|
||||
- Golden test: the 64 vectors this changes (38 PercentGood + 26 DurationGood, all with an effective `TreatUncertainAsBad = false` over data with Uncertain regions) are exempted **exactly** — the PercentGood / DurationGood vectors whose baseline output differs from the same case's baseline vector with `TreatUncertainAsBad = true`, where 1.5.378.182 already counted Good regions only — and each is pinned to that twin's output. The pin is therefore baseline evidence, never regenerated from the application's calculator; any further drift in any of the 6,144 cases fails. `Sdk2AggregateCompatibilityTests.Issue809_*` also pins a hand-computed case.
|
||||
- Live-vs-history parity: `SdkAggregateParityTests.Live_AggregateFilter_PercentGood_and_DurationGood_equal_HistoryRead_over_Uncertain_samples` compares the registered calculator with `TimeWeightedAggregator` over Good / Uncertain / Bad mixes with no masks, and checks that the data discriminates against the pre-#809 baseline.
|
||||
|
||||
### #821 — TimeAverage / Total integrate only the covered part when there is no start bound (owner-approved 2026-11-11)
|
||||
|
||||
When the data starts inside an interval (no usable value before it) the baseline's interpolated start bound is `BadNoData`, so the region before the first value is Bad and left out — Part 13 §5.4.3.6/§5.4.3.8 "No Start Bound: Calculate Partial Interval", and what the HistoryRead evaluator (`TimeWeightedAggregator`, SqlServer + QuestDB) does. But when that first value is **Uncertain**, `AggregateCalculator.GetRegionsInValueSet` (ignoreBadData, sloped) overwrites the region's Bad status with `UncertainDataSubNormal` (the one branch without the not-Bad guard), so `AverageAggregateCalculator` integrated the uncovered span as a ramp from 0 and divided TimeAverage by the whole interval (QP: Total 188.96 for 163.96, TimeAverage 9.448 for 10.931). SDK 2.0.1's stock calculator does the same. ScadaBridge uses neither AggregateFilter nor these aggregates.
|
||||
|
||||
- The copied upstream files above are **unchanged** (their SHA-256s still match `baseline-sources.json`).
|
||||
- `CoveredTimeAverageAggregateCalculator.cs` (application code) subclasses the baseline `AverageAggregateCalculator` and recomputes only TimeAverage / Total: the baseline's interpolated-bounds values and regions, with a region that starts at a Bad point (only the start bound can) kept Bad. Values change only for such an interval; status (Uncertain either way, except `BadNoData` when nothing at all is covered), timestamps, value type and Calculated / Partial bits are the baseline's. Stepped variables and `TreatUncertainAsBad = true` never reached the defect and are unchanged. `CompatibilityAggregateFactory.Create` routes exactly those two IDs to it; TimeAverage2 / Total2 and the other 31 IDs are the baseline unchanged.
|
||||
- Golden test: **no** fixture vector changes — every fixture data set has a usable value at −10 s, before every window, so no fixture interval lacks a start bound; all 384 TimeAverage / Total vectors stay held to their own output (`Issue821FixtureExemptions = 0`). The change is pinned over late-start variants of every data set (cut at +5 s and +25 s, first remaining non-Bad sample made Uncertain, every direction / stepping / configuration / incremental mode): 150 and 142 of the 384 change, each held to a baseline-derived oracle — the untouched baseline's value on the same variant with that sample Good, its status / timestamps / bits on the variant itself. `Sdk2AggregateCompatibilityTests.Issue821_*` also pins a hand-computed case.
|
||||
- Live-vs-history parity: `SdkAggregateParityTests.Live_AggregateFilter_TimeAverage_and_Total_equal_HistoryRead_over_an_Uncertain_first_late_start` compares the registered calculator with `TimeWeightedAggregator`, forward and reverse, with no masks, and checks that the data discriminates against the pre-#821 baseline; `QualityMixConformance` now includes TimeAverage / Total over QP on both SQL backends.
|
||||
|
||||
@@ -31,8 +31,12 @@ namespace ZB.MOM.WW.OtOpcUa.OpcUaServer;
|
||||
/// (owner-approved 2026-11-11) overrides exactly those two (<c>GoodOnlyStatusAggregateCalculator</c>), so live and
|
||||
/// HistoryRead now agree on them — pinned by
|
||||
/// <c>SdkAggregateParityTests.Live_AggregateFilter_PercentGood_and_DurationGood_equal_HistoryRead_over_Uncertain_samples</c>.
|
||||
/// Other known live-vs-HistoryRead differences (e.g. #821, an Uncertain first sample in a late-start interval) are
|
||||
/// documented in <c>docs/Historian.md</c>.
|
||||
/// Likewise the frozen <c>AverageAggregateCalculator</c> integrated the span before an <b>Uncertain</b> first value
|
||||
/// as a ramp from 0 when the data starts inside an interval (no start bound) — live TimeAverage / Total disagreed
|
||||
/// with HistoryRead and Part 13 ("No Start Bound: Calculate Partial Interval"). #821 (owner-approved 2026-11-11)
|
||||
/// overrides exactly those two (<c>CoveredTimeAverageAggregateCalculator</c>) — pinned by
|
||||
/// <c>SdkAggregateParityTests.Live_AggregateFilter_TimeAverage_and_Total_equal_HistoryRead_over_an_Uncertain_first_late_start</c>.
|
||||
/// The remaining documented live-vs-HistoryRead differences are listed in <c>docs/Historian.md</c>.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// A request asking for <c>UseServerCapabilitiesDefaults</c>, or spelling out exactly these values, is served;
|
||||
|
||||
@@ -237,15 +237,16 @@ public abstract class QualityMixConformance : ConformanceBase
|
||||
public async Task Processed_reads_over_partial_data_match_the_Part_13_oracle(double from, double to, double intervalS)
|
||||
{
|
||||
// QP's start-of-data and end-of-data intervals against the oracle, forward and reverse (#821). Left out, each a documented or pinned divergence:
|
||||
// TimeAverage2 / Total2 (the SDK's 1 ms end-of-data region), the min/max family (the SDK sets Partial on a Bad
|
||||
// result), and TimeAverage / Total (pinned by Time_weighted_values_over_partial_data_cover_only_the_data below).
|
||||
// TimeAverage2 / Total2 (the SDK's 1 ms end-of-data region) and the min/max family (the SDK sets Partial on a Bad
|
||||
// result). TimeAverage / Total are in since #821: the live calculator no longer integrates the span before QP's
|
||||
// Uncertain first sample (the hand-computed values are pinned below as well).
|
||||
await B.SeedAsync("QP", QP);
|
||||
await AssertMatchesOracle("QP", Rounded(QP), from, to, intervalS,
|
||||
[
|
||||
HistoryAggregateType.PercentGood, HistoryAggregateType.PercentBad, HistoryAggregateType.DurationGood,
|
||||
HistoryAggregateType.DurationBad, HistoryAggregateType.Count,
|
||||
HistoryAggregateType.Average, HistoryAggregateType.Start, HistoryAggregateType.End, HistoryAggregateType.Delta,
|
||||
HistoryAggregateType.Interpolative,
|
||||
HistoryAggregateType.Interpolative, HistoryAggregateType.TimeAverage, HistoryAggregateType.Total,
|
||||
]);
|
||||
}
|
||||
|
||||
@@ -255,9 +256,9 @@ public abstract class QualityMixConformance : ConformanceBase
|
||||
// Part 13 §5.4.3.3/§5.4.3.8 over QP's [0,20) (data from +5): the interpolated-bounding signal is defined from +5
|
||||
// only — 10 → 11 over [5,12) (area 73.5) and 11 → 11 + 8/13 over [12,20) (area 4·(22 + 8/13)) — so
|
||||
// Total = 73.5 + 88 + 32/13 and TimeAverage = Total ÷ 15 s; Uncertain (an Uncertain value used, not fully covered),
|
||||
// Partial (the data starts inside). Divergence found by #810: the 1.5.378.182 baseline, whose first sample is
|
||||
// Uncertain here, integrates the uncovered [0,5) as a ramp from 0 (Total +25) — so these are pinned by hand, not
|
||||
// against the oracle.
|
||||
// Partial (the data starts inside). Found by #810: the untouched 1.5.378.182 baseline, whose first sample is
|
||||
// Uncertain here, integrates the uncovered [0,5) as a ramp from 0 (Total +25); since #821 the live calculator (the
|
||||
// oracle) does not.
|
||||
await B.SeedAsync("QP", QP);
|
||||
var src = B.CreateDataSource();
|
||||
const double total = 73.5 + 88 + 32d / 13;
|
||||
@@ -268,8 +269,10 @@ public abstract class QualityMixConformance : ConformanceBase
|
||||
((double)avg.Value!).ShouldBe(total / 15, 1e-6);
|
||||
avg.StatusCode.ShouldBe(UncertainCalc | Partial);
|
||||
|
||||
var sdk = SdkAggregateOracle.Run(HistoryAggregateType.Total, B.Round(At(0)), B.Round(At(20)), Twenty, Rounded(QP)).Single();
|
||||
Convert.ToDouble(sdk.Value).ShouldBe(total + 25, 1e-6, "baseline divergence — if this moves, compare QP's TimeAverage/Total to the oracle");
|
||||
var live = SdkAggregateOracle.Run(HistoryAggregateType.Total, B.Round(At(0)), B.Round(At(20)), Twenty, Rounded(QP)).Single();
|
||||
Convert.ToDouble(live.Value).ShouldBe(total, 1e-6, "live AggregateFilter equals HistoryRead (#821)");
|
||||
var before = SdkAggregateOracle.RunSdkAsIs(HistoryAggregateType.Total, B.Round(At(0)), B.Round(At(20)), Twenty, Rounded(QP)).Single();
|
||||
Convert.ToDouble(before.Value).ShouldBe(total + 25, 1e-6, "the untouched baseline (live before #821)");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -277,8 +280,8 @@ public abstract class QualityMixConformance : ConformanceBase
|
||||
{
|
||||
// #821: the reverse-order variant of the late-start case above — (0,20] read 20 → 0. The same Part 13 integral and
|
||||
// average over the covered [5,20); Uncertain; NOT Partial (a reverse read keys start-of-data on the latest sample,
|
||||
// which is past the window — the SDK's rule, SdkAggregateParityTests); stamped with the later edge (#677). The
|
||||
// baseline's reverse answer carries the same +25 ramp divergence.
|
||||
// which is past the window — the SDK's rule, SdkAggregateParityTests); stamped with the later edge (#677). The live
|
||||
// calculator agrees since #821; the untouched baseline's reverse answer carries the same +25 ramp.
|
||||
await B.SeedAsync("QP", QP);
|
||||
var src = B.CreateDataSource();
|
||||
const double total = 73.5 + 88 + 32d / 13;
|
||||
@@ -290,8 +293,10 @@ public abstract class QualityMixConformance : ConformanceBase
|
||||
((double)avg.Value!).ShouldBe(total / 15, 1e-6);
|
||||
avg.StatusCode.ShouldBe(UncertainCalc);
|
||||
|
||||
var sdk = SdkAggregateOracle.Run(HistoryAggregateType.Total, B.Round(At(20)), B.Round(At(0)), Twenty, Rounded(QP)).Single();
|
||||
Convert.ToDouble(sdk.Value).ShouldBe(total + 25, 1e-6, "baseline divergence (reverse) — #821");
|
||||
var live = SdkAggregateOracle.Run(HistoryAggregateType.Total, B.Round(At(20)), B.Round(At(0)), Twenty, Rounded(QP)).Single();
|
||||
Convert.ToDouble(live.Value).ShouldBe(total, 1e-6, "live AggregateFilter equals HistoryRead (reverse) — #821");
|
||||
var before = SdkAggregateOracle.RunSdkAsIs(HistoryAggregateType.Total, B.Round(At(20)), B.Round(At(0)), Twenty, Rounded(QP)).Single();
|
||||
Convert.ToDouble(before.Value).ShouldBe(total + 25, 1e-6, "the untouched baseline (live before #821, reverse)");
|
||||
}
|
||||
|
||||
private async Task AssertMatchesOracle(string tag, (DateTime, HistorianValue, uint)[] rows, double from, double to,
|
||||
|
||||
@@ -28,8 +28,15 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Conformance;
|
||||
/// <c>GoodOnlyStatusAggregateCalculator</c> (the same baseline region machinery — simple bounds, end-of-data 1 ms
|
||||
/// region, status and Partial bits — counting only Good regions), so <see cref="Run"/> / <see cref="RunLive"/> are
|
||||
/// exactly what a live AggregateFilter computes, for every aggregate. PercentBad / DurationBad count Bad regions only
|
||||
/// (unchanged by .182). <see cref="RunSdkAsIs"/> runs the untouched baseline (what live served before #809), for the
|
||||
/// tests that pin that divergence.
|
||||
/// (unchanged by .182). <see cref="RunSdkAsIs"/> runs the untouched baseline (what live served before #809 / #821),
|
||||
/// for the tests that pin those divergences.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>TimeAverage / Total with no start bound (#821).</b> When the data starts inside an interval with an
|
||||
/// <b>Uncertain</b> first value, the baseline integrates the uncovered span before it as a ramp from 0 (its region
|
||||
/// machinery overwrites that region's Bad status with Uncertain). Part 13 ("No Start Bound: Calculate Partial
|
||||
/// Interval") and ours integrate only the covered part, as the baseline itself does after a Good first value. Since
|
||||
/// #821 the live calculators follow Part 13 here too (<c>CoveredTimeAverageAggregateCalculator</c>).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Not <c>100 − PercentBad</c> under TreatUncertainAsBad = true: that flag also changes the region machinery
|
||||
@@ -56,8 +63,8 @@ public static class SdkAggregateOracle
|
||||
|
||||
/// <summary>
|
||||
/// Runs the Part 13 oracle for <paramref name="aggregate"/> over <paramref name="samples"/>: the live calculator
|
||||
/// (<see cref="RunLive"/>) — the baseline, except PercentGood / DurationGood, which count Good regions only (see the
|
||||
/// remarks, #796/#820/#809).
|
||||
/// (<see cref="RunLive"/>) — the baseline, except PercentGood / DurationGood, which count Good regions only, and
|
||||
/// TimeAverage / Total, which leave the span before an Uncertain first value out (see the remarks, #796/#820/#809/#821).
|
||||
/// </summary>
|
||||
/// <param name="aggregate">The aggregate.</param>
|
||||
/// <param name="start">Request start (later than <paramref name="end"/> for a reverse read).</param>
|
||||
@@ -71,7 +78,8 @@ public static class SdkAggregateOracle
|
||||
|
||||
/// <summary>
|
||||
/// Runs exactly the calculator the server registers for live AggregateFilter monitored items
|
||||
/// (<c>CompatibilityAggregateFactory.Create</c>, the #809 Good-only PercentGood / DurationGood included).
|
||||
/// (<c>CompatibilityAggregateFactory.Create</c>, the #809 Good-only PercentGood / DurationGood and the #821
|
||||
/// covered-only TimeAverage / Total included).
|
||||
/// </summary>
|
||||
/// <param name="aggregate">The aggregate.</param>
|
||||
/// <param name="start">Request start (later than <paramref name="end"/> for a reverse read).</param>
|
||||
@@ -89,7 +97,8 @@ public static class SdkAggregateOracle
|
||||
/// <summary>
|
||||
/// Runs the untouched baseline (1.5.378.182) calculator for <paramref name="aggregate"/> exactly as it computes it,
|
||||
/// under <see cref="DefaultConfiguration"/> — including its PercentGood / DurationGood rule (Uncertain counted as
|
||||
/// Good) that neither HistoryRead nor, since #809, the live calculators use.
|
||||
/// Good) and its TimeAverage / Total ramp before an Uncertain first value, which neither HistoryRead nor, since
|
||||
/// #809 / #821, the live calculators use.
|
||||
/// </summary>
|
||||
/// <param name="aggregate">The aggregate.</param>
|
||||
/// <param name="start">Request start (later than <paramref name="end"/> for a reverse read).</param>
|
||||
|
||||
+72
-8
@@ -525,17 +525,19 @@ public sealed class SdkAggregateParityTests
|
||||
public void An_Uncertain_first_sample_in_a_late_start_interval_makes_the_SDK_integrate_the_time_before_the_data(
|
||||
double from, double to)
|
||||
{
|
||||
// #821, pinned divergence (pending the #821 owner decision — this test records, it does not choose).
|
||||
// Part 13 §5.4.3.3/§5.4.3.8: the interpolated-bounding signal is defined from the first sample only — 10 → 11 over
|
||||
// [5,12) (73.5) and 11 → 11 + 8/13 over [12,20) (4·(22 + 8/13)) ⇒ Total 163.96…, TimeAverage = Total ÷ 15 s.
|
||||
// #821, decided 2026-11-11: live follows Part 13. Part 13 §5.4.3.6/§5.4.3.8 ("No Start Bound: Calculate Partial
|
||||
// Interval"): the interpolated-bounding signal is defined from the first sample only — 10 → 11 over [5,12) (73.5)
|
||||
// and 11 → 11 + 8/13 over [12,20) (4·(22 + 8/13)) ⇒ Total 163.96…, TimeAverage = Total ÷ 15 s.
|
||||
const double part13Total = 73.5 + 88 + 32d / 13;
|
||||
Pct(Ours(HistoryAggregateType.Total, from, to, 0, UncertainFirstLateStart)).ShouldBe(part13Total, 1e-9);
|
||||
Pct(Ours(HistoryAggregateType.TimeAverage, from, to, 0, UncertainFirstLateStart)).ShouldBe(part13Total / 15, 1e-9);
|
||||
// The live AggregateFilter calculators (CompatibilityAggregateFactory — CoveredTimeAverageAggregateCalculator since
|
||||
// #821; Sdk2AggregateCompatibilityTests proves the runtime manager serves the same code) answer the same.
|
||||
Pct(Sdk(HistoryAggregateType.Total, from, to, 0, UncertainFirstLateStart)).ShouldBe(part13Total, 1e-9);
|
||||
Pct(Sdk(HistoryAggregateType.TimeAverage, from, to, 0, UncertainFirstLateStart)).ShouldBe(part13Total / 15, 1e-9);
|
||||
|
||||
// The frozen 1.5.378.182 calculators — for Total / TimeAverage exactly what the server registers for live
|
||||
// AggregateFilter items (CompatibilityAggregateFactory; Sdk2AggregateCompatibilityTests proves the runtime manager
|
||||
// serves the same code)
|
||||
// — integrate the uncovered [0,5) as a ramp 0 → 10 (+25) and divide TimeAverage by the WHOLE 20 s interval.
|
||||
// The untouched frozen 1.5.378.182 calculators (live before #821) integrate the uncovered [0,5) as a ramp 0 → 10
|
||||
// (+25) and divide TimeAverage by the WHOLE 20 s interval.
|
||||
Pct(AsIs(HistoryAggregateType.Total, from, to, UncertainFirstLateStart)).ShouldBe(part13Total + 25, 1e-9);
|
||||
Pct(AsIs(HistoryAggregateType.TimeAverage, from, to, UncertainFirstLateStart)).ShouldBe((part13Total + 25) / 20, 1e-9);
|
||||
// SDK 2.0.1's own stock calculators answer the same: the upgrade did not change this.
|
||||
@@ -546,15 +548,18 @@ public sealed class SdkAggregateParityTests
|
||||
// Status agrees everywhere (Uncertain — an Uncertain value is used and the interval is not fully covered).
|
||||
(AsIs(HistoryAggregateType.Total, from, to, UncertainFirstLateStart).Single().StatusCode & StatusBits.CodeBitsMask)
|
||||
.ShouldBe(HistorianStatusCodes.UncertainDataSubNormal);
|
||||
Sdk(HistoryAggregateType.Total, from, to, 0, UncertainFirstLateStart).Single().StatusCode
|
||||
.ShouldBe(AsIs(HistoryAggregateType.Total, from, to, UncertainFirstLateStart).Single().StatusCode);
|
||||
(Ours(HistoryAggregateType.Total, from, to, 0, UncertainFirstLateStart).Single().StatusCode & StatusBits.CodeBitsMask)
|
||||
.ShouldBe(HistorianStatusCodes.UncertainDataSubNormal);
|
||||
|
||||
// Control: the SAME data with a Good first sample — all three agree on the Part 13 answer, so the trigger is the
|
||||
// Control: the SAME data with a Good first sample — all agree on the Part 13 answer, so the trigger is the
|
||||
// Uncertain first sample, not the late start.
|
||||
foreach (var agg in new[] { HistoryAggregateType.Total, HistoryAggregateType.TimeAverage })
|
||||
{
|
||||
var expected = agg == HistoryAggregateType.Total ? part13Total : part13Total / 15;
|
||||
Pct(Ours(agg, from, to, 0, GoodFirstLateStart)).ShouldBe(expected, 1e-9);
|
||||
Pct(Sdk(agg, from, to, 0, GoodFirstLateStart)).ShouldBe(expected, 1e-9);
|
||||
Pct(AsIs(agg, from, to, GoodFirstLateStart)).ShouldBe(expected, 1e-9);
|
||||
Stock201(agg, from, to, GoodFirstLateStart).Single().Value!.Value.ShouldBe(expected, 1e-9);
|
||||
}
|
||||
@@ -661,6 +666,65 @@ public sealed class SdkAggregateParityTests
|
||||
.Select(r => Convert.ToDouble(r.Value)).ShouldBe([7000d, 10000, 10000, 10000, 5000, 10000, 10000, 7000], tolerance: 1e-9);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// #821: late-start data whose first usable sample is Uncertain (+7, after every window's start), with Good / Uncertain / Bad samples after it
|
||||
/// and a Good sample past every window's end (+90), so only the start of the data is in question. No sample sits on
|
||||
/// an interval edge of <see cref="WholeIntervalWindows"/> (the reverse on-edge divergence, class remarks).
|
||||
/// </summary>
|
||||
private static readonly StoredSample[] UncertainFirstLate =
|
||||
[
|
||||
S(7, HistorianValue.FromDouble(10), Uncertain),
|
||||
S(12, HistorianValue.FromDouble(11), Good),
|
||||
S(25, HistorianValue.FromDouble(12), Uncertain),
|
||||
S(33, HistorianValue.FromDouble(14), Good),
|
||||
S(47, HistorianValue.FromDouble(13), Bad),
|
||||
S(52, HistorianValue.FromDouble(15), Good),
|
||||
S(90, HistorianValue.FromDouble(16), Good),
|
||||
];
|
||||
|
||||
/// <summary>#821: the same, preceded by Bad samples only (an outage before the data — still no start bound).</summary>
|
||||
private static readonly StoredSample[] BadThenUncertainFirstLate =
|
||||
[
|
||||
S(-10, HistorianValue.Empty, BadNoCommunication),
|
||||
S(2, HistorianValue.FromDouble(7), Bad),
|
||||
.. UncertainFirstLate,
|
||||
];
|
||||
|
||||
[Theory]
|
||||
[MemberData(nameof(WholeIntervalWindows))]
|
||||
public void Live_AggregateFilter_TimeAverage_and_Total_equal_HistoryRead_over_an_Uncertain_first_late_start(
|
||||
double from, double to, double intervalS)
|
||||
{
|
||||
// #821: a live AggregateFilter monitored item and a HistoryRead-Processed (SqlServer / QuestDB share
|
||||
// TimeWeightedAggregator) answer TimeAverage / Total identically when the data starts inside an interval with an
|
||||
// Uncertain value — same values, timestamps, status and Partial / Calculated bits, NO documented-divergence mask.
|
||||
// RunLive is exactly CompatibilityAggregateFactory.Create, the calculator the server registers.
|
||||
var interval = intervalS > 0 ? TimeSpan.FromSeconds(intervalS) : (At(to) - At(from)).Duration();
|
||||
var failures = new List<string>();
|
||||
var discriminated = 0;
|
||||
foreach (var agg in new[] { HistoryAggregateType.TimeAverage, HistoryAggregateType.Total })
|
||||
foreach (var (name, data) in new[] { ("UncertainFirstLate", UncertainFirstLate), ("BadThenUncertainFirstLate", BadThenUncertainFirstLate) })
|
||||
{
|
||||
var rows = data.Select(x => (x.TimeUtc, x.Value, x.Status)).ToArray();
|
||||
var live = SdkAggregateOracle.RunLive(agg, At(from), At(to), interval, rows);
|
||||
try
|
||||
{
|
||||
SdkAggregateOracle.ShouldMatch(Ours(agg, from, to, intervalS, data), live, $"live {agg} {name} {from}→{to}/{intervalS}s");
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
failures.Add(ex.Message);
|
||||
}
|
||||
// The data must discriminate: the pre-#821 live calculator (the untouched baseline) integrates the span before
|
||||
// the data, so it disagrees on the late-start interval — or this parity would pass against the old behaviour too.
|
||||
var before = SdkAggregateOracle.RunSdkAsIs(agg, At(from), At(to), interval, rows);
|
||||
if (before.Zip(live).Any(p => Math.Abs(Convert.ToDouble(p.First.Value) - Convert.ToDouble(p.Second.Value)) > 1e-9))
|
||||
discriminated++;
|
||||
}
|
||||
failures.ShouldBeEmpty(string.Join("\n", failures));
|
||||
discriminated.ShouldBe(4, "every aggregate × data set must differ from the pre-#821 live answer somewhere");
|
||||
}
|
||||
|
||||
/// <summary>#684: Good 10 / 30 with an Uncertain value BELOW the Good minimum (+4) and one ABOVE the maximum (+6).</summary>
|
||||
private static readonly StoredSample[] UncertainBeyond =
|
||||
[
|
||||
|
||||
+10
-8
@@ -6,10 +6,11 @@ namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests;
|
||||
|
||||
/// <summary>
|
||||
/// #821 / #809 facts, through the BOOTED server's runtime <c>AggregateManager</c> — the calculators a live
|
||||
/// AggregateFilter monitored item gets, under the server's own default configuration. The #821 Total / TimeAverage
|
||||
/// facts pin current behaviour for the owner decision; they do not choose. #809 is decided (2026-11-11): live
|
||||
/// PercentGood / DurationGood count Good time only, equal to HistoryRead. The HistoryRead (Part 13) answers and SDK
|
||||
/// 2.0.1's stock answers for the same input are pinned beside it in <c>SdkAggregateParityTests</c>.
|
||||
/// AggregateFilter monitored item gets, under the server's own default configuration. Both are decided (owner,
|
||||
/// 2026-11-11): live PercentGood / DurationGood count Good time only (#809), and live Total / TimeAverage integrate only
|
||||
/// the covered part of an interval the data starts inside, even with an Uncertain first sample (#821) — equal to
|
||||
/// HistoryRead. The HistoryRead (Part 13) answers, the untouched baseline's and SDK 2.0.1's stock answers for the same
|
||||
/// input are pinned beside it in <c>SdkAggregateParityTests</c>.
|
||||
/// </summary>
|
||||
public sealed class LiveAggregateLateStartFactsTests
|
||||
{
|
||||
@@ -41,11 +42,12 @@ public sealed class LiveAggregateLateStartFactsTests
|
||||
return d;
|
||||
}
|
||||
|
||||
// Part 13 (HistoryRead on SqlServer/QuestDB): Total = 73.5 + 88 + 32/13 = 163.96…, TimeAverage = Total ÷ 15 s.
|
||||
// The live calculators integrate the uncovered [0,5) as a ramp from 0 (+25) and divide by the whole 20 s.
|
||||
// #821: Part 13 (HistoryRead on SqlServer/QuestDB): Total = 73.5 + 88 + 32/13 = 163.96…, TimeAverage = Total ÷ 15 s
|
||||
// — live as in HistoryRead (before #821 the live calculator integrated the uncovered [0,5) as a ramp from 0 (+25)
|
||||
// and divided by the whole 20 s: 188.96… / 9.448…).
|
||||
const double part13Total = 73.5 + 88 + 32d / 13;
|
||||
One(ObjectIds.AggregateFunction_Total).ShouldBe(part13Total + 25, 1e-9); // 188.96…
|
||||
One(ObjectIds.AggregateFunction_TimeAverage).ShouldBe((part13Total + 25) / 20, 1e-9); // 9.448… (Part 13: 10.93…)
|
||||
One(ObjectIds.AggregateFunction_Total).ShouldBe(part13Total, 1e-9); // 163.96…
|
||||
One(ObjectIds.AggregateFunction_TimeAverage).ShouldBe(part13Total / 15, 1e-9); // 10.93…
|
||||
// #809: the Uncertain [5,12) is not Good — only [12,20) is, live as in HistoryRead (8 000 ms / 40 %; before #809
|
||||
// the live calculator counted the Uncertain time too: 15 000 ms / 75 %).
|
||||
One(ObjectIds.AggregateFunction_DurationGood).ShouldBe(8_000d, 1e-9);
|
||||
|
||||
+133
-3
@@ -15,6 +15,13 @@ namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests;
|
||||
/// case computed by the baseline SDK with <c>TreatUncertainAsBad = true</c> (its "twin": there 1.5.378.182 already
|
||||
/// counted Good regions only) — and each is pinned to that twin instead. The twin is baseline evidence, not output of
|
||||
/// the application's calculator, so the pin never regenerates expectations from the implementation.
|
||||
/// <para>
|
||||
/// #821 (owner-approved 2026-11-11): TimeAverage / Total no longer integrate the span before an Uncertain first value
|
||||
/// when there is no start bound. No fixture vector changes (every data set has a usable value at −10 s, before every
|
||||
/// window, so no fixture interval lacks a start bound) — all 384 TimeAverage / Total vectors stay held to their own
|
||||
/// output. The change is pinned instead over late-start variants of every data set against a baseline-derived oracle
|
||||
/// (<see cref="Issue821_late_start_with_an_uncertain_first_sample_integrates_only_the_covered_part"/>) and by hand.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public sealed class Sdk2AggregateCompatibilityTests
|
||||
{
|
||||
@@ -22,6 +29,12 @@ public sealed class Sdk2AggregateCompatibilityTests
|
||||
private const string DurationGood = "AggregateFunction_DurationGood";
|
||||
private const int Issue809PercentGoodExemptions = 38;
|
||||
private const int Issue809DurationGoodExemptions = 26;
|
||||
private const string TimeAverage = "AggregateFunction_TimeAverage";
|
||||
private const string Total = "AggregateFunction_Total";
|
||||
/// <summary>#821: fixture vectors changed — none (every data set has a usable value before the window; class remarks).</summary>
|
||||
private const int Issue821FixtureExemptions = 0;
|
||||
private const int Issue821Cut5Changed = 150;
|
||||
private const int Issue821Cut25Changed = 142;
|
||||
|
||||
[Theory]
|
||||
[InlineData(false)]
|
||||
@@ -44,6 +57,8 @@ public sealed class Sdk2AggregateCompatibilityTests
|
||||
baseline.Vectors.Select(x => x.Aggregate).Distinct().Count().ShouldBe(37);
|
||||
exemptions.Keys.Count(x => x.Aggregate == PercentGood).ShouldBe(Issue809PercentGoodExemptions);
|
||||
exemptions.Keys.Count(x => x.Aggregate == DurationGood).ShouldBe(Issue809DurationGoodExemptions);
|
||||
// #821: no TimeAverage / Total fixture vector is exempt — all 384 are held to their own baseline output above.
|
||||
exemptions.Keys.Count(x => x.Aggregate is TimeAverage or Total).ShouldBe(Issue821FixtureExemptions);
|
||||
failures.ShouldBeEmpty("The application must retain baseline values, CLR types, complete status bits, timestamps and incremental processing (#809 exemptions: the uncertain-as-bad twin).");
|
||||
}
|
||||
|
||||
@@ -97,6 +112,104 @@ public sealed class Sdk2AggregateCompatibilityTests
|
||||
output.Select(x => double.Parse(x.Payload!, CultureInfo.InvariantCulture)).ShouldBe(expected);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// #821 pin over late-start variants of every fixture data set (the fixture itself has none — see the class remarks):
|
||||
/// each TimeAverage / Total vector's data set cut at <paramref name="cutSeconds"/> (no usable value before the data
|
||||
/// starts, so no start bound) with its first remaining non-Bad sample made Uncertain, run through the application's
|
||||
/// calculator under the vector's own direction, stepping, configuration and incremental draining.
|
||||
/// <para>
|
||||
/// The oracle is the untouched BASELINE, never the application's calculator: under TreatUncertainAsBad = false
|
||||
/// the baseline treats an Uncertain value exactly like a Good one except for statuses, and with a Good first
|
||||
/// value it already leaves the uncovered span out (the #821 control). So the expected value of each interval is
|
||||
/// the baseline's on the same variant with that first sample Good, and the expected status, timestamps and bits
|
||||
/// the baseline's on the variant itself (the region before the data is non-Good either way) — except an
|
||||
/// interval the Good twin answers Bad (nothing covered: only the uncovered span ever fed it), which is the twin
|
||||
/// whole. Stepped variables and TreatUncertainAsBad = true never reached the baseline's defect and must be the
|
||||
/// baseline byte for byte.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
[Theory]
|
||||
[InlineData(5, Issue821Cut5Changed)]
|
||||
[InlineData(25, Issue821Cut25Changed)]
|
||||
public void Issue821_late_start_with_an_uncertain_first_sample_integrates_only_the_covered_part(double cutSeconds, int expectedChanged)
|
||||
{
|
||||
var baseline = LoadBaseline();
|
||||
var failures = new List<string>();
|
||||
var changed = 0;
|
||||
// Why the fixture needs no #821 exemption: every data set with a usable value has one before the window (−10 s), so
|
||||
// no fixture interval lacks a start bound; the variants below remove it.
|
||||
baseline.Data.Values.Where(d => d.Any(s => !StatusCode.IsBad(s.Status)))
|
||||
.ShouldAllBe(d => d.Any(s => s.Seconds < 0 && !StatusCode.IsBad(s.Status)));
|
||||
var vectors = baseline.Vectors.Where(v => v.Aggregate is TimeAverage or Total).ToList();
|
||||
foreach (var vector in vectors)
|
||||
{
|
||||
var (variant, goodTwin) = LateStartVariant(baseline.Data[vector.DataSet], cutSeconds);
|
||||
var asIs = Run(baseline, vector, rig: null, out var asIsError, variant, baselineAsIs: true);
|
||||
var twin = Run(baseline, vector, rig: null, out var twinError, goodTwin, baselineAsIs: true);
|
||||
var actual = Run(baseline, vector, rig: null, out var error, variant);
|
||||
asIsError.ShouldBeNull(Describe(vector));
|
||||
twinError.ShouldBeNull(Describe(vector));
|
||||
twin.Count.ShouldBe(asIs.Count, Describe(vector));
|
||||
|
||||
Value[] expected = EffectiveTreatUncertainAsBad(vector) || vector.Stepped
|
||||
? [.. asIs]
|
||||
: [.. asIs.Zip(twin, (a, g) =>
|
||||
{
|
||||
g.SourceTicks.ShouldBe(a.SourceTicks, Describe(vector));
|
||||
return StatusCode.IsBad(g.Status) ? g : a with { Type = g.Type, Payload = g.Payload };
|
||||
})];
|
||||
if (!expected.SequenceEqual(asIs)) changed++;
|
||||
if (error is not null || !actual.SequenceEqual(expected))
|
||||
failures.Add($"{Describe(vector)} cut {cutSeconds}: error {error}; expected {JsonSerializer.Serialize(expected)} actual {JsonSerializer.Serialize(actual.ToArray())}");
|
||||
}
|
||||
vectors.Count.ShouldBe(384);
|
||||
failures.ShouldBeEmpty("Live TimeAverage / Total must leave the span before an Uncertain first value out (Part 13 'No Start Bound: Calculate Partial Interval', #821).");
|
||||
changed.ShouldBe(expectedChanged, "the late-start variants the #821 override changes from the untouched baseline");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// #821 hand-computed pin, independent of every calculator: the "uncertain" data set from +5 (Uncertain 100 at 5,
|
||||
/// Good 18 at 15, Bad at 25, Good 19 at 35, …), forward, sloped, [0,20). Covered [5,20): 100 → 18 over 10 s = 590,
|
||||
/// then 18 → the bound at 20 (sloped 18 → 19 across the Bad sample: 18.25) over 5 s = 90.625 ⇒ Total 680.625,
|
||||
/// TimeAverage 680.625 ÷ 15 = 45.375. The baseline added the ramp 0 → 100 over [0,5) (+250): 930.625 and 46.53125.
|
||||
/// Uncertain either way (an Uncertain value is used, a Bad one skipped, the interval not covered).
|
||||
/// </summary>
|
||||
[Theory]
|
||||
[InlineData(Total, 680.625, 930.625)]
|
||||
[InlineData(TimeAverage, 45.375, 46.53125)]
|
||||
public void Issue821_late_start_hand_computed(string aggregate, double part13, double baselineValue)
|
||||
{
|
||||
var baseline = LoadBaseline();
|
||||
var vector = baseline.Vectors.Single(v => v.Aggregate == aggregate && v.DataSet == "uncertain" && !v.Reverse && !v.Stepped
|
||||
&& !v.UncertainAsBad && !v.Incremental && v.Configuration is null);
|
||||
var (variant, _) = LateStartVariant(baseline.Data["uncertain"], 5);
|
||||
variant[0].Status.ShouldBe(0x40A40000u, "the data set's own Uncertain 100 at +5");
|
||||
var output = Run(baseline, vector, rig: null, out var error, variant);
|
||||
var asIs = Run(baseline, vector, rig: null, out _, variant, baselineAsIs: true);
|
||||
error.ShouldBeNull();
|
||||
double.Parse(output[0].Payload!, CultureInfo.InvariantCulture).ShouldBe(part13, 1e-9);
|
||||
double.Parse(asIs[0].Payload!, CultureInfo.InvariantCulture).ShouldBe(baselineValue, 1e-9);
|
||||
(output[0].Status & 0xFFFF0000u).ShouldBe(StatusCodes.UncertainDataSubNormal.Code);
|
||||
output[0].Status.ShouldBe(asIs[0].Status);
|
||||
output.Skip(1).ShouldBe(asIs.Skip(1), "only the late-start interval changes");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A data set cut at <paramref name="cutSeconds"/> with its first non-Bad sample Uncertain (kept Uncertain if it already
|
||||
/// is), and the same with that sample Good.
|
||||
/// </summary>
|
||||
private static (Sample[] Variant, Sample[] GoodTwin) LateStartVariant(Sample[] data, double cutSeconds)
|
||||
{
|
||||
var cut = data.Where(s => s.Seconds >= cutSeconds).OrderBy(s => s.Seconds).ToArray();
|
||||
var first = Array.FindIndex(cut, s => !StatusCode.IsBad(s.Status));
|
||||
if (first < 0) return (cut, cut);
|
||||
var variant = cut.ToArray();
|
||||
var good = cut.ToArray();
|
||||
variant[first] = cut[first] with { Status = StatusCode.IsUncertain(cut[first].Status) ? cut[first].Status : 0x40A40000u };
|
||||
good[first] = cut[first] with { Status = 0u };
|
||||
return (variant, good);
|
||||
}
|
||||
|
||||
private static Baseline LoadBaseline() =>
|
||||
JsonSerializer.Deserialize<Baseline>(File.ReadAllText(Path.Combine(AppContext.BaseDirectory, "Fixtures", "opcua-1.5.378.182-aggregates.json")), new JsonSerializerOptions { PropertyNameCaseInsensitive = true })!;
|
||||
|
||||
@@ -126,7 +239,8 @@ public sealed class Sdk2AggregateCompatibilityTests
|
||||
: null;
|
||||
}
|
||||
|
||||
private static RunOutput Run(Baseline baseline, Vector vector, ServerEventRig? rig, out string? error)
|
||||
private static RunOutput Run(Baseline baseline, Vector vector, ServerEventRig? rig, out string? error,
|
||||
Sample[]? data = null, bool baselineAsIs = false)
|
||||
{
|
||||
var telemetry = DefaultTelemetry.Create(_ => { });
|
||||
var epoch = new DateTime(baseline.EpochTicks, DateTimeKind.Utc);
|
||||
@@ -142,13 +256,15 @@ public sealed class Sdk2AggregateCompatibilityTests
|
||||
};
|
||||
var calculator = rig is not null
|
||||
? rig.Server.CurrentInstance.AggregateManager.CreateCalculator(id, start, end, baseline.IntervalMilliseconds, vector.Stepped, config)
|
||||
: CompatibilityAggregateFactory.Create(id, start, end, baseline.IntervalMilliseconds, vector.Stepped, config, telemetry);
|
||||
: baselineAsIs
|
||||
? new BaselineAdapter(Aggregators.CreateStandardCalculator(id, start, end, baseline.IntervalMilliseconds, vector.Stepped, config, telemetry)!)
|
||||
: CompatibilityAggregateFactory.Create(id, start, end, baseline.IntervalMilliseconds, vector.Stepped, config, telemetry);
|
||||
calculator.ShouldNotBeNull($"Runtime registration must serve {vector.Aggregate}.");
|
||||
var output = new RunOutput();
|
||||
error = null;
|
||||
try
|
||||
{
|
||||
var samples = baseline.Data[vector.DataSet].OrderBy(x => x.Seconds);
|
||||
var samples = (data ?? baseline.Data[vector.DataSet]).OrderBy(x => x.Seconds);
|
||||
foreach (var sample in vector.Reverse ? samples.Reverse() : samples)
|
||||
{
|
||||
var time = epoch.AddSeconds(sample.Seconds);
|
||||
@@ -185,6 +301,20 @@ public sealed class Sdk2AggregateCompatibilityTests
|
||||
"Double" => double.Parse(sample.Payload!, CultureInfo.InvariantCulture), "Int64" => long.Parse(sample.Payload!, CultureInfo.InvariantCulture), "Boolean" => bool.Parse(sample.Payload!), _ => null };
|
||||
static string? Format(object? value) => value switch { null => null, double number => number.ToString("R", CultureInfo.InvariantCulture), float number => number.ToString("R", CultureInfo.InvariantCulture), IFormattable f => f.ToString(null, CultureInfo.InvariantCulture), _ => value.ToString() };
|
||||
private sealed class RunOutput : List<Value> { public List<bool> Accepted { get; } = []; }
|
||||
|
||||
/// <summary>The untouched baseline calculator behind the SDK 2 protocol (what live served before #809 / #821).</summary>
|
||||
private sealed class BaselineAdapter(AggregateCalculator calculator) : Opc.Ua.Server.IAggregateCalculator
|
||||
{
|
||||
public NodeId AggregateId => calculator.AggregateId;
|
||||
public bool QueueRawValue(DataValue value) => calculator.QueueRawValue(LegacyDataValue.FromSdk(value));
|
||||
public bool HasEndTimePassed(DateTimeUtc currentTime) => calculator.HasEndTimePassed((DateTime)currentTime);
|
||||
public bool TryGetProcessedValue(bool returnPartial, out DataValue value)
|
||||
{
|
||||
var processed = calculator.GetProcessedValue(returnPartial);
|
||||
value = processed is null ? default : processed.ToSdk();
|
||||
return processed is not null;
|
||||
}
|
||||
}
|
||||
record Baseline(long EpochTicks, int IntervalMilliseconds, Dictionary<string,Sample[]> Data, Vector[] Vectors);
|
||||
record Sample(double Seconds,string Type,string? Payload,uint Status,ushort SourcePicoseconds = 0,ushort ServerPicoseconds = 0,long ServerOffsetTicks = 0);
|
||||
record Value(bool PartialRequested,string? Type,string? Payload,uint Status,long SourceTicks,long ServerTicks,ushort SourcePicoseconds,ushort ServerPicoseconds);
|
||||
|
||||
Reference in New Issue
Block a user