fix(historian): #671 review — raw-row cap, reverse Interpolative, SDK status rule, Partial bit

M1: the time-weighted family (incl. Total, formerly a pushed-down SUM) streams raw window rows into the C# evaluator. New shared knob ServerHistorian:MaxTimeWeightedRawRows (default 1,000,000; <=0 unbounded with a Validate warning) flows through HistorianBackendRegistry into both providers. The window query is capped at cap+1 (SQL Server TOP (@maxRows), QuestDB LIMIT $4); one row past the cap throws the new Core.Abstractions HistoryReadLimitExceededException, which the node manager maps to BadTooManyOperations for that node. Not counted as a backend health failure; bound rows are not counted; per-sample aggregates are not capped. M2: reverse Interpolative is anchored at the later time (Part 13): points at End - i*interval, emitted latest first. 10:00->09:00 @1h yields 10:00. L1: Partial bit (0x4, = Opc.Ua.AggregateBits.Partial) on non-Bad results of a truncated last interval (time-weighted family; per-sample family is a follow-up). L2/L3: aligned with the SDK AggregateCalculator.GetTimeBasedStatusCode (decompiled from Opc.Ua.Server 1.5.378.106): TreatUncertainAsBad=false counts Uncertain time (Uncertain samples, held-into-outage, extrapolation) as Good time; data but no non-Bad time => generic Bad (0x80000000); BadNoData only when there is no data at all. Consequence: a live tag's latest interval reads Good (documented). L5: release note no longer claims Total matches the Gateway Integral exactly (AVEVA's divisor is unverified); it notes Total is now evaluated in C# and subject to the cap. Tests: unit (reverse Interpolative x3, Partial, Bad vs BadNoData incl. an open-ended outage, cap helpers, SDK bit oracles); conformance (reverse Interpolative, raw-row cap at/over/0, bounds not counted, health unaffected); node manager BadTooManyOperations; options default + warning. Each new behaviour falsified by mutation (aggregator x9, data-source cap checks on both backends against the real DBs, node-manager catch).
2026-10-11 09:10:26 -04:00
parent 1f88669304
commit 56bb4855bb
24 changed files with 460 additions and 86 deletions
@@ -22,6 +22,10 @@ public interface IConformanceBackend : IAsyncDisposable
    /// <param name="clock">Its "now"; null means the system clock.</param>
    IHistorianDataSource CreateDataSource(TimeProvider? clock = null);

+    /// <summary>A fresh data source whose time-weighted raw-row cap (<c>ServerHistorian:MaxTimeWeightedRawRows</c>) is <paramref name="maxTimeWeightedRawRows"/>.</summary>
+    /// <param name="maxTimeWeightedRawRows">The cap (&lt;= 0 = unbounded).</param>
+    IHistorianDataSource CreateDataSource(int maxTimeWeightedRawRows);
+
    /// <summary>A fresh live-value writer over this backend's store.</summary>
    /// <param name="utcNow">The clock used to stamp values that carry no timestamp; null means the system clock.</param>
    IHistorianValueWriter CreateValueWriter(Func<DateTime>? utcNow = null);
@@ -346,6 +346,7 @@ public abstract class ProcessedConformance : ConformanceBase
    private const uint GoodInterp = 0x00000402u;        // Good | InfoType 0x400 | Interpolated 0x2
    private const uint UncertainInterp = 0x40A40402u;   // UncertainDataSubNormal | 0x402
    private const uint NoData = 0x809B0000u;
+    private const uint BadCalc = 0x80000401u;           // generic Bad | Calculated (an all-outage interval)

    /// <summary>
    ///     Sparse tag W: −30 = 0 G (bound before the window), +20 = 40 G, +30 = outage marker, +50 = 20 G, +100 = 20 G
@@ -405,12 +406,13 @@ public abstract class ProcessedConformance : ConformanceBase
    public async Task TimeAverage2_over_a_sparse_tag_with_narrow_intervals()
    {
        await SeedWAsync();
-        // 10 s intervals (narrower than the gaps): [0,10) 24→32 = 28 G; [10,20) 32→40 = 36 G; [20,30) 40 held, Uncertain;
-        // [30,40) and [40,50) inside the outage ⇒ BadNoData; [50,60) … [70,80) the flat 20 up to the bound after, Good.
+        // 10 s intervals (narrower than the gaps): [0,10) 24→32 = 28; [10,20) 32→40 = 36; [20,30) 40 held up to the
+        // outage (Uncertain time, which counts as Good: TreatUncertainAsBad = false); [30,40) and [40,50) inside the outage
+        // ⇒ generic Bad (data, but none non-Bad); [50,60) … [70,80) the flat 20 up to the bound after, Good.
        var s = await ReadW(HistoryAggregateType.TimeAverage2, 10);
        Values(s, 28, 36, 40, null, null, 20, 20, 20);
        s.Select(x => x.StatusCode).ShouldBe(
-            [GoodCalc, GoodCalc, UncertainCalc, NoDataCalc, NoDataCalc, GoodCalc, GoodCalc, GoodCalc]);
+            [GoodCalc, GoodCalc, GoodCalc, BadCalc, BadCalc, GoodCalc, GoodCalc, GoodCalc]);
        // Total2 of the first interval: 28 × 10 s.
        ((double)(await ReadW(HistoryAggregateType.Total2, 10))[0].Value!).ShouldBe(280d, 1e-9);
    }
@@ -427,6 +429,43 @@ public abstract class ProcessedConformance : ConformanceBase
            [GoodInterp, GoodInterp, GoodRaw, UncertainInterp, UncertainInterp, GoodRaw, GoodInterp, GoodInterp]);
    }

+    [Fact]
+    public async Task A_reverse_Interpolative_read_is_anchored_at_its_later_time()
+    {
+        await SeedWAsync();
+        // At(80) → At(0), 40 s: points At(80) (on the Good 20→20 segment to the bound at +100 ⇒ 20) and At(40) (inside
+        // the outage ⇒ bridged 80/3, Uncertain), latest first. A forward read would answer At(0) and At(40) instead.
+        var s = (await B.CreateDataSource().ReadProcessedAsync("W", At(80), At(0), TimeSpan.FromSeconds(40),
+            HistoryAggregateType.Interpolative, Ct)).Samples;
+        s.Select(x => x.SourceTimestampUtc).ShouldBe([B.Round(At(80)), B.Round(At(40))]);
+        Values(s, 20, 80d / 3d);
+        s.Select(x => x.StatusCode).ShouldBe([GoodInterp, UncertainInterp]);
+    }
+
+    [Fact]
+    public async Task The_time_weighted_raw_row_cap_fails_the_read_past_the_cap_only()
+    {
+        // Tag A holds five samples in [0,80); W holds three in [0,80) plus two bound rows outside it.
+        await ConformanceData.SeedAAsync(B);
+        await SeedWAsync();
+        var atCap = B.CreateDataSource(5);
+        (await atCap.ReadProcessedAsync("A", At(0), At(80), Twenty, HistoryAggregateType.Total, Ct)).Samples.Count.ShouldBe(4);
+
+        var under = B.CreateDataSource(4);
+        var ex = await Should.ThrowAsync<HistoryReadLimitExceededException>(
+            () => under.ReadProcessedAsync("A", At(0), At(80), Twenty, HistoryAggregateType.Total, Ct));
+        ex.Limit.ShouldBe(4);
+        // Pushed-down (per-sample) aggregates are not capped; the bound rows outside the window do not count.
+        (await under.ReadProcessedAsync("A", At(0), At(80), Twenty, HistoryAggregateType.Average, Ct)).Samples.Count.ShouldBe(4);
+        (await B.CreateDataSource(3).ReadProcessedAsync("W", At(0), At(80), Twenty, HistoryAggregateType.TimeAverage, Ct))
+            .Samples.Count.ShouldBe(4);
+        // A client-request limit, not a backend fault.
+        under.GetHealthSnapshot().TotalFailures.ShouldBe(0);
+        // Non-positive = unbounded.
+        (await B.CreateDataSource(0).ReadProcessedAsync("A", At(0), At(80), Twenty, HistoryAggregateType.Total, Ct))
+            .Samples.Count.ShouldBe(4);
+    }
+
    [Fact]
    public async Task Outages_straddling_both_window_edges_use_the_four_bound_seeks()
    {