Joseph Doherty
a3ee917fb4
With StartTime at or before the epoch, RequestStartFor has no margin left, so AVEVA's synthetic no-data/boundary row is stamped exactly at StartTime and was kept: raw reads returned a fabricated Bad sample at 1970-01-01 and the per-sample aggregates put it in bucket 0 (Start/End fabricated, Min/Max status Uncertain). IsWindowSample and the aggregate loop now drop any row with t <= ProbeFloor (RawAggregateReader.IsAfterFloor), matching the #686 walk-back's assumption.
616 lines
34 KiB
C#
616 lines
34 KiB
C#
using Google.Protobuf.WellKnownTypes;
|
||
using Grpc.Core;
|
||
using ZB.MOM.WW.HistorianGateway.Client;
|
||
using Microsoft.Extensions.Logging.Abstractions;
|
||
using Shouldly;
|
||
using Xunit;
|
||
using ZB.MOM.WW.HistorianGateway.Contracts.Grpc;
|
||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||
using ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common;
|
||
|
||
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests;
|
||
|
||
/// <summary>
|
||
/// #681: the Gateway backend serves the per-sample aggregates from raw reads with the SQL backends' shared evaluator.
|
||
/// The parity oracle is what SqlServer/QuestDB answer for the same stored samples: per-bucket statistics built
|
||
/// independently in memory (LINQ, <see cref="BucketPlan.IndexOf"/>) — the C# statement of the backends' SQL that
|
||
/// <c>Sql.Common.Tests.SdkAggregateParityTests</c> pins against the OPC UA SDK's own Part 13 calculators — evaluated by
|
||
/// <see cref="AggregateEvaluator.EvaluateAll"/> with the data range the SQL backends query (earliest sample ≤ the
|
||
/// window's later edge, latest ≥ its earlier edge, over EVERY stored sample). The fake gateway serves raw reads from
|
||
/// the same stored samples, optionally emitting AVEVA's initial-value row stamped at the query start.
|
||
/// </summary>
|
||
public sealed class GatewayRawAggregateTests
|
||
{
|
||
private static readonly CancellationToken Ct = CancellationToken.None;
|
||
|
||
/// <summary>2026-10-09 08:00:00.1234567 UTC — not millisecond-aligned (the SQL fixtures' T0).</summary>
|
||
private static readonly DateTime T0 = new DateTime(2026, 10, 9, 8, 0, 0, DateTimeKind.Utc).AddTicks(1_234_567);
|
||
|
||
private static readonly DateTime Now = At(3600);
|
||
|
||
private const uint Good = 0x00000000u;
|
||
private const uint Bad = 0x80000000u;
|
||
private const uint Uncertain = 0x40000000u;
|
||
private const uint GoodCalc = 0x00000401u;
|
||
private const uint UncertainCalc = 0x40A40401u;
|
||
private const uint NoDataCalc = 0x809B0401u;
|
||
private const uint InvalidInputsCalc = 0x80D60401u;
|
||
private const uint Partial = 0x4u;
|
||
|
||
private static DateTime At(double seconds) => T0.AddTicks((long)(seconds * TimeSpan.TicksPerSecond));
|
||
|
||
private static StoredSample S(double seconds, HistorianValue v, uint status) => new(At(seconds), v, status);
|
||
|
||
private static StoredSample D(double seconds, double v, uint status = Good) => S(seconds, HistorianValue.FromDouble(v), status);
|
||
|
||
// ---- datasets (statuses restricted to codes that round-trip the OPC DA quality byte: Good, Uncertain, Bad) ----------
|
||
|
||
/// <summary>Tag A of the conformance suite: +0=10, +10=20, +20=Bad(30), +30=40, +40=50.</summary>
|
||
private static readonly StoredSample[] A = [D(0, 10), D(10, 20), D(20, 30, Bad), D(30, 40), D(40, 50)];
|
||
|
||
/// <summary>Every 2.5 s from −10 to +100, values cycling, one Bad sample at +22.5.</summary>
|
||
private static readonly StoredSample[] Dense = Enumerable.Range(0, 45)
|
||
.Select(i => (t: -10 + i * 2.5, i))
|
||
.Select(x => D(x.t, 10 + (x.i * 7 % 13), x.t == 22.5 ? Bad : Good))
|
||
.ToArray();
|
||
|
||
private static readonly StoredSample[] Gappy =
|
||
[D(1, 10), D(2, 11, Uncertain), D(25, 12, Bad), D(45, 13), D(70, 1)];
|
||
|
||
private static readonly StoredSample[] Bracketed =
|
||
[
|
||
D(-5, 1), D(1, 10, Bad), D(3, 11), D(5, 12, Bad), D(7, 13, Uncertain), D(9, 15), D(12, 20),
|
||
S(14, HistorianValue.Empty, Bad), D(16, 21), S(19, HistorianValue.Empty, Bad), D(40, 1),
|
||
];
|
||
|
||
private static readonly StoredSample[] EarlyEnd = [D(-5, 1), D(3, 2), D(25, 3)];
|
||
private static readonly StoredSample[] EndOnEdge = [D(-5, 1), D(3, 2), D(20, 3)];
|
||
|
||
private static readonly StoredSample[] LateStart = Enumerable.Range(0, 20)
|
||
.Select(i => D(7 + i * 5, i % 2 == 0 ? 20 : 40)).ToArray();
|
||
|
||
private static readonly StoredSample[] OutageEnds = [D(-5, 1), D(3, 2), S(25, HistorianValue.Empty, Bad)];
|
||
|
||
/// <summary>A usable String sample among numbers (numeric aggregates ⇒ BadAggregateInvalidInputs in its bucket).</summary>
|
||
private static readonly StoredSample[] Mixed =
|
||
[D(-5, 1), D(2, 4), S(12, HistorianValue.FromString("x"), Good), D(15, 6), D(33, 9, Uncertain)];
|
||
|
||
/// <summary>Equal extremes in one bucket (ActualTime ties ⇒ earliest).</summary>
|
||
private static readonly StoredSample[] Ties = [D(1, 5), D(3, 9), D(4, 1), D(6, 9), D(8, 1), D(25, 2)];
|
||
|
||
/// <summary>#684 (Sql.Common's DenseUncertain): Dense with every 5th sample an in-range Uncertain 16.</summary>
|
||
private static readonly StoredSample[] DenseUncertain = Enumerable.Range(0, 45)
|
||
.Select(i => (t: -10 + i * 2.5, i))
|
||
.Select(x => x.i % 5 == 0 && x.t != 22.5
|
||
? D(x.t, 16, Uncertain)
|
||
: D(x.t, 10 + (x.i * 7 % 13), x.t == 22.5 ? Bad : Good))
|
||
.ToArray();
|
||
|
||
/// <summary>#684 (Sql.Common's UncertainBeyond): Uncertain values below and above the Good extremes in one interval.</summary>
|
||
private static readonly StoredSample[] UncertainBeyond = [D(-5, 20), D(2, 10), D(4, 5, Uncertain), D(6, 99, Uncertain), D(8, 30), D(25, 20)];
|
||
|
||
/// <summary>#684: an Uncertain string among Good numbers (does not refuse the Good-value aggregates) and a Good one.</summary>
|
||
private static readonly StoredSample[] UncertainText =
|
||
[
|
||
D(-5, 1), D(2, 4), S(4, HistorianValue.FromString("u"), Uncertain), D(6, 8), S(14, HistorianValue.FromString("g"), Good),
|
||
D(16, 3),
|
||
];
|
||
|
||
private static readonly (string Name, StoredSample[] Data)[] Datasets =
|
||
[
|
||
("A", A), ("Dense", Dense), ("Gappy", Gappy), ("Bracketed", Bracketed), ("EarlyEnd", EarlyEnd),
|
||
("EndOnEdge", EndOnEdge), ("LateStart", LateStart), ("OutageEnds", OutageEnds), ("Mixed", Mixed), ("Ties", Ties),
|
||
("DenseUncertain", DenseUncertain), ("UncertainBeyond", UncertainBeyond), ("UncertainText", UncertainText),
|
||
];
|
||
|
||
private static readonly (double From, double To, double Interval)[] Windows =
|
||
[
|
||
(0, 80, 20), (80, 0, 20), (0, 50, 20), (50, 0, 20), (5, 80, 15), (80, 5, 15), (3, 77, 20), (77, 3, 20),
|
||
(0, 60, 0), (60, 0, 0), (0, 60, 10), (60, 0, 10), (-10, 1, 5), (1, -10, 5), (70, 85, 5), (85, 70, 5), (0, 1, 1),
|
||
(0, 40, 10), (40, 0, 10), (0, 40, 25), (40, 0, 25),
|
||
];
|
||
|
||
private static readonly HistoryAggregateType[] PerSample =
|
||
[
|
||
HistoryAggregateType.Average, HistoryAggregateType.Minimum, HistoryAggregateType.Maximum,
|
||
HistoryAggregateType.Count, HistoryAggregateType.Range, HistoryAggregateType.Start, HistoryAggregateType.End,
|
||
HistoryAggregateType.Delta, HistoryAggregateType.MinimumActualTime, HistoryAggregateType.MaximumActualTime,
|
||
];
|
||
|
||
// ---- the fake store ----------------------------------------------------------------------------------------------
|
||
|
||
private static HistorianSample ToWire(StoredSample s)
|
||
{
|
||
var w = new HistorianSample
|
||
{
|
||
Tag = "T",
|
||
Timestamp = Timestamp.FromDateTime(s.TimeUtc),
|
||
OpcQuality = s.Status switch { Good => 192u, Uncertain => 64u, _ => 0u },
|
||
};
|
||
// The REAL wire shape (histsdk): every numeric row carries NumericValue — a NULL / no-data row arrives as 0 with
|
||
// OPC quality 0 — and a string row always carries StringValue.
|
||
if (s.Value.Kind == HistorianValueKind.String) w.StringValue = s.Value.Text;
|
||
else w.NumericValue = s.Value.TryGetDouble(out var d) ? d : 0d;
|
||
return w;
|
||
}
|
||
|
||
/// <summary>
|
||
/// A gateway over a stored sample set: ReadRaw returns the samples in <c>[start, end]</c> ascending, capped at
|
||
/// <c>maxValues</c>. With <paramref name="avevaBoundary"/> it first emits what an AVEVA <c>Full</c> read does at
|
||
/// the query start when nothing is stored exactly there: a row stamped AT the query start carrying the value in
|
||
/// effect (the last stored sample before it, with its quality; <c>Quality = 133</c>) — or, with no earlier data, a
|
||
/// NULL row in the real wire shape: <c>NumericValue = 0</c> present, <c>OpcQuality = 0</c>. With
|
||
/// <paramref name="terminalRow"/> the small (2-row) reads also end with a NULL row stamped at the query END (a
|
||
/// hypothetical shape the probe / look-ahead must ignore; the window read relies on there being none — live leg).
|
||
/// </summary>
|
||
private static FakeHistorianGatewayClient Store(IReadOnlyList<StoredSample> data, bool avevaBoundary = false,
|
||
bool terminalRow = false)
|
||
{
|
||
var sorted = data.OrderBy(s => s.TimeUtc).ToList();
|
||
return new FakeHistorianGatewayClient
|
||
{
|
||
RawSource = (start, end, max) =>
|
||
{
|
||
var rows = new List<HistorianSample>();
|
||
if (avevaBoundary && sorted.All(s => s.TimeUtc != start))
|
||
{
|
||
var prior = sorted.LastOrDefault(s => s.TimeUtc < start);
|
||
var boundary = prior.TimeUtc == default
|
||
? new HistorianSample { Tag = "T", NumericValue = 0, OpcQuality = 0, Quality = 1 }
|
||
: ToWire(prior);
|
||
boundary.Timestamp = Timestamp.FromDateTime(start);
|
||
if (prior.TimeUtc != default) boundary.Quality = 133;
|
||
rows.Add(boundary);
|
||
}
|
||
rows.AddRange(sorted.Where(s => s.TimeUtc >= start && s.TimeUtc <= end).Select(ToWire));
|
||
if (terminalRow && max <= 2 && end > start && sorted.All(s => s.TimeUtc != end))
|
||
rows.Add(new HistorianSample
|
||
{ Tag = "T", Timestamp = Timestamp.FromDateTime(end), NumericValue = 0, OpcQuality = 0, Quality = 1 });
|
||
return rows.Take(max);
|
||
},
|
||
};
|
||
}
|
||
|
||
private sealed class FixedClock(DateTime now) : TimeProvider
|
||
{
|
||
public override DateTimeOffset GetUtcNow() => new(now);
|
||
}
|
||
|
||
private static GatewayHistorianDataSource Source(FakeHistorianGatewayClient client, int maxRawRows = 1_000_000) =>
|
||
new(client, NullLogger<GatewayHistorianDataSource>.Instance, maxRawRows, new FixedClock(Now));
|
||
|
||
// ---- the oracle: what the SQL backends answer --------------------------------------------------------------------
|
||
|
||
/// <summary>The SQL backends' answer for <paramref name="data"/> (statuses as the gateway quality byte carries them).</summary>
|
||
private static IReadOnlyList<DataValueSnapshot> SqlBackendAnswer(HistoryAggregateType agg, BucketPlan plan,
|
||
IReadOnlyList<StoredSample> data)
|
||
{
|
||
// Same clamp the backends apply (an Empty sample is Bad).
|
||
var stored = data.Select(s => StoredSample.FromStored(s.TimeUtc, s.Value, s.Status)).ToList();
|
||
var earliest = stored.Where(s => s.TimeUtc <= plan.EndUtc).Select(s => (DateTime?)s.TimeUtc).Min();
|
||
var latest = stored.Where(s => s.TimeUtc >= plan.StartUtc).Select(s => (DateTime?)s.TimeUtc).Max();
|
||
return AggregateEvaluator.EvaluateAll(agg, plan, InMemoryRows(plan, stored, agg == HistoryAggregateType.MaximumActualTime),
|
||
earliest, latest, Now);
|
||
}
|
||
|
||
/// <summary>Independent LINQ statement of the per-bucket statistics (copied from Sql.Common.Tests, #684 semantics).</summary>
|
||
private static IEnumerable<BucketRow> InMemoryRows(BucketPlan plan, IEnumerable<StoredSample> samples, bool maximum)
|
||
{
|
||
foreach (var g in samples.Select(s => (s, b: plan.IndexOf(s.TimeUtc))).Where(x => x.b is not null)
|
||
.GroupBy(x => x.b!.Value).OrderBy(g => g.Key))
|
||
{
|
||
var all = g.Select(x => x.s).OrderBy(s => s.TimeUtc).ToList();
|
||
var usable = all.Where(s => s.IsUsable).ToList();
|
||
var numeric = usable.Where(s => s.Value.Kind is HistorianValueKind.Double or HistorianValueKind.Int64
|
||
or HistorianValueKind.Boolean).ToList();
|
||
var good = numeric.Where(s => s.IsGood).ToList();
|
||
var uncertain = numeric.Where(s => !s.IsGood).ToList();
|
||
var extreme = good.OrderBy(s => maximum ? -s.Value.Number : s.Value.Number).ThenBy(s => s.TimeUtc)
|
||
.Cast<StoredSample?>().FirstOrDefault();
|
||
yield return new BucketRow
|
||
{
|
||
Bucket = g.Key,
|
||
Total = all.Count,
|
||
Usable = usable.Count,
|
||
Good = all.Count(s => s.IsGood),
|
||
Bad = all.Count - usable.Count,
|
||
NonNumeric = usable.Count - numeric.Count,
|
||
GoodNonNumeric = usable.Count(s => s.IsGood) - numeric.Count(s => s.IsGood),
|
||
Avg = good.Count == 0 ? null : good.Average(s => s.Value.Number),
|
||
Min = good.Count == 0 ? null : good.Min(s => s.Value.Number),
|
||
Max = good.Count == 0 ? null : good.Max(s => s.Value.Number),
|
||
Sum = good.Count == 0 ? null : good.Sum(s => s.Value.Number),
|
||
UncertainMin = uncertain.Count == 0 ? null : uncertain.Min(s => s.Value.Number),
|
||
UncertainMax = uncertain.Count == 0 ? null : uncertain.Max(s => s.Value.Number),
|
||
First = usable.Count == 0 ? HistorianValue.Empty : usable[0].Value,
|
||
Last = usable.Count == 0 ? HistorianValue.Empty : usable[^1].Value,
|
||
Extreme = extreme?.Value.Number,
|
||
ExtremeTime = extreme?.TimeUtc,
|
||
FirstAny = all[0],
|
||
LastAny = all[^1],
|
||
};
|
||
}
|
||
}
|
||
|
||
private static string Describe(IReadOnlyList<DataValueSnapshot> v) =>
|
||
string.Join(" | ", v.Select(x => $"{x.Value ?? "null"} 0x{x.StatusCode:X8} {(x.SourceTimestampUtc!.Value - T0).TotalSeconds}s"));
|
||
|
||
// ---- parity ------------------------------------------------------------------------------------------------------
|
||
|
||
[Theory]
|
||
[InlineData(false)]
|
||
[InlineData(true)]
|
||
public async Task Every_per_sample_aggregate_matches_the_SQL_backends_forward_and_reverse(bool avevaBoundary)
|
||
{
|
||
var failures = new List<string>();
|
||
var compared = 0;
|
||
foreach (var agg in PerSample)
|
||
foreach (var (name, data) in Datasets)
|
||
foreach (var (from, to, interval) in Windows)
|
||
{
|
||
var plan = BucketPlan.Create(At(from), At(to), TimeSpan.FromSeconds(interval));
|
||
var expected = SqlBackendAnswer(agg, plan, data);
|
||
var fake = Store(data, avevaBoundary);
|
||
var actual = (await Source(fake)
|
||
.ReadProcessedAsync("T", At(from), At(to), TimeSpan.FromSeconds(interval), agg, Ct)).Samples;
|
||
compared++;
|
||
if (fake.RawStreamsCompleted != fake.ReadRawCallCount)
|
||
failures.Add($"{agg} {name} {from}→{to}/{interval}s: a raw stream was abandoned mid-way");
|
||
var same = actual.Count == expected.Count && actual.Zip(expected).All(p =>
|
||
Equals(p.First.Value, p.Second.Value) && p.First.StatusCode == p.Second.StatusCode
|
||
&& p.First.SourceTimestampUtc == p.Second.SourceTimestampUtc);
|
||
if (!same)
|
||
failures.Add($"{agg} {name} {from}→{to}/{interval}s\n ours {Describe(actual)}\n sql {Describe(expected)}");
|
||
}
|
||
compared.ShouldBe(PerSample.Length * Datasets.Length * Windows.Length);
|
||
failures.ShouldBeEmpty(string.Join("\n", failures));
|
||
}
|
||
|
||
[Theory]
|
||
[InlineData(HistoryAggregateType.Average, 15d, 40d, 50d)]
|
||
[InlineData(HistoryAggregateType.Minimum, 10d, 40d, 50d)]
|
||
[InlineData(HistoryAggregateType.Maximum, 20d, 40d, 50d)]
|
||
[InlineData(HistoryAggregateType.Delta, 10d, 0d, 0d)]
|
||
[InlineData(HistoryAggregateType.Range, 10d, 0d, 0d)]
|
||
public async Task Numeric_aggregates_match_the_conformance_suites_hand_computed_literals(HistoryAggregateType agg,
|
||
double b0, double b1, double b2)
|
||
{
|
||
// The literals of ProcessedConformance.Numeric_aggregates_over_the_fixture (SqlServer + QuestDB), over tag A.
|
||
var s = (await Source(Store(A, avevaBoundary: true))
|
||
.ReadProcessedAsync("A", At(0), At(80), TimeSpan.FromSeconds(20), agg, Ct)).Samples;
|
||
s.Select(x => x.Value).ShouldBe([b0, b1, b2, null]);
|
||
var b2Status = agg == HistoryAggregateType.Average ? GoodCalc : GoodCalc | Partial;
|
||
s.Select(x => x.StatusCode).ShouldBe([GoodCalc, UncertainCalc, b2Status, NoDataCalc]);
|
||
s.Select(x => x.SourceTimestampUtc).ShouldBe([At(0), At(20), At(40), At(60)]); // interval STARTS (Part 13)
|
||
}
|
||
|
||
[Fact]
|
||
public async Task Count_is_a_Good_sample_count_as_int_and_zero_inside_the_data_range()
|
||
{
|
||
// #679 made Count BadAggregateNotSupported on Gateway; #681 serves it with the SQL backends' #680 semantics.
|
||
var s = (await Source(Store(Gappy)).ReadProcessedAsync("T", At(0), At(60), TimeSpan.FromSeconds(10),
|
||
HistoryAggregateType.Count, Ct)).Samples;
|
||
// [0,10): +1 G, +2 U ⇒ 1, Uncertain (data starts inside ⇒ Partial); [10,20) empty inside the range ⇒ 0 Good;
|
||
// [20,30): +25 Bad only ⇒ generic Bad; [30,40) 0; [40,50) +45 ⇒ 1; [50,60) 0 (+70 is later data).
|
||
s.Select(x => x.Value).ShouldBe([1, 0, null, 0, 1, 0]);
|
||
s.Select(x => x.StatusCode).ShouldBe([UncertainCalc | Partial, GoodCalc, 0x80000401u, GoodCalc, GoodCalc, GoodCalc]);
|
||
}
|
||
|
||
[Fact]
|
||
public async Task A_usable_string_sample_makes_a_numeric_aggregate_BadAggregateInvalidInputs()
|
||
{
|
||
var s = (await Source(Store(Mixed)).ReadProcessedAsync("T", At(0), At(40), TimeSpan.FromSeconds(10),
|
||
HistoryAggregateType.Average, Ct)).Samples;
|
||
s[1].StatusCode.ShouldBe(InvalidInputsCalc);
|
||
s[1].Value.ShouldBeNull();
|
||
}
|
||
|
||
[Fact]
|
||
public async Task ActualTime_carries_the_extreme_samples_own_time_ties_earliest()
|
||
{
|
||
var max = (await Source(Store(Ties)).ReadProcessedAsync("T", At(0), At(10), TimeSpan.Zero,
|
||
HistoryAggregateType.MaximumActualTime, Ct)).Samples.Single();
|
||
max.Value.ShouldBe(9d);
|
||
max.SourceTimestampUtc.ShouldBe(At(3));
|
||
var min = (await Source(Store(Ties)).ReadProcessedAsync("T", At(0), At(10), TimeSpan.Zero,
|
||
HistoryAggregateType.MinimumActualTime, Ct)).Samples.Single();
|
||
min.Value.ShouldBe(1d);
|
||
min.SourceTimestampUtc.ShouldBe(At(4));
|
||
}
|
||
|
||
[Fact]
|
||
public async Task A_reverse_read_is_anchored_at_its_later_time_and_returned_latest_first()
|
||
{
|
||
// #677: 50 → 0 at 20 s ⇒ (30,50], (10,30], (0,10] stamped 50, 30, 10.
|
||
var s = (await Source(Store(A)).ReadProcessedAsync("A", At(50), At(0), TimeSpan.FromSeconds(20),
|
||
HistoryAggregateType.Maximum, Ct)).Samples;
|
||
s.Select(x => x.SourceTimestampUtc).ShouldBe([At(50), At(30), At(10)]);
|
||
s.Select(x => x.Value).ShouldBe([50d, 40d, 20d]);
|
||
}
|
||
|
||
// ---- the raw request ---------------------------------------------------------------------------------------------
|
||
|
||
[Fact]
|
||
public async Task The_raw_stream_starts_one_millisecond_before_the_window_so_the_AVEVA_boundary_row_is_never_a_sample()
|
||
{
|
||
// Data before the window, nothing inside [0,10) but +5: the boundary row (value 1, stamped at the query start) must
|
||
// not be counted. Count over [0,10) is 1, not 2.
|
||
var fake = Store([D(-5, 1), D(5, 2)], avevaBoundary: true);
|
||
var s = (await Source(fake).ReadProcessedAsync("T", At(0), At(10), TimeSpan.Zero, HistoryAggregateType.Count, Ct))
|
||
.Samples.Single();
|
||
s.Value.ShouldBe(1);
|
||
fake.RawCalls[0].Start.ShouldBe(At(0) - TimeSpan.FromMilliseconds(1));
|
||
fake.RawCalls.ShouldNotContain(c => c.Start == DateTime.UnixEpoch); // the Good boundary row proved earlier data: no probe
|
||
}
|
||
|
||
[Fact]
|
||
public async Task Without_boundary_evidence_a_probe_asks_for_data_before_the_window()
|
||
{
|
||
// No boundary row (a historian that emits none): the second call probes [epoch, start − 1 tick] for 2 rows, and the
|
||
// sample at −5 it finds keeps the first interval free of the start-of-data Partial bit.
|
||
var fake = Store([D(-5, 1), D(5, 2), D(15, 3)]);
|
||
var s = (await Source(fake).ReadProcessedAsync("T", At(0), At(20), TimeSpan.FromSeconds(10),
|
||
HistoryAggregateType.Minimum, Ct)).Samples;
|
||
fake.RawCalls[1].ShouldBe((DateTime.UnixEpoch, At(0).AddTicks(-1), 2));
|
||
(s[0].StatusCode & Partial).ShouldBe(0u);
|
||
|
||
// Data that really starts inside the first interval: the probe finds nothing, the interval is Partial.
|
||
var late = Store([D(5, 2), D(15, 3)], avevaBoundary: true);
|
||
var l = (await Source(late).ReadProcessedAsync("T", At(0), At(20), TimeSpan.FromSeconds(10),
|
||
HistoryAggregateType.Minimum, Ct)).Samples;
|
||
late.RawCalls[1].Start.ShouldBe(DateTime.UnixEpoch); // the NULL boundary row (value 0, OPC quality 0) is no evidence
|
||
(l[0].StatusCode & Partial).ShouldBe(Partial);
|
||
}
|
||
|
||
[Fact]
|
||
public async Task The_raw_request_looks_ahead_past_the_window_only_when_the_data_range_matters()
|
||
{
|
||
var count = Store(A);
|
||
await Source(count).ReadProcessedAsync("A", At(0), At(80), TimeSpan.FromSeconds(20), HistoryAggregateType.Count, Ct);
|
||
// The window read ends AT the window end; probe + look-ahead are separate 2-row reads, all consumed to completion.
|
||
count.RawCalls.ShouldBe([
|
||
(At(0) - TimeSpan.FromMilliseconds(1), At(80), 1_000_000), // cap + 18, clamped at the gateway default ceiling (#688)
|
||
(DateTime.UnixEpoch, At(0).AddTicks(-1), 2),
|
||
(At(80), Now + TimeSpan.FromDays(1), 2),
|
||
]);
|
||
count.RawStreamsCompleted.ShouldBe(3);
|
||
|
||
var average = Store(A);
|
||
await Source(average).ReadProcessedAsync("A", At(0), At(80), TimeSpan.FromSeconds(20), HistoryAggregateType.Average, Ct);
|
||
average.RawCalls.Single().End.ShouldBe(At(80)); // Average needs no data range: no look-ahead, no probe
|
||
average.ReadAggregateCallCount.ShouldBe(0);
|
||
|
||
var unbounded = Store(A);
|
||
await Source(unbounded, maxRawRows: 0).ReadProcessedAsync("A", At(0), At(80), TimeSpan.FromSeconds(20),
|
||
HistoryAggregateType.Count, Ct);
|
||
unbounded.RawCalls[0].MaxValues.ShouldBe(int.MaxValue);
|
||
}
|
||
|
||
[Fact]
|
||
public async Task No_gateway_stream_is_ever_abandoned_and_later_data_comes_from_a_two_row_look_ahead()
|
||
{
|
||
// #681 review: breaking out of a gateway stream cancels the call, which the gateway treats as a connection fault
|
||
// (it evicts its shared historian session). 1000 samples after the window: the window read never sees them, the
|
||
// 2-row look-ahead reads two, and every stream runs to its end.
|
||
var data = Enumerable.Range(0, 1000).Select(i => D(100 + i, i)).Prepend(D(-5, 0)).Prepend(D(5, 1)).ToArray();
|
||
var fake = Store(data, avevaBoundary: true);
|
||
var s = (await Source(fake).ReadProcessedAsync("T", At(0), At(50), TimeSpan.FromSeconds(10),
|
||
HistoryAggregateType.Minimum, Ct)).Samples;
|
||
fake.RawStreamsCompleted.ShouldBe(fake.ReadRawCallCount);
|
||
fake.RawCalls.Last().ShouldBe((At(50), Now + TimeSpan.FromDays(1), 2));
|
||
fake.RawYieldedCount.ShouldBe(2 + 2); // window read: boundary + +5; look-ahead: boundary at +50 + first sample
|
||
(s[0].StatusCode & Partial).ShouldBe(0u); // data continues after the window: no end-of-data Partial
|
||
}
|
||
|
||
[Fact]
|
||
public async Task Rows_stamped_at_a_small_reads_query_start_or_end_are_not_samples()
|
||
{
|
||
// Data ends at +15: the look-ahead's NULL rows at its own start (+20, boundary) and end (now + 1 day, terminal)
|
||
// must not count as later data, so (10,20] / [10,20) holds the end of the data ⇒ Partial.
|
||
var fake = Store([D(-5, 1), D(5, 2), D(15, 3)], avevaBoundary: true, terminalRow: true);
|
||
var s = (await Source(fake).ReadProcessedAsync("T", At(0), At(20), TimeSpan.FromSeconds(10),
|
||
HistoryAggregateType.Maximum, Ct)).Samples;
|
||
(s[1].StatusCode & Partial).ShouldBe(Partial);
|
||
fake.RawCalls.Last().Start.ShouldBe(At(20));
|
||
}
|
||
|
||
[Fact]
|
||
public async Task A_window_over_the_raw_row_cap_fails_the_node_without_counting_a_backend_failure()
|
||
{
|
||
var data = Enumerable.Range(0, 10).Select(i => D(i, i)).ToArray(); // 10 rows in [0,10)
|
||
var ds = Source(Store(data), maxRawRows: 9);
|
||
var ex = await Should.ThrowAsync<HistoryReadLimitExceededException>(() =>
|
||
ds.ReadProcessedAsync("T", At(0), At(10), TimeSpan.Zero, HistoryAggregateType.Average, Ct));
|
||
ex.Limit.ShouldBe(9);
|
||
ds.GetHealthSnapshot().TotalFailures.ShouldBe(0);
|
||
|
||
// Exactly at the cap is served.
|
||
var ok = await Source(Store(data), maxRawRows: 10)
|
||
.ReadProcessedAsync("T", At(0), At(10), TimeSpan.Zero, HistoryAggregateType.Average, Ct);
|
||
ok.Samples.Single().Value.ShouldBe(4.5);
|
||
}
|
||
|
||
[Fact]
|
||
public async Task A_stream_that_exhausts_its_row_budget_before_the_window_is_never_answered_from_a_truncated_window()
|
||
{
|
||
// 40 rows inside the 1 ms margin before the window eat the whole budget (cap 5 ⇒ 23 rows): the window could not be
|
||
// read, so the node fails rather than reporting BadNoData.
|
||
var margin = Enumerable.Range(0, 40).Select(i => new StoredSample(At(0).AddTicks(-1 - i), HistorianValue.FromDouble(i), Good));
|
||
var data = margin.Append(D(5, 1)).ToArray();
|
||
await Should.ThrowAsync<HistoryReadLimitExceededException>(() => Source(Store(data), maxRawRows: 5)
|
||
.ReadProcessedAsync("T", At(0), At(10), TimeSpan.Zero, HistoryAggregateType.Count, Ct));
|
||
}
|
||
|
||
[Fact]
|
||
public async Task Past_the_cap_the_window_stream_is_drained_not_abandoned()
|
||
{
|
||
var data = Enumerable.Range(0, 10).Select(i => D(i, i)).ToArray();
|
||
var fake = Store(data);
|
||
await Should.ThrowAsync<HistoryReadLimitExceededException>(() => Source(fake, maxRawRows: 3)
|
||
.ReadProcessedAsync("T", At(0), At(10), TimeSpan.Zero, HistoryAggregateType.Average, Ct));
|
||
fake.RawStreamsCompleted.ShouldBe(1);
|
||
fake.RawYieldedCount.ShouldBe(10);
|
||
}
|
||
|
||
[Fact]
|
||
public async Task The_gateways_max_values_ceiling_rejection_is_a_request_limit_not_a_backend_fault()
|
||
{
|
||
// Historian:ReadLimits:MaxValuesCeiling (1,000,000) with AllowStreamingBeyondCeiling=false refuses the default
|
||
// cap + 18 budget with InvalidArgument; the client package wraps it in HistorianGatewayException.
|
||
var rpc = new RpcException(new Status(StatusCode.InvalidArgument,
|
||
"max_values exceeds the server ceiling of 1000000; read large ranges by time-slicing ..."));
|
||
var fake = new FakeHistorianGatewayClient { ThrowOnRead = new HistorianGatewayException(rpc.Status.Detail, rpc) };
|
||
var ds = Source(fake);
|
||
await Should.ThrowAsync<HistoryReadLimitExceededException>(() =>
|
||
ds.ReadProcessedAsync("T", At(0), At(10), TimeSpan.Zero, HistoryAggregateType.Count, Ct));
|
||
ds.GetHealthSnapshot().TotalFailures.ShouldBe(0);
|
||
|
||
// Any other InvalidArgument is still a backend fault.
|
||
var other = new RpcException(new Status(StatusCode.InvalidArgument, "start_time must be less than or equal to end_time."));
|
||
var ds2 = Source(new FakeHistorianGatewayClient { ThrowOnRead = new HistorianGatewayException(other.Status.Detail, other) });
|
||
await Should.ThrowAsync<HistorianGatewayException>(() =>
|
||
ds2.ReadProcessedAsync("T", At(0), At(10), TimeSpan.Zero, HistoryAggregateType.Count, Ct));
|
||
ds2.GetHealthSnapshot().TotalFailures.ShouldBe(1);
|
||
}
|
||
|
||
[Theory] // a window starting AT the epoch gets no 1 ms margin, so AVEVA's no-data row is stamped at the window start:
|
||
// it must not become a stored Bad sample of bucket 0
|
||
[InlineData(HistoryAggregateType.Start)]
|
||
[InlineData(HistoryAggregateType.Count)]
|
||
[InlineData(HistoryAggregateType.Minimum)]
|
||
public async Task A_window_starting_at_the_epoch_does_not_evaluate_the_fabricated_row(HistoryAggregateType agg)
|
||
{
|
||
var epoch = DateTime.UnixEpoch;
|
||
var data = new[] { new StoredSample(epoch.AddSeconds(5), HistorianValue.FromDouble(7), Good) };
|
||
var fake = Store(data, avevaBoundary: true);
|
||
var s = (await Source(fake).ReadProcessedAsync("T", epoch, epoch.AddSeconds(20), TimeSpan.FromSeconds(20), agg, Ct))
|
||
.Samples.Single();
|
||
var expected = SqlBackendAnswer(agg, BucketPlan.Create(epoch, epoch.AddSeconds(20), TimeSpan.FromSeconds(20)), data).Single();
|
||
(s.Value, s.StatusCode).ShouldBe((expected.Value, expected.StatusCode));
|
||
fake.RawCalls[0].Start.ShouldBe(epoch); // the request start cannot go before the epoch
|
||
}
|
||
|
||
[Fact] // #688: the default cap's budget fits the gateway's default ceiling, so a strict gateway serves the default config
|
||
public async Task The_default_caps_row_budget_does_not_exceed_the_gateways_default_ceiling()
|
||
{
|
||
var fake = Store([D(-5, 1), D(5, 2)], avevaBoundary: true);
|
||
await Source(fake).ReadProcessedAsync("T", At(0), At(10), TimeSpan.Zero, HistoryAggregateType.Average, Ct);
|
||
TimeWeightedAggregator.DefaultMaxWindowRows.ShouldBe(1_000_000);
|
||
fake.RawCalls[0].MaxValues.ShouldBe(RawAggregateReader.GatewayDefaultMaxValuesCeiling);
|
||
}
|
||
|
||
[Theory] // #688: the + 18 allowance is clamped at the gateway's default ceiling, never below the cap itself
|
||
[InlineData(10, 28)]
|
||
[InlineData(999_982, 1_000_000)]
|
||
[InlineData(999_990, 1_000_000)]
|
||
[InlineData(1_000_000, 1_000_000)]
|
||
[InlineData(2_000_000, 2_000_018)] // a cap above the default ceiling is a deliberate choice for a streaming gateway
|
||
[InlineData(0, int.MaxValue)]
|
||
[InlineData(-1, int.MaxValue)]
|
||
public void The_window_row_budget_is_clamped_at_the_gateways_default_ceiling(int cap, int expected) =>
|
||
RawAggregateReader.MaxValuesFor(cap).ShouldBe(expected);
|
||
|
||
[Fact] // #688: near the ceiling the window is still served when it is not full
|
||
public async Task A_cap_at_the_ceiling_still_serves_a_window_below_it()
|
||
{
|
||
var data = Enumerable.Range(0, 10).Select(i => D(i, i)).ToArray();
|
||
var fake = Store(data, avevaBoundary: true);
|
||
var r = await Source(fake, maxRawRows: 1_000_000)
|
||
.ReadProcessedAsync("T", At(0), At(10), TimeSpan.Zero, HistoryAggregateType.Average, Ct);
|
||
r.Samples.Single().Value.ShouldBe(4.5);
|
||
}
|
||
|
||
[Fact]
|
||
public async Task A_caller_cancellation_is_not_counted_as_a_backend_failure()
|
||
{
|
||
using var cts = new CancellationTokenSource();
|
||
await cts.CancelAsync();
|
||
var ds = Source(new FakeHistorianGatewayClient { ThrowOnRead = new OperationCanceledException(cts.Token) });
|
||
await Should.ThrowAsync<OperationCanceledException>(() =>
|
||
ds.ReadProcessedAsync("T", At(0), At(10), TimeSpan.Zero, HistoryAggregateType.Count, cts.Token));
|
||
ds.GetHealthSnapshot().TotalFailures.ShouldBe(0);
|
||
|
||
// A cancellation the caller did NOT ask for (e.g. the gateway cancelled the call) is still a backend fault.
|
||
var ds2 = Source(new FakeHistorianGatewayClient { ThrowOnRead = new OperationCanceledException() });
|
||
await Should.ThrowAsync<OperationCanceledException>(() =>
|
||
ds2.ReadProcessedAsync("T", At(0), At(10), TimeSpan.Zero, HistoryAggregateType.Count, Ct));
|
||
ds2.GetHealthSnapshot().TotalFailures.ShouldBe(1);
|
||
}
|
||
|
||
[Fact]
|
||
public async Task A_gateway_fault_on_the_raw_read_is_recorded_and_rethrown()
|
||
{
|
||
var fake = new FakeHistorianGatewayClient { ThrowOnRead = new InvalidOperationException("down") };
|
||
var ds = Source(fake);
|
||
await Should.ThrowAsync<InvalidOperationException>(() =>
|
||
ds.ReadProcessedAsync("T", At(0), At(10), TimeSpan.Zero, HistoryAggregateType.Count, Ct));
|
||
ds.GetHealthSnapshot().TotalFailures.ShouldBe(1);
|
||
}
|
||
|
||
[Fact]
|
||
public async Task An_empty_window_calls_nothing_and_returns_no_intervals()
|
||
{
|
||
var fake = Store(A);
|
||
var r = await Source(fake).ReadProcessedAsync("A", At(10), At(10), TimeSpan.Zero, HistoryAggregateType.Count, Ct);
|
||
r.Samples.ShouldBeEmpty();
|
||
fake.ReadRawCallCount.ShouldBe(0);
|
||
}
|
||
|
||
// ---- the served set ----------------------------------------------------------------------------------------------
|
||
|
||
[Fact]
|
||
public void The_served_set_is_the_per_sample_families_plus_the_two_native_modes()
|
||
{
|
||
GatewayHistorianDataSource.ServedAggregates.OrderBy(a => a).ShouldBe(
|
||
PerSample.Append(HistoryAggregateType.TimeAverage).Append(HistoryAggregateType.Total).OrderBy(a => a));
|
||
}
|
||
|
||
[Fact]
|
||
public async Task Every_member_outside_the_served_set_is_refused_without_calling_the_gateway_and_every_member_inside_is_served()
|
||
{
|
||
foreach (var agg in System.Enum.GetValues<HistoryAggregateType>())
|
||
{
|
||
var fake = Store(A);
|
||
var ds = Source(fake);
|
||
if (GatewayHistorianDataSource.ServedAggregates.Contains(agg))
|
||
{
|
||
await ds.ReadProcessedAsync("A", At(0), At(40), TimeSpan.FromSeconds(20), agg, Ct);
|
||
(fake.ReadRawCallCount + fake.ReadAggregateCallCount).ShouldBeGreaterThan(0, agg.ToString());
|
||
}
|
||
else
|
||
{
|
||
await Should.ThrowAsync<HistoryAggregateNotSupportedException>(() =>
|
||
ds.ReadProcessedAsync("A", At(0), At(40), TimeSpan.FromSeconds(20), agg, Ct));
|
||
(fake.ReadRawCallCount + fake.ReadAggregateCallCount).ShouldBe(0, agg.ToString());
|
||
ds.GetHealthSnapshot().TotalQueries.ShouldBe(0);
|
||
}
|
||
}
|
||
}
|
||
|
||
// ---- the accumulator against the independent oracle, sample order shuffled ---------------------------------------
|
||
|
||
[Fact]
|
||
public void The_bucket_statistics_do_not_depend_on_arrival_order()
|
||
{
|
||
var plan = BucketPlan.Create(At(-10), At(100), TimeSpan.FromSeconds(10));
|
||
foreach (var maximum in new[] { false, true })
|
||
{
|
||
var acc = new RawBucketAccumulator(maximum);
|
||
foreach (var s in DenseUncertain.Reverse())
|
||
if (plan.IndexOf(s.TimeUtc) is { } b) acc.Add(b, s);
|
||
var expected = InMemoryRows(plan, DenseUncertain, maximum).ToList();
|
||
var actual = acc.Rows().ToList();
|
||
actual.Count.ShouldBe(expected.Count);
|
||
foreach (var (a, e) in actual.Zip(expected))
|
||
{
|
||
(a.Bucket, a.Total, a.Usable, a.Good, a.Bad, a.NonNumeric).ShouldBe((e.Bucket, e.Total, e.Usable, e.Good, e.Bad, e.NonNumeric));
|
||
(a.Min, a.Max, a.Extreme, a.ExtremeTime).ShouldBe((e.Min, e.Max, e.Extreme, e.ExtremeTime));
|
||
(a.UncertainMin, a.UncertainMax, a.GoodNonNumeric).ShouldBe((e.UncertainMin, e.UncertainMax, e.GoodNonNumeric));
|
||
(a.Avg is null).ShouldBe(e.Avg is null);
|
||
if (a.Avg is { } avg) avg.ShouldBe(e.Avg!.Value, 1e-9);
|
||
(a.First, a.Last, a.FirstAny, a.LastAny).ShouldBe((e.First, e.Last, e.FirstAny, e.LastAny));
|
||
}
|
||
}
|
||
}
|
||
}
|