feat(historian): Gateway serves the per-sample aggregates from raw reads with the shared Sql.Common evaluator (#681)

Average, Count, Minimum, Maximum, Range, Start, End, Delta and Minimum/MaximumActualTime are evaluated
on the Gateway backend from forward ReadRaw reads with BucketPlan + AggregateEvaluator, so Gateway
matches SqlServer/QuestDB exactly (values, status bits, interval-start stamps, #677 reverse).

- Window read [start - 1 ms, end] so AVEVA's initial-value row never lands in a bucket; a pre-window
  row proves earlier data only with a non-Bad OPC quality (histsdk puts a value on every NULL row).
- Earlier data otherwise from a 2-row [1970, start) probe; later data from a 2-row [end, now+1d]
  look-ahead; both ignore rows stamped at their query start/end.
- No gateway stream is ever abandoned (a client-side cancel makes the gateway evict its shared
  historian session); past ServerHistorian:MaxTimeWeightedRawRows the bounded stream is drained, then
  BadTooManyOperations. The gateway's max_values-ceiling InvalidArgument maps to the same limit; a
  caller cancellation is not a backend fault.
- TimeAverage/Total stay AVEVA-native (no prior-sample read for exact Part 13 bounds), now stamped with
  the interval start (PENDING the live leg: histsdk's tail+43 start_time field is unverified) and
  carrying the Calculated bits.
- Served set: GatewayHistorianDataSource.ServedAggregates (12). Live AVEVA leg open (VPN).
This commit is contained in:
Joseph Doherty
2026-10-12 09:55:32 -04:00
parent dbdf29c8fc
commit 78f604e0cd
16 changed files with 1187 additions and 77 deletions
@@ -19,6 +19,17 @@ public sealed class FakeHistorianGatewayClient : IHistorianGatewayClient
public DateTime LastReadRawEndUtc;
public int LastReadRawMaxValues;
public int ReadRawCallCount;
/// <summary>
/// When set, ReadRaw serves <c>RawSource(start, end, maxValues)</c> instead of <see cref="RawSamples"/> (a
/// store-like fake honouring the window, #681).
/// </summary>
public Func<DateTime, DateTime, int, IEnumerable<HistorianSample>>? RawSource;
/// <summary>Raw samples actually pulled by the consumer across every ReadRaw stream.</summary>
public int RawYieldedCount;
/// <summary>ReadRaw streams the consumer enumerated to the END (never broke out of — #681 review).</summary>
public int RawStreamsCompleted;
/// <summary>Every ReadRaw call's (start, end, maxValues), in order.</summary>
public readonly List<(DateTime Start, DateTime End, int MaxValues)> RawCalls = new();
// ---- ReadAggregate -------------------------------------------------------------------------
public IReadOnlyList<HistorianAggregateSample> AggregateSamples = Array.Empty<HistorianAggregateSample>();
@@ -90,7 +101,9 @@ public sealed class FakeHistorianGatewayClient : IHistorianGatewayClient
LastReadRawEndUtc = endUtc;
LastReadRawMaxValues = maxValues;
ReadRawCallCount++;
return ToAsyncStream(RawSamples, ThrowOnRead, ct);
RawCalls.Add((startUtc, endUtc, maxValues));
var items = RawSource is null ? RawSamples : RawSource(startUtc, endUtc, maxValues).ToList();
return CountingStream(ToAsyncStream(items, ThrowOnRead, ct));
}
public IAsyncEnumerable<HistorianAggregateSample> ReadAggregateAsync(
@@ -193,6 +206,16 @@ public sealed class FakeHistorianGatewayClient : IHistorianGatewayClient
return ValueTask.CompletedTask;
}
private async IAsyncEnumerable<HistorianSample> CountingStream(IAsyncEnumerable<HistorianSample> inner)
{
await foreach (var item in inner.ConfigureAwait(false))
{
RawYieldedCount++;
yield return item;
}
RawStreamsCompleted++;
}
private static async IAsyncEnumerable<T> ToAsyncStream<T>(
IReadOnlyList<T> items,
Exception? error,
@@ -32,11 +32,29 @@ public sealed class GatewayHistorianDataSourceTests
{
var fake = new FakeHistorianGatewayClient();
var ds = new GatewayHistorianDataSource(fake, NullLogger<GatewayHistorianDataSource>.Instance);
await ds.ReadProcessedAsync("T", default, default, TimeSpan.FromSeconds(60), HistoryAggregateType.Minimum, TestContext.Current.CancellationToken);
Assert.Equal(RetrievalMode.MinimumWithTime, fake.LastAggregateMode);
await ds.ReadProcessedAsync("T", default, default, TimeSpan.FromSeconds(60), HistoryAggregateType.Total, TestContext.Current.CancellationToken);
Assert.Equal(RetrievalMode.Integral, fake.LastAggregateMode);
Assert.Equal(TimeSpan.FromSeconds(60), fake.LastAggregateInterval);
}
[Fact] // #681: the native buckets are stamped with the interval START and carry the Calculated bits
public async Task ReadProcessed_native_buckets_are_stamped_at_the_interval_start_with_the_Calculated_bit()
{
var start = new DateTime(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc);
var fake = new FakeHistorianGatewayClient
{
AggregateSamples = new[]
{
new HistorianAggregateSample { Value = 7.0, OpcQuality = 192, StartTime = Timestamp.FromDateTime(start), EndTime = Timestamp.FromDateTime(start.AddMinutes(1)) },
new HistorianAggregateSample { OpcQuality = 0, StartTime = Timestamp.FromDateTime(start.AddMinutes(1)), EndTime = Timestamp.FromDateTime(start.AddMinutes(2)) },
},
};
var ds = new GatewayHistorianDataSource(fake, NullLogger<GatewayHistorianDataSource>.Instance);
var r = await ds.ReadProcessedAsync("T", start, start.AddMinutes(2), TimeSpan.FromMinutes(1), HistoryAggregateType.TimeAverage, TestContext.Current.CancellationToken);
Assert.Equal(new DateTime?[] { start, start.AddMinutes(1) }, r.Samples.Select(x => x.SourceTimestampUtc).ToArray());
Assert.Equal(new[] { 0x00000401u, 0x809B0401u }, r.Samples.Select(x => x.StatusCode).ToArray());
}
[Fact]
public async Task ReadProcessed_TimeAverage_requests_TimeWeightedAverage()
{
@@ -46,9 +64,24 @@ public sealed class GatewayHistorianDataSourceTests
Assert.Equal(RetrievalMode.TimeWeightedAverage, fake.LastAggregateMode);
}
[Theory] // #679: no AVEVA equivalent — not supported, gateway never called, no backend failure recorded
[Theory] // #681 (was #679 BadAggregateNotSupported): served from a raw read, never a native retrieval mode
[InlineData(HistoryAggregateType.Average)]
[InlineData(HistoryAggregateType.Count)]
[InlineData(HistoryAggregateType.Minimum)]
[InlineData(HistoryAggregateType.Maximum)]
public async Task ReadProcessed_per_sample_aggregate_reads_raw_not_a_retrieval_mode(HistoryAggregateType aggregate)
{
var fake = new FakeHistorianGatewayClient();
var ds = new GatewayHistorianDataSource(fake, NullLogger<GatewayHistorianDataSource>.Instance);
var t0 = new DateTime(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc);
await ds.ReadProcessedAsync("T", t0, t0.AddMinutes(1), TimeSpan.FromSeconds(60), aggregate, TestContext.Current.CancellationToken);
Assert.Equal(0, fake.ReadAggregateCallCount);
Assert.True(fake.ReadRawCallCount >= 1); // the window, plus the earlier-data probe when the range matters
}
[Theory] // no AVEVA mode and not a per-sample aggregate: not supported, gateway never called, no failure recorded
[InlineData(HistoryAggregateType.TimeAverage2)]
[InlineData(HistoryAggregateType.PercentGood)]
public async Task ReadProcessed_unsupported_aggregate_throws_without_calling_gateway_or_recording_failure(HistoryAggregateType aggregate)
{
var fake = new FakeHistorianGatewayClient();
@@ -56,6 +89,7 @@ public sealed class GatewayHistorianDataSourceTests
await Assert.ThrowsAsync<HistoryAggregateNotSupportedException>(() => ds.ReadProcessedAsync(
"T", default, default, TimeSpan.FromSeconds(60), aggregate, TestContext.Current.CancellationToken));
Assert.Equal(0, fake.ReadAggregateCallCount);
Assert.Equal(0, fake.ReadRawCallCount);
var h = ds.GetHealthSnapshot();
Assert.Equal(0, h.TotalFailures);
Assert.Equal(0, h.TotalQueries);
@@ -0,0 +1,542 @@
using Google.Protobuf.WellKnownTypes;
using Grpc.Core;
using ZB.MOM.WW.HistorianGateway.Client;
using Microsoft.Extensions.Logging.Abstractions;
using Shouldly;
using Xunit;
using ZB.MOM.WW.HistorianGateway.Contracts.Grpc;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common;
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests;
/// <summary>
/// #681: the Gateway backend serves the per-sample aggregates from raw reads with the SQL backends' shared evaluator.
/// The parity oracle is what SqlServer/QuestDB answer for the same stored samples: per-bucket statistics built
/// independently in memory (LINQ, <see cref="BucketPlan.IndexOf"/>) — the C# statement of the backends' SQL that
/// <c>Sql.Common.Tests.SdkAggregateParityTests</c> pins against the OPC UA SDK's own Part 13 calculators — evaluated by
/// <see cref="AggregateEvaluator.EvaluateAll"/> with the data range the SQL backends query (earliest sample ≤ the
/// window's later edge, latest ≥ its earlier edge, over EVERY stored sample). The fake gateway serves raw reads from
/// the same stored samples, optionally emitting AVEVA's initial-value row stamped at the query start.
/// </summary>
public sealed class GatewayRawAggregateTests
{
private static readonly CancellationToken Ct = CancellationToken.None;
/// <summary>2026-10-09 08:00:00.1234567 UTC — not millisecond-aligned (the SQL fixtures' T0).</summary>
private static readonly DateTime T0 = new DateTime(2026, 10, 9, 8, 0, 0, DateTimeKind.Utc).AddTicks(1_234_567);
private static readonly DateTime Now = At(3600);
private const uint Good = 0x00000000u;
private const uint Bad = 0x80000000u;
private const uint Uncertain = 0x40000000u;
private const uint GoodCalc = 0x00000401u;
private const uint UncertainCalc = 0x40A40401u;
private const uint NoDataCalc = 0x809B0401u;
private const uint InvalidInputsCalc = 0x80D60401u;
private const uint Partial = 0x4u;
private static DateTime At(double seconds) => T0.AddTicks((long)(seconds * TimeSpan.TicksPerSecond));
private static StoredSample S(double seconds, HistorianValue v, uint status) => new(At(seconds), v, status);
private static StoredSample D(double seconds, double v, uint status = Good) => S(seconds, HistorianValue.FromDouble(v), status);
// ---- datasets (statuses restricted to codes that round-trip the OPC DA quality byte: Good, Uncertain, Bad) ----------
/// <summary>Tag A of the conformance suite: +0=10, +10=20, +20=Bad(30), +30=40, +40=50.</summary>
private static readonly StoredSample[] A = [D(0, 10), D(10, 20), D(20, 30, Bad), D(30, 40), D(40, 50)];
/// <summary>Every 2.5 s from −10 to +100, values cycling, one Bad sample at +22.5.</summary>
private static readonly StoredSample[] Dense = Enumerable.Range(0, 45)
.Select(i => (t: -10 + i * 2.5, i))
.Select(x => D(x.t, 10 + (x.i * 7 % 13), x.t == 22.5 ? Bad : Good))
.ToArray();
private static readonly StoredSample[] Gappy =
[D(1, 10), D(2, 11, Uncertain), D(25, 12, Bad), D(45, 13), D(70, 1)];
private static readonly StoredSample[] Bracketed =
[
D(-5, 1), D(1, 10, Bad), D(3, 11), D(5, 12, Bad), D(7, 13, Uncertain), D(9, 15), D(12, 20),
S(14, HistorianValue.Empty, Bad), D(16, 21), S(19, HistorianValue.Empty, Bad), D(40, 1),
];
private static readonly StoredSample[] EarlyEnd = [D(-5, 1), D(3, 2), D(25, 3)];
private static readonly StoredSample[] EndOnEdge = [D(-5, 1), D(3, 2), D(20, 3)];
private static readonly StoredSample[] LateStart = Enumerable.Range(0, 20)
.Select(i => D(7 + i * 5, i % 2 == 0 ? 20 : 40)).ToArray();
private static readonly StoredSample[] OutageEnds = [D(-5, 1), D(3, 2), S(25, HistorianValue.Empty, Bad)];
/// <summary>A usable String sample among numbers (numeric aggregates ⇒ BadAggregateInvalidInputs in its bucket).</summary>
private static readonly StoredSample[] Mixed =
[D(-5, 1), D(2, 4), S(12, HistorianValue.FromString("x"), Good), D(15, 6), D(33, 9, Uncertain)];
/// <summary>Equal extremes in one bucket (ActualTime ties ⇒ earliest).</summary>
private static readonly StoredSample[] Ties = [D(1, 5), D(3, 9), D(4, 1), D(6, 9), D(8, 1), D(25, 2)];
private static readonly (string Name, StoredSample[] Data)[] Datasets =
[
("A", A), ("Dense", Dense), ("Gappy", Gappy), ("Bracketed", Bracketed), ("EarlyEnd", EarlyEnd),
("EndOnEdge", EndOnEdge), ("LateStart", LateStart), ("OutageEnds", OutageEnds), ("Mixed", Mixed), ("Ties", Ties),
];
private static readonly (double From, double To, double Interval)[] Windows =
[
(0, 80, 20), (80, 0, 20), (0, 50, 20), (50, 0, 20), (5, 80, 15), (80, 5, 15), (3, 77, 20), (77, 3, 20),
(0, 60, 0), (60, 0, 0), (0, 60, 10), (60, 0, 10), (-10, 1, 5), (1, -10, 5), (70, 85, 5), (85, 70, 5), (0, 1, 1),
(0, 40, 10), (40, 0, 10), (0, 40, 25), (40, 0, 25),
];
private static readonly HistoryAggregateType[] PerSample =
[
HistoryAggregateType.Average, HistoryAggregateType.Minimum, HistoryAggregateType.Maximum,
HistoryAggregateType.Count, HistoryAggregateType.Range, HistoryAggregateType.Start, HistoryAggregateType.End,
HistoryAggregateType.Delta, HistoryAggregateType.MinimumActualTime, HistoryAggregateType.MaximumActualTime,
];
// ---- the fake store ----------------------------------------------------------------------------------------------
private static HistorianSample ToWire(StoredSample s)
{
var w = new HistorianSample
{
Tag = "T",
Timestamp = Timestamp.FromDateTime(s.TimeUtc),
OpcQuality = s.Status switch { Good => 192u, Uncertain => 64u, _ => 0u },
};
// The REAL wire shape (histsdk): every numeric row carries NumericValue — a NULL / no-data row arrives as 0 with
// OPC quality 0 — and a string row always carries StringValue.
if (s.Value.Kind == HistorianValueKind.String) w.StringValue = s.Value.Text;
else w.NumericValue = s.Value.TryGetDouble(out var d) ? d : 0d;
return w;
}
/// <summary>
/// A gateway over a stored sample set: ReadRaw returns the samples in <c>[start, end]</c> ascending, capped at
/// <c>maxValues</c>. With <paramref name="avevaBoundary"/> it first emits what an AVEVA <c>Full</c> read does at
/// the query start when nothing is stored exactly there: a row stamped AT the query start carrying the value in
/// effect (the last stored sample before it, with its quality; <c>Quality = 133</c>) — or, with no earlier data, a
/// NULL row in the real wire shape: <c>NumericValue = 0</c> present, <c>OpcQuality = 0</c>. With
/// <paramref name="terminalRow"/> the small (2-row) reads also end with a NULL row stamped at the query END (a
/// hypothetical shape the probe / look-ahead must ignore; the window read relies on there being none — live leg).
/// </summary>
private static FakeHistorianGatewayClient Store(IReadOnlyList<StoredSample> data, bool avevaBoundary = false,
bool terminalRow = false)
{
var sorted = data.OrderBy(s => s.TimeUtc).ToList();
return new FakeHistorianGatewayClient
{
RawSource = (start, end, max) =>
{
var rows = new List<HistorianSample>();
if (avevaBoundary && sorted.All(s => s.TimeUtc != start))
{
var prior = sorted.LastOrDefault(s => s.TimeUtc < start);
var boundary = prior.TimeUtc == default
? new HistorianSample { Tag = "T", NumericValue = 0, OpcQuality = 0, Quality = 1 }
: ToWire(prior);
boundary.Timestamp = Timestamp.FromDateTime(start);
if (prior.TimeUtc != default) boundary.Quality = 133;
rows.Add(boundary);
}
rows.AddRange(sorted.Where(s => s.TimeUtc >= start && s.TimeUtc <= end).Select(ToWire));
if (terminalRow && max <= 2 && end > start && sorted.All(s => s.TimeUtc != end))
rows.Add(new HistorianSample
{ Tag = "T", Timestamp = Timestamp.FromDateTime(end), NumericValue = 0, OpcQuality = 0, Quality = 1 });
return rows.Take(max);
},
};
}
private sealed class FixedClock(DateTime now) : TimeProvider
{
public override DateTimeOffset GetUtcNow() => new(now);
}
private static GatewayHistorianDataSource Source(FakeHistorianGatewayClient client, int maxRawRows = 1_000_000) =>
new(client, NullLogger<GatewayHistorianDataSource>.Instance, maxRawRows, new FixedClock(Now));
// ---- the oracle: what the SQL backends answer --------------------------------------------------------------------
/// <summary>The SQL backends' answer for <paramref name="data"/> (statuses as the gateway quality byte carries them).</summary>
private static IReadOnlyList<DataValueSnapshot> SqlBackendAnswer(HistoryAggregateType agg, BucketPlan plan,
IReadOnlyList<StoredSample> data)
{
// Same clamp the backends apply (an Empty sample is Bad).
var stored = data.Select(s => StoredSample.FromStored(s.TimeUtc, s.Value, s.Status)).ToList();
var earliest = stored.Where(s => s.TimeUtc <= plan.EndUtc).Select(s => (DateTime?)s.TimeUtc).Min();
var latest = stored.Where(s => s.TimeUtc >= plan.StartUtc).Select(s => (DateTime?)s.TimeUtc).Max();
return AggregateEvaluator.EvaluateAll(agg, plan, InMemoryRows(plan, stored, agg == HistoryAggregateType.MaximumActualTime),
earliest, latest, Now);
}
/// <summary>Independent LINQ statement of the per-bucket statistics (copied from Sql.Common.Tests).</summary>
private static IEnumerable<BucketRow> InMemoryRows(BucketPlan plan, IEnumerable<StoredSample> samples, bool maximum)
{
foreach (var g in samples.Select(s => (s, b: plan.IndexOf(s.TimeUtc))).Where(x => x.b is not null)
.GroupBy(x => x.b!.Value).OrderBy(g => g.Key))
{
var all = g.Select(x => x.s).OrderBy(s => s.TimeUtc).ToList();
var usable = all.Where(s => s.IsUsable).ToList();
var numeric = usable.Where(s => s.Value.Kind is HistorianValueKind.Double or HistorianValueKind.Int64
or HistorianValueKind.Boolean).ToList();
var extreme = numeric.OrderBy(s => maximum ? -s.Value.Number : s.Value.Number).ThenBy(s => s.TimeUtc)
.Cast<StoredSample?>().FirstOrDefault();
yield return new BucketRow
{
Bucket = g.Key,
Total = all.Count,
Usable = usable.Count,
Good = all.Count(s => s.IsGood),
Bad = all.Count - usable.Count,
NonNumeric = usable.Count - numeric.Count,
Avg = numeric.Count == 0 ? null : numeric.Average(s => s.Value.Number),
Min = numeric.Count == 0 ? null : numeric.Min(s => s.Value.Number),
Max = numeric.Count == 0 ? null : numeric.Max(s => s.Value.Number),
Sum = numeric.Count == 0 ? null : numeric.Sum(s => s.Value.Number),
First = usable.Count == 0 ? HistorianValue.Empty : usable[0].Value,
Last = usable.Count == 0 ? HistorianValue.Empty : usable[^1].Value,
Extreme = extreme?.Value.Number,
ExtremeTime = extreme?.TimeUtc,
FirstAny = all[0],
LastAny = all[^1],
};
}
}
private static string Describe(IReadOnlyList<DataValueSnapshot> v) =>
string.Join(" | ", v.Select(x => $"{x.Value ?? "null"} 0x{x.StatusCode:X8} {(x.SourceTimestampUtc!.Value - T0).TotalSeconds}s"));
// ---- parity ------------------------------------------------------------------------------------------------------
[Theory]
[InlineData(false)]
[InlineData(true)]
public async Task Every_per_sample_aggregate_matches_the_SQL_backends_forward_and_reverse(bool avevaBoundary)
{
var failures = new List<string>();
var compared = 0;
foreach (var agg in PerSample)
foreach (var (name, data) in Datasets)
foreach (var (from, to, interval) in Windows)
{
var plan = BucketPlan.Create(At(from), At(to), TimeSpan.FromSeconds(interval));
var expected = SqlBackendAnswer(agg, plan, data);
var fake = Store(data, avevaBoundary);
var actual = (await Source(fake)
.ReadProcessedAsync("T", At(from), At(to), TimeSpan.FromSeconds(interval), agg, Ct)).Samples;
compared++;
if (fake.RawStreamsCompleted != fake.ReadRawCallCount)
failures.Add($"{agg} {name} {from}→{to}/{interval}s: a raw stream was abandoned mid-way");
var same = actual.Count == expected.Count && actual.Zip(expected).All(p =>
Equals(p.First.Value, p.Second.Value) && p.First.StatusCode == p.Second.StatusCode
&& p.First.SourceTimestampUtc == p.Second.SourceTimestampUtc);
if (!same)
failures.Add($"{agg} {name} {from}→{to}/{interval}s\n ours {Describe(actual)}\n sql {Describe(expected)}");
}
compared.ShouldBe(PerSample.Length * Datasets.Length * Windows.Length);
failures.ShouldBeEmpty(string.Join("\n", failures));
}
[Theory]
[InlineData(HistoryAggregateType.Average, 15d, 40d, 50d)]
[InlineData(HistoryAggregateType.Minimum, 10d, 40d, 50d)]
[InlineData(HistoryAggregateType.Maximum, 20d, 40d, 50d)]
[InlineData(HistoryAggregateType.Delta, 10d, 0d, 0d)]
[InlineData(HistoryAggregateType.Range, 10d, 0d, 0d)]
public async Task Numeric_aggregates_match_the_conformance_suites_hand_computed_literals(HistoryAggregateType agg,
double b0, double b1, double b2)
{
// The literals of ProcessedConformance.Numeric_aggregates_over_the_fixture (SqlServer + QuestDB), over tag A.
var s = (await Source(Store(A, avevaBoundary: true))
.ReadProcessedAsync("A", At(0), At(80), TimeSpan.FromSeconds(20), agg, Ct)).Samples;
s.Select(x => x.Value).ShouldBe([b0, b1, b2, null]);
var b2Status = agg == HistoryAggregateType.Average ? GoodCalc : GoodCalc | Partial;
s.Select(x => x.StatusCode).ShouldBe([GoodCalc, UncertainCalc, b2Status, NoDataCalc]);
s.Select(x => x.SourceTimestampUtc).ShouldBe([At(0), At(20), At(40), At(60)]); // interval STARTS (Part 13)
}
[Fact]
public async Task Count_is_a_Good_sample_count_as_int_and_zero_inside_the_data_range()
{
// #679 made Count BadAggregateNotSupported on Gateway; #681 serves it with the SQL backends' #680 semantics.
var s = (await Source(Store(Gappy)).ReadProcessedAsync("T", At(0), At(60), TimeSpan.FromSeconds(10),
HistoryAggregateType.Count, Ct)).Samples;
// [0,10): +1 G, +2 U ⇒ 1, Uncertain (data starts inside ⇒ Partial); [10,20) empty inside the range ⇒ 0 Good;
// [20,30): +25 Bad only ⇒ generic Bad; [30,40) 0; [40,50) +45 ⇒ 1; [50,60) 0 (+70 is later data).
s.Select(x => x.Value).ShouldBe([1, 0, null, 0, 1, 0]);
s.Select(x => x.StatusCode).ShouldBe([UncertainCalc | Partial, GoodCalc, 0x80000401u, GoodCalc, GoodCalc, GoodCalc]);
}
[Fact]
public async Task A_usable_string_sample_makes_a_numeric_aggregate_BadAggregateInvalidInputs()
{
var s = (await Source(Store(Mixed)).ReadProcessedAsync("T", At(0), At(40), TimeSpan.FromSeconds(10),
HistoryAggregateType.Average, Ct)).Samples;
s[1].StatusCode.ShouldBe(InvalidInputsCalc);
s[1].Value.ShouldBeNull();
}
[Fact]
public async Task ActualTime_carries_the_extreme_samples_own_time_ties_earliest()
{
var max = (await Source(Store(Ties)).ReadProcessedAsync("T", At(0), At(10), TimeSpan.Zero,
HistoryAggregateType.MaximumActualTime, Ct)).Samples.Single();
max.Value.ShouldBe(9d);
max.SourceTimestampUtc.ShouldBe(At(3));
var min = (await Source(Store(Ties)).ReadProcessedAsync("T", At(0), At(10), TimeSpan.Zero,
HistoryAggregateType.MinimumActualTime, Ct)).Samples.Single();
min.Value.ShouldBe(1d);
min.SourceTimestampUtc.ShouldBe(At(4));
}
[Fact]
public async Task A_reverse_read_is_anchored_at_its_later_time_and_returned_latest_first()
{
// #677: 50 → 0 at 20 s ⇒ (30,50], (10,30], (0,10] stamped 50, 30, 10.
var s = (await Source(Store(A)).ReadProcessedAsync("A", At(50), At(0), TimeSpan.FromSeconds(20),
HistoryAggregateType.Maximum, Ct)).Samples;
s.Select(x => x.SourceTimestampUtc).ShouldBe([At(50), At(30), At(10)]);
s.Select(x => x.Value).ShouldBe([50d, 40d, 20d]);
}
// ---- the raw request ---------------------------------------------------------------------------------------------
[Fact]
public async Task The_raw_stream_starts_one_millisecond_before_the_window_so_the_AVEVA_boundary_row_is_never_a_sample()
{
// Data before the window, nothing inside [0,10) but +5: the boundary row (value 1, stamped at the query start) must
// not be counted. Count over [0,10) is 1, not 2.
var fake = Store([D(-5, 1), D(5, 2)], avevaBoundary: true);
var s = (await Source(fake).ReadProcessedAsync("T", At(0), At(10), TimeSpan.Zero, HistoryAggregateType.Count, Ct))
.Samples.Single();
s.Value.ShouldBe(1);
fake.RawCalls[0].Start.ShouldBe(At(0) - TimeSpan.FromMilliseconds(1));
fake.RawCalls.ShouldNotContain(c => c.Start == DateTime.UnixEpoch); // the Good boundary row proved earlier data: no probe
}
[Fact]
public async Task Without_boundary_evidence_a_probe_asks_for_data_before_the_window()
{
// No boundary row (a historian that emits none): the second call probes [epoch, start − 1 tick] for 2 rows, and the
// sample at −5 it finds keeps the first interval free of the start-of-data Partial bit.
var fake = Store([D(-5, 1), D(5, 2), D(15, 3)]);
var s = (await Source(fake).ReadProcessedAsync("T", At(0), At(20), TimeSpan.FromSeconds(10),
HistoryAggregateType.Minimum, Ct)).Samples;
fake.RawCalls[1].ShouldBe((DateTime.UnixEpoch, At(0).AddTicks(-1), 2));
(s[0].StatusCode & Partial).ShouldBe(0u);
// Data that really starts inside the first interval: the probe finds nothing, the interval is Partial.
var late = Store([D(5, 2), D(15, 3)], avevaBoundary: true);
var l = (await Source(late).ReadProcessedAsync("T", At(0), At(20), TimeSpan.FromSeconds(10),
HistoryAggregateType.Minimum, Ct)).Samples;
late.RawCalls[1].Start.ShouldBe(DateTime.UnixEpoch); // the NULL boundary row (value 0, OPC quality 0) is no evidence
(l[0].StatusCode & Partial).ShouldBe(Partial);
}
[Fact]
public async Task The_raw_request_looks_ahead_past_the_window_only_when_the_data_range_matters()
{
var count = Store(A);
await Source(count).ReadProcessedAsync("A", At(0), At(80), TimeSpan.FromSeconds(20), HistoryAggregateType.Count, Ct);
// The window read ends AT the window end; probe + look-ahead are separate 2-row reads, all consumed to completion.
count.RawCalls.ShouldBe([
(At(0) - TimeSpan.FromMilliseconds(1), At(80), 1_000_000 + 2 + 16),
(DateTime.UnixEpoch, At(0).AddTicks(-1), 2),
(At(80), Now + TimeSpan.FromDays(1), 2),
]);
count.RawStreamsCompleted.ShouldBe(3);
var average = Store(A);
await Source(average).ReadProcessedAsync("A", At(0), At(80), TimeSpan.FromSeconds(20), HistoryAggregateType.Average, Ct);
average.RawCalls.Single().End.ShouldBe(At(80)); // Average needs no data range: no look-ahead, no probe
average.ReadAggregateCallCount.ShouldBe(0);
var unbounded = Store(A);
await Source(unbounded, maxRawRows: 0).ReadProcessedAsync("A", At(0), At(80), TimeSpan.FromSeconds(20),
HistoryAggregateType.Count, Ct);
unbounded.RawCalls[0].MaxValues.ShouldBe(int.MaxValue);
}
[Fact]
public async Task No_gateway_stream_is_ever_abandoned_and_later_data_comes_from_a_two_row_look_ahead()
{
// #681 review: breaking out of a gateway stream cancels the call, which the gateway treats as a connection fault
// (it evicts its shared historian session). 1000 samples after the window: the window read never sees them, the
// 2-row look-ahead reads two, and every stream runs to its end.
var data = Enumerable.Range(0, 1000).Select(i => D(100 + i, i)).Prepend(D(-5, 0)).Prepend(D(5, 1)).ToArray();
var fake = Store(data, avevaBoundary: true);
var s = (await Source(fake).ReadProcessedAsync("T", At(0), At(50), TimeSpan.FromSeconds(10),
HistoryAggregateType.Minimum, Ct)).Samples;
fake.RawStreamsCompleted.ShouldBe(fake.ReadRawCallCount);
fake.RawCalls.Last().ShouldBe((At(50), Now + TimeSpan.FromDays(1), 2));
fake.RawYieldedCount.ShouldBe(2 + 2); // window read: boundary + +5; look-ahead: boundary at +50 + first sample
(s[0].StatusCode & Partial).ShouldBe(0u); // data continues after the window: no end-of-data Partial
}
[Fact]
public async Task Rows_stamped_at_a_small_reads_query_start_or_end_are_not_samples()
{
// Data ends at +15: the look-ahead's NULL rows at its own start (+20, boundary) and end (now + 1 day, terminal)
// must not count as later data, so (10,20] / [10,20) holds the end of the data ⇒ Partial.
var fake = Store([D(-5, 1), D(5, 2), D(15, 3)], avevaBoundary: true, terminalRow: true);
var s = (await Source(fake).ReadProcessedAsync("T", At(0), At(20), TimeSpan.FromSeconds(10),
HistoryAggregateType.Maximum, Ct)).Samples;
(s[1].StatusCode & Partial).ShouldBe(Partial);
fake.RawCalls.Last().Start.ShouldBe(At(20));
}
[Fact]
public async Task A_window_over_the_raw_row_cap_fails_the_node_without_counting_a_backend_failure()
{
var data = Enumerable.Range(0, 10).Select(i => D(i, i)).ToArray(); // 10 rows in [0,10)
var ds = Source(Store(data), maxRawRows: 9);
var ex = await Should.ThrowAsync<HistoryReadLimitExceededException>(() =>
ds.ReadProcessedAsync("T", At(0), At(10), TimeSpan.Zero, HistoryAggregateType.Average, Ct));
ex.Limit.ShouldBe(9);
ds.GetHealthSnapshot().TotalFailures.ShouldBe(0);
// Exactly at the cap is served.
var ok = await Source(Store(data), maxRawRows: 10)
.ReadProcessedAsync("T", At(0), At(10), TimeSpan.Zero, HistoryAggregateType.Average, Ct);
ok.Samples.Single().Value.ShouldBe(4.5);
}
[Fact]
public async Task A_stream_that_exhausts_its_row_budget_before_the_window_is_never_answered_from_a_truncated_window()
{
// 40 rows inside the 1 ms margin before the window eat the whole budget (cap 5 ⇒ 23 rows): the window could not be
// read, so the node fails rather than reporting BadNoData.
var margin = Enumerable.Range(0, 40).Select(i => new StoredSample(At(0).AddTicks(-1 - i), HistorianValue.FromDouble(i), Good));
var data = margin.Append(D(5, 1)).ToArray();
await Should.ThrowAsync<HistoryReadLimitExceededException>(() => Source(Store(data), maxRawRows: 5)
.ReadProcessedAsync("T", At(0), At(10), TimeSpan.Zero, HistoryAggregateType.Count, Ct));
}
[Fact]
public async Task Past_the_cap_the_window_stream_is_drained_not_abandoned()
{
var data = Enumerable.Range(0, 10).Select(i => D(i, i)).ToArray();
var fake = Store(data);
await Should.ThrowAsync<HistoryReadLimitExceededException>(() => Source(fake, maxRawRows: 3)
.ReadProcessedAsync("T", At(0), At(10), TimeSpan.Zero, HistoryAggregateType.Average, Ct));
fake.RawStreamsCompleted.ShouldBe(1);
fake.RawYieldedCount.ShouldBe(10);
}
[Fact]
public async Task The_gateways_max_values_ceiling_rejection_is_a_request_limit_not_a_backend_fault()
{
// Historian:ReadLimits:MaxValuesCeiling (1,000,000) with AllowStreamingBeyondCeiling=false refuses the default
// cap + 18 budget with InvalidArgument; the client package wraps it in HistorianGatewayException.
var rpc = new RpcException(new Status(StatusCode.InvalidArgument,
"max_values exceeds the server ceiling of 1000000; read large ranges by time-slicing ..."));
var fake = new FakeHistorianGatewayClient { ThrowOnRead = new HistorianGatewayException(rpc.Status.Detail, rpc) };
var ds = Source(fake);
await Should.ThrowAsync<HistoryReadLimitExceededException>(() =>
ds.ReadProcessedAsync("T", At(0), At(10), TimeSpan.Zero, HistoryAggregateType.Count, Ct));
ds.GetHealthSnapshot().TotalFailures.ShouldBe(0);
// Any other InvalidArgument is still a backend fault.
var other = new RpcException(new Status(StatusCode.InvalidArgument, "start_time must be less than or equal to end_time."));
var ds2 = Source(new FakeHistorianGatewayClient { ThrowOnRead = new HistorianGatewayException(other.Status.Detail, other) });
await Should.ThrowAsync<HistorianGatewayException>(() =>
ds2.ReadProcessedAsync("T", At(0), At(10), TimeSpan.Zero, HistoryAggregateType.Count, Ct));
ds2.GetHealthSnapshot().TotalFailures.ShouldBe(1);
}
[Fact]
public async Task A_caller_cancellation_is_not_counted_as_a_backend_failure()
{
using var cts = new CancellationTokenSource();
await cts.CancelAsync();
var ds = Source(new FakeHistorianGatewayClient { ThrowOnRead = new OperationCanceledException(cts.Token) });
await Should.ThrowAsync<OperationCanceledException>(() =>
ds.ReadProcessedAsync("T", At(0), At(10), TimeSpan.Zero, HistoryAggregateType.Count, cts.Token));
ds.GetHealthSnapshot().TotalFailures.ShouldBe(0);
// A cancellation the caller did NOT ask for (e.g. the gateway cancelled the call) is still a backend fault.
var ds2 = Source(new FakeHistorianGatewayClient { ThrowOnRead = new OperationCanceledException() });
await Should.ThrowAsync<OperationCanceledException>(() =>
ds2.ReadProcessedAsync("T", At(0), At(10), TimeSpan.Zero, HistoryAggregateType.Count, Ct));
ds2.GetHealthSnapshot().TotalFailures.ShouldBe(1);
}
[Fact]
public async Task A_gateway_fault_on_the_raw_read_is_recorded_and_rethrown()
{
var fake = new FakeHistorianGatewayClient { ThrowOnRead = new InvalidOperationException("down") };
var ds = Source(fake);
await Should.ThrowAsync<InvalidOperationException>(() =>
ds.ReadProcessedAsync("T", At(0), At(10), TimeSpan.Zero, HistoryAggregateType.Count, Ct));
ds.GetHealthSnapshot().TotalFailures.ShouldBe(1);
}
[Fact]
public async Task An_empty_window_calls_nothing_and_returns_no_intervals()
{
var fake = Store(A);
var r = await Source(fake).ReadProcessedAsync("A", At(10), At(10), TimeSpan.Zero, HistoryAggregateType.Count, Ct);
r.Samples.ShouldBeEmpty();
fake.ReadRawCallCount.ShouldBe(0);
}
// ---- the served set ----------------------------------------------------------------------------------------------
[Fact]
public void The_served_set_is_the_per_sample_families_plus_the_two_native_modes()
{
GatewayHistorianDataSource.ServedAggregates.OrderBy(a => a).ShouldBe(
PerSample.Append(HistoryAggregateType.TimeAverage).Append(HistoryAggregateType.Total).OrderBy(a => a));
}
[Fact]
public async Task Every_member_outside_the_served_set_is_refused_without_calling_the_gateway_and_every_member_inside_is_served()
{
foreach (var agg in System.Enum.GetValues<HistoryAggregateType>())
{
var fake = Store(A);
var ds = Source(fake);
if (GatewayHistorianDataSource.ServedAggregates.Contains(agg))
{
await ds.ReadProcessedAsync("A", At(0), At(40), TimeSpan.FromSeconds(20), agg, Ct);
(fake.ReadRawCallCount + fake.ReadAggregateCallCount).ShouldBeGreaterThan(0, agg.ToString());
}
else
{
await Should.ThrowAsync<HistoryAggregateNotSupportedException>(() =>
ds.ReadProcessedAsync("A", At(0), At(40), TimeSpan.FromSeconds(20), agg, Ct));
(fake.ReadRawCallCount + fake.ReadAggregateCallCount).ShouldBe(0, agg.ToString());
ds.GetHealthSnapshot().TotalQueries.ShouldBe(0);
}
}
}
// ---- the accumulator against the independent oracle, sample order shuffled ---------------------------------------
[Fact]
public void The_bucket_statistics_do_not_depend_on_arrival_order()
{
var plan = BucketPlan.Create(At(-10), At(100), TimeSpan.FromSeconds(10));
foreach (var maximum in new[] { false, true })
{
var acc = new RawBucketAccumulator(maximum);
foreach (var s in Dense.Reverse())
if (plan.IndexOf(s.TimeUtc) is { } b) acc.Add(b, s);
var expected = InMemoryRows(plan, Dense, maximum).ToList();
var actual = acc.Rows().ToList();
actual.Count.ShouldBe(expected.Count);
foreach (var (a, e) in actual.Zip(expected))
{
(a.Bucket, a.Total, a.Usable, a.Good, a.Bad, a.NonNumeric).ShouldBe((e.Bucket, e.Total, e.Usable, e.Good, e.Bad, e.NonNumeric));
(a.Min, a.Max, a.Extreme, a.ExtremeTime).ShouldBe((e.Min, e.Max, e.Extreme, e.ExtremeTime));
a.Avg!.Value.ShouldBe(e.Avg!.Value, 1e-9);
(a.First, a.Last, a.FirstAny, a.LastAny).ShouldBe((e.First, e.Last, e.FirstAny, e.LastAny));
}
}
}
}
@@ -9,8 +9,6 @@ public sealed class AggregateModeMapperTests
{
[Theory]
[InlineData(HistoryAggregateType.TimeAverage, RetrievalMode.TimeWeightedAverage)]
[InlineData(HistoryAggregateType.Minimum, RetrievalMode.MinimumWithTime)]
[InlineData(HistoryAggregateType.Maximum, RetrievalMode.MaximumWithTime)]
[InlineData(HistoryAggregateType.Total, RetrievalMode.Integral)]
public void Maps_each_aggregate(HistoryAggregateType a, RetrievalMode expected)
=> Assert.Equal(expected, AggregateModeMapper.ToRetrievalMode(a));
@@ -18,11 +16,8 @@ public sealed class AggregateModeMapperTests
[Fact] // matrix guard: every member is either mapped or explicitly reported as not supported
public void Every_aggregate_member_is_mapped_or_reports_not_supported()
{
var original = new[]
{
HistoryAggregateType.Minimum, HistoryAggregateType.Maximum,
HistoryAggregateType.Total, HistoryAggregateType.TimeAverage,
};
var original = new[] { HistoryAggregateType.Total, HistoryAggregateType.TimeAverage };
Assert.Equal(original.OrderBy(a => a), AggregateModeMapper.NativeAggregates.OrderBy(a => a));
foreach (var a in Enum.GetValues<HistoryAggregateType>())
{
if (original.Contains(a))
@@ -32,26 +27,33 @@ public sealed class AggregateModeMapperTests
}
}
[Theory] // #679: AVEVA has no arithmetic mean and no sample count (Counter is a rollover increase)
[Theory] // #679: AVEVA has no arithmetic mean and no sample count; #681: the per-sample aggregates are raw-evaluated, not native
[InlineData(HistoryAggregateType.Average)]
[InlineData(HistoryAggregateType.Count)]
public void Average_and_Count_are_not_supported(HistoryAggregateType a)
[InlineData(HistoryAggregateType.Minimum)]
[InlineData(HistoryAggregateType.Maximum)]
public void Per_sample_aggregates_have_no_native_mode(HistoryAggregateType a)
=> Assert.Throws<HistoryAggregateNotSupportedException>(() => AggregateModeMapper.ToRetrievalMode(a));
[Fact] // #682: the served (= advertised) set is exactly what the mapper maps, and the data source returns it
[Fact] // #682/#681: the served (= advertised) set is the native members plus the raw-evaluated per-sample families,
// and the data source returns exactly that static
public void Served_is_exactly_the_mapped_members_and_the_data_source_declares_it()
{
Assert.Equal(
new[] { HistoryAggregateType.Minimum, HistoryAggregateType.Maximum, HistoryAggregateType.Total, HistoryAggregateType.TimeAverage }
.OrderBy(a => a),
AggregateModeMapper.Served.OrderBy(a => a));
foreach (var a in Enum.GetValues<HistoryAggregateType>().Where(a => !AggregateModeMapper.Served.Contains(a)))
Assert.Throws<HistoryAggregateNotSupportedException>(() => AggregateModeMapper.ToRetrievalMode(a));
foreach (var a in Enum.GetValues<HistoryAggregateType>())
{
var native = AggregateModeMapper.NativeAggregates.Contains(a);
var perSample = ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common.AggregateFamilies.FamilyOf(a) is
ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common.AggregateFamily.Simple
or ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common.AggregateFamily.StartEnd
or ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common.AggregateFamily.ActualTime;
Assert.Equal(native || perSample, GatewayHistorianDataSource.ServedAggregates.Contains(a));
if (!native) Assert.Throws<HistoryAggregateNotSupportedException>(() => AggregateModeMapper.ToRetrievalMode(a));
}
Assert.Equal(12, GatewayHistorianDataSource.ServedAggregates.Count);
var source = new GatewayHistorianDataSource(
new FakeHistorianGatewayClient(), Microsoft.Extensions.Logging.Abstractions.NullLogger<GatewayHistorianDataSource>.Instance);
Assert.Same(GatewayHistorianDataSource.ServedAggregates, source.SupportedAggregates);
Assert.Same(AggregateModeMapper.Served, GatewayHistorianDataSource.ServedAggregates);
}
[Fact]
@@ -37,7 +37,7 @@ public sealed class SampleMapperTests
{
var a = new HistorianAggregateSample { Tag = "T", /* Value unset, no Good quality */ EndTime = Ts(2026, 1, 1, 0, 0, 0) };
var snap = SampleMapper.ToAggregateSnapshot(a);
Assert.Equal(0x809B0000u, snap.StatusCode); // BadNoData
Assert.Equal(0x809B0401u, snap.StatusCode); // BadNoData + Calculated (#681)
Assert.Null(snap.Value);
}
@@ -46,10 +46,20 @@ public sealed class SampleMapperTests
{
var a = new HistorianAggregateSample { Tag = "T", Value = 42.0, OpcQuality = 192, EndTime = Ts(2026, 1, 1, 0, 0, 0) };
var snap = SampleMapper.ToAggregateSnapshot(a);
Assert.Equal(0x00000000u, snap.StatusCode); // Good
Assert.Equal(0x00000401u, snap.StatusCode); // Good + Calculated (#681)
Assert.Equal(42.0, Assert.IsType<double>(snap.Value));
}
[Fact] // #681: Part 13 stamps a processed value with its interval START (it used to be the end)
public void Aggregate_bucket_is_stamped_with_its_interval_start()
{
var a = new HistorianAggregateSample
{
Tag = "T", Value = 1.0, OpcQuality = 192, StartTime = Ts(2026, 1, 1, 0, 0, 0), EndTime = Ts(2026, 1, 1, 0, 1, 0),
};
Assert.Equal(new DateTime(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc), SampleMapper.ToAggregateSnapshot(a).SourceTimestampUtc);
}
// Ts(...) builds a Google.Protobuf.WellKnownTypes.Timestamp from UTC parts.
private static Timestamp Ts(int y, int mo, int d, int h, int mi, int s)
=> Timestamp.FromDateTime(new DateTime(y, mo, d, h, mi, s, DateTimeKind.Utc));