dohertj2/lmxopcua
1
0
Fork 0
Code Issues 14 Pull Requests Actions Packages Projects Releases Wiki Activity

fix(historian): #693 review — bisecting walk-back, small jump reads, read cap, DateTime/NaN hints

Browse Source 
H1: the walk-back reset its width after a full segment and discarded a full
whole-remainder read, so a dense tag that stopped reporting weeks ago gave up
after 65 reads (BadTooManyOperations). It now gallops (x4, whole-remainder read
after two empties) and then BISECTS once any stored row is known below — a full
segment's (incl. a full whole-remainder read's) last row, or an earlier window
read's last row: (mid, hi] empty => hi = mid; full => known = its last row;
complete with rows => done; one density-aimed segment after a full read until an
aimed segment comes back empty; below 2 ms the whole stretch from the known row
(never a 100 ns step). Stopped 1 h..365 d with 5 y history: 6-18 reads at
1/min, 21-25 at 1 Hz, 25-31 at 10 Hz.
M1: a jump read asks for 64 rows (1,000 times over 10 y: ~66k rows, was ~1M).
L1: per-request read cap 64 + 4 x distinct times with a clear message; a strict
ceiling below 4 rows is refused naming the ceiling.
L2: DateTime hints are stepped. L3: under an integer hint a NaN/inf or
out-of-Int64-range value is unusable, never saturated Good.
L5: conformance Int64 AtTime asserts the type. L4: docs name the backend
differences. Nit: the window read end is clamped to now + 1 day.

Reviewer harnesses (Scratch693Fuzz/Cost, not committed): 0 differential diffs
over 2,400 cases.
This commit is contained in:
Joseph Doherty
2026-10-13 01:09:54 -04:00
parent 0d249f3ee9
commit 68a1f31397
4 changed files with 379 additions and 96 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/Drivers/Historian.Conformance/AtTimeConformance.cs
+1
View File
@@ -139,6 +139,7 @@ public abstract class AtTimeConformance : ConformanceBase
(At(10), HistorianValue.FromInt64(15), Good));
var s = (await B.CreateDataSource().ReadAtTimeAsync("I", [At(5), At(2)], Ct)).Samples;
s[0].Value.ShouldBe(13L); // 12.5 ⇒ 13 (away from zero)
s[0].Value.ShouldBeOfType<long>(); // ShouldBe compares numbers across types — the type needs its own check (#693)
s[1].Value.ShouldBe(11L); // 11.0
s[0].StatusCode.ShouldBe(GoodInterpolated);
}
tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests/GatewayAtTimeTests.cs
+181 -1
View File
@@ -409,7 +409,7 @@ public sealed class GatewayAtTimeTests
var s = (await AtTime(fake, T0.AddMinutes(-5))).Single();
s.Value.ShouldBe(599_999d);
s.StatusCode.ShouldBe(GoodInterpolated); // stepped from the last sample (before now)
fake.ReadRawCallCount.ShouldBe(7); // window, 1m, 4m, remainder (truncated), 16m (truncated), 256 ms, probe
fake.ReadRawCallCount.ShouldBe(6); // window, 1m, 4m, remainder (full: seeds the bisection), one aimed segment, probe
store.RowsStreamed.ShouldBeLessThan(10_000);
}
@@ -446,6 +446,186 @@ public sealed class GatewayAtTimeTests
ds.GetHealthSnapshot().TotalFailures.ShouldBe(1);
}
// ---- long histories (review H1/M1/L1): a generated store, too large to materialise -----------------------------
/// <summary>
/// A regular series (value = sample index) over <c>[from, to]</c> every <paramref name="step"/>, optionally with a
/// hole <c>(gapFrom, gapTo)</c>, served in the AVEVA wire shape (boundary row at the query start, ms bounds only,
/// <c>max_values &gt; 0</c>, <c>start &lt;= end</c>, nothing past now + 2 days).
/// </summary>
private sealed class SeriesStore(DateTime from, DateTime to, TimeSpan step, DateTime? gapFrom = null, DateTime? gapTo = null)
{
public long RowsStreamed;
private bool InGap(DateTime t) => gapFrom is { } a && gapTo is { } b && t > a && t < b;
private DateTime? FirstAtOrAfter(DateTime t)
{
if (t > to) return null;
var k = t <= from ? 0 : ((t - from).Ticks + step.Ticks - 1) / step.Ticks;
var r = from.AddTicks(k * step.Ticks);
if (InGap(r)) return FirstAtOrAfter(gapTo!.Value);
return r <= to ? r : null;
}
private DateTime? LastBefore(DateTime t)
{
if (t <= from) return null;
var k = ((Min(t, to.AddTicks(1)) - from).Ticks - 1) / step.Ticks;
var r = from.AddTicks(k * step.Ticks);
return InGap(r) ? LastBefore(gapFrom!.Value.AddTicks(1)) : r;
}
private static DateTime Min(DateTime a, DateTime b) => a <= b ? a : b;
/// <summary>The stored row at <paramref name="t"/>, stamped <paramref name="at"/> (the boundary row) when given.</summary>
private HistorianSample Row(DateTime t, uint quality = 0, DateTime? at = null) => new()
{
Tag = "T", Timestamp = Timestamp.FromDateTime(at ?? t), NumericValue = (t - from).Ticks / step.Ticks, OpcQuality = 192,
Quality = quality,
};
public FakeHistorianGatewayClient Client(int? ceiling = null) => new() { RawSource = Serve, MaxValuesCeiling = ceiling };
private IEnumerable<HistorianSample> Serve(DateTime start, DateTime end, int max)
{
if (max <= 0 || end < start || end > Now.AddDays(2)
|| start.Ticks % TimeSpan.TicksPerMillisecond != 0 || end.Ticks % TimeSpan.TicksPerMillisecond != 0)
throw new Grpc.Core.RpcException(new Grpc.Core.Status(Grpc.Core.StatusCode.InvalidArgument, "refused"));
var rows = new List<HistorianSample>();
if (FirstAtOrAfter(start) != start)
rows.Add(LastBefore(start) is { } p
? Row(p, 133, start)
: new HistorianSample { Tag = "T", Timestamp = Timestamp.FromDateTime(start), OpcQuality = 0, Quality = 1, NumericValue = 0 });
for (var t = FirstAtOrAfter(start); t is { } u && u <= end && rows.Count < max; t = FirstAtOrAfter(u.AddTicks(1)))
rows.Add(Row(u));
RowsStreamed += rows.Count;
return rows;
}
}
[Theory] // H1: a dense tag that stopped reporting — the walk crosses the silence by bisection, not by regrowing segments
[InlineData(1.0, 30)]
[InlineData(1.0, 365)]
[InlineData(10.0, 30)]
[InlineData(10.0, 365)]
public async Task A_dense_tag_that_stopped_long_ago_is_found_in_a_bounded_number_of_reads(double hz, int stoppedDaysAgo)
{
var step = TimeSpan.FromTicks((long)(TimeSpan.TicksPerSecond / hz));
var end = Now.AddDays(-stoppedDaysAgo);
var fake = new SeriesStore(end.AddYears(-5), end, step).Client();
var s = (await AtTime(fake, Now.AddMinutes(-1))).Single();
s.Value.ShouldBe((double)((end - end.AddYears(-5)).Ticks / step.Ticks)); // the last sample, held
s.StatusCode.ShouldBe(GoodInterpolated);
fake.ReadRawCallCount.ShouldBeLessThan(40);
}
[Fact] // H1: a time inside a month-long hole in the middle of a dense history
public async Task A_time_inside_a_mid_history_gap_interpolates_across_it()
{
var from = Now.AddYears(-2);
var gapFrom = Now.AddDays(-200);
var gapTo = gapFrom.AddDays(30);
var fake = new SeriesStore(from, Now, TimeSpan.FromSeconds(1), gapFrom, gapTo).Client();
var t = gapFrom.AddDays(15);
var s = (await AtTime(fake, t)).Single();
var before = (gapFrom - from).Ticks / TimeSpan.TicksPerSecond;
var after = (gapTo - from).Ticks / TimeSpan.TicksPerSecond;
((double)s.Value!).ShouldBe(before + (after - before) * 0.5, 1e-6);
s.StatusCode.ShouldBe(GoodInterpolated);
fake.ReadRawCallCount.ShouldBeLessThan(40);
}
[Fact] // H1: a jump to a time in a gap — the walk starts from the earlier read's last row, far below
public async Task A_jump_into_a_gap_bisects_up_from_the_earlier_reads_last_row()
{
var from = Now.AddDays(-30);
var gapFrom = from.AddDays(2);
var gapTo = gapFrom.AddDays(10);
var fake = new SeriesStore(from, Now, TimeSpan.FromSeconds(1), gapFrom, gapTo).Client();
var t0 = from.AddSeconds(10.5);
var t1 = gapFrom.AddDays(5);
var s = await AtTime(fake, t0, t1);
s[0].Value.ShouldBe(10.5d);
var before = (gapFrom - from).Ticks / TimeSpan.TicksPerSecond;
var after = (gapTo - from).Ticks / TimeSpan.TicksPerSecond;
((double)s[1].Value!).ShouldBe(before + (after - before) * 0.5, 1e-6);
fake.ReadRawCallCount.ShouldBeLessThan(40);
}
[Fact] // M1: widely spread times cost one SMALL read each, not one 1,024-row read each
public async Task A_thousand_times_over_ten_years_cost_one_small_read_each()
{
var store = new SeriesStore(Now.AddYears(-10), Now, TimeSpan.FromSeconds(1));
var fake = store.Client();
var rnd = new Random(1);
var times = Enumerable.Range(0, 1000).Select(_ => Now.AddYears(-9).AddTicks((long)(rnd.NextDouble() * TimeSpan.TicksPerDay * 3000))).ToArray();
var s = await AtTime(fake, times);
s.ShouldAllBe(x => x.StatusCode == GoodInterpolated);
fake.ReadRawCallCount.ShouldBeLessThanOrEqualTo(1010);
store.RowsStreamed.ShouldBeLessThan(80_000); // ~64 rows a read (was ~1,000,000)
}
[Fact] // L1: a strict-gateway ceiling too small to work with is refused with a message naming it
public async Task A_ceiling_below_a_usable_read_is_refused_naming_the_ceiling()
{
var fake = new SeriesStore(Now.AddDays(-1), Now, TimeSpan.FromSeconds(1)).Client(ceiling: 2);
var ex = await Should.ThrowAsync<HistoryReadLimitExceededException>(
() => Source(fake).ReadAtTimeAsync("T", [Now.AddHours(-1)], Ct));
ex.Message.ShouldContain("row ceiling (2");
}
[Fact] // L1: a small ceiling is served (paging), within the read cap
public async Task A_small_ceiling_is_served_within_the_read_cap()
{
var fake = new SeriesStore(Now.AddYears(-1), Now, TimeSpan.FromSeconds(1)).Client(ceiling: 10);
var times = Enumerable.Range(0, 50).Select(i => Now.AddDays(-300 + i * 5).AddSeconds(0.5)).ToArray();
var s = await AtTime(fake, times);
s.ShouldAllBe(x => x.StatusCode == GoodInterpolated);
fake.ReadRawCallCount.ShouldBeLessThanOrEqualTo(GatewayAtTimeReader.BaseReads + GatewayAtTimeReader.ReadsPerTime * 50);
}
[Fact] // L1: the request's read cap stops a walk with a clear error, never a hang (here a cap of 3 vs. a 5-read walk)
public async Task The_read_cap_stops_a_request_with_a_clear_error()
{
var fake = new AvevaStore([(T0.AddDays(-400), 42, false), (T0.AddDays(-400).AddSeconds(10), 43, false)]).Client();
var ex = await Should.ThrowAsync<HistoryReadLimitExceededException>(() => GatewayAtTimeReader.ReadAsync(
fake, new GatewayRowCeiling(), "T", [T0], Now, Ct, maxReads: 3));
ex.Message.ShouldContain("needed more than 3 raw reads");
fake.ReadRawCallCount.ShouldBe(3);
fake.RawStreamsCompleted.ShouldBe(3);
}
[Fact] // the default cap scales with the request: 64 + 4 per distinct time
public async Task The_default_read_cap_is_64_plus_4_per_distinct_time()
{
(GatewayAtTimeReader.BaseReads, GatewayAtTimeReader.ReadsPerTime).ShouldBe((64, 4));
await Task.CompletedTask;
}
// ---- type hint edge cases (review L2/L3) ------------------------------------------------------------------------
[Fact]
public async Task A_DateTime_node_is_stepped()
{
var fake = new AvevaStore([(At(0), 0, false), (At(10), 10, false)]).Client();
var s = (await AtTimeTyped(fake, TypeCode.DateTime, At(5))).Single();
s.Value.ShouldBe(0d);
s.StatusCode.ShouldBe(GoodInterpolated);
}
[Theory]
[InlineData(double.PositiveInfinity)]
[InlineData(double.NaN)]
[InlineData(1e30)]
public async Task Under_an_integer_hint_a_non_integer_next_value_is_unusable_not_saturated(double next)
{
var fake = new AvevaStore([(At(0), 100, false), (At(10), next, false)]).Client();
var s = (await AtTimeTyped(fake, TypeCode.Int32, At(5))).Single();
s.Value.ShouldBeOfType<int>().ShouldBe(100); // held before an unusable neighbour
s.StatusCode.ShouldBe(UncertainInterpolated);
}
// ---- kill switch -----------------------------------------------------------------------------------------------
[Fact]