fix(historian): #687 re-review — a row at/before the Unix epoch is never a stored sample

With StartTime at or before the epoch, RequestStartFor has no margin left, so
AVEVA's synthetic no-data/boundary row is stamped exactly at StartTime and was
kept: raw reads returned a fabricated Bad sample at 1970-01-01 and the per-sample
aggregates put it in bucket 0 (Start/End fabricated, Min/Max status Uncertain).
IsWindowSample and the aggregate loop now drop any row with t <= ProbeFloor
(RawAggregateReader.IsAfterFloor), matching the #686 walk-back's assumption.
This commit is contained in:
Joseph Doherty
2026-10-12 15:47:39 -04:00
parent 8114f13a28
commit a3ee917fb4
4 changed files with 43 additions and 2 deletions
+3 -1
View File
@@ -686,7 +686,9 @@ data.
the gateway is asked from **1 ms before** the window (never before the Unix epoch, the start of the historian's time encoding),
so the boundary row lands before it, and only rows `RawAggregateReader.IsWindowSample` are returned. The decision is by
**timestamp**, not by value or quality — a stored NULL sample looks exactly like the no-data boundary row on the wire (value 0,
Bad), and a stored NULL inside the window is still returned (Bad), as it is a stored sample.
Bad), and a stored NULL inside the window is still returned (Bad), as it is a stored sample. A row stamped **at or before the
epoch** is never a stored sample (AVEVA stores nothing there — the #686 walk-back's floor too): a window starting at or before
the epoch gets no margin, so AVEVA's no-data row lands exactly at its start; raw reads and the per-sample aggregates both drop it.
- **The forward end is exclusive** (`[StartTime, EndTime)`, the `IHistorianDataSource` contract and what the SQL backends return;
the gateway's own `ReadRaw` end is inclusive, so a sample stored exactly at EndTime used to be returned). A point read
(`StartTime == EndTime` — the server's tie-cluster over-fetch) still returns the samples at that instant. The #686 reverse
@@ -104,9 +104,18 @@ internal static class RawAggregateReader
/// <param name="windowEndUtc">The window's end: exclusive, or inclusive when it equals the start.</param>
/// <returns>Whether the row belongs to the window.</returns>
public static bool IsWindowSample(HistorianSample raw, DateTime windowStartUtc, DateTime windowEndUtc) =>
raw.Timestamp is { } ts && ts.ToDateTime() is var t && t >= windowStartUtc
raw.Timestamp is { } ts && ts.ToDateTime() is var t && IsAfterFloor(t) && t >= windowStartUtc
&& (t < windowEndUtc || (t == windowEndUtc && windowStartUtc == windowEndUtc));
/// <summary>
/// A row stamped at or before <see cref="ProbeFloor"/> (the Unix epoch) is never a stored sample: AVEVA stores nothing
/// there (the server's #686 walk-back assumes the same), and a window starting at or before the epoch gets no 1 ms
/// margin (<see cref="RequestStartFor"/>), so AVEVA's synthetic boundary / no-data row lands exactly at its start.
/// </summary>
/// <param name="t">The row's timestamp.</param>
/// <returns>Whether the row can be a stored sample.</returns>
internal static bool IsAfterFloor(DateTime t) => t > ProbeFloor;
/// <summary>True for the aggregates this reader serves: the per-sample families of <see cref="AggregateFamilies"/>.</summary>
/// <param name="aggregate">Requested aggregate.</param>
/// <returns>Whether the aggregate is evaluated from raw samples.</returns>
@@ -184,6 +193,7 @@ internal static class RawAggregateReader
if (overCap || raw.Timestamp is null) continue;
var s = ToStored(raw);
var t = s.TimeUtc;
if (!IsAfterFloor(t)) continue; // the boundary row of a window starting at/before the epoch, never a sample
if (plan.IndexOf(t) is { } bucket)
{
if (maxWindowRows > 0 && ++windowRows > maxWindowRows)
@@ -473,6 +473,23 @@ public sealed class GatewayRawAggregateTests
ds2.GetHealthSnapshot().TotalFailures.ShouldBe(1);
}
[Theory] // a window starting AT the epoch gets no 1 ms margin, so AVEVA's no-data row is stamped at the window start:
// it must not become a stored Bad sample of bucket 0
[InlineData(HistoryAggregateType.Start)]
[InlineData(HistoryAggregateType.Count)]
[InlineData(HistoryAggregateType.Minimum)]
public async Task A_window_starting_at_the_epoch_does_not_evaluate_the_fabricated_row(HistoryAggregateType agg)
{
var epoch = DateTime.UnixEpoch;
var data = new[] { new StoredSample(epoch.AddSeconds(5), HistorianValue.FromDouble(7), Good) };
var fake = Store(data, avevaBoundary: true);
var s = (await Source(fake).ReadProcessedAsync("T", epoch, epoch.AddSeconds(20), TimeSpan.FromSeconds(20), agg, Ct))
.Samples.Single();
var expected = SqlBackendAnswer(agg, BucketPlan.Create(epoch, epoch.AddSeconds(20), TimeSpan.FromSeconds(20)), data).Single();
(s.Value, s.StatusCode).ShouldBe((expected.Value, expected.StatusCode));
fake.RawCalls[0].Start.ShouldBe(epoch); // the request start cannot go before the epoch
}
[Fact] // #688: the default cap's budget fits the gateway's default ceiling, so a strict gateway serves the default config
public async Task The_default_caps_row_budget_does_not_exceed_the_gateways_default_ceiling()
{
@@ -292,4 +292,16 @@ public sealed class GatewayRawBoundaryRowTests
RawAggregateReader.RequestStartFor(start).ShouldBe(DateTime.UnixEpoch);
RawAggregateReader.RequestStartFor(At(0)).ShouldBe(At(0) - RawAggregateReader.BoundaryMargin);
}
[Theory] // with no margin left before the epoch, AVEVA's no-data row lands AT StartTime: a row at/before the epoch is
// never a stored sample (AVEVA stores nothing there — the #686 walk-back's floor too)
[InlineData(0L)]
[InlineData(-10_000_000L)]
public async Task A_read_starting_at_or_before_the_epoch_does_not_return_the_fabricated_row(long startTicksFromEpoch)
{
var epoch = DateTime.UnixEpoch;
var fake = Aveva([(epoch.AddSeconds(10), 7d, false)]);
var r = await Source(fake).ReadRawAsync("T", epoch.AddTicks(startTicksFromEpoch), epoch.AddSeconds(60), 5, Ct);
r.Samples.Select(s => s.SourceTimestampUtc).ShouldBe(new DateTime?[] { epoch.AddSeconds(10) });
}
}