fix(historian): #687 re-review — a row at/before the Unix epoch is never a stored sample
With StartTime at or before the epoch, RequestStartFor has no margin left, so AVEVA's synthetic no-data/boundary row is stamped exactly at StartTime and was kept: raw reads returned a fabricated Bad sample at 1970-01-01 and the per-sample aggregates put it in bucket 0 (Start/End fabricated, Min/Max status Uncertain). IsWindowSample and the aggregate loop now drop any row with t <= ProbeFloor (RawAggregateReader.IsAfterFloor), matching the #686 walk-back's assumption.
This commit is contained in:
+3
-1
@@ -686,7 +686,9 @@ data.
|
||||
the gateway is asked from **1 ms before** the window (never before the Unix epoch, the start of the historian's time encoding),
|
||||
so the boundary row lands before it, and only rows `RawAggregateReader.IsWindowSample` are returned. The decision is by
|
||||
**timestamp**, not by value or quality — a stored NULL sample looks exactly like the no-data boundary row on the wire (value 0,
|
||||
Bad), and a stored NULL inside the window is still returned (Bad), as it is a stored sample.
|
||||
Bad), and a stored NULL inside the window is still returned (Bad), as it is a stored sample. A row stamped **at or before the
|
||||
epoch** is never a stored sample (AVEVA stores nothing there — the #686 walk-back's floor too): a window starting at or before
|
||||
the epoch gets no margin, so AVEVA's no-data row lands exactly at its start; raw reads and the per-sample aggregates both drop it.
|
||||
- **The forward end is exclusive** (`[StartTime, EndTime)`, the `IHistorianDataSource` contract and what the SQL backends return;
|
||||
the gateway's own `ReadRaw` end is inclusive, so a sample stored exactly at EndTime used to be returned). A point read
|
||||
(`StartTime == EndTime` — the server's tie-cluster over-fetch) still returns the samples at that instant. The #686 reverse
|
||||
|
||||
@@ -104,9 +104,18 @@ internal static class RawAggregateReader
|
||||
/// <param name="windowEndUtc">The window's end: exclusive, or inclusive when it equals the start.</param>
|
||||
/// <returns>Whether the row belongs to the window.</returns>
|
||||
public static bool IsWindowSample(HistorianSample raw, DateTime windowStartUtc, DateTime windowEndUtc) =>
|
||||
raw.Timestamp is { } ts && ts.ToDateTime() is var t && t >= windowStartUtc
|
||||
raw.Timestamp is { } ts && ts.ToDateTime() is var t && IsAfterFloor(t) && t >= windowStartUtc
|
||||
&& (t < windowEndUtc || (t == windowEndUtc && windowStartUtc == windowEndUtc));
|
||||
|
||||
/// <summary>
|
||||
/// A row stamped at or before <see cref="ProbeFloor"/> (the Unix epoch) is never a stored sample: AVEVA stores nothing
|
||||
/// there (the server's #686 walk-back assumes the same), and a window starting at or before the epoch gets no 1 ms
|
||||
/// margin (<see cref="RequestStartFor"/>), so AVEVA's synthetic boundary / no-data row lands exactly at its start.
|
||||
/// </summary>
|
||||
/// <param name="t">The row's timestamp.</param>
|
||||
/// <returns>Whether the row can be a stored sample.</returns>
|
||||
internal static bool IsAfterFloor(DateTime t) => t > ProbeFloor;
|
||||
|
||||
/// <summary>True for the aggregates this reader serves: the per-sample families of <see cref="AggregateFamilies"/>.</summary>
|
||||
/// <param name="aggregate">Requested aggregate.</param>
|
||||
/// <returns>Whether the aggregate is evaluated from raw samples.</returns>
|
||||
@@ -184,6 +193,7 @@ internal static class RawAggregateReader
|
||||
if (overCap || raw.Timestamp is null) continue;
|
||||
var s = ToStored(raw);
|
||||
var t = s.TimeUtc;
|
||||
if (!IsAfterFloor(t)) continue; // the boundary row of a window starting at/before the epoch, never a sample
|
||||
if (plan.IndexOf(t) is { } bucket)
|
||||
{
|
||||
if (maxWindowRows > 0 && ++windowRows > maxWindowRows)
|
||||
|
||||
+17
@@ -473,6 +473,23 @@ public sealed class GatewayRawAggregateTests
|
||||
ds2.GetHealthSnapshot().TotalFailures.ShouldBe(1);
|
||||
}
|
||||
|
||||
[Theory] // a window starting AT the epoch gets no 1 ms margin, so AVEVA's no-data row is stamped at the window start:
|
||||
// it must not become a stored Bad sample of bucket 0
|
||||
[InlineData(HistoryAggregateType.Start)]
|
||||
[InlineData(HistoryAggregateType.Count)]
|
||||
[InlineData(HistoryAggregateType.Minimum)]
|
||||
public async Task A_window_starting_at_the_epoch_does_not_evaluate_the_fabricated_row(HistoryAggregateType agg)
|
||||
{
|
||||
var epoch = DateTime.UnixEpoch;
|
||||
var data = new[] { new StoredSample(epoch.AddSeconds(5), HistorianValue.FromDouble(7), Good) };
|
||||
var fake = Store(data, avevaBoundary: true);
|
||||
var s = (await Source(fake).ReadProcessedAsync("T", epoch, epoch.AddSeconds(20), TimeSpan.FromSeconds(20), agg, Ct))
|
||||
.Samples.Single();
|
||||
var expected = SqlBackendAnswer(agg, BucketPlan.Create(epoch, epoch.AddSeconds(20), TimeSpan.FromSeconds(20)), data).Single();
|
||||
(s.Value, s.StatusCode).ShouldBe((expected.Value, expected.StatusCode));
|
||||
fake.RawCalls[0].Start.ShouldBe(epoch); // the request start cannot go before the epoch
|
||||
}
|
||||
|
||||
[Fact] // #688: the default cap's budget fits the gateway's default ceiling, so a strict gateway serves the default config
|
||||
public async Task The_default_caps_row_budget_does_not_exceed_the_gateways_default_ceiling()
|
||||
{
|
||||
|
||||
+12
@@ -292,4 +292,16 @@ public sealed class GatewayRawBoundaryRowTests
|
||||
RawAggregateReader.RequestStartFor(start).ShouldBe(DateTime.UnixEpoch);
|
||||
RawAggregateReader.RequestStartFor(At(0)).ShouldBe(At(0) - RawAggregateReader.BoundaryMargin);
|
||||
}
|
||||
|
||||
[Theory] // with no margin left before the epoch, AVEVA's no-data row lands AT StartTime: a row at/before the epoch is
|
||||
// never a stored sample (AVEVA stores nothing there — the #686 walk-back's floor too)
|
||||
[InlineData(0L)]
|
||||
[InlineData(-10_000_000L)]
|
||||
public async Task A_read_starting_at_or_before_the_epoch_does_not_return_the_fabricated_row(long startTicksFromEpoch)
|
||||
{
|
||||
var epoch = DateTime.UnixEpoch;
|
||||
var fake = Aveva([(epoch.AddSeconds(10), 7d, false)]);
|
||||
var r = await Source(fake).ReadRawAsync("T", epoch.AddTicks(startTicksFromEpoch), epoch.AddSeconds(60), 5, Ct);
|
||||
r.Samples.Select(s => s.SourceTimestampUtc).ShouldBe(new DateTime?[] { epoch.AddSeconds(10) });
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user