Joseph Doherty
68a1f31397
H1: the walk-back reset its width after a full segment and discarded a full whole-remainder read, so a dense tag that stopped reporting weeks ago gave up after 65 reads (BadTooManyOperations). It now gallops (x4, whole-remainder read after two empties) and then BISECTS once any stored row is known below — a full segment's (incl. a full whole-remainder read's) last row, or an earlier window read's last row: (mid, hi] empty => hi = mid; full => known = its last row; complete with rows => done; one density-aimed segment after a full read until an aimed segment comes back empty; below 2 ms the whole stretch from the known row (never a 100 ns step). Stopped 1 h..365 d with 5 y history: 6-18 reads at 1/min, 21-25 at 1 Hz, 25-31 at 10 Hz. M1: a jump read asks for 64 rows (1,000 times over 10 y: ~66k rows, was ~1M). L1: per-request read cap 64 + 4 x distinct times with a clear message; a strict ceiling below 4 rows is refused naming the ceiling. L2: DateTime hints are stepped. L3: under an integer hint a NaN/inf or out-of-Int64-range value is unusable, never saturated Good. L5: conformance Int64 AtTime asserts the type. L4: docs name the backend differences. Nit: the window read end is clamped to now + 1 day. Reviewer harnesses (Scratch693Fuzz/Cost, not committed): 0 differential diffs over 2,400 cases.
643 lines
31 KiB
C#
643 lines
31 KiB
C#
using Google.Protobuf.WellKnownTypes;
|
|
using Microsoft.Extensions.Logging.Abstractions;
|
|
using Shouldly;
|
|
using Xunit;
|
|
using ZB.MOM.WW.HistorianGateway.Contracts.Grpc;
|
|
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
|
|
|
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests;
|
|
|
|
/// <summary>
|
|
/// Gitea #693: Gateway HistoryRead AtTime is served from RAW reads and evaluated with the SQL backends' shared
|
|
/// <c>AtTimeEvaluator</c> (Part 11 §6.5.4) — never AVEVA's own interpolated answer, which the live gate caught returning
|
|
/// the previous value as Good at a stored NULL's own time, null + Bad at a Good sample right after a NULL, a drifted value
|
|
/// at an exact stored sample, and no Interpolated bit. The fake is AVEVA-shaped (the real wire shape): ReadRaw over
|
|
/// <c>[start, end]</c> ascending, capped, led by the synthetic boundary row at the query start (the value in effect there,
|
|
/// <c>Quality = 133</c>, or a NULL row), NULL rows as <c>NumericValue = 0</c> + <c>q=1 opc=0</c>, millisecond timestamps;
|
|
/// and it REJECTS what the real gateway rejects (<c>max_values <= 0</c>, <c>end < start</c>).
|
|
/// </summary>
|
|
public sealed class GatewayAtTimeTests
|
|
{
|
|
private static CancellationToken Ct => TestContext.Current.CancellationToken;
|
|
|
|
private const uint Good = 0u;
|
|
private const uint Bad = 0x80000000u;
|
|
private const uint BadNoData = 0x809B0000u;
|
|
private const uint GoodInterpolated = 0x00000402u;
|
|
private const uint UncertainInterpolated = 0x40A40402u; // UncertainDataSubNormal | InfoType DataValue | Interpolated
|
|
|
|
/// <summary>The live gate's tag <c>HistGW.LiveTest.Gate685</c>: 19:00:03 is a stored NULL.</summary>
|
|
private static readonly DateTime T0 = new(2026, 10, 12, 19, 0, 0, DateTimeKind.Utc);
|
|
|
|
private static readonly DateTime Now = T0.AddHours(1);
|
|
|
|
private static DateTime At(double seconds) => T0.AddTicks((long)(seconds * TimeSpan.TicksPerSecond));
|
|
|
|
private static List<(DateTime T, double V, bool Null)> Gate685() =>
|
|
[(At(0), 100, false), (At(2), 107, false), (At(3), 0, true), (At(5), 108, false)];
|
|
|
|
/// <summary>An AVEVA-shaped store over sorted rows, counting reads and rejecting what the real gateway rejects.</summary>
|
|
private sealed class AvevaStore
|
|
{
|
|
private readonly (DateTime T, double V, bool Null)[] _rows;
|
|
|
|
public AvevaStore(IEnumerable<(DateTime T, double V, bool Null)> rows) => _rows = rows.OrderBy(r => r.T).ToArray();
|
|
|
|
/// <summary>Also emit a row at the exact query END carrying the value in effect there (unverified AVEVA behaviour).</summary>
|
|
public bool TerminalRow { get; init; }
|
|
|
|
/// <summary>Total rows streamed (cost).</summary>
|
|
public long RowsStreamed;
|
|
|
|
public FakeHistorianGatewayClient Client(int? ceiling = null) => new()
|
|
{
|
|
RawSource = Serve,
|
|
MaxValuesCeiling = ceiling,
|
|
};
|
|
|
|
private static HistorianSample Row(DateTime t, double v, bool isNull, uint quality = 0) => new()
|
|
{
|
|
Tag = "T",
|
|
Timestamp = Timestamp.FromDateTime(t),
|
|
NumericValue = isNull ? 0 : v,
|
|
OpcQuality = isNull ? 0u : 192u,
|
|
Quality = isNull ? 1u : quality,
|
|
};
|
|
|
|
private int FirstAtOrAfter(DateTime t)
|
|
{
|
|
int lo = 0, hi = _rows.Length;
|
|
while (lo < hi)
|
|
{
|
|
var mid = (lo + hi) / 2;
|
|
if (_rows[mid].T < t) lo = mid + 1; else hi = mid;
|
|
}
|
|
return lo;
|
|
}
|
|
|
|
private HistorianSample InEffect(DateTime t)
|
|
{
|
|
var i = FirstAtOrAfter(t) - 1;
|
|
if (i < 0) return new HistorianSample { Tag = "T", Timestamp = Timestamp.FromDateTime(t), NumericValue = 0, OpcQuality = 0, Quality = 1 };
|
|
var p = _rows[i];
|
|
return Row(t, p.V, p.Null, p.Null ? 1u : 133u);
|
|
}
|
|
|
|
private IEnumerable<HistorianSample> Serve(DateTime start, DateTime end, int max)
|
|
{
|
|
if (max <= 0)
|
|
throw new Grpc.Core.RpcException(new Grpc.Core.Status(Grpc.Core.StatusCode.InvalidArgument, "max_values must be greater than zero."));
|
|
if (end < start)
|
|
throw new Grpc.Core.RpcException(new Grpc.Core.Status(Grpc.Core.StatusCode.InvalidArgument,
|
|
"start_time must be less than or equal to end_time."));
|
|
if (end > Now.AddDays(2))
|
|
throw new Grpc.Core.RpcException(new Grpc.Core.Status(Grpc.Core.StatusCode.InvalidArgument,
|
|
"end_time is outside the historian's time range"));
|
|
if (start.Ticks % TimeSpan.TicksPerMillisecond != 0 || end.Ticks % TimeSpan.TicksPerMillisecond != 0)
|
|
throw new InvalidOperationException("AVEVA truncates request times to the millisecond; send millisecond bounds.");
|
|
|
|
var rows = new List<HistorianSample>();
|
|
var i = FirstAtOrAfter(start);
|
|
if (i >= _rows.Length || _rows[i].T != start) rows.Add(InEffect(start));
|
|
for (; i < _rows.Length && _rows[i].T <= end && rows.Count < max; i++)
|
|
rows.Add(Row(_rows[i].T, _rows[i].V, _rows[i].Null));
|
|
if (TerminalRow && rows.Count < max && (i >= _rows.Length || _rows[i].T > end) && _rows.All(r => r.T != end))
|
|
rows.Add(InEffect(end));
|
|
var served = rows.Take(max).ToList();
|
|
RowsStreamed += served.Count;
|
|
return served;
|
|
}
|
|
}
|
|
|
|
private static GatewayHistorianDataSource Source(FakeHistorianGatewayClient fake) =>
|
|
new(fake, NullLogger<GatewayHistorianDataSource>.Instance, timeProvider: new FixedTime(Now));
|
|
|
|
private sealed class FixedTime(DateTime now) : TimeProvider
|
|
{
|
|
public override DateTimeOffset GetUtcNow() => new(now);
|
|
}
|
|
|
|
private static async Task<IReadOnlyList<DataValueSnapshot>> AtTime(FakeHistorianGatewayClient fake, params DateTime[] times)
|
|
{
|
|
var r = await Source(fake).ReadAtTimeAsync("T", times, Ct);
|
|
r.Samples.Count.ShouldBe(times.Length);
|
|
for (var i = 0; i < times.Length; i++) r.Samples[i].SourceTimestampUtc.ShouldBe(times[i]);
|
|
var refused = fake.MaxValuesCeiling is { } c ? fake.RawCalls.Count(x => x.MaxValues > c) : 0;
|
|
fake.RawStreamsCompleted.ShouldBe(fake.ReadRawCallCount - refused); // never abandons a gateway stream
|
|
fake.ReadAtTimeCallCount.ShouldBe(0); // never AVEVA's own interpolation
|
|
return r.Samples;
|
|
}
|
|
|
|
// ---- the four live findings ------------------------------------------------------------------------------------
|
|
|
|
[Fact] // live: 19:00:03 (a stored NULL) ⇒ 107 Good
|
|
public async Task At_a_stored_NULLs_own_time_is_null_and_Bad()
|
|
{
|
|
var s = (await AtTime(new AvevaStore(Gate685()).Client(), At(3))).Single();
|
|
s.Value.ShouldBeNull();
|
|
s.StatusCode.ShouldBe(Bad);
|
|
}
|
|
|
|
[Fact] // live: 19:00:05 (stored 108, right after the NULL) ⇒ null + Bad
|
|
public async Task At_a_Good_sample_right_after_a_NULL_is_the_stored_value()
|
|
{
|
|
var s = (await AtTime(new AvevaStore(Gate685()).Client(), At(5))).Single();
|
|
s.Value.ShouldBe(108d);
|
|
s.StatusCode.ShouldBe(Good);
|
|
}
|
|
|
|
[Fact] // live: 99.99999999944 for a stored 100
|
|
public async Task At_an_exact_stored_sample_is_the_exact_stored_value_without_the_Interpolated_bit()
|
|
{
|
|
var s = await AtTime(new AvevaStore(Gate685()).Client(), At(0), At(2));
|
|
s[0].Value.ShouldBe(100d);
|
|
s[0].StatusCode.ShouldBe(Good);
|
|
s[1].Value.ShouldBe(107d);
|
|
s[1].StatusCode.ShouldBe(Good);
|
|
}
|
|
|
|
[Fact] // live: no Interpolated bit
|
|
public async Task Between_two_Good_samples_is_linear_with_the_Interpolated_bit()
|
|
{
|
|
var s = (await AtTime(new AvevaStore(Gate685()).Client(), At(1))).Single();
|
|
s.Value.ShouldBe(103.5d);
|
|
s.StatusCode.ShouldBe(GoodInterpolated);
|
|
new Opc.Ua.StatusCode(s.StatusCode).AggregateBits.ShouldBe(Opc.Ua.AggregateBits.Interpolated);
|
|
}
|
|
|
|
// ---- the rest of the AtTime rules --------------------------------------------------------------------------------
|
|
|
|
[Fact]
|
|
public async Task Before_a_Bad_neighbour_the_previous_value_is_held_Uncertain_and_Interpolated()
|
|
{
|
|
var s = (await AtTime(new AvevaStore(Gate685()).Client(), At(2.5))).Single();
|
|
s.Value.ShouldBe(107d);
|
|
s.StatusCode.ShouldBe(UncertainInterpolated);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Inside_an_outage_is_null_with_the_outages_Bad_code()
|
|
{
|
|
var s = (await AtTime(new AvevaStore(Gate685()).Client(), At(4))).Single();
|
|
s.Value.ShouldBeNull();
|
|
s.StatusCode.ShouldBe(Bad);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Before_the_first_sample_is_BadNoData_not_the_boundary_rows_zero()
|
|
{
|
|
var s = await AtTime(new AvevaStore(Gate685()).Client(), At(-1), At(-0.001), At(-3600));
|
|
s.ShouldAllBe(x => x.StatusCode == BadNoData && x.Value == null);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task After_the_last_sample_is_stepped_Good_until_now_then_Uncertain()
|
|
{
|
|
var s = await AtTime(new AvevaStore(Gate685()).Client(), At(10), Now.AddMinutes(5), Now.AddDays(3));
|
|
s[0].Value.ShouldBe(108d);
|
|
s[0].StatusCode.ShouldBe(GoodInterpolated);
|
|
s[1].Value.ShouldBe(108d);
|
|
s[1].StatusCode.ShouldBe(UncertainInterpolated); // later than now: not known to still hold
|
|
s[2].Value.ShouldBe(108d);
|
|
s[2].StatusCode.ShouldBe(UncertainInterpolated); // beyond the horizon: never sent to the gateway (it refuses)
|
|
}
|
|
|
|
[Fact]
|
|
public async Task A_mixed_request_keeps_order_and_duplicates()
|
|
{
|
|
var s = await AtTime(new AvevaStore(Gate685()).Client(), At(5), At(1), At(-1), At(3), At(1), At(2.5));
|
|
s.Select(x => x.Value).ShouldBe(new object?[] { 108d, 103.5d, null, null, 103.5d, 107d });
|
|
s.Select(x => x.StatusCode).ShouldBe(new[] { Good, GoodInterpolated, BadNoData, Bad, GoodInterpolated, UncertainInterpolated });
|
|
}
|
|
|
|
[Fact]
|
|
public async Task An_empty_request_reads_nothing()
|
|
{
|
|
var fake = new AvevaStore(Gate685()).Client();
|
|
(await Source(fake).ReadAtTimeAsync("T", [], Ct)).Samples.ShouldBeEmpty();
|
|
fake.ReadRawCallCount.ShouldBe(0);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task A_sub_millisecond_request_time_interpolates_between_the_millisecond_samples()
|
|
{
|
|
var s = (await AtTime(new AvevaStore(Gate685()).Client(), At(0.0005))).Single();
|
|
((double)s.Value!).ShouldBe(100.00175d, 1e-9);
|
|
s.StatusCode.ShouldBe(GoodInterpolated);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task A_row_at_the_query_end_never_counts_as_a_stored_sample()
|
|
{
|
|
var store = new AvevaStore(Gate685()) { TerminalRow = true };
|
|
var s = await AtTime(store.Client(), At(1), At(2.5), At(4), At(10));
|
|
s.Select(x => x.Value).ShouldBe(new object?[] { 103.5d, 107d, null, 108d });
|
|
s.Select(x => x.StatusCode).ShouldBe(new[] { GoodInterpolated, UncertainInterpolated, Bad, GoodInterpolated });
|
|
}
|
|
|
|
[Fact]
|
|
public async Task A_string_tag_is_stepped()
|
|
{
|
|
var fake = new FakeHistorianGatewayClient
|
|
{
|
|
RawSource = (start, end, max) =>
|
|
{
|
|
var rows = new List<HistorianSample>
|
|
{
|
|
new() { Tag = "T", Timestamp = Timestamp.FromDateTime(start), StringValue = "", OpcQuality = 0, Quality = 1 },
|
|
};
|
|
foreach (var (t, v) in new[] { (At(0), "a"), (At(10), "b") })
|
|
if (t >= start && t <= end)
|
|
rows.Add(new HistorianSample { Tag = "T", Timestamp = Timestamp.FromDateTime(t), StringValue = v, OpcQuality = 192 });
|
|
return rows.Take(max);
|
|
},
|
|
};
|
|
var s = (await AtTime(fake, At(5))).Single();
|
|
s.Value.ShouldBe("a");
|
|
s.StatusCode.ShouldBe(GoodInterpolated);
|
|
}
|
|
|
|
// ---- the node's value type (the node manager's hint; AVEVA stores a Boolean as a 0/1 Int2) ------------------------
|
|
|
|
/// <summary>Reads through the INTERFACE's hinted overload, as the node manager does.</summary>
|
|
private static async Task<IReadOnlyList<DataValueSnapshot>> AtTimeTyped(
|
|
FakeHistorianGatewayClient fake, TypeCode? valueType, params DateTime[] times)
|
|
{
|
|
IHistorianDataSource source = Source(fake);
|
|
var r = await source.ReadAtTimeAsync("T", times, valueType, Ct);
|
|
r.Samples.Count.ShouldBe(times.Length);
|
|
fake.ReadAtTimeCallCount.ShouldBe(0);
|
|
return r.Samples;
|
|
}
|
|
|
|
[Fact]
|
|
public async Task A_Boolean_node_is_stepped_never_a_fraction()
|
|
{
|
|
var fake = new AvevaStore([(At(0), 0, false), (At(10), 1, false), (At(20), 0, false)]).Client();
|
|
var s = await AtTimeTyped(fake, TypeCode.Boolean, At(5), At(10), At(15), At(25));
|
|
s.Select(x => x.Value).ShouldBe(new object?[] { false, true, true, false });
|
|
s.ShouldAllBe(x => x.Value is bool);
|
|
s.Select(x => x.StatusCode).ShouldBe(new[] { GoodInterpolated, Good, GoodInterpolated, GoodInterpolated });
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Without_a_hint_a_Boolean_tag_interpolates_as_before()
|
|
{
|
|
var fake = new AvevaStore([(At(0), 0, false), (At(10), 1, false)]).Client();
|
|
(await AtTimeTyped(fake, null, At(5))).Single().Value.ShouldBeOfType<double>().ShouldBe(0.5d);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task An_Int32_node_interpolates_rounds_half_away_from_zero_and_keeps_its_type()
|
|
{
|
|
var s = await AtTimeTyped(new AvevaStore(Gate685()).Client(), TypeCode.Int32, At(0), At(0.5), At(1), At(2.5), At(3));
|
|
s[0].Value.ShouldBe(100); // exact hit, as stored, Int32
|
|
s[0].StatusCode.ShouldBe(Good);
|
|
s[1].Value.ShouldBe(102); // 101.75 ⇒ 102
|
|
s[2].Value.ShouldBe(104); // 103.5 ⇒ 104 (half away from zero)
|
|
s[1].StatusCode.ShouldBe(GoodInterpolated);
|
|
s[3].Value.ShouldBe(107); // held before the NULL, Uncertain
|
|
s[3].StatusCode.ShouldBe(UncertainInterpolated);
|
|
s[4].Value.ShouldBeNull(); // the NULL itself
|
|
s[4].StatusCode.ShouldBe(Bad);
|
|
s.Take(4).Select(x => x.Value + ":" + x.Value!.GetType().Name).ShouldBe(["100:Int32", "102:Int32", "104:Int32", "107:Int32"]);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task A_negative_midpoint_rounds_away_from_zero()
|
|
{
|
|
var fake = new AvevaStore([(At(0), -100, false), (At(2), -107, false)]).Client();
|
|
(await AtTimeTyped(fake, TypeCode.Int16, At(1))).Single().Value.ShouldBeOfType<short>().ShouldBe((short)-104);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Other_integer_nodes_keep_their_own_type()
|
|
{
|
|
var fake = new AvevaStore(Gate685()).Client();
|
|
(await AtTimeTyped(fake, TypeCode.UInt16, At(1))).Single().Value.ShouldBeOfType<ushort>().ShouldBe((ushort)104);
|
|
(await AtTimeTyped(fake, TypeCode.Int64, At(1))).Single().Value.ShouldBeOfType<long>().ShouldBe(104L);
|
|
(await AtTimeTyped(fake, TypeCode.Byte, At(1))).Single().Value.ShouldBeOfType<byte>().ShouldBe((byte)104);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Float_and_Double_nodes_interpolate_unchanged()
|
|
{
|
|
var fake = new AvevaStore(Gate685()).Client();
|
|
(await AtTimeTyped(fake, TypeCode.Double, At(1))).Single().Value.ShouldBeOfType<double>().ShouldBe(103.5d);
|
|
(await AtTimeTyped(fake, TypeCode.Single, At(1))).Single().Value.ShouldBeOfType<double>().ShouldBe(103.5d); // as the SQL backends (Double kind)
|
|
}
|
|
|
|
// ---- cost ------------------------------------------------------------------------------------------------------
|
|
|
|
private static IEnumerable<(DateTime T, double V, bool Null)> OneHertz(DateTime from, int seconds) =>
|
|
Enumerable.Range(0, seconds).Select(i => (from.AddSeconds(i), (double)i, false));
|
|
|
|
[Fact]
|
|
public async Task Ten_exact_hits_on_dense_data_are_one_read()
|
|
{
|
|
var fake = new AvevaStore(OneHertz(T0.AddHours(-1), 3600)).Client();
|
|
var times = Enumerable.Range(0, 10).Select(i => T0.AddMinutes(-30).AddSeconds(i)).ToArray();
|
|
var s = await AtTime(fake, times);
|
|
s.Select(x => x.Value).ShouldBe(Enumerable.Range(1800, 10).Select(i => (object?)(double)i));
|
|
s.ShouldAllBe(x => x.StatusCode == Good);
|
|
fake.ReadRawCallCount.ShouldBe(1);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Ten_interpolated_times_on_dense_data_are_three_reads()
|
|
{
|
|
var fake = new AvevaStore(OneHertz(T0.AddHours(-1), 3600)).Client();
|
|
var times = Enumerable.Range(0, 10).Select(i => T0.AddMinutes(-30).AddSeconds(i + 0.5)).ToArray();
|
|
var s = await AtTime(fake, times);
|
|
s.Select(x => x.Value).ShouldBe(Enumerable.Range(1800, 10).Select(i => (object?)(i + 0.5)));
|
|
s.ShouldAllBe(x => x.StatusCode == GoodInterpolated);
|
|
fake.ReadRawCallCount.ShouldBe(3); // the window, the sample before the first time, the sample after the last
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Ten_times_an_hour_apart_on_dense_data_cost_about_one_read_each_not_the_rows_between()
|
|
{
|
|
var store = new AvevaStore(OneHertz(T0.AddHours(-12), 12 * 3600));
|
|
var fake = store.Client();
|
|
var times = Enumerable.Range(0, 10).Select(i => T0.AddHours(-11).AddHours(i).AddSeconds(0.5)).ToArray();
|
|
var s = await AtTime(fake, times);
|
|
s.Select(x => x.Value).ShouldBe(Enumerable.Range(0, 10).Select(i => (object?)(3600.0 * (i + 1) + 0.5)));
|
|
s.ShouldAllBe(x => x.StatusCode == GoodInterpolated);
|
|
fake.ReadRawCallCount.ShouldBe(12); // 10 window reads (each finds its own prev) + 1 walk-back + 1 probe
|
|
store.RowsStreamed.ShouldBeLessThan(10 * 1100); // never the ~32,400 rows between the first and last time
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Sparse_history_far_back_is_found_in_a_bounded_number_of_reads()
|
|
{
|
|
var fake = new AvevaStore([(T0.AddDays(-400), 42, false), (T0.AddDays(-400).AddSeconds(10), 43, false)]).Client();
|
|
var s = (await AtTime(fake, T0)).Single();
|
|
s.Value.ShouldBe(43d);
|
|
s.StatusCode.ShouldBe(GoodInterpolated);
|
|
fake.ReadRawCallCount.ShouldBe(5); // window, 1-min + 4-min segments, the whole remainder, the probe after
|
|
}
|
|
|
|
[Fact]
|
|
public async Task A_tag_with_no_history_is_BadNoData_in_a_bounded_number_of_reads()
|
|
{
|
|
var fake = new AvevaStore([]).Client();
|
|
var s = await AtTime(fake, At(1), At(2));
|
|
s.ShouldAllBe(x => x.StatusCode == BadNoData);
|
|
fake.ReadRawCallCount.ShouldBe(4); // window, two segments, the whole remainder (no probe: nothing to interpolate)
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Dense_data_far_denser_than_one_read_still_finds_the_neighbours()
|
|
{
|
|
// 1 kHz for 10 minutes: a 1-minute walk-back segment holds 60,000 rows (truncated) — it must narrow, not fail.
|
|
var rows = Enumerable.Range(0, 600_000).Select(i => (T0.AddMinutes(-20).AddMilliseconds(i), (double)i, false));
|
|
var fake = new AvevaStore(rows).Client();
|
|
var t = T0.AddMinutes(-15).AddTicks(5000); // 0.5 ms past sample 300,000
|
|
var s = (await AtTime(fake, t)).Single();
|
|
s.Value.ShouldBe(300_000.5d);
|
|
s.StatusCode.ShouldBe(GoodInterpolated);
|
|
fake.ReadRawCallCount.ShouldBe(2); // the window (holds the prev) + the probe
|
|
}
|
|
|
|
[Fact]
|
|
public async Task A_walk_back_into_data_far_denser_than_one_read_narrows_instead_of_paging()
|
|
{
|
|
// 1 kHz for 10 minutes, then a 10-minute gap: the walk-back crosses the gap and lands in 600,000 rows.
|
|
var rows = Enumerable.Range(0, 600_000).Select(i => (T0.AddMinutes(-20).AddMilliseconds(i), (double)i, false));
|
|
var store = new AvevaStore(rows);
|
|
var fake = store.Client();
|
|
var s = (await AtTime(fake, T0.AddMinutes(-5))).Single();
|
|
s.Value.ShouldBe(599_999d);
|
|
s.StatusCode.ShouldBe(GoodInterpolated); // stepped from the last sample (before now)
|
|
fake.ReadRawCallCount.ShouldBe(6); // window, 1m, 4m, remainder (full: seeds the bisection), one aimed segment, probe
|
|
store.RowsStreamed.ShouldBeLessThan(10_000);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task A_strict_gateway_with_a_small_ceiling_is_still_served()
|
|
{
|
|
var store = new AvevaStore(OneHertz(T0.AddHours(-1), 3600));
|
|
var fake = store.Client(ceiling: 100);
|
|
var times = Enumerable.Range(0, 10).Select(i => T0.AddMinutes(-30).AddSeconds(i * 30 + 0.5)).ToArray();
|
|
var s = await AtTime(fake, times);
|
|
s.Select(x => x.Value).ShouldBe(Enumerable.Range(0, 10).Select(i => (object?)(1800 + i * 30 + 0.5)));
|
|
s.ShouldAllBe(x => x.StatusCode == GoodInterpolated);
|
|
fake.RawCalls.Count(c => c.MaxValues > 100).ShouldBe(1); // learned once, then remembered — a short page is not an error
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Success_is_recorded_once()
|
|
{
|
|
var fake = new AvevaStore(Gate685()).Client();
|
|
var ds = Source(fake);
|
|
await ds.ReadAtTimeAsync("T", [At(1), At(4)], Ct);
|
|
var h = ds.GetHealthSnapshot();
|
|
h.TotalQueries.ShouldBe(1);
|
|
h.TotalSuccesses.ShouldBe(1);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task A_gateway_fault_is_a_recorded_failure()
|
|
{
|
|
var fake = new AvevaStore(Gate685()).Client();
|
|
fake.ThrowOnRead = new Grpc.Core.RpcException(new Grpc.Core.Status(Grpc.Core.StatusCode.Unavailable, "down"));
|
|
var ds = Source(fake);
|
|
await Should.ThrowAsync<Exception>(() => ds.ReadAtTimeAsync("T", [At(1)], Ct));
|
|
ds.GetHealthSnapshot().TotalFailures.ShouldBe(1);
|
|
}
|
|
|
|
// ---- long histories (review H1/M1/L1): a generated store, too large to materialise -----------------------------
|
|
|
|
/// <summary>
|
|
/// A regular series (value = sample index) over <c>[from, to]</c> every <paramref name="step"/>, optionally with a
|
|
/// hole <c>(gapFrom, gapTo)</c>, served in the AVEVA wire shape (boundary row at the query start, ms bounds only,
|
|
/// <c>max_values > 0</c>, <c>start <= end</c>, nothing past now + 2 days).
|
|
/// </summary>
|
|
private sealed class SeriesStore(DateTime from, DateTime to, TimeSpan step, DateTime? gapFrom = null, DateTime? gapTo = null)
|
|
{
|
|
public long RowsStreamed;
|
|
|
|
private bool InGap(DateTime t) => gapFrom is { } a && gapTo is { } b && t > a && t < b;
|
|
|
|
private DateTime? FirstAtOrAfter(DateTime t)
|
|
{
|
|
if (t > to) return null;
|
|
var k = t <= from ? 0 : ((t - from).Ticks + step.Ticks - 1) / step.Ticks;
|
|
var r = from.AddTicks(k * step.Ticks);
|
|
if (InGap(r)) return FirstAtOrAfter(gapTo!.Value);
|
|
return r <= to ? r : null;
|
|
}
|
|
|
|
private DateTime? LastBefore(DateTime t)
|
|
{
|
|
if (t <= from) return null;
|
|
var k = ((Min(t, to.AddTicks(1)) - from).Ticks - 1) / step.Ticks;
|
|
var r = from.AddTicks(k * step.Ticks);
|
|
return InGap(r) ? LastBefore(gapFrom!.Value.AddTicks(1)) : r;
|
|
}
|
|
|
|
private static DateTime Min(DateTime a, DateTime b) => a <= b ? a : b;
|
|
|
|
/// <summary>The stored row at <paramref name="t"/>, stamped <paramref name="at"/> (the boundary row) when given.</summary>
|
|
private HistorianSample Row(DateTime t, uint quality = 0, DateTime? at = null) => new()
|
|
{
|
|
Tag = "T", Timestamp = Timestamp.FromDateTime(at ?? t), NumericValue = (t - from).Ticks / step.Ticks, OpcQuality = 192,
|
|
Quality = quality,
|
|
};
|
|
|
|
public FakeHistorianGatewayClient Client(int? ceiling = null) => new() { RawSource = Serve, MaxValuesCeiling = ceiling };
|
|
|
|
private IEnumerable<HistorianSample> Serve(DateTime start, DateTime end, int max)
|
|
{
|
|
if (max <= 0 || end < start || end > Now.AddDays(2)
|
|
|| start.Ticks % TimeSpan.TicksPerMillisecond != 0 || end.Ticks % TimeSpan.TicksPerMillisecond != 0)
|
|
throw new Grpc.Core.RpcException(new Grpc.Core.Status(Grpc.Core.StatusCode.InvalidArgument, "refused"));
|
|
var rows = new List<HistorianSample>();
|
|
if (FirstAtOrAfter(start) != start)
|
|
rows.Add(LastBefore(start) is { } p
|
|
? Row(p, 133, start)
|
|
: new HistorianSample { Tag = "T", Timestamp = Timestamp.FromDateTime(start), OpcQuality = 0, Quality = 1, NumericValue = 0 });
|
|
for (var t = FirstAtOrAfter(start); t is { } u && u <= end && rows.Count < max; t = FirstAtOrAfter(u.AddTicks(1)))
|
|
rows.Add(Row(u));
|
|
RowsStreamed += rows.Count;
|
|
return rows;
|
|
}
|
|
}
|
|
|
|
[Theory] // H1: a dense tag that stopped reporting — the walk crosses the silence by bisection, not by regrowing segments
|
|
[InlineData(1.0, 30)]
|
|
[InlineData(1.0, 365)]
|
|
[InlineData(10.0, 30)]
|
|
[InlineData(10.0, 365)]
|
|
public async Task A_dense_tag_that_stopped_long_ago_is_found_in_a_bounded_number_of_reads(double hz, int stoppedDaysAgo)
|
|
{
|
|
var step = TimeSpan.FromTicks((long)(TimeSpan.TicksPerSecond / hz));
|
|
var end = Now.AddDays(-stoppedDaysAgo);
|
|
var fake = new SeriesStore(end.AddYears(-5), end, step).Client();
|
|
var s = (await AtTime(fake, Now.AddMinutes(-1))).Single();
|
|
s.Value.ShouldBe((double)((end - end.AddYears(-5)).Ticks / step.Ticks)); // the last sample, held
|
|
s.StatusCode.ShouldBe(GoodInterpolated);
|
|
fake.ReadRawCallCount.ShouldBeLessThan(40);
|
|
}
|
|
|
|
[Fact] // H1: a time inside a month-long hole in the middle of a dense history
|
|
public async Task A_time_inside_a_mid_history_gap_interpolates_across_it()
|
|
{
|
|
var from = Now.AddYears(-2);
|
|
var gapFrom = Now.AddDays(-200);
|
|
var gapTo = gapFrom.AddDays(30);
|
|
var fake = new SeriesStore(from, Now, TimeSpan.FromSeconds(1), gapFrom, gapTo).Client();
|
|
var t = gapFrom.AddDays(15);
|
|
var s = (await AtTime(fake, t)).Single();
|
|
var before = (gapFrom - from).Ticks / TimeSpan.TicksPerSecond;
|
|
var after = (gapTo - from).Ticks / TimeSpan.TicksPerSecond;
|
|
((double)s.Value!).ShouldBe(before + (after - before) * 0.5, 1e-6);
|
|
s.StatusCode.ShouldBe(GoodInterpolated);
|
|
fake.ReadRawCallCount.ShouldBeLessThan(40);
|
|
}
|
|
|
|
[Fact] // H1: a jump to a time in a gap — the walk starts from the earlier read's last row, far below
|
|
public async Task A_jump_into_a_gap_bisects_up_from_the_earlier_reads_last_row()
|
|
{
|
|
var from = Now.AddDays(-30);
|
|
var gapFrom = from.AddDays(2);
|
|
var gapTo = gapFrom.AddDays(10);
|
|
var fake = new SeriesStore(from, Now, TimeSpan.FromSeconds(1), gapFrom, gapTo).Client();
|
|
var t0 = from.AddSeconds(10.5);
|
|
var t1 = gapFrom.AddDays(5);
|
|
var s = await AtTime(fake, t0, t1);
|
|
s[0].Value.ShouldBe(10.5d);
|
|
var before = (gapFrom - from).Ticks / TimeSpan.TicksPerSecond;
|
|
var after = (gapTo - from).Ticks / TimeSpan.TicksPerSecond;
|
|
((double)s[1].Value!).ShouldBe(before + (after - before) * 0.5, 1e-6);
|
|
fake.ReadRawCallCount.ShouldBeLessThan(40);
|
|
}
|
|
|
|
[Fact] // M1: widely spread times cost one SMALL read each, not one 1,024-row read each
|
|
public async Task A_thousand_times_over_ten_years_cost_one_small_read_each()
|
|
{
|
|
var store = new SeriesStore(Now.AddYears(-10), Now, TimeSpan.FromSeconds(1));
|
|
var fake = store.Client();
|
|
var rnd = new Random(1);
|
|
var times = Enumerable.Range(0, 1000).Select(_ => Now.AddYears(-9).AddTicks((long)(rnd.NextDouble() * TimeSpan.TicksPerDay * 3000))).ToArray();
|
|
var s = await AtTime(fake, times);
|
|
s.ShouldAllBe(x => x.StatusCode == GoodInterpolated);
|
|
fake.ReadRawCallCount.ShouldBeLessThanOrEqualTo(1010);
|
|
store.RowsStreamed.ShouldBeLessThan(80_000); // ~64 rows a read (was ~1,000,000)
|
|
}
|
|
|
|
[Fact] // L1: a strict-gateway ceiling too small to work with is refused with a message naming it
|
|
public async Task A_ceiling_below_a_usable_read_is_refused_naming_the_ceiling()
|
|
{
|
|
var fake = new SeriesStore(Now.AddDays(-1), Now, TimeSpan.FromSeconds(1)).Client(ceiling: 2);
|
|
var ex = await Should.ThrowAsync<HistoryReadLimitExceededException>(
|
|
() => Source(fake).ReadAtTimeAsync("T", [Now.AddHours(-1)], Ct));
|
|
ex.Message.ShouldContain("row ceiling (2");
|
|
}
|
|
|
|
[Fact] // L1: a small ceiling is served (paging), within the read cap
|
|
public async Task A_small_ceiling_is_served_within_the_read_cap()
|
|
{
|
|
var fake = new SeriesStore(Now.AddYears(-1), Now, TimeSpan.FromSeconds(1)).Client(ceiling: 10);
|
|
var times = Enumerable.Range(0, 50).Select(i => Now.AddDays(-300 + i * 5).AddSeconds(0.5)).ToArray();
|
|
var s = await AtTime(fake, times);
|
|
s.ShouldAllBe(x => x.StatusCode == GoodInterpolated);
|
|
fake.ReadRawCallCount.ShouldBeLessThanOrEqualTo(GatewayAtTimeReader.BaseReads + GatewayAtTimeReader.ReadsPerTime * 50);
|
|
}
|
|
|
|
[Fact] // L1: the request's read cap stops a walk with a clear error, never a hang (here a cap of 3 vs. a 5-read walk)
|
|
public async Task The_read_cap_stops_a_request_with_a_clear_error()
|
|
{
|
|
var fake = new AvevaStore([(T0.AddDays(-400), 42, false), (T0.AddDays(-400).AddSeconds(10), 43, false)]).Client();
|
|
var ex = await Should.ThrowAsync<HistoryReadLimitExceededException>(() => GatewayAtTimeReader.ReadAsync(
|
|
fake, new GatewayRowCeiling(), "T", [T0], Now, Ct, maxReads: 3));
|
|
ex.Message.ShouldContain("needed more than 3 raw reads");
|
|
fake.ReadRawCallCount.ShouldBe(3);
|
|
fake.RawStreamsCompleted.ShouldBe(3);
|
|
}
|
|
|
|
[Fact] // the default cap scales with the request: 64 + 4 per distinct time
|
|
public async Task The_default_read_cap_is_64_plus_4_per_distinct_time()
|
|
{
|
|
(GatewayAtTimeReader.BaseReads, GatewayAtTimeReader.ReadsPerTime).ShouldBe((64, 4));
|
|
await Task.CompletedTask;
|
|
}
|
|
|
|
// ---- type hint edge cases (review L2/L3) ------------------------------------------------------------------------
|
|
|
|
[Fact]
|
|
public async Task A_DateTime_node_is_stepped()
|
|
{
|
|
var fake = new AvevaStore([(At(0), 0, false), (At(10), 10, false)]).Client();
|
|
var s = (await AtTimeTyped(fake, TypeCode.DateTime, At(5))).Single();
|
|
s.Value.ShouldBe(0d);
|
|
s.StatusCode.ShouldBe(GoodInterpolated);
|
|
}
|
|
|
|
[Theory]
|
|
[InlineData(double.PositiveInfinity)]
|
|
[InlineData(double.NaN)]
|
|
[InlineData(1e30)]
|
|
public async Task Under_an_integer_hint_a_non_integer_next_value_is_unusable_not_saturated(double next)
|
|
{
|
|
var fake = new AvevaStore([(At(0), 100, false), (At(10), next, false)]).Client();
|
|
var s = (await AtTimeTyped(fake, TypeCode.Int32, At(5))).Single();
|
|
s.Value.ShouldBeOfType<int>().ShouldBe(100); // held before an unusable neighbour
|
|
s.StatusCode.ShouldBe(UncertainInterpolated);
|
|
}
|
|
|
|
// ---- kill switch -----------------------------------------------------------------------------------------------
|
|
|
|
[Fact]
|
|
public async Task The_kill_switch_restores_AVEVAs_own_answer()
|
|
{
|
|
var fake = new AvevaStore(Gate685()).Client();
|
|
fake.AtTimeSamples = [new HistorianSample { NumericValue = 107, OpcQuality = 192, Timestamp = Timestamp.FromDateTime(At(3)) }];
|
|
var ds = new GatewayHistorianDataSource(fake, NullLogger<GatewayHistorianDataSource>.Instance, atTimeFromRaw: false);
|
|
var s = (await ds.ReadAtTimeAsync("T", [At(3)], Ct)).Samples.Single();
|
|
s.Value.ShouldBe(107d);
|
|
fake.ReadAtTimeCallCount.ShouldBe(1);
|
|
fake.ReadRawCallCount.ShouldBe(0);
|
|
}
|
|
}
|