Merge feat/668-modified-values: Part 11 modified-value history (IsReadModified) on SqlServer + QuestDB (#668)

This commit is contained in:
Joseph Doherty
2026-10-11 09:32:13 -04:00
51 changed files with 2173 additions and 195 deletions
@@ -52,7 +52,7 @@ public sealed class BootstrapTests(QuestDbFixture fx)
var p = fx.UniquePrefix();
var (c, boot, _) = Make(p);
await boot.EnsureAsync(default);
(await TablesAsync(c, p)).ShouldBe([p + "event", p + "sample", p + "tag"]);
(await TablesAsync(c, p)).ShouldBe([p + "event", p + "sample", p + "sample_modified", p + "tag"]);
}
[Fact]
@@ -69,7 +69,7 @@ public sealed class BootstrapTests(QuestDbFixture fx)
await Task.Delay(300); // an ALTER would have been applied by now
(await SequencerTxnAsync(c2, p + "sample")).ShouldBe(before);
(await SequencerTxnAsync(c2, p + "event")).ShouldBe(await SequencerTxnAsync(c, p + "event"));
(await TablesAsync(c2, p)).Count.ShouldBe(3);
(await TablesAsync(c2, p)).Count.ShouldBe(4);
}
[Fact]
@@ -84,7 +84,7 @@ public sealed class BootstrapTests(QuestDbFixture fx)
await Task.WhenAll(
Task.Run(() => a.Boot.EnsureAsync(default)),
Task.Run(() => b.Boot.EnsureAsync(default)));
(await TablesAsync(a.Conn, p)).Count.ShouldBe(3);
(await TablesAsync(a.Conn, p)).Count.ShouldBe(4);
}
}
@@ -97,6 +97,8 @@ public sealed class BootstrapTests(QuestDbFixture fx)
await b1.EnsureAsync(default);
QuestDbTables.TtlHours((await TtlAsync(c1, withTtl + "sample")).Value, (await TtlAsync(c1, withTtl + "sample")).Unit).ShouldBe(168);
QuestDbTables.TtlHours((await TtlAsync(c1, withTtl + "event")).Value, (await TtlAsync(c1, withTtl + "event")).Unit).ShouldBe(168);
// #668: the modification trail ages out with the samples it describes.
QuestDbTables.TtlHours((await TtlAsync(c1, withTtl + "sample_modified")).Value, (await TtlAsync(c1, withTtl + "sample_modified")).Unit).ShouldBe(168);
(await TtlAsync(c1, withTtl + "tag")).Value.ShouldBe(0); // the tag catalog never expires
var noTtl = fx.UniquePrefix();
@@ -53,3 +53,10 @@ public sealed class QuestDbTagNameConformance(QuestDbFixture fx) : TagNameConfor
/// <inheritdoc/>
protected override Task<IConformanceBackend> CreateBackendAsync() => QuestDbConformanceBackend.CreateAsync(fx);
}
[Collection(QuestDbCollection.Name)]
public sealed class QuestDbModifiedValueConformance(QuestDbFixture fx) : ModifiedValueConformance
{
/// <inheritdoc/>
protected override Task<IConformanceBackend> CreateBackendAsync() => QuestDbConformanceBackend.CreateAsync(fx);
}
@@ -26,7 +26,7 @@ public sealed partial class TombstoneTests
];
(await writer.WriteLiveValuesAsync("r", batch, default)).ShouldBeTrue();
(await rig.Updater.DeleteAtTimeAsync("r", [T0], default)).ShouldBe([HistorianStatusCodes.Good]);
(await rig.Updater.DeleteAtTimeAsync("r", [T0], UpdateTests.Who, default)).ShouldBe([HistorianStatusCodes.Good]);
(await writer.WriteLiveValuesAsync("r", batch, default)).ShouldBeTrue(); // the redelivery
(await UpdateTests.RowsAsync(rig, "r", liveOnly: true)).Select(r => r.Num).ShouldBe([2d]);
@@ -17,8 +17,8 @@ public sealed partial class TombstoneTests(QuestDbFixture fx)
fx.SkipIfUnavailable();
var rig = UpdateTests.MakeRig(fx);
await rig.Updater.UpdateDataAsync("t", HistoryUpdateMode.Insert,
[UpdateTests.V(T0, 1), UpdateTests.V(T0.AddSeconds(1), 2)], default);
(await rig.Updater.DeleteAtTimeAsync("t", [T0], default)).ShouldBe([HistorianStatusCodes.Good]);
[UpdateTests.V(T0, 1), UpdateTests.V(T0.AddSeconds(1), 2)], UpdateTests.Who, default);
(await rig.Updater.DeleteAtTimeAsync("t", [T0], UpdateTests.Who, default)).ShouldBe([HistorianStatusCodes.Good]);
(await UpdateTests.RowsAsync(rig, "t", liveOnly: true)).Select(r => r.Num).ShouldBe([2d]);
var all = await UpdateTests.RowsAsync(rig, "t");
@@ -35,12 +35,12 @@ public sealed partial class TombstoneTests(QuestDbFixture fx)
{
fx.SkipIfUnavailable();
var rig = UpdateTests.MakeRig(fx);
await rig.Updater.UpdateDataAsync("t", HistoryUpdateMode.Insert, [UpdateTests.V(T0, 1)], default);
await rig.Updater.DeleteAtTimeAsync("t", [T0], default);
await rig.Updater.UpdateDataAsync("t", HistoryUpdateMode.Insert, [UpdateTests.V(T0, 1)], UpdateTests.Who, default);
await rig.Updater.DeleteAtTimeAsync("t", [T0], UpdateTests.Who, default);
(await rig.Updater.UpdateDataAsync("t", HistoryUpdateMode.Replace, [UpdateTests.V(T0, 5)], default))
(await rig.Updater.UpdateDataAsync("t", HistoryUpdateMode.Replace, [UpdateTests.V(T0, 5)], UpdateTests.Who, default))
.ShouldBe([HistorianStatusCodes.BadNoEntryExists]); // a tombstone is no entry for Replace
(await rig.Updater.UpdateDataAsync("t", HistoryUpdateMode.Insert, [UpdateTests.V(T0, 9)], default))
(await rig.Updater.UpdateDataAsync("t", HistoryUpdateMode.Insert, [UpdateTests.V(T0, 9)], UpdateTests.Who, default))
.ShouldBe([HistorianStatusCodes.GoodEntryInserted]);
var live = await UpdateTests.RowsAsync(rig, "t", liveOnly: true);
live.Count.ShouldBe(1);
@@ -24,6 +24,9 @@ public sealed class UpdateTests(QuestDbFixture fx)
return new Rig(c, new QuestDbHistorianDataUpdater(c, tags, wal), tags);
}
/// <summary>The HistoryUpdate caller these tests pass to the updater (recorded in the modification trail).</summary>
internal static readonly HistoryUpdateContext Who = new("questdb-test", null);
internal static HistorizationValue V(DateTime? t, double v, uint status = Good) =>
new(t, HistorianValue.FromDouble(v), status);
@@ -57,10 +60,10 @@ public sealed class UpdateTests(QuestDbFixture fx)
{
fx.SkipIfUnavailable();
var rig = MakeRig(fx);
(await rig.Updater.UpdateDataAsync("t1", HistoryUpdateMode.Insert, [V(T0, 1)], default))
(await rig.Updater.UpdateDataAsync("t1", HistoryUpdateMode.Insert, [V(T0, 1)], UpdateTests.Who, default))
.ShouldBe([HistorianStatusCodes.GoodEntryInserted]);
var codes = await rig.Updater.UpdateDataAsync("t1", HistoryUpdateMode.Insert,
[V(T0, 99), V(T0.AddSeconds(1), 2)], default);
[V(T0, 99), V(T0.AddSeconds(1), 2)], UpdateTests.Who, default);
codes.ShouldBe([HistorianStatusCodes.BadEntryExists, HistorianStatusCodes.GoodEntryInserted]);
var rows = await RowsAsync(rig, "t1"); // read right after return: read-your-writes
rows.Select(r => r.Num).ShouldBe([1d, 2d]);
@@ -72,14 +75,14 @@ public sealed class UpdateTests(QuestDbFixture fx)
{
fx.SkipIfUnavailable();
var rig = MakeRig(fx);
(await rig.Updater.UpdateDataAsync("nope", HistoryUpdateMode.Replace, [V(T0, 1)], default))
(await rig.Updater.UpdateDataAsync("nope", HistoryUpdateMode.Replace, [V(T0, 1)], UpdateTests.Who, default))
.ShouldBe([HistorianStatusCodes.BadNoEntryExists]);
(await TagRowsAsync(rig, "nope")).ShouldBe(0);
(await RowsAsync(rig, "nope")).ShouldBeEmpty();
await rig.Updater.UpdateDataAsync("t", HistoryUpdateMode.Insert, [V(T0, 1)], default);
await rig.Updater.UpdateDataAsync("t", HistoryUpdateMode.Insert, [V(T0, 1)], UpdateTests.Who, default);
var codes = await rig.Updater.UpdateDataAsync("t", HistoryUpdateMode.Replace,
[V(T0, 5), V(T0.AddSeconds(1), 6)], default);
[V(T0, 5), V(T0.AddSeconds(1), 6)], UpdateTests.Who, default);
codes.ShouldBe([HistorianStatusCodes.GoodEntryReplaced, HistorianStatusCodes.BadNoEntryExists]);
(await RowsAsync(rig, "t")).Select(r => r.Num).ShouldBe([5d]);
}
@@ -89,9 +92,9 @@ public sealed class UpdateTests(QuestDbFixture fx)
{
fx.SkipIfUnavailable();
var rig = MakeRig(fx);
await rig.Updater.UpdateDataAsync("t", HistoryUpdateMode.Insert, [V(T0, 1)], default);
await rig.Updater.UpdateDataAsync("t", HistoryUpdateMode.Insert, [V(T0, 1)], UpdateTests.Who, default);
var codes = await rig.Updater.UpdateDataAsync("t", HistoryUpdateMode.Update,
[V(T0, 2), V(T0.AddSeconds(1), 3), V(T0.AddSeconds(1).AddTicks(3), 4)], default);
[V(T0, 2), V(T0.AddSeconds(1), 3), V(T0.AddSeconds(1).AddTicks(3), 4)], UpdateTests.Who, default);
codes.ShouldBe([HistorianStatusCodes.GoodEntryReplaced, HistorianStatusCodes.GoodEntryInserted,
HistorianStatusCodes.GoodEntryInserted]);
(await RowsAsync(rig, "t")).Select(r => r.Num).ShouldBe([2d, 4d]);
@@ -104,7 +107,7 @@ public sealed class UpdateTests(QuestDbFixture fx)
var rig = MakeRig(fx);
// offsets 0 and 3 ticks sit inside one 10-tick window => the same stored µs
var codes = await rig.Updater.UpdateDataAsync("t", HistoryUpdateMode.Insert,
[V(T0, 1), V(T0.AddTicks(3), 2)], default);
[V(T0, 1), V(T0.AddTicks(3), 2)], UpdateTests.Who, default);
codes.ShouldBe([HistorianStatusCodes.GoodEntryInserted, HistorianStatusCodes.BadEntryExists]);
(await RowsAsync(rig, "t")).Select(r => r.Num).ShouldBe([1d]);
}
@@ -119,7 +122,7 @@ public sealed class UpdateTests(QuestDbFixture fx)
V(null, 1), V(DateTime.MinValue, 1), V(T0, double.NaN),
new HistorizationValue(T0.AddSeconds(1), HistorianValue.Empty, Good),
V(T0.AddSeconds(2), 7),
], default);
], UpdateTests.Who, default);
codes.ShouldBe([HistorianStatusCodes.BadInvalidTimestamp, HistorianStatusCodes.BadInvalidTimestamp,
HistorianStatusCodes.BadInvalidArgument, HistorianStatusCodes.BadInvalidArgument,
HistorianStatusCodes.GoodEntryInserted]);
@@ -132,10 +135,10 @@ public sealed class UpdateTests(QuestDbFixture fx)
fx.SkipIfUnavailable();
var rig = MakeRig(fx);
var tag = new string('x', HistorianTagNames.MaxLength + 1);
(await rig.Updater.UpdateDataAsync(tag, HistoryUpdateMode.Insert, [V(T0, 1)], default))
(await rig.Updater.UpdateDataAsync(tag, HistoryUpdateMode.Insert, [V(T0, 1)], UpdateTests.Who, default))
.ShouldBe([HistorianStatusCodes.BadInvalidArgument]);
(await rig.Updater.DeleteAtTimeAsync(tag, [T0], default)).ShouldBe([HistorianStatusCodes.BadInvalidArgument]);
(await rig.Updater.DeleteRawAsync(tag, T0, T0.AddHours(1), default)).ShouldBe(HistorianStatusCodes.BadInvalidArgument);
(await rig.Updater.DeleteAtTimeAsync(tag, [T0], UpdateTests.Who, default)).ShouldBe([HistorianStatusCodes.BadInvalidArgument]);
(await rig.Updater.DeleteRawAsync(tag, T0, T0.AddHours(1), UpdateTests.Who, default)).ShouldBe(HistorianStatusCodes.BadInvalidArgument);
}
[Fact]
@@ -146,15 +149,15 @@ public sealed class UpdateTests(QuestDbFixture fx)
var pre = new DateTime(1969, 12, 31, 23, 59, 59, DateTimeKind.Utc);
foreach (var mode in new[] { HistoryUpdateMode.Insert, HistoryUpdateMode.Replace, HistoryUpdateMode.Update })
{
var codes = await rig.Updater.UpdateDataAsync("t", mode, [V(pre, 1), V(DateTime.MinValue, 1), V(T0, 2)], default);
var codes = await rig.Updater.UpdateDataAsync("t", mode, [V(pre, 1), V(DateTime.MinValue, 1), V(T0, 2)], UpdateTests.Who, default);
codes[0].ShouldBe(HistorianStatusCodes.BadOutOfRange);
codes[1].ShouldBe(HistorianStatusCodes.BadInvalidTimestamp);
}
// Pre-1970 only: the whole batch is refused, no tag row is created, no INSERT would succeed anyway.
(await rig.Updater.UpdateDataAsync("only-pre", HistoryUpdateMode.Update, [V(pre, 1)], default))
(await rig.Updater.UpdateDataAsync("only-pre", HistoryUpdateMode.Update, [V(pre, 1)], UpdateTests.Who, default))
.ShouldBe([HistorianStatusCodes.BadOutOfRange]);
(await TagRowsAsync(rig, "only-pre")).ShouldBe(0);
(await rig.Updater.DeleteAtTimeAsync("t", [pre], default)).ShouldBe([HistorianStatusCodes.BadNoEntryExists]);
(await rig.Updater.DeleteAtTimeAsync("t", [pre], UpdateTests.Who, default)).ShouldBe([HistorianStatusCodes.BadNoEntryExists]);
}
[Fact]
@@ -162,15 +165,15 @@ public sealed class UpdateTests(QuestDbFixture fx)
{
fx.SkipIfUnavailable();
var rig = MakeRig(fx);
await rig.Updater.UpdateDataAsync("t", HistoryUpdateMode.Insert, [V(T0, 1), V(T0.AddSeconds(1), 2)], default);
await rig.Updater.UpdateDataAsync("t", HistoryUpdateMode.Insert, [V(T0, 1), V(T0.AddSeconds(1), 2)], UpdateTests.Who, default);
var codes = await rig.Updater.DeleteAtTimeAsync("t",
[T0, T0.AddSeconds(5), T0, DateTime.MinValue], default);
[T0, T0.AddSeconds(5), T0, DateTime.MinValue], UpdateTests.Who, default);
codes.ShouldBe([Good, HistorianStatusCodes.BadNoEntryExists, HistorianStatusCodes.BadNoEntryExists,
HistorianStatusCodes.BadInvalidTimestamp]);
(await RowsAsync(rig, "t", liveOnly: true)).Select(r => r.Num).ShouldBe([2d]);
// a tombstone is not a live row: deleting it again finds nothing
(await rig.Updater.DeleteAtTimeAsync("t", [T0], default)).ShouldBe([HistorianStatusCodes.BadNoEntryExists]);
(await rig.Updater.DeleteAtTimeAsync("unknown", [T0], default)).ShouldBe([HistorianStatusCodes.BadNoEntryExists]);
(await rig.Updater.DeleteAtTimeAsync("t", [T0], UpdateTests.Who, default)).ShouldBe([HistorianStatusCodes.BadNoEntryExists]);
(await rig.Updater.DeleteAtTimeAsync("unknown", [T0], UpdateTests.Who, default)).ShouldBe([HistorianStatusCodes.BadNoEntryExists]);
}
[Fact]
@@ -178,9 +181,9 @@ public sealed class UpdateTests(QuestDbFixture fx)
{
fx.SkipIfUnavailable();
var rig = MakeRig(fx);
await rig.Updater.UpdateDataAsync("t", HistoryUpdateMode.Insert, [V(T0, 1)], default);
await rig.Updater.UpdateDataAsync("t", HistoryUpdateMode.Insert, [V(T0, 1)], UpdateTests.Who, default);
// +7 ticks is inside the same µs; the row lives at T0.
(await rig.Updater.DeleteAtTimeAsync("t", [T0.AddTicks(7)], default)).ShouldBe([Good]);
(await rig.Updater.DeleteAtTimeAsync("t", [T0.AddTicks(7)], UpdateTests.Who, default)).ShouldBe([Good]);
(await RowsAsync(rig, "t", liveOnly: true)).ShouldBeEmpty();
}
@@ -189,18 +192,18 @@ public sealed class UpdateTests(QuestDbFixture fx)
{
fx.SkipIfUnavailable();
var rig = MakeRig(fx);
(await rig.Updater.DeleteRawAsync("unknown", T0, T0.AddHours(1), default)).ShouldBe(HistorianStatusCodes.BadNoData);
(await rig.Updater.DeleteRawAsync("unknown", T0, T0.AddHours(1), UpdateTests.Who, default)).ShouldBe(HistorianStatusCodes.BadNoData);
// 2,500 rows => 3 pages of at most 1,000; plus one row exactly at the (exclusive) end.
var values = Enumerable.Range(0, 2500).Select(i => V(T0.AddSeconds(i), i)).ToList();
values.Add(V(T0.AddSeconds(2500), -1));
(await rig.Updater.UpdateDataAsync("big", HistoryUpdateMode.Insert, values, default))
(await rig.Updater.UpdateDataAsync("big", HistoryUpdateMode.Insert, values, UpdateTests.Who, default))
.ShouldAllBe(c => c == HistorianStatusCodes.GoodEntryInserted);
(await rig.Updater.DeleteRawAsync("big", T0, T0.AddSeconds(2500), default)).ShouldBe(Good);
(await rig.Updater.DeleteRawAsync("big", T0, T0.AddSeconds(2500), UpdateTests.Who, default)).ShouldBe(Good);
var live = await RowsAsync(rig, "big", liveOnly: true);
live.Count.ShouldBe(1);
live[0].Num.ShouldBe(-1d);
(await rig.Updater.DeleteRawAsync("big", T0, T0.AddSeconds(2500), default)).ShouldBe(HistorianStatusCodes.BadNoData);
(await rig.Updater.DeleteRawAsync("big", T0, T0.AddSeconds(2500), UpdateTests.Who, default)).ShouldBe(HistorianStatusCodes.BadNoData);
}
[Fact]
@@ -208,12 +211,12 @@ public sealed class UpdateTests(QuestDbFixture fx)
{
fx.SkipIfUnavailable();
var rig = MakeRig(fx);
await rig.Updater.UpdateDataAsync("t", HistoryUpdateMode.Insert, [V(T0, 1), V(T0.AddSeconds(1), 2)], default);
(await rig.Updater.DeleteRawAsync("t", new DateTime(1900, 1, 1, 0, 0, 0, DateTimeKind.Utc), T0.AddSeconds(1), default))
await rig.Updater.UpdateDataAsync("t", HistoryUpdateMode.Insert, [V(T0, 1), V(T0.AddSeconds(1), 2)], UpdateTests.Who, default);
(await rig.Updater.DeleteRawAsync("t", new DateTime(1900, 1, 1, 0, 0, 0, DateTimeKind.Utc), T0.AddSeconds(1), UpdateTests.Who, default))
.ShouldBe(Good);
(await RowsAsync(rig, "t", liveOnly: true)).Select(r => r.Num).ShouldBe([2d]);
// DateTime.MinValue as the start is nudged inside, not bound as -infinity
(await rig.Updater.DeleteRawAsync("t", DateTime.MinValue, DateTime.MaxValue, default)).ShouldBe(Good);
(await rig.Updater.DeleteRawAsync("t", DateTime.MinValue, DateTime.MaxValue, UpdateTests.Who, default)).ShouldBe(Good);
(await RowsAsync(rig, "t", liveOnly: true)).ShouldBeEmpty();
}
@@ -226,9 +229,9 @@ public sealed class UpdateTests(QuestDbFixture fx)
for (var i = 0; i < 40; i++)
{
var t = T0.AddSeconds(i);
(await rig.Updater.UpdateDataAsync("t", HistoryUpdateMode.Insert, [V(t, i)], default))
(await rig.Updater.UpdateDataAsync("t", HistoryUpdateMode.Insert, [V(t, i)], UpdateTests.Who, default))
.ShouldBe([HistorianStatusCodes.GoodEntryInserted]);
(await rig.Updater.UpdateDataAsync("t", HistoryUpdateMode.Replace, [V(t, i + 1000)], default))
(await rig.Updater.UpdateDataAsync("t", HistoryUpdateMode.Replace, [V(t, i + 1000)], UpdateTests.Who, default))
.ShouldBe([HistorianStatusCodes.GoodEntryReplaced], $"iteration {i}: the insert was not visible");
}
}
@@ -242,8 +245,8 @@ public sealed class UpdateTests(QuestDbFixture fx)
{
var t = T0.AddSeconds(i);
var codes = await Task.WhenAll(
Task.Run(() => rig.Updater.UpdateDataAsync("race", HistoryUpdateMode.Insert, [V(t, 1)], default)),
Task.Run(() => rig.Updater.UpdateDataAsync("race", HistoryUpdateMode.Insert, [V(t, 2)], default)));
Task.Run(() => rig.Updater.UpdateDataAsync("race", HistoryUpdateMode.Insert, [V(t, 1)], UpdateTests.Who, default)),
Task.Run(() => rig.Updater.UpdateDataAsync("race", HistoryUpdateMode.Insert, [V(t, 2)], UpdateTests.Who, default)));
codes.Select(c => c[0]).OrderBy(c => c).ToArray().ShouldBe(
new[] { HistorianStatusCodes.GoodEntryInserted, HistorianStatusCodes.BadEntryExists }, $"iteration {i}");
}
@@ -256,10 +259,10 @@ public sealed class UpdateTests(QuestDbFixture fx)
var rig = MakeRig(fx);
// 1 Hz seed across 2 h = 7,200 live rows; the request touches 3 instants 1 h apart.
var seed = Enumerable.Range(0, 7200).Select(i => V(T0.AddSeconds(i), i)).ToList();
await rig.Updater.UpdateDataAsync("sparse", HistoryUpdateMode.Insert, seed, default);
await rig.Updater.UpdateDataAsync("sparse", HistoryUpdateMode.Insert, seed, UpdateTests.Who, default);
var before = rig.Updater.LiveRowsRead;
var codes = await rig.Updater.UpdateDataAsync("sparse", HistoryUpdateMode.Replace,
[V(T0, 1), V(T0.AddHours(1), 2), V(T0.AddHours(2).AddSeconds(-1), 3), V(T0.AddHours(1).AddMilliseconds(500), 4)], default);
[V(T0, 1), V(T0.AddHours(1), 2), V(T0.AddHours(2).AddSeconds(-1), 3), V(T0.AddHours(1).AddMilliseconds(500), 4)], UpdateTests.Who, default);
codes.ShouldBe([HistorianStatusCodes.GoodEntryReplaced, HistorianStatusCodes.GoodEntryReplaced,
HistorianStatusCodes.GoodEntryReplaced, HistorianStatusCodes.BadNoEntryExists]);
// 3 clusters of a single instant each (T0+1h and +1h.5s are one cluster of 2 instants, 1 live row)