dohertj2/lmxopcua
1
0
Fork 0
Code Issues 32 Pull Requests Actions Packages Projects Releases Wiki Activity

feat(historian): Part 11 modified-value history on SqlServer + QuestDB (#668)

Browse Source 
Every HistoryUpdate a SQL backend applies now leaves a modification trail
(prior value, principal, Part 11 update type, modification time), and
HistoryReadRaw with IsReadModified=true serves it as HistoryModifiedData.

- Core: HistoryUpdateContext threaded into all three IHistorianDataUpdater
  methods; IHistorianDataSource.ReadModifiedAsync (default NotSupported =>
  BadHistoryOperationUnsupported on Gateway/Null); HistoryModified* contracts.
- SqlServer: schema v2 (002_sample_modified.sql, RequiredVersion=2, docs copy);
  trail written by OUTPUT ... INTO on the same MERGE/DELETE statement, so a
  failed update leaves no audit row; retention purges SampleModified per tag on
  the source-time cutoff; ReadModifiedAsync keyed on (SourceTimeUtc, ModifiedId).
- QuestDB: append-only {prefix}sample_modified (TTL like samples), trail row
  written BEFORE the change (orphan-on-crash, never an untracked change);
  existence reads now carry the prior value; ReadModifiedAsync.
- Node manager: modified reads dispatch + page via the #669 continuation store
  (HistoryContinuationState gains HistoryCursorKind + LastModifiedId), gated by
  the existing HistoryRead decision; session principal / key id passed to the
  updater. IsDeleteModified stays unsupported.
- Client.CLI: historyread --modified.
- Tests: node-manager dispatch/paging/cursor binding/principal threading,
  over-the-wire HistoryModifiedData, shared ModifiedValueConformance on both
  backends, SqlServer atomicity (forced trail failure => no change) + v1->v2
  upgrade + retention trail purge.
- Docs: Historian.md (Modified-value history section, schema v2 upgrade note),
  security.md, Client.CLI.md, CLAUDE.md.
This commit is contained in:
Joseph Doherty
2026-10-11 09:26:36 -04:00
parent d803e0eab4
commit cfd403710e
51 changed files with 2173 additions and 195 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb.IntegrationTests/UpdateTests.cs
+37 -34
View File
@@ -24,6 +24,9 @@ public sealed class UpdateTests(QuestDbFixture fx)
return new Rig(c, new QuestDbHistorianDataUpdater(c, tags, wal), tags);
}
/// <summary>The HistoryUpdate caller these tests pass to the updater (recorded in the modification trail).</summary>
internal static readonly HistoryUpdateContext Who = new("questdb-test", null);
internal static HistorizationValue V(DateTime? t, double v, uint status = Good) =>
new(t, HistorianValue.FromDouble(v), status);
@@ -57,10 +60,10 @@ public sealed class UpdateTests(QuestDbFixture fx)
{
fx.SkipIfUnavailable();
var rig = MakeRig(fx);
(await rig.Updater.UpdateDataAsync("t1", HistoryUpdateMode.Insert, [V(T0, 1)], default))
(await rig.Updater.UpdateDataAsync("t1", HistoryUpdateMode.Insert, [V(T0, 1)], UpdateTests.Who, default))
.ShouldBe([HistorianStatusCodes.GoodEntryInserted]);
var codes = await rig.Updater.UpdateDataAsync("t1", HistoryUpdateMode.Insert,
[V(T0, 99), V(T0.AddSeconds(1), 2)], default);
[V(T0, 99), V(T0.AddSeconds(1), 2)], UpdateTests.Who, default);
codes.ShouldBe([HistorianStatusCodes.BadEntryExists, HistorianStatusCodes.GoodEntryInserted]);
var rows = await RowsAsync(rig, "t1"); // read right after return: read-your-writes
rows.Select(r => r.Num).ShouldBe([1d, 2d]);
@@ -72,14 +75,14 @@ public sealed class UpdateTests(QuestDbFixture fx)
{
fx.SkipIfUnavailable();
var rig = MakeRig(fx);
(await rig.Updater.UpdateDataAsync("nope", HistoryUpdateMode.Replace, [V(T0, 1)], default))
(await rig.Updater.UpdateDataAsync("nope", HistoryUpdateMode.Replace, [V(T0, 1)], UpdateTests.Who, default))
.ShouldBe([HistorianStatusCodes.BadNoEntryExists]);
(await TagRowsAsync(rig, "nope")).ShouldBe(0);
(await RowsAsync(rig, "nope")).ShouldBeEmpty();
await rig.Updater.UpdateDataAsync("t", HistoryUpdateMode.Insert, [V(T0, 1)], default);
await rig.Updater.UpdateDataAsync("t", HistoryUpdateMode.Insert, [V(T0, 1)], UpdateTests.Who, default);
var codes = await rig.Updater.UpdateDataAsync("t", HistoryUpdateMode.Replace,
[V(T0, 5), V(T0.AddSeconds(1), 6)], default);
[V(T0, 5), V(T0.AddSeconds(1), 6)], UpdateTests.Who, default);
codes.ShouldBe([HistorianStatusCodes.GoodEntryReplaced, HistorianStatusCodes.BadNoEntryExists]);
(await RowsAsync(rig, "t")).Select(r => r.Num).ShouldBe([5d]);
}
@@ -89,9 +92,9 @@ public sealed class UpdateTests(QuestDbFixture fx)
{
fx.SkipIfUnavailable();
var rig = MakeRig(fx);
await rig.Updater.UpdateDataAsync("t", HistoryUpdateMode.Insert, [V(T0, 1)], default);
await rig.Updater.UpdateDataAsync("t", HistoryUpdateMode.Insert, [V(T0, 1)], UpdateTests.Who, default);
var codes = await rig.Updater.UpdateDataAsync("t", HistoryUpdateMode.Update,
[V(T0, 2), V(T0.AddSeconds(1), 3), V(T0.AddSeconds(1).AddTicks(3), 4)], default);
[V(T0, 2), V(T0.AddSeconds(1), 3), V(T0.AddSeconds(1).AddTicks(3), 4)], UpdateTests.Who, default);
codes.ShouldBe([HistorianStatusCodes.GoodEntryReplaced, HistorianStatusCodes.GoodEntryInserted,
HistorianStatusCodes.GoodEntryInserted]);
(await RowsAsync(rig, "t")).Select(r => r.Num).ShouldBe([2d, 4d]);
@@ -104,7 +107,7 @@ public sealed class UpdateTests(QuestDbFixture fx)
var rig = MakeRig(fx);
// offsets 0 and 3 ticks sit inside one 10-tick window => the same stored µs
var codes = await rig.Updater.UpdateDataAsync("t", HistoryUpdateMode.Insert,
[V(T0, 1), V(T0.AddTicks(3), 2)], default);
[V(T0, 1), V(T0.AddTicks(3), 2)], UpdateTests.Who, default);
codes.ShouldBe([HistorianStatusCodes.GoodEntryInserted, HistorianStatusCodes.BadEntryExists]);
(await RowsAsync(rig, "t")).Select(r => r.Num).ShouldBe([1d]);
}
@@ -119,7 +122,7 @@ public sealed class UpdateTests(QuestDbFixture fx)
V(null, 1), V(DateTime.MinValue, 1), V(T0, double.NaN),
new HistorizationValue(T0.AddSeconds(1), HistorianValue.Empty, Good),
V(T0.AddSeconds(2), 7),
], default);
], UpdateTests.Who, default);
codes.ShouldBe([HistorianStatusCodes.BadInvalidTimestamp, HistorianStatusCodes.BadInvalidTimestamp,
HistorianStatusCodes.BadInvalidArgument, HistorianStatusCodes.BadInvalidArgument,
HistorianStatusCodes.GoodEntryInserted]);
@@ -132,10 +135,10 @@ public sealed class UpdateTests(QuestDbFixture fx)
fx.SkipIfUnavailable();
var rig = MakeRig(fx);
var tag = new string('x', HistorianTagNames.MaxLength + 1);
(await rig.Updater.UpdateDataAsync(tag, HistoryUpdateMode.Insert, [V(T0, 1)], default))
(await rig.Updater.UpdateDataAsync(tag, HistoryUpdateMode.Insert, [V(T0, 1)], UpdateTests.Who, default))
.ShouldBe([HistorianStatusCodes.BadInvalidArgument]);
(await rig.Updater.DeleteAtTimeAsync(tag, [T0], default)).ShouldBe([HistorianStatusCodes.BadInvalidArgument]);
(await rig.Updater.DeleteRawAsync(tag, T0, T0.AddHours(1), default)).ShouldBe(HistorianStatusCodes.BadInvalidArgument);
(await rig.Updater.DeleteAtTimeAsync(tag, [T0], UpdateTests.Who, default)).ShouldBe([HistorianStatusCodes.BadInvalidArgument]);
(await rig.Updater.DeleteRawAsync(tag, T0, T0.AddHours(1), UpdateTests.Who, default)).ShouldBe(HistorianStatusCodes.BadInvalidArgument);
}
[Fact]
@@ -146,15 +149,15 @@ public sealed class UpdateTests(QuestDbFixture fx)
var pre = new DateTime(1969, 12, 31, 23, 59, 59, DateTimeKind.Utc);
foreach (var mode in new[] { HistoryUpdateMode.Insert, HistoryUpdateMode.Replace, HistoryUpdateMode.Update })
{
var codes = await rig.Updater.UpdateDataAsync("t", mode, [V(pre, 1), V(DateTime.MinValue, 1), V(T0, 2)], default);
var codes = await rig.Updater.UpdateDataAsync("t", mode, [V(pre, 1), V(DateTime.MinValue, 1), V(T0, 2)], UpdateTests.Who, default);
codes[0].ShouldBe(HistorianStatusCodes.BadOutOfRange);
codes[1].ShouldBe(HistorianStatusCodes.BadInvalidTimestamp);
}
// Pre-1970 only: the whole batch is refused, no tag row is created, no INSERT would succeed anyway.
(await rig.Updater.UpdateDataAsync("only-pre", HistoryUpdateMode.Update, [V(pre, 1)], default))
(await rig.Updater.UpdateDataAsync("only-pre", HistoryUpdateMode.Update, [V(pre, 1)], UpdateTests.Who, default))
.ShouldBe([HistorianStatusCodes.BadOutOfRange]);
(await TagRowsAsync(rig, "only-pre")).ShouldBe(0);
(await rig.Updater.DeleteAtTimeAsync("t", [pre], default)).ShouldBe([HistorianStatusCodes.BadNoEntryExists]);
(await rig.Updater.DeleteAtTimeAsync("t", [pre], UpdateTests.Who, default)).ShouldBe([HistorianStatusCodes.BadNoEntryExists]);
}
[Fact]
@@ -162,15 +165,15 @@ public sealed class UpdateTests(QuestDbFixture fx)
{
fx.SkipIfUnavailable();
var rig = MakeRig(fx);
await rig.Updater.UpdateDataAsync("t", HistoryUpdateMode.Insert, [V(T0, 1), V(T0.AddSeconds(1), 2)], default);
await rig.Updater.UpdateDataAsync("t", HistoryUpdateMode.Insert, [V(T0, 1), V(T0.AddSeconds(1), 2)], UpdateTests.Who, default);
var codes = await rig.Updater.DeleteAtTimeAsync("t",
[T0, T0.AddSeconds(5), T0, DateTime.MinValue], default);
[T0, T0.AddSeconds(5), T0, DateTime.MinValue], UpdateTests.Who, default);
codes.ShouldBe([Good, HistorianStatusCodes.BadNoEntryExists, HistorianStatusCodes.BadNoEntryExists,
HistorianStatusCodes.BadInvalidTimestamp]);
(await RowsAsync(rig, "t", liveOnly: true)).Select(r => r.Num).ShouldBe([2d]);
// a tombstone is not a live row: deleting it again finds nothing
(await rig.Updater.DeleteAtTimeAsync("t", [T0], default)).ShouldBe([HistorianStatusCodes.BadNoEntryExists]);
(await rig.Updater.DeleteAtTimeAsync("unknown", [T0], default)).ShouldBe([HistorianStatusCodes.BadNoEntryExists]);
(await rig.Updater.DeleteAtTimeAsync("t", [T0], UpdateTests.Who, default)).ShouldBe([HistorianStatusCodes.BadNoEntryExists]);
(await rig.Updater.DeleteAtTimeAsync("unknown", [T0], UpdateTests.Who, default)).ShouldBe([HistorianStatusCodes.BadNoEntryExists]);
}
[Fact]
@@ -178,9 +181,9 @@ public sealed class UpdateTests(QuestDbFixture fx)
{
fx.SkipIfUnavailable();
var rig = MakeRig(fx);
await rig.Updater.UpdateDataAsync("t", HistoryUpdateMode.Insert, [V(T0, 1)], default);
await rig.Updater.UpdateDataAsync("t", HistoryUpdateMode.Insert, [V(T0, 1)], UpdateTests.Who, default);
// +7 ticks is inside the same µs; the row lives at T0.
(await rig.Updater.DeleteAtTimeAsync("t", [T0.AddTicks(7)], default)).ShouldBe([Good]);
(await rig.Updater.DeleteAtTimeAsync("t", [T0.AddTicks(7)], UpdateTests.Who, default)).ShouldBe([Good]);
(await RowsAsync(rig, "t", liveOnly: true)).ShouldBeEmpty();
}
@@ -189,18 +192,18 @@ public sealed class UpdateTests(QuestDbFixture fx)
{
fx.SkipIfUnavailable();
var rig = MakeRig(fx);
(await rig.Updater.DeleteRawAsync("unknown", T0, T0.AddHours(1), default)).ShouldBe(HistorianStatusCodes.BadNoData);
(await rig.Updater.DeleteRawAsync("unknown", T0, T0.AddHours(1), UpdateTests.Who, default)).ShouldBe(HistorianStatusCodes.BadNoData);
// 2,500 rows => 3 pages of at most 1,000; plus one row exactly at the (exclusive) end.
var values = Enumerable.Range(0, 2500).Select(i => V(T0.AddSeconds(i), i)).ToList();
values.Add(V(T0.AddSeconds(2500), -1));
(await rig.Updater.UpdateDataAsync("big", HistoryUpdateMode.Insert, values, default))
(await rig.Updater.UpdateDataAsync("big", HistoryUpdateMode.Insert, values, UpdateTests.Who, default))
.ShouldAllBe(c => c == HistorianStatusCodes.GoodEntryInserted);
(await rig.Updater.DeleteRawAsync("big", T0, T0.AddSeconds(2500), default)).ShouldBe(Good);
(await rig.Updater.DeleteRawAsync("big", T0, T0.AddSeconds(2500), UpdateTests.Who, default)).ShouldBe(Good);
var live = await RowsAsync(rig, "big", liveOnly: true);
live.Count.ShouldBe(1);
live[0].Num.ShouldBe(-1d);
(await rig.Updater.DeleteRawAsync("big", T0, T0.AddSeconds(2500), default)).ShouldBe(HistorianStatusCodes.BadNoData);
(await rig.Updater.DeleteRawAsync("big", T0, T0.AddSeconds(2500), UpdateTests.Who, default)).ShouldBe(HistorianStatusCodes.BadNoData);
}
[Fact]
@@ -208,12 +211,12 @@ public sealed class UpdateTests(QuestDbFixture fx)
{
fx.SkipIfUnavailable();
var rig = MakeRig(fx);
await rig.Updater.UpdateDataAsync("t", HistoryUpdateMode.Insert, [V(T0, 1), V(T0.AddSeconds(1), 2)], default);
(await rig.Updater.DeleteRawAsync("t", new DateTime(1900, 1, 1, 0, 0, 0, DateTimeKind.Utc), T0.AddSeconds(1), default))
await rig.Updater.UpdateDataAsync("t", HistoryUpdateMode.Insert, [V(T0, 1), V(T0.AddSeconds(1), 2)], UpdateTests.Who, default);
(await rig.Updater.DeleteRawAsync("t", new DateTime(1900, 1, 1, 0, 0, 0, DateTimeKind.Utc), T0.AddSeconds(1), UpdateTests.Who, default))
.ShouldBe(Good);
(await RowsAsync(rig, "t", liveOnly: true)).Select(r => r.Num).ShouldBe([2d]);
// DateTime.MinValue as the start is nudged inside, not bound as -infinity
(await rig.Updater.DeleteRawAsync("t", DateTime.MinValue, DateTime.MaxValue, default)).ShouldBe(Good);
(await rig.Updater.DeleteRawAsync("t", DateTime.MinValue, DateTime.MaxValue, UpdateTests.Who, default)).ShouldBe(Good);
(await RowsAsync(rig, "t", liveOnly: true)).ShouldBeEmpty();
}
@@ -226,9 +229,9 @@ public sealed class UpdateTests(QuestDbFixture fx)
for (var i = 0; i < 40; i++)
{
var t = T0.AddSeconds(i);
(await rig.Updater.UpdateDataAsync("t", HistoryUpdateMode.Insert, [V(t, i)], default))
(await rig.Updater.UpdateDataAsync("t", HistoryUpdateMode.Insert, [V(t, i)], UpdateTests.Who, default))
.ShouldBe([HistorianStatusCodes.GoodEntryInserted]);
(await rig.Updater.UpdateDataAsync("t", HistoryUpdateMode.Replace, [V(t, i + 1000)], default))
(await rig.Updater.UpdateDataAsync("t", HistoryUpdateMode.Replace, [V(t, i + 1000)], UpdateTests.Who, default))
.ShouldBe([HistorianStatusCodes.GoodEntryReplaced], $"iteration {i}: the insert was not visible");
}
}
@@ -242,8 +245,8 @@ public sealed class UpdateTests(QuestDbFixture fx)
{
var t = T0.AddSeconds(i);
var codes = await Task.WhenAll(
Task.Run(() => rig.Updater.UpdateDataAsync("race", HistoryUpdateMode.Insert, [V(t, 1)], default)),
Task.Run(() => rig.Updater.UpdateDataAsync("race", HistoryUpdateMode.Insert, [V(t, 2)], default)));
Task.Run(() => rig.Updater.UpdateDataAsync("race", HistoryUpdateMode.Insert, [V(t, 1)], UpdateTests.Who, default)),
Task.Run(() => rig.Updater.UpdateDataAsync("race", HistoryUpdateMode.Insert, [V(t, 2)], UpdateTests.Who, default)));
codes.Select(c => c[0]).OrderBy(c => c).ToArray().ShouldBe(
new[] { HistorianStatusCodes.GoodEntryInserted, HistorianStatusCodes.BadEntryExists }, $"iteration {i}");
}
@@ -256,10 +259,10 @@ public sealed class UpdateTests(QuestDbFixture fx)
var rig = MakeRig(fx);
// 1 Hz seed across 2 h = 7,200 live rows; the request touches 3 instants 1 h apart.
var seed = Enumerable.Range(0, 7200).Select(i => V(T0.AddSeconds(i), i)).ToList();
await rig.Updater.UpdateDataAsync("sparse", HistoryUpdateMode.Insert, seed, default);
await rig.Updater.UpdateDataAsync("sparse", HistoryUpdateMode.Insert, seed, UpdateTests.Who, default);
var before = rig.Updater.LiveRowsRead;
var codes = await rig.Updater.UpdateDataAsync("sparse", HistoryUpdateMode.Replace,
[V(T0, 1), V(T0.AddHours(1), 2), V(T0.AddHours(2).AddSeconds(-1), 3), V(T0.AddHours(1).AddMilliseconds(500), 4)], default);
[V(T0, 1), V(T0.AddHours(1), 2), V(T0.AddHours(2).AddSeconds(-1), 3), V(T0.AddHours(1).AddMilliseconds(500), 4)], UpdateTests.Who, default);
codes.ShouldBe([HistorianStatusCodes.GoodEntryReplaced, HistorianStatusCodes.GoodEntryReplaced,
HistorianStatusCodes.GoodEntryReplaced, HistorianStatusCodes.BadNoEntryExists]);
// 3 clusters of a single instant each (T0+1h and +1h.5s are one cluster of 2 instants, 1 live row)