dohertj2/lmxopcua
1
0
Fork 0
Code Issues 32 Pull Requests Actions Packages Projects Releases Wiki Activity

feat(historian): Part 11 modified-value history on SqlServer + QuestDB (#668)

Browse Source 
Every HistoryUpdate a SQL backend applies now leaves a modification trail
(prior value, principal, Part 11 update type, modification time), and
HistoryReadRaw with IsReadModified=true serves it as HistoryModifiedData.

- Core: HistoryUpdateContext threaded into all three IHistorianDataUpdater
  methods; IHistorianDataSource.ReadModifiedAsync (default NotSupported =>
  BadHistoryOperationUnsupported on Gateway/Null); HistoryModified* contracts.
- SqlServer: schema v2 (002_sample_modified.sql, RequiredVersion=2, docs copy);
  trail written by OUTPUT ... INTO on the same MERGE/DELETE statement, so a
  failed update leaves no audit row; retention purges SampleModified per tag on
  the source-time cutoff; ReadModifiedAsync keyed on (SourceTimeUtc, ModifiedId).
- QuestDB: append-only {prefix}sample_modified (TTL like samples), trail row
  written BEFORE the change (orphan-on-crash, never an untracked change);
  existence reads now carry the prior value; ReadModifiedAsync.
- Node manager: modified reads dispatch + page via the #669 continuation store
  (HistoryContinuationState gains HistoryCursorKind + LastModifiedId), gated by
  the existing HistoryRead decision; session principal / key id passed to the
  updater. IsDeleteModified stays unsupported.
- Client.CLI: historyread --modified.
- Tests: node-manager dispatch/paging/cursor binding/principal threading,
  over-the-wire HistoryModifiedData, shared ModifiedValueConformance on both
  backends, SqlServer atomicity (forced trail failure => no change) + v1->v2
  upgrade + retention trail purge.
- Docs: Historian.md (Modified-value history section, schema v2 upgrade note),
  security.md, Client.CLI.md, CLAUDE.md.
This commit is contained in:
Joseph Doherty
2026-10-11 09:26:36 -04:00
parent d803e0eab4
commit cfd403710e
51 changed files with 2173 additions and 195 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/Drivers/Historian.Conformance/UpdateConformance.cs
+29 -29
View File
@@ -23,10 +23,10 @@ public abstract class UpdateConformance : ConformanceBase
public async Task Insert_new_is_GoodEntryInserted_and_existing_is_BadEntryExists_in_order()
{
var u = B.CreateUpdater();
var r1 = await u.UpdateDataAsync("t", HistoryUpdateMode.Insert, [V(T(1), 1), V(T(2), 2)], Ct);
var r1 = await u.UpdateDataAsync("t", HistoryUpdateMode.Insert, [V(T(1), 1), V(T(2), 2)], ConformanceData.Editor, Ct);
r1.ShouldBe([HistorianStatusCodes.GoodEntryInserted, HistorianStatusCodes.GoodEntryInserted]);
// Mixed: T(2) exists, T(3) new, T(1) exists - out of timestamp order on purpose.
var r2 = await u.UpdateDataAsync("t", HistoryUpdateMode.Insert, [V(T(3), 3), V(T(2), 99), V(T(1), 98)], Ct);
var r2 = await u.UpdateDataAsync("t", HistoryUpdateMode.Insert, [V(T(3), 3), V(T(2), 99), V(T(1), 98)], ConformanceData.Editor, Ct);
r2.ShouldBe([HistorianStatusCodes.GoodEntryInserted, HistorianStatusCodes.BadEntryExists, HistorianStatusCodes.BadEntryExists]);
var rows = await B.ReadBackAsync("t");
rows.Select(x => (double?)x.V.Number).ShouldBe([1d, 2d, 3d]); // existing values untouched
@@ -37,9 +37,9 @@ public abstract class UpdateConformance : ConformanceBase
public async Task Replace_existing_is_GoodEntryReplaced_with_new_value_and_status_missing_is_BadNoEntryExists()
{
var u = B.CreateUpdater();
await u.UpdateDataAsync("t", HistoryUpdateMode.Insert, [V(T(1), 1), V(T(2), 2)], Ct);
await u.UpdateDataAsync("t", HistoryUpdateMode.Insert, [V(T(1), 1), V(T(2), 2)], ConformanceData.Editor, Ct);
var r = await u.UpdateDataAsync("t", HistoryUpdateMode.Replace,
[V(T(2), 22, UncertainStatus), V(T(5), 5), V(T(1), 11)], Ct);
[V(T(2), 22, UncertainStatus), V(T(5), 5), V(T(1), 11)], ConformanceData.Editor, Ct);
r.ShouldBe([HistorianStatusCodes.GoodEntryReplaced, HistorianStatusCodes.BadNoEntryExists, HistorianStatusCodes.GoodEntryReplaced]);
var rows = await B.ReadBackAsync("t");
rows.Count.ShouldBe(2); // T(5) was not created
@@ -53,8 +53,8 @@ public abstract class UpdateConformance : ConformanceBase
public async Task Update_upserts_inserted_or_replaced_per_item()
{
var u = B.CreateUpdater();
await u.UpdateDataAsync("t", HistoryUpdateMode.Insert, [V(T(1), 1)], Ct);
var r = await u.UpdateDataAsync("t", HistoryUpdateMode.Update, [V(T(2), 2), V(T(1), 10)], Ct);
await u.UpdateDataAsync("t", HistoryUpdateMode.Insert, [V(T(1), 1)], ConformanceData.Editor, Ct);
var r = await u.UpdateDataAsync("t", HistoryUpdateMode.Update, [V(T(2), 2), V(T(1), 10)], ConformanceData.Editor, Ct);
r.ShouldBe([HistorianStatusCodes.GoodEntryInserted, HistorianStatusCodes.GoodEntryReplaced]);
(await B.ReadBackAsync("t")).Select(x => (double?)x.V.Number).ShouldBe([10d, 2d]);
}
@@ -63,15 +63,15 @@ public abstract class UpdateConformance : ConformanceBase
public async Task Unknown_tag_Insert_and_Update_create_it_but_Replace_and_deletes_do_not()
{
var u = B.CreateUpdater();
(await u.UpdateDataAsync("rep", HistoryUpdateMode.Replace, [V(T(1), 1)], Ct))
(await u.UpdateDataAsync("rep", HistoryUpdateMode.Replace, [V(T(1), 1)], ConformanceData.Editor, Ct))
.ShouldBe([HistorianStatusCodes.BadNoEntryExists]);
(await u.DeleteRawAsync("del", T(0), T(9), Ct)).ShouldBe(HistorianStatusCodes.BadNoData);
(await u.DeleteAtTimeAsync("del2", [T(1)], Ct)).ShouldBe([HistorianStatusCodes.BadNoEntryExists]);
(await u.DeleteRawAsync("del", T(0), T(9), ConformanceData.Editor, Ct)).ShouldBe(HistorianStatusCodes.BadNoData);
(await u.DeleteAtTimeAsync("del2", [T(1)], ConformanceData.Editor, Ct)).ShouldBe([HistorianStatusCodes.BadNoEntryExists]);
foreach (var name in new[] { "rep", "del", "del2" })
(await B.TagExistsAsync(name)).ShouldBeFalse(name); // none of those created a tag
(await u.UpdateDataAsync("ins", HistoryUpdateMode.Insert, [V(T(1), 1)], Ct)).ShouldBe([HistorianStatusCodes.GoodEntryInserted]);
(await u.UpdateDataAsync("upd", HistoryUpdateMode.Update, [V(T(1), 1)], Ct)).ShouldBe([HistorianStatusCodes.GoodEntryInserted]);
(await u.UpdateDataAsync("ins", HistoryUpdateMode.Insert, [V(T(1), 1)], ConformanceData.Editor, Ct)).ShouldBe([HistorianStatusCodes.GoodEntryInserted]);
(await u.UpdateDataAsync("upd", HistoryUpdateMode.Update, [V(T(1), 1)], ConformanceData.Editor, Ct)).ShouldBe([HistorianStatusCodes.GoodEntryInserted]);
(await B.TagExistsAsync("ins")).ShouldBeTrue();
(await B.TagExistsAsync("upd")).ShouldBeTrue();
}
@@ -81,14 +81,14 @@ public abstract class UpdateConformance : ConformanceBase
{
var u = B.CreateUpdater();
var r = await u.UpdateDataAsync("t", HistoryUpdateMode.Insert,
[V(null, 1), V(T(1), 2), V(DateTime.MinValue, 3)], Ct);
[V(null, 1), V(T(1), 2), V(DateTime.MinValue, 3)], ConformanceData.Editor, Ct);
r.ShouldBe([HistorianStatusCodes.BadInvalidTimestamp, HistorianStatusCodes.GoodEntryInserted, HistorianStatusCodes.BadInvalidTimestamp]);
(await B.ReadBackAsync("t")).Count.ShouldBe(1);
// All-invalid request: no DB work, no tag created, still one code per item.
var u2 = B.CreateUpdater();
await u2.UpdateDataAsync("seed", HistoryUpdateMode.Insert, [V(T(1), 1)], Ct); // bootstraps the schema
(await u2.UpdateDataAsync("x", HistoryUpdateMode.Insert, [V(null, 1)], Ct)).ShouldBe([HistorianStatusCodes.BadInvalidTimestamp]);
await u2.UpdateDataAsync("seed", HistoryUpdateMode.Insert, [V(T(1), 1)], ConformanceData.Editor, Ct); // bootstraps the schema
(await u2.UpdateDataAsync("x", HistoryUpdateMode.Insert, [V(null, 1)], ConformanceData.Editor, Ct)).ShouldBe([HistorianStatusCodes.BadInvalidTimestamp]);
(await B.TagExistsAsync("seed")).ShouldBeTrue();
(await B.TagExistsAsync("x")).ShouldBeFalse(); // "x" was never created
}
@@ -98,7 +98,7 @@ public abstract class UpdateConformance : ConformanceBase
{
var u = B.CreateUpdater();
var r = await u.UpdateDataAsync("t", HistoryUpdateMode.Insert,
[V(T(1), double.NaN), V(T(2), 2), V(T(3), double.PositiveInfinity), V(T(4), double.NegativeInfinity)], Ct);
[V(T(1), double.NaN), V(T(2), 2), V(T(3), double.PositiveInfinity), V(T(4), double.NegativeInfinity)], ConformanceData.Editor, Ct);
r.ShouldBe([HistorianStatusCodes.BadInvalidArgument, HistorianStatusCodes.GoodEntryInserted,
HistorianStatusCodes.BadInvalidArgument, HistorianStatusCodes.BadInvalidArgument]);
(await B.ReadBackAsync("t")).Select(x => (double?)x.V.Number).ShouldBe([2d]);
@@ -108,7 +108,7 @@ public abstract class UpdateConformance : ConformanceBase
public async Task Duplicate_timestamp_in_one_Insert_request_second_is_BadEntryExists_first_wins()
{
var u = B.CreateUpdater();
var r = await u.UpdateDataAsync("t", HistoryUpdateMode.Insert, [V(T(1), 1), V(T(2), 2), V(T(1), 99)], Ct);
var r = await u.UpdateDataAsync("t", HistoryUpdateMode.Insert, [V(T(1), 1), V(T(2), 2), V(T(1), 99)], ConformanceData.Editor, Ct);
r.ShouldBe([HistorianStatusCodes.GoodEntryInserted, HistorianStatusCodes.GoodEntryInserted, HistorianStatusCodes.BadEntryExists]);
(await B.ReadBackAsync("t")).Select(x => (double?)x.V.Number).ShouldBe([1d, 2d]);
}
@@ -117,12 +117,12 @@ public abstract class UpdateConformance : ConformanceBase
public async Task Duplicate_timestamp_in_one_Replace_or_Update_request_last_wins_and_all_share_the_code()
{
var u = B.CreateUpdater();
await u.UpdateDataAsync("t", HistoryUpdateMode.Insert, [V(T(1), 1)], Ct);
var rep = await u.UpdateDataAsync("t", HistoryUpdateMode.Replace, [V(T(1), 5), V(T(1), 6)], Ct);
await u.UpdateDataAsync("t", HistoryUpdateMode.Insert, [V(T(1), 1)], ConformanceData.Editor, Ct);
var rep = await u.UpdateDataAsync("t", HistoryUpdateMode.Replace, [V(T(1), 5), V(T(1), 6)], ConformanceData.Editor, Ct);
rep.ShouldBe([HistorianStatusCodes.GoodEntryReplaced, HistorianStatusCodes.GoodEntryReplaced]);
(await B.ReadBackAsync("t")).Single().V.Number.ShouldBe(6d);
var upd = await u.UpdateDataAsync("t", HistoryUpdateMode.Update, [V(T(7), 7), V(T(7), 8)], Ct);
var upd = await u.UpdateDataAsync("t", HistoryUpdateMode.Update, [V(T(7), 7), V(T(7), 8)], ConformanceData.Editor, Ct);
upd.ShouldBe([HistorianStatusCodes.GoodEntryInserted, HistorianStatusCodes.GoodEntryInserted]);
(await B.ReadBackAsync("t")).Last().V.Number.ShouldBe(8d);
}
@@ -131,9 +131,9 @@ public abstract class UpdateConformance : ConformanceBase
public async Task Replace_and_Update_can_change_the_value_kind_and_clear_stale_columns()
{
var u = B.CreateUpdater();
await u.UpdateDataAsync("t", HistoryUpdateMode.Insert, [new HistorizationValue(T(1), HistorianValue.FromDouble(1.5), Good)], Ct);
await u.UpdateDataAsync("t", HistoryUpdateMode.Insert, [new HistorizationValue(T(1), HistorianValue.FromDouble(1.5), Good)], ConformanceData.Editor, Ct);
(await u.UpdateDataAsync("t", HistoryUpdateMode.Replace,
[new HistorizationValue(T(1), HistorianValue.FromString("hi"), Good)], Ct)).ShouldBe([HistorianStatusCodes.GoodEntryReplaced]);
[new HistorizationValue(T(1), HistorianValue.FromString("hi"), Good)], ConformanceData.Editor, Ct)).ShouldBe([HistorianStatusCodes.GoodEntryReplaced]);
var rows = await B.ReadBackAsync("t");
rows.Single().V.ShouldBe(HistorianValue.FromString("hi")); // Number cleared (0), not left at 1.5
}
@@ -142,13 +142,13 @@ public abstract class UpdateConformance : ConformanceBase
public async Task DeleteRaw_is_half_open_Good_when_rows_removed_BadNoData_when_none()
{
var u = B.CreateUpdater();
await u.UpdateDataAsync("t", HistoryUpdateMode.Insert, [V(T(1), 1), V(T(2), 2), V(T(3), 3)], Ct);
await u.UpdateDataAsync("t", HistoryUpdateMode.Insert, [V(T(1), 1), V(T(2), 2), V(T(3), 3)], ConformanceData.Editor, Ct);
// [T(1), T(3)) removes T(1) and T(2); T(3) (the end) survives.
(await u.DeleteRawAsync("t", T(1), T(3), Ct)).ShouldBe(Good);
(await u.DeleteRawAsync("t", T(1), T(3), ConformanceData.Editor, Ct)).ShouldBe(Good);
(await B.ReadBackAsync("t")).Select(x => x.T).ShouldBe([B.Round(T(3))]);
(await u.DeleteRawAsync("t", T(1), T(3), Ct)).ShouldBe(HistorianStatusCodes.BadNoData);
(await u.DeleteRawAsync("t", T(1), T(3), ConformanceData.Editor, Ct)).ShouldBe(HistorianStatusCodes.BadNoData);
// sub-ms boundary: a start one resolution unit after T(3) excludes it.
(await u.DeleteRawAsync("t", T(3).AddTicks(B.ResolutionTicks), T(3).AddTicks(100 * B.ResolutionTicks), Ct)).ShouldBe(HistorianStatusCodes.BadNoData);
(await u.DeleteRawAsync("t", T(3).AddTicks(B.ResolutionTicks), T(3).AddTicks(100 * B.ResolutionTicks), ConformanceData.Editor, Ct)).ShouldBe(HistorianStatusCodes.BadNoData);
(await B.ReadBackAsync("t")).Count.ShouldBe(1);
}
@@ -156,8 +156,8 @@ public abstract class UpdateConformance : ConformanceBase
public async Task DeleteAtTime_reports_per_time_in_order_with_exact_tick_matching()
{
var u = B.CreateUpdater();
await u.UpdateDataAsync("t", HistoryUpdateMode.Insert, [V(T(1), 1), V(T(2), 2), V(T(3), 3)], Ct);
var r = await u.DeleteAtTimeAsync("t", [T(3), T(2).AddTicks(B.ResolutionTicks) /* near miss */, T(1), DateTime.MinValue], Ct);
await u.UpdateDataAsync("t", HistoryUpdateMode.Insert, [V(T(1), 1), V(T(2), 2), V(T(3), 3)], ConformanceData.Editor, Ct);
var r = await u.DeleteAtTimeAsync("t", [T(3), T(2).AddTicks(B.ResolutionTicks) /* near miss */, T(1), DateTime.MinValue], ConformanceData.Editor, Ct);
r.ShouldBe([Good, HistorianStatusCodes.BadNoEntryExists, Good, HistorianStatusCodes.BadInvalidTimestamp]);
(await B.ReadBackAsync("t")).Select(x => x.T).ShouldBe([B.Round(T(2))]);
}
@@ -166,8 +166,8 @@ public abstract class UpdateConformance : ConformanceBase
public async Task DeleteAtTime_duplicate_time_first_reports_Good_later_BadNoEntryExists()
{
var u = B.CreateUpdater();
await u.UpdateDataAsync("t", HistoryUpdateMode.Insert, [V(T(1), 1)], Ct);
(await u.DeleteAtTimeAsync("t", [T(1), T(1)], Ct)).ShouldBe([Good, HistorianStatusCodes.BadNoEntryExists]);
await u.UpdateDataAsync("t", HistoryUpdateMode.Insert, [V(T(1), 1)], ConformanceData.Editor, Ct);
(await u.DeleteAtTimeAsync("t", [T(1), T(1)], ConformanceData.Editor, Ct)).ShouldBe([Good, HistorianStatusCodes.BadNoEntryExists]);
(await B.ReadBackAsync("t")).ShouldBeEmpty();
}
}