dohertj2/lmxopcua
1
0
Fork 0
Code Issues 32 Pull Requests Actions Packages Projects Releases Wiki Activity

feat(historian): Part 11 modified-value history on SqlServer + QuestDB (#668)

Browse Source 
Every HistoryUpdate a SQL backend applies now leaves a modification trail
(prior value, principal, Part 11 update type, modification time), and
HistoryReadRaw with IsReadModified=true serves it as HistoryModifiedData.

- Core: HistoryUpdateContext threaded into all three IHistorianDataUpdater
  methods; IHistorianDataSource.ReadModifiedAsync (default NotSupported =>
  BadHistoryOperationUnsupported on Gateway/Null); HistoryModified* contracts.
- SqlServer: schema v2 (002_sample_modified.sql, RequiredVersion=2, docs copy);
  trail written by OUTPUT ... INTO on the same MERGE/DELETE statement, so a
  failed update leaves no audit row; retention purges SampleModified per tag on
  the source-time cutoff; ReadModifiedAsync keyed on (SourceTimeUtc, ModifiedId).
- QuestDB: append-only {prefix}sample_modified (TTL like samples), trail row
  written BEFORE the change (orphan-on-crash, never an untracked change);
  existence reads now carry the prior value; ReadModifiedAsync.
- Node manager: modified reads dispatch + page via the #669 continuation store
  (HistoryContinuationState gains HistoryCursorKind + LastModifiedId), gated by
  the existing HistoryRead decision; session principal / key id passed to the
  updater. IsDeleteModified stays unsupported.
- Client.CLI: historyread --modified.
- Tests: node-manager dispatch/paging/cursor binding/principal threading,
  over-the-wire HistoryModifiedData, shared ModifiedValueConformance on both
  backends, SqlServer atomicity (forced trail failure => no change) + v1->v2
  upgrade + retention trail purge.
- Docs: Historian.md (Modified-value history section, schema v2 upgrade note),
  security.md, Client.CLI.md, CLAUDE.md.
This commit is contained in:
Joseph Doherty
2026-10-11 09:26:36 -04:00
parent d803e0eab4
commit cfd403710e
51 changed files with 2173 additions and 195 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer.IntegrationTests/RetentionTests.cs
+20 -2
View File
@@ -56,6 +56,9 @@ public sealed class RetentionTests(SqlHistorianFixture fx)
sql.Append($"SET @i = 0; WHILE @i < 2 BEGIN INSERT [{s}].Sample (TagId, SourceTimeUtc, ValueKind, NumValue, StatusCode, ServerTimeUtc) SELECT TagId, DATEADD(minute, @i, '{fresh}'), 1, 1.0, 0, SYSUTCDATETIME() FROM [{s}].Tag; SET @i += 1; END;");
sql.Append($"SET @i = 0; WHILE @i < 4 BEGIN INSERT [{s}].Event (EventId, SourceName, AlarmId, AlarmName, AlarmTypeName, Severity, EventKind, EventTimeUtc, ReceivedTimeUtc) VALUES (CONCAT('old', @i), 's', 'a', 'n', 't', 1, 'Activated', DATEADD(minute, @i, '{old}'), SYSUTCDATETIME()); SET @i += 1; END;");
sql.Append($"INSERT [{s}].Event (EventId, SourceName, AlarmId, AlarmName, AlarmTypeName, Severity, EventKind, EventTimeUtc, ReceivedTimeUtc) VALUES ('new0', 's', 'a', 'n', 't', 1, 'Activated', '{fresh}', SYSUTCDATETIME());");
// #668 modification trail: 2 old + 1 fresh row per tag (by SOURCE time — the modification time is "now" on all).
sql.Append($"SET @i = 0; WHILE @i < 2 BEGIN INSERT [{s}].SampleModified (TagId, SourceTimeUtc, ModificationTimeUtc, UpdateType, Principal, ValueKind, NumValue, StatusCode, ServerTimeUtc) SELECT TagId, DATEADD(minute, @i, '{old}'), SYSUTCDATETIME(), 2, N'p', 1, 1.0, 0, SYSUTCDATETIME() FROM [{s}].Tag; SET @i += 1; END;");
sql.Append($"INSERT [{s}].SampleModified (TagId, SourceTimeUtc, ModificationTimeUtc, UpdateType, Principal) SELECT TagId, '{fresh}', SYSUTCDATETIME(), 1, N'p' FROM [{s}].Tag;");
await ExecAsync(sql.ToString());
}
@@ -67,11 +70,22 @@ public sealed class RetentionTests(SqlHistorianFixture fx)
fx.SkipIfUnavailable();
var (c, b, _, s) = Make();
await SeedAsync(b, s);
var identBefore = await CountAsync($"SELECT IDENT_CURRENT('[{s}].SampleModified')");
var purger = new SqlServerHistorianRetention(c, b, TimeSpan.FromHours(1), 30, batch, time: new FixedTime(Now));
var results = await purger.PurgeAsync(CancellationToken.None);
results.ShouldBe([new RetentionPurgeResult("Sample", 15), new RetentionPurgeResult("Event", 4)]);
// #668: a purge never WRITES to the modification trail. The identity seed proves no row was ever inserted —
// a row-count check could not, since the trail purge would delete an old-source-time audit row again.
(await CountAsync($"SELECT IDENT_CURRENT('[{s}].SampleModified')")).ShouldBe(identBefore);
results.ShouldBe([
new RetentionPurgeResult("Sample", 15),
new RetentionPurgeResult("SampleModified", 6),
new RetentionPurgeResult("Event", 4),
]);
(await CountAsync($"SELECT COUNT(*) FROM [{s}].SampleModified")).ShouldBe(3);
(await CountAsync($"SELECT COUNT(*) FROM [{s}].SampleModified WHERE SourceTimeUtc < '{Now.AddDays(-30):yyyy-MM-ddTHH:mm:ss}'")).ShouldBe(0);
(await CountAsync($"SELECT COUNT(*) FROM [{s}].Sample")).ShouldBe(6);
(await CountAsync($"SELECT COUNT(*) FROM [{s}].Sample WHERE SourceTimeUtc < '{Now.AddDays(-30):yyyy-MM-ddTHH:mm:ss}'")).ShouldBe(0);
(await CountAsync($"SELECT COUNT(*) FROM [{s}].Event")).ShouldBe(1);
@@ -87,7 +101,11 @@ public sealed class RetentionTests(SqlHistorianFixture fx)
var purger = new SqlServerHistorianRetention(c, b, TimeSpan.FromHours(1), 30, 4000, time: new FixedTime(Now));
await purger.PurgeAsync(CancellationToken.None);
var again = await purger.PurgeAsync(CancellationToken.None);
again.ShouldBe([new RetentionPurgeResult("Sample", 0), new RetentionPurgeResult("Event", 0)]);
again.ShouldBe([
new RetentionPurgeResult("Sample", 0),
new RetentionPurgeResult("SampleModified", 0),
new RetentionPurgeResult("Event", 0),
]);
}
[Fact]