dohertj2/lmxopcua
1
0
Fork 0
Code Issues 32 Pull Requests Actions Packages Projects Releases Wiki Activity

Merge feat/668-modified-values: Part 11 modified-value history (IsReadModified) on SqlServer + QuestDB (#668)

Browse Source
This commit is contained in:
Joseph Doherty
2026-10-11 09:32:13 -04:00
parent 699871040a cfd403710e
commit 062086f9a6
51 changed files with 2173 additions and 195 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer.IntegrationTests/ModifiedHistoryAtomicityTests.cs
+112
View File
@@ -0,0 +1,112 @@
using Microsoft.Data.SqlClient;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Driver.Historian.Conformance;
using ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common;
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer.IntegrationTests;
/// <summary>
/// #668, SQL Server only: the modification-trail row is written by the SAME statement that changes
/// <c>hist.Sample</c> (<c>OUTPUT … INTO</c>), so a change and its trail row commit or roll back together. The failure
/// is forced INSIDE the audit insert (a unique index on the trail that the second modification of one timestamp
/// violates), i.e. after the sample row was already changed within the statement — the strongest place to prove it.
/// Each test has a positive control: the same operation succeeds, with its trail row, once the index is dropped.
/// </summary>
[Collection(SqlHistorianCollection.Name)]
public sealed class ModifiedHistoryAtomicityTests(SqlHistorianFixture fx)
{
private static readonly DateTime T1 = new(2026, 1, 1, 0, 0, 1, DateTimeKind.Utc);
private static readonly DateTime T2 = T1.AddSeconds(1);
private static readonly HistoryUpdateContext Who = ConformanceData.Editor;
private static HistorizationValue V(DateTime t, double d) => new(t, HistorianValue.FromDouble(d), 0u);
private async Task ExecAsync(ReadTestDb db, string sql)
{
await using var c = await db.Conn.OpenAsync(CancellationToken.None);
await using var cmd = db.Conn.CreateCommand(c, sql);
await cmd.ExecuteNonQueryAsync();
}
private async Task<long> CountTrailAsync(ReadTestDb db)
{
await using var c = await db.Conn.OpenAsync(CancellationToken.None);
await using var cmd = db.Conn.CreateCommand(c, $"SELECT COUNT(*) FROM {db.Conn.Q("SampleModified")}");
return Convert.ToInt64(await cmd.ExecuteScalarAsync());
}
private Task BreakTheTrailAsync(ReadTestDb db) => ExecAsync(db,
$"CREATE UNIQUE INDEX UX_Break ON {db.Conn.Q("SampleModified")} (TagId, SourceTimeUtc)");
private Task RepairTheTrailAsync(ReadTestDb db) => ExecAsync(db,
$"DROP INDEX UX_Break ON {db.Conn.Q("SampleModified")}");
private static async Task<double> ValueAtAsync(ReadTestDb db, string tag, DateTime t)
{
using var src = db.Source();
var r = await src.ReadRawAsync(tag, t, t, 0, CancellationToken.None);
return (double)r.Samples.ShouldHaveSingleItem().Value!;
}
[Fact]
public async Task A_replace_whose_trail_insert_fails_changes_nothing()
{
var db = await ReadTestDb.CreateAsync(fx);
var u = new SqlServerHistorianDataUpdater(db.Conn, new SqlServerTagCatalog(db.Conn, db.Boot));
(await u.UpdateDataAsync("t", HistoryUpdateMode.Insert, [V(T1, 1)], Who, CancellationToken.None))
.ShouldBe([HistorianStatusCodes.GoodEntryInserted]);
(await CountTrailAsync(db)).ShouldBe(1);
await BreakTheTrailAsync(db);
await Should.ThrowAsync<SqlException>(
() => u.UpdateDataAsync("t", HistoryUpdateMode.Replace, [V(T1, 99)], Who, CancellationToken.None));
(await ValueAtAsync(db, "t", T1)).ShouldBe(1d, "the sample change rolled back with its trail row");
(await CountTrailAsync(db)).ShouldBe(1, "a failed update leaves no trail row");
// Positive control: the identical replace succeeds, with its trail row, once the trail accepts it.
await RepairTheTrailAsync(db);
(await u.UpdateDataAsync("t", HistoryUpdateMode.Replace, [V(T1, 99)], Who, CancellationToken.None))
.ShouldBe([HistorianStatusCodes.GoodEntryReplaced]);
(await ValueAtAsync(db, "t", T1)).ShouldBe(99d);
(await CountTrailAsync(db)).ShouldBe(2);
}
[Fact]
public async Task Deletes_whose_trail_insert_fails_delete_nothing()
{
var db = await ReadTestDb.CreateAsync(fx);
var u = new SqlServerHistorianDataUpdater(db.Conn, new SqlServerTagCatalog(db.Conn, db.Boot));
await u.UpdateDataAsync("t", HistoryUpdateMode.Insert, [V(T1, 1), V(T2, 2)], Who, CancellationToken.None);
await BreakTheTrailAsync(db);
await Should.ThrowAsync<SqlException>(() => u.DeleteAtTimeAsync("t", [T1], Who, CancellationToken.None));
await Should.ThrowAsync<SqlException>(() => u.DeleteRawAsync("t", T1, T2.AddSeconds(1), Who, CancellationToken.None));
(await ValueAtAsync(db, "t", T1)).ShouldBe(1d);
(await ValueAtAsync(db, "t", T2)).ShouldBe(2d);
(await CountTrailAsync(db)).ShouldBe(2, "only the two inserts");
await RepairTheTrailAsync(db);
(await u.DeleteAtTimeAsync("t", [T1], Who, CancellationToken.None)).ShouldBe([HistorianStatusCodes.Good]);
(await u.DeleteRawAsync("t", T1, T2.AddSeconds(1), Who, CancellationToken.None)).ShouldBe(HistorianStatusCodes.Good);
(await CountTrailAsync(db)).ShouldBe(4);
}
[Fact]
public async Task An_over_long_principal_is_truncated_never_refused()
{
var db = await ReadTestDb.CreateAsync(fx);
var u = new SqlServerHistorianDataUpdater(db.Conn, new SqlServerTagCatalog(db.Conn, db.Boot));
var who = new HistoryUpdateContext(new string('p', 500), new string('k', 500));
(await u.UpdateDataAsync("t", HistoryUpdateMode.Insert, [V(T1, 1)], who, CancellationToken.None))
.ShouldBe([HistorianStatusCodes.GoodEntryInserted]);
using var src = db.Source();
var trail = await src.ReadModifiedAsync("t", T1.AddHours(-1), T1.AddHours(1), 0, null, CancellationToken.None);
trail.Values.ShouldHaveSingleItem().UserName.ShouldBe(new string('p', HistoryUpdateContext.MaxIdentityLength));
}
}
tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer.IntegrationTests/RawReadTests.cs
+1 -1
View File
@@ -30,7 +30,7 @@ public sealed class RawReadTests(SqlHistorianFixture fx)
var h = src.GetHealthSnapshot();
h.TotalFailures.ShouldBe(2);
h.ConsecutiveFailures.ShouldBe(2);
h.LastError!.ShouldContain("missing version 1");
h.LastError!.ShouldContain($"missing version {SqlServerSchemaBootstrapper.RequiredVersion}");
h.ProcessConnectionOpen.ShouldBeFalse();
// Another node creates the schema; the same data source then succeeds (bootstrap failure is not latched).
tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer.IntegrationTests/RetentionTests.cs
+20 -2
View File
@@ -56,6 +56,9 @@ public sealed class RetentionTests(SqlHistorianFixture fx)
sql.Append($"SET @i = 0; WHILE @i < 2 BEGIN INSERT [{s}].Sample (TagId, SourceTimeUtc, ValueKind, NumValue, StatusCode, ServerTimeUtc) SELECT TagId, DATEADD(minute, @i, '{fresh}'), 1, 1.0, 0, SYSUTCDATETIME() FROM [{s}].Tag; SET @i += 1; END;");
sql.Append($"SET @i = 0; WHILE @i < 4 BEGIN INSERT [{s}].Event (EventId, SourceName, AlarmId, AlarmName, AlarmTypeName, Severity, EventKind, EventTimeUtc, ReceivedTimeUtc) VALUES (CONCAT('old', @i), 's', 'a', 'n', 't', 1, 'Activated', DATEADD(minute, @i, '{old}'), SYSUTCDATETIME()); SET @i += 1; END;");
sql.Append($"INSERT [{s}].Event (EventId, SourceName, AlarmId, AlarmName, AlarmTypeName, Severity, EventKind, EventTimeUtc, ReceivedTimeUtc) VALUES ('new0', 's', 'a', 'n', 't', 1, 'Activated', '{fresh}', SYSUTCDATETIME());");
// #668 modification trail: 2 old + 1 fresh row per tag (by SOURCE time — the modification time is "now" on all).
sql.Append($"SET @i = 0; WHILE @i < 2 BEGIN INSERT [{s}].SampleModified (TagId, SourceTimeUtc, ModificationTimeUtc, UpdateType, Principal, ValueKind, NumValue, StatusCode, ServerTimeUtc) SELECT TagId, DATEADD(minute, @i, '{old}'), SYSUTCDATETIME(), 2, N'p', 1, 1.0, 0, SYSUTCDATETIME() FROM [{s}].Tag; SET @i += 1; END;");
sql.Append($"INSERT [{s}].SampleModified (TagId, SourceTimeUtc, ModificationTimeUtc, UpdateType, Principal) SELECT TagId, '{fresh}', SYSUTCDATETIME(), 1, N'p' FROM [{s}].Tag;");
await ExecAsync(sql.ToString());
}
@@ -67,11 +70,22 @@ public sealed class RetentionTests(SqlHistorianFixture fx)
fx.SkipIfUnavailable();
var (c, b, _, s) = Make();
await SeedAsync(b, s);
var identBefore = await CountAsync($"SELECT IDENT_CURRENT('[{s}].SampleModified')");
var purger = new SqlServerHistorianRetention(c, b, TimeSpan.FromHours(1), 30, batch, time: new FixedTime(Now));
var results = await purger.PurgeAsync(CancellationToken.None);
results.ShouldBe([new RetentionPurgeResult("Sample", 15), new RetentionPurgeResult("Event", 4)]);
// #668: a purge never WRITES to the modification trail. The identity seed proves no row was ever inserted —
// a row-count check could not, since the trail purge would delete an old-source-time audit row again.
(await CountAsync($"SELECT IDENT_CURRENT('[{s}].SampleModified')")).ShouldBe(identBefore);
results.ShouldBe([
new RetentionPurgeResult("Sample", 15),
new RetentionPurgeResult("SampleModified", 6),
new RetentionPurgeResult("Event", 4),
]);
(await CountAsync($"SELECT COUNT(*) FROM [{s}].SampleModified")).ShouldBe(3);
(await CountAsync($"SELECT COUNT(*) FROM [{s}].SampleModified WHERE SourceTimeUtc < '{Now.AddDays(-30):yyyy-MM-ddTHH:mm:ss}'")).ShouldBe(0);
(await CountAsync($"SELECT COUNT(*) FROM [{s}].Sample")).ShouldBe(6);
(await CountAsync($"SELECT COUNT(*) FROM [{s}].Sample WHERE SourceTimeUtc < '{Now.AddDays(-30):yyyy-MM-ddTHH:mm:ss}'")).ShouldBe(0);
(await CountAsync($"SELECT COUNT(*) FROM [{s}].Event")).ShouldBe(1);
@@ -87,7 +101,11 @@ public sealed class RetentionTests(SqlHistorianFixture fx)
var purger = new SqlServerHistorianRetention(c, b, TimeSpan.FromHours(1), 30, 4000, time: new FixedTime(Now));
await purger.PurgeAsync(CancellationToken.None);
var again = await purger.PurgeAsync(CancellationToken.None);
again.ShouldBe([new RetentionPurgeResult("Sample", 0), new RetentionPurgeResult("Event", 0)]);
again.ShouldBe([
new RetentionPurgeResult("Sample", 0),
new RetentionPurgeResult("SampleModified", 0),
new RetentionPurgeResult("Event", 0),
]);
}
[Fact]
tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer.IntegrationTests/SchemaBootstrapIntegrationTests.cs
+44 -4
View File
@@ -32,7 +32,7 @@ public sealed class SchemaBootstrapIntegrationTests(SqlHistorianFixture fx)
var s = SqlHistorianFixture.UniqueSchema();
await Boot(Opts(s)).EnsureAsync(CancellationToken.None);
await Boot(Opts(s)).EnsureAsync(CancellationToken.None); // fresh instance => re-runs the guarded path
(await ScalarAsync($"SELECT COUNT(*) FROM [{s}].SchemaVersion")).ShouldBe(1);
(await ScalarAsync($"SELECT COUNT(*) FROM [{s}].SchemaVersion")).ShouldBe(SqlServerSchemaBootstrapper.RequiredVersion);
(await ScalarAsync($"SELECT COUNT(*) FROM sys.indexes WHERE name='CX_Event_Time' AND object_id=OBJECT_ID('[{s}].Event')")).ShouldBe(1);
}
@@ -51,7 +51,47 @@ public sealed class SchemaBootstrapIntegrationTests(SqlHistorianFixture fx)
cmd.CommandText = b;
await cmd.ExecuteNonQueryAsync();
}
(await ScalarAsync($"SELECT COUNT(*) FROM [{s}].SchemaVersion")).ShouldBe(1);
(await ScalarAsync($"SELECT COUNT(*) FROM [{s}].SchemaVersion")).ShouldBe(SqlServerSchemaBootstrapper.RequiredVersion);
}
/// <summary>
/// #668 upgrade path: a schema left at version 1 (only <c>001_initial.sql</c> applied, as every pre-#668 install is)
/// is upgraded in place by an auto-creating bootstrap — <c>SampleModified</c> appears, both version rows exist, and
/// existing samples are untouched. With <c>AutoCreateSchema=false</c> the same v1 schema is refused.
/// </summary>
[Fact]
public async Task A_version_1_schema_is_upgraded_to_version_2_in_place()
{
fx.SkipIfUnavailable();
var s = SqlHistorianFixture.UniqueSchema();
var v1 = new StreamReader(typeof(SqlServerSchemaBootstrapper).Assembly.GetManifestResourceStream("Schema.001_initial.sql")!).ReadToEnd();
await using (var c = new SqlConnection(fx.ConnectionString))
{
await c.OpenAsync();
foreach (var b in SqlServerSchemaBootstrapper.SplitBatches(v1.Replace("$(Schema)", s)))
{
await using var cmd = c.CreateCommand();
cmd.CommandText = b;
await cmd.ExecuteNonQueryAsync();
}
await using var seed = c.CreateCommand();
seed.CommandText = $"INSERT [{s}].Tag (Name, DataType, CreatedUtc, UpdatedUtc) VALUES (N'old', 1, SYSUTCDATETIME(), SYSUTCDATETIME()); " +
$"INSERT [{s}].Sample (TagId, SourceTimeUtc, ValueKind, NumValue, StatusCode, ServerTimeUtc) SELECT TagId, '2026-01-01', 1, 4.0, 0, SYSUTCDATETIME() FROM [{s}].Tag";
await seed.ExecuteNonQueryAsync();
}
(await ScalarAsync($"SELECT COUNT(*) FROM sys.tables WHERE name = 'SampleModified' AND schema_id = SCHEMA_ID('{s}')")).ShouldBe(0);
var refused = await Should.ThrowAsync<HistorianSchemaMissingException>(
() => Boot(Opts(s, auto: false)).EnsureAsync(CancellationToken.None));
refused.Message.ShouldContain("missing version 2 (found 1)");
await Boot(Opts(s)).EnsureAsync(CancellationToken.None);
(await ScalarAsync($"SELECT COUNT(*) FROM [{s}].SchemaVersion")).ShouldBe(2);
(await ScalarAsync($"SELECT MAX(Version) FROM [{s}].SchemaVersion")).ShouldBe(2);
(await ScalarAsync($"SELECT COUNT(*) FROM [{s}].SampleModified")).ShouldBe(0);
(await ScalarAsync($"SELECT COUNT(*) FROM [{s}].Sample")).ShouldBe(1);
await Boot(Opts(s, auto: false)).EnsureAsync(CancellationToken.None); // now satisfies the verify-only path
}
[Fact]
@@ -61,7 +101,7 @@ public sealed class SchemaBootstrapIntegrationTests(SqlHistorianFixture fx)
var s = SqlHistorianFixture.UniqueSchema();
var tasks = Enumerable.Range(0, 4).Select(_ => Task.Run(() => Boot(Opts(s)).EnsureAsync(CancellationToken.None))).ToArray();
await Task.WhenAll(tasks);
(await ScalarAsync($"SELECT COUNT(*) FROM [{s}].SchemaVersion")).ShouldBe(1);
(await ScalarAsync($"SELECT COUNT(*) FROM [{s}].SchemaVersion")).ShouldBe(SqlServerSchemaBootstrapper.RequiredVersion);
}
[Fact]
@@ -70,7 +110,7 @@ public sealed class SchemaBootstrapIntegrationTests(SqlHistorianFixture fx)
fx.SkipIfUnavailable();
var ex = await Should.ThrowAsync<HistorianSchemaMissingException>(
() => Boot(Opts(SqlHistorianFixture.UniqueSchema(), auto: false)).EnsureAsync(CancellationToken.None));
ex.Message.ShouldContain("missing version 1");
ex.Message.ShouldContain("missing version 2");
}
[Fact]
tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer.IntegrationTests/SqlServerConformance.cs
+7
View File
@@ -53,3 +53,10 @@ public sealed class SqlServerTagNameConformance(SqlHistorianFixture fx) : TagNam
/// <inheritdoc/>
protected override Task<IConformanceBackend> CreateBackendAsync() => SqlServerConformanceBackend.CreateAsync(fx);
}
[Collection(SqlHistorianCollection.Name)]
public sealed class SqlServerModifiedValueConformance(SqlHistorianFixture fx) : ModifiedValueConformance
{
/// <inheritdoc/>
protected override Task<IConformanceBackend> CreateBackendAsync() => SqlServerConformanceBackend.CreateAsync(fx);
}