Joseph Doherty
cfd403710e
Every HistoryUpdate a SQL backend applies now leaves a modification trail
(prior value, principal, Part 11 update type, modification time), and
HistoryReadRaw with IsReadModified=true serves it as HistoryModifiedData.
- Core: HistoryUpdateContext threaded into all three IHistorianDataUpdater
methods; IHistorianDataSource.ReadModifiedAsync (default NotSupported =>
BadHistoryOperationUnsupported on Gateway/Null); HistoryModified* contracts.
- SqlServer: schema v2 (002_sample_modified.sql, RequiredVersion=2, docs copy);
trail written by OUTPUT ... INTO on the same MERGE/DELETE statement, so a
failed update leaves no audit row; retention purges SampleModified per tag on
the source-time cutoff; ReadModifiedAsync keyed on (SourceTimeUtc, ModifiedId).
- QuestDB: append-only {prefix}sample_modified (TTL like samples), trail row
written BEFORE the change (orphan-on-crash, never an untracked change);
existence reads now carry the prior value; ReadModifiedAsync.
- Node manager: modified reads dispatch + page via the #669 continuation store
(HistoryContinuationState gains HistoryCursorKind + LastModifiedId), gated by
the existing HistoryRead decision; session principal / key id passed to the
updater. IsDeleteModified stays unsupported.
- Client.CLI: historyread --modified.
- Tests: node-manager dispatch/paging/cursor binding/principal threading,
over-the-wire HistoryModifiedData, shared ModifiedValueConformance on both
backends, SqlServer atomicity (forced trail failure => no change) + v1->v2
upgrade + retention trail purge.
- Docs: Historian.md (Modified-value history section, schema v2 upgrade note),
security.md, Client.CLI.md, CLAUDE.md.
208 lines
10 KiB
C#
208 lines
10 KiB
C#
using Microsoft.Data.SqlClient;
|
|
using Shouldly;
|
|
using Xunit;
|
|
using ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common;
|
|
|
|
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer.IntegrationTests;
|
|
|
|
[Collection(SqlHistorianCollection.Name)]
|
|
public sealed class SchemaBootstrapIntegrationTests(SqlHistorianFixture fx)
|
|
{
|
|
private SqlServerHistorianOptions Opts(string schema, bool auto = true) => new()
|
|
{
|
|
ConnectionString = fx.ConnectionString, Schema = schema, AutoCreateSchema = auto,
|
|
};
|
|
|
|
private static SqlServerSchemaBootstrapper Boot(SqlServerHistorianOptions o) =>
|
|
new(new SqlHistorianConnection(o), o);
|
|
|
|
private async Task<long> ScalarAsync(string sql)
|
|
{
|
|
await using var c = new SqlConnection(fx.ConnectionString);
|
|
await c.OpenAsync();
|
|
await using var cmd = c.CreateCommand();
|
|
cmd.CommandText = sql;
|
|
return Convert.ToInt64(await cmd.ExecuteScalarAsync());
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Bootstrapping_twice_is_a_no_op_with_one_version_row()
|
|
{
|
|
fx.SkipIfUnavailable();
|
|
var s = SqlHistorianFixture.UniqueSchema();
|
|
await Boot(Opts(s)).EnsureAsync(CancellationToken.None);
|
|
await Boot(Opts(s)).EnsureAsync(CancellationToken.None); // fresh instance => re-runs the guarded path
|
|
(await ScalarAsync($"SELECT COUNT(*) FROM [{s}].SchemaVersion")).ShouldBe(SqlServerSchemaBootstrapper.RequiredVersion);
|
|
(await ScalarAsync($"SELECT COUNT(*) FROM sys.indexes WHERE name='CX_Event_Time' AND object_id=OBJECT_ID('[{s}].Event')")).ShouldBe(1);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task A_re_run_of_the_raw_script_is_idempotent()
|
|
{
|
|
fx.SkipIfUnavailable();
|
|
var s = SqlHistorianFixture.UniqueSchema();
|
|
await Boot(Opts(s)).EnsureAsync(CancellationToken.None);
|
|
var script = new StreamReader(typeof(SqlServerSchemaBootstrapper).Assembly.GetManifestResourceStream("Schema.001_initial.sql")!).ReadToEnd();
|
|
await using var c = new SqlConnection(fx.ConnectionString);
|
|
await c.OpenAsync();
|
|
foreach (var b in SqlServerSchemaBootstrapper.SplitBatches(script.Replace("$(Schema)", s)))
|
|
{
|
|
await using var cmd = c.CreateCommand();
|
|
cmd.CommandText = b;
|
|
await cmd.ExecuteNonQueryAsync();
|
|
}
|
|
(await ScalarAsync($"SELECT COUNT(*) FROM [{s}].SchemaVersion")).ShouldBe(SqlServerSchemaBootstrapper.RequiredVersion);
|
|
}
|
|
|
|
/// <summary>
|
|
/// #668 upgrade path: a schema left at version 1 (only <c>001_initial.sql</c> applied, as every pre-#668 install is)
|
|
/// is upgraded in place by an auto-creating bootstrap — <c>SampleModified</c> appears, both version rows exist, and
|
|
/// existing samples are untouched. With <c>AutoCreateSchema=false</c> the same v1 schema is refused.
|
|
/// </summary>
|
|
[Fact]
|
|
public async Task A_version_1_schema_is_upgraded_to_version_2_in_place()
|
|
{
|
|
fx.SkipIfUnavailable();
|
|
var s = SqlHistorianFixture.UniqueSchema();
|
|
var v1 = new StreamReader(typeof(SqlServerSchemaBootstrapper).Assembly.GetManifestResourceStream("Schema.001_initial.sql")!).ReadToEnd();
|
|
await using (var c = new SqlConnection(fx.ConnectionString))
|
|
{
|
|
await c.OpenAsync();
|
|
foreach (var b in SqlServerSchemaBootstrapper.SplitBatches(v1.Replace("$(Schema)", s)))
|
|
{
|
|
await using var cmd = c.CreateCommand();
|
|
cmd.CommandText = b;
|
|
await cmd.ExecuteNonQueryAsync();
|
|
}
|
|
await using var seed = c.CreateCommand();
|
|
seed.CommandText = $"INSERT [{s}].Tag (Name, DataType, CreatedUtc, UpdatedUtc) VALUES (N'old', 1, SYSUTCDATETIME(), SYSUTCDATETIME()); " +
|
|
$"INSERT [{s}].Sample (TagId, SourceTimeUtc, ValueKind, NumValue, StatusCode, ServerTimeUtc) SELECT TagId, '2026-01-01', 1, 4.0, 0, SYSUTCDATETIME() FROM [{s}].Tag";
|
|
await seed.ExecuteNonQueryAsync();
|
|
}
|
|
(await ScalarAsync($"SELECT COUNT(*) FROM sys.tables WHERE name = 'SampleModified' AND schema_id = SCHEMA_ID('{s}')")).ShouldBe(0);
|
|
|
|
var refused = await Should.ThrowAsync<HistorianSchemaMissingException>(
|
|
() => Boot(Opts(s, auto: false)).EnsureAsync(CancellationToken.None));
|
|
refused.Message.ShouldContain("missing version 2 (found 1)");
|
|
|
|
await Boot(Opts(s)).EnsureAsync(CancellationToken.None);
|
|
|
|
(await ScalarAsync($"SELECT COUNT(*) FROM [{s}].SchemaVersion")).ShouldBe(2);
|
|
(await ScalarAsync($"SELECT MAX(Version) FROM [{s}].SchemaVersion")).ShouldBe(2);
|
|
(await ScalarAsync($"SELECT COUNT(*) FROM [{s}].SampleModified")).ShouldBe(0);
|
|
(await ScalarAsync($"SELECT COUNT(*) FROM [{s}].Sample")).ShouldBe(1);
|
|
await Boot(Opts(s, auto: false)).EnsureAsync(CancellationToken.None); // now satisfies the verify-only path
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Concurrent_bootstrappers_both_succeed_with_one_version_row()
|
|
{
|
|
fx.SkipIfUnavailable();
|
|
var s = SqlHistorianFixture.UniqueSchema();
|
|
var tasks = Enumerable.Range(0, 4).Select(_ => Task.Run(() => Boot(Opts(s)).EnsureAsync(CancellationToken.None))).ToArray();
|
|
await Task.WhenAll(tasks);
|
|
(await ScalarAsync($"SELECT COUNT(*) FROM [{s}].SchemaVersion")).ShouldBe(SqlServerSchemaBootstrapper.RequiredVersion);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task AutoCreateSchema_false_on_an_empty_schema_throws_missing()
|
|
{
|
|
fx.SkipIfUnavailable();
|
|
var ex = await Should.ThrowAsync<HistorianSchemaMissingException>(
|
|
() => Boot(Opts(SqlHistorianFixture.UniqueSchema(), auto: false)).EnsureAsync(CancellationToken.None));
|
|
ex.Message.ShouldContain("missing version 2");
|
|
}
|
|
|
|
[Fact]
|
|
public async Task AutoCreateSchema_false_passes_once_the_schema_exists_and_failure_is_not_latched()
|
|
{
|
|
fx.SkipIfUnavailable();
|
|
var s = SqlHistorianFixture.UniqueSchema();
|
|
var clock = new FixedClock(DateTimeOffset.UtcNow);
|
|
var o = Opts(s, auto: false);
|
|
var checker = new SqlServerSchemaBootstrapper(new SqlHistorianConnection(o), o, timeProvider: clock);
|
|
await Should.ThrowAsync<HistorianSchemaMissingException>(() => checker.EnsureAsync(CancellationToken.None));
|
|
await Boot(Opts(s)).EnsureAsync(CancellationToken.None);
|
|
clock.Now += BootstrapFailureCache.DefaultDuration;
|
|
await checker.EnsureAsync(CancellationToken.None); // same instance retries once the failure cache expires
|
|
}
|
|
|
|
/// <summary>
|
|
/// M-2: within <see cref="BootstrapFailureCache.DefaultDuration"/> of a failure the bootstrapper
|
|
/// re-throws it without touching SQL — proven by creating the schema in between: a call that went to SQL
|
|
/// would now pass.
|
|
/// </summary>
|
|
[Fact]
|
|
public async Task A_bootstrap_failure_is_cached_for_five_seconds_then_retried()
|
|
{
|
|
fx.SkipIfUnavailable();
|
|
var s = SqlHistorianFixture.UniqueSchema();
|
|
var clock = new FixedClock(DateTimeOffset.UtcNow);
|
|
var o = Opts(s, auto: false);
|
|
var checker = new SqlServerSchemaBootstrapper(new SqlHistorianConnection(o), o, timeProvider: clock);
|
|
await Should.ThrowAsync<HistorianSchemaMissingException>(() => checker.EnsureAsync(CancellationToken.None));
|
|
|
|
await Boot(Opts(s)).EnsureAsync(CancellationToken.None); // the schema now exists
|
|
|
|
clock.Now += BootstrapFailureCache.DefaultDuration - TimeSpan.FromTicks(1);
|
|
await Should.ThrowAsync<HistorianSchemaMissingException>(() => checker.EnsureAsync(CancellationToken.None));
|
|
|
|
clock.Now += TimeSpan.FromTicks(1);
|
|
await checker.EnsureAsync(CancellationToken.None);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Concurrent_get_or_create_of_one_new_tag_yields_one_row_and_one_id()
|
|
{
|
|
fx.SkipIfUnavailable();
|
|
var s = SqlHistorianFixture.UniqueSchema();
|
|
var o = Opts(s);
|
|
var conn = new SqlHistorianConnection(o);
|
|
var boot = new SqlServerSchemaBootstrapper(conn, o);
|
|
await boot.EnsureAsync(CancellationToken.None);
|
|
// Separate catalogs => no shared cache, so every task really races the MERGE. Several rounds on fresh
|
|
// names, all callers released together, raise the odds of hitting the create window.
|
|
for (var round = 0; round < 10; round++)
|
|
{
|
|
var name = $"Plant/Tag{round}";
|
|
using var start = new ManualResetEventSlim(false);
|
|
var ids = Enumerable.Range(0, 32).Select(_ => Task.Run(async () =>
|
|
{
|
|
var cat = new SqlServerTagCatalog(conn, boot);
|
|
await cat.TryGetTagIdAsync("warm", CancellationToken.None); // opens the pool + bootstrap before the race
|
|
start.Wait();
|
|
return await cat.GetOrCreateTagIdAsync(name, 1, CancellationToken.None);
|
|
})).ToArray();
|
|
await Task.Delay(200);
|
|
start.Set();
|
|
(await Task.WhenAll(ids)).Distinct().Count().ShouldBe(1);
|
|
(await ScalarAsync($"SELECT COUNT(*) FROM [{s}].Tag WHERE Name = N'{name}'")).ShouldBe(1);
|
|
}
|
|
(await new SqlServerTagCatalog(conn, boot).TryGetTagIdAsync("nope", CancellationToken.None)).ShouldBeNull();
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Sample_tvp_keeps_sub_millisecond_ticks_end_to_end()
|
|
{
|
|
fx.SkipIfUnavailable();
|
|
var s = SqlHistorianFixture.UniqueSchema();
|
|
var o = Opts(s);
|
|
var conn = new SqlHistorianConnection(o);
|
|
var boot = new SqlServerSchemaBootstrapper(conn, o);
|
|
var tagId = await new SqlServerTagCatalog(conn, boot).GetOrCreateTagIdAsync("T", 1, CancellationToken.None);
|
|
var t = new DateTime(2026, 10, 9, 0, 0, 0, DateTimeKind.Utc).AddTicks(1234567);
|
|
await using var c = await conn.OpenAsync(CancellationToken.None);
|
|
await using var cmd = conn.CreateCommand(c,
|
|
$"INSERT {conn.Q("Sample")} (TagId, SourceTimeUtc, ValueKind, NumValue, IntValue, StrValue, StatusCode, ServerTimeUtc) " +
|
|
"SELECT @id, SourceTimeUtc, ValueKind, NumValue, IntValue, StrValue, StatusCode, SYSUTCDATETIME() FROM @rows");
|
|
cmd.Parameters.AddWithValue("@id", tagId);
|
|
conn.AddStructured(cmd, "@rows", "SampleRow",
|
|
SampleRowMapper.ToSampleRecords([new HistorianSampleRow(t, ZB.MOM.WW.OtOpcUa.Core.Abstractions.HistorianValue.FromDouble(1), 0u)]));
|
|
await cmd.ExecuteNonQueryAsync();
|
|
await using var q = conn.CreateCommand(c, $"SELECT SourceTimeUtc FROM {conn.Q("Sample")}");
|
|
await using var r = await q.ExecuteReaderAsync();
|
|
(await r.ReadAsync()).ShouldBeTrue();
|
|
SqlHistorianConnection.GetUtc(r, 0).Ticks.ShouldBe(t.Ticks);
|
|
}
|
|
}
|