Joseph Doherty
31ca9407a6
The ScadaBridge central shared-store live gate proved SqlServerSecretsStoreMigrator's
"safe to run concurrently" claim false on a virgin database: Serializable isolation does
not serialize the IF NOT EXISTS metadata probes against another session's in-flight
CREATE, so of two central nodes booting together the loser died with SqlException 2714
("There is already an object named 'zbsecrets'") — the retry filter covered only
deadlock 1205. Losing such a race means the object now exists (the desired end state),
so 2714 (schema/table), 1913 (index) and 2627 (schema_version MERGE-insert PK) are now
retryable alongside 1205; the retry re-runs the idempotent IF NOT EXISTS path.
Classification scans the whole SqlException.Errors collection (2714 arrives stacked with
2759), the retry cap keeps genuine conflicts failing closed, and 2705 is deliberately
excluded (unreachable — columns live in a single CREATE TABLE statement). Predicate
extracted as an internal static pinned by offline tests; live suite gains an env-gated
4-concurrent-migrators virgin-database test. Six packages pushed to the Gitea feed at
0.5.1.
Claude-Session: https://claude.ai/code/session_014WNM4vjoVksyyBraTXSZE1
397 lines
17 KiB
C#
397 lines
17 KiB
C#
using ZB.MOM.WW.Secrets.Abstractions;
|
|
|
|
namespace ZB.MOM.WW.Secrets.Replicator.SqlServer.Tests.Live;
|
|
|
|
/// <summary>
|
|
/// The SQLite store's test suite, ported case-for-case onto the SQL-Server store.
|
|
/// </summary>
|
|
/// <remarks>
|
|
/// Ported rather than newly written on purpose. The two stores must be behaviourally identical —
|
|
/// a cluster can hold both at once — so the strongest available check is that the assertions
|
|
/// written against SQLite pass unchanged here. Anywhere the T-SQL diverges in observable behaviour,
|
|
/// one of these fails.
|
|
/// </remarks>
|
|
[Collection("live-sqlserver")]
|
|
public sealed class SqlServerSecretStoreLiveTests : IAsyncLifetime
|
|
{
|
|
private readonly string _schema = LiveSqlServer.NewSchemaName();
|
|
private SqlServerSecretStore _store = null!;
|
|
|
|
public async Task InitializeAsync()
|
|
{
|
|
if (!LiveSqlServer.IsEnabled)
|
|
{
|
|
return;
|
|
}
|
|
|
|
var factory = new SecretsSqlServerConnectionFactory(LiveSqlServer.OptionsFor(_schema));
|
|
await new SqlServerSecretsStoreMigrator(factory).MigrateAsync(CancellationToken.None);
|
|
_store = new SqlServerSecretStore(factory);
|
|
}
|
|
|
|
public Task DisposeAsync() => LiveSqlServer.DropSchemaAsync(_schema);
|
|
|
|
private static StoredSecret MakeSecret(
|
|
string name,
|
|
long revision = 0,
|
|
byte[]? ciphertext = null,
|
|
DateTimeOffset? updatedUtc = null,
|
|
DateTimeOffset? createdUtc = null,
|
|
bool isDeleted = false,
|
|
DateTimeOffset? deletedUtc = null,
|
|
string? createdBy = "alice",
|
|
string? updatedBy = "alice") => new()
|
|
{
|
|
Name = new SecretName(name),
|
|
Description = "desc",
|
|
ContentType = SecretContentType.ConnectionString,
|
|
Ciphertext = ciphertext ?? [1, 2, 3],
|
|
Nonce = [4, 5, 6],
|
|
Tag = [7, 8, 9],
|
|
WrappedDek = [10, 11, 12],
|
|
WrapNonce = [13, 14, 15],
|
|
WrapTag = [16, 17, 18],
|
|
KekId = "kek-1",
|
|
Revision = revision,
|
|
IsDeleted = isDeleted,
|
|
DeletedUtc = deletedUtc,
|
|
CreatedUtc = createdUtc ?? DateTimeOffset.UtcNow,
|
|
UpdatedUtc = updatedUtc ?? DateTimeOffset.UtcNow,
|
|
CreatedBy = createdBy,
|
|
UpdatedBy = updatedBy,
|
|
};
|
|
|
|
[SkippableFact]
|
|
public async Task Upsert_Then_Get_RoundTrips()
|
|
{
|
|
Skip.IfNot(LiveSqlServer.IsEnabled, LiveSqlServer.SkipReason);
|
|
|
|
await _store.UpsertAsync(MakeSecret("app/db-conn"), CancellationToken.None);
|
|
|
|
StoredSecret? got = await _store.GetAsync(new SecretName("app/db-conn"), CancellationToken.None);
|
|
|
|
Assert.NotNull(got);
|
|
Assert.Equal("app/db-conn", got!.Name.Value);
|
|
Assert.Equal("desc", got.Description);
|
|
Assert.Equal(SecretContentType.ConnectionString, got.ContentType);
|
|
Assert.Equal(new byte[] { 1, 2, 3 }, got.Ciphertext);
|
|
Assert.Equal(new byte[] { 4, 5, 6 }, got.Nonce);
|
|
Assert.Equal(new byte[] { 7, 8, 9 }, got.Tag);
|
|
Assert.Equal(new byte[] { 10, 11, 12 }, got.WrappedDek);
|
|
Assert.Equal(new byte[] { 13, 14, 15 }, got.WrapNonce);
|
|
Assert.Equal(new byte[] { 16, 17, 18 }, got.WrapTag);
|
|
Assert.Equal("kek-1", got.KekId);
|
|
Assert.Equal(0, got.Revision);
|
|
Assert.False(got.IsDeleted);
|
|
Assert.Null(got.DeletedUtc);
|
|
Assert.Equal("alice", got.CreatedBy);
|
|
Assert.Equal("alice", got.UpdatedBy);
|
|
}
|
|
|
|
[SkippableFact]
|
|
public async Task Get_ReturnsNull_WhenAbsent()
|
|
{
|
|
Skip.IfNot(LiveSqlServer.IsEnabled, LiveSqlServer.SkipReason);
|
|
|
|
Assert.Null(await _store.GetAsync(new SecretName("nope"), CancellationToken.None));
|
|
}
|
|
|
|
[SkippableFact]
|
|
public async Task Upsert_Existing_OverwritesInPlace_BumpsRevision()
|
|
{
|
|
Skip.IfNot(LiveSqlServer.IsEnabled, LiveSqlServer.SkipReason);
|
|
|
|
await _store.UpsertAsync(
|
|
MakeSecret("app/rotating", ciphertext: [1, 1, 1], createdBy: "alice", updatedBy: "alice"),
|
|
CancellationToken.None);
|
|
|
|
StoredSecret afterFirst =
|
|
(await _store.GetAsync(new SecretName("app/rotating"), CancellationToken.None))!;
|
|
Assert.Equal(0, afterFirst.Revision);
|
|
DateTimeOffset originalCreatedUtc = afterFirst.CreatedUtc;
|
|
|
|
await _store.UpsertAsync(
|
|
MakeSecret("app/rotating", ciphertext: [9, 9, 9], createdBy: "bob", updatedBy: "bob"),
|
|
CancellationToken.None);
|
|
|
|
StoredSecret afterSecond =
|
|
(await _store.GetAsync(new SecretName("app/rotating"), CancellationToken.None))!;
|
|
Assert.Equal(1, afterSecond.Revision);
|
|
Assert.Equal(new byte[] { 9, 9, 9 }, afterSecond.Ciphertext);
|
|
Assert.Equal(originalCreatedUtc, afterSecond.CreatedUtc);
|
|
Assert.Equal("alice", afterSecond.CreatedBy);
|
|
Assert.Equal("bob", afterSecond.UpdatedBy);
|
|
Assert.False(afterSecond.IsDeleted);
|
|
}
|
|
|
|
[SkippableFact]
|
|
public async Task List_ExcludesTombstoned_ByDefault()
|
|
{
|
|
Skip.IfNot(LiveSqlServer.IsEnabled, LiveSqlServer.SkipReason);
|
|
|
|
await _store.UpsertAsync(MakeSecret("keep"), CancellationToken.None);
|
|
await _store.UpsertAsync(MakeSecret("gone"), CancellationToken.None);
|
|
await _store.DeleteAsync(new SecretName("gone"), "carol", CancellationToken.None);
|
|
|
|
IReadOnlyList<SecretMetadata> visible =
|
|
await _store.ListAsync(includeDeleted: false, CancellationToken.None);
|
|
Assert.DoesNotContain(visible, m => m.Name.Value == "gone");
|
|
Assert.Contains(visible, m => m.Name.Value == "keep");
|
|
|
|
IReadOnlyList<SecretMetadata> all =
|
|
await _store.ListAsync(includeDeleted: true, CancellationToken.None);
|
|
Assert.Contains(all, m => m.Name.Value == "gone");
|
|
Assert.Contains(all, m => m.Name.Value == "keep");
|
|
}
|
|
|
|
[SkippableFact]
|
|
public async Task Delete_SetsTombstone_BumpsRevision_ReturnsFalseWhenAbsent()
|
|
{
|
|
Skip.IfNot(LiveSqlServer.IsEnabled, LiveSqlServer.SkipReason);
|
|
|
|
await _store.UpsertAsync(MakeSecret("app/secret"), CancellationToken.None);
|
|
|
|
Assert.True(await _store.DeleteAsync(new SecretName("app/secret"), "carol", CancellationToken.None));
|
|
|
|
StoredSecret tombstoned =
|
|
(await _store.GetAsync(new SecretName("app/secret"), CancellationToken.None))!;
|
|
Assert.True(tombstoned.IsDeleted);
|
|
Assert.NotNull(tombstoned.DeletedUtc);
|
|
Assert.Equal(1, tombstoned.Revision);
|
|
Assert.Equal("carol", tombstoned.UpdatedBy);
|
|
|
|
Assert.False(await _store.DeleteAsync(new SecretName("app/secret"), "carol", CancellationToken.None));
|
|
Assert.False(await _store.DeleteAsync(new SecretName("never-existed"), "carol", CancellationToken.None));
|
|
}
|
|
|
|
[SkippableFact]
|
|
public async Task GetManifest_ReturnsAllRows()
|
|
{
|
|
Skip.IfNot(LiveSqlServer.IsEnabled, LiveSqlServer.SkipReason);
|
|
|
|
await _store.UpsertAsync(MakeSecret("a"), CancellationToken.None);
|
|
await _store.UpsertAsync(MakeSecret("b"), CancellationToken.None);
|
|
await _store.DeleteAsync(new SecretName("b"), "carol", CancellationToken.None);
|
|
|
|
IReadOnlyList<SecretManifestEntry> manifest = await _store.GetManifestAsync(CancellationToken.None);
|
|
|
|
Assert.Equal(2, manifest.Count);
|
|
SecretManifestEntry a = manifest.Single(e => e.Name.Value == "a");
|
|
SecretManifestEntry b = manifest.Single(e => e.Name.Value == "b");
|
|
Assert.False(a.IsDeleted);
|
|
Assert.Equal(0, a.Revision);
|
|
Assert.True(b.IsDeleted);
|
|
Assert.Equal(1, b.Revision);
|
|
}
|
|
|
|
[SkippableFact]
|
|
public async Task ApplyReplicated_AppliesNewer_IgnoresStale()
|
|
{
|
|
Skip.IfNot(LiveSqlServer.IsEnabled, LiveSqlServer.SkipReason);
|
|
|
|
DateTimeOffset t1 = new(2026, 1, 1, 0, 0, 0, TimeSpan.Zero);
|
|
DateTimeOffset t2 = new(2026, 1, 2, 0, 0, 0, TimeSpan.Zero);
|
|
DateTimeOffset t3 = new(2026, 1, 3, 0, 0, 0, TimeSpan.Zero);
|
|
|
|
await _store.ApplyReplicatedAsync(
|
|
MakeSecret("repl", revision: 5, ciphertext: [5, 5, 5], updatedUtc: t2, createdUtc: t1),
|
|
CancellationToken.None);
|
|
|
|
Assert.Equal(5, (await _store.GetAsync(new SecretName("repl"), CancellationToken.None))!.Revision);
|
|
|
|
await _store.ApplyReplicatedAsync(
|
|
MakeSecret("repl", revision: 6, ciphertext: [6, 6, 6], updatedUtc: t3, createdUtc: t1),
|
|
CancellationToken.None);
|
|
|
|
StoredSecret afterNewer = (await _store.GetAsync(new SecretName("repl"), CancellationToken.None))!;
|
|
Assert.Equal(6, afterNewer.Revision);
|
|
Assert.Equal(new byte[] { 6, 6, 6 }, afterNewer.Ciphertext);
|
|
Assert.Equal(t3, afterNewer.UpdatedUtc);
|
|
|
|
await _store.ApplyReplicatedAsync(
|
|
MakeSecret("repl", revision: 4, ciphertext: [4, 4, 4], updatedUtc: t1, createdUtc: t1),
|
|
CancellationToken.None);
|
|
|
|
StoredSecret afterStale = (await _store.GetAsync(new SecretName("repl"), CancellationToken.None))!;
|
|
Assert.Equal(6, afterStale.Revision);
|
|
Assert.Equal(new byte[] { 6, 6, 6 }, afterStale.Ciphertext);
|
|
}
|
|
|
|
[SkippableFact]
|
|
public async Task ApplyRewrap_ChangesOnlyWrapEnvelopeAndKekId_PreservingRevisionBodyAndTimestamps()
|
|
{
|
|
Skip.IfNot(LiveSqlServer.IsEnabled, LiveSqlServer.SkipReason);
|
|
|
|
await _store.UpsertAsync(
|
|
MakeSecret("app/rewrap", ciphertext: [1, 2, 3], createdUtc: new DateTimeOffset(2026, 1, 1, 0, 0, 0, TimeSpan.Zero)),
|
|
CancellationToken.None);
|
|
await _store.UpsertAsync(MakeSecret("app/rewrap", ciphertext: [1, 2, 3]), CancellationToken.None);
|
|
|
|
StoredSecret before = (await _store.GetAsync(new SecretName("app/rewrap"), CancellationToken.None))!;
|
|
Assert.Equal(1, before.Revision);
|
|
|
|
bool applied = await _store.ApplyRewrapAsync(
|
|
before with
|
|
{
|
|
WrappedDek = [90, 91, 92],
|
|
WrapNonce = [93, 94, 95],
|
|
WrapTag = [96, 97, 98],
|
|
KekId = "kek-2",
|
|
},
|
|
before.WrappedDek,
|
|
CancellationToken.None);
|
|
Assert.True(applied);
|
|
|
|
StoredSecret after = (await _store.GetAsync(new SecretName("app/rewrap"), CancellationToken.None))!;
|
|
Assert.Equal(new byte[] { 90, 91, 92 }, after.WrappedDek);
|
|
Assert.Equal(new byte[] { 93, 94, 95 }, after.WrapNonce);
|
|
Assert.Equal(new byte[] { 96, 97, 98 }, after.WrapTag);
|
|
Assert.Equal("kek-2", after.KekId);
|
|
Assert.Equal(before.Revision, after.Revision);
|
|
Assert.Equal(before.UpdatedUtc, after.UpdatedUtc);
|
|
Assert.Equal(before.CreatedUtc, after.CreatedUtc);
|
|
Assert.Equal(before.UpdatedBy, after.UpdatedBy);
|
|
Assert.Equal(new byte[] { 1, 2, 3 }, after.Ciphertext);
|
|
Assert.Equal(before.Nonce, after.Nonce);
|
|
Assert.Equal(before.Tag, after.Tag);
|
|
}
|
|
|
|
[SkippableFact]
|
|
public async Task ApplyRewrap_RewrapsTombstonedRows()
|
|
{
|
|
Skip.IfNot(LiveSqlServer.IsEnabled, LiveSqlServer.SkipReason);
|
|
|
|
await _store.UpsertAsync(MakeSecret("app/dead"), CancellationToken.None);
|
|
await _store.DeleteAsync(new SecretName("app/dead"), "carol", CancellationToken.None);
|
|
|
|
StoredSecret tomb = (await _store.GetAsync(new SecretName("app/dead"), CancellationToken.None))!;
|
|
Assert.True(tomb.IsDeleted);
|
|
|
|
Assert.True(await _store.ApplyRewrapAsync(
|
|
tomb with { WrappedDek = [1], WrapNonce = [2], WrapTag = [3], KekId = "kek-2" },
|
|
tomb.WrappedDek,
|
|
CancellationToken.None));
|
|
|
|
StoredSecret after = (await _store.GetAsync(new SecretName("app/dead"), CancellationToken.None))!;
|
|
Assert.Equal("kek-2", after.KekId);
|
|
Assert.True(after.IsDeleted);
|
|
Assert.Equal(tomb.Revision, after.Revision);
|
|
}
|
|
|
|
[SkippableFact]
|
|
public async Task ApplyRewrap_ReturnsFalse_WhenRowAbsent()
|
|
{
|
|
Skip.IfNot(LiveSqlServer.IsEnabled, LiveSqlServer.SkipReason);
|
|
|
|
StoredSecret ghost = MakeSecret("never-existed") with { KekId = "kek-2" };
|
|
|
|
Assert.False(await _store.ApplyRewrapAsync(ghost, ghost.WrappedDek, CancellationToken.None));
|
|
}
|
|
|
|
[SkippableFact]
|
|
public async Task ApplyRewrap_ReturnsFalse_WhenCurrentWrapChanged_UnderConcurrentWrite()
|
|
{
|
|
Skip.IfNot(LiveSqlServer.IsEnabled, LiveSqlServer.SkipReason);
|
|
|
|
await _store.UpsertAsync(MakeSecret("app/cas", ciphertext: [1, 2, 3]), CancellationToken.None);
|
|
StoredSecret before = (await _store.GetAsync(new SecretName("app/cas"), CancellationToken.None))!;
|
|
|
|
await _store.UpsertAsync(
|
|
MakeSecret("app/cas", ciphertext: [9, 9, 9]) with { WrappedDek = [77, 78, 79] },
|
|
CancellationToken.None);
|
|
|
|
bool applied = await _store.ApplyRewrapAsync(
|
|
before with { WrappedDek = [1], WrapNonce = [2], WrapTag = [3], KekId = "kek-2" },
|
|
before.WrappedDek,
|
|
CancellationToken.None);
|
|
|
|
Assert.False(applied);
|
|
StoredSecret after = (await _store.GetAsync(new SecretName("app/cas"), CancellationToken.None))!;
|
|
Assert.Equal(new byte[] { 77, 78, 79 }, after.WrappedDek);
|
|
Assert.Equal("kek-1", after.KekId);
|
|
}
|
|
|
|
[SkippableFact]
|
|
public async Task GetMany_returns_only_the_requested_rows()
|
|
{
|
|
Skip.IfNot(LiveSqlServer.IsEnabled, LiveSqlServer.SkipReason);
|
|
|
|
await _store.UpsertAsync(MakeSecret("a/one"), CancellationToken.None);
|
|
await _store.UpsertAsync(MakeSecret("b/two"), CancellationToken.None);
|
|
await _store.UpsertAsync(MakeSecret("c/three"), CancellationToken.None);
|
|
|
|
IReadOnlyList<StoredSecret> rows = await _store.GetManyAsync(
|
|
[new SecretName("a/one"), new SecretName("c/three"), new SecretName("absent/name")],
|
|
CancellationToken.None);
|
|
|
|
Assert.Equal(2, rows.Count);
|
|
Assert.Contains(rows, r => r.Name.Value == "a/one");
|
|
Assert.Contains(rows, r => r.Name.Value == "c/three");
|
|
}
|
|
|
|
[SkippableFact]
|
|
public async Task GetMany_short_circuits_on_an_empty_request()
|
|
{
|
|
Skip.IfNot(LiveSqlServer.IsEnabled, LiveSqlServer.SkipReason);
|
|
|
|
// Guards the IN () clause: an empty list would otherwise produce invalid T-SQL.
|
|
Assert.Empty(await _store.GetManyAsync([], CancellationToken.None));
|
|
}
|
|
|
|
[SkippableFact]
|
|
public async Task Migrator_is_idempotent()
|
|
{
|
|
Skip.IfNot(LiveSqlServer.IsEnabled, LiveSqlServer.SkipReason);
|
|
|
|
var factory = new SecretsSqlServerConnectionFactory(LiveSqlServer.OptionsFor(_schema));
|
|
var migrator = new SqlServerSecretsStoreMigrator(factory);
|
|
|
|
// Every node runs this at every startup — re-running must be a no-op, not an error.
|
|
await migrator.MigrateAsync(CancellationToken.None);
|
|
await migrator.MigrateAsync(CancellationToken.None);
|
|
|
|
await _store.UpsertAsync(MakeSecret("survives/migration"), CancellationToken.None);
|
|
await migrator.MigrateAsync(CancellationToken.None);
|
|
|
|
Assert.NotNull(await _store.GetAsync(new SecretName("survives/migration"), CancellationToken.None));
|
|
}
|
|
|
|
[SkippableFact]
|
|
public async Task Migrator_converges_when_many_nodes_provision_a_virgin_database_concurrently()
|
|
{
|
|
Skip.IfNot(LiveSqlServer.IsEnabled, LiveSqlServer.SkipReason);
|
|
|
|
// The ScadaBridge central shared-store live gate (2026-08-07): an ordinary compose up
|
|
// recreated both central nodes at once, both ran the boot migrator against the virgin
|
|
// database, and the loser died with SqlException 2714 ("There is already an object named
|
|
// 'zbsecrets'") — the retry filter then covered only deadlock 1205. Deliberately NOT the
|
|
// class's shared schema: that one is already migrated by InitializeAsync, and this race
|
|
// only exists against a virgin database.
|
|
string schema = LiveSqlServer.NewSchemaName();
|
|
try
|
|
{
|
|
await Task.WhenAll(Enumerable.Range(0, 4).Select(_ => Task.Run(() =>
|
|
new SqlServerSecretsStoreMigrator(
|
|
new SecretsSqlServerConnectionFactory(LiveSqlServer.OptionsFor(schema)))
|
|
.MigrateAsync(CancellationToken.None))));
|
|
|
|
// Every "node" survived; prove the converged schema is actually usable.
|
|
var store = new SqlServerSecretStore(
|
|
new SecretsSqlServerConnectionFactory(LiveSqlServer.OptionsFor(schema)));
|
|
await store.UpsertAsync(MakeSecret("concurrent/provision"), CancellationToken.None);
|
|
Assert.NotNull(await store.GetAsync(new SecretName("concurrent/provision"), CancellationToken.None));
|
|
}
|
|
finally
|
|
{
|
|
await LiveSqlServer.DropSchemaAsync(schema);
|
|
}
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Serializes the live suite: the tests share one database server and each provisions its own
|
|
/// schema, so running them in parallel would multiply connection load for no benefit.
|
|
/// </summary>
|
|
[CollectionDefinition("live-sqlserver", DisableParallelization = true)]
|
|
public sealed class LiveSqlServerCollection;
|