Files
scadaproj/ZB.MOM.WW.Secrets/tests/ZB.MOM.WW.Secrets.Tests/Sqlite/SqliteSecretsStoreMigratorTests.cs
T
Joseph Doherty f21f2e551d fix(secrets): make concurrent SQLite store initialisation safe; bump to 0.6.4 (unpublished) (scadaproj#25)
Several setups migrating one SQLite secret store at once (mxaccessgw's parallel test hosts on the
per-process store) failed intermittently on Windows with "cannot start a transaction within a
transaction" from SqliteSecretsStoreMigrator.MigrateAsync. That message means the BEGIN ran on a
handle that was already in a transaction: the pinned Microsoft.Data.Sqlite 10.0.7 can hand one
pooled handle to two owners (dotnet/efcore#39009, fixed in 10.0.13) and no 10.0.x refuses to
re-pool a handle whose rollback failed (dotnet/efcore#39082).

The migrator now serialises migrations of one store per process behind a gate keyed on the
normalised path, runs each attempt on a dedicated unpooled connection inside BEGIN IMMEDIATE, and
retries SQLITE_BUSY/SQLITE_LOCKED (another process past the busy timeout) with equal-jitter
exponential backoff. Store operations and the Microsoft.Data.Sqlite pin are unchanged.

Tests: concurrent-init (16 hosts x 20 rounds on fresh and existing stores, two path spellings,
16 AddZbSecrets containers running the pre-host migrator + hosted service), gate sharing/waiting,
BUSY retry past the 5 s busy timeout, transient-code classification.

Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
2026-11-09 13:57:53 -05:00

202 lines
8.0 KiB
C#

using Microsoft.Data.Sqlite;
using ZB.MOM.WW.Secrets.Abstractions;
using ZB.MOM.WW.Secrets.Sqlite;
namespace ZB.MOM.WW.Secrets.Tests.Sqlite;
public sealed class SqliteSecretsStoreMigratorTests : IDisposable
{
private readonly string _dbPath =
Path.Combine(Path.GetTempPath(), $"zb-secrets-migrator-{Guid.NewGuid():N}.db");
[Fact]
public async Task Migrate_IsIdempotent()
{
var factory = new SecretsSqliteConnectionFactory(_dbPath);
var migrator = new SqliteSecretsStoreMigrator(factory);
// Running twice must not throw and must leave the schema version at CurrentVersion.
await migrator.MigrateAsync(CancellationToken.None);
await migrator.MigrateAsync(CancellationToken.None);
long version = await ReadSchemaVersionAsync(factory);
Assert.Equal(SqliteSecretsSchema.CurrentVersion, version);
Assert.Equal(1, version);
}
[Fact]
public async Task Migrate_CreatesSecretTableWithExpectedColumns()
{
var factory = new SecretsSqliteConnectionFactory(_dbPath);
var migrator = new SqliteSecretsStoreMigrator(factory);
await migrator.MigrateAsync(CancellationToken.None);
HashSet<string> columns = await ReadSecretColumnsAsync(factory);
string[] expected =
[
"name", "description", "content_type", "ciphertext", "nonce", "tag",
"wrapped_dek", "wrap_nonce", "wrap_tag", "kek_id", "revision",
"is_deleted", "deleted_utc", "created_utc", "updated_utc",
"created_by", "updated_by",
];
Assert.Equal(17, expected.Length);
Assert.Equal(17, columns.Count);
foreach (string column in expected)
{
Assert.Contains(column, columns);
}
}
[Fact]
public async Task Migrate_RefusesNewerSchema()
{
var factory = new SecretsSqliteConnectionFactory(_dbPath);
// Manually stamp an on-disk version newer than this build supports.
await using (SqliteConnection connection = await factory.OpenConnectionAsync(CancellationToken.None))
{
await using SqliteCommand create = connection.CreateCommand();
create.CommandText = SqliteSecretsSchema.CreateSchemaVersionTable;
await create.ExecuteNonQueryAsync();
await using SqliteCommand insert = connection.CreateCommand();
insert.CommandText = """
INSERT INTO schema_version (id, version, applied_utc)
VALUES (1, $version, $applied_utc);
""";
insert.Parameters.AddWithValue("$version", 999);
insert.Parameters.AddWithValue("$applied_utc", DateTimeOffset.UtcNow.ToString("O"));
await insert.ExecuteNonQueryAsync();
}
var migrator = new SqliteSecretsStoreMigrator(factory);
await Assert.ThrowsAsync<SecretStoreMigrationException>(
() => migrator.MigrateAsync(CancellationToken.None));
}
[Fact]
public void NormalizedPath_DifferentSpellingsOfOneFile_ShareOneMigrationGate()
{
string dir = Path.GetDirectoryName(_dbPath)!;
string detour = Path.Combine(dir, "zb-detour", "..", Path.GetFileName(_dbPath));
var canonical = new SecretsSqliteConnectionFactory(_dbPath);
var other = new SecretsSqliteConnectionFactory(detour);
Assert.Equal(canonical.NormalizedPath, other.NormalizedPath);
Assert.Same(
SqliteSecretsStoreMigrator.GateFor(canonical.NormalizedPath),
SqliteSecretsStoreMigrator.GateFor(other.NormalizedPath));
Assert.NotSame(
SqliteSecretsStoreMigrator.GateFor(canonical.NormalizedPath),
SqliteSecretsStoreMigrator.GateFor(canonical.NormalizedPath + ".other"));
}
[Fact]
public async Task Migrate_WaitsForTheProcessWideGateOfItsStore()
{
var factory = new SecretsSqliteConnectionFactory(_dbPath);
SemaphoreSlim gate = SqliteSecretsStoreMigrator.GateFor(factory.NormalizedPath);
await gate.WaitAsync();
Task migration;
try
{
// A second factory/migrator pair, as a separate DI container would build.
migration = new SqliteSecretsStoreMigrator(new SecretsSqliteConnectionFactory(_dbPath))
.MigrateAsync(CancellationToken.None);
await Task.Delay(300);
Assert.False(migration.IsCompleted, "migration ran while another migration of the same store held the gate");
}
finally
{
gate.Release();
}
await migration.WaitAsync(TimeSpan.FromSeconds(30));
Assert.Equal(SqliteSecretsSchema.CurrentVersion, await ReadSchemaVersionAsync(factory));
}
[Fact]
public async Task Migrate_RetriesWhenAnotherWriterHoldsTheStorePastTheBusyTimeout()
{
var factory = new SecretsSqliteConnectionFactory(_dbPath);
// Another "process": an unpooled connection holding the write lock (BEGIN IMMEDIATE) for longer
// than the 5 s busy timeout, so the first migration attempt surfaces SQLITE_BUSY.
await using SqliteConnection holder = await factory.OpenUnpooledConnectionAsync(CancellationToken.None);
SqliteTransaction lockTx = holder.BeginTransaction(System.Data.IsolationLevel.Serializable, deferred: false);
Task migration = new SqliteSecretsStoreMigrator(factory).MigrateAsync(CancellationToken.None);
await Task.Delay(TimeSpan.FromSeconds(6.5));
Assert.False(migration.IsCompleted, "migration finished while the write lock was still held");
lockTx.Rollback();
lockTx.Dispose();
await migration.WaitAsync(TimeSpan.FromSeconds(30));
Assert.Equal(SqliteSecretsSchema.CurrentVersion, await ReadSchemaVersionAsync(factory));
}
[Theory]
[InlineData(5, true)] // SQLITE_BUSY
[InlineData(6, true)] // SQLITE_LOCKED
[InlineData(261, true)] // SQLITE_BUSY_RECOVERY (extended code of BUSY)
[InlineData(1, false)] // SQLITE_ERROR, e.g. "cannot start a transaction within a transaction"
[InlineData(11, false)] // SQLITE_CORRUPT
public void IsTransient_OnlyBusyAndLockedAreRetried(int errorCode, bool expected) =>
Assert.Equal(expected, SqliteSecretsStoreMigrator.IsTransient(new SqliteException("x", errorCode)));
private static async Task<long> ReadSchemaVersionAsync(SecretsSqliteConnectionFactory factory)
{
await using SqliteConnection connection = await factory.OpenConnectionAsync(CancellationToken.None);
await using SqliteCommand command = connection.CreateCommand();
command.CommandText = "SELECT version FROM schema_version WHERE id = 1;";
object? result = await command.ExecuteScalarAsync();
return Convert.ToInt64(result);
}
private static async Task<HashSet<string>> ReadSecretColumnsAsync(SecretsSqliteConnectionFactory factory)
{
await using SqliteConnection connection = await factory.OpenConnectionAsync(CancellationToken.None);
await using SqliteCommand command = connection.CreateCommand();
command.CommandText = $"PRAGMA table_info({SqliteSecretsSchema.SecretTable});";
var columns = new HashSet<string>(StringComparer.Ordinal);
await using SqliteDataReader reader = await command.ExecuteReaderAsync();
while (await reader.ReadAsync())
{
// PRAGMA table_info columns: cid(0), name(1), type(2), ...
columns.Add(reader.GetString(1));
}
return columns;
}
public void Dispose()
{
// Drop pooled connections so the WAL/-shm/-wal sidecars release before we delete.
SqliteConnection.ClearAllPools();
foreach (string path in new[] { _dbPath, _dbPath + "-wal", _dbPath + "-shm" })
{
try
{
if (File.Exists(path))
{
File.Delete(path);
}
}
catch (IOException)
{
// Best-effort temp cleanup; a leaked temp file is not a test failure.
}
}
}
}