f21f2e551d
Several setups migrating one SQLite secret store at once (mxaccessgw's parallel test hosts on the per-process store) failed intermittently on Windows with "cannot start a transaction within a transaction" from SqliteSecretsStoreMigrator.MigrateAsync. That message means the BEGIN ran on a handle that was already in a transaction: the pinned Microsoft.Data.Sqlite 10.0.7 can hand one pooled handle to two owners (dotnet/efcore#39009, fixed in 10.0.13) and no 10.0.x refuses to re-pool a handle whose rollback failed (dotnet/efcore#39082). The migrator now serialises migrations of one store per process behind a gate keyed on the normalised path, runs each attempt on a dedicated unpooled connection inside BEGIN IMMEDIATE, and retries SQLITE_BUSY/SQLITE_LOCKED (another process past the busy timeout) with equal-jitter exponential backoff. Store operations and the Microsoft.Data.Sqlite pin are unchanged. Tests: concurrent-init (16 hosts x 20 rounds on fresh and existing stores, two path spellings, 16 AddZbSecrets containers running the pre-host migrator + hosted service), gate sharing/waiting, BUSY retry past the 5 s busy timeout, transient-code classification. Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
202 lines
8.0 KiB
C#
202 lines
8.0 KiB
C#
using Microsoft.Data.Sqlite;
|
|
using ZB.MOM.WW.Secrets.Abstractions;
|
|
using ZB.MOM.WW.Secrets.Sqlite;
|
|
|
|
namespace ZB.MOM.WW.Secrets.Tests.Sqlite;
|
|
|
|
public sealed class SqliteSecretsStoreMigratorTests : IDisposable
|
|
{
|
|
private readonly string _dbPath =
|
|
Path.Combine(Path.GetTempPath(), $"zb-secrets-migrator-{Guid.NewGuid():N}.db");
|
|
|
|
[Fact]
|
|
public async Task Migrate_IsIdempotent()
|
|
{
|
|
var factory = new SecretsSqliteConnectionFactory(_dbPath);
|
|
var migrator = new SqliteSecretsStoreMigrator(factory);
|
|
|
|
// Running twice must not throw and must leave the schema version at CurrentVersion.
|
|
await migrator.MigrateAsync(CancellationToken.None);
|
|
await migrator.MigrateAsync(CancellationToken.None);
|
|
|
|
long version = await ReadSchemaVersionAsync(factory);
|
|
Assert.Equal(SqliteSecretsSchema.CurrentVersion, version);
|
|
Assert.Equal(1, version);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Migrate_CreatesSecretTableWithExpectedColumns()
|
|
{
|
|
var factory = new SecretsSqliteConnectionFactory(_dbPath);
|
|
var migrator = new SqliteSecretsStoreMigrator(factory);
|
|
|
|
await migrator.MigrateAsync(CancellationToken.None);
|
|
|
|
HashSet<string> columns = await ReadSecretColumnsAsync(factory);
|
|
|
|
string[] expected =
|
|
[
|
|
"name", "description", "content_type", "ciphertext", "nonce", "tag",
|
|
"wrapped_dek", "wrap_nonce", "wrap_tag", "kek_id", "revision",
|
|
"is_deleted", "deleted_utc", "created_utc", "updated_utc",
|
|
"created_by", "updated_by",
|
|
];
|
|
|
|
Assert.Equal(17, expected.Length);
|
|
Assert.Equal(17, columns.Count);
|
|
foreach (string column in expected)
|
|
{
|
|
Assert.Contains(column, columns);
|
|
}
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Migrate_RefusesNewerSchema()
|
|
{
|
|
var factory = new SecretsSqliteConnectionFactory(_dbPath);
|
|
|
|
// Manually stamp an on-disk version newer than this build supports.
|
|
await using (SqliteConnection connection = await factory.OpenConnectionAsync(CancellationToken.None))
|
|
{
|
|
await using SqliteCommand create = connection.CreateCommand();
|
|
create.CommandText = SqliteSecretsSchema.CreateSchemaVersionTable;
|
|
await create.ExecuteNonQueryAsync();
|
|
|
|
await using SqliteCommand insert = connection.CreateCommand();
|
|
insert.CommandText = """
|
|
INSERT INTO schema_version (id, version, applied_utc)
|
|
VALUES (1, $version, $applied_utc);
|
|
""";
|
|
insert.Parameters.AddWithValue("$version", 999);
|
|
insert.Parameters.AddWithValue("$applied_utc", DateTimeOffset.UtcNow.ToString("O"));
|
|
await insert.ExecuteNonQueryAsync();
|
|
}
|
|
|
|
var migrator = new SqliteSecretsStoreMigrator(factory);
|
|
|
|
await Assert.ThrowsAsync<SecretStoreMigrationException>(
|
|
() => migrator.MigrateAsync(CancellationToken.None));
|
|
}
|
|
|
|
[Fact]
|
|
public void NormalizedPath_DifferentSpellingsOfOneFile_ShareOneMigrationGate()
|
|
{
|
|
string dir = Path.GetDirectoryName(_dbPath)!;
|
|
string detour = Path.Combine(dir, "zb-detour", "..", Path.GetFileName(_dbPath));
|
|
|
|
var canonical = new SecretsSqliteConnectionFactory(_dbPath);
|
|
var other = new SecretsSqliteConnectionFactory(detour);
|
|
|
|
Assert.Equal(canonical.NormalizedPath, other.NormalizedPath);
|
|
Assert.Same(
|
|
SqliteSecretsStoreMigrator.GateFor(canonical.NormalizedPath),
|
|
SqliteSecretsStoreMigrator.GateFor(other.NormalizedPath));
|
|
Assert.NotSame(
|
|
SqliteSecretsStoreMigrator.GateFor(canonical.NormalizedPath),
|
|
SqliteSecretsStoreMigrator.GateFor(canonical.NormalizedPath + ".other"));
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Migrate_WaitsForTheProcessWideGateOfItsStore()
|
|
{
|
|
var factory = new SecretsSqliteConnectionFactory(_dbPath);
|
|
SemaphoreSlim gate = SqliteSecretsStoreMigrator.GateFor(factory.NormalizedPath);
|
|
|
|
await gate.WaitAsync();
|
|
Task migration;
|
|
try
|
|
{
|
|
// A second factory/migrator pair, as a separate DI container would build.
|
|
migration = new SqliteSecretsStoreMigrator(new SecretsSqliteConnectionFactory(_dbPath))
|
|
.MigrateAsync(CancellationToken.None);
|
|
|
|
await Task.Delay(300);
|
|
Assert.False(migration.IsCompleted, "migration ran while another migration of the same store held the gate");
|
|
}
|
|
finally
|
|
{
|
|
gate.Release();
|
|
}
|
|
|
|
await migration.WaitAsync(TimeSpan.FromSeconds(30));
|
|
Assert.Equal(SqliteSecretsSchema.CurrentVersion, await ReadSchemaVersionAsync(factory));
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Migrate_RetriesWhenAnotherWriterHoldsTheStorePastTheBusyTimeout()
|
|
{
|
|
var factory = new SecretsSqliteConnectionFactory(_dbPath);
|
|
|
|
// Another "process": an unpooled connection holding the write lock (BEGIN IMMEDIATE) for longer
|
|
// than the 5 s busy timeout, so the first migration attempt surfaces SQLITE_BUSY.
|
|
await using SqliteConnection holder = await factory.OpenUnpooledConnectionAsync(CancellationToken.None);
|
|
SqliteTransaction lockTx = holder.BeginTransaction(System.Data.IsolationLevel.Serializable, deferred: false);
|
|
|
|
Task migration = new SqliteSecretsStoreMigrator(factory).MigrateAsync(CancellationToken.None);
|
|
await Task.Delay(TimeSpan.FromSeconds(6.5));
|
|
Assert.False(migration.IsCompleted, "migration finished while the write lock was still held");
|
|
|
|
lockTx.Rollback();
|
|
lockTx.Dispose();
|
|
|
|
await migration.WaitAsync(TimeSpan.FromSeconds(30));
|
|
Assert.Equal(SqliteSecretsSchema.CurrentVersion, await ReadSchemaVersionAsync(factory));
|
|
}
|
|
|
|
[Theory]
|
|
[InlineData(5, true)] // SQLITE_BUSY
|
|
[InlineData(6, true)] // SQLITE_LOCKED
|
|
[InlineData(261, true)] // SQLITE_BUSY_RECOVERY (extended code of BUSY)
|
|
[InlineData(1, false)] // SQLITE_ERROR, e.g. "cannot start a transaction within a transaction"
|
|
[InlineData(11, false)] // SQLITE_CORRUPT
|
|
public void IsTransient_OnlyBusyAndLockedAreRetried(int errorCode, bool expected) =>
|
|
Assert.Equal(expected, SqliteSecretsStoreMigrator.IsTransient(new SqliteException("x", errorCode)));
|
|
|
|
private static async Task<long> ReadSchemaVersionAsync(SecretsSqliteConnectionFactory factory)
|
|
{
|
|
await using SqliteConnection connection = await factory.OpenConnectionAsync(CancellationToken.None);
|
|
await using SqliteCommand command = connection.CreateCommand();
|
|
command.CommandText = "SELECT version FROM schema_version WHERE id = 1;";
|
|
object? result = await command.ExecuteScalarAsync();
|
|
return Convert.ToInt64(result);
|
|
}
|
|
|
|
private static async Task<HashSet<string>> ReadSecretColumnsAsync(SecretsSqliteConnectionFactory factory)
|
|
{
|
|
await using SqliteConnection connection = await factory.OpenConnectionAsync(CancellationToken.None);
|
|
await using SqliteCommand command = connection.CreateCommand();
|
|
command.CommandText = $"PRAGMA table_info({SqliteSecretsSchema.SecretTable});";
|
|
|
|
var columns = new HashSet<string>(StringComparer.Ordinal);
|
|
await using SqliteDataReader reader = await command.ExecuteReaderAsync();
|
|
while (await reader.ReadAsync())
|
|
{
|
|
// PRAGMA table_info columns: cid(0), name(1), type(2), ...
|
|
columns.Add(reader.GetString(1));
|
|
}
|
|
|
|
return columns;
|
|
}
|
|
|
|
public void Dispose()
|
|
{
|
|
// Drop pooled connections so the WAL/-shm/-wal sidecars release before we delete.
|
|
SqliteConnection.ClearAllPools();
|
|
|
|
foreach (string path in new[] { _dbPath, _dbPath + "-wal", _dbPath + "-shm" })
|
|
{
|
|
try
|
|
{
|
|
if (File.Exists(path))
|
|
{
|
|
File.Delete(path);
|
|
}
|
|
}
|
|
catch (IOException)
|
|
{
|
|
// Best-effort temp cleanup; a leaked temp file is not a test failure.
|
|
}
|
|
}
|
|
}
|
|
}
|