fix(secrets): make concurrent SQLite store initialisation safe; bump to 0.6.4 (unpublished) (scadaproj#25)

Several setups migrating one SQLite secret store at once (mxaccessgw's parallel test hosts on the
per-process store) failed intermittently on Windows with "cannot start a transaction within a
transaction" from SqliteSecretsStoreMigrator.MigrateAsync. That message means the BEGIN ran on a
handle that was already in a transaction: the pinned Microsoft.Data.Sqlite 10.0.7 can hand one
pooled handle to two owners (dotnet/efcore#39009, fixed in 10.0.13) and no 10.0.x refuses to
re-pool a handle whose rollback failed (dotnet/efcore#39082).

The migrator now serialises migrations of one store per process behind a gate keyed on the
normalised path, runs each attempt on a dedicated unpooled connection inside BEGIN IMMEDIATE, and
retries SQLITE_BUSY/SQLITE_LOCKED (another process past the busy timeout) with equal-jitter
exponential backoff. Store operations and the Microsoft.Data.Sqlite pin are unchanged.

Tests: concurrent-init (16 hosts x 20 rounds on fresh and existing stores, two path spellings,
16 AddZbSecrets containers running the pre-host migrator + hosted service), gate sharing/waiting,
BUSY retry past the 5 s busy timeout, transient-code classification.

Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
Joseph Doherty
2026-11-09 13:57:53 -05:00
parent 01c6dc1d02
commit f21f2e551d
7 changed files with 420 additions and 9 deletions
@@ -78,6 +78,80 @@ public sealed class SqliteSecretsStoreMigratorTests : IDisposable
() => migrator.MigrateAsync(CancellationToken.None));
}
[Fact]
public void NormalizedPath_DifferentSpellingsOfOneFile_ShareOneMigrationGate()
{
string dir = Path.GetDirectoryName(_dbPath)!;
string detour = Path.Combine(dir, "zb-detour", "..", Path.GetFileName(_dbPath));
var canonical = new SecretsSqliteConnectionFactory(_dbPath);
var other = new SecretsSqliteConnectionFactory(detour);
Assert.Equal(canonical.NormalizedPath, other.NormalizedPath);
Assert.Same(
SqliteSecretsStoreMigrator.GateFor(canonical.NormalizedPath),
SqliteSecretsStoreMigrator.GateFor(other.NormalizedPath));
Assert.NotSame(
SqliteSecretsStoreMigrator.GateFor(canonical.NormalizedPath),
SqliteSecretsStoreMigrator.GateFor(canonical.NormalizedPath + ".other"));
}
[Fact]
public async Task Migrate_WaitsForTheProcessWideGateOfItsStore()
{
var factory = new SecretsSqliteConnectionFactory(_dbPath);
SemaphoreSlim gate = SqliteSecretsStoreMigrator.GateFor(factory.NormalizedPath);
await gate.WaitAsync();
Task migration;
try
{
// A second factory/migrator pair, as a separate DI container would build.
migration = new SqliteSecretsStoreMigrator(new SecretsSqliteConnectionFactory(_dbPath))
.MigrateAsync(CancellationToken.None);
await Task.Delay(300);
Assert.False(migration.IsCompleted, "migration ran while another migration of the same store held the gate");
}
finally
{
gate.Release();
}
await migration.WaitAsync(TimeSpan.FromSeconds(30));
Assert.Equal(SqliteSecretsSchema.CurrentVersion, await ReadSchemaVersionAsync(factory));
}
[Fact]
public async Task Migrate_RetriesWhenAnotherWriterHoldsTheStorePastTheBusyTimeout()
{
var factory = new SecretsSqliteConnectionFactory(_dbPath);
// Another "process": an unpooled connection holding the write lock (BEGIN IMMEDIATE) for longer
// than the 5 s busy timeout, so the first migration attempt surfaces SQLITE_BUSY.
await using SqliteConnection holder = await factory.OpenUnpooledConnectionAsync(CancellationToken.None);
SqliteTransaction lockTx = holder.BeginTransaction(System.Data.IsolationLevel.Serializable, deferred: false);
Task migration = new SqliteSecretsStoreMigrator(factory).MigrateAsync(CancellationToken.None);
await Task.Delay(TimeSpan.FromSeconds(6.5));
Assert.False(migration.IsCompleted, "migration finished while the write lock was still held");
lockTx.Rollback();
lockTx.Dispose();
await migration.WaitAsync(TimeSpan.FromSeconds(30));
Assert.Equal(SqliteSecretsSchema.CurrentVersion, await ReadSchemaVersionAsync(factory));
}
[Theory]
[InlineData(5, true)] // SQLITE_BUSY
[InlineData(6, true)] // SQLITE_LOCKED
[InlineData(261, true)] // SQLITE_BUSY_RECOVERY (extended code of BUSY)
[InlineData(1, false)] // SQLITE_ERROR, e.g. "cannot start a transaction within a transaction"
[InlineData(11, false)] // SQLITE_CORRUPT
public void IsTransient_OnlyBusyAndLockedAreRetried(int errorCode, bool expected) =>
Assert.Equal(expected, SqliteSecretsStoreMigrator.IsTransient(new SqliteException("x", errorCode)));
private static async Task<long> ReadSchemaVersionAsync(SecretsSqliteConnectionFactory factory)
{
await using SqliteConnection connection = await factory.OpenConnectionAsync(CancellationToken.None);