fix(secrets): make concurrent SQLite store initialisation safe; bump to 0.6.4 (unpublished) (scadaproj#25)
Several setups migrating one SQLite secret store at once (mxaccessgw's parallel test hosts on the per-process store) failed intermittently on Windows with "cannot start a transaction within a transaction" from SqliteSecretsStoreMigrator.MigrateAsync. That message means the BEGIN ran on a handle that was already in a transaction: the pinned Microsoft.Data.Sqlite 10.0.7 can hand one pooled handle to two owners (dotnet/efcore#39009, fixed in 10.0.13) and no 10.0.x refuses to re-pool a handle whose rollback failed (dotnet/efcore#39082). The migrator now serialises migrations of one store per process behind a gate keyed on the normalised path, runs each attempt on a dedicated unpooled connection inside BEGIN IMMEDIATE, and retries SQLITE_BUSY/SQLITE_LOCKED (another process past the busy timeout) with equal-jitter exponential backoff. Store operations and the Microsoft.Data.Sqlite pin are unchanged. Tests: concurrent-init (16 hosts x 20 rounds on fresh and existing stores, two path spellings, 16 AddZbSecrets containers running the pre-host migrator + hosted service), gate sharing/waiting, BUSY retry past the 5 s busy timeout, transient-code classification. Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
+217
@@ -0,0 +1,217 @@
|
||||
using Microsoft.Data.Sqlite;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using ZB.MOM.WW.Secrets.Abstractions;
|
||||
using ZB.MOM.WW.Secrets.DependencyInjection;
|
||||
using ZB.MOM.WW.Secrets.Sqlite;
|
||||
|
||||
namespace ZB.MOM.WW.Secrets.Tests.Sqlite;
|
||||
|
||||
/// <summary>
|
||||
/// scadaproj#25: several hosts in one process initialising the SAME SQLite secret store at once
|
||||
/// (mxaccessgw's parallel test classes, each building a host whose migration hosted service and
|
||||
/// store point at the per-process default path). On Windows this surfaced as
|
||||
/// "cannot start a transaction within a transaction". Each simulated host owns its own
|
||||
/// connection factory, migrator and store, exactly as separate DI containers would.
|
||||
/// </summary>
|
||||
public sealed class SqliteSecretsStoreConcurrentInitTests : IDisposable
|
||||
{
|
||||
private const int Hosts = 16;
|
||||
private const int Rounds = 20;
|
||||
|
||||
private readonly string _root =
|
||||
Path.Combine(Path.GetTempPath(), $"zb-secrets-concurrent-{Guid.NewGuid():N}");
|
||||
|
||||
[Fact]
|
||||
public async Task ConcurrentInitialisation_OfOneFreshStore_AllHostsSucceed()
|
||||
{
|
||||
for (int round = 0; round < Rounds; round++)
|
||||
{
|
||||
string path = Path.Combine(_root, $"round-{round}", "secrets.db");
|
||||
await RunHostsAsync(path, round);
|
||||
await AssertStoreAsync(path, expectedSecrets: Hosts);
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ConcurrentInitialisation_OfOneExistingStore_AllHostsSucceed()
|
||||
{
|
||||
string path = Path.Combine(_root, "existing", "secrets.db");
|
||||
await new SqliteSecretsStoreMigrator(new SecretsSqliteConnectionFactory(path))
|
||||
.MigrateAsync(CancellationToken.None);
|
||||
|
||||
for (int round = 0; round < Rounds; round++)
|
||||
{
|
||||
await RunHostsAsync(path, round);
|
||||
}
|
||||
|
||||
await AssertStoreAsync(path, expectedSecrets: Hosts * Rounds);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ConcurrentInitialisation_ThroughDifferentSpellingsOfOnePath_AllHostsSucceed()
|
||||
{
|
||||
// The per-path gate keys on the NORMALISED path: a relative spelling and a "..\" detour
|
||||
// must land on the same gate as the canonical absolute path.
|
||||
string dir = Path.Combine(_root, "spellings");
|
||||
string canonical = Path.Combine(dir, "secrets.db");
|
||||
string detour = Path.Combine(dir, "sub", "..", "secrets.db");
|
||||
string[] spellings = [canonical, detour];
|
||||
|
||||
Task[] hosts = new Task[Hosts];
|
||||
TaskCompletionSource start = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
for (int i = 0; i < Hosts; i++)
|
||||
{
|
||||
string spelling = spellings[i % spellings.Length];
|
||||
int host = i;
|
||||
hosts[i] = Task.Run(async () =>
|
||||
{
|
||||
await start.Task;
|
||||
await RunOneHostAsync(spelling, $"spell/{host}");
|
||||
});
|
||||
}
|
||||
|
||||
start.SetResult();
|
||||
await Task.WhenAll(hosts);
|
||||
await AssertStoreAsync(canonical, expectedSecrets: Hosts);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ConcurrentInitialisation_ThroughSeparateAddZbSecretsContainers_AllSucceed()
|
||||
{
|
||||
// The exact mxaccessgw shape: every host builds its own container with AddZbSecrets, runs the
|
||||
// pre-host migrator, then starts the SecretsMigrationHostedService — all on one store path.
|
||||
string path = Path.Combine(_root, "di", "secrets.db");
|
||||
IConfiguration configuration = new ConfigurationBuilder()
|
||||
.AddInMemoryCollection(new Dictionary<string, string?>
|
||||
{
|
||||
["Secrets:SqlitePath"] = path,
|
||||
["Secrets:RunMigrationsOnStartup"] = "true",
|
||||
})
|
||||
.Build();
|
||||
|
||||
TaskCompletionSource start = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
Task[] hosts = new Task[Hosts];
|
||||
for (int i = 0; i < Hosts; i++)
|
||||
{
|
||||
hosts[i] = Task.Run(async () =>
|
||||
{
|
||||
await using ServiceProvider provider = new ServiceCollection()
|
||||
.AddZbSecrets(configuration, "Secrets")
|
||||
.BuildServiceProvider();
|
||||
await start.Task;
|
||||
|
||||
provider.GetRequiredService<SqliteSecretsStoreMigrator>()
|
||||
.MigrateAsync(default).GetAwaiter().GetResult();
|
||||
foreach (IHostedService hosted in provider.GetServices<IHostedService>())
|
||||
{
|
||||
if (hosted.GetType().Name == "SecretsMigrationHostedService")
|
||||
{
|
||||
await hosted.StartAsync(CancellationToken.None);
|
||||
}
|
||||
}
|
||||
|
||||
_ = await provider.GetRequiredService<ISecretStore>().ListAsync(includeDeleted: true, CancellationToken.None);
|
||||
});
|
||||
}
|
||||
|
||||
start.SetResult();
|
||||
await Task.WhenAll(hosts);
|
||||
await AssertStoreAsync(path, expectedSecrets: 0);
|
||||
}
|
||||
|
||||
private static async Task RunHostsAsync(string path, int round)
|
||||
{
|
||||
TaskCompletionSource start = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
Task[] hosts = new Task[Hosts];
|
||||
for (int i = 0; i < Hosts; i++)
|
||||
{
|
||||
int host = i;
|
||||
hosts[i] = Task.Run(async () =>
|
||||
{
|
||||
await start.Task;
|
||||
await RunOneHostAsync(path, $"r{round}/h{host}");
|
||||
});
|
||||
}
|
||||
|
||||
// Release every host at once so their migrations genuinely overlap.
|
||||
start.SetResult();
|
||||
await Task.WhenAll(hosts);
|
||||
}
|
||||
|
||||
private static async Task RunOneHostAsync(string path, string secretName)
|
||||
{
|
||||
// One "host": its own factory, migrator and store, as a separate DI container would build.
|
||||
SecretsSqliteConnectionFactory factory = new(path);
|
||||
await new SqliteSecretsStoreMigrator(factory).MigrateAsync(CancellationToken.None);
|
||||
|
||||
SqliteSecretStore store = new(factory);
|
||||
StoredSecret row = MakeSecret(secretName);
|
||||
await store.UpsertAsync(row, CancellationToken.None);
|
||||
await store.ApplyReplicatedAsync(row with { Revision = 5, UpdatedUtc = row.UpdatedUtc.AddSeconds(1) }, CancellationToken.None);
|
||||
Assert.NotNull(await store.GetAsync(row.Name, CancellationToken.None));
|
||||
_ = await store.ListAsync(includeDeleted: false, CancellationToken.None);
|
||||
}
|
||||
|
||||
private static async Task AssertStoreAsync(string path, int expectedSecrets)
|
||||
{
|
||||
SecretsSqliteConnectionFactory factory = new(path);
|
||||
await using SqliteConnection connection = await factory.OpenConnectionAsync(CancellationToken.None);
|
||||
|
||||
await using SqliteCommand version = connection.CreateCommand();
|
||||
version.CommandText = "SELECT version FROM schema_version WHERE id = 1;";
|
||||
Assert.Equal((long)SqliteSecretsSchema.CurrentVersion, Convert.ToInt64(await version.ExecuteScalarAsync()));
|
||||
|
||||
await using SqliteCommand count = connection.CreateCommand();
|
||||
count.CommandText = "SELECT COUNT(*) FROM secret;";
|
||||
Assert.Equal((long)expectedSecrets, Convert.ToInt64(await count.ExecuteScalarAsync()));
|
||||
|
||||
// No pooled connection may be handed back with a transaction still open: a fresh
|
||||
// BEGIN on a pooled connection is exactly what failed on Windows.
|
||||
await using SqliteTransaction tx = (SqliteTransaction)
|
||||
await connection.BeginTransactionAsync(System.Data.IsolationLevel.Serializable);
|
||||
await tx.CommitAsync();
|
||||
}
|
||||
|
||||
private static StoredSecret MakeSecret(string name) => new()
|
||||
{
|
||||
Name = new SecretName(name),
|
||||
Description = "concurrent-init",
|
||||
ContentType = SecretContentType.ConnectionString,
|
||||
Ciphertext = [1, 2, 3],
|
||||
Nonce = [4, 5, 6],
|
||||
Tag = [7, 8, 9],
|
||||
WrappedDek = [10, 11, 12],
|
||||
WrapNonce = [13, 14, 15],
|
||||
WrapTag = [16, 17, 18],
|
||||
KekId = "kek-1",
|
||||
Revision = 0,
|
||||
IsDeleted = false,
|
||||
DeletedUtc = null,
|
||||
CreatedUtc = DateTimeOffset.UtcNow,
|
||||
UpdatedUtc = DateTimeOffset.UtcNow,
|
||||
CreatedBy = "concurrent",
|
||||
UpdatedBy = "concurrent",
|
||||
};
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
SqliteConnection.ClearAllPools();
|
||||
try
|
||||
{
|
||||
if (Directory.Exists(_root))
|
||||
{
|
||||
Directory.Delete(_root, recursive: true);
|
||||
}
|
||||
}
|
||||
catch (IOException)
|
||||
{
|
||||
// Best-effort temp cleanup.
|
||||
}
|
||||
catch (UnauthorizedAccessException)
|
||||
{
|
||||
// Best-effort temp cleanup.
|
||||
}
|
||||
}
|
||||
}
|
||||
+74
@@ -78,6 +78,80 @@ public sealed class SqliteSecretsStoreMigratorTests : IDisposable
|
||||
() => migrator.MigrateAsync(CancellationToken.None));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void NormalizedPath_DifferentSpellingsOfOneFile_ShareOneMigrationGate()
|
||||
{
|
||||
string dir = Path.GetDirectoryName(_dbPath)!;
|
||||
string detour = Path.Combine(dir, "zb-detour", "..", Path.GetFileName(_dbPath));
|
||||
|
||||
var canonical = new SecretsSqliteConnectionFactory(_dbPath);
|
||||
var other = new SecretsSqliteConnectionFactory(detour);
|
||||
|
||||
Assert.Equal(canonical.NormalizedPath, other.NormalizedPath);
|
||||
Assert.Same(
|
||||
SqliteSecretsStoreMigrator.GateFor(canonical.NormalizedPath),
|
||||
SqliteSecretsStoreMigrator.GateFor(other.NormalizedPath));
|
||||
Assert.NotSame(
|
||||
SqliteSecretsStoreMigrator.GateFor(canonical.NormalizedPath),
|
||||
SqliteSecretsStoreMigrator.GateFor(canonical.NormalizedPath + ".other"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Migrate_WaitsForTheProcessWideGateOfItsStore()
|
||||
{
|
||||
var factory = new SecretsSqliteConnectionFactory(_dbPath);
|
||||
SemaphoreSlim gate = SqliteSecretsStoreMigrator.GateFor(factory.NormalizedPath);
|
||||
|
||||
await gate.WaitAsync();
|
||||
Task migration;
|
||||
try
|
||||
{
|
||||
// A second factory/migrator pair, as a separate DI container would build.
|
||||
migration = new SqliteSecretsStoreMigrator(new SecretsSqliteConnectionFactory(_dbPath))
|
||||
.MigrateAsync(CancellationToken.None);
|
||||
|
||||
await Task.Delay(300);
|
||||
Assert.False(migration.IsCompleted, "migration ran while another migration of the same store held the gate");
|
||||
}
|
||||
finally
|
||||
{
|
||||
gate.Release();
|
||||
}
|
||||
|
||||
await migration.WaitAsync(TimeSpan.FromSeconds(30));
|
||||
Assert.Equal(SqliteSecretsSchema.CurrentVersion, await ReadSchemaVersionAsync(factory));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Migrate_RetriesWhenAnotherWriterHoldsTheStorePastTheBusyTimeout()
|
||||
{
|
||||
var factory = new SecretsSqliteConnectionFactory(_dbPath);
|
||||
|
||||
// Another "process": an unpooled connection holding the write lock (BEGIN IMMEDIATE) for longer
|
||||
// than the 5 s busy timeout, so the first migration attempt surfaces SQLITE_BUSY.
|
||||
await using SqliteConnection holder = await factory.OpenUnpooledConnectionAsync(CancellationToken.None);
|
||||
SqliteTransaction lockTx = holder.BeginTransaction(System.Data.IsolationLevel.Serializable, deferred: false);
|
||||
|
||||
Task migration = new SqliteSecretsStoreMigrator(factory).MigrateAsync(CancellationToken.None);
|
||||
await Task.Delay(TimeSpan.FromSeconds(6.5));
|
||||
Assert.False(migration.IsCompleted, "migration finished while the write lock was still held");
|
||||
|
||||
lockTx.Rollback();
|
||||
lockTx.Dispose();
|
||||
|
||||
await migration.WaitAsync(TimeSpan.FromSeconds(30));
|
||||
Assert.Equal(SqliteSecretsSchema.CurrentVersion, await ReadSchemaVersionAsync(factory));
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(5, true)] // SQLITE_BUSY
|
||||
[InlineData(6, true)] // SQLITE_LOCKED
|
||||
[InlineData(261, true)] // SQLITE_BUSY_RECOVERY (extended code of BUSY)
|
||||
[InlineData(1, false)] // SQLITE_ERROR, e.g. "cannot start a transaction within a transaction"
|
||||
[InlineData(11, false)] // SQLITE_CORRUPT
|
||||
public void IsTransient_OnlyBusyAndLockedAreRetried(int errorCode, bool expected) =>
|
||||
Assert.Equal(expected, SqliteSecretsStoreMigrator.IsTransient(new SqliteException("x", errorCode)));
|
||||
|
||||
private static async Task<long> ReadSchemaVersionAsync(SecretsSqliteConnectionFactory factory)
|
||||
{
|
||||
await using SqliteConnection connection = await factory.OpenConnectionAsync(CancellationToken.None);
|
||||
|
||||
Reference in New Issue
Block a user