scadaproj/ZB.MOM.WW.Secrets/tests/ZB.MOM.WW.Secrets.Tests/Sqlite/SqliteSecretsStoreConcurrentInitTests.cs

using Microsoft.Data.Sqlite;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using ZB.MOM.WW.Secrets.Abstractions;
using ZB.MOM.WW.Secrets.DependencyInjection;
using ZB.MOM.WW.Secrets.Sqlite;

namespace ZB.MOM.WW.Secrets.Tests.Sqlite;

/// <summary>
/// scadaproj#25: several hosts in one process initialising the SAME SQLite secret store at once
/// (mxaccessgw's parallel test classes, each building a host whose migration hosted service and
/// store point at the per-process default path). On Windows this surfaced as
/// "cannot start a transaction within a transaction". Each simulated host owns its own
/// connection factory, migrator and store, exactly as separate DI containers would.
/// </summary>
public sealed class SqliteSecretsStoreConcurrentInitTests : IDisposable
{
    private const int Hosts = 16;
    private const int Rounds = 20;

    private readonly string _root =
        Path.Combine(Path.GetTempPath(), $"zb-secrets-concurrent-{Guid.NewGuid():N}");

    [Fact]
    public async Task ConcurrentInitialisation_OfOneFreshStore_AllHostsSucceed()
    {
        for (int round = 0; round < Rounds; round++)
        {
            string path = Path.Combine(_root, $"round-{round}", "secrets.db");
            await RunHostsAsync(path, round);
            await AssertStoreAsync(path, expectedSecrets: Hosts);
        }
    }

    [Fact]
    public async Task ConcurrentInitialisation_OfOneExistingStore_AllHostsSucceed()
    {
        string path = Path.Combine(_root, "existing", "secrets.db");
        await new SqliteSecretsStoreMigrator(new SecretsSqliteConnectionFactory(path))
            .MigrateAsync(CancellationToken.None);

        for (int round = 0; round < Rounds; round++)
        {
            await RunHostsAsync(path, round);
        }

        await AssertStoreAsync(path, expectedSecrets: Hosts * Rounds);
    }

    [Fact]
    public async Task ConcurrentInitialisation_ThroughDifferentSpellingsOfOnePath_AllHostsSucceed()
    {
        // The per-path gate keys on the NORMALISED path: a relative spelling and a "..\" detour
        // must land on the same gate as the canonical absolute path.
        string dir = Path.Combine(_root, "spellings");
        string canonical = Path.Combine(dir, "secrets.db");
        string detour = Path.Combine(dir, "sub", "..", "secrets.db");
        string[] spellings = [canonical, detour];

        Task[] hosts = new Task[Hosts];
        TaskCompletionSource start = new(TaskCreationOptions.RunContinuationsAsynchronously);
        for (int i = 0; i < Hosts; i++)
        {
            string spelling = spellings[i % spellings.Length];
            int host = i;
            hosts[i] = Task.Run(async () =>
            {
                await start.Task;
                await RunOneHostAsync(spelling, $"spell/{host}");
            });
        }

        start.SetResult();
        await Task.WhenAll(hosts);
        await AssertStoreAsync(canonical, expectedSecrets: Hosts);
    }

    [Fact]
    public async Task ConcurrentInitialisation_ThroughSeparateAddZbSecretsContainers_AllSucceed()
    {
        // The exact mxaccessgw shape: every host builds its own container with AddZbSecrets, runs the
        // pre-host migrator, then starts the SecretsMigrationHostedService — all on one store path.
        string path = Path.Combine(_root, "di", "secrets.db");
        IConfiguration configuration = new ConfigurationBuilder()
            .AddInMemoryCollection(new Dictionary<string, string?>
            {
                ["Secrets:SqlitePath"] = path,
                ["Secrets:RunMigrationsOnStartup"] = "true",
            })
            .Build();

        TaskCompletionSource start = new(TaskCreationOptions.RunContinuationsAsynchronously);
        Task[] hosts = new Task[Hosts];
        for (int i = 0; i < Hosts; i++)
        {
            hosts[i] = Task.Run(async () =>
            {
                await using ServiceProvider provider = new ServiceCollection()
                    .AddZbSecrets(configuration, "Secrets")
                    .BuildServiceProvider();
                await start.Task;

                provider.GetRequiredService<SqliteSecretsStoreMigrator>()
                    .MigrateAsync(default).GetAwaiter().GetResult();
                foreach (IHostedService hosted in provider.GetServices<IHostedService>())
                {
                    if (hosted.GetType().Name == "SecretsMigrationHostedService")
                    {
                        await hosted.StartAsync(CancellationToken.None);
                    }
                }

                _ = await provider.GetRequiredService<ISecretStore>().ListAsync(includeDeleted: true, CancellationToken.None);
            });
        }

        start.SetResult();
        await Task.WhenAll(hosts);
        await AssertStoreAsync(path, expectedSecrets: 0);
    }

    private static async Task RunHostsAsync(string path, int round)
    {
        TaskCompletionSource start = new(TaskCreationOptions.RunContinuationsAsynchronously);
        Task[] hosts = new Task[Hosts];
        for (int i = 0; i < Hosts; i++)
        {
            int host = i;
            hosts[i] = Task.Run(async () =>
            {
                await start.Task;
                await RunOneHostAsync(path, $"r{round}/h{host}");
            });
        }

        // Release every host at once so their migrations genuinely overlap.
        start.SetResult();
        await Task.WhenAll(hosts);
    }

    private static async Task RunOneHostAsync(string path, string secretName)
    {
        // One "host": its own factory, migrator and store, as a separate DI container would build.
        SecretsSqliteConnectionFactory factory = new(path);
        await new SqliteSecretsStoreMigrator(factory).MigrateAsync(CancellationToken.None);

        SqliteSecretStore store = new(factory);
        StoredSecret row = MakeSecret(secretName);
        await store.UpsertAsync(row, CancellationToken.None);
        await store.ApplyReplicatedAsync(row with { Revision = 5, UpdatedUtc = row.UpdatedUtc.AddSeconds(1) }, CancellationToken.None);
        Assert.NotNull(await store.GetAsync(row.Name, CancellationToken.None));
        _ = await store.ListAsync(includeDeleted: false, CancellationToken.None);
    }

    private static async Task AssertStoreAsync(string path, int expectedSecrets)
    {
        SecretsSqliteConnectionFactory factory = new(path);
        await using SqliteConnection connection = await factory.OpenConnectionAsync(CancellationToken.None);

        await using SqliteCommand version = connection.CreateCommand();
        version.CommandText = "SELECT version FROM schema_version WHERE id = 1;";
        Assert.Equal((long)SqliteSecretsSchema.CurrentVersion, Convert.ToInt64(await version.ExecuteScalarAsync()));

        await using SqliteCommand count = connection.CreateCommand();
        count.CommandText = "SELECT COUNT(*) FROM secret;";
        Assert.Equal((long)expectedSecrets, Convert.ToInt64(await count.ExecuteScalarAsync()));

        // No pooled connection may be handed back with a transaction still open: a fresh
        // BEGIN on a pooled connection is exactly what failed on Windows.
        await using SqliteTransaction tx = (SqliteTransaction)
            await connection.BeginTransactionAsync(System.Data.IsolationLevel.Serializable);
        await tx.CommitAsync();
    }

    private static StoredSecret MakeSecret(string name) => new()
    {
        Name = new SecretName(name),
        Description = "concurrent-init",
        ContentType = SecretContentType.ConnectionString,
        Ciphertext = [1, 2, 3],
        Nonce = [4, 5, 6],
        Tag = [7, 8, 9],
        WrappedDek = [10, 11, 12],
        WrapNonce = [13, 14, 15],
        WrapTag = [16, 17, 18],
        KekId = "kek-1",
        Revision = 0,
        IsDeleted = false,
        DeletedUtc = null,
        CreatedUtc = DateTimeOffset.UtcNow,
        UpdatedUtc = DateTimeOffset.UtcNow,
        CreatedBy = "concurrent",
        UpdatedBy = "concurrent",
    };

    public void Dispose()
    {
        SqliteConnection.ClearAllPools();
        try
        {
            if (Directory.Exists(_root))
            {
                Directory.Delete(_root, recursive: true);
            }
        }
        catch (IOException)
        {
            // Best-effort temp cleanup.
        }
        catch (UnauthorizedAccessException)
        {
            // Best-effort temp cleanup.
        }
    }
}