feat(auth.apikeys.sqlserver): zbauth schema v1 + concurrent-boot-safe migrator (retry on 1205/2714/1913/2627, newer-version fail-closed); live tests gated on AUTH_SQLSERVER_CONNSTR

Claude-Session: https://claude.ai/code/session_013L3XdmVii9i5sY4FChU8pB
2026-09-02 14:22:25 -04:00
parent 2969805e4e
commit c1e34c307e
7 changed files with 712 additions and 2 deletions
@@ -0,0 +1,101 @@
+using Microsoft.Data.SqlClient;
+
+namespace ZB.MOM.WW.Auth.ApiKeys.SqlServer.Tests.Live;
+
+/// <summary>
+/// Shared connection details for the env-gated live SQL Server suite.
+/// </summary>
+/// <remarks>
+/// Gated on <c>AUTH_SQLSERVER_CONNSTR</c> and skips cleanly when it is unset, matching the family
+/// live-test idiom (see <c>ZB.MOM.WW.Secrets.Replicator.SqlServer</c>) — the offline suite must stay
+/// runnable on any machine. Set it to a database the test is allowed to create and drop a schema in,
+/// for example <c>Server=localhost,1433;Database=ZbAuthTest;User Id=sa;Password=...;TrustServerCertificate=true</c>.
+/// Each test class works in a throwaway <em>schema</em> rather than a throwaway database: it needs no
+/// CREATE DATABASE right, and a virgin schema still reproduces the CREATE SCHEMA race the migrator's
+/// retry loop exists for.
+/// </remarks>
+public static class LiveSqlServer
+{
+    /// <summary>Environment variable holding the live connection string.</summary>
+    public const string ConnectionStringVariable = "AUTH_SQLSERVER_CONNSTR";
+
+    /// <summary>The configured connection string, or <see langword="null"/> when the suite is not enabled.</summary>
+    public static string? ConnectionString =>
+        Environment.GetEnvironmentVariable(ConnectionStringVariable);
+
+    /// <summary>Whether the live suite should run.</summary>
+    public static bool IsEnabled => !string.IsNullOrWhiteSpace(ConnectionString);
+
+    /// <summary>Skip reason shown when the suite is not enabled.</summary>
+    public const string SkipReason =
+        "Live SQL Server suite disabled; set AUTH_SQLSERVER_CONNSTR to enable.";
+
+    /// <summary>
+    /// Builds options against a throwaway schema so parallel runs and repeat runs cannot collide,
+    /// and so a failed run leaves no trace in a shared database.
+    /// </summary>
+    /// <param name="schemaName">The unique schema name for this test.</param>
+    /// <returns>Options targeting that schema.</returns>
+    public static SqlServerApiKeyOptions OptionsFor(string schemaName) => new()
+    {
+        ConnectionString = ConnectionString!,
+        Schema = schemaName,
+    };
+
+    /// <summary>Generates a unique, allow-list-legal schema name for one test.</summary>
+    /// <returns>A fresh schema name.</returns>
+    public static string NewSchemaName() => $"zbauth_test_{Guid.NewGuid():N}"[..32];
+
+    /// <summary>Drops the throwaway schema and its three tables.</summary>
+    /// <param name="schemaName">The schema to drop.</param>
+    /// <returns>A task that completes when the schema is gone.</returns>
+    public static async Task DropSchemaAsync(string schemaName)
+    {
+        if (!IsEnabled)
+        {
+            return;
+        }
+
+        await using var connection = new SqlConnection(ConnectionString);
+        await connection.OpenAsync();
+
+        await using SqlCommand command = connection.CreateCommand();
+        command.CommandText = $"""
+            IF OBJECT_ID(N'[{schemaName}].[api_key_audit]', N'U') IS NOT NULL DROP TABLE [{schemaName}].[api_key_audit];
+            IF OBJECT_ID(N'[{schemaName}].[api_keys]', N'U') IS NOT NULL DROP TABLE [{schemaName}].[api_keys];
+            IF OBJECT_ID(N'[{schemaName}].[schema_version]', N'U') IS NOT NULL DROP TABLE [{schemaName}].[schema_version];
+            IF EXISTS (SELECT 1 FROM sys.schemas WHERE name = N'{schemaName}') EXEC(N'DROP SCHEMA [{schemaName}]');
+            """;
+
+        await command.ExecuteNonQueryAsync();
+    }
+
+    /// <summary>Runs a scalar query against the live database.</summary>
+    /// <param name="commandText">The query text.</param>
+    /// <returns>The first column of the first row, or <see langword="null"/>.</returns>
+    public static async Task<object?> ScalarAsync(string commandText)
+    {
+        await using var connection = new SqlConnection(ConnectionString);
+        await connection.OpenAsync();
+
+        await using SqlCommand command = connection.CreateCommand();
+        command.CommandText = commandText;
+
+        object? value = await command.ExecuteScalarAsync();
+        return value == DBNull.Value ? null : value;
+    }
+
+    /// <summary>Runs a non-query statement against the live database.</summary>
+    /// <param name="commandText">The statement text.</param>
+    /// <returns>A task that completes when the statement has run.</returns>
+    public static async Task ExecuteAsync(string commandText)
+    {
+        await using var connection = new SqlConnection(ConnectionString);
+        await connection.OpenAsync();
+
+        await using SqlCommand command = connection.CreateCommand();
+        command.CommandText = commandText;
+
+        await command.ExecuteNonQueryAsync();
+    }
+}
@@ -0,0 +1,145 @@
+using System.Globalization;
+using ZB.MOM.WW.Auth.ApiKeys.Sqlite;
+
+namespace ZB.MOM.WW.Auth.ApiKeys.SqlServer.Tests.Live;
+
+/// <summary>
+/// Live provisioning tests for <see cref="SqlServerApiKeyStoreMigrator"/>.
+/// </summary>
+/// <remarks>
+/// Every test provisions its own throwaway schema — the concurrency case in particular needs a
+/// virgin one, because the race the retry loop exists for only happens while the schema and tables
+/// do not yet exist. All schemas created by the class are dropped in <see cref="DisposeAsync"/>.
+/// </remarks>
+[Collection("live-sqlserver")]
+public sealed class SqlServerApiKeyStoreMigratorLiveTests : IAsyncLifetime
+{
+    private readonly List<string> _schemas = [];
+
+    /// <inheritdoc />
+    public Task InitializeAsync() => Task.CompletedTask;
+
+    /// <inheritdoc />
+    public async Task DisposeAsync()
+    {
+        foreach (string schema in _schemas)
+        {
+            await LiveSqlServer.DropSchemaAsync(schema);
+        }
+    }
+
+    private string NewSchema()
+    {
+        string schema = LiveSqlServer.NewSchemaName();
+        _schemas.Add(schema);
+        return schema;
+    }
+
+    private static SqlServerApiKeyStoreMigrator MigratorFor(string schema) =>
+        new(new AuthSqlServerConnectionFactory(LiveSqlServer.OptionsFor(schema)));
+
+    [SkippableFact]
+    public async Task Migrate_OnEmptyDatabase_CreatesSchemaTablesAndVersionRow()
+    {
+        Skip.IfNot(LiveSqlServer.IsEnabled, LiveSqlServer.SkipReason);
+
+        string schema = NewSchema();
+
+        await MigratorFor(schema).MigrateAsync(CancellationToken.None);
+
+        Assert.Equal(
+            1,
+            Convert.ToInt32(
+                await LiveSqlServer.ScalarAsync(
+                    $"SELECT COUNT(*) FROM sys.schemas WHERE name = N'{schema}'"),
+                CultureInfo.InvariantCulture));
+
+        foreach (string table in new[]
+        {
+            SqlServerAuthSchema.SchemaVersionTable,
+            SqlServerAuthSchema.ApiKeysTable,
+            SqlServerAuthSchema.ApiKeyAuditTable,
+        })
+        {
+            Assert.NotNull(
+                await LiveSqlServer.ScalarAsync($"SELECT OBJECT_ID(N'[{schema}].[{table}]', N'U')"));
+        }
+
+        Assert.Equal(
+            SqlServerAuthSchema.CurrentVersion,
+            Convert.ToInt32(
+                await LiveSqlServer.ScalarAsync(
+                    $"SELECT version FROM [{schema}].[{SqlServerAuthSchema.SchemaVersionTable}] WHERE id = 1"),
+                CultureInfo.InvariantCulture));
+    }
+
+    [SkippableFact]
+    public async Task Migrate_Twice_IsIdempotent()
+    {
+        Skip.IfNot(LiveSqlServer.IsEnabled, LiveSqlServer.SkipReason);
+
+        string schema = NewSchema();
+        SqlServerApiKeyStoreMigrator migrator = MigratorFor(schema);
+
+        await migrator.MigrateAsync(CancellationToken.None);
+        await migrator.MigrateAsync(CancellationToken.None);
+
+        Assert.Equal(
+            1,
+            Convert.ToInt32(
+                await LiveSqlServer.ScalarAsync(
+                    $"SELECT COUNT(*) FROM [{schema}].[{SqlServerAuthSchema.SchemaVersionTable}]"),
+                CultureInfo.InvariantCulture));
+        Assert.Equal(
+            SqlServerAuthSchema.CurrentVersion,
+            Convert.ToInt32(
+                await LiveSqlServer.ScalarAsync(
+                    $"SELECT version FROM [{schema}].[{SqlServerAuthSchema.SchemaVersionTable}] WHERE id = 1"),
+                CultureInfo.InvariantCulture));
+    }
+
+    [SkippableFact]
+    public async Task Migrate_Concurrently_FromEightTasks_AllSucceed()
+    {
+        Skip.IfNot(LiveSqlServer.IsEnabled, LiveSqlServer.SkipReason);
+
+        // A virgin schema provisioned by eight nodes at once — the central-pair first boot, scaled
+        // up. Each task gets its own factory (and therefore its own connection) so the DDL really
+        // does run on eight sessions.
+        string schema = NewSchema();
+
+        await Task.WhenAll(Enumerable.Range(0, 8).Select(_ =>
+            Task.Run(() => MigratorFor(schema).MigrateAsync(CancellationToken.None))));
+
+        Assert.Equal(
+            1,
+            Convert.ToInt32(
+                await LiveSqlServer.ScalarAsync(
+                    $"SELECT COUNT(*) FROM [{schema}].[{SqlServerAuthSchema.SchemaVersionTable}]"),
+                CultureInfo.InvariantCulture));
+        Assert.Equal(
+            SqlServerAuthSchema.CurrentVersion,
+            Convert.ToInt32(
+                await LiveSqlServer.ScalarAsync(
+                    $"SELECT version FROM [{schema}].[{SqlServerAuthSchema.SchemaVersionTable}] WHERE id = 1"),
+                CultureInfo.InvariantCulture));
+    }
+
+    [SkippableFact]
+    public async Task Migrate_WhenStoredVersionIsNewer_Throws()
+    {
+        Skip.IfNot(LiveSqlServer.IsEnabled, LiveSqlServer.SkipReason);
+
+        string schema = NewSchema();
+        SqlServerApiKeyStoreMigrator migrator = MigratorFor(schema);
+
+        await migrator.MigrateAsync(CancellationToken.None);
+        await LiveSqlServer.ExecuteAsync(
+            $"UPDATE [{schema}].[{SqlServerAuthSchema.SchemaVersionTable}] SET version = 99 WHERE id = 1");
+
+        AuthStoreMigrationException exception = await Assert.ThrowsAsync<AuthStoreMigrationException>(
+            () => migrator.MigrateAsync(CancellationToken.None));
+
+        Assert.Contains("99", exception.Message, StringComparison.Ordinal);
+    }
+}
@@ -0,0 +1,43 @@
+namespace ZB.MOM.WW.Auth.ApiKeys.SqlServer.Tests;
+
+/// <summary>
+/// Offline pins for the migrator's retry classification. A <see cref="Microsoft.Data.SqlClient.SqlException"/>
+/// cannot be constructed normally, so the error-number predicate is extracted as an internal static
+/// and pinned here by number; the concurrent-provision behaviour itself is exercised by the
+/// env-gated live suite. These run on any machine, so the retry filter stays pinned even when the
+/// live suite skips.
+/// </summary>
+public sealed class SqlServerApiKeyStoreMigratorTests
+{
+    /// <summary>
+    /// The provisioning-race outcomes the ScadaBridge central shared-store live gate (2026-08-07)
+    /// proved reachable — or that the multi-statement DDL makes reachable — when N nodes provision
+    /// one virgin database at once. Losing such a race means the object now exists, which is the
+    /// desired end state, so a retry of the idempotent IF NOT EXISTS path must be taken.
+    /// </summary>
+    /// <param name="errorNumber">The SQL Server error number under test.</param>
+    [Theory]
+    [InlineData(1205)] // deadlock victim
+    [InlineData(2714)] // "There is already an object named ..." — lost CREATE SCHEMA/TABLE (the live-gate defect)
+    [InlineData(1913)] // "index ... already exists" — lost CREATE INDEX
+    [InlineData(2627)] // PK violation — lost schema_version MERGE-insert
+    public void Provisioning_race_error_numbers_are_retryable(int errorNumber)
+        => Assert.True(SqlServerApiKeyStoreMigrator.IsRetryableMigrationErrorNumber(errorNumber));
+
+    /// <summary>
+    /// Errors that reproduce identically on retry must propagate so the node fails closed instead
+    /// of spinning through the retry budget against a genuine misconfiguration.
+    /// </summary>
+    /// <param name="errorNumber">The SQL Server error number under test.</param>
+    [Theory]
+    [InlineData(2705)] // duplicate column — unreachable from the race (single CREATE TABLE statement)
+    [InlineData(208)] // invalid object name
+    [InlineData(229)] // permission denied
+    [InlineData(2759)] // "CREATE SCHEMA failed due to previous errors" — retryable only via its companion 2714
+    [InlineData(4060)] // cannot open database
+    [InlineData(18456)] // login failed
+    [InlineData(0)]
+    [InlineData(-2)] // client-side timeout
+    public void Non_race_error_numbers_are_not_retryable(int errorNumber)
+        => Assert.False(SqlServerApiKeyStoreMigrator.IsRetryableMigrationErrorNumber(errorNumber));
+}