using System.Data; using System.Globalization; using Microsoft.Data.SqlClient; using ZB.MOM.WW.Auth.Abstractions.ApiKeys; using ZB.MOM.WW.Auth.ApiKeys.Sqlite; namespace ZB.MOM.WW.Auth.ApiKeys.SqlServer; /// /// Creates the shared SQL Server API-key schema and records the applied version. Idempotent: safe /// to run on every node at every startup, including concurrently — lost provisioning races surface /// as retryable s (see ) and /// the retry re-runs the IF NOT EXISTS path against the objects the winner just created. /// Refuses to run against a database whose recorded version is newer than this build supports. /// /// Factory for connections to the shared API-key database. public sealed class SqlServerApiKeyStoreMigrator(AuthSqlServerConnectionFactory connectionFactory) : IApiKeyStoreMigrator { // Every node runs this at every startup, so a virgin database being provisioned by N nodes at // once is the NORMAL case, not a rare race — and an unretried failure aborts that node's whole // host at startup. Two failure shapes come out of that race, and retrying is correct for both // precisely because the migration is idempotent: // // - Deadlock victim (1205): concurrent Serializable DDL against the system catalogs can pick // a victim even though the IF NOT EXISTS guards make the logic itself safe. // - Lost CREATE / lost INSERT (2714, 1913, 2627): Serializable range locks do NOT serialize // the metadata existence checks against another session's in-flight CREATE — observed live // on the sibling shared secret store (ScadaBridge central-pair first boot, 2026-08-07): // both nodes passed the sys.schemas IF NOT EXISTS probe, the loser's CREATE SCHEMA then // died with 2714 "There is already an object named 'zbsecrets'". The object the loser tried // to create now exists, which is the desired end state — the retry's IF NOT EXISTS pass // skips it and proceeds. // // The retry cap keeps this from masking a GENUINE conflict (e.g. an unrelated pre-existing // object colliding with the schema name): that reproduces on every attempt, so after // MaxMigrationRetries the real SqlException propagates and the node fails closed. private const int MaxMigrationRetries = 5; /// Deadlock victim. private const int SqlDeadlockVictimErrorNumber = 1205; /// "There is already an object named '%ls'" — lost CREATE SCHEMA / CREATE TABLE race. private const int SqlObjectAlreadyExistsErrorNumber = 2714; /// "An index or statistics with name '%ls' already exists" — lost CREATE INDEX race. private const int SqlIndexAlreadyExistsErrorNumber = 1913; /// Primary-key violation — lost version-row MERGE-insert race. private const int SqlPrimaryKeyViolationErrorNumber = 2627; /// /// /// The recorded schema version is newer than . /// public async Task MigrateAsync(CancellationToken cancellationToken) { for (int attempt = 1; ; attempt++) { try { await MigrateOnceAsync(cancellationToken).ConfigureAwait(false); return; } catch (SqlException ex) when (IsRetryableMigrationError(ex) && attempt < MaxMigrationRetries) { await Task.Delay(BackoffFor(attempt), cancellationToken).ConfigureAwait(false); } } } /// /// Back-off before retry attempt + 1: a growing base plus RANDOM /// jitter. The jitter is the load-bearing half. The sibling secret-store migrator this is ported /// from delays a flat 100 * attempt, which is the SAME value for every victim of the same /// round — so N deadlock victims wake together and re-collide, exactly the lockstep its comment /// claims to prevent. Measured here against a live SQL Server with eight concurrent migrators on /// a virgin schema, the unjittered form exhausted the five-attempt budget on repeated 1205s in /// roughly 1 run in 60; with jitter it did not. /// /// The 1-based attempt that just failed. /// How long to wait before the next attempt. private static TimeSpan BackoffFor(int attempt) => TimeSpan.FromMilliseconds((100 * attempt) + Random.Shared.Next(0, 100 * attempt)); /// /// Whether any error carried by is a retryable provisioning-race /// outcome. Scans the whole collection rather than only /// because a failed DDL batch stacks follow-on errors (e.g. /// 2714 arrives with 2759 "CREATE SCHEMA failed due to previous errors") and the retryable one /// is not guaranteed to be first. /// /// The exception the migration attempt failed with. /// when a retry may succeed. internal static bool IsRetryableMigrationError(SqlException exception) { foreach (SqlError error in exception.Errors) { if (IsRetryableMigrationErrorNumber(error.Number)) { return true; } } return IsRetryableMigrationErrorNumber(exception.Number); } /// /// Classifies a SQL Server error number as a retryable outcome of N nodes provisioning the same /// (possibly virgin) database at once. Everything here is a race whose retry re-runs the /// idempotent IF NOT EXISTS path; anything else (login failures, missing database, /// permissions, syntax) reproduces identically on retry and must propagate. 2705 (duplicate /// column) is deliberately absent: every column lives inside a single CREATE TABLE /// statement, so no cross-session race can produce it. /// /// The to classify. /// when the number is a known provisioning-race outcome. internal static bool IsRetryableMigrationErrorNumber(int errorNumber) => errorNumber is SqlDeadlockVictimErrorNumber or SqlObjectAlreadyExistsErrorNumber or SqlIndexAlreadyExistsErrorNumber or SqlPrimaryKeyViolationErrorNumber; private async Task MigrateOnceAsync(CancellationToken cancellationToken) { string schema = connectionFactory.Schema; await using SqlConnection connection = await connectionFactory.OpenConnectionAsync(cancellationToken).ConfigureAwait(false); // Serializable: unlike the single-writer SQLite store, EVERY node in the cluster runs this // migration at startup, so concurrent first-boots are the normal case rather than a rare // race. The isolation level narrows the race window but does NOT close it — the IF NOT // EXISTS metadata probes are not serialized against another session's in-flight CREATE, so // a loser can still fail with "object exists" (2714/1913) or a deadlock (1205). The retry // filter in MigrateAsync is what makes simultaneous provision converge: by the time the // loser retries, the winner's objects exist and every guard here skips them. await using SqlTransaction transaction = (SqlTransaction) await connection.BeginTransactionAsync(IsolationLevel.Serializable, cancellationToken) .ConfigureAwait(false); int existingVersion = await ReadExistingSchemaVersionAsync(connection, transaction, schema, cancellationToken) .ConfigureAwait(false); if (existingVersion > SqlServerAuthSchema.CurrentVersion) { throw new AuthStoreMigrationException( $"Shared API-key database schema version {existingVersion} is newer than supported version " + $"{SqlServerAuthSchema.CurrentVersion}. Upgrade this node before it writes to the shared store."); } await using (SqlCommand ddl = connectionFactory.CreateCommand( connection, SqlServerAuthSchema.CreateSchemaDdl(schema), transaction)) { await ddl.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); } // MERGE rather than insert-if-missing: WHEN MATCHED re-stamps the row, so a future schema // version actually lands in the table instead of leaving the first-ever value behind. await using (SqlCommand version = connectionFactory.CreateCommand( connection, $""" MERGE [{schema}].[{SqlServerAuthSchema.SchemaVersionTable}] AS target USING (SELECT 1 AS id, @version AS version, @applied_utc AS applied_utc) AS source ON target.id = source.id WHEN MATCHED THEN UPDATE SET version = source.version, applied_utc = source.applied_utc WHEN NOT MATCHED THEN INSERT (id, version, applied_utc) VALUES (1, source.version, source.applied_utc); """, transaction)) { version.Parameters.Add("@version", SqlDbType.Int).Value = SqlServerAuthSchema.CurrentVersion; version.Parameters.Add("@applied_utc", SqlDbType.DateTimeOffset).Value = DateTimeOffset.UtcNow; await version.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); } await transaction.CommitAsync(cancellationToken).ConfigureAwait(false); } private async Task ReadExistingSchemaVersionAsync( SqlConnection connection, SqlTransaction transaction, string schema, CancellationToken cancellationToken) { // OBJECT_ID returns NULL for an absent table, so this reads 0 on a virgin database without // needing the table to exist first. await using SqlCommand command = connectionFactory.CreateCommand( connection, $""" IF OBJECT_ID(N'[{schema}].[{SqlServerAuthSchema.SchemaVersionTable}]', N'U') IS NULL SELECT 0; ELSE SELECT ISNULL((SELECT version FROM [{schema}].[{SqlServerAuthSchema.SchemaVersionTable}] WHERE id = 1), 0); """, transaction); object? version = await command.ExecuteScalarAsync(cancellationToken).ConfigureAwait(false); return version is null || version == DBNull.Value ? 0 : Convert.ToInt32(version, CultureInfo.InvariantCulture); } }