c1e34c307e
Claude-Session: https://claude.ai/code/session_013L3XdmVii9i5sY4FChU8pB
208 lines
11 KiB
C#
208 lines
11 KiB
C#
using System.Data;
|
|
using System.Globalization;
|
|
using Microsoft.Data.SqlClient;
|
|
using ZB.MOM.WW.Auth.Abstractions.ApiKeys;
|
|
using ZB.MOM.WW.Auth.ApiKeys.Sqlite;
|
|
|
|
namespace ZB.MOM.WW.Auth.ApiKeys.SqlServer;
|
|
|
|
/// <summary>
|
|
/// Creates the shared SQL Server API-key schema and records the applied version. Idempotent: safe
|
|
/// to run on every node at every startup, including concurrently — lost provisioning races surface
|
|
/// as retryable <see cref="SqlException"/>s (see <see cref="IsRetryableMigrationErrorNumber"/>) and
|
|
/// the retry re-runs the <c>IF NOT EXISTS</c> path against the objects the winner just created.
|
|
/// Refuses to run against a database whose recorded version is newer than this build supports.
|
|
/// </summary>
|
|
/// <param name="connectionFactory">Factory for connections to the shared API-key database.</param>
|
|
public sealed class SqlServerApiKeyStoreMigrator(AuthSqlServerConnectionFactory connectionFactory)
|
|
: IApiKeyStoreMigrator
|
|
{
|
|
// Every node runs this at every startup, so a virgin database being provisioned by N nodes at
|
|
// once is the NORMAL case, not a rare race — and an unretried failure aborts that node's whole
|
|
// host at startup. Two failure shapes come out of that race, and retrying is correct for both
|
|
// precisely because the migration is idempotent:
|
|
//
|
|
// - Deadlock victim (1205): concurrent Serializable DDL against the system catalogs can pick
|
|
// a victim even though the IF NOT EXISTS guards make the logic itself safe.
|
|
// - Lost CREATE / lost INSERT (2714, 1913, 2627): Serializable range locks do NOT serialize
|
|
// the metadata existence checks against another session's in-flight CREATE — observed live
|
|
// on the sibling shared secret store (ScadaBridge central-pair first boot, 2026-08-07):
|
|
// both nodes passed the sys.schemas IF NOT EXISTS probe, the loser's CREATE SCHEMA then
|
|
// died with 2714 "There is already an object named 'zbsecrets'". The object the loser tried
|
|
// to create now exists, which is the desired end state — the retry's IF NOT EXISTS pass
|
|
// skips it and proceeds.
|
|
//
|
|
// The retry cap keeps this from masking a GENUINE conflict (e.g. an unrelated pre-existing
|
|
// object colliding with the schema name): that reproduces on every attempt, so after
|
|
// MaxMigrationRetries the real SqlException propagates and the node fails closed.
|
|
private const int MaxMigrationRetries = 5;
|
|
|
|
/// <summary>Deadlock victim.</summary>
|
|
private const int SqlDeadlockVictimErrorNumber = 1205;
|
|
|
|
/// <summary>"There is already an object named '%ls'" — lost CREATE SCHEMA / CREATE TABLE race.</summary>
|
|
private const int SqlObjectAlreadyExistsErrorNumber = 2714;
|
|
|
|
/// <summary>"An index or statistics with name '%ls' already exists" — lost CREATE INDEX race.</summary>
|
|
private const int SqlIndexAlreadyExistsErrorNumber = 1913;
|
|
|
|
/// <summary>Primary-key violation — lost version-row MERGE-insert race.</summary>
|
|
private const int SqlPrimaryKeyViolationErrorNumber = 2627;
|
|
|
|
/// <inheritdoc />
|
|
/// <exception cref="AuthStoreMigrationException">
|
|
/// The recorded schema version is newer than <see cref="SqlServerAuthSchema.CurrentVersion"/>.
|
|
/// </exception>
|
|
public async Task MigrateAsync(CancellationToken cancellationToken)
|
|
{
|
|
for (int attempt = 1; ; attempt++)
|
|
{
|
|
try
|
|
{
|
|
await MigrateOnceAsync(cancellationToken).ConfigureAwait(false);
|
|
return;
|
|
}
|
|
catch (SqlException ex)
|
|
when (IsRetryableMigrationError(ex) && attempt < MaxMigrationRetries)
|
|
{
|
|
await Task.Delay(BackoffFor(attempt), cancellationToken).ConfigureAwait(false);
|
|
}
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Back-off before retry attempt <paramref name="attempt"/> + 1: a growing base plus RANDOM
|
|
/// jitter. The jitter is the load-bearing half. The sibling secret-store migrator this is ported
|
|
/// from delays a flat <c>100 * attempt</c>, which is the SAME value for every victim of the same
|
|
/// round — so N deadlock victims wake together and re-collide, exactly the lockstep its comment
|
|
/// claims to prevent. Measured here against a live SQL Server with eight concurrent migrators on
|
|
/// a virgin schema, the unjittered form exhausted the five-attempt budget on repeated 1205s in
|
|
/// roughly 1 run in 60; with jitter it did not.
|
|
/// </summary>
|
|
/// <param name="attempt">The 1-based attempt that just failed.</param>
|
|
/// <returns>How long to wait before the next attempt.</returns>
|
|
private static TimeSpan BackoffFor(int attempt) =>
|
|
TimeSpan.FromMilliseconds((100 * attempt) + Random.Shared.Next(0, 100 * attempt));
|
|
|
|
/// <summary>
|
|
/// Whether any error carried by <paramref name="exception"/> is a retryable provisioning-race
|
|
/// outcome. Scans the whole <see cref="SqlException.Errors"/> collection rather than only
|
|
/// <see cref="SqlException.Number"/> because a failed DDL batch stacks follow-on errors (e.g.
|
|
/// 2714 arrives with 2759 "CREATE SCHEMA failed due to previous errors") and the retryable one
|
|
/// is not guaranteed to be first.
|
|
/// </summary>
|
|
/// <param name="exception">The exception the migration attempt failed with.</param>
|
|
/// <returns><see langword="true"/> when a retry may succeed.</returns>
|
|
internal static bool IsRetryableMigrationError(SqlException exception)
|
|
{
|
|
foreach (SqlError error in exception.Errors)
|
|
{
|
|
if (IsRetryableMigrationErrorNumber(error.Number))
|
|
{
|
|
return true;
|
|
}
|
|
}
|
|
|
|
return IsRetryableMigrationErrorNumber(exception.Number);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Classifies a SQL Server error number as a retryable outcome of N nodes provisioning the same
|
|
/// (possibly virgin) database at once. Everything here is a race whose retry re-runs the
|
|
/// idempotent <c>IF NOT EXISTS</c> path; anything else (login failures, missing database,
|
|
/// permissions, syntax) reproduces identically on retry and must propagate. 2705 (duplicate
|
|
/// column) is deliberately absent: every column lives inside a single <c>CREATE TABLE</c>
|
|
/// statement, so no cross-session race can produce it.
|
|
/// </summary>
|
|
/// <param name="errorNumber">The <see cref="SqlError.Number"/> to classify.</param>
|
|
/// <returns><see langword="true"/> when the number is a known provisioning-race outcome.</returns>
|
|
internal static bool IsRetryableMigrationErrorNumber(int errorNumber) => errorNumber
|
|
is SqlDeadlockVictimErrorNumber
|
|
or SqlObjectAlreadyExistsErrorNumber
|
|
or SqlIndexAlreadyExistsErrorNumber
|
|
or SqlPrimaryKeyViolationErrorNumber;
|
|
|
|
private async Task MigrateOnceAsync(CancellationToken cancellationToken)
|
|
{
|
|
string schema = connectionFactory.Schema;
|
|
|
|
await using SqlConnection connection =
|
|
await connectionFactory.OpenConnectionAsync(cancellationToken).ConfigureAwait(false);
|
|
|
|
// Serializable: unlike the single-writer SQLite store, EVERY node in the cluster runs this
|
|
// migration at startup, so concurrent first-boots are the normal case rather than a rare
|
|
// race. The isolation level narrows the race window but does NOT close it — the IF NOT
|
|
// EXISTS metadata probes are not serialized against another session's in-flight CREATE, so
|
|
// a loser can still fail with "object exists" (2714/1913) or a deadlock (1205). The retry
|
|
// filter in MigrateAsync is what makes simultaneous provision converge: by the time the
|
|
// loser retries, the winner's objects exist and every guard here skips them.
|
|
await using SqlTransaction transaction = (SqlTransaction)
|
|
await connection.BeginTransactionAsync(IsolationLevel.Serializable, cancellationToken)
|
|
.ConfigureAwait(false);
|
|
|
|
int existingVersion =
|
|
await ReadExistingSchemaVersionAsync(connection, transaction, schema, cancellationToken)
|
|
.ConfigureAwait(false);
|
|
|
|
if (existingVersion > SqlServerAuthSchema.CurrentVersion)
|
|
{
|
|
throw new AuthStoreMigrationException(
|
|
$"Shared API-key database schema version {existingVersion} is newer than supported version " +
|
|
$"{SqlServerAuthSchema.CurrentVersion}. Upgrade this node before it writes to the shared store.");
|
|
}
|
|
|
|
await using (SqlCommand ddl = connectionFactory.CreateCommand(
|
|
connection, SqlServerAuthSchema.CreateSchemaDdl(schema), transaction))
|
|
{
|
|
await ddl.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
|
|
}
|
|
|
|
// MERGE rather than insert-if-missing: WHEN MATCHED re-stamps the row, so a future schema
|
|
// version actually lands in the table instead of leaving the first-ever value behind.
|
|
await using (SqlCommand version = connectionFactory.CreateCommand(
|
|
connection,
|
|
$"""
|
|
MERGE [{schema}].[{SqlServerAuthSchema.SchemaVersionTable}] AS target
|
|
USING (SELECT 1 AS id, @version AS version, @applied_utc AS applied_utc) AS source
|
|
ON target.id = source.id
|
|
WHEN MATCHED THEN
|
|
UPDATE SET version = source.version, applied_utc = source.applied_utc
|
|
WHEN NOT MATCHED THEN
|
|
INSERT (id, version, applied_utc) VALUES (1, source.version, source.applied_utc);
|
|
""",
|
|
transaction))
|
|
{
|
|
version.Parameters.Add("@version", SqlDbType.Int).Value = SqlServerAuthSchema.CurrentVersion;
|
|
version.Parameters.Add("@applied_utc", SqlDbType.DateTimeOffset).Value = DateTimeOffset.UtcNow;
|
|
await version.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
|
|
}
|
|
|
|
await transaction.CommitAsync(cancellationToken).ConfigureAwait(false);
|
|
}
|
|
|
|
private async Task<int> ReadExistingSchemaVersionAsync(
|
|
SqlConnection connection,
|
|
SqlTransaction transaction,
|
|
string schema,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
// OBJECT_ID returns NULL for an absent table, so this reads 0 on a virgin database without
|
|
// needing the table to exist first.
|
|
await using SqlCommand command = connectionFactory.CreateCommand(
|
|
connection,
|
|
$"""
|
|
IF OBJECT_ID(N'[{schema}].[{SqlServerAuthSchema.SchemaVersionTable}]', N'U') IS NULL
|
|
SELECT 0;
|
|
ELSE
|
|
SELECT ISNULL((SELECT version FROM [{schema}].[{SqlServerAuthSchema.SchemaVersionTable}] WHERE id = 1), 0);
|
|
""",
|
|
transaction);
|
|
|
|
object? version = await command.ExecuteScalarAsync(cancellationToken).ConfigureAwait(false);
|
|
|
|
return version is null || version == DBNull.Value
|
|
? 0
|
|
: Convert.ToInt32(version, CultureInfo.InvariantCulture);
|
|
}
|
|
}
|