using Microsoft.Data.SqlClient; using Microsoft.EntityFrameworkCore; namespace ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests.Migrations; /// /// Per-test-class MSSQL fixture for the Bundle C integration tests (#23 M1). /// /// Creates a fresh, uniquely-named test database on the running infra/mssql /// container, applies the EF migrations against it, and drops it on dispose. /// When MSSQL is not reachable (CI without the container), /// is set to false and describes why — tests pair /// [SkippableFact] with Skip.IfNot(_fixture.Available, _fixture.SkipReason) /// so the runner reports them as Skipped (not silently Passed). /// /// /// /// xunit 2.9.x has no native Assert.Skip/Assert.SkipUnless (those /// are v3); the project uses the Xunit.SkippableFact package as the canonical /// equivalent. The fixture attempts connect + create-db + migrate once at /// construct time. Only connect-failure exceptions (SqlException, plus the /// InvalidOperationException SqlClient raises from OpenAsync) flip Available to /// false — every other exception bubbles up so a real bug is not silently /// swallowed. /// /// /// Cross-process throttling (deferred-work register row 82). Database /// naming was already isolated (a guid per fixture instance), but nothing /// bounded how many fixtures ran CREATE DATABASE + the full migration /// chain against the single dev SQL Server at the same time. xunit runs /// processor-count test collections in parallel per process, and a /// full-solution run starts several MSSQL-backed test processes at once — so the /// server saw dozens of concurrent fresh-database migrations. That is pure /// resource saturation (CPU / log IO / tempdb on one container), not object-level /// locking, and it pushed individual migration DDL statements past EF's default /// 30-second command timeout. Every observed failure had the identical /// fingerprint: MsSqlMigrationFixture..ctor → Migrator.MigrateAsync /// → SqlException: Execution Timeout Expired (error 258), which xunit /// reports as "Class fixture type … threw in its constructor" against every test /// in the class — a false red that vanished on a re-run in isolation. /// /// /// Two changes close it. First, bounds the number of /// simultaneous create+migrate windows across every process on the machine /// using SQL Server application locks (sp_getapplock) on master as a /// counting semaphore — an xunit collection attribute cannot do this, because the /// contention is between separate test processes. Second, the create/migrate /// commands get timeouts sized for a queued-but-progressing server rather than an /// idle one, so a slot holder waits instead of failing. The work always completed; /// it just did not complete within 30 seconds. /// /// public sealed class MsSqlMigrationFixture : IDisposable { // Same credentials infra/mssql/setup.sql + docker-compose use. Not a committed // production secret — this is a local dev container connection string. // Connect Timeout=3 makes the connectivity probe fail fast (~3s per attempt) in a // no-container environment rather than hanging on SqlClient's default 30s. private const string DefaultAdminConnectionString = "Server=localhost,1433;User Id=sa;Password=ScadaBridge_Dev1#;TrustServerCertificate=true;Encrypt=false;Connect Timeout=3"; private const string AdminEnvVar = "SCADABRIDGE_MSSQL_TEST_CONN"; /// /// Command timeout for the migration chain. Deliberately far above EF's 30s /// default: under a bounded queue the server is busy but progressing, and the /// correct response to a slow DDL statement is to wait for it, not to fail the /// whole class. See the register-row-82 note in the type remarks. /// private const int MigrationCommandTimeoutSeconds = 300; /// /// Command timeout for the CREATE / DROP DATABASE statements. Lower than the /// migration chain (one statement, not fifty-odd) but still well clear of the /// 30s default, because CREATE DATABASE zeroes a fresh log file and that is the /// single most IO-bound step in the fixture. /// private const int DatabaseDdlCommandTimeoutSeconds = 120; /// /// Number of connectivity-probe attempts before declaring MSSQL unavailable. /// A single 3s attempt is not proof of absence once the server is under load — /// treating a busy server as an absent one would silently turn every MSSQL test /// into a Skip, which reads as green and is worse than a red. /// private const int ConnectProbeAttempts = 3; /// /// Databases whose name matches the fixture prefix and that are older than this /// are assumed to be strays from a crashed run and are dropped at process start. /// Comfortably longer than any legitimate fixture lifetime, so a live fixture in /// a concurrently-running process is never targeted. /// private static readonly TimeSpan OrphanDatabaseMaxAge = TimeSpan.FromHours(2); private const string DatabaseNamePrefix = "ScadaBridgeAuditMigTest_"; /// Guards the once-per-process orphan sweep. private static int _orphanSweepStarted; public string DatabaseName { get; } public string ConnectionString { get; } /// /// True when the MSSQL container was reachable at fixture construction /// time AND the per-fixture test database was successfully created. When /// false, the integration tests using this fixture must early-return. /// public bool Available { get; } /// /// Populated when is false; describes why the /// fixture chose to skip (env var unset, connect failed, etc.). /// public string SkipReason { get; } private readonly string _adminConnectionString; public MsSqlMigrationFixture() { // Short, lowercase guid suffix keeps the database identifier under SQL Server's // 128-char limit and safe for raw concatenation (no quoting required). DatabaseName = $"{DatabaseNamePrefix}{Guid.NewGuid():N}".Substring(0, 38); // Env var lets CI / power users override the admin endpoint; absent // defaults to the local docker dev container's sa connection. var fromEnv = Environment.GetEnvironmentVariable(AdminEnvVar); _adminConnectionString = string.IsNullOrWhiteSpace(fromEnv) ? DefaultAdminConnectionString : fromEnv; // Step 1: probe connectivity. This is the only step that may legitimately // fail when MSSQL is absent; SqlException + the rare InvalidOperationException // from Open are the connect-failure surfaces we tolerate. Everything else // (CREATE DATABASE, MigrateAsync) is treated as a hard fixture failure once we // *have* a connection. var probeFailure = ProbeConnectivity(_adminConnectionString); if (probeFailure is not null) { ConnectionString = string.Empty; Available = false; SkipReason = probeFailure; return; } // Step 2: best-effort, once per process — drop databases stranded by a run // that crashed before Dispose. The guid suffix means they never collide, but // they do accumulate and consume server resources, which feeds the very // saturation this fixture now throttles against. SweepOrphanDatabasesOnce(_adminConnectionString); try { // Step 3: take a create+migrate slot. Held across BOTH the CREATE DATABASE // and the migration chain, then released — this is the machine-wide bound // on how much fresh-database DDL the dev server is asked to do at once. using (MigrationGate.Acquire(_adminConnectionString)) { using (var connection = new SqlConnection(_adminConnectionString)) { connection.Open(); using var createCmd = connection.CreateCommand(); createCmd.CommandTimeout = DatabaseDdlCommandTimeoutSeconds; createCmd.CommandText = $"CREATE DATABASE [{DatabaseName}];"; createCmd.ExecuteNonQuery(); } ConnectionString = BuildPerDbConnectionString(_adminConnectionString, DatabaseName); // Apply the EF migrations once at fixture construction so each test // can read from a fully-migrated database without per-test setup. // Failures here are real bugs — let them bubble. ApplyMigrationsCore(ConnectionString, CancellationToken.None).GetAwaiter().GetResult(); } Available = true; SkipReason = string.Empty; } catch { // Best-effort cleanup if we created the database but failed before // setting Available — otherwise Dispose() would skip the drop. TryDropOrphanDatabase(); throw; } } /// /// Opens the admin connection up to times. /// Returns null when a connection was established, or the skip reason when every /// attempt failed. /// private static string? ProbeConnectivity(string adminConnectionString) { string lastFailure = "MSSQL unavailable (no connect attempt was made)"; for (var attempt = 1; attempt <= ConnectProbeAttempts; attempt++) { try { using var connection = new SqlConnection(adminConnectionString); connection.Open(); return null; } catch (SqlException ex) { lastFailure = $"MSSQL unavailable (connect failed after {attempt} attempt(s): SqlException {ex.Number}: {ex.Message})"; } catch (InvalidOperationException ex) { lastFailure = $"MSSQL unavailable (Open threw after {attempt} attempt(s): {ex.Message})"; } if (attempt < ConnectProbeAttempts) { Thread.Sleep(TimeSpan.FromMilliseconds(250 * attempt)); } } return lastFailure; } /// /// Drops fixture databases left behind by a run that died before /// . Runs at most once per test process and swallows every /// failure — an uncleaned stray is untidy, not fatal, and must never turn into a /// fixture error. /// private static void SweepOrphanDatabasesOnce(string adminConnectionString) { if (Interlocked.Exchange(ref _orphanSweepStarted, 1) != 0) { return; } try { using var connection = new SqlConnection(adminConnectionString); connection.Open(); var stale = new List(); using (var listCmd = connection.CreateCommand()) { listCmd.CommandTimeout = DatabaseDdlCommandTimeoutSeconds; listCmd.CommandText = "SELECT name FROM sys.databases " + "WHERE name LIKE @Prefix + '%' AND create_date < DATEADD(minute, -@AgeMinutes, GETDATE());"; listCmd.Parameters.AddWithValue("@Prefix", DatabaseNamePrefix); listCmd.Parameters.AddWithValue("@AgeMinutes", (int)OrphanDatabaseMaxAge.TotalMinutes); using var reader = listCmd.ExecuteReader(); while (reader.Read()) { stale.Add(reader.GetString(0)); } } foreach (var name in stale) { try { using var dropCmd = connection.CreateCommand(); dropCmd.CommandTimeout = DatabaseDdlCommandTimeoutSeconds; // Name comes from sys.databases filtered to our own guid-suffixed // prefix, so it cannot carry an injection payload. dropCmd.CommandText = $"ALTER DATABASE [{name}] SET SINGLE_USER WITH ROLLBACK IMMEDIATE; " + $"DROP DATABASE [{name}];"; dropCmd.ExecuteNonQuery(); } catch { // Another process may be mid-drop, or the database may be in use. // Best-effort by design. } } } catch { // Best-effort by design. } } private void TryDropOrphanDatabase() { if (string.IsNullOrEmpty(ConnectionString)) { return; } DropSelf(); } /// /// Applies the EF migrations to the per-fixture test database via a freshly /// constructed pointed at it. Uses the /// schema-only single-argument constructor — the AuditLog migration does /// not write secret-bearing columns at apply time. Called once from the /// constructor; tests do not invoke this directly. /// private static async Task ApplyMigrationsCore(string connectionString, CancellationToken cancellationToken) { var options = new DbContextOptionsBuilder() // The explicit command timeout is the register-row-82 fix: EF's 30s default // is what actually failed under cross-process load, and every migration // statement in the chain inherits this value. .UseSqlServer(connectionString, sql => sql.CommandTimeout(MigrationCommandTimeoutSeconds)) .Options; await using var context = new ScadaBridgeDbContext(options); await context.Database.MigrateAsync(cancellationToken); } /// /// Convenience for opening a fresh to the test /// database. Caller is responsible for disposal. /// public SqlConnection OpenConnection() { ThrowIfUnavailable(); var connection = new SqlConnection(ConnectionString); connection.Open(); return connection; } public void Dispose() { if (!Available) { return; } DropSelf(); } /// /// Best-effort drop of this fixture's database — never let a teardown failure /// pollute later runs. SINGLE_USER WITH ROLLBACK IMMEDIATE detaches lingering /// pooled connections so the DROP DATABASE doesn't fail with "database is in use". /// Deliberately NOT gated by : holding a create+migrate /// slot at teardown would starve the fixtures still trying to start. /// private void DropSelf() { try { // Connection-pool cleanup is necessary because EF's MigrateAsync leaves // pooled connections behind; SqlConnection.ClearAllPools() forces them // closed so the SINGLE_USER + DROP sequence below can complete. SqlConnection.ClearAllPools(); using var connection = new SqlConnection(_adminConnectionString); connection.Open(); using var cmd = connection.CreateCommand(); cmd.CommandTimeout = DatabaseDdlCommandTimeoutSeconds; cmd.CommandText = $"IF DB_ID(N'{DatabaseName}') IS NOT NULL " + $"BEGIN " + $" ALTER DATABASE [{DatabaseName}] SET SINGLE_USER WITH ROLLBACK IMMEDIATE; " + $" DROP DATABASE [{DatabaseName}]; " + $"END"; cmd.ExecuteNonQuery(); } catch { // Swallow — the database name carries a random guid suffix so a // stranded test database does not collide with future runs, and the // next process's orphan sweep will clear it. } } /// /// Throws an when invoked on an /// unavailable fixture; tests should branch on /// before reaching this code path. /// private void ThrowIfUnavailable() { if (!Available) { throw new InvalidOperationException( $"MsSqlMigrationFixture is not Available: {SkipReason}"); } } private static string BuildPerDbConnectionString(string adminConnectionString, string databaseName) { var builder = new SqlConnectionStringBuilder(adminConnectionString) { InitialCatalog = databaseName, }; return builder.ConnectionString; } /// /// A machine-wide counting semaphore over the create+migrate window, built from /// SQL Server session-scoped application locks on master /// (deferred-work register row 82). /// /// /// /// Why not an xunit collection? xunit's parallelism controls only bound /// the collections inside ONE test assembly running in ONE process. The /// contention here is between processes — AuditLog.Tests, /// ConfigurationDatabase.Tests and anything else a solution-level /// dotnet test starts in parallel all hammer the same container. The /// arbiter therefore has to live on the shared resource itself, which is the /// SQL Server instance. /// /// /// sp_getapplock is a binary mutex, so a counting semaphore is emulated /// with distinctly-named lock resources: a caller walks /// the slots with a short per-slot timeout and takes the first free one, looping /// until it wins or the overall deadline passes. Application locks are /// database-scoped, so every gate connection targets master to make /// sure all processes contend over the same namespace regardless of which test /// database they are about to build. /// /// /// Pooling is disabled on the gate connection. A session-scoped application lock /// is released by sp_reset_connection, which a pooled connection only /// issues when it is next reused — so a pooled gate connection could keep /// a slot locked long after Dispose. Closing an unpooled connection ends the /// session outright, and the explicit sp_releaseapplock makes the normal /// path deterministic rather than relying on that. A crashed process drops its /// connection, and SQL Server releases the lock with it, so the gate cannot /// wedge the suite. /// /// /// The gate never fails a test: if the deadline passes without a slot, it returns /// an ungated instance and lets the (now generous) command timeouts carry the /// run. A throttle that could itself become a source of red is not worth having. /// /// private sealed class MigrationGate : IDisposable { /// /// How many fixtures may build a database at once, machine-wide. Sized to keep /// the dev container busy without saturating it; override with /// SCADABRIDGE_MSSQL_TEST_MIGRATE_SLOTS. /// private const int DefaultSlotCount = 4; private const string SlotEnvVar = "SCADABRIDGE_MSSQL_TEST_MIGRATE_SLOTS"; private const string ResourcePrefix = "ScadaBridgeTestMigrate/"; /// Per-slot wait before moving on to try the next slot. private const int PerSlotLockTimeoutMs = 500; /// /// Overall wait for a slot. Long enough to outlast a full queue of migration /// chains on a loaded server; past it the gate gives up and lets the caller /// proceed ungated rather than failing. /// private static readonly TimeSpan AcquireDeadline = TimeSpan.FromMinutes(10); private static int SlotCount { get { var raw = Environment.GetEnvironmentVariable(SlotEnvVar); return int.TryParse(raw, out var parsed) && parsed > 0 ? parsed : DefaultSlotCount; } } private readonly SqlConnection? _connection; private readonly string? _resource; private MigrationGate(SqlConnection? connection, string? resource) { _connection = connection; _resource = resource; } public static MigrationGate Acquire(string adminConnectionString) { var slots = SlotCount; SqlConnection? connection = null; try { connection = new SqlConnection(BuildGateConnectionString(adminConnectionString)); connection.Open(); var deadline = DateTime.UtcNow + AcquireDeadline; // Start each caller at a different slot so they spread out instead of // all queueing behind slot 0. var offset = Random.Shared.Next(slots); while (DateTime.UtcNow < deadline) { for (var i = 0; i < slots; i++) { var resource = ResourcePrefix + ((offset + i) % slots); if (TryAcquireSlot(connection, resource)) { return new MigrationGate(connection, resource); } } } } catch { // The gate is an optimisation, not a correctness requirement — if it // cannot be established (permissions, an older server, a transient // fault) the caller proceeds ungated. } connection?.Dispose(); return new MigrationGate(null, null); } private static bool TryAcquireSlot(SqlConnection connection, string resource) { using var cmd = connection.CreateCommand(); cmd.CommandType = System.Data.CommandType.StoredProcedure; cmd.CommandText = "sp_getapplock"; cmd.CommandTimeout = (PerSlotLockTimeoutMs / 1000) + 30; cmd.Parameters.AddWithValue("@Resource", resource); cmd.Parameters.AddWithValue("@LockMode", "Exclusive"); cmd.Parameters.AddWithValue("@LockOwner", "Session"); cmd.Parameters.AddWithValue("@LockTimeout", PerSlotLockTimeoutMs); var result = cmd.Parameters.Add("@Result", System.Data.SqlDbType.Int); result.Direction = System.Data.ParameterDirection.ReturnValue; cmd.ExecuteNonQuery(); // 0 = granted immediately, 1 = granted after waiting; everything else // (-1 timeout, -2 cancelled, -3 deadlock victim, -999 error) means "not ours". var code = result.Value as int? ?? -999; return code >= 0; } private static string BuildGateConnectionString(string adminConnectionString) { return new SqlConnectionStringBuilder(adminConnectionString) { // Application locks are database-scoped: every process must contend in // the same database for the slots to mean anything. InitialCatalog = "master", // See the remarks — a pooled connection defers the implicit release of a // session-scoped app lock until the connection is reused. Pooling = false, }.ConnectionString; } public void Dispose() { if (_connection is null) { return; } try { if (_resource is not null) { using var cmd = _connection.CreateCommand(); cmd.CommandType = System.Data.CommandType.StoredProcedure; cmd.CommandText = "sp_releaseapplock"; cmd.Parameters.AddWithValue("@Resource", _resource); cmd.Parameters.AddWithValue("@LockOwner", "Session"); cmd.ExecuteNonQuery(); } } catch { // Closing the unpooled connection below ends the session, which releases // the lock regardless. } finally { _connection.Dispose(); } } } }