using Microsoft.Data.SqlClient;
using Microsoft.EntityFrameworkCore;
namespace ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests.Migrations;
///
/// Per-test-class MSSQL fixture for the Bundle C integration tests (#23 M1).
///
/// Creates a fresh, uniquely-named test database on the running infra/mssql
/// container, applies the EF migrations against it, and drops it on dispose.
/// When MSSQL is not reachable (CI without the container),
/// is set to false and describes why — tests pair
/// [SkippableFact] with Skip.IfNot(_fixture.Available, _fixture.SkipReason)
/// so the runner reports them as Skipped (not silently Passed).
///
///
///
/// xunit 2.9.x has no native Assert.Skip/Assert.SkipUnless (those
/// are v3); the project uses the Xunit.SkippableFact package as the canonical
/// equivalent. The fixture attempts connect + create-db + migrate once at
/// construct time. Only connect-failure exceptions (SqlException, plus the
/// InvalidOperationException SqlClient raises from OpenAsync) flip Available to
/// false — every other exception bubbles up so a real bug is not silently
/// swallowed.
///
///
/// Cross-process throttling (deferred-work register row 82). Database
/// naming was already isolated (a guid per fixture instance), but nothing
/// bounded how many fixtures ran CREATE DATABASE + the full migration
/// chain against the single dev SQL Server at the same time. xunit runs
/// processor-count test collections in parallel per process, and a
/// full-solution run starts several MSSQL-backed test processes at once — so the
/// server saw dozens of concurrent fresh-database migrations. That is pure
/// resource saturation (CPU / log IO / tempdb on one container), not object-level
/// locking, and it pushed individual migration DDL statements past EF's default
/// 30-second command timeout. Every observed failure had the identical
/// fingerprint: MsSqlMigrationFixture..ctor → Migrator.MigrateAsync
/// → SqlException: Execution Timeout Expired (error 258), which xunit
/// reports as "Class fixture type … threw in its constructor" against every test
/// in the class — a false red that vanished on a re-run in isolation.
///
///
/// Two changes close it. First, bounds the number of
/// simultaneous create+migrate windows across every process on the machine
/// using SQL Server application locks (sp_getapplock) on master as a
/// counting semaphore — an xunit collection attribute cannot do this, because the
/// contention is between separate test processes. Second, the create/migrate
/// commands get timeouts sized for a queued-but-progressing server rather than an
/// idle one, so a slot holder waits instead of failing. The work always completed;
/// it just did not complete within 30 seconds.
///
///
public sealed class MsSqlMigrationFixture : IDisposable
{
// Same credentials infra/mssql/setup.sql + docker-compose use. Not a committed
// production secret — this is a local dev container connection string.
// Connect Timeout=3 makes the connectivity probe fail fast (~3s per attempt) in a
// no-container environment rather than hanging on SqlClient's default 30s.
private const string DefaultAdminConnectionString =
"Server=localhost,1433;User Id=sa;Password=ScadaBridge_Dev1#;TrustServerCertificate=true;Encrypt=false;Connect Timeout=3";
private const string AdminEnvVar = "SCADABRIDGE_MSSQL_TEST_CONN";
///
/// Command timeout for the migration chain. Deliberately far above EF's 30s
/// default: under a bounded queue the server is busy but progressing, and the
/// correct response to a slow DDL statement is to wait for it, not to fail the
/// whole class. See the register-row-82 note in the type remarks.
///
private const int MigrationCommandTimeoutSeconds = 300;
///
/// Command timeout for the CREATE / DROP DATABASE statements. Lower than the
/// migration chain (one statement, not fifty-odd) but still well clear of the
/// 30s default, because CREATE DATABASE zeroes a fresh log file and that is the
/// single most IO-bound step in the fixture.
///
private const int DatabaseDdlCommandTimeoutSeconds = 120;
///
/// Number of connectivity-probe attempts before declaring MSSQL unavailable.
/// A single 3s attempt is not proof of absence once the server is under load —
/// treating a busy server as an absent one would silently turn every MSSQL test
/// into a Skip, which reads as green and is worse than a red.
///
private const int ConnectProbeAttempts = 3;
///
/// Databases whose name matches the fixture prefix and that are older than this
/// are assumed to be strays from a crashed run and are dropped at process start.
/// Comfortably longer than any legitimate fixture lifetime, so a live fixture in
/// a concurrently-running process is never targeted.
///
private static readonly TimeSpan OrphanDatabaseMaxAge = TimeSpan.FromHours(2);
private const string DatabaseNamePrefix = "ScadaBridgeAuditMigTest_";
/// Guards the once-per-process orphan sweep.
private static int _orphanSweepStarted;
public string DatabaseName { get; }
public string ConnectionString { get; }
///
/// True when the MSSQL container was reachable at fixture construction
/// time AND the per-fixture test database was successfully created. When
/// false, the integration tests using this fixture must early-return.
///
public bool Available { get; }
///
/// Populated when is false; describes why the
/// fixture chose to skip (env var unset, connect failed, etc.).
///
public string SkipReason { get; }
private readonly string _adminConnectionString;
public MsSqlMigrationFixture()
{
// Short, lowercase guid suffix keeps the database identifier under SQL Server's
// 128-char limit and safe for raw concatenation (no quoting required).
DatabaseName = $"{DatabaseNamePrefix}{Guid.NewGuid():N}".Substring(0, 38);
// Env var lets CI / power users override the admin endpoint; absent
// defaults to the local docker dev container's sa connection.
var fromEnv = Environment.GetEnvironmentVariable(AdminEnvVar);
_adminConnectionString = string.IsNullOrWhiteSpace(fromEnv)
? DefaultAdminConnectionString
: fromEnv;
// Step 1: probe connectivity. This is the only step that may legitimately
// fail when MSSQL is absent; SqlException + the rare InvalidOperationException
// from Open are the connect-failure surfaces we tolerate. Everything else
// (CREATE DATABASE, MigrateAsync) is treated as a hard fixture failure once we
// *have* a connection.
var probeFailure = ProbeConnectivity(_adminConnectionString);
if (probeFailure is not null)
{
ConnectionString = string.Empty;
Available = false;
SkipReason = probeFailure;
return;
}
// Step 2: best-effort, once per process — drop databases stranded by a run
// that crashed before Dispose. The guid suffix means they never collide, but
// they do accumulate and consume server resources, which feeds the very
// saturation this fixture now throttles against.
SweepOrphanDatabasesOnce(_adminConnectionString);
try
{
// Step 3: take a create+migrate slot. Held across BOTH the CREATE DATABASE
// and the migration chain, then released — this is the machine-wide bound
// on how much fresh-database DDL the dev server is asked to do at once.
using (MigrationGate.Acquire(_adminConnectionString))
{
using (var connection = new SqlConnection(_adminConnectionString))
{
connection.Open();
using var createCmd = connection.CreateCommand();
createCmd.CommandTimeout = DatabaseDdlCommandTimeoutSeconds;
createCmd.CommandText = $"CREATE DATABASE [{DatabaseName}];";
createCmd.ExecuteNonQuery();
}
ConnectionString = BuildPerDbConnectionString(_adminConnectionString, DatabaseName);
// Apply the EF migrations once at fixture construction so each test
// can read from a fully-migrated database without per-test setup.
// Failures here are real bugs — let them bubble.
ApplyMigrationsCore(ConnectionString, CancellationToken.None).GetAwaiter().GetResult();
}
Available = true;
SkipReason = string.Empty;
}
catch
{
// Best-effort cleanup if we created the database but failed before
// setting Available — otherwise Dispose() would skip the drop.
TryDropOrphanDatabase();
throw;
}
}
///
/// Opens the admin connection up to times.
/// Returns null when a connection was established, or the skip reason when every
/// attempt failed.
///
private static string? ProbeConnectivity(string adminConnectionString)
{
string lastFailure = "MSSQL unavailable (no connect attempt was made)";
for (var attempt = 1; attempt <= ConnectProbeAttempts; attempt++)
{
try
{
using var connection = new SqlConnection(adminConnectionString);
connection.Open();
return null;
}
catch (SqlException ex)
{
lastFailure =
$"MSSQL unavailable (connect failed after {attempt} attempt(s): SqlException {ex.Number}: {ex.Message})";
}
catch (InvalidOperationException ex)
{
lastFailure =
$"MSSQL unavailable (Open threw after {attempt} attempt(s): {ex.Message})";
}
if (attempt < ConnectProbeAttempts)
{
Thread.Sleep(TimeSpan.FromMilliseconds(250 * attempt));
}
}
return lastFailure;
}
///
/// Drops fixture databases left behind by a run that died before
/// . Runs at most once per test process and swallows every
/// failure — an uncleaned stray is untidy, not fatal, and must never turn into a
/// fixture error.
///
private static void SweepOrphanDatabasesOnce(string adminConnectionString)
{
if (Interlocked.Exchange(ref _orphanSweepStarted, 1) != 0)
{
return;
}
try
{
using var connection = new SqlConnection(adminConnectionString);
connection.Open();
var stale = new List();
using (var listCmd = connection.CreateCommand())
{
listCmd.CommandTimeout = DatabaseDdlCommandTimeoutSeconds;
listCmd.CommandText =
"SELECT name FROM sys.databases " +
"WHERE name LIKE @Prefix + '%' AND create_date < DATEADD(minute, -@AgeMinutes, GETDATE());";
listCmd.Parameters.AddWithValue("@Prefix", DatabaseNamePrefix);
listCmd.Parameters.AddWithValue("@AgeMinutes", (int)OrphanDatabaseMaxAge.TotalMinutes);
using var reader = listCmd.ExecuteReader();
while (reader.Read())
{
stale.Add(reader.GetString(0));
}
}
foreach (var name in stale)
{
try
{
using var dropCmd = connection.CreateCommand();
dropCmd.CommandTimeout = DatabaseDdlCommandTimeoutSeconds;
// Name comes from sys.databases filtered to our own guid-suffixed
// prefix, so it cannot carry an injection payload.
dropCmd.CommandText =
$"ALTER DATABASE [{name}] SET SINGLE_USER WITH ROLLBACK IMMEDIATE; " +
$"DROP DATABASE [{name}];";
dropCmd.ExecuteNonQuery();
}
catch
{
// Another process may be mid-drop, or the database may be in use.
// Best-effort by design.
}
}
}
catch
{
// Best-effort by design.
}
}
private void TryDropOrphanDatabase()
{
if (string.IsNullOrEmpty(ConnectionString))
{
return;
}
DropSelf();
}
///
/// Applies the EF migrations to the per-fixture test database via a freshly
/// constructed pointed at it. Uses the
/// schema-only single-argument constructor — the AuditLog migration does
/// not write secret-bearing columns at apply time. Called once from the
/// constructor; tests do not invoke this directly.
///
private static async Task ApplyMigrationsCore(string connectionString, CancellationToken cancellationToken)
{
var options = new DbContextOptionsBuilder()
// The explicit command timeout is the register-row-82 fix: EF's 30s default
// is what actually failed under cross-process load, and every migration
// statement in the chain inherits this value.
.UseSqlServer(connectionString, sql => sql.CommandTimeout(MigrationCommandTimeoutSeconds))
.Options;
await using var context = new ScadaBridgeDbContext(options);
await context.Database.MigrateAsync(cancellationToken);
}
///
/// Convenience for opening a fresh to the test
/// database. Caller is responsible for disposal.
///
public SqlConnection OpenConnection()
{
ThrowIfUnavailable();
var connection = new SqlConnection(ConnectionString);
connection.Open();
return connection;
}
public void Dispose()
{
if (!Available)
{
return;
}
DropSelf();
}
///
/// Best-effort drop of this fixture's database — never let a teardown failure
/// pollute later runs. SINGLE_USER WITH ROLLBACK IMMEDIATE detaches lingering
/// pooled connections so the DROP DATABASE doesn't fail with "database is in use".
/// Deliberately NOT gated by : holding a create+migrate
/// slot at teardown would starve the fixtures still trying to start.
///
private void DropSelf()
{
try
{
// Connection-pool cleanup is necessary because EF's MigrateAsync leaves
// pooled connections behind; SqlConnection.ClearAllPools() forces them
// closed so the SINGLE_USER + DROP sequence below can complete.
SqlConnection.ClearAllPools();
using var connection = new SqlConnection(_adminConnectionString);
connection.Open();
using var cmd = connection.CreateCommand();
cmd.CommandTimeout = DatabaseDdlCommandTimeoutSeconds;
cmd.CommandText =
$"IF DB_ID(N'{DatabaseName}') IS NOT NULL " +
$"BEGIN " +
$" ALTER DATABASE [{DatabaseName}] SET SINGLE_USER WITH ROLLBACK IMMEDIATE; " +
$" DROP DATABASE [{DatabaseName}]; " +
$"END";
cmd.ExecuteNonQuery();
}
catch
{
// Swallow — the database name carries a random guid suffix so a
// stranded test database does not collide with future runs, and the
// next process's orphan sweep will clear it.
}
}
///
/// Throws an when invoked on an
/// unavailable fixture; tests should branch on
/// before reaching this code path.
///
private void ThrowIfUnavailable()
{
if (!Available)
{
throw new InvalidOperationException(
$"MsSqlMigrationFixture is not Available: {SkipReason}");
}
}
private static string BuildPerDbConnectionString(string adminConnectionString, string databaseName)
{
var builder = new SqlConnectionStringBuilder(adminConnectionString)
{
InitialCatalog = databaseName,
};
return builder.ConnectionString;
}
///
/// A machine-wide counting semaphore over the create+migrate window, built from
/// SQL Server session-scoped application locks on master
/// (deferred-work register row 82).
///
///
///
/// Why not an xunit collection? xunit's parallelism controls only bound
/// the collections inside ONE test assembly running in ONE process. The
/// contention here is between processes — AuditLog.Tests,
/// ConfigurationDatabase.Tests and anything else a solution-level
/// dotnet test starts in parallel all hammer the same container. The
/// arbiter therefore has to live on the shared resource itself, which is the
/// SQL Server instance.
///
///
/// sp_getapplock is a binary mutex, so a counting semaphore is emulated
/// with distinctly-named lock resources: a caller walks
/// the slots with a short per-slot timeout and takes the first free one, looping
/// until it wins or the overall deadline passes. Application locks are
/// database-scoped, so every gate connection targets master to make
/// sure all processes contend over the same namespace regardless of which test
/// database they are about to build.
///
///
/// Pooling is disabled on the gate connection. A session-scoped application lock
/// is released by sp_reset_connection, which a pooled connection only
/// issues when it is next reused — so a pooled gate connection could keep
/// a slot locked long after Dispose. Closing an unpooled connection ends the
/// session outright, and the explicit sp_releaseapplock makes the normal
/// path deterministic rather than relying on that. A crashed process drops its
/// connection, and SQL Server releases the lock with it, so the gate cannot
/// wedge the suite.
///
///
/// The gate never fails a test: if the deadline passes without a slot, it returns
/// an ungated instance and lets the (now generous) command timeouts carry the
/// run. A throttle that could itself become a source of red is not worth having.
///
///
private sealed class MigrationGate : IDisposable
{
///
/// How many fixtures may build a database at once, machine-wide. Sized to keep
/// the dev container busy without saturating it; override with
/// SCADABRIDGE_MSSQL_TEST_MIGRATE_SLOTS.
///
private const int DefaultSlotCount = 4;
private const string SlotEnvVar = "SCADABRIDGE_MSSQL_TEST_MIGRATE_SLOTS";
private const string ResourcePrefix = "ScadaBridgeTestMigrate/";
/// Per-slot wait before moving on to try the next slot.
private const int PerSlotLockTimeoutMs = 500;
///
/// Overall wait for a slot. Long enough to outlast a full queue of migration
/// chains on a loaded server; past it the gate gives up and lets the caller
/// proceed ungated rather than failing.
///
private static readonly TimeSpan AcquireDeadline = TimeSpan.FromMinutes(10);
private static int SlotCount
{
get
{
var raw = Environment.GetEnvironmentVariable(SlotEnvVar);
return int.TryParse(raw, out var parsed) && parsed > 0 ? parsed : DefaultSlotCount;
}
}
private readonly SqlConnection? _connection;
private readonly string? _resource;
private MigrationGate(SqlConnection? connection, string? resource)
{
_connection = connection;
_resource = resource;
}
public static MigrationGate Acquire(string adminConnectionString)
{
var slots = SlotCount;
SqlConnection? connection = null;
try
{
connection = new SqlConnection(BuildGateConnectionString(adminConnectionString));
connection.Open();
var deadline = DateTime.UtcNow + AcquireDeadline;
// Start each caller at a different slot so they spread out instead of
// all queueing behind slot 0.
var offset = Random.Shared.Next(slots);
while (DateTime.UtcNow < deadline)
{
for (var i = 0; i < slots; i++)
{
var resource = ResourcePrefix + ((offset + i) % slots);
if (TryAcquireSlot(connection, resource))
{
return new MigrationGate(connection, resource);
}
}
}
}
catch
{
// The gate is an optimisation, not a correctness requirement — if it
// cannot be established (permissions, an older server, a transient
// fault) the caller proceeds ungated.
}
connection?.Dispose();
return new MigrationGate(null, null);
}
private static bool TryAcquireSlot(SqlConnection connection, string resource)
{
using var cmd = connection.CreateCommand();
cmd.CommandType = System.Data.CommandType.StoredProcedure;
cmd.CommandText = "sp_getapplock";
cmd.CommandTimeout = (PerSlotLockTimeoutMs / 1000) + 30;
cmd.Parameters.AddWithValue("@Resource", resource);
cmd.Parameters.AddWithValue("@LockMode", "Exclusive");
cmd.Parameters.AddWithValue("@LockOwner", "Session");
cmd.Parameters.AddWithValue("@LockTimeout", PerSlotLockTimeoutMs);
var result = cmd.Parameters.Add("@Result", System.Data.SqlDbType.Int);
result.Direction = System.Data.ParameterDirection.ReturnValue;
cmd.ExecuteNonQuery();
// 0 = granted immediately, 1 = granted after waiting; everything else
// (-1 timeout, -2 cancelled, -3 deadlock victim, -999 error) means "not ours".
var code = result.Value as int? ?? -999;
return code >= 0;
}
private static string BuildGateConnectionString(string adminConnectionString)
{
return new SqlConnectionStringBuilder(adminConnectionString)
{
// Application locks are database-scoped: every process must contend in
// the same database for the slots to mean anything.
InitialCatalog = "master",
// See the remarks — a pooled connection defers the implicit release of a
// session-scoped app lock until the connection is reused.
Pooling = false,
}.ConnectionString;
}
public void Dispose()
{
if (_connection is null)
{
return;
}
try
{
if (_resource is not null)
{
using var cmd = _connection.CreateCommand();
cmd.CommandType = System.Data.CommandType.StoredProcedure;
cmd.CommandText = "sp_releaseapplock";
cmd.Parameters.AddWithValue("@Resource", _resource);
cmd.Parameters.AddWithValue("@LockOwner", "Session");
cmd.ExecuteNonQuery();
}
}
catch
{
// Closing the unpooled connection below ends the session, which releases
// the lock regardless.
}
finally
{
_connection.Dispose();
}
}
}
}