diff --git a/docs/test_infra/test_infra_db.md b/docs/test_infra/test_infra_db.md index 2d58da6c..1cda1c23 100644 --- a/docs/test_infra/test_infra_db.md +++ b/docs/test_infra/test_infra_db.md @@ -26,6 +26,40 @@ The test database uses Microsoft SQL Server 2022 Developer Edition running in Do Both databases are created by `infra/mssql/setup.sql`. EF Core migrations populate the `ScadaBridgeConfig` schema. The `ScadaBridgeMachineData` database is seeded with sample tables and stored procedures by `infra/mssql/machinedata_seed.sql`. +### Ephemeral per-test-class databases + +`MsSqlMigrationFixture` (`tests/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests/Migrations/`) creates a +throwaway `ScadaBridgeAuditMigTest_` database per test class, applies the full EF migration chain +to it, and drops it on teardown. Both `ConfigurationDatabase.Tests` and `AuditLog.Tests` use it — the +latter by project reference, so there is one fixture implementation, not a copy. Seeing a handful of +these databases come and go while tests run is normal; seeing them survive a run is not (see below). + +Three behaviours are worth knowing when you look at this server during a test run: + +- **Concurrent create+migrate is throttled machine-wide.** Fixtures take a slot from a counting + semaphore built on `sp_getapplock` in `master` (resources `ScadaBridgeTestMigrate/0..N-1`) and hold it + across `CREATE DATABASE` + the migration chain. The default is 4 slots; override with + `SCADABRIDGE_MSSQL_TEST_MIGRATE_SLOTS`. An xunit collection attribute cannot do this job — a + solution-level `dotnet test` runs several MSSQL-backed test *processes* at once, so the throttle has + to live on the shared server. Without it, dozens of simultaneous fresh-database migrations saturated + the container and individual DDL statements blew past EF's default 30-second command timeout, + failing whole test classes at fixture construction (deferred-work register row 82). +- **Migration commands use a 300s timeout**, and CREATE/DROP DATABASE use 120s. Under a bounded queue + the server is busy but progressing, so the correct response to a slow statement is to wait. +- **Strays are swept.** Each test process drops, once at start and best-effort, any + `ScadaBridgeAuditMigTest_*` database older than 2 hours, so a run killed before teardown does not + leave databases accumulating. To clear them by hand: + +```bash +docker exec -i scadabridge-mssql /opt/mssql-tools18/bin/sqlcmd \ + -S localhost -U sa -P 'ScadaBridge_Dev1#' -C \ + -Q "SELECT name, create_date FROM sys.databases WHERE name LIKE 'ScadaBridgeAuditMigTest_%'" +``` + +Overriding the server the fixture targets (CI, a non-default port) is done with +`SCADABRIDGE_MSSQL_TEST_CONN`. When no server answers, the fixture reports `Available = false` and the +tests report as **Skipped**, not passed. + ## Data Persistence SQL data is stored in the named Docker volume `scadabridge-mssql-data`. Data survives container restarts and `docker compose down`. To reset the database completely: diff --git a/tests/ZB.MOM.WW.ScadaBridge.AuditLog.Tests/Integration/NotifyDispatcherAuditTrailTests.cs b/tests/ZB.MOM.WW.ScadaBridge.AuditLog.Tests/Integration/NotifyDispatcherAuditTrailTests.cs index 7e498def..365be0ce 100644 --- a/tests/ZB.MOM.WW.ScadaBridge.AuditLog.Tests/Integration/NotifyDispatcherAuditTrailTests.cs +++ b/tests/ZB.MOM.WW.ScadaBridge.AuditLog.Tests/Integration/NotifyDispatcherAuditTrailTests.cs @@ -200,6 +200,39 @@ public class NotifyDispatcherAuditTrailTests : TestKit, IClassFixture + /// Stops a dispatcher actor and blocks until it is actually dead. + /// + /// + /// + /// Deferred-work register row 82. The dispatcher claims every due row in the + /// Notifications table — it is not scoped to the per-test siteId — and the + /// tests in this class share one database. The + /// actors were only ever stopped implicitly, by 's + /// Dispose → Shutdown, which is bounded by a timeout and does not fail + /// when that timeout expires. On a loaded machine an earlier test's dispatcher was + /// therefore still alive when the next test seeded its notification, claimed that + /// row, and delivered it through its OWN exhausted + /// — which returns a transient outcome, leaving + /// the row in Retrying (observed: Expected: Delivered, Actual: Retrying) + /// and the audit-row counts short (Range: (3 - 4), Actual: 2). + /// + /// + /// Stopping the actor inside the test that created it removes the dependency on + /// ActorSystem-shutdown timing, so no dispatcher outlives the rows it is allowed to + /// touch. + /// + /// + private void StopDispatcher(IActorRef actor) + { + Watch(actor); + Sys.Stop(actor); + // A stop is processed after the in-flight tick completes, so this also + // guarantees no delivery is still running against the fixture DB. + ExpectTerminated(actor, TimeSpan.FromSeconds(30)); + Unwatch(actor); + } + private static NotificationOutboxOptions LongDispatchOptions() => // 1h dispatch + 24h purge so PreStart's timers never fire during the // test; the test drives the dispatcher with explicit DispatchTick. @@ -280,6 +313,10 @@ public class NotifyDispatcherAuditTrailTests : TestKit, IClassFixture diff --git a/tests/ZB.MOM.WW.ScadaBridge.AuditLog.Tests/Integration/PartitionPurgeTests.cs b/tests/ZB.MOM.WW.ScadaBridge.AuditLog.Tests/Integration/PartitionPurgeTests.cs index 59c702e4..6b228b2a 100644 --- a/tests/ZB.MOM.WW.ScadaBridge.AuditLog.Tests/Integration/PartitionPurgeTests.cs +++ b/tests/ZB.MOM.WW.ScadaBridge.AuditLog.Tests/Integration/PartitionPurgeTests.cs @@ -156,6 +156,41 @@ SELECT NullLogger.Instance))); } + /// + /// Stops a purge actor and blocks until it is actually dead, then disposes the + /// service provider that backed it. + /// + /// + /// + /// Deferred-work register row 82. Every actor these tests create ticks every + /// 100 ms () and switches + /// out the Jan-2026 partition each time it finds one eligible. The tests relied on + /// 's Dispose → Shutdown to stop them between + /// test methods, but that shutdown is bounded by a timeout and does NOT fail when + /// it expires — so on a loaded machine a previous test's actor was still ticking + /// against the (per-class, shared) fixture database while the next test ran. + /// + /// + /// That is what made PerChannelOverride_DeletesOnlyOverriddenChannelsOldRows + /// flake under a parallel full-solution run: it seeds a Jan-15-2026 row and expects + /// its own per-channel DELETE to remove exactly one row, but a leaked actor could + /// switch the whole Jan-2026 partition out first, leaving nothing to delete + /// (observed: Assert.Equal() Failure … Expected: 1, Actual: 0). Stopping the + /// actor inside the test that created it removes the dependency on shutdown timing + /// altogether. + /// + /// + private void StopPurgeActor(IActorRef actor, ServiceProvider sp) + { + Watch(actor); + Sys.Stop(actor); + // A stop is processed after the in-flight tick completes, so this also + // guarantees no partition switch is still running against the fixture DB. + ExpectTerminated(actor, TimeSpan.FromSeconds(30)); + Unwatch(actor); + sp.Dispose(); + } + /// /// Returns three seed timestamps and a computed RetentionDays value that /// keep the purge-intent date-independent regardless of when the test runs. @@ -246,7 +281,7 @@ SELECT }; var auditOptions = new AuditLogOptions { RetentionDays = retentionDays }; - CreateActor(sp, purgeOptions, auditOptions); + var actor = CreateActor(sp, purgeOptions, auditOptions); // The Jan-2026 partition boundary is the only eligible one in this fixture DB. var janBoundary = new DateTime(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc); @@ -256,6 +291,10 @@ SELECT Assert.True(matched.RowsDeleted >= 1, $"Expected RowsDeleted >= 1 for Jan-2026 boundary; got {matched.RowsDeleted}."); + // Stop the actor before verifying: its 100ms tick must not outlive this test + // and start purging the next one's seed rows (register row 82). + StopPurgeActor(actor, sp); + // Allow a brief settle in case the actor re-enumerates quickly. await Task.Delay(TimeSpan.FromMilliseconds(500)); @@ -303,7 +342,7 @@ SELECT var probe = CreateTestProbe(); Sys.EventStream.Subscribe(probe.Ref, typeof(AuditLogPurgedEvent)); - CreateActor( + var actor = CreateActor( sp, new AuditLogPurgeOptions { @@ -317,6 +356,10 @@ SELECT isMessage: m => m.MonthBoundary == janBoundary, max: TimeSpan.FromSeconds(30)); + // Stop the actor before verifying: its 100ms tick must not outlive this test + // and start purging the next one's seed rows (register row 82). + StopPurgeActor(actor, sp); + // Open a fresh connection (the actor's pool is owned by EF) and // assert the index is present post-purge. await using var check = _fixture.OpenConnection(); @@ -357,7 +400,7 @@ SELECT var probe = CreateTestProbe(); Sys.EventStream.Subscribe(probe.Ref, typeof(AuditLogPurgedEvent)); - CreateActor( + var actor = CreateActor( sp, new AuditLogPurgeOptions { @@ -371,6 +414,10 @@ SELECT isMessage: m => m.MonthBoundary == janBoundary, max: TimeSpan.FromSeconds(30)); + // Stop the actor before verifying: its 100ms tick must not outlive this test + // and start purging the next one's seed rows (register row 82). + StopPurgeActor(actor, sp); + // Settle then exercise InsertIfNotExistsAsync twice for the same // EventId. The repository's idempotency relies on the aligned clustered // PK (EventId, OccurredAtUtc) being intact so the IF NOT EXISTS … INSERT diff --git a/tests/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests/Migrations/MsSqlMigrationFixture.cs b/tests/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests/Migrations/MsSqlMigrationFixture.cs index abc2e368..0c9fd5e8 100644 --- a/tests/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests/Migrations/MsSqlMigrationFixture.cs +++ b/tests/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests/Migrations/MsSqlMigrationFixture.cs @@ -14,27 +14,90 @@ namespace ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests.Migrations; /// so the runner reports them as Skipped (not silently Passed). /// /// +/// /// xunit 2.9.x has no native Assert.Skip/Assert.SkipUnless (those /// are v3); the project uses the Xunit.SkippableFact package as the canonical /// equivalent. The fixture attempts connect + create-db + migrate once at -/// construct time. The Connect Timeout=3 in -/// makes the fixture fail fast in a no-container environment (under ~5s total) -/// instead of hanging 30s on SqlClient's default. Only connect-failure exceptions -/// (SqlException, plus the InvalidOperationException SqlClient raises from -/// OpenAsync) flip Available to false — every other exception bubbles up so a -/// real bug is not silently swallowed. +/// construct time. Only connect-failure exceptions (SqlException, plus the +/// InvalidOperationException SqlClient raises from OpenAsync) flip Available to +/// false — every other exception bubbles up so a real bug is not silently +/// swallowed. +/// +/// +/// Cross-process throttling (deferred-work register row 82). Database +/// naming was already isolated (a guid per fixture instance), but nothing +/// bounded how many fixtures ran CREATE DATABASE + the full migration +/// chain against the single dev SQL Server at the same time. xunit runs +/// processor-count test collections in parallel per process, and a +/// full-solution run starts several MSSQL-backed test processes at once — so the +/// server saw dozens of concurrent fresh-database migrations. That is pure +/// resource saturation (CPU / log IO / tempdb on one container), not object-level +/// locking, and it pushed individual migration DDL statements past EF's default +/// 30-second command timeout. Every observed failure had the identical +/// fingerprint: MsSqlMigrationFixture..ctor → Migrator.MigrateAsync +/// → SqlException: Execution Timeout Expired (error 258), which xunit +/// reports as "Class fixture type … threw in its constructor" against every test +/// in the class — a false red that vanished on a re-run in isolation. +/// +/// +/// Two changes close it. First, bounds the number of +/// simultaneous create+migrate windows across every process on the machine +/// using SQL Server application locks (sp_getapplock) on master as a +/// counting semaphore — an xunit collection attribute cannot do this, because the +/// contention is between separate test processes. Second, the create/migrate +/// commands get timeouts sized for a queued-but-progressing server rather than an +/// idle one, so a slot holder waits instead of failing. The work always completed; +/// it just did not complete within 30 seconds. +/// /// public sealed class MsSqlMigrationFixture : IDisposable { // Same credentials infra/mssql/setup.sql + docker-compose use. Not a committed // production secret — this is a local dev container connection string. - // Connect Timeout=3 makes the fixture fail fast (~3s) in a no-container - // environment rather than hanging on SqlClient's default 30s connect timeout. + // Connect Timeout=3 makes the connectivity probe fail fast (~3s per attempt) in a + // no-container environment rather than hanging on SqlClient's default 30s. private const string DefaultAdminConnectionString = "Server=localhost,1433;User Id=sa;Password=ScadaBridge_Dev1#;TrustServerCertificate=true;Encrypt=false;Connect Timeout=3"; private const string AdminEnvVar = "SCADABRIDGE_MSSQL_TEST_CONN"; + /// + /// Command timeout for the migration chain. Deliberately far above EF's 30s + /// default: under a bounded queue the server is busy but progressing, and the + /// correct response to a slow DDL statement is to wait for it, not to fail the + /// whole class. See the register-row-82 note in the type remarks. + /// + private const int MigrationCommandTimeoutSeconds = 300; + + /// + /// Command timeout for the CREATE / DROP DATABASE statements. Lower than the + /// migration chain (one statement, not fifty-odd) but still well clear of the + /// 30s default, because CREATE DATABASE zeroes a fresh log file and that is the + /// single most IO-bound step in the fixture. + /// + private const int DatabaseDdlCommandTimeoutSeconds = 120; + + /// + /// Number of connectivity-probe attempts before declaring MSSQL unavailable. + /// A single 3s attempt is not proof of absence once the server is under load — + /// treating a busy server as an absent one would silently turn every MSSQL test + /// into a Skip, which reads as green and is worse than a red. + /// + private const int ConnectProbeAttempts = 3; + + /// + /// Databases whose name matches the fixture prefix and that are older than this + /// are assumed to be strays from a crashed run and are dropped at process start. + /// Comfortably longer than any legitimate fixture lifetime, so a live fixture in + /// a concurrently-running process is never targeted. + /// + private static readonly TimeSpan OrphanDatabaseMaxAge = TimeSpan.FromHours(2); + + private const string DatabaseNamePrefix = "ScadaBridgeAuditMigTest_"; + + /// Guards the once-per-process orphan sweep. + private static int _orphanSweepStarted; + public string DatabaseName { get; } public string ConnectionString { get; } @@ -58,7 +121,7 @@ public sealed class MsSqlMigrationFixture : IDisposable { // Short, lowercase guid suffix keeps the database identifier under SQL Server's // 128-char limit and safe for raw concatenation (no quoting required). - DatabaseName = $"ScadaBridgeAuditMigTest_{Guid.NewGuid():N}".Substring(0, 38); + DatabaseName = $"{DatabaseNamePrefix}{Guid.NewGuid():N}".Substring(0, 38); // Env var lets CI / power users override the admin endpoint; absent // defaults to the local docker dev container's sa connection. @@ -67,46 +130,50 @@ public sealed class MsSqlMigrationFixture : IDisposable ? DefaultAdminConnectionString : fromEnv; - // Step 1: open the admin connection. This is the only step that may - // legitimately fail when MSSQL is absent; SqlException + the rare - // InvalidOperationException from OpenAsync are the connect-failure - // surfaces we tolerate. Everything else (CREATE DATABASE, MigrateAsync) - // is treated as a hard fixture failure once we *have* a connection. + // Step 1: probe connectivity. This is the only step that may legitimately + // fail when MSSQL is absent; SqlException + the rare InvalidOperationException + // from Open are the connect-failure surfaces we tolerate. Everything else + // (CREATE DATABASE, MigrateAsync) is treated as a hard fixture failure once we + // *have* a connection. + var probeFailure = ProbeConnectivity(_adminConnectionString); + if (probeFailure is not null) + { + ConnectionString = string.Empty; + Available = false; + SkipReason = probeFailure; + return; + } + + // Step 2: best-effort, once per process — drop databases stranded by a run + // that crashed before Dispose. The guid suffix means they never collide, but + // they do accumulate and consume server resources, which feeds the very + // saturation this fixture now throttles against. + SweepOrphanDatabasesOnce(_adminConnectionString); + try { - using var connection = new SqlConnection(_adminConnectionString); - try + // Step 3: take a create+migrate slot. Held across BOTH the CREATE DATABASE + // and the migration chain, then released — this is the machine-wide bound + // on how much fresh-database DDL the dev server is asked to do at once. + using (MigrationGate.Acquire(_adminConnectionString)) { - connection.Open(); - } - catch (SqlException ex) - { - ConnectionString = string.Empty; - Available = false; - SkipReason = $"MSSQL unavailable (connect failed: SqlException {ex.Number}: {ex.Message})"; - return; - } - catch (InvalidOperationException ex) - { - ConnectionString = string.Empty; - Available = false; - SkipReason = $"MSSQL unavailable (OpenAsync threw: {ex.Message})"; - return; - } + using (var connection = new SqlConnection(_adminConnectionString)) + { + connection.Open(); + using var createCmd = connection.CreateCommand(); + createCmd.CommandTimeout = DatabaseDdlCommandTimeoutSeconds; + createCmd.CommandText = $"CREATE DATABASE [{DatabaseName}];"; + createCmd.ExecuteNonQuery(); + } - using (var createCmd = connection.CreateCommand()) - { - createCmd.CommandText = $"CREATE DATABASE [{DatabaseName}];"; - createCmd.ExecuteNonQuery(); + ConnectionString = BuildPerDbConnectionString(_adminConnectionString, DatabaseName); + + // Apply the EF migrations once at fixture construction so each test + // can read from a fully-migrated database without per-test setup. + // Failures here are real bugs — let them bubble. + ApplyMigrationsCore(ConnectionString, CancellationToken.None).GetAwaiter().GetResult(); } - ConnectionString = BuildPerDbConnectionString(_adminConnectionString, DatabaseName); - - // Apply the EF migrations once at fixture construction so each test - // can read from a fully-migrated database without per-test setup. - // Failures here are real bugs — let them bubble. - ApplyMigrationsCore(ConnectionString, CancellationToken.None).GetAwaiter().GetResult(); - Available = true; SkipReason = string.Empty; } @@ -119,6 +186,104 @@ public sealed class MsSqlMigrationFixture : IDisposable } } + /// + /// Opens the admin connection up to times. + /// Returns null when a connection was established, or the skip reason when every + /// attempt failed. + /// + private static string? ProbeConnectivity(string adminConnectionString) + { + string lastFailure = "MSSQL unavailable (no connect attempt was made)"; + + for (var attempt = 1; attempt <= ConnectProbeAttempts; attempt++) + { + try + { + using var connection = new SqlConnection(adminConnectionString); + connection.Open(); + return null; + } + catch (SqlException ex) + { + lastFailure = + $"MSSQL unavailable (connect failed after {attempt} attempt(s): SqlException {ex.Number}: {ex.Message})"; + } + catch (InvalidOperationException ex) + { + lastFailure = + $"MSSQL unavailable (Open threw after {attempt} attempt(s): {ex.Message})"; + } + + if (attempt < ConnectProbeAttempts) + { + Thread.Sleep(TimeSpan.FromMilliseconds(250 * attempt)); + } + } + + return lastFailure; + } + + /// + /// Drops fixture databases left behind by a run that died before + /// . Runs at most once per test process and swallows every + /// failure — an uncleaned stray is untidy, not fatal, and must never turn into a + /// fixture error. + /// + private static void SweepOrphanDatabasesOnce(string adminConnectionString) + { + if (Interlocked.Exchange(ref _orphanSweepStarted, 1) != 0) + { + return; + } + + try + { + using var connection = new SqlConnection(adminConnectionString); + connection.Open(); + + var stale = new List(); + using (var listCmd = connection.CreateCommand()) + { + listCmd.CommandTimeout = DatabaseDdlCommandTimeoutSeconds; + listCmd.CommandText = + "SELECT name FROM sys.databases " + + "WHERE name LIKE @Prefix + '%' AND create_date < DATEADD(minute, -@AgeMinutes, GETDATE());"; + listCmd.Parameters.AddWithValue("@Prefix", DatabaseNamePrefix); + listCmd.Parameters.AddWithValue("@AgeMinutes", (int)OrphanDatabaseMaxAge.TotalMinutes); + + using var reader = listCmd.ExecuteReader(); + while (reader.Read()) + { + stale.Add(reader.GetString(0)); + } + } + + foreach (var name in stale) + { + try + { + using var dropCmd = connection.CreateCommand(); + dropCmd.CommandTimeout = DatabaseDdlCommandTimeoutSeconds; + // Name comes from sys.databases filtered to our own guid-suffixed + // prefix, so it cannot carry an injection payload. + dropCmd.CommandText = + $"ALTER DATABASE [{name}] SET SINGLE_USER WITH ROLLBACK IMMEDIATE; " + + $"DROP DATABASE [{name}];"; + dropCmd.ExecuteNonQuery(); + } + catch + { + // Another process may be mid-drop, or the database may be in use. + // Best-effort by design. + } + } + } + catch + { + // Best-effort by design. + } + } + private void TryDropOrphanDatabase() { if (string.IsNullOrEmpty(ConnectionString)) @@ -126,24 +291,7 @@ public sealed class MsSqlMigrationFixture : IDisposable return; } - try - { - SqlConnection.ClearAllPools(); - using var connection = new SqlConnection(_adminConnectionString); - connection.Open(); - using var cmd = connection.CreateCommand(); - cmd.CommandText = - $"IF DB_ID(N'{DatabaseName}') IS NOT NULL " + - $"BEGIN " + - $" ALTER DATABASE [{DatabaseName}] SET SINGLE_USER WITH ROLLBACK IMMEDIATE; " + - $" DROP DATABASE [{DatabaseName}]; " + - $"END"; - cmd.ExecuteNonQuery(); - } - catch - { - // Best-effort — orphan databases carry a random guid suffix. - } + DropSelf(); } /// @@ -156,7 +304,10 @@ public sealed class MsSqlMigrationFixture : IDisposable private static async Task ApplyMigrationsCore(string connectionString, CancellationToken cancellationToken) { var options = new DbContextOptionsBuilder() - .UseSqlServer(connectionString) + // The explicit command timeout is the register-row-82 fix: EF's 30s default + // is what actually failed under cross-process load, and every migration + // statement in the chain inherits this value. + .UseSqlServer(connectionString, sql => sql.CommandTimeout(MigrationCommandTimeoutSeconds)) .Options; await using var context = new ScadaBridgeDbContext(options); @@ -183,9 +334,18 @@ public sealed class MsSqlMigrationFixture : IDisposable return; } - // Best-effort drop — never let a teardown failure pollute later runs. - // SINGLE_USER WITH ROLLBACK IMMEDIATE detaches lingering pooled connections - // so the DROP DATABASE doesn't fail with "database is in use". + DropSelf(); + } + + /// + /// Best-effort drop of this fixture's database — never let a teardown failure + /// pollute later runs. SINGLE_USER WITH ROLLBACK IMMEDIATE detaches lingering + /// pooled connections so the DROP DATABASE doesn't fail with "database is in use". + /// Deliberately NOT gated by : holding a create+migrate + /// slot at teardown would starve the fixtures still trying to start. + /// + private void DropSelf() + { try { // Connection-pool cleanup is necessary because EF's MigrateAsync leaves @@ -197,6 +357,7 @@ public sealed class MsSqlMigrationFixture : IDisposable connection.Open(); using var cmd = connection.CreateCommand(); + cmd.CommandTimeout = DatabaseDdlCommandTimeoutSeconds; cmd.CommandText = $"IF DB_ID(N'{DatabaseName}') IS NOT NULL " + $"BEGIN " + @@ -208,7 +369,8 @@ public sealed class MsSqlMigrationFixture : IDisposable catch { // Swallow — the database name carries a random guid suffix so a - // stranded test database does not collide with future runs. + // stranded test database does not collide with future runs, and the + // next process's orphan sweep will clear it. } } @@ -234,4 +396,189 @@ public sealed class MsSqlMigrationFixture : IDisposable }; return builder.ConnectionString; } + + /// + /// A machine-wide counting semaphore over the create+migrate window, built from + /// SQL Server session-scoped application locks on master + /// (deferred-work register row 82). + /// + /// + /// + /// Why not an xunit collection? xunit's parallelism controls only bound + /// the collections inside ONE test assembly running in ONE process. The + /// contention here is between processes — AuditLog.Tests, + /// ConfigurationDatabase.Tests and anything else a solution-level + /// dotnet test starts in parallel all hammer the same container. The + /// arbiter therefore has to live on the shared resource itself, which is the + /// SQL Server instance. + /// + /// + /// sp_getapplock is a binary mutex, so a counting semaphore is emulated + /// with distinctly-named lock resources: a caller walks + /// the slots with a short per-slot timeout and takes the first free one, looping + /// until it wins or the overall deadline passes. Application locks are + /// database-scoped, so every gate connection targets master to make + /// sure all processes contend over the same namespace regardless of which test + /// database they are about to build. + /// + /// + /// Pooling is disabled on the gate connection. A session-scoped application lock + /// is released by sp_reset_connection, which a pooled connection only + /// issues when it is next reused — so a pooled gate connection could keep + /// a slot locked long after Dispose. Closing an unpooled connection ends the + /// session outright, and the explicit sp_releaseapplock makes the normal + /// path deterministic rather than relying on that. A crashed process drops its + /// connection, and SQL Server releases the lock with it, so the gate cannot + /// wedge the suite. + /// + /// + /// The gate never fails a test: if the deadline passes without a slot, it returns + /// an ungated instance and lets the (now generous) command timeouts carry the + /// run. A throttle that could itself become a source of red is not worth having. + /// + /// + private sealed class MigrationGate : IDisposable + { + /// + /// How many fixtures may build a database at once, machine-wide. Sized to keep + /// the dev container busy without saturating it; override with + /// SCADABRIDGE_MSSQL_TEST_MIGRATE_SLOTS. + /// + private const int DefaultSlotCount = 4; + + private const string SlotEnvVar = "SCADABRIDGE_MSSQL_TEST_MIGRATE_SLOTS"; + + private const string ResourcePrefix = "ScadaBridgeTestMigrate/"; + + /// Per-slot wait before moving on to try the next slot. + private const int PerSlotLockTimeoutMs = 500; + + /// + /// Overall wait for a slot. Long enough to outlast a full queue of migration + /// chains on a loaded server; past it the gate gives up and lets the caller + /// proceed ungated rather than failing. + /// + private static readonly TimeSpan AcquireDeadline = TimeSpan.FromMinutes(10); + + private static int SlotCount + { + get + { + var raw = Environment.GetEnvironmentVariable(SlotEnvVar); + return int.TryParse(raw, out var parsed) && parsed > 0 ? parsed : DefaultSlotCount; + } + } + + private readonly SqlConnection? _connection; + private readonly string? _resource; + + private MigrationGate(SqlConnection? connection, string? resource) + { + _connection = connection; + _resource = resource; + } + + public static MigrationGate Acquire(string adminConnectionString) + { + var slots = SlotCount; + SqlConnection? connection = null; + + try + { + connection = new SqlConnection(BuildGateConnectionString(adminConnectionString)); + connection.Open(); + + var deadline = DateTime.UtcNow + AcquireDeadline; + // Start each caller at a different slot so they spread out instead of + // all queueing behind slot 0. + var offset = Random.Shared.Next(slots); + + while (DateTime.UtcNow < deadline) + { + for (var i = 0; i < slots; i++) + { + var resource = ResourcePrefix + ((offset + i) % slots); + if (TryAcquireSlot(connection, resource)) + { + return new MigrationGate(connection, resource); + } + } + } + } + catch + { + // The gate is an optimisation, not a correctness requirement — if it + // cannot be established (permissions, an older server, a transient + // fault) the caller proceeds ungated. + } + + connection?.Dispose(); + return new MigrationGate(null, null); + } + + private static bool TryAcquireSlot(SqlConnection connection, string resource) + { + using var cmd = connection.CreateCommand(); + cmd.CommandType = System.Data.CommandType.StoredProcedure; + cmd.CommandText = "sp_getapplock"; + cmd.CommandTimeout = (PerSlotLockTimeoutMs / 1000) + 30; + cmd.Parameters.AddWithValue("@Resource", resource); + cmd.Parameters.AddWithValue("@LockMode", "Exclusive"); + cmd.Parameters.AddWithValue("@LockOwner", "Session"); + cmd.Parameters.AddWithValue("@LockTimeout", PerSlotLockTimeoutMs); + + var result = cmd.Parameters.Add("@Result", System.Data.SqlDbType.Int); + result.Direction = System.Data.ParameterDirection.ReturnValue; + + cmd.ExecuteNonQuery(); + + // 0 = granted immediately, 1 = granted after waiting; everything else + // (-1 timeout, -2 cancelled, -3 deadlock victim, -999 error) means "not ours". + var code = result.Value as int? ?? -999; + return code >= 0; + } + + private static string BuildGateConnectionString(string adminConnectionString) + { + return new SqlConnectionStringBuilder(adminConnectionString) + { + // Application locks are database-scoped: every process must contend in + // the same database for the slots to mean anything. + InitialCatalog = "master", + // See the remarks — a pooled connection defers the implicit release of a + // session-scoped app lock until the connection is reused. + Pooling = false, + }.ConnectionString; + } + + public void Dispose() + { + if (_connection is null) + { + return; + } + + try + { + if (_resource is not null) + { + using var cmd = _connection.CreateCommand(); + cmd.CommandType = System.Data.CommandType.StoredProcedure; + cmd.CommandText = "sp_releaseapplock"; + cmd.Parameters.AddWithValue("@Resource", _resource); + cmd.Parameters.AddWithValue("@LockOwner", "Session"); + cmd.ExecuteNonQuery(); + } + } + catch + { + // Closing the unpooled connection below ends the session, which releases + // the lock regardless. + } + finally + { + _connection.Dispose(); + } + } + } }