test(sql): reap >24h-old orphaned test databases at fixture startup (AdminUI + Configuration prefixes)

2026-08-17 05:37:22 -04:00
parent 9f09a62ad2
commit 8946644bb1
3 changed files with 460 additions and 22 deletions
@@ -1,3 +1,5 @@
+using System.Data.Common;
+using System.Globalization;
 using Microsoft.Data.SqlClient;
 using Microsoft.EntityFrameworkCore;
 using Xunit;
@@ -13,9 +15,17 @@ namespace ZB.MOM.WW.OtOpcUa.Configuration.Tests;
 /// </summary>
 public sealed class SchemaComplianceFixture : IDisposable
 {
+    /// <summary>The name prefix every Configuration test database carries; also the reaper's LIKE anchor.</summary>
+    internal const string TestDatabasePrefix = "OtOpcUaConfig_Test_";
+
    private const string DefaultServer = "10.100.0.35,14330";
    private const string DefaultSaPassword = "OtOpcUaDev_2026!";

+    /// <summary>How long a test database must have existed before the reaper considers it certainly orphaned.</summary>
+    private static readonly TimeSpan OrphanAge = TimeSpan.FromHours(24);
+
+    private static readonly Lazy<int> Reaper = new(RunReaper, LazyThreadSafetyMode.ExecutionAndPublication);
+
    /// <summary>Gets the name of the test database.</summary>
    public string DatabaseName { get; }
    /// <summary>Gets the connection string for the test database.</summary>
@@ -27,7 +37,13 @@ public sealed class SchemaComplianceFixture : IDisposable
        var server = Environment.GetEnvironmentVariable("OTOPCUA_CONFIG_TEST_SERVER") ?? DefaultServer;
        var saPassword = Environment.GetEnvironmentVariable("OTOPCUA_CONFIG_TEST_SA_PASSWORD") ?? DefaultSaPassword;

-        DatabaseName = $"OtOpcUaConfig_Test_{DateTime.UtcNow:yyyyMMddHHmmss}_{Guid.NewGuid():N}";
+        // A killed test process never reaches Dispose, so its database outlives the run. Reap the ones old
+        // enough to be certainly nobody's, once per process. This fixture has no reachability probe, so an
+        // unreachable server must leave behaviour exactly as it was: the reaper silently does nothing and
+        // Migrate() below throws as it always has.
+        _ = Reaper.Value;
+
+        DatabaseName = $"{TestDatabasePrefix}{DateTime.UtcNow:yyyyMMddHHmmss}_{Guid.NewGuid():N}";
        ConnectionString =
            $"Server={server};Database={DatabaseName};User Id=sa;Password={saPassword};TrustServerCertificate=True;Encrypt=False;";

@@ -48,6 +64,16 @@ public sealed class SchemaComplianceFixture : IDisposable
        return conn;
    }

+    private static int RunReaper()
+    {
+        var server = Environment.GetEnvironmentVariable("OTOPCUA_CONFIG_TEST_SERVER") ?? DefaultServer;
+        var saPassword = Environment.GetEnvironmentVariable("OTOPCUA_CONFIG_TEST_SA_PASSWORD") ?? DefaultSaPassword;
+        var master =
+            $"Server={server};Database=master;User Id=sa;Password={saPassword};TrustServerCertificate=True;Encrypt=False;";
+
+        return SqlTestDatabaseReaper.ReapOrphans(master, TestDatabasePrefix, DateTime.UtcNow - OrphanAge).Count;
+    }
+
    /// <summary>Disposes the fixture and drops the test database.</summary>
    public void Dispose()
    {
@@ -68,6 +94,156 @@ END";
    }
 }

+/// <summary>
+///     Drops test databases that a killed test process left behind on the <b>shared</b> SQL Server.
+/// </summary>
+/// <remarks>
+///     <para>
+///         <see cref="SchemaComplianceFixture"/> names its database
+///         <c>{prefix}{yyyyMMddHHmmss-UTC}_{guid:N}</c> and drops it in <c>Dispose</c> — which a
+///         <c>Ctrl-C</c>, a crashed runner or a killed container never reaches, so the database survives
+///         forever on an instance nobody gardens.
+///     </para>
+///     <para>
+///         This is <c>DROP DATABASE</c> against a server other people share, so the rules are deliberately
+///         narrow and every one of them is load-bearing:
+///     </para>
+///     <list type="bullet">
+///         <item>
+///             the <c>LIKE</c> is an <b>exact prefix</b> with <c>_</c> escaped (an unescaped <c>_</c> is a
+///             single-character wildcard in T-SQL) — never widen it;
+///         </item>
+///         <item>
+///             a name whose embedded timestamp does not parse is <b>left alone</b>, because an unparseable
+///             name is by definition not one this code created;
+///         </item>
+///         <item>
+///             only names <b>strictly older than the cutoff</b> are dropped. At 24 hours that is far beyond
+///             any run, so the reaper can never race a live sibling run on the same server; and
+///         </item>
+///         <item>
+///             nothing here ever throws. Unlike this project's <c>Dispose</c>, which deliberately lets a
+///             teardown failure surface, opportunistic housekeeping must never fail a green run.
+///         </item>
+///     </list>
+///     <para>
+///         Deliberately duplicated from the AdminUI.Tests copy rather than shared: the two suites have no
+///         common test-support assembly, and a new one purely for ~100 lines of best-effort housekeeping
+///         costs more than the duplication.
+///     </para>
+/// </remarks>
+internal static class SqlTestDatabaseReaper
+{
+    private const string TimestampFormat = "yyyyMMddHHmmss";
+
+    /// <summary>
+    ///     Drops every database matching <paramref name="prefix"/> whose embedded creation timestamp is
+    ///     strictly older than <paramref name="cutoffUtc"/>.
+    /// </summary>
+    /// <param name="masterConnectionString">A connection string pointed at <c>master</c>.</param>
+    /// <param name="prefix">The exact test-database name prefix, e.g. <c>OtOpcUaConfig_Test_</c>.</param>
+    /// <param name="cutoffUtc">Databases created strictly before this UTC instant are dropped.</param>
+    /// <returns>The names actually dropped; empty when there was nothing to reap or anything went wrong.</returns>
+    internal static IReadOnlyList<string> ReapOrphans(string masterConnectionString, string prefix, DateTime cutoffUtc)
+    {
+        var reaped = new List<string>();
+
+        try
+        {
+            foreach (var name in ListCandidates(masterConnectionString, prefix))
+            {
+                if (!TryParseCreatedUtc(name, prefix, out var createdUtc)) continue;
+                if (createdUtc >= cutoffUtc) continue;
+
+                try
+                {
+                    if (TryDropDatabase(masterConnectionString, name)) reaped.Add(name);
+                }
+                catch (Exception)
+                {
+                    // One stubborn database must not stop the sweep, nor fail the run that triggered it.
+                }
+            }
+        }
+        catch (Exception)
+        {
+            // Enumeration failed (server down, no permission). Reaping is opportunistic housekeeping.
+        }
+
+        if (reaped.Count > 0)
+        {
+            Console.WriteLine(
+                $"[sql-test-reaper] dropped {reaped.Count} orphaned test database(s) older than " +
+                $"{cutoffUtc:yyyy-MM-dd HH:mm:ss}Z: {string.Join(", ", reaped)}");
+        }
+
+        return reaped;
+    }
+
+    /// <summary>Drops the named database, swallowing database errors — reaping is best-effort.</summary>
+    /// <param name="masterConnectionString">A connection string pointed at <c>master</c>.</param>
+    /// <param name="databaseName">The database to drop.</param>
+    /// <returns><see langword="true"/> when the statement ran without error.</returns>
+    internal static bool TryDropDatabase(string masterConnectionString, string databaseName)
+    {
+        try
+        {
+            using var conn = new SqlConnection(masterConnectionString);
+            conn.Open();
+
+            using var cmd = conn.CreateCommand();
+            cmd.CommandText = $@"
+IF DB_ID(N'{databaseName}') IS NOT NULL
+BEGIN
+    ALTER DATABASE [{databaseName}] SET SINGLE_USER WITH ROLLBACK IMMEDIATE;
+    DROP DATABASE [{databaseName}];
+END";
+            cmd.ExecuteNonQuery();
+            return true;
+        }
+        catch (DbException)
+        {
+            return false;
+        }
+    }
+
+    private static List<string> ListCandidates(string masterConnectionString, string prefix)
+    {
+        var names = new List<string>();
+
+        using var conn = new SqlConnection(masterConnectionString);
+        conn.Open();
+
+        using var cmd = conn.CreateCommand();
+        cmd.CommandText = @"SELECT name FROM sys.databases WHERE name LIKE @pattern ESCAPE '\'";
+        cmd.Parameters.Add(new SqlParameter("@pattern", prefix.Replace("_", @"\_", StringComparison.Ordinal) + "%"));
+
+        using var reader = cmd.ExecuteReader();
+        while (reader.Read()) names.Add(reader.GetString(0));
+
+        return names;
+    }
+
+    private static bool TryParseCreatedUtc(string databaseName, string prefix, out DateTime createdUtc)
+    {
+        createdUtc = default;
+
+        if (!databaseName.StartsWith(prefix, StringComparison.Ordinal)) return false;
+
+        var rest = databaseName.AsSpan(prefix.Length);
+
+        // Expect exactly "<14-digit timestamp>_<guid>" — anything else is not a name this code minted.
+        if (rest.Length < TimestampFormat.Length + 1 || rest[TimestampFormat.Length] != '_') return false;
+
+        return DateTime.TryParseExact(
+            rest[..TimestampFormat.Length],
+            TimestampFormat,
+            CultureInfo.InvariantCulture,
+            DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal,
+            out createdUtc);
+    }
+}
+
 [CollectionDefinition(nameof(SchemaComplianceCollection))]
 public sealed class SchemaComplianceCollection : ICollectionFixture<SchemaComplianceFixture>
 {
@@ -1,4 +1,5 @@
 using System.Data.Common;
+using System.Globalization;
 using Microsoft.Data.SqlClient;
 using Microsoft.EntityFrameworkCore;
 using Xunit;
@@ -74,6 +75,10 @@ public sealed class AdminUiSqlServerFixture : IDisposable
        SkipReason = AdminUiSqlServer.SkipReason;
        if (SkipReason is not null) return;

+        // A killed test process never reaches Dispose, so its database outlives the run. Reap the ones
+        // old enough to be certainly nobody's (once per process, and only after the probe answered).
+        AdminUiSqlServer.ReapOrphanedTestDatabasesOnce();
+
        DatabaseName = $"OtOpcUaAdminUi_Test_{DateTime.UtcNow:yyyyMMddHHmmss}_{Guid.NewGuid():N}";
        ConnectionString = AdminUiSqlServer.ConnectionStringFor(DatabaseName);

@@ -106,27 +111,8 @@ public sealed class AdminUiSqlServerFixture : IDisposable
    {
        if (ConnectionString is null || DatabaseName is null) return;

-        try
-        {
-            var master = new SqlConnectionStringBuilder(ConnectionString) { InitialCatalog = "master" }.ConnectionString;
-
-            using var conn = new SqlConnection(master);
-            conn.Open();
-
-            using var cmd = conn.CreateCommand();
-            cmd.CommandText = $@"
-IF DB_ID(N'{DatabaseName}') IS NOT NULL
-BEGIN
-    ALTER DATABASE [{DatabaseName}] SET SINGLE_USER WITH ROLLBACK IMMEDIATE;
-    DROP DATABASE [{DatabaseName}];
-END";
-            cmd.ExecuteNonQuery();
-        }
-        catch (DbException)
-        {
-            // Teardown is best-effort: a dropped connection here must not fail an otherwise-green run.
-            // The database name carries a timestamp, so a leaked one is identifiable.
-        }
+        var master = new SqlConnectionStringBuilder(ConnectionString) { InitialCatalog = "master" }.ConnectionString;
+        SqlTestDatabaseReaper.TryDropDatabase(master, DatabaseName);
    }

    private sealed class SqlServerContextFactory(string connectionString)
@@ -150,15 +136,30 @@ END";
 /// </summary>
 internal static class AdminUiSqlServer
 {
+    /// <summary>The name prefix every AdminUI test database carries; also the reaper's LIKE anchor.</summary>
+    public const string TestDatabasePrefix = "OtOpcUaAdminUi_Test_";
+
    private const string DefaultServer = "10.100.0.35,14330";
    private const string DefaultSaPassword = "OtOpcUaDev_2026!";
    private const int ProbeTimeoutSeconds = 3;

+    /// <summary>How long a test database must have existed before the reaper considers it certainly orphaned.</summary>
+    private static readonly TimeSpan OrphanAge = TimeSpan.FromHours(24);
+
    private static readonly Lazy<string?> Probe = new(RunProbe, LazyThreadSafetyMode.ExecutionAndPublication);

+    private static readonly Lazy<int> Reaper = new(RunReaper, LazyThreadSafetyMode.ExecutionAndPublication);
+
    /// <summary>Gets the reason SQL Server is unusable, or <see langword="null"/> when it answered.</summary>
    public static string? SkipReason => Probe.Value;

+    /// <summary>
+    ///     Reaps orphaned AdminUI test databases older than 24 hours, at most once per process. Call only
+    ///     after <see cref="SkipReason"/> came back <see langword="null"/> — the reaper never throws, but
+    ///     against a dead server it would just burn a connect timeout for nothing.
+    /// </summary>
+    public static void ReapOrphanedTestDatabasesOnce() => _ = Reaper.Value;
+
    /// <summary>Builds a connection string for the named database on the configured test server.</summary>
    /// <param name="database">The database to connect to.</param>
    /// <returns>The connection string.</returns>
@@ -171,6 +172,12 @@ internal static class AdminUiSqlServer
               "TrustServerCertificate=True;Encrypt=False;";
    }

+    private static int RunReaper() =>
+        SqlTestDatabaseReaper.ReapOrphans(
+            ConnectionStringFor("master"),
+            TestDatabasePrefix,
+            DateTime.UtcNow - OrphanAge).Count;
+
    private static string? RunProbe()
    {
        var server = Environment.GetEnvironmentVariable("OTOPCUA_CONFIG_TEST_SERVER") ?? DefaultServer;
@@ -194,6 +201,152 @@ internal static class AdminUiSqlServer
    }
 }

+/// <summary>
+///     Drops test databases that a killed test process left behind on the <b>shared</b> SQL Server.
+/// </summary>
+/// <remarks>
+///     <para>
+///         Every fixture here names its database <c>{prefix}{yyyyMMddHHmmss-UTC}_{guid:N}</c> and drops it
+///         in <c>Dispose</c> — which a <c>Ctrl-C</c>, a crashed runner or a killed container never reaches,
+///         so the database survives forever on an instance nobody gardens.
+///     </para>
+///     <para>
+///         This is <c>DROP DATABASE</c> against a server other people share, so the rules are deliberately
+///         narrow and every one of them is load-bearing:
+///     </para>
+///     <list type="bullet">
+///         <item>
+///             the <c>LIKE</c> is an <b>exact prefix</b> with <c>_</c> escaped (an unescaped <c>_</c> is a
+///             single-character wildcard in T-SQL) — never widen it;
+///         </item>
+///         <item>
+///             a name whose embedded timestamp does not parse is <b>left alone</b>, because an unparseable
+///             name is by definition not one this code created;
+///         </item>
+///         <item>
+///             only names <b>strictly older than the cutoff</b> are dropped. At 24 hours that is far beyond
+///             any run, so the reaper can never race a live sibling run on the same server; and
+///         </item>
+///         <item>
+///             nothing here ever throws. A reaper that fails a green run would be worse than the leak it
+///             cleans up.
+///         </item>
+///     </list>
+/// </remarks>
+internal static class SqlTestDatabaseReaper
+{
+    private const string TimestampFormat = "yyyyMMddHHmmss";
+
+    /// <summary>
+    ///     Drops every database matching <paramref name="prefix"/> whose embedded creation timestamp is
+    ///     strictly older than <paramref name="cutoffUtc"/>.
+    /// </summary>
+    /// <param name="masterConnectionString">A connection string pointed at <c>master</c>.</param>
+    /// <param name="prefix">The exact test-database name prefix, e.g. <c>OtOpcUaAdminUi_Test_</c>.</param>
+    /// <param name="cutoffUtc">Databases created strictly before this UTC instant are dropped.</param>
+    /// <returns>The names actually dropped; empty when there was nothing to reap or anything went wrong.</returns>
+    internal static IReadOnlyList<string> ReapOrphans(string masterConnectionString, string prefix, DateTime cutoffUtc)
+    {
+        var reaped = new List<string>();
+
+        try
+        {
+            foreach (var name in ListCandidates(masterConnectionString, prefix))
+            {
+                if (!TryParseCreatedUtc(name, prefix, out var createdUtc)) continue;
+                if (createdUtc >= cutoffUtc) continue;
+
+                try
+                {
+                    if (TryDropDatabase(masterConnectionString, name)) reaped.Add(name);
+                }
+                catch (Exception)
+                {
+                    // One stubborn database must not stop the sweep, nor fail the run that triggered it.
+                }
+            }
+        }
+        catch (Exception)
+        {
+            // Enumeration failed (server down, no permission). Reaping is opportunistic housekeeping.
+        }
+
+        if (reaped.Count > 0)
+        {
+            Console.WriteLine(
+                $"[sql-test-reaper] dropped {reaped.Count} orphaned test database(s) older than " +
+                $"{cutoffUtc:yyyy-MM-dd HH:mm:ss}Z: {string.Join(", ", reaped)}");
+        }
+
+        return reaped;
+    }
+
+    /// <summary>Drops the named database, swallowing database errors — teardown is best-effort.</summary>
+    /// <param name="masterConnectionString">A connection string pointed at <c>master</c>.</param>
+    /// <param name="databaseName">The database to drop.</param>
+    /// <returns><see langword="true"/> when the statement ran without error.</returns>
+    internal static bool TryDropDatabase(string masterConnectionString, string databaseName)
+    {
+        try
+        {
+            using var conn = new SqlConnection(masterConnectionString);
+            conn.Open();
+
+            using var cmd = conn.CreateCommand();
+            cmd.CommandText = $@"
+IF DB_ID(N'{databaseName}') IS NOT NULL
+BEGIN
+    ALTER DATABASE [{databaseName}] SET SINGLE_USER WITH ROLLBACK IMMEDIATE;
+    DROP DATABASE [{databaseName}];
+END";
+            cmd.ExecuteNonQuery();
+            return true;
+        }
+        catch (DbException)
+        {
+            // Teardown is best-effort: a dropped connection here must not fail an otherwise-green run.
+            // The database name carries a timestamp, so a leaked one is reaped by the next run instead.
+            return false;
+        }
+    }
+
+    private static List<string> ListCandidates(string masterConnectionString, string prefix)
+    {
+        var names = new List<string>();
+
+        using var conn = new SqlConnection(masterConnectionString);
+        conn.Open();
+
+        using var cmd = conn.CreateCommand();
+        cmd.CommandText = @"SELECT name FROM sys.databases WHERE name LIKE @pattern ESCAPE '\'";
+        cmd.Parameters.Add(new SqlParameter("@pattern", prefix.Replace("_", @"\_", StringComparison.Ordinal) + "%"));
+
+        using var reader = cmd.ExecuteReader();
+        while (reader.Read()) names.Add(reader.GetString(0));
+
+        return names;
+    }
+
+    private static bool TryParseCreatedUtc(string databaseName, string prefix, out DateTime createdUtc)
+    {
+        createdUtc = default;
+
+        if (!databaseName.StartsWith(prefix, StringComparison.Ordinal)) return false;
+
+        var rest = databaseName.AsSpan(prefix.Length);
+
+        // Expect exactly "<14-digit timestamp>_<guid>" — anything else is not a name this code minted.
+        if (rest.Length < TimestampFormat.Length + 1 || rest[TimestampFormat.Length] != '_') return false;
+
+        return DateTime.TryParseExact(
+            rest[..TimestampFormat.Length],
+            TimestampFormat,
+            CultureInfo.InvariantCulture,
+            DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal,
+            out createdUtc);
+    }
+}
+
 /// <summary>
 ///     A <see cref="FactAttribute"/> that skips itself when the shared test SQL Server is unreachable.
 ///     The xunit v2 equivalent of xunit v3's <c>Assert.Skip</c> — see
@@ -0,0 +1,109 @@
+using Microsoft.Data.SqlClient;
+using Shouldly;
+
+namespace ZB.MOM.WW.OtOpcUa.AdminUI.Tests;
+
+/// <summary>
+///     Pins <see cref="SqlTestDatabaseReaper"/> against the real shared SQL Server: an aged orphan is
+///     dropped, a fresh sibling is not.
+/// </summary>
+/// <remarks>
+///     A <b>pin, not a red-first test</b>. The reaper is time-gated and the behaviour it protects — "a
+///     database created by a run that is long over gets dropped, one created by a run that may still be
+///     going does not" — has no observable failing state to start from; what it needs is a guard against
+///     someone later widening the age window, the <c>LIKE</c>, or the timestamp parse. It therefore drives
+///     the internal core with an explicit cutoff rather than waiting 24 hours.
+/// </remarks>
+public sealed class SqlTestDatabaseReaperTests
+{
+    [SqlServerFact]
+    public void Reap_drops_databases_older_than_the_cutoff_and_leaves_fresh_ones_alone()
+    {
+        var master = AdminUiSqlServer.ConnectionStringFor("master");
+
+        // A name that carries a timestamp from 2025 — unambiguously older than any cutoff this suite uses.
+        var aged = $"{AdminUiSqlServer.TestDatabasePrefix}20250101000000_{Guid.NewGuid():N}";
+        var fresh = $"{AdminUiSqlServer.TestDatabasePrefix}{DateTime.UtcNow:yyyyMMddHHmmss}_{Guid.NewGuid():N}";
+
+        CreateDatabase(master, aged);
+
+        try
+        {
+            CreateDatabase(master, fresh);
+
+            try
+            {
+                var reaped = SqlTestDatabaseReaper.ReapOrphans(
+                    master,
+                    AdminUiSqlServer.TestDatabasePrefix,
+                    DateTime.UtcNow - TimeSpan.FromHours(24));
+
+                reaped.ShouldContain(aged);
+                DatabaseExists(master, aged).ShouldBeFalse("the aged orphan should have been reaped");
+
+                reaped.ShouldNotContain(fresh);
+                DatabaseExists(master, fresh).ShouldBeTrue("a database from a possibly-live run must survive");
+            }
+            finally
+            {
+                SqlTestDatabaseReaper.TryDropDatabase(master, fresh);
+            }
+        }
+        finally
+        {
+            // No-op when the reaper already took it; belt-and-braces if an assertion failed first.
+            SqlTestDatabaseReaper.TryDropDatabase(master, aged);
+        }
+    }
+
+    [SqlServerFact]
+    public void Reap_leaves_a_matching_name_whose_timestamp_does_not_parse_alone()
+    {
+        var master = AdminUiSqlServer.ConnectionStringFor("master");
+        var unparseable = $"{AdminUiSqlServer.TestDatabasePrefix}notatimestamp_{Guid.NewGuid():N}";
+
+        CreateDatabase(master, unparseable);
+
+        try
+        {
+            // The cutoff stays at the production 24 hours — deliberately NOT a far-future one, which would
+            // sweep away the live database of an AdminUI fixture running in parallel on this shared server.
+            // It still bites: an unparseable name that fell through to default(DateTime) would read as year
+            // 0001 and be dropped.
+            var reaped = SqlTestDatabaseReaper.ReapOrphans(
+                master,
+                AdminUiSqlServer.TestDatabasePrefix,
+                DateTime.UtcNow - TimeSpan.FromHours(24));
+
+            reaped.ShouldNotContain(unparseable);
+            DatabaseExists(master, unparseable)
+                .ShouldBeTrue("a name this code could not have minted is never dropped");
+        }
+        finally
+        {
+            SqlTestDatabaseReaper.TryDropDatabase(master, unparseable);
+        }
+    }
+
+    private static void CreateDatabase(string masterConnectionString, string databaseName)
+    {
+        using var conn = new SqlConnection(masterConnectionString);
+        conn.Open();
+
+        using var cmd = conn.CreateCommand();
+        cmd.CommandText = $"CREATE DATABASE [{databaseName}];";
+        cmd.ExecuteNonQuery();
+    }
+
+    private static bool DatabaseExists(string masterConnectionString, string databaseName)
+    {
+        using var conn = new SqlConnection(masterConnectionString);
+        conn.Open();
+
+        using var cmd = conn.CreateCommand();
+        cmd.CommandText = "SELECT DB_ID(@name);";
+        cmd.Parameters.Add(new SqlParameter("@name", databaseName));
+
+        return cmd.ExecuteScalar() is not (null or DBNull);
+    }
+}