From 5da779db17c9daa92d96e8dcb0cbe807c8cf90ae Mon Sep 17 00:00:00 2001
From: Joseph Doherty <dohejw01@gmail.com>
Date: Fri, 8 May 2026 09:33:59 -0400
Subject: [PATCH] fix(host): wait for configuration database before applying
 migrations
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Central nodes crashed at startup with `CREATE DATABASE permission denied`
when MSSQL accepted connections before recovering user databases —
DB_ID(@db) returned null, so EF Core's MigrateAsync fell through to
SqlServerDatabaseCreator.CreateAsync. The non-privileged app login then
failed CREATE DATABASE and the host terminated with FTL, leaving Traefik's
/health/active probe unable to find an upstream ("no available server" at
localhost:9000).

Add MigrationHelper.WaitForDatabaseReadyAsync that polls
Database.CanConnectAsync() for up to 60s before invoking MigrateAsync, and
thread an ILogger through so retry attempts surface in normal logs. This
removes the startup race without requiring depends_on across compose stacks
or granting dbcreator to the app login.
---
 .../MigrationHelper.cs                        | 53 +++++++++++++++++++
 src/ScadaLink.Host/Program.cs                 |  5 +-
 2 files changed, 57 insertions(+), 1 deletion(-)

diff --git a/src/ScadaLink.ConfigurationDatabase/MigrationHelper.cs b/src/ScadaLink.ConfigurationDatabase/MigrationHelper.cs
index 4acec5e..b38c4bd 100644
--- a/src/ScadaLink.ConfigurationDatabase/MigrationHelper.cs
+++ b/src/ScadaLink.ConfigurationDatabase/MigrationHelper.cs
@@ -16,12 +16,21 @@ public static class MigrationHelper
     /// </summary>
     /// <param name="dbContext">The database context to migrate or validate.</param>
     /// <param name="isDevelopment">When true, auto-applies migrations. When false, validates schema version matches.</param>
+    /// <param name="logger">Optional logger for readiness-wait diagnostics.</param>
     /// <param name="cancellationToken">Cancellation token.</param>
     public static async Task ApplyOrValidateMigrationsAsync(
         ScadaLinkDbContext dbContext,
         bool isDevelopment,
+        ILogger? logger = null,
         CancellationToken cancellationToken = default)
     {
+        // Wait for the target database to accept connections before invoking MigrateAsync.
+        // On a fresh MSSQL container, user databases recover asynchronously after the server
+        // starts accepting connections — DB_ID(@dbName) returns null until recovery completes.
+        // Without this wait, MigrateAsync sees the database as missing and falls through to
+        // CREATE DATABASE, which fails for non-privileged app logins.
+        await WaitForDatabaseReadyAsync(dbContext, logger, cancellationToken);
+
         if (isDevelopment)
         {
             await dbContext.Database.MigrateAsync(cancellationToken);
@@ -38,4 +47,48 @@ public static class MigrationHelper
             }
         }
     }
+
+    private static async Task WaitForDatabaseReadyAsync(
+        ScadaLinkDbContext dbContext,
+        ILogger? logger,
+        CancellationToken cancellationToken)
+    {
+        var timeout = TimeSpan.FromSeconds(60);
+        var pollInterval = TimeSpan.FromSeconds(2);
+        var deadline = DateTimeOffset.UtcNow + timeout;
+        var attempt = 0;
+        Exception? lastException = null;
+
+        while (DateTimeOffset.UtcNow < deadline)
+        {
+            attempt++;
+            try
+            {
+                if (await dbContext.Database.CanConnectAsync(cancellationToken))
+                {
+                    if (attempt > 1)
+                    {
+                        logger?.LogInformation(
+                            "Configuration database ready after {Attempt} attempt(s).", attempt);
+                    }
+                    return;
+                }
+                logger?.LogDebug(
+                    "Configuration database not yet reachable (attempt {Attempt}).", attempt);
+            }
+            catch (Exception ex)
+            {
+                lastException = ex;
+                logger?.LogDebug(ex,
+                    "Configuration database not yet reachable (attempt {Attempt}).", attempt);
+            }
+
+            await Task.Delay(pollInterval, cancellationToken);
+        }
+
+        throw new InvalidOperationException(
+            $"Configuration database not ready after {timeout.TotalSeconds:N0}s ({attempt} attempts). " +
+            "Verify SQL Server is running and the configuration database exists and is attached.",
+            lastException);
+    }
 }
diff --git a/src/ScadaLink.Host/Program.cs b/src/ScadaLink.Host/Program.cs
index 29116f8..637fdba 100644
--- a/src/ScadaLink.Host/Program.cs
+++ b/src/ScadaLink.Host/Program.cs
@@ -109,7 +109,10 @@ try
             using (var scope = app.Services.CreateScope())
             {
                 var dbContext = scope.ServiceProvider.GetRequiredService<ScadaLinkDbContext>();
-                await MigrationHelper.ApplyOrValidateMigrationsAsync(dbContext, isDevelopment);
+                var migrationLogger = scope.ServiceProvider
+                    .GetRequiredService<ILoggerFactory>()
+                    .CreateLogger(typeof(MigrationHelper).FullName!);
+                await MigrationHelper.ApplyOrValidateMigrationsAsync(dbContext, isDevelopment, migrationLogger);
             }
         }