fix(host): wait for configuration database before applying migrations

Central nodes crashed at startup with `CREATE DATABASE permission denied`
when MSSQL accepted connections before recovering user databases —
DB_ID(@db) returned null, so EF Core's MigrateAsync fell through to
SqlServerDatabaseCreator.CreateAsync. The non-privileged app login then
failed CREATE DATABASE and the host terminated with FTL, leaving Traefik's
/health/active probe unable to find an upstream ("no available server" at
localhost:9000).

Add MigrationHelper.WaitForDatabaseReadyAsync that polls
Database.CanConnectAsync() for up to 60s before invoking MigrateAsync, and
thread an ILogger through so retry attempts surface in normal logs. This
removes the startup race without requiring depends_on across compose stacks
or granting dbcreator to the app login.
This commit is contained in:
Joseph Doherty
2026-05-08 09:33:59 -04:00
parent 9dccf8e72f
commit 5da779db17
2 changed files with 57 additions and 1 deletions

View File

@@ -16,12 +16,21 @@ public static class MigrationHelper
/// </summary>
/// <param name="dbContext">The database context to migrate or validate.</param>
/// <param name="isDevelopment">When true, auto-applies migrations. When false, validates schema version matches.</param>
/// <param name="logger">Optional logger for readiness-wait diagnostics.</param>
/// <param name="cancellationToken">Cancellation token.</param>
public static async Task ApplyOrValidateMigrationsAsync(
ScadaLinkDbContext dbContext,
bool isDevelopment,
ILogger? logger = null,
CancellationToken cancellationToken = default)
{
// Wait for the target database to accept connections before invoking MigrateAsync.
// On a fresh MSSQL container, user databases recover asynchronously after the server
// starts accepting connections — DB_ID(@dbName) returns null until recovery completes.
// Without this wait, MigrateAsync sees the database as missing and falls through to
// CREATE DATABASE, which fails for non-privileged app logins.
await WaitForDatabaseReadyAsync(dbContext, logger, cancellationToken);
if (isDevelopment)
{
await dbContext.Database.MigrateAsync(cancellationToken);
@@ -38,4 +47,48 @@ public static class MigrationHelper
}
}
}
private static async Task WaitForDatabaseReadyAsync(
ScadaLinkDbContext dbContext,
ILogger? logger,
CancellationToken cancellationToken)
{
var timeout = TimeSpan.FromSeconds(60);
var pollInterval = TimeSpan.FromSeconds(2);
var deadline = DateTimeOffset.UtcNow + timeout;
var attempt = 0;
Exception? lastException = null;
while (DateTimeOffset.UtcNow < deadline)
{
attempt++;
try
{
if (await dbContext.Database.CanConnectAsync(cancellationToken))
{
if (attempt > 1)
{
logger?.LogInformation(
"Configuration database ready after {Attempt} attempt(s).", attempt);
}
return;
}
logger?.LogDebug(
"Configuration database not yet reachable (attempt {Attempt}).", attempt);
}
catch (Exception ex)
{
lastException = ex;
logger?.LogDebug(ex,
"Configuration database not yet reachable (attempt {Attempt}).", attempt);
}
await Task.Delay(pollInterval, cancellationToken);
}
throw new InvalidOperationException(
$"Configuration database not ready after {timeout.TotalSeconds:N0}s ({attempt} attempts). " +
"Verify SQL Server is running and the configuration database exists and is attached.",
lastException);
}
}

View File

@@ -109,7 +109,10 @@ try
using (var scope = app.Services.CreateScope())
{
var dbContext = scope.ServiceProvider.GetRequiredService<ScadaLinkDbContext>();
await MigrationHelper.ApplyOrValidateMigrationsAsync(dbContext, isDevelopment);
var migrationLogger = scope.ServiceProvider
.GetRequiredService<ILoggerFactory>()
.CreateLogger(typeof(MigrationHelper).FullName!);
await MigrationHelper.ApplyOrValidateMigrationsAsync(dbContext, isDevelopment, migrationLogger);
}
}