namespace ZB.MOM.WW.ScadaBridge.Host;
///
/// Bounded retry-with-backoff for startup preconditions.
///
/// REQ-HOST-4a: a Central node applies/validates database migrations
/// before the host begins serving traffic. In container orchestration the database
/// and the app frequently start together, so the database may be briefly
/// unreachable. Rather than crashing the process on the first connection failure,
/// the migration step is wrapped in this bounded exponential backoff: it tolerates a
/// short outage and only fails fatally once attempts are exhausted.
///
/// Only transient faults are retried. The optional
/// isTransient predicate classifies each exception; a permanent failure
/// (e.g. a database schema-version mismatch — which no amount of waiting can fix)
/// is rethrown immediately rather than being retried for minutes before the
/// inevitable fatal exit.
///
public static class StartupRetry
{
///
/// Executes an asynchronous operation with bounded exponential backoff, retrying only transient faults.
///
/// Human-readable name of the operation, used in log messages.
/// The operation to attempt.
/// Maximum number of attempts before the exception propagates.
/// Delay before the second attempt; doubled on each subsequent retry, capped at 30 seconds.
/// Logger for retry warnings.
/// Optional predicate classifying an exception as transient; null means all exceptions are transient.
/// Cancellation token that aborts the retry loop immediately.
/// A task that completes when the operation succeeds, or faults on a permanent or final-attempt failure.
public static Task ExecuteWithRetryAsync(
string operationName,
Func operation,
int maxAttempts,
TimeSpan initialDelay,
ILogger logger,
Func? isTransient = null,
CancellationToken cancellationToken = default)
=> ExecuteWithRetryAsync(operationName, _ => operation(), maxAttempts, initialDelay, logger, isTransient, cancellationToken);
///
/// Executes an asynchronous operation with bounded exponential backoff, retrying only transient faults.
/// Overload that forwards the retry-loop cancellation token to the operation itself —
/// needed so callers (e.g. the database-migration step) can honour
/// IHostApplicationLifetime.ApplicationStopping inside the operation as well
/// as inside the inter-attempt Task.Delay.
///
/// Human-readable name of the operation, used in log messages.
/// The async operation to attempt; receives the cancellation token on each try.
/// Maximum number of attempts before the exception propagates.
/// Delay before the second attempt; doubled on each subsequent retry, capped at 30 seconds.
/// Logger for retry warnings and success messages.
/// Optional predicate classifying an exception as transient; null means all exceptions are transient.
/// Token to observe for cancellation; aborts the retry loop immediately.
/// A task that completes when the operation succeeds, or faults on a permanent or final-attempt failure.
public static async Task ExecuteWithRetryAsync(
string operationName,
Func operation,
int maxAttempts,
TimeSpan initialDelay,
ILogger logger,
Func? isTransient = null,
CancellationToken cancellationToken = default)
{
// Default: treat every exception as transient (preserves the pre-existing
// behaviour for callers that do not classify faults).
isTransient ??= static _ => true;
var delay = initialDelay;
for (var attempt = 1; ; attempt++)
{
cancellationToken.ThrowIfCancellationRequested();
try
{
await operation(cancellationToken);
if (attempt > 1)
logger.LogInformation(
"Startup operation '{Operation}' succeeded on attempt {Attempt}.",
operationName, attempt);
return;
}
catch (Exception ex) when (attempt < maxAttempts && isTransient(ex))
{
logger.LogWarning(ex,
"Startup operation '{Operation}' failed (transient) on attempt {Attempt}/{MaxAttempts}; " +
"retrying in {Delay}.",
operationName, attempt, maxAttempts, delay);
await Task.Delay(delay, cancellationToken);
// Exponential backoff, capped so the total wait stays bounded.
delay = TimeSpan.FromTicks(Math.Min(delay.Ticks * 2, TimeSpan.FromSeconds(30).Ticks));
}
}
}
///
/// Transient-fault classifier for the database-migration startup step.
/// Returns true only for connection-class faults that a brief wait can
/// resolve — a SQL connection/transport error or a timeout — and false
/// for everything else (notably schema-validation s
/// raised by MigrationHelper.ApplyOrValidateMigrationsAsync, which are
/// permanent and must fail fast).
///
/// The exception to classify.
/// true if the exception is a transient connection or timeout fault; false otherwise.
public static bool IsTransientDatabaseFault(Exception ex)
{
// Unwrap a single layer of aggregation so a faulted Task surfaces correctly.
if (ex is AggregateException agg && agg.InnerException != null)
ex = agg.InnerException;
if (ex is TimeoutException)
return true;
// Socket / network errors raised while opening the connection.
if (ex is System.Net.Sockets.SocketException)
return true;
// Microsoft.Data.SqlClient throws SqlException; matching by type name keeps
// the Host free of a direct SqlClient package reference. A SqlException at
// the migration stage is, in practice, a connection failure (the server is
// not yet reachable) rather than a schema fault — schema mismatches surface
// as InvalidOperationException from the migration helper.
var typeName = ex.GetType().FullName;
if (typeName != null &&
(typeName.EndsWith("SqlException", StringComparison.Ordinal) ||
typeName.EndsWith("DbException", StringComparison.Ordinal)))
return true;
return false;
}
}