fix(secrets): jittered exponential backoff in SqlServerSecretsStoreMigrator + 8-way concurrent-migrate live test

Flat 100*attempt backoff let concurrent boots retry in lockstep (Auth's twin loop
exhausted retries ~1/60 eight-way boots). Backoff is now exponential with equal
jitter, via an injectable Random for deterministic tests.

Refs scadaproj#18
This commit is contained in:
Joseph Doherty
2026-11-09 12:05:33 -05:00
parent a16567b8bf
commit d0d3058651
3 changed files with 84 additions and 5 deletions
@@ -13,9 +13,22 @@ namespace ZB.MOM.WW.Secrets.Replicator.SqlServer;
/// to run against a database whose recorded version is newer than this build supports.
/// </summary>
/// <param name="connectionFactory">Factory for connections to the shared secret database.</param>
public sealed class SqlServerSecretsStoreMigrator(SecretsSqlServerConnectionFactory connectionFactory)
: ISecretsStoreMigrator
/// <param name="random">Jitter source for the retry back-off; inject a seeded instance for deterministic tests.</param>
public sealed class SqlServerSecretsStoreMigrator(
SecretsSqlServerConnectionFactory connectionFactory,
Random random) : ISecretsStoreMigrator
{
/// <summary>
/// Initializes the migrator with the process-wide <see cref="Random.Shared"/> as its jitter source.
/// </summary>
/// <param name="connectionFactory">Factory for connections to the shared secret database.</param>
public SqlServerSecretsStoreMigrator(SecretsSqlServerConnectionFactory connectionFactory)
: this(connectionFactory, Random.Shared)
{
}
private const int BaseBackoffMilliseconds = 100;
// Every node runs this at every startup, so a virgin database being provisioned by N nodes at
// once is the NORMAL case, not a rare race — and an unretried failure aborts that node's whole
// host at startup. Two failure shapes come out of that race, and retrying is correct for both
@@ -63,13 +76,29 @@ public sealed class SqlServerSecretsStoreMigrator(SecretsSqlServerConnectionFact
catch (SqlException ex)
when (IsRetryableMigrationError(ex) && attempt < MaxMigrationRetries)
{
// Staggered back-off so two victims do not retry in lockstep and race again.
await Task.Delay(TimeSpan.FromMilliseconds(100 * attempt), cancellationToken)
// Jittered back-off so N victims do not retry in lockstep and race again (a flat
// 100 * attempt schedule exhausted its retries ~1 in 60 eight-way boots in Auth's
// identical loop; scadaproj#18).
await Task.Delay(ComputeBackoff(attempt, random), cancellationToken)
.ConfigureAwait(false);
}
}
}
/// <summary>
/// Computes the delay before retry number <paramref name="attempt"/> using exponential back-off
/// with "equal jitter": half the exponential window is fixed, the other half is random, so the
/// result lies in [window/2, window) where window = 100 ms * 2^(attempt - 1).
/// </summary>
/// <param name="attempt">The 1-based attempt that just failed.</param>
/// <param name="random">Jitter source; inject a seeded instance for deterministic tests.</param>
/// <returns>The delay to wait before the next attempt.</returns>
internal static TimeSpan ComputeBackoff(int attempt, Random random)
{
double window = BaseBackoffMilliseconds * Math.Pow(2, attempt - 1);
return TimeSpan.FromMilliseconds((window / 2) + (random.NextDouble() * window / 2));
}
/// <summary>
/// Whether any error carried by <paramref name="exception"/> is a retryable provisioning-race
/// outcome. Scans the whole <see cref="SqlException.Errors"/> collection rather than only