Joseph Doherty
d0d3058651
Flat 100*attempt backoff let concurrent boots retry in lockstep (Auth's twin loop exhausted retries ~1/60 eight-way boots). Backoff is now exponential with equal jitter, via an injectable Random for deterministic tests. Refs scadaproj#18
77 lines
3.6 KiB
C#
77 lines
3.6 KiB
C#
namespace ZB.MOM.WW.Secrets.Replicator.SqlServer.Tests;
|
|
|
|
/// <summary>
|
|
/// Offline pins for the migrator's retry classification. A <see cref="Microsoft.Data.SqlClient.SqlException"/>
|
|
/// cannot be constructed normally, so the error-number predicate is extracted as an internal static
|
|
/// and pinned here by number; the concurrent-provision behaviour itself is exercised by the
|
|
/// env-gated live suite.
|
|
/// </summary>
|
|
public sealed class SqlServerSecretsStoreMigratorTests
|
|
{
|
|
/// <summary>
|
|
/// The provisioning-race outcomes the ScadaBridge central shared-store live gate (2026-08-07)
|
|
/// proved reachable — or that the multi-statement DDL makes reachable — when N nodes provision
|
|
/// one virgin database at once. Losing such a race means the object now exists, which is the
|
|
/// desired end state, so a retry of the idempotent IF NOT EXISTS path must be taken.
|
|
/// </summary>
|
|
[Theory]
|
|
[InlineData(1205)] // deadlock victim (pre-0.5.1 was the ONLY retried number)
|
|
[InlineData(2714)] // "There is already an object named ..." — lost CREATE SCHEMA/TABLE (the live-gate defect)
|
|
[InlineData(1913)] // "index ... already exists" — lost CREATE INDEX
|
|
[InlineData(2627)] // PK violation — lost schema_version MERGE-insert
|
|
public void Provisioning_race_error_numbers_are_retryable(int errorNumber)
|
|
=> Assert.True(SqlServerSecretsStoreMigrator.IsRetryableMigrationErrorNumber(errorNumber));
|
|
|
|
/// <summary>
|
|
/// Errors that reproduce identically on retry must propagate so the node fails closed instead
|
|
/// of spinning through the retry budget against a genuine misconfiguration.
|
|
/// </summary>
|
|
[Theory]
|
|
[InlineData(2705)] // duplicate column — unreachable from the race (single CREATE TABLE statement)
|
|
[InlineData(208)] // invalid object name
|
|
[InlineData(229)] // permission denied
|
|
[InlineData(2759)] // "CREATE SCHEMA failed due to previous errors" — retryable only via its companion 2714
|
|
[InlineData(4060)] // cannot open database
|
|
[InlineData(18456)] // login failed
|
|
[InlineData(0)]
|
|
[InlineData(-2)] // client-side timeout
|
|
public void Non_race_error_numbers_are_not_retryable(int errorNumber)
|
|
=> Assert.False(SqlServerSecretsStoreMigrator.IsRetryableMigrationErrorNumber(errorNumber));
|
|
|
|
/// <summary>The jittered back-off stays inside [window/2, window) and grows with the attempt.</summary>
|
|
[Theory]
|
|
[InlineData(1, 50, 100)]
|
|
[InlineData(2, 100, 200)]
|
|
[InlineData(4, 400, 800)]
|
|
public void Backoff_is_bounded_by_the_exponential_window(int attempt, double minMs, double maxMs)
|
|
{
|
|
var random = new Random(12345);
|
|
for (int i = 0; i < 1000; i++)
|
|
{
|
|
double ms = SqlServerSecretsStoreMigrator.ComputeBackoff(attempt, random).TotalMilliseconds;
|
|
Assert.InRange(ms, minMs, maxMs);
|
|
}
|
|
}
|
|
|
|
/// <summary>Different draws give different delays (the lockstep-retry defect was a constant schedule).</summary>
|
|
[Fact]
|
|
public void Backoff_is_jittered_not_constant()
|
|
{
|
|
var random = new Random(42);
|
|
var delays = Enumerable.Range(0, 8)
|
|
.Select(_ => SqlServerSecretsStoreMigrator.ComputeBackoff(3, random))
|
|
.Distinct()
|
|
.ToList();
|
|
Assert.True(delays.Count > 1);
|
|
}
|
|
|
|
/// <summary>A fixed jitter source yields an exactly predictable delay.</summary>
|
|
[Fact]
|
|
public void Backoff_is_deterministic_for_a_seeded_source()
|
|
{
|
|
Assert.Equal(
|
|
SqlServerSecretsStoreMigrator.ComputeBackoff(3, new Random(7)),
|
|
SqlServerSecretsStoreMigrator.ComputeBackoff(3, new Random(7)));
|
|
}
|
|
}
|