Joseph Doherty
103 lines
4.0 KiB
C#
103 lines
4.0 KiB
C#
namespace ZB.MOM.WW.LocalDb.Tests.Convergence;
|
|
|
|
/// <summary>
|
|
/// Randomized property test: a large stream of mixed insert/update/delete operations on both nodes
|
|
/// over a small shared key space, interleaved with genuine transport disconnect/reconnect cycles,
|
|
/// must always converge to byte-identical state. Seeded Random only (no time-based seeds) so every
|
|
/// failure is reproducible from the logged seed.
|
|
/// </summary>
|
|
public sealed class RandomOpsConvergenceTests
|
|
{
|
|
private const int OpsPerNode = 500;
|
|
private const int KeySpace = 40;
|
|
|
|
[Theory]
|
|
[InlineData(1701)]
|
|
[InlineData(42)]
|
|
[InlineData(7)]
|
|
public async Task RandomInterleavedOps_Converge(int seed)
|
|
{
|
|
var rng = new Random(seed);
|
|
|
|
// Segment plan derived up front from the seeded RNG (so the whole run is deterministic):
|
|
// steady, OUTAGE-1, steady, OUTAGE-2, steady — summing to OpsPerNode ops on each node.
|
|
var seg1 = rng.Next(80, 150);
|
|
var outage1 = rng.Next(20, 50);
|
|
var seg2 = rng.Next(80, 150);
|
|
var outage2 = rng.Next(20, 50);
|
|
var tail = OpsPerNode - (seg1 + outage1 + seg2 + outage2);
|
|
|
|
await using var fx = new ConvergenceFixture();
|
|
await fx.StartAsync();
|
|
|
|
try
|
|
{
|
|
await RunOpsAsync(fx, rng, seg1);
|
|
await OutageAsync(fx, rng, outage1);
|
|
await RunOpsAsync(fx, rng, seg2);
|
|
await OutageAsync(fx, rng, outage2);
|
|
await RunOpsAsync(fx, rng, tail);
|
|
|
|
await fx.AssertConvergedAsync();
|
|
}
|
|
catch (Exception ex) when (ex is not Xunit.Sdk.XunitException)
|
|
{
|
|
throw new Xunit.Sdk.XunitException($"seed {seed}: run faulted.\n{ex}");
|
|
}
|
|
catch (Xunit.Sdk.XunitException ex)
|
|
{
|
|
throw new Xunit.Sdk.XunitException($"seed {seed}: convergence failed.\n{ex.Message}");
|
|
}
|
|
|
|
// Both outage windows actually severed and re-dialed the transport (initial dial + 2
|
|
// reconnects) — proof the run exercised real offline accumulation + resync.
|
|
Assert.True(fx.InitiatorConnectionAttempts >= 3,
|
|
$"seed {seed}: expected >=3 connection attempts (2 reconnects), got {fx.InitiatorConnectionAttempts}");
|
|
}
|
|
|
|
// Severs the transport, accumulates offline writes on BOTH nodes, waits until the initiator has
|
|
// actually observed the loss (a reconnect attempt started), then restores the transport.
|
|
private static async Task OutageAsync(ConvergenceFixture fx, Random rng, int ops)
|
|
{
|
|
var baseline = fx.InitiatorConnectionAttempts;
|
|
await fx.KillTransportAsync();
|
|
await RunOpsAsync(fx, rng, ops);
|
|
await WaitForAsync(() => Task.FromResult(fx.InitiatorConnectionAttempts > baseline),
|
|
TimeSpan.FromSeconds(10));
|
|
await fx.RestartTransportAsync();
|
|
}
|
|
|
|
private static async Task RunOpsAsync(ConvergenceFixture fx, Random rng, int count)
|
|
{
|
|
// One op per node per iteration; no sleeps — the sessions' pumps drain concurrently.
|
|
for (var i = 0; i < count; i++)
|
|
{
|
|
await ApplyRandomOpAsync(fx.A, rng);
|
|
await ApplyRandomOpAsync(fx.B, rng);
|
|
}
|
|
}
|
|
|
|
private static Task ApplyRandomOpAsync(ILocalDb db, Random rng)
|
|
{
|
|
var id = rng.Next(1, KeySpace + 1);
|
|
return rng.Next(3) switch
|
|
{
|
|
// insert / update both resolve through the supported upsert helper (plain UPDATE-else-INSERT).
|
|
0 or 1 => ConvergenceFixture.UpsertAsync(db, id, "v" + rng.Next(1000), rng.Next(10_000)),
|
|
_ => ConvergenceFixture.DeleteAsync(db, id),
|
|
};
|
|
}
|
|
|
|
private static async Task WaitForAsync(Func<Task<bool>> predicate, TimeSpan timeout)
|
|
{
|
|
var deadline = DateTime.UtcNow + timeout;
|
|
while (DateTime.UtcNow < deadline)
|
|
{
|
|
if (await predicate())
|
|
return;
|
|
await Task.Delay(20);
|
|
}
|
|
throw new TimeoutException("Condition not reached within " + timeout);
|
|
}
|
|
}
|