namespace ZB.MOM.WW.LocalDb.Tests.Convergence; /// /// Randomized property test: a large stream of mixed insert/update/delete operations on both nodes /// over a small shared key space, interleaved with genuine transport disconnect/reconnect cycles, /// must always converge to byte-identical state. Seeded Random only (no time-based seeds) so every /// failure is reproducible from the logged seed. /// public sealed class RandomOpsConvergenceTests { private const int OpsPerNode = 500; private const int KeySpace = 40; [Theory] [InlineData(1701)] [InlineData(42)] [InlineData(7)] public async Task RandomInterleavedOps_Converge(int seed) { var rng = new Random(seed); // Segment plan derived up front from the seeded RNG (so the whole run is deterministic): // steady, OUTAGE-1, steady, OUTAGE-2, steady — summing to OpsPerNode ops on each node. var seg1 = rng.Next(80, 150); var outage1 = rng.Next(20, 50); var seg2 = rng.Next(80, 150); var outage2 = rng.Next(20, 50); var tail = OpsPerNode - (seg1 + outage1 + seg2 + outage2); await using var fx = new ConvergenceFixture(); await fx.StartAsync(); try { await RunOpsAsync(fx, rng, seg1); await OutageAsync(fx, rng, outage1); await RunOpsAsync(fx, rng, seg2); await OutageAsync(fx, rng, outage2); await RunOpsAsync(fx, rng, tail); await fx.AssertConvergedAsync(); } catch (Exception ex) when (ex is not Xunit.Sdk.XunitException) { throw new Xunit.Sdk.XunitException($"seed {seed}: run faulted.\n{ex}"); } catch (Xunit.Sdk.XunitException ex) { throw new Xunit.Sdk.XunitException($"seed {seed}: convergence failed.\n{ex.Message}"); } // Both outage windows actually severed and re-dialed the transport (initial dial + 2 // reconnects) — proof the run exercised real offline accumulation + resync. Assert.True(fx.InitiatorConnectionAttempts >= 3, $"seed {seed}: expected >=3 connection attempts (2 reconnects), got {fx.InitiatorConnectionAttempts}"); } // Severs the transport, accumulates offline writes on BOTH nodes, waits until the initiator has // actually observed the loss (a reconnect attempt started), then restores the transport. private static async Task OutageAsync(ConvergenceFixture fx, Random rng, int ops) { var baseline = fx.InitiatorConnectionAttempts; await fx.KillTransportAsync(); await RunOpsAsync(fx, rng, ops); await WaitForAsync(() => Task.FromResult(fx.InitiatorConnectionAttempts > baseline), TimeSpan.FromSeconds(10)); await fx.RestartTransportAsync(); } private static async Task RunOpsAsync(ConvergenceFixture fx, Random rng, int count) { // One op per node per iteration; no sleeps — the sessions' pumps drain concurrently. for (var i = 0; i < count; i++) { await ApplyRandomOpAsync(fx.A, rng); await ApplyRandomOpAsync(fx.B, rng); } } private static Task ApplyRandomOpAsync(ILocalDb db, Random rng) { var id = rng.Next(1, KeySpace + 1); return rng.Next(3) switch { // insert / update both resolve through the supported upsert helper (plain UPDATE-else-INSERT). 0 or 1 => ConvergenceFixture.UpsertAsync(db, id, "v" + rng.Next(1000), rng.Next(10_000)), _ => ConvergenceFixture.DeleteAsync(db, id), }; } private static async Task WaitForAsync(Func> predicate, TimeSpan timeout) { var deadline = DateTime.UtcNow + timeout; while (DateTime.UtcNow < deadline) { if (await predicate()) return; await Task.Delay(20); } throw new TimeoutException("Condition not reached within " + timeout); } }