test(localdb): bidirectional convergence + randomized property tests

Claude-Session: https://claude.ai/code/session_01BL2Vu1ESDQ9SCN4gVKkdts
2026-07-18 00:51:16 -04:00
parent eb40cd8f29
commit 801b042208
3 changed files with 630 additions and 0 deletions
@@ -0,0 +1,102 @@
+namespace ZB.MOM.WW.LocalDb.Tests.Convergence;
+
+/// <summary>
+/// Randomized property test: a large stream of mixed insert/update/delete operations on both nodes
+/// over a small shared key space, interleaved with genuine transport disconnect/reconnect cycles,
+/// must always converge to byte-identical state. Seeded Random only (no time-based seeds) so every
+/// failure is reproducible from the logged seed.
+/// </summary>
+public sealed class RandomOpsConvergenceTests
+{
+    private const int OpsPerNode = 500;
+    private const int KeySpace = 40;
+
+    [Theory]
+    [InlineData(1701)]
+    [InlineData(42)]
+    [InlineData(7)]
+    public async Task RandomInterleavedOps_Converge(int seed)
+    {
+        var rng = new Random(seed);
+
+        // Segment plan derived up front from the seeded RNG (so the whole run is deterministic):
+        // steady, OUTAGE-1, steady, OUTAGE-2, steady — summing to OpsPerNode ops on each node.
+        var seg1 = rng.Next(80, 150);
+        var outage1 = rng.Next(20, 50);
+        var seg2 = rng.Next(80, 150);
+        var outage2 = rng.Next(20, 50);
+        var tail = OpsPerNode - (seg1 + outage1 + seg2 + outage2);
+
+        await using var fx = new ConvergenceFixture();
+        await fx.StartAsync();
+
+        try
+        {
+            await RunOpsAsync(fx, rng, seg1);
+            await OutageAsync(fx, rng, outage1);
+            await RunOpsAsync(fx, rng, seg2);
+            await OutageAsync(fx, rng, outage2);
+            await RunOpsAsync(fx, rng, tail);
+
+            await fx.AssertConvergedAsync();
+        }
+        catch (Exception ex) when (ex is not Xunit.Sdk.XunitException)
+        {
+            throw new Xunit.Sdk.XunitException($"seed {seed}: run faulted.\n{ex}");
+        }
+        catch (Xunit.Sdk.XunitException ex)
+        {
+            throw new Xunit.Sdk.XunitException($"seed {seed}: convergence failed.\n{ex.Message}");
+        }
+
+        // Both outage windows actually severed and re-dialed the transport (initial dial + 2
+        // reconnects) — proof the run exercised real offline accumulation + resync.
+        Assert.True(fx.InitiatorConnectionAttempts >= 3,
+            $"seed {seed}: expected >=3 connection attempts (2 reconnects), got {fx.InitiatorConnectionAttempts}");
+    }
+
+    // Severs the transport, accumulates offline writes on BOTH nodes, waits until the initiator has
+    // actually observed the loss (a reconnect attempt started), then restores the transport.
+    private static async Task OutageAsync(ConvergenceFixture fx, Random rng, int ops)
+    {
+        var baseline = fx.InitiatorConnectionAttempts;
+        await fx.KillTransportAsync();
+        await RunOpsAsync(fx, rng, ops);
+        await WaitForAsync(() => Task.FromResult(fx.InitiatorConnectionAttempts > baseline),
+            TimeSpan.FromSeconds(10));
+        await fx.RestartTransportAsync();
+    }
+
+    private static async Task RunOpsAsync(ConvergenceFixture fx, Random rng, int count)
+    {
+        // One op per node per iteration; no sleeps — the sessions' pumps drain concurrently.
+        for (var i = 0; i < count; i++)
+        {
+            await ApplyRandomOpAsync(fx.A, rng);
+            await ApplyRandomOpAsync(fx.B, rng);
+        }
+    }
+
+    private static Task ApplyRandomOpAsync(ILocalDb db, Random rng)
+    {
+        var id = rng.Next(1, KeySpace + 1);
+        return rng.Next(3) switch
+        {
+            // insert / update both resolve through the supported upsert helper (plain UPDATE-else-INSERT).
+            0 or 1 => ConvergenceFixture.UpsertAsync(db, id, "v" + rng.Next(1000), rng.Next(10_000)),
+            _ => ConvergenceFixture.DeleteAsync(db, id),
+        };
+    }
+
+    private static async Task WaitForAsync(Func<Task<bool>> predicate, TimeSpan timeout)
+    {
+        var deadline = DateTime.UtcNow + timeout;
+        while (DateTime.UtcNow < deadline)
+        {
+            if (await predicate())
+                return;
+            await Task.Delay(20);
+        }
+        throw new TimeoutException("Condition not reached within " + timeout);
+    }
+}