Implement in-process multi-dataset sync isolation across core, network, persistence, and tests

2026-02-22 11:58:34 -05:00
parent c06b56172a
commit 8e97061ab8
60 changed files with 4519 additions and 559 deletions
--- a/tests/ZB.MOM.WW.CDBBC.E2E.Benchmark.Tests/OfflineResyncThroughputBenchmarks.cs
+++ b/tests/ZB.MOM.WW.CDBBC.E2E.Benchmark.Tests/OfflineResyncThroughputBenchmarks.cs
@@ -0,0 +1,142 @@
+using BenchmarkDotNet.Attributes;
+using System.Net;
+using System.Net.Sockets;
+using ZB.MOM.WW.CBDDC.Core.Network;
+using ZB.MOM.WW.CBDDC.Sample.Console;
+
+namespace ZB.MOM.WW.CDBBC.E2E.Benchmark.Tests;
+
+[MemoryDiagnoser]
+[SimpleJob(launchCount: 1, warmupCount: 0, iterationCount: 1)]
+public class OfflineResyncThroughputBenchmarks
+{
+    private const int BacklogOperationCount = 10_000;
+    private BenchmarkPeerNode _onlineNode = null!;
+    private BenchmarkPeerNode _offlineNode = null!;
+    private int _runSequence;
+    private string _currentPrefix = string.Empty;
+
+    [GlobalSetup]
+    public Task GlobalSetupAsync()
+    {
+        return Task.CompletedTask;
+    }
+
+    [GlobalCleanup]
+    public Task GlobalCleanupAsync()
+    {
+        // Avoid explicit node disposal in BenchmarkDotNet child processes due Surreal embedded callback race.
+        // Process teardown releases resources after benchmark completion.
+        return Task.CompletedTask;
+    }
+
+    [IterationSetup(Target = nameof(OfflineBacklogWriteThroughput100k))]
+    public void SetupOfflineWriteThroughput()
+    {
+        _currentPrefix = $"offline-write-{Interlocked.Increment(ref _runSequence):D6}";
+        InitializeIterationNodesAsync().GetAwaiter().GetResult();
+    }
+
+    [Benchmark(Description = "Offline backlog write throughput (10K ops)", OperationsPerInvoke = BacklogOperationCount)]
+    public async Task OfflineBacklogWriteThroughput100k()
+    {
+        await WriteBatchAsync(_currentPrefix, BacklogOperationCount);
+    }
+
+    [IterationSetup(Target = nameof(OfflineNodeResyncDurationAfter100kBacklog))]
+    public void SetupOfflineResyncBenchmark()
+    {
+        _currentPrefix = $"offline-resync-{Interlocked.Increment(ref _runSequence):D6}";
+        InitializeIterationNodesAsync().GetAwaiter().GetResult();
+        WriteBatchAsync(_currentPrefix, BacklogOperationCount).GetAwaiter().GetResult();
+    }
+
+    [Benchmark(Description = "Offline node re-sync duration after 10K backlog")]
+    public async Task OfflineNodeResyncDurationAfter100kBacklog()
+    {
+        await _offlineNode.StartAsync();
+        await WaitForReplicationAsync(_currentPrefix, BacklogOperationCount, TimeSpan.FromMinutes(3));
+    }
+
+    private async Task WriteBatchAsync(string prefix, int count)
+    {
+        for (var i = 0; i < count; i++)
+        {
+            string userId = $"{prefix}-{i:D6}";
+            await _onlineNode.UpsertUserAsync(CreateUser(userId));
+        }
+    }
+
+    private async Task WaitForReplicationAsync(string prefix, int expectedCount, TimeSpan timeout)
+    {
+        DateTime deadline = DateTime.UtcNow.Add(timeout);
+        while (DateTime.UtcNow < deadline)
+        {
+            if (_offlineNode.CountUsersWithPrefix(prefix) >= expectedCount)
+                return;
+
+            await Task.Delay(250);
+        }
+
+        int replicatedCount = _offlineNode.CountUsersWithPrefix(prefix);
+        throw new TimeoutException(
+            $"Timed out waiting for re-sync. Expected {expectedCount}, replicated {replicatedCount}.");
+    }
+
+    private static User CreateUser(string userId)
+    {
+        return new User
+        {
+            Id = userId,
+            Name = $"user-{userId}",
+            Age = 30,
+            Address = new Address { City = "OfflineBenchmarkCity" }
+        };
+    }
+
+    private static int GetAvailableTcpPort()
+    {
+        using var listener = new TcpListener(IPAddress.Loopback, 0);
+        listener.Start();
+        return ((IPEndPoint)listener.LocalEndpoint).Port;
+    }
+
+    private async Task InitializeIterationNodesAsync()
+    {
+        int onlinePort = GetAvailableTcpPort();
+        int offlinePort = GetAvailableTcpPort();
+        while (offlinePort == onlinePort)
+            offlinePort = GetAvailableTcpPort();
+
+        string clusterToken = Guid.NewGuid().ToString("N");
+
+        _onlineNode = BenchmarkPeerNode.Create(
+            "offline-benchmark-online",
+            onlinePort,
+            clusterToken,
+            [
+                new KnownPeerConfiguration
+                {
+                    NodeId = "offline-benchmark-offline",
+                    Host = "127.0.0.1",
+                    Port = offlinePort
+                }
+            ]);
+
+        _offlineNode = BenchmarkPeerNode.Create(
+            "offline-benchmark-offline",
+            offlinePort,
+            clusterToken,
+            [
+                new KnownPeerConfiguration
+                {
+                    NodeId = "offline-benchmark-online",
+                    Host = "127.0.0.1",
+                    Port = onlinePort
+                }
+            ]);
+
+        await _onlineNode.StartAsync();
+        await Task.Delay(250);
+    }
+}