Implement in-process multi-dataset sync isolation across core, network, persistence, and tests

tests/ZB.MOM.WW.CBDDC.Network.Tests/MultiDatasetRegistrationTests.cs

@@ -0,0 +1,47 @@
+using Microsoft.Extensions.DependencyInjection;
+using ZB.MOM.WW.CBDDC.Core.Network;
+using ZB.MOM.WW.CBDDC.Core.Storage;
+
+namespace ZB.MOM.WW.CBDDC.Network.Tests;
+
+public class MultiDatasetRegistrationTests
+{
+    [Fact]
+    public void AddCBDDCMultiDataset_ShouldRegisterCoordinatorAndReplaceSyncOrchestrator()
+    {
+        var services = new ServiceCollection();
+
+        services.AddCBDDCNetwork<TestPeerNodeConfigurationProvider>(useHostedService: false);
+        services.AddCBDDCMultiDataset(options =>
+        {
+            options.EnableMultiDatasetSync = true;
+            options.EnableDatasetPrimary = true;
+            options.EnableDatasetLogs = true;
+            options.EnableDatasetTimeseries = true;
+        });
+
+        services.Any(descriptor => descriptor.ServiceType == typeof(IMultiDatasetSyncOrchestrator)).ShouldBeTrue();
+
+        var syncDescriptor = services.Last(descriptor => descriptor.ServiceType == typeof(ISyncOrchestrator));
+        syncDescriptor.ImplementationFactory.ShouldNotBeNull();
+    }
+
+    private sealed class TestPeerNodeConfigurationProvider : IPeerNodeConfigurationProvider
+    {
+        public event PeerNodeConfigurationChangedEventHandler? ConfigurationChanged
+        {
+            add { }
+            remove { }
+        }
+
+        public Task<PeerNodeConfiguration> GetConfiguration()
+        {
+            return Task.FromResult(new PeerNodeConfiguration
+            {
+                NodeId = "node-test",
+                TcpPort = 9000,
+                AuthToken = "auth"
+            });
+        }
+    }
+}

tests/ZB.MOM.WW.CBDDC.Network.Tests/MultiDatasetSyncOrchestratorTests.cs

@@ -0,0 +1,102 @@
+using Microsoft.Extensions.Logging.Abstractions;
+using ZB.MOM.WW.CBDDC.Core;
+using ZB.MOM.WW.CBDDC.Core.Network;
+using ZB.MOM.WW.CBDDC.Core.Storage;
+
+namespace ZB.MOM.WW.CBDDC.Network.Tests;
+
+public class MultiDatasetSyncOrchestratorTests
+{
+    [Fact]
+    public void Constructor_WhenMultiDatasetDisabled_ShouldOnlyCreatePrimaryContext()
+    {
+        var sut = CreateSut(
+            [
+                new DatasetSyncOptions { DatasetId = DatasetId.Primary, Enabled = true },
+                new DatasetSyncOptions { DatasetId = DatasetId.Logs, Enabled = true }
+            ],
+            new MultiDatasetRuntimeOptions
+            {
+                EnableMultiDatasetSync = false,
+                EnableDatasetPrimary = true,
+                EnableDatasetLogs = true
+            });
+
+        var datasetIds = sut.Contexts.Select(c => c.DatasetId).ToList();
+        datasetIds.Count.ShouldBe(1);
+        datasetIds[0].ShouldBe(DatasetId.Primary);
+    }
+
+    [Fact]
+    public async Task StartStop_WhenOneDatasetThrows_ShouldContinueOtherDatasets()
+    {
+        var orchestrators = new Dictionary<string, TrackingSyncOrchestrator>(StringComparer.Ordinal)
+        {
+            [DatasetId.Primary] = new TrackingSyncOrchestrator(),
+            [DatasetId.Logs] = new TrackingSyncOrchestrator(startException: new InvalidOperationException("boom")),
+            [DatasetId.Timeseries] = new TrackingSyncOrchestrator()
+        };
+
+        var sut = CreateSut(
+            [],
+            new MultiDatasetRuntimeOptions
+            {
+                EnableMultiDatasetSync = true,
+                EnableDatasetPrimary = true,
+                EnableDatasetLogs = true,
+                EnableDatasetTimeseries = true
+            },
+            options => orchestrators[DatasetId.Normalize(options.DatasetId)]);
+
+        await sut.Start();
+        await sut.Stop();
+
+        orchestrators[DatasetId.Primary].StartCalls.ShouldBe(1);
+        orchestrators[DatasetId.Primary].StopCalls.ShouldBe(1);
+
+        orchestrators[DatasetId.Logs].StartCalls.ShouldBe(1);
+        orchestrators[DatasetId.Logs].StopCalls.ShouldBe(1);
+
+        orchestrators[DatasetId.Timeseries].StartCalls.ShouldBe(1);
+        orchestrators[DatasetId.Timeseries].StopCalls.ShouldBe(1);
+    }
+
+    private static MultiDatasetSyncOrchestrator CreateSut(
+        IEnumerable<DatasetSyncOptions> datasetOptions,
+        MultiDatasetRuntimeOptions runtimeOptions,
+        Func<DatasetSyncOptions, ISyncOrchestrator>? orchestratorFactory = null)
+    {
+        return new MultiDatasetSyncOrchestrator(
+            Substitute.For<IDiscoveryService>(),
+            Substitute.For<IOplogStore>(),
+            Substitute.For<IDocumentStore>(),
+            Substitute.For<ISnapshotMetadataStore>(),
+            Substitute.For<ISnapshotService>(),
+            Substitute.For<IPeerNodeConfigurationProvider>(),
+            NullLoggerFactory.Instance,
+            datasetOptions,
+            runtimeOptions,
+            orchestratorFactory: orchestratorFactory);
+    }
+
+    private sealed class TrackingSyncOrchestrator(Exception? startException = null, Exception? stopException = null)
+        : ISyncOrchestrator
+    {
+        public int StartCalls { get; private set; }
+        public int StopCalls { get; private set; }
+
+        public Task Start()
+        {
+            StartCalls++;
+            if (startException != null) throw startException;
+            return Task.CompletedTask;
+        }
+
+        public Task Stop()
+        {
+            StopCalls++;
+            if (stopException != null) throw stopException;
+            return Task.CompletedTask;
+        }
+    }
+}

tests/ZB.MOM.WW.CBDDC.Network.Tests/ProtocolTests.cs

@@ -1,4 +1,6 @@
 using Microsoft.Extensions.Logging.Abstractions;
+using Google.Protobuf;
+using ZB.MOM.WW.CBDDC.Core;
 using ZB.MOM.WW.CBDDC.Network.Proto;
 using ZB.MOM.WW.CBDDC.Network.Protocol;
 using ZB.MOM.WW.CBDDC.Network.Security;
@@ -145,6 +147,44 @@ public class ProtocolTests
         decoded.NodeId.ShouldBe("fragmented");
     }
 
+    /// <summary>
+    ///     Verifies that dataset-aware protocol fields are serialized and parsed correctly.
+    /// </summary>
+    [Fact]
+    public void DatasetAwareMessages_ShouldRoundTripDatasetFields()
+    {
+        var request = new PullChangesRequest
+        {
+            SinceWall = 10,
+            SinceLogic = 1,
+            SinceNode = "node-a",
+            DatasetId = "logs"
+        };
+
+        byte[] payload = request.ToByteArray();
+        var decoded = PullChangesRequest.Parser.ParseFrom(payload);
+
+        decoded.DatasetId.ShouldBe("logs");
+    }
+
+    /// <summary>
+    ///     Verifies that legacy messages with no dataset id default to the primary dataset.
+    /// </summary>
+    [Fact]
+    public void DatasetAwareMessages_WhenMissingDataset_ShouldDefaultToPrimary()
+    {
+        var legacy = new HandshakeRequest
+        {
+            NodeId = "node-legacy",
+            AuthToken = "token"
+        };
+
+        byte[] payload = legacy.ToByteArray();
+        var decoded = HandshakeRequest.Parser.ParseFrom(payload);
+
+        DatasetId.Normalize(decoded.DatasetId).ShouldBe(DatasetId.Primary);
+    }
+
     // Helper Stream for fragmentation test
     private class FragmentedMemoryStream : MemoryStream
     {
@@ -169,4 +209,4 @@ public class ProtocolTests
             return await base.ReadAsync(buffer, offset, toRead, cancellationToken);
         }
     }
-}
+}

tests/ZB.MOM.WW.CBDDC.Network.Tests/SnapshotReconnectRegressionTests.cs

@@ -16,6 +16,8 @@ public class SnapshotReconnectRegressionTests
             .Returns((SnapshotMetadata?)null);
         snapshotMetadataStore.GetSnapshotHashAsync(Arg.Any<string>(), Arg.Any<CancellationToken>())
             .Returns((string?)null);
+        snapshotMetadataStore.GetSnapshotHashAsync(Arg.Any<string>(), Arg.Any<string>(), Arg.Any<CancellationToken>())
+            .Returns((string?)null);
         snapshotMetadataStore.GetAllSnapshotMetadataAsync(Arg.Any<CancellationToken>())
             .Returns(Array.Empty<SnapshotMetadata>());
         return snapshotMetadataStore;
@@ -30,6 +32,10 @@ public class SnapshotReconnectRegressionTests
             .Returns(Task.CompletedTask);
         snapshotService.MergeSnapshotAsync(Arg.Any<Stream>(), Arg.Any<CancellationToken>())
             .Returns(Task.CompletedTask);
+        snapshotService.ReplaceDatabaseAsync(Arg.Any<Stream>(), Arg.Any<string>(), Arg.Any<CancellationToken>())
+            .Returns(Task.CompletedTask);
+        snapshotService.MergeSnapshotAsync(Arg.Any<Stream>(), Arg.Any<string>(), Arg.Any<CancellationToken>())
+            .Returns(Task.CompletedTask);
         return snapshotService;
     }
 
@@ -69,8 +75,12 @@ public class SnapshotReconnectRegressionTests
         var oplogStore = Substitute.For<IOplogStore>();
         oplogStore.GetLastEntryHashAsync(Arg.Any<string>(), Arg.Any<CancellationToken>())
             .Returns(localHeadHash);
+        oplogStore.GetLastEntryHashAsync(Arg.Any<string>(), Arg.Any<string>(), Arg.Any<CancellationToken>())
+            .Returns(localHeadHash);
         oplogStore.ApplyBatchAsync(Arg.Any<IEnumerable<OplogEntry>>(), Arg.Any<CancellationToken>())
             .Returns(Task.CompletedTask);
+        oplogStore.ApplyBatchAsync(Arg.Any<IEnumerable<OplogEntry>>(), Arg.Any<string>(), Arg.Any<CancellationToken>())
+            .Returns(Task.CompletedTask);
         return oplogStore;
     }
 
@@ -84,6 +94,8 @@ public class SnapshotReconnectRegressionTests
             null);
         client.GetChainRangeAsync(Arg.Any<string>(), Arg.Any<string>(), Arg.Any<CancellationToken>())
             .Returns(_ => Task.FromException<List<OplogEntry>>(new SnapshotRequiredException()));
+        client.GetChainRangeAsync(Arg.Any<string>(), Arg.Any<string>(), Arg.Any<string>(), Arg.Any<CancellationToken>())
+            .Returns(_ => Task.FromException<List<OplogEntry>>(new SnapshotRequiredException()));
         return client;
     }
 
@@ -109,19 +121,38 @@ public class SnapshotReconnectRegressionTests
         store.EnsurePeerRegisteredAsync(Arg.Any<string>(), Arg.Any<string>(), Arg.Any<PeerType>(),
                 Arg.Any<CancellationToken>())
             .Returns(Task.CompletedTask);
+        store.EnsurePeerRegisteredAsync(Arg.Any<string>(), Arg.Any<string>(), Arg.Any<PeerType>(), Arg.Any<string>(),
+                Arg.Any<CancellationToken>())
+            .Returns(Task.CompletedTask);
         store.UpdateConfirmationAsync(Arg.Any<string>(), Arg.Any<string>(), Arg.Any<HlcTimestamp>(), Arg.Any<string>(),
                 Arg.Any<CancellationToken>())
             .Returns(Task.CompletedTask);
+        store.UpdateConfirmationAsync(Arg.Any<string>(), Arg.Any<string>(), Arg.Any<HlcTimestamp>(), Arg.Any<string>(),
+                Arg.Any<string>(), Arg.Any<CancellationToken>())
+            .Returns(Task.CompletedTask);
         store.GetConfirmationsAsync(Arg.Any<CancellationToken>()).Returns(Array.Empty<PeerOplogConfirmation>());
+        store.GetConfirmationsAsync(Arg.Any<string>(), Arg.Any<CancellationToken>())
+            .Returns(Array.Empty<PeerOplogConfirmation>());
         store.GetConfirmationsForPeerAsync(Arg.Any<string>(), Arg.Any<CancellationToken>())
             .Returns(Array.Empty<PeerOplogConfirmation>());
+        store.GetConfirmationsForPeerAsync(Arg.Any<string>(), Arg.Any<string>(), Arg.Any<CancellationToken>())
+            .Returns(Array.Empty<PeerOplogConfirmation>());
         store.RemovePeerTrackingAsync(Arg.Any<string>(), Arg.Any<CancellationToken>()).Returns(Task.CompletedTask);
+        store.RemovePeerTrackingAsync(Arg.Any<string>(), Arg.Any<string>(), Arg.Any<CancellationToken>())
+            .Returns(Task.CompletedTask);
         store.GetActiveTrackedPeersAsync(Arg.Any<CancellationToken>()).Returns(Array.Empty<string>());
+        store.GetActiveTrackedPeersAsync(Arg.Any<string>(), Arg.Any<CancellationToken>()).Returns(Array.Empty<string>());
         store.ExportAsync(Arg.Any<CancellationToken>()).Returns(Array.Empty<PeerOplogConfirmation>());
+        store.ExportAsync(Arg.Any<string>(), Arg.Any<CancellationToken>())
+            .Returns(Array.Empty<PeerOplogConfirmation>());
         store.ImportAsync(Arg.Any<IEnumerable<PeerOplogConfirmation>>(), Arg.Any<CancellationToken>())
             .Returns(Task.CompletedTask);
+        store.ImportAsync(Arg.Any<IEnumerable<PeerOplogConfirmation>>(), Arg.Any<string>(), Arg.Any<CancellationToken>())
+            .Returns(Task.CompletedTask);
         store.MergeAsync(Arg.Any<IEnumerable<PeerOplogConfirmation>>(), Arg.Any<CancellationToken>())
             .Returns(Task.CompletedTask);
+        store.MergeAsync(Arg.Any<IEnumerable<PeerOplogConfirmation>>(), Arg.Any<string>(), Arg.Any<CancellationToken>())
+            .Returns(Task.CompletedTask);
         return store;
     }
 
@@ -136,6 +167,8 @@ public class SnapshotReconnectRegressionTests
         var snapshotMetadataStore = CreateSnapshotMetadataStore();
         snapshotMetadataStore.GetSnapshotHashAsync(Arg.Any<string>(), Arg.Any<CancellationToken>())
             .Returns("snapshot-boundary-hash");
+        snapshotMetadataStore.GetSnapshotHashAsync(Arg.Any<string>(), Arg.Any<string>(), Arg.Any<CancellationToken>())
+            .Returns("snapshot-boundary-hash");
         var snapshotService = CreateSnapshotService();
 
         var orch = new TestableSyncOrchestrator(
@@ -165,7 +198,7 @@ public class SnapshotReconnectRegressionTests
         // Assert
         result.ShouldBe("Success");
         await client.DidNotReceive()
-            .GetChainRangeAsync(Arg.Any<string>(), Arg.Any<string>(), Arg.Any<CancellationToken>());
+            .GetChainRangeAsync(Arg.Any<string>(), Arg.Any<string>(), Arg.Any<string>(), Arg.Any<CancellationToken>());
     }
 
     /// <summary>
@@ -179,6 +212,8 @@ public class SnapshotReconnectRegressionTests
         var snapshotMetadataStore = CreateSnapshotMetadataStore();
         snapshotMetadataStore.GetSnapshotHashAsync(Arg.Any<string>(), Arg.Any<CancellationToken>())
             .Returns("snapshot-boundary-hash");
+        snapshotMetadataStore.GetSnapshotHashAsync(Arg.Any<string>(), Arg.Any<string>(), Arg.Any<CancellationToken>())
+            .Returns("snapshot-boundary-hash");
         var snapshotService = CreateSnapshotService();
 
         var orch = new TestableSyncOrchestrator(
@@ -208,7 +243,11 @@ public class SnapshotReconnectRegressionTests
         await Should.ThrowAsync<SnapshotRequiredException>(async () =>
             await orch.TestProcessInboundBatchAsync(client, "remote-node", entries, CancellationToken.None));
 
-        await client.Received(1).GetChainRangeAsync(Arg.Any<string>(), Arg.Any<string>(), Arg.Any<CancellationToken>());
+        await client.Received(1).GetChainRangeAsync(
+            Arg.Any<string>(),
+            Arg.Any<string>(),
+            Arg.Any<string>(),
+            Arg.Any<CancellationToken>());
     }
 
     // Subclass to expose private method
@@ -283,4 +322,4 @@ public class SnapshotReconnectRegressionTests
             }
         }
     }
-}
+}

tests/ZB.MOM.WW.CBDDC.Network.Tests/SyncOrchestratorConfirmationTests.cs

@@ -39,18 +39,21 @@ public class SyncOrchestratorConfirmationTests
             "peer-a",
             "10.0.0.1:9000",
             PeerType.LanDiscovered,
+            DatasetId.Primary,
             Arg.Any<CancellationToken>());
 
         await confirmationStore.Received(1).EnsurePeerRegisteredAsync(
             "peer-b",
             "10.0.0.2:9010",
             PeerType.StaticRemote,
+            DatasetId.Primary,
             Arg.Any<CancellationToken>());
 
         await confirmationStore.DidNotReceive().EnsurePeerRegisteredAsync(
             "local",
             Arg.Any<string>(),
             Arg.Any<PeerType>(),
+            Arg.Any<string>(),
             Arg.Any<CancellationToken>());
     }
 
@@ -89,6 +92,7 @@ public class SyncOrchestratorConfirmationTests
             "peer-new",
             "10.0.0.25:9010",
             PeerType.LanDiscovered,
+            DatasetId.Primary,
             Arg.Any<CancellationToken>());
     }
 
@@ -114,9 +118,9 @@ public class SyncOrchestratorConfirmationTests
         remote.SetTimestamp("node-behind", new HlcTimestamp(299, 9, "node-behind"));
         remote.SetTimestamp("node-remote-only", new HlcTimestamp(900, 0, "node-remote-only"));
 
-        oplogStore.GetLastEntryHashAsync("node-equal", Arg.Any<CancellationToken>())
+        oplogStore.GetLastEntryHashAsync("node-equal", DatasetId.Primary, Arg.Any<CancellationToken>())
             .Returns("hash-equal");
-        oplogStore.GetLastEntryHashAsync("node-ahead", Arg.Any<CancellationToken>())
+        oplogStore.GetLastEntryHashAsync("node-ahead", DatasetId.Primary, Arg.Any<CancellationToken>())
             .Returns((string?)null);
 
         await orchestrator.AdvanceConfirmationsFromVectorClockAsync("peer-1", local, remote, CancellationToken.None);
@@ -126,6 +130,7 @@ public class SyncOrchestratorConfirmationTests
             "node-equal",
             new HlcTimestamp(100, 1, "node-equal"),
             "hash-equal",
+            DatasetId.Primary,
             Arg.Any<CancellationToken>());
 
         await confirmationStore.Received(1).UpdateConfirmationAsync(
@@ -133,6 +138,7 @@ public class SyncOrchestratorConfirmationTests
             "node-ahead",
             new HlcTimestamp(200, 0, "node-ahead"),
             string.Empty,
+            DatasetId.Primary,
             Arg.Any<CancellationToken>());
 
         await confirmationStore.DidNotReceive().UpdateConfirmationAsync(
@@ -140,6 +146,7 @@ public class SyncOrchestratorConfirmationTests
             "node-behind",
             Arg.Any<HlcTimestamp>(),
             Arg.Any<string>(),
+            Arg.Any<string>(),
             Arg.Any<CancellationToken>());
 
         await confirmationStore.DidNotReceive().UpdateConfirmationAsync(
@@ -147,6 +154,7 @@ public class SyncOrchestratorConfirmationTests
             "node-local-only",
             Arg.Any<HlcTimestamp>(),
             Arg.Any<string>(),
+            Arg.Any<string>(),
             Arg.Any<CancellationToken>());
 
         await confirmationStore.DidNotReceive().UpdateConfirmationAsync(
@@ -154,6 +162,7 @@ public class SyncOrchestratorConfirmationTests
             "node-remote-only",
             Arg.Any<HlcTimestamp>(),
             Arg.Any<string>(),
+            Arg.Any<string>(),
             Arg.Any<CancellationToken>());
     }
 
@@ -182,6 +191,7 @@ public class SyncOrchestratorConfirmationTests
             "source-1",
             new HlcTimestamp(120, 1, "source-1"),
             "hash-120",
+            DatasetId.Primary,
             Arg.Any<CancellationToken>());
     }
 
@@ -206,6 +216,7 @@ public class SyncOrchestratorConfirmationTests
             Arg.Any<string>(),
             Arg.Any<HlcTimestamp>(),
             Arg.Any<string>(),
+            Arg.Any<string>(),
             Arg.Any<CancellationToken>());
     }
 
@@ -245,4 +256,4 @@ public class SyncOrchestratorConfirmationTests
             string.Empty,
             hash);
     }
-}
+}

tests/ZB.MOM.WW.CBDDC.Network.Tests/SyncOrchestratorMaintenancePruningTests.cs

@@ -138,7 +138,10 @@ public class SyncOrchestratorMaintenancePruningTests
 
         await orchestrator.RunMaintenanceIfDueAsync(config, DateTime.UtcNow, CancellationToken.None);
 
-        await oplogStore.DidNotReceive().PruneOplogAsync(Arg.Any<HlcTimestamp>(), Arg.Any<CancellationToken>());
+        await oplogStore.DidNotReceive().PruneOplogAsync(
+            Arg.Any<HlcTimestamp>(),
+            Arg.Any<string>(),
+            Arg.Any<CancellationToken>());
     }
 
     /// <summary>
@@ -187,6 +190,7 @@ public class SyncOrchestratorMaintenancePruningTests
                 timestamp.PhysicalTime == 100 &&
                 timestamp.LogicalCounter == 0 &&
                 string.Equals(timestamp.NodeId, "node-local", StringComparison.Ordinal)),
+            DatasetId.Primary,
             Arg.Any<CancellationToken>());
     }
 
@@ -228,7 +232,10 @@ public class SyncOrchestratorMaintenancePruningTests
 
         var now = DateTime.UtcNow;
         await orchestrator.RunMaintenanceIfDueAsync(config, now, CancellationToken.None);
-        await oplogStore.DidNotReceive().PruneOplogAsync(Arg.Any<HlcTimestamp>(), Arg.Any<CancellationToken>());
+        await oplogStore.DidNotReceive().PruneOplogAsync(
+            Arg.Any<HlcTimestamp>(),
+            Arg.Any<string>(),
+            Arg.Any<CancellationToken>());
 
         await orchestrator.RunMaintenanceIfDueAsync(config, now.AddMinutes(2), CancellationToken.None);
 
@@ -237,6 +244,7 @@ public class SyncOrchestratorMaintenancePruningTests
                 timestamp.PhysicalTime == 100 &&
                 timestamp.LogicalCounter == 0 &&
                 string.Equals(timestamp.NodeId, "node-local", StringComparison.Ordinal)),
+            DatasetId.Primary,
             Arg.Any<CancellationToken>());
     }
 
@@ -286,4 +294,4 @@ public class SyncOrchestratorMaintenancePruningTests
             IsActive = isActive
         };
     }
-}
+}