From 8e97061ab839a722a71d959afb29471e038bafe1 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Sun, 22 Feb 2026 11:58:34 -0500 Subject: [PATCH] Implement in-process multi-dataset sync isolation across core, network, persistence, and tests --- docs/architecture.md | 7 + docs/features/README.md | 1 + docs/features/multi-dataset-sync.md | 67 +++ docs/persistence-providers.md | 8 + docs/runbook.md | 9 + .../ConsoleInteractiveService.cs | 50 +- .../ZB.MOM.WW.CBDDC.Sample.Console/Program.cs | 36 +- .../SampleDbContext.cs | 20 + .../SampleDocumentStore.cs | 38 ++ .../TelemetryData.cs | 57 +++ .../appsettings.json | 22 +- separate.md | 323 +++++++++++++ src/ZB.MOM.WW.CBDDC.Core/DatasetId.cs | 34 ++ .../DatasetSyncOptions.cs | 45 ++ src/ZB.MOM.WW.CBDDC.Core/OplogEntry.cs | 57 ++- .../PeerOplogConfirmation.cs | 7 +- src/ZB.MOM.WW.CBDDC.Core/SnapshotMetadata.cs | 7 +- .../Storage/IDatasetSyncContext.cs | 42 ++ .../Storage/IDocumentMetadataStore.cs | 151 +++++- .../Storage/IMultiDatasetSyncOrchestrator.cs | 25 + .../Storage/IOplogStore.cs | 175 ++++++- .../Storage/IPeerOplogConfirmationStore.cs | 92 +++- .../Storage/ISnapshotMetadataStore.cs | 73 ++- .../Storage/ISnapshotService.cs | 50 +- .../Storage/ISnapshotable.cs | 70 ++- .../DatasetSyncContext.cs | 67 +++ .../MultiDatasetRuntimeOptions.cs | 34 ++ .../MultiDatasetSyncOrchestrator.cs | 199 ++++++++ .../MultiDatasetSyncOrchestratorAdapter.cs | 32 ++ .../PeerDbNetworkExtensions.cs | 54 ++- .../SyncOrchestrator.cs | 214 +++++---- src/ZB.MOM.WW.CBDDC.Network/TcpPeerClient.cs | 288 ++++++++---- src/ZB.MOM.WW.CBDDC.Network/TcpSyncServer.cs | 173 ++++--- src/ZB.MOM.WW.CBDDC.Network/sync.proto | 82 ++-- .../Snapshot/SnapshotDto.cs | 27 +- .../SnapshotStore.cs | 86 +++- .../Surreal/CBDDCSurrealEmbeddedExtensions.cs | 57 +++ .../Surreal/CBDDCSurrealSchemaInitializer.cs | 26 +- .../ISurrealCdcCheckpointPersistence.cs | 56 +++ .../SurrealCdcCheckpointPersistence.cs | 67 ++- .../Surreal/SurrealDocumentMetadataStore.cs | 182 +++++++- .../Surreal/SurrealDocumentStore.cs | 17 +- .../Surreal/SurrealOplogStore.cs | 321 ++++++++++++- .../SurrealPeerOplogConfirmationStore.cs | 202 +++++++- .../Surreal/SurrealSnapshotMetadataStore.cs | 164 ++++++- .../Surreal/SurrealStoreRecords.cs | 62 ++- .../DatasetAwareModelTests.cs | 43 ++ .../OplogEntryTests.cs | 40 +- .../MultiDatasetRegistrationTests.cs | 47 ++ .../MultiDatasetSyncOrchestratorTests.cs | 102 ++++ .../ProtocolTests.cs | 42 +- .../SnapshotReconnectRegressionTests.cs | 45 +- .../SyncOrchestratorConfirmationTests.cs | 17 +- ...SyncOrchestratorMaintenancePruningTests.cs | 14 +- .../MultiDatasetConfigParsingTests.cs | 38 ++ .../SurrealStoreContractTests.cs | 170 +++++++ .../BenchmarkPeerNode.cs | 12 +- .../OfflineResyncThroughputBenchmarks.cs | 142 ++++++ .../SerilogLogEntry.cs | 50 ++ .../SurrealLogStorageBenchmarks.cs | 440 ++++++++++++++++++ 60 files changed, 4519 insertions(+), 559 deletions(-) create mode 100644 docs/features/multi-dataset-sync.md create mode 100644 samples/ZB.MOM.WW.CBDDC.Sample.Console/TelemetryData.cs create mode 100644 separate.md create mode 100644 src/ZB.MOM.WW.CBDDC.Core/DatasetId.cs create mode 100644 src/ZB.MOM.WW.CBDDC.Core/DatasetSyncOptions.cs create mode 100644 src/ZB.MOM.WW.CBDDC.Core/Storage/IDatasetSyncContext.cs create mode 100644 src/ZB.MOM.WW.CBDDC.Core/Storage/IMultiDatasetSyncOrchestrator.cs create mode 100644 src/ZB.MOM.WW.CBDDC.Network/DatasetSyncContext.cs create mode 100644 src/ZB.MOM.WW.CBDDC.Network/MultiDatasetRuntimeOptions.cs create mode 100644 src/ZB.MOM.WW.CBDDC.Network/MultiDatasetSyncOrchestrator.cs create mode 100644 src/ZB.MOM.WW.CBDDC.Network/MultiDatasetSyncOrchestratorAdapter.cs create mode 100644 tests/ZB.MOM.WW.CBDDC.Core.Tests/DatasetAwareModelTests.cs create mode 100644 tests/ZB.MOM.WW.CBDDC.Network.Tests/MultiDatasetRegistrationTests.cs create mode 100644 tests/ZB.MOM.WW.CBDDC.Network.Tests/MultiDatasetSyncOrchestratorTests.cs create mode 100644 tests/ZB.MOM.WW.CBDDC.Sample.Console.Tests/MultiDatasetConfigParsingTests.cs create mode 100644 tests/ZB.MOM.WW.CDBBC.E2E.Benchmark.Tests/OfflineResyncThroughputBenchmarks.cs create mode 100644 tests/ZB.MOM.WW.CDBBC.E2E.Benchmark.Tests/SerilogLogEntry.cs create mode 100644 tests/ZB.MOM.WW.CDBBC.E2E.Benchmark.Tests/SurrealLogStorageBenchmarks.cs diff --git a/docs/architecture.md b/docs/architecture.md index 17e0279..d9d983e 100755 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -30,6 +30,13 @@ To optimize reconnection, each node maintains a **Snapshot** of the last known s - If the chain hash matches, they only exchange the delta. - This avoids re-processing the entire operation history and ensures efficient gap recovery. +### Multi-Dataset Sync +CBDDC supports per-dataset sync pipelines in one process. + +- Dataset identity (`datasetId`) is propagated in protocol and persistence records. +- Each dataset has independent oplog reads, confirmation state, and maintenance cadence. +- Legacy peers without dataset fields interoperate on `primary`. + ### Peer-Confirmed Oplog Pruning CBDDC maintenance pruning now uses a two-cutoff model: diff --git a/docs/features/README.md b/docs/features/README.md index 3aa9a29..ac910b1 100644 --- a/docs/features/README.md +++ b/docs/features/README.md @@ -8,6 +8,7 @@ This index tracks CBDDC major functionality. Each feature has one canonical docu - [Peer-to-Peer Gossip Sync](peer-to-peer-gossip-sync.md) - [Secure Peer Transport](secure-peer-transport.md) - [Peer-Confirmed Pruning](peer-confirmed-pruning.md) +- [Multi-Dataset Sync](multi-dataset-sync.md) ## Maintenance Rules diff --git a/docs/features/multi-dataset-sync.md b/docs/features/multi-dataset-sync.md new file mode 100644 index 0000000..97c6401 --- /dev/null +++ b/docs/features/multi-dataset-sync.md @@ -0,0 +1,67 @@ +# Multi-Dataset Sync + +## Summary + +CBDDC can run multiple sync pipelines inside one process by assigning each pipeline a `datasetId` (for example `primary`, `logs`, `timeseries`). +Each dataset pipeline has independent oplog state, vector-clock reads, peer confirmation watermarks, and maintenance scheduling. + +## Why Use It + +- Keep primary business data sync latency stable during high telemetry volume. +- Isolate append-only streams (`logs`, `timeseries`) from CRUD-heavy collections. +- Roll out incrementally using runtime flags and per-dataset enablement. + +## Configuration + +Register dataset options and enable the runtime coordinator: + +```csharp +services.AddCBDDCSurrealEmbedded(sp => options) + .AddCBDDCSurrealEmbeddedDataset("primary", o => + { + o.InterestingCollections = ["Users", "TodoLists"]; + }) + .AddCBDDCSurrealEmbeddedDataset("logs", o => + { + o.InterestingCollections = ["Logs"]; + o.SyncLoopDelay = TimeSpan.FromMilliseconds(500); + }) + .AddCBDDCSurrealEmbeddedDataset("timeseries", o => + { + o.InterestingCollections = ["Timeseries"]; + o.SyncLoopDelay = TimeSpan.FromMilliseconds(500); + }) + .AddCBDDCNetwork(); + +services.AddCBDDCMultiDataset(options => +{ + options.EnableMultiDatasetSync = true; + options.EnableDatasetPrimary = true; + options.EnableDatasetLogs = true; + options.EnableDatasetTimeseries = true; +}); +``` + +## Wire and Storage Compatibility + +- Protocol messages include optional `dataset_id` fields. +- Missing `dataset_id` is treated as `primary`. +- Surreal persistence records include `datasetId`; legacy rows without `datasetId` are read as `primary`. + +## Operational Notes + +- Each dataset runs its own `SyncOrchestrator` instance. +- Maintenance pruning is dataset-scoped (`datasetId` + cutoff). +- Snapshot APIs support dataset-scoped operations (`CreateSnapshotAsync(stream, datasetId)`). + +## Migration + +1. Deploy with `EnableMultiDatasetSync = false`. +2. Enable multi-dataset mode with only `primary` enabled. +3. Enable `logs`, verify primary sync SLO. +4. Enable `timeseries`, verify primary sync SLO again. + +## Rollback + +- Set `EnableDatasetLogs = false` and `EnableDatasetTimeseries = false` first. +- If needed, set `EnableMultiDatasetSync = false` to return to the single `primary` sync path. diff --git a/docs/persistence-providers.md b/docs/persistence-providers.md index ac4dc9d..2679647 100755 --- a/docs/persistence-providers.md +++ b/docs/persistence-providers.md @@ -221,6 +221,14 @@ services.AddCBDDCCore() }); ``` +### Multi-Dataset Partitioning + +Surreal persistence now stores `datasetId` on oplog, metadata, snapshot metadata, confirmation, and CDC checkpoint records. + +- Composite indexes include `datasetId` to prevent cross-dataset reads. +- Legacy rows missing `datasetId` are interpreted as `primary` during reads. +- Dataset-scoped store APIs (`ExportAsync(datasetId)`, `GetOplogAfterAsync(..., datasetId, ...)`) enforce isolation. + ### CDC Durability Notes 1. **Checkpoint semantics**: each consumer id has an independent durable cursor (`timestamp + hash`). diff --git a/docs/runbook.md b/docs/runbook.md index d53e16f..40792df 100644 --- a/docs/runbook.md +++ b/docs/runbook.md @@ -27,6 +27,15 @@ Capture these artifacts before remediation: - Current runtime configuration (excluding secrets). - Most recent deployment identifier and change window. +## Multi-Dataset Gates + +Before enabling telemetry datasets in production: + +1. Enable `primary` only and record baseline primary sync lag. +2. Enable `logs`; confirm primary lag remains within SLO. +3. Enable `timeseries`; confirm primary lag remains within SLO. +4. If primary SLO regresses, disable telemetry datasets first before broader rollback. + ## Recovery Plays ### Peer unreachable or lagging diff --git a/samples/ZB.MOM.WW.CBDDC.Sample.Console/ConsoleInteractiveService.cs b/samples/ZB.MOM.WW.CBDDC.Sample.Console/ConsoleInteractiveService.cs index b2ceeba..bf76ea6 100755 --- a/samples/ZB.MOM.WW.CBDDC.Sample.Console/ConsoleInteractiveService.cs +++ b/samples/ZB.MOM.WW.CBDDC.Sample.Console/ConsoleInteractiveService.cs @@ -112,6 +112,7 @@ public class ConsoleInteractiveService : BackgroundService System.Console.WriteLine("Commands:"); System.Console.WriteLine(" [p]ut, [g]et, [d]elete, [f]ind, [l]ist peers, [q]uit"); System.Console.WriteLine(" [n]ew (auto), [s]pam (5x), [c]ount, [t]odos"); + System.Console.WriteLine(" log [count], ts [count] (append telemetry load)"); System.Console.WriteLine(" [h]ealth, cac[h]e"); System.Console.WriteLine(" [r]esolver [lww|merge], [demo] conflict"); } @@ -156,8 +157,12 @@ public class ConsoleInteractiveService : BackgroundService { int userCount = _db.Users.FindAll().Count(); int todoCount = _db.TodoLists.FindAll().Count(); + int logCount = _db.Logs.FindAll().Count(); + int timeseriesCount = _db.Timeseries.FindAll().Count(); System.Console.WriteLine($"Collection 'Users': {userCount} documents"); System.Console.WriteLine($"Collection 'TodoLists': {todoCount} documents"); + System.Console.WriteLine($"Collection 'Logs': {logCount} documents"); + System.Console.WriteLine($"Collection 'Timeseries': {timeseriesCount} documents"); } else if (input.StartsWith("p")) { @@ -212,6 +217,42 @@ public class ConsoleInteractiveService : BackgroundService var results = _db.Users.Find(u => u.Age > 28); foreach (var u in results) System.Console.WriteLine($"Found: {u.Name} ({u.Age})"); } + else if (input.StartsWith("log", StringComparison.OrdinalIgnoreCase)) + { + int count = ParseCount(input, 100); + for (var i = 0; i < count; i++) + { + var entry = new TelemetryLogEntry + { + Id = Guid.NewGuid().ToString("N"), + Level = i % 25 == 0 ? "Warning" : "Information", + Message = $"sample-log-{DateTimeOffset.UtcNow:O}-{i}", + CreatedUtc = DateTime.UtcNow + }; + await _db.Logs.InsertAsync(entry); + } + + await _db.SaveChangesAsync(); + System.Console.WriteLine($"Appended {count} log entries."); + } + else if (input.StartsWith("ts", StringComparison.OrdinalIgnoreCase)) + { + int count = ParseCount(input, 100); + for (var i = 0; i < count; i++) + { + var point = new TimeseriesPoint + { + Id = Guid.NewGuid().ToString("N"), + Metric = i % 2 == 0 ? "cpu" : "latency", + Value = Random.Shared.NextDouble() * 100, + RecordedUtc = DateTime.UtcNow + }; + await _db.Timeseries.InsertAsync(point); + } + + await _db.SaveChangesAsync(); + System.Console.WriteLine($"Appended {count} timeseries points."); + } else if (input.StartsWith("h")) { var health = await _healthCheck.CheckAsync(); @@ -283,6 +324,13 @@ public class ConsoleInteractiveService : BackgroundService } } + private static int ParseCount(string input, int fallback) + { + string[] parts = input.Split(' ', StringSplitOptions.RemoveEmptyEntries); + if (parts.Length < 2) return fallback; + return int.TryParse(parts[1], out int parsed) && parsed > 0 ? parsed : fallback; + } + private async Task RunConflictDemo() { System.Console.WriteLine("\n=== Conflict Resolution Demo ==="); @@ -355,4 +403,4 @@ public class ConsoleInteractiveService : BackgroundService System.Console.WriteLine("\n✓ Demo complete. Run 'todos' to see all lists.\n"); } -} \ No newline at end of file +} diff --git a/samples/ZB.MOM.WW.CBDDC.Sample.Console/Program.cs b/samples/ZB.MOM.WW.CBDDC.Sample.Console/Program.cs index 396b37c..5533ba3 100755 --- a/samples/ZB.MOM.WW.CBDDC.Sample.Console/Program.cs +++ b/samples/ZB.MOM.WW.CBDDC.Sample.Console/Program.cs @@ -4,6 +4,7 @@ using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using Serilog; using System.Text.Json; +using ZB.MOM.WW.CBDDC.Core; using ZB.MOM.WW.CBDDC.Core.Network; using ZB.MOM.WW.CBDDC.Core.Storage; using ZB.MOM.WW.CBDDC.Core.Sync; @@ -62,11 +63,22 @@ internal class Program Directory.CreateDirectory(dataPath); string databasePath = Path.Combine(dataPath, $"{nodeId}.rocksdb"); string surrealDatabase = nodeId.Replace("-", "_", StringComparison.Ordinal); + var multiDatasetOptions = builder.Configuration + .GetSection("CBDDC:MultiDataset") + .Get() + ?? new MultiDatasetRuntimeOptions + { + EnableMultiDatasetSync = true, + EnableDatasetPrimary = true, + EnableDatasetLogs = true, + EnableDatasetTimeseries = true + }; // Register CBDDC services with embedded Surreal (RocksDB). builder.Services.AddSingleton(); builder.Services.AddSingleton(); - builder.Services.AddCBDDCCore() + builder.Services + .AddCBDDCCore() .AddCBDDCSurrealEmbedded(_ => new CBDDCSurrealEmbeddedOptions { Endpoint = "rocksdb://local", @@ -74,8 +86,30 @@ internal class Program Namespace = "cbddc_sample", Database = surrealDatabase }) + .AddCBDDCSurrealEmbeddedDataset(DatasetId.Primary, options => + { + options.InterestingCollections = ["Users", "TodoLists"]; + }) + .AddCBDDCSurrealEmbeddedDataset(DatasetId.Logs, options => + { + options.InterestingCollections = ["Logs"]; + }) + .AddCBDDCSurrealEmbeddedDataset(DatasetId.Timeseries, options => + { + options.InterestingCollections = ["Timeseries"]; + }) .AddCBDDCNetwork(); // useHostedService = true by default + if (multiDatasetOptions.EnableMultiDatasetSync) + builder.Services.AddCBDDCMultiDataset(options => + { + options.EnableMultiDatasetSync = multiDatasetOptions.EnableMultiDatasetSync; + options.EnableDatasetPrimary = multiDatasetOptions.EnableDatasetPrimary; + options.EnableDatasetLogs = multiDatasetOptions.EnableDatasetLogs; + options.EnableDatasetTimeseries = multiDatasetOptions.EnableDatasetTimeseries; + options.AdditionalDatasets = multiDatasetOptions.AdditionalDatasets.ToList(); + }); + builder.Services.AddHostedService(); // Runs the Input Loop var host = builder.Build(); diff --git a/samples/ZB.MOM.WW.CBDDC.Sample.Console/SampleDbContext.cs b/samples/ZB.MOM.WW.CBDDC.Sample.Console/SampleDbContext.cs index b6fd508..e1ae08b 100755 --- a/samples/ZB.MOM.WW.CBDDC.Sample.Console/SampleDbContext.cs +++ b/samples/ZB.MOM.WW.CBDDC.Sample.Console/SampleDbContext.cs @@ -11,6 +11,8 @@ public class SampleDbContext : IDisposable { private const string UsersTable = "sample_users"; private const string TodoListsTable = "sample_todo_lists"; + private const string LogsTable = "sample_logs"; + private const string TimeseriesTable = "sample_timeseries"; private readonly bool _ownsClient; @@ -28,6 +30,8 @@ public class SampleDbContext : IDisposable Users = new SampleSurrealCollection(UsersTable, u => u.Id, SurrealEmbeddedClient, SchemaInitializer); TodoLists = new SampleSurrealCollection(TodoListsTable, t => t.Id, SurrealEmbeddedClient, SchemaInitializer); + Logs = new SampleSurrealCollection(LogsTable, e => e.Id, SurrealEmbeddedClient, SchemaInitializer); + Timeseries = new SampleSurrealCollection(TimeseriesTable, p => p.Id, SurrealEmbeddedClient, SchemaInitializer); OplogEntries = new SampleSurrealReadOnlyCollection( CBDDCSurrealSchemaNames.OplogEntriesTable, SurrealEmbeddedClient, @@ -57,6 +61,8 @@ public class SampleDbContext : IDisposable Users = new SampleSurrealCollection(UsersTable, u => u.Id, SurrealEmbeddedClient, SchemaInitializer); TodoLists = new SampleSurrealCollection(TodoListsTable, t => t.Id, SurrealEmbeddedClient, SchemaInitializer); + Logs = new SampleSurrealCollection(LogsTable, e => e.Id, SurrealEmbeddedClient, SchemaInitializer); + Timeseries = new SampleSurrealCollection(TimeseriesTable, p => p.Id, SurrealEmbeddedClient, SchemaInitializer); OplogEntries = new SampleSurrealReadOnlyCollection( CBDDCSurrealSchemaNames.OplogEntriesTable, SurrealEmbeddedClient, @@ -88,6 +94,16 @@ public class SampleDbContext : IDisposable /// public SampleSurrealReadOnlyCollection OplogEntries { get; private set; } + /// + /// Gets the append-only telemetry logs collection. + /// + public SampleSurrealCollection Logs { get; private set; } + + /// + /// Gets the append-only timeseries collection. + /// + public SampleSurrealCollection Timeseries { get; private set; } + /// /// Ensures schema changes are applied before persisting updates. /// @@ -102,6 +118,8 @@ public class SampleDbContext : IDisposable { Users.Dispose(); TodoLists.Dispose(); + Logs.Dispose(); + Timeseries.Dispose(); if (_ownsClient) SurrealEmbeddedClient.Dispose(); } @@ -126,6 +144,8 @@ public sealed class SampleSurrealSchemaInitializer : ICBDDCSurrealSchemaInitiali private const string SampleSchemaSql = """ DEFINE TABLE OVERWRITE sample_users SCHEMALESS CHANGEFEED 7d; DEFINE TABLE OVERWRITE sample_todo_lists SCHEMALESS CHANGEFEED 7d; + DEFINE TABLE OVERWRITE sample_logs SCHEMALESS CHANGEFEED 7d; + DEFINE TABLE OVERWRITE sample_timeseries SCHEMALESS CHANGEFEED 7d; """; private readonly ICBDDCSurrealEmbeddedClient _client; private int _initialized; diff --git a/samples/ZB.MOM.WW.CBDDC.Sample.Console/SampleDocumentStore.cs b/samples/ZB.MOM.WW.CBDDC.Sample.Console/SampleDocumentStore.cs index ef11042..205a062 100755 --- a/samples/ZB.MOM.WW.CBDDC.Sample.Console/SampleDocumentStore.cs +++ b/samples/ZB.MOM.WW.CBDDC.Sample.Console/SampleDocumentStore.cs @@ -14,6 +14,8 @@ public class SampleDocumentStore : SurrealDocumentStore { private const string UsersCollection = "Users"; private const string TodoListsCollection = "TodoLists"; + private const string LogsCollection = "Logs"; + private const string TimeseriesCollection = "Timeseries"; /// /// Initializes a new instance of the class. @@ -40,6 +42,8 @@ public class SampleDocumentStore : SurrealDocumentStore { WatchCollection(UsersCollection, context.Users, u => u.Id); WatchCollection(TodoListsCollection, context.TodoLists, t => t.Id); + WatchCollection(LogsCollection, context.Logs, entry => entry.Id); + WatchCollection(TimeseriesCollection, context.Timeseries, point => point.Id); } /// @@ -71,6 +75,8 @@ public class SampleDocumentStore : SurrealDocumentStore { UsersCollection => SerializeEntity(await _context.Users.FindByIdAsync(key, cancellationToken)), TodoListsCollection => SerializeEntity(await _context.TodoLists.FindByIdAsync(key, cancellationToken)), + LogsCollection => SerializeEntity(await _context.Logs.FindByIdAsync(key, cancellationToken)), + TimeseriesCollection => SerializeEntity(await _context.Timeseries.FindByIdAsync(key, cancellationToken)), _ => null }; } @@ -106,6 +112,12 @@ public class SampleDocumentStore : SurrealDocumentStore TodoListsCollection => (await _context.TodoLists.FindAllAsync(cancellationToken)) .Select(t => (t.Id, SerializeEntity(t)!.Value)) .ToList(), + LogsCollection => (await _context.Logs.FindAllAsync(cancellationToken)) + .Select(entry => (entry.Id, SerializeEntity(entry)!.Value)) + .ToList(), + TimeseriesCollection => (await _context.Timeseries.FindAllAsync(cancellationToken)) + .Select(point => (point.Id, SerializeEntity(point)!.Value)) + .ToList(), _ => [] }; } @@ -137,6 +149,26 @@ public class SampleDocumentStore : SurrealDocumentStore await _context.TodoLists.UpdateAsync(todo, cancellationToken); break; + case LogsCollection: + var logEntry = content.Deserialize() ?? + throw new InvalidOperationException("Failed to deserialize telemetry log."); + logEntry.Id = key; + if (await _context.Logs.FindByIdAsync(key, cancellationToken) == null) + await _context.Logs.InsertAsync(logEntry, cancellationToken); + else + await _context.Logs.UpdateAsync(logEntry, cancellationToken); + break; + + case TimeseriesCollection: + var point = content.Deserialize() ?? + throw new InvalidOperationException("Failed to deserialize timeseries point."); + point.Id = key; + if (await _context.Timeseries.FindByIdAsync(key, cancellationToken) == null) + await _context.Timeseries.InsertAsync(point, cancellationToken); + else + await _context.Timeseries.UpdateAsync(point, cancellationToken); + break; + default: throw new NotSupportedException($"Collection '{collection}' is not supported for sync."); } @@ -152,6 +184,12 @@ public class SampleDocumentStore : SurrealDocumentStore case TodoListsCollection: await _context.TodoLists.DeleteAsync(key, cancellationToken); break; + case LogsCollection: + await _context.Logs.DeleteAsync(key, cancellationToken); + break; + case TimeseriesCollection: + await _context.Timeseries.DeleteAsync(key, cancellationToken); + break; default: _logger.LogWarning("Attempted to remove entity from unsupported collection: {Collection}", collection); break; diff --git a/samples/ZB.MOM.WW.CBDDC.Sample.Console/TelemetryData.cs b/samples/ZB.MOM.WW.CBDDC.Sample.Console/TelemetryData.cs new file mode 100644 index 0000000..a79b8af --- /dev/null +++ b/samples/ZB.MOM.WW.CBDDC.Sample.Console/TelemetryData.cs @@ -0,0 +1,57 @@ +using System.ComponentModel.DataAnnotations; + +namespace ZB.MOM.WW.CBDDC.Sample.Console; + +/// +/// Append-only telemetry log entry used for high-volume sync scenarios. +/// +public class TelemetryLogEntry +{ + /// + /// Gets or sets the unique log identifier. + /// + [Key] + public string Id { get; set; } = Guid.NewGuid().ToString("N"); + + /// + /// Gets or sets the log level. + /// + public string Level { get; set; } = "Information"; + + /// + /// Gets or sets the log message. + /// + public string Message { get; set; } = string.Empty; + + /// + /// Gets or sets the UTC timestamp. + /// + public DateTime CreatedUtc { get; set; } = DateTime.UtcNow; +} + +/// +/// Append-only timeseries metric point used for telemetry sync scenarios. +/// +public class TimeseriesPoint +{ + /// + /// Gets or sets the unique metric point identifier. + /// + [Key] + public string Id { get; set; } = Guid.NewGuid().ToString("N"); + + /// + /// Gets or sets the metric name. + /// + public string Metric { get; set; } = "cpu"; + + /// + /// Gets or sets the metric value. + /// + public double Value { get; set; } + + /// + /// Gets or sets the UTC timestamp. + /// + public DateTime RecordedUtc { get; set; } = DateTime.UtcNow; +} diff --git a/samples/ZB.MOM.WW.CBDDC.Sample.Console/appsettings.json b/samples/ZB.MOM.WW.CBDDC.Sample.Console/appsettings.json index 52ec43c..f74b654 100755 --- a/samples/ZB.MOM.WW.CBDDC.Sample.Console/appsettings.json +++ b/samples/ZB.MOM.WW.CBDDC.Sample.Console/appsettings.json @@ -28,13 +28,19 @@ "BackupPath": "backups/", "BusyTimeoutMs": 5000 }, - "Sync": { - "SyncIntervalMs": 5000, - "BatchSize": 100, - "EnableOfflineQueue": true, - "MaxQueueSize": 1000 - }, - "Logging": { + "Sync": { + "SyncIntervalMs": 5000, + "BatchSize": 100, + "EnableOfflineQueue": true, + "MaxQueueSize": 1000 + }, + "MultiDataset": { + "EnableMultiDatasetSync": true, + "EnableDatasetPrimary": true, + "EnableDatasetLogs": true, + "EnableDatasetTimeseries": true + }, + "Logging": { "LogLevel": "Information", "LogFilePath": "logs/cbddc.log", "MaxLogFileSizeMb": 10, @@ -48,4 +54,4 @@ } ] } -} \ No newline at end of file +} diff --git a/separate.md b/separate.md new file mode 100644 index 0000000..48b88bd --- /dev/null +++ b/separate.md @@ -0,0 +1,323 @@ +# In-Process Multi-Dataset Sync Plan (Worktree Execution) + +## Goal + +Add true in-process multi-dataset sync so primary business data can sync independently from high-volume append-only datasets (logs, timeseries), with separate state, scheduling, and backpressure behavior. + +## Desired Outcome + +1. Primary dataset sync throughput/latency is not materially impacted by telemetry dataset volume. +2. Log and timeseries datasets use independent sync pipelines in the same process. +3. Existing single-dataset apps continue to work with minimal/no code changes. +4. Test coverage explicitly verifies isolation and no cross-dataset leakage. + +## Current Baseline (Why This Change Is Needed) + +1. Current host wiring registers a single `IDocumentStore`, `IOplogStore`, and `ISyncOrchestrator` graph. +2. Collection filtering exists, but all collections still share one orchestrator/sync loop and one oplog/vector clock lifecycle. +3. Protocol filters by collection only; there is no dataset identity boundary. +4. Surreal schema objects are fixed names per configured namespace/database and are not dataset-aware by design. + +## Proposed Target Architecture + +## New Concepts + +1. `DatasetId`: + - Stable identifier (`primary`, `logs`, `timeseries`, etc). + - Included in all sync-state-bearing entities and wire messages. +2. `DatasetSyncContext`: + - Encapsulates one dataset's services: document store adapter, oplog store, snapshot metadata, peer confirmation state, orchestrator configuration. +3. `IMultiDatasetSyncOrchestrator`: + - Host-level coordinator that starts/stops one `ISyncOrchestrator` per dataset. +4. `DatasetSyncOptions`: + - Per-dataset scheduling and limits (loop delay, max peers, optional bandwidth/entry caps, maintenance interval override). + +## Isolation Model + +1. Independent per-dataset oplog stream and vector clock. +2. Independent per-dataset peer confirmation watermarks for pruning. +3. Independent per-dataset transport filtering (handshake and pull/push include dataset id). +4. Independent per-dataset observability counters. + +## Compatibility Strategy + +1. Backward compatible wire changes: + - Add optional `dataset_id` fields; default to `"primary"` when absent. +2. Backward compatible storage: + - Add `datasetId` columns/fields where needed. + - Existing rows default to `"primary"` during migration/read fallback. +3. API defaults: + - Existing single-store registration maps to dataset `"primary"` with no functional change. + +## Git Worktree Execution Plan + +## 0. Worktree Preparation + +1. Create worktree and branch: + - `git worktree add ../CBDDC-multidataset -b codex/multidataset-sync` +2. Build baseline in worktree: + - `dotnet build CBDDC.slnx` +3. Capture baseline tests (save output artifact in worktree): + - `dotnet test CBDDC.slnx` + +Deliverable: +1. Clean baseline build/test result captured before changes. + +## 1. Design and Contract Layer + +### Code Changes + +1. Add dataset contracts in `src/ZB.MOM.WW.CBDDC.Core`: + - `DatasetId` value object or constants. + - `DatasetSyncOptions`. + - `IDatasetSyncContext`/`IMultiDatasetSyncOrchestrator`. +2. Extend domain models where sync identity is required: + - `OplogEntry` add `DatasetId` (constructor defaults to `"primary"`). + - Any metadata types used for causal state/pruning that need dataset partitioning. +3. Extend store interfaces (minimally invasive): + - Keep existing methods as compatibility overloads. + - Add dataset-aware variants where cross-dataset ambiguity exists. + +### Test Work + +1. Add Core unit tests: + - `OplogEntry` hash stability with `DatasetId`. + - Defaulting behavior to `"primary"`. + - Equality/serialization behavior for dataset-aware records. +2. Update existing Core tests that construct `OplogEntry` directly. + +Exit Criteria: +1. Core tests compile and pass with default dataset behavior unchanged. + +## 2. Persistence Partitioning (Surreal) + +### Code Changes + +1. Add dataset partition key to persistence records: + - Oplog rows. + - Document metadata rows. + - Snapshot metadata rows (if used in dataset-scoped recoveries). + - Peer confirmation records. + - CDC checkpoints (consumer id should include dataset id or add dedicated field). +2. Update schema initializer: + - Add `datasetId` fields and composite indexes (`datasetId + existing key dimensions`). +3. Update queries in all Surreal stores: + - Enforce dataset filter in every select/update/delete path. + - Guard against full-table scans that omit dataset filter. +4. Add migration/read fallback: + - If `datasetId` missing on older records, treat as `"primary"` during transitional reads. + +### Test Work + +1. Extend `SurrealStoreContractTests`: + - Write records in two datasets and verify strict isolation. + - Verify prune/merge/export/import scoped by dataset. +2. Add regression tests: + - Legacy records without `datasetId` load as `"primary"` only. +3. Update durability tests: + - CDC checkpoints do not collide between datasets. + +Exit Criteria: +1. Persistence tests prove no cross-dataset reads/writes. + +## 3. Network Protocol Dataset Awareness + +### Code Changes + +1. Update `sync.proto` (backward compatible): + - Add `dataset_id` to `HandshakeRequest`, `HandshakeResponse`, `PullChangesRequest`, `PushChangesRequest`, and optionally snapshot requests. +2. Regenerate protocol classes and adapt transport handlers: + - `TcpPeerClient` sends dataset id for every dataset pipeline. + - `TcpSyncServer` routes requests to correct dataset context. +3. Defaulting rules: + - Missing/empty `dataset_id` => `"primary"`. +4. Add explicit rejection semantics: + - If remote peer does not support requested dataset, return accepted handshake but with dataset capability mismatch response path (or reject per dataset connection). + +### Test Work + +1. Add protocol-level unit tests: + - Message parse/serialize with and without dataset field. +2. Update network tests: + - Handshake stores remote interests per dataset. + - Pull/push operations do not cross datasets. + - Backward compatibility with no dataset id present. + +Exit Criteria: +1. Network tests pass for both new and legacy message shapes. + +## 4. Multi-Orchestrator Runtime and DI + +### Code Changes + +1. Add multi-dataset DI registration extensions: + - `AddCBDDCSurrealEmbeddedDataset(...)` + - `AddCBDDCMultiDataset(...)` +2. Build `MultiDatasetSyncOrchestrator`: + - Start/stop orchestrators for configured datasets. + - Isolated cancellation tokens, loops, and failure handling per dataset. +3. Ensure hosting services (`CBDDCNodeService`, `TcpSyncServerHostedService`) initialize dataset contexts deterministically. +4. Add per-dataset knobs: + - Sync interval, max entries per cycle, maintenance interval, optional parallelism limits. + +### Test Work + +1. Add Hosting tests: + - Multiple datasets register/start/stop cleanly. + - Failure in one dataset does not stop others. +2. Add orchestrator tests: + - Scheduling fairness and per-dataset failure backoff isolation. +3. Update `NoOp`/fallback tests for multi-dataset mode. + +Exit Criteria: +1. Runtime starts N dataset pipelines with independent lifecycle behavior. + +## 5. Snapshot and Recovery Semantics + +### Code Changes + +1. Define snapshot scope options: + - Per-dataset snapshot and full multi-dataset snapshot. +2. Update snapshot service APIs and implementations to support: + - Export/import/merge by dataset id. +3. Ensure emergency recovery paths in orchestrator are dataset-scoped. + +### Test Work + +1. Add snapshot tests: + - Replace/merge for one dataset leaves others untouched. +2. Update reconnect regression tests: + - Snapshot-required flow only affects targeted dataset pipeline. + +Exit Criteria: +1. Recovery operations preserve dataset isolation. + +## 6. Sample App and Developer Experience + +### Code Changes + +1. Add sample configuration for three datasets: + - `primary`, `logs`, `timeseries`. +2. Implement append-only sample stores for `logs` and `timeseries`. +3. Expose sample CLI commands to emit load independently per dataset. + +### Test Work + +1. Add sample integration tests: + - Heavy append load on logs/timeseries does not significantly delay primary data convergence. +2. Add benchmark harness cases: + - Single-dataset baseline vs multi-dataset under telemetry load. + +Exit Criteria: +1. Demonstrable isolation in sample workload. + +## 7. Documentation and Migration Guides + +### Code/Docs Changes + +1. New doc: `docs/features/multi-dataset-sync.md`. +2. Update: + - `docs/architecture.md` + - `docs/persistence-providers.md` + - `docs/runbook.md` +3. Add migration notes: + - From single pipeline to multi-dataset configuration. + - Backward compatibility and rollout toggles. + +### Test Work + +1. Doc examples compile check (if applicable). +2. Add config parsing tests for dataset option sections. + +Exit Criteria: +1. Operators have explicit rollout and rollback steps. + +## 8. Rollout Strategy (Safe Adoption) + +1. Feature flags: + - `EnableMultiDatasetSync` (global). + - `EnableDatasetPrimary/Logs/Timeseries`. +2. Rollout sequence: + - Stage 1: Deploy with flag off. + - Stage 2: Enable `primary` only in new runtime path. + - Stage 3: Enable `logs`, then `timeseries`. +3. Observability gates: + - Primary sync latency SLO must remain within threshold before enabling telemetry datasets. + +## 9. Test Plan (Comprehensive Coverage Matrix) + +## Unit Tests + +1. Core model defaults and hash behavior with dataset id. +2. Dataset routing logic in orchestrator dispatcher. +3. Protocol adapters default `dataset_id` to `"primary"` when absent. +4. Persistence query builders always include dataset predicate. + +## Integration Tests + +1. Surreal stores: + - Same key/collection in different datasets remains isolated. +2. Network: + - Pull/push with mixed datasets never cross-stream. +3. Hosting: + - Independent orchestrator lifecycle and failure isolation. + +## E2E Tests + +1. Multi-node cluster: + - Primary converges under heavy append-only telemetry load. +2. Snapshot/recovery: + - Dataset-scoped restore preserves other datasets. +3. Backward compatibility: + - Legacy node (no dataset id) interoperates on `"primary"`. + +## Non-Functional Tests + +1. Throughput and latency benchmarks: + - Compare primary p95 sync lag before/after. +2. Resource isolation: + - CPU/memory pressure from telemetry datasets should not break primary SLO. + +## Test Update Checklist (Existing Tests to Modify) + +1. `tests/ZB.MOM.WW.CBDDC.Core.Tests`: + - Update direct `OplogEntry` constructions. +2. `tests/ZB.MOM.WW.CBDDC.Network.Tests`: + - Handshake/connection/vector-clock tests for dataset-aware flows. +3. `tests/ZB.MOM.WW.CBDDC.Hosting.Tests`: + - Add multi-dataset startup/shutdown/failure cases. +4. `tests/ZB.MOM.WW.CBDDC.Sample.Console.Tests`: + - Extend Surreal contract and durability tests for dataset partitioning. +5. `tests/ZB.MOM.WW.CBDDC.E2E.Tests`: + - Add multi-dataset convergence + interference tests. + +## Worktree Task Breakdown (Execution Order) + +1. `Phase-A`: Contracts + Core model updates + unit tests. +2. `Phase-B`: Surreal schema/store partitioning + persistence tests. +3. `Phase-C`: Protocol and network routing + network tests. +4. `Phase-D`: Multi-orchestrator DI/runtime + hosting tests. +5. `Phase-E`: Snapshot/recovery updates + regression tests. +6. `Phase-F`: Sample/bench/docs + end-to-end verification. + +Each phase should be committed separately in the worktree to keep reviewable deltas. + +## Validation Commands (Run in Worktree) + +1. `dotnet build /Users/dohertj2/Desktop/CBDDC/CBDDC.slnx` +2. `dotnet test /Users/dohertj2/Desktop/CBDDC/CBDDC.slnx` +3. Focused suites during implementation: + - `dotnet test /Users/dohertj2/Desktop/CBDDC/tests/ZB.MOM.WW.CBDDC.Core.Tests/ZB.MOM.WW.CBDDC.Core.Tests.csproj` + - `dotnet test /Users/dohertj2/Desktop/CBDDC/tests/ZB.MOM.WW.CBDDC.Network.Tests/ZB.MOM.WW.CBDDC.Network.Tests.csproj` + - `dotnet test /Users/dohertj2/Desktop/CBDDC/tests/ZB.MOM.WW.CBDDC.Hosting.Tests/ZB.MOM.WW.CBDDC.Hosting.Tests.csproj` + - `dotnet test /Users/dohertj2/Desktop/CBDDC/tests/ZB.MOM.WW.CBDDC.Sample.Console.Tests/ZB.MOM.WW.CBDDC.Sample.Console.Tests.csproj` + - `dotnet test /Users/dohertj2/Desktop/CBDDC/tests/ZB.MOM.WW.CBDDC.E2E.Tests/ZB.MOM.WW.CBDDC.E2E.Tests.csproj` + +## Definition of Done + +1. Multi-dataset mode runs `primary`, `logs`, and `timeseries` in one process with independent sync paths. +2. No cross-dataset data movement in persistence, protocol, or runtime. +3. Single-dataset existing usage still works via default `"primary"` dataset. +4. Added/updated unit, integration, and E2E tests pass in CI. +5. Docs include migration and operational guidance. + diff --git a/src/ZB.MOM.WW.CBDDC.Core/DatasetId.cs b/src/ZB.MOM.WW.CBDDC.Core/DatasetId.cs new file mode 100644 index 0000000..3518f41 --- /dev/null +++ b/src/ZB.MOM.WW.CBDDC.Core/DatasetId.cs @@ -0,0 +1,34 @@ +namespace ZB.MOM.WW.CBDDC.Core; + +/// +/// Provides well-known dataset identifiers and normalization helpers. +/// +public static class DatasetId +{ + /// + /// The default dataset identifier used by legacy single-dataset deployments. + /// + public const string Primary = "primary"; + + /// + /// A high-volume append-only telemetry/log dataset identifier. + /// + public const string Logs = "logs"; + + /// + /// A high-volume append-only timeseries dataset identifier. + /// + public const string Timeseries = "timeseries"; + + /// + /// Normalizes a dataset identifier and applies the default when missing. + /// + /// The raw dataset identifier. + /// The normalized dataset identifier. + public static string Normalize(string? datasetId) + { + return string.IsNullOrWhiteSpace(datasetId) + ? Primary + : datasetId.Trim().ToLowerInvariant(); + } +} diff --git a/src/ZB.MOM.WW.CBDDC.Core/DatasetSyncOptions.cs b/src/ZB.MOM.WW.CBDDC.Core/DatasetSyncOptions.cs new file mode 100644 index 0000000..ed898fc --- /dev/null +++ b/src/ZB.MOM.WW.CBDDC.Core/DatasetSyncOptions.cs @@ -0,0 +1,45 @@ +using System; +using System.Collections.Generic; + +namespace ZB.MOM.WW.CBDDC.Core; + +/// +/// Configures synchronization behavior for a single dataset pipeline. +/// +public sealed class DatasetSyncOptions +{ + /// + /// Gets or sets the dataset identifier. + /// + public string DatasetId { get; set; } = global::ZB.MOM.WW.CBDDC.Core.DatasetId.Primary; + + /// + /// Gets or sets a value indicating whether this dataset pipeline is enabled. + /// + public bool Enabled { get; set; } = true; + + /// + /// Gets or sets the delay between sync loop cycles. + /// + public TimeSpan SyncLoopDelay { get; set; } = TimeSpan.FromSeconds(2); + + /// + /// Gets or sets the maximum number of peers selected per cycle. + /// + public int MaxPeersPerCycle { get; set; } = 3; + + /// + /// Gets or sets the maximum number of oplog entries pushed or pulled in one cycle. + /// + public int? MaxEntriesPerCycle { get; set; } + + /// + /// Gets or sets an optional maintenance interval override for this dataset. + /// + public TimeSpan? MaintenanceIntervalOverride { get; set; } + + /// + /// Gets or sets collection interests for this dataset pipeline. + /// + public List InterestingCollections { get; set; } = []; +} diff --git a/src/ZB.MOM.WW.CBDDC.Core/OplogEntry.cs b/src/ZB.MOM.WW.CBDDC.Core/OplogEntry.cs index dbac33a..c45b762 100755 --- a/src/ZB.MOM.WW.CBDDC.Core/OplogEntry.cs +++ b/src/ZB.MOM.WW.CBDDC.Core/OplogEntry.cs @@ -19,14 +19,16 @@ public static class OplogEntryExtensions /// /// The oplog entry to hash. /// The lowercase hexadecimal SHA-256 hash of the entry. - public static string ComputeHash(this OplogEntry entry) - { - using var sha256 = SHA256.Create(); - var sb = new StringBuilder(); - - sb.Append(entry.Collection); - sb.Append('|'); - sb.Append(entry.Key); + public static string ComputeHash(this OplogEntry entry) + { + using var sha256 = SHA256.Create(); + var sb = new StringBuilder(); + + sb.Append(DatasetId.Normalize(entry.DatasetId)); + sb.Append('|'); + sb.Append(entry.Collection); + sb.Append('|'); + sb.Append(entry.Key); sb.Append('|'); // Ensure stable string representation for Enum (integer value) sb.Append(((int)entry.Operation).ToString(CultureInfo.InvariantCulture)); @@ -56,24 +58,31 @@ public class OplogEntry /// The document key. /// The operation type. /// The serialized payload. - /// The logical timestamp. - /// The previous entry hash. - /// The current entry hash. If null, it is computed. - public OplogEntry(string collection, string key, OperationType operation, JsonElement? payload, - HlcTimestamp timestamp, string previousHash, string? hash = null) - { - Collection = collection; - Key = key; - Operation = operation; + /// The logical timestamp. + /// The previous entry hash. + /// The current entry hash. If null, it is computed. + /// The dataset identifier for this entry. Defaults to primary. + public OplogEntry(string collection, string key, OperationType operation, JsonElement? payload, + HlcTimestamp timestamp, string previousHash, string? hash = null, string? datasetId = null) + { + DatasetId = global::ZB.MOM.WW.CBDDC.Core.DatasetId.Normalize(datasetId); + Collection = collection; + Key = key; + Operation = operation; Payload = payload; Timestamp = timestamp; PreviousHash = previousHash ?? string.Empty; - Hash = hash ?? this.ComputeHash(); - } - - /// - /// Gets the collection name associated with this entry. - /// + Hash = hash ?? this.ComputeHash(); + } + + /// + /// Gets the dataset identifier associated with this entry. + /// + public string DatasetId { get; } + + /// + /// Gets the collection name associated with this entry. + /// public string Collection { get; } /// @@ -113,4 +122,4 @@ public class OplogEntry { return Hash == this.ComputeHash(); } -} \ No newline at end of file +} diff --git a/src/ZB.MOM.WW.CBDDC.Core/PeerOplogConfirmation.cs b/src/ZB.MOM.WW.CBDDC.Core/PeerOplogConfirmation.cs index 89d74bd..f57260b 100644 --- a/src/ZB.MOM.WW.CBDDC.Core/PeerOplogConfirmation.cs +++ b/src/ZB.MOM.WW.CBDDC.Core/PeerOplogConfirmation.cs @@ -7,6 +7,11 @@ namespace ZB.MOM.WW.CBDDC.Core; /// public class PeerOplogConfirmation { + /// + /// Gets or sets the dataset identifier associated with this confirmation record. + /// + public string DatasetId { get; set; } = global::ZB.MOM.WW.CBDDC.Core.DatasetId.Primary; + /// /// Gets or sets the tracked peer node identifier. /// @@ -41,4 +46,4 @@ public class PeerOplogConfirmation /// Gets or sets whether this tracked peer is active for pruning/sync gating. /// public bool IsActive { get; set; } = true; -} \ No newline at end of file +} diff --git a/src/ZB.MOM.WW.CBDDC.Core/SnapshotMetadata.cs b/src/ZB.MOM.WW.CBDDC.Core/SnapshotMetadata.cs index 30736f8..a9df447 100755 --- a/src/ZB.MOM.WW.CBDDC.Core/SnapshotMetadata.cs +++ b/src/ZB.MOM.WW.CBDDC.Core/SnapshotMetadata.cs @@ -2,6 +2,11 @@ namespace ZB.MOM.WW.CBDDC.Core; public class SnapshotMetadata { + /// + /// Gets or sets the dataset identifier associated with the snapshot metadata. + /// + public string DatasetId { get; set; } = global::ZB.MOM.WW.CBDDC.Core.DatasetId.Primary; + /// /// Gets or sets the node identifier associated with the snapshot. /// @@ -21,4 +26,4 @@ public class SnapshotMetadata /// Gets or sets the snapshot hash. /// public string Hash { get; set; } = ""; -} \ No newline at end of file +} diff --git a/src/ZB.MOM.WW.CBDDC.Core/Storage/IDatasetSyncContext.cs b/src/ZB.MOM.WW.CBDDC.Core/Storage/IDatasetSyncContext.cs new file mode 100644 index 0000000..29832fc --- /dev/null +++ b/src/ZB.MOM.WW.CBDDC.Core/Storage/IDatasetSyncContext.cs @@ -0,0 +1,42 @@ +namespace ZB.MOM.WW.CBDDC.Core.Storage; + +/// +/// Represents the storage and runtime contracts bound to a specific dataset pipeline. +/// +public interface IDatasetSyncContext +{ + /// + /// Gets the dataset identifier represented by this context. + /// + string DatasetId { get; } + + /// + /// Gets the per-dataset synchronization options. + /// + DatasetSyncOptions Options { get; } + + /// + /// Gets the dataset-scoped document store. + /// + IDocumentStore DocumentStore { get; } + + /// + /// Gets the dataset-scoped oplog store. + /// + IOplogStore OplogStore { get; } + + /// + /// Gets the dataset-scoped snapshot metadata store. + /// + ISnapshotMetadataStore SnapshotMetadataStore { get; } + + /// + /// Gets the dataset-scoped snapshot service. + /// + ISnapshotService SnapshotService { get; } + + /// + /// Gets the optional dataset-scoped peer confirmation store. + /// + IPeerOplogConfirmationStore? PeerOplogConfirmationStore { get; } +} diff --git a/src/ZB.MOM.WW.CBDDC.Core/Storage/IDocumentMetadataStore.cs b/src/ZB.MOM.WW.CBDDC.Core/Storage/IDocumentMetadataStore.cs index f8491f4..2e8b017 100755 --- a/src/ZB.MOM.WW.CBDDC.Core/Storage/IDocumentMetadataStore.cs +++ b/src/ZB.MOM.WW.CBDDC.Core/Storage/IDocumentMetadataStore.cs @@ -17,8 +17,25 @@ public interface IDocumentMetadataStore : ISnapshotable /// The document key. /// A cancellation token. /// The document metadata if found; otherwise null. - Task GetMetadataAsync(string collection, string key, - CancellationToken cancellationToken = default); + Task GetMetadataAsync(string collection, string key, + CancellationToken cancellationToken = default); + + /// + /// Gets metadata for a specific document within a dataset. + /// + /// The collection name. + /// The document key. + /// The dataset identifier. + /// A cancellation token. + /// The matching metadata when found. + Task GetMetadataAsync( + string collection, + string key, + string datasetId, + CancellationToken cancellationToken = default) + { + return GetMetadataAsync(collection, key, cancellationToken); + } /// /// Gets metadata for all documents in a collection. @@ -26,23 +43,66 @@ public interface IDocumentMetadataStore : ISnapshotable /// The collection name. /// A cancellation token. /// Enumerable of document metadata for the collection. - Task> GetMetadataByCollectionAsync(string collection, - CancellationToken cancellationToken = default); + Task> GetMetadataByCollectionAsync(string collection, + CancellationToken cancellationToken = default); + + /// + /// Gets metadata for all documents in a collection for the specified dataset. + /// + /// The collection name. + /// The dataset identifier. + /// A cancellation token. + /// Enumerable of metadata rows. + Task> GetMetadataByCollectionAsync( + string collection, + string datasetId, + CancellationToken cancellationToken = default) + { + return GetMetadataByCollectionAsync(collection, cancellationToken); + } /// /// Upserts (inserts or updates) metadata for a document. /// /// The metadata to upsert. /// A cancellation token. - Task UpsertMetadataAsync(DocumentMetadata metadata, CancellationToken cancellationToken = default); + Task UpsertMetadataAsync(DocumentMetadata metadata, CancellationToken cancellationToken = default); + + /// + /// Upserts metadata for a specific dataset. + /// + /// The metadata to upsert. + /// The dataset identifier. + /// A cancellation token. + Task UpsertMetadataAsync( + DocumentMetadata metadata, + string datasetId, + CancellationToken cancellationToken = default) + { + return UpsertMetadataAsync(metadata, cancellationToken); + } /// /// Upserts metadata for multiple documents in batch. /// /// The metadata items to upsert. /// A cancellation token. - Task UpsertMetadataBatchAsync(IEnumerable metadatas, - CancellationToken cancellationToken = default); + Task UpsertMetadataBatchAsync(IEnumerable metadatas, + CancellationToken cancellationToken = default); + + /// + /// Upserts metadata batch for a specific dataset. + /// + /// The metadata items. + /// The dataset identifier. + /// A cancellation token. + Task UpsertMetadataBatchAsync( + IEnumerable metadatas, + string datasetId, + CancellationToken cancellationToken = default) + { + return UpsertMetadataBatchAsync(metadatas, cancellationToken); + } /// /// Marks a document as deleted by setting IsDeleted=true and updating the timestamp. @@ -51,8 +111,26 @@ public interface IDocumentMetadataStore : ISnapshotable /// The document key. /// The HLC timestamp of the deletion. /// A cancellation token. - Task MarkDeletedAsync(string collection, string key, HlcTimestamp timestamp, - CancellationToken cancellationToken = default); + Task MarkDeletedAsync(string collection, string key, HlcTimestamp timestamp, + CancellationToken cancellationToken = default); + + /// + /// Marks a document as deleted in a specific dataset. + /// + /// The collection name. + /// The document key. + /// The deletion timestamp. + /// The dataset identifier. + /// A cancellation token. + Task MarkDeletedAsync( + string collection, + string key, + HlcTimestamp timestamp, + string datasetId, + CancellationToken cancellationToken = default) + { + return MarkDeletedAsync(collection, key, timestamp, cancellationToken); + } /// /// Gets all document metadata with timestamps after the specified timestamp. @@ -62,8 +140,25 @@ public interface IDocumentMetadataStore : ISnapshotable /// Optional collection filter. /// A cancellation token. /// Documents modified after the specified timestamp. - Task> GetMetadataAfterAsync(HlcTimestamp since, - IEnumerable? collections = null, CancellationToken cancellationToken = default); + Task> GetMetadataAfterAsync(HlcTimestamp since, + IEnumerable? collections = null, CancellationToken cancellationToken = default); + + /// + /// Gets document metadata modified after a timestamp for a specific dataset. + /// + /// The lower-bound timestamp. + /// The dataset identifier. + /// Optional collection filter. + /// A cancellation token. + /// Documents modified after the specified timestamp. + Task> GetMetadataAfterAsync( + HlcTimestamp since, + string datasetId, + IEnumerable? collections = null, + CancellationToken cancellationToken = default) + { + return GetMetadataAfterAsync(since, collections, cancellationToken); + } } /// @@ -84,14 +179,22 @@ public class DocumentMetadata /// The collection name. /// The document key. /// The last update timestamp. - /// Whether the document is marked as deleted. - public DocumentMetadata(string collection, string key, HlcTimestamp updatedAt, bool isDeleted = false) - { - Collection = collection; - Key = key; - UpdatedAt = updatedAt; - IsDeleted = isDeleted; - } + /// Whether the document is marked as deleted. + /// The dataset identifier. Defaults to primary. + public DocumentMetadata(string collection, string key, HlcTimestamp updatedAt, bool isDeleted = false, + string? datasetId = null) + { + Collection = collection; + Key = key; + UpdatedAt = updatedAt; + IsDeleted = isDeleted; + DatasetId = global::ZB.MOM.WW.CBDDC.Core.DatasetId.Normalize(datasetId); + } + + /// + /// Gets or sets the dataset identifier. + /// + public string DatasetId { get; set; } = global::ZB.MOM.WW.CBDDC.Core.DatasetId.Primary; /// /// Gets or sets the collection name. @@ -108,8 +211,8 @@ public class DocumentMetadata /// public HlcTimestamp UpdatedAt { get; set; } - /// - /// Gets or sets whether this document is marked as deleted (tombstone). - /// - public bool IsDeleted { get; set; } -} \ No newline at end of file + /// + /// Gets or sets whether this document is marked as deleted (tombstone). + /// + public bool IsDeleted { get; set; } +} diff --git a/src/ZB.MOM.WW.CBDDC.Core/Storage/IMultiDatasetSyncOrchestrator.cs b/src/ZB.MOM.WW.CBDDC.Core/Storage/IMultiDatasetSyncOrchestrator.cs new file mode 100644 index 0000000..c73bb1e --- /dev/null +++ b/src/ZB.MOM.WW.CBDDC.Core/Storage/IMultiDatasetSyncOrchestrator.cs @@ -0,0 +1,25 @@ +using System.Collections.Generic; +using System.Threading.Tasks; + +namespace ZB.MOM.WW.CBDDC.Core.Storage; + +/// +/// Coordinates the lifecycle of multiple dataset synchronization pipelines. +/// +public interface IMultiDatasetSyncOrchestrator +{ + /// + /// Gets the registered dataset contexts. + /// + IReadOnlyCollection Contexts { get; } + + /// + /// Starts synchronization for all configured datasets. + /// + Task Start(); + + /// + /// Stops synchronization for all configured datasets. + /// + Task Stop(); +} diff --git a/src/ZB.MOM.WW.CBDDC.Core/Storage/IOplogStore.cs b/src/ZB.MOM.WW.CBDDC.Core/Storage/IOplogStore.cs index b6414bc..9a60cef 100755 --- a/src/ZB.MOM.WW.CBDDC.Core/Storage/IOplogStore.cs +++ b/src/ZB.MOM.WW.CBDDC.Core/Storage/IOplogStore.cs @@ -15,13 +15,28 @@ public interface IOplogStore : ISnapshotable /// event EventHandler ChangesApplied; - /// - /// Appends a new entry to the operation log asynchronously. + /// + /// Appends a new entry to the operation log asynchronously. /// /// The operation log entry to append. Cannot be null. /// A cancellation token that can be used to cancel the append operation. - /// A task that represents the asynchronous append operation. - Task AppendOplogEntryAsync(OplogEntry entry, CancellationToken cancellationToken = default); + /// A task that represents the asynchronous append operation. + Task AppendOplogEntryAsync(OplogEntry entry, CancellationToken cancellationToken = default); + + /// + /// Appends a new entry to the operation log asynchronously for a specific dataset. + /// + /// The operation log entry to append. + /// The dataset identifier. + /// A cancellation token. + /// A task that represents the asynchronous append operation. + Task AppendOplogEntryAsync( + OplogEntry entry, + string datasetId, + CancellationToken cancellationToken = default) + { + return AppendOplogEntryAsync(entry, cancellationToken); + } /// /// Asynchronously retrieves all oplog entries that occurred after the specified timestamp. @@ -30,22 +45,61 @@ public interface IOplogStore : ISnapshotable /// An optional collection of collection names to filter the results. /// A cancellation token that can be used to cancel the asynchronous operation. /// A task that represents the asynchronous operation containing matching oplog entries. - Task> GetOplogAfterAsync(HlcTimestamp timestamp, IEnumerable? collections = null, - CancellationToken cancellationToken = default); + Task> GetOplogAfterAsync(HlcTimestamp timestamp, IEnumerable? collections = null, + CancellationToken cancellationToken = default); + + /// + /// Asynchronously retrieves oplog entries after the specified timestamp for a specific dataset. + /// + /// The lower-bound timestamp. + /// The dataset identifier. + /// Optional collection filter. + /// A cancellation token. + /// A task containing matching oplog entries. + Task> GetOplogAfterAsync( + HlcTimestamp timestamp, + string datasetId, + IEnumerable? collections = null, + CancellationToken cancellationToken = default) + { + return GetOplogAfterAsync(timestamp, collections, cancellationToken); + } /// /// Asynchronously retrieves the latest observed hybrid logical clock (HLC) timestamp. /// /// A cancellation token that can be used to cancel the operation. /// A task that represents the asynchronous operation containing the latest HLC timestamp. - Task GetLatestTimestampAsync(CancellationToken cancellationToken = default); + Task GetLatestTimestampAsync(CancellationToken cancellationToken = default); + + /// + /// Asynchronously retrieves the latest observed timestamp for a specific dataset. + /// + /// The dataset identifier. + /// A cancellation token. + /// A task containing the latest timestamp. + Task GetLatestTimestampAsync(string datasetId, CancellationToken cancellationToken = default) + { + return GetLatestTimestampAsync(cancellationToken); + } /// /// Asynchronously retrieves the current vector clock representing the state of distributed events. /// /// A cancellation token that can be used to cancel the asynchronous operation. /// A task that represents the asynchronous operation containing the current vector clock. - Task GetVectorClockAsync(CancellationToken cancellationToken = default); + Task GetVectorClockAsync(CancellationToken cancellationToken = default); + + /// + /// Asynchronously retrieves the vector clock for a specific dataset. + /// + /// The dataset identifier. + /// A cancellation token. + /// A task containing the vector clock. + Task GetVectorClockAsync(string datasetId, CancellationToken cancellationToken = default) + { + return GetVectorClockAsync(cancellationToken); + } /// /// Retrieves a collection of oplog entries for the specified node that occurred after the given timestamp. @@ -55,8 +109,27 @@ public interface IOplogStore : ISnapshotable /// An optional collection of collection names to filter the oplog entries. /// A cancellation token that can be used to cancel the asynchronous operation. /// A task that represents the asynchronous operation containing oplog entries for the specified node. - Task> GetOplogForNodeAfterAsync(string nodeId, HlcTimestamp since, - IEnumerable? collections = null, CancellationToken cancellationToken = default); + Task> GetOplogForNodeAfterAsync(string nodeId, HlcTimestamp since, + IEnumerable? collections = null, CancellationToken cancellationToken = default); + + /// + /// Retrieves oplog entries for the specified node and dataset after the provided timestamp. + /// + /// The node identifier. + /// The lower-bound timestamp. + /// The dataset identifier. + /// Optional collection filter. + /// A cancellation token. + /// A task containing matching oplog entries. + Task> GetOplogForNodeAfterAsync( + string nodeId, + HlcTimestamp since, + string datasetId, + IEnumerable? collections = null, + CancellationToken cancellationToken = default) + { + return GetOplogForNodeAfterAsync(nodeId, since, collections, cancellationToken); + } /// /// Asynchronously retrieves the hash of the most recent entry for the specified node. @@ -67,7 +140,19 @@ public interface IOplogStore : ISnapshotable /// /// A cancellation token that can be used to cancel the operation. /// A task that represents the asynchronous operation containing the hash string of the last entry or null. - Task GetLastEntryHashAsync(string nodeId, CancellationToken cancellationToken = default); + Task GetLastEntryHashAsync(string nodeId, CancellationToken cancellationToken = default); + + /// + /// Asynchronously retrieves the last entry hash for a node within a specific dataset. + /// + /// The node identifier. + /// The dataset identifier. + /// A cancellation token. + /// A task containing the last hash or null. + Task GetLastEntryHashAsync(string nodeId, string datasetId, CancellationToken cancellationToken = default) + { + return GetLastEntryHashAsync(nodeId, cancellationToken); + } /// /// Asynchronously retrieves a sequence of oplog entries representing the chain between the specified start and end @@ -77,8 +162,25 @@ public interface IOplogStore : ISnapshotable /// The hash of the last entry in the chain range. Cannot be null or empty. /// A cancellation token that can be used to cancel the asynchronous operation. /// A task that represents the asynchronous operation containing OplogEntry objects in chain order. - Task> GetChainRangeAsync(string startHash, string endHash, - CancellationToken cancellationToken = default); + Task> GetChainRangeAsync(string startHash, string endHash, + CancellationToken cancellationToken = default); + + /// + /// Asynchronously retrieves a chain range for a specific dataset. + /// + /// The start hash. + /// The end hash. + /// The dataset identifier. + /// A cancellation token. + /// A task containing chain entries. + Task> GetChainRangeAsync( + string startHash, + string endHash, + string datasetId, + CancellationToken cancellationToken = default) + { + return GetChainRangeAsync(startHash, endHash, cancellationToken); + } /// /// Asynchronously retrieves the oplog entry associated with the specified hash value. @@ -86,7 +188,19 @@ public interface IOplogStore : ISnapshotable /// The hash string identifying the oplog entry to retrieve. Cannot be null or empty. /// A cancellation token that can be used to cancel the asynchronous operation. /// A task representing the asynchronous operation containing the OplogEntry if found, otherwise null. - Task GetEntryByHashAsync(string hash, CancellationToken cancellationToken = default); + Task GetEntryByHashAsync(string hash, CancellationToken cancellationToken = default); + + /// + /// Asynchronously retrieves an entry by hash for a specific dataset. + /// + /// The entry hash. + /// The dataset identifier. + /// A cancellation token. + /// A task containing the entry when found. + Task GetEntryByHashAsync(string hash, string datasetId, CancellationToken cancellationToken = default) + { + return GetEntryByHashAsync(hash, cancellationToken); + } /// /// Applies a batch of oplog entries asynchronously to the target data store. @@ -94,7 +208,22 @@ public interface IOplogStore : ISnapshotable /// A collection of OplogEntry objects representing the operations to apply. Cannot be null. /// A cancellation token that can be used to cancel the batch operation. /// A task that represents the asynchronous batch apply operation. - Task ApplyBatchAsync(IEnumerable oplogEntries, CancellationToken cancellationToken = default); + Task ApplyBatchAsync(IEnumerable oplogEntries, CancellationToken cancellationToken = default); + + /// + /// Applies a batch of oplog entries asynchronously for a specific dataset. + /// + /// The entries to apply. + /// The dataset identifier. + /// A cancellation token. + /// A task that represents the apply operation. + Task ApplyBatchAsync( + IEnumerable oplogEntries, + string datasetId, + CancellationToken cancellationToken = default) + { + return ApplyBatchAsync(oplogEntries, cancellationToken); + } /// /// Asynchronously removes entries from the oplog that are older than the specified cutoff timestamp. @@ -102,5 +231,17 @@ public interface IOplogStore : ISnapshotable /// The timestamp that defines the upper bound for entries to be pruned. /// A cancellation token that can be used to cancel the prune operation. /// A task that represents the asynchronous prune operation. - Task PruneOplogAsync(HlcTimestamp cutoff, CancellationToken cancellationToken = default); -} \ No newline at end of file + Task PruneOplogAsync(HlcTimestamp cutoff, CancellationToken cancellationToken = default); + + /// + /// Asynchronously removes entries from the specified dataset oplog that are older than the cutoff. + /// + /// The prune cutoff timestamp. + /// The dataset identifier. + /// A cancellation token. + /// A task that represents the prune operation. + Task PruneOplogAsync(HlcTimestamp cutoff, string datasetId, CancellationToken cancellationToken = default) + { + return PruneOplogAsync(cutoff, cancellationToken); + } +} diff --git a/src/ZB.MOM.WW.CBDDC.Core/Storage/IPeerOplogConfirmationStore.cs b/src/ZB.MOM.WW.CBDDC.Core/Storage/IPeerOplogConfirmationStore.cs index 8c27597..8e9199b 100644 --- a/src/ZB.MOM.WW.CBDDC.Core/Storage/IPeerOplogConfirmationStore.cs +++ b/src/ZB.MOM.WW.CBDDC.Core/Storage/IPeerOplogConfirmationStore.cs @@ -23,6 +23,24 @@ public interface IPeerOplogConfirmationStore : ISnapshotable + /// Ensures the specified peer is tracked for a given dataset. + /// + /// The peer node identifier. + /// The peer address. + /// The peer type. + /// The dataset identifier. + /// A cancellation token. + Task EnsurePeerRegisteredAsync( + string peerNodeId, + string address, + PeerType type, + string datasetId, + CancellationToken cancellationToken = default) + { + return EnsurePeerRegisteredAsync(peerNodeId, address, type, cancellationToken); + } + /// /// Updates the confirmation watermark for a tracked peer and source node. /// @@ -38,6 +56,26 @@ public interface IPeerOplogConfirmationStore : ISnapshotable + /// Updates the confirmation watermark for a tracked peer and dataset. + /// + /// The tracked peer node identifier. + /// The source node identifier. + /// The confirmed timestamp. + /// The confirmed hash. + /// The dataset identifier. + /// A cancellation token. + Task UpdateConfirmationAsync( + string peerNodeId, + string sourceNodeId, + HlcTimestamp timestamp, + string hash, + string datasetId, + CancellationToken cancellationToken = default) + { + return UpdateConfirmationAsync(peerNodeId, sourceNodeId, timestamp, hash, cancellationToken); + } + /// /// Gets all persisted peer confirmations. /// @@ -45,6 +83,19 @@ public interface IPeerOplogConfirmationStore : ISnapshotableAll peer confirmations. Task> GetConfirmationsAsync(CancellationToken cancellationToken = default); + /// + /// Gets all persisted peer confirmations for a specific dataset. + /// + /// The dataset identifier. + /// A cancellation token. + /// All peer confirmations in the dataset. + Task> GetConfirmationsAsync( + string datasetId, + CancellationToken cancellationToken = default) + { + return GetConfirmationsAsync(cancellationToken); + } + /// /// Gets persisted confirmations for a specific tracked peer. /// @@ -55,6 +106,21 @@ public interface IPeerOplogConfirmationStore : ISnapshotable + /// Gets persisted confirmations for a specific peer and dataset. + /// + /// The peer node identifier. + /// The dataset identifier. + /// A cancellation token. + /// Peer confirmations for the requested peer and dataset. + Task> GetConfirmationsForPeerAsync( + string peerNodeId, + string datasetId, + CancellationToken cancellationToken = default) + { + return GetConfirmationsForPeerAsync(peerNodeId, cancellationToken); + } + /// /// Deactivates tracking for the specified peer. /// @@ -62,10 +128,34 @@ public interface IPeerOplogConfirmationStore : ISnapshotableA cancellation token. Task RemovePeerTrackingAsync(string peerNodeId, CancellationToken cancellationToken = default); + /// + /// Deactivates tracking for a peer in a specific dataset. + /// + /// The peer node identifier. + /// The dataset identifier. + /// A cancellation token. + Task RemovePeerTrackingAsync(string peerNodeId, string datasetId, CancellationToken cancellationToken = default) + { + return RemovePeerTrackingAsync(peerNodeId, cancellationToken); + } + /// /// Gets all active tracked peer identifiers. /// /// A cancellation token. /// Distinct active tracked peer identifiers. Task> GetActiveTrackedPeersAsync(CancellationToken cancellationToken = default); -} \ No newline at end of file + + /// + /// Gets active tracked peers for a specific dataset. + /// + /// The dataset identifier. + /// A cancellation token. + /// Distinct active peer identifiers. + Task> GetActiveTrackedPeersAsync( + string datasetId, + CancellationToken cancellationToken = default) + { + return GetActiveTrackedPeersAsync(cancellationToken); + } +} diff --git a/src/ZB.MOM.WW.CBDDC.Core/Storage/ISnapshotMetadataStore.cs b/src/ZB.MOM.WW.CBDDC.Core/Storage/ISnapshotMetadataStore.cs index ed836d4..d228fca 100755 --- a/src/ZB.MOM.WW.CBDDC.Core/Storage/ISnapshotMetadataStore.cs +++ b/src/ZB.MOM.WW.CBDDC.Core/Storage/ISnapshotMetadataStore.cs @@ -20,6 +20,21 @@ public interface ISnapshotMetadataStore : ISnapshotable /// Task GetSnapshotMetadataAsync(string nodeId, CancellationToken cancellationToken = default); + /// + /// Asynchronously retrieves snapshot metadata for a node within a specific dataset. + /// + /// The node identifier. + /// The dataset identifier. + /// A cancellation token. + /// The snapshot metadata when available. + Task GetSnapshotMetadataAsync( + string nodeId, + string datasetId, + CancellationToken cancellationToken = default) + { + return GetSnapshotMetadataAsync(nodeId, cancellationToken); + } + /// /// Asynchronously inserts the specified snapshot metadata into the data store. /// @@ -28,6 +43,20 @@ public interface ISnapshotMetadataStore : ISnapshotable /// A task that represents the asynchronous insert operation. Task InsertSnapshotMetadataAsync(SnapshotMetadata metadata, CancellationToken cancellationToken = default); + /// + /// Asynchronously inserts snapshot metadata for a specific dataset. + /// + /// The metadata to insert. + /// The dataset identifier. + /// A cancellation token. + Task InsertSnapshotMetadataAsync( + SnapshotMetadata metadata, + string datasetId, + CancellationToken cancellationToken = default) + { + return InsertSnapshotMetadataAsync(metadata, cancellationToken); + } + /// /// Asynchronously updates the metadata for an existing snapshot. /// @@ -36,6 +65,20 @@ public interface ISnapshotMetadataStore : ISnapshotable /// A task that represents the asynchronous update operation. Task UpdateSnapshotMetadataAsync(SnapshotMetadata existingMeta, CancellationToken cancellationToken = default); + /// + /// Asynchronously updates snapshot metadata for a specific dataset. + /// + /// The metadata to update. + /// The dataset identifier. + /// A cancellation token. + Task UpdateSnapshotMetadataAsync( + SnapshotMetadata existingMeta, + string datasetId, + CancellationToken cancellationToken = default) + { + return UpdateSnapshotMetadataAsync(existingMeta, cancellationToken); + } + /// /// Asynchronously retrieves the hash of the current snapshot for the specified node. /// @@ -44,10 +87,38 @@ public interface ISnapshotMetadataStore : ISnapshotable /// A task containing the snapshot hash as a string, or null if no snapshot is available. Task GetSnapshotHashAsync(string nodeId, CancellationToken cancellationToken = default); + /// + /// Asynchronously retrieves the snapshot hash for a node within a specific dataset. + /// + /// The node identifier. + /// The dataset identifier. + /// A cancellation token. + /// The snapshot hash when available. + Task GetSnapshotHashAsync( + string nodeId, + string datasetId, + CancellationToken cancellationToken = default) + { + return GetSnapshotHashAsync(nodeId, cancellationToken); + } + /// /// Gets all snapshot metadata entries. Used for initializing VectorClock cache. /// /// A cancellation token. /// All snapshot metadata entries. Task> GetAllSnapshotMetadataAsync(CancellationToken cancellationToken = default); -} \ No newline at end of file + + /// + /// Gets all snapshot metadata entries for a specific dataset. + /// + /// The dataset identifier. + /// A cancellation token. + /// All snapshot metadata entries for the dataset. + Task> GetAllSnapshotMetadataAsync( + string datasetId, + CancellationToken cancellationToken = default) + { + return GetAllSnapshotMetadataAsync(cancellationToken); + } +} diff --git a/src/ZB.MOM.WW.CBDDC.Core/Storage/ISnapshotService.cs b/src/ZB.MOM.WW.CBDDC.Core/Storage/ISnapshotService.cs index cf391ce..2a383d6 100755 --- a/src/ZB.MOM.WW.CBDDC.Core/Storage/ISnapshotService.cs +++ b/src/ZB.MOM.WW.CBDDC.Core/Storage/ISnapshotService.cs @@ -14,22 +14,58 @@ public interface ISnapshotService /// /// The stream to which the snapshot data will be written. /// A cancellation token that can be used to cancel the snapshot creation operation. - /// A task that represents the asynchronous snapshot creation operation. - Task CreateSnapshotAsync(Stream destination, CancellationToken cancellationToken = default); + /// A task that represents the asynchronous snapshot creation operation. + Task CreateSnapshotAsync(Stream destination, CancellationToken cancellationToken = default); + + /// + /// Asynchronously creates a snapshot scoped to a specific dataset and writes it to the destination stream. + /// + /// The stream receiving serialized snapshot content. + /// The dataset identifier. + /// A cancellation token. + /// A task that represents the asynchronous snapshot creation operation. + Task CreateSnapshotAsync(Stream destination, string datasetId, CancellationToken cancellationToken = default) + { + return CreateSnapshotAsync(destination, cancellationToken); + } /// /// Replaces the existing database with the contents provided in the specified stream asynchronously. /// /// A stream containing the new database data to be used for replacement. /// A cancellation token that can be used to cancel the operation. - /// A task that represents the asynchronous database replacement operation. - Task ReplaceDatabaseAsync(Stream databaseStream, CancellationToken cancellationToken = default); + /// A task that represents the asynchronous database replacement operation. + Task ReplaceDatabaseAsync(Stream databaseStream, CancellationToken cancellationToken = default); + + /// + /// Replaces data for a specific dataset with the contents provided in the stream. + /// + /// The stream containing replacement data. + /// The dataset identifier. + /// A cancellation token. + /// A task that represents the asynchronous replace operation. + Task ReplaceDatabaseAsync(Stream databaseStream, string datasetId, CancellationToken cancellationToken = default) + { + return ReplaceDatabaseAsync(databaseStream, cancellationToken); + } /// /// Merges the provided snapshot stream into the current data store asynchronously. /// /// A stream containing the snapshot data to be merged. /// A cancellation token that can be used to cancel the merge operation. - /// A task that represents the asynchronous merge operation. - Task MergeSnapshotAsync(Stream snapshotStream, CancellationToken cancellationToken = default); -} \ No newline at end of file + /// A task that represents the asynchronous merge operation. + Task MergeSnapshotAsync(Stream snapshotStream, CancellationToken cancellationToken = default); + + /// + /// Merges a snapshot stream into a specific dataset asynchronously. + /// + /// The stream containing snapshot data. + /// The dataset identifier. + /// A cancellation token. + /// A task that represents the asynchronous merge operation. + Task MergeSnapshotAsync(Stream snapshotStream, string datasetId, CancellationToken cancellationToken = default) + { + return MergeSnapshotAsync(snapshotStream, cancellationToken); + } +} diff --git a/src/ZB.MOM.WW.CBDDC.Core/Storage/ISnapshotable.cs b/src/ZB.MOM.WW.CBDDC.Core/Storage/ISnapshotable.cs index 9c12ca7..f2f4c67 100755 --- a/src/ZB.MOM.WW.CBDDC.Core/Storage/ISnapshotable.cs +++ b/src/ZB.MOM.WW.CBDDC.Core/Storage/ISnapshotable.cs @@ -13,9 +13,20 @@ public interface ISnapshotable /// /// After calling this method, the data store and all stored data will be permanently removed. /// This operation cannot be undone. Any further operations on the data store may result in errors. - /// - /// A task that represents the asynchronous drop operation. - Task DropAsync(CancellationToken cancellationToken = default); + /// + /// A task that represents the asynchronous drop operation. + Task DropAsync(CancellationToken cancellationToken = default); + + /// + /// Asynchronously deletes data for the specified dataset. + /// + /// The dataset identifier. + /// A cancellation token. + /// A task that represents the asynchronous drop operation. + Task DropAsync(string datasetId, CancellationToken cancellationToken = default) + { + return DropAsync(cancellationToken); + } /// /// Asynchronously exports a collection of items of type T. @@ -24,16 +35,39 @@ public interface ISnapshotable /// /// A task that represents the asynchronous export operation. The task result contains an enumerable collection of /// exported items of type T. - /// - Task> ExportAsync(CancellationToken cancellationToken = default); + /// + Task> ExportAsync(CancellationToken cancellationToken = default); + + /// + /// Asynchronously exports items for the specified dataset. + /// + /// The dataset identifier. + /// A cancellation token. + /// An enumerable collection of exported items. + Task> ExportAsync(string datasetId, CancellationToken cancellationToken = default) + { + return ExportAsync(cancellationToken); + } /// /// Imports the specified collection of items asynchronously. /// /// The collection of items to import. Cannot be null. Each item will be processed in sequence. - /// A cancellation token that can be used to cancel the import operation. - /// A task that represents the asynchronous import operation. - Task ImportAsync(IEnumerable items, CancellationToken cancellationToken = default); + /// A cancellation token that can be used to cancel the import operation. + /// A task that represents the asynchronous import operation. + Task ImportAsync(IEnumerable items, CancellationToken cancellationToken = default); + + /// + /// Imports items into the specified dataset asynchronously. + /// + /// The items to import. + /// The dataset identifier. + /// A cancellation token. + /// A task that represents the asynchronous import operation. + Task ImportAsync(IEnumerable items, string datasetId, CancellationToken cancellationToken = default) + { + return ImportAsync(items, cancellationToken); + } /// /// Merges the specified collection of items into the target data store asynchronously. @@ -44,7 +78,19 @@ public interface ISnapshotable /// implementation. /// /// The collection of items to merge into the data store. Cannot be null. - /// A cancellation token that can be used to cancel the merge operation. - /// A task that represents the asynchronous merge operation. - Task MergeAsync(IEnumerable items, CancellationToken cancellationToken = default); -} \ No newline at end of file + /// A cancellation token that can be used to cancel the merge operation. + /// A task that represents the asynchronous merge operation. + Task MergeAsync(IEnumerable items, CancellationToken cancellationToken = default); + + /// + /// Merges items into the specified dataset asynchronously. + /// + /// The items to merge. + /// The dataset identifier. + /// A cancellation token. + /// A task that represents the asynchronous merge operation. + Task MergeAsync(IEnumerable items, string datasetId, CancellationToken cancellationToken = default) + { + return MergeAsync(items, cancellationToken); + } +} diff --git a/src/ZB.MOM.WW.CBDDC.Network/DatasetSyncContext.cs b/src/ZB.MOM.WW.CBDDC.Network/DatasetSyncContext.cs new file mode 100644 index 0000000..5f08b8e --- /dev/null +++ b/src/ZB.MOM.WW.CBDDC.Network/DatasetSyncContext.cs @@ -0,0 +1,67 @@ +using ZB.MOM.WW.CBDDC.Core; +using ZB.MOM.WW.CBDDC.Core.Storage; + +namespace ZB.MOM.WW.CBDDC.Network; + +/// +/// Default dataset sync context implementation used by the runtime coordinator. +/// +public sealed class DatasetSyncContext : IDatasetSyncContext +{ + /// + /// Initializes a new instance of the class. + /// + /// The dataset identifier. + /// The dataset options. + /// The document store. + /// The oplog store. + /// The snapshot metadata store. + /// The snapshot service. + /// The optional peer confirmation store. + /// The dataset orchestrator. + public DatasetSyncContext( + string datasetId, + DatasetSyncOptions options, + IDocumentStore documentStore, + IOplogStore oplogStore, + ISnapshotMetadataStore snapshotMetadataStore, + ISnapshotService snapshotService, + IPeerOplogConfirmationStore? peerOplogConfirmationStore, + ISyncOrchestrator orchestrator) + { + DatasetId = global::ZB.MOM.WW.CBDDC.Core.DatasetId.Normalize(datasetId); + Options = options; + DocumentStore = documentStore; + OplogStore = oplogStore; + SnapshotMetadataStore = snapshotMetadataStore; + SnapshotService = snapshotService; + PeerOplogConfirmationStore = peerOplogConfirmationStore; + Orchestrator = orchestrator; + } + + /// + public string DatasetId { get; } + + /// + public DatasetSyncOptions Options { get; } + + /// + public IDocumentStore DocumentStore { get; } + + /// + public IOplogStore OplogStore { get; } + + /// + public ISnapshotMetadataStore SnapshotMetadataStore { get; } + + /// + public ISnapshotService SnapshotService { get; } + + /// + public IPeerOplogConfirmationStore? PeerOplogConfirmationStore { get; } + + /// + /// Gets the orchestrator instance for this dataset context. + /// + public ISyncOrchestrator Orchestrator { get; } +} diff --git a/src/ZB.MOM.WW.CBDDC.Network/MultiDatasetRuntimeOptions.cs b/src/ZB.MOM.WW.CBDDC.Network/MultiDatasetRuntimeOptions.cs new file mode 100644 index 0000000..c09bce8 --- /dev/null +++ b/src/ZB.MOM.WW.CBDDC.Network/MultiDatasetRuntimeOptions.cs @@ -0,0 +1,34 @@ +using ZB.MOM.WW.CBDDC.Core; + +namespace ZB.MOM.WW.CBDDC.Network; + +/// +/// Feature flags and defaults for multi-dataset sync runtime activation. +/// +public sealed class MultiDatasetRuntimeOptions +{ + /// + /// Gets or sets a value indicating whether multi-dataset sync runtime is enabled. + /// + public bool EnableMultiDatasetSync { get; set; } + + /// + /// Gets or sets a value indicating whether the primary dataset pipeline is enabled. + /// + public bool EnableDatasetPrimary { get; set; } = true; + + /// + /// Gets or sets a value indicating whether the logs dataset pipeline is enabled. + /// + public bool EnableDatasetLogs { get; set; } + + /// + /// Gets or sets a value indicating whether the timeseries dataset pipeline is enabled. + /// + public bool EnableDatasetTimeseries { get; set; } + + /// + /// Gets or sets additional dataset-specific runtime options. + /// + public List AdditionalDatasets { get; set; } = []; +} diff --git a/src/ZB.MOM.WW.CBDDC.Network/MultiDatasetSyncOrchestrator.cs b/src/ZB.MOM.WW.CBDDC.Network/MultiDatasetSyncOrchestrator.cs new file mode 100644 index 0000000..155dc83 --- /dev/null +++ b/src/ZB.MOM.WW.CBDDC.Network/MultiDatasetSyncOrchestrator.cs @@ -0,0 +1,199 @@ +using ZB.MOM.WW.CBDDC.Core; +using ZB.MOM.WW.CBDDC.Core.Network; +using ZB.MOM.WW.CBDDC.Core.Storage; +using Microsoft.Extensions.Logging; +using ZB.MOM.WW.CBDDC.Network.Security; +using ZB.MOM.WW.CBDDC.Network.Telemetry; + +namespace ZB.MOM.WW.CBDDC.Network; + +/// +/// Coordinates one sync orchestrator per dataset in-process. +/// +public sealed class MultiDatasetSyncOrchestrator : IMultiDatasetSyncOrchestrator +{ + private readonly ILoggerFactory _loggerFactory; + private readonly ILogger _logger; + + /// + /// Initializes a new instance of the class. + /// + /// The discovery service. + /// The shared oplog store supporting dataset partitioning. + /// The document store. + /// The snapshot metadata store. + /// The snapshot service. + /// The peer configuration provider. + /// The logger factory. + /// Registered dataset options from DI. + /// Runtime feature flags. + /// Optional peer confirmation store. + /// Optional handshake service. + /// Optional telemetry service. + /// Optional prune cutoff calculator. + /// Optional factory override for dataset orchestrators (used by tests). + public MultiDatasetSyncOrchestrator( + IDiscoveryService discovery, + IOplogStore oplogStore, + IDocumentStore documentStore, + ISnapshotMetadataStore snapshotMetadataStore, + ISnapshotService snapshotService, + IPeerNodeConfigurationProvider peerNodeConfigurationProvider, + ILoggerFactory loggerFactory, + IEnumerable registeredDatasetOptions, + MultiDatasetRuntimeOptions runtimeOptions, + IPeerOplogConfirmationStore? peerOplogConfirmationStore = null, + IPeerHandshakeService? handshakeService = null, + INetworkTelemetryService? telemetry = null, + IOplogPruneCutoffCalculator? oplogPruneCutoffCalculator = null, + Func? orchestratorFactory = null) + { + _loggerFactory = loggerFactory; + _logger = loggerFactory.CreateLogger(); + + List datasets = BuildDatasetList(registeredDatasetOptions, runtimeOptions); + Contexts = datasets + .Select(options => + { + var orchestrator = orchestratorFactory?.Invoke(options) ?? new SyncOrchestrator( + discovery, + oplogStore, + documentStore, + snapshotMetadataStore, + snapshotService, + peerNodeConfigurationProvider, + loggerFactory, + peerOplogConfirmationStore, + handshakeService, + telemetry, + oplogPruneCutoffCalculator, + options); + + return (IDatasetSyncContext)new DatasetSyncContext( + options.DatasetId, + options, + documentStore, + oplogStore, + snapshotMetadataStore, + snapshotService, + peerOplogConfirmationStore, + orchestrator); + }) + .ToList(); + } + + /// + public IReadOnlyCollection Contexts { get; } + + /// + public async Task Start() + { + foreach (var context in Contexts) + { + if (!context.Options.Enabled) continue; + + try + { + var runtimeContext = (DatasetSyncContext)context; + await runtimeContext.Orchestrator.Start(); + _logger.LogInformation("Started dataset orchestrator: {DatasetId}", context.DatasetId); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to start dataset orchestrator: {DatasetId}", context.DatasetId); + } + } + } + + /// + public async Task Stop() + { + foreach (var context in Contexts) + try + { + var runtimeContext = (DatasetSyncContext)context; + await runtimeContext.Orchestrator.Stop(); + _logger.LogInformation("Stopped dataset orchestrator: {DatasetId}", context.DatasetId); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to stop dataset orchestrator: {DatasetId}", context.DatasetId); + } + } + + private static List BuildDatasetList( + IEnumerable registeredDatasetOptions, + MultiDatasetRuntimeOptions runtimeOptions) + { + var optionMap = registeredDatasetOptions + .Select(Clone) + .GroupBy(o => DatasetId.Normalize(o.DatasetId)) + .ToDictionary(g => g.Key, g => g.Last(), StringComparer.Ordinal); + + foreach (var additional in runtimeOptions.AdditionalDatasets) + optionMap[DatasetId.Normalize(additional.DatasetId)] = Clone(additional); + + if (!runtimeOptions.EnableMultiDatasetSync) + return + [ + ResolveOrDefault(optionMap, DatasetId.Primary, enabled: true) + ]; + + var result = new List(); + if (runtimeOptions.EnableDatasetPrimary) + result.Add(ResolveOrDefault(optionMap, DatasetId.Primary, enabled: true)); + + if (runtimeOptions.EnableDatasetLogs) + result.Add(ResolveOrDefault(optionMap, DatasetId.Logs, enabled: true)); + + if (runtimeOptions.EnableDatasetTimeseries) + result.Add(ResolveOrDefault(optionMap, DatasetId.Timeseries, enabled: true)); + + foreach (var kvp in optionMap) + { + if (string.Equals(kvp.Key, DatasetId.Primary, StringComparison.Ordinal)) continue; + if (string.Equals(kvp.Key, DatasetId.Logs, StringComparison.Ordinal)) continue; + if (string.Equals(kvp.Key, DatasetId.Timeseries, StringComparison.Ordinal)) continue; + result.Add(kvp.Value); + } + + return result + .GroupBy(o => DatasetId.Normalize(o.DatasetId), StringComparer.Ordinal) + .Select(g => g.Last()) + .ToList(); + } + + private static DatasetSyncOptions ResolveOrDefault( + IReadOnlyDictionary optionMap, + string datasetId, + bool enabled) + { + var normalized = DatasetId.Normalize(datasetId); + if (optionMap.TryGetValue(normalized, out var existing)) + { + existing.DatasetId = normalized; + existing.Enabled = enabled && existing.Enabled; + return existing; + } + + return new DatasetSyncOptions + { + DatasetId = normalized, + Enabled = enabled + }; + } + + private static DatasetSyncOptions Clone(DatasetSyncOptions options) + { + return new DatasetSyncOptions + { + DatasetId = DatasetId.Normalize(options.DatasetId), + Enabled = options.Enabled, + SyncLoopDelay = options.SyncLoopDelay, + MaxPeersPerCycle = options.MaxPeersPerCycle, + MaxEntriesPerCycle = options.MaxEntriesPerCycle, + MaintenanceIntervalOverride = options.MaintenanceIntervalOverride, + InterestingCollections = options.InterestingCollections.ToList() + }; + } +} diff --git a/src/ZB.MOM.WW.CBDDC.Network/MultiDatasetSyncOrchestratorAdapter.cs b/src/ZB.MOM.WW.CBDDC.Network/MultiDatasetSyncOrchestratorAdapter.cs new file mode 100644 index 0000000..04888d0 --- /dev/null +++ b/src/ZB.MOM.WW.CBDDC.Network/MultiDatasetSyncOrchestratorAdapter.cs @@ -0,0 +1,32 @@ +using ZB.MOM.WW.CBDDC.Core.Storage; + +namespace ZB.MOM.WW.CBDDC.Network; + +/// +/// Adapts to . +/// +internal sealed class MultiDatasetSyncOrchestratorAdapter : ISyncOrchestrator +{ + private readonly IMultiDatasetSyncOrchestrator _inner; + + /// + /// Initializes a new instance of the class. + /// + /// The multi-dataset orchestrator. + public MultiDatasetSyncOrchestratorAdapter(IMultiDatasetSyncOrchestrator inner) + { + _inner = inner; + } + + /// + public Task Start() + { + return _inner.Start(); + } + + /// + public Task Stop() + { + return _inner.Stop(); + } +} diff --git a/src/ZB.MOM.WW.CBDDC.Network/PeerDbNetworkExtensions.cs b/src/ZB.MOM.WW.CBDDC.Network/PeerDbNetworkExtensions.cs index 71d3395..2ed7ea3 100755 --- a/src/ZB.MOM.WW.CBDDC.Network/PeerDbNetworkExtensions.cs +++ b/src/ZB.MOM.WW.CBDDC.Network/PeerDbNetworkExtensions.cs @@ -1,9 +1,11 @@ -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.DependencyInjection.Extensions; -using Microsoft.Extensions.Logging; -using ZB.MOM.WW.CBDDC.Core.Network; -using ZB.MOM.WW.CBDDC.Network.Security; -using ZB.MOM.WW.CBDDC.Network.Telemetry; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Logging; +using ZB.MOM.WW.CBDDC.Core; +using ZB.MOM.WW.CBDDC.Core.Storage; +using ZB.MOM.WW.CBDDC.Core.Network; +using ZB.MOM.WW.CBDDC.Network.Security; +using ZB.MOM.WW.CBDDC.Network.Telemetry; // For IMeshNetwork if we implement it namespace ZB.MOM.WW.CBDDC.Network; @@ -50,6 +52,40 @@ public static class CBDDCNetworkExtensions // Optionally register hosted service for automatic node lifecycle management if (useHostedService) services.AddHostedService(); - return services; - } -} \ No newline at end of file + return services; + } + + /// + /// Enables multi-dataset orchestration and replaces the single orchestrator with a dataset coordinator. + /// + /// The service collection. + /// Optional configuration for runtime feature flags and dataset defaults. + /// The service collection for chaining. + public static IServiceCollection AddCBDDCMultiDataset( + this IServiceCollection services, + Action? configure = null) + { + var runtimeOptions = new MultiDatasetRuntimeOptions + { + EnableMultiDatasetSync = true, + EnableDatasetPrimary = true + }; + configure?.Invoke(runtimeOptions); + + services.AddSingleton(runtimeOptions); + + // Ensure a primary dataset option exists for compatibility when callers did not register dataset options. + if (!services.Any(descriptor => descriptor.ServiceType == typeof(DatasetSyncOptions))) + services.AddSingleton(new DatasetSyncOptions + { + DatasetId = DatasetId.Primary, + Enabled = true + }); + + services.TryAddSingleton(); + services.Replace(ServiceDescriptor.Singleton(sp => + new MultiDatasetSyncOrchestratorAdapter(sp.GetRequiredService()))); + + return services; + } +} diff --git a/src/ZB.MOM.WW.CBDDC.Network/SyncOrchestrator.cs b/src/ZB.MOM.WW.CBDDC.Network/SyncOrchestrator.cs index a88c604..fb11c54 100755 --- a/src/ZB.MOM.WW.CBDDC.Network/SyncOrchestrator.cs +++ b/src/ZB.MOM.WW.CBDDC.Network/SyncOrchestrator.cs @@ -31,10 +31,12 @@ public class SyncOrchestrator : ISyncOrchestrator private readonly ConcurrentDictionary _peerStates = new(); private readonly Random _random = new(); private readonly ISnapshotMetadataStore _snapshotMetadataStore; - private readonly ISnapshotService _snapshotService; - private readonly object _startStopLock = new(); - private readonly INetworkTelemetryService? _telemetry; - private CancellationTokenSource? _cts; + private readonly ISnapshotService _snapshotService; + private readonly object _startStopLock = new(); + private readonly INetworkTelemetryService? _telemetry; + private readonly DatasetSyncOptions _datasetSyncOptions; + private readonly string _datasetId; + private CancellationTokenSource? _cts; private DateTime _lastMaintenanceTime = DateTime.MinValue; @@ -49,24 +51,26 @@ public class SyncOrchestrator : ISyncOrchestrator /// The peer configuration provider. /// The logger factory. /// The optional peer confirmation watermark store. - /// The optional peer handshake service. - /// The optional network telemetry service. - /// The optional cutoff calculator for safe maintenance pruning. - public SyncOrchestrator( - IDiscoveryService discovery, - IOplogStore oplogStore, + /// The optional peer handshake service. + /// The optional network telemetry service. + /// The optional cutoff calculator for safe maintenance pruning. + /// The optional per-dataset synchronization options. + public SyncOrchestrator( + IDiscoveryService discovery, + IOplogStore oplogStore, IDocumentStore documentStore, ISnapshotMetadataStore snapshotStore, ISnapshotService snapshotService, IPeerNodeConfigurationProvider peerNodeConfigurationProvider, - ILoggerFactory loggerFactory, - IPeerOplogConfirmationStore? peerOplogConfirmationStore = null, - IPeerHandshakeService? handshakeService = null, - INetworkTelemetryService? telemetry = null, - IOplogPruneCutoffCalculator? oplogPruneCutoffCalculator = null) - { - _discovery = discovery; - _oplogStore = oplogStore; + ILoggerFactory loggerFactory, + IPeerOplogConfirmationStore? peerOplogConfirmationStore = null, + IPeerHandshakeService? handshakeService = null, + INetworkTelemetryService? telemetry = null, + IOplogPruneCutoffCalculator? oplogPruneCutoffCalculator = null, + DatasetSyncOptions? datasetSyncOptions = null) + { + _discovery = discovery; + _oplogStore = oplogStore; _oplogPruneCutoffCalculator = oplogPruneCutoffCalculator; _peerOplogConfirmationStore = peerOplogConfirmationStore; _documentStore = documentStore; @@ -74,10 +78,12 @@ public class SyncOrchestrator : ISyncOrchestrator _snapshotService = snapshotService; _peerNodeConfigurationProvider = peerNodeConfigurationProvider; _loggerFactory = loggerFactory; - _logger = loggerFactory.CreateLogger(); - _handshakeService = handshakeService; - _telemetry = telemetry; - } + _logger = loggerFactory.CreateLogger(); + _handshakeService = handshakeService; + _telemetry = telemetry; + _datasetSyncOptions = datasetSyncOptions ?? new DatasetSyncOptions(); + _datasetId = DatasetId.Normalize(_datasetSyncOptions.DatasetId); + } /// /// Starts the synchronization orchestrator loop. @@ -164,11 +170,11 @@ public class SyncOrchestrator : ISyncOrchestrator /// /// Main synchronization loop. Periodically selects random peers to gossip with. /// - private async Task SyncLoopAsync(CancellationToken token) - { - _logger.LogInformation("Sync Orchestrator Started (Parallel P2P)"); - while (!token.IsCancellationRequested) - { + private async Task SyncLoopAsync(CancellationToken token) + { + _logger.LogInformation("Sync Orchestrator Started (Parallel P2P, Dataset: {DatasetId})", _datasetId); + while (!token.IsCancellationRequested) + { var config = await _peerNodeConfigurationProvider.GetConfiguration(); try { @@ -192,13 +198,13 @@ public class SyncOrchestrator : ISyncOrchestrator return true; }).ToList(); - // Interest-Aware Gossip: Prioritize peers sharing interests with us - var localInterests = _documentStore.InterestedCollection.ToList(); - var targets = eligiblePeers - .OrderByDescending(p => p.InterestingCollections.Any(ci => localInterests.Contains(ci))) - .ThenBy(x => _random.Next()) - .Take(3) - .ToList(); + // Interest-Aware Gossip: Prioritize peers sharing interests with us + var localInterests = GetDatasetInterests(); + var targets = eligiblePeers + .OrderByDescending(p => p.InterestingCollections.Any(ci => localInterests.Contains(ci))) + .ThenBy(x => _random.Next()) + .Take(Math.Max(1, _datasetSyncOptions.MaxPeersPerCycle)) + .ToList(); // NetStandard 2.0 fallback: Use Task.WhenAll var tasks = targets.Select(peer => TrySyncWithPeer(peer, token)); @@ -216,10 +222,10 @@ public class SyncOrchestrator : ISyncOrchestrator _logger.LogError(ex, "Sync Loop Error"); } - try - { - await Task.Delay(2000, token); - } + try + { + await Task.Delay(_datasetSyncOptions.SyncLoopDelay, token); + } catch (OperationCanceledException) { break; @@ -234,9 +240,10 @@ public class SyncOrchestrator : ISyncOrchestrator /// The current UTC time used for interval evaluation. /// The cancellation token. /// A task that represents the asynchronous maintenance operation. - internal async Task RunMaintenanceIfDueAsync(PeerNodeConfiguration config, DateTime now, CancellationToken token) - { - var maintenanceInterval = TimeSpan.FromMinutes(config.MaintenanceIntervalMinutes); + internal async Task RunMaintenanceIfDueAsync(PeerNodeConfiguration config, DateTime now, CancellationToken token) + { + var maintenanceInterval = _datasetSyncOptions.MaintenanceIntervalOverride ?? + TimeSpan.FromMinutes(config.MaintenanceIntervalMinutes); if (now - _lastMaintenanceTime < maintenanceInterval) return; _logger.LogInformation("Running periodic maintenance (Oplog pruning)..."); @@ -253,7 +260,7 @@ public class SyncOrchestrator : ISyncOrchestrator return; } - await _oplogStore.PruneOplogAsync(cutoffDecision.EffectiveCutoff.Value, token); + await _oplogStore.PruneOplogAsync(cutoffDecision.EffectiveCutoff.Value, _datasetId, token); _lastMaintenanceTime = now; if (cutoffDecision.ConfirmationCutoff.HasValue) @@ -311,7 +318,8 @@ public class SyncOrchestrator : ISyncOrchestrator try { - var config = await _peerNodeConfigurationProvider.GetConfiguration(); + var config = await _peerNodeConfigurationProvider.GetConfiguration(); + var localInterests = GetDatasetInterests(); // Get or create persistent client client = _clients.GetOrAdd(peer.NodeId, id => new TcpPeerClient( @@ -323,18 +331,18 @@ public class SyncOrchestrator : ISyncOrchestrator // Reconnect if disconnected if (!client.IsConnected) await client.ConnectAsync(token); - // Handshake (idempotent) - if (!await client.HandshakeAsync(config.NodeId, config.AuthToken, _documentStore.InterestedCollection, - token)) - { - _logger.LogWarning("Handshake rejected by {NodeId}", peer.NodeId); - shouldRemoveClient = true; - throw new Exception("Handshake rejected"); - } - - // 1. Exchange Vector Clocks - var remoteVectorClock = await client.GetVectorClockAsync(token); - var localVectorClock = await _oplogStore.GetVectorClockAsync(token); + // Handshake (idempotent) + if (!await client.HandshakeAsync(config.NodeId, config.AuthToken, localInterests, + _datasetId, token)) + { + _logger.LogWarning("Handshake rejected by {NodeId}", peer.NodeId); + shouldRemoveClient = true; + throw new Exception("Handshake rejected"); + } + + // 1. Exchange Vector Clocks + var remoteVectorClock = await client.GetVectorClockAsync(token); + var localVectorClock = await _oplogStore.GetVectorClockAsync(_datasetId, token); _logger.LogDebug("Vector Clock - Local: {Local}, Remote: {Remote}", localVectorClock, remoteVectorClock); @@ -359,9 +367,9 @@ public class SyncOrchestrator : ISyncOrchestrator _logger.LogDebug("Pulling Node {NodeId}: Local={LocalTs}, Remote={RemoteTs}", nodeId, localTs, remoteTs); - // PASS LOCAL INTERESTS TO PULL - var changes = await client.PullChangesFromNodeAsync(nodeId, localTs, - _documentStore.InterestedCollection, token); + // PASS LOCAL INTERESTS TO PULL + var changes = await client.PullChangesFromNodeAsync(nodeId, localTs, + localInterests, _datasetId, token); if (changes != null && changes.Count > 0) { var result = await ProcessInboundBatchAsync(client, peer.NodeId, changes, token); @@ -389,17 +397,17 @@ public class SyncOrchestrator : ISyncOrchestrator // PUSH FILTERING: Pass remote receiver's interests to oplogStore for efficient retrieval var remoteInterests = client.RemoteInterests; - var changes = - (await _oplogStore.GetOplogForNodeAfterAsync(nodeId, remoteTs, remoteInterests, token)) - .ToList(); + var changes = + (await _oplogStore.GetOplogForNodeAfterAsync(nodeId, remoteTs, _datasetId, remoteInterests, token)) + .ToList(); if (changes.Any()) { - _logger.LogDebug("Pushing {Count} filtered changes for Node {NodeId}", changes.Count, nodeId); - await client.PushChangesAsync(changes, token); - await AdvanceConfirmationForPushedBatchAsync(peer.NodeId, nodeId, changes, token); - } - } + _logger.LogDebug("Pushing {Count} filtered changes for Node {NodeId}", changes.Count, nodeId); + await client.PushChangesAsync(changes, _datasetId, token); + await AdvanceConfirmationForPushedBatchAsync(peer.NodeId, nodeId, changes, token); + } + } } // 5. Handle Concurrent/Equal cases @@ -551,8 +559,8 @@ public class SyncOrchestrator : ISyncOrchestrator try { - await _peerOplogConfirmationStore.EnsurePeerRegisteredAsync(peer.NodeId, peer.Address, peer.Type, - token); + await _peerOplogConfirmationStore.EnsurePeerRegisteredAsync(peer.NodeId, peer.Address, peer.Type, + _datasetId, token); } catch (OperationCanceledException) when (token.IsCancellationRequested) { @@ -618,12 +626,13 @@ public class SyncOrchestrator : ISyncOrchestrator try { - await _peerOplogConfirmationStore.UpdateConfirmationAsync( - peerNodeId, - sourceNodeId, - maxPushed.Timestamp, - maxPushed.Hash ?? string.Empty, - token); + await _peerOplogConfirmationStore.UpdateConfirmationAsync( + peerNodeId, + sourceNodeId, + maxPushed.Timestamp, + maxPushed.Hash ?? string.Empty, + _datasetId, + token); } catch (OperationCanceledException) when (token.IsCancellationRequested) { @@ -647,9 +656,15 @@ public class SyncOrchestrator : ISyncOrchestrator try { - // Best-effort hash lookup: IOplogStore exposes latest hash per source node. - string hash = await _oplogStore.GetLastEntryHashAsync(sourceNodeId, token) ?? string.Empty; - await _peerOplogConfirmationStore.UpdateConfirmationAsync(peerNodeId, sourceNodeId, timestamp, hash, token); + // Best-effort hash lookup: IOplogStore exposes latest hash per source node. + string hash = await _oplogStore.GetLastEntryHashAsync(sourceNodeId, _datasetId, token) ?? string.Empty; + await _peerOplogConfirmationStore.UpdateConfirmationAsync( + peerNodeId, + sourceNodeId, + timestamp, + hash, + _datasetId, + token); } catch (OperationCanceledException) when (token.IsCancellationRequested) { @@ -716,7 +731,7 @@ public class SyncOrchestrator : ISyncOrchestrator // Check linkage with Local State var firstEntry = authorChain[0]; - string? localHeadHash = await _oplogStore.GetLastEntryHashAsync(authorNodeId, token); + string? localHeadHash = await _oplogStore.GetLastEntryHashAsync(authorNodeId, _datasetId, token); _logger.LogDebug( "Processing chain for Node {AuthorId}: FirstEntry.PrevHash={PrevHash}, FirstEntry.Hash={Hash}, LocalHeadHash={LocalHead}", @@ -725,7 +740,7 @@ public class SyncOrchestrator : ISyncOrchestrator if (localHeadHash != null && firstEntry.PreviousHash != localHeadHash) { // Check if entry starts from snapshot boundary (valid case after pruning) - string? snapshotHash = await _snapshotMetadataStore.GetSnapshotHashAsync(authorNodeId, token); + string? snapshotHash = await _snapshotMetadataStore.GetSnapshotHashAsync(authorNodeId, _datasetId, token); if (snapshotHash != null && firstEntry.PreviousHash == snapshotHash) { @@ -748,7 +763,7 @@ public class SyncOrchestrator : ISyncOrchestrator List? missingChain = null; try { - missingChain = await client.GetChainRangeAsync(localHeadHash, firstEntry.PreviousHash, token); + missingChain = await client.GetChainRangeAsync(localHeadHash, firstEntry.PreviousHash, _datasetId, token); } catch (SnapshotRequiredException) { @@ -779,7 +794,7 @@ public class SyncOrchestrator : ISyncOrchestrator } // Apply Missing Chain First - await _oplogStore.ApplyBatchAsync(missingChain, token); + await _oplogStore.ApplyBatchAsync(missingChain, _datasetId, token); _logger.LogInformation("Gap Recovery Applied Successfully."); } else @@ -808,13 +823,13 @@ public class SyncOrchestrator : ISyncOrchestrator } // Apply original batch (grouped by node for clarity, but oplogStore usually handles bulk) - await _oplogStore.ApplyBatchAsync(authorChain, token); + await _oplogStore.ApplyBatchAsync(authorChain, _datasetId, token); } return SyncBatchResult.Success; } - private async Task PerformSnapshotSyncAsync(TcpPeerClient client, bool mergeOnly, CancellationToken token) + private async Task PerformSnapshotSyncAsync(TcpPeerClient client, bool mergeOnly, CancellationToken token) { _logger.LogInformation(mergeOnly ? "Starting Snapshot Merge..." : "Starting Full Database Replacement..."); @@ -824,17 +839,17 @@ public class SyncOrchestrator : ISyncOrchestrator _logger.LogInformation("Downloading snapshot to {TempFile}...", tempFile); using (var fs = File.Create(tempFile)) { - await client.GetSnapshotAsync(fs, token); + await client.GetSnapshotAsync(fs, _datasetId, token); } _logger.LogInformation("Snapshot Downloaded. applying to store..."); using (var fs = File.OpenRead(tempFile)) { - if (mergeOnly) - await _snapshotService.MergeSnapshotAsync(fs, token); - else - await _snapshotService.ReplaceDatabaseAsync(fs, token); + if (mergeOnly) + await _snapshotService.MergeSnapshotAsync(fs, _datasetId, token); + else + await _snapshotService.ReplaceDatabaseAsync(fs, _datasetId, token); } _logger.LogInformation("Snapshot applied successfully."); @@ -855,11 +870,22 @@ public class SyncOrchestrator : ISyncOrchestrator { _logger.LogWarning(ex, "Failed to delete temporary snapshot file {TempFile}", tempFile); } - } - } - - private class PeerStatus - { + } + } + + private List GetDatasetInterests() + { + if (_datasetSyncOptions.InterestingCollections.Count == 0) + return _documentStore.InterestedCollection.ToList(); + + return _datasetSyncOptions.InterestingCollections + .Where(value => !string.IsNullOrWhiteSpace(value)) + .Distinct(StringComparer.Ordinal) + .ToList(); + } + + private class PeerStatus + { /// /// Gets or sets the number of consecutive failures for the peer. /// @@ -882,4 +908,4 @@ public class SyncOrchestrator : ISyncOrchestrator IntegrityError, ChainBroken } -} \ No newline at end of file +} diff --git a/src/ZB.MOM.WW.CBDDC.Network/TcpPeerClient.cs b/src/ZB.MOM.WW.CBDDC.Network/TcpPeerClient.cs index 3080a5b..9a49088 100755 --- a/src/ZB.MOM.WW.CBDDC.Network/TcpPeerClient.cs +++ b/src/ZB.MOM.WW.CBDDC.Network/TcpPeerClient.cs @@ -168,10 +168,10 @@ public class TcpPeerClient : IDisposable /// The authentication token. /// Cancellation token. /// True if handshake was accepted, false otherwise. - public async Task HandshakeAsync(string myNodeId, string authToken, CancellationToken token) - { - return await HandshakeAsync(myNodeId, authToken, null, token); - } + public async Task HandshakeAsync(string myNodeId, string authToken, CancellationToken token) + { + return await HandshakeAsync(myNodeId, authToken, null, DatasetId.Primary, token); + } /// /// Performs authentication handshake with the remote peer, including collection interests. @@ -181,17 +181,38 @@ public class TcpPeerClient : IDisposable /// Optional collection names this node is interested in receiving. /// Cancellation token. /// if handshake was accepted; otherwise . - public async Task HandshakeAsync(string myNodeId, string authToken, - IEnumerable? interestingCollections, CancellationToken token) - { - if (HasHandshaked) return true; - - if (_handshakeService != null) + public async Task HandshakeAsync(string myNodeId, string authToken, + IEnumerable? interestingCollections, CancellationToken token) + { + return await HandshakeAsync(myNodeId, authToken, interestingCollections, DatasetId.Primary, token); + } + + /// + /// Performs authentication handshake with the remote peer for a specific dataset. + /// + /// The local node identifier. + /// The authentication token. + /// Optional collection names this node is interested in receiving. + /// The dataset identifier. + /// Cancellation token. + /// if handshake was accepted; otherwise . + public async Task HandshakeAsync(string myNodeId, string authToken, + IEnumerable? interestingCollections, string datasetId, CancellationToken token) + { + if (HasHandshaked) return true; + string normalizedDatasetId = DatasetId.Normalize(datasetId); + + if (_handshakeService != null) // Perform secure handshake if service is available // We assume we are initiator here _cipherState = await _handshakeService.HandshakeAsync(_stream!, true, myNodeId, token); - var req = new HandshakeRequest { NodeId = myNodeId, AuthToken = authToken ?? "" }; + var req = new HandshakeRequest + { + NodeId = myNodeId, + AuthToken = authToken ?? "", + DatasetId = normalizedDatasetId + }; if (interestingCollections != null) foreach (string coll in interestingCollections) @@ -207,10 +228,17 @@ public class TcpPeerClient : IDisposable if (type != MessageType.HandshakeRes) return false; - var res = HandshakeResponse.Parser.ParseFrom(payload); - - // Store remote interests - _remoteInterests = res.InterestingCollections.ToList(); + var res = HandshakeResponse.Parser.ParseFrom(payload); + string resolvedResponseDatasetId = DatasetId.Normalize(res.DatasetId); + if ((res.HasDatasetSupported && !res.DatasetSupported) || + !string.Equals(resolvedResponseDatasetId, normalizedDatasetId, StringComparison.Ordinal)) + { + HasHandshaked = false; + return false; + } + + // Store remote interests + _remoteInterests = res.InterestingCollections.ToList(); // Negotiation Result if (res.SelectedCompression == "brotli") @@ -274,10 +302,10 @@ public class TcpPeerClient : IDisposable /// The starting timestamp for requested changes. /// Cancellation token. /// The list of oplog entries returned by the remote peer. - public async Task> PullChangesAsync(HlcTimestamp since, CancellationToken token) - { - return await PullChangesAsync(since, null, token); - } + public async Task> PullChangesAsync(HlcTimestamp since, CancellationToken token) + { + return await PullChangesAsync(since, null, DatasetId.Primary, token); + } /// /// Pulls oplog changes from the remote peer since the specified timestamp, filtered by collections. @@ -286,16 +314,32 @@ public class TcpPeerClient : IDisposable /// Optional collection names used to filter the returned entries. /// Cancellation token. /// The list of oplog entries returned by the remote peer. - public async Task> PullChangesAsync(HlcTimestamp since, IEnumerable? collections, - CancellationToken token) - { - var req = new PullChangesRequest - { - SinceWall = since.PhysicalTime, - SinceLogic = since.LogicalCounter, - // Empty SinceNode indicates a global pull (not source-node filtered). - SinceNode = string.Empty - }; + public async Task> PullChangesAsync(HlcTimestamp since, IEnumerable? collections, + CancellationToken token) + { + return await PullChangesAsync(since, collections, DatasetId.Primary, token); + } + + /// + /// Pulls oplog changes from the remote peer for a specific dataset. + /// + /// The starting timestamp for requested changes. + /// Optional collection names used to filter the returned entries. + /// The dataset identifier. + /// Cancellation token. + /// The list of oplog entries returned by the remote peer. + public async Task> PullChangesAsync(HlcTimestamp since, IEnumerable? collections, + string datasetId, CancellationToken token) + { + string normalizedDatasetId = DatasetId.Normalize(datasetId); + var req = new PullChangesRequest + { + SinceWall = since.PhysicalTime, + SinceLogic = since.LogicalCounter, + // Empty SinceNode indicates a global pull (not source-node filtered). + SinceNode = string.Empty, + DatasetId = normalizedDatasetId + }; if (collections != null) foreach (string coll in collections) req.Collections.Add(coll); @@ -311,13 +355,14 @@ public class TcpPeerClient : IDisposable return res.Entries.Select(e => new OplogEntry( e.Collection, e.Key, - ParseOp(e.Operation), - string.IsNullOrEmpty(e.JsonData) ? default : JsonSerializer.Deserialize(e.JsonData), - new HlcTimestamp(e.HlcWall, e.HlcLogic, e.HlcNode), - e.PreviousHash, - e.Hash // Pass the received hash to preserve integrity reference - )).ToList(); - } + ParseOp(e.Operation), + string.IsNullOrEmpty(e.JsonData) ? default : JsonSerializer.Deserialize(e.JsonData), + new HlcTimestamp(e.HlcWall, e.HlcLogic, e.HlcNode), + e.PreviousHash, + e.Hash, // Pass the received hash to preserve integrity reference + string.IsNullOrWhiteSpace(e.DatasetId) ? normalizedDatasetId : e.DatasetId + )).ToList(); + } /// /// Pulls oplog changes for a specific node from the remote peer since the specified timestamp. @@ -326,11 +371,11 @@ public class TcpPeerClient : IDisposable /// The starting timestamp for requested changes. /// Cancellation token. /// The list of oplog entries returned by the remote peer. - public async Task> PullChangesFromNodeAsync(string nodeId, HlcTimestamp since, - CancellationToken token) - { - return await PullChangesFromNodeAsync(nodeId, since, null, token); - } + public async Task> PullChangesFromNodeAsync(string nodeId, HlcTimestamp since, + CancellationToken token) + { + return await PullChangesFromNodeAsync(nodeId, since, null, DatasetId.Primary, token); + } /// /// Pulls oplog changes for a specific node from the remote peer since the specified timestamp, filtered by @@ -341,15 +386,32 @@ public class TcpPeerClient : IDisposable /// Optional collection names used to filter the returned entries. /// Cancellation token. /// The list of oplog entries returned by the remote peer. - public async Task> PullChangesFromNodeAsync(string nodeId, HlcTimestamp since, - IEnumerable? collections, CancellationToken token) - { - var req = new PullChangesRequest - { - SinceNode = nodeId, - SinceWall = since.PhysicalTime, - SinceLogic = since.LogicalCounter - }; + public async Task> PullChangesFromNodeAsync(string nodeId, HlcTimestamp since, + IEnumerable? collections, CancellationToken token) + { + return await PullChangesFromNodeAsync(nodeId, since, collections, DatasetId.Primary, token); + } + + /// + /// Pulls oplog changes for a specific node and dataset from the remote peer. + /// + /// The node identifier to filter changes by. + /// The starting timestamp for requested changes. + /// Optional collection names used to filter the returned entries. + /// The dataset identifier. + /// Cancellation token. + /// The list of oplog entries returned by the remote peer. + public async Task> PullChangesFromNodeAsync(string nodeId, HlcTimestamp since, + IEnumerable? collections, string datasetId, CancellationToken token) + { + string normalizedDatasetId = DatasetId.Normalize(datasetId); + var req = new PullChangesRequest + { + SinceNode = nodeId, + SinceWall = since.PhysicalTime, + SinceLogic = since.LogicalCounter, + DatasetId = normalizedDatasetId + }; if (collections != null) foreach (string coll in collections) req.Collections.Add(coll); @@ -365,13 +427,14 @@ public class TcpPeerClient : IDisposable return res.Entries.Select(e => new OplogEntry( e.Collection, e.Key, - ParseOp(e.Operation), - string.IsNullOrEmpty(e.JsonData) ? default : JsonSerializer.Deserialize(e.JsonData), - new HlcTimestamp(e.HlcWall, e.HlcLogic, e.HlcNode), - e.PreviousHash, - e.Hash - )).ToList(); - } + ParseOp(e.Operation), + string.IsNullOrEmpty(e.JsonData) ? default : JsonSerializer.Deserialize(e.JsonData), + new HlcTimestamp(e.HlcWall, e.HlcLogic, e.HlcNode), + e.PreviousHash, + e.Hash, + string.IsNullOrWhiteSpace(e.DatasetId) ? normalizedDatasetId : e.DatasetId + )).ToList(); + } /// /// Retrieves a range of oplog entries connecting two hashes (Gap Recovery). @@ -380,10 +443,30 @@ public class TcpPeerClient : IDisposable /// The ending hash in the chain. /// Cancellation token. /// The chain entries connecting the requested hash range. - public virtual async Task> GetChainRangeAsync(string startHash, string endHash, - CancellationToken token) - { - var req = new GetChainRangeRequest { StartHash = startHash, EndHash = endHash }; + public virtual async Task> GetChainRangeAsync(string startHash, string endHash, + CancellationToken token) + { + return await GetChainRangeAsync(startHash, endHash, DatasetId.Primary, token); + } + + /// + /// Retrieves a range of oplog entries connecting two hashes for a specific dataset. + /// + /// The starting hash in the chain. + /// The ending hash in the chain. + /// The dataset identifier. + /// Cancellation token. + /// The chain entries connecting the requested hash range. + public virtual async Task> GetChainRangeAsync(string startHash, string endHash, + string datasetId, CancellationToken token) + { + string normalizedDatasetId = DatasetId.Normalize(datasetId); + var req = new GetChainRangeRequest + { + StartHash = startHash, + EndHash = endHash, + DatasetId = normalizedDatasetId + }; await _protocol.SendMessageAsync(_stream!, MessageType.GetChainRangeReq, req, _useCompression, _cipherState, token); @@ -397,13 +480,14 @@ public class TcpPeerClient : IDisposable return res.Entries.Select(e => new OplogEntry( e.Collection, e.Key, - ParseOp(e.Operation), - string.IsNullOrEmpty(e.JsonData) ? default : JsonSerializer.Deserialize(e.JsonData), - new HlcTimestamp(e.HlcWall, e.HlcLogic, e.HlcNode), - e.PreviousHash, - e.Hash - )).ToList(); - } + ParseOp(e.Operation), + string.IsNullOrEmpty(e.JsonData) ? default : JsonSerializer.Deserialize(e.JsonData), + new HlcTimestamp(e.HlcWall, e.HlcLogic, e.HlcNode), + e.PreviousHash, + e.Hash, + string.IsNullOrWhiteSpace(e.DatasetId) ? normalizedDatasetId : e.DatasetId + )).ToList(); + } /// /// Pushes local oplog changes to the remote peer. @@ -411,11 +495,27 @@ public class TcpPeerClient : IDisposable /// The oplog entries to push. /// Cancellation token. /// A task that represents the asynchronous push operation. - public async Task PushChangesAsync(IEnumerable entries, CancellationToken token) - { - var req = new PushChangesRequest(); - var entryList = entries.ToList(); - if (entryList.Count == 0) return; + public async Task PushChangesAsync(IEnumerable entries, CancellationToken token) + { + await PushChangesAsync(entries, DatasetId.Primary, token); + } + + /// + /// Pushes local oplog changes to the remote peer for a specific dataset. + /// + /// The oplog entries to push. + /// The dataset identifier. + /// Cancellation token. + /// A task that represents the asynchronous push operation. + public async Task PushChangesAsync(IEnumerable entries, string datasetId, CancellationToken token) + { + string normalizedDatasetId = DatasetId.Normalize(datasetId); + var req = new PushChangesRequest + { + DatasetId = normalizedDatasetId + }; + var entryList = entries.ToList(); + if (entryList.Count == 0) return; foreach (var e in entryList) req.Entries.Add(new ProtoOplogEntry @@ -425,11 +525,12 @@ public class TcpPeerClient : IDisposable Operation = e.Operation.ToString(), JsonData = e.Payload?.GetRawText() ?? "", HlcWall = e.Timestamp.PhysicalTime, - HlcLogic = e.Timestamp.LogicalCounter, - HlcNode = e.Timestamp.NodeId, - Hash = e.Hash, - PreviousHash = e.PreviousHash - }); + HlcLogic = e.Timestamp.LogicalCounter, + HlcNode = e.Timestamp.NodeId, + Hash = e.Hash, + PreviousHash = e.PreviousHash, + DatasetId = string.IsNullOrWhiteSpace(e.DatasetId) ? normalizedDatasetId : e.DatasetId + }); await _protocol.SendMessageAsync(_stream!, MessageType.PushChangesReq, req, _useCompression, _cipherState, token); @@ -453,10 +554,25 @@ public class TcpPeerClient : IDisposable /// The stream that receives snapshot bytes. /// Cancellation token. /// A task that represents the asynchronous snapshot transfer operation. - public async Task GetSnapshotAsync(Stream destination, CancellationToken token) - { - await _protocol.SendMessageAsync(_stream!, MessageType.GetSnapshotReq, new GetSnapshotRequest(), - _useCompression, _cipherState, token); + public async Task GetSnapshotAsync(Stream destination, CancellationToken token) + { + await GetSnapshotAsync(destination, DatasetId.Primary, token); + } + + /// + /// Downloads a full snapshot for a specific dataset from the remote peer. + /// + /// The stream that receives snapshot bytes. + /// The dataset identifier. + /// Cancellation token. + /// A task that represents the asynchronous snapshot transfer operation. + public async Task GetSnapshotAsync(Stream destination, string datasetId, CancellationToken token) + { + await _protocol.SendMessageAsync(_stream!, MessageType.GetSnapshotReq, new GetSnapshotRequest + { + DatasetId = DatasetId.Normalize(datasetId) + }, + _useCompression, _cipherState, token); while (true) { @@ -468,9 +584,9 @@ public class TcpPeerClient : IDisposable if (chunk.Data.Length > 0) await destination.WriteAsync(chunk.Data.ToByteArray(), 0, chunk.Data.Length, token); - if (chunk.IsLast) break; - } - } + if (chunk.IsLast) break; + } + } } public class SnapshotRequiredException : Exception @@ -481,4 +597,4 @@ public class SnapshotRequiredException : Exception public SnapshotRequiredException() : base("Peer requires a full snapshot sync.") { } -} \ No newline at end of file +} diff --git a/src/ZB.MOM.WW.CBDDC.Network/TcpSyncServer.cs b/src/ZB.MOM.WW.CBDDC.Network/TcpSyncServer.cs index 7aa9aa0..d4fa211 100755 --- a/src/ZB.MOM.WW.CBDDC.Network/TcpSyncServer.cs +++ b/src/ZB.MOM.WW.CBDDC.Network/TcpSyncServer.cs @@ -245,9 +245,10 @@ internal class TcpSyncServer : ISyncServer var protocol = new ProtocolHandler(_logger, _telemetry); - var useCompression = false; - CipherState? cipherState = null; - List remoteInterests = new(); + var useCompression = false; + CipherState? cipherState = null; + List remoteInterests = new(); + string currentDatasetId = DatasetId.Primary; // Perform Secure Handshake (if service is available) var config = await _configProvider.GetConfiguration(); @@ -276,11 +277,13 @@ internal class TcpSyncServer : ISyncServer // Handshake Loop if (type == MessageType.HandshakeReq) { - var hReq = HandshakeRequest.Parser.ParseFrom(payload); - _logger.LogDebug("Received HandshakeReq from Node {NodeId}", hReq.NodeId); - - // Track remote peer interests - remoteInterests = hReq.InterestingCollections.ToList(); + var hReq = HandshakeRequest.Parser.ParseFrom(payload); + _logger.LogDebug("Received HandshakeReq from Node {NodeId}", hReq.NodeId); + string requestedDatasetId = DatasetId.Normalize(hReq.DatasetId); + currentDatasetId = requestedDatasetId; + + // Track remote peer interests + remoteInterests = hReq.InterestingCollections.ToList(); bool valid = await _authenticator.ValidateAsync(hReq.NodeId, hReq.AuthToken); if (!valid) @@ -292,7 +295,13 @@ internal class TcpSyncServer : ISyncServer return; } - var hRes = new HandshakeResponse { NodeId = config.NodeId, Accepted = true }; + var hRes = new HandshakeResponse + { + NodeId = config.NodeId, + Accepted = true, + DatasetId = requestedDatasetId, + DatasetSupported = true + }; // Include local interests from IDocumentStore in response for push filtering foreach (string coll in _documentStore.InterestedCollection) @@ -314,10 +323,10 @@ internal class TcpSyncServer : ISyncServer switch (type) { - case MessageType.GetClockReq: - var clock = await _oplogStore.GetLatestTimestampAsync(token); - response = new ClockResponse - { + case MessageType.GetClockReq: + var clock = await _oplogStore.GetLatestTimestampAsync(currentDatasetId, token); + response = new ClockResponse + { HlcWall = clock.PhysicalTime, HlcLogic = clock.LogicalCounter, HlcNode = clock.NodeId @@ -325,8 +334,8 @@ internal class TcpSyncServer : ISyncServer resType = MessageType.ClockRes; break; - case MessageType.GetVectorClockReq: - var vectorClock = await _oplogStore.GetVectorClockAsync(token); + case MessageType.GetVectorClockReq: + var vectorClock = await _oplogStore.GetVectorClockAsync(currentDatasetId, token); var vcRes = new VectorClockResponse(); foreach (string nodeId in vectorClock.NodeIds) { @@ -343,59 +352,74 @@ internal class TcpSyncServer : ISyncServer resType = MessageType.VectorClockRes; break; - case MessageType.PullChangesReq: - var pReq = PullChangesRequest.Parser.ParseFrom(payload); - var since = new HlcTimestamp(pReq.SinceWall, pReq.SinceLogic, pReq.SinceNode); - - // Use collection filter from request - var filter = pReq.Collections.Any() ? pReq.Collections : null; - var oplog = string.IsNullOrWhiteSpace(pReq.SinceNode) - ? await _oplogStore.GetOplogAfterAsync(since, filter, token) - : await _oplogStore.GetOplogForNodeAfterAsync(pReq.SinceNode, since, filter, token); - - var csRes = new ChangeSetResponse(); - foreach (var e in oplog) - csRes.Entries.Add(new ProtoOplogEntry - { + case MessageType.PullChangesReq: + var pReq = PullChangesRequest.Parser.ParseFrom(payload); + string pullDatasetId = string.IsNullOrWhiteSpace(pReq.DatasetId) + ? currentDatasetId + : DatasetId.Normalize(pReq.DatasetId); + var since = new HlcTimestamp(pReq.SinceWall, pReq.SinceLogic, pReq.SinceNode); + + // Use collection filter from request + var filter = pReq.Collections.Any() ? pReq.Collections : null; + var oplog = string.IsNullOrWhiteSpace(pReq.SinceNode) + ? await _oplogStore.GetOplogAfterAsync(since, pullDatasetId, filter, token) + : await _oplogStore.GetOplogForNodeAfterAsync(pReq.SinceNode, since, pullDatasetId, filter, token); + + var csRes = new ChangeSetResponse(); + foreach (var e in oplog) + csRes.Entries.Add(new ProtoOplogEntry + { Collection = e.Collection, Key = e.Key, Operation = e.Operation.ToString(), JsonData = e.Payload?.GetRawText() ?? "", HlcWall = e.Timestamp.PhysicalTime, HlcLogic = e.Timestamp.LogicalCounter, - HlcNode = e.Timestamp.NodeId, - Hash = e.Hash, - PreviousHash = e.PreviousHash - }); - response = csRes; - resType = MessageType.ChangeSetRes; - break; - - case MessageType.PushChangesReq: - var pushReq = PushChangesRequest.Parser.ParseFrom(payload); - var entries = pushReq.Entries.Select(e => new OplogEntry( - e.Collection, - e.Key, - (OperationType)Enum.Parse(typeof(OperationType), e.Operation), - string.IsNullOrEmpty(e.JsonData) - ? null - : JsonSerializer.Deserialize(e.JsonData), - new HlcTimestamp(e.HlcWall, e.HlcLogic, e.HlcNode), - e.PreviousHash, // Restore PreviousHash - e.Hash // Restore Hash - )); - - await _oplogStore.ApplyBatchAsync(entries, token); + HlcNode = e.Timestamp.NodeId, + Hash = e.Hash, + PreviousHash = e.PreviousHash, + DatasetId = e.DatasetId + }); + response = csRes; + resType = MessageType.ChangeSetRes; + break; + + case MessageType.PushChangesReq: + var pushReq = PushChangesRequest.Parser.ParseFrom(payload); + string pushDatasetId = string.IsNullOrWhiteSpace(pushReq.DatasetId) + ? currentDatasetId + : DatasetId.Normalize(pushReq.DatasetId); + var entries = pushReq.Entries.Select(e => new OplogEntry( + e.Collection, + e.Key, + (OperationType)Enum.Parse(typeof(OperationType), e.Operation), + string.IsNullOrEmpty(e.JsonData) + ? null + : JsonSerializer.Deserialize(e.JsonData), + new HlcTimestamp(e.HlcWall, e.HlcLogic, e.HlcNode), + e.PreviousHash, // Restore PreviousHash + e.Hash, // Restore Hash + string.IsNullOrWhiteSpace(e.DatasetId) ? pushDatasetId : e.DatasetId + )); + + await _oplogStore.ApplyBatchAsync(entries, pushDatasetId, token); response = new AckResponse { Success = true }; resType = MessageType.AckRes; break; - case MessageType.GetChainRangeReq: - var rangeReq = GetChainRangeRequest.Parser.ParseFrom(payload); - var rangeEntries = - await _oplogStore.GetChainRangeAsync(rangeReq.StartHash, rangeReq.EndHash, token); - var rangeRes = new ChainRangeResponse(); + case MessageType.GetChainRangeReq: + var rangeReq = GetChainRangeRequest.Parser.ParseFrom(payload); + string chainDatasetId = string.IsNullOrWhiteSpace(rangeReq.DatasetId) + ? currentDatasetId + : DatasetId.Normalize(rangeReq.DatasetId); + var rangeEntries = + await _oplogStore.GetChainRangeAsync( + rangeReq.StartHash, + rangeReq.EndHash, + chainDatasetId, + token); + var rangeRes = new ChainRangeResponse(); if (!rangeEntries.Any() && rangeReq.StartHash != rangeReq.EndHash) // Gap cannot be filled (likely pruned or unknown branch) @@ -410,25 +434,30 @@ internal class TcpSyncServer : ISyncServer JsonData = e.Payload?.GetRawText() ?? "", HlcWall = e.Timestamp.PhysicalTime, HlcLogic = e.Timestamp.LogicalCounter, - HlcNode = e.Timestamp.NodeId, - Hash = e.Hash, - PreviousHash = e.PreviousHash - }); + HlcNode = e.Timestamp.NodeId, + Hash = e.Hash, + PreviousHash = e.PreviousHash, + DatasetId = e.DatasetId + }); response = rangeRes; resType = MessageType.ChainRangeRes; break; - case MessageType.GetSnapshotReq: - _logger.LogInformation("Processing GetSnapshotReq from {Endpoint}", remoteEp); - string tempFile = Path.GetTempFileName(); - try - { - // Create backup - using (var fs = File.Create(tempFile)) - { - await _snapshotStore.CreateSnapshotAsync(fs, token); - } + case MessageType.GetSnapshotReq: + var snapshotRequest = GetSnapshotRequest.Parser.ParseFrom(payload); + string snapshotDatasetId = string.IsNullOrWhiteSpace(snapshotRequest.DatasetId) + ? currentDatasetId + : DatasetId.Normalize(snapshotRequest.DatasetId); + _logger.LogInformation("Processing GetSnapshotReq from {Endpoint}", remoteEp); + string tempFile = Path.GetTempFileName(); + try + { + // Create backup + using (var fs = File.Create(tempFile)) + { + await _snapshotStore.CreateSnapshotAsync(fs, snapshotDatasetId, token); + } using (var fs = File.OpenRead(tempFile)) { @@ -472,4 +501,4 @@ internal class TcpSyncServer : ISyncServer _logger.LogDebug("Client Disconnected: {Endpoint}", remoteEp); } } -} \ No newline at end of file +} diff --git a/src/ZB.MOM.WW.CBDDC.Network/sync.proto b/src/ZB.MOM.WW.CBDDC.Network/sync.proto index e851b23..a6eb26a 100755 --- a/src/ZB.MOM.WW.CBDDC.Network/sync.proto +++ b/src/ZB.MOM.WW.CBDDC.Network/sync.proto @@ -4,19 +4,22 @@ package ZB.MOM.WW.CBDDC.Network.Proto; option csharp_namespace = "ZB.MOM.WW.CBDDC.Network.Proto"; -message HandshakeRequest { - string node_id = 1; - string auth_token = 2; - repeated string supported_compression = 3; // v4 - repeated string interesting_collections = 4; // v5 -} - -message HandshakeResponse { - string node_id = 1; - bool accepted = 2; - string selected_compression = 3; // v4 - repeated string interesting_collections = 4; // v5 -} +message HandshakeRequest { + string node_id = 1; + string auth_token = 2; + repeated string supported_compression = 3; // v4 + repeated string interesting_collections = 4; // v5 + string dataset_id = 5; // v6 +} + +message HandshakeResponse { + string node_id = 1; + bool accepted = 2; + string selected_compression = 3; // v4 + repeated string interesting_collections = 4; // v5 + string dataset_id = 5; // v6 + optional bool dataset_supported = 6; // v6 +} message GetClockRequest { } @@ -40,25 +43,28 @@ message VectorClockEntry { int32 hlc_logic = 3; } -message PullChangesRequest { - int64 since_wall = 1; - int32 since_logic = 2; - string since_node = 3; - repeated string collections = 4; // v5: Filter by collection -} +message PullChangesRequest { + int64 since_wall = 1; + int32 since_logic = 2; + string since_node = 3; + repeated string collections = 4; // v5: Filter by collection + string dataset_id = 5; // v6 +} message ChangeSetResponse { repeated ProtoOplogEntry entries = 1; } -message PushChangesRequest { - repeated ProtoOplogEntry entries = 1; -} - -message GetChainRangeRequest { - string start_hash = 1; - string end_hash = 2; -} +message PushChangesRequest { + repeated ProtoOplogEntry entries = 1; + string dataset_id = 2; // v6 +} + +message GetChainRangeRequest { + string start_hash = 1; + string end_hash = 2; + string dataset_id = 3; // v6 +} message ChainRangeResponse { repeated ProtoOplogEntry entries = 1; @@ -70,20 +76,22 @@ message AckResponse { bool snapshot_required = 2; } -message ProtoOplogEntry { - string collection = 1; - string key = 2; - string operation = 3; // "Put" or "Delete" +message ProtoOplogEntry { + string collection = 1; + string key = 2; + string operation = 3; // "Put" or "Delete" string json_data = 4; int64 hlc_wall = 5; int32 hlc_logic = 6; string hlc_node = 7; - string hash = 8; - string previous_hash = 9; -} - -message GetSnapshotRequest { -} + string hash = 8; + string previous_hash = 9; + string dataset_id = 10; // v6 +} + +message GetSnapshotRequest { + string dataset_id = 1; // v6 +} message SnapshotChunk { bytes data = 1; diff --git a/src/ZB.MOM.WW.CBDDC.Persistence/Snapshot/SnapshotDto.cs b/src/ZB.MOM.WW.CBDDC.Persistence/Snapshot/SnapshotDto.cs index 3b04f4b..87778fe 100755 --- a/src/ZB.MOM.WW.CBDDC.Persistence/Snapshot/SnapshotDto.cs +++ b/src/ZB.MOM.WW.CBDDC.Persistence/Snapshot/SnapshotDto.cs @@ -20,6 +20,11 @@ public class SnapshotDto /// public string NodeId { get; set; } = ""; + /// + /// Gets or sets the dataset identifier represented by this snapshot payload. + /// + public string DatasetId { get; set; } = "primary"; + /// /// Gets or sets the serialized document records. /// @@ -48,6 +53,11 @@ public class SnapshotDto public class DocumentDto { + /// + /// Gets or sets the dataset identifier. + /// + public string DatasetId { get; set; } = "primary"; + /// /// Gets or sets the document collection name. /// @@ -86,6 +96,11 @@ public class DocumentDto public class OplogDto { + /// + /// Gets or sets the dataset identifier. + /// + public string DatasetId { get; set; } = "primary"; + /// /// Gets or sets the collection associated with the operation. /// @@ -134,6 +149,11 @@ public class OplogDto public class SnapshotMetadataDto { + /// + /// Gets or sets the dataset identifier. + /// + public string DatasetId { get; set; } = "primary"; + /// /// Gets or sets the node identifier. /// @@ -180,6 +200,11 @@ public class RemotePeerDto public class PeerOplogConfirmationDto { + /// + /// Gets or sets the dataset identifier. + /// + public string DatasetId { get; set; } = "primary"; + /// /// Gets or sets the tracked peer node identifier. /// @@ -214,4 +239,4 @@ public class PeerOplogConfirmationDto /// Gets or sets a value indicating whether the tracked peer is active. /// public bool IsActive { get; set; } -} \ No newline at end of file +} diff --git a/src/ZB.MOM.WW.CBDDC.Persistence/SnapshotStore.cs b/src/ZB.MOM.WW.CBDDC.Persistence/SnapshotStore.cs index 379bf4c..40030e5 100755 --- a/src/ZB.MOM.WW.CBDDC.Persistence/SnapshotStore.cs +++ b/src/ZB.MOM.WW.CBDDC.Persistence/SnapshotStore.cs @@ -92,24 +92,43 @@ public class SnapshotStore : ISnapshotService /// public async Task CreateSnapshotAsync(Stream destination, CancellationToken cancellationToken = default) { + await CreateSnapshotAsync(destination, DatasetId.Primary, cancellationToken); + } + + /// + public async Task CreateSnapshotAsync(Stream destination, string datasetId, + CancellationToken cancellationToken = default) + { + string normalizedDatasetId = DatasetId.Normalize(datasetId); _logger.LogInformation("Creating snapshot..."); var documents = await _documentStore.ExportAsync(cancellationToken); var remotePeers = await _peerConfigurationStore.ExportAsync(cancellationToken); - var oplogEntries = await _oplogStore.ExportAsync(cancellationToken); + var oplogEntries = (await _oplogStore.ExportAsync(normalizedDatasetId, cancellationToken)).ToList(); var peerConfirmations = _peerOplogConfirmationStore == null ? [] - : await _peerOplogConfirmationStore.ExportAsync(cancellationToken); + : await _peerOplogConfirmationStore.ExportAsync(normalizedDatasetId, cancellationToken); + + if (!string.Equals(normalizedDatasetId, DatasetId.Primary, StringComparison.Ordinal)) + { + var datasetDocumentKeys = oplogEntries + .Select(o => (o.Collection, o.Key)) + .ToHashSet(); + documents = documents.Where(document => datasetDocumentKeys.Contains((document.Collection, document.Key))); + remotePeers = []; + } var snapshot = new SnapshotDto { Version = "1.0", CreatedAt = DateTime.UtcNow.ToString("O"), NodeId = "", // Will be set by caller if needed + DatasetId = normalizedDatasetId, Documents = [ .. documents.Select(d => new DocumentDto { + DatasetId = normalizedDatasetId, Collection = d.Collection, Key = d.Key, JsonData = d.Content.GetRawText(), @@ -123,6 +142,7 @@ public class SnapshotStore : ISnapshotService [ .. oplogEntries.Select(o => new OplogDto { + DatasetId = o.DatasetId, Collection = o.Collection, Key = o.Key, Operation = (int)o.Operation, @@ -149,6 +169,7 @@ public class SnapshotStore : ISnapshotService [ .. peerConfirmations.Select(c => new PeerOplogConfirmationDto { + DatasetId = c.DatasetId, PeerNodeId = c.PeerNodeId, SourceNodeId = c.SourceNodeId, ConfirmedWall = c.ConfirmedWall, @@ -174,9 +195,17 @@ public class SnapshotStore : ISnapshotService /// public async Task ReplaceDatabaseAsync(Stream databaseStream, CancellationToken cancellationToken = default) { + await ReplaceDatabaseAsync(databaseStream, DatasetId.Primary, cancellationToken); + } + + /// + public async Task ReplaceDatabaseAsync(Stream databaseStream, string datasetId, + CancellationToken cancellationToken = default) + { + string normalizedDatasetId = DatasetId.Normalize(datasetId); _logger.LogWarning("Replacing data from snapshot stream..."); - await ClearAllDataAsync(cancellationToken); + await ClearAllDataAsync(normalizedDatasetId, cancellationToken); var snapshot = await JsonSerializer.DeserializeAsync(databaseStream, cancellationToken: cancellationToken); @@ -198,7 +227,8 @@ public class SnapshotStore : ISnapshotService : JsonSerializer.Deserialize(o.JsonData), new HlcTimestamp(o.HlcWall, o.HlcLogic, o.HlcNode), o.PreviousHash ?? string.Empty, - string.IsNullOrWhiteSpace(o.Hash) ? null : o.Hash)).ToList(); + string.IsNullOrWhiteSpace(o.Hash) ? null : o.Hash, + string.IsNullOrWhiteSpace(o.DatasetId) ? normalizedDatasetId : o.DatasetId)).ToList(); var remotePeers = snapshot.RemotePeers.Select(p => new RemotePeerConfiguration { @@ -210,6 +240,7 @@ public class SnapshotStore : ISnapshotService }).ToList(); var peerConfirmations = (snapshot.PeerConfirmations ?? []).Select(c => new PeerOplogConfirmation { + DatasetId = string.IsNullOrWhiteSpace(c.DatasetId) ? normalizedDatasetId : c.DatasetId, PeerNodeId = c.PeerNodeId, SourceNodeId = c.SourceNodeId, ConfirmedWall = c.ConfirmedWall, @@ -219,11 +250,15 @@ public class SnapshotStore : ISnapshotService IsActive = c.IsActive }).ToList(); - await _documentStore.ImportAsync(documents, cancellationToken); - await _oplogStore.ImportAsync(oplogEntries, cancellationToken); - await _peerConfigurationStore.ImportAsync(remotePeers, cancellationToken); + if (string.Equals(normalizedDatasetId, DatasetId.Primary, StringComparison.Ordinal)) + await _documentStore.ImportAsync(documents, cancellationToken); + else + await _documentStore.MergeAsync(documents, cancellationToken); + await _oplogStore.ImportAsync(oplogEntries, normalizedDatasetId, cancellationToken); + if (string.Equals(normalizedDatasetId, DatasetId.Primary, StringComparison.Ordinal)) + await _peerConfigurationStore.ImportAsync(remotePeers, cancellationToken); if (_peerOplogConfirmationStore != null) - await _peerOplogConfirmationStore.ImportAsync(peerConfirmations, cancellationToken); + await _peerOplogConfirmationStore.ImportAsync(peerConfirmations, normalizedDatasetId, cancellationToken); _logger.LogInformation("Database replaced successfully."); } @@ -231,6 +266,14 @@ public class SnapshotStore : ISnapshotService /// public async Task MergeSnapshotAsync(Stream snapshotStream, CancellationToken cancellationToken = default) { + await MergeSnapshotAsync(snapshotStream, DatasetId.Primary, cancellationToken); + } + + /// + public async Task MergeSnapshotAsync(Stream snapshotStream, string datasetId, + CancellationToken cancellationToken = default) + { + string normalizedDatasetId = DatasetId.Normalize(datasetId); _logger.LogInformation("Merging snapshot from stream..."); var snapshot = await JsonSerializer.DeserializeAsync(snapshotStream, cancellationToken: cancellationToken); @@ -250,7 +293,8 @@ public class SnapshotStore : ISnapshotService : JsonSerializer.Deserialize(o.JsonData), new HlcTimestamp(o.HlcWall, o.HlcLogic, o.HlcNode), o.PreviousHash ?? string.Empty, - string.IsNullOrWhiteSpace(o.Hash) ? null : o.Hash)).ToList(); + string.IsNullOrWhiteSpace(o.Hash) ? null : o.Hash, + string.IsNullOrWhiteSpace(o.DatasetId) ? normalizedDatasetId : o.DatasetId)).ToList(); var remotePeers = snapshot.RemotePeers.Select(p => new RemotePeerConfiguration { NodeId = p.NodeId, @@ -261,6 +305,7 @@ public class SnapshotStore : ISnapshotService }).ToList(); var peerConfirmations = (snapshot.PeerConfirmations ?? []).Select(c => new PeerOplogConfirmation { + DatasetId = string.IsNullOrWhiteSpace(c.DatasetId) ? normalizedDatasetId : c.DatasetId, PeerNodeId = c.PeerNodeId, SourceNodeId = c.SourceNodeId, ConfirmedWall = c.ConfirmedWall, @@ -271,19 +316,24 @@ public class SnapshotStore : ISnapshotService }).ToList(); await _documentStore.MergeAsync(documents, cancellationToken); - await _oplogStore.MergeAsync(oplogEntries, cancellationToken); - await _peerConfigurationStore.MergeAsync(remotePeers, cancellationToken); + await _oplogStore.MergeAsync(oplogEntries, normalizedDatasetId, cancellationToken); + if (string.Equals(normalizedDatasetId, DatasetId.Primary, StringComparison.Ordinal)) + await _peerConfigurationStore.MergeAsync(remotePeers, cancellationToken); if (_peerOplogConfirmationStore != null) - await _peerOplogConfirmationStore.MergeAsync(peerConfirmations, cancellationToken); + await _peerOplogConfirmationStore.MergeAsync(peerConfirmations, normalizedDatasetId, cancellationToken); _logger.LogInformation("Snapshot merged successfully."); } - private async Task ClearAllDataAsync(CancellationToken cancellationToken = default) + private async Task ClearAllDataAsync(string datasetId, CancellationToken cancellationToken = default) { - await _documentStore.DropAsync(cancellationToken); - await _peerConfigurationStore.DropAsync(cancellationToken); - await _oplogStore.DropAsync(cancellationToken); - if (_peerOplogConfirmationStore != null) await _peerOplogConfirmationStore.DropAsync(cancellationToken); + await _oplogStore.DropAsync(datasetId, cancellationToken); + if (_peerOplogConfirmationStore != null) await _peerOplogConfirmationStore.DropAsync(datasetId, cancellationToken); + + if (string.Equals(datasetId, DatasetId.Primary, StringComparison.Ordinal)) + { + await _documentStore.DropAsync(cancellationToken); + await _peerConfigurationStore.DropAsync(cancellationToken); + } } -} \ No newline at end of file +} diff --git a/src/ZB.MOM.WW.CBDDC.Persistence/Surreal/CBDDCSurrealEmbeddedExtensions.cs b/src/ZB.MOM.WW.CBDDC.Persistence/Surreal/CBDDCSurrealEmbeddedExtensions.cs index aff56f4..d763ccd 100644 --- a/src/ZB.MOM.WW.CBDDC.Persistence/Surreal/CBDDCSurrealEmbeddedExtensions.cs +++ b/src/ZB.MOM.WW.CBDDC.Persistence/Surreal/CBDDCSurrealEmbeddedExtensions.cs @@ -1,5 +1,6 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; +using ZB.MOM.WW.CBDDC.Core; using ZB.MOM.WW.CBDDC.Core.Network; using SurrealDb.Net; using ZB.MOM.WW.CBDDC.Core.Storage; @@ -46,6 +47,62 @@ public static class CBDDCSurrealEmbeddedExtensions return services; } + /// + /// Registers dataset synchronization options for a Surreal-backed dataset pipeline. + /// + /// The service collection. + /// The dataset identifier. + /// Optional per-dataset option overrides. + /// The service collection for chaining. + public static IServiceCollection AddCBDDCSurrealEmbeddedDataset( + this IServiceCollection services, + string datasetId, + Action? configure = null) + { + if (services == null) throw new ArgumentNullException(nameof(services)); + + var options = new DatasetSyncOptions + { + DatasetId = DatasetId.Normalize(datasetId), + Enabled = true + }; + configure?.Invoke(options); + options.DatasetId = DatasetId.Normalize(options.DatasetId); + + services.AddSingleton(options); + return services; + } + + /// + /// Registers dataset synchronization options for a Surreal-backed dataset pipeline. + /// + /// The service collection. + /// Configuration delegate for dataset options. + /// The service collection for chaining. + public static IServiceCollection AddCBDDCSurrealEmbeddedDataset( + this IServiceCollection services, + Action configure) + { + if (configure == null) throw new ArgumentNullException(nameof(configure)); + + var options = new DatasetSyncOptions + { + DatasetId = DatasetId.Primary, + Enabled = true + }; + configure(options); + + return services.AddCBDDCSurrealEmbeddedDataset(options.DatasetId, configured => + { + configured.Enabled = options.Enabled; + configured.SyncLoopDelay = options.SyncLoopDelay; + configured.MaxPeersPerCycle = options.MaxPeersPerCycle; + configured.MaxEntriesPerCycle = options.MaxEntriesPerCycle; + configured.MaintenanceIntervalOverride = options.MaintenanceIntervalOverride; + configured.InterestingCollections = options.InterestingCollections.ToList(); + }); + } + private static void RegisterCoreServices( IServiceCollection services, Func optionsFactory) diff --git a/src/ZB.MOM.WW.CBDDC.Persistence/Surreal/CBDDCSurrealSchemaInitializer.cs b/src/ZB.MOM.WW.CBDDC.Persistence/Surreal/CBDDCSurrealSchemaInitializer.cs index d3123df..e859866 100644 --- a/src/ZB.MOM.WW.CBDDC.Persistence/Surreal/CBDDCSurrealSchemaInitializer.cs +++ b/src/ZB.MOM.WW.CBDDC.Persistence/Surreal/CBDDCSurrealSchemaInitializer.cs @@ -67,31 +67,31 @@ public sealed class CBDDCSurrealSchemaInitializer : ICBDDCSurrealSchemaInitializ { return $""" DEFINE TABLE OVERWRITE {CBDDCSurrealSchemaNames.OplogEntriesTable} SCHEMAFULL CHANGEFEED {_changefeedRetentionLiteral}; - DEFINE INDEX OVERWRITE {CBDDCSurrealSchemaNames.OplogHashIndex} ON TABLE {CBDDCSurrealSchemaNames.OplogEntriesTable} COLUMNS hash UNIQUE; - DEFINE INDEX OVERWRITE {CBDDCSurrealSchemaNames.OplogHlcIndex} ON TABLE {CBDDCSurrealSchemaNames.OplogEntriesTable} COLUMNS timestampPhysicalTime, timestampLogicalCounter, timestampNodeId; - DEFINE INDEX OVERWRITE {CBDDCSurrealSchemaNames.OplogCollectionIndex} ON TABLE {CBDDCSurrealSchemaNames.OplogEntriesTable} COLUMNS collection; + DEFINE INDEX OVERWRITE {CBDDCSurrealSchemaNames.OplogHashIndex} ON TABLE {CBDDCSurrealSchemaNames.OplogEntriesTable} COLUMNS datasetId, hash UNIQUE; + DEFINE INDEX OVERWRITE {CBDDCSurrealSchemaNames.OplogHlcIndex} ON TABLE {CBDDCSurrealSchemaNames.OplogEntriesTable} COLUMNS datasetId, timestampPhysicalTime, timestampLogicalCounter, timestampNodeId; + DEFINE INDEX OVERWRITE {CBDDCSurrealSchemaNames.OplogCollectionIndex} ON TABLE {CBDDCSurrealSchemaNames.OplogEntriesTable} COLUMNS datasetId, collection; DEFINE TABLE OVERWRITE {CBDDCSurrealSchemaNames.SnapshotMetadataTable} SCHEMAFULL CHANGEFEED {_changefeedRetentionLiteral}; - DEFINE INDEX OVERWRITE {CBDDCSurrealSchemaNames.SnapshotNodeIdIndex} ON TABLE {CBDDCSurrealSchemaNames.SnapshotMetadataTable} COLUMNS nodeId UNIQUE; - DEFINE INDEX OVERWRITE {CBDDCSurrealSchemaNames.SnapshotHlcIndex} ON TABLE {CBDDCSurrealSchemaNames.SnapshotMetadataTable} COLUMNS timestampPhysicalTime, timestampLogicalCounter; + DEFINE INDEX OVERWRITE {CBDDCSurrealSchemaNames.SnapshotNodeIdIndex} ON TABLE {CBDDCSurrealSchemaNames.SnapshotMetadataTable} COLUMNS datasetId, nodeId UNIQUE; + DEFINE INDEX OVERWRITE {CBDDCSurrealSchemaNames.SnapshotHlcIndex} ON TABLE {CBDDCSurrealSchemaNames.SnapshotMetadataTable} COLUMNS datasetId, timestampPhysicalTime, timestampLogicalCounter; DEFINE TABLE OVERWRITE {CBDDCSurrealSchemaNames.RemotePeerConfigurationsTable} SCHEMAFULL CHANGEFEED {_changefeedRetentionLiteral}; DEFINE INDEX OVERWRITE {CBDDCSurrealSchemaNames.PeerNodeIdIndex} ON TABLE {CBDDCSurrealSchemaNames.RemotePeerConfigurationsTable} COLUMNS nodeId UNIQUE; DEFINE INDEX OVERWRITE {CBDDCSurrealSchemaNames.PeerEnabledIndex} ON TABLE {CBDDCSurrealSchemaNames.RemotePeerConfigurationsTable} COLUMNS isEnabled; DEFINE TABLE OVERWRITE {CBDDCSurrealSchemaNames.DocumentMetadataTable} SCHEMAFULL CHANGEFEED {_changefeedRetentionLiteral}; - DEFINE INDEX OVERWRITE {CBDDCSurrealSchemaNames.DocumentMetadataCollectionKeyIndex} ON TABLE {CBDDCSurrealSchemaNames.DocumentMetadataTable} COLUMNS collection, key UNIQUE; - DEFINE INDEX OVERWRITE {CBDDCSurrealSchemaNames.DocumentMetadataHlcIndex} ON TABLE {CBDDCSurrealSchemaNames.DocumentMetadataTable} COLUMNS hlcPhysicalTime, hlcLogicalCounter, hlcNodeId; - DEFINE INDEX OVERWRITE {CBDDCSurrealSchemaNames.DocumentMetadataCollectionIndex} ON TABLE {CBDDCSurrealSchemaNames.DocumentMetadataTable} COLUMNS collection; + DEFINE INDEX OVERWRITE {CBDDCSurrealSchemaNames.DocumentMetadataCollectionKeyIndex} ON TABLE {CBDDCSurrealSchemaNames.DocumentMetadataTable} COLUMNS datasetId, collection, key UNIQUE; + DEFINE INDEX OVERWRITE {CBDDCSurrealSchemaNames.DocumentMetadataHlcIndex} ON TABLE {CBDDCSurrealSchemaNames.DocumentMetadataTable} COLUMNS datasetId, hlcPhysicalTime, hlcLogicalCounter, hlcNodeId; + DEFINE INDEX OVERWRITE {CBDDCSurrealSchemaNames.DocumentMetadataCollectionIndex} ON TABLE {CBDDCSurrealSchemaNames.DocumentMetadataTable} COLUMNS datasetId, collection; DEFINE TABLE OVERWRITE {CBDDCSurrealSchemaNames.PeerOplogConfirmationsTable} SCHEMAFULL CHANGEFEED {_changefeedRetentionLiteral}; - DEFINE INDEX OVERWRITE {CBDDCSurrealSchemaNames.PeerConfirmationPairIndex} ON TABLE {CBDDCSurrealSchemaNames.PeerOplogConfirmationsTable} COLUMNS peerNodeId, sourceNodeId UNIQUE; - DEFINE INDEX OVERWRITE {CBDDCSurrealSchemaNames.PeerConfirmationActiveIndex} ON TABLE {CBDDCSurrealSchemaNames.PeerOplogConfirmationsTable} COLUMNS isActive; - DEFINE INDEX OVERWRITE {CBDDCSurrealSchemaNames.PeerConfirmationSourceHlcIndex} ON TABLE {CBDDCSurrealSchemaNames.PeerOplogConfirmationsTable} COLUMNS sourceNodeId, confirmedWall, confirmedLogic; + DEFINE INDEX OVERWRITE {CBDDCSurrealSchemaNames.PeerConfirmationPairIndex} ON TABLE {CBDDCSurrealSchemaNames.PeerOplogConfirmationsTable} COLUMNS datasetId, peerNodeId, sourceNodeId UNIQUE; + DEFINE INDEX OVERWRITE {CBDDCSurrealSchemaNames.PeerConfirmationActiveIndex} ON TABLE {CBDDCSurrealSchemaNames.PeerOplogConfirmationsTable} COLUMNS datasetId, isActive; + DEFINE INDEX OVERWRITE {CBDDCSurrealSchemaNames.PeerConfirmationSourceHlcIndex} ON TABLE {CBDDCSurrealSchemaNames.PeerOplogConfirmationsTable} COLUMNS datasetId, sourceNodeId, confirmedWall, confirmedLogic; DEFINE TABLE OVERWRITE {_checkpointTable} SCHEMAFULL; - DEFINE INDEX OVERWRITE {CBDDCSurrealSchemaNames.CdcCheckpointConsumerIndex} ON TABLE {_checkpointTable} COLUMNS consumerId UNIQUE; - DEFINE INDEX OVERWRITE {CBDDCSurrealSchemaNames.CdcCheckpointVersionstampIndex} ON TABLE {_checkpointTable} COLUMNS versionstampCursor; + DEFINE INDEX OVERWRITE {CBDDCSurrealSchemaNames.CdcCheckpointConsumerIndex} ON TABLE {_checkpointTable} COLUMNS datasetId, consumerId UNIQUE; + DEFINE INDEX OVERWRITE {CBDDCSurrealSchemaNames.CdcCheckpointVersionstampIndex} ON TABLE {_checkpointTable} COLUMNS datasetId, versionstampCursor; """; } diff --git a/src/ZB.MOM.WW.CBDDC.Persistence/Surreal/ISurrealCdcCheckpointPersistence.cs b/src/ZB.MOM.WW.CBDDC.Persistence/Surreal/ISurrealCdcCheckpointPersistence.cs index 00be6f6..6747667 100644 --- a/src/ZB.MOM.WW.CBDDC.Persistence/Surreal/ISurrealCdcCheckpointPersistence.cs +++ b/src/ZB.MOM.WW.CBDDC.Persistence/Surreal/ISurrealCdcCheckpointPersistence.cs @@ -7,6 +7,11 @@ namespace ZB.MOM.WW.CBDDC.Persistence.Surreal; /// public sealed class SurrealCdcCheckpoint { + /// + /// Gets or sets the dataset identifier. + /// + public string DatasetId { get; set; } = global::ZB.MOM.WW.CBDDC.Core.DatasetId.Primary; + /// /// Gets or sets the logical consumer identifier. /// @@ -48,6 +53,21 @@ public interface ISurrealCdcCheckpointPersistence string? consumerId = null, CancellationToken cancellationToken = default); + /// + /// Reads the checkpoint for a consumer within a specific dataset. + /// + /// The dataset identifier. + /// Optional consumer id. + /// A cancellation token. + /// The checkpoint if found; otherwise . + Task GetCheckpointAsync( + string datasetId, + string? consumerId, + CancellationToken cancellationToken = default) + { + return GetCheckpointAsync(consumerId, cancellationToken); + } + /// /// Upserts checkpoint progress for a consumer. /// @@ -63,6 +83,26 @@ public interface ISurrealCdcCheckpointPersistence CancellationToken cancellationToken = default, long? versionstampCursor = null); + /// + /// Upserts checkpoint progress for a consumer and dataset. + /// + /// The dataset identifier. + /// The last processed timestamp. + /// The last processed hash. + /// Optional consumer id. + /// A cancellation token. + /// Optional changefeed versionstamp cursor. + Task UpsertCheckpointAsync( + string datasetId, + HlcTimestamp timestamp, + string lastHash, + string? consumerId = null, + CancellationToken cancellationToken = default, + long? versionstampCursor = null) + { + return UpsertCheckpointAsync(timestamp, lastHash, consumerId, cancellationToken, versionstampCursor); + } + /// /// Advances checkpoint progress from an oplog entry. /// @@ -73,4 +113,20 @@ public interface ISurrealCdcCheckpointPersistence OplogEntry entry, string? consumerId = null, CancellationToken cancellationToken = default); + + /// + /// Advances checkpoint progress for a specific dataset. + /// + /// The dataset identifier. + /// The oplog entry that was processed. + /// Optional consumer id. + /// A cancellation token. + Task AdvanceCheckpointAsync( + string datasetId, + OplogEntry entry, + string? consumerId = null, + CancellationToken cancellationToken = default) + { + return AdvanceCheckpointAsync(entry, consumerId, cancellationToken); + } } diff --git a/src/ZB.MOM.WW.CBDDC.Persistence/Surreal/SurrealCdcCheckpointPersistence.cs b/src/ZB.MOM.WW.CBDDC.Persistence/Surreal/SurrealCdcCheckpointPersistence.cs index 1908e6b..1166b39 100644 --- a/src/ZB.MOM.WW.CBDDC.Persistence/Surreal/SurrealCdcCheckpointPersistence.cs +++ b/src/ZB.MOM.WW.CBDDC.Persistence/Surreal/SurrealCdcCheckpointPersistence.cs @@ -49,11 +49,21 @@ public sealed class SurrealCdcCheckpointPersistence : ISurrealCdcCheckpointPersi public async Task GetCheckpointAsync( string? consumerId = null, CancellationToken cancellationToken = default) + { + return await GetCheckpointAsync(DatasetId.Primary, consumerId, cancellationToken); + } + + /// + public async Task GetCheckpointAsync( + string datasetId, + string? consumerId, + CancellationToken cancellationToken = default) { if (!_enabled) return null; string resolvedConsumerId = ResolveConsumerId(consumerId); - var existing = await FindByConsumerIdAsync(resolvedConsumerId, cancellationToken); + string resolvedDatasetId = DatasetId.Normalize(datasetId); + var existing = await FindByConsumerIdAsync(resolvedDatasetId, resolvedConsumerId, cancellationToken); return existing?.ToDomain(); } @@ -64,26 +74,47 @@ public sealed class SurrealCdcCheckpointPersistence : ISurrealCdcCheckpointPersi string? consumerId = null, CancellationToken cancellationToken = default, long? versionstampCursor = null) + { + await UpsertCheckpointAsync( + DatasetId.Primary, + timestamp, + lastHash, + consumerId, + cancellationToken, + versionstampCursor); + } + + /// + public async Task UpsertCheckpointAsync( + string datasetId, + HlcTimestamp timestamp, + string lastHash, + string? consumerId = null, + CancellationToken cancellationToken = default, + long? versionstampCursor = null) { if (!_enabled) return; string resolvedConsumerId = ResolveConsumerId(consumerId); + string resolvedDatasetId = DatasetId.Normalize(datasetId); await EnsureReadyAsync(cancellationToken); long? effectiveVersionstampCursor = versionstampCursor; if (!effectiveVersionstampCursor.HasValue) { var existing = await FindByConsumerIdAsync( + resolvedDatasetId, resolvedConsumerId, cancellationToken, ensureInitialized: false); effectiveVersionstampCursor = existing?.VersionstampCursor; } - RecordId recordId = RecordId.From(_checkpointTable, ComputeConsumerKey(resolvedConsumerId)); + RecordId recordId = RecordId.From(_checkpointTable, ComputeConsumerKey(resolvedDatasetId, resolvedConsumerId)); var record = new SurrealCdcCheckpointRecord { + DatasetId = resolvedDatasetId, ConsumerId = resolvedConsumerId, TimestampPhysicalTime = timestamp.PhysicalTime, TimestampLogicalCounter = timestamp.LogicalCounter, @@ -106,7 +137,18 @@ public sealed class SurrealCdcCheckpointPersistence : ISurrealCdcCheckpointPersi CancellationToken cancellationToken = default) { ArgumentNullException.ThrowIfNull(entry); - return UpsertCheckpointAsync(entry.Timestamp, entry.Hash, consumerId, cancellationToken); + return UpsertCheckpointAsync(entry.DatasetId, entry.Timestamp, entry.Hash, consumerId, cancellationToken); + } + + /// + public Task AdvanceCheckpointAsync( + string datasetId, + OplogEntry entry, + string? consumerId = null, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(entry); + return UpsertCheckpointAsync(datasetId, entry.Timestamp, entry.Hash, consumerId, cancellationToken); } private string ResolveConsumerId(string? consumerId) @@ -124,32 +166,44 @@ public sealed class SurrealCdcCheckpointPersistence : ISurrealCdcCheckpointPersi } private async Task FindByConsumerIdAsync( + string datasetId, string consumerId, CancellationToken cancellationToken, bool ensureInitialized = true) { + string normalizedDatasetId = DatasetId.Normalize(datasetId); if (ensureInitialized) await EnsureReadyAsync(cancellationToken); - RecordId deterministicId = RecordId.From(_checkpointTable, ComputeConsumerKey(consumerId)); + RecordId deterministicId = RecordId.From(_checkpointTable, ComputeConsumerKey(normalizedDatasetId, consumerId)); var deterministic = await _surrealClient.Select(deterministicId, cancellationToken); if (deterministic != null && + string.Equals(deterministic.DatasetId, normalizedDatasetId, StringComparison.Ordinal) && string.Equals(deterministic.ConsumerId, consumerId, StringComparison.Ordinal)) return deterministic; var all = await _surrealClient.Select(_checkpointTable, cancellationToken); return all?.FirstOrDefault(c => + (string.IsNullOrWhiteSpace(c.DatasetId) + ? string.Equals(normalizedDatasetId, DatasetId.Primary, StringComparison.Ordinal) + : string.Equals(c.DatasetId, normalizedDatasetId, StringComparison.Ordinal)) && string.Equals(c.ConsumerId, consumerId, StringComparison.Ordinal)); } - private static string ComputeConsumerKey(string consumerId) + private static string ComputeConsumerKey(string datasetId, string consumerId) { - byte[] input = Encoding.UTF8.GetBytes(consumerId); + byte[] input = Encoding.UTF8.GetBytes($"{datasetId}\n{consumerId}"); return Convert.ToHexString(SHA256.HashData(input)).ToLowerInvariant(); } } internal sealed class SurrealCdcCheckpointRecord : Record { + /// + /// Gets or sets the dataset identifier. + /// + [JsonPropertyName("datasetId")] + public string DatasetId { get; set; } = global::ZB.MOM.WW.CBDDC.Core.DatasetId.Primary; + /// /// Gets or sets the CDC consumer identifier. /// @@ -204,6 +258,7 @@ internal static class SurrealCdcCheckpointRecordMappers { return new SurrealCdcCheckpoint { + DatasetId = string.IsNullOrWhiteSpace(record.DatasetId) ? DatasetId.Primary : record.DatasetId, ConsumerId = record.ConsumerId, Timestamp = new HlcTimestamp( record.TimestampPhysicalTime, diff --git a/src/ZB.MOM.WW.CBDDC.Persistence/Surreal/SurrealDocumentMetadataStore.cs b/src/ZB.MOM.WW.CBDDC.Persistence/Surreal/SurrealDocumentMetadataStore.cs index 91f4e19..3926c63 100644 --- a/src/ZB.MOM.WW.CBDDC.Persistence/Surreal/SurrealDocumentMetadataStore.cs +++ b/src/ZB.MOM.WW.CBDDC.Persistence/Surreal/SurrealDocumentMetadataStore.cs @@ -9,6 +9,7 @@ namespace ZB.MOM.WW.CBDDC.Persistence.Surreal; public class SurrealDocumentMetadataStore : DocumentMetadataStore { + private const string PrimaryDatasetId = DatasetId.Primary; private readonly ILogger _logger; private readonly ICBDDCSurrealSchemaInitializer _schemaInitializer; private readonly ISurrealDbClient _surrealClient; @@ -30,11 +31,35 @@ public class SurrealDocumentMetadataStore : DocumentMetadataStore _logger = logger ?? NullLogger.Instance; } + private static string NormalizeDatasetId(string? datasetId) + { + return DatasetId.Normalize(datasetId); + } + + private static bool MatchesDataset(string? recordDatasetId, string datasetId) + { + if (string.IsNullOrWhiteSpace(recordDatasetId)) + return string.Equals(datasetId, PrimaryDatasetId, StringComparison.Ordinal); + + return string.Equals(NormalizeDatasetId(recordDatasetId), datasetId, StringComparison.Ordinal); + } + /// public override async Task GetMetadataAsync(string collection, string key, CancellationToken cancellationToken = default) { - var existing = await FindByCollectionKeyAsync(collection, key, cancellationToken); + var existing = await FindByCollectionKeyAsync(collection, key, PrimaryDatasetId, cancellationToken); + return existing?.ToDomain(); + } + + /// + public async Task GetMetadataAsync( + string collection, + string key, + string datasetId, + CancellationToken cancellationToken = default) + { + var existing = await FindByCollectionKeyAsync(collection, key, NormalizeDatasetId(datasetId), cancellationToken); return existing?.ToDomain(); } @@ -42,7 +67,20 @@ public class SurrealDocumentMetadataStore : DocumentMetadataStore public override async Task> GetMetadataByCollectionAsync(string collection, CancellationToken cancellationToken = default) { - var all = await SelectAllAsync(cancellationToken); + var all = await SelectAllAsync(PrimaryDatasetId, cancellationToken); + return all + .Where(m => string.Equals(m.Collection, collection, StringComparison.Ordinal)) + .Select(m => m.ToDomain()) + .ToList(); + } + + /// + public async Task> GetMetadataByCollectionAsync( + string collection, + string datasetId, + CancellationToken cancellationToken = default) + { + var all = await SelectAllAsync(NormalizeDatasetId(datasetId), cancellationToken); return all .Where(m => string.Equals(m.Collection, collection, StringComparison.Ordinal)) .Select(m => m.ToDomain()) @@ -53,10 +91,26 @@ public class SurrealDocumentMetadataStore : DocumentMetadataStore public override async Task UpsertMetadataAsync(DocumentMetadata metadata, CancellationToken cancellationToken = default) { + await UpsertMetadataAsync(metadata, PrimaryDatasetId, cancellationToken); + } + + /// + public async Task UpsertMetadataAsync( + DocumentMetadata metadata, + string datasetId, + CancellationToken cancellationToken = default) + { + string normalizedDatasetId = NormalizeDatasetId(datasetId); + metadata.DatasetId = normalizedDatasetId; await EnsureReadyAsync(cancellationToken); - var existing = await FindByCollectionKeyAsync(metadata.Collection, metadata.Key, cancellationToken); - RecordId recordId = existing?.Id ?? SurrealStoreRecordIds.DocumentMetadata(metadata.Collection, metadata.Key); + var existing = + await FindByCollectionKeyAsync(metadata.Collection, metadata.Key, normalizedDatasetId, cancellationToken); + RecordId recordId = existing?.Id ?? + SurrealStoreRecordIds.DocumentMetadata( + metadata.Collection, + metadata.Key, + normalizedDatasetId); await _surrealClient.Upsert( recordId, @@ -67,24 +121,46 @@ public class SurrealDocumentMetadataStore : DocumentMetadataStore /// public override async Task UpsertMetadataBatchAsync(IEnumerable metadatas, CancellationToken cancellationToken = default) + { + await UpsertMetadataBatchAsync(metadatas, PrimaryDatasetId, cancellationToken); + } + + /// + public async Task UpsertMetadataBatchAsync(IEnumerable metadatas, + string datasetId, + CancellationToken cancellationToken = default) { foreach (var metadata in metadatas) - await UpsertMetadataAsync(metadata, cancellationToken); + await UpsertMetadataAsync(metadata, datasetId, cancellationToken); } /// public override async Task MarkDeletedAsync(string collection, string key, HlcTimestamp timestamp, CancellationToken cancellationToken = default) { - var metadata = new DocumentMetadata(collection, key, timestamp, true); - await UpsertMetadataAsync(metadata, cancellationToken); + await MarkDeletedAsync(collection, key, timestamp, PrimaryDatasetId, cancellationToken); + } + + /// + public async Task MarkDeletedAsync(string collection, string key, HlcTimestamp timestamp, string datasetId, + CancellationToken cancellationToken = default) + { + var metadata = new DocumentMetadata(collection, key, timestamp, true, datasetId); + await UpsertMetadataAsync(metadata, datasetId, cancellationToken); } /// public override async Task> GetMetadataAfterAsync(HlcTimestamp since, IEnumerable? collections = null, CancellationToken cancellationToken = default) { - var all = await SelectAllAsync(cancellationToken); + return await GetMetadataAfterAsync(since, PrimaryDatasetId, collections, cancellationToken); + } + + /// + public async Task> GetMetadataAfterAsync(HlcTimestamp since, string datasetId, + IEnumerable? collections = null, CancellationToken cancellationToken = default) + { + var all = await SelectAllAsync(NormalizeDatasetId(datasetId), cancellationToken); HashSet? collectionSet = collections != null ? new HashSet(collections) : null; return all @@ -101,14 +177,45 @@ public class SurrealDocumentMetadataStore : DocumentMetadataStore /// public override async Task DropAsync(CancellationToken cancellationToken = default) { - await EnsureReadyAsync(cancellationToken); - await _surrealClient.Delete(CBDDCSurrealSchemaNames.DocumentMetadataTable, cancellationToken); + await DropAsync(PrimaryDatasetId, cancellationToken); + } + + /// + /// Drops all metadata rows for a dataset. + /// + /// The dataset identifier. + /// A cancellation token. + public async Task DropAsync(string datasetId, CancellationToken cancellationToken = default) + { + var rows = await SelectAllAsync(NormalizeDatasetId(datasetId), cancellationToken); + foreach (var row in rows) + { + var recordId = row.Id ?? + SurrealStoreRecordIds.DocumentMetadata( + row.Collection, + row.Key, + NormalizeDatasetId(datasetId)); + await _surrealClient.Delete(recordId, cancellationToken); + } } /// public override async Task> ExportAsync(CancellationToken cancellationToken = default) { - var all = await SelectAllAsync(cancellationToken); + var all = await SelectAllAsync(PrimaryDatasetId, cancellationToken); + return all.Select(m => m.ToDomain()).ToList(); + } + + /// + /// Exports metadata rows for a dataset. + /// + /// The dataset identifier. + /// A cancellation token. + /// The exported metadata rows. + public async Task> ExportAsync(string datasetId, + CancellationToken cancellationToken = default) + { + var all = await SelectAllAsync(NormalizeDatasetId(datasetId), cancellationToken); return all.Select(m => m.ToDomain()).ToList(); } @@ -116,16 +223,42 @@ public class SurrealDocumentMetadataStore : DocumentMetadataStore public override async Task ImportAsync(IEnumerable items, CancellationToken cancellationToken = default) { - foreach (var item in items) await UpsertMetadataAsync(item, cancellationToken); + await ImportAsync(items, PrimaryDatasetId, cancellationToken); + } + + /// + /// Imports metadata rows into a dataset. + /// + /// The metadata items. + /// The dataset identifier. + /// A cancellation token. + public async Task ImportAsync(IEnumerable items, string datasetId, + CancellationToken cancellationToken = default) + { + foreach (var item in items) await UpsertMetadataAsync(item, datasetId, cancellationToken); } /// public override async Task MergeAsync(IEnumerable items, CancellationToken cancellationToken = default) { + await MergeAsync(items, PrimaryDatasetId, cancellationToken); + } + + /// + /// Merges metadata rows into a dataset. + /// + /// The metadata items. + /// The dataset identifier. + /// A cancellation token. + public async Task MergeAsync(IEnumerable items, string datasetId, + CancellationToken cancellationToken = default) + { + string normalizedDatasetId = NormalizeDatasetId(datasetId); foreach (var item in items) { - var existing = await FindByCollectionKeyAsync(item.Collection, item.Key, cancellationToken); + item.DatasetId = normalizedDatasetId; + var existing = await FindByCollectionKeyAsync(item.Collection, item.Key, normalizedDatasetId, cancellationToken); if (existing == null) { @@ -138,7 +271,8 @@ public class SurrealDocumentMetadataStore : DocumentMetadataStore if (item.UpdatedAt.CompareTo(existingTimestamp) <= 0) continue; - RecordId recordId = existing.Id ?? SurrealStoreRecordIds.DocumentMetadata(item.Collection, item.Key); + RecordId recordId = + existing.Id ?? SurrealStoreRecordIds.DocumentMetadata(item.Collection, item.Key, normalizedDatasetId); await EnsureReadyAsync(cancellationToken); await _surrealClient.Upsert( recordId, @@ -152,27 +286,37 @@ public class SurrealDocumentMetadataStore : DocumentMetadataStore await _schemaInitializer.EnsureInitializedAsync(cancellationToken); } - private async Task> SelectAllAsync(CancellationToken cancellationToken) + private async Task> SelectAllAsync(string datasetId, + CancellationToken cancellationToken) { + string normalizedDatasetId = NormalizeDatasetId(datasetId); await EnsureReadyAsync(cancellationToken); var rows = await _surrealClient.Select( CBDDCSurrealSchemaNames.DocumentMetadataTable, cancellationToken); - return rows?.ToList() ?? []; + return rows? + .Where(row => MatchesDataset(row.DatasetId, normalizedDatasetId)) + .ToList() + ?? []; } - private async Task FindByCollectionKeyAsync(string collection, string key, + private async Task FindByCollectionKeyAsync( + string collection, + string key, + string datasetId, CancellationToken cancellationToken) { + string normalizedDatasetId = NormalizeDatasetId(datasetId); await EnsureReadyAsync(cancellationToken); - RecordId deterministicId = SurrealStoreRecordIds.DocumentMetadata(collection, key); + RecordId deterministicId = SurrealStoreRecordIds.DocumentMetadata(collection, key, normalizedDatasetId); var deterministic = await _surrealClient.Select(deterministicId, cancellationToken); if (deterministic != null && + MatchesDataset(deterministic.DatasetId, normalizedDatasetId) && string.Equals(deterministic.Collection, collection, StringComparison.Ordinal) && string.Equals(deterministic.Key, key, StringComparison.Ordinal)) return deterministic; - var all = await SelectAllAsync(cancellationToken); + var all = await SelectAllAsync(normalizedDatasetId, cancellationToken); return all.FirstOrDefault(m => string.Equals(m.Collection, collection, StringComparison.Ordinal) && string.Equals(m.Key, key, StringComparison.Ordinal)); diff --git a/src/ZB.MOM.WW.CBDDC.Persistence/Surreal/SurrealDocumentStore.cs b/src/ZB.MOM.WW.CBDDC.Persistence/Surreal/SurrealDocumentStore.cs index 98d9d66..3cd6ce0 100644 --- a/src/ZB.MOM.WW.CBDDC.Persistence/Surreal/SurrealDocumentStore.cs +++ b/src/ZB.MOM.WW.CBDDC.Persistence/Surreal/SurrealDocumentStore.cs @@ -1205,7 +1205,10 @@ public abstract class SurrealDocumentStore : IDocumentStore, ISurrealC { ["oplogRecordId"] = SurrealStoreRecordIds.Oplog(oplogEntry.Hash), ["oplogRecord"] = oplogEntry.ToSurrealRecord(), - ["metadataRecordId"] = SurrealStoreRecordIds.DocumentMetadata(metadata.Collection, metadata.Key), + ["metadataRecordId"] = SurrealStoreRecordIds.DocumentMetadata( + metadata.Collection, + metadata.Key, + metadata.DatasetId), ["metadataRecord"] = metadata.ToSurrealRecord() }; @@ -1261,10 +1264,12 @@ public abstract class SurrealDocumentStore : IDocumentStore, ISurrealC checkpointRecord = new Dictionary(); if (!TryGetCheckpointSettings(out string checkpointTable, out string consumerId)) return false; - string consumerKey = ComputeConsumerKey(consumerId); + const string datasetId = DatasetId.Primary; + string consumerKey = ComputeConsumerKey(datasetId, consumerId); checkpointRecordId = RecordId.From(checkpointTable, consumerKey); checkpointRecord = new Dictionary { + ["datasetId"] = datasetId, ["consumerId"] = consumerId, ["timestampPhysicalTime"] = oplogEntry.Timestamp.PhysicalTime, ["timestampLogicalCounter"] = oplogEntry.Timestamp.LogicalCounter, @@ -1294,10 +1299,12 @@ public abstract class SurrealDocumentStore : IDocumentStore, ISurrealC ? long.MaxValue : (long)pendingCursorCheckpoint.Value.Cursor; - string consumerKey = ComputeConsumerKey(cursorConsumerId); + const string datasetId = DatasetId.Primary; + string consumerKey = ComputeConsumerKey(datasetId, cursorConsumerId); checkpointRecordId = RecordId.From(checkpointTable, consumerKey); checkpointRecord = new Dictionary { + ["datasetId"] = datasetId, ["consumerId"] = cursorConsumerId, ["timestampPhysicalTime"] = encodedCursor, ["timestampLogicalCounter"] = 0, @@ -1329,9 +1336,9 @@ public abstract class SurrealDocumentStore : IDocumentStore, ISurrealC return true; } - private static string ComputeConsumerKey(string consumerId) + private static string ComputeConsumerKey(string datasetId, string consumerId) { - byte[] bytes = Encoding.UTF8.GetBytes(consumerId); + byte[] bytes = Encoding.UTF8.GetBytes($"{datasetId}\n{consumerId}"); return Convert.ToHexString(SHA256.HashData(bytes)).ToLowerInvariant(); } diff --git a/src/ZB.MOM.WW.CBDDC.Persistence/Surreal/SurrealOplogStore.cs b/src/ZB.MOM.WW.CBDDC.Persistence/Surreal/SurrealOplogStore.cs index 162b1db..7f9ea26 100644 --- a/src/ZB.MOM.WW.CBDDC.Persistence/Surreal/SurrealOplogStore.cs +++ b/src/ZB.MOM.WW.CBDDC.Persistence/Surreal/SurrealOplogStore.cs @@ -10,6 +10,7 @@ namespace ZB.MOM.WW.CBDDC.Persistence.Surreal; public class SurrealOplogStore : OplogStore { + private const string PrimaryDatasetId = DatasetId.Primary; private readonly ILogger _logger; private readonly ICBDDCSurrealSchemaInitializer? _schemaInitializer; private readonly ISurrealDbClient? _surrealClient; @@ -46,17 +47,38 @@ public class SurrealOplogStore : OplogStore InitializeVectorClock(); } + private static string NormalizeDatasetId(string? datasetId) + { + return DatasetId.Normalize(datasetId); + } + + private static bool MatchesDataset(string? recordDatasetId, string datasetId) + { + if (string.IsNullOrWhiteSpace(recordDatasetId)) + return string.Equals(datasetId, PrimaryDatasetId, StringComparison.Ordinal); + + return string.Equals(NormalizeDatasetId(recordDatasetId), datasetId, StringComparison.Ordinal); + } + /// public override async Task> GetChainRangeAsync(string startHash, string endHash, CancellationToken cancellationToken = default) { - var startRow = await FindByHashAsync(startHash, cancellationToken); - var endRow = await FindByHashAsync(endHash, cancellationToken); + return await GetChainRangeAsync(startHash, endHash, PrimaryDatasetId, cancellationToken); + } + + /// + public async Task> GetChainRangeAsync(string startHash, string endHash, string datasetId, + CancellationToken cancellationToken = default) + { + string normalizedDatasetId = NormalizeDatasetId(datasetId); + var startRow = await FindByHashAsync(startHash, normalizedDatasetId, cancellationToken); + var endRow = await FindByHashAsync(endHash, normalizedDatasetId, cancellationToken); if (startRow == null || endRow == null) return []; string nodeId = startRow.TimestampNodeId; - var all = await SelectAllAsync(cancellationToken); + var all = await SelectAllAsync(normalizedDatasetId, cancellationToken); return all .Where(o => string.Equals(o.TimestampNodeId, nodeId, StringComparison.Ordinal) && @@ -75,7 +97,16 @@ public class SurrealOplogStore : OplogStore /// public override async Task GetEntryByHashAsync(string hash, CancellationToken cancellationToken = default) { - var existing = await FindByHashAsync(hash, cancellationToken); + var existing = await FindByHashAsync(hash, PrimaryDatasetId, cancellationToken); + return existing?.ToDomain(); + } + + /// + public async Task GetEntryByHashAsync(string hash, string datasetId, + CancellationToken cancellationToken = default) + { + string normalizedDatasetId = NormalizeDatasetId(datasetId); + var existing = await FindByHashAsync(hash, normalizedDatasetId, cancellationToken); return existing?.ToDomain(); } @@ -83,7 +114,15 @@ public class SurrealOplogStore : OplogStore public override async Task> GetOplogAfterAsync(HlcTimestamp timestamp, IEnumerable? collections = null, CancellationToken cancellationToken = default) { - var all = await SelectAllAsync(cancellationToken); + return await GetOplogAfterAsync(timestamp, PrimaryDatasetId, collections, cancellationToken); + } + + /// + public async Task> GetOplogAfterAsync(HlcTimestamp timestamp, string datasetId, + IEnumerable? collections = null, CancellationToken cancellationToken = default) + { + string normalizedDatasetId = NormalizeDatasetId(datasetId); + var all = await SelectAllAsync(normalizedDatasetId, cancellationToken); HashSet? collectionSet = collections != null ? new HashSet(collections) : null; return all @@ -102,7 +141,15 @@ public class SurrealOplogStore : OplogStore public override async Task> GetOplogForNodeAfterAsync(string nodeId, HlcTimestamp since, IEnumerable? collections = null, CancellationToken cancellationToken = default) { - var all = await SelectAllAsync(cancellationToken); + return await GetOplogForNodeAfterAsync(nodeId, since, PrimaryDatasetId, collections, cancellationToken); + } + + /// + public async Task> GetOplogForNodeAfterAsync(string nodeId, HlcTimestamp since, + string datasetId, IEnumerable? collections = null, CancellationToken cancellationToken = default) + { + string normalizedDatasetId = NormalizeDatasetId(datasetId); + var all = await SelectAllAsync(normalizedDatasetId, cancellationToken); HashSet? collectionSet = collections != null ? new HashSet(collections) : null; return all @@ -121,7 +168,15 @@ public class SurrealOplogStore : OplogStore /// public override async Task PruneOplogAsync(HlcTimestamp cutoff, CancellationToken cancellationToken = default) { - var all = await SelectAllAsync(cancellationToken); + await PruneOplogAsync(cutoff, PrimaryDatasetId, cancellationToken); + } + + /// + public async Task PruneOplogAsync(HlcTimestamp cutoff, string datasetId, + CancellationToken cancellationToken = default) + { + string normalizedDatasetId = NormalizeDatasetId(datasetId); + var all = await SelectAllAsync(normalizedDatasetId, cancellationToken); var toDelete = all .Where(o => o.TimestampPhysicalTime < cutoff.PhysicalTime || (o.TimestampPhysicalTime == cutoff.PhysicalTime && @@ -139,38 +194,114 @@ public class SurrealOplogStore : OplogStore /// public override async Task DropAsync(CancellationToken cancellationToken = default) { - await EnsureReadyAsync(cancellationToken); - await _surrealClient!.Delete(CBDDCSurrealSchemaNames.OplogEntriesTable, cancellationToken); + await DropAsync(PrimaryDatasetId, cancellationToken); _vectorClock.Invalidate(); } + /// + /// Drops all oplog rows for the specified dataset. + /// + /// The dataset identifier. + /// A cancellation token. + public async Task DropAsync(string datasetId, CancellationToken cancellationToken = default) + { + var rows = await SelectAllAsync(NormalizeDatasetId(datasetId), cancellationToken); + foreach (var row in rows) + { + RecordId recordId = row.Id ?? SurrealStoreRecordIds.Oplog(row.Hash); + await EnsureReadyAsync(cancellationToken); + await _surrealClient!.Delete(recordId, cancellationToken); + } + } + /// public override async Task> ExportAsync(CancellationToken cancellationToken = default) { - var all = await SelectAllAsync(cancellationToken); + var all = await SelectAllAsync(PrimaryDatasetId, cancellationToken); + return all.Select(o => o.ToDomain()).ToList(); + } + + /// + /// Exports all oplog entries for a dataset. + /// + /// The dataset identifier. + /// A cancellation token. + /// All dataset-scoped oplog entries. + public async Task> ExportAsync(string datasetId, CancellationToken cancellationToken = default) + { + var all = await SelectAllAsync(NormalizeDatasetId(datasetId), cancellationToken); return all.Select(o => o.ToDomain()).ToList(); } /// public override async Task ImportAsync(IEnumerable items, CancellationToken cancellationToken = default) { + await ImportAsync(items, PrimaryDatasetId, cancellationToken); + } + + /// + /// Imports oplog entries for the specified dataset. + /// + /// The entries to import. + /// The dataset identifier. + /// A cancellation token. + public async Task ImportAsync(IEnumerable items, string datasetId, + CancellationToken cancellationToken = default) + { + string normalizedDatasetId = NormalizeDatasetId(datasetId); + Dictionary existingByHash = + await LoadOplogRecordIdsByHashAsync(normalizedDatasetId, cancellationToken); foreach (var item in items) { - var existing = await FindByHashAsync(item.Hash, cancellationToken); - RecordId recordId = existing?.Id ?? SurrealStoreRecordIds.Oplog(item.Hash); - await UpsertAsync(item, recordId, cancellationToken); + var normalizedItem = new OplogEntry( + item.Collection, + item.Key, + item.Operation, + item.Payload, + item.Timestamp, + item.PreviousHash, + item.Hash, + normalizedDatasetId); + RecordId recordId = existingByHash.TryGetValue(item.Hash, out RecordId? existingRecordId) + ? existingRecordId + : SurrealStoreRecordIds.Oplog(item.Hash); + await UpsertAsync(normalizedItem, recordId, cancellationToken); + existingByHash[item.Hash] = recordId; } } /// public override async Task MergeAsync(IEnumerable items, CancellationToken cancellationToken = default) { + await MergeAsync(items, PrimaryDatasetId, cancellationToken); + } + + /// + /// Merges oplog entries into the specified dataset. + /// + /// The entries to merge. + /// The dataset identifier. + /// A cancellation token. + public async Task MergeAsync(IEnumerable items, string datasetId, + CancellationToken cancellationToken = default) + { + string normalizedDatasetId = NormalizeDatasetId(datasetId); + HashSet existingHashes = await LoadOplogHashesAsync(normalizedDatasetId, cancellationToken); foreach (var item in items) { - var existing = await FindByHashAsync(item.Hash, cancellationToken); - if (existing != null) continue; + if (!existingHashes.Add(item.Hash)) + continue; - await UpsertAsync(item, SurrealStoreRecordIds.Oplog(item.Hash), cancellationToken); + var normalizedItem = new OplogEntry( + item.Collection, + item.Key, + item.Operation, + item.Payload, + item.Timestamp, + item.PreviousHash, + item.Hash, + normalizedDatasetId); + await UpsertAsync(normalizedItem, SurrealStoreRecordIds.Oplog(normalizedItem.Hash), cancellationToken); } } @@ -190,6 +321,8 @@ public class SurrealOplogStore : OplogStore { var snapshots = _snapshotMetadataStore.GetAllSnapshotMetadataAsync().GetAwaiter().GetResult(); foreach (var snapshot in snapshots) + { + if (!MatchesDataset(snapshot.DatasetId, PrimaryDatasetId)) continue; _vectorClock.UpdateNode( snapshot.NodeId, new HlcTimestamp( @@ -197,6 +330,7 @@ public class SurrealOplogStore : OplogStore snapshot.TimestampLogicalCounter, snapshot.NodeId), snapshot.Hash ?? ""); + } } catch { @@ -209,6 +343,7 @@ public class SurrealOplogStore : OplogStore ?? []; var latestPerNode = all + .Where(x => MatchesDataset(x.DatasetId, PrimaryDatasetId)) .Where(x => !string.IsNullOrWhiteSpace(x.TimestampNodeId)) .GroupBy(x => x.TimestampNodeId) .Select(g => g @@ -229,17 +364,27 @@ public class SurrealOplogStore : OplogStore /// protected override async Task InsertOplogEntryAsync(OplogEntry entry, CancellationToken cancellationToken = default) { - var existing = await FindByHashAsync(entry.Hash, cancellationToken); + string datasetId = NormalizeDatasetId(entry.DatasetId); + var normalizedEntry = new OplogEntry( + entry.Collection, + entry.Key, + entry.Operation, + entry.Payload, + entry.Timestamp, + entry.PreviousHash, + entry.Hash, + datasetId); + var existing = await FindByHashAsync(normalizedEntry.Hash, datasetId, cancellationToken); if (existing != null) return; - await UpsertAsync(entry, SurrealStoreRecordIds.Oplog(entry.Hash), cancellationToken); + await UpsertAsync(normalizedEntry, SurrealStoreRecordIds.Oplog(normalizedEntry.Hash), cancellationToken); } /// protected override async Task QueryLastHashForNodeAsync(string nodeId, CancellationToken cancellationToken = default) { - var all = await SelectAllAsync(cancellationToken); + var all = await SelectAllAsync(PrimaryDatasetId, cancellationToken); var lastEntry = all .Where(o => string.Equals(o.TimestampNodeId, nodeId, StringComparison.Ordinal)) .OrderByDescending(o => o.TimestampPhysicalTime) @@ -252,11 +397,106 @@ public class SurrealOplogStore : OplogStore protected override async Task<(long Wall, int Logic)?> QueryLastHashTimestampFromOplogAsync(string hash, CancellationToken cancellationToken = default) { - var existing = await FindByHashAsync(hash, cancellationToken); + var existing = await FindByHashAsync(hash, PrimaryDatasetId, cancellationToken); if (existing == null) return null; return (existing.TimestampPhysicalTime, existing.TimestampLogicalCounter); } + /// + public async Task AppendOplogEntryAsync( + OplogEntry entry, + string datasetId, + CancellationToken cancellationToken = default) + { + string normalizedDatasetId = NormalizeDatasetId(datasetId); + var normalizedEntry = new OplogEntry( + entry.Collection, + entry.Key, + entry.Operation, + entry.Payload, + entry.Timestamp, + entry.PreviousHash, + entry.Hash, + normalizedDatasetId); + await AppendOplogEntryAsync(normalizedEntry, cancellationToken); + } + + /// + public async Task GetLatestTimestampAsync(string datasetId, CancellationToken cancellationToken = default) + { + string normalizedDatasetId = NormalizeDatasetId(datasetId); + var all = await SelectAllAsync(normalizedDatasetId, cancellationToken); + var latest = all + .OrderByDescending(o => o.TimestampPhysicalTime) + .ThenByDescending(o => o.TimestampLogicalCounter) + .FirstOrDefault(); + + return latest == null + ? new HlcTimestamp(0, 0, "") + : new HlcTimestamp(latest.TimestampPhysicalTime, latest.TimestampLogicalCounter, latest.TimestampNodeId); + } + + /// + public async Task GetVectorClockAsync(string datasetId, CancellationToken cancellationToken = default) + { + string normalizedDatasetId = NormalizeDatasetId(datasetId); + var all = await SelectAllAsync(normalizedDatasetId, cancellationToken); + var vectorClock = new VectorClock(); + foreach (var latest in all + .Where(o => !string.IsNullOrWhiteSpace(o.TimestampNodeId)) + .GroupBy(o => o.TimestampNodeId) + .Select(g => g.OrderByDescending(o => o.TimestampPhysicalTime) + .ThenByDescending(o => o.TimestampLogicalCounter) + .First())) + vectorClock.SetTimestamp( + latest.TimestampNodeId, + new HlcTimestamp(latest.TimestampPhysicalTime, latest.TimestampLogicalCounter, latest.TimestampNodeId)); + + return vectorClock; + } + + /// + public async Task GetLastEntryHashAsync(string nodeId, string datasetId, + CancellationToken cancellationToken = default) + { + string normalizedDatasetId = NormalizeDatasetId(datasetId); + var all = await SelectAllAsync(normalizedDatasetId, cancellationToken); + var latest = all + .Where(o => string.Equals(o.TimestampNodeId, nodeId, StringComparison.Ordinal)) + .OrderByDescending(o => o.TimestampPhysicalTime) + .ThenByDescending(o => o.TimestampLogicalCounter) + .FirstOrDefault(); + + if (latest != null) return latest.Hash; + + if (_snapshotMetadataStore == null) return null; + var snapshotHash = + await _snapshotMetadataStore.GetSnapshotHashAsync(nodeId, normalizedDatasetId, cancellationToken); + return snapshotHash; + } + + /// + public async Task ApplyBatchAsync( + IEnumerable oplogEntries, + string datasetId, + CancellationToken cancellationToken = default) + { + string normalizedDatasetId = NormalizeDatasetId(datasetId); + var normalizedEntries = oplogEntries + .Select(entry => new OplogEntry( + entry.Collection, + entry.Key, + entry.Operation, + entry.Payload, + entry.Timestamp, + entry.PreviousHash, + entry.Hash, + normalizedDatasetId)) + .ToList(); + + await ApplyBatchAsync(normalizedEntries, cancellationToken); + } + private async Task UpsertAsync(OplogEntry entry, RecordId recordId, CancellationToken cancellationToken) { await EnsureReadyAsync(cancellationToken); @@ -271,25 +511,56 @@ public class SurrealOplogStore : OplogStore await _schemaInitializer!.EnsureInitializedAsync(cancellationToken); } - private async Task> SelectAllAsync(CancellationToken cancellationToken) + private async Task> SelectAllAsync(string datasetId, CancellationToken cancellationToken) { + string normalizedDatasetId = NormalizeDatasetId(datasetId); await EnsureReadyAsync(cancellationToken); var rows = await _surrealClient!.Select( CBDDCSurrealSchemaNames.OplogEntriesTable, cancellationToken); - return rows?.ToList() ?? []; + return rows? + .Where(row => MatchesDataset(row.DatasetId, normalizedDatasetId)) + .ToList() + ?? []; } - private async Task FindByHashAsync(string hash, CancellationToken cancellationToken) + private async Task FindByHashAsync(string hash, string datasetId, CancellationToken cancellationToken) { + string normalizedDatasetId = NormalizeDatasetId(datasetId); await EnsureReadyAsync(cancellationToken); RecordId deterministicId = SurrealStoreRecordIds.Oplog(hash); var deterministic = await _surrealClient!.Select(deterministicId, cancellationToken); - if (deterministic != null && string.Equals(deterministic.Hash, hash, StringComparison.Ordinal)) + if (deterministic != null && + string.Equals(deterministic.Hash, hash, StringComparison.Ordinal) && + MatchesDataset(deterministic.DatasetId, normalizedDatasetId)) return deterministic; - var all = await SelectAllAsync(cancellationToken); + var all = await SelectAllAsync(normalizedDatasetId, cancellationToken); return all.FirstOrDefault(o => string.Equals(o.Hash, hash, StringComparison.Ordinal)); } + + private async Task> LoadOplogHashesAsync(string datasetId, CancellationToken cancellationToken) + { + var rows = await SelectAllAsync(datasetId, cancellationToken); + return rows + .Where(r => !string.IsNullOrWhiteSpace(r.Hash)) + .Select(r => r.Hash) + .ToHashSet(StringComparer.Ordinal); + } + + private async Task> LoadOplogRecordIdsByHashAsync(string datasetId, + CancellationToken cancellationToken) + { + var rows = await SelectAllAsync(datasetId, cancellationToken); + var records = new Dictionary(StringComparer.Ordinal); + + foreach (var row in rows) + { + if (string.IsNullOrWhiteSpace(row.Hash)) continue; + records[row.Hash] = row.Id ?? SurrealStoreRecordIds.Oplog(row.Hash); + } + + return records; + } } diff --git a/src/ZB.MOM.WW.CBDDC.Persistence/Surreal/SurrealPeerOplogConfirmationStore.cs b/src/ZB.MOM.WW.CBDDC.Persistence/Surreal/SurrealPeerOplogConfirmationStore.cs index 66dcf80..6e692c4 100644 --- a/src/ZB.MOM.WW.CBDDC.Persistence/Surreal/SurrealPeerOplogConfirmationStore.cs +++ b/src/ZB.MOM.WW.CBDDC.Persistence/Surreal/SurrealPeerOplogConfirmationStore.cs @@ -10,6 +10,7 @@ namespace ZB.MOM.WW.CBDDC.Persistence.Surreal; public class SurrealPeerOplogConfirmationStore : PeerOplogConfirmationStore { internal const string RegistrationSourceNodeId = "__peer_registration__"; + private const string PrimaryDatasetId = DatasetId.Primary; private readonly ILogger _logger; private readonly ICBDDCSurrealSchemaInitializer _schemaInitializer; @@ -32,6 +33,19 @@ public class SurrealPeerOplogConfirmationStore : PeerOplogConfirmationStore _logger = logger ?? NullLogger.Instance; } + private static string NormalizeDatasetId(string? datasetId) + { + return DatasetId.Normalize(datasetId); + } + + private static bool MatchesDataset(string? recordDatasetId, string datasetId) + { + if (string.IsNullOrWhiteSpace(recordDatasetId)) + return string.Equals(datasetId, PrimaryDatasetId, StringComparison.Ordinal); + + return string.Equals(NormalizeDatasetId(recordDatasetId), datasetId, StringComparison.Ordinal); + } + /// public override async Task EnsurePeerRegisteredAsync( string peerNodeId, @@ -39,16 +53,29 @@ public class SurrealPeerOplogConfirmationStore : PeerOplogConfirmationStore PeerType type, CancellationToken cancellationToken = default) { + await EnsurePeerRegisteredAsync(peerNodeId, address, type, PrimaryDatasetId, cancellationToken); + } + + /// + public async Task EnsurePeerRegisteredAsync( + string peerNodeId, + string address, + PeerType type, + string datasetId, + CancellationToken cancellationToken = default) + { + string normalizedDatasetId = NormalizeDatasetId(datasetId); if (string.IsNullOrWhiteSpace(peerNodeId)) throw new ArgumentException("Peer node id is required.", nameof(peerNodeId)); var existing = - await FindByPairAsync(peerNodeId, RegistrationSourceNodeId, cancellationToken); + await FindByPairAsync(peerNodeId, RegistrationSourceNodeId, normalizedDatasetId, cancellationToken); if (existing == null) { var created = new PeerOplogConfirmation { + DatasetId = normalizedDatasetId, PeerNodeId = peerNodeId, SourceNodeId = RegistrationSourceNodeId, ConfirmedWall = 0, @@ -58,7 +85,9 @@ public class SurrealPeerOplogConfirmationStore : PeerOplogConfirmationStore IsActive = true }; - await UpsertAsync(created, SurrealStoreRecordIds.PeerOplogConfirmation(peerNodeId, RegistrationSourceNodeId), + await UpsertAsync( + created, + SurrealStoreRecordIds.PeerOplogConfirmation(peerNodeId, RegistrationSourceNodeId, normalizedDatasetId), cancellationToken); _logger.LogDebug("Registered peer confirmation tracking for {PeerNodeId} ({Address}, {Type}).", peerNodeId, @@ -71,7 +100,8 @@ public class SurrealPeerOplogConfirmationStore : PeerOplogConfirmationStore existing.IsActive = true; existing.LastConfirmedUtcMs = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(); RecordId recordId = - existing.Id ?? SurrealStoreRecordIds.PeerOplogConfirmation(peerNodeId, RegistrationSourceNodeId); + existing.Id ?? + SurrealStoreRecordIds.PeerOplogConfirmation(peerNodeId, RegistrationSourceNodeId, normalizedDatasetId); await UpsertAsync(existing, recordId, cancellationToken); } @@ -83,19 +113,33 @@ public class SurrealPeerOplogConfirmationStore : PeerOplogConfirmationStore string hash, CancellationToken cancellationToken = default) { + await UpdateConfirmationAsync(peerNodeId, sourceNodeId, timestamp, hash, PrimaryDatasetId, cancellationToken); + } + + /// + public async Task UpdateConfirmationAsync( + string peerNodeId, + string sourceNodeId, + HlcTimestamp timestamp, + string hash, + string datasetId, + CancellationToken cancellationToken = default) + { + string normalizedDatasetId = NormalizeDatasetId(datasetId); if (string.IsNullOrWhiteSpace(peerNodeId)) throw new ArgumentException("Peer node id is required.", nameof(peerNodeId)); if (string.IsNullOrWhiteSpace(sourceNodeId)) throw new ArgumentException("Source node id is required.", nameof(sourceNodeId)); - var existing = await FindByPairAsync(peerNodeId, sourceNodeId, cancellationToken); + var existing = await FindByPairAsync(peerNodeId, sourceNodeId, normalizedDatasetId, cancellationToken); long nowMs = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(); if (existing == null) { var created = new PeerOplogConfirmation { + DatasetId = normalizedDatasetId, PeerNodeId = peerNodeId, SourceNodeId = sourceNodeId, ConfirmedWall = timestamp.PhysicalTime, @@ -104,7 +148,7 @@ public class SurrealPeerOplogConfirmationStore : PeerOplogConfirmationStore LastConfirmedUtc = DateTimeOffset.FromUnixTimeMilliseconds(nowMs), IsActive = true }; - await UpsertAsync(created, SurrealStoreRecordIds.PeerOplogConfirmation(peerNodeId, sourceNodeId), + await UpsertAsync(created, SurrealStoreRecordIds.PeerOplogConfirmation(peerNodeId, sourceNodeId, normalizedDatasetId), cancellationToken); return; } @@ -122,7 +166,7 @@ public class SurrealPeerOplogConfirmationStore : PeerOplogConfirmationStore existing.LastConfirmedUtcMs = nowMs; existing.IsActive = true; - RecordId recordId = existing.Id ?? SurrealStoreRecordIds.PeerOplogConfirmation(peerNodeId, sourceNodeId); + RecordId recordId = existing.Id ?? SurrealStoreRecordIds.PeerOplogConfirmation(peerNodeId, sourceNodeId, normalizedDatasetId); await UpsertAsync(existing, recordId, cancellationToken); } @@ -130,7 +174,15 @@ public class SurrealPeerOplogConfirmationStore : PeerOplogConfirmationStore public override async Task> GetConfirmationsAsync( CancellationToken cancellationToken = default) { - var all = await SelectAllAsync(cancellationToken); + return await GetConfirmationsAsync(PrimaryDatasetId, cancellationToken); + } + + /// + public async Task> GetConfirmationsAsync( + string datasetId, + CancellationToken cancellationToken = default) + { + var all = await SelectAllAsync(NormalizeDatasetId(datasetId), cancellationToken); return all .Where(c => !string.Equals(c.SourceNodeId, RegistrationSourceNodeId, StringComparison.Ordinal)) .Select(c => c.ToDomain()) @@ -141,11 +193,20 @@ public class SurrealPeerOplogConfirmationStore : PeerOplogConfirmationStore public override async Task> GetConfirmationsForPeerAsync( string peerNodeId, CancellationToken cancellationToken = default) + { + return await GetConfirmationsForPeerAsync(peerNodeId, PrimaryDatasetId, cancellationToken); + } + + /// + public async Task> GetConfirmationsForPeerAsync( + string peerNodeId, + string datasetId, + CancellationToken cancellationToken = default) { if (string.IsNullOrWhiteSpace(peerNodeId)) throw new ArgumentException("Peer node id is required.", nameof(peerNodeId)); - var all = await SelectAllAsync(cancellationToken); + var all = await SelectAllAsync(NormalizeDatasetId(datasetId), cancellationToken); return all .Where(c => string.Equals(c.PeerNodeId, peerNodeId, StringComparison.Ordinal) && !string.Equals(c.SourceNodeId, RegistrationSourceNodeId, StringComparison.Ordinal)) @@ -156,10 +217,18 @@ public class SurrealPeerOplogConfirmationStore : PeerOplogConfirmationStore /// public override async Task RemovePeerTrackingAsync(string peerNodeId, CancellationToken cancellationToken = default) { + await RemovePeerTrackingAsync(peerNodeId, PrimaryDatasetId, cancellationToken); + } + + /// + public async Task RemovePeerTrackingAsync(string peerNodeId, string datasetId, + CancellationToken cancellationToken = default) + { + string normalizedDatasetId = NormalizeDatasetId(datasetId); if (string.IsNullOrWhiteSpace(peerNodeId)) throw new ArgumentException("Peer node id is required.", nameof(peerNodeId)); - var matches = (await SelectAllAsync(cancellationToken)) + var matches = (await SelectAllAsync(normalizedDatasetId, cancellationToken)) .Where(c => string.Equals(c.PeerNodeId, peerNodeId, StringComparison.Ordinal)) .ToList(); @@ -173,7 +242,11 @@ public class SurrealPeerOplogConfirmationStore : PeerOplogConfirmationStore match.IsActive = false; match.LastConfirmedUtcMs = nowMs; - RecordId recordId = match.Id ?? SurrealStoreRecordIds.PeerOplogConfirmation(match.PeerNodeId, match.SourceNodeId); + RecordId recordId = match.Id ?? + SurrealStoreRecordIds.PeerOplogConfirmation( + match.PeerNodeId, + match.SourceNodeId, + normalizedDatasetId); await UpsertAsync(match, recordId, cancellationToken); } } @@ -182,7 +255,15 @@ public class SurrealPeerOplogConfirmationStore : PeerOplogConfirmationStore public override async Task> GetActiveTrackedPeersAsync( CancellationToken cancellationToken = default) { - var all = await SelectAllAsync(cancellationToken); + return await GetActiveTrackedPeersAsync(PrimaryDatasetId, cancellationToken); + } + + /// + public async Task> GetActiveTrackedPeersAsync( + string datasetId, + CancellationToken cancellationToken = default) + { + var all = await SelectAllAsync(NormalizeDatasetId(datasetId), cancellationToken); return all .Where(c => c.IsActive) .Select(c => c.PeerNodeId) @@ -193,14 +274,45 @@ public class SurrealPeerOplogConfirmationStore : PeerOplogConfirmationStore /// public override async Task DropAsync(CancellationToken cancellationToken = default) { - await EnsureReadyAsync(cancellationToken); - await _surrealClient.Delete(CBDDCSurrealSchemaNames.PeerOplogConfirmationsTable, cancellationToken); + await DropAsync(PrimaryDatasetId, cancellationToken); + } + + /// + /// Drops all peer confirmation rows for a dataset. + /// + /// The dataset identifier. + /// A cancellation token. + public async Task DropAsync(string datasetId, CancellationToken cancellationToken = default) + { + var rows = await SelectAllAsync(NormalizeDatasetId(datasetId), cancellationToken); + foreach (var row in rows) + { + RecordId recordId = row.Id ?? + SurrealStoreRecordIds.PeerOplogConfirmation( + row.PeerNodeId, + row.SourceNodeId, + NormalizeDatasetId(datasetId)); + await _surrealClient.Delete(recordId, cancellationToken); + } } /// public override async Task> ExportAsync(CancellationToken cancellationToken = default) { - var all = await SelectAllAsync(cancellationToken); + var all = await SelectAllAsync(PrimaryDatasetId, cancellationToken); + return all.Select(c => c.ToDomain()).ToList(); + } + + /// + /// Exports peer confirmations for a dataset. + /// + /// The dataset identifier. + /// A cancellation token. + /// The exported confirmations. + public async Task> ExportAsync(string datasetId, + CancellationToken cancellationToken = default) + { + var all = await SelectAllAsync(NormalizeDatasetId(datasetId), cancellationToken); return all.Select(c => c.ToDomain()).ToList(); } @@ -208,11 +320,25 @@ public class SurrealPeerOplogConfirmationStore : PeerOplogConfirmationStore public override async Task ImportAsync(IEnumerable items, CancellationToken cancellationToken = default) { + await ImportAsync(items, PrimaryDatasetId, cancellationToken); + } + + /// + /// Imports peer confirmation rows into a dataset. + /// + /// The confirmation items. + /// The dataset identifier. + /// A cancellation token. + public async Task ImportAsync(IEnumerable items, string datasetId, + CancellationToken cancellationToken = default) + { + string normalizedDatasetId = NormalizeDatasetId(datasetId); foreach (var item in items) { - var existing = await FindByPairAsync(item.PeerNodeId, item.SourceNodeId, cancellationToken); + item.DatasetId = normalizedDatasetId; + var existing = await FindByPairAsync(item.PeerNodeId, item.SourceNodeId, normalizedDatasetId, cancellationToken); RecordId recordId = - existing?.Id ?? SurrealStoreRecordIds.PeerOplogConfirmation(item.PeerNodeId, item.SourceNodeId); + existing?.Id ?? SurrealStoreRecordIds.PeerOplogConfirmation(item.PeerNodeId, item.SourceNodeId, normalizedDatasetId); await UpsertAsync(item, recordId, cancellationToken); } } @@ -221,12 +347,26 @@ public class SurrealPeerOplogConfirmationStore : PeerOplogConfirmationStore public override async Task MergeAsync(IEnumerable items, CancellationToken cancellationToken = default) { + await MergeAsync(items, PrimaryDatasetId, cancellationToken); + } + + /// + /// Merges peer confirmations into a dataset. + /// + /// The confirmation items. + /// The dataset identifier. + /// A cancellation token. + public async Task MergeAsync(IEnumerable items, string datasetId, + CancellationToken cancellationToken = default) + { + string normalizedDatasetId = NormalizeDatasetId(datasetId); foreach (var item in items) { - var existing = await FindByPairAsync(item.PeerNodeId, item.SourceNodeId, cancellationToken); + item.DatasetId = normalizedDatasetId; + var existing = await FindByPairAsync(item.PeerNodeId, item.SourceNodeId, normalizedDatasetId, cancellationToken); if (existing == null) { - await UpsertAsync(item, SurrealStoreRecordIds.PeerOplogConfirmation(item.PeerNodeId, item.SourceNodeId), + await UpsertAsync(item, SurrealStoreRecordIds.PeerOplogConfirmation(item.PeerNodeId, item.SourceNodeId, normalizedDatasetId), cancellationToken); continue; } @@ -259,7 +399,11 @@ public class SurrealPeerOplogConfirmationStore : PeerOplogConfirmationStore if (!changed) continue; RecordId recordId = - existing.Id ?? SurrealStoreRecordIds.PeerOplogConfirmation(existing.PeerNodeId, existing.SourceNodeId); + existing.Id ?? + SurrealStoreRecordIds.PeerOplogConfirmation( + existing.PeerNodeId, + existing.SourceNodeId, + normalizedDatasetId); await UpsertAsync(existing, recordId, cancellationToken); } } @@ -288,27 +432,37 @@ public class SurrealPeerOplogConfirmationStore : PeerOplogConfirmationStore await _schemaInitializer.EnsureInitializedAsync(cancellationToken); } - private async Task> SelectAllAsync(CancellationToken cancellationToken) + private async Task> SelectAllAsync(string datasetId, + CancellationToken cancellationToken) { + string normalizedDatasetId = NormalizeDatasetId(datasetId); await EnsureReadyAsync(cancellationToken); var rows = await _surrealClient.Select( CBDDCSurrealSchemaNames.PeerOplogConfirmationsTable, cancellationToken); - return rows?.ToList() ?? []; + return rows? + .Where(row => MatchesDataset(row.DatasetId, normalizedDatasetId)) + .ToList() + ?? []; } - private async Task FindByPairAsync(string peerNodeId, string sourceNodeId, + private async Task FindByPairAsync( + string peerNodeId, + string sourceNodeId, + string datasetId, CancellationToken cancellationToken) { + string normalizedDatasetId = NormalizeDatasetId(datasetId); await EnsureReadyAsync(cancellationToken); - RecordId deterministicId = SurrealStoreRecordIds.PeerOplogConfirmation(peerNodeId, sourceNodeId); + RecordId deterministicId = SurrealStoreRecordIds.PeerOplogConfirmation(peerNodeId, sourceNodeId, normalizedDatasetId); var deterministic = await _surrealClient.Select(deterministicId, cancellationToken); if (deterministic != null && + MatchesDataset(deterministic.DatasetId, normalizedDatasetId) && string.Equals(deterministic.PeerNodeId, peerNodeId, StringComparison.Ordinal) && string.Equals(deterministic.SourceNodeId, sourceNodeId, StringComparison.Ordinal)) return deterministic; - var all = await SelectAllAsync(cancellationToken); + var all = await SelectAllAsync(normalizedDatasetId, cancellationToken); return all.FirstOrDefault(c => string.Equals(c.PeerNodeId, peerNodeId, StringComparison.Ordinal) && string.Equals(c.SourceNodeId, sourceNodeId, StringComparison.Ordinal)); diff --git a/src/ZB.MOM.WW.CBDDC.Persistence/Surreal/SurrealSnapshotMetadataStore.cs b/src/ZB.MOM.WW.CBDDC.Persistence/Surreal/SurrealSnapshotMetadataStore.cs index cd20999..ca440d0 100644 --- a/src/ZB.MOM.WW.CBDDC.Persistence/Surreal/SurrealSnapshotMetadataStore.cs +++ b/src/ZB.MOM.WW.CBDDC.Persistence/Surreal/SurrealSnapshotMetadataStore.cs @@ -8,6 +8,7 @@ namespace ZB.MOM.WW.CBDDC.Persistence.Surreal; public class SurrealSnapshotMetadataStore : SnapshotMetadataStore { + private const string PrimaryDatasetId = DatasetId.Primary; private readonly ILogger _logger; private readonly ICBDDCSurrealSchemaInitializer _schemaInitializer; private readonly ISurrealDbClient _surrealClient; @@ -29,17 +30,56 @@ public class SurrealSnapshotMetadataStore : SnapshotMetadataStore _logger = logger ?? NullLogger.Instance; } + private static string NormalizeDatasetId(string? datasetId) + { + return DatasetId.Normalize(datasetId); + } + + private static bool MatchesDataset(string? recordDatasetId, string datasetId) + { + if (string.IsNullOrWhiteSpace(recordDatasetId)) + return string.Equals(datasetId, PrimaryDatasetId, StringComparison.Ordinal); + + return string.Equals(NormalizeDatasetId(recordDatasetId), datasetId, StringComparison.Ordinal); + } + /// public override async Task DropAsync(CancellationToken cancellationToken = default) { - await EnsureReadyAsync(cancellationToken); - await _surrealClient.Delete(CBDDCSurrealSchemaNames.SnapshotMetadataTable, cancellationToken); + await DropAsync(PrimaryDatasetId, cancellationToken); + } + + /// + /// Drops snapshot metadata rows for a dataset. + /// + /// The dataset identifier. + /// A cancellation token. + public async Task DropAsync(string datasetId, CancellationToken cancellationToken = default) + { + var rows = await SelectAllAsync(NormalizeDatasetId(datasetId), cancellationToken); + foreach (var row in rows) + { + RecordId recordId = row.Id ?? SurrealStoreRecordIds.SnapshotMetadata(row.NodeId, NormalizeDatasetId(datasetId)); + await _surrealClient.Delete(recordId, cancellationToken); + } } /// public override async Task> ExportAsync(CancellationToken cancellationToken = default) { - var all = await SelectAllAsync(cancellationToken); + var all = await SelectAllAsync(PrimaryDatasetId, cancellationToken); + return all.Select(m => m.ToDomain()).ToList(); + } + + /// + /// Exports snapshot metadata rows for a dataset. + /// + /// The dataset identifier. + /// A cancellation token. + /// Dataset-scoped snapshot metadata rows. + public async Task> ExportAsync(string datasetId, CancellationToken cancellationToken = default) + { + var all = await SelectAllAsync(NormalizeDatasetId(datasetId), cancellationToken); return all.Select(m => m.ToDomain()).ToList(); } @@ -47,14 +87,32 @@ public class SurrealSnapshotMetadataStore : SnapshotMetadataStore public override async Task GetSnapshotMetadataAsync(string nodeId, CancellationToken cancellationToken = default) { - var existing = await FindByNodeIdAsync(nodeId, cancellationToken); + var existing = await FindByNodeIdAsync(nodeId, PrimaryDatasetId, cancellationToken); + return existing?.ToDomain(); + } + + /// + public async Task GetSnapshotMetadataAsync( + string nodeId, + string datasetId, + CancellationToken cancellationToken = default) + { + var existing = await FindByNodeIdAsync(nodeId, NormalizeDatasetId(datasetId), cancellationToken); return existing?.ToDomain(); } /// public override async Task GetSnapshotHashAsync(string nodeId, CancellationToken cancellationToken = default) { - var existing = await FindByNodeIdAsync(nodeId, cancellationToken); + var existing = await FindByNodeIdAsync(nodeId, PrimaryDatasetId, cancellationToken); + return existing?.Hash; + } + + /// + public async Task GetSnapshotHashAsync(string nodeId, string datasetId, + CancellationToken cancellationToken = default) + { + var existing = await FindByNodeIdAsync(nodeId, NormalizeDatasetId(datasetId), cancellationToken); return existing?.Hash; } @@ -62,10 +120,24 @@ public class SurrealSnapshotMetadataStore : SnapshotMetadataStore public override async Task ImportAsync(IEnumerable items, CancellationToken cancellationToken = default) { + await ImportAsync(items, PrimaryDatasetId, cancellationToken); + } + + /// + /// Imports snapshot metadata rows into a dataset. + /// + /// Snapshot metadata items. + /// The dataset identifier. + /// A cancellation token. + public async Task ImportAsync(IEnumerable items, string datasetId, + CancellationToken cancellationToken = default) + { + string normalizedDatasetId = NormalizeDatasetId(datasetId); foreach (var item in items) { - var existing = await FindByNodeIdAsync(item.NodeId, cancellationToken); - RecordId recordId = existing?.Id ?? SurrealStoreRecordIds.SnapshotMetadata(item.NodeId); + item.DatasetId = normalizedDatasetId; + var existing = await FindByNodeIdAsync(item.NodeId, normalizedDatasetId, cancellationToken); + RecordId recordId = existing?.Id ?? SurrealStoreRecordIds.SnapshotMetadata(item.NodeId, normalizedDatasetId); await UpsertAsync(item, recordId, cancellationToken); } } @@ -74,20 +146,45 @@ public class SurrealSnapshotMetadataStore : SnapshotMetadataStore public override async Task InsertSnapshotMetadataAsync(SnapshotMetadata metadata, CancellationToken cancellationToken = default) { - var existing = await FindByNodeIdAsync(metadata.NodeId, cancellationToken); - RecordId recordId = existing?.Id ?? SurrealStoreRecordIds.SnapshotMetadata(metadata.NodeId); + await InsertSnapshotMetadataAsync(metadata, PrimaryDatasetId, cancellationToken); + } + + /// + public async Task InsertSnapshotMetadataAsync( + SnapshotMetadata metadata, + string datasetId, + CancellationToken cancellationToken = default) + { + string normalizedDatasetId = NormalizeDatasetId(datasetId); + metadata.DatasetId = normalizedDatasetId; + var existing = await FindByNodeIdAsync(metadata.NodeId, normalizedDatasetId, cancellationToken); + RecordId recordId = existing?.Id ?? SurrealStoreRecordIds.SnapshotMetadata(metadata.NodeId, normalizedDatasetId); await UpsertAsync(metadata, recordId, cancellationToken); } /// public override async Task MergeAsync(IEnumerable items, CancellationToken cancellationToken = default) { + await MergeAsync(items, PrimaryDatasetId, cancellationToken); + } + + /// + /// Merges snapshot metadata rows into a dataset. + /// + /// Snapshot metadata items. + /// The dataset identifier. + /// A cancellation token. + public async Task MergeAsync(IEnumerable items, string datasetId, + CancellationToken cancellationToken = default) + { + string normalizedDatasetId = NormalizeDatasetId(datasetId); foreach (var metadata in items) { - var existing = await FindByNodeIdAsync(metadata.NodeId, cancellationToken); + metadata.DatasetId = normalizedDatasetId; + var existing = await FindByNodeIdAsync(metadata.NodeId, normalizedDatasetId, cancellationToken); if (existing == null) { - await UpsertAsync(metadata, SurrealStoreRecordIds.SnapshotMetadata(metadata.NodeId), cancellationToken); + await UpsertAsync(metadata, SurrealStoreRecordIds.SnapshotMetadata(metadata.NodeId, normalizedDatasetId), cancellationToken); continue; } @@ -96,7 +193,7 @@ public class SurrealSnapshotMetadataStore : SnapshotMetadataStore metadata.TimestampLogicalCounter <= existing.TimestampLogicalCounter)) continue; - RecordId recordId = existing.Id ?? SurrealStoreRecordIds.SnapshotMetadata(metadata.NodeId); + RecordId recordId = existing.Id ?? SurrealStoreRecordIds.SnapshotMetadata(metadata.NodeId, normalizedDatasetId); await UpsertAsync(metadata, recordId, cancellationToken); } } @@ -105,10 +202,21 @@ public class SurrealSnapshotMetadataStore : SnapshotMetadataStore public override async Task UpdateSnapshotMetadataAsync(SnapshotMetadata existingMeta, CancellationToken cancellationToken) { - var existing = await FindByNodeIdAsync(existingMeta.NodeId, cancellationToken); + await UpdateSnapshotMetadataAsync(existingMeta, PrimaryDatasetId, cancellationToken); + } + + /// + public async Task UpdateSnapshotMetadataAsync( + SnapshotMetadata existingMeta, + string datasetId, + CancellationToken cancellationToken = default) + { + string normalizedDatasetId = NormalizeDatasetId(datasetId); + existingMeta.DatasetId = normalizedDatasetId; + var existing = await FindByNodeIdAsync(existingMeta.NodeId, normalizedDatasetId, cancellationToken); if (existing == null) return; - RecordId recordId = existing.Id ?? SurrealStoreRecordIds.SnapshotMetadata(existingMeta.NodeId); + RecordId recordId = existing.Id ?? SurrealStoreRecordIds.SnapshotMetadata(existingMeta.NodeId, normalizedDatasetId); await UpsertAsync(existingMeta, recordId, cancellationToken); } @@ -116,7 +224,15 @@ public class SurrealSnapshotMetadataStore : SnapshotMetadataStore public override async Task> GetAllSnapshotMetadataAsync( CancellationToken cancellationToken = default) { - return await ExportAsync(cancellationToken); + return await ExportAsync(PrimaryDatasetId, cancellationToken); + } + + /// + public async Task> GetAllSnapshotMetadataAsync( + string datasetId, + CancellationToken cancellationToken = default) + { + return await ExportAsync(datasetId, cancellationToken); } private async Task UpsertAsync(SnapshotMetadata metadata, RecordId recordId, CancellationToken cancellationToken) @@ -133,25 +249,33 @@ public class SurrealSnapshotMetadataStore : SnapshotMetadataStore await _schemaInitializer.EnsureInitializedAsync(cancellationToken); } - private async Task> SelectAllAsync(CancellationToken cancellationToken) + private async Task> SelectAllAsync(string datasetId, + CancellationToken cancellationToken) { + string normalizedDatasetId = NormalizeDatasetId(datasetId); await EnsureReadyAsync(cancellationToken); var rows = await _surrealClient.Select( CBDDCSurrealSchemaNames.SnapshotMetadataTable, cancellationToken); - return rows?.ToList() ?? []; + return rows? + .Where(row => MatchesDataset(row.DatasetId, normalizedDatasetId)) + .ToList() + ?? []; } - private async Task FindByNodeIdAsync(string nodeId, CancellationToken cancellationToken) + private async Task FindByNodeIdAsync(string nodeId, string datasetId, + CancellationToken cancellationToken) { + string normalizedDatasetId = NormalizeDatasetId(datasetId); await EnsureReadyAsync(cancellationToken); - RecordId deterministicId = SurrealStoreRecordIds.SnapshotMetadata(nodeId); + RecordId deterministicId = SurrealStoreRecordIds.SnapshotMetadata(nodeId, normalizedDatasetId); var deterministic = await _surrealClient.Select(deterministicId, cancellationToken); if (deterministic != null && + MatchesDataset(deterministic.DatasetId, normalizedDatasetId) && string.Equals(deterministic.NodeId, nodeId, StringComparison.Ordinal)) return deterministic; - var all = await SelectAllAsync(cancellationToken); + var all = await SelectAllAsync(normalizedDatasetId, cancellationToken); return all.FirstOrDefault(m => string.Equals(m.NodeId, nodeId, StringComparison.Ordinal)); } } diff --git a/src/ZB.MOM.WW.CBDDC.Persistence/Surreal/SurrealStoreRecords.cs b/src/ZB.MOM.WW.CBDDC.Persistence/Surreal/SurrealStoreRecords.cs index a26234f..2c74c1e 100644 --- a/src/ZB.MOM.WW.CBDDC.Persistence/Surreal/SurrealStoreRecords.cs +++ b/src/ZB.MOM.WW.CBDDC.Persistence/Surreal/SurrealStoreRecords.cs @@ -26,22 +26,28 @@ internal static class SurrealStoreRecordIds /// /// The document collection name. /// The document key. + /// The dataset identifier. /// The SurrealDB record identifier. - public static RecordId DocumentMetadata(string collection, string key) + public static RecordId DocumentMetadata(string collection, string key, string datasetId) { + string normalizedDatasetId = DatasetId.Normalize(datasetId); return RecordId.From( CBDDCSurrealSchemaNames.DocumentMetadataTable, - CompositeKey("docmeta", collection, key)); + CompositeKey("docmeta", normalizedDatasetId, collection, key)); } /// /// Creates the record identifier for snapshot metadata. /// /// The node identifier. + /// The dataset identifier. /// The SurrealDB record identifier. - public static RecordId SnapshotMetadata(string nodeId) + public static RecordId SnapshotMetadata(string nodeId, string datasetId) { - return RecordId.From(CBDDCSurrealSchemaNames.SnapshotMetadataTable, nodeId); + string normalizedDatasetId = DatasetId.Normalize(datasetId); + return RecordId.From( + CBDDCSurrealSchemaNames.SnapshotMetadataTable, + CompositeKey("snapshotmeta", normalizedDatasetId, nodeId)); } /// @@ -59,12 +65,14 @@ internal static class SurrealStoreRecordIds /// /// The peer node identifier. /// The source node identifier. + /// The dataset identifier. /// The SurrealDB record identifier. - public static RecordId PeerOplogConfirmation(string peerNodeId, string sourceNodeId) + public static RecordId PeerOplogConfirmation(string peerNodeId, string sourceNodeId, string datasetId) { + string normalizedDatasetId = DatasetId.Normalize(datasetId); return RecordId.From( CBDDCSurrealSchemaNames.PeerOplogConfirmationsTable, - CompositeKey("peerconfirm", peerNodeId, sourceNodeId)); + CompositeKey("peerconfirm", normalizedDatasetId, peerNodeId, sourceNodeId)); } private static string CompositeKey(string prefix, string first, string second) @@ -72,10 +80,22 @@ internal static class SurrealStoreRecordIds byte[] bytes = Encoding.UTF8.GetBytes($"{prefix}\n{first}\n{second}"); return Convert.ToHexString(SHA256.HashData(bytes)).ToLowerInvariant(); } + + private static string CompositeKey(string prefix, string first, string second, string third) + { + byte[] bytes = Encoding.UTF8.GetBytes($"{prefix}\n{first}\n{second}\n{third}"); + return Convert.ToHexString(SHA256.HashData(bytes)).ToLowerInvariant(); + } } internal sealed class SurrealOplogRecord : Record { + /// + /// Gets or sets the dataset identifier. + /// + [JsonPropertyName("datasetId")] + public string DatasetId { get; set; } = ""; + /// /// Gets or sets the collection name. /// @@ -133,6 +153,12 @@ internal sealed class SurrealOplogRecord : Record internal sealed class SurrealDocumentMetadataRecord : Record { + /// + /// Gets or sets the dataset identifier. + /// + [JsonPropertyName("datasetId")] + public string DatasetId { get; set; } = ""; + /// /// Gets or sets the collection name. /// @@ -205,6 +231,12 @@ internal sealed class SurrealRemotePeerRecord : Record internal sealed class SurrealPeerOplogConfirmationRecord : Record { + /// + /// Gets or sets the dataset identifier. + /// + [JsonPropertyName("datasetId")] + public string DatasetId { get; set; } = ""; + /// /// Gets or sets the peer node identifier. /// @@ -250,6 +282,12 @@ internal sealed class SurrealPeerOplogConfirmationRecord : Record internal sealed class SurrealSnapshotMetadataRecord : Record { + /// + /// Gets or sets the dataset identifier. + /// + [JsonPropertyName("datasetId")] + public string DatasetId { get; set; } = ""; + /// /// Gets or sets the node identifier. /// @@ -286,6 +324,7 @@ internal static class SurrealStoreRecordMappers { return new SurrealOplogRecord { + DatasetId = DatasetId.Normalize(entry.DatasetId), Collection = entry.Collection, Key = entry.Key, Operation = (int)entry.Operation, @@ -316,7 +355,8 @@ internal static class SurrealStoreRecordMappers payload, new HlcTimestamp(record.TimestampPhysicalTime, record.TimestampLogicalCounter, record.TimestampNodeId), record.PreviousHash, - record.Hash); + record.Hash, + record.DatasetId); } /// @@ -328,6 +368,7 @@ internal static class SurrealStoreRecordMappers { return new SurrealDocumentMetadataRecord { + DatasetId = DatasetId.Normalize(metadata.DatasetId), Collection = metadata.Collection, Key = metadata.Key, HlcPhysicalTime = metadata.UpdatedAt.PhysicalTime, @@ -348,7 +389,8 @@ internal static class SurrealStoreRecordMappers record.Collection, record.Key, new HlcTimestamp(record.HlcPhysicalTime, record.HlcLogicalCounter, record.HlcNodeId), - record.IsDeleted); + record.IsDeleted, + record.DatasetId); } /// @@ -401,6 +443,7 @@ internal static class SurrealStoreRecordMappers { return new SurrealPeerOplogConfirmationRecord { + DatasetId = DatasetId.Normalize(confirmation.DatasetId), PeerNodeId = confirmation.PeerNodeId, SourceNodeId = confirmation.SourceNodeId, ConfirmedWall = confirmation.ConfirmedWall, @@ -420,6 +463,7 @@ internal static class SurrealStoreRecordMappers { return new PeerOplogConfirmation { + DatasetId = DatasetId.Normalize(record.DatasetId), PeerNodeId = record.PeerNodeId, SourceNodeId = record.SourceNodeId, ConfirmedWall = record.ConfirmedWall, @@ -439,6 +483,7 @@ internal static class SurrealStoreRecordMappers { return new SurrealSnapshotMetadataRecord { + DatasetId = DatasetId.Normalize(metadata.DatasetId), NodeId = metadata.NodeId, TimestampPhysicalTime = metadata.TimestampPhysicalTime, TimestampLogicalCounter = metadata.TimestampLogicalCounter, @@ -455,6 +500,7 @@ internal static class SurrealStoreRecordMappers { return new SnapshotMetadata { + DatasetId = DatasetId.Normalize(record.DatasetId), NodeId = record.NodeId, TimestampPhysicalTime = record.TimestampPhysicalTime, TimestampLogicalCounter = record.TimestampLogicalCounter, diff --git a/tests/ZB.MOM.WW.CBDDC.Core.Tests/DatasetAwareModelTests.cs b/tests/ZB.MOM.WW.CBDDC.Core.Tests/DatasetAwareModelTests.cs new file mode 100644 index 0000000..9d0f130 --- /dev/null +++ b/tests/ZB.MOM.WW.CBDDC.Core.Tests/DatasetAwareModelTests.cs @@ -0,0 +1,43 @@ +using System.Text.Json; +using ZB.MOM.WW.CBDDC.Core.Storage; + +namespace ZB.MOM.WW.CBDDC.Core.Tests; + +public class DatasetAwareModelTests +{ + [Fact] + public void DocumentMetadata_ShouldDefaultDatasetId_ToPrimary() + { + var metadata = new DocumentMetadata("Users", "42", new HlcTimestamp(100, 0, "node")); + + metadata.DatasetId.ShouldBe(DatasetId.Primary); + } + + [Fact] + public void DocumentMetadata_SerializationRoundTrip_ShouldPreserveDatasetId() + { + var original = new DocumentMetadata("Users", "42", new HlcTimestamp(100, 0, "node"), false, "logs"); + + string json = JsonSerializer.Serialize(original); + var restored = JsonSerializer.Deserialize(json); + + restored.ShouldNotBeNull(); + restored.DatasetId.ShouldBe("logs"); + } + + [Fact] + public void SnapshotMetadata_ShouldDefaultDatasetId_ToPrimary() + { + var metadata = new SnapshotMetadata(); + + metadata.DatasetId.ShouldBe(DatasetId.Primary); + } + + [Fact] + public void PeerOplogConfirmation_ShouldDefaultDatasetId_ToPrimary() + { + var confirmation = new PeerOplogConfirmation(); + + confirmation.DatasetId.ShouldBe(DatasetId.Primary); + } +} diff --git a/tests/ZB.MOM.WW.CBDDC.Core.Tests/OplogEntryTests.cs b/tests/ZB.MOM.WW.CBDDC.Core.Tests/OplogEntryTests.cs index 2c33e0f..ef9b05e 100755 --- a/tests/ZB.MOM.WW.CBDDC.Core.Tests/OplogEntryTests.cs +++ b/tests/ZB.MOM.WW.CBDDC.Core.Tests/OplogEntryTests.cs @@ -63,11 +63,35 @@ public class OplogEntryTests /// Verifies that an entry is valid when its stored hash matches computed content. /// [Fact] - public void IsValid_ShouldReturnTrue_WhenHashMatches() - { - var timestamp = new HlcTimestamp(100, 0, "node-1"); - var entry = new OplogEntry("col", "key", OperationType.Put, null, timestamp, "prev"); - - entry.IsValid().ShouldBeTrue(); - } -} \ No newline at end of file + public void IsValid_ShouldReturnTrue_WhenHashMatches() + { + var timestamp = new HlcTimestamp(100, 0, "node-1"); + var entry = new OplogEntry("col", "key", OperationType.Put, null, timestamp, "prev"); + + entry.IsValid().ShouldBeTrue(); + } + + /// + /// Verifies that entries default to the primary dataset when dataset is omitted. + /// + [Fact] + public void Constructor_ShouldDefaultDatasetId_ToPrimary() + { + var entry = new OplogEntry("col", "key", OperationType.Put, null, new HlcTimestamp(1, 0, "node"), "prev"); + + entry.DatasetId.ShouldBe(DatasetId.Primary); + } + + /// + /// Verifies that hash computation includes dataset identity to prevent cross-dataset collisions. + /// + [Fact] + public void ComputeHash_ShouldDiffer_WhenDatasetDiffers() + { + var timestamp = new HlcTimestamp(100, 0, "node-1"); + var primary = new OplogEntry("col", "key", OperationType.Put, null, timestamp, "prev", datasetId: "primary"); + var logs = new OplogEntry("col", "key", OperationType.Put, null, timestamp, "prev", datasetId: "logs"); + + logs.Hash.ShouldNotBe(primary.Hash); + } +} diff --git a/tests/ZB.MOM.WW.CBDDC.Network.Tests/MultiDatasetRegistrationTests.cs b/tests/ZB.MOM.WW.CBDDC.Network.Tests/MultiDatasetRegistrationTests.cs new file mode 100644 index 0000000..0d25786 --- /dev/null +++ b/tests/ZB.MOM.WW.CBDDC.Network.Tests/MultiDatasetRegistrationTests.cs @@ -0,0 +1,47 @@ +using Microsoft.Extensions.DependencyInjection; +using ZB.MOM.WW.CBDDC.Core.Network; +using ZB.MOM.WW.CBDDC.Core.Storage; + +namespace ZB.MOM.WW.CBDDC.Network.Tests; + +public class MultiDatasetRegistrationTests +{ + [Fact] + public void AddCBDDCMultiDataset_ShouldRegisterCoordinatorAndReplaceSyncOrchestrator() + { + var services = new ServiceCollection(); + + services.AddCBDDCNetwork(useHostedService: false); + services.AddCBDDCMultiDataset(options => + { + options.EnableMultiDatasetSync = true; + options.EnableDatasetPrimary = true; + options.EnableDatasetLogs = true; + options.EnableDatasetTimeseries = true; + }); + + services.Any(descriptor => descriptor.ServiceType == typeof(IMultiDatasetSyncOrchestrator)).ShouldBeTrue(); + + var syncDescriptor = services.Last(descriptor => descriptor.ServiceType == typeof(ISyncOrchestrator)); + syncDescriptor.ImplementationFactory.ShouldNotBeNull(); + } + + private sealed class TestPeerNodeConfigurationProvider : IPeerNodeConfigurationProvider + { + public event PeerNodeConfigurationChangedEventHandler? ConfigurationChanged + { + add { } + remove { } + } + + public Task GetConfiguration() + { + return Task.FromResult(new PeerNodeConfiguration + { + NodeId = "node-test", + TcpPort = 9000, + AuthToken = "auth" + }); + } + } +} diff --git a/tests/ZB.MOM.WW.CBDDC.Network.Tests/MultiDatasetSyncOrchestratorTests.cs b/tests/ZB.MOM.WW.CBDDC.Network.Tests/MultiDatasetSyncOrchestratorTests.cs new file mode 100644 index 0000000..357022b --- /dev/null +++ b/tests/ZB.MOM.WW.CBDDC.Network.Tests/MultiDatasetSyncOrchestratorTests.cs @@ -0,0 +1,102 @@ +using Microsoft.Extensions.Logging.Abstractions; +using ZB.MOM.WW.CBDDC.Core; +using ZB.MOM.WW.CBDDC.Core.Network; +using ZB.MOM.WW.CBDDC.Core.Storage; + +namespace ZB.MOM.WW.CBDDC.Network.Tests; + +public class MultiDatasetSyncOrchestratorTests +{ + [Fact] + public void Constructor_WhenMultiDatasetDisabled_ShouldOnlyCreatePrimaryContext() + { + var sut = CreateSut( + [ + new DatasetSyncOptions { DatasetId = DatasetId.Primary, Enabled = true }, + new DatasetSyncOptions { DatasetId = DatasetId.Logs, Enabled = true } + ], + new MultiDatasetRuntimeOptions + { + EnableMultiDatasetSync = false, + EnableDatasetPrimary = true, + EnableDatasetLogs = true + }); + + var datasetIds = sut.Contexts.Select(c => c.DatasetId).ToList(); + datasetIds.Count.ShouldBe(1); + datasetIds[0].ShouldBe(DatasetId.Primary); + } + + [Fact] + public async Task StartStop_WhenOneDatasetThrows_ShouldContinueOtherDatasets() + { + var orchestrators = new Dictionary(StringComparer.Ordinal) + { + [DatasetId.Primary] = new TrackingSyncOrchestrator(), + [DatasetId.Logs] = new TrackingSyncOrchestrator(startException: new InvalidOperationException("boom")), + [DatasetId.Timeseries] = new TrackingSyncOrchestrator() + }; + + var sut = CreateSut( + [], + new MultiDatasetRuntimeOptions + { + EnableMultiDatasetSync = true, + EnableDatasetPrimary = true, + EnableDatasetLogs = true, + EnableDatasetTimeseries = true + }, + options => orchestrators[DatasetId.Normalize(options.DatasetId)]); + + await sut.Start(); + await sut.Stop(); + + orchestrators[DatasetId.Primary].StartCalls.ShouldBe(1); + orchestrators[DatasetId.Primary].StopCalls.ShouldBe(1); + + orchestrators[DatasetId.Logs].StartCalls.ShouldBe(1); + orchestrators[DatasetId.Logs].StopCalls.ShouldBe(1); + + orchestrators[DatasetId.Timeseries].StartCalls.ShouldBe(1); + orchestrators[DatasetId.Timeseries].StopCalls.ShouldBe(1); + } + + private static MultiDatasetSyncOrchestrator CreateSut( + IEnumerable datasetOptions, + MultiDatasetRuntimeOptions runtimeOptions, + Func? orchestratorFactory = null) + { + return new MultiDatasetSyncOrchestrator( + Substitute.For(), + Substitute.For(), + Substitute.For(), + Substitute.For(), + Substitute.For(), + Substitute.For(), + NullLoggerFactory.Instance, + datasetOptions, + runtimeOptions, + orchestratorFactory: orchestratorFactory); + } + + private sealed class TrackingSyncOrchestrator(Exception? startException = null, Exception? stopException = null) + : ISyncOrchestrator + { + public int StartCalls { get; private set; } + public int StopCalls { get; private set; } + + public Task Start() + { + StartCalls++; + if (startException != null) throw startException; + return Task.CompletedTask; + } + + public Task Stop() + { + StopCalls++; + if (stopException != null) throw stopException; + return Task.CompletedTask; + } + } +} diff --git a/tests/ZB.MOM.WW.CBDDC.Network.Tests/ProtocolTests.cs b/tests/ZB.MOM.WW.CBDDC.Network.Tests/ProtocolTests.cs index 0724fdd..3b6432f 100755 --- a/tests/ZB.MOM.WW.CBDDC.Network.Tests/ProtocolTests.cs +++ b/tests/ZB.MOM.WW.CBDDC.Network.Tests/ProtocolTests.cs @@ -1,4 +1,6 @@ using Microsoft.Extensions.Logging.Abstractions; +using Google.Protobuf; +using ZB.MOM.WW.CBDDC.Core; using ZB.MOM.WW.CBDDC.Network.Proto; using ZB.MOM.WW.CBDDC.Network.Protocol; using ZB.MOM.WW.CBDDC.Network.Security; @@ -145,6 +147,44 @@ public class ProtocolTests decoded.NodeId.ShouldBe("fragmented"); } + /// + /// Verifies that dataset-aware protocol fields are serialized and parsed correctly. + /// + [Fact] + public void DatasetAwareMessages_ShouldRoundTripDatasetFields() + { + var request = new PullChangesRequest + { + SinceWall = 10, + SinceLogic = 1, + SinceNode = "node-a", + DatasetId = "logs" + }; + + byte[] payload = request.ToByteArray(); + var decoded = PullChangesRequest.Parser.ParseFrom(payload); + + decoded.DatasetId.ShouldBe("logs"); + } + + /// + /// Verifies that legacy messages with no dataset id default to the primary dataset. + /// + [Fact] + public void DatasetAwareMessages_WhenMissingDataset_ShouldDefaultToPrimary() + { + var legacy = new HandshakeRequest + { + NodeId = "node-legacy", + AuthToken = "token" + }; + + byte[] payload = legacy.ToByteArray(); + var decoded = HandshakeRequest.Parser.ParseFrom(payload); + + DatasetId.Normalize(decoded.DatasetId).ShouldBe(DatasetId.Primary); + } + // Helper Stream for fragmentation test private class FragmentedMemoryStream : MemoryStream { @@ -169,4 +209,4 @@ public class ProtocolTests return await base.ReadAsync(buffer, offset, toRead, cancellationToken); } } -} \ No newline at end of file +} diff --git a/tests/ZB.MOM.WW.CBDDC.Network.Tests/SnapshotReconnectRegressionTests.cs b/tests/ZB.MOM.WW.CBDDC.Network.Tests/SnapshotReconnectRegressionTests.cs index 13723d2..a863f0a 100755 --- a/tests/ZB.MOM.WW.CBDDC.Network.Tests/SnapshotReconnectRegressionTests.cs +++ b/tests/ZB.MOM.WW.CBDDC.Network.Tests/SnapshotReconnectRegressionTests.cs @@ -16,6 +16,8 @@ public class SnapshotReconnectRegressionTests .Returns((SnapshotMetadata?)null); snapshotMetadataStore.GetSnapshotHashAsync(Arg.Any(), Arg.Any()) .Returns((string?)null); + snapshotMetadataStore.GetSnapshotHashAsync(Arg.Any(), Arg.Any(), Arg.Any()) + .Returns((string?)null); snapshotMetadataStore.GetAllSnapshotMetadataAsync(Arg.Any()) .Returns(Array.Empty()); return snapshotMetadataStore; @@ -30,6 +32,10 @@ public class SnapshotReconnectRegressionTests .Returns(Task.CompletedTask); snapshotService.MergeSnapshotAsync(Arg.Any(), Arg.Any()) .Returns(Task.CompletedTask); + snapshotService.ReplaceDatabaseAsync(Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(Task.CompletedTask); + snapshotService.MergeSnapshotAsync(Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(Task.CompletedTask); return snapshotService; } @@ -69,8 +75,12 @@ public class SnapshotReconnectRegressionTests var oplogStore = Substitute.For(); oplogStore.GetLastEntryHashAsync(Arg.Any(), Arg.Any()) .Returns(localHeadHash); + oplogStore.GetLastEntryHashAsync(Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(localHeadHash); oplogStore.ApplyBatchAsync(Arg.Any>(), Arg.Any()) .Returns(Task.CompletedTask); + oplogStore.ApplyBatchAsync(Arg.Any>(), Arg.Any(), Arg.Any()) + .Returns(Task.CompletedTask); return oplogStore; } @@ -84,6 +94,8 @@ public class SnapshotReconnectRegressionTests null); client.GetChainRangeAsync(Arg.Any(), Arg.Any(), Arg.Any()) .Returns(_ => Task.FromException>(new SnapshotRequiredException())); + client.GetChainRangeAsync(Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(_ => Task.FromException>(new SnapshotRequiredException())); return client; } @@ -109,19 +121,38 @@ public class SnapshotReconnectRegressionTests store.EnsurePeerRegisteredAsync(Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) .Returns(Task.CompletedTask); + store.EnsurePeerRegisteredAsync(Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any(), + Arg.Any()) + .Returns(Task.CompletedTask); store.UpdateConfirmationAsync(Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) .Returns(Task.CompletedTask); + store.UpdateConfirmationAsync(Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any(), + Arg.Any(), Arg.Any()) + .Returns(Task.CompletedTask); store.GetConfirmationsAsync(Arg.Any()).Returns(Array.Empty()); + store.GetConfirmationsAsync(Arg.Any(), Arg.Any()) + .Returns(Array.Empty()); store.GetConfirmationsForPeerAsync(Arg.Any(), Arg.Any()) .Returns(Array.Empty()); + store.GetConfirmationsForPeerAsync(Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(Array.Empty()); store.RemovePeerTrackingAsync(Arg.Any(), Arg.Any()).Returns(Task.CompletedTask); + store.RemovePeerTrackingAsync(Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(Task.CompletedTask); store.GetActiveTrackedPeersAsync(Arg.Any()).Returns(Array.Empty()); + store.GetActiveTrackedPeersAsync(Arg.Any(), Arg.Any()).Returns(Array.Empty()); store.ExportAsync(Arg.Any()).Returns(Array.Empty()); + store.ExportAsync(Arg.Any(), Arg.Any()) + .Returns(Array.Empty()); store.ImportAsync(Arg.Any>(), Arg.Any()) .Returns(Task.CompletedTask); + store.ImportAsync(Arg.Any>(), Arg.Any(), Arg.Any()) + .Returns(Task.CompletedTask); store.MergeAsync(Arg.Any>(), Arg.Any()) .Returns(Task.CompletedTask); + store.MergeAsync(Arg.Any>(), Arg.Any(), Arg.Any()) + .Returns(Task.CompletedTask); return store; } @@ -136,6 +167,8 @@ public class SnapshotReconnectRegressionTests var snapshotMetadataStore = CreateSnapshotMetadataStore(); snapshotMetadataStore.GetSnapshotHashAsync(Arg.Any(), Arg.Any()) .Returns("snapshot-boundary-hash"); + snapshotMetadataStore.GetSnapshotHashAsync(Arg.Any(), Arg.Any(), Arg.Any()) + .Returns("snapshot-boundary-hash"); var snapshotService = CreateSnapshotService(); var orch = new TestableSyncOrchestrator( @@ -165,7 +198,7 @@ public class SnapshotReconnectRegressionTests // Assert result.ShouldBe("Success"); await client.DidNotReceive() - .GetChainRangeAsync(Arg.Any(), Arg.Any(), Arg.Any()); + .GetChainRangeAsync(Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()); } /// @@ -179,6 +212,8 @@ public class SnapshotReconnectRegressionTests var snapshotMetadataStore = CreateSnapshotMetadataStore(); snapshotMetadataStore.GetSnapshotHashAsync(Arg.Any(), Arg.Any()) .Returns("snapshot-boundary-hash"); + snapshotMetadataStore.GetSnapshotHashAsync(Arg.Any(), Arg.Any(), Arg.Any()) + .Returns("snapshot-boundary-hash"); var snapshotService = CreateSnapshotService(); var orch = new TestableSyncOrchestrator( @@ -208,7 +243,11 @@ public class SnapshotReconnectRegressionTests await Should.ThrowAsync(async () => await orch.TestProcessInboundBatchAsync(client, "remote-node", entries, CancellationToken.None)); - await client.Received(1).GetChainRangeAsync(Arg.Any(), Arg.Any(), Arg.Any()); + await client.Received(1).GetChainRangeAsync( + Arg.Any(), + Arg.Any(), + Arg.Any(), + Arg.Any()); } // Subclass to expose private method @@ -283,4 +322,4 @@ public class SnapshotReconnectRegressionTests } } } -} \ No newline at end of file +} diff --git a/tests/ZB.MOM.WW.CBDDC.Network.Tests/SyncOrchestratorConfirmationTests.cs b/tests/ZB.MOM.WW.CBDDC.Network.Tests/SyncOrchestratorConfirmationTests.cs index 114ed76..04b56a2 100644 --- a/tests/ZB.MOM.WW.CBDDC.Network.Tests/SyncOrchestratorConfirmationTests.cs +++ b/tests/ZB.MOM.WW.CBDDC.Network.Tests/SyncOrchestratorConfirmationTests.cs @@ -39,18 +39,21 @@ public class SyncOrchestratorConfirmationTests "peer-a", "10.0.0.1:9000", PeerType.LanDiscovered, + DatasetId.Primary, Arg.Any()); await confirmationStore.Received(1).EnsurePeerRegisteredAsync( "peer-b", "10.0.0.2:9010", PeerType.StaticRemote, + DatasetId.Primary, Arg.Any()); await confirmationStore.DidNotReceive().EnsurePeerRegisteredAsync( "local", Arg.Any(), Arg.Any(), + Arg.Any(), Arg.Any()); } @@ -89,6 +92,7 @@ public class SyncOrchestratorConfirmationTests "peer-new", "10.0.0.25:9010", PeerType.LanDiscovered, + DatasetId.Primary, Arg.Any()); } @@ -114,9 +118,9 @@ public class SyncOrchestratorConfirmationTests remote.SetTimestamp("node-behind", new HlcTimestamp(299, 9, "node-behind")); remote.SetTimestamp("node-remote-only", new HlcTimestamp(900, 0, "node-remote-only")); - oplogStore.GetLastEntryHashAsync("node-equal", Arg.Any()) + oplogStore.GetLastEntryHashAsync("node-equal", DatasetId.Primary, Arg.Any()) .Returns("hash-equal"); - oplogStore.GetLastEntryHashAsync("node-ahead", Arg.Any()) + oplogStore.GetLastEntryHashAsync("node-ahead", DatasetId.Primary, Arg.Any()) .Returns((string?)null); await orchestrator.AdvanceConfirmationsFromVectorClockAsync("peer-1", local, remote, CancellationToken.None); @@ -126,6 +130,7 @@ public class SyncOrchestratorConfirmationTests "node-equal", new HlcTimestamp(100, 1, "node-equal"), "hash-equal", + DatasetId.Primary, Arg.Any()); await confirmationStore.Received(1).UpdateConfirmationAsync( @@ -133,6 +138,7 @@ public class SyncOrchestratorConfirmationTests "node-ahead", new HlcTimestamp(200, 0, "node-ahead"), string.Empty, + DatasetId.Primary, Arg.Any()); await confirmationStore.DidNotReceive().UpdateConfirmationAsync( @@ -140,6 +146,7 @@ public class SyncOrchestratorConfirmationTests "node-behind", Arg.Any(), Arg.Any(), + Arg.Any(), Arg.Any()); await confirmationStore.DidNotReceive().UpdateConfirmationAsync( @@ -147,6 +154,7 @@ public class SyncOrchestratorConfirmationTests "node-local-only", Arg.Any(), Arg.Any(), + Arg.Any(), Arg.Any()); await confirmationStore.DidNotReceive().UpdateConfirmationAsync( @@ -154,6 +162,7 @@ public class SyncOrchestratorConfirmationTests "node-remote-only", Arg.Any(), Arg.Any(), + Arg.Any(), Arg.Any()); } @@ -182,6 +191,7 @@ public class SyncOrchestratorConfirmationTests "source-1", new HlcTimestamp(120, 1, "source-1"), "hash-120", + DatasetId.Primary, Arg.Any()); } @@ -206,6 +216,7 @@ public class SyncOrchestratorConfirmationTests Arg.Any(), Arg.Any(), Arg.Any(), + Arg.Any(), Arg.Any()); } @@ -245,4 +256,4 @@ public class SyncOrchestratorConfirmationTests string.Empty, hash); } -} \ No newline at end of file +} diff --git a/tests/ZB.MOM.WW.CBDDC.Network.Tests/SyncOrchestratorMaintenancePruningTests.cs b/tests/ZB.MOM.WW.CBDDC.Network.Tests/SyncOrchestratorMaintenancePruningTests.cs index cc6e36e..c875581 100644 --- a/tests/ZB.MOM.WW.CBDDC.Network.Tests/SyncOrchestratorMaintenancePruningTests.cs +++ b/tests/ZB.MOM.WW.CBDDC.Network.Tests/SyncOrchestratorMaintenancePruningTests.cs @@ -138,7 +138,10 @@ public class SyncOrchestratorMaintenancePruningTests await orchestrator.RunMaintenanceIfDueAsync(config, DateTime.UtcNow, CancellationToken.None); - await oplogStore.DidNotReceive().PruneOplogAsync(Arg.Any(), Arg.Any()); + await oplogStore.DidNotReceive().PruneOplogAsync( + Arg.Any(), + Arg.Any(), + Arg.Any()); } /// @@ -187,6 +190,7 @@ public class SyncOrchestratorMaintenancePruningTests timestamp.PhysicalTime == 100 && timestamp.LogicalCounter == 0 && string.Equals(timestamp.NodeId, "node-local", StringComparison.Ordinal)), + DatasetId.Primary, Arg.Any()); } @@ -228,7 +232,10 @@ public class SyncOrchestratorMaintenancePruningTests var now = DateTime.UtcNow; await orchestrator.RunMaintenanceIfDueAsync(config, now, CancellationToken.None); - await oplogStore.DidNotReceive().PruneOplogAsync(Arg.Any(), Arg.Any()); + await oplogStore.DidNotReceive().PruneOplogAsync( + Arg.Any(), + Arg.Any(), + Arg.Any()); await orchestrator.RunMaintenanceIfDueAsync(config, now.AddMinutes(2), CancellationToken.None); @@ -237,6 +244,7 @@ public class SyncOrchestratorMaintenancePruningTests timestamp.PhysicalTime == 100 && timestamp.LogicalCounter == 0 && string.Equals(timestamp.NodeId, "node-local", StringComparison.Ordinal)), + DatasetId.Primary, Arg.Any()); } @@ -286,4 +294,4 @@ public class SyncOrchestratorMaintenancePruningTests IsActive = isActive }; } -} \ No newline at end of file +} diff --git a/tests/ZB.MOM.WW.CBDDC.Sample.Console.Tests/MultiDatasetConfigParsingTests.cs b/tests/ZB.MOM.WW.CBDDC.Sample.Console.Tests/MultiDatasetConfigParsingTests.cs new file mode 100644 index 0000000..6cdc6d5 --- /dev/null +++ b/tests/ZB.MOM.WW.CBDDC.Sample.Console.Tests/MultiDatasetConfigParsingTests.cs @@ -0,0 +1,38 @@ +using System.Text; +using Microsoft.Extensions.Configuration; +using ZB.MOM.WW.CBDDC.Network; + +namespace ZB.MOM.WW.CBDDC.Sample.Console.Tests; + +public class MultiDatasetConfigParsingTests +{ + [Fact] + public void MultiDatasetSection_ShouldBindRuntimeOptions() + { + const string json = """ + { + "CBDDC": { + "MultiDataset": { + "EnableMultiDatasetSync": true, + "EnableDatasetPrimary": true, + "EnableDatasetLogs": true, + "EnableDatasetTimeseries": false + } + } + } + """; + + using var stream = new MemoryStream(Encoding.UTF8.GetBytes(json)); + var config = new ConfigurationBuilder() + .AddJsonStream(stream) + .Build(); + + var options = config.GetSection("CBDDC:MultiDataset").Get(); + + options.ShouldNotBeNull(); + options.EnableMultiDatasetSync.ShouldBeTrue(); + options.EnableDatasetPrimary.ShouldBeTrue(); + options.EnableDatasetLogs.ShouldBeTrue(); + options.EnableDatasetTimeseries.ShouldBeFalse(); + } +} diff --git a/tests/ZB.MOM.WW.CBDDC.Sample.Console.Tests/SurrealStoreContractTests.cs b/tests/ZB.MOM.WW.CBDDC.Sample.Console.Tests/SurrealStoreContractTests.cs index dfbf6fd..bc1f8ad 100644 --- a/tests/ZB.MOM.WW.CBDDC.Sample.Console.Tests/SurrealStoreContractTests.cs +++ b/tests/ZB.MOM.WW.CBDDC.Sample.Console.Tests/SurrealStoreContractTests.cs @@ -54,6 +54,68 @@ public class SurrealOplogStoreContractTests (await store.ExportAsync()).ShouldBeEmpty(); } + /// + /// Verifies oplog reads and writes are isolated by dataset identifier. + /// + [Fact] + public async Task OplogStore_DatasetIsolation_Works() + { + await using var harness = new SurrealTestHarness(); + var store = harness.CreateOplogStore(); + + var primaryEntry = CreateOplogEntry("Users", "p1", "node-a", 100, 0, ""); + var logsEntry = CreateOplogEntry("Users", "l1", "node-a", 100, 1, ""); + + await store.AppendOplogEntryAsync(primaryEntry, DatasetId.Primary); + await store.AppendOplogEntryAsync(logsEntry, DatasetId.Logs); + + var primary = (await store.ExportAsync(DatasetId.Primary)).ToList(); + var logs = (await store.ExportAsync(DatasetId.Logs)).ToList(); + + primary.Count.ShouldBe(1); + primary[0].DatasetId.ShouldBe(DatasetId.Primary); + + logs.Count.ShouldBe(1); + logs[0].DatasetId.ShouldBe(DatasetId.Logs); + } + + /// + /// Verifies legacy oplog rows without dataset id are treated as primary dataset. + /// + [Fact] + public async Task OplogStore_LegacyRowsWithoutDatasetId_MapToPrimaryOnly() + { + await using var harness = new SurrealTestHarness(); + var store = harness.CreateOplogStore(); + + await harness.SurrealEmbeddedClient.InitializeAsync(); + await harness.SurrealEmbeddedClient.RawQueryAsync( + """ + UPSERT type::thing($table, $id) CONTENT { + collection: "Users", + key: "legacy", + operation: 0, + payloadJson: "{}", + timestampPhysicalTime: 10, + timestampLogicalCounter: 0, + timestampNodeId: "node-legacy", + hash: "legacy-hash", + previousHash: "" + }; + """, + new Dictionary + { + ["table"] = CBDDCSurrealSchemaNames.OplogEntriesTable, + ["id"] = "legacy-hash" + }); + + var primary = (await store.GetOplogAfterAsync(new HlcTimestamp(0, 0, ""), DatasetId.Primary)).ToList(); + var logs = (await store.GetOplogAfterAsync(new HlcTimestamp(0, 0, ""), DatasetId.Logs)).ToList(); + + primary.Any(entry => entry.Hash == "legacy-hash").ShouldBeTrue(); + logs.Any(entry => entry.Hash == "legacy-hash").ShouldBeFalse(); + } + private static OplogEntry CreateOplogEntry( string collection, string key, @@ -110,6 +172,34 @@ public class SurrealDocumentMetadataStoreContractTests var exported = (await store.ExportAsync()).ToList(); exported.Count.ShouldBe(3); } + + /// + /// Verifies document metadata records do not leak across datasets. + /// + [Fact] + public async Task DocumentMetadataStore_DatasetIsolation_Works() + { + await using var harness = new SurrealTestHarness(); + var store = harness.CreateDocumentMetadataStore(); + + await store.UpsertMetadataAsync( + new DocumentMetadata("Users", "doc-shared", new HlcTimestamp(100, 0, "node-a"), false, DatasetId.Primary), + DatasetId.Primary); + await store.UpsertMetadataAsync( + new DocumentMetadata("Users", "doc-shared", new HlcTimestamp(101, 0, "node-a"), false, DatasetId.Logs), + DatasetId.Logs); + + var primary = await store.GetMetadataAsync("Users", "doc-shared", DatasetId.Primary); + var logs = await store.GetMetadataAsync("Users", "doc-shared", DatasetId.Logs); + + primary.ShouldNotBeNull(); + primary.DatasetId.ShouldBe(DatasetId.Primary); + primary.UpdatedAt.ShouldBe(new HlcTimestamp(100, 0, "node-a")); + + logs.ShouldNotBeNull(); + logs.DatasetId.ShouldBe(DatasetId.Logs); + logs.UpdatedAt.ShouldBe(new HlcTimestamp(101, 0, "node-a")); + } } public class SurrealPeerConfigurationStoreContractTests @@ -206,6 +296,42 @@ public class SurrealPeerOplogConfirmationStoreContractTests afterDeactivate.All(x => x.IsActive == false).ShouldBeTrue(); } + /// + /// Verifies peer confirmations are isolated by dataset. + /// + [Fact] + public async Task PeerOplogConfirmationStore_DatasetIsolation_Works() + { + await using var harness = new SurrealTestHarness(); + var store = harness.CreatePeerOplogConfirmationStore(); + + await store.EnsurePeerRegisteredAsync("peer-a", "10.0.0.10:5050", PeerType.StaticRemote, DatasetId.Primary); + await store.EnsurePeerRegisteredAsync("peer-a", "10.0.0.10:5050", PeerType.StaticRemote, DatasetId.Logs); + await store.UpdateConfirmationAsync( + "peer-a", + "source-1", + new HlcTimestamp(100, 1, "source-1"), + "hash-primary", + DatasetId.Primary); + await store.UpdateConfirmationAsync( + "peer-a", + "source-1", + new HlcTimestamp(200, 1, "source-1"), + "hash-logs", + DatasetId.Logs); + + var primary = (await store.GetConfirmationsForPeerAsync("peer-a", DatasetId.Primary)).ToList(); + var logs = (await store.GetConfirmationsForPeerAsync("peer-a", DatasetId.Logs)).ToList(); + + primary.Count.ShouldBe(1); + primary[0].ConfirmedHash.ShouldBe("hash-primary"); + primary[0].DatasetId.ShouldBe(DatasetId.Primary); + + logs.Count.ShouldBe(1); + logs[0].ConfirmedHash.ShouldBe("hash-logs"); + logs[0].DatasetId.ShouldBe(DatasetId.Logs); + } + /// /// Verifies merge semantics prefer newer confirmations and preserve active-state transitions. /// @@ -343,6 +469,45 @@ public class SurrealSnapshotMetadataStoreContractTests all[0].NodeId.ShouldBe("node-a"); all[1].NodeId.ShouldBe("node-b"); } + + /// + /// Verifies snapshot metadata rows are isolated by dataset. + /// + [Fact] + public async Task SnapshotMetadataStore_DatasetIsolation_Works() + { + await using var harness = new SurrealTestHarness(); + var store = harness.CreateSnapshotMetadataStore(); + + await store.InsertSnapshotMetadataAsync(new SnapshotMetadata + { + DatasetId = DatasetId.Primary, + NodeId = "node-a", + TimestampPhysicalTime = 100, + TimestampLogicalCounter = 0, + Hash = "hash-primary" + }, DatasetId.Primary); + + await store.InsertSnapshotMetadataAsync(new SnapshotMetadata + { + DatasetId = DatasetId.Logs, + NodeId = "node-a", + TimestampPhysicalTime = 200, + TimestampLogicalCounter = 0, + Hash = "hash-logs" + }, DatasetId.Logs); + + var primary = await store.GetSnapshotMetadataAsync("node-a", DatasetId.Primary); + var logs = await store.GetSnapshotMetadataAsync("node-a", DatasetId.Logs); + + primary.ShouldNotBeNull(); + primary.Hash.ShouldBe("hash-primary"); + primary.DatasetId.ShouldBe(DatasetId.Primary); + + logs.ShouldNotBeNull(); + logs.Hash.ShouldBe("hash-logs"); + logs.DatasetId.ShouldBe(DatasetId.Logs); + } } internal sealed class SurrealTestHarness : IAsyncDisposable @@ -431,6 +596,11 @@ internal sealed class SurrealTestHarness : IAsyncDisposable NullLogger.Instance); } + /// + /// Gets the embedded Surreal client used by this harness. + /// + public ICBDDCSurrealEmbeddedClient SurrealEmbeddedClient => _client; + /// public async ValueTask DisposeAsync() { diff --git a/tests/ZB.MOM.WW.CDBBC.E2E.Benchmark.Tests/BenchmarkPeerNode.cs b/tests/ZB.MOM.WW.CDBBC.E2E.Benchmark.Tests/BenchmarkPeerNode.cs index 9357578..b44d2dc 100644 --- a/tests/ZB.MOM.WW.CDBBC.E2E.Benchmark.Tests/BenchmarkPeerNode.cs +++ b/tests/ZB.MOM.WW.CDBBC.E2E.Benchmark.Tests/BenchmarkPeerNode.cs @@ -113,12 +113,7 @@ internal sealed class BenchmarkPeerNode : IAsyncDisposable public async Task UpsertUserAsync(User user) { - User? existing = Context.Users.Find(u => u.Id == user.Id).FirstOrDefault(); - if (existing == null) - await Context.Users.InsertAsync(user); - else - await Context.Users.UpdateAsync(user); - + await Context.Users.UpdateAsync(user); await Context.SaveChangesAsync(); } @@ -127,6 +122,11 @@ internal sealed class BenchmarkPeerNode : IAsyncDisposable return Context.Users.Find(u => u.Id == userId).Any(); } + public int CountUsersWithPrefix(string prefix) + { + return Context.Users.FindAll().Count(u => u.Id.StartsWith(prefix, StringComparison.Ordinal)); + } + public async ValueTask DisposeAsync() { try diff --git a/tests/ZB.MOM.WW.CDBBC.E2E.Benchmark.Tests/OfflineResyncThroughputBenchmarks.cs b/tests/ZB.MOM.WW.CDBBC.E2E.Benchmark.Tests/OfflineResyncThroughputBenchmarks.cs new file mode 100644 index 0000000..8cc07f5 --- /dev/null +++ b/tests/ZB.MOM.WW.CDBBC.E2E.Benchmark.Tests/OfflineResyncThroughputBenchmarks.cs @@ -0,0 +1,142 @@ +using BenchmarkDotNet.Attributes; +using System.Net; +using System.Net.Sockets; +using ZB.MOM.WW.CBDDC.Core.Network; +using ZB.MOM.WW.CBDDC.Sample.Console; + +namespace ZB.MOM.WW.CDBBC.E2E.Benchmark.Tests; + +[MemoryDiagnoser] +[SimpleJob(launchCount: 1, warmupCount: 0, iterationCount: 1)] +public class OfflineResyncThroughputBenchmarks +{ + private const int BacklogOperationCount = 10_000; + private BenchmarkPeerNode _onlineNode = null!; + private BenchmarkPeerNode _offlineNode = null!; + private int _runSequence; + private string _currentPrefix = string.Empty; + + [GlobalSetup] + public Task GlobalSetupAsync() + { + return Task.CompletedTask; + } + + [GlobalCleanup] + public Task GlobalCleanupAsync() + { + // Avoid explicit node disposal in BenchmarkDotNet child processes due Surreal embedded callback race. + // Process teardown releases resources after benchmark completion. + return Task.CompletedTask; + } + + [IterationSetup(Target = nameof(OfflineBacklogWriteThroughput100k))] + public void SetupOfflineWriteThroughput() + { + _currentPrefix = $"offline-write-{Interlocked.Increment(ref _runSequence):D6}"; + InitializeIterationNodesAsync().GetAwaiter().GetResult(); + } + + [Benchmark(Description = "Offline backlog write throughput (10K ops)", OperationsPerInvoke = BacklogOperationCount)] + public async Task OfflineBacklogWriteThroughput100k() + { + await WriteBatchAsync(_currentPrefix, BacklogOperationCount); + } + + [IterationSetup(Target = nameof(OfflineNodeResyncDurationAfter100kBacklog))] + public void SetupOfflineResyncBenchmark() + { + _currentPrefix = $"offline-resync-{Interlocked.Increment(ref _runSequence):D6}"; + InitializeIterationNodesAsync().GetAwaiter().GetResult(); + WriteBatchAsync(_currentPrefix, BacklogOperationCount).GetAwaiter().GetResult(); + } + + [Benchmark(Description = "Offline node re-sync duration after 10K backlog")] + public async Task OfflineNodeResyncDurationAfter100kBacklog() + { + await _offlineNode.StartAsync(); + await WaitForReplicationAsync(_currentPrefix, BacklogOperationCount, TimeSpan.FromMinutes(3)); + } + + private async Task WriteBatchAsync(string prefix, int count) + { + for (var i = 0; i < count; i++) + { + string userId = $"{prefix}-{i:D6}"; + await _onlineNode.UpsertUserAsync(CreateUser(userId)); + } + } + + private async Task WaitForReplicationAsync(string prefix, int expectedCount, TimeSpan timeout) + { + DateTime deadline = DateTime.UtcNow.Add(timeout); + while (DateTime.UtcNow < deadline) + { + if (_offlineNode.CountUsersWithPrefix(prefix) >= expectedCount) + return; + + await Task.Delay(250); + } + + int replicatedCount = _offlineNode.CountUsersWithPrefix(prefix); + throw new TimeoutException( + $"Timed out waiting for re-sync. Expected {expectedCount}, replicated {replicatedCount}."); + } + + private static User CreateUser(string userId) + { + return new User + { + Id = userId, + Name = $"user-{userId}", + Age = 30, + Address = new Address { City = "OfflineBenchmarkCity" } + }; + } + + private static int GetAvailableTcpPort() + { + using var listener = new TcpListener(IPAddress.Loopback, 0); + listener.Start(); + return ((IPEndPoint)listener.LocalEndpoint).Port; + } + + private async Task InitializeIterationNodesAsync() + { + int onlinePort = GetAvailableTcpPort(); + int offlinePort = GetAvailableTcpPort(); + while (offlinePort == onlinePort) + offlinePort = GetAvailableTcpPort(); + + string clusterToken = Guid.NewGuid().ToString("N"); + + _onlineNode = BenchmarkPeerNode.Create( + "offline-benchmark-online", + onlinePort, + clusterToken, + [ + new KnownPeerConfiguration + { + NodeId = "offline-benchmark-offline", + Host = "127.0.0.1", + Port = offlinePort + } + ]); + + _offlineNode = BenchmarkPeerNode.Create( + "offline-benchmark-offline", + offlinePort, + clusterToken, + [ + new KnownPeerConfiguration + { + NodeId = "offline-benchmark-online", + Host = "127.0.0.1", + Port = onlinePort + } + ]); + + await _onlineNode.StartAsync(); + await Task.Delay(250); + } +} diff --git a/tests/ZB.MOM.WW.CDBBC.E2E.Benchmark.Tests/SerilogLogEntry.cs b/tests/ZB.MOM.WW.CDBBC.E2E.Benchmark.Tests/SerilogLogEntry.cs new file mode 100644 index 0000000..0a5df9d --- /dev/null +++ b/tests/ZB.MOM.WW.CDBBC.E2E.Benchmark.Tests/SerilogLogEntry.cs @@ -0,0 +1,50 @@ +using System; +using System.Collections.Generic; + +namespace ZB.MOM.WW.CDBBC.E2E.Benchmark.Tests; + +/// +/// Represents a typical Serilog log entry. +/// +public sealed class SerilogLogEntry +{ + /// + /// Timestamp when the log event was written. + /// + public DateTimeOffset Timestamp { get; set; } + + /// + /// Log level (for example: Information, Warning, Error). + /// + public string Level { get; set; } = "Information"; + + /// + /// Name of the logger/category (typically the class name). + /// + public string? LoggerName { get; set; } + + /// + /// Correlation context identifier used to tie log entries to a request. + /// + public string? ContextId { get; set; } + + /// + /// Original message template used by Serilog. + /// + public string MessageTemplate { get; set; } = string.Empty; + + /// + /// Fully rendered message text. + /// + public string? RenderedMessage { get; set; } + + /// + /// Exception details if one was logged. + /// + public string? Exception { get; set; } + + /// + /// Structured context values captured from Serilog context. + /// + public Dictionary ContextProperties { get; set; } = new(StringComparer.Ordinal); +} diff --git a/tests/ZB.MOM.WW.CDBBC.E2E.Benchmark.Tests/SurrealLogStorageBenchmarks.cs b/tests/ZB.MOM.WW.CDBBC.E2E.Benchmark.Tests/SurrealLogStorageBenchmarks.cs new file mode 100644 index 0000000..4e9373a --- /dev/null +++ b/tests/ZB.MOM.WW.CDBBC.E2E.Benchmark.Tests/SurrealLogStorageBenchmarks.cs @@ -0,0 +1,440 @@ +using System.Diagnostics; +using System.Text; +using BenchmarkDotNet.Attributes; +using SurrealDb.Net.Models; +using SurrealDb.Net.Models.Response; +using ZB.MOM.WW.CBDDC.Persistence.Surreal; + +namespace ZB.MOM.WW.CDBBC.E2E.Benchmark.Tests; + +[MemoryDiagnoser] +[SimpleJob(launchCount: 1, warmupCount: 1, iterationCount: 3)] +public class SurrealLogStorageBenchmarks +{ + private const int LogRecordCount = 100_000; + private const int InsertBatchSize = 500; + private const string LogTable = "benchmark_log_entry"; + private const string LogKvTable = "benchmark_log_kv"; + + private static readonly string[] LoggerNames = + [ + "Api.RequestHandler", + "Api.AuthController", + "Api.OrderController", + "Api.InventoryController", + "Api.CustomerController", + "Workers.OutboxPublisher", + "Workers.NotificationDispatcher", + "Infrastructure.SqlRepository", + "Infrastructure.CacheService", + "Infrastructure.HttpClient", + "Domain.OrderService", + "Domain.PricingService" + ]; + + private static readonly string[] TenantIds = + [ + "tenant-01", + "tenant-02", + "tenant-03", + "tenant-04", + "tenant-05", + "tenant-06", + "tenant-07", + "tenant-08" + ]; + + private CBDDCSurrealEmbeddedClient _surrealClient = null!; + private string _databasePath = string.Empty; + private string _workDir = string.Empty; + private DateTime _seedBaseUtc; + private DateTime _queryRangeStartUtc; + private DateTime _queryRangeEndUtc; + private string _contextIdQueryValue = string.Empty; + private string _loggerQueryValue = string.Empty; + private string _contextStringKeyQueryValue = string.Empty; + private string _contextStringValueQueryValue = string.Empty; + private string _contextNumericKeyQueryValue = string.Empty; + private int _contextNumericValueQueryValue; + + [GlobalSetup] + public async Task GlobalSetupAsync() + { + _workDir = Path.Combine(Path.GetTempPath(), $"cbddc-serilog-benchmark-{Guid.NewGuid():N}"); + Directory.CreateDirectory(_workDir); + _databasePath = Path.Combine(_workDir, "serilog.rocksdb"); + + _surrealClient = new CBDDCSurrealEmbeddedClient( + new CBDDCSurrealEmbeddedOptions + { + Endpoint = "rocksdb://local", + DatabasePath = _databasePath, + Namespace = "cbddc_benchmark", + Database = $"serilog_{Guid.NewGuid():N}", + Cdc = new CBDDCSurrealCdcOptions { Enabled = false } + }); + + await _surrealClient.InitializeAsync(); + await DefineSchemaAndIndexesAsync(); + + _seedBaseUtc = DateTime.UtcNow.AddDays(-1); + _contextIdQueryValue = BuildContextId(LogRecordCount / 2); + _loggerQueryValue = LoggerNames[3]; + _contextStringKeyQueryValue = "tenantId"; + _contextStringValueQueryValue = TenantIds[5]; + _contextNumericKeyQueryValue = "statusCode"; + _contextNumericValueQueryValue = 500; + _queryRangeStartUtc = _seedBaseUtc.AddMinutes(6); + _queryRangeEndUtc = _queryRangeStartUtc.AddSeconds(30); + + var seedTimer = Stopwatch.StartNew(); + await InsertLogRecordsAsync(); + seedTimer.Stop(); + + long sizeBytes = CalculatePathSizeBytes(_databasePath); + Console.WriteLine( + $"Seeded {LogRecordCount:N0} records in {seedTimer.Elapsed.TotalSeconds:F2}s. " + + $"RocksDB size: {sizeBytes / (1024d * 1024d):F2} MiB ({sizeBytes:N0} bytes). Path: {_databasePath}"); + } + + [GlobalCleanup] + public Task GlobalCleanupAsync() + { + // Avoid explicit Surreal embedded disposal in benchmark child processes due known native callback race. + return Task.CompletedTask; + } + + [Benchmark(Description = "Query by contextId (latest 200 rows)")] + public async Task QueryByContextIdAsync() + { + await ExecuteQueryAsync( + $""" + SELECT * FROM {LogTable} + WHERE contextId = $contextId + ORDER BY timestamp DESC + LIMIT 200; + """, + new Dictionary { ["contextId"] = _contextIdQueryValue }); + } + + [Benchmark(Description = "Query by loggerName + timestamp range (latest 200 rows)")] + public async Task QueryByLoggerAndTimestampAsync() + { + await ExecuteQueryAsync( + $""" + SELECT * FROM {LogTable} + WHERE loggerName = $loggerName + AND timestamp >= $fromTs + AND timestamp <= $toTs + ORDER BY timestamp DESC + LIMIT 200; + """, + new Dictionary + { + ["loggerName"] = _loggerQueryValue, + ["fromTs"] = _queryRangeStartUtc, + ["toTs"] = _queryRangeEndUtc + }); + } + + [Benchmark(Description = "Query by loggerName + timestamp + arbitrary context string key/value")] + public async Task QueryByLoggerTimestampAndContextKeyAsync() + { + await ExecuteQueryAsync( + $""" + LET $logIds = ( + SELECT VALUE logId FROM {LogKvTable} + WHERE loggerName = $loggerName + AND key = $contextKey + AND valueStr = $contextValueStr + AND timestamp >= $fromTs + AND timestamp <= $toTs + ORDER BY timestamp DESC + LIMIT 200 + ); + SELECT * FROM {LogTable} + WHERE id INSIDE $logIds + ORDER BY timestamp DESC + LIMIT 200; + """, + new Dictionary + { + ["loggerName"] = _loggerQueryValue, + ["fromTs"] = _queryRangeStartUtc, + ["toTs"] = _queryRangeEndUtc, + ["contextKey"] = _contextStringKeyQueryValue, + ["contextValueStr"] = _contextStringValueQueryValue + }); + } + + [Benchmark(Description = "Query by loggerName + timestamp + arbitrary context number key/value")] + public async Task QueryByLoggerTimestampAndNumericContextKeyAsync() + { + await ExecuteQueryAsync( + $""" + LET $logIds = ( + SELECT VALUE logId FROM {LogKvTable} + WHERE loggerName = $loggerName + AND key = $contextKey + AND valueNum = $contextValueNum + AND timestamp >= $fromTs + AND timestamp <= $toTs + ORDER BY timestamp DESC + LIMIT 200 + ); + SELECT * FROM {LogTable} + WHERE id INSIDE $logIds + ORDER BY timestamp DESC + LIMIT 200; + """, + new Dictionary + { + ["loggerName"] = _loggerQueryValue, + ["fromTs"] = _queryRangeStartUtc, + ["toTs"] = _queryRangeEndUtc, + ["contextKey"] = _contextNumericKeyQueryValue, + ["contextValueNum"] = _contextNumericValueQueryValue + }); + } + + [Benchmark(Description = "RocksDB size (bytes)")] + public long GetDatabaseFileSizeBytes() + { + return CalculatePathSizeBytes(_databasePath); + } + + private async Task DefineSchemaAndIndexesAsync() + { + string schemaSql = + $""" + DEFINE TABLE OVERWRITE {LogTable} SCHEMAFULL; + DEFINE FIELD OVERWRITE timestamp ON TABLE {LogTable} TYPE datetime; + DEFINE FIELD OVERWRITE level ON TABLE {LogTable} TYPE string; + DEFINE FIELD OVERWRITE loggerName ON TABLE {LogTable} TYPE option; + DEFINE FIELD OVERWRITE contextId ON TABLE {LogTable} TYPE option; + DEFINE FIELD OVERWRITE messageTemplate ON TABLE {LogTable} TYPE string; + DEFINE FIELD OVERWRITE renderedMessage ON TABLE {LogTable} TYPE option; + DEFINE FIELD OVERWRITE exception ON TABLE {LogTable} TYPE option; + DEFINE FIELD OVERWRITE contextValues ON TABLE {LogTable} TYPE object FLEXIBLE; + + DEFINE INDEX OVERWRITE idx_log_contextid_ts + ON TABLE {LogTable} COLUMNS contextId, timestamp; + + DEFINE INDEX OVERWRITE idx_log_logger_ts + ON TABLE {LogTable} COLUMNS loggerName, timestamp; + + DEFINE TABLE OVERWRITE {LogKvTable} SCHEMAFULL; + DEFINE FIELD OVERWRITE logId ON TABLE {LogKvTable} TYPE record<{LogTable}>; + DEFINE FIELD OVERWRITE loggerName ON TABLE {LogKvTable} TYPE string; + DEFINE FIELD OVERWRITE timestamp ON TABLE {LogKvTable} TYPE datetime; + DEFINE FIELD OVERWRITE key ON TABLE {LogKvTable} TYPE string; + DEFINE FIELD OVERWRITE valueStr ON TABLE {LogKvTable} TYPE option; + DEFINE FIELD OVERWRITE valueNum ON TABLE {LogKvTable} TYPE option; + DEFINE FIELD OVERWRITE valueBool ON TABLE {LogKvTable} TYPE option; + + DEFINE INDEX OVERWRITE idx_logkv_logger_key_str_ts + ON TABLE {LogKvTable} COLUMNS loggerName, key, valueStr, timestamp; + + DEFINE INDEX OVERWRITE idx_logkv_logger_key_num_ts + ON TABLE {LogKvTable} COLUMNS loggerName, key, valueNum, timestamp; + + DEFINE INDEX OVERWRITE idx_logkv_logger_key_bool_ts + ON TABLE {LogKvTable} COLUMNS loggerName, key, valueBool, timestamp; + + DEFINE INDEX OVERWRITE idx_logkv_logid + ON TABLE {LogKvTable} COLUMNS logId; + """; + + var response = await _surrealClient.RawQueryAsync(schemaSql); + EnsureSuccessfulResponse(response, "Schema definition"); + } + + private async Task InsertLogRecordsAsync() + { + for (var batchStart = 0; batchStart < LogRecordCount; batchStart += InsertBatchSize) + { + int batchCount = Math.Min(InsertBatchSize, LogRecordCount - batchStart); + var sqlBuilder = new StringBuilder(); + sqlBuilder.AppendLine("BEGIN TRANSACTION;"); + + var parameters = new Dictionary(batchCount * 2); + for (var offset = 0; offset < batchCount; offset++) + { + int sequence = batchStart + offset; + string idParameterName = $"id{offset}"; + string recordParameterName = $"record{offset}"; + string logId = $"log-{sequence:D8}"; + RecordId logRecordId = RecordId.From(LogTable, logId); + IReadOnlyDictionary logRecord = CreateLogRecord(sequence); + + parameters[idParameterName] = logRecordId; + parameters[recordParameterName] = logRecord; + sqlBuilder.Append("UPSERT $") + .Append(idParameterName) + .Append(" CONTENT $") + .Append(recordParameterName) + .AppendLine(";"); + + int kvOrdinal = 0; + foreach (IReadOnlyDictionary kvRow in CreateKvRows(logId, logRecordId, logRecord)) + { + string kvIdParameterName = $"kvid{offset}_{kvOrdinal}"; + string kvRecordParameterName = $"kvrecord{offset}_{kvOrdinal}"; + parameters[kvIdParameterName] = RecordId.From(LogKvTable, $"{logId}-{kvOrdinal:D2}"); + parameters[kvRecordParameterName] = kvRow; + + sqlBuilder.Append("UPSERT $") + .Append(kvIdParameterName) + .Append(" CONTENT $") + .Append(kvRecordParameterName) + .AppendLine(";"); + + kvOrdinal++; + } + } + + sqlBuilder.AppendLine("COMMIT TRANSACTION;"); + + var response = await _surrealClient.RawQueryAsync(sqlBuilder.ToString(), parameters); + EnsureSuccessfulResponse(response, $"Insert batch starting at row {batchStart}"); + } + } + + private IReadOnlyDictionary CreateLogRecord(int sequence) + { + DateTime timestamp = _seedBaseUtc.AddMilliseconds(sequence * 10L); + string loggerName = LoggerNames[sequence % LoggerNames.Length]; + string tenantId = TenantIds[sequence % TenantIds.Length]; + bool isBackground = sequence % 7 == 0; + string? exception = sequence % 2_500 == 0 + ? "System.InvalidOperationException: simulated benchmark exception." + : null; + + var record = new Dictionary + { + ["timestamp"] = timestamp, + ["level"] = ResolveLogLevel(sequence), + ["loggerName"] = loggerName, + ["contextId"] = BuildContextId(sequence), + ["messageTemplate"] = "Processed request {RequestId} for {Route}", + ["renderedMessage"] = $"Processed request req-{sequence:D8} for /api/items/{sequence % 250}", + ["contextValues"] = new Dictionary + { + ["tenantId"] = tenantId, + ["requestId"] = $"req-{sequence:D8}", + ["route"] = $"/api/items/{sequence % 250}", + ["statusCode"] = sequence % 20 == 0 ? 500 : 200, + ["elapsedMs"] = 5 + (sequence % 200), + ["nodeId"] = $"node-{sequence % 8:D2}", + ["isBackground"] = isBackground + } + }; + + if (!string.IsNullOrEmpty(exception)) + record["exception"] = exception; + + return record; + } + + private static IEnumerable> CreateKvRows( + string logId, + RecordId logRecordId, + IReadOnlyDictionary logRecord) + { + if (!logRecord.TryGetValue("loggerName", out object? loggerNameValue) || loggerNameValue is not string loggerName) + yield break; + if (!logRecord.TryGetValue("timestamp", out object? timestampValue) || timestampValue is not DateTime timestamp) + yield break; + if (!logRecord.TryGetValue("contextValues", out object? contextValuesObject) || + contextValuesObject is not IReadOnlyDictionary contextValues) + yield break; + + foreach ((string key, object? value) in contextValues) + { + if (value == null) continue; + + var row = new Dictionary + { + ["logId"] = logRecordId, + ["loggerName"] = loggerName, + ["timestamp"] = timestamp, + ["key"] = key + }; + + switch (value) + { + case string stringValue: + row["valueStr"] = stringValue; + break; + case bool boolValue: + row["valueBool"] = boolValue; + break; + case sbyte or byte or short or ushort or int or uint or long or ulong or float or double or decimal: + row["valueNum"] = Convert.ToDouble(value); + break; + default: + row["valueStr"] = value.ToString(); + break; + } + + yield return row; + } + } + + private async Task ExecuteQueryAsync(string query, IReadOnlyDictionary parameters) + { + var response = await _surrealClient.RawQueryAsync(query, parameters); + EnsureSuccessfulResponse(response, "Query execution"); + } + + private static string BuildContextId(int sequence) + { + return $"ctx-{sequence / 10:D6}"; + } + + private static string ResolveLogLevel(int sequence) + { + if (sequence % 2_500 == 0) return "Error"; + if (sequence % 500 == 0) return "Warning"; + return "Information"; + } + + private static long CalculatePathSizeBytes(string path) + { + if (File.Exists(path)) return new FileInfo(path).Length; + if (!Directory.Exists(path)) return 0; + + long size = 0; + foreach (string file in Directory.EnumerateFiles(path, "*", SearchOption.AllDirectories)) + size += new FileInfo(file).Length; + + return size; + } + + private static void EnsureSuccessfulResponse(SurrealDbResponse response, string operation) + { + if (!response.HasErrors) return; + + string errorSummary = string.Join( + " | ", + response.Errors.Take(3).Select((error, index) => DescribeError(error, index))); + + throw new InvalidOperationException( + $"{operation} failed with SurrealDB errors. Details: {errorSummary}"); + } + + private static string DescribeError(object error, int index) + { + Type errorType = error.GetType(); + string[] fieldsToExtract = ["Status", "Details", "Description", "Information", "Code"]; + var extracted = new List(); + foreach (string field in fieldsToExtract) + { + object? value = errorType.GetProperty(field)?.GetValue(error); + if (value != null) extracted.Add($"{field}={value}"); + } + + if (extracted.Count == 0) return $"error[{index}] type={errorType.Name}"; + return $"error[{index}] type={errorType.Name} {string.Join(", ", extracted)}"; + } +}