Harden Surreal migration with retry/coverage fixes and XML docs cleanup

tests/ZB.MOM.WW.CBDDC.Sample.Console.Tests/SurrealCdcDurabilityTests.cs

@@ -13,6 +13,9 @@ namespace ZB.MOM.WW.CBDDC.Sample.Console.Tests;
 [Collection("SurrealCdcDurability")]
 public class SurrealCdcDurabilityTests
 {
+    /// <summary>
+    ///     Verifies checkpoints persist latest local changes per consumer across restarts.
+    /// </summary>
     [Fact]
     public async Task CheckpointPersistence_ShouldTrackLatestLocalChange_AndPersistPerConsumer()
     {
@@ -85,6 +88,9 @@ public class SurrealCdcDurabilityTests
         }
     }
 
+    /// <summary>
+    ///     Verifies recovery resumes from a persisted checkpoint and advances after catch-up.
+    /// </summary>
     [Fact]
     public async Task RestartRecovery_ShouldResumeCatchUpFromPersistedCheckpoint_InRocksDb()
     {
@@ -153,6 +159,9 @@ public class SurrealCdcDurabilityTests
         }
     }
 
+    /// <summary>
+    ///     Verifies duplicate remote apply windows are idempotent without loopback entries.
+    /// </summary>
     [Fact]
     public async Task RemoteApply_ShouldBeIdempotentAcrossDuplicateWindow_WithoutLoopbackEntries()
     {
@@ -207,6 +216,9 @@ public class SurrealCdcDurabilityTests
         }
     }
 
+    /// <summary>
+    ///     Verifies local deletes persist tombstone metadata and advance checkpoints.
+    /// </summary>
     [Fact]
     public async Task LocalDelete_ShouldPersistTombstoneMetadata_AndAdvanceCheckpoint()
     {
@@ -358,21 +370,46 @@ internal sealed class CdcTestHarness : IAsyncDisposable
             NullLogger<SurrealDocumentMetadataStore>.Instance);
     }
 
+    /// <summary>
+    ///     Gets the sample database context.
+    /// </summary>
     public SampleDbContext Context { get; }
 
+    /// <summary>
+    ///     Gets the checkpointed sample document store.
+    /// </summary>
     public CheckpointedSampleDocumentStore DocumentStore { get; }
 
+    /// <summary>
+    ///     Gets the oplog store used by the harness.
+    /// </summary>
     public SurrealOplogStore OplogStore { get; }
 
+    /// <summary>
+    ///     Gets the document metadata store.
+    /// </summary>
     public SurrealDocumentMetadataStore MetadataStore { get; }
 
+    /// <summary>
+    ///     Gets checkpoint persistence used for CDC progress tracking.
+    /// </summary>
     public ISurrealCdcCheckpointPersistence CheckpointPersistence { get; }
 
+    /// <summary>
+    ///     Polls CDC once through the document store.
+    /// </summary>
     public async Task PollAsync()
     {
         await DocumentStore.PollCdcOnceAsync();
     }
 
+    /// <summary>
+    ///     Creates a harness instance with retries for transient RocksDB lock contention.
+    /// </summary>
+    /// <param name="databasePath">Database directory path.</param>
+    /// <param name="nodeId">Node identifier.</param>
+    /// <param name="consumerId">CDC consumer identifier.</param>
+    /// <returns>Initialized test harness.</returns>
     public static async Task<CdcTestHarness> OpenWithRetriesAsync(
         string databasePath,
         string nodeId,
@@ -391,6 +428,11 @@ internal sealed class CdcTestHarness : IAsyncDisposable
         throw new InvalidOperationException("Unable to acquire RocksDB lock for test harness.");
     }
 
+    /// <summary>
+    ///     Gets oplog entries for a collection ordered by timestamp.
+    /// </summary>
+    /// <param name="collection">Collection name.</param>
+    /// <returns>Ordered oplog entries.</returns>
     public async Task<List<OplogEntry>> GetEntriesByCollectionAsync(string collection)
     {
         return (await OplogStore.ExportAsync())
@@ -400,6 +442,12 @@ internal sealed class CdcTestHarness : IAsyncDisposable
             .ToList();
     }
 
+    /// <summary>
+    ///     Gets oplog entries for a collection key ordered by timestamp.
+    /// </summary>
+    /// <param name="collection">Collection name.</param>
+    /// <param name="key">Document key.</param>
+    /// <returns>Ordered oplog entries.</returns>
     public async Task<List<OplogEntry>> GetEntriesByKeyAsync(string collection, string key)
     {
         return (await OplogStore.ExportAsync())
@@ -410,6 +458,7 @@ internal sealed class CdcTestHarness : IAsyncDisposable
             .ToList();
     }
 
+    /// <inheritdoc />
     public async ValueTask DisposeAsync()
     {
         DocumentStore.Dispose();
@@ -428,6 +477,15 @@ internal sealed class CheckpointedSampleDocumentStore : SurrealDocumentStore<Sam
     private const string UsersCollection = "Users";
     private const string TodoListsCollection = "TodoLists";
 
+    /// <summary>
+    ///     Initializes a new instance of the <see cref="CheckpointedSampleDocumentStore"/> class.
+    /// </summary>
+    /// <param name="context">Sample database context.</param>
+    /// <param name="configProvider">Peer configuration provider.</param>
+    /// <param name="vectorClockService">Vector clock service.</param>
+    /// <param name="checkpointPersistence">Checkpoint persistence implementation.</param>
+    /// <param name="surrealOptions">Optional Surreal embedded options.</param>
+    /// <param name="logger">Optional logger.</param>
     public CheckpointedSampleDocumentStore(
         SampleDbContext context,
         IPeerNodeConfigurationProvider configProvider,
@@ -450,6 +508,7 @@ internal sealed class CheckpointedSampleDocumentStore : SurrealDocumentStore<Sam
         WatchCollection(TodoListsCollection, context.TodoLists, t => t.Id, subscribeForInMemoryEvents: false);
     }
 
+    /// <inheritdoc />
     protected override async Task ApplyContentToEntityAsync(
         string collection,
         string key,
@@ -459,6 +518,7 @@ internal sealed class CheckpointedSampleDocumentStore : SurrealDocumentStore<Sam
         await UpsertEntityAsync(collection, key, content, cancellationToken);
     }
 
+    /// <inheritdoc />
     protected override async Task ApplyContentToEntitiesBatchAsync(
         IEnumerable<(string Collection, string Key, JsonElement Content)> documents,
         CancellationToken cancellationToken)
@@ -467,6 +527,7 @@ internal sealed class CheckpointedSampleDocumentStore : SurrealDocumentStore<Sam
             await UpsertEntityAsync(collection, key, content, cancellationToken);
     }
 
+    /// <inheritdoc />
     protected override async Task<JsonElement?> GetEntityAsJsonAsync(
         string collection,
         string key,
@@ -480,6 +541,7 @@ internal sealed class CheckpointedSampleDocumentStore : SurrealDocumentStore<Sam
         };
     }
 
+    /// <inheritdoc />
     protected override async Task RemoveEntityAsync(
         string collection,
         string key,
@@ -488,6 +550,7 @@ internal sealed class CheckpointedSampleDocumentStore : SurrealDocumentStore<Sam
         await DeleteEntityAsync(collection, key, cancellationToken);
     }
 
+    /// <inheritdoc />
     protected override async Task RemoveEntitiesBatchAsync(
         IEnumerable<(string Collection, string Key)> documents,
         CancellationToken cancellationToken)
@@ -496,6 +559,7 @@ internal sealed class CheckpointedSampleDocumentStore : SurrealDocumentStore<Sam
             await DeleteEntityAsync(collection, key, cancellationToken);
     }
 
+    /// <inheritdoc />
     protected override async Task<IEnumerable<(string Key, JsonElement Content)>> GetAllEntitiesAsJsonAsync(
         string collection,
         CancellationToken cancellationToken)

tests/ZB.MOM.WW.CBDDC.Sample.Console.Tests/SurrealCdcMatrixCompletionTests.cs

@@ -13,6 +13,11 @@ namespace ZB.MOM.WW.CBDDC.Sample.Console.Tests;
 
 public class SurrealCdcMatrixCompletionTests
 {
+    /// <summary>
+    ///     Verifies retention-boundary classifier behavior across expected exception message patterns.
+    /// </summary>
+    /// <param name="message">The exception message sample.</param>
+    /// <param name="expected">Expected classifier outcome.</param>
     [Theory]
     [InlineData("versionstamp is outside the configured retention window", true)]
     [InlineData("change feed history since cursor is unavailable", true)]
@@ -29,6 +34,9 @@ public class SurrealCdcMatrixCompletionTests
         actual.ShouldBe(expected);
     }
 
+    /// <summary>
+    ///     Verifies a local write produces exactly one oplog entry.
+    /// </summary>
     [Fact]
     public async Task LocalWrite_ShouldEmitExactlyOneOplogEntry()
     {
@@ -63,6 +71,9 @@ public class SurrealCdcMatrixCompletionTests
         }
     }
 
+    /// <summary>
+    ///     Verifies checkpoint persistence does not advance when atomic write fails.
+    /// </summary>
     [Fact]
     public async Task Checkpoint_ShouldNotAdvance_WhenAtomicWriteFails()
     {
@@ -139,6 +150,14 @@ public class SurrealCdcMatrixCompletionTests
 
 internal sealed class FailureInjectedDocumentStore : SurrealDocumentStore<object>
 {
+    /// <summary>
+    ///     Initializes a new instance of the <see cref="FailureInjectedDocumentStore" /> class.
+    /// </summary>
+    /// <param name="surrealEmbeddedClient">The embedded Surreal client provider.</param>
+    /// <param name="schemaInitializer">The schema initializer.</param>
+    /// <param name="configProvider">The node configuration provider.</param>
+    /// <param name="vectorClockService">The vector clock service.</param>
+    /// <param name="checkpointPersistence">The CDC checkpoint persistence dependency.</param>
     public FailureInjectedDocumentStore(
         ICBDDCSurrealEmbeddedClient surrealEmbeddedClient,
         ICBDDCSurrealSchemaInitializer schemaInitializer,
@@ -158,6 +177,15 @@ internal sealed class FailureInjectedDocumentStore : SurrealDocumentStore<object
     {
     }
 
+    /// <summary>
+    ///     Triggers local change handling for testing failure scenarios.
+    /// </summary>
+    /// <param name="collection">The collection name.</param>
+    /// <param name="key">The document key.</param>
+    /// <param name="operationType">The operation type.</param>
+    /// <param name="content">Optional document content payload.</param>
+    /// <param name="cancellationToken">Cancellation token.</param>
+    /// <returns>A task that completes when processing is finished.</returns>
     public Task TriggerLocalChangeAsync(
         string collection,
         string key,
@@ -174,6 +202,7 @@ internal sealed class FailureInjectedDocumentStore : SurrealDocumentStore<object
             cancellationToken);
     }
 
+    /// <inheritdoc />
     protected override Task ApplyContentToEntityAsync(
         string collection,
         string key,
@@ -183,6 +212,7 @@ internal sealed class FailureInjectedDocumentStore : SurrealDocumentStore<object
         return Task.CompletedTask;
     }
 
+    /// <inheritdoc />
     protected override Task ApplyContentToEntitiesBatchAsync(
         IEnumerable<(string Collection, string Key, JsonElement Content)> documents,
         CancellationToken cancellationToken)
@@ -190,6 +220,7 @@ internal sealed class FailureInjectedDocumentStore : SurrealDocumentStore<object
         return Task.CompletedTask;
     }
 
+    /// <inheritdoc />
     protected override Task<JsonElement?> GetEntityAsJsonAsync(
         string collection,
         string key,
@@ -198,11 +229,13 @@ internal sealed class FailureInjectedDocumentStore : SurrealDocumentStore<object
         return Task.FromResult<JsonElement?>(null);
     }
 
+    /// <inheritdoc />
     protected override Task RemoveEntityAsync(string collection, string key, CancellationToken cancellationToken)
     {
         return Task.CompletedTask;
     }
 
+    /// <inheritdoc />
     protected override Task RemoveEntitiesBatchAsync(
         IEnumerable<(string Collection, string Key)> documents,
         CancellationToken cancellationToken)
@@ -210,6 +243,7 @@ internal sealed class FailureInjectedDocumentStore : SurrealDocumentStore<object
         return Task.CompletedTask;
     }
 
+    /// <inheritdoc />
     protected override Task<IEnumerable<(string Key, JsonElement Content)>> GetAllEntitiesAsJsonAsync(
         string collection,
         CancellationToken cancellationToken)

tests/ZB.MOM.WW.CBDDC.Sample.Console.Tests/SurrealInfrastructureReadinessTests.cs

@@ -0,0 +1,98 @@
+using SurrealDb.Net.Models.Response;
+using ZB.MOM.WW.CBDDC.Persistence.Surreal;
+
+namespace ZB.MOM.WW.CBDDC.Sample.Console.Tests;
+
+public class SurrealInfrastructureReadinessTests
+{
+    /// <summary>
+    ///     Verifies schema initialization runs only once for repeated calls.
+    /// </summary>
+    [Fact]
+    public async Task SchemaInitializer_ShouldApplySchemaOnlyOnce()
+    {
+        var embeddedClient = Substitute.For<ICBDDCSurrealEmbeddedClient>();
+        embeddedClient.RawQueryAsync(
+                Arg.Any<string>(),
+                Arg.Any<IReadOnlyDictionary<string, object?>>(),
+                Arg.Any<CancellationToken>())
+            .Returns(Task.FromResult(default(SurrealDbResponse)!));
+
+        var options = new CBDDCSurrealEmbeddedOptions
+        {
+            Cdc = new CBDDCSurrealCdcOptions
+            {
+                CheckpointTable = "custom_checkpoint",
+                RetentionDuration = TimeSpan.FromHours(12)
+            }
+        };
+
+        var initializer = new CBDDCSurrealSchemaInitializer(embeddedClient, options);
+        await initializer.EnsureInitializedAsync();
+        await initializer.EnsureInitializedAsync();
+
+        await embeddedClient.Received(1).RawQueryAsync(
+            Arg.Is<string>(sql =>
+                sql.Contains("DEFINE TABLE OVERWRITE cbddc_oplog_entries", StringComparison.Ordinal) &&
+                sql.Contains("CHANGEFEED 12h", StringComparison.Ordinal) &&
+                sql.Contains("DEFINE TABLE OVERWRITE custom_checkpoint", StringComparison.Ordinal)),
+            Arg.Any<IReadOnlyDictionary<string, object?>>(),
+            Arg.Any<CancellationToken>());
+    }
+
+    /// <summary>
+    ///     Verifies invalid checkpoint table identifiers are rejected.
+    /// </summary>
+    [Fact]
+    public void SchemaInitializer_WhenCheckpointIdentifierIsInvalid_ThrowsArgumentException()
+    {
+        var embeddedClient = Substitute.For<ICBDDCSurrealEmbeddedClient>();
+        var options = new CBDDCSurrealEmbeddedOptions
+        {
+            Cdc = new CBDDCSurrealCdcOptions
+            {
+                CheckpointTable = "invalid-checkpoint"
+            }
+        };
+
+        Should.Throw<ArgumentException>(() => new CBDDCSurrealSchemaInitializer(embeddedClient, options));
+    }
+
+    /// <summary>
+    ///     Verifies readiness returns true when schema init and health checks succeed.
+    /// </summary>
+    [Fact]
+    public async Task ReadinessProbe_WhenSchemaAndHealthSucceed_ReturnsTrue()
+    {
+        var embeddedClient = Substitute.For<ICBDDCSurrealEmbeddedClient>();
+        embeddedClient.HealthAsync(Arg.Any<CancellationToken>()).Returns(true);
+
+        var schemaInitializer = Substitute.For<ICBDDCSurrealSchemaInitializer>();
+        schemaInitializer.EnsureInitializedAsync(Arg.Any<CancellationToken>()).Returns(Task.CompletedTask);
+
+        var probe = new CBDDCSurrealReadinessProbe(embeddedClient, schemaInitializer);
+        bool isReady = await probe.IsReadyAsync();
+
+        isReady.ShouldBeTrue();
+        await schemaInitializer.Received(1).EnsureInitializedAsync(Arg.Any<CancellationToken>());
+        await embeddedClient.Received(1).HealthAsync(Arg.Any<CancellationToken>());
+    }
+
+    /// <summary>
+    ///     Verifies readiness returns false when schema initialization fails.
+    /// </summary>
+    [Fact]
+    public async Task ReadinessProbe_WhenSchemaInitializationThrows_ReturnsFalse()
+    {
+        var embeddedClient = Substitute.For<ICBDDCSurrealEmbeddedClient>();
+        var schemaInitializer = Substitute.For<ICBDDCSurrealSchemaInitializer>();
+        schemaInitializer.EnsureInitializedAsync(Arg.Any<CancellationToken>())
+            .Returns(Task.FromException(new InvalidOperationException("boom")));
+
+        var probe = new CBDDCSurrealReadinessProbe(embeddedClient, schemaInitializer);
+        bool isReady = await probe.IsReadyAsync();
+
+        isReady.ShouldBeFalse();
+        await embeddedClient.DidNotReceive().HealthAsync(Arg.Any<CancellationToken>());
+    }
+}

tests/ZB.MOM.WW.CBDDC.Sample.Console.Tests/SurrealStoreContractTests.cs

@@ -11,6 +11,9 @@ namespace ZB.MOM.WW.CBDDC.Sample.Console.Tests;
 
 public class SurrealOplogStoreContractTests
 {
+    /// <summary>
+    ///     Verifies append, range query, merge, drop, and last-hash behavior for the oplog store.
+    /// </summary>
     [Fact]
     public async Task OplogStore_AppendQueryMergeDrop_AndLastHash_Works()
     {
@@ -71,6 +74,9 @@ public class SurrealOplogStoreContractTests
 
 public class SurrealDocumentMetadataStoreContractTests
 {
+    /// <summary>
+    ///     Verifies upsert, deletion marking, incremental reads, and merge precedence for document metadata.
+    /// </summary>
     [Fact]
     public async Task DocumentMetadataStore_UpsertMarkDeletedGetAfterAndMergeNewer_Works()
     {
@@ -108,6 +114,9 @@ public class SurrealDocumentMetadataStoreContractTests
 
 public class SurrealPeerConfigurationStoreContractTests
 {
+    /// <summary>
+    ///     Verifies save, read, remove, and merge behavior for remote peer configuration records.
+    /// </summary>
     [Fact]
     public async Task PeerConfigurationStore_SaveGetRemoveAndMerge_Works()
     {
@@ -163,6 +172,9 @@ public class SurrealPeerConfigurationStoreContractTests
 
 public class SurrealPeerOplogConfirmationStoreContractTests
 {
+    /// <summary>
+    ///     Verifies peer registration, confirmation updates, and peer deactivation semantics.
+    /// </summary>
     [Fact]
     public async Task PeerOplogConfirmationStore_EnsureUpdateAndDeactivate_Works()
     {
@@ -194,6 +206,9 @@ public class SurrealPeerOplogConfirmationStoreContractTests
         afterDeactivate.All(x => x.IsActive == false).ShouldBeTrue();
     }
 
+    /// <summary>
+    ///     Verifies merge semantics prefer newer confirmations and preserve active-state transitions.
+    /// </summary>
     [Fact]
     public async Task PeerOplogConfirmationStore_Merge_UsesNewerAndActiveStateSemantics()
     {
@@ -262,6 +277,9 @@ public class SurrealPeerOplogConfirmationStoreContractTests
 
 public class SurrealSnapshotMetadataStoreContractTests
 {
+    /// <summary>
+    ///     Verifies insert, update, merge, and hash lookup behavior for snapshot metadata records.
+    /// </summary>
     [Fact]
     public async Task SnapshotMetadataStore_InsertUpdateMergeAndHashLookup_Works()
     {
@@ -333,6 +351,9 @@ internal sealed class SurrealTestHarness : IAsyncDisposable
     private readonly string _rootPath;
     private readonly ICBDDCSurrealSchemaInitializer _schemaInitializer;
 
+    /// <summary>
+    ///     Initializes a temporary embedded Surreal environment for contract tests.
+    /// </summary>
     public SurrealTestHarness()
     {
         string suffix = Guid.NewGuid().ToString("N");
@@ -351,6 +372,9 @@ internal sealed class SurrealTestHarness : IAsyncDisposable
         _schemaInitializer = new TestSurrealSchemaInitializer(_client);
     }
 
+    /// <summary>
+    ///     Creates a document metadata store instance bound to the test harness database.
+    /// </summary>
     public SurrealDocumentMetadataStore CreateDocumentMetadataStore()
     {
         return new SurrealDocumentMetadataStore(
@@ -359,6 +383,9 @@ internal sealed class SurrealTestHarness : IAsyncDisposable
             NullLogger<SurrealDocumentMetadataStore>.Instance);
     }
 
+    /// <summary>
+    ///     Creates an oplog store instance bound to the test harness database.
+    /// </summary>
     public SurrealOplogStore CreateOplogStore()
     {
         return new SurrealOplogStore(
@@ -371,6 +398,9 @@ internal sealed class SurrealTestHarness : IAsyncDisposable
             NullLogger<SurrealOplogStore>.Instance);
     }
 
+    /// <summary>
+    ///     Creates a peer configuration store instance bound to the test harness database.
+    /// </summary>
     public SurrealPeerConfigurationStore CreatePeerConfigurationStore()
     {
         return new SurrealPeerConfigurationStore(
@@ -379,6 +409,9 @@ internal sealed class SurrealTestHarness : IAsyncDisposable
             NullLogger<SurrealPeerConfigurationStore>.Instance);
     }
 
+    /// <summary>
+    ///     Creates a peer oplog confirmation store instance bound to the test harness database.
+    /// </summary>
     public SurrealPeerOplogConfirmationStore CreatePeerOplogConfirmationStore()
     {
         return new SurrealPeerOplogConfirmationStore(
@@ -387,6 +420,9 @@ internal sealed class SurrealTestHarness : IAsyncDisposable
             NullLogger<SurrealPeerOplogConfirmationStore>.Instance);
     }
 
+    /// <summary>
+    ///     Creates a snapshot metadata store instance bound to the test harness database.
+    /// </summary>
     public SurrealSnapshotMetadataStore CreateSnapshotMetadataStore()
     {
         return new SurrealSnapshotMetadataStore(
@@ -395,6 +431,7 @@ internal sealed class SurrealTestHarness : IAsyncDisposable
             NullLogger<SurrealSnapshotMetadataStore>.Instance);
     }
 
+    /// <inheritdoc />
     public async ValueTask DisposeAsync()
     {
         await _client.DisposeAsync();
@@ -421,11 +458,16 @@ internal sealed class TestSurrealSchemaInitializer : ICBDDCSurrealSchemaInitiali
     private readonly ICBDDCSurrealEmbeddedClient _client;
     private int _initialized;
 
+    /// <summary>
+    ///     Initializes a new instance of the <see cref="TestSurrealSchemaInitializer" /> class.
+    /// </summary>
+    /// <param name="client">The embedded client to initialize.</param>
     public TestSurrealSchemaInitializer(ICBDDCSurrealEmbeddedClient client)
     {
         _client = client;
     }
 
+    /// <inheritdoc />
     public async Task EnsureInitializedAsync(CancellationToken cancellationToken = default)
     {
         if (Interlocked.Exchange(ref _initialized, 1) == 1) return;