Harden Surreal migration with retry/coverage fixes and XML docs cleanup

tests/ZB.MOM.WW.CBDDC.Core.Tests/DocumentCacheTests.cs

@@ -0,0 +1,87 @@
+using System.Text.Json;
+using ZB.MOM.WW.CBDDC.Core.Cache;
+using ZB.MOM.WW.CBDDC.Core.Network;
+
+namespace ZB.MOM.WW.CBDDC.Core.Tests;
+
+public class DocumentCacheTests
+{
+    /// <summary>
+    ///     Verifies cache hit/miss statistics after get and set operations.
+    /// </summary>
+    [Fact]
+    public async Task GetAndSet_ShouldTrackCacheHitsAndMisses()
+    {
+        var cache = new DocumentCache(CreateConfigProvider(maxDocumentCacheSize: 2));
+
+        Document? missing = await cache.Get("users", "1");
+        missing.ShouldBeNull();
+
+        var document = CreateDocument("users", "1");
+        await cache.Set("users", "1", document);
+        Document? hit = await cache.Get("users", "1");
+
+        hit.ShouldNotBeNull();
+        hit.Key.ShouldBe("1");
+        var stats = cache.GetStatistics();
+        stats.Hits.ShouldBe(1);
+        stats.Misses.ShouldBe(1);
+        stats.Size.ShouldBe(1);
+        stats.HitRate.ShouldBe(0.5d);
+    }
+
+    /// <summary>
+    ///     Verifies least-recently-used eviction when cache capacity is reached.
+    /// </summary>
+    [Fact]
+    public async Task Set_WhenCacheIsFull_EvictsLeastRecentlyUsedEntry()
+    {
+        var cache = new DocumentCache(CreateConfigProvider(maxDocumentCacheSize: 2));
+        await cache.Set("users", "1", CreateDocument("users", "1"));
+        await cache.Set("users", "2", CreateDocument("users", "2"));
+
+        // Touch key 1 so key 2 becomes the LRU entry.
+        (await cache.Get("users", "1")).ShouldNotBeNull();
+
+        await cache.Set("users", "3", CreateDocument("users", "3"));
+
+        (await cache.Get("users", "2")).ShouldBeNull();
+        (await cache.Get("users", "1")).ShouldNotBeNull();
+        (await cache.Get("users", "3")).ShouldNotBeNull();
+    }
+
+    /// <summary>
+    ///     Verifies remove and clear operations delete entries from the cache.
+    /// </summary>
+    [Fact]
+    public async Task RemoveAndClear_ShouldDeleteEntriesFromCache()
+    {
+        var cache = new DocumentCache(CreateConfigProvider(maxDocumentCacheSize: 3));
+        await cache.Set("users", "1", CreateDocument("users", "1"));
+        await cache.Set("users", "2", CreateDocument("users", "2"));
+        cache.GetStatistics().Size.ShouldBe(2);
+
+        cache.Remove("users", "1");
+        (await cache.Get("users", "1")).ShouldBeNull();
+        cache.GetStatistics().Size.ShouldBe(1);
+
+        cache.Clear();
+        cache.GetStatistics().Size.ShouldBe(0);
+    }
+
+    private static Document CreateDocument(string collection, string key)
+    {
+        using var json = JsonDocument.Parse("""{"name":"test"}""");
+        return new Document(collection, key, json.RootElement.Clone(), new HlcTimestamp(1, 0, "node-a"), false);
+    }
+
+    private static IPeerNodeConfigurationProvider CreateConfigProvider(int maxDocumentCacheSize)
+    {
+        var configProvider = Substitute.For<IPeerNodeConfigurationProvider>();
+        configProvider.GetConfiguration().Returns(new PeerNodeConfiguration
+        {
+            MaxDocumentCacheSize = maxDocumentCacheSize
+        });
+        return configProvider;
+    }
+}

tests/ZB.MOM.WW.CBDDC.Core.Tests/OfflineQueueTests.cs

@@ -0,0 +1,92 @@
+using ZB.MOM.WW.CBDDC.Core.Network;
+using ZB.MOM.WW.CBDDC.Core.Sync;
+
+namespace ZB.MOM.WW.CBDDC.Core.Tests;
+
+public class OfflineQueueTests
+{
+    /// <summary>
+    /// Verifies that enqueuing beyond capacity drops the oldest operation.
+    /// </summary>
+    [Fact]
+    public async Task Enqueue_WhenQueueIsFull_DropsOldestOperation()
+    {
+        var queue = new OfflineQueue(CreateConfigProvider(maxQueueSize: 2));
+        await queue.Enqueue(CreateOperation("1"));
+        await queue.Enqueue(CreateOperation("2"));
+        await queue.Enqueue(CreateOperation("3"));
+
+        var flushed = new List<string>();
+        (int successful, int failed) = await queue.FlushAsync(op =>
+        {
+            flushed.Add(op.Key);
+            return Task.CompletedTask;
+        });
+
+        successful.ShouldBe(2);
+        failed.ShouldBe(0);
+        flushed.ShouldBe(["2", "3"]);
+    }
+
+    /// <summary>
+    /// Verifies that flush continues when an executor throws and returns the failure count.
+    /// </summary>
+    [Fact]
+    public async Task FlushAsync_WhenExecutorThrows_ContinuesAndReturnsFailureCount()
+    {
+        var queue = new OfflineQueue(CreateConfigProvider(maxQueueSize: 10));
+        await queue.Enqueue(CreateOperation("1"));
+        await queue.Enqueue(CreateOperation("2"));
+
+        (int successful, int failed) = await queue.FlushAsync(op =>
+        {
+            if (op.Key == "1") throw new InvalidOperationException("boom");
+            return Task.CompletedTask;
+        });
+
+        successful.ShouldBe(1);
+        failed.ShouldBe(1);
+        queue.Count.ShouldBe(0);
+    }
+
+    /// <summary>
+    /// Verifies that clear removes all queued operations.
+    /// </summary>
+    [Fact]
+    public async Task Clear_RemovesAllQueuedOperations()
+    {
+        var queue = new OfflineQueue(CreateConfigProvider(maxQueueSize: 10));
+        await queue.Enqueue(CreateOperation("1"));
+        await queue.Enqueue(CreateOperation("2"));
+        queue.Count.ShouldBe(2);
+
+        await queue.Clear();
+
+        queue.Count.ShouldBe(0);
+        (int successful, int failed) = await queue.FlushAsync(_ => Task.CompletedTask);
+        successful.ShouldBe(0);
+        failed.ShouldBe(0);
+    }
+
+    private static PendingOperation CreateOperation(string key)
+    {
+        return new PendingOperation
+        {
+            Type = "upsert",
+            Collection = "users",
+            Key = key,
+            Data = new { Value = key },
+            QueuedAt = DateTime.UtcNow
+        };
+    }
+
+    private static IPeerNodeConfigurationProvider CreateConfigProvider(int maxQueueSize)
+    {
+        var configProvider = Substitute.For<IPeerNodeConfigurationProvider>();
+        configProvider.GetConfiguration().Returns(new PeerNodeConfiguration
+        {
+            MaxQueueSize = maxQueueSize
+        });
+        return configProvider;
+    }
+}

tests/ZB.MOM.WW.CBDDC.Core.Tests/RetryPolicyTests.cs

@@ -0,0 +1,78 @@
+using ZB.MOM.WW.CBDDC.Core.Exceptions;
+using ZB.MOM.WW.CBDDC.Core.Network;
+using ZB.MOM.WW.CBDDC.Core.Resilience;
+
+namespace ZB.MOM.WW.CBDDC.Core.Tests;
+
+public class RetryPolicyTests
+{
+    /// <summary>
+    ///     Verifies transient failures are retried until a successful result is returned.
+    /// </summary>
+    [Fact]
+    public async Task ExecuteAsync_WhenTransientFailureEventuallySucceeds_RetriesAndReturnsResult()
+    {
+        var policy = new RetryPolicy(CreateConfigProvider(retryAttempts: 3, retryDelayMs: 1));
+        var attempts = 0;
+
+        int result = await policy.ExecuteAsync(async () =>
+        {
+            attempts++;
+            if (attempts < 3) throw new NetworkException("transient");
+            await Task.CompletedTask;
+            return 42;
+        }, "test-op");
+
+        result.ShouldBe(42);
+        attempts.ShouldBe(3);
+    }
+
+    /// <summary>
+    ///     Verifies transient failures throw retry exhausted when all retries are consumed.
+    /// </summary>
+    [Fact]
+    public async Task ExecuteAsync_WhenTransientFailureExhausted_ThrowsRetryExhaustedException()
+    {
+        var policy = new RetryPolicy(CreateConfigProvider(retryAttempts: 2, retryDelayMs: 1));
+        var attempts = 0;
+
+        var ex = await Should.ThrowAsync<CBDDCException>(() => policy.ExecuteAsync<int>(() =>
+        {
+            attempts++;
+            throw new NetworkException("still transient");
+        }, "test-op"));
+
+        ex.ErrorCode.ShouldBe("RETRY_EXHAUSTED");
+        ex.InnerException.ShouldBeOfType<NetworkException>();
+        attempts.ShouldBe(2);
+    }
+
+    /// <summary>
+    ///     Verifies non-transient failures are not retried.
+    /// </summary>
+    [Fact]
+    public async Task ExecuteAsync_WhenFailureIsNonTransient_DoesNotRetry()
+    {
+        var policy = new RetryPolicy(CreateConfigProvider(retryAttempts: 3, retryDelayMs: 1));
+        var attempts = 0;
+
+        await Should.ThrowAsync<InvalidOperationException>(() => policy.ExecuteAsync<int>(() =>
+        {
+            attempts++;
+            throw new InvalidOperationException("non-transient");
+        }, "test-op"));
+
+        attempts.ShouldBe(1);
+    }
+
+    private static IPeerNodeConfigurationProvider CreateConfigProvider(int retryAttempts, int retryDelayMs)
+    {
+        var configProvider = Substitute.For<IPeerNodeConfigurationProvider>();
+        configProvider.GetConfiguration().Returns(new PeerNodeConfiguration
+        {
+            RetryAttempts = retryAttempts,
+            RetryDelayMs = retryDelayMs
+        });
+        return configProvider;
+    }
+}

tests/ZB.MOM.WW.CBDDC.E2E.Tests/ClusterCrudSyncE2ETests.cs

@@ -428,12 +428,12 @@ public class ClusterCrudSyncE2ETests
                            && replicated.Address?.City == payload.Address?.City;
                 }, 60, "Node B did not converge after crash-window recovery.", () => BuildDiagnostics(recoveredNodeA, nodeB));
 
-            await AssertEventuallyAsync(
-                () => recoveredNodeA.GetOplogCountForKey("Users", userId) == 1 &&
-                      nodeB.GetOplogCountForKey("Users", userId) == 1,
-                60,
-                "Crash-window recovery created duplicate oplog entries.",
-                () => BuildDiagnostics(recoveredNodeA, nodeB));
+                await AssertEventuallyAsync(
+                    () => recoveredNodeA.GetOplogCountForKey("Users", userId) == 1 &&
+                          nodeB.GetOplogCountForKey("Users", userId) == 1,
+                    60,
+                    "Crash-window recovery created duplicate oplog entries.",
+                    () => BuildDiagnostics(recoveredNodeA, nodeB));
             }
         }
         finally
@@ -569,6 +569,9 @@ public class ClusterCrudSyncE2ETests
         /// <param name="tcpPort">The TCP port used by the node listener.</param>
         /// <param name="authToken">The cluster authentication token.</param>
         /// <param name="knownPeers">The known peers this node can connect to.</param>
+        /// <param name="workDirOverride">An optional working directory override for test artifacts.</param>
+        /// <param name="preserveWorkDirOnDispose">A value indicating whether to preserve the working directory on dispose.</param>
+        /// <param name="useFaultInjectedCheckpointStore">A value indicating whether to inject a checkpoint persistence that fails once.</param>
         /// <returns>A configured <see cref="TestPeerNode" /> instance.</returns>
         public static TestPeerNode Create(
             string nodeId,
@@ -690,6 +693,12 @@ public class ClusterCrudSyncE2ETests
             return Context.Users.Find(u => u.Id == userId).FirstOrDefault();
         }
 
+        /// <summary>
+        ///     Gets the local oplog entry count for a collection/key pair produced by this node.
+        /// </summary>
+        /// <param name="collection">The collection name.</param>
+        /// <param name="key">The document key.</param>
+        /// <returns>The number of local oplog entries matching the key.</returns>
         public int GetLocalOplogCountForKey(string collection, string key)
         {
             return Context.OplogEntries.FindAll()
@@ -699,6 +708,12 @@ public class ClusterCrudSyncE2ETests
                     string.Equals(e.TimestampNodeId, _nodeId, StringComparison.Ordinal));
         }
 
+        /// <summary>
+        ///     Gets the total oplog entry count for a collection/key pair across nodes.
+        /// </summary>
+        /// <param name="collection">The collection name.</param>
+        /// <param name="key">The document key.</param>
+        /// <returns>The number of oplog entries matching the key.</returns>
         public int GetOplogCountForKey(string collection, string key)
         {
             return Context.OplogEntries.FindAll()
@@ -824,6 +839,14 @@ public class ClusterCrudSyncE2ETests
         private const string UsersCollection = "Users";
         private const string TodoListsCollection = "TodoLists";
 
+        /// <summary>
+        ///     Initializes a new instance of the <see cref="FaultInjectedSampleDocumentStore" /> class.
+        /// </summary>
+        /// <param name="context">The sample database context.</param>
+        /// <param name="configProvider">The peer node configuration provider.</param>
+        /// <param name="vectorClockService">The vector clock service.</param>
+        /// <param name="checkpointPersistence">The checkpoint persistence implementation.</param>
+        /// <param name="logger">The optional logger instance.</param>
         public FaultInjectedSampleDocumentStore(
             SampleDbContext context,
             IPeerNodeConfigurationProvider configProvider,
@@ -849,6 +872,7 @@ public class ClusterCrudSyncE2ETests
             WatchCollection(TodoListsCollection, context.TodoLists, t => t.Id);
         }
 
+        /// <inheritdoc />
         protected override async Task ApplyContentToEntityAsync(
             string collection,
             string key,
@@ -858,6 +882,7 @@ public class ClusterCrudSyncE2ETests
             await UpsertEntityAsync(collection, key, content, cancellationToken);
         }
 
+        /// <inheritdoc />
         protected override async Task ApplyContentToEntitiesBatchAsync(
             IEnumerable<(string Collection, string Key, JsonElement Content)> documents,
             CancellationToken cancellationToken)
@@ -866,6 +891,7 @@ public class ClusterCrudSyncE2ETests
                 await UpsertEntityAsync(collection, key, content, cancellationToken);
         }
 
+        /// <inheritdoc />
         protected override async Task<JsonElement?> GetEntityAsJsonAsync(
             string collection,
             string key,
@@ -879,6 +905,7 @@ public class ClusterCrudSyncE2ETests
             };
         }
 
+        /// <inheritdoc />
         protected override async Task RemoveEntityAsync(
             string collection,
             string key,
@@ -887,6 +914,7 @@ public class ClusterCrudSyncE2ETests
             await DeleteEntityAsync(collection, key, cancellationToken);
         }
 
+        /// <inheritdoc />
         protected override async Task RemoveEntitiesBatchAsync(
             IEnumerable<(string Collection, string Key)> documents,
             CancellationToken cancellationToken)
@@ -895,6 +923,7 @@ public class ClusterCrudSyncE2ETests
                 await DeleteEntityAsync(collection, key, cancellationToken);
         }
 
+        /// <inheritdoc />
         protected override async Task<IEnumerable<(string Key, JsonElement Content)>> GetAllEntitiesAsJsonAsync(
             string collection,
             CancellationToken cancellationToken)
@@ -967,6 +996,7 @@ public class ClusterCrudSyncE2ETests
     {
         private int _failOnNextAdvance = 1;
 
+        /// <inheritdoc />
         public Task<SurrealCdcCheckpoint?> GetCheckpointAsync(
             string? consumerId = null,
             CancellationToken cancellationToken = default)
@@ -974,6 +1004,7 @@ public class ClusterCrudSyncE2ETests
             return Task.FromResult<SurrealCdcCheckpoint?>(null);
         }
 
+        /// <inheritdoc />
         public Task UpsertCheckpointAsync(
             HlcTimestamp timestamp,
             string lastHash,
@@ -984,6 +1015,7 @@ public class ClusterCrudSyncE2ETests
             return Task.CompletedTask;
         }
 
+        /// <inheritdoc />
         public Task AdvanceCheckpointAsync(
             OplogEntry entry,
             string? consumerId = null,

tests/ZB.MOM.WW.CBDDC.Sample.Console.Tests/SurrealCdcDurabilityTests.cs

@@ -13,6 +13,9 @@ namespace ZB.MOM.WW.CBDDC.Sample.Console.Tests;
 [Collection("SurrealCdcDurability")]
 public class SurrealCdcDurabilityTests
 {
+    /// <summary>
+    ///     Verifies checkpoints persist latest local changes per consumer across restarts.
+    /// </summary>
     [Fact]
     public async Task CheckpointPersistence_ShouldTrackLatestLocalChange_AndPersistPerConsumer()
     {
@@ -85,6 +88,9 @@ public class SurrealCdcDurabilityTests
         }
     }
 
+    /// <summary>
+    ///     Verifies recovery resumes from a persisted checkpoint and advances after catch-up.
+    /// </summary>
     [Fact]
     public async Task RestartRecovery_ShouldResumeCatchUpFromPersistedCheckpoint_InRocksDb()
     {
@@ -153,6 +159,9 @@ public class SurrealCdcDurabilityTests
         }
     }
 
+    /// <summary>
+    ///     Verifies duplicate remote apply windows are idempotent without loopback entries.
+    /// </summary>
     [Fact]
     public async Task RemoteApply_ShouldBeIdempotentAcrossDuplicateWindow_WithoutLoopbackEntries()
     {
@@ -207,6 +216,9 @@ public class SurrealCdcDurabilityTests
         }
     }
 
+    /// <summary>
+    ///     Verifies local deletes persist tombstone metadata and advance checkpoints.
+    /// </summary>
     [Fact]
     public async Task LocalDelete_ShouldPersistTombstoneMetadata_AndAdvanceCheckpoint()
     {
@@ -358,21 +370,46 @@ internal sealed class CdcTestHarness : IAsyncDisposable
             NullLogger<SurrealDocumentMetadataStore>.Instance);
     }
 
+    /// <summary>
+    ///     Gets the sample database context.
+    /// </summary>
     public SampleDbContext Context { get; }
 
+    /// <summary>
+    ///     Gets the checkpointed sample document store.
+    /// </summary>
     public CheckpointedSampleDocumentStore DocumentStore { get; }
 
+    /// <summary>
+    ///     Gets the oplog store used by the harness.
+    /// </summary>
     public SurrealOplogStore OplogStore { get; }
 
+    /// <summary>
+    ///     Gets the document metadata store.
+    /// </summary>
     public SurrealDocumentMetadataStore MetadataStore { get; }
 
+    /// <summary>
+    ///     Gets checkpoint persistence used for CDC progress tracking.
+    /// </summary>
     public ISurrealCdcCheckpointPersistence CheckpointPersistence { get; }
 
+    /// <summary>
+    ///     Polls CDC once through the document store.
+    /// </summary>
     public async Task PollAsync()
     {
         await DocumentStore.PollCdcOnceAsync();
     }
 
+    /// <summary>
+    ///     Creates a harness instance with retries for transient RocksDB lock contention.
+    /// </summary>
+    /// <param name="databasePath">Database directory path.</param>
+    /// <param name="nodeId">Node identifier.</param>
+    /// <param name="consumerId">CDC consumer identifier.</param>
+    /// <returns>Initialized test harness.</returns>
     public static async Task<CdcTestHarness> OpenWithRetriesAsync(
         string databasePath,
         string nodeId,
@@ -391,6 +428,11 @@ internal sealed class CdcTestHarness : IAsyncDisposable
         throw new InvalidOperationException("Unable to acquire RocksDB lock for test harness.");
     }
 
+    /// <summary>
+    ///     Gets oplog entries for a collection ordered by timestamp.
+    /// </summary>
+    /// <param name="collection">Collection name.</param>
+    /// <returns>Ordered oplog entries.</returns>
     public async Task<List<OplogEntry>> GetEntriesByCollectionAsync(string collection)
     {
         return (await OplogStore.ExportAsync())
@@ -400,6 +442,12 @@ internal sealed class CdcTestHarness : IAsyncDisposable
             .ToList();
     }
 
+    /// <summary>
+    ///     Gets oplog entries for a collection key ordered by timestamp.
+    /// </summary>
+    /// <param name="collection">Collection name.</param>
+    /// <param name="key">Document key.</param>
+    /// <returns>Ordered oplog entries.</returns>
     public async Task<List<OplogEntry>> GetEntriesByKeyAsync(string collection, string key)
     {
         return (await OplogStore.ExportAsync())
@@ -410,6 +458,7 @@ internal sealed class CdcTestHarness : IAsyncDisposable
             .ToList();
     }
 
+    /// <inheritdoc />
     public async ValueTask DisposeAsync()
     {
         DocumentStore.Dispose();
@@ -428,6 +477,15 @@ internal sealed class CheckpointedSampleDocumentStore : SurrealDocumentStore<Sam
     private const string UsersCollection = "Users";
     private const string TodoListsCollection = "TodoLists";
 
+    /// <summary>
+    ///     Initializes a new instance of the <see cref="CheckpointedSampleDocumentStore"/> class.
+    /// </summary>
+    /// <param name="context">Sample database context.</param>
+    /// <param name="configProvider">Peer configuration provider.</param>
+    /// <param name="vectorClockService">Vector clock service.</param>
+    /// <param name="checkpointPersistence">Checkpoint persistence implementation.</param>
+    /// <param name="surrealOptions">Optional Surreal embedded options.</param>
+    /// <param name="logger">Optional logger.</param>
     public CheckpointedSampleDocumentStore(
         SampleDbContext context,
         IPeerNodeConfigurationProvider configProvider,
@@ -450,6 +508,7 @@ internal sealed class CheckpointedSampleDocumentStore : SurrealDocumentStore<Sam
         WatchCollection(TodoListsCollection, context.TodoLists, t => t.Id, subscribeForInMemoryEvents: false);
     }
 
+    /// <inheritdoc />
     protected override async Task ApplyContentToEntityAsync(
         string collection,
         string key,
@@ -459,6 +518,7 @@ internal sealed class CheckpointedSampleDocumentStore : SurrealDocumentStore<Sam
         await UpsertEntityAsync(collection, key, content, cancellationToken);
     }
 
+    /// <inheritdoc />
     protected override async Task ApplyContentToEntitiesBatchAsync(
         IEnumerable<(string Collection, string Key, JsonElement Content)> documents,
         CancellationToken cancellationToken)
@@ -467,6 +527,7 @@ internal sealed class CheckpointedSampleDocumentStore : SurrealDocumentStore<Sam
             await UpsertEntityAsync(collection, key, content, cancellationToken);
     }
 
+    /// <inheritdoc />
     protected override async Task<JsonElement?> GetEntityAsJsonAsync(
         string collection,
         string key,
@@ -480,6 +541,7 @@ internal sealed class CheckpointedSampleDocumentStore : SurrealDocumentStore<Sam
         };
     }
 
+    /// <inheritdoc />
     protected override async Task RemoveEntityAsync(
         string collection,
         string key,
@@ -488,6 +550,7 @@ internal sealed class CheckpointedSampleDocumentStore : SurrealDocumentStore<Sam
         await DeleteEntityAsync(collection, key, cancellationToken);
     }
 
+    /// <inheritdoc />
     protected override async Task RemoveEntitiesBatchAsync(
         IEnumerable<(string Collection, string Key)> documents,
         CancellationToken cancellationToken)
@@ -496,6 +559,7 @@ internal sealed class CheckpointedSampleDocumentStore : SurrealDocumentStore<Sam
             await DeleteEntityAsync(collection, key, cancellationToken);
     }
 
+    /// <inheritdoc />
     protected override async Task<IEnumerable<(string Key, JsonElement Content)>> GetAllEntitiesAsJsonAsync(
         string collection,
         CancellationToken cancellationToken)

tests/ZB.MOM.WW.CBDDC.Sample.Console.Tests/SurrealCdcMatrixCompletionTests.cs

@@ -13,6 +13,11 @@ namespace ZB.MOM.WW.CBDDC.Sample.Console.Tests;
 
 public class SurrealCdcMatrixCompletionTests
 {
+    /// <summary>
+    ///     Verifies retention-boundary classifier behavior across expected exception message patterns.
+    /// </summary>
+    /// <param name="message">The exception message sample.</param>
+    /// <param name="expected">Expected classifier outcome.</param>
     [Theory]
     [InlineData("versionstamp is outside the configured retention window", true)]
     [InlineData("change feed history since cursor is unavailable", true)]
@@ -29,6 +34,9 @@ public class SurrealCdcMatrixCompletionTests
         actual.ShouldBe(expected);
     }
 
+    /// <summary>
+    ///     Verifies a local write produces exactly one oplog entry.
+    /// </summary>
     [Fact]
     public async Task LocalWrite_ShouldEmitExactlyOneOplogEntry()
     {
@@ -63,6 +71,9 @@ public class SurrealCdcMatrixCompletionTests
         }
     }
 
+    /// <summary>
+    ///     Verifies checkpoint persistence does not advance when atomic write fails.
+    /// </summary>
     [Fact]
     public async Task Checkpoint_ShouldNotAdvance_WhenAtomicWriteFails()
     {
@@ -139,6 +150,14 @@ public class SurrealCdcMatrixCompletionTests
 
 internal sealed class FailureInjectedDocumentStore : SurrealDocumentStore<object>
 {
+    /// <summary>
+    ///     Initializes a new instance of the <see cref="FailureInjectedDocumentStore" /> class.
+    /// </summary>
+    /// <param name="surrealEmbeddedClient">The embedded Surreal client provider.</param>
+    /// <param name="schemaInitializer">The schema initializer.</param>
+    /// <param name="configProvider">The node configuration provider.</param>
+    /// <param name="vectorClockService">The vector clock service.</param>
+    /// <param name="checkpointPersistence">The CDC checkpoint persistence dependency.</param>
     public FailureInjectedDocumentStore(
         ICBDDCSurrealEmbeddedClient surrealEmbeddedClient,
         ICBDDCSurrealSchemaInitializer schemaInitializer,
@@ -158,6 +177,15 @@ internal sealed class FailureInjectedDocumentStore : SurrealDocumentStore<object
     {
     }
 
+    /// <summary>
+    ///     Triggers local change handling for testing failure scenarios.
+    /// </summary>
+    /// <param name="collection">The collection name.</param>
+    /// <param name="key">The document key.</param>
+    /// <param name="operationType">The operation type.</param>
+    /// <param name="content">Optional document content payload.</param>
+    /// <param name="cancellationToken">Cancellation token.</param>
+    /// <returns>A task that completes when processing is finished.</returns>
     public Task TriggerLocalChangeAsync(
         string collection,
         string key,
@@ -174,6 +202,7 @@ internal sealed class FailureInjectedDocumentStore : SurrealDocumentStore<object
             cancellationToken);
     }
 
+    /// <inheritdoc />
     protected override Task ApplyContentToEntityAsync(
         string collection,
         string key,
@@ -183,6 +212,7 @@ internal sealed class FailureInjectedDocumentStore : SurrealDocumentStore<object
         return Task.CompletedTask;
     }
 
+    /// <inheritdoc />
     protected override Task ApplyContentToEntitiesBatchAsync(
         IEnumerable<(string Collection, string Key, JsonElement Content)> documents,
         CancellationToken cancellationToken)
@@ -190,6 +220,7 @@ internal sealed class FailureInjectedDocumentStore : SurrealDocumentStore<object
         return Task.CompletedTask;
     }
 
+    /// <inheritdoc />
     protected override Task<JsonElement?> GetEntityAsJsonAsync(
         string collection,
         string key,
@@ -198,11 +229,13 @@ internal sealed class FailureInjectedDocumentStore : SurrealDocumentStore<object
         return Task.FromResult<JsonElement?>(null);
     }
 
+    /// <inheritdoc />
     protected override Task RemoveEntityAsync(string collection, string key, CancellationToken cancellationToken)
     {
         return Task.CompletedTask;
     }
 
+    /// <inheritdoc />
     protected override Task RemoveEntitiesBatchAsync(
         IEnumerable<(string Collection, string Key)> documents,
         CancellationToken cancellationToken)
@@ -210,6 +243,7 @@ internal sealed class FailureInjectedDocumentStore : SurrealDocumentStore<object
         return Task.CompletedTask;
     }
 
+    /// <inheritdoc />
     protected override Task<IEnumerable<(string Key, JsonElement Content)>> GetAllEntitiesAsJsonAsync(
         string collection,
         CancellationToken cancellationToken)

tests/ZB.MOM.WW.CBDDC.Sample.Console.Tests/SurrealInfrastructureReadinessTests.cs

@@ -0,0 +1,98 @@
+using SurrealDb.Net.Models.Response;
+using ZB.MOM.WW.CBDDC.Persistence.Surreal;
+
+namespace ZB.MOM.WW.CBDDC.Sample.Console.Tests;
+
+public class SurrealInfrastructureReadinessTests
+{
+    /// <summary>
+    ///     Verifies schema initialization runs only once for repeated calls.
+    /// </summary>
+    [Fact]
+    public async Task SchemaInitializer_ShouldApplySchemaOnlyOnce()
+    {
+        var embeddedClient = Substitute.For<ICBDDCSurrealEmbeddedClient>();
+        embeddedClient.RawQueryAsync(
+                Arg.Any<string>(),
+                Arg.Any<IReadOnlyDictionary<string, object?>>(),
+                Arg.Any<CancellationToken>())
+            .Returns(Task.FromResult(default(SurrealDbResponse)!));
+
+        var options = new CBDDCSurrealEmbeddedOptions
+        {
+            Cdc = new CBDDCSurrealCdcOptions
+            {
+                CheckpointTable = "custom_checkpoint",
+                RetentionDuration = TimeSpan.FromHours(12)
+            }
+        };
+
+        var initializer = new CBDDCSurrealSchemaInitializer(embeddedClient, options);
+        await initializer.EnsureInitializedAsync();
+        await initializer.EnsureInitializedAsync();
+
+        await embeddedClient.Received(1).RawQueryAsync(
+            Arg.Is<string>(sql =>
+                sql.Contains("DEFINE TABLE OVERWRITE cbddc_oplog_entries", StringComparison.Ordinal) &&
+                sql.Contains("CHANGEFEED 12h", StringComparison.Ordinal) &&
+                sql.Contains("DEFINE TABLE OVERWRITE custom_checkpoint", StringComparison.Ordinal)),
+            Arg.Any<IReadOnlyDictionary<string, object?>>(),
+            Arg.Any<CancellationToken>());
+    }
+
+    /// <summary>
+    ///     Verifies invalid checkpoint table identifiers are rejected.
+    /// </summary>
+    [Fact]
+    public void SchemaInitializer_WhenCheckpointIdentifierIsInvalid_ThrowsArgumentException()
+    {
+        var embeddedClient = Substitute.For<ICBDDCSurrealEmbeddedClient>();
+        var options = new CBDDCSurrealEmbeddedOptions
+        {
+            Cdc = new CBDDCSurrealCdcOptions
+            {
+                CheckpointTable = "invalid-checkpoint"
+            }
+        };
+
+        Should.Throw<ArgumentException>(() => new CBDDCSurrealSchemaInitializer(embeddedClient, options));
+    }
+
+    /// <summary>
+    ///     Verifies readiness returns true when schema init and health checks succeed.
+    /// </summary>
+    [Fact]
+    public async Task ReadinessProbe_WhenSchemaAndHealthSucceed_ReturnsTrue()
+    {
+        var embeddedClient = Substitute.For<ICBDDCSurrealEmbeddedClient>();
+        embeddedClient.HealthAsync(Arg.Any<CancellationToken>()).Returns(true);
+
+        var schemaInitializer = Substitute.For<ICBDDCSurrealSchemaInitializer>();
+        schemaInitializer.EnsureInitializedAsync(Arg.Any<CancellationToken>()).Returns(Task.CompletedTask);
+
+        var probe = new CBDDCSurrealReadinessProbe(embeddedClient, schemaInitializer);
+        bool isReady = await probe.IsReadyAsync();
+
+        isReady.ShouldBeTrue();
+        await schemaInitializer.Received(1).EnsureInitializedAsync(Arg.Any<CancellationToken>());
+        await embeddedClient.Received(1).HealthAsync(Arg.Any<CancellationToken>());
+    }
+
+    /// <summary>
+    ///     Verifies readiness returns false when schema initialization fails.
+    /// </summary>
+    [Fact]
+    public async Task ReadinessProbe_WhenSchemaInitializationThrows_ReturnsFalse()
+    {
+        var embeddedClient = Substitute.For<ICBDDCSurrealEmbeddedClient>();
+        var schemaInitializer = Substitute.For<ICBDDCSurrealSchemaInitializer>();
+        schemaInitializer.EnsureInitializedAsync(Arg.Any<CancellationToken>())
+            .Returns(Task.FromException(new InvalidOperationException("boom")));
+
+        var probe = new CBDDCSurrealReadinessProbe(embeddedClient, schemaInitializer);
+        bool isReady = await probe.IsReadyAsync();
+
+        isReady.ShouldBeFalse();
+        await embeddedClient.DidNotReceive().HealthAsync(Arg.Any<CancellationToken>());
+    }
+}

tests/ZB.MOM.WW.CBDDC.Sample.Console.Tests/SurrealStoreContractTests.cs

@@ -11,6 +11,9 @@ namespace ZB.MOM.WW.CBDDC.Sample.Console.Tests;
 
 public class SurrealOplogStoreContractTests
 {
+    /// <summary>
+    ///     Verifies append, range query, merge, drop, and last-hash behavior for the oplog store.
+    /// </summary>
     [Fact]
     public async Task OplogStore_AppendQueryMergeDrop_AndLastHash_Works()
     {
@@ -71,6 +74,9 @@ public class SurrealOplogStoreContractTests
 
 public class SurrealDocumentMetadataStoreContractTests
 {
+    /// <summary>
+    ///     Verifies upsert, deletion marking, incremental reads, and merge precedence for document metadata.
+    /// </summary>
     [Fact]
     public async Task DocumentMetadataStore_UpsertMarkDeletedGetAfterAndMergeNewer_Works()
     {
@@ -108,6 +114,9 @@ public class SurrealDocumentMetadataStoreContractTests
 
 public class SurrealPeerConfigurationStoreContractTests
 {
+    /// <summary>
+    ///     Verifies save, read, remove, and merge behavior for remote peer configuration records.
+    /// </summary>
     [Fact]
     public async Task PeerConfigurationStore_SaveGetRemoveAndMerge_Works()
     {
@@ -163,6 +172,9 @@ public class SurrealPeerConfigurationStoreContractTests
 
 public class SurrealPeerOplogConfirmationStoreContractTests
 {
+    /// <summary>
+    ///     Verifies peer registration, confirmation updates, and peer deactivation semantics.
+    /// </summary>
     [Fact]
     public async Task PeerOplogConfirmationStore_EnsureUpdateAndDeactivate_Works()
     {
@@ -194,6 +206,9 @@ public class SurrealPeerOplogConfirmationStoreContractTests
         afterDeactivate.All(x => x.IsActive == false).ShouldBeTrue();
     }
 
+    /// <summary>
+    ///     Verifies merge semantics prefer newer confirmations and preserve active-state transitions.
+    /// </summary>
     [Fact]
     public async Task PeerOplogConfirmationStore_Merge_UsesNewerAndActiveStateSemantics()
     {
@@ -262,6 +277,9 @@ public class SurrealPeerOplogConfirmationStoreContractTests
 
 public class SurrealSnapshotMetadataStoreContractTests
 {
+    /// <summary>
+    ///     Verifies insert, update, merge, and hash lookup behavior for snapshot metadata records.
+    /// </summary>
     [Fact]
     public async Task SnapshotMetadataStore_InsertUpdateMergeAndHashLookup_Works()
     {
@@ -333,6 +351,9 @@ internal sealed class SurrealTestHarness : IAsyncDisposable
     private readonly string _rootPath;
     private readonly ICBDDCSurrealSchemaInitializer _schemaInitializer;
 
+    /// <summary>
+    ///     Initializes a temporary embedded Surreal environment for contract tests.
+    /// </summary>
     public SurrealTestHarness()
     {
         string suffix = Guid.NewGuid().ToString("N");
@@ -351,6 +372,9 @@ internal sealed class SurrealTestHarness : IAsyncDisposable
         _schemaInitializer = new TestSurrealSchemaInitializer(_client);
     }
 
+    /// <summary>
+    ///     Creates a document metadata store instance bound to the test harness database.
+    /// </summary>
     public SurrealDocumentMetadataStore CreateDocumentMetadataStore()
     {
         return new SurrealDocumentMetadataStore(
@@ -359,6 +383,9 @@ internal sealed class SurrealTestHarness : IAsyncDisposable
             NullLogger<SurrealDocumentMetadataStore>.Instance);
     }
 
+    /// <summary>
+    ///     Creates an oplog store instance bound to the test harness database.
+    /// </summary>
     public SurrealOplogStore CreateOplogStore()
     {
         return new SurrealOplogStore(
@@ -371,6 +398,9 @@ internal sealed class SurrealTestHarness : IAsyncDisposable
             NullLogger<SurrealOplogStore>.Instance);
     }
 
+    /// <summary>
+    ///     Creates a peer configuration store instance bound to the test harness database.
+    /// </summary>
     public SurrealPeerConfigurationStore CreatePeerConfigurationStore()
     {
         return new SurrealPeerConfigurationStore(
@@ -379,6 +409,9 @@ internal sealed class SurrealTestHarness : IAsyncDisposable
             NullLogger<SurrealPeerConfigurationStore>.Instance);
     }
 
+    /// <summary>
+    ///     Creates a peer oplog confirmation store instance bound to the test harness database.
+    /// </summary>
     public SurrealPeerOplogConfirmationStore CreatePeerOplogConfirmationStore()
     {
         return new SurrealPeerOplogConfirmationStore(
@@ -387,6 +420,9 @@ internal sealed class SurrealTestHarness : IAsyncDisposable
             NullLogger<SurrealPeerOplogConfirmationStore>.Instance);
     }
 
+    /// <summary>
+    ///     Creates a snapshot metadata store instance bound to the test harness database.
+    /// </summary>
     public SurrealSnapshotMetadataStore CreateSnapshotMetadataStore()
     {
         return new SurrealSnapshotMetadataStore(
@@ -395,6 +431,7 @@ internal sealed class SurrealTestHarness : IAsyncDisposable
             NullLogger<SurrealSnapshotMetadataStore>.Instance);
     }
 
+    /// <inheritdoc />
     public async ValueTask DisposeAsync()
     {
         await _client.DisposeAsync();
@@ -421,11 +458,16 @@ internal sealed class TestSurrealSchemaInitializer : ICBDDCSurrealSchemaInitiali
     private readonly ICBDDCSurrealEmbeddedClient _client;
     private int _initialized;
 
+    /// <summary>
+    ///     Initializes a new instance of the <see cref="TestSurrealSchemaInitializer" /> class.
+    /// </summary>
+    /// <param name="client">The embedded client to initialize.</param>
     public TestSurrealSchemaInitializer(ICBDDCSurrealEmbeddedClient client)
     {
         _client = client;
     }
 
+    /// <inheritdoc />
     public async Task EnsureInitializedAsync(CancellationToken cancellationToken = default)
     {
         if (Interlocked.Exchange(ref _initialized, 1) == 1) return;