using System.Text.Json; using System.Reflection; using Microsoft.Extensions.Logging.Abstractions; using SurrealDb.Net; using SurrealDb.Net.Models.Response; using ZB.MOM.WW.CBDDC.Core; using ZB.MOM.WW.CBDDC.Core.Network; using ZB.MOM.WW.CBDDC.Core.Storage; using ZB.MOM.WW.CBDDC.Core.Sync; using ZB.MOM.WW.CBDDC.Persistence.Surreal; namespace ZB.MOM.WW.CBDDC.Sample.Console.Tests; public class SurrealCdcMatrixCompletionTests { /// /// Verifies retention-boundary classifier behavior across expected exception message patterns. /// /// The exception message sample. /// Expected classifier outcome. [Theory] [InlineData("versionstamp is outside the configured retention window", true)] [InlineData("change feed history since cursor is unavailable", true)] [InlineData("socket closed unexpectedly", false)] public void RetentionBoundaryClassifier_ShouldDetectExpectedPatterns(string message, bool expected) { var closedType = typeof(SurrealDocumentStore<>).MakeGenericType(typeof(object)); var classifier = closedType.GetMethod( "IsLikelyChangefeedRetentionBoundary", BindingFlags.NonPublic | BindingFlags.Static); classifier.ShouldNotBeNull(); bool actual = (bool)classifier!.Invoke(null, [new InvalidOperationException(message)])!; actual.ShouldBe(expected); } /// /// Verifies a local write produces exactly one oplog entry. /// [Fact] public async Task LocalWrite_ShouldEmitExactlyOneOplogEntry() { string dbPath = Path.Combine(Path.GetTempPath(), $"cbddc-cdc-matrix-{Guid.NewGuid():N}.rocksdb"); try { await using var harness = await CdcTestHarness.OpenWithRetriesAsync(dbPath, "node-single-write", "consumer-single"); await harness.Context.Users.InsertAsync(new User { Id = "single-write-user", Name = "Single Write", Age = 25, Address = new Address { City = "Bologna" } }); await harness.Context.SaveChangesAsync(); await harness.PollAsync(); await WaitForConditionAsync( async () => (await harness.GetEntriesByKeyAsync("Users", "single-write-user")).Count == 1, "Timed out waiting for exactly one local oplog entry."); var entries = await harness.GetEntriesByKeyAsync("Users", "single-write-user"); entries.Count.ShouldBe(1); entries[0].Operation.ShouldBe(OperationType.Put); entries[0].Timestamp.NodeId.ShouldBe("node-single-write"); } finally { await DeleteDirectoryWithRetriesAsync(dbPath); } } /// /// Verifies checkpoint persistence does not advance when atomic write fails. /// [Fact] public async Task Checkpoint_ShouldNotAdvance_WhenAtomicWriteFails() { var surrealClient = Substitute.For(); surrealClient.RawQuery( Arg.Any(), Arg.Any>(), Arg.Any()) .Returns(Task.FromException(new InvalidOperationException("forced atomic write failure"))); var embeddedClient = Substitute.For(); embeddedClient.Client.Returns(surrealClient); var schemaInitializer = Substitute.For(); schemaInitializer.EnsureInitializedAsync(Arg.Any()).Returns(Task.CompletedTask); var configProvider = Substitute.For(); configProvider.GetConfiguration().Returns(new PeerNodeConfiguration { NodeId = "node-failure", TcpPort = 0, AuthToken = "test-token" }); var checkpointPersistence = Substitute.For(); var vectorClock = Substitute.For(); vectorClock.GetLastHash(Arg.Any()).Returns("seed-hash"); var store = new FailureInjectedDocumentStore( embeddedClient, schemaInitializer, configProvider, vectorClock, checkpointPersistence); var payload = JsonSerializer.SerializeToElement(new { Id = "failure-user", Value = "x" }); await Should.ThrowAsync( () => store.TriggerLocalChangeAsync("Users", "failure-user", OperationType.Put, payload)); checkpointPersistence.ReceivedCalls().ShouldBeEmpty(); } private static async Task WaitForConditionAsync( Func> predicate, string failureMessage, int timeoutMs = 6000, int pollMs = 50) { DateTimeOffset deadline = DateTimeOffset.UtcNow.AddMilliseconds(timeoutMs); while (DateTimeOffset.UtcNow < deadline) { if (await predicate()) return; await Task.Delay(pollMs); } throw new TimeoutException(failureMessage); } private static async Task DeleteDirectoryWithRetriesAsync(string path) { for (var attempt = 0; attempt < 5; attempt++) try { if (Directory.Exists(path)) Directory.Delete(path, true); return; } catch when (attempt < 4) { await Task.Delay(50); } } } internal sealed class FailureInjectedDocumentStore : SurrealDocumentStore { /// /// Initializes a new instance of the class. /// /// The embedded Surreal client provider. /// The schema initializer. /// The node configuration provider. /// The vector clock service. /// The CDC checkpoint persistence dependency. public FailureInjectedDocumentStore( ICBDDCSurrealEmbeddedClient surrealEmbeddedClient, ICBDDCSurrealSchemaInitializer schemaInitializer, IPeerNodeConfigurationProvider configProvider, IVectorClockService vectorClockService, ISurrealCdcCheckpointPersistence checkpointPersistence) : base( new object(), surrealEmbeddedClient, schemaInitializer, configProvider, vectorClockService, new LastWriteWinsConflictResolver(), checkpointPersistence, new SurrealCdcPollingOptions { Enabled = false }, NullLogger.Instance) { } /// /// Triggers local change handling for testing failure scenarios. /// /// The collection name. /// The document key. /// The operation type. /// Optional document content payload. /// Cancellation token. /// A task that completes when processing is finished. public Task TriggerLocalChangeAsync( string collection, string key, OperationType operationType, JsonElement? content, CancellationToken cancellationToken = default) { return OnLocalChangeDetectedAsync( collection, key, operationType, content, pendingCursorCheckpoint: null, cancellationToken); } /// protected override Task ApplyContentToEntityAsync( string collection, string key, JsonElement content, CancellationToken cancellationToken) { return Task.CompletedTask; } /// protected override Task ApplyContentToEntitiesBatchAsync( IEnumerable<(string Collection, string Key, JsonElement Content)> documents, CancellationToken cancellationToken) { return Task.CompletedTask; } /// protected override Task GetEntityAsJsonAsync( string collection, string key, CancellationToken cancellationToken) { return Task.FromResult(null); } /// protected override Task RemoveEntityAsync(string collection, string key, CancellationToken cancellationToken) { return Task.CompletedTask; } /// protected override Task RemoveEntitiesBatchAsync( IEnumerable<(string Collection, string Key)> documents, CancellationToken cancellationToken) { return Task.CompletedTask; } /// protected override Task> GetAllEntitiesAsJsonAsync( string collection, CancellationToken cancellationToken) { return Task.FromResult>([]); } }