CBDDC/tests/ZB.MOM.WW.CBDDC.Sample.Console.Tests/SurrealCdcMatrixCompletionTests.cs

using System.Text.Json;
using System.Reflection;
using Microsoft.Extensions.Logging.Abstractions;
using SurrealDb.Net;
using SurrealDb.Net.Models.Response;
using ZB.MOM.WW.CBDDC.Core;
using ZB.MOM.WW.CBDDC.Core.Network;
using ZB.MOM.WW.CBDDC.Core.Storage;
using ZB.MOM.WW.CBDDC.Core.Sync;
using ZB.MOM.WW.CBDDC.Persistence.Surreal;

namespace ZB.MOM.WW.CBDDC.Sample.Console.Tests;

public class SurrealCdcMatrixCompletionTests
{
    /// <summary>
    ///     Verifies retention-boundary classifier behavior across expected exception message patterns.
    /// </summary>
    /// <param name="message">The exception message sample.</param>
    /// <param name="expected">Expected classifier outcome.</param>
    [Theory]
    [InlineData("versionstamp is outside the configured retention window", true)]
    [InlineData("change feed history since cursor is unavailable", true)]
    [InlineData("socket closed unexpectedly", false)]
    public void RetentionBoundaryClassifier_ShouldDetectExpectedPatterns(string message, bool expected)
    {
        var closedType = typeof(SurrealDocumentStore<>).MakeGenericType(typeof(object));
        var classifier = closedType.GetMethod(
            "IsLikelyChangefeedRetentionBoundary",
            BindingFlags.NonPublic | BindingFlags.Static);

        classifier.ShouldNotBeNull();
        bool actual = (bool)classifier!.Invoke(null, [new InvalidOperationException(message)])!;
        actual.ShouldBe(expected);
    }

    /// <summary>
    ///     Verifies a local write produces exactly one oplog entry.
    /// </summary>
    [Fact]
    public async Task LocalWrite_ShouldEmitExactlyOneOplogEntry()
    {
        string dbPath = Path.Combine(Path.GetTempPath(), $"cbddc-cdc-matrix-{Guid.NewGuid():N}.rocksdb");

        try
        {
            await using var harness = await CdcTestHarness.OpenWithRetriesAsync(dbPath, "node-single-write", "consumer-single");

            await harness.Context.Users.InsertAsync(new User
            {
                Id = "single-write-user",
                Name = "Single Write",
                Age = 25,
                Address = new Address { City = "Bologna" }
            });
            await harness.Context.SaveChangesAsync();
            await harness.PollAsync();

            await WaitForConditionAsync(
                async () => (await harness.GetEntriesByKeyAsync("Users", "single-write-user")).Count == 1,
                "Timed out waiting for exactly one local oplog entry.");

            var entries = await harness.GetEntriesByKeyAsync("Users", "single-write-user");
            entries.Count.ShouldBe(1);
            entries[0].Operation.ShouldBe(OperationType.Put);
            entries[0].Timestamp.NodeId.ShouldBe("node-single-write");
        }
        finally
        {
            await DeleteDirectoryWithRetriesAsync(dbPath);
        }
    }

    /// <summary>
    ///     Verifies checkpoint persistence does not advance when atomic write fails.
    /// </summary>
    [Fact]
    public async Task Checkpoint_ShouldNotAdvance_WhenAtomicWriteFails()
    {
        var surrealClient = Substitute.For<ISurrealDbClient>();
        surrealClient.RawQuery(
                Arg.Any<string>(),
                Arg.Any<IReadOnlyDictionary<string, object?>>(),
                Arg.Any<CancellationToken>())
            .Returns(Task.FromException<SurrealDbResponse>(new InvalidOperationException("forced atomic write failure")));

        var embeddedClient = Substitute.For<ICBDDCSurrealEmbeddedClient>();
        embeddedClient.Client.Returns(surrealClient);

        var schemaInitializer = Substitute.For<ICBDDCSurrealSchemaInitializer>();
        schemaInitializer.EnsureInitializedAsync(Arg.Any<CancellationToken>()).Returns(Task.CompletedTask);

        var configProvider = Substitute.For<IPeerNodeConfigurationProvider>();
        configProvider.GetConfiguration().Returns(new PeerNodeConfiguration
        {
            NodeId = "node-failure",
            TcpPort = 0,
            AuthToken = "test-token"
        });

        var checkpointPersistence = Substitute.For<ISurrealCdcCheckpointPersistence>();
        var vectorClock = Substitute.For<IVectorClockService>();
        vectorClock.GetLastHash(Arg.Any<string>()).Returns("seed-hash");

        var store = new FailureInjectedDocumentStore(
            embeddedClient,
            schemaInitializer,
            configProvider,
            vectorClock,
            checkpointPersistence);

        var payload = JsonSerializer.SerializeToElement(new { Id = "failure-user", Value = "x" });

        await Should.ThrowAsync<InvalidOperationException>(
            () => store.TriggerLocalChangeAsync("Users", "failure-user", OperationType.Put, payload));

        checkpointPersistence.ReceivedCalls().ShouldBeEmpty();
    }

    private static async Task WaitForConditionAsync(
        Func<Task<bool>> predicate,
        string failureMessage,
        int timeoutMs = 6000,
        int pollMs = 50)
    {
        DateTimeOffset deadline = DateTimeOffset.UtcNow.AddMilliseconds(timeoutMs);
        while (DateTimeOffset.UtcNow < deadline)
        {
            if (await predicate()) return;
            await Task.Delay(pollMs);
        }

        throw new TimeoutException(failureMessage);
    }

    private static async Task DeleteDirectoryWithRetriesAsync(string path)
    {
        for (var attempt = 0; attempt < 5; attempt++)
            try
            {
                if (Directory.Exists(path)) Directory.Delete(path, true);
                return;
            }
            catch when (attempt < 4)
            {
                await Task.Delay(50);
            }
    }
}

internal sealed class FailureInjectedDocumentStore : SurrealDocumentStore<object>
{
    /// <summary>
    ///     Initializes a new instance of the <see cref="FailureInjectedDocumentStore" /> class.
    /// </summary>
    /// <param name="surrealEmbeddedClient">The embedded Surreal client provider.</param>
    /// <param name="schemaInitializer">The schema initializer.</param>
    /// <param name="configProvider">The node configuration provider.</param>
    /// <param name="vectorClockService">The vector clock service.</param>
    /// <param name="checkpointPersistence">The CDC checkpoint persistence dependency.</param>
    public FailureInjectedDocumentStore(
        ICBDDCSurrealEmbeddedClient surrealEmbeddedClient,
        ICBDDCSurrealSchemaInitializer schemaInitializer,
        IPeerNodeConfigurationProvider configProvider,
        IVectorClockService vectorClockService,
        ISurrealCdcCheckpointPersistence checkpointPersistence)
        : base(
            new object(),
            surrealEmbeddedClient,
            schemaInitializer,
            configProvider,
            vectorClockService,
            new LastWriteWinsConflictResolver(),
            checkpointPersistence,
            new SurrealCdcPollingOptions { Enabled = false },
            NullLogger<FailureInjectedDocumentStore>.Instance)
    {
    }

    /// <summary>
    ///     Triggers local change handling for testing failure scenarios.
    /// </summary>
    /// <param name="collection">The collection name.</param>
    /// <param name="key">The document key.</param>
    /// <param name="operationType">The operation type.</param>
    /// <param name="content">Optional document content payload.</param>
    /// <param name="cancellationToken">Cancellation token.</param>
    /// <returns>A task that completes when processing is finished.</returns>
    public Task TriggerLocalChangeAsync(
        string collection,
        string key,
        OperationType operationType,
        JsonElement? content,
        CancellationToken cancellationToken = default)
    {
        return OnLocalChangeDetectedAsync(
            collection,
            key,
            operationType,
            content,
            pendingCursorCheckpoint: null,
            cancellationToken);
    }

    /// <inheritdoc />
    protected override Task ApplyContentToEntityAsync(
        string collection,
        string key,
        JsonElement content,
        CancellationToken cancellationToken)
    {
        return Task.CompletedTask;
    }

    /// <inheritdoc />
    protected override Task ApplyContentToEntitiesBatchAsync(
        IEnumerable<(string Collection, string Key, JsonElement Content)> documents,
        CancellationToken cancellationToken)
    {
        return Task.CompletedTask;
    }

    /// <inheritdoc />
    protected override Task<JsonElement?> GetEntityAsJsonAsync(
        string collection,
        string key,
        CancellationToken cancellationToken)
    {
        return Task.FromResult<JsonElement?>(null);
    }

    /// <inheritdoc />
    protected override Task RemoveEntityAsync(string collection, string key, CancellationToken cancellationToken)
    {
        return Task.CompletedTask;
    }

    /// <inheritdoc />
    protected override Task RemoveEntitiesBatchAsync(
        IEnumerable<(string Collection, string Key)> documents,
        CancellationToken cancellationToken)
    {
        return Task.CompletedTask;
    }

    /// <inheritdoc />
    protected override Task<IEnumerable<(string Key, JsonElement Content)>> GetAllEntitiesAsJsonAsync(
        string collection,
        CancellationToken cancellationToken)
    {
        return Task.FromResult<IEnumerable<(string Key, JsonElement Content)>>([]);
    }
}