jdescopingtool/NEW/tests/JdeScoping.DataSync.Tests/TableSyncOperationTests.cs

using System.Diagnostics.Metrics;
using System.Linq.Expressions;
using System.Runtime.CompilerServices;
using JdeScoping.Core.Interfaces;
using JdeScoping.Core.Models;
using JdeScoping.Core.Models.Enums;
using JdeScoping.DataAccess.Interfaces;
using JdeScoping.DataSync.Options;
using JdeScoping.DataSync.Contracts;
using JdeScoping.DataSync.Models;
using JdeScoping.DataSync.Services;
using JdeScoping.DataSync.Telemetry;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using NSubstitute;
using NSubstitute.ExceptionExtensions;
using Shouldly;

namespace JdeScoping.DataSync.Tests;

/// <summary>
/// Unit tests for TableSyncOperation.
/// Tests mass/incremental paths, batching, and post-processor execution.
/// </summary>
public class TableSyncOperationTests
{
    private readonly IDbConnectionFactory _connectionFactory;
    private readonly IDataUpdateRepository _updateRepository;
    private readonly IBulkMergeHelper _bulkMergeHelper;
    private readonly IMergeConfigurationRegistry _configRegistry;
    private readonly IOptions<DataSyncOptions> _options;
    private readonly DataSyncMetrics _metrics;
    private readonly IServiceProvider _serviceProvider;

    public TableSyncOperationTests()
    {
        _connectionFactory = Substitute.For<IDbConnectionFactory>();
        _updateRepository = Substitute.For<IDataUpdateRepository>();
        _bulkMergeHelper = Substitute.For<IBulkMergeHelper>();
        _configRegistry = Substitute.For<IMergeConfigurationRegistry>();

        _options = Microsoft.Extensions.Options.Options.Create(new DataSyncOptions
        {
            BatchSize = 1000,
            BulkCopyBatchSize = 100
        });

        var services = new ServiceCollection();
        services.AddMetrics();
        var provider = services.BuildServiceProvider();
        var meterFactory = provider.GetRequiredService<IMeterFactory>();
        _metrics = new DataSyncMetrics(meterFactory);

        _serviceProvider = Substitute.For<IServiceProvider>();
    }

    #region Update Logging Tests

    [Fact]
    public async Task ExecuteAsync_StartsUpdateWithInProgressMarker()
    {
        // Arrange
        var task = CreateTask("WorkOrder", UpdateTypes.Mass);

        _updateRepository.StartUpdateAsync(
            Arg.Any<string>(),
            Arg.Any<string>(),
            Arg.Any<string>(),
            Arg.Any<UpdateTypes>(),
            Arg.Any<CancellationToken>())
            .Returns(123);

        SetupMockFetcher(task, AsyncEnumerable.Empty<TestEntity>());
        SetupMockMergeConfiguration();

        _bulkMergeHelper.MassInsertAsync(
            Arg.Any<IAsyncEnumerable<TestEntity>>(),
            Arg.Any<string>(),
            Arg.Any<bool>(),
            Arg.Any<int>(),
            Arg.Any<CancellationToken>())
            .Returns(new MassInsertResult(0, TimeSpan.Zero, true));

        var sut = CreateSut();

        // Act
        await sut.ExecuteAsync(task);

        // Assert
        await _updateRepository.Received(1).StartUpdateAsync(
            task.SourceSystem,
            task.SourceData,
            task.TableName,
            task.UpdateType,
            Arg.Any<CancellationToken>());
    }

    [Fact]
    public async Task ExecuteAsync_OnSuccess_CompletesUpdateWithRecordCount()
    {
        // Arrange
        var task = CreateTask("WorkOrder", UpdateTypes.Mass);

        _updateRepository.StartUpdateAsync(
            Arg.Any<string>(),
            Arg.Any<string>(),
            Arg.Any<string>(),
            Arg.Any<UpdateTypes>(),
            Arg.Any<CancellationToken>())
            .Returns(123);

        SetupMockFetcher(task, AsyncEnumerable.Empty<TestEntity>());
        SetupMockMergeConfiguration();

        _bulkMergeHelper.MassInsertAsync(
            Arg.Any<IAsyncEnumerable<TestEntity>>(),
            Arg.Any<string>(),
            Arg.Any<bool>(),
            Arg.Any<int>(),
            Arg.Any<CancellationToken>())
            .Returns(new MassInsertResult(500, TimeSpan.Zero, true));

        var sut = CreateSut();

        // Act
        await sut.ExecuteAsync(task);

        // Assert
        await _updateRepository.Received(1).CompleteUpdateAsync(
            123,
            500L,
            true,
            Arg.Any<CancellationToken>());
    }

    [Fact]
    public async Task ExecuteAsync_OnFailure_CompletesUpdateWithFailureMarker()
    {
        // Arrange
        var task = CreateTask("WorkOrder", UpdateTypes.Mass);

        _updateRepository.StartUpdateAsync(
            Arg.Any<string>(),
            Arg.Any<string>(),
            Arg.Any<string>(),
            Arg.Any<UpdateTypes>(),
            Arg.Any<CancellationToken>())
            .Returns(123);

        SetupMockFetcher(task, AsyncEnumerable.Empty<TestEntity>());
        SetupMockMergeConfiguration();

        _bulkMergeHelper.MassInsertAsync(
            Arg.Any<IAsyncEnumerable<TestEntity>>(),
            Arg.Any<string>(),
            Arg.Any<bool>(),
            Arg.Any<int>(),
            Arg.Any<CancellationToken>())
            .ThrowsAsync(new Exception("Database error"));

        var sut = CreateSut();

        // Act & Assert
        await Should.ThrowAsync<Exception>(() => sut.ExecuteAsync(task));

        // Verify update was marked as failed
        await _updateRepository.Received(1).CompleteUpdateAsync(
            123,
            -1,
            false,
            Arg.Any<CancellationToken>());
    }

    [Fact]
    public async Task ExecuteAsync_OnCancellation_MarksUpdateAsFailed()
    {
        // Arrange
        var task = CreateTask("WorkOrder", UpdateTypes.Mass);
        var cts = new CancellationTokenSource();

        _updateRepository.StartUpdateAsync(
            Arg.Any<string>(),
            Arg.Any<string>(),
            Arg.Any<string>(),
            Arg.Any<UpdateTypes>(),
            Arg.Any<CancellationToken>())
            .Returns(123);

        SetupMockFetcher(task, AsyncEnumerable.Empty<TestEntity>());
        SetupMockMergeConfiguration();

        _bulkMergeHelper.MassInsertAsync(
            Arg.Any<IAsyncEnumerable<TestEntity>>(),
            Arg.Any<string>(),
            Arg.Any<bool>(),
            Arg.Any<int>(),
            Arg.Any<CancellationToken>())
            .Returns(async callInfo =>
            {
                cts.Cancel();
                callInfo.Arg<CancellationToken>().ThrowIfCancellationRequested();
                return new MassInsertResult(0, TimeSpan.Zero, true);
            });

        var sut = CreateSut();

        // Act & Assert
        await Should.ThrowAsync<OperationCanceledException>(() => sut.ExecuteAsync(task, cts.Token));

        // Verify update was marked as failed
        await _updateRepository.Received(1).CompleteUpdateAsync(
            123,
            -1,
            false,
            Arg.Any<CancellationToken>());
    }

    #endregion

    #region Mass Update Path Tests

    [Fact]
    public async Task ExecuteAsync_MassWithPrepurge_UsesMassUpdatePath()
    {
        // Arrange
        var task = CreateTask("WorkOrder", UpdateTypes.Mass, prepurge: true);

        _updateRepository.StartUpdateAsync(
            Arg.Any<string>(),
            Arg.Any<string>(),
            Arg.Any<string>(),
            Arg.Any<UpdateTypes>(),
            Arg.Any<CancellationToken>())
            .Returns(1);

        SetupMockFetcher(task, AsyncEnumerable.Empty<TestEntity>());
        SetupMockMergeConfiguration();

        _bulkMergeHelper.MassInsertAsync(
            Arg.Any<IAsyncEnumerable<TestEntity>>(),
            Arg.Any<string>(),
            Arg.Any<bool>(),
            Arg.Any<int>(),
            Arg.Any<CancellationToken>())
            .Returns(new MassInsertResult(100, TimeSpan.Zero, true));

        var sut = CreateSut();

        // Act
        await sut.ExecuteAsync(task);

        // Assert: Should use mass insert path
        await _bulkMergeHelper.Received(1).MassInsertAsync(
            Arg.Any<IAsyncEnumerable<TestEntity>>(),
            "TestTable",
            task.ScheduleConfig.ReIndexData,
            _options.Value.BulkCopyBatchSize,
            Arg.Any<CancellationToken>());

        // Should NOT use merge path
        await _bulkMergeHelper.DidNotReceive().MergeAsync(
            Arg.Any<IAsyncEnumerable<TestEntity>>(),
            Arg.Any<string>(),
            Arg.Any<Expression<Func<TestEntity, object>>>(),
            Arg.Any<Expression<Func<TestEntity, object>>>(),
            Arg.Any<Expression<Func<TestEntity, TestEntity, bool>>>(),
            Arg.Any<Expression<Func<TestEntity, object>>>(),
            Arg.Any<string>(),
            Arg.Any<int>(),
            Arg.Any<bool>(),
            Arg.Any<CancellationToken>());
    }

    #endregion

    #region Incremental Update Path Tests

    // Note: The following tests are marked as integration tests because they require
    // complex reflection-based fetcher resolution that's difficult to unit test.
    // These scenarios should be covered by integration tests with a real test database.
    //
    // Scenarios covered by integration tests:
    // - ExecuteAsync_DailyUpdate_UsesIncrementalPath
    // - ExecuteAsync_HourlyUpdate_UsesIncrementalPath
    // - ExecuteAsync_LargeDataset_ProcessesInBatches (incremental path)

    [Fact]
    public void IncrementalUpdatePath_RequiresIntegrationTest()
    {
        // This test documents that incremental update scenarios require integration testing
        // because the TableSyncOperation uses reflection to resolve and invoke fetchers.
        //
        // Integration test should verify:
        // 1. Daily updates use staging table → merge path
        // 2. Hourly updates use staging table → merge path
        // 3. Staging tables are created with unique suffixes
        // 4. MERGE correctly handles INSERT/UPDATE based on LastUpdateDT
        // 5. Staging tables are cleaned up after success and failure
        Assert.True(true, "See integration tests for incremental update path coverage");
    }

    #endregion

    #region Batching Tests

    // Note: Batching tests for incremental updates require integration testing
    // because they depend on the reflection-based fetcher resolution.
    // The batching logic is tested implicitly through integration tests.
    //
    // Test scenario to verify:
    // - Large dataset (25 entities) with BatchSize=10 should create 3 batches

    #endregion

    #region Post-Processor Tests

    [Fact]
    public async Task ExecuteAsync_WithPostProcessor_InvokesPostProcessor()
    {
        // Arrange
        var task = CreateTask("MisData", UpdateTypes.Mass, postProcessor: nameof(MockPostProcessor));

        _updateRepository.StartUpdateAsync(
            Arg.Any<string>(),
            Arg.Any<string>(),
            Arg.Any<string>(),
            Arg.Any<UpdateTypes>(),
            Arg.Any<CancellationToken>())
            .Returns(1);

        SetupMockFetcher(task, AsyncEnumerable.Empty<TestEntity>());
        SetupMockMergeConfiguration();

        _bulkMergeHelper.MassInsertAsync(
            Arg.Any<IAsyncEnumerable<TestEntity>>(),
            Arg.Any<string>(),
            Arg.Any<bool>(),
            Arg.Any<int>(),
            Arg.Any<CancellationToken>())
            .Returns(new MassInsertResult(0, TimeSpan.Zero, true));

        var mockPostProcessor = new MockPostProcessor();
        _serviceProvider.GetService(typeof(MockPostProcessor)).Returns(mockPostProcessor);

        var sut = CreateSut();

        // Act
        await sut.ExecuteAsync(task);

        // Assert
        mockPostProcessor.WasInvoked.ShouldBeTrue();
        mockPostProcessor.TableNameReceived.ShouldBe("MisData");
    }

    [Fact]
    public async Task ExecuteAsync_WithoutPostProcessor_SkipsPostProcessing()
    {
        // Arrange
        var task = CreateTask("WorkOrder", UpdateTypes.Mass, postProcessor: null);

        _updateRepository.StartUpdateAsync(
            Arg.Any<string>(),
            Arg.Any<string>(),
            Arg.Any<string>(),
            Arg.Any<UpdateTypes>(),
            Arg.Any<CancellationToken>())
            .Returns(1);

        SetupMockFetcher(task, AsyncEnumerable.Empty<TestEntity>());
        SetupMockMergeConfiguration();

        _bulkMergeHelper.MassInsertAsync(
            Arg.Any<IAsyncEnumerable<TestEntity>>(),
            Arg.Any<string>(),
            Arg.Any<bool>(),
            Arg.Any<int>(),
            Arg.Any<CancellationToken>())
            .Returns(new MassInsertResult(0, TimeSpan.Zero, true));

        var sut = CreateSut();

        // Act
        await sut.ExecuteAsync(task);

        // Assert: No post-processor service resolution should occur
        _serviceProvider.DidNotReceive().GetService(typeof(IPostProcessor));
    }

    #endregion

    #region Metrics Tests

    [Fact]
    public async Task ExecuteAsync_RecordsOperationStartedMetric()
    {
        // Arrange
        var task = CreateTask("WorkOrder", UpdateTypes.Mass);

        _updateRepository.StartUpdateAsync(
            Arg.Any<string>(),
            Arg.Any<string>(),
            Arg.Any<string>(),
            Arg.Any<UpdateTypes>(),
            Arg.Any<CancellationToken>())
            .Returns(1);

        SetupMockFetcher(task, AsyncEnumerable.Empty<TestEntity>());
        SetupMockMergeConfiguration();

        _bulkMergeHelper.MassInsertAsync(
            Arg.Any<IAsyncEnumerable<TestEntity>>(),
            Arg.Any<string>(),
            Arg.Any<bool>(),
            Arg.Any<int>(),
            Arg.Any<CancellationToken>())
            .Returns(new MassInsertResult(100, TimeSpan.Zero, true));

        var sut = CreateSut();

        // Act
        await sut.ExecuteAsync(task);

        // Assert: Metrics were recorded (using real metrics, just verify no exceptions)
        // Detailed metric verification is in DataSyncMetricsTests
    }

    #endregion

    #region Staging Table Cleanup Tests

    // Note: Staging table cleanup tests require integration testing because
    // they depend on the incremental update path with reflection-based fetcher resolution.
    //
    // Test scenarios to verify:
    // - On successful merge, staging table is dropped
    // - On merge failure, staging table is still dropped (finally block)
    // - On bulk copy failure, staging table is still dropped

    #endregion

    #region Helper Methods

    private TableSyncOperation CreateSut()
    {
        return new TableSyncOperation(
            _serviceProvider,
            _connectionFactory,
            _updateRepository,
            _bulkMergeHelper,
            _configRegistry,
            _options,
            NullLogger<TableSyncOperation>.Instance,
            _metrics);
    }

    private static DataUpdateTask CreateTask(
        string tableName,
        UpdateTypes updateType,
        bool prepurge = true,
        bool reindex = true,
        string? postProcessor = null)
    {
        return new DataUpdateTask
        {
            TableName = tableName,
            SourceSystem = "JDE",
            SourceData = tableName.ToUpper(),
            UpdateType = updateType,
            MinimumDt = updateType == UpdateTypes.Mass ? null : DateTime.UtcNow.AddDays(-1),
            Config = new DataSourceConfig
            {
                TableName = tableName,
                SourceSystem = "JDE",
                SourceData = tableName.ToUpper(),
                FetcherTypeName = nameof(MockDataFetcher),
                PostProcessorTypeName = postProcessor,
                IsEnabled = true,
                MassConfig = new ScheduleConfig
                {
                    Enabled = true,
                    IntervalMinutes = 10080,
                    PrepurgeData = prepurge,
                    ReIndexData = reindex
                },
                DailyConfig = new ScheduleConfig { Enabled = true, IntervalMinutes = 1440 },
                HourlyConfig = new ScheduleConfig { Enabled = true, IntervalMinutes = 60 }
            }
        };
    }

    private void SetupMockFetcher(DataUpdateTask task, IAsyncEnumerable<TestEntity> entities, int batchSize = 1000)
    {
        var fetcher = new MockDataFetcher(entities);
        _serviceProvider.GetService(typeof(MockDataFetcher)).Returns(fetcher);
    }

    private void SetupMockMergeConfiguration()
    {
        var mockConfig = Substitute.For<IMergeConfiguration<TestEntity>>();
        mockConfig.TableName.Returns("TestTable");
        mockConfig.MatchOn.Returns(x => x.Id);
        mockConfig.UpdateColumns.Returns(x => new { x.Name, x.LastUpdateDT });
        mockConfig.UpdateWhen.Returns((Expression<Func<TestEntity, TestEntity, bool>>?)null);
        mockConfig.InsertColumns.Returns((Expression<Func<TestEntity, object>>?)null);

        _configRegistry.GetConfiguration<TestEntity>().Returns(mockConfig);
    }

    #endregion
}

#region Test Support Classes

public class TestEntity
{
    public int Id { get; set; }
    public string Name { get; set; } = string.Empty;
    public DateTime LastUpdateDT { get; set; } = DateTime.UtcNow;
}

public class MockDataFetcher : IDataFetcher<TestEntity>
{
    private readonly IAsyncEnumerable<TestEntity> _entities;

    public MockDataFetcher(IAsyncEnumerable<TestEntity>? entities = null)
    {
        _entities = entities ?? AsyncEnumerable.Empty<TestEntity>();
    }

    public IAsyncEnumerable<TestEntity> FetchAsync(DateTime? minimumDt, CancellationToken cancellationToken = default)
    {
        return _entities;
    }
}

public class MockPostProcessor : IPostProcessor
{
    public bool WasInvoked { get; private set; }
    public string? TableNameReceived { get; private set; }

    public Task ProcessAsync(string tableName, CancellationToken cancellationToken = default)
    {
        WasInvoked = true;
        TableNameReceived = tableName;
        return Task.CompletedTask;
    }
}

#endregion