jdescopingtool/NEW/tests/JdeScoping.DataSync.Tests/SyncOrchestratorTests.cs

using System.Diagnostics.Metrics;
using JdeScoping.Core.Models.Enums;
using JdeScoping.DataSync.Options;
using JdeScoping.DataSync.Contracts;
using JdeScoping.DataSync.Models;
using JdeScoping.DataSync.Services;
using JdeScoping.DataSync.Telemetry;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using NSubstitute;
using NSubstitute.ExceptionExtensions;
using Shouldly;

namespace JdeScoping.DataSync.Tests;

/// <summary>
/// Unit tests for SyncOrchestrator.
/// Tests parallel execution, cancellation, and scope isolation.
/// </summary>
public class SyncOrchestratorTests
{
    private readonly IScheduleChecker _scheduleChecker;
    private readonly IOptions<DataSyncOptions> _options;
    private readonly DataSyncMetrics _metrics;

    public SyncOrchestratorTests()
    {
        _scheduleChecker = Substitute.For<IScheduleChecker>();
        _options = Microsoft.Extensions.Options.Options.Create(new DataSyncOptions
        {
            MaxDegreeOfParallelism = 4
        });

        var services = new ServiceCollection();
        services.AddMetrics();
        var provider = services.BuildServiceProvider();
        var meterFactory = provider.GetRequiredService<IMeterFactory>();
        _metrics = new DataSyncMetrics(meterFactory);
    }

    #region No Pending Tasks

    [Fact]
    public async Task ExecutePendingSyncsAsync_NoPendingTasks_ReturnsImmediately()
    {
        // Arrange
        _scheduleChecker.GetPendingTasksAsync(Arg.Any<CancellationToken>())
            .Returns(new List<DataUpdateTask>());

        var operation = Substitute.For<ITableSyncOperation>();
        var services = new ServiceCollection();
        services.AddSingleton(_scheduleChecker);
        services.AddScoped(_ => operation);
        var serviceProvider = services.BuildServiceProvider();
        var scopeFactory = serviceProvider.GetRequiredService<IServiceScopeFactory>();

        var sut = new SyncOrchestrator(
            scopeFactory,
            _scheduleChecker,
            _options,
            NullLogger<SyncOrchestrator>.Instance,
            _metrics);

        // Act
        await sut.ExecutePendingSyncsAsync();

        // Assert: Operation should never be called
        await operation.DidNotReceive().ExecuteAsync(Arg.Any<DataUpdateTask>(), Arg.Any<CancellationToken>());
    }

    #endregion

    #region Single Task Execution

    [Fact]
    public async Task ExecutePendingSyncsAsync_SingleTask_ExecutesOperation()
    {
        // Arrange
        var task = CreateTask("WorkOrder", UpdateTypes.Mass);
        _scheduleChecker.GetPendingTasksAsync(Arg.Any<CancellationToken>())
            .Returns(new List<DataUpdateTask> { task });

        var executedTasks = new List<string>();
        var operation = Substitute.For<ITableSyncOperation>();
        operation.ExecuteAsync(Arg.Any<DataUpdateTask>(), Arg.Any<CancellationToken>())
            .Returns(callInfo =>
            {
                var t = callInfo.Arg<DataUpdateTask>();
                executedTasks.Add(t.TableName);
                return Task.CompletedTask;
            });

        var services = new ServiceCollection();
        services.AddSingleton(_scheduleChecker);
        services.AddScoped(_ => operation);
        var serviceProvider = services.BuildServiceProvider();
        var scopeFactory = serviceProvider.GetRequiredService<IServiceScopeFactory>();

        var sut = new SyncOrchestrator(
            scopeFactory,
            _scheduleChecker,
            _options,
            NullLogger<SyncOrchestrator>.Instance,
            _metrics);

        // Act
        await sut.ExecutePendingSyncsAsync();

        // Assert
        executedTasks.ShouldContain("WorkOrder");
    }

    #endregion

    #region Parallel Execution

    [Fact]
    public async Task ExecutePendingSyncsAsync_MultipleTasks_ExecutesInParallel()
    {
        // Arrange
        var tasks = new List<DataUpdateTask>
        {
            CreateTask("WorkOrder", UpdateTypes.Mass),
            CreateTask("LotUsage", UpdateTypes.Mass),
            CreateTask("Item", UpdateTypes.Mass),
            CreateTask("Lot", UpdateTypes.Mass)
        };

        _scheduleChecker.GetPendingTasksAsync(Arg.Any<CancellationToken>())
            .Returns(tasks);

        var executingConcurrently = 0;
        var maxConcurrent = 0;
        var lockObj = new object();

        var operation = Substitute.For<ITableSyncOperation>();
        operation.ExecuteAsync(Arg.Any<DataUpdateTask>(), Arg.Any<CancellationToken>())
            .Returns(async callInfo =>
            {
                lock (lockObj)
                {
                    executingConcurrently++;
                    maxConcurrent = Math.Max(maxConcurrent, executingConcurrently);
                }

                await Task.Delay(50); // Simulate work

                lock (lockObj)
                {
                    executingConcurrently--;
                }
            });

        var services = new ServiceCollection();
        services.AddSingleton(_scheduleChecker);
        services.AddScoped(_ => operation);
        var serviceProvider = services.BuildServiceProvider();
        var scopeFactory = serviceProvider.GetRequiredService<IServiceScopeFactory>();

        var sut = new SyncOrchestrator(
            scopeFactory,
            _scheduleChecker,
            _options,
            NullLogger<SyncOrchestrator>.Instance,
            _metrics);

        // Act
        await sut.ExecutePendingSyncsAsync();

        // Assert: Should have executed multiple tasks concurrently
        maxConcurrent.ShouldBeGreaterThan(1);
    }

    [Fact]
    public async Task ExecutePendingSyncsAsync_RespectsMaxDegreeOfParallelism()
    {
        // Arrange: Create 10 tasks but limit parallelism to 2
        var options = Microsoft.Extensions.Options.Options.Create(new DataSyncOptions
        {
            MaxDegreeOfParallelism = 2
        });

        var tasks = Enumerable.Range(0, 10)
            .Select(i => CreateTask($"Table{i}", UpdateTypes.Mass))
            .ToList();

        _scheduleChecker.GetPendingTasksAsync(Arg.Any<CancellationToken>())
            .Returns(tasks);

        var executingConcurrently = 0;
        var maxConcurrent = 0;
        var lockObj = new object();

        var operation = Substitute.For<ITableSyncOperation>();
        operation.ExecuteAsync(Arg.Any<DataUpdateTask>(), Arg.Any<CancellationToken>())
            .Returns(async callInfo =>
            {
                lock (lockObj)
                {
                    executingConcurrently++;
                    maxConcurrent = Math.Max(maxConcurrent, executingConcurrently);
                }

                await Task.Delay(50);

                lock (lockObj)
                {
                    executingConcurrently--;
                }
            });

        var services = new ServiceCollection();
        services.AddSingleton(_scheduleChecker);
        services.AddScoped(_ => operation);
        var serviceProvider = services.BuildServiceProvider();
        var scopeFactory = serviceProvider.GetRequiredService<IServiceScopeFactory>();

        var sut = new SyncOrchestrator(
            scopeFactory,
            _scheduleChecker,
            options,
            NullLogger<SyncOrchestrator>.Instance,
            _metrics);

        // Act
        await sut.ExecutePendingSyncsAsync();

        // Assert: Should not exceed MaxDegreeOfParallelism
        maxConcurrent.ShouldBeLessThanOrEqualTo(2);
    }

    #endregion

    #region Scope Isolation

    [Fact]
    public async Task ExecutePendingSyncsAsync_EachTaskGetsOwnScope()
    {
        // Arrange
        var tasks = new List<DataUpdateTask>
        {
            CreateTask("WorkOrder", UpdateTypes.Mass),
            CreateTask("LotUsage", UpdateTypes.Mass)
        };

        _scheduleChecker.GetPendingTasksAsync(Arg.Any<CancellationToken>())
            .Returns(tasks);

        var scopeCount = 0;
        var services = new ServiceCollection();
        services.AddSingleton(_scheduleChecker);
        services.AddScoped<ITableSyncOperation>(sp =>
        {
            Interlocked.Increment(ref scopeCount);
            var mock = Substitute.For<ITableSyncOperation>();
            mock.ExecuteAsync(Arg.Any<DataUpdateTask>(), Arg.Any<CancellationToken>())
                .Returns(Task.CompletedTask);
            return mock;
        });
        var serviceProvider = services.BuildServiceProvider();
        var scopeFactory = serviceProvider.GetRequiredService<IServiceScopeFactory>();

        var sut = new SyncOrchestrator(
            scopeFactory,
            _scheduleChecker,
            _options,
            NullLogger<SyncOrchestrator>.Instance,
            _metrics);

        // Act
        await sut.ExecutePendingSyncsAsync();

        // Assert: Each task should create its own scope
        scopeCount.ShouldBe(2);
    }

    #endregion

    #region Cancellation Handling

    [Fact]
    public async Task ExecutePendingSyncsAsync_WhenCancelled_PropagatesCancellation()
    {
        // Arrange
        var tasks = new List<DataUpdateTask>
        {
            CreateTask("WorkOrder", UpdateTypes.Mass),
            CreateTask("LotUsage", UpdateTypes.Mass),
            CreateTask("Item", UpdateTypes.Mass)
        };

        _scheduleChecker.GetPendingTasksAsync(Arg.Any<CancellationToken>())
            .Returns(tasks);

        var cts = new CancellationTokenSource();
        var operationsStarted = 0;

        var operation = Substitute.For<ITableSyncOperation>();
        operation.ExecuteAsync(Arg.Any<DataUpdateTask>(), Arg.Any<CancellationToken>())
            .Returns(async callInfo =>
            {
                Interlocked.Increment(ref operationsStarted);

                // Cancel after first operation starts
                if (operationsStarted == 1)
                {
                    cts.Cancel();
                }

                var token = callInfo.Arg<CancellationToken>();
                await Task.Delay(500, token); // Will throw if cancelled
            });

        var services = new ServiceCollection();
        services.AddSingleton(_scheduleChecker);
        services.AddScoped(_ => operation);
        var serviceProvider = services.BuildServiceProvider();
        var scopeFactory = serviceProvider.GetRequiredService<IServiceScopeFactory>();

        var sut = new SyncOrchestrator(
            scopeFactory,
            _scheduleChecker,
            _options,
            NullLogger<SyncOrchestrator>.Instance,
            _metrics);

        // Act & Assert
        await Should.ThrowAsync<OperationCanceledException>(
            () => sut.ExecutePendingSyncsAsync(cts.Token));
    }

    [Fact]
    public async Task ExecutePendingSyncsAsync_PassesCancellationTokenToOperations()
    {
        // Arrange
        var task = CreateTask("WorkOrder", UpdateTypes.Mass);
        _scheduleChecker.GetPendingTasksAsync(Arg.Any<CancellationToken>())
            .Returns(new List<DataUpdateTask> { task });

        CancellationToken receivedToken = default;
        var operation = Substitute.For<ITableSyncOperation>();
        operation.ExecuteAsync(Arg.Any<DataUpdateTask>(), Arg.Any<CancellationToken>())
            .Returns(callInfo =>
            {
                receivedToken = callInfo.Arg<CancellationToken>();
                return Task.CompletedTask;
            });

        var services = new ServiceCollection();
        services.AddSingleton(_scheduleChecker);
        services.AddScoped(_ => operation);
        var serviceProvider = services.BuildServiceProvider();
        var scopeFactory = serviceProvider.GetRequiredService<IServiceScopeFactory>();

        var sut = new SyncOrchestrator(
            scopeFactory,
            _scheduleChecker,
            _options,
            NullLogger<SyncOrchestrator>.Instance,
            _metrics);

        using var cts = new CancellationTokenSource();

        // Act
        await sut.ExecutePendingSyncsAsync(cts.Token);

        // Assert
        receivedToken.ShouldNotBe(CancellationToken.None);
    }

    #endregion

    #region Error Handling

    [Fact]
    public async Task ExecutePendingSyncsAsync_OneTaskFails_OthersContinue()
    {
        // Arrange
        var tasks = new List<DataUpdateTask>
        {
            CreateTask("WorkOrder", UpdateTypes.Mass),
            CreateTask("LotUsage", UpdateTypes.Mass),
            CreateTask("Item", UpdateTypes.Mass)
        };

        _scheduleChecker.GetPendingTasksAsync(Arg.Any<CancellationToken>())
            .Returns(tasks);

        var executedTables = new List<string>();
        var operation = Substitute.For<ITableSyncOperation>();
        operation.ExecuteAsync(Arg.Any<DataUpdateTask>(), Arg.Any<CancellationToken>())
            .Returns(callInfo =>
            {
                var t = callInfo.Arg<DataUpdateTask>();
                executedTables.Add(t.TableName);

                if (t.TableName == "LotUsage")
                {
                    throw new Exception("Sync failed for LotUsage");
                }

                return Task.CompletedTask;
            });

        var services = new ServiceCollection();
        services.AddSingleton(_scheduleChecker);
        services.AddScoped(_ => operation);
        var serviceProvider = services.BuildServiceProvider();
        var scopeFactory = serviceProvider.GetRequiredService<IServiceScopeFactory>();

        var sut = new SyncOrchestrator(
            scopeFactory,
            _scheduleChecker,
            _options,
            NullLogger<SyncOrchestrator>.Instance,
            _metrics);

        // Act
        await sut.ExecutePendingSyncsAsync();

        // Assert: All tasks should have been attempted
        executedTables.Count.ShouldBe(3);
        executedTables.ShouldContain("WorkOrder");
        executedTables.ShouldContain("LotUsage");
        executedTables.ShouldContain("Item");
    }

    [Fact]
    public async Task ExecutePendingSyncsAsync_MultipleTasksFail_AllAttemptsComplete()
    {
        // Arrange
        var tasks = new List<DataUpdateTask>
        {
            CreateTask("WorkOrder", UpdateTypes.Mass),
            CreateTask("LotUsage", UpdateTypes.Mass),
            CreateTask("Item", UpdateTypes.Mass),
            CreateTask("Lot", UpdateTypes.Mass)
        };

        _scheduleChecker.GetPendingTasksAsync(Arg.Any<CancellationToken>())
            .Returns(tasks);

        var executedCount = 0;
        var operation = Substitute.For<ITableSyncOperation>();
        operation.ExecuteAsync(Arg.Any<DataUpdateTask>(), Arg.Any<CancellationToken>())
            .Returns(callInfo =>
            {
                Interlocked.Increment(ref executedCount);
                var t = callInfo.Arg<DataUpdateTask>();

                // Fail odd-numbered tables
                if (t.TableName is "WorkOrder" or "Item")
                {
                    throw new Exception($"Sync failed for {t.TableName}");
                }

                return Task.CompletedTask;
            });

        var services = new ServiceCollection();
        services.AddSingleton(_scheduleChecker);
        services.AddScoped(_ => operation);
        var serviceProvider = services.BuildServiceProvider();
        var scopeFactory = serviceProvider.GetRequiredService<IServiceScopeFactory>();

        var sut = new SyncOrchestrator(
            scopeFactory,
            _scheduleChecker,
            _options,
            NullLogger<SyncOrchestrator>.Instance,
            _metrics);

        // Act
        await sut.ExecutePendingSyncsAsync();

        // Assert: All 4 tasks should have been attempted
        executedCount.ShouldBe(4);
    }

    #endregion

    #region Metrics Recording

    [Fact]
    public async Task ExecutePendingSyncsAsync_RecordsCycleMetrics()
    {
        // Arrange
        var tasks = new List<DataUpdateTask>
        {
            CreateTask("WorkOrder", UpdateTypes.Mass),
            CreateTask("LotUsage", UpdateTypes.Mass)
        };

        _scheduleChecker.GetPendingTasksAsync(Arg.Any<CancellationToken>())
            .Returns(tasks);

        var operation = Substitute.For<ITableSyncOperation>();
        operation.ExecuteAsync(Arg.Any<DataUpdateTask>(), Arg.Any<CancellationToken>())
            .Returns(callInfo =>
            {
                var t = callInfo.Arg<DataUpdateTask>();
                if (t.TableName == "LotUsage")
                {
                    throw new Exception("Failed");
                }
                return Task.CompletedTask;
            });

        var services = new ServiceCollection();
        services.AddSingleton(_scheduleChecker);
        services.AddScoped(_ => operation);
        var serviceProvider = services.BuildServiceProvider();
        var scopeFactory = serviceProvider.GetRequiredService<IServiceScopeFactory>();

        var sut = new SyncOrchestrator(
            scopeFactory,
            _scheduleChecker,
            _options,
            NullLogger<SyncOrchestrator>.Instance,
            _metrics);

        // Act
        await sut.ExecutePendingSyncsAsync();

        // Assert: Metrics should have been recorded (we're using real metrics, so just verify no exceptions)
        // More detailed metrics testing is in DataSyncMetricsTests
    }

    #endregion

    #region Helper Methods

    private static DataUpdateTask CreateTask(string tableName, UpdateTypes updateType)
    {
        return new DataUpdateTask
        {
            TableName = tableName,
            SourceSystem = "JDE",
            SourceData = tableName.ToUpper(),
            UpdateType = updateType,
            MinimumDt = null,
            Config = new DataSourceConfig
            {
                TableName = tableName,
                SourceSystem = "JDE",
                SourceData = tableName.ToUpper(),
                FetcherTypeName = $"Jde{tableName}Fetcher",
                IsEnabled = true,
                MassConfig = new ScheduleConfig { Enabled = true, IntervalMinutes = 10080 },
                DailyConfig = new ScheduleConfig { Enabled = true, IntervalMinutes = 1440 },
                HourlyConfig = new ScheduleConfig { Enabled = true, IntervalMinutes = 60 }
            }
        };
    }

    #endregion
}