jdescopingtool/NEW/tests/JdeScoping.DataSync.Tests/ScheduleCheckerTests.cs

using JdeScoping.Core.Models;
using JdeScoping.Core.Models.Enums;
using JdeScoping.Core.Models.Infrastructure;
using JdeScoping.DataSync.Configuration;
using JdeScoping.DataSync.Options;
using JdeScoping.DataSync.Contracts;
using JdeScoping.DataSync.Services;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using NSubstitute;
using Shouldly;

namespace JdeScoping.DataSync.Tests;

/// <summary>
/// Unit tests for ScheduleChecker service.
/// </summary>
public class ScheduleCheckerTests
{
    private readonly IDataUpdateRepository _repository;
    private readonly IPipelineRegistry _pipelineRegistry;
    private readonly IOptions<DataSyncOptions> _options;
    private readonly List<EtlPipelineConfig> _pipelines;
    private readonly ScheduleChecker _sut;

    public ScheduleCheckerTests()
    {
        _repository = Substitute.For<IDataUpdateRepository>();
        _pipelineRegistry = Substitute.For<IPipelineRegistry>();
        _pipelines = [];
        _options = Microsoft.Extensions.Options.Options.Create(new DataSyncOptions
        {
            LookbackMultiplier = 3
        });

        // Setup pipeline registry to return our pipeline list
        _pipelineRegistry.GetEnabledPipelines().Returns(_ => _pipelines);

        _sut = new ScheduleChecker(
            _repository,
            _pipelineRegistry,
            _options,
            NullLogger<ScheduleChecker>.Instance);
    }

    #region Priority Tests - Mass > Daily > Hourly

    [Fact]
    public async Task GetPendingTasksAsync_WhenMassNeverRun_ReturnsMassTask()
    {
        // Arrange
        var pipeline = CreatePipeline("WorkOrder", massInterval: 10080, dailyInterval: 1440, hourlyInterval: 60);
        _pipelines.Add(pipeline);
        _repository.GetLastDataUpdatesAsync(Arg.Any<CancellationToken>())
            .Returns(new Dictionary<string, DataUpdate>());

        // Act
        var tasks = await _sut.GetPendingTasksAsync();

        // Assert
        tasks.ShouldHaveSingleItem();
        tasks[0].TableName.ShouldBe("WorkOrder");
        tasks[0].UpdateType.ShouldBe(UpdateTypes.Mass);
        tasks[0].MinimumDt.ShouldBeNull(); // Mass updates don't have MinimumDT
    }

    [Fact]
    public async Task GetPendingTasksAsync_WhenMassDue_ReturnsMassOverDaily()
    {
        // Arrange
        var pipeline = CreatePipeline("WorkOrder", massInterval: 60, dailyInterval: 1440);
        _pipelines.Add(pipeline);

        var now = DateTime.UtcNow;
        var lastMass = CreateDataUpdate("WorkOrder", UpdateTypes.Mass, now.AddMinutes(-120), success: true);

        _repository.GetLastDataUpdatesAsync(Arg.Any<CancellationToken>())
            .Returns(new Dictionary<string, DataUpdate>
            {
                { "WorkOrder_3", lastMass } // 3 = UpdateTypes.Mass
            });

        // Act
        var tasks = await _sut.GetPendingTasksAsync();

        // Assert
        tasks.ShouldHaveSingleItem();
        tasks[0].UpdateType.ShouldBe(UpdateTypes.Mass);
    }

    [Fact]
    public async Task GetPendingTasksAsync_WhenMassNotDue_ChecksDailyAndHourly()
    {
        // Arrange
        var pipeline = CreatePipeline("WorkOrder", massInterval: 10080, dailyInterval: 1440, hourlyInterval: 60);
        _pipelines.Add(pipeline);

        var now = DateTime.UtcNow;
        var lastMass = CreateDataUpdate("WorkOrder", UpdateTypes.Mass, now.AddHours(-1), success: true);
        var lastDaily = CreateDataUpdate("WorkOrder", UpdateTypes.Daily, now.AddHours(-25), success: true);

        _repository.GetLastDataUpdatesAsync(Arg.Any<CancellationToken>())
            .Returns(new Dictionary<string, DataUpdate>
            {
                { "WorkOrder_3", lastMass },  // Mass not due
                { "WorkOrder_2", lastDaily }  // Daily is due (25 hrs > 1440 min)
            });

        // Act
        var tasks = await _sut.GetPendingTasksAsync();

        // Assert
        tasks.ShouldHaveSingleItem();
        tasks[0].UpdateType.ShouldBe(UpdateTypes.Daily);
    }

    [Fact]
    public async Task GetPendingTasksAsync_WhenDailyDue_ReturnsDailyOverHourly()
    {
        // Arrange
        var pipeline = CreatePipeline("WorkOrder", massInterval: 10080, dailyInterval: 1440, hourlyInterval: 60);
        _pipelines.Add(pipeline);

        var now = DateTime.UtcNow;
        var lastMass = CreateDataUpdate("WorkOrder", UpdateTypes.Mass, now.AddDays(-1), success: true);
        var lastDaily = CreateDataUpdate("WorkOrder", UpdateTypes.Daily, now.AddHours(-25), success: true);
        var lastHourly = CreateDataUpdate("WorkOrder", UpdateTypes.Hourly, now.AddHours(-2), success: true);

        _repository.GetLastDataUpdatesAsync(Arg.Any<CancellationToken>())
            .Returns(new Dictionary<string, DataUpdate>
            {
                { "WorkOrder_3", lastMass },
                { "WorkOrder_2", lastDaily },
                { "WorkOrder_1", lastHourly }
            });

        // Act
        var tasks = await _sut.GetPendingTasksAsync();

        // Assert
        tasks.ShouldHaveSingleItem();
        tasks[0].UpdateType.ShouldBe(UpdateTypes.Daily);
    }

    [Fact]
    public async Task GetPendingTasksAsync_WhenOnlyHourlyDue_ReturnsHourly()
    {
        // Arrange
        var pipeline = CreatePipeline("WorkOrder", massInterval: 10080, dailyInterval: 1440, hourlyInterval: 60);
        _pipelines.Add(pipeline);

        var now = DateTime.UtcNow;
        var lastMass = CreateDataUpdate("WorkOrder", UpdateTypes.Mass, now.AddDays(-1), success: true);
        var lastDaily = CreateDataUpdate("WorkOrder", UpdateTypes.Daily, now.AddHours(-12), success: true);
        var lastHourly = CreateDataUpdate("WorkOrder", UpdateTypes.Hourly, now.AddHours(-2), success: true);

        _repository.GetLastDataUpdatesAsync(Arg.Any<CancellationToken>())
            .Returns(new Dictionary<string, DataUpdate>
            {
                { "WorkOrder_3", lastMass },
                { "WorkOrder_2", lastDaily },
                { "WorkOrder_1", lastHourly }
            });

        // Act
        var tasks = await _sut.GetPendingTasksAsync();

        // Assert
        tasks.ShouldHaveSingleItem();
        tasks[0].UpdateType.ShouldBe(UpdateTypes.Hourly);
    }

    #endregion

    #region MinimumDT Calculation with Lookback

    [Fact]
    public async Task GetPendingTasksAsync_DailySync_CalculatesMinimumDTWithLookback()
    {
        // Arrange: LookbackMultiplier = 3, daily interval = 1440 min
        // MinimumDT = lastDaily.EndDT - (3 * 1440) = lastDaily.EndDT - 4320 min = 3 days before lastDaily
        var pipeline = CreatePipeline("WorkOrder", massInterval: 10080, dailyInterval: 1440);
        _pipelines.Add(pipeline);

        var now = DateTime.UtcNow;
        var lastMass = CreateDataUpdate("WorkOrder", UpdateTypes.Mass, now.AddDays(-2), success: true);
        var lastDaily = CreateDataUpdate("WorkOrder", UpdateTypes.Daily, now.AddHours(-25), success: true);

        _repository.GetLastDataUpdatesAsync(Arg.Any<CancellationToken>())
            .Returns(new Dictionary<string, DataUpdate>
            {
                { "WorkOrder_3", lastMass },
                { "WorkOrder_2", lastDaily }
            });

        // Act
        var tasks = await _sut.GetPendingTasksAsync();

        // Assert
        tasks.ShouldHaveSingleItem();
        tasks[0].UpdateType.ShouldBe(UpdateTypes.Daily);
        tasks[0].MinimumDt.ShouldNotBeNull();

        // Expected: lastDaily.EndDT - (3 * 1440 min) = lastDaily.EndDT - 3 days
        var expectedMinimumDt = lastDaily.EndDt!.Value.AddMinutes(-3 * 1440);
        tasks[0].MinimumDt!.Value.ShouldBe(expectedMinimumDt, TimeSpan.FromSeconds(1));
    }

    [Fact]
    public async Task GetPendingTasksAsync_HourlySync_UsesHourlyTimestampForMinimumDT()
    {
        // Arrange: Hourly uses its own timestamp and interval for MinimumDT calculation
        var pipeline = CreatePipeline("WorkOrder", massInterval: 10080, dailyInterval: 1440, hourlyInterval: 60);
        _pipelines.Add(pipeline);

        var now = DateTime.UtcNow;
        var lastMass = CreateDataUpdate("WorkOrder", UpdateTypes.Mass, now.AddDays(-1), success: true);
        var lastDaily = CreateDataUpdate("WorkOrder", UpdateTypes.Daily, now.AddHours(-12), success: true);
        var lastHourly = CreateDataUpdate("WorkOrder", UpdateTypes.Hourly, now.AddHours(-2), success: true);

        _repository.GetLastDataUpdatesAsync(Arg.Any<CancellationToken>())
            .Returns(new Dictionary<string, DataUpdate>
            {
                { "WorkOrder_3", lastMass },
                { "WorkOrder_2", lastDaily },
                { "WorkOrder_1", lastHourly }
            });

        // Act
        var tasks = await _sut.GetPendingTasksAsync();

        // Assert
        tasks.ShouldHaveSingleItem();
        tasks[0].UpdateType.ShouldBe(UpdateTypes.Hourly);
        tasks[0].MinimumDt.ShouldNotBeNull();

        // Hourly uses hourly's timestamp and hourly's interval for lookback calculation
        var expectedMinimumDt = lastHourly.EndDt!.Value.AddMinutes(-3 * 60);
        tasks[0].MinimumDt!.Value.ShouldBe(expectedMinimumDt, TimeSpan.FromSeconds(1));
    }

    [Fact]
    public async Task GetPendingTasksAsync_WithDifferentLookbackMultiplier_CalculatesCorrectly()
    {
        // Arrange: Test with multiplier = 5
        _options.Value.LookbackMultiplier = 5;
        var pipeline = CreatePipeline("WorkOrder", massInterval: 10080, dailyInterval: 1440);
        _pipelines.Add(pipeline);

        var now = DateTime.UtcNow;
        var lastMass = CreateDataUpdate("WorkOrder", UpdateTypes.Mass, now.AddDays(-2), success: true);
        var lastDaily = CreateDataUpdate("WorkOrder", UpdateTypes.Daily, now.AddHours(-25), success: true);

        _repository.GetLastDataUpdatesAsync(Arg.Any<CancellationToken>())
            .Returns(new Dictionary<string, DataUpdate>
            {
                { "WorkOrder_3", lastMass },
                { "WorkOrder_2", lastDaily }
            });

        // Act
        var tasks = await _sut.GetPendingTasksAsync();

        // Assert
        var expectedMinimumDt = lastDaily.EndDt!.Value.AddMinutes(-5 * 1440);
        tasks[0].MinimumDt!.Value.ShouldBe(expectedMinimumDt, TimeSpan.FromSeconds(1));
    }

    #endregion

    #region Manual-Only Pipelines

    [Fact]
    public async Task GetPendingTasksAsync_ManualOnlyPipeline_ReturnsNoTasks()
    {
        // Arrange
        var pipeline = CreatePipeline("WorkOrder", massInterval: 10080);
        pipeline.IsManualOnly = true;
        _pipelines.Add(pipeline);

        _repository.GetLastDataUpdatesAsync(Arg.Any<CancellationToken>())
            .Returns(new Dictionary<string, DataUpdate>());

        // Act
        var tasks = await _sut.GetPendingTasksAsync();

        // Assert
        tasks.ShouldBeEmpty();
    }

    #endregion

    #region First Sync (No Prior Updates) Scenario

    [Fact]
    public async Task GetPendingTasksAsync_NoPriorUpdates_RequiresMassFirst()
    {
        // Arrange: Never synced before, all schedules enabled
        var pipeline = CreatePipeline("WorkOrder", massInterval: 10080, dailyInterval: 1440, hourlyInterval: 60);
        _pipelines.Add(pipeline);

        _repository.GetLastDataUpdatesAsync(Arg.Any<CancellationToken>())
            .Returns(new Dictionary<string, DataUpdate>());

        // Act
        var tasks = await _sut.GetPendingTasksAsync();

        // Assert: Must do Mass first
        tasks.ShouldHaveSingleItem();
        tasks[0].UpdateType.ShouldBe(UpdateTypes.Mass);
        tasks[0].MinimumDt.ShouldBeNull();
    }

    [Fact]
    public async Task GetPendingTasksAsync_OnlyMassCompleted_DailyHasNullMinimumDT()
    {
        // Arrange: Mass completed, no daily yet
        var pipeline = CreatePipeline("WorkOrder", massInterval: 10080, dailyInterval: 1440);
        _pipelines.Add(pipeline);

        var now = DateTime.UtcNow;
        var lastMass = CreateDataUpdate("WorkOrder", UpdateTypes.Mass, now.AddHours(-1), success: true);

        _repository.GetLastDataUpdatesAsync(Arg.Any<CancellationToken>())
            .Returns(new Dictionary<string, DataUpdate>
            {
                { "WorkOrder_3", lastMass }
            });

        // Act
        var tasks = await _sut.GetPendingTasksAsync();

        // Assert: Should return Daily with null MinimumDT (no prior daily to calculate from)
        tasks.ShouldHaveSingleItem();
        tasks[0].UpdateType.ShouldBe(UpdateTypes.Daily);
        tasks[0].MinimumDt.ShouldBeNull();
    }

    [Fact]
    public async Task GetPendingTasksAsync_NeverHadMass_DoesNotReturnDailyOrHourly()
    {
        // Arrange: Daily and Hourly enabled but no Mass ever run
        var pipeline = CreatePipeline("WorkOrder", massInterval: 10080, dailyInterval: 1440, hourlyInterval: 60);
        _pipelines.Add(pipeline);

        _repository.GetLastDataUpdatesAsync(Arg.Any<CancellationToken>())
            .Returns(new Dictionary<string, DataUpdate>());

        // Act
        var tasks = await _sut.GetPendingTasksAsync();

        // Assert: Only Mass should be returned - can't do daily/hourly without initial mass
        tasks.ShouldHaveSingleItem();
        tasks[0].UpdateType.ShouldBe(UpdateTypes.Mass);
    }

    #endregion

    #region Failed Sync Recovery

    [Fact]
    public async Task GetPendingTasksAsync_FailedMass_ReturnsMassAgain()
    {
        // Arrange
        var pipeline = CreatePipeline("WorkOrder", massInterval: 10080);
        _pipelines.Add(pipeline);

        var now = DateTime.UtcNow;
        var lastMass = CreateDataUpdate("WorkOrder", UpdateTypes.Mass, now.AddMinutes(-5), success: false);

        _repository.GetLastDataUpdatesAsync(Arg.Any<CancellationToken>())
            .Returns(new Dictionary<string, DataUpdate>
            {
                { "WorkOrder_3", lastMass }
            });

        // Act
        var tasks = await _sut.GetPendingTasksAsync();

        // Assert: Failed mass should trigger retry regardless of interval
        tasks.ShouldHaveSingleItem();
        tasks[0].UpdateType.ShouldBe(UpdateTypes.Mass);
    }

    [Fact]
    public async Task GetPendingTasksAsync_FailedDaily_ReturnsDailyAgain()
    {
        // Arrange
        var pipeline = CreatePipeline("WorkOrder", massInterval: 10080, dailyInterval: 1440);
        _pipelines.Add(pipeline);

        var now = DateTime.UtcNow;
        var lastMass = CreateDataUpdate("WorkOrder", UpdateTypes.Mass, now.AddDays(-1), success: true);
        var lastDaily = CreateDataUpdate("WorkOrder", UpdateTypes.Daily, now.AddMinutes(-5), success: false);

        _repository.GetLastDataUpdatesAsync(Arg.Any<CancellationToken>())
            .Returns(new Dictionary<string, DataUpdate>
            {
                { "WorkOrder_3", lastMass },
                { "WorkOrder_2", lastDaily }
            });

        // Act
        var tasks = await _sut.GetPendingTasksAsync();

        // Assert
        tasks.ShouldHaveSingleItem();
        tasks[0].UpdateType.ShouldBe(UpdateTypes.Daily);
    }

    [Fact]
    public async Task GetPendingTasksAsync_FailedHourly_ReturnsHourlyAgain()
    {
        // Arrange
        var pipeline = CreatePipeline("WorkOrder", massInterval: 10080, dailyInterval: 1440, hourlyInterval: 60);
        _pipelines.Add(pipeline);

        var now = DateTime.UtcNow;
        var lastMass = CreateDataUpdate("WorkOrder", UpdateTypes.Mass, now.AddDays(-1), success: true);
        var lastDaily = CreateDataUpdate("WorkOrder", UpdateTypes.Daily, now.AddHours(-1), success: true);
        var lastHourly = CreateDataUpdate("WorkOrder", UpdateTypes.Hourly, now.AddMinutes(-5), success: false);

        _repository.GetLastDataUpdatesAsync(Arg.Any<CancellationToken>())
            .Returns(new Dictionary<string, DataUpdate>
            {
                { "WorkOrder_3", lastMass },
                { "WorkOrder_2", lastDaily },
                { "WorkOrder_1", lastHourly }
            });

        // Act
        var tasks = await _sut.GetPendingTasksAsync();

        // Assert
        tasks.ShouldHaveSingleItem();
        tasks[0].UpdateType.ShouldBe(UpdateTypes.Hourly);
    }

    #endregion

    #region Multiple Tables

    [Fact]
    public async Task GetPendingTasksAsync_MultipleTables_ReturnsTasksForEach()
    {
        // Arrange
        var pipeline1 = CreatePipeline("WorkOrder", massInterval: 60);
        var pipeline2 = CreatePipeline("LotUsage", massInterval: 60);
        _pipelines.Add(pipeline1);
        _pipelines.Add(pipeline2);

        _repository.GetLastDataUpdatesAsync(Arg.Any<CancellationToken>())
            .Returns(new Dictionary<string, DataUpdate>());

        // Act
        var tasks = await _sut.GetPendingTasksAsync();

        // Assert
        tasks.Count.ShouldBe(2);
        tasks.ShouldContain(t => t.TableName == "WorkOrder");
        tasks.ShouldContain(t => t.TableName == "LotUsage");
    }

    [Fact]
    public async Task GetPendingTasksAsync_MultipleTables_DifferentSchedulesDue()
    {
        // Arrange
        var pipeline1 = CreatePipeline("WorkOrder", massInterval: 10080, dailyInterval: 1440);
        var pipeline2 = CreatePipeline("LotUsage", massInterval: 60);
        _pipelines.Add(pipeline1);
        _pipelines.Add(pipeline2);

        var now = DateTime.UtcNow;
        var lastMassWorkOrder = CreateDataUpdate("WorkOrder", UpdateTypes.Mass, now.AddDays(-1), success: true);
        var lastDailyWorkOrder = CreateDataUpdate("WorkOrder", UpdateTypes.Daily, now.AddHours(-25), success: true);
        var lastMassLotUsage = CreateDataUpdate("LotUsage", UpdateTypes.Mass, now.AddHours(-2), success: true);

        _repository.GetLastDataUpdatesAsync(Arg.Any<CancellationToken>())
            .Returns(new Dictionary<string, DataUpdate>
            {
                { "WorkOrder_3", lastMassWorkOrder },
                { "WorkOrder_2", lastDailyWorkOrder },
                { "LotUsage_3", lastMassLotUsage }
            });

        // Act
        var tasks = await _sut.GetPendingTasksAsync();

        // Assert
        tasks.Count.ShouldBe(2);
        tasks.ShouldContain(t => t.TableName == "WorkOrder" && t.UpdateType == UpdateTypes.Daily);
        tasks.ShouldContain(t => t.TableName == "LotUsage" && t.UpdateType == UpdateTypes.Mass);
    }

    #endregion

    #region Edge Cases

    [Fact]
    public async Task GetPendingTasksAsync_NothingDue_ReturnsEmptyList()
    {
        // Arrange
        var pipeline = CreatePipeline("WorkOrder", massInterval: 10080, dailyInterval: 1440, hourlyInterval: 60);
        _pipelines.Add(pipeline);

        var now = DateTime.UtcNow;
        // All syncs completed recently
        var lastMass = CreateDataUpdate("WorkOrder", UpdateTypes.Mass, now.AddMinutes(-5), success: true);
        var lastDaily = CreateDataUpdate("WorkOrder", UpdateTypes.Daily, now.AddMinutes(-5), success: true);
        var lastHourly = CreateDataUpdate("WorkOrder", UpdateTypes.Hourly, now.AddMinutes(-5), success: true);

        _repository.GetLastDataUpdatesAsync(Arg.Any<CancellationToken>())
            .Returns(new Dictionary<string, DataUpdate>
            {
                { "WorkOrder_3", lastMass },
                { "WorkOrder_2", lastDaily },
                { "WorkOrder_1", lastHourly }
            });

        // Act
        var tasks = await _sut.GetPendingTasksAsync();

        // Assert
        tasks.ShouldBeEmpty();
    }

    [Fact]
    public async Task GetPendingTasksAsync_NoPipelines_ReturnsEmptyList()
    {
        // Arrange: No pipelines configured
        _repository.GetLastDataUpdatesAsync(Arg.Any<CancellationToken>())
            .Returns(new Dictionary<string, DataUpdate>());

        // Act
        var tasks = await _sut.GetPendingTasksAsync();

        // Assert
        tasks.ShouldBeEmpty();
    }

    #endregion

    #region Helper Methods

    private static EtlPipelineConfig CreatePipeline(
        string name,
        int? massInterval = null,
        int? dailyInterval = null,
        int? hourlyInterval = null)
    {
        return new EtlPipelineConfig
        {
            Name = name,
            IsEnabled = true,
            IsManualOnly = false,
            MassSyncIntervalMinutes = massInterval,
            DailySyncIntervalMinutes = dailyInterval,
            HourlySyncIntervalMinutes = hourlyInterval,
            Source = new SourceElement
            {
                Connection = "jde",
                Query = "SELECT * FROM Test"
            },
            Destination = new DestinationElement
            {
                Table = name,
                MatchColumns = ["Id"]
            }
        };
    }

    private static DataUpdate CreateDataUpdate(
        string tableName,
        UpdateTypes updateType,
        DateTime endDt,
        bool success)
    {
        return new DataUpdate
        {
            Id = 1,
            TableName = tableName,
            SourceSystem = "JDE",
            SourceData = tableName.ToUpper(),
            UpdateType = updateType,
            StartDt = endDt.AddMinutes(-5),
            EndDt = endDt,
            WasSuccessful = success,
            NumberRecords = success ? 1000 : -1
        };
    }

    #endregion
}