using System.Diagnostics.Metrics; using JdeScoping.Core.Models.Enums; using JdeScoping.DataSync.Options; using JdeScoping.DataSync.Contracts; using JdeScoping.DataSync.Models; using JdeScoping.DataSync.Services; using JdeScoping.DataSync.Telemetry; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Options; using NSubstitute; using NSubstitute.ExceptionExtensions; using Shouldly; namespace JdeScoping.DataSync.Tests; /// /// Unit tests for SyncOrchestrator. /// Tests parallel execution, cancellation, and scope isolation. /// public class SyncOrchestratorTests { private readonly IScheduleChecker _scheduleChecker; private readonly IOptions _options; private readonly DataSyncMetrics _metrics; public SyncOrchestratorTests() { _scheduleChecker = Substitute.For(); _options = Microsoft.Extensions.Options.Options.Create(new DataSyncOptions { MaxDegreeOfParallelism = 4 }); var services = new ServiceCollection(); services.AddMetrics(); var provider = services.BuildServiceProvider(); var meterFactory = provider.GetRequiredService(); _metrics = new DataSyncMetrics(meterFactory); } #region No Pending Tasks [Fact] public async Task ExecutePendingSyncsAsync_NoPendingTasks_ReturnsImmediately() { // Arrange _scheduleChecker.GetPendingTasksAsync(Arg.Any()) .Returns(new List()); var operation = Substitute.For(); var services = new ServiceCollection(); services.AddSingleton(_scheduleChecker); services.AddScoped(_ => operation); var serviceProvider = services.BuildServiceProvider(); var scopeFactory = serviceProvider.GetRequiredService(); var sut = new SyncOrchestrator( scopeFactory, _scheduleChecker, _options, NullLogger.Instance, _metrics); // Act await sut.ExecutePendingSyncsAsync(); // Assert: Operation should never be called await operation.DidNotReceive().ExecuteAsync(Arg.Any(), Arg.Any()); } #endregion #region Single Task Execution [Fact] public async Task ExecutePendingSyncsAsync_SingleTask_ExecutesOperation() { // Arrange var task = CreateTask("WorkOrder", UpdateTypes.Mass); _scheduleChecker.GetPendingTasksAsync(Arg.Any()) .Returns(new List { task }); var executedTasks = new List(); var operation = Substitute.For(); operation.ExecuteAsync(Arg.Any(), Arg.Any()) .Returns(callInfo => { var t = callInfo.Arg(); executedTasks.Add(t.TableName); return Task.CompletedTask; }); var services = new ServiceCollection(); services.AddSingleton(_scheduleChecker); services.AddScoped(_ => operation); var serviceProvider = services.BuildServiceProvider(); var scopeFactory = serviceProvider.GetRequiredService(); var sut = new SyncOrchestrator( scopeFactory, _scheduleChecker, _options, NullLogger.Instance, _metrics); // Act await sut.ExecutePendingSyncsAsync(); // Assert executedTasks.ShouldContain("WorkOrder"); } #endregion #region Parallel Execution [Fact] public async Task ExecutePendingSyncsAsync_MultipleTasks_ExecutesInParallel() { // Arrange var tasks = new List { CreateTask("WorkOrder", UpdateTypes.Mass), CreateTask("LotUsage", UpdateTypes.Mass), CreateTask("Item", UpdateTypes.Mass), CreateTask("Lot", UpdateTypes.Mass) }; _scheduleChecker.GetPendingTasksAsync(Arg.Any()) .Returns(tasks); var executingConcurrently = 0; var maxConcurrent = 0; var lockObj = new object(); var operation = Substitute.For(); operation.ExecuteAsync(Arg.Any(), Arg.Any()) .Returns(async callInfo => { lock (lockObj) { executingConcurrently++; maxConcurrent = Math.Max(maxConcurrent, executingConcurrently); } await Task.Delay(50); // Simulate work lock (lockObj) { executingConcurrently--; } }); var services = new ServiceCollection(); services.AddSingleton(_scheduleChecker); services.AddScoped(_ => operation); var serviceProvider = services.BuildServiceProvider(); var scopeFactory = serviceProvider.GetRequiredService(); var sut = new SyncOrchestrator( scopeFactory, _scheduleChecker, _options, NullLogger.Instance, _metrics); // Act await sut.ExecutePendingSyncsAsync(); // Assert: Should have executed multiple tasks concurrently maxConcurrent.ShouldBeGreaterThan(1); } [Fact] public async Task ExecutePendingSyncsAsync_RespectsMaxDegreeOfParallelism() { // Arrange: Create 10 tasks but limit parallelism to 2 var options = Microsoft.Extensions.Options.Options.Create(new DataSyncOptions { MaxDegreeOfParallelism = 2 }); var tasks = Enumerable.Range(0, 10) .Select(i => CreateTask($"Table{i}", UpdateTypes.Mass)) .ToList(); _scheduleChecker.GetPendingTasksAsync(Arg.Any()) .Returns(tasks); var executingConcurrently = 0; var maxConcurrent = 0; var lockObj = new object(); var operation = Substitute.For(); operation.ExecuteAsync(Arg.Any(), Arg.Any()) .Returns(async callInfo => { lock (lockObj) { executingConcurrently++; maxConcurrent = Math.Max(maxConcurrent, executingConcurrently); } await Task.Delay(50); lock (lockObj) { executingConcurrently--; } }); var services = new ServiceCollection(); services.AddSingleton(_scheduleChecker); services.AddScoped(_ => operation); var serviceProvider = services.BuildServiceProvider(); var scopeFactory = serviceProvider.GetRequiredService(); var sut = new SyncOrchestrator( scopeFactory, _scheduleChecker, options, NullLogger.Instance, _metrics); // Act await sut.ExecutePendingSyncsAsync(); // Assert: Should not exceed MaxDegreeOfParallelism maxConcurrent.ShouldBeLessThanOrEqualTo(2); } #endregion #region Scope Isolation [Fact] public async Task ExecutePendingSyncsAsync_EachTaskGetsOwnScope() { // Arrange var tasks = new List { CreateTask("WorkOrder", UpdateTypes.Mass), CreateTask("LotUsage", UpdateTypes.Mass) }; _scheduleChecker.GetPendingTasksAsync(Arg.Any()) .Returns(tasks); var scopeCount = 0; var services = new ServiceCollection(); services.AddSingleton(_scheduleChecker); services.AddScoped(sp => { Interlocked.Increment(ref scopeCount); var mock = Substitute.For(); mock.ExecuteAsync(Arg.Any(), Arg.Any()) .Returns(Task.CompletedTask); return mock; }); var serviceProvider = services.BuildServiceProvider(); var scopeFactory = serviceProvider.GetRequiredService(); var sut = new SyncOrchestrator( scopeFactory, _scheduleChecker, _options, NullLogger.Instance, _metrics); // Act await sut.ExecutePendingSyncsAsync(); // Assert: Each task should create its own scope scopeCount.ShouldBe(2); } #endregion #region Cancellation Handling [Fact] public async Task ExecutePendingSyncsAsync_WhenCancelled_PropagatesCancellation() { // Arrange var tasks = new List { CreateTask("WorkOrder", UpdateTypes.Mass), CreateTask("LotUsage", UpdateTypes.Mass), CreateTask("Item", UpdateTypes.Mass) }; _scheduleChecker.GetPendingTasksAsync(Arg.Any()) .Returns(tasks); var cts = new CancellationTokenSource(); var operationsStarted = 0; var operation = Substitute.For(); operation.ExecuteAsync(Arg.Any(), Arg.Any()) .Returns(async callInfo => { Interlocked.Increment(ref operationsStarted); // Cancel after first operation starts if (operationsStarted == 1) { cts.Cancel(); } var token = callInfo.Arg(); await Task.Delay(500, token); // Will throw if cancelled }); var services = new ServiceCollection(); services.AddSingleton(_scheduleChecker); services.AddScoped(_ => operation); var serviceProvider = services.BuildServiceProvider(); var scopeFactory = serviceProvider.GetRequiredService(); var sut = new SyncOrchestrator( scopeFactory, _scheduleChecker, _options, NullLogger.Instance, _metrics); // Act & Assert await Should.ThrowAsync( () => sut.ExecutePendingSyncsAsync(cts.Token)); } [Fact] public async Task ExecutePendingSyncsAsync_PassesCancellationTokenToOperations() { // Arrange var task = CreateTask("WorkOrder", UpdateTypes.Mass); _scheduleChecker.GetPendingTasksAsync(Arg.Any()) .Returns(new List { task }); CancellationToken receivedToken = default; var operation = Substitute.For(); operation.ExecuteAsync(Arg.Any(), Arg.Any()) .Returns(callInfo => { receivedToken = callInfo.Arg(); return Task.CompletedTask; }); var services = new ServiceCollection(); services.AddSingleton(_scheduleChecker); services.AddScoped(_ => operation); var serviceProvider = services.BuildServiceProvider(); var scopeFactory = serviceProvider.GetRequiredService(); var sut = new SyncOrchestrator( scopeFactory, _scheduleChecker, _options, NullLogger.Instance, _metrics); using var cts = new CancellationTokenSource(); // Act await sut.ExecutePendingSyncsAsync(cts.Token); // Assert receivedToken.ShouldNotBe(CancellationToken.None); } #endregion #region Error Handling [Fact] public async Task ExecutePendingSyncsAsync_OneTaskFails_OthersContinue() { // Arrange var tasks = new List { CreateTask("WorkOrder", UpdateTypes.Mass), CreateTask("LotUsage", UpdateTypes.Mass), CreateTask("Item", UpdateTypes.Mass) }; _scheduleChecker.GetPendingTasksAsync(Arg.Any()) .Returns(tasks); var executedTables = new List(); var operation = Substitute.For(); operation.ExecuteAsync(Arg.Any(), Arg.Any()) .Returns(callInfo => { var t = callInfo.Arg(); executedTables.Add(t.TableName); if (t.TableName == "LotUsage") { throw new Exception("Sync failed for LotUsage"); } return Task.CompletedTask; }); var services = new ServiceCollection(); services.AddSingleton(_scheduleChecker); services.AddScoped(_ => operation); var serviceProvider = services.BuildServiceProvider(); var scopeFactory = serviceProvider.GetRequiredService(); var sut = new SyncOrchestrator( scopeFactory, _scheduleChecker, _options, NullLogger.Instance, _metrics); // Act await sut.ExecutePendingSyncsAsync(); // Assert: All tasks should have been attempted executedTables.Count.ShouldBe(3); executedTables.ShouldContain("WorkOrder"); executedTables.ShouldContain("LotUsage"); executedTables.ShouldContain("Item"); } [Fact] public async Task ExecutePendingSyncsAsync_MultipleTasksFail_AllAttemptsComplete() { // Arrange var tasks = new List { CreateTask("WorkOrder", UpdateTypes.Mass), CreateTask("LotUsage", UpdateTypes.Mass), CreateTask("Item", UpdateTypes.Mass), CreateTask("Lot", UpdateTypes.Mass) }; _scheduleChecker.GetPendingTasksAsync(Arg.Any()) .Returns(tasks); var executedCount = 0; var operation = Substitute.For(); operation.ExecuteAsync(Arg.Any(), Arg.Any()) .Returns(callInfo => { Interlocked.Increment(ref executedCount); var t = callInfo.Arg(); // Fail odd-numbered tables if (t.TableName is "WorkOrder" or "Item") { throw new Exception($"Sync failed for {t.TableName}"); } return Task.CompletedTask; }); var services = new ServiceCollection(); services.AddSingleton(_scheduleChecker); services.AddScoped(_ => operation); var serviceProvider = services.BuildServiceProvider(); var scopeFactory = serviceProvider.GetRequiredService(); var sut = new SyncOrchestrator( scopeFactory, _scheduleChecker, _options, NullLogger.Instance, _metrics); // Act await sut.ExecutePendingSyncsAsync(); // Assert: All 4 tasks should have been attempted executedCount.ShouldBe(4); } #endregion #region Metrics Recording [Fact] public async Task ExecutePendingSyncsAsync_RecordsCycleMetrics() { // Arrange var tasks = new List { CreateTask("WorkOrder", UpdateTypes.Mass), CreateTask("LotUsage", UpdateTypes.Mass) }; _scheduleChecker.GetPendingTasksAsync(Arg.Any()) .Returns(tasks); var operation = Substitute.For(); operation.ExecuteAsync(Arg.Any(), Arg.Any()) .Returns(callInfo => { var t = callInfo.Arg(); if (t.TableName == "LotUsage") { throw new Exception("Failed"); } return Task.CompletedTask; }); var services = new ServiceCollection(); services.AddSingleton(_scheduleChecker); services.AddScoped(_ => operation); var serviceProvider = services.BuildServiceProvider(); var scopeFactory = serviceProvider.GetRequiredService(); var sut = new SyncOrchestrator( scopeFactory, _scheduleChecker, _options, NullLogger.Instance, _metrics); // Act await sut.ExecutePendingSyncsAsync(); // Assert: Metrics should have been recorded (we're using real metrics, so just verify no exceptions) // More detailed metrics testing is in DataSyncMetricsTests } #endregion #region Helper Methods private static DataUpdateTask CreateTask(string tableName, UpdateTypes updateType) { return new DataUpdateTask { TableName = tableName, SourceSystem = "JDE", SourceData = tableName.ToUpper(), UpdateType = updateType, MinimumDt = null, Config = new DataSourceConfig { TableName = tableName, SourceSystem = "JDE", SourceData = tableName.ToUpper(), FetcherTypeName = $"Jde{tableName}Fetcher", IsEnabled = true, MassConfig = new ScheduleConfig { Enabled = true, IntervalMinutes = 10080 }, DailyConfig = new ScheduleConfig { Enabled = true, IntervalMinutes = 1440 }, HourlyConfig = new ScheduleConfig { Enabled = true, IntervalMinutes = 60 } } }; } #endregion }