feat: implement ETL pipeline redesign and ConfigManager improvements
- Add pipeline registry with JSON-based configuration and hot-reload support - Implement manual sync request feature with API, client UI, and database - Improve ConfigManager: connection string dropdown in pipeline editor, step delete/reorder functionality, and fix JSON parsing for ConnectionStrings
This commit is contained in:
Joseph Doherty
@@ -1,6 +1,7 @@
|
||||
using JdeScoping.Core.Models;
|
||||
using JdeScoping.Core.Models.Enums;
|
||||
using JdeScoping.Core.Models.Infrastructure;
|
||||
using JdeScoping.DataSync.Configuration;
|
||||
using JdeScoping.DataSync.Options;
|
||||
using JdeScoping.DataSync.Contracts;
|
||||
using JdeScoping.DataSync.Services;
|
||||
@@ -17,19 +18,28 @@ namespace JdeScoping.DataSync.Tests;
|
||||
public class ScheduleCheckerTests
|
||||
{
|
||||
private readonly IDataUpdateRepository _repository;
|
||||
private readonly IPipelineRegistry _pipelineRegistry;
|
||||
private readonly IOptions<DataSyncOptions> _options;
|
||||
private readonly List<EtlPipelineConfig> _pipelines;
|
||||
private readonly ScheduleChecker _sut;
|
||||
|
||||
public ScheduleCheckerTests()
|
||||
{
|
||||
_repository = Substitute.For<IDataUpdateRepository>();
|
||||
_pipelineRegistry = Substitute.For<IPipelineRegistry>();
|
||||
_pipelines = [];
|
||||
_options = Microsoft.Extensions.Options.Options.Create(new DataSyncOptions
|
||||
{
|
||||
LookbackMultiplier = 3,
|
||||
DataSources = []
|
||||
});
|
||||
|
||||
// Setup pipeline registry to return our pipeline list
|
||||
_pipelineRegistry.GetEnabledPipelines().Returns(_ => _pipelines);
|
||||
|
||||
_sut = new ScheduleChecker(
|
||||
_repository,
|
||||
_pipelineRegistry,
|
||||
_options,
|
||||
NullLogger<ScheduleChecker>.Instance);
|
||||
}
|
||||
@@ -40,8 +50,8 @@ public class ScheduleCheckerTests
|
||||
public async Task GetPendingTasksAsync_WhenMassNeverRun_ReturnsMassTask()
|
||||
{
|
||||
// Arrange
|
||||
var config = CreateDataSourceConfig("WorkOrder", massEnabled: true, dailyEnabled: true, hourlyEnabled: true);
|
||||
_options.Value.DataSources.Add(config);
|
||||
var pipeline = CreatePipeline("WorkOrder", massInterval: 10080, dailyInterval: 1440, hourlyInterval: 60);
|
||||
_pipelines.Add(pipeline);
|
||||
_repository.GetLastDataUpdatesAsync(Arg.Any<CancellationToken>())
|
||||
.Returns(new Dictionary<string, DataUpdate>());
|
||||
|
||||
@@ -59,10 +69,8 @@ public class ScheduleCheckerTests
|
||||
public async Task GetPendingTasksAsync_WhenMassDue_ReturnsMassOverDaily()
|
||||
{
|
||||
// Arrange
|
||||
var config = CreateDataSourceConfig("WorkOrder",
|
||||
massEnabled: true, massInterval: 60,
|
||||
dailyEnabled: true, dailyInterval: 1440);
|
||||
_options.Value.DataSources.Add(config);
|
||||
var pipeline = CreatePipeline("WorkOrder", massInterval: 60, dailyInterval: 1440);
|
||||
_pipelines.Add(pipeline);
|
||||
|
||||
var now = DateTime.UtcNow;
|
||||
var lastMass = CreateDataUpdate("WorkOrder", UpdateTypes.Mass, now.AddMinutes(-120), success: true);
|
||||
@@ -85,11 +93,8 @@ public class ScheduleCheckerTests
|
||||
public async Task GetPendingTasksAsync_WhenMassNotDue_ChecksDailyAndHourly()
|
||||
{
|
||||
// Arrange
|
||||
var config = CreateDataSourceConfig("WorkOrder",
|
||||
massEnabled: true, massInterval: 10080, // weekly
|
||||
dailyEnabled: true, dailyInterval: 1440,
|
||||
hourlyEnabled: true, hourlyInterval: 60);
|
||||
_options.Value.DataSources.Add(config);
|
||||
var pipeline = CreatePipeline("WorkOrder", massInterval: 10080, dailyInterval: 1440, hourlyInterval: 60);
|
||||
_pipelines.Add(pipeline);
|
||||
|
||||
var now = DateTime.UtcNow;
|
||||
var lastMass = CreateDataUpdate("WorkOrder", UpdateTypes.Mass, now.AddHours(-1), success: true);
|
||||
@@ -114,11 +119,8 @@ public class ScheduleCheckerTests
|
||||
public async Task GetPendingTasksAsync_WhenDailyDue_ReturnsDailyOverHourly()
|
||||
{
|
||||
// Arrange
|
||||
var config = CreateDataSourceConfig("WorkOrder",
|
||||
massEnabled: true, massInterval: 10080,
|
||||
dailyEnabled: true, dailyInterval: 1440,
|
||||
hourlyEnabled: true, hourlyInterval: 60);
|
||||
_options.Value.DataSources.Add(config);
|
||||
var pipeline = CreatePipeline("WorkOrder", massInterval: 10080, dailyInterval: 1440, hourlyInterval: 60);
|
||||
_pipelines.Add(pipeline);
|
||||
|
||||
var now = DateTime.UtcNow;
|
||||
var lastMass = CreateDataUpdate("WorkOrder", UpdateTypes.Mass, now.AddDays(-1), success: true);
|
||||
@@ -145,11 +147,8 @@ public class ScheduleCheckerTests
|
||||
public async Task GetPendingTasksAsync_WhenOnlyHourlyDue_ReturnsHourly()
|
||||
{
|
||||
// Arrange
|
||||
var config = CreateDataSourceConfig("WorkOrder",
|
||||
massEnabled: true, massInterval: 10080,
|
||||
dailyEnabled: true, dailyInterval: 1440,
|
||||
hourlyEnabled: true, hourlyInterval: 60);
|
||||
_options.Value.DataSources.Add(config);
|
||||
var pipeline = CreatePipeline("WorkOrder", massInterval: 10080, dailyInterval: 1440, hourlyInterval: 60);
|
||||
_pipelines.Add(pipeline);
|
||||
|
||||
var now = DateTime.UtcNow;
|
||||
var lastMass = CreateDataUpdate("WorkOrder", UpdateTypes.Mass, now.AddDays(-1), success: true);
|
||||
@@ -181,10 +180,8 @@ public class ScheduleCheckerTests
|
||||
{
|
||||
// Arrange: LookbackMultiplier = 3, daily interval = 1440 min
|
||||
// MinimumDT = lastDaily.EndDT - (3 * 1440) = lastDaily.EndDT - 4320 min = 3 days before lastDaily
|
||||
var config = CreateDataSourceConfig("WorkOrder",
|
||||
massEnabled: true, massInterval: 10080,
|
||||
dailyEnabled: true, dailyInterval: 1440);
|
||||
_options.Value.DataSources.Add(config);
|
||||
var pipeline = CreatePipeline("WorkOrder", massInterval: 10080, dailyInterval: 1440);
|
||||
_pipelines.Add(pipeline);
|
||||
|
||||
var now = DateTime.UtcNow;
|
||||
var lastMass = CreateDataUpdate("WorkOrder", UpdateTypes.Mass, now.AddDays(-2), success: true);
|
||||
@@ -214,11 +211,8 @@ public class ScheduleCheckerTests
|
||||
public async Task GetPendingTasksAsync_HourlySync_UsesHourlyTimestampForMinimumDT()
|
||||
{
|
||||
// Arrange: Hourly uses its own timestamp and interval for MinimumDT calculation
|
||||
var config = CreateDataSourceConfig("WorkOrder",
|
||||
massEnabled: true, massInterval: 10080,
|
||||
dailyEnabled: true, dailyInterval: 1440,
|
||||
hourlyEnabled: true, hourlyInterval: 60);
|
||||
_options.Value.DataSources.Add(config);
|
||||
var pipeline = CreatePipeline("WorkOrder", massInterval: 10080, dailyInterval: 1440, hourlyInterval: 60);
|
||||
_pipelines.Add(pipeline);
|
||||
|
||||
var now = DateTime.UtcNow;
|
||||
var lastMass = CreateDataUpdate("WorkOrder", UpdateTypes.Mass, now.AddDays(-1), success: true);
|
||||
@@ -251,10 +245,8 @@ public class ScheduleCheckerTests
|
||||
{
|
||||
// Arrange: Test with multiplier = 5
|
||||
_options.Value.LookbackMultiplier = 5;
|
||||
var config = CreateDataSourceConfig("WorkOrder",
|
||||
massEnabled: true, massInterval: 10080,
|
||||
dailyEnabled: true, dailyInterval: 1440);
|
||||
_options.Value.DataSources.Add(config);
|
||||
var pipeline = CreatePipeline("WorkOrder", massInterval: 10080, dailyInterval: 1440);
|
||||
_pipelines.Add(pipeline);
|
||||
|
||||
var now = DateTime.UtcNow;
|
||||
var lastMass = CreateDataUpdate("WorkOrder", UpdateTypes.Mass, now.AddDays(-2), success: true);
|
||||
@@ -277,92 +269,15 @@ public class ScheduleCheckerTests
|
||||
|
||||
#endregion
|
||||
|
||||
#region Disabled Table Handling
|
||||
#region Manual-Only Pipelines
|
||||
|
||||
[Fact]
|
||||
public async Task GetPendingTasksAsync_DisabledDataSource_ReturnsNoTasks()
|
||||
public async Task GetPendingTasksAsync_ManualOnlyPipeline_ReturnsNoTasks()
|
||||
{
|
||||
// Arrange
|
||||
var config = CreateDataSourceConfig("WorkOrder", massEnabled: true);
|
||||
config.IsEnabled = false;
|
||||
_options.Value.DataSources.Add(config);
|
||||
|
||||
_repository.GetLastDataUpdatesAsync(Arg.Any<CancellationToken>())
|
||||
.Returns(new Dictionary<string, DataUpdate>());
|
||||
|
||||
// Act
|
||||
var tasks = await _sut.GetPendingTasksAsync();
|
||||
|
||||
// Assert
|
||||
tasks.ShouldBeEmpty();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetPendingTasksAsync_DisabledMassSchedule_SkipsMass()
|
||||
{
|
||||
// Arrange: Mass disabled, Daily enabled
|
||||
var config = CreateDataSourceConfig("WorkOrder",
|
||||
massEnabled: false,
|
||||
dailyEnabled: true, dailyInterval: 1440);
|
||||
_options.Value.DataSources.Add(config);
|
||||
|
||||
var now = DateTime.UtcNow;
|
||||
// Even with no mass ever run, if mass is disabled, should NOT require mass first
|
||||
// However, current logic requires mass before daily, so this tests that properly
|
||||
var lastMass = CreateDataUpdate("WorkOrder", UpdateTypes.Mass, now.AddDays(-1), success: true);
|
||||
|
||||
_repository.GetLastDataUpdatesAsync(Arg.Any<CancellationToken>())
|
||||
.Returns(new Dictionary<string, DataUpdate>
|
||||
{
|
||||
{ "WorkOrder_3", lastMass }
|
||||
});
|
||||
|
||||
// Act
|
||||
var tasks = await _sut.GetPendingTasksAsync();
|
||||
|
||||
// Assert: Should return Daily since mass is disabled but already ran before
|
||||
tasks.ShouldHaveSingleItem();
|
||||
tasks[0].UpdateType.ShouldBe(UpdateTypes.Daily);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetPendingTasksAsync_DisabledDailySchedule_SkipsDaily()
|
||||
{
|
||||
// Arrange
|
||||
var config = CreateDataSourceConfig("WorkOrder",
|
||||
massEnabled: true, massInterval: 10080,
|
||||
dailyEnabled: false,
|
||||
hourlyEnabled: true, hourlyInterval: 60);
|
||||
_options.Value.DataSources.Add(config);
|
||||
|
||||
var now = DateTime.UtcNow;
|
||||
var lastMass = CreateDataUpdate("WorkOrder", UpdateTypes.Mass, now.AddDays(-1), success: true);
|
||||
var lastHourly = CreateDataUpdate("WorkOrder", UpdateTypes.Hourly, now.AddHours(-2), success: true);
|
||||
|
||||
_repository.GetLastDataUpdatesAsync(Arg.Any<CancellationToken>())
|
||||
.Returns(new Dictionary<string, DataUpdate>
|
||||
{
|
||||
{ "WorkOrder_3", lastMass },
|
||||
{ "WorkOrder_1", lastHourly }
|
||||
});
|
||||
|
||||
// Act
|
||||
var tasks = await _sut.GetPendingTasksAsync();
|
||||
|
||||
// Assert: Should return Hourly, skipping Daily
|
||||
tasks.ShouldHaveSingleItem();
|
||||
tasks[0].UpdateType.ShouldBe(UpdateTypes.Hourly);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetPendingTasksAsync_AllSchedulesDisabled_ReturnsNoTasks()
|
||||
{
|
||||
// Arrange
|
||||
var config = CreateDataSourceConfig("WorkOrder",
|
||||
massEnabled: false,
|
||||
dailyEnabled: false,
|
||||
hourlyEnabled: false);
|
||||
_options.Value.DataSources.Add(config);
|
||||
var pipeline = CreatePipeline("WorkOrder", massInterval: 10080);
|
||||
pipeline.IsManualOnly = true;
|
||||
_pipelines.Add(pipeline);
|
||||
|
||||
_repository.GetLastDataUpdatesAsync(Arg.Any<CancellationToken>())
|
||||
.Returns(new Dictionary<string, DataUpdate>());
|
||||
@@ -382,11 +297,8 @@ public class ScheduleCheckerTests
|
||||
public async Task GetPendingTasksAsync_NoPriorUpdates_RequiresMassFirst()
|
||||
{
|
||||
// Arrange: Never synced before, all schedules enabled
|
||||
var config = CreateDataSourceConfig("WorkOrder",
|
||||
massEnabled: true, massInterval: 10080,
|
||||
dailyEnabled: true, dailyInterval: 1440,
|
||||
hourlyEnabled: true, hourlyInterval: 60);
|
||||
_options.Value.DataSources.Add(config);
|
||||
var pipeline = CreatePipeline("WorkOrder", massInterval: 10080, dailyInterval: 1440, hourlyInterval: 60);
|
||||
_pipelines.Add(pipeline);
|
||||
|
||||
_repository.GetLastDataUpdatesAsync(Arg.Any<CancellationToken>())
|
||||
.Returns(new Dictionary<string, DataUpdate>());
|
||||
@@ -404,10 +316,8 @@ public class ScheduleCheckerTests
|
||||
public async Task GetPendingTasksAsync_OnlyMassCompleted_DailyHasNullMinimumDT()
|
||||
{
|
||||
// Arrange: Mass completed, no daily yet
|
||||
var config = CreateDataSourceConfig("WorkOrder",
|
||||
massEnabled: true, massInterval: 10080,
|
||||
dailyEnabled: true, dailyInterval: 1440);
|
||||
_options.Value.DataSources.Add(config);
|
||||
var pipeline = CreatePipeline("WorkOrder", massInterval: 10080, dailyInterval: 1440);
|
||||
_pipelines.Add(pipeline);
|
||||
|
||||
var now = DateTime.UtcNow;
|
||||
var lastMass = CreateDataUpdate("WorkOrder", UpdateTypes.Mass, now.AddHours(-1), success: true);
|
||||
@@ -431,11 +341,8 @@ public class ScheduleCheckerTests
|
||||
public async Task GetPendingTasksAsync_NeverHadMass_DoesNotReturnDailyOrHourly()
|
||||
{
|
||||
// Arrange: Daily and Hourly enabled but no Mass ever run
|
||||
var config = CreateDataSourceConfig("WorkOrder",
|
||||
massEnabled: true, massInterval: 10080,
|
||||
dailyEnabled: true, dailyInterval: 1440,
|
||||
hourlyEnabled: true, hourlyInterval: 60);
|
||||
_options.Value.DataSources.Add(config);
|
||||
var pipeline = CreatePipeline("WorkOrder", massInterval: 10080, dailyInterval: 1440, hourlyInterval: 60);
|
||||
_pipelines.Add(pipeline);
|
||||
|
||||
_repository.GetLastDataUpdatesAsync(Arg.Any<CancellationToken>())
|
||||
.Returns(new Dictionary<string, DataUpdate>());
|
||||
@@ -456,9 +363,8 @@ public class ScheduleCheckerTests
|
||||
public async Task GetPendingTasksAsync_FailedMass_ReturnsMassAgain()
|
||||
{
|
||||
// Arrange
|
||||
var config = CreateDataSourceConfig("WorkOrder",
|
||||
massEnabled: true, massInterval: 10080);
|
||||
_options.Value.DataSources.Add(config);
|
||||
var pipeline = CreatePipeline("WorkOrder", massInterval: 10080);
|
||||
_pipelines.Add(pipeline);
|
||||
|
||||
var now = DateTime.UtcNow;
|
||||
var lastMass = CreateDataUpdate("WorkOrder", UpdateTypes.Mass, now.AddMinutes(-5), success: false);
|
||||
@@ -481,10 +387,8 @@ public class ScheduleCheckerTests
|
||||
public async Task GetPendingTasksAsync_FailedDaily_ReturnsDailyAgain()
|
||||
{
|
||||
// Arrange
|
||||
var config = CreateDataSourceConfig("WorkOrder",
|
||||
massEnabled: true, massInterval: 10080,
|
||||
dailyEnabled: true, dailyInterval: 1440);
|
||||
_options.Value.DataSources.Add(config);
|
||||
var pipeline = CreatePipeline("WorkOrder", massInterval: 10080, dailyInterval: 1440);
|
||||
_pipelines.Add(pipeline);
|
||||
|
||||
var now = DateTime.UtcNow;
|
||||
var lastMass = CreateDataUpdate("WorkOrder", UpdateTypes.Mass, now.AddDays(-1), success: true);
|
||||
@@ -509,11 +413,8 @@ public class ScheduleCheckerTests
|
||||
public async Task GetPendingTasksAsync_FailedHourly_ReturnsHourlyAgain()
|
||||
{
|
||||
// Arrange
|
||||
var config = CreateDataSourceConfig("WorkOrder",
|
||||
massEnabled: true, massInterval: 10080,
|
||||
dailyEnabled: true, dailyInterval: 1440,
|
||||
hourlyEnabled: true, hourlyInterval: 60);
|
||||
_options.Value.DataSources.Add(config);
|
||||
var pipeline = CreatePipeline("WorkOrder", massInterval: 10080, dailyInterval: 1440, hourlyInterval: 60);
|
||||
_pipelines.Add(pipeline);
|
||||
|
||||
var now = DateTime.UtcNow;
|
||||
var lastMass = CreateDataUpdate("WorkOrder", UpdateTypes.Mass, now.AddDays(-1), success: true);
|
||||
@@ -544,10 +445,10 @@ public class ScheduleCheckerTests
|
||||
public async Task GetPendingTasksAsync_MultipleTables_ReturnsTasksForEach()
|
||||
{
|
||||
// Arrange
|
||||
var config1 = CreateDataSourceConfig("WorkOrder", massEnabled: true, massInterval: 60);
|
||||
var config2 = CreateDataSourceConfig("LotUsage", massEnabled: true, massInterval: 60);
|
||||
_options.Value.DataSources.Add(config1);
|
||||
_options.Value.DataSources.Add(config2);
|
||||
var pipeline1 = CreatePipeline("WorkOrder", massInterval: 60);
|
||||
var pipeline2 = CreatePipeline("LotUsage", massInterval: 60);
|
||||
_pipelines.Add(pipeline1);
|
||||
_pipelines.Add(pipeline2);
|
||||
|
||||
_repository.GetLastDataUpdatesAsync(Arg.Any<CancellationToken>())
|
||||
.Returns(new Dictionary<string, DataUpdate>());
|
||||
@@ -565,13 +466,10 @@ public class ScheduleCheckerTests
|
||||
public async Task GetPendingTasksAsync_MultipleTables_DifferentSchedulesDue()
|
||||
{
|
||||
// Arrange
|
||||
var config1 = CreateDataSourceConfig("WorkOrder",
|
||||
massEnabled: true, massInterval: 10080,
|
||||
dailyEnabled: true, dailyInterval: 1440);
|
||||
var config2 = CreateDataSourceConfig("LotUsage",
|
||||
massEnabled: true, massInterval: 60);
|
||||
_options.Value.DataSources.Add(config1);
|
||||
_options.Value.DataSources.Add(config2);
|
||||
var pipeline1 = CreatePipeline("WorkOrder", massInterval: 10080, dailyInterval: 1440);
|
||||
var pipeline2 = CreatePipeline("LotUsage", massInterval: 60);
|
||||
_pipelines.Add(pipeline1);
|
||||
_pipelines.Add(pipeline2);
|
||||
|
||||
var now = DateTime.UtcNow;
|
||||
var lastMassWorkOrder = CreateDataUpdate("WorkOrder", UpdateTypes.Mass, now.AddDays(-1), success: true);
|
||||
@@ -603,11 +501,8 @@ public class ScheduleCheckerTests
|
||||
public async Task GetPendingTasksAsync_NothingDue_ReturnsEmptyList()
|
||||
{
|
||||
// Arrange
|
||||
var config = CreateDataSourceConfig("WorkOrder",
|
||||
massEnabled: true, massInterval: 10080,
|
||||
dailyEnabled: true, dailyInterval: 1440,
|
||||
hourlyEnabled: true, hourlyInterval: 60);
|
||||
_options.Value.DataSources.Add(config);
|
||||
var pipeline = CreatePipeline("WorkOrder", massInterval: 10080, dailyInterval: 1440, hourlyInterval: 60);
|
||||
_pipelines.Add(pipeline);
|
||||
|
||||
var now = DateTime.UtcNow;
|
||||
// All syncs completed recently
|
||||
@@ -631,9 +526,9 @@ public class ScheduleCheckerTests
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetPendingTasksAsync_NoDataSources_ReturnsEmptyList()
|
||||
public async Task GetPendingTasksAsync_NoPipelines_ReturnsEmptyList()
|
||||
{
|
||||
// Arrange: No data sources configured
|
||||
// Arrange: No pipelines configured
|
||||
_repository.GetLastDataUpdatesAsync(Arg.Any<CancellationToken>())
|
||||
.Returns(new Dictionary<string, DataUpdate>());
|
||||
|
||||
@@ -648,35 +543,29 @@ public class ScheduleCheckerTests
|
||||
|
||||
#region Helper Methods
|
||||
|
||||
private static DataSourceConfig CreateDataSourceConfig(
|
||||
string tableName,
|
||||
bool massEnabled = false,
|
||||
int massInterval = 10080,
|
||||
bool dailyEnabled = false,
|
||||
int dailyInterval = 1440,
|
||||
bool hourlyEnabled = false,
|
||||
int hourlyInterval = 60)
|
||||
private static EtlPipelineConfig CreatePipeline(
|
||||
string name,
|
||||
int? massInterval = null,
|
||||
int? dailyInterval = null,
|
||||
int? hourlyInterval = null)
|
||||
{
|
||||
return new DataSourceConfig
|
||||
return new EtlPipelineConfig
|
||||
{
|
||||
TableName = tableName,
|
||||
SourceSystem = "JDE",
|
||||
SourceData = tableName.ToUpper(),
|
||||
Name = name,
|
||||
IsEnabled = true,
|
||||
MassConfig = new ScheduleConfig
|
||||
IsManualOnly = false,
|
||||
MassSyncIntervalMinutes = massInterval,
|
||||
DailySyncIntervalMinutes = dailyInterval,
|
||||
HourlySyncIntervalMinutes = hourlyInterval,
|
||||
Source = new SourceElement
|
||||
{
|
||||
Enabled = massEnabled,
|
||||
IntervalMinutes = massInterval
|
||||
Connection = "jde",
|
||||
Query = "SELECT * FROM Test"
|
||||
},
|
||||
DailyConfig = new ScheduleConfig
|
||||
Destination = new DestinationElement
|
||||
{
|
||||
Enabled = dailyEnabled,
|
||||
IntervalMinutes = dailyInterval
|
||||
},
|
||||
HourlyConfig = new ScheduleConfig
|
||||
{
|
||||
Enabled = hourlyEnabled,
|
||||
IntervalMinutes = hourlyInterval
|
||||
Table = name,
|
||||
MatchColumns = ["Id"]
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
@@ -0,0 +1,434 @@
|
||||
using JdeScoping.DataSync.Configuration;
|
||||
using JdeScoping.DataSync.Options;
|
||||
using JdeScoping.DataSync.Services;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using NSubstitute;
|
||||
using Shouldly;
|
||||
using System.Text.Json;
|
||||
using Xunit;
|
||||
using MsOptions = Microsoft.Extensions.Options.Options;
|
||||
|
||||
namespace JdeScoping.DataSync.Tests.Services;
|
||||
|
||||
public class PipelineRegistryTests : IDisposable
|
||||
{
|
||||
private readonly string _testDirectory;
|
||||
private readonly IPipelineValidator _validator;
|
||||
private readonly IOptions<DataSyncOptions> _options;
|
||||
private readonly ILogger<PipelineRegistry> _logger;
|
||||
private readonly IHostEnvironment _environment;
|
||||
|
||||
public PipelineRegistryTests()
|
||||
{
|
||||
_testDirectory = Path.Combine(Path.GetTempPath(), $"PipelineRegistryTests_{Guid.NewGuid():N}");
|
||||
Directory.CreateDirectory(_testDirectory);
|
||||
|
||||
_validator = new PipelineValidator();
|
||||
_logger = Substitute.For<ILogger<PipelineRegistry>>();
|
||||
_environment = Substitute.For<IHostEnvironment>();
|
||||
_environment.ContentRootPath.Returns(_testDirectory);
|
||||
|
||||
_options = MsOptions.Create(new DataSyncOptions
|
||||
{
|
||||
PipelinesDirectory = "Pipelines",
|
||||
StrictPipelineValidation = false
|
||||
});
|
||||
|
||||
// Create the Pipelines subdirectory
|
||||
Directory.CreateDirectory(Path.Combine(_testDirectory, "Pipelines"));
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
if (Directory.Exists(_testDirectory))
|
||||
{
|
||||
Directory.Delete(_testDirectory, recursive: true);
|
||||
}
|
||||
}
|
||||
|
||||
private string PipelinesDir => Path.Combine(_testDirectory, "Pipelines");
|
||||
|
||||
#region Loading Tests
|
||||
|
||||
[Fact]
|
||||
public async Task LoadPipelines_ValidDirectory_LoadsAll()
|
||||
{
|
||||
// Arrange
|
||||
CreatePipelineFile("Pipeline1", true);
|
||||
CreatePipelineFile("Pipeline2", true);
|
||||
CreatePipelineFile("Pipeline3", true);
|
||||
|
||||
var registry = CreateRegistry();
|
||||
|
||||
// Act
|
||||
var result = await registry.ReloadAsync();
|
||||
|
||||
// Assert
|
||||
result.Success.ShouldBeTrue();
|
||||
result.PipelinesLoaded.ShouldBe(3);
|
||||
registry.GetAllPipelines().Count.ShouldBe(3);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task LoadPipelines_EmptyDirectory_ReturnsEmpty()
|
||||
{
|
||||
// Arrange - empty directory (Pipelines subdirectory is already created but empty)
|
||||
var registry = CreateRegistry();
|
||||
|
||||
// Act
|
||||
var result = await registry.ReloadAsync();
|
||||
|
||||
// Assert
|
||||
result.Success.ShouldBeTrue();
|
||||
result.PipelinesLoaded.ShouldBe(0);
|
||||
registry.GetAllPipelines().ShouldBeEmpty();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task LoadPipelines_OnlyLoadsJsonFiles()
|
||||
{
|
||||
// Arrange
|
||||
CreatePipelineFile("ValidPipeline", true);
|
||||
File.WriteAllText(Path.Combine(PipelinesDir, "readme.txt"), "Some text");
|
||||
File.WriteAllText(Path.Combine(PipelinesDir, "config.xml"), "<xml/>");
|
||||
|
||||
var registry = CreateRegistry();
|
||||
|
||||
// Act
|
||||
var result = await registry.ReloadAsync();
|
||||
|
||||
// Assert
|
||||
result.PipelinesLoaded.ShouldBe(1);
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Error Handling Tests
|
||||
|
||||
[Fact]
|
||||
public async Task LoadPipelines_InvalidJson_ReturnsError()
|
||||
{
|
||||
// Arrange
|
||||
File.WriteAllText(Path.Combine(PipelinesDir, "pipeline.Invalid.json"), "{ invalid json }");
|
||||
|
||||
var registry = CreateRegistry();
|
||||
|
||||
// Act
|
||||
var result = await registry.ReloadAsync();
|
||||
|
||||
// Assert
|
||||
result.Errors.ShouldContain(e => e.ErrorType.Contains("parse", StringComparison.OrdinalIgnoreCase));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task LoadPipelines_DuplicateNames_ReturnsError()
|
||||
{
|
||||
// Arrange - two files with same pipeline name
|
||||
CreatePipelineFile("DuplicateName", true, "pipeline.First.json");
|
||||
CreatePipelineFile("DuplicateName", true, "pipeline.Second.json");
|
||||
|
||||
var registry = CreateRegistry();
|
||||
|
||||
// Act
|
||||
var result = await registry.ReloadAsync();
|
||||
|
||||
// Assert
|
||||
result.Errors.ShouldContain(e => e.ErrorType.Contains("validation", StringComparison.OrdinalIgnoreCase)
|
||||
&& e.Messages.Any(m => m.Contains("Duplicate", StringComparison.OrdinalIgnoreCase)));
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Retrieval Tests
|
||||
|
||||
[Fact]
|
||||
public async Task GetPipeline_ByName_ReturnsCorrect()
|
||||
{
|
||||
// Arrange
|
||||
CreatePipelineFile("TestPipeline", true);
|
||||
var registry = CreateRegistry();
|
||||
await registry.ReloadAsync();
|
||||
|
||||
// Act
|
||||
var pipeline = registry.GetPipeline("TestPipeline");
|
||||
|
||||
// Assert
|
||||
pipeline.ShouldNotBeNull();
|
||||
pipeline.Name.ShouldBe("TestPipeline");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetPipeline_CaseInsensitive_ReturnsCorrect()
|
||||
{
|
||||
// Arrange
|
||||
CreatePipelineFile("MyPipeline", true);
|
||||
var registry = CreateRegistry();
|
||||
await registry.ReloadAsync();
|
||||
|
||||
// Act
|
||||
var pipeline = registry.GetPipeline("mypipeline");
|
||||
|
||||
// Assert
|
||||
pipeline.ShouldNotBeNull();
|
||||
pipeline.Name.ShouldBe("MyPipeline");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetPipeline_NotFound_ReturnsNull()
|
||||
{
|
||||
// Arrange
|
||||
CreatePipelineFile("ExistingPipeline", true);
|
||||
var registry = CreateRegistry();
|
||||
await registry.ReloadAsync();
|
||||
|
||||
// Act
|
||||
var pipeline = registry.GetPipeline("NonExistentPipeline");
|
||||
|
||||
// Assert
|
||||
pipeline.ShouldBeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetEnabledPipelines_OnlyReturnsEnabled()
|
||||
{
|
||||
// Arrange
|
||||
CreatePipelineFile("EnabledPipeline1", true);
|
||||
CreatePipelineFile("EnabledPipeline2", true);
|
||||
CreatePipelineFile("DisabledPipeline", false);
|
||||
|
||||
var registry = CreateRegistry();
|
||||
await registry.ReloadAsync();
|
||||
|
||||
// Act
|
||||
var enabledPipelines = registry.GetEnabledPipelines();
|
||||
|
||||
// Assert
|
||||
enabledPipelines.Count.ShouldBe(2);
|
||||
enabledPipelines.ShouldAllBe(p => p.IsEnabled);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetAllPipelines_IncludesDisabled()
|
||||
{
|
||||
// Arrange
|
||||
CreatePipelineFile("EnabledPipeline", true);
|
||||
CreatePipelineFile("DisabledPipeline", false);
|
||||
|
||||
var registry = CreateRegistry();
|
||||
await registry.ReloadAsync();
|
||||
|
||||
// Act
|
||||
var allPipelines = registry.GetAllPipelines();
|
||||
|
||||
// Assert
|
||||
allPipelines.Count.ShouldBe(2);
|
||||
allPipelines.ShouldContain(p => !p.IsEnabled);
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Reload Tests
|
||||
|
||||
[Fact]
|
||||
public async Task ReloadAsync_UpdatesSnapshot()
|
||||
{
|
||||
// Arrange
|
||||
CreatePipelineFile("Pipeline1", true);
|
||||
var registry = CreateRegistry();
|
||||
await registry.ReloadAsync();
|
||||
|
||||
// Add another pipeline
|
||||
CreatePipelineFile("Pipeline2", true);
|
||||
|
||||
// Act
|
||||
await registry.ReloadAsync();
|
||||
|
||||
// Assert
|
||||
registry.GetAllPipelines().Count.ShouldBe(2);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ReloadAsync_IncrementsVersion()
|
||||
{
|
||||
// Arrange
|
||||
CreatePipelineFile("Pipeline1", true);
|
||||
var registry = CreateRegistry();
|
||||
|
||||
// Act
|
||||
await registry.ReloadAsync();
|
||||
var version1 = registry.Version;
|
||||
|
||||
await registry.ReloadAsync();
|
||||
var version2 = registry.Version;
|
||||
|
||||
// Assert
|
||||
version2.ShouldBe(version1 + 1);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ReloadAsync_UpdatesLastLoadedAt()
|
||||
{
|
||||
// Arrange
|
||||
CreatePipelineFile("Pipeline1", true);
|
||||
var registry = CreateRegistry();
|
||||
|
||||
// Act
|
||||
var before = DateTime.UtcNow;
|
||||
await registry.ReloadAsync();
|
||||
var after = DateTime.UtcNow;
|
||||
|
||||
// Assert
|
||||
registry.LastLoadedAt.ShouldNotBeNull();
|
||||
registry.LastLoadedAt.Value.ShouldBeGreaterThanOrEqualTo(before.AddSeconds(-1));
|
||||
registry.LastLoadedAt.Value.ShouldBeLessThanOrEqualTo(after.AddSeconds(1));
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region IsValidPipelineAndSyncType Tests
|
||||
|
||||
[Fact]
|
||||
public async Task IsValidPipelineAndSyncType_ValidCombination_ReturnsTrue()
|
||||
{
|
||||
// Arrange
|
||||
CreatePipelineFile("TestPipeline", true, massSyncInterval: 1440, dailySyncInterval: 60);
|
||||
var registry = CreateRegistry();
|
||||
await registry.ReloadAsync();
|
||||
|
||||
// Act & Assert
|
||||
registry.IsValidPipelineAndSyncType("TestPipeline", "mass").ShouldBeTrue();
|
||||
registry.IsValidPipelineAndSyncType("TestPipeline", "daily").ShouldBeTrue();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task IsValidPipelineAndSyncType_UnsupportedSyncType_ReturnsFalse()
|
||||
{
|
||||
// Arrange - only mass sync
|
||||
CreatePipelineFile("TestPipeline", true, massSyncInterval: 1440);
|
||||
var registry = CreateRegistry();
|
||||
await registry.ReloadAsync();
|
||||
|
||||
// Act & Assert
|
||||
registry.IsValidPipelineAndSyncType("TestPipeline", "hourly").ShouldBeFalse();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task IsValidPipelineAndSyncType_UnknownPipeline_ReturnsFalse()
|
||||
{
|
||||
// Arrange
|
||||
CreatePipelineFile("TestPipeline", true);
|
||||
var registry = CreateRegistry();
|
||||
await registry.ReloadAsync();
|
||||
|
||||
// Act & Assert
|
||||
registry.IsValidPipelineAndSyncType("UnknownPipeline", "mass").ShouldBeFalse();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task IsValidPipelineAndSyncType_DisabledPipeline_ReturnsFalse()
|
||||
{
|
||||
// Arrange
|
||||
CreatePipelineFile("DisabledPipeline", false, massSyncInterval: 1440);
|
||||
var registry = CreateRegistry();
|
||||
await registry.ReloadAsync();
|
||||
|
||||
// Act & Assert
|
||||
registry.IsValidPipelineAndSyncType("DisabledPipeline", "mass").ShouldBeFalse();
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Thread Safety Tests
|
||||
|
||||
[Fact]
|
||||
public async Task ReadOperations_ThreadSafe()
|
||||
{
|
||||
// Arrange
|
||||
for (int i = 0; i < 10; i++)
|
||||
{
|
||||
CreatePipelineFile($"Pipeline{i}", true);
|
||||
}
|
||||
|
||||
var registry = CreateRegistry();
|
||||
await registry.ReloadAsync();
|
||||
|
||||
// Act - multiple concurrent reads
|
||||
var tasks = Enumerable.Range(0, 100).Select(_ => Task.Run(() =>
|
||||
{
|
||||
var all = registry.GetAllPipelines();
|
||||
var enabled = registry.GetEnabledPipelines();
|
||||
var specific = registry.GetPipeline("Pipeline5");
|
||||
return all.Count + enabled.Count + (specific != null ? 1 : 0);
|
||||
}));
|
||||
|
||||
// Assert - no exceptions
|
||||
var results = await Task.WhenAll(tasks);
|
||||
results.ShouldAllBe(r => r > 0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ConcurrentReloads_Serialized()
|
||||
{
|
||||
// Arrange
|
||||
CreatePipelineFile("Pipeline1", true);
|
||||
var registry = CreateRegistry();
|
||||
|
||||
// Act - multiple concurrent reloads
|
||||
var tasks = Enumerable.Range(0, 10).Select(_ => registry.ReloadAsync());
|
||||
|
||||
// Assert - no exceptions and final state is valid
|
||||
await Task.WhenAll(tasks);
|
||||
registry.GetAllPipelines().Count.ShouldBe(1);
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Helper Methods
|
||||
|
||||
private PipelineRegistry CreateRegistry() =>
|
||||
new PipelineRegistry(_options, _validator, _logger, _environment);
|
||||
|
||||
private void CreatePipelineFile(
|
||||
string name,
|
||||
bool isEnabled,
|
||||
string? fileName = null,
|
||||
int? massSyncInterval = 1440,
|
||||
int? dailySyncInterval = null,
|
||||
int? hourlySyncInterval = null)
|
||||
{
|
||||
var pipeline = new
|
||||
{
|
||||
name,
|
||||
isEnabled,
|
||||
isManualOnly = !massSyncInterval.HasValue && !dailySyncInterval.HasValue && !hourlySyncInterval.HasValue && !isEnabled,
|
||||
massSyncIntervalMinutes = massSyncInterval,
|
||||
dailySyncIntervalMinutes = dailySyncInterval,
|
||||
hourlySyncIntervalMinutes = hourlySyncInterval,
|
||||
source = new
|
||||
{
|
||||
connection = "jde",
|
||||
query = "SELECT * FROM TestTable"
|
||||
},
|
||||
destination = new
|
||||
{
|
||||
table = $"{name}_Table",
|
||||
matchColumns = new[] { "Id" }
|
||||
}
|
||||
};
|
||||
|
||||
var json = JsonSerializer.Serialize(pipeline, new JsonSerializerOptions
|
||||
{
|
||||
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
|
||||
WriteIndented = true,
|
||||
DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull
|
||||
});
|
||||
|
||||
var actualFileName = fileName ?? $"pipeline.{name}.json";
|
||||
var filePath = Path.Combine(PipelinesDir, actualFileName);
|
||||
File.WriteAllText(filePath, json);
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
@@ -0,0 +1,483 @@
|
||||
using JdeScoping.DataSync.Configuration;
|
||||
using JdeScoping.DataSync.Services;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
|
||||
namespace JdeScoping.DataSync.Tests.Services;
|
||||
|
||||
public class PipelineValidatorTests
|
||||
{
|
||||
private readonly IPipelineValidator _validator;
|
||||
|
||||
public PipelineValidatorTests()
|
||||
{
|
||||
_validator = new PipelineValidator();
|
||||
}
|
||||
|
||||
#region Name/Filename Matching
|
||||
|
||||
[Fact]
|
||||
public void Validate_NameMatchesFilename_Passes()
|
||||
{
|
||||
// Arrange
|
||||
var pipeline = CreateValidPipeline("TestPipeline");
|
||||
|
||||
// Act
|
||||
var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json");
|
||||
|
||||
// Assert
|
||||
result.IsValid.ShouldBeTrue();
|
||||
result.Errors.ShouldBeEmpty();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Validate_NameMismatchFilename_Fails()
|
||||
{
|
||||
// Arrange
|
||||
var pipeline = CreateValidPipeline("WrongName");
|
||||
|
||||
// Act
|
||||
var result = _validator.Validate(pipeline, "pipeline.CorrectName.json");
|
||||
|
||||
// Assert
|
||||
result.IsValid.ShouldBeFalse();
|
||||
result.Errors.ShouldContain(e => e.Contains("does not match filename", StringComparison.OrdinalIgnoreCase));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Validate_NameCaseInsensitive_Passes()
|
||||
{
|
||||
// Arrange
|
||||
var pipeline = CreateValidPipeline("testpipeline");
|
||||
|
||||
// Act
|
||||
var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json");
|
||||
|
||||
// Assert
|
||||
result.IsValid.ShouldBeTrue();
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Source Validation
|
||||
|
||||
[Fact]
|
||||
public void Validate_MissingSource_Fails()
|
||||
{
|
||||
// Arrange
|
||||
var pipeline = new EtlPipelineConfig
|
||||
{
|
||||
Name = "TestPipeline",
|
||||
IsEnabled = true,
|
||||
MassSyncIntervalMinutes = 1440,
|
||||
Source = null!,
|
||||
Destination = CreateValidDestination()
|
||||
};
|
||||
|
||||
// Act
|
||||
var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json");
|
||||
|
||||
// Assert
|
||||
result.IsValid.ShouldBeFalse();
|
||||
result.Errors.ShouldContain(e => e.Contains("Source is required", StringComparison.OrdinalIgnoreCase));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Validate_MissingSourceConnection_Fails()
|
||||
{
|
||||
// Arrange
|
||||
var pipeline = new EtlPipelineConfig
|
||||
{
|
||||
Name = "TestPipeline",
|
||||
IsEnabled = true,
|
||||
MassSyncIntervalMinutes = 1440,
|
||||
Source = new SourceElement
|
||||
{
|
||||
Connection = "",
|
||||
Query = "SELECT * FROM table"
|
||||
},
|
||||
Destination = CreateValidDestination()
|
||||
};
|
||||
|
||||
// Act
|
||||
var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json");
|
||||
|
||||
// Assert
|
||||
result.IsValid.ShouldBeFalse();
|
||||
result.Errors.ShouldContain(e => e.Contains("Connection is required", StringComparison.OrdinalIgnoreCase));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Validate_InvalidConnection_Fails()
|
||||
{
|
||||
// Arrange
|
||||
var pipeline = new EtlPipelineConfig
|
||||
{
|
||||
Name = "TestPipeline",
|
||||
IsEnabled = true,
|
||||
MassSyncIntervalMinutes = 1440,
|
||||
Source = new SourceElement
|
||||
{
|
||||
Connection = "invalid_db",
|
||||
Query = "SELECT * FROM table"
|
||||
},
|
||||
Destination = CreateValidDestination()
|
||||
};
|
||||
|
||||
// Act
|
||||
var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json");
|
||||
|
||||
// Assert
|
||||
result.IsValid.ShouldBeFalse();
|
||||
result.Errors.ShouldContain(e => e.Contains("not valid", StringComparison.OrdinalIgnoreCase));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Validate_ValidConnections_Pass()
|
||||
{
|
||||
// Arrange & Act & Assert
|
||||
foreach (var connection in new[] { "jde", "cms", "giw", "lotfinder" })
|
||||
{
|
||||
var pipeline = CreateValidPipeline("TestPipeline");
|
||||
pipeline.Source.Connection = connection;
|
||||
|
||||
var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json");
|
||||
result.IsValid.ShouldBeTrue($"Connection '{connection}' should be valid");
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Validate_MissingSourceQuery_Fails()
|
||||
{
|
||||
// Arrange
|
||||
var pipeline = new EtlPipelineConfig
|
||||
{
|
||||
Name = "TestPipeline",
|
||||
IsEnabled = true,
|
||||
MassSyncIntervalMinutes = 1440,
|
||||
Source = new SourceElement
|
||||
{
|
||||
Connection = "jde",
|
||||
Query = ""
|
||||
},
|
||||
Destination = CreateValidDestination()
|
||||
};
|
||||
|
||||
// Act
|
||||
var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json");
|
||||
|
||||
// Assert
|
||||
result.IsValid.ShouldBeFalse();
|
||||
result.Errors.ShouldContain(e => e.Contains("Query is required", StringComparison.OrdinalIgnoreCase));
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Destination Validation
|
||||
|
||||
[Fact]
|
||||
public void Validate_MissingDestination_Fails()
|
||||
{
|
||||
// Arrange
|
||||
var pipeline = new EtlPipelineConfig
|
||||
{
|
||||
Name = "TestPipeline",
|
||||
IsEnabled = true,
|
||||
MassSyncIntervalMinutes = 1440,
|
||||
Source = CreateValidSource(),
|
||||
Destination = null!
|
||||
};
|
||||
|
||||
// Act
|
||||
var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json");
|
||||
|
||||
// Assert
|
||||
result.IsValid.ShouldBeFalse();
|
||||
result.Errors.ShouldContain(e => e.Contains("Destination is required", StringComparison.OrdinalIgnoreCase));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Validate_MissingDestinationTable_Fails()
|
||||
{
|
||||
// Arrange
|
||||
var pipeline = new EtlPipelineConfig
|
||||
{
|
||||
Name = "TestPipeline",
|
||||
IsEnabled = true,
|
||||
MassSyncIntervalMinutes = 1440,
|
||||
Source = CreateValidSource(),
|
||||
Destination = new DestinationElement
|
||||
{
|
||||
Table = "",
|
||||
MatchColumns = ["Id"]
|
||||
}
|
||||
};
|
||||
|
||||
// Act
|
||||
var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json");
|
||||
|
||||
// Assert
|
||||
result.IsValid.ShouldBeFalse();
|
||||
result.Errors.ShouldContain(e => e.Contains("Table is required", StringComparison.OrdinalIgnoreCase));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Validate_EmptyMatchColumns_Fails()
|
||||
{
|
||||
// Arrange
|
||||
var pipeline = new EtlPipelineConfig
|
||||
{
|
||||
Name = "TestPipeline",
|
||||
IsEnabled = true,
|
||||
MassSyncIntervalMinutes = 1440,
|
||||
Source = CreateValidSource(),
|
||||
Destination = new DestinationElement
|
||||
{
|
||||
Table = "TestTable",
|
||||
MatchColumns = []
|
||||
}
|
||||
};
|
||||
|
||||
// Act
|
||||
var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json");
|
||||
|
||||
// Assert
|
||||
result.IsValid.ShouldBeFalse();
|
||||
result.Errors.ShouldContain(e => e.Contains("MatchColumns", StringComparison.OrdinalIgnoreCase));
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Interval Validation
|
||||
|
||||
[Fact]
|
||||
public void Validate_EnabledWithoutAnyInterval_Fails()
|
||||
{
|
||||
// Arrange
|
||||
var pipeline = new EtlPipelineConfig
|
||||
{
|
||||
Name = "TestPipeline",
|
||||
IsEnabled = true,
|
||||
IsManualOnly = false,
|
||||
// No intervals set
|
||||
Source = CreateValidSource(),
|
||||
Destination = CreateValidDestination()
|
||||
};
|
||||
|
||||
// Act
|
||||
var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json");
|
||||
|
||||
// Assert
|
||||
result.IsValid.ShouldBeFalse();
|
||||
result.Errors.ShouldContain(e => e.Contains("At least one sync interval", StringComparison.OrdinalIgnoreCase));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Validate_ManualOnlyWithoutInterval_Passes()
|
||||
{
|
||||
// Arrange
|
||||
var pipeline = new EtlPipelineConfig
|
||||
{
|
||||
Name = "TestPipeline",
|
||||
IsEnabled = true,
|
||||
IsManualOnly = true,
|
||||
// No intervals set - ok for manual-only
|
||||
Source = CreateValidSource(),
|
||||
Destination = CreateValidDestination()
|
||||
};
|
||||
|
||||
// Act
|
||||
var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json");
|
||||
|
||||
// Assert
|
||||
result.IsValid.ShouldBeTrue();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Validate_DisabledWithoutInterval_Passes()
|
||||
{
|
||||
// Arrange
|
||||
var pipeline = new EtlPipelineConfig
|
||||
{
|
||||
Name = "TestPipeline",
|
||||
IsEnabled = false,
|
||||
// No intervals set - ok for disabled
|
||||
Source = CreateValidSource(),
|
||||
Destination = CreateValidDestination()
|
||||
};
|
||||
|
||||
// Act
|
||||
var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json");
|
||||
|
||||
// Assert
|
||||
result.IsValid.ShouldBeTrue();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Validate_ZeroMassInterval_Fails()
|
||||
{
|
||||
// Arrange
|
||||
var pipeline = CreateValidPipeline("TestPipeline");
|
||||
pipeline.MassSyncIntervalMinutes = 0;
|
||||
|
||||
// Act
|
||||
var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json");
|
||||
|
||||
// Assert
|
||||
result.IsValid.ShouldBeFalse();
|
||||
result.Errors.ShouldContain(e => e.Contains("MassSyncIntervalMinutes must be greater than 0", StringComparison.OrdinalIgnoreCase));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Validate_NegativeInterval_Fails()
|
||||
{
|
||||
// Arrange
|
||||
var pipeline = CreateValidPipeline("TestPipeline");
|
||||
pipeline.DailySyncIntervalMinutes = -60;
|
||||
|
||||
// Act
|
||||
var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json");
|
||||
|
||||
// Assert
|
||||
result.IsValid.ShouldBeFalse();
|
||||
result.Errors.ShouldContain(e => e.Contains("DailySyncIntervalMinutes must be greater than 0", StringComparison.OrdinalIgnoreCase));
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Warning Cases
|
||||
|
||||
[Fact]
|
||||
public void Validate_HourlyWithoutDaily_ReturnsWarning()
|
||||
{
|
||||
// Arrange
|
||||
var pipeline = new EtlPipelineConfig
|
||||
{
|
||||
Name = "TestPipeline",
|
||||
IsEnabled = true,
|
||||
HourlySyncIntervalMinutes = 15,
|
||||
// No daily interval
|
||||
Source = CreateValidSource(),
|
||||
Destination = CreateValidDestination()
|
||||
};
|
||||
|
||||
// Act
|
||||
var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json");
|
||||
|
||||
// Assert
|
||||
result.IsValid.ShouldBeTrue(); // Warnings don't fail validation
|
||||
result.Warnings.ShouldContain(w => w.Contains("daily", StringComparison.OrdinalIgnoreCase));
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Script Validation
|
||||
|
||||
[Fact]
|
||||
public void Validate_PreScriptWithEmptyScript_Fails()
|
||||
{
|
||||
// Arrange
|
||||
var pipeline = CreateValidPipeline("TestPipeline");
|
||||
pipeline.PreScripts = [new ScriptElement { Script = "" }];
|
||||
|
||||
// Act
|
||||
var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json");
|
||||
|
||||
// Assert
|
||||
result.IsValid.ShouldBeFalse();
|
||||
result.Errors.ShouldContain(e => e.Contains("PreScripts", StringComparison.OrdinalIgnoreCase));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Validate_PostScriptWithEmptyScript_Fails()
|
||||
{
|
||||
// Arrange
|
||||
var pipeline = CreateValidPipeline("TestPipeline");
|
||||
pipeline.PostScripts = [new ScriptElement { Script = "" }];
|
||||
|
||||
// Act
|
||||
var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json");
|
||||
|
||||
// Assert
|
||||
result.IsValid.ShouldBeFalse();
|
||||
result.Errors.ShouldContain(e => e.Contains("PostScripts", StringComparison.OrdinalIgnoreCase));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Validate_ValidScripts_Passes()
|
||||
{
|
||||
// Arrange
|
||||
var pipeline = CreateValidPipeline("TestPipeline");
|
||||
pipeline.PreScripts = [new ScriptElement { Script = "TRUNCATE TABLE Staging" }];
|
||||
pipeline.PostScripts = [new ScriptElement { Script = "EXEC ProcessData" }];
|
||||
|
||||
// Act
|
||||
var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json");
|
||||
|
||||
// Assert
|
||||
result.IsValid.ShouldBeTrue();
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Complete Valid Pipeline
|
||||
|
||||
[Fact]
|
||||
public void Validate_CompleteValidPipeline_Passes()
|
||||
{
|
||||
// Arrange
|
||||
var pipeline = new EtlPipelineConfig
|
||||
{
|
||||
Name = "WorkOrder_Curr",
|
||||
IsEnabled = true,
|
||||
MassSyncIntervalMinutes = 1440,
|
||||
DailySyncIntervalMinutes = 60,
|
||||
HourlySyncIntervalMinutes = 15,
|
||||
Source = new SourceElement
|
||||
{
|
||||
Connection = "jde",
|
||||
Query = "SELECT * FROM WorkOrders WHERE ModDate > @lastSync",
|
||||
MassQuery = "SELECT * FROM WorkOrders"
|
||||
},
|
||||
Destination = new DestinationElement
|
||||
{
|
||||
Table = "WorkOrder_Curr",
|
||||
MatchColumns = ["OrderNumber"]
|
||||
}
|
||||
};
|
||||
|
||||
// Act
|
||||
var result = _validator.Validate(pipeline, "pipeline.WorkOrder_Curr.json");
|
||||
|
||||
// Assert
|
||||
result.IsValid.ShouldBeTrue();
|
||||
result.Errors.ShouldBeEmpty();
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Helper Methods
|
||||
|
||||
private static EtlPipelineConfig CreateValidPipeline(string name) => new()
|
||||
{
|
||||
Name = name,
|
||||
IsEnabled = true,
|
||||
MassSyncIntervalMinutes = 1440,
|
||||
Source = CreateValidSource(),
|
||||
Destination = CreateValidDestination()
|
||||
};
|
||||
|
||||
private static SourceElement CreateValidSource() => new()
|
||||
{
|
||||
Connection = "jde",
|
||||
Query = "SELECT * FROM TestTable"
|
||||
};
|
||||
|
||||
private static DestinationElement CreateValidDestination() => new()
|
||||
{
|
||||
Table = "TestTable",
|
||||
MatchColumns = ["Id"]
|
||||
};
|
||||
|
||||
#endregion
|
||||
}
|
||||
Reference in New Issue
Block a user