dohertj2/jdescopingtool
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: implement ETL pipeline redesign and ConfigManager improvements

Browse Source 
- Add pipeline registry with JSON-based configuration and hot-reload support
- Implement manual sync request feature with API, client UI, and database
- Improve ConfigManager: connection string dropdown in pipeline editor,
  step delete/reorder functionality, and fix JSON parsing for ConnectionStrings
This commit is contained in:
Joseph Doherty
2026-01-22 17:48:33 -05:00
parent 5a332232d0
commit 29ac56006d
82 changed files with 6257 additions and 296 deletions
Download Patch File Download Diff File Expand all files Collapse all files
NEW/tests/JdeScoping.DataSync.Tests/ScheduleCheckerTests.cs
+72 -183
View File
@@ -1,6 +1,7 @@
using JdeScoping.Core.Models;
using JdeScoping.Core.Models.Enums;
using JdeScoping.Core.Models.Infrastructure;
using JdeScoping.DataSync.Configuration;
using JdeScoping.DataSync.Options;
using JdeScoping.DataSync.Contracts;
using JdeScoping.DataSync.Services;
@@ -17,19 +18,28 @@ namespace JdeScoping.DataSync.Tests;
public class ScheduleCheckerTests
{
private readonly IDataUpdateRepository _repository;
private readonly IPipelineRegistry _pipelineRegistry;
private readonly IOptions<DataSyncOptions> _options;
private readonly List<EtlPipelineConfig> _pipelines;
private readonly ScheduleChecker _sut;
public ScheduleCheckerTests()
{
_repository = Substitute.For<IDataUpdateRepository>();
_pipelineRegistry = Substitute.For<IPipelineRegistry>();
_pipelines = [];
_options = Microsoft.Extensions.Options.Options.Create(new DataSyncOptions
{
LookbackMultiplier = 3,
DataSources = []
});
// Setup pipeline registry to return our pipeline list
_pipelineRegistry.GetEnabledPipelines().Returns(_ => _pipelines);
_sut = new ScheduleChecker(
_repository,
_pipelineRegistry,
_options,
NullLogger<ScheduleChecker>.Instance);
}
@@ -40,8 +50,8 @@ public class ScheduleCheckerTests
public async Task GetPendingTasksAsync_WhenMassNeverRun_ReturnsMassTask()
{
// Arrange
var config = CreateDataSourceConfig("WorkOrder", massEnabled: true, dailyEnabled: true, hourlyEnabled: true);
_options.Value.DataSources.Add(config);
var pipeline = CreatePipeline("WorkOrder", massInterval: 10080, dailyInterval: 1440, hourlyInterval: 60);
_pipelines.Add(pipeline);
_repository.GetLastDataUpdatesAsync(Arg.Any<CancellationToken>())
.Returns(new Dictionary<string, DataUpdate>());
@@ -59,10 +69,8 @@ public class ScheduleCheckerTests
public async Task GetPendingTasksAsync_WhenMassDue_ReturnsMassOverDaily()
{
// Arrange
var config = CreateDataSourceConfig("WorkOrder",
massEnabled: true, massInterval: 60,
dailyEnabled: true, dailyInterval: 1440);
_options.Value.DataSources.Add(config);
var pipeline = CreatePipeline("WorkOrder", massInterval: 60, dailyInterval: 1440);
_pipelines.Add(pipeline);
var now = DateTime.UtcNow;
var lastMass = CreateDataUpdate("WorkOrder", UpdateTypes.Mass, now.AddMinutes(-120), success: true);
@@ -85,11 +93,8 @@ public class ScheduleCheckerTests
public async Task GetPendingTasksAsync_WhenMassNotDue_ChecksDailyAndHourly()
{
// Arrange
var config = CreateDataSourceConfig("WorkOrder",
massEnabled: true, massInterval: 10080, // weekly
dailyEnabled: true, dailyInterval: 1440,
hourlyEnabled: true, hourlyInterval: 60);
_options.Value.DataSources.Add(config);
var pipeline = CreatePipeline("WorkOrder", massInterval: 10080, dailyInterval: 1440, hourlyInterval: 60);
_pipelines.Add(pipeline);
var now = DateTime.UtcNow;
var lastMass = CreateDataUpdate("WorkOrder", UpdateTypes.Mass, now.AddHours(-1), success: true);
@@ -114,11 +119,8 @@ public class ScheduleCheckerTests
public async Task GetPendingTasksAsync_WhenDailyDue_ReturnsDailyOverHourly()
{
// Arrange
var config = CreateDataSourceConfig("WorkOrder",
massEnabled: true, massInterval: 10080,
dailyEnabled: true, dailyInterval: 1440,
hourlyEnabled: true, hourlyInterval: 60);
_options.Value.DataSources.Add(config);
var pipeline = CreatePipeline("WorkOrder", massInterval: 10080, dailyInterval: 1440, hourlyInterval: 60);
_pipelines.Add(pipeline);
var now = DateTime.UtcNow;
var lastMass = CreateDataUpdate("WorkOrder", UpdateTypes.Mass, now.AddDays(-1), success: true);
@@ -145,11 +147,8 @@ public class ScheduleCheckerTests
public async Task GetPendingTasksAsync_WhenOnlyHourlyDue_ReturnsHourly()
{
// Arrange
var config = CreateDataSourceConfig("WorkOrder",
massEnabled: true, massInterval: 10080,
dailyEnabled: true, dailyInterval: 1440,
hourlyEnabled: true, hourlyInterval: 60);
_options.Value.DataSources.Add(config);
var pipeline = CreatePipeline("WorkOrder", massInterval: 10080, dailyInterval: 1440, hourlyInterval: 60);
_pipelines.Add(pipeline);
var now = DateTime.UtcNow;
var lastMass = CreateDataUpdate("WorkOrder", UpdateTypes.Mass, now.AddDays(-1), success: true);
@@ -181,10 +180,8 @@ public class ScheduleCheckerTests
{
// Arrange: LookbackMultiplier = 3, daily interval = 1440 min
// MinimumDT = lastDaily.EndDT - (3 * 1440) = lastDaily.EndDT - 4320 min = 3 days before lastDaily
var config = CreateDataSourceConfig("WorkOrder",
massEnabled: true, massInterval: 10080,
dailyEnabled: true, dailyInterval: 1440);
_options.Value.DataSources.Add(config);
var pipeline = CreatePipeline("WorkOrder", massInterval: 10080, dailyInterval: 1440);
_pipelines.Add(pipeline);
var now = DateTime.UtcNow;
var lastMass = CreateDataUpdate("WorkOrder", UpdateTypes.Mass, now.AddDays(-2), success: true);
@@ -214,11 +211,8 @@ public class ScheduleCheckerTests
public async Task GetPendingTasksAsync_HourlySync_UsesHourlyTimestampForMinimumDT()
{
// Arrange: Hourly uses its own timestamp and interval for MinimumDT calculation
var config = CreateDataSourceConfig("WorkOrder",
massEnabled: true, massInterval: 10080,
dailyEnabled: true, dailyInterval: 1440,
hourlyEnabled: true, hourlyInterval: 60);
_options.Value.DataSources.Add(config);
var pipeline = CreatePipeline("WorkOrder", massInterval: 10080, dailyInterval: 1440, hourlyInterval: 60);
_pipelines.Add(pipeline);
var now = DateTime.UtcNow;
var lastMass = CreateDataUpdate("WorkOrder", UpdateTypes.Mass, now.AddDays(-1), success: true);
@@ -251,10 +245,8 @@ public class ScheduleCheckerTests
{
// Arrange: Test with multiplier = 5
_options.Value.LookbackMultiplier = 5;
var config = CreateDataSourceConfig("WorkOrder",
massEnabled: true, massInterval: 10080,
dailyEnabled: true, dailyInterval: 1440);
_options.Value.DataSources.Add(config);
var pipeline = CreatePipeline("WorkOrder", massInterval: 10080, dailyInterval: 1440);
_pipelines.Add(pipeline);
var now = DateTime.UtcNow;
var lastMass = CreateDataUpdate("WorkOrder", UpdateTypes.Mass, now.AddDays(-2), success: true);
@@ -277,92 +269,15 @@ public class ScheduleCheckerTests
#endregion
#region Disabled Table Handling
#region Manual-Only Pipelines
[Fact]
public async Task GetPendingTasksAsync_DisabledDataSource_ReturnsNoTasks()
public async Task GetPendingTasksAsync_ManualOnlyPipeline_ReturnsNoTasks()
{
// Arrange
var config = CreateDataSourceConfig("WorkOrder", massEnabled: true);
config.IsEnabled = false;
_options.Value.DataSources.Add(config);
_repository.GetLastDataUpdatesAsync(Arg.Any<CancellationToken>())
.Returns(new Dictionary<string, DataUpdate>());
// Act
var tasks = await _sut.GetPendingTasksAsync();
// Assert
tasks.ShouldBeEmpty();
}
[Fact]
public async Task GetPendingTasksAsync_DisabledMassSchedule_SkipsMass()
{
// Arrange: Mass disabled, Daily enabled
var config = CreateDataSourceConfig("WorkOrder",
massEnabled: false,
dailyEnabled: true, dailyInterval: 1440);
_options.Value.DataSources.Add(config);
var now = DateTime.UtcNow;
// Even with no mass ever run, if mass is disabled, should NOT require mass first
// However, current logic requires mass before daily, so this tests that properly
var lastMass = CreateDataUpdate("WorkOrder", UpdateTypes.Mass, now.AddDays(-1), success: true);
_repository.GetLastDataUpdatesAsync(Arg.Any<CancellationToken>())
.Returns(new Dictionary<string, DataUpdate>
{
{ "WorkOrder_3", lastMass }
});
// Act
var tasks = await _sut.GetPendingTasksAsync();
// Assert: Should return Daily since mass is disabled but already ran before
tasks.ShouldHaveSingleItem();
tasks[0].UpdateType.ShouldBe(UpdateTypes.Daily);
}
[Fact]
public async Task GetPendingTasksAsync_DisabledDailySchedule_SkipsDaily()
{
// Arrange
var config = CreateDataSourceConfig("WorkOrder",
massEnabled: true, massInterval: 10080,
dailyEnabled: false,
hourlyEnabled: true, hourlyInterval: 60);
_options.Value.DataSources.Add(config);
var now = DateTime.UtcNow;
var lastMass = CreateDataUpdate("WorkOrder", UpdateTypes.Mass, now.AddDays(-1), success: true);
var lastHourly = CreateDataUpdate("WorkOrder", UpdateTypes.Hourly, now.AddHours(-2), success: true);
_repository.GetLastDataUpdatesAsync(Arg.Any<CancellationToken>())
.Returns(new Dictionary<string, DataUpdate>
{
{ "WorkOrder_3", lastMass },
{ "WorkOrder_1", lastHourly }
});
// Act
var tasks = await _sut.GetPendingTasksAsync();
// Assert: Should return Hourly, skipping Daily
tasks.ShouldHaveSingleItem();
tasks[0].UpdateType.ShouldBe(UpdateTypes.Hourly);
}
[Fact]
public async Task GetPendingTasksAsync_AllSchedulesDisabled_ReturnsNoTasks()
{
// Arrange
var config = CreateDataSourceConfig("WorkOrder",
massEnabled: false,
dailyEnabled: false,
hourlyEnabled: false);
_options.Value.DataSources.Add(config);
var pipeline = CreatePipeline("WorkOrder", massInterval: 10080);
pipeline.IsManualOnly = true;
_pipelines.Add(pipeline);
_repository.GetLastDataUpdatesAsync(Arg.Any<CancellationToken>())
.Returns(new Dictionary<string, DataUpdate>());
@@ -382,11 +297,8 @@ public class ScheduleCheckerTests
public async Task GetPendingTasksAsync_NoPriorUpdates_RequiresMassFirst()
{
// Arrange: Never synced before, all schedules enabled
var config = CreateDataSourceConfig("WorkOrder",
massEnabled: true, massInterval: 10080,
dailyEnabled: true, dailyInterval: 1440,
hourlyEnabled: true, hourlyInterval: 60);
_options.Value.DataSources.Add(config);
var pipeline = CreatePipeline("WorkOrder", massInterval: 10080, dailyInterval: 1440, hourlyInterval: 60);
_pipelines.Add(pipeline);
_repository.GetLastDataUpdatesAsync(Arg.Any<CancellationToken>())
.Returns(new Dictionary<string, DataUpdate>());
@@ -404,10 +316,8 @@ public class ScheduleCheckerTests
public async Task GetPendingTasksAsync_OnlyMassCompleted_DailyHasNullMinimumDT()
{
// Arrange: Mass completed, no daily yet
var config = CreateDataSourceConfig("WorkOrder",
massEnabled: true, massInterval: 10080,
dailyEnabled: true, dailyInterval: 1440);
_options.Value.DataSources.Add(config);
var pipeline = CreatePipeline("WorkOrder", massInterval: 10080, dailyInterval: 1440);
_pipelines.Add(pipeline);
var now = DateTime.UtcNow;
var lastMass = CreateDataUpdate("WorkOrder", UpdateTypes.Mass, now.AddHours(-1), success: true);
@@ -431,11 +341,8 @@ public class ScheduleCheckerTests
public async Task GetPendingTasksAsync_NeverHadMass_DoesNotReturnDailyOrHourly()
{
// Arrange: Daily and Hourly enabled but no Mass ever run
var config = CreateDataSourceConfig("WorkOrder",
massEnabled: true, massInterval: 10080,
dailyEnabled: true, dailyInterval: 1440,
hourlyEnabled: true, hourlyInterval: 60);
_options.Value.DataSources.Add(config);
var pipeline = CreatePipeline("WorkOrder", massInterval: 10080, dailyInterval: 1440, hourlyInterval: 60);
_pipelines.Add(pipeline);
_repository.GetLastDataUpdatesAsync(Arg.Any<CancellationToken>())
.Returns(new Dictionary<string, DataUpdate>());
@@ -456,9 +363,8 @@ public class ScheduleCheckerTests
public async Task GetPendingTasksAsync_FailedMass_ReturnsMassAgain()
{
// Arrange
var config = CreateDataSourceConfig("WorkOrder",
massEnabled: true, massInterval: 10080);
_options.Value.DataSources.Add(config);
var pipeline = CreatePipeline("WorkOrder", massInterval: 10080);
_pipelines.Add(pipeline);
var now = DateTime.UtcNow;
var lastMass = CreateDataUpdate("WorkOrder", UpdateTypes.Mass, now.AddMinutes(-5), success: false);
@@ -481,10 +387,8 @@ public class ScheduleCheckerTests
public async Task GetPendingTasksAsync_FailedDaily_ReturnsDailyAgain()
{
// Arrange
var config = CreateDataSourceConfig("WorkOrder",
massEnabled: true, massInterval: 10080,
dailyEnabled: true, dailyInterval: 1440);
_options.Value.DataSources.Add(config);
var pipeline = CreatePipeline("WorkOrder", massInterval: 10080, dailyInterval: 1440);
_pipelines.Add(pipeline);
var now = DateTime.UtcNow;
var lastMass = CreateDataUpdate("WorkOrder", UpdateTypes.Mass, now.AddDays(-1), success: true);
@@ -509,11 +413,8 @@ public class ScheduleCheckerTests
public async Task GetPendingTasksAsync_FailedHourly_ReturnsHourlyAgain()
{
// Arrange
var config = CreateDataSourceConfig("WorkOrder",
massEnabled: true, massInterval: 10080,
dailyEnabled: true, dailyInterval: 1440,
hourlyEnabled: true, hourlyInterval: 60);
_options.Value.DataSources.Add(config);
var pipeline = CreatePipeline("WorkOrder", massInterval: 10080, dailyInterval: 1440, hourlyInterval: 60);
_pipelines.Add(pipeline);
var now = DateTime.UtcNow;
var lastMass = CreateDataUpdate("WorkOrder", UpdateTypes.Mass, now.AddDays(-1), success: true);
@@ -544,10 +445,10 @@ public class ScheduleCheckerTests
public async Task GetPendingTasksAsync_MultipleTables_ReturnsTasksForEach()
{
// Arrange
var config1 = CreateDataSourceConfig("WorkOrder", massEnabled: true, massInterval: 60);
var config2 = CreateDataSourceConfig("LotUsage", massEnabled: true, massInterval: 60);
_options.Value.DataSources.Add(config1);
_options.Value.DataSources.Add(config2);
var pipeline1 = CreatePipeline("WorkOrder", massInterval: 60);
var pipeline2 = CreatePipeline("LotUsage", massInterval: 60);
_pipelines.Add(pipeline1);
_pipelines.Add(pipeline2);
_repository.GetLastDataUpdatesAsync(Arg.Any<CancellationToken>())
.Returns(new Dictionary<string, DataUpdate>());
@@ -565,13 +466,10 @@ public class ScheduleCheckerTests
public async Task GetPendingTasksAsync_MultipleTables_DifferentSchedulesDue()
{
// Arrange
var config1 = CreateDataSourceConfig("WorkOrder",
massEnabled: true, massInterval: 10080,
dailyEnabled: true, dailyInterval: 1440);
var config2 = CreateDataSourceConfig("LotUsage",
massEnabled: true, massInterval: 60);
_options.Value.DataSources.Add(config1);
_options.Value.DataSources.Add(config2);
var pipeline1 = CreatePipeline("WorkOrder", massInterval: 10080, dailyInterval: 1440);
var pipeline2 = CreatePipeline("LotUsage", massInterval: 60);
_pipelines.Add(pipeline1);
_pipelines.Add(pipeline2);
var now = DateTime.UtcNow;
var lastMassWorkOrder = CreateDataUpdate("WorkOrder", UpdateTypes.Mass, now.AddDays(-1), success: true);
@@ -603,11 +501,8 @@ public class ScheduleCheckerTests
public async Task GetPendingTasksAsync_NothingDue_ReturnsEmptyList()
{
// Arrange
var config = CreateDataSourceConfig("WorkOrder",
massEnabled: true, massInterval: 10080,
dailyEnabled: true, dailyInterval: 1440,
hourlyEnabled: true, hourlyInterval: 60);
_options.Value.DataSources.Add(config);
var pipeline = CreatePipeline("WorkOrder", massInterval: 10080, dailyInterval: 1440, hourlyInterval: 60);
_pipelines.Add(pipeline);
var now = DateTime.UtcNow;
// All syncs completed recently
@@ -631,9 +526,9 @@ public class ScheduleCheckerTests
}
[Fact]
public async Task GetPendingTasksAsync_NoDataSources_ReturnsEmptyList()
public async Task GetPendingTasksAsync_NoPipelines_ReturnsEmptyList()
{
// Arrange: No data sources configured
// Arrange: No pipelines configured
_repository.GetLastDataUpdatesAsync(Arg.Any<CancellationToken>())
.Returns(new Dictionary<string, DataUpdate>());
@@ -648,35 +543,29 @@ public class ScheduleCheckerTests
#region Helper Methods
private static DataSourceConfig CreateDataSourceConfig(
string tableName,
bool massEnabled = false,
int massInterval = 10080,
bool dailyEnabled = false,
int dailyInterval = 1440,
bool hourlyEnabled = false,
int hourlyInterval = 60)
private static EtlPipelineConfig CreatePipeline(
string name,
int? massInterval = null,
int? dailyInterval = null,
int? hourlyInterval = null)
{
return new DataSourceConfig
return new EtlPipelineConfig
{
TableName = tableName,
SourceSystem = "JDE",
SourceData = tableName.ToUpper(),
Name = name,
IsEnabled = true,
MassConfig = new ScheduleConfig
IsManualOnly = false,
MassSyncIntervalMinutes = massInterval,
DailySyncIntervalMinutes = dailyInterval,
HourlySyncIntervalMinutes = hourlyInterval,
Source = new SourceElement
{
Enabled = massEnabled,
IntervalMinutes = massInterval
Connection = "jde",
Query = "SELECT * FROM Test"
},
DailyConfig = new ScheduleConfig
Destination = new DestinationElement
{
Enabled = dailyEnabled,
IntervalMinutes = dailyInterval
},
HourlyConfig = new ScheduleConfig
{
Enabled = hourlyEnabled,
IntervalMinutes = hourlyInterval
Table = name,
MatchColumns = ["Id"]
}
};
}