dohertj2/jdescopingtool
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor(configmanager): migrate to per-file pipeline system

Browse Source 
Align ConfigManager with DataSync's per-file pipeline format (pipeline.*.json)
by reusing EtlPipelineConfig types directly, eliminating duplicate models and
simplifying the codebase. Removes ~3200 lines of obsolete code.
This commit is contained in:
Joseph Doherty
2026-01-23 02:30:48 -05:00
parent 1b7bb26def
commit ba54a87be5
49 changed files with 1429 additions and 4396 deletions
Download Patch File Download Diff File Expand all files Collapse all files
NEW/tests/JdeScoping.DataSync.Tests/Services/TableSyncOperationTests.cs
+99 -111
View File
@@ -1,6 +1,7 @@
using System.Data;
using System.Diagnostics.Metrics;
using JdeScoping.Core.Models.Enums;
using JdeScoping.DataSync.Configuration;
using JdeScoping.DataSync.Contracts;
using JdeScoping.DataSync.Etl.Contracts;
using JdeScoping.DataSync.Etl.Pipeline;
@@ -19,7 +20,7 @@ namespace JdeScoping.DataSync.Tests.Services;
/// <summary>
/// Unit tests for TableSyncOperation.
/// Tests that the operation correctly uses the ETL pipeline with UpdateTypes.
/// Tests that the operation correctly uses the ETL pipeline builder.
/// </summary>
public class TableSyncOperationTests
{
@@ -48,33 +49,26 @@ public class TableSyncOperationTests
_metrics = new DataSyncMetrics(meterFactory);
}
#region WithUpdateType Tests
#region Pipeline Builder Tests
[Fact]
public async Task ExecuteAsync_WithUpdateTypesDaily_CallsWithUpdateTypeWithDaily()
public async Task ExecuteAsync_WithDailyUpdateType_CallsBuildWithDailyUpdateType()
{
// Arrange
var task = CreateTask("TestTable", UpdateTypes.Daily);
UpdateTypes? receivedUpdateType = null;
// Pre-create the test pipeline to avoid NSubstitute issues
var testPipeline = CreateTestPipeline();
var mockBuilder = Substitute.For<IEtlPipelineBuilder>();
mockBuilder.WithUpdateType(Arg.Any<UpdateTypes>())
.Returns(callInfo =>
{
receivedUpdateType = callInfo.Arg<UpdateTypes>();
return mockBuilder;
});
mockBuilder.WithMinimumDate(Arg.Any<DateTime?>()).Returns(mockBuilder);
mockBuilder.Build().Returns(testPipeline);
var mockFactory = Substitute.For<IEtlPipelineFactory>();
mockFactory.ForTable(Arg.Any<string>()).Returns(mockBuilder);
mockBuilder.Build(
Arg.Any<EtlPipelineConfig>(),
Arg.Do<UpdateTypes>(ut => receivedUpdateType = ut),
Arg.Any<DateTime?>())
.Returns(testPipeline);
var sut = new TableSyncOperation(
mockFactory,
mockBuilder,
_updateRepository,
_options,
NullLogger<TableSyncOperation>.Instance,
@@ -83,35 +77,28 @@ public class TableSyncOperationTests
// Act
await sut.ExecuteAsync(task);
// Assert - Verify WithUpdateType was called with Daily (not mapped to Incremental)
// Assert
receivedUpdateType.ShouldBe(UpdateTypes.Daily);
}
[Fact]
public async Task ExecuteAsync_WithUpdateTypesHourly_CallsWithUpdateTypeWithHourly()
public async Task ExecuteAsync_WithHourlyUpdateType_CallsBuildWithHourlyUpdateType()
{
// Arrange
var task = CreateTask("TestTable", UpdateTypes.Hourly);
UpdateTypes? receivedUpdateType = null;
// Pre-create the test pipeline to avoid NSubstitute issues
var testPipeline = CreateTestPipeline();
var mockBuilder = Substitute.For<IEtlPipelineBuilder>();
mockBuilder.WithUpdateType(Arg.Any<UpdateTypes>())
.Returns(callInfo =>
{
receivedUpdateType = callInfo.Arg<UpdateTypes>();
return mockBuilder;
});
mockBuilder.WithMinimumDate(Arg.Any<DateTime?>()).Returns(mockBuilder);
mockBuilder.Build().Returns(testPipeline);
var mockFactory = Substitute.For<IEtlPipelineFactory>();
mockFactory.ForTable(Arg.Any<string>()).Returns(mockBuilder);
mockBuilder.Build(
Arg.Any<EtlPipelineConfig>(),
Arg.Do<UpdateTypes>(ut => receivedUpdateType = ut),
Arg.Any<DateTime?>())
.Returns(testPipeline);
var sut = new TableSyncOperation(
mockFactory,
mockBuilder,
_updateRepository,
_options,
NullLogger<TableSyncOperation>.Instance,
@@ -125,30 +112,23 @@ public class TableSyncOperationTests
}
[Fact]
public async Task ExecuteAsync_WithUpdateTypesMass_CallsWithUpdateTypeWithMass()
public async Task ExecuteAsync_WithMassUpdateType_CallsBuildWithMassUpdateType()
{
// Arrange
var task = CreateTask("TestTable", UpdateTypes.Mass);
UpdateTypes? receivedUpdateType = null;
// Pre-create the test pipeline to avoid NSubstitute issues
var testPipeline = CreateTestPipeline();
var mockBuilder = Substitute.For<IEtlPipelineBuilder>();
mockBuilder.WithUpdateType(Arg.Any<UpdateTypes>())
.Returns(callInfo =>
{
receivedUpdateType = callInfo.Arg<UpdateTypes>();
return mockBuilder;
});
mockBuilder.WithMinimumDate(Arg.Any<DateTime?>()).Returns(mockBuilder);
mockBuilder.Build().Returns(testPipeline);
var mockFactory = Substitute.For<IEtlPipelineFactory>();
mockFactory.ForTable(Arg.Any<string>()).Returns(mockBuilder);
mockBuilder.Build(
Arg.Any<EtlPipelineConfig>(),
Arg.Do<UpdateTypes>(ut => receivedUpdateType = ut),
Arg.Any<DateTime?>())
.Returns(testPipeline);
var sut = new TableSyncOperation(
mockFactory,
mockBuilder,
_updateRepository,
_options,
NullLogger<TableSyncOperation>.Instance,
@@ -161,35 +141,24 @@ public class TableSyncOperationTests
receivedUpdateType.ShouldBe(UpdateTypes.Mass);
}
#endregion
#region Pipeline Execution Tests
[Fact]
public async Task ExecuteAsync_CallsForTableWithCorrectTableName()
public async Task ExecuteAsync_CallsBuildWithCorrectPipelineConfig()
{
// Arrange
var task = CreateTask("WorkOrder", UpdateTypes.Daily);
string? receivedTableName = null;
EtlPipelineConfig? receivedConfig = null;
// Pre-create the test pipeline to avoid NSubstitute issues
var testPipeline = CreateTestPipeline();
var mockBuilder = Substitute.For<IEtlPipelineBuilder>();
mockBuilder.WithUpdateType(Arg.Any<UpdateTypes>()).Returns(mockBuilder);
mockBuilder.WithMinimumDate(Arg.Any<DateTime?>()).Returns(mockBuilder);
mockBuilder.Build().Returns(testPipeline);
var mockFactory = Substitute.For<IEtlPipelineFactory>();
mockFactory.ForTable(Arg.Any<string>())
.Returns(callInfo =>
{
receivedTableName = callInfo.Arg<string>();
return mockBuilder;
});
mockBuilder.Build(
Arg.Do<EtlPipelineConfig>(c => receivedConfig = c),
Arg.Any<UpdateTypes>(),
Arg.Any<DateTime?>())
.Returns(testPipeline);
var sut = new TableSyncOperation(
mockFactory,
mockBuilder,
_updateRepository,
_options,
NullLogger<TableSyncOperation>.Instance,
@@ -199,35 +168,29 @@ public class TableSyncOperationTests
await sut.ExecuteAsync(task);
// Assert
receivedTableName.ShouldBe("WorkOrder");
receivedConfig.ShouldNotBeNull();
receivedConfig.Name.ShouldBe("WorkOrder");
}
[Fact]
public async Task ExecuteAsync_CallsWithMinimumDateWithTaskMinimumDt()
public async Task ExecuteAsync_CallsBuildWithCorrectMinimumDate()
{
// Arrange
var minDt = new DateTime(2024, 1, 15, 10, 30, 0, DateTimeKind.Utc);
var task = CreateTask("TestTable", UpdateTypes.Daily, minDt);
DateTime? receivedMinDt = null;
// Pre-create the test pipeline to avoid NSubstitute issues
var testPipeline = CreateTestPipeline();
var mockBuilder = Substitute.For<IEtlPipelineBuilder>();
mockBuilder.WithUpdateType(Arg.Any<UpdateTypes>()).Returns(mockBuilder);
mockBuilder.WithMinimumDate(Arg.Any<DateTime?>())
.Returns(callInfo =>
{
receivedMinDt = callInfo.Arg<DateTime?>();
return mockBuilder;
});
mockBuilder.Build().Returns(testPipeline);
var mockFactory = Substitute.For<IEtlPipelineFactory>();
mockFactory.ForTable(Arg.Any<string>()).Returns(mockBuilder);
mockBuilder.Build(
Arg.Any<EtlPipelineConfig>(),
Arg.Any<UpdateTypes>(),
Arg.Do<DateTime?>(dt => receivedMinDt = dt))
.Returns(testPipeline);
var sut = new TableSyncOperation(
mockFactory,
mockBuilder,
_updateRepository,
_options,
NullLogger<TableSyncOperation>.Instance,
@@ -240,25 +203,54 @@ public class TableSyncOperationTests
receivedMinDt.ShouldBe(minDt);
}
[Fact]
public async Task ExecuteAsync_TaskWithNoPipeline_ThrowsInvalidOperationException()
{
// Arrange
var task = new DataUpdateTask
{
TableName = "TestTable",
SourceSystem = "JDE",
SourceData = "TESTTABLE",
UpdateType = UpdateTypes.Daily,
Pipeline = null // No pipeline!
};
var mockBuilder = Substitute.For<IEtlPipelineBuilder>();
var sut = new TableSyncOperation(
mockBuilder,
_updateRepository,
_options,
NullLogger<TableSyncOperation>.Instance,
_metrics);
// Act & Assert
var ex = await Should.ThrowAsync<InvalidOperationException>(() => sut.ExecuteAsync(task));
ex.Message.ShouldContain("No pipeline configuration");
ex.Message.ShouldContain("TestTable");
}
#endregion
#region Pipeline Execution Tests
[Fact]
public async Task ExecuteAsync_SuccessfulPipeline_CompletesUpdateAsSuccess()
{
// Arrange
var task = CreateTask("TestTable", UpdateTypes.Daily);
// Pre-create the test pipeline to avoid NSubstitute issues
var testPipeline = CreateTestPipeline(totalRows: 100);
var mockBuilder = Substitute.For<IEtlPipelineBuilder>();
mockBuilder.WithUpdateType(Arg.Any<UpdateTypes>()).Returns(mockBuilder);
mockBuilder.WithMinimumDate(Arg.Any<DateTime?>()).Returns(mockBuilder);
mockBuilder.Build().Returns(testPipeline);
var mockFactory = Substitute.For<IEtlPipelineFactory>();
mockFactory.ForTable(Arg.Any<string>()).Returns(mockBuilder);
mockBuilder.Build(
Arg.Any<EtlPipelineConfig>(),
Arg.Any<UpdateTypes>(),
Arg.Any<DateTime?>())
.Returns(testPipeline);
var sut = new TableSyncOperation(
mockFactory,
mockBuilder,
_updateRepository,
_options,
NullLogger<TableSyncOperation>.Instance,
@@ -295,15 +287,14 @@ public class TableSyncOperationTests
var testPipeline = CreateTestPipeline();
var mockBuilder = Substitute.For<IEtlPipelineBuilder>();
mockBuilder.WithUpdateType(Arg.Any<UpdateTypes>()).Returns(mockBuilder);
mockBuilder.WithMinimumDate(Arg.Any<DateTime?>()).Returns(mockBuilder);
mockBuilder.Build().Returns(testPipeline);
var mockFactory = Substitute.For<IEtlPipelineFactory>();
mockFactory.ForTable(Arg.Any<string>()).Returns(mockBuilder);
mockBuilder.Build(
Arg.Any<EtlPipelineConfig>(),
Arg.Any<UpdateTypes>(),
Arg.Any<DateTime?>())
.Returns(testPipeline);
var sut = new TableSyncOperation(
mockFactory,
mockBuilder,
_updateRepository,
_options,
NullLogger<TableSyncOperation>.Instance,
@@ -324,20 +315,17 @@ public class TableSyncOperationTests
{
// Arrange
var task = CreateTask("TestTable", UpdateTypes.Daily);
// Pre-create the test pipeline to avoid NSubstitute issues
var testPipeline = CreateTestPipeline(success: false);
var mockBuilder = Substitute.For<IEtlPipelineBuilder>();
mockBuilder.WithUpdateType(Arg.Any<UpdateTypes>()).Returns(mockBuilder);
mockBuilder.WithMinimumDate(Arg.Any<DateTime?>()).Returns(mockBuilder);
mockBuilder.Build().Returns(testPipeline);
var mockFactory = Substitute.For<IEtlPipelineFactory>();
mockFactory.ForTable(Arg.Any<string>()).Returns(mockBuilder);
mockBuilder.Build(
Arg.Any<EtlPipelineConfig>(),
Arg.Any<UpdateTypes>(),
Arg.Any<DateTime?>())
.Returns(testPipeline);
var sut = new TableSyncOperation(
mockFactory,
mockBuilder,
_updateRepository,
_options,
NullLogger<TableSyncOperation>.Instance,
@@ -366,15 +354,15 @@ public class TableSyncOperationTests
SourceData = tableName.ToUpper(),
UpdateType = updateType,
MinimumDt = minDt,
Config = new DataSourceConfig
Pipeline = new EtlPipelineConfig
{
TableName = tableName,
SourceSystem = "JDE",
SourceData = tableName.ToUpper(),
Name = tableName,
IsEnabled = true,
MassConfig = new ScheduleConfig { Enabled = true, IntervalMinutes = 10080 },
DailyConfig = new ScheduleConfig { Enabled = true, IntervalMinutes = 1440 },
HourlyConfig = new ScheduleConfig { Enabled = true, IntervalMinutes = 60 }
MassSyncIntervalMinutes = 10080,
DailySyncIntervalMinutes = 1440,
HourlySyncIntervalMinutes = 60,
Source = new SourceElement { Connection = "JDE", Query = "SELECT 1" },
Destination = new DestinationElement { Table = tableName, MatchColumns = ["Id"] }
}
};
}