dohertj2/jdescopingtool
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(datasync): add WithUpdateType to IEtlPipelineBuilder

Browse Source 
- Add WithUpdateType(UpdateTypes) method to IEtlPipelineBuilder interface
- Mark existing WithMode(SyncMode) as [Obsolete("Use WithUpdateType instead")]
- Update PipelineBuilder to store UpdateTypes instead of SyncMode
- Add GetEffectiveScheduleConfig method to merge pipeline schedules with defaults
- Add BuildWithSchedules method for new Schedules-based config
- Update validation to support both old SyncModes and new Schedules formats
- Pass ScheduleDefaults from PipelinesRoot to PipelineBuilder
- For Mass mode: use massQuery, apply prePurge/reIndex from schedule config
- For Daily/Hourly: use regular query with date parameters
- Add 8 new tests for WithUpdateType functionality
This commit is contained in:
Joseph Doherty
2026-01-07 01:15:18 -05:00
parent 4b3c900d52
commit bb54994f2d
3 changed files with 392 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
NEW/tests/JdeScoping.DataSync.Tests/Services/EtlPipelineFactoryTests.cs
+164
View File
@@ -1,3 +1,4 @@
using JdeScoping.Core.Models.Enums;
using JdeScoping.DataAccess.Interfaces;
using JdeScoping.DataSync.Configuration;
using JdeScoping.DataSync.Contracts;
@@ -156,6 +157,141 @@ public class EtlPipelineFactoryTests
#endregion
#region Builder WithUpdateType Tests
[Fact]
public void Builder_WithUpdateTypesMass_BuildsPipeline()
{
// Arrange
var config = CreateValidConfigWithSchedules();
var factory = CreateFactory(config);
// Act
var pipeline = factory.ForTable("TestTable")
.WithUpdateType(UpdateTypes.Mass)
.Build();
// Assert
pipeline.ShouldNotBeNull();
pipeline.PipelineName.ShouldBe("TestTable");
}
[Fact]
public void Builder_WithUpdateTypesDaily_BuildsPipeline()
{
// Arrange
var config = CreateValidConfigWithSchedules();
var factory = CreateFactory(config);
// Act
var pipeline = factory.ForTable("TestTable")
.WithUpdateType(UpdateTypes.Daily)
.Build();
// Assert
pipeline.ShouldNotBeNull();
pipeline.PipelineName.ShouldBe("TestTable");
}
[Fact]
public void Builder_WithUpdateTypesHourly_BuildsPipeline()
{
// Arrange
var config = CreateValidConfigWithSchedules();
var factory = CreateFactory(config);
// Act
var pipeline = factory.ForTable("TestTable")
.WithUpdateType(UpdateTypes.Hourly)
.Build();
// Assert
pipeline.ShouldNotBeNull();
pipeline.PipelineName.ShouldBe("TestTable");
}
[Fact]
public void Builder_WithUpdateTypesMass_UsesMassQuery()
{
// Arrange - config with massQuery should use it for Mass update type
var config = CreateValidConfigWithSchedules();
var factory = CreateFactory(config);
// Act
var pipeline = factory.ForTable("TestTable")
.WithUpdateType(UpdateTypes.Mass)
.Build();
// Assert
pipeline.ShouldNotBeNull();
}
[Fact]
public void Builder_WithUpdateTypesDaily_UsesRegularQuery()
{
// Arrange - Daily should use regular query with date filtering
var config = CreateValidConfigWithSchedules();
var factory = CreateFactory(config);
// Act
var pipeline = factory.ForTable("TestTable")
.WithUpdateType(UpdateTypes.Daily)
.Build();
// Assert
pipeline.ShouldNotBeNull();
}
[Fact]
public void Builder_WithUpdateTypesMass_AppliesPrePurgeFromScheduleConfig()
{
// Arrange - Mass schedule should have prePurge=true from defaults
var config = CreateValidConfigWithSchedules();
var factory = CreateFactory(config);
// Act - should not throw and should include truncate pre-script
var pipeline = factory.ForTable("TestTable")
.WithUpdateType(UpdateTypes.Mass)
.Build();
// Assert
pipeline.ShouldNotBeNull();
}
[Fact]
public void Builder_WithUpdateTypesMass_AppliesReIndexFromScheduleConfig()
{
// Arrange - Mass schedule should have reIndex=true from defaults
var config = CreateValidConfigWithSchedules();
var factory = CreateFactory(config);
// Act - should not throw and should include reindex post-script
var pipeline = factory.ForTable("TestTable")
.WithUpdateType(UpdateTypes.Mass)
.Build();
// Assert
pipeline.ShouldNotBeNull();
}
[Fact]
public void Builder_WithUpdateTypesHourly_UsesUpdateWhenFromDefaults()
{
// Arrange - Hourly should use updateWhen from defaults
var config = CreateValidConfigWithSchedules();
var factory = CreateFactory(config);
// Act
var pipeline = factory.ForTable("TestTable")
.WithUpdateType(UpdateTypes.Hourly)
.Build();
// Assert
pipeline.ShouldNotBeNull();
}
#endregion
#region Builder WithMinimumDate Tests
[Fact]
@@ -839,6 +975,34 @@ public class EtlPipelineFactoryTests
});
}
private PipelinesRoot CreateValidConfigWithSchedules()
{
return new PipelinesRoot(
new PipelineSettings("UTC"),
new ScheduleDefaults(),
new Dictionary<string, PipelineConfig>
{
["TestTable"] = new PipelineConfig(
new SourceConfig("lotfinder", "SELECT * FROM Test WHERE UpdateDt >= @MinDt",
new Dictionary<string, ParameterConfig>
{
["minDt"] = new ParameterConfig("@MinDt", null, "offset", null)
},
"SELECT * FROM Test"), // MassQuery
null, // No old SyncModes
new PipelineSchedules
{
Mass = new ScheduleConfig { PrePurge = true, ReIndex = true },
Daily = new ScheduleConfig(),
Hourly = new ScheduleConfig()
},
null, // Transformers
new DestinationConfig("TestTable", ["Id"], null),
null,
null)
});
}
private EtlPipelineFactory CreateFactory(PipelinesRoot config)
{
return new EtlPipelineFactory(_connectionFactory, config, _logger);