dohertj2/jdescopingtool
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor(datasync): remove deprecated SyncMode and SyncModeConfig

Browse Source 
- Delete SyncMode.cs enum file
- Remove SyncModes property from PipelineConfig
- Remove SyncModeConfig and DestinationOverride records
- Remove WithMode(SyncMode) from IEtlPipelineBuilder
- Remove BuildWithSyncModes() and related methods from EtlPipelineFactory
- Remove syncModes sections from all pipelines in pipelines.json
- Update tests to use schedules-only configuration

All pipelines now require 'schedules' format (mass/daily/hourly).
WithUpdateType(UpdateTypes) is the only way to set update type.
This commit is contained in:
Joseph Doherty
2026-01-07 05:16:20 -05:00
parent 1618b6664d
commit c814a7294b
8 changed files with 122 additions and 670 deletions
Download Patch File Download Diff File Expand all files Collapse all files
NEW/tests/JdeScoping.DataSync.Tests/Configuration/PipelinesRootTests.cs
+5 -6
View File
@@ -72,14 +72,13 @@ public class PipelinesRootTests
{
var config = new PipelineConfig(
new SourceConfig("jde", "SELECT 1", null, null),
null, // Old SyncModes - deprecated
new PipelineSchedules
{
Mass = new ScheduleConfig { PrePurge = true, ReIndex = true },
Daily = new ScheduleConfig { Enabled = true },
Hourly = new ScheduleConfig { Enabled = false }
},
null,
null, // Transformers
new DestinationConfig("TestTable", ["Id"], null),
null,
null);
@@ -93,12 +92,12 @@ public class PipelinesRootTests
{
return new PipelineConfig(
new SourceConfig("lotfinder", "SELECT 1", null, null),
new Dictionary<string, SyncModeConfig>
new PipelineSchedules
{
["mass"] = new SyncModeConfig(null, true, true),
["incremental"] = new SyncModeConfig("-1d")
Mass = new ScheduleConfig { PrePurge = true, ReIndex = true },
Daily = new ScheduleConfig(),
Hourly = new ScheduleConfig()
},
null, // Schedules
null, // Transformers
new DestinationConfig("TestTable", ["Id"], null),
null,
NEW/tests/JdeScoping.DataSync.Tests/Services/EtlPipelineFactoryTests.cs
+108 -325
View File
@@ -28,7 +28,7 @@ public class EtlPipelineFactoryTests
public void ForTable_WithValidTable_ReturnsBuilder()
{
// Arrange
var config = CreateValidConfig();
var config = CreateValidConfigWithSchedules();
var factory = CreateFactory(config);
// Act
@@ -43,7 +43,7 @@ public class EtlPipelineFactoryTests
public void ForTable_WithUnknownTable_ThrowsInvalidOperationException()
{
// Arrange
var config = CreateValidConfig();
var config = CreateValidConfigWithSchedules();
var factory = CreateFactory(config);
// Act & Assert
@@ -56,7 +56,7 @@ public class EtlPipelineFactoryTests
public void ForTable_WithNullTableName_ThrowsArgumentException()
{
// Arrange
var config = CreateValidConfig();
var config = CreateValidConfigWithSchedules();
var factory = CreateFactory(config);
// Act & Assert
@@ -67,7 +67,7 @@ public class EtlPipelineFactoryTests
public void ForTable_WithEmptyTableName_ThrowsArgumentException()
{
// Arrange
var config = CreateValidConfig();
var config = CreateValidConfigWithSchedules();
var factory = CreateFactory(config);
// Act & Assert
@@ -76,87 +76,6 @@ public class EtlPipelineFactoryTests
#endregion
#region Builder WithMode Tests
[Fact]
public void Builder_WithMassMode_BuildsPipeline()
{
// Arrange
var config = CreateValidConfig();
var factory = CreateFactory(config);
// Act
var pipeline = factory.ForTable("TestTable")
.WithMode(SyncMode.Mass)
.Build();
// Assert
pipeline.ShouldNotBeNull();
pipeline.PipelineName.ShouldBe("TestTable");
}
[Fact]
public void Builder_WithIncrementalMode_BuildsPipeline()
{
// Arrange
var config = CreateValidConfig();
var factory = CreateFactory(config);
// Act
var pipeline = factory.ForTable("TestTable")
.WithMode(SyncMode.Incremental)
.Build();
// Assert
pipeline.ShouldNotBeNull();
pipeline.PipelineName.ShouldBe("TestTable");
}
[Fact]
public void Builder_DefaultMode_IsIncremental()
{
// Arrange
var config = CreateValidConfig();
var factory = CreateFactory(config);
// Act - don't call WithMode()
var pipeline = factory.ForTable("TestTable")
.Build();
// Assert - should work because incremental mode is defined
pipeline.ShouldNotBeNull();
}
[Fact]
public void Builder_WithUndefinedSyncMode_ThrowsInvalidOperationException()
{
// Arrange - config with only mass mode
var config = new PipelinesRoot(
new PipelineSettings("UTC"),
null,
new Dictionary<string, PipelineConfig>
{
["TestTable"] = new PipelineConfig(
new SourceConfig("lotfinder", "SELECT * FROM Test", null),
new Dictionary<string, SyncModeConfig>
{
["mass"] = new SyncModeConfig("-365.00:00:00", PrePurge: true)
// No incremental mode defined
},
null, // Schedules
null, // Transformers
new DestinationConfig("TestTable", ["Id"], null),
null,
null)
});
// Act & Assert - validation fails at factory creation
var ex = Should.Throw<InvalidOperationException>(() => CreateFactory(config));
ex.Message.ShouldContain("missing required 'incremental' sync mode");
}
#endregion
#region Builder WithUpdateType Tests
[Fact]
@@ -290,6 +209,21 @@ public class EtlPipelineFactoryTests
pipeline.ShouldNotBeNull();
}
[Fact]
public void Builder_DefaultMode_IsHourly()
{
// Arrange
var config = CreateValidConfigWithSchedules();
var factory = CreateFactory(config);
// Act - don't call WithUpdateType()
var pipeline = factory.ForTable("TestTable")
.Build();
// Assert - should work because hourly mode is defined
pipeline.ShouldNotBeNull();
}
#endregion
#region Builder WithMinimumDate Tests
@@ -298,13 +232,13 @@ public class EtlPipelineFactoryTests
public void Builder_WithMinimumDate_OverridesConfigOffset()
{
// Arrange
var config = CreateValidConfig();
var config = CreateValidConfigWithSchedules();
var factory = CreateFactory(config);
var customDate = new DateTime(2024, 1, 1, 0, 0, 0, DateTimeKind.Utc);
// Act - should not throw even though we're overriding
var pipeline = factory.ForTable("TestTable")
.WithMode(SyncMode.Incremental)
.WithUpdateType(UpdateTypes.Hourly)
.WithMinimumDate(customDate)
.Build();
@@ -316,12 +250,12 @@ public class EtlPipelineFactoryTests
public void Builder_WithNullMinimumDate_UsesConfigOffset()
{
// Arrange
var config = CreateValidConfig();
var config = CreateValidConfigWithSchedules();
var factory = CreateFactory(config);
// Act - null minDt means use config offset
var pipeline = factory.ForTable("TestTable")
.WithMode(SyncMode.Incremental)
.WithUpdateType(UpdateTypes.Hourly)
.WithMinimumDate(null)
.Build();
@@ -334,7 +268,7 @@ public class EtlPipelineFactoryTests
#region Config Validation Tests
[Fact]
public void Validate_ConfigMissingMassMode_ThrowsInvalidOperationException()
public void Validate_ConfigMissingSchedules_ThrowsInvalidOperationException()
{
// Arrange
var config = new PipelinesRoot(
@@ -344,12 +278,7 @@ public class EtlPipelineFactoryTests
{
["TestTable"] = new PipelineConfig(
new SourceConfig("lotfinder", "SELECT * FROM Test", null),
new Dictionary<string, SyncModeConfig>
{
// Missing mass mode
["incremental"] = new SyncModeConfig("-1.00:00:00")
},
null, // Schedules
null, // Schedules - null means invalid
null, // Transformers
new DestinationConfig("TestTable", ["Id"], null),
null,
@@ -358,7 +287,7 @@ public class EtlPipelineFactoryTests
// Act & Assert
var ex = Should.Throw<InvalidOperationException>(() => CreateFactory(config));
ex.Message.ShouldContain("missing required 'mass' sync mode");
ex.Message.ShouldContain("must define 'schedules'");
}
[Fact]
@@ -376,12 +305,12 @@ public class EtlPipelineFactoryTests
{
["id"] = new ParameterConfig("@Id", null, "runtime", null)
}),
new Dictionary<string, SyncModeConfig>
new PipelineSchedules
{
["mass"] = new SyncModeConfig("-365.00:00:00"),
["incremental"] = new SyncModeConfig("-1.00:00:00")
Mass = new ScheduleConfig(),
Daily = new ScheduleConfig(),
Hourly = new ScheduleConfig()
},
null, // Schedules
null, // Transformers
new DestinationConfig("TestTable", ["Id"], null),
null,
@@ -398,32 +327,15 @@ public class EtlPipelineFactoryTests
#region Destination Type Tests
[Fact]
public void Builder_MassMode_DefaultsToBulkImport()
public void Builder_MassMode_WithPrePurge_UsesBulkImport()
{
// Arrange - no destination override
var config = new PipelinesRoot(
new PipelineSettings("UTC"),
null,
new Dictionary<string, PipelineConfig>
{
["TestTable"] = new PipelineConfig(
new SourceConfig("lotfinder", "SELECT * FROM Test", null),
new Dictionary<string, SyncModeConfig>
{
["mass"] = new SyncModeConfig("-365.00:00:00", PrePurge: true),
["incremental"] = new SyncModeConfig("-1.00:00:00")
},
null, // Schedules
null, // Transformers
new DestinationConfig("TestTable", ["Id"], null),
null,
null)
});
// Arrange - Mass with prePurge defaults to bulkImport
var config = CreateValidConfigWithSchedules();
var factory = CreateFactory(config);
// Act - should use bulkImport for mass mode
// Act - should use bulkImport for mass mode with prePurge
var pipeline = factory.ForTable("TestTable")
.WithMode(SyncMode.Mass)
.WithUpdateType(UpdateTypes.Mass)
.Build();
// Assert
@@ -431,49 +343,15 @@ public class EtlPipelineFactoryTests
}
[Fact]
public void Builder_IncrementalMode_DefaultsToBulkMerge()
public void Builder_HourlyMode_UsesBulkMerge()
{
// Arrange - no destination override
var config = CreateValidConfig();
// Arrange - Hourly without prePurge uses bulkMerge
var config = CreateValidConfigWithSchedules();
var factory = CreateFactory(config);
// Act - should use bulkMerge for incremental mode
// Act - should use bulkMerge for hourly mode
var pipeline = factory.ForTable("TestTable")
.WithMode(SyncMode.Incremental)
.Build();
// Assert
pipeline.ShouldNotBeNull();
}
[Fact]
public void Builder_ModeWithDestinationOverride_UsesOverride()
{
// Arrange - mass mode with bulkMerge override
var config = new PipelinesRoot(
new PipelineSettings("UTC"),
null,
new Dictionary<string, PipelineConfig>
{
["TestTable"] = new PipelineConfig(
new SourceConfig("lotfinder", "SELECT * FROM Test", null),
new Dictionary<string, SyncModeConfig>
{
["mass"] = new SyncModeConfig("-365.00:00:00",
Destination: new DestinationOverride("bulkMerge", null, null)),
["incremental"] = new SyncModeConfig("-1.00:00:00")
},
null, // Schedules
null, // Transformers
new DestinationConfig("TestTable", ["Id"], null),
null,
null)
});
var factory = CreateFactory(config);
// Act - mass mode should use bulkMerge due to override
var pipeline = factory.ForTable("TestTable")
.WithMode(SyncMode.Mass)
.WithUpdateType(UpdateTypes.Hourly)
.Build();
// Assert
@@ -486,17 +364,17 @@ public class EtlPipelineFactoryTests
// Arrange - bulkMerge needs matchColumns
var config = new PipelinesRoot(
new PipelineSettings("UTC"),
null,
new ScheduleDefaults(),
new Dictionary<string, PipelineConfig>
{
["TestTable"] = new PipelineConfig(
new SourceConfig("lotfinder", "SELECT * FROM Test", null),
new Dictionary<string, SyncModeConfig>
new PipelineSchedules
{
["mass"] = new SyncModeConfig("-365.00:00:00", PrePurge: true),
["incremental"] = new SyncModeConfig("-1.00:00:00")
Mass = new ScheduleConfig { PrePurge = true },
Daily = new ScheduleConfig(),
Hourly = new ScheduleConfig()
},
null, // Schedules
null, // Transformers
new DestinationConfig("TestTable", null, null), // No matchColumns!
null,
@@ -507,7 +385,7 @@ public class EtlPipelineFactoryTests
// Act & Assert
var ex = Should.Throw<InvalidOperationException>(() =>
factory.ForTable("TestTable")
.WithMode(SyncMode.Incremental) // Uses bulkMerge
.WithUpdateType(UpdateTypes.Hourly) // Uses bulkMerge
.Build());
ex.Message.ShouldContain("matchColumns required for bulkMerge");
}
@@ -522,7 +400,7 @@ public class EtlPipelineFactoryTests
// Arrange
var config = new PipelinesRoot(
new PipelineSettings("UTC"),
null,
new ScheduleDefaults(),
new Dictionary<string, PipelineConfig>
{
["TestTable"] = new PipelineConfig(
@@ -531,12 +409,12 @@ public class EtlPipelineFactoryTests
{
["minDt"] = new ParameterConfig("@MinDt", null, "offset", null)
}),
new Dictionary<string, SyncModeConfig>
new PipelineSchedules
{
["mass"] = new SyncModeConfig("-365.00:00:00", PrePurge: true),
["incremental"] = new SyncModeConfig("-1.00:00:00")
Mass = new ScheduleConfig { PrePurge = true },
Daily = new ScheduleConfig(),
Hourly = new ScheduleConfig()
},
null, // Schedules
null, // Transformers
new DestinationConfig("TestTable", ["Id"], null),
null,
@@ -546,7 +424,7 @@ public class EtlPipelineFactoryTests
// Act
var pipeline = factory.ForTable("TestTable")
.WithMode(SyncMode.Incremental)
.WithUpdateType(UpdateTypes.Hourly)
.Build();
// Assert
@@ -559,7 +437,7 @@ public class EtlPipelineFactoryTests
// Arrange
var config = new PipelinesRoot(
new PipelineSettings("UTC"),
null,
new ScheduleDefaults(),
new Dictionary<string, PipelineConfig>
{
["TestTable"] = new PipelineConfig(
@@ -568,12 +446,12 @@ public class EtlPipelineFactoryTests
{
["minDt"] = new ParameterConfig(":dateUpdated", "jdeJulian", "offset", null)
}),
new Dictionary<string, SyncModeConfig>
new PipelineSchedules
{
["mass"] = new SyncModeConfig("-365.00:00:00", PrePurge: true),
["incremental"] = new SyncModeConfig("-1.00:00:00")
Mass = new ScheduleConfig { PrePurge = true },
Daily = new ScheduleConfig(),
Hourly = new ScheduleConfig()
},
null, // Schedules
null, // Transformers
new DestinationConfig("TestTable", ["Id"], null),
null,
@@ -583,7 +461,7 @@ public class EtlPipelineFactoryTests
// Act
var pipeline = factory.ForTable("TestTable")
.WithMode(SyncMode.Incremental)
.WithUpdateType(UpdateTypes.Hourly)
.Build();
// Assert
@@ -596,7 +474,7 @@ public class EtlPipelineFactoryTests
// Arrange
var config = new PipelinesRoot(
new PipelineSettings("UTC"),
null,
new ScheduleDefaults(),
new Dictionary<string, PipelineConfig>
{
["TestTable"] = new PipelineConfig(
@@ -605,12 +483,12 @@ public class EtlPipelineFactoryTests
{
["status"] = new ParameterConfig("@Status", null, "static", "Active")
}),
new Dictionary<string, SyncModeConfig>
new PipelineSchedules
{
["mass"] = new SyncModeConfig("-365.00:00:00", PrePurge: true),
["incremental"] = new SyncModeConfig("-1.00:00:00")
Mass = new ScheduleConfig { PrePurge = true },
Daily = new ScheduleConfig(),
Hourly = new ScheduleConfig()
},
null, // Schedules
null, // Transformers
new DestinationConfig("TestTable", ["Id"], null),
null,
@@ -620,7 +498,7 @@ public class EtlPipelineFactoryTests
// Act
var pipeline = factory.ForTable("TestTable")
.WithMode(SyncMode.Incremental)
.WithUpdateType(UpdateTypes.Hourly)
.Build();
// Assert
@@ -633,7 +511,7 @@ public class EtlPipelineFactoryTests
// Arrange
var config = new PipelinesRoot(
new PipelineSettings("UTC"),
null,
new ScheduleDefaults(),
new Dictionary<string, PipelineConfig>
{
["TestTable"] = new PipelineConfig(
@@ -642,12 +520,12 @@ public class EtlPipelineFactoryTests
{
["status"] = new ParameterConfig("@Status", null, "static", null) // No value!
}),
new Dictionary<string, SyncModeConfig>
new PipelineSchedules
{
["mass"] = new SyncModeConfig("-365.00:00:00", PrePurge: true),
["incremental"] = new SyncModeConfig("-1.00:00:00")
Mass = new ScheduleConfig { PrePurge = true },
Daily = new ScheduleConfig(),
Hourly = new ScheduleConfig()
},
null, // Schedules
null, // Transformers
new DestinationConfig("TestTable", ["Id"], null),
null,
@@ -655,10 +533,11 @@ public class EtlPipelineFactoryTests
});
var factory = CreateFactory(config);
// Act & Assert
// Act & Assert - must provide minDt for parameters to be processed
var ex = Should.Throw<InvalidOperationException>(() =>
factory.ForTable("TestTable")
.WithMode(SyncMode.Incremental)
.WithUpdateType(UpdateTypes.Hourly)
.WithMinimumDate(DateTime.UtcNow.AddDays(-1))
.Build());
ex.Message.ShouldContain("Static parameter '@Status' requires a value");
}
@@ -673,17 +552,17 @@ public class EtlPipelineFactoryTests
// Arrange
var config = new PipelinesRoot(
new PipelineSettings("UTC"),
null,
new ScheduleDefaults(),
new Dictionary<string, PipelineConfig>
{
["TestTable"] = new PipelineConfig(
new SourceConfig("lotfinder", "SELECT * FROM Test", null),
new Dictionary<string, SyncModeConfig>
new PipelineSchedules
{
["mass"] = new SyncModeConfig("-365.00:00:00", PrePurge: true),
["incremental"] = new SyncModeConfig("-1.00:00:00")
Mass = new ScheduleConfig { PrePurge = true },
Daily = new ScheduleConfig(),
Hourly = new ScheduleConfig()
},
null, // Schedules
null, // Transformers
new DestinationConfig("TestTable", ["Id"], null),
null,
@@ -693,7 +572,7 @@ public class EtlPipelineFactoryTests
// Act
var pipeline = factory.ForTable("TestTable")
.WithMode(SyncMode.Mass)
.WithUpdateType(UpdateTypes.Mass)
.Build();
// Assert
@@ -706,17 +585,17 @@ public class EtlPipelineFactoryTests
// Arrange
var config = new PipelinesRoot(
new PipelineSettings("UTC"),
null,
new ScheduleDefaults(),
new Dictionary<string, PipelineConfig>
{
["TestTable"] = new PipelineConfig(
new SourceConfig("lotfinder", "SELECT * FROM Test", null),
new Dictionary<string, SyncModeConfig>
new PipelineSchedules
{
["mass"] = new SyncModeConfig("-365.00:00:00", PrePurge: true, ReIndex: true),
["incremental"] = new SyncModeConfig("-1.00:00:00")
Mass = new ScheduleConfig { PrePurge = true, ReIndex = true },
Daily = new ScheduleConfig(),
Hourly = new ScheduleConfig()
},
null, // Schedules
null, // Transformers
new DestinationConfig("TestTable", ["Id"], null),
null,
@@ -726,7 +605,7 @@ public class EtlPipelineFactoryTests
// Act
var pipeline = factory.ForTable("TestTable")
.WithMode(SyncMode.Mass)
.WithUpdateType(UpdateTypes.Mass)
.Build();
// Assert
@@ -739,17 +618,17 @@ public class EtlPipelineFactoryTests
// Arrange
var config = new PipelinesRoot(
new PipelineSettings("UTC"),
null,
new ScheduleDefaults(),
new Dictionary<string, PipelineConfig>
{
["TestTable"] = new PipelineConfig(
new SourceConfig("lotfinder", "SELECT * FROM Test", null),
new Dictionary<string, SyncModeConfig>
new PipelineSchedules
{
["mass"] = new SyncModeConfig("-365.00:00:00", PrePurge: true),
["incremental"] = new SyncModeConfig("-1.00:00:00")
Mass = new ScheduleConfig { PrePurge = true },
Daily = new ScheduleConfig(),
Hourly = new ScheduleConfig()
},
null, // Schedules
null, // Transformers
new DestinationConfig("TestTable", ["Id"], null),
["EXEC sp_BeforeSync"],
@@ -759,7 +638,7 @@ public class EtlPipelineFactoryTests
// Act
var pipeline = factory.ForTable("TestTable")
.WithMode(SyncMode.Mass)
.WithUpdateType(UpdateTypes.Mass)
.Build();
// Assert
@@ -772,17 +651,17 @@ public class EtlPipelineFactoryTests
// Arrange
var config = new PipelinesRoot(
new PipelineSettings("UTC"),
null,
new ScheduleDefaults(),
new Dictionary<string, PipelineConfig>
{
["TestTable"] = new PipelineConfig(
new SourceConfig("lotfinder", "SELECT * FROM Test", null),
new Dictionary<string, SyncModeConfig>
new PipelineSchedules
{
["mass"] = new SyncModeConfig("-365.00:00:00", PrePurge: true),
["incremental"] = new SyncModeConfig("-1.00:00:00")
Mass = new ScheduleConfig { PrePurge = true },
Daily = new ScheduleConfig(),
Hourly = new ScheduleConfig()
},
null, // Schedules
null, // Transformers
new DestinationConfig("TestTable", ["Id"], null),
null,
@@ -792,7 +671,7 @@ public class EtlPipelineFactoryTests
// Act
var pipeline = factory.ForTable("TestTable")
.WithMode(SyncMode.Incremental)
.WithUpdateType(UpdateTypes.Hourly)
.Build();
// Assert
@@ -812,17 +691,17 @@ public class EtlPipelineFactoryTests
// Arrange
var config = new PipelinesRoot(
new PipelineSettings("UTC"),
null,
new ScheduleDefaults(),
new Dictionary<string, PipelineConfig>
{
["TestTable"] = new PipelineConfig(
new SourceConfig(connectionType, "SELECT * FROM Test", null),
new Dictionary<string, SyncModeConfig>
new PipelineSchedules
{
["mass"] = new SyncModeConfig("-365.00:00:00", PrePurge: true),
["incremental"] = new SyncModeConfig("-1.00:00:00")
Mass = new ScheduleConfig { PrePurge = true },
Daily = new ScheduleConfig(),
Hourly = new ScheduleConfig()
},
null, // Schedules
null, // Transformers
new DestinationConfig("TestTable", ["Id"], null),
null,
@@ -832,7 +711,7 @@ public class EtlPipelineFactoryTests
// Act
var pipeline = factory.ForTable("TestTable")
.WithMode(SyncMode.Mass)
.WithUpdateType(UpdateTypes.Mass)
.Build();
// Assert
@@ -849,17 +728,17 @@ public class EtlPipelineFactoryTests
// Arrange - null settings should use defaults
var config = new PipelinesRoot(
null, // Null settings
null,
new ScheduleDefaults(),
new Dictionary<string, PipelineConfig>
{
["TestTable"] = new PipelineConfig(
new SourceConfig("lotfinder", "SELECT * FROM Test", null),
new Dictionary<string, SyncModeConfig>
new PipelineSchedules
{
["mass"] = new SyncModeConfig("-365.00:00:00", PrePurge: true),
["incremental"] = new SyncModeConfig("-1.00:00:00")
Mass = new ScheduleConfig { PrePurge = true },
Daily = new ScheduleConfig(),
Hourly = new ScheduleConfig()
},
null, // Schedules
null, // Transformers
new DestinationConfig("TestTable", ["Id"], null),
null,
@@ -869,76 +748,7 @@ public class EtlPipelineFactoryTests
// Act
var pipeline = factory.ForTable("TestTable")
.WithMode(SyncMode.Incremental)
.Build();
// Assert
pipeline.ShouldNotBeNull();
}
#endregion
#region MinDtOffset Format Tests
[Fact]
public void Builder_WithInvalidMinDtOffsetFormat_ThrowsInvalidOperationException()
{
// Arrange
var config = new PipelinesRoot(
new PipelineSettings("UTC"),
null,
new Dictionary<string, PipelineConfig>
{
["TestTable"] = new PipelineConfig(
new SourceConfig("lotfinder", "SELECT * FROM Test", null),
new Dictionary<string, SyncModeConfig>
{
["mass"] = new SyncModeConfig("-365.00:00:00", PrePurge: true),
["incremental"] = new SyncModeConfig("not-a-valid-timespan") // Invalid!
},
null, // Schedules
null, // Transformers
new DestinationConfig("TestTable", ["Id"], null),
null,
null)
});
var factory = CreateFactory(config);
// Act & Assert
var ex = Should.Throw<InvalidOperationException>(() =>
factory.ForTable("TestTable")
.WithMode(SyncMode.Incremental)
.Build());
ex.Message.ShouldContain("Invalid minDtOffset format");
}
[Fact]
public void Builder_WithNullMinDtOffset_DoesNotThrow()
{
// Arrange - null offset should be valid (no date filtering)
var config = new PipelinesRoot(
new PipelineSettings("UTC"),
null,
new Dictionary<string, PipelineConfig>
{
["TestTable"] = new PipelineConfig(
new SourceConfig("lotfinder", "SELECT * FROM Test", null),
new Dictionary<string, SyncModeConfig>
{
["mass"] = new SyncModeConfig(null, PrePurge: true), // Null offset
["incremental"] = new SyncModeConfig("-1.00:00:00")
},
null, // Schedules
null, // Transformers
new DestinationConfig("TestTable", ["Id"], null),
null,
null)
});
var factory = CreateFactory(config);
// Act
var pipeline = factory.ForTable("TestTable")
.WithMode(SyncMode.Mass)
.WithUpdateType(UpdateTypes.Hourly)
.Build();
// Assert
@@ -949,32 +759,6 @@ public class EtlPipelineFactoryTests
#region Helper Methods
private PipelinesRoot CreateValidConfig()
{
return new PipelinesRoot(
new PipelineSettings("UTC"),
null,
new Dictionary<string, PipelineConfig>
{
["TestTable"] = new PipelineConfig(
new SourceConfig("lotfinder", "SELECT * FROM Test WHERE UpdateDt >= @MinDt",
new Dictionary<string, ParameterConfig>
{
["minDt"] = new ParameterConfig("@MinDt", null, "offset", null)
}),
new Dictionary<string, SyncModeConfig>
{
["mass"] = new SyncModeConfig("-365.00:00:00", PrePurge: true, ReIndex: true),
["incremental"] = new SyncModeConfig("-1.00:00:00")
},
null, // Schedules
null, // Transformers
new DestinationConfig("TestTable", ["Id"], null),
null,
null)
});
}
private PipelinesRoot CreateValidConfigWithSchedules()
{
return new PipelinesRoot(
@@ -989,7 +773,6 @@ public class EtlPipelineFactoryTests
["minDt"] = new ParameterConfig("@MinDt", null, "offset", null)
},
"SELECT * FROM Test"), // MassQuery
null, // No old SyncModes
new PipelineSchedules
{
Mass = new ScheduleConfig { PrePurge = true, ReIndex = true },
NEW/tests/JdeScoping.DataSync.Tests/Services/TableSyncOperationTests.cs
-38
View File
@@ -160,44 +160,6 @@ public class TableSyncOperationTests
receivedUpdateType.ShouldBe(UpdateTypes.Mass);
}
[Fact]
public async Task ExecuteAsync_DoesNotCallObsoleteWithModeMethod()
{
// Arrange
var task = CreateTask("TestTable", UpdateTypes.Daily);
var withModeCalled = false;
// Pre-create the test pipeline to avoid NSubstitute issues
var testPipeline = CreateTestPipeline();
var mockBuilder = Substitute.For<IEtlPipelineBuilder>();
mockBuilder.WithUpdateType(Arg.Any<UpdateTypes>()).Returns(mockBuilder);
mockBuilder.WithMode(Arg.Any<SyncMode>())
.Returns(callInfo =>
{
withModeCalled = true;
return mockBuilder;
});
mockBuilder.WithMinimumDate(Arg.Any<DateTime?>()).Returns(mockBuilder);
mockBuilder.Build().Returns(testPipeline);
var mockFactory = Substitute.For<IEtlPipelineFactory>();
mockFactory.ForTable(Arg.Any<string>()).Returns(mockBuilder);
var sut = new TableSyncOperation(
mockFactory,
_updateRepository,
_options,
NullLogger<TableSyncOperation>.Instance,
_metrics);
// Act
await sut.ExecuteAsync(task);
// Assert - Verify the obsolete WithMode method was NOT called
withModeCalled.ShouldBeFalse();
}
#endregion
#region Pipeline Execution Tests