dohertj2/jdescopingtool
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor(datasync): remove deprecated SyncMode and SyncModeConfig

Browse Source 
- Delete SyncMode.cs enum file
- Remove SyncModes property from PipelineConfig
- Remove SyncModeConfig and DestinationOverride records
- Remove WithMode(SyncMode) from IEtlPipelineBuilder
- Remove BuildWithSyncModes() and related methods from EtlPipelineFactory
- Remove syncModes sections from all pipelines in pipelines.json
- Update tests to use schedules-only configuration

All pipelines now require 'schedules' format (mass/daily/hourly).
WithUpdateType(UpdateTypes) is the only way to set update type.
This commit is contained in:
Joseph Doherty
2026-01-07 05:16:20 -05:00
parent 1618b6664d
commit c814a7294b
8 changed files with 122 additions and 670 deletions
Download Patch File Download Diff File Expand all files Collapse all files
NEW/src/JdeScoping.DataSync/Configuration/PipelineConfig.cs
-18
View File
@@ -5,8 +5,6 @@ namespace JdeScoping.DataSync.Configuration;
/// </summary>
public record PipelineConfig(
SourceConfig Source,
[property: Obsolete("Use Schedules property instead. SyncModes will be removed in a future version.")]
Dictionary<string, SyncModeConfig>? SyncModes,
PipelineSchedules? Schedules,
List<TransformerConfig>? Transformers,
DestinationConfig Destination,
@@ -25,22 +23,6 @@ public record ParameterConfig(
string Source = "offset",
string? Value = null);
/// <summary>
/// Sync mode configuration (legacy format).
/// </summary>
[Obsolete("Use ScheduleConfig instead. SyncModeConfig will be removed in a future version.")]
public record SyncModeConfig(
string? MinDtOffset,
bool PrePurge = false,
bool ReIndex = false,
string? UpdateWhen = null,
DestinationOverride? Destination = null);
public record DestinationOverride(
string? Type,
List<string>? MatchColumns,
List<string>? ExcludeFromUpdate);
public record TransformerConfig(
string Type,
List<string>? Columns,
NEW/src/JdeScoping.DataSync/Contracts/IEtlPipelineFactory.cs
-8
View File
@@ -10,14 +10,6 @@ public interface IEtlPipelineFactory
public interface IEtlPipelineBuilder
{
/// <summary>
/// Sets the sync mode for this pipeline.
/// </summary>
/// <param name="mode">The sync mode (Mass or Incremental).</param>
/// <returns>The builder for chaining.</returns>
[Obsolete("Use WithUpdateType instead")]
IEtlPipelineBuilder WithMode(SyncMode mode);
/// <summary>
/// Sets the update type for this pipeline (Mass, Daily, or Hourly).
/// </summary>
NEW/src/JdeScoping.DataSync/Contracts/SyncMode.cs
-11
View File
@@ -1,11 +0,0 @@
namespace JdeScoping.DataSync.Contracts;
/// <summary>
/// Sync mode for ETL pipelines.
/// </summary>
[Obsolete("Use UpdateTypes enum and WithUpdateType() instead. SyncMode will be removed in a future version.")]
public enum SyncMode
{
Mass,
Incremental
}
NEW/src/JdeScoping.DataSync/Pipelines/pipelines.json
-68
View File
@@ -18,10 +18,6 @@
"timeUpdated": { "name": ":timeUpdated", "format": "jdeTime", "source": "offset" }
}
},
"syncModes": {
"mass": { "minDtOffset": "-365.00:00:00", "prePurge": true, "reIndex": true },
"incremental": { "minDtOffset": "-7.00:00:00", "updateWhen": "src.LastUpdateDt > tgt.LastUpdateDt" }
},
"schedules": {
"mass": {},
"daily": {},
@@ -43,10 +39,6 @@
"timeUpdated": { "name": ":timeUpdated", "format": "jdeTime", "source": "offset" }
}
},
"syncModes": {
"mass": { "minDtOffset": "-365.00:00:00", "prePurge": true, "reIndex": true },
"incremental": { "minDtOffset": "-7.00:00:00", "updateWhen": "src.LastUpdateDt > tgt.LastUpdateDt" }
},
"schedules": {
"mass": {},
"daily": {},
@@ -68,10 +60,6 @@
"timeUpdated": { "name": ":timeUpdated", "format": "jdeTime", "source": "offset" }
}
},
"syncModes": {
"mass": { "minDtOffset": "-365.00:00:00", "prePurge": true, "reIndex": true },
"incremental": { "minDtOffset": "-7.00:00:00", "updateWhen": "src.LastUpdateDt > tgt.LastUpdateDt" }
},
"schedules": {
"mass": {},
"daily": {},
@@ -93,10 +81,6 @@
"timeUpdated": { "name": ":timeUpdated", "format": "jdeTime", "source": "offset" }
}
},
"syncModes": {
"mass": { "minDtOffset": "-365.00:00:00", "prePurge": true, "reIndex": true },
"incremental": { "minDtOffset": "-7.00:00:00", "updateWhen": "src.LastUpdateDt > tgt.LastUpdateDt" }
},
"schedules": {
"mass": {},
"daily": {},
@@ -118,10 +102,6 @@
"timeUpdated": { "name": ":timeUpdated", "format": "jdeTime", "source": "offset" }
}
},
"syncModes": {
"mass": { "minDtOffset": "-365.00:00:00", "prePurge": true, "reIndex": true },
"incremental": { "minDtOffset": "-7.00:00:00", "updateWhen": "src.LastUpdateDt > tgt.LastUpdateDt" }
},
"schedules": {
"mass": {},
"daily": {},
@@ -143,10 +123,6 @@
"timeUpdated": { "name": ":timeUpdated", "format": "jdeTime", "source": "offset" }
}
},
"syncModes": {
"mass": { "minDtOffset": "-365.00:00:00", "prePurge": true, "reIndex": true },
"incremental": { "minDtOffset": "-7.00:00:00", "updateWhen": "src.LastUpdateDt > tgt.LastUpdateDt" }
},
"schedules": {
"mass": {},
"daily": {},
@@ -165,10 +141,6 @@
"massQuery": "WITH USER_CTE AS (SELECT ab.ABAN8 AS AddressNumber, TRIM(pro.ULUSER) AS UserId, TRIM(ab.ABALPH) AS FullName, ab.ABUPMJ AS LastUpdateDate, ab.ABUPMT AS LastUpdateTime, ROW_NUMBER() OVER (PARTITION BY ab.ABAN8 ORDER BY ab.ABUPMJ DESC, ab.ABUPMT DESC) RN FROM {ProductionSchema}.F0101 ab LEFT OUTER JOIN {ProductionSchema}.F0092 pro ON (ab.ABAN8 = pro.ULAN8) WHERE ab.ABATE = 'Y') SELECT AddressNumber, UserId, FullName, LastUpdateDate, LastUpdateTime FROM USER_CTE WHERE RN = 1",
"parameters": {}
},
"syncModes": {
"mass": { "minDtOffset": "-365.00:00:00", "prePurge": true, "reIndex": true },
"incremental": { "minDtOffset": "-7.00:00:00", "updateWhen": "src.LastUpdateDt > tgt.LastUpdateDt" }
},
"schedules": {
"mass": {},
"daily": {},
@@ -190,10 +162,6 @@
"timeUpdated": { "name": ":timeUpdated", "format": "jdeTime", "source": "offset" }
}
},
"syncModes": {
"mass": { "minDtOffset": "-365.00:00:00", "prePurge": true, "reIndex": true },
"incremental": { "minDtOffset": "-7.00:00:00", "updateWhen": "src.LastUpdateDt > tgt.LastUpdateDt" }
},
"schedules": {
"mass": {},
"daily": {},
@@ -214,10 +182,6 @@
"lastUpdateDT": { "name": ":lastUpdateDT", "format": null, "source": "offset" }
}
},
"syncModes": {
"mass": { "prePurge": true, "reIndex": true },
"incremental": { "minDtOffset": "-7.00:00:00" }
},
"schedules": {
"mass": { "intervalMinutes": 100800 },
"daily": {},
@@ -243,10 +207,6 @@
"timeUpdated": { "name": ":timeUpdated", "format": "jdeTime", "source": "offset" }
}
},
"syncModes": {
"mass": { "minDtOffset": "-365.00:00:00", "prePurge": true, "reIndex": true },
"incremental": { "minDtOffset": "-7.00:00:00", "updateWhen": "src.LastUpdateDt > tgt.LastUpdateDt" }
},
"schedules": {
"mass": {},
"daily": {},
@@ -268,10 +228,6 @@
"timeUpdated": { "name": ":timeUpdated", "format": "jdeTime", "source": "offset" }
}
},
"syncModes": {
"mass": { "minDtOffset": "-365.00:00:00", "prePurge": true, "reIndex": true },
"incremental": { "minDtOffset": "-7.00:00:00", "updateWhen": "src.LastUpdateDt > tgt.LastUpdateDt" }
},
"schedules": {
"mass": {},
"daily": {},
@@ -293,10 +249,6 @@
"timeUpdated": { "name": ":timeUpdated", "format": "jdeTime", "source": "offset" }
}
},
"syncModes": {
"mass": { "minDtOffset": "-365.00:00:00", "prePurge": true, "reIndex": true },
"incremental": { "minDtOffset": "-7.00:00:00", "updateWhen": "src.LastUpdateDt > tgt.LastUpdateDt" }
},
"schedules": {
"mass": {},
"daily": {},
@@ -318,10 +270,6 @@
"timeUpdated": { "name": ":timeUpdated", "format": "jdeTime", "source": "offset" }
}
},
"syncModes": {
"mass": { "minDtOffset": "-365.00:00:00", "prePurge": true, "reIndex": true },
"incremental": { "minDtOffset": "-7.00:00:00", "updateWhen": "src.LastUpdateDt > tgt.LastUpdateDt" }
},
"schedules": {
"mass": {},
"daily": {},
@@ -343,10 +291,6 @@
"timeUpdated": { "name": ":timeUpdated", "format": "jdeTime", "source": "offset" }
}
},
"syncModes": {
"mass": { "minDtOffset": "-365.00:00:00", "prePurge": true, "reIndex": true },
"incremental": { "minDtOffset": "-7.00:00:00", "updateWhen": "src.LastUpdateDt > tgt.LastUpdateDt" }
},
"schedules": {
"mass": {},
"daily": {},
@@ -368,10 +312,6 @@
"timeUpdated": { "name": ":timeUpdated", "format": "jdeTime", "source": "offset" }
}
},
"syncModes": {
"mass": { "minDtOffset": "-365.00:00:00", "prePurge": true, "reIndex": true },
"incremental": { "minDtOffset": "-7.00:00:00", "updateWhen": "src.LastUpdateDt > tgt.LastUpdateDt" }
},
"schedules": {
"mass": {},
"daily": {},
@@ -393,10 +333,6 @@
"timeUpdated": { "name": ":timeUpdated", "format": "jdeTime", "source": "offset" }
}
},
"syncModes": {
"mass": { "minDtOffset": "-365.00:00:00", "prePurge": true, "reIndex": true },
"incremental": { "minDtOffset": "-7.00:00:00", "updateWhen": "src.LastUpdateDt > tgt.LastUpdateDt" }
},
"schedules": {
"mass": {},
"daily": {},
@@ -415,10 +351,6 @@
"massQuery": "SELECT Code, TRIM(LISTAGG(Description, ' ') WITHIN GROUP(ORDER BY Description) || CASE WHEN MAX(total_lengthb) > 4000 THEN '...' ELSE '' END) Description, SYSDATE AS LastUpdateDT FROM (SELECT TRIM(fc.CFKY) AS Code, TRIM(ASCIISTR(fc.CFDS80)) AS Description, SUM(LENGTHB(TRIM(fc.CFDS80))+1) OVER(PARTITION BY TRIM(fc.CFKY) ORDER BY TRIM(fc.CFDS80)) - 1 cumul_lengthb, SUM(LENGTHB(TRIM(fc.CFDS80))+1) OVER(PARTITION BY TRIM(fc.CFKY)) - 1 total_lengthb, COUNT(*) OVER(PARTITION BY TRIM(fc.CFKY)) num_values FROM PRODDTA.F00192 fc WHERE TRIM(fc.CFKY) IS NOT NULL) WHERE total_lengthb <= 4000 OR cumul_lengthb <= 4000 - length('...') GROUP BY Code",
"parameters": {}
},
"syncModes": {
"mass": { "prePurge": true, "reIndex": true },
"incremental": { "prePurge": true, "reIndex": true }
},
"schedules": {
"mass": { "prePurge": true, "reIndex": true },
"daily": { "prePurge": true, "reIndex": true },
NEW/src/JdeScoping.DataSync/Services/EtlPipelineFactory.cs
+9 -196
View File
@@ -115,34 +115,11 @@ public class EtlPipelineFactory : IEtlPipelineFactory
{
foreach (var (name, config) in root.Pipelines)
{
// Accept either old SyncModes or new Schedules format
var hasOldConfig = config.SyncModes != null &&
config.SyncModes.ContainsKey("mass") &&
config.SyncModes.ContainsKey("incremental");
var hasNewConfig = config.Schedules != null;
if (!hasOldConfig && !hasNewConfig)
// Schedules are now required
if (config.Schedules == null)
{
// If neither format is present, check for the old partial config for backward-compat error messages
if (config.SyncModes != null)
{
if (!config.SyncModes.ContainsKey("mass"))
{
throw new InvalidOperationException(
$"Pipeline '{name}' missing required 'mass' sync mode.");
}
if (!config.SyncModes.ContainsKey("incremental"))
{
throw new InvalidOperationException(
$"Pipeline '{name}' missing required 'incremental' sync mode.");
}
}
else
{
throw new InvalidOperationException(
$"Pipeline '{name}' must define either 'syncModes' (mass+incremental) or 'schedules'.");
}
throw new InvalidOperationException(
$"Pipeline '{name}' must define 'schedules'.");
}
// Validate no runtime parameters (not yet supported)
@@ -188,14 +165,6 @@ public class EtlPipelineFactory : IEtlPipelineFactory
_logger = logger;
}
[Obsolete("Use WithUpdateType instead")]
public IEtlPipelineBuilder WithMode(SyncMode mode)
{
// Map old SyncMode to new UpdateTypes for backward compatibility
_updateType = mode == SyncMode.Mass ? UpdateTypes.Mass : UpdateTypes.Hourly;
return this;
}
public IEtlPipelineBuilder WithUpdateType(UpdateTypes updateType)
{
_updateType = updateType;
@@ -210,15 +179,7 @@ public class EtlPipelineFactory : IEtlPipelineFactory
public EtlPipeline Build()
{
// Check if using new Schedules format or old SyncModes format
if (_config.Schedules != null)
{
return BuildWithSchedules();
}
else
{
return BuildWithSyncModes();
}
return BuildWithSchedules();
}
private EtlPipeline BuildWithSchedules()
@@ -232,11 +193,11 @@ public class EtlPipelineFactory : IEtlPipelineFactory
var useMassQuery = _updateType == UpdateTypes.Mass && !string.IsNullOrEmpty(_config.Source.MassQuery);
// Create source with parameter substitution
var source = CreateSourceWithUpdateType(_config.Source, minDt, useMassQuery);
var source = CreateSource(_config.Source, minDt, useMassQuery);
// Determine destination type (Mass with prePurge = bulkImport, others = bulkMerge unless prePurge)
var destType = scheduleConfig.PrePurge ? "bulkImport" : "bulkMerge";
var destination = CreateDestinationWithSchedule(destType, _config.Destination, scheduleConfig);
var destination = CreateDestination(destType, _config.Destination, scheduleConfig);
// Build pipeline with scripts
var builder = new EtlPipelineBuilder()
@@ -272,69 +233,6 @@ public class EtlPipelineFactory : IEtlPipelineFactory
return builder.Build();
}
private EtlPipeline BuildWithSyncModes()
{
// Map UpdateTypes to old sync mode keys for backward compatibility
var modeKey = _updateType == UpdateTypes.Mass ? "mass" : "incremental";
if (!_config.SyncModes!.TryGetValue(modeKey, out var modeConfig))
{
throw new InvalidOperationException(
$"Sync mode '{modeKey}' not defined for table '{_tableName}'.");
}
// Compute MinDt from offset or override
var minDt = _minDtOverride ?? ComputeMinDt(modeConfig.MinDtOffset);
// Convert UpdateTypes to SyncMode for backward compatibility with CreateSource
var syncMode = _updateType == UpdateTypes.Mass ? SyncMode.Mass : SyncMode.Incremental;
// Create source with parameter substitution
var source = CreateSource(_config.Source, minDt, syncMode);
// Determine destination type (mode override > default by mode)
var destType = modeConfig.Destination?.Type
?? (_updateType == UpdateTypes.Mass ? "bulkImport" : "bulkMerge");
var destination = CreateDestination(destType, _config.Destination, modeConfig);
// Build pipeline with scripts
var builder = new EtlPipelineBuilder()
.WithName(_tableName)
.WithSource(source)
.WithDestination(destination)
.WithLogger(_logger);
// Add pre-scripts: config scripts first, then prePurge
foreach (var script in _config.PreScripts ?? [])
{
builder.WithPreScript(new SqlScriptRunner(_connectionFactory, script, $"PreScript:{script.Substring(0, Math.Min(30, script.Length))}"));
}
if (modeConfig.PrePurge)
{
var truncateSql = $"TRUNCATE TABLE [{_config.Destination.Table}]";
builder.WithPreScript(new SqlScriptRunner(_connectionFactory, truncateSql, "PrePurge"));
}
// Add post-scripts: reIndex first, then config scripts
if (modeConfig.ReIndex)
{
var reindexSql = $"ALTER INDEX ALL ON [{_config.Destination.Table}] REBUILD";
builder.WithPostScript(new SqlScriptRunner(_connectionFactory, reindexSql, "ReIndex"));
}
foreach (var script in _config.PostScripts ?? [])
{
builder.WithPostScript(new SqlScriptRunner(_connectionFactory, script, $"PostScript:{script.Substring(0, Math.Min(30, script.Length))}"));
}
// Transformers are not yet implemented - placeholder for future
// foreach (var t in _config.Transformers ?? [])
// builder.WithTransformer(CreateTransformer(t));
return builder.Build();
}
private Configuration.ScheduleConfig GetEffectiveScheduleConfig(UpdateTypes updateType)
{
// Get default for this update type
@@ -359,92 +257,7 @@ public class EtlPipelineFactory : IEtlPipelineFactory
return pipelineConfig?.MergeWith(defaultConfig) ?? defaultConfig;
}
private DateTime? ComputeMinDt(string? minDtOffset)
{
if (string.IsNullOrEmpty(minDtOffset))
return null;
if (!TimeSpan.TryParse(minDtOffset, out var offset))
{
throw new InvalidOperationException(
$"Invalid minDtOffset format: '{minDtOffset}'. Expected TimeSpan format (e.g., '-7.00:00:00').");
}
return DateTime.UtcNow.Add(offset);
}
private IImportSource CreateSource(SourceConfig sourceConfig, DateTime? minDt, SyncMode mode)
{
// Use massQuery if available and in mass mode, otherwise use the default query
var query = (mode == SyncMode.Mass && !string.IsNullOrEmpty(sourceConfig.MassQuery))
? sourceConfig.MassQuery
: sourceConfig.Query;
var parameters = new Dictionary<string, object>();
var converter = new ParameterFormatConverter(_settings.Timezone);
// Only add parameters for incremental mode or when using the default query
// Mass mode with massQuery typically doesn't need date parameters
var needsParameters = mode != SyncMode.Mass || string.IsNullOrEmpty(sourceConfig.MassQuery);
if (sourceConfig.Parameters != null && minDt.HasValue && needsParameters)
{
foreach (var (_, paramConfig) in sourceConfig.Parameters)
{
var paramValue = paramConfig.Source.ToLowerInvariant() switch
{
"offset" => converter.Convert(minDt.Value, paramConfig.Format),
"static" => paramConfig.Value
?? throw new InvalidOperationException(
$"Static parameter '{paramConfig.Name}' requires a value."),
_ => throw new NotSupportedException(
$"Parameter source '{paramConfig.Source}' is not supported.")
};
// Use the parameter name exactly as configured (provider-specific)
parameters[paramConfig.Name] = paramValue;
}
}
return new DbQuerySource(
_connectionFactory,
sourceConfig.Connection,
query,
parameters);
}
private IImportDestination CreateDestination(
string destType,
DestinationConfig baseConfig,
SyncModeConfig modeConfig)
{
var tableName = baseConfig.Table;
// Merge mode-specific destination config with base
var matchColumns = modeConfig.Destination?.MatchColumns?.ToArray()
?? baseConfig.MatchColumns?.ToArray();
var excludeFromUpdate = modeConfig.Destination?.ExcludeFromUpdate?.ToArray()
?? baseConfig.ExcludeFromUpdate?.ToArray();
return destType.ToLowerInvariant() switch
{
"bulkimport" => new DbBulkImportDestination(_connectionFactory, tableName),
"bulkmerge" => new DbBulkMergeDestination(
_connectionFactory,
tableName,
matchColumns ?? throw new InvalidOperationException(
$"matchColumns required for bulkMerge destination on table '{tableName}'."),
updateColumns: null,
excludeFromUpdate: excludeFromUpdate,
updateCondition: modeConfig.UpdateWhen),
_ => throw new InvalidOperationException(
$"Unknown destination type: '{destType}'. Expected 'bulkImport' or 'bulkMerge'.")
};
}
private IImportSource CreateSourceWithUpdateType(SourceConfig sourceConfig, DateTime? minDt, bool useMassQuery)
private IImportSource CreateSource(SourceConfig sourceConfig, DateTime? minDt, bool useMassQuery)
{
// Use massQuery if specified, otherwise use the default query
var query = useMassQuery ? sourceConfig.MassQuery! : sourceConfig.Query;
@@ -481,7 +294,7 @@ public class EtlPipelineFactory : IEtlPipelineFactory
parameters);
}
private IImportDestination CreateDestinationWithSchedule(
private IImportDestination CreateDestination(
string destType,
DestinationConfig baseConfig,
Configuration.ScheduleConfig scheduleConfig)
NEW/tests/JdeScoping.DataSync.Tests/Configuration/PipelinesRootTests.cs
+5 -6
View File
@@ -72,14 +72,13 @@ public class PipelinesRootTests
{
var config = new PipelineConfig(
new SourceConfig("jde", "SELECT 1", null, null),
null, // Old SyncModes - deprecated
new PipelineSchedules
{
Mass = new ScheduleConfig { PrePurge = true, ReIndex = true },
Daily = new ScheduleConfig { Enabled = true },
Hourly = new ScheduleConfig { Enabled = false }
},
null,
null, // Transformers
new DestinationConfig("TestTable", ["Id"], null),
null,
null);
@@ -93,12 +92,12 @@ public class PipelinesRootTests
{
return new PipelineConfig(
new SourceConfig("lotfinder", "SELECT 1", null, null),
new Dictionary<string, SyncModeConfig>
new PipelineSchedules
{
["mass"] = new SyncModeConfig(null, true, true),
["incremental"] = new SyncModeConfig("-1d")
Mass = new ScheduleConfig { PrePurge = true, ReIndex = true },
Daily = new ScheduleConfig(),
Hourly = new ScheduleConfig()
},
null, // Schedules
null, // Transformers
new DestinationConfig("TestTable", ["Id"], null),
null,
NEW/tests/JdeScoping.DataSync.Tests/Services/EtlPipelineFactoryTests.cs
+108 -325
View File
@@ -28,7 +28,7 @@ public class EtlPipelineFactoryTests
public void ForTable_WithValidTable_ReturnsBuilder()
{
// Arrange
var config = CreateValidConfig();
var config = CreateValidConfigWithSchedules();
var factory = CreateFactory(config);
// Act
@@ -43,7 +43,7 @@ public class EtlPipelineFactoryTests
public void ForTable_WithUnknownTable_ThrowsInvalidOperationException()
{
// Arrange
var config = CreateValidConfig();
var config = CreateValidConfigWithSchedules();
var factory = CreateFactory(config);
// Act & Assert
@@ -56,7 +56,7 @@ public class EtlPipelineFactoryTests
public void ForTable_WithNullTableName_ThrowsArgumentException()
{
// Arrange
var config = CreateValidConfig();
var config = CreateValidConfigWithSchedules();
var factory = CreateFactory(config);
// Act & Assert
@@ -67,7 +67,7 @@ public class EtlPipelineFactoryTests
public void ForTable_WithEmptyTableName_ThrowsArgumentException()
{
// Arrange
var config = CreateValidConfig();
var config = CreateValidConfigWithSchedules();
var factory = CreateFactory(config);
// Act & Assert
@@ -76,87 +76,6 @@ public class EtlPipelineFactoryTests
#endregion
#region Builder WithMode Tests
[Fact]
public void Builder_WithMassMode_BuildsPipeline()
{
// Arrange
var config = CreateValidConfig();
var factory = CreateFactory(config);
// Act
var pipeline = factory.ForTable("TestTable")
.WithMode(SyncMode.Mass)
.Build();
// Assert
pipeline.ShouldNotBeNull();
pipeline.PipelineName.ShouldBe("TestTable");
}
[Fact]
public void Builder_WithIncrementalMode_BuildsPipeline()
{
// Arrange
var config = CreateValidConfig();
var factory = CreateFactory(config);
// Act
var pipeline = factory.ForTable("TestTable")
.WithMode(SyncMode.Incremental)
.Build();
// Assert
pipeline.ShouldNotBeNull();
pipeline.PipelineName.ShouldBe("TestTable");
}
[Fact]
public void Builder_DefaultMode_IsIncremental()
{
// Arrange
var config = CreateValidConfig();
var factory = CreateFactory(config);
// Act - don't call WithMode()
var pipeline = factory.ForTable("TestTable")
.Build();
// Assert - should work because incremental mode is defined
pipeline.ShouldNotBeNull();
}
[Fact]
public void Builder_WithUndefinedSyncMode_ThrowsInvalidOperationException()
{
// Arrange - config with only mass mode
var config = new PipelinesRoot(
new PipelineSettings("UTC"),
null,
new Dictionary<string, PipelineConfig>
{
["TestTable"] = new PipelineConfig(
new SourceConfig("lotfinder", "SELECT * FROM Test", null),
new Dictionary<string, SyncModeConfig>
{
["mass"] = new SyncModeConfig("-365.00:00:00", PrePurge: true)
// No incremental mode defined
},
null, // Schedules
null, // Transformers
new DestinationConfig("TestTable", ["Id"], null),
null,
null)
});
// Act & Assert - validation fails at factory creation
var ex = Should.Throw<InvalidOperationException>(() => CreateFactory(config));
ex.Message.ShouldContain("missing required 'incremental' sync mode");
}
#endregion
#region Builder WithUpdateType Tests
[Fact]
@@ -290,6 +209,21 @@ public class EtlPipelineFactoryTests
pipeline.ShouldNotBeNull();
}
[Fact]
public void Builder_DefaultMode_IsHourly()
{
// Arrange
var config = CreateValidConfigWithSchedules();
var factory = CreateFactory(config);
// Act - don't call WithUpdateType()
var pipeline = factory.ForTable("TestTable")
.Build();
// Assert - should work because hourly mode is defined
pipeline.ShouldNotBeNull();
}
#endregion
#region Builder WithMinimumDate Tests
@@ -298,13 +232,13 @@ public class EtlPipelineFactoryTests
public void Builder_WithMinimumDate_OverridesConfigOffset()
{
// Arrange
var config = CreateValidConfig();
var config = CreateValidConfigWithSchedules();
var factory = CreateFactory(config);
var customDate = new DateTime(2024, 1, 1, 0, 0, 0, DateTimeKind.Utc);
// Act - should not throw even though we're overriding
var pipeline = factory.ForTable("TestTable")
.WithMode(SyncMode.Incremental)
.WithUpdateType(UpdateTypes.Hourly)
.WithMinimumDate(customDate)
.Build();
@@ -316,12 +250,12 @@ public class EtlPipelineFactoryTests
public void Builder_WithNullMinimumDate_UsesConfigOffset()
{
// Arrange
var config = CreateValidConfig();
var config = CreateValidConfigWithSchedules();
var factory = CreateFactory(config);
// Act - null minDt means use config offset
var pipeline = factory.ForTable("TestTable")
.WithMode(SyncMode.Incremental)
.WithUpdateType(UpdateTypes.Hourly)
.WithMinimumDate(null)
.Build();
@@ -334,7 +268,7 @@ public class EtlPipelineFactoryTests
#region Config Validation Tests
[Fact]
public void Validate_ConfigMissingMassMode_ThrowsInvalidOperationException()
public void Validate_ConfigMissingSchedules_ThrowsInvalidOperationException()
{
// Arrange
var config = new PipelinesRoot(
@@ -344,12 +278,7 @@ public class EtlPipelineFactoryTests
{
["TestTable"] = new PipelineConfig(
new SourceConfig("lotfinder", "SELECT * FROM Test", null),
new Dictionary<string, SyncModeConfig>
{
// Missing mass mode
["incremental"] = new SyncModeConfig("-1.00:00:00")
},
null, // Schedules
null, // Schedules - null means invalid
null, // Transformers
new DestinationConfig("TestTable", ["Id"], null),
null,
@@ -358,7 +287,7 @@ public class EtlPipelineFactoryTests
// Act & Assert
var ex = Should.Throw<InvalidOperationException>(() => CreateFactory(config));
ex.Message.ShouldContain("missing required 'mass' sync mode");
ex.Message.ShouldContain("must define 'schedules'");
}
[Fact]
@@ -376,12 +305,12 @@ public class EtlPipelineFactoryTests
{
["id"] = new ParameterConfig("@Id", null, "runtime", null)
}),
new Dictionary<string, SyncModeConfig>
new PipelineSchedules
{
["mass"] = new SyncModeConfig("-365.00:00:00"),
["incremental"] = new SyncModeConfig("-1.00:00:00")
Mass = new ScheduleConfig(),
Daily = new ScheduleConfig(),
Hourly = new ScheduleConfig()
},
null, // Schedules
null, // Transformers
new DestinationConfig("TestTable", ["Id"], null),
null,
@@ -398,32 +327,15 @@ public class EtlPipelineFactoryTests
#region Destination Type Tests
[Fact]
public void Builder_MassMode_DefaultsToBulkImport()
public void Builder_MassMode_WithPrePurge_UsesBulkImport()
{
// Arrange - no destination override
var config = new PipelinesRoot(
new PipelineSettings("UTC"),
null,
new Dictionary<string, PipelineConfig>
{
["TestTable"] = new PipelineConfig(
new SourceConfig("lotfinder", "SELECT * FROM Test", null),
new Dictionary<string, SyncModeConfig>
{
["mass"] = new SyncModeConfig("-365.00:00:00", PrePurge: true),
["incremental"] = new SyncModeConfig("-1.00:00:00")
},
null, // Schedules
null, // Transformers
new DestinationConfig("TestTable", ["Id"], null),
null,
null)
});
// Arrange - Mass with prePurge defaults to bulkImport
var config = CreateValidConfigWithSchedules();
var factory = CreateFactory(config);
// Act - should use bulkImport for mass mode
// Act - should use bulkImport for mass mode with prePurge
var pipeline = factory.ForTable("TestTable")
.WithMode(SyncMode.Mass)
.WithUpdateType(UpdateTypes.Mass)
.Build();
// Assert
@@ -431,49 +343,15 @@ public class EtlPipelineFactoryTests
}
[Fact]
public void Builder_IncrementalMode_DefaultsToBulkMerge()
public void Builder_HourlyMode_UsesBulkMerge()
{
// Arrange - no destination override
var config = CreateValidConfig();
// Arrange - Hourly without prePurge uses bulkMerge
var config = CreateValidConfigWithSchedules();
var factory = CreateFactory(config);
// Act - should use bulkMerge for incremental mode
// Act - should use bulkMerge for hourly mode
var pipeline = factory.ForTable("TestTable")
.WithMode(SyncMode.Incremental)
.Build();
// Assert
pipeline.ShouldNotBeNull();
}
[Fact]
public void Builder_ModeWithDestinationOverride_UsesOverride()
{
// Arrange - mass mode with bulkMerge override
var config = new PipelinesRoot(
new PipelineSettings("UTC"),
null,
new Dictionary<string, PipelineConfig>
{
["TestTable"] = new PipelineConfig(
new SourceConfig("lotfinder", "SELECT * FROM Test", null),
new Dictionary<string, SyncModeConfig>
{
["mass"] = new SyncModeConfig("-365.00:00:00",
Destination: new DestinationOverride("bulkMerge", null, null)),
["incremental"] = new SyncModeConfig("-1.00:00:00")
},
null, // Schedules
null, // Transformers
new DestinationConfig("TestTable", ["Id"], null),
null,
null)
});
var factory = CreateFactory(config);
// Act - mass mode should use bulkMerge due to override
var pipeline = factory.ForTable("TestTable")
.WithMode(SyncMode.Mass)
.WithUpdateType(UpdateTypes.Hourly)
.Build();
// Assert
@@ -486,17 +364,17 @@ public class EtlPipelineFactoryTests
// Arrange - bulkMerge needs matchColumns
var config = new PipelinesRoot(
new PipelineSettings("UTC"),
null,
new ScheduleDefaults(),
new Dictionary<string, PipelineConfig>
{
["TestTable"] = new PipelineConfig(
new SourceConfig("lotfinder", "SELECT * FROM Test", null),
new Dictionary<string, SyncModeConfig>
new PipelineSchedules
{
["mass"] = new SyncModeConfig("-365.00:00:00", PrePurge: true),
["incremental"] = new SyncModeConfig("-1.00:00:00")
Mass = new ScheduleConfig { PrePurge = true },
Daily = new ScheduleConfig(),
Hourly = new ScheduleConfig()
},
null, // Schedules
null, // Transformers
new DestinationConfig("TestTable", null, null), // No matchColumns!
null,
@@ -507,7 +385,7 @@ public class EtlPipelineFactoryTests
// Act & Assert
var ex = Should.Throw<InvalidOperationException>(() =>
factory.ForTable("TestTable")
.WithMode(SyncMode.Incremental) // Uses bulkMerge
.WithUpdateType(UpdateTypes.Hourly) // Uses bulkMerge
.Build());
ex.Message.ShouldContain("matchColumns required for bulkMerge");
}
@@ -522,7 +400,7 @@ public class EtlPipelineFactoryTests
// Arrange
var config = new PipelinesRoot(
new PipelineSettings("UTC"),
null,
new ScheduleDefaults(),
new Dictionary<string, PipelineConfig>
{
["TestTable"] = new PipelineConfig(
@@ -531,12 +409,12 @@ public class EtlPipelineFactoryTests
{
["minDt"] = new ParameterConfig("@MinDt", null, "offset", null)
}),
new Dictionary<string, SyncModeConfig>
new PipelineSchedules
{
["mass"] = new SyncModeConfig("-365.00:00:00", PrePurge: true),
["incremental"] = new SyncModeConfig("-1.00:00:00")
Mass = new ScheduleConfig { PrePurge = true },
Daily = new ScheduleConfig(),
Hourly = new ScheduleConfig()
},
null, // Schedules
null, // Transformers
new DestinationConfig("TestTable", ["Id"], null),
null,
@@ -546,7 +424,7 @@ public class EtlPipelineFactoryTests
// Act
var pipeline = factory.ForTable("TestTable")
.WithMode(SyncMode.Incremental)
.WithUpdateType(UpdateTypes.Hourly)
.Build();
// Assert
@@ -559,7 +437,7 @@ public class EtlPipelineFactoryTests
// Arrange
var config = new PipelinesRoot(
new PipelineSettings("UTC"),
null,
new ScheduleDefaults(),
new Dictionary<string, PipelineConfig>
{
["TestTable"] = new PipelineConfig(
@@ -568,12 +446,12 @@ public class EtlPipelineFactoryTests
{
["minDt"] = new ParameterConfig(":dateUpdated", "jdeJulian", "offset", null)
}),
new Dictionary<string, SyncModeConfig>
new PipelineSchedules
{
["mass"] = new SyncModeConfig("-365.00:00:00", PrePurge: true),
["incremental"] = new SyncModeConfig("-1.00:00:00")
Mass = new ScheduleConfig { PrePurge = true },
Daily = new ScheduleConfig(),
Hourly = new ScheduleConfig()
},
null, // Schedules
null, // Transformers
new DestinationConfig("TestTable", ["Id"], null),
null,
@@ -583,7 +461,7 @@ public class EtlPipelineFactoryTests
// Act
var pipeline = factory.ForTable("TestTable")
.WithMode(SyncMode.Incremental)
.WithUpdateType(UpdateTypes.Hourly)
.Build();
// Assert
@@ -596,7 +474,7 @@ public class EtlPipelineFactoryTests
// Arrange
var config = new PipelinesRoot(
new PipelineSettings("UTC"),
null,
new ScheduleDefaults(),
new Dictionary<string, PipelineConfig>
{
["TestTable"] = new PipelineConfig(
@@ -605,12 +483,12 @@ public class EtlPipelineFactoryTests
{
["status"] = new ParameterConfig("@Status", null, "static", "Active")
}),
new Dictionary<string, SyncModeConfig>
new PipelineSchedules
{
["mass"] = new SyncModeConfig("-365.00:00:00", PrePurge: true),
["incremental"] = new SyncModeConfig("-1.00:00:00")
Mass = new ScheduleConfig { PrePurge = true },
Daily = new ScheduleConfig(),
Hourly = new ScheduleConfig()
},
null, // Schedules
null, // Transformers
new DestinationConfig("TestTable", ["Id"], null),
null,
@@ -620,7 +498,7 @@ public class EtlPipelineFactoryTests
// Act
var pipeline = factory.ForTable("TestTable")
.WithMode(SyncMode.Incremental)
.WithUpdateType(UpdateTypes.Hourly)
.Build();
// Assert
@@ -633,7 +511,7 @@ public class EtlPipelineFactoryTests
// Arrange
var config = new PipelinesRoot(
new PipelineSettings("UTC"),
null,
new ScheduleDefaults(),
new Dictionary<string, PipelineConfig>
{
["TestTable"] = new PipelineConfig(
@@ -642,12 +520,12 @@ public class EtlPipelineFactoryTests
{
["status"] = new ParameterConfig("@Status", null, "static", null) // No value!
}),
new Dictionary<string, SyncModeConfig>
new PipelineSchedules
{
["mass"] = new SyncModeConfig("-365.00:00:00", PrePurge: true),
["incremental"] = new SyncModeConfig("-1.00:00:00")
Mass = new ScheduleConfig { PrePurge = true },
Daily = new ScheduleConfig(),
Hourly = new ScheduleConfig()
},
null, // Schedules
null, // Transformers
new DestinationConfig("TestTable", ["Id"], null),
null,
@@ -655,10 +533,11 @@ public class EtlPipelineFactoryTests
});
var factory = CreateFactory(config);
// Act & Assert
// Act & Assert - must provide minDt for parameters to be processed
var ex = Should.Throw<InvalidOperationException>(() =>
factory.ForTable("TestTable")
.WithMode(SyncMode.Incremental)
.WithUpdateType(UpdateTypes.Hourly)
.WithMinimumDate(DateTime.UtcNow.AddDays(-1))
.Build());
ex.Message.ShouldContain("Static parameter '@Status' requires a value");
}
@@ -673,17 +552,17 @@ public class EtlPipelineFactoryTests
// Arrange
var config = new PipelinesRoot(
new PipelineSettings("UTC"),
null,
new ScheduleDefaults(),
new Dictionary<string, PipelineConfig>
{
["TestTable"] = new PipelineConfig(
new SourceConfig("lotfinder", "SELECT * FROM Test", null),
new Dictionary<string, SyncModeConfig>
new PipelineSchedules
{
["mass"] = new SyncModeConfig("-365.00:00:00", PrePurge: true),
["incremental"] = new SyncModeConfig("-1.00:00:00")
Mass = new ScheduleConfig { PrePurge = true },
Daily = new ScheduleConfig(),
Hourly = new ScheduleConfig()
},
null, // Schedules
null, // Transformers
new DestinationConfig("TestTable", ["Id"], null),
null,
@@ -693,7 +572,7 @@ public class EtlPipelineFactoryTests
// Act
var pipeline = factory.ForTable("TestTable")
.WithMode(SyncMode.Mass)
.WithUpdateType(UpdateTypes.Mass)
.Build();
// Assert
@@ -706,17 +585,17 @@ public class EtlPipelineFactoryTests
// Arrange
var config = new PipelinesRoot(
new PipelineSettings("UTC"),
null,
new ScheduleDefaults(),
new Dictionary<string, PipelineConfig>
{
["TestTable"] = new PipelineConfig(
new SourceConfig("lotfinder", "SELECT * FROM Test", null),
new Dictionary<string, SyncModeConfig>
new PipelineSchedules
{
["mass"] = new SyncModeConfig("-365.00:00:00", PrePurge: true, ReIndex: true),
["incremental"] = new SyncModeConfig("-1.00:00:00")
Mass = new ScheduleConfig { PrePurge = true, ReIndex = true },
Daily = new ScheduleConfig(),
Hourly = new ScheduleConfig()
},
null, // Schedules
null, // Transformers
new DestinationConfig("TestTable", ["Id"], null),
null,
@@ -726,7 +605,7 @@ public class EtlPipelineFactoryTests
// Act
var pipeline = factory.ForTable("TestTable")
.WithMode(SyncMode.Mass)
.WithUpdateType(UpdateTypes.Mass)
.Build();
// Assert
@@ -739,17 +618,17 @@ public class EtlPipelineFactoryTests
// Arrange
var config = new PipelinesRoot(
new PipelineSettings("UTC"),
null,
new ScheduleDefaults(),
new Dictionary<string, PipelineConfig>
{
["TestTable"] = new PipelineConfig(
new SourceConfig("lotfinder", "SELECT * FROM Test", null),
new Dictionary<string, SyncModeConfig>
new PipelineSchedules
{
["mass"] = new SyncModeConfig("-365.00:00:00", PrePurge: true),
["incremental"] = new SyncModeConfig("-1.00:00:00")
Mass = new ScheduleConfig { PrePurge = true },
Daily = new ScheduleConfig(),
Hourly = new ScheduleConfig()
},
null, // Schedules
null, // Transformers
new DestinationConfig("TestTable", ["Id"], null),
["EXEC sp_BeforeSync"],
@@ -759,7 +638,7 @@ public class EtlPipelineFactoryTests
// Act
var pipeline = factory.ForTable("TestTable")
.WithMode(SyncMode.Mass)
.WithUpdateType(UpdateTypes.Mass)
.Build();
// Assert
@@ -772,17 +651,17 @@ public class EtlPipelineFactoryTests
// Arrange
var config = new PipelinesRoot(
new PipelineSettings("UTC"),
null,
new ScheduleDefaults(),
new Dictionary<string, PipelineConfig>
{
["TestTable"] = new PipelineConfig(
new SourceConfig("lotfinder", "SELECT * FROM Test", null),
new Dictionary<string, SyncModeConfig>
new PipelineSchedules
{
["mass"] = new SyncModeConfig("-365.00:00:00", PrePurge: true),
["incremental"] = new SyncModeConfig("-1.00:00:00")
Mass = new ScheduleConfig { PrePurge = true },
Daily = new ScheduleConfig(),
Hourly = new ScheduleConfig()
},
null, // Schedules
null, // Transformers
new DestinationConfig("TestTable", ["Id"], null),
null,
@@ -792,7 +671,7 @@ public class EtlPipelineFactoryTests
// Act
var pipeline = factory.ForTable("TestTable")
.WithMode(SyncMode.Incremental)
.WithUpdateType(UpdateTypes.Hourly)
.Build();
// Assert
@@ -812,17 +691,17 @@ public class EtlPipelineFactoryTests
// Arrange
var config = new PipelinesRoot(
new PipelineSettings("UTC"),
null,
new ScheduleDefaults(),
new Dictionary<string, PipelineConfig>
{
["TestTable"] = new PipelineConfig(
new SourceConfig(connectionType, "SELECT * FROM Test", null),
new Dictionary<string, SyncModeConfig>
new PipelineSchedules
{
["mass"] = new SyncModeConfig("-365.00:00:00", PrePurge: true),
["incremental"] = new SyncModeConfig("-1.00:00:00")
Mass = new ScheduleConfig { PrePurge = true },
Daily = new ScheduleConfig(),
Hourly = new ScheduleConfig()
},
null, // Schedules
null, // Transformers
new DestinationConfig("TestTable", ["Id"], null),
null,
@@ -832,7 +711,7 @@ public class EtlPipelineFactoryTests
// Act
var pipeline = factory.ForTable("TestTable")
.WithMode(SyncMode.Mass)
.WithUpdateType(UpdateTypes.Mass)
.Build();
// Assert
@@ -849,17 +728,17 @@ public class EtlPipelineFactoryTests
// Arrange - null settings should use defaults
var config = new PipelinesRoot(
null, // Null settings
null,
new ScheduleDefaults(),
new Dictionary<string, PipelineConfig>
{
["TestTable"] = new PipelineConfig(
new SourceConfig("lotfinder", "SELECT * FROM Test", null),
new Dictionary<string, SyncModeConfig>
new PipelineSchedules
{
["mass"] = new SyncModeConfig("-365.00:00:00", PrePurge: true),
["incremental"] = new SyncModeConfig("-1.00:00:00")
Mass = new ScheduleConfig { PrePurge = true },
Daily = new ScheduleConfig(),
Hourly = new ScheduleConfig()
},
null, // Schedules
null, // Transformers
new DestinationConfig("TestTable", ["Id"], null),
null,
@@ -869,76 +748,7 @@ public class EtlPipelineFactoryTests
// Act
var pipeline = factory.ForTable("TestTable")
.WithMode(SyncMode.Incremental)
.Build();
// Assert
pipeline.ShouldNotBeNull();
}
#endregion
#region MinDtOffset Format Tests
[Fact]
public void Builder_WithInvalidMinDtOffsetFormat_ThrowsInvalidOperationException()
{
// Arrange
var config = new PipelinesRoot(
new PipelineSettings("UTC"),
null,
new Dictionary<string, PipelineConfig>
{
["TestTable"] = new PipelineConfig(
new SourceConfig("lotfinder", "SELECT * FROM Test", null),
new Dictionary<string, SyncModeConfig>
{
["mass"] = new SyncModeConfig("-365.00:00:00", PrePurge: true),
["incremental"] = new SyncModeConfig("not-a-valid-timespan") // Invalid!
},
null, // Schedules
null, // Transformers
new DestinationConfig("TestTable", ["Id"], null),
null,
null)
});
var factory = CreateFactory(config);
// Act & Assert
var ex = Should.Throw<InvalidOperationException>(() =>
factory.ForTable("TestTable")
.WithMode(SyncMode.Incremental)
.Build());
ex.Message.ShouldContain("Invalid minDtOffset format");
}
[Fact]
public void Builder_WithNullMinDtOffset_DoesNotThrow()
{
// Arrange - null offset should be valid (no date filtering)
var config = new PipelinesRoot(
new PipelineSettings("UTC"),
null,
new Dictionary<string, PipelineConfig>
{
["TestTable"] = new PipelineConfig(
new SourceConfig("lotfinder", "SELECT * FROM Test", null),
new Dictionary<string, SyncModeConfig>
{
["mass"] = new SyncModeConfig(null, PrePurge: true), // Null offset
["incremental"] = new SyncModeConfig("-1.00:00:00")
},
null, // Schedules
null, // Transformers
new DestinationConfig("TestTable", ["Id"], null),
null,
null)
});
var factory = CreateFactory(config);
// Act
var pipeline = factory.ForTable("TestTable")
.WithMode(SyncMode.Mass)
.WithUpdateType(UpdateTypes.Hourly)
.Build();
// Assert
@@ -949,32 +759,6 @@ public class EtlPipelineFactoryTests
#region Helper Methods
private PipelinesRoot CreateValidConfig()
{
return new PipelinesRoot(
new PipelineSettings("UTC"),
null,
new Dictionary<string, PipelineConfig>
{
["TestTable"] = new PipelineConfig(
new SourceConfig("lotfinder", "SELECT * FROM Test WHERE UpdateDt >= @MinDt",
new Dictionary<string, ParameterConfig>
{
["minDt"] = new ParameterConfig("@MinDt", null, "offset", null)
}),
new Dictionary<string, SyncModeConfig>
{
["mass"] = new SyncModeConfig("-365.00:00:00", PrePurge: true, ReIndex: true),
["incremental"] = new SyncModeConfig("-1.00:00:00")
},
null, // Schedules
null, // Transformers
new DestinationConfig("TestTable", ["Id"], null),
null,
null)
});
}
private PipelinesRoot CreateValidConfigWithSchedules()
{
return new PipelinesRoot(
@@ -989,7 +773,6 @@ public class EtlPipelineFactoryTests
["minDt"] = new ParameterConfig("@MinDt", null, "offset", null)
},
"SELECT * FROM Test"), // MassQuery
null, // No old SyncModes
new PipelineSchedules
{
Mass = new ScheduleConfig { PrePurge = true, ReIndex = true },
NEW/tests/JdeScoping.DataSync.Tests/Services/TableSyncOperationTests.cs
-38
View File
@@ -160,44 +160,6 @@ public class TableSyncOperationTests
receivedUpdateType.ShouldBe(UpdateTypes.Mass);
}
[Fact]
public async Task ExecuteAsync_DoesNotCallObsoleteWithModeMethod()
{
// Arrange
var task = CreateTask("TestTable", UpdateTypes.Daily);
var withModeCalled = false;
// Pre-create the test pipeline to avoid NSubstitute issues
var testPipeline = CreateTestPipeline();
var mockBuilder = Substitute.For<IEtlPipelineBuilder>();
mockBuilder.WithUpdateType(Arg.Any<UpdateTypes>()).Returns(mockBuilder);
mockBuilder.WithMode(Arg.Any<SyncMode>())
.Returns(callInfo =>
{
withModeCalled = true;
return mockBuilder;
});
mockBuilder.WithMinimumDate(Arg.Any<DateTime?>()).Returns(mockBuilder);
mockBuilder.Build().Returns(testPipeline);
var mockFactory = Substitute.For<IEtlPipelineFactory>();
mockFactory.ForTable(Arg.Any<string>()).Returns(mockBuilder);
var sut = new TableSyncOperation(
mockFactory,
_updateRepository,
_options,
NullLogger<TableSyncOperation>.Instance,
_metrics);
// Act
await sut.ExecuteAsync(task);
// Assert - Verify the obsolete WithMode method was NOT called
withModeCalled.ShouldBeFalse();
}
#endregion
#region Pipeline Execution Tests