dohertj2/jdescopingtool
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor(datasync): remove deprecated SyncMode and SyncModeConfig

Browse Source 
- Delete SyncMode.cs enum file
- Remove SyncModes property from PipelineConfig
- Remove SyncModeConfig and DestinationOverride records
- Remove WithMode(SyncMode) from IEtlPipelineBuilder
- Remove BuildWithSyncModes() and related methods from EtlPipelineFactory
- Remove syncModes sections from all pipelines in pipelines.json
- Update tests to use schedules-only configuration

All pipelines now require 'schedules' format (mass/daily/hourly).
WithUpdateType(UpdateTypes) is the only way to set update type.
This commit is contained in:
Joseph Doherty
2026-01-07 05:16:20 -05:00
parent 1618b6664d
commit c814a7294b
8 changed files with 122 additions and 670 deletions
Download Patch File Download Diff File Expand all files Collapse all files
NEW/src/JdeScoping.DataSync/Services/EtlPipelineFactory.cs
+9 -196
View File
@@ -115,34 +115,11 @@ public class EtlPipelineFactory : IEtlPipelineFactory
{
foreach (var (name, config) in root.Pipelines)
{
// Accept either old SyncModes or new Schedules format
var hasOldConfig = config.SyncModes != null &&
config.SyncModes.ContainsKey("mass") &&
config.SyncModes.ContainsKey("incremental");
var hasNewConfig = config.Schedules != null;
if (!hasOldConfig && !hasNewConfig)
// Schedules are now required
if (config.Schedules == null)
{
// If neither format is present, check for the old partial config for backward-compat error messages
if (config.SyncModes != null)
{
if (!config.SyncModes.ContainsKey("mass"))
{
throw new InvalidOperationException(
$"Pipeline '{name}' missing required 'mass' sync mode.");
}
if (!config.SyncModes.ContainsKey("incremental"))
{
throw new InvalidOperationException(
$"Pipeline '{name}' missing required 'incremental' sync mode.");
}
}
else
{
throw new InvalidOperationException(
$"Pipeline '{name}' must define either 'syncModes' (mass+incremental) or 'schedules'.");
}
throw new InvalidOperationException(
$"Pipeline '{name}' must define 'schedules'.");
}
// Validate no runtime parameters (not yet supported)
@@ -188,14 +165,6 @@ public class EtlPipelineFactory : IEtlPipelineFactory
_logger = logger;
}
[Obsolete("Use WithUpdateType instead")]
public IEtlPipelineBuilder WithMode(SyncMode mode)
{
// Map old SyncMode to new UpdateTypes for backward compatibility
_updateType = mode == SyncMode.Mass ? UpdateTypes.Mass : UpdateTypes.Hourly;
return this;
}
public IEtlPipelineBuilder WithUpdateType(UpdateTypes updateType)
{
_updateType = updateType;
@@ -210,15 +179,7 @@ public class EtlPipelineFactory : IEtlPipelineFactory
public EtlPipeline Build()
{
// Check if using new Schedules format or old SyncModes format
if (_config.Schedules != null)
{
return BuildWithSchedules();
}
else
{
return BuildWithSyncModes();
}
return BuildWithSchedules();
}
private EtlPipeline BuildWithSchedules()
@@ -232,11 +193,11 @@ public class EtlPipelineFactory : IEtlPipelineFactory
var useMassQuery = _updateType == UpdateTypes.Mass && !string.IsNullOrEmpty(_config.Source.MassQuery);
// Create source with parameter substitution
var source = CreateSourceWithUpdateType(_config.Source, minDt, useMassQuery);
var source = CreateSource(_config.Source, minDt, useMassQuery);
// Determine destination type (Mass with prePurge = bulkImport, others = bulkMerge unless prePurge)
var destType = scheduleConfig.PrePurge ? "bulkImport" : "bulkMerge";
var destination = CreateDestinationWithSchedule(destType, _config.Destination, scheduleConfig);
var destination = CreateDestination(destType, _config.Destination, scheduleConfig);
// Build pipeline with scripts
var builder = new EtlPipelineBuilder()
@@ -272,69 +233,6 @@ public class EtlPipelineFactory : IEtlPipelineFactory
return builder.Build();
}
private EtlPipeline BuildWithSyncModes()
{
// Map UpdateTypes to old sync mode keys for backward compatibility
var modeKey = _updateType == UpdateTypes.Mass ? "mass" : "incremental";
if (!_config.SyncModes!.TryGetValue(modeKey, out var modeConfig))
{
throw new InvalidOperationException(
$"Sync mode '{modeKey}' not defined for table '{_tableName}'.");
}
// Compute MinDt from offset or override
var minDt = _minDtOverride ?? ComputeMinDt(modeConfig.MinDtOffset);
// Convert UpdateTypes to SyncMode for backward compatibility with CreateSource
var syncMode = _updateType == UpdateTypes.Mass ? SyncMode.Mass : SyncMode.Incremental;
// Create source with parameter substitution
var source = CreateSource(_config.Source, minDt, syncMode);
// Determine destination type (mode override > default by mode)
var destType = modeConfig.Destination?.Type
?? (_updateType == UpdateTypes.Mass ? "bulkImport" : "bulkMerge");
var destination = CreateDestination(destType, _config.Destination, modeConfig);
// Build pipeline with scripts
var builder = new EtlPipelineBuilder()
.WithName(_tableName)
.WithSource(source)
.WithDestination(destination)
.WithLogger(_logger);
// Add pre-scripts: config scripts first, then prePurge
foreach (var script in _config.PreScripts ?? [])
{
builder.WithPreScript(new SqlScriptRunner(_connectionFactory, script, $"PreScript:{script.Substring(0, Math.Min(30, script.Length))}"));
}
if (modeConfig.PrePurge)
{
var truncateSql = $"TRUNCATE TABLE [{_config.Destination.Table}]";
builder.WithPreScript(new SqlScriptRunner(_connectionFactory, truncateSql, "PrePurge"));
}
// Add post-scripts: reIndex first, then config scripts
if (modeConfig.ReIndex)
{
var reindexSql = $"ALTER INDEX ALL ON [{_config.Destination.Table}] REBUILD";
builder.WithPostScript(new SqlScriptRunner(_connectionFactory, reindexSql, "ReIndex"));
}
foreach (var script in _config.PostScripts ?? [])
{
builder.WithPostScript(new SqlScriptRunner(_connectionFactory, script, $"PostScript:{script.Substring(0, Math.Min(30, script.Length))}"));
}
// Transformers are not yet implemented - placeholder for future
// foreach (var t in _config.Transformers ?? [])
// builder.WithTransformer(CreateTransformer(t));
return builder.Build();
}
private Configuration.ScheduleConfig GetEffectiveScheduleConfig(UpdateTypes updateType)
{
// Get default for this update type
@@ -359,92 +257,7 @@ public class EtlPipelineFactory : IEtlPipelineFactory
return pipelineConfig?.MergeWith(defaultConfig) ?? defaultConfig;
}
private DateTime? ComputeMinDt(string? minDtOffset)
{
if (string.IsNullOrEmpty(minDtOffset))
return null;
if (!TimeSpan.TryParse(minDtOffset, out var offset))
{
throw new InvalidOperationException(
$"Invalid minDtOffset format: '{minDtOffset}'. Expected TimeSpan format (e.g., '-7.00:00:00').");
}
return DateTime.UtcNow.Add(offset);
}
private IImportSource CreateSource(SourceConfig sourceConfig, DateTime? minDt, SyncMode mode)
{
// Use massQuery if available and in mass mode, otherwise use the default query
var query = (mode == SyncMode.Mass && !string.IsNullOrEmpty(sourceConfig.MassQuery))
? sourceConfig.MassQuery
: sourceConfig.Query;
var parameters = new Dictionary<string, object>();
var converter = new ParameterFormatConverter(_settings.Timezone);
// Only add parameters for incremental mode or when using the default query
// Mass mode with massQuery typically doesn't need date parameters
var needsParameters = mode != SyncMode.Mass || string.IsNullOrEmpty(sourceConfig.MassQuery);
if (sourceConfig.Parameters != null && minDt.HasValue && needsParameters)
{
foreach (var (_, paramConfig) in sourceConfig.Parameters)
{
var paramValue = paramConfig.Source.ToLowerInvariant() switch
{
"offset" => converter.Convert(minDt.Value, paramConfig.Format),
"static" => paramConfig.Value
?? throw new InvalidOperationException(
$"Static parameter '{paramConfig.Name}' requires a value."),
_ => throw new NotSupportedException(
$"Parameter source '{paramConfig.Source}' is not supported.")
};
// Use the parameter name exactly as configured (provider-specific)
parameters[paramConfig.Name] = paramValue;
}
}
return new DbQuerySource(
_connectionFactory,
sourceConfig.Connection,
query,
parameters);
}
private IImportDestination CreateDestination(
string destType,
DestinationConfig baseConfig,
SyncModeConfig modeConfig)
{
var tableName = baseConfig.Table;
// Merge mode-specific destination config with base
var matchColumns = modeConfig.Destination?.MatchColumns?.ToArray()
?? baseConfig.MatchColumns?.ToArray();
var excludeFromUpdate = modeConfig.Destination?.ExcludeFromUpdate?.ToArray()
?? baseConfig.ExcludeFromUpdate?.ToArray();
return destType.ToLowerInvariant() switch
{
"bulkimport" => new DbBulkImportDestination(_connectionFactory, tableName),
"bulkmerge" => new DbBulkMergeDestination(
_connectionFactory,
tableName,
matchColumns ?? throw new InvalidOperationException(
$"matchColumns required for bulkMerge destination on table '{tableName}'."),
updateColumns: null,
excludeFromUpdate: excludeFromUpdate,
updateCondition: modeConfig.UpdateWhen),
_ => throw new InvalidOperationException(
$"Unknown destination type: '{destType}'. Expected 'bulkImport' or 'bulkMerge'.")
};
}
private IImportSource CreateSourceWithUpdateType(SourceConfig sourceConfig, DateTime? minDt, bool useMassQuery)
private IImportSource CreateSource(SourceConfig sourceConfig, DateTime? minDt, bool useMassQuery)
{
// Use massQuery if specified, otherwise use the default query
var query = useMassQuery ? sourceConfig.MassQuery! : sourceConfig.Query;
@@ -481,7 +294,7 @@ public class EtlPipelineFactory : IEtlPipelineFactory
parameters);
}
private IImportDestination CreateDestinationWithSchedule(
private IImportDestination CreateDestination(
string destType,
DestinationConfig baseConfig,
Configuration.ScheduleConfig scheduleConfig)