dohertj2/jdescopingtool
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor(datasync): remove deprecated SyncMode and SyncModeConfig

Browse Source 
- Delete SyncMode.cs enum file
- Remove SyncModes property from PipelineConfig
- Remove SyncModeConfig and DestinationOverride records
- Remove WithMode(SyncMode) from IEtlPipelineBuilder
- Remove BuildWithSyncModes() and related methods from EtlPipelineFactory
- Remove syncModes sections from all pipelines in pipelines.json
- Update tests to use schedules-only configuration

All pipelines now require 'schedules' format (mass/daily/hourly).
WithUpdateType(UpdateTypes) is the only way to set update type.
This commit is contained in:
Joseph Doherty
2026-01-07 05:16:20 -05:00
parent 1618b6664d
commit c814a7294b
8 changed files with 122 additions and 670 deletions
Download Patch File Download Diff File Expand all files Collapse all files
NEW/src/JdeScoping.DataSync/Configuration/PipelineConfig.cs
-18
View File
@@ -5,8 +5,6 @@ namespace JdeScoping.DataSync.Configuration;
/// </summary>
public record PipelineConfig(
SourceConfig Source,
[property: Obsolete("Use Schedules property instead. SyncModes will be removed in a future version.")]
Dictionary<string, SyncModeConfig>? SyncModes,
PipelineSchedules? Schedules,
List<TransformerConfig>? Transformers,
DestinationConfig Destination,
@@ -25,22 +23,6 @@ public record ParameterConfig(
string Source = "offset",
string? Value = null);
/// <summary>
/// Sync mode configuration (legacy format).
/// </summary>
[Obsolete("Use ScheduleConfig instead. SyncModeConfig will be removed in a future version.")]
public record SyncModeConfig(
string? MinDtOffset,
bool PrePurge = false,
bool ReIndex = false,
string? UpdateWhen = null,
DestinationOverride? Destination = null);
public record DestinationOverride(
string? Type,
List<string>? MatchColumns,
List<string>? ExcludeFromUpdate);
public record TransformerConfig(
string Type,
List<string>? Columns,
NEW/src/JdeScoping.DataSync/Contracts/IEtlPipelineFactory.cs
-8
View File
@@ -10,14 +10,6 @@ public interface IEtlPipelineFactory
public interface IEtlPipelineBuilder
{
/// <summary>
/// Sets the sync mode for this pipeline.
/// </summary>
/// <param name="mode">The sync mode (Mass or Incremental).</param>
/// <returns>The builder for chaining.</returns>
[Obsolete("Use WithUpdateType instead")]
IEtlPipelineBuilder WithMode(SyncMode mode);
/// <summary>
/// Sets the update type for this pipeline (Mass, Daily, or Hourly).
/// </summary>
NEW/src/JdeScoping.DataSync/Contracts/SyncMode.cs
-11
View File
@@ -1,11 +0,0 @@
namespace JdeScoping.DataSync.Contracts;
/// <summary>
/// Sync mode for ETL pipelines.
/// </summary>
[Obsolete("Use UpdateTypes enum and WithUpdateType() instead. SyncMode will be removed in a future version.")]
public enum SyncMode
{
Mass,
Incremental
}
NEW/src/JdeScoping.DataSync/Pipelines/pipelines.json
-68
View File
@@ -18,10 +18,6 @@
"timeUpdated": { "name": ":timeUpdated", "format": "jdeTime", "source": "offset" }
}
},
"syncModes": {
"mass": { "minDtOffset": "-365.00:00:00", "prePurge": true, "reIndex": true },
"incremental": { "minDtOffset": "-7.00:00:00", "updateWhen": "src.LastUpdateDt > tgt.LastUpdateDt" }
},
"schedules": {
"mass": {},
"daily": {},
@@ -43,10 +39,6 @@
"timeUpdated": { "name": ":timeUpdated", "format": "jdeTime", "source": "offset" }
}
},
"syncModes": {
"mass": { "minDtOffset": "-365.00:00:00", "prePurge": true, "reIndex": true },
"incremental": { "minDtOffset": "-7.00:00:00", "updateWhen": "src.LastUpdateDt > tgt.LastUpdateDt" }
},
"schedules": {
"mass": {},
"daily": {},
@@ -68,10 +60,6 @@
"timeUpdated": { "name": ":timeUpdated", "format": "jdeTime", "source": "offset" }
}
},
"syncModes": {
"mass": { "minDtOffset": "-365.00:00:00", "prePurge": true, "reIndex": true },
"incremental": { "minDtOffset": "-7.00:00:00", "updateWhen": "src.LastUpdateDt > tgt.LastUpdateDt" }
},
"schedules": {
"mass": {},
"daily": {},
@@ -93,10 +81,6 @@
"timeUpdated": { "name": ":timeUpdated", "format": "jdeTime", "source": "offset" }
}
},
"syncModes": {
"mass": { "minDtOffset": "-365.00:00:00", "prePurge": true, "reIndex": true },
"incremental": { "minDtOffset": "-7.00:00:00", "updateWhen": "src.LastUpdateDt > tgt.LastUpdateDt" }
},
"schedules": {
"mass": {},
"daily": {},
@@ -118,10 +102,6 @@
"timeUpdated": { "name": ":timeUpdated", "format": "jdeTime", "source": "offset" }
}
},
"syncModes": {
"mass": { "minDtOffset": "-365.00:00:00", "prePurge": true, "reIndex": true },
"incremental": { "minDtOffset": "-7.00:00:00", "updateWhen": "src.LastUpdateDt > tgt.LastUpdateDt" }
},
"schedules": {
"mass": {},
"daily": {},
@@ -143,10 +123,6 @@
"timeUpdated": { "name": ":timeUpdated", "format": "jdeTime", "source": "offset" }
}
},
"syncModes": {
"mass": { "minDtOffset": "-365.00:00:00", "prePurge": true, "reIndex": true },
"incremental": { "minDtOffset": "-7.00:00:00", "updateWhen": "src.LastUpdateDt > tgt.LastUpdateDt" }
},
"schedules": {
"mass": {},
"daily": {},
@@ -165,10 +141,6 @@
"massQuery": "WITH USER_CTE AS (SELECT ab.ABAN8 AS AddressNumber, TRIM(pro.ULUSER) AS UserId, TRIM(ab.ABALPH) AS FullName, ab.ABUPMJ AS LastUpdateDate, ab.ABUPMT AS LastUpdateTime, ROW_NUMBER() OVER (PARTITION BY ab.ABAN8 ORDER BY ab.ABUPMJ DESC, ab.ABUPMT DESC) RN FROM {ProductionSchema}.F0101 ab LEFT OUTER JOIN {ProductionSchema}.F0092 pro ON (ab.ABAN8 = pro.ULAN8) WHERE ab.ABATE = 'Y') SELECT AddressNumber, UserId, FullName, LastUpdateDate, LastUpdateTime FROM USER_CTE WHERE RN = 1",
"parameters": {}
},
"syncModes": {
"mass": { "minDtOffset": "-365.00:00:00", "prePurge": true, "reIndex": true },
"incremental": { "minDtOffset": "-7.00:00:00", "updateWhen": "src.LastUpdateDt > tgt.LastUpdateDt" }
},
"schedules": {
"mass": {},
"daily": {},
@@ -190,10 +162,6 @@
"timeUpdated": { "name": ":timeUpdated", "format": "jdeTime", "source": "offset" }
}
},
"syncModes": {
"mass": { "minDtOffset": "-365.00:00:00", "prePurge": true, "reIndex": true },
"incremental": { "minDtOffset": "-7.00:00:00", "updateWhen": "src.LastUpdateDt > tgt.LastUpdateDt" }
},
"schedules": {
"mass": {},
"daily": {},
@@ -214,10 +182,6 @@
"lastUpdateDT": { "name": ":lastUpdateDT", "format": null, "source": "offset" }
}
},
"syncModes": {
"mass": { "prePurge": true, "reIndex": true },
"incremental": { "minDtOffset": "-7.00:00:00" }
},
"schedules": {
"mass": { "intervalMinutes": 100800 },
"daily": {},
@@ -243,10 +207,6 @@
"timeUpdated": { "name": ":timeUpdated", "format": "jdeTime", "source": "offset" }
}
},
"syncModes": {
"mass": { "minDtOffset": "-365.00:00:00", "prePurge": true, "reIndex": true },
"incremental": { "minDtOffset": "-7.00:00:00", "updateWhen": "src.LastUpdateDt > tgt.LastUpdateDt" }
},
"schedules": {
"mass": {},
"daily": {},
@@ -268,10 +228,6 @@
"timeUpdated": { "name": ":timeUpdated", "format": "jdeTime", "source": "offset" }
}
},
"syncModes": {
"mass": { "minDtOffset": "-365.00:00:00", "prePurge": true, "reIndex": true },
"incremental": { "minDtOffset": "-7.00:00:00", "updateWhen": "src.LastUpdateDt > tgt.LastUpdateDt" }
},
"schedules": {
"mass": {},
"daily": {},
@@ -293,10 +249,6 @@
"timeUpdated": { "name": ":timeUpdated", "format": "jdeTime", "source": "offset" }
}
},
"syncModes": {
"mass": { "minDtOffset": "-365.00:00:00", "prePurge": true, "reIndex": true },
"incremental": { "minDtOffset": "-7.00:00:00", "updateWhen": "src.LastUpdateDt > tgt.LastUpdateDt" }
},
"schedules": {
"mass": {},
"daily": {},
@@ -318,10 +270,6 @@
"timeUpdated": { "name": ":timeUpdated", "format": "jdeTime", "source": "offset" }
}
},
"syncModes": {
"mass": { "minDtOffset": "-365.00:00:00", "prePurge": true, "reIndex": true },
"incremental": { "minDtOffset": "-7.00:00:00", "updateWhen": "src.LastUpdateDt > tgt.LastUpdateDt" }
},
"schedules": {
"mass": {},
"daily": {},
@@ -343,10 +291,6 @@
"timeUpdated": { "name": ":timeUpdated", "format": "jdeTime", "source": "offset" }
}
},
"syncModes": {
"mass": { "minDtOffset": "-365.00:00:00", "prePurge": true, "reIndex": true },
"incremental": { "minDtOffset": "-7.00:00:00", "updateWhen": "src.LastUpdateDt > tgt.LastUpdateDt" }
},
"schedules": {
"mass": {},
"daily": {},
@@ -368,10 +312,6 @@
"timeUpdated": { "name": ":timeUpdated", "format": "jdeTime", "source": "offset" }
}
},
"syncModes": {
"mass": { "minDtOffset": "-365.00:00:00", "prePurge": true, "reIndex": true },
"incremental": { "minDtOffset": "-7.00:00:00", "updateWhen": "src.LastUpdateDt > tgt.LastUpdateDt" }
},
"schedules": {
"mass": {},
"daily": {},
@@ -393,10 +333,6 @@
"timeUpdated": { "name": ":timeUpdated", "format": "jdeTime", "source": "offset" }
}
},
"syncModes": {
"mass": { "minDtOffset": "-365.00:00:00", "prePurge": true, "reIndex": true },
"incremental": { "minDtOffset": "-7.00:00:00", "updateWhen": "src.LastUpdateDt > tgt.LastUpdateDt" }
},
"schedules": {
"mass": {},
"daily": {},
@@ -415,10 +351,6 @@
"massQuery": "SELECT Code, TRIM(LISTAGG(Description, ' ') WITHIN GROUP(ORDER BY Description) || CASE WHEN MAX(total_lengthb) > 4000 THEN '...' ELSE '' END) Description, SYSDATE AS LastUpdateDT FROM (SELECT TRIM(fc.CFKY) AS Code, TRIM(ASCIISTR(fc.CFDS80)) AS Description, SUM(LENGTHB(TRIM(fc.CFDS80))+1) OVER(PARTITION BY TRIM(fc.CFKY) ORDER BY TRIM(fc.CFDS80)) - 1 cumul_lengthb, SUM(LENGTHB(TRIM(fc.CFDS80))+1) OVER(PARTITION BY TRIM(fc.CFKY)) - 1 total_lengthb, COUNT(*) OVER(PARTITION BY TRIM(fc.CFKY)) num_values FROM PRODDTA.F00192 fc WHERE TRIM(fc.CFKY) IS NOT NULL) WHERE total_lengthb <= 4000 OR cumul_lengthb <= 4000 - length('...') GROUP BY Code",
"parameters": {}
},
"syncModes": {
"mass": { "prePurge": true, "reIndex": true },
"incremental": { "prePurge": true, "reIndex": true }
},
"schedules": {
"mass": { "prePurge": true, "reIndex": true },
"daily": { "prePurge": true, "reIndex": true },
NEW/src/JdeScoping.DataSync/Services/EtlPipelineFactory.cs
+9 -196
View File
@@ -115,34 +115,11 @@ public class EtlPipelineFactory : IEtlPipelineFactory
{
foreach (var (name, config) in root.Pipelines)
{
// Accept either old SyncModes or new Schedules format
var hasOldConfig = config.SyncModes != null &&
config.SyncModes.ContainsKey("mass") &&
config.SyncModes.ContainsKey("incremental");
var hasNewConfig = config.Schedules != null;
if (!hasOldConfig && !hasNewConfig)
// Schedules are now required
if (config.Schedules == null)
{
// If neither format is present, check for the old partial config for backward-compat error messages
if (config.SyncModes != null)
{
if (!config.SyncModes.ContainsKey("mass"))
{
throw new InvalidOperationException(
$"Pipeline '{name}' missing required 'mass' sync mode.");
}
if (!config.SyncModes.ContainsKey("incremental"))
{
throw new InvalidOperationException(
$"Pipeline '{name}' missing required 'incremental' sync mode.");
}
}
else
{
throw new InvalidOperationException(
$"Pipeline '{name}' must define either 'syncModes' (mass+incremental) or 'schedules'.");
}
throw new InvalidOperationException(
$"Pipeline '{name}' must define 'schedules'.");
}
// Validate no runtime parameters (not yet supported)
@@ -188,14 +165,6 @@ public class EtlPipelineFactory : IEtlPipelineFactory
_logger = logger;
}
[Obsolete("Use WithUpdateType instead")]
public IEtlPipelineBuilder WithMode(SyncMode mode)
{
// Map old SyncMode to new UpdateTypes for backward compatibility
_updateType = mode == SyncMode.Mass ? UpdateTypes.Mass : UpdateTypes.Hourly;
return this;
}
public IEtlPipelineBuilder WithUpdateType(UpdateTypes updateType)
{
_updateType = updateType;
@@ -210,15 +179,7 @@ public class EtlPipelineFactory : IEtlPipelineFactory
public EtlPipeline Build()
{
// Check if using new Schedules format or old SyncModes format
if (_config.Schedules != null)
{
return BuildWithSchedules();
}
else
{
return BuildWithSyncModes();
}
return BuildWithSchedules();
}
private EtlPipeline BuildWithSchedules()
@@ -232,11 +193,11 @@ public class EtlPipelineFactory : IEtlPipelineFactory
var useMassQuery = _updateType == UpdateTypes.Mass && !string.IsNullOrEmpty(_config.Source.MassQuery);
// Create source with parameter substitution
var source = CreateSourceWithUpdateType(_config.Source, minDt, useMassQuery);
var source = CreateSource(_config.Source, minDt, useMassQuery);
// Determine destination type (Mass with prePurge = bulkImport, others = bulkMerge unless prePurge)
var destType = scheduleConfig.PrePurge ? "bulkImport" : "bulkMerge";
var destination = CreateDestinationWithSchedule(destType, _config.Destination, scheduleConfig);
var destination = CreateDestination(destType, _config.Destination, scheduleConfig);
// Build pipeline with scripts
var builder = new EtlPipelineBuilder()
@@ -272,69 +233,6 @@ public class EtlPipelineFactory : IEtlPipelineFactory
return builder.Build();
}
private EtlPipeline BuildWithSyncModes()
{
// Map UpdateTypes to old sync mode keys for backward compatibility
var modeKey = _updateType == UpdateTypes.Mass ? "mass" : "incremental";
if (!_config.SyncModes!.TryGetValue(modeKey, out var modeConfig))
{
throw new InvalidOperationException(
$"Sync mode '{modeKey}' not defined for table '{_tableName}'.");
}
// Compute MinDt from offset or override
var minDt = _minDtOverride ?? ComputeMinDt(modeConfig.MinDtOffset);
// Convert UpdateTypes to SyncMode for backward compatibility with CreateSource
var syncMode = _updateType == UpdateTypes.Mass ? SyncMode.Mass : SyncMode.Incremental;
// Create source with parameter substitution
var source = CreateSource(_config.Source, minDt, syncMode);
// Determine destination type (mode override > default by mode)
var destType = modeConfig.Destination?.Type
?? (_updateType == UpdateTypes.Mass ? "bulkImport" : "bulkMerge");
var destination = CreateDestination(destType, _config.Destination, modeConfig);
// Build pipeline with scripts
var builder = new EtlPipelineBuilder()
.WithName(_tableName)
.WithSource(source)
.WithDestination(destination)
.WithLogger(_logger);
// Add pre-scripts: config scripts first, then prePurge
foreach (var script in _config.PreScripts ?? [])
{
builder.WithPreScript(new SqlScriptRunner(_connectionFactory, script, $"PreScript:{script.Substring(0, Math.Min(30, script.Length))}"));
}
if (modeConfig.PrePurge)
{
var truncateSql = $"TRUNCATE TABLE [{_config.Destination.Table}]";
builder.WithPreScript(new SqlScriptRunner(_connectionFactory, truncateSql, "PrePurge"));
}
// Add post-scripts: reIndex first, then config scripts
if (modeConfig.ReIndex)
{
var reindexSql = $"ALTER INDEX ALL ON [{_config.Destination.Table}] REBUILD";
builder.WithPostScript(new SqlScriptRunner(_connectionFactory, reindexSql, "ReIndex"));
}
foreach (var script in _config.PostScripts ?? [])
{
builder.WithPostScript(new SqlScriptRunner(_connectionFactory, script, $"PostScript:{script.Substring(0, Math.Min(30, script.Length))}"));
}
// Transformers are not yet implemented - placeholder for future
// foreach (var t in _config.Transformers ?? [])
// builder.WithTransformer(CreateTransformer(t));
return builder.Build();
}
private Configuration.ScheduleConfig GetEffectiveScheduleConfig(UpdateTypes updateType)
{
// Get default for this update type
@@ -359,92 +257,7 @@ public class EtlPipelineFactory : IEtlPipelineFactory
return pipelineConfig?.MergeWith(defaultConfig) ?? defaultConfig;
}
private DateTime? ComputeMinDt(string? minDtOffset)
{
if (string.IsNullOrEmpty(minDtOffset))
return null;
if (!TimeSpan.TryParse(minDtOffset, out var offset))
{
throw new InvalidOperationException(
$"Invalid minDtOffset format: '{minDtOffset}'. Expected TimeSpan format (e.g., '-7.00:00:00').");
}
return DateTime.UtcNow.Add(offset);
}
private IImportSource CreateSource(SourceConfig sourceConfig, DateTime? minDt, SyncMode mode)
{
// Use massQuery if available and in mass mode, otherwise use the default query
var query = (mode == SyncMode.Mass && !string.IsNullOrEmpty(sourceConfig.MassQuery))
? sourceConfig.MassQuery
: sourceConfig.Query;
var parameters = new Dictionary<string, object>();
var converter = new ParameterFormatConverter(_settings.Timezone);
// Only add parameters for incremental mode or when using the default query
// Mass mode with massQuery typically doesn't need date parameters
var needsParameters = mode != SyncMode.Mass || string.IsNullOrEmpty(sourceConfig.MassQuery);
if (sourceConfig.Parameters != null && minDt.HasValue && needsParameters)
{
foreach (var (_, paramConfig) in sourceConfig.Parameters)
{
var paramValue = paramConfig.Source.ToLowerInvariant() switch
{
"offset" => converter.Convert(minDt.Value, paramConfig.Format),
"static" => paramConfig.Value
?? throw new InvalidOperationException(
$"Static parameter '{paramConfig.Name}' requires a value."),
_ => throw new NotSupportedException(
$"Parameter source '{paramConfig.Source}' is not supported.")
};
// Use the parameter name exactly as configured (provider-specific)
parameters[paramConfig.Name] = paramValue;
}
}
return new DbQuerySource(
_connectionFactory,
sourceConfig.Connection,
query,
parameters);
}
private IImportDestination CreateDestination(
string destType,
DestinationConfig baseConfig,
SyncModeConfig modeConfig)
{
var tableName = baseConfig.Table;
// Merge mode-specific destination config with base
var matchColumns = modeConfig.Destination?.MatchColumns?.ToArray()
?? baseConfig.MatchColumns?.ToArray();
var excludeFromUpdate = modeConfig.Destination?.ExcludeFromUpdate?.ToArray()
?? baseConfig.ExcludeFromUpdate?.ToArray();
return destType.ToLowerInvariant() switch
{
"bulkimport" => new DbBulkImportDestination(_connectionFactory, tableName),
"bulkmerge" => new DbBulkMergeDestination(
_connectionFactory,
tableName,
matchColumns ?? throw new InvalidOperationException(
$"matchColumns required for bulkMerge destination on table '{tableName}'."),
updateColumns: null,
excludeFromUpdate: excludeFromUpdate,
updateCondition: modeConfig.UpdateWhen),
_ => throw new InvalidOperationException(
$"Unknown destination type: '{destType}'. Expected 'bulkImport' or 'bulkMerge'.")
};
}
private IImportSource CreateSourceWithUpdateType(SourceConfig sourceConfig, DateTime? minDt, bool useMassQuery)
private IImportSource CreateSource(SourceConfig sourceConfig, DateTime? minDt, bool useMassQuery)
{
// Use massQuery if specified, otherwise use the default query
var query = useMassQuery ? sourceConfig.MassQuery! : sourceConfig.Query;
@@ -481,7 +294,7 @@ public class EtlPipelineFactory : IEtlPipelineFactory
parameters);
}
private IImportDestination CreateDestinationWithSchedule(
private IImportDestination CreateDestination(
string destType,
DestinationConfig baseConfig,
Configuration.ScheduleConfig scheduleConfig)