dohertj2/jdescopingtool
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(datasync): add Schedules property to PipelineConfig

Browse Source 
Add PipelineSchedules? Schedules parameter to PipelineConfig record between
SyncModes and Transformers for new schedule-based configuration. The existing
SyncModes property is now nullable for backward compatibility during the
transition to the new schedule system.
This commit is contained in:
Joseph Doherty
2026-01-07 00:41:03 -05:00
parent 15cfc1a010
commit 21f598f25c
3 changed files with 64 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files
NEW/src/JdeScoping.DataSync/Configuration/PipelineConfig.cs
+2 -1
View File
@@ -2,7 +2,8 @@ namespace JdeScoping.DataSync.Configuration;
public record PipelineConfig( public record PipelineConfig(
SourceConfig Source, SourceConfig Source,
Dictionary<string, SyncModeConfig> SyncModes, Dictionary<string, SyncModeConfig>? SyncModes,
PipelineSchedules? Schedules,
List<TransformerConfig>? Transformers, List<TransformerConfig>? Transformers,
DestinationConfig Destination, DestinationConfig Destination,
List<string>? PreScripts, List<string>? PreScripts,
NEW/tests/JdeScoping.DataSync.Tests/Configuration/PipelinesRootTests.cs
+24 -1
View File
@@ -67,6 +67,28 @@ public class PipelinesRootTests
root.Pipelines["TestTable"].Destination.Table.ShouldBe("TestTable"); root.Pipelines["TestTable"].Destination.Table.ShouldBe("TestTable");
} }
[Fact]
public void PipelineConfig_WithSchedules_ParsesCorrectly()
{
var config = new PipelineConfig(
new SourceConfig("jde", "SELECT 1", null, null),
null, // Old SyncModes - deprecated
new PipelineSchedules
{
Mass = new ScheduleConfig { PrePurge = true, ReIndex = true },
Daily = new ScheduleConfig { Enabled = true },
Hourly = new ScheduleConfig { Enabled = false }
},
null,
new DestinationConfig("TestTable", ["Id"], null),
null,
null);
config.Schedules.ShouldNotBeNull();
config.Schedules!.Mass!.PrePurge.ShouldBeTrue();
config.Schedules!.Hourly!.Enabled.ShouldBeFalse();
}
private static PipelineConfig CreateMinimalPipelineConfig() private static PipelineConfig CreateMinimalPipelineConfig()
{ {
return new PipelineConfig( return new PipelineConfig(
@@ -76,7 +98,8 @@ public class PipelinesRootTests
["mass"] = new SyncModeConfig(null, true, true), ["mass"] = new SyncModeConfig(null, true, true),
["incremental"] = new SyncModeConfig("-1d") ["incremental"] = new SyncModeConfig("-1d")
}, },
null, null, // Schedules
null, // Transformers
new DestinationConfig("TestTable", ["Id"], null), new DestinationConfig("TestTable", ["Id"], null),
null, null,
null); null);
NEW/tests/JdeScoping.DataSync.Tests/Services/EtlPipelineFactoryTests.cs
+38 -19
View File
@@ -142,7 +142,8 @@ public class EtlPipelineFactoryTests
["mass"] = new SyncModeConfig("-365.00:00:00", PrePurge: true) ["mass"] = new SyncModeConfig("-365.00:00:00", PrePurge: true)
// No incremental mode defined // No incremental mode defined
}, },
null, null, // Schedules
null, // Transformers
new DestinationConfig("TestTable", ["Id"], null), new DestinationConfig("TestTable", ["Id"], null),
null, null,
null) null)
@@ -212,7 +213,8 @@ public class EtlPipelineFactoryTests
// Missing mass mode // Missing mass mode
["incremental"] = new SyncModeConfig("-1.00:00:00") ["incremental"] = new SyncModeConfig("-1.00:00:00")
}, },
null, null, // Schedules
null, // Transformers
new DestinationConfig("TestTable", ["Id"], null), new DestinationConfig("TestTable", ["Id"], null),
null, null,
null) null)
@@ -243,7 +245,8 @@ public class EtlPipelineFactoryTests
["mass"] = new SyncModeConfig("-365.00:00:00"), ["mass"] = new SyncModeConfig("-365.00:00:00"),
["incremental"] = new SyncModeConfig("-1.00:00:00") ["incremental"] = new SyncModeConfig("-1.00:00:00")
}, },
null, null, // Schedules
null, // Transformers
new DestinationConfig("TestTable", ["Id"], null), new DestinationConfig("TestTable", ["Id"], null),
null, null,
null) null)
@@ -274,7 +277,8 @@ public class EtlPipelineFactoryTests
["mass"] = new SyncModeConfig("-365.00:00:00", PrePurge: true), ["mass"] = new SyncModeConfig("-365.00:00:00", PrePurge: true),
["incremental"] = new SyncModeConfig("-1.00:00:00") ["incremental"] = new SyncModeConfig("-1.00:00:00")
}, },
null, null, // Schedules
null, // Transformers
new DestinationConfig("TestTable", ["Id"], null), new DestinationConfig("TestTable", ["Id"], null),
null, null,
null) null)
@@ -323,7 +327,8 @@ public class EtlPipelineFactoryTests
Destination: new DestinationOverride("bulkMerge", null, null)), Destination: new DestinationOverride("bulkMerge", null, null)),
["incremental"] = new SyncModeConfig("-1.00:00:00") ["incremental"] = new SyncModeConfig("-1.00:00:00")
}, },
null, null, // Schedules
null, // Transformers
new DestinationConfig("TestTable", ["Id"], null), new DestinationConfig("TestTable", ["Id"], null),
null, null,
null) null)
@@ -355,7 +360,8 @@ public class EtlPipelineFactoryTests
["mass"] = new SyncModeConfig("-365.00:00:00", PrePurge: true), ["mass"] = new SyncModeConfig("-365.00:00:00", PrePurge: true),
["incremental"] = new SyncModeConfig("-1.00:00:00") ["incremental"] = new SyncModeConfig("-1.00:00:00")
}, },
null, null, // Schedules
null, // Transformers
new DestinationConfig("TestTable", null, null), // No matchColumns! new DestinationConfig("TestTable", null, null), // No matchColumns!
null, null,
null) null)
@@ -394,7 +400,8 @@ public class EtlPipelineFactoryTests
["mass"] = new SyncModeConfig("-365.00:00:00", PrePurge: true), ["mass"] = new SyncModeConfig("-365.00:00:00", PrePurge: true),
["incremental"] = new SyncModeConfig("-1.00:00:00") ["incremental"] = new SyncModeConfig("-1.00:00:00")
}, },
null, null, // Schedules
null, // Transformers
new DestinationConfig("TestTable", ["Id"], null), new DestinationConfig("TestTable", ["Id"], null),
null, null,
null) null)
@@ -430,7 +437,8 @@ public class EtlPipelineFactoryTests
["mass"] = new SyncModeConfig("-365.00:00:00", PrePurge: true), ["mass"] = new SyncModeConfig("-365.00:00:00", PrePurge: true),
["incremental"] = new SyncModeConfig("-1.00:00:00") ["incremental"] = new SyncModeConfig("-1.00:00:00")
}, },
null, null, // Schedules
null, // Transformers
new DestinationConfig("TestTable", ["Id"], null), new DestinationConfig("TestTable", ["Id"], null),
null, null,
null) null)
@@ -466,7 +474,8 @@ public class EtlPipelineFactoryTests
["mass"] = new SyncModeConfig("-365.00:00:00", PrePurge: true), ["mass"] = new SyncModeConfig("-365.00:00:00", PrePurge: true),
["incremental"] = new SyncModeConfig("-1.00:00:00") ["incremental"] = new SyncModeConfig("-1.00:00:00")
}, },
null, null, // Schedules
null, // Transformers
new DestinationConfig("TestTable", ["Id"], null), new DestinationConfig("TestTable", ["Id"], null),
null, null,
null) null)
@@ -502,7 +511,8 @@ public class EtlPipelineFactoryTests
["mass"] = new SyncModeConfig("-365.00:00:00", PrePurge: true), ["mass"] = new SyncModeConfig("-365.00:00:00", PrePurge: true),
["incremental"] = new SyncModeConfig("-1.00:00:00") ["incremental"] = new SyncModeConfig("-1.00:00:00")
}, },
null, null, // Schedules
null, // Transformers
new DestinationConfig("TestTable", ["Id"], null), new DestinationConfig("TestTable", ["Id"], null),
null, null,
null) null)
@@ -537,7 +547,8 @@ public class EtlPipelineFactoryTests
["mass"] = new SyncModeConfig("-365.00:00:00", PrePurge: true), ["mass"] = new SyncModeConfig("-365.00:00:00", PrePurge: true),
["incremental"] = new SyncModeConfig("-1.00:00:00") ["incremental"] = new SyncModeConfig("-1.00:00:00")
}, },
null, null, // Schedules
null, // Transformers
new DestinationConfig("TestTable", ["Id"], null), new DestinationConfig("TestTable", ["Id"], null),
null, null,
null) null)
@@ -569,7 +580,8 @@ public class EtlPipelineFactoryTests
["mass"] = new SyncModeConfig("-365.00:00:00", PrePurge: true, ReIndex: true), ["mass"] = new SyncModeConfig("-365.00:00:00", PrePurge: true, ReIndex: true),
["incremental"] = new SyncModeConfig("-1.00:00:00") ["incremental"] = new SyncModeConfig("-1.00:00:00")
}, },
null, null, // Schedules
null, // Transformers
new DestinationConfig("TestTable", ["Id"], null), new DestinationConfig("TestTable", ["Id"], null),
null, null,
null) null)
@@ -601,7 +613,8 @@ public class EtlPipelineFactoryTests
["mass"] = new SyncModeConfig("-365.00:00:00", PrePurge: true), ["mass"] = new SyncModeConfig("-365.00:00:00", PrePurge: true),
["incremental"] = new SyncModeConfig("-1.00:00:00") ["incremental"] = new SyncModeConfig("-1.00:00:00")
}, },
null, null, // Schedules
null, // Transformers
new DestinationConfig("TestTable", ["Id"], null), new DestinationConfig("TestTable", ["Id"], null),
["EXEC sp_BeforeSync"], ["EXEC sp_BeforeSync"],
null) null)
@@ -633,7 +646,8 @@ public class EtlPipelineFactoryTests
["mass"] = new SyncModeConfig("-365.00:00:00", PrePurge: true), ["mass"] = new SyncModeConfig("-365.00:00:00", PrePurge: true),
["incremental"] = new SyncModeConfig("-1.00:00:00") ["incremental"] = new SyncModeConfig("-1.00:00:00")
}, },
null, null, // Schedules
null, // Transformers
new DestinationConfig("TestTable", ["Id"], null), new DestinationConfig("TestTable", ["Id"], null),
null, null,
["UPDATE TestTable SET ProcessedFlag = 1 WHERE ProcessedFlag IS NULL"]) ["UPDATE TestTable SET ProcessedFlag = 1 WHERE ProcessedFlag IS NULL"])
@@ -672,7 +686,8 @@ public class EtlPipelineFactoryTests
["mass"] = new SyncModeConfig("-365.00:00:00", PrePurge: true), ["mass"] = new SyncModeConfig("-365.00:00:00", PrePurge: true),
["incremental"] = new SyncModeConfig("-1.00:00:00") ["incremental"] = new SyncModeConfig("-1.00:00:00")
}, },
null, null, // Schedules
null, // Transformers
new DestinationConfig("TestTable", ["Id"], null), new DestinationConfig("TestTable", ["Id"], null),
null, null,
null) null)
@@ -708,7 +723,8 @@ public class EtlPipelineFactoryTests
["mass"] = new SyncModeConfig("-365.00:00:00", PrePurge: true), ["mass"] = new SyncModeConfig("-365.00:00:00", PrePurge: true),
["incremental"] = new SyncModeConfig("-1.00:00:00") ["incremental"] = new SyncModeConfig("-1.00:00:00")
}, },
null, null, // Schedules
null, // Transformers
new DestinationConfig("TestTable", ["Id"], null), new DestinationConfig("TestTable", ["Id"], null),
null, null,
null) null)
@@ -744,7 +760,8 @@ public class EtlPipelineFactoryTests
["mass"] = new SyncModeConfig("-365.00:00:00", PrePurge: true), ["mass"] = new SyncModeConfig("-365.00:00:00", PrePurge: true),
["incremental"] = new SyncModeConfig("not-a-valid-timespan") // Invalid! ["incremental"] = new SyncModeConfig("not-a-valid-timespan") // Invalid!
}, },
null, null, // Schedules
null, // Transformers
new DestinationConfig("TestTable", ["Id"], null), new DestinationConfig("TestTable", ["Id"], null),
null, null,
null) null)
@@ -775,7 +792,8 @@ public class EtlPipelineFactoryTests
["mass"] = new SyncModeConfig(null, PrePurge: true), // Null offset ["mass"] = new SyncModeConfig(null, PrePurge: true), // Null offset
["incremental"] = new SyncModeConfig("-1.00:00:00") ["incremental"] = new SyncModeConfig("-1.00:00:00")
}, },
null, null, // Schedules
null, // Transformers
new DestinationConfig("TestTable", ["Id"], null), new DestinationConfig("TestTable", ["Id"], null),
null, null,
null) null)
@@ -813,7 +831,8 @@ public class EtlPipelineFactoryTests
["mass"] = new SyncModeConfig("-365.00:00:00", PrePurge: true, ReIndex: true), ["mass"] = new SyncModeConfig("-365.00:00:00", PrePurge: true, ReIndex: true),
["incremental"] = new SyncModeConfig("-1.00:00:00") ["incremental"] = new SyncModeConfig("-1.00:00:00")
}, },
null, null, // Schedules
null, // Transformers
new DestinationConfig("TestTable", ["Id"], null), new DestinationConfig("TestTable", ["Id"], null),
null, null,
null) null)