dohertj2/jdescopingtool
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor(configmanager): migrate to per-file pipeline system

Browse Source 
Align ConfigManager with DataSync's per-file pipeline format (pipeline.*.json)
by reusing EtlPipelineConfig types directly, eliminating duplicate models and
simplifying the codebase. Removes ~3200 lines of obsolete code.
This commit is contained in:
Joseph Doherty
2026-01-23 02:30:48 -05:00
parent 1b7bb26def
commit ba54a87be5
49 changed files with 1429 additions and 4396 deletions
Download Patch File Download Diff File Expand all files Collapse all files
NEW/tests/JdeScoping.DataSync.Tests/Configuration/PipelinesRootTests.cs
-106
View File
@@ -1,106 +0,0 @@
using JdeScoping.DataSync.Configuration;
using Shouldly;
namespace JdeScoping.DataSync.Tests.Configuration;
public class PipelinesRootTests
{
[Fact]
public void EffectiveScheduleDefaults_WhenNull_ReturnsDefaults()
{
var root = new PipelinesRoot(null, null, new Dictionary<string, PipelineConfig>());
var defaults = root.EffectiveScheduleDefaults;
defaults.ShouldNotBeNull();
defaults.Mass.IntervalMinutes.ShouldBe(10080);
defaults.Daily.IntervalMinutes.ShouldBe(1440);
defaults.Hourly.IntervalMinutes.ShouldBe(60);
}
[Fact]
public void EffectiveScheduleDefaults_WhenProvided_ReturnsProvided()
{
var customDefaults = new ScheduleDefaults
{
Mass = new ScheduleConfig { IntervalMinutes = 20000 }
};
var root = new PipelinesRoot(null, customDefaults, new Dictionary<string, PipelineConfig>());
var defaults = root.EffectiveScheduleDefaults;
defaults.Mass.IntervalMinutes.ShouldBe(20000);
}
[Fact]
public void EffectiveSettings_WhenNull_ReturnsDefaults()
{
var root = new PipelinesRoot(null, null, new Dictionary<string, PipelineConfig>());
var settings = root.EffectiveSettings;
settings.ShouldNotBeNull();
settings.Timezone.ShouldBe("UTC");
}
[Fact]
public void EffectiveSettings_WhenProvided_ReturnsProvided()
{
var customSettings = new PipelineSettings("America/New_York");
var root = new PipelinesRoot(customSettings, null, new Dictionary<string, PipelineConfig>());
var settings = root.EffectiveSettings;
settings.Timezone.ShouldBe("America/New_York");
}
[Fact]
public void Pipelines_WhenProvided_StoresCorrectly()
{
var pipelines = new Dictionary<string, PipelineConfig>
{
["TestTable"] = CreateMinimalPipelineConfig()
};
var root = new PipelinesRoot(null, null, pipelines);
root.Pipelines.ShouldContainKey("TestTable");
root.Pipelines["TestTable"].Destination.Table.ShouldBe("TestTable");
}
[Fact]
public void PipelineConfig_WithSchedules_ParsesCorrectly()
{
var config = new PipelineConfig(
new SourceConfig("jde", "SELECT 1", null, null),
new PipelineSchedules
{
Mass = new ScheduleConfig { PrePurge = true, ReIndex = true },
Daily = new ScheduleConfig { Enabled = true },
Hourly = new ScheduleConfig { Enabled = false }
},
null, // Transformers
new DestinationConfig("TestTable", ["Id"], null),
null,
null);
config.Schedules.ShouldNotBeNull();
config.Schedules!.Mass!.PrePurge.ShouldBeTrue();
config.Schedules!.Hourly!.Enabled.ShouldBeFalse();
}
private static PipelineConfig CreateMinimalPipelineConfig()
{
return new PipelineConfig(
new SourceConfig("lotfinder", "SELECT 1", null, null),
new PipelineSchedules
{
Mass = new ScheduleConfig { PrePurge = true, ReIndex = true },
Daily = new ScheduleConfig(),
Hourly = new ScheduleConfig()
},
null, // Transformers
new DestinationConfig("TestTable", ["Id"], null),
null,
null);
}
}
NEW/tests/JdeScoping.DataSync.Tests/Configuration/ScheduleConfigTests.cs
-176
View File
@@ -1,176 +0,0 @@
using JdeScoping.DataSync.Configuration;
using Shouldly;
namespace JdeScoping.DataSync.Tests.Configuration;
public class ScheduleConfigTests
{
[Fact]
public void ScheduleConfig_DefaultValues_AreCorrect()
{
var config = new ScheduleConfig();
config.Enabled.ShouldBeTrue();
config.IntervalMinutes.ShouldBe(0);
config.PrePurge.ShouldBeFalse();
config.ReIndex.ShouldBeFalse();
config.UpdateWhen.ShouldBeNull();
}
[Fact]
public void ScheduleConfig_WithValues_StoresCorrectly()
{
var config = new ScheduleConfig
{
Enabled = false,
IntervalMinutes = 60,
PrePurge = true,
ReIndex = true,
UpdateWhen = "src.LastUpdateDt > tgt.LastUpdateDt"
};
config.Enabled.ShouldBeFalse();
config.IntervalMinutes.ShouldBe(60);
config.PrePurge.ShouldBeTrue();
config.ReIndex.ShouldBeTrue();
config.UpdateWhen.ShouldBe("src.LastUpdateDt > tgt.LastUpdateDt");
}
[Fact]
public void ScheduleDefaults_HasCorrectDefaultValues()
{
var defaults = new ScheduleDefaults();
defaults.Mass.ShouldNotBeNull();
defaults.Daily.ShouldNotBeNull();
defaults.Hourly.ShouldNotBeNull();
}
[Fact]
public void ScheduleDefaults_Mass_HasCorrectValues()
{
var defaults = new ScheduleDefaults();
defaults.Mass.Enabled.ShouldBeTrue();
defaults.Mass.IntervalMinutes.ShouldBe(10080); // Weekly
defaults.Mass.PrePurge.ShouldBeTrue();
defaults.Mass.ReIndex.ShouldBeTrue();
defaults.Mass.UpdateWhen.ShouldBeNull();
}
[Fact]
public void ScheduleDefaults_Daily_HasCorrectValues()
{
var defaults = new ScheduleDefaults();
defaults.Daily.Enabled.ShouldBeTrue();
defaults.Daily.IntervalMinutes.ShouldBe(1440); // Daily
defaults.Daily.PrePurge.ShouldBeFalse();
defaults.Daily.ReIndex.ShouldBeFalse();
defaults.Daily.UpdateWhen.ShouldBe("src.LastUpdateDt > tgt.LastUpdateDt");
}
[Fact]
public void ScheduleDefaults_Hourly_HasCorrectValues()
{
var defaults = new ScheduleDefaults();
defaults.Hourly.Enabled.ShouldBeTrue();
defaults.Hourly.IntervalMinutes.ShouldBe(60); // Hourly
defaults.Hourly.PrePurge.ShouldBeFalse();
defaults.Hourly.ReIndex.ShouldBeFalse();
defaults.Hourly.UpdateWhen.ShouldBe("src.LastUpdateDt > tgt.LastUpdateDt");
}
[Fact]
public void PipelineSchedules_AllPropertiesNullable()
{
var schedules = new PipelineSchedules();
schedules.Mass.ShouldBeNull();
schedules.Daily.ShouldBeNull();
schedules.Hourly.ShouldBeNull();
}
[Fact]
public void PipelineSchedules_WithValues_StoresCorrectly()
{
var schedules = new PipelineSchedules
{
Mass = new ScheduleConfig { PrePurge = true },
Daily = new ScheduleConfig { Enabled = true },
Hourly = new ScheduleConfig { Enabled = false }
};
schedules.Mass.ShouldNotBeNull();
schedules.Mass!.PrePurge.ShouldBeTrue();
schedules.Daily.ShouldNotBeNull();
schedules.Daily!.Enabled.ShouldBeTrue();
schedules.Hourly.ShouldNotBeNull();
schedules.Hourly!.Enabled.ShouldBeFalse();
}
[Fact]
public void MergeWith_WhenConfigHasNoOverrides_ReturnsDefaultValues()
{
var config = new ScheduleConfig();
var defaults = new ScheduleConfig
{
Enabled = true,
IntervalMinutes = 60,
PrePurge = false,
ReIndex = false,
UpdateWhen = "src.LastUpdateDt > tgt.LastUpdateDt"
};
var merged = config.MergeWith(defaults);
merged.IntervalMinutes.ShouldBe(60);
merged.UpdateWhen.ShouldBe("src.LastUpdateDt > tgt.LastUpdateDt");
}
[Fact]
public void MergeWith_WhenConfigHasOverrides_UsesOverrideValues()
{
var config = new ScheduleConfig
{
IntervalMinutes = 120,
PrePurge = true,
UpdateWhen = "custom condition"
};
var defaults = new ScheduleConfig
{
IntervalMinutes = 60,
PrePurge = false,
UpdateWhen = "default condition"
};
var merged = config.MergeWith(defaults);
merged.IntervalMinutes.ShouldBe(120);
merged.PrePurge.ShouldBeTrue();
merged.UpdateWhen.ShouldBe("custom condition");
}
[Fact]
public void MergeWith_PreservesEnabledFromConfig()
{
var config = new ScheduleConfig { Enabled = false };
var defaults = new ScheduleConfig { Enabled = true };
var merged = config.MergeWith(defaults);
merged.Enabled.ShouldBeFalse();
}
[Fact]
public void MergeWith_WhenIntervalZero_UsesDefaultInterval()
{
var config = new ScheduleConfig { IntervalMinutes = 0 };
var defaults = new ScheduleConfig { IntervalMinutes = 1440 };
var merged = config.MergeWith(defaults);
merged.IntervalMinutes.ShouldBe(1440);
}
}
NEW/tests/JdeScoping.DataSync.Tests/ScheduleCheckerTests.cs
+1 -2
View File
@@ -30,8 +30,7 @@ public class ScheduleCheckerTests
_pipelines = [];
_options = Microsoft.Extensions.Options.Options.Create(new DataSyncOptions
{
LookbackMultiplier = 3,
DataSources = []
LookbackMultiplier = 3
});
// Setup pipeline registry to return our pipeline list
NEW/tests/JdeScoping.DataSync.Tests/Services/EtlPipelineFactoryTests.cs
-795
View File
@@ -1,795 +0,0 @@
using JdeScoping.Core.Models.Enums;
using JdeScoping.DataAccess.Interfaces;
using JdeScoping.DataSync.Configuration;
using JdeScoping.DataSync.Contracts;
using JdeScoping.DataSync.Etl.Pipeline;
using JdeScoping.DataSync.Services;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
using NSubstitute;
using Shouldly;
namespace JdeScoping.DataSync.Tests.Services;
public class EtlPipelineFactoryTests
{
private readonly IDbConnectionFactory _connectionFactory;
private readonly ILogger<EtlPipeline> _logger;
public EtlPipelineFactoryTests()
{
_connectionFactory = Substitute.For<IDbConnectionFactory>();
_logger = NullLogger<EtlPipeline>.Instance;
}
#region ForTable Tests
[Fact]
public void ForTable_WithValidTable_ReturnsBuilder()
{
// Arrange
var config = CreateValidConfigWithSchedules();
var factory = CreateFactory(config);
// Act
var builder = factory.ForTable("TestTable");
// Assert
builder.ShouldNotBeNull();
builder.ShouldBeAssignableTo<IEtlPipelineBuilder>();
}
[Fact]
public void ForTable_WithUnknownTable_ThrowsInvalidOperationException()
{
// Arrange
var config = CreateValidConfigWithSchedules();
var factory = CreateFactory(config);
// Act & Assert
var ex = Should.Throw<InvalidOperationException>(() => factory.ForTable("NonExistentTable"));
ex.Message.ShouldContain("No pipeline configured for table: NonExistentTable");
ex.Message.ShouldContain("TestTable"); // Should list available tables
}
[Fact]
public void ForTable_WithNullTableName_ThrowsArgumentException()
{
// Arrange
var config = CreateValidConfigWithSchedules();
var factory = CreateFactory(config);
// Act & Assert
Should.Throw<ArgumentException>(() => factory.ForTable(null!));
}
[Fact]
public void ForTable_WithEmptyTableName_ThrowsArgumentException()
{
// Arrange
var config = CreateValidConfigWithSchedules();
var factory = CreateFactory(config);
// Act & Assert
Should.Throw<ArgumentException>(() => factory.ForTable(""));
}
#endregion
#region Builder WithUpdateType Tests
[Fact]
public void Builder_WithUpdateTypesMass_BuildsPipeline()
{
// Arrange
var config = CreateValidConfigWithSchedules();
var factory = CreateFactory(config);
// Act
var pipeline = factory.ForTable("TestTable")
.WithUpdateType(UpdateTypes.Mass)
.Build();
// Assert
pipeline.ShouldNotBeNull();
pipeline.PipelineName.ShouldBe("TestTable");
}
[Fact]
public void Builder_WithUpdateTypesDaily_BuildsPipeline()
{
// Arrange
var config = CreateValidConfigWithSchedules();
var factory = CreateFactory(config);
// Act
var pipeline = factory.ForTable("TestTable")
.WithUpdateType(UpdateTypes.Daily)
.Build();
// Assert
pipeline.ShouldNotBeNull();
pipeline.PipelineName.ShouldBe("TestTable");
}
[Fact]
public void Builder_WithUpdateTypesHourly_BuildsPipeline()
{
// Arrange
var config = CreateValidConfigWithSchedules();
var factory = CreateFactory(config);
// Act
var pipeline = factory.ForTable("TestTable")
.WithUpdateType(UpdateTypes.Hourly)
.Build();
// Assert
pipeline.ShouldNotBeNull();
pipeline.PipelineName.ShouldBe("TestTable");
}
[Fact]
public void Builder_WithUpdateTypesMass_UsesMassQuery()
{
// Arrange - config with massQuery should use it for Mass update type
var config = CreateValidConfigWithSchedules();
var factory = CreateFactory(config);
// Act
var pipeline = factory.ForTable("TestTable")
.WithUpdateType(UpdateTypes.Mass)
.Build();
// Assert
pipeline.ShouldNotBeNull();
}
[Fact]
public void Builder_WithUpdateTypesDaily_UsesRegularQuery()
{
// Arrange - Daily should use regular query with date filtering
var config = CreateValidConfigWithSchedules();
var factory = CreateFactory(config);
// Act
var pipeline = factory.ForTable("TestTable")
.WithUpdateType(UpdateTypes.Daily)
.Build();
// Assert
pipeline.ShouldNotBeNull();
}
[Fact]
public void Builder_WithUpdateTypesMass_AppliesPrePurgeFromScheduleConfig()
{
// Arrange - Mass schedule should have prePurge=true from defaults
var config = CreateValidConfigWithSchedules();
var factory = CreateFactory(config);
// Act - should not throw and should include truncate pre-script
var pipeline = factory.ForTable("TestTable")
.WithUpdateType(UpdateTypes.Mass)
.Build();
// Assert
pipeline.ShouldNotBeNull();
}
[Fact]
public void Builder_WithUpdateTypesMass_AppliesReIndexFromScheduleConfig()
{
// Arrange - Mass schedule should have reIndex=true from defaults
var config = CreateValidConfigWithSchedules();
var factory = CreateFactory(config);
// Act - should not throw and should include reindex post-script
var pipeline = factory.ForTable("TestTable")
.WithUpdateType(UpdateTypes.Mass)
.Build();
// Assert
pipeline.ShouldNotBeNull();
}
[Fact]
public void Builder_WithUpdateTypesHourly_UsesUpdateWhenFromDefaults()
{
// Arrange - Hourly should use updateWhen from defaults
var config = CreateValidConfigWithSchedules();
var factory = CreateFactory(config);
// Act
var pipeline = factory.ForTable("TestTable")
.WithUpdateType(UpdateTypes.Hourly)
.Build();
// Assert
pipeline.ShouldNotBeNull();
}
[Fact]
public void Builder_DefaultMode_IsHourly()
{
// Arrange
var config = CreateValidConfigWithSchedules();
var factory = CreateFactory(config);
// Act - don't call WithUpdateType()
var pipeline = factory.ForTable("TestTable")
.Build();
// Assert - should work because hourly mode is defined
pipeline.ShouldNotBeNull();
}
#endregion
#region Builder WithMinimumDate Tests
[Fact]
public void Builder_WithMinimumDate_OverridesConfigOffset()
{
// Arrange
var config = CreateValidConfigWithSchedules();
var factory = CreateFactory(config);
var customDate = new DateTime(2024, 1, 1, 0, 0, 0, DateTimeKind.Utc);
// Act - should not throw even though we're overriding
var pipeline = factory.ForTable("TestTable")
.WithUpdateType(UpdateTypes.Hourly)
.WithMinimumDate(customDate)
.Build();
// Assert
pipeline.ShouldNotBeNull();
}
[Fact]
public void Builder_WithNullMinimumDate_UsesConfigOffset()
{
// Arrange
var config = CreateValidConfigWithSchedules();
var factory = CreateFactory(config);
// Act - null minDt means use config offset
var pipeline = factory.ForTable("TestTable")
.WithUpdateType(UpdateTypes.Hourly)
.WithMinimumDate(null)
.Build();
// Assert
pipeline.ShouldNotBeNull();
}
#endregion
#region Config Validation Tests
[Fact]
public void Validate_ConfigMissingSchedules_ThrowsInvalidOperationException()
{
// Arrange
var config = new PipelinesRoot(
new PipelineSettings("UTC"),
null,
new Dictionary<string, PipelineConfig>
{
["TestTable"] = new PipelineConfig(
new SourceConfig("lotfinder", "SELECT * FROM Test", null),
null, // Schedules - null means invalid
null, // Transformers
new DestinationConfig("TestTable", ["Id"], null),
null,
null)
});
// Act & Assert
var ex = Should.Throw<InvalidOperationException>(() => CreateFactory(config));
ex.Message.ShouldContain("must define 'schedules'");
}
[Fact]
public void Validate_ConfigWithRuntimeParameter_ThrowsNotSupportedException()
{
// Arrange
var config = new PipelinesRoot(
new PipelineSettings("UTC"),
null,
new Dictionary<string, PipelineConfig>
{
["TestTable"] = new PipelineConfig(
new SourceConfig("lotfinder", "SELECT * FROM Test WHERE Id = @Id",
new Dictionary<string, ParameterConfig>
{
["id"] = new ParameterConfig("@Id", null, "runtime", null)
}),
new PipelineSchedules
{
Mass = new ScheduleConfig(),
Daily = new ScheduleConfig(),
Hourly = new ScheduleConfig()
},
null, // Transformers
new DestinationConfig("TestTable", ["Id"], null),
null,
null)
});
// Act & Assert
var ex = Should.Throw<NotSupportedException>(() => CreateFactory(config));
ex.Message.ShouldContain("runtime parameter source is not yet supported");
}
#endregion
#region Destination Type Tests
[Fact]
public void Builder_MassMode_WithPrePurge_UsesBulkImport()
{
// Arrange - Mass with prePurge defaults to bulkImport
var config = CreateValidConfigWithSchedules();
var factory = CreateFactory(config);
// Act - should use bulkImport for mass mode with prePurge
var pipeline = factory.ForTable("TestTable")
.WithUpdateType(UpdateTypes.Mass)
.Build();
// Assert
pipeline.ShouldNotBeNull();
}
[Fact]
public void Builder_HourlyMode_UsesBulkMerge()
{
// Arrange - Hourly without prePurge uses bulkMerge
var config = CreateValidConfigWithSchedules();
var factory = CreateFactory(config);
// Act - should use bulkMerge for hourly mode
var pipeline = factory.ForTable("TestTable")
.WithUpdateType(UpdateTypes.Hourly)
.Build();
// Assert
pipeline.ShouldNotBeNull();
}
[Fact]
public void Builder_BulkMergeWithoutMatchColumns_ThrowsInvalidOperationException()
{
// Arrange - bulkMerge needs matchColumns
var config = new PipelinesRoot(
new PipelineSettings("UTC"),
new ScheduleDefaults(),
new Dictionary<string, PipelineConfig>
{
["TestTable"] = new PipelineConfig(
new SourceConfig("lotfinder", "SELECT * FROM Test", null),
new PipelineSchedules
{
Mass = new ScheduleConfig { PrePurge = true },
Daily = new ScheduleConfig(),
Hourly = new ScheduleConfig()
},
null, // Transformers
new DestinationConfig("TestTable", null, null), // No matchColumns!
null,
null)
});
var factory = CreateFactory(config);
// Act & Assert
var ex = Should.Throw<InvalidOperationException>(() =>
factory.ForTable("TestTable")
.WithUpdateType(UpdateTypes.Hourly) // Uses bulkMerge
.Build());
ex.Message.ShouldContain("matchColumns required for bulkMerge");
}
#endregion
#region Parameter Tests
[Fact]
public void Builder_WithOffsetParameter_CreatesSource()
{
// Arrange
var config = new PipelinesRoot(
new PipelineSettings("UTC"),
new ScheduleDefaults(),
new Dictionary<string, PipelineConfig>
{
["TestTable"] = new PipelineConfig(
new SourceConfig("lotfinder", "SELECT * FROM Test WHERE UpdateDt >= @MinDt",
new Dictionary<string, ParameterConfig>
{
["minDt"] = new ParameterConfig("@MinDt", null, "offset", null)
}),
new PipelineSchedules
{
Mass = new ScheduleConfig { PrePurge = true },
Daily = new ScheduleConfig(),
Hourly = new ScheduleConfig()
},
null, // Transformers
new DestinationConfig("TestTable", ["Id"], null),
null,
null)
});
var factory = CreateFactory(config);
// Act
var pipeline = factory.ForTable("TestTable")
.WithUpdateType(UpdateTypes.Hourly)
.Build();
// Assert
pipeline.ShouldNotBeNull();
}
[Fact]
public void Builder_WithJdeJulianParameter_CreatesSource()
{
// Arrange
var config = new PipelinesRoot(
new PipelineSettings("UTC"),
new ScheduleDefaults(),
new Dictionary<string, PipelineConfig>
{
["TestTable"] = new PipelineConfig(
new SourceConfig("jde", "SELECT * FROM Test WHERE UPMJ >= :dateUpdated",
new Dictionary<string, ParameterConfig>
{
["minDt"] = new ParameterConfig(":dateUpdated", "jdeJulian", "offset", null)
}),
new PipelineSchedules
{
Mass = new ScheduleConfig { PrePurge = true },
Daily = new ScheduleConfig(),
Hourly = new ScheduleConfig()
},
null, // Transformers
new DestinationConfig("TestTable", ["Id"], null),
null,
null)
});
var factory = CreateFactory(config);
// Act
var pipeline = factory.ForTable("TestTable")
.WithUpdateType(UpdateTypes.Hourly)
.Build();
// Assert
pipeline.ShouldNotBeNull();
}
[Fact]
public void Builder_WithStaticParameter_UsesConfiguredValue()
{
// Arrange
var config = new PipelinesRoot(
new PipelineSettings("UTC"),
new ScheduleDefaults(),
new Dictionary<string, PipelineConfig>
{
["TestTable"] = new PipelineConfig(
new SourceConfig("lotfinder", "SELECT * FROM Test WHERE Status = @Status",
new Dictionary<string, ParameterConfig>
{
["status"] = new ParameterConfig("@Status", null, "static", "Active")
}),
new PipelineSchedules
{
Mass = new ScheduleConfig { PrePurge = true },
Daily = new ScheduleConfig(),
Hourly = new ScheduleConfig()
},
null, // Transformers
new DestinationConfig("TestTable", ["Id"], null),
null,
null)
});
var factory = CreateFactory(config);
// Act
var pipeline = factory.ForTable("TestTable")
.WithUpdateType(UpdateTypes.Hourly)
.Build();
// Assert
pipeline.ShouldNotBeNull();
}
[Fact]
public void Builder_WithStaticParameterNoValue_ThrowsInvalidOperationException()
{
// Arrange
var config = new PipelinesRoot(
new PipelineSettings("UTC"),
new ScheduleDefaults(),
new Dictionary<string, PipelineConfig>
{
["TestTable"] = new PipelineConfig(
new SourceConfig("lotfinder", "SELECT * FROM Test WHERE Status = @Status",
new Dictionary<string, ParameterConfig>
{
["status"] = new ParameterConfig("@Status", null, "static", null) // No value!
}),
new PipelineSchedules
{
Mass = new ScheduleConfig { PrePurge = true },
Daily = new ScheduleConfig(),
Hourly = new ScheduleConfig()
},
null, // Transformers
new DestinationConfig("TestTable", ["Id"], null),
null,
null)
});
var factory = CreateFactory(config);
// Act & Assert - must provide minDt for parameters to be processed
var ex = Should.Throw<InvalidOperationException>(() =>
factory.ForTable("TestTable")
.WithUpdateType(UpdateTypes.Hourly)
.WithMinimumDate(DateTime.UtcNow.AddDays(-1))
.Build());
ex.Message.ShouldContain("Static parameter '@Status' requires a value");
}
#endregion
#region Script Tests
[Fact]
public void Builder_WithPrePurge_AddsTruncateScript()
{
// Arrange
var config = new PipelinesRoot(
new PipelineSettings("UTC"),
new ScheduleDefaults(),
new Dictionary<string, PipelineConfig>
{
["TestTable"] = new PipelineConfig(
new SourceConfig("lotfinder", "SELECT * FROM Test", null),
new PipelineSchedules
{
Mass = new ScheduleConfig { PrePurge = true },
Daily = new ScheduleConfig(),
Hourly = new ScheduleConfig()
},
null, // Transformers
new DestinationConfig("TestTable", ["Id"], null),
null,
null)
});
var factory = CreateFactory(config);
// Act
var pipeline = factory.ForTable("TestTable")
.WithUpdateType(UpdateTypes.Mass)
.Build();
// Assert
pipeline.ShouldNotBeNull();
}
[Fact]
public void Builder_WithReIndex_AddsRebuildScript()
{
// Arrange
var config = new PipelinesRoot(
new PipelineSettings("UTC"),
new ScheduleDefaults(),
new Dictionary<string, PipelineConfig>
{
["TestTable"] = new PipelineConfig(
new SourceConfig("lotfinder", "SELECT * FROM Test", null),
new PipelineSchedules
{
Mass = new ScheduleConfig { PrePurge = true, ReIndex = true },
Daily = new ScheduleConfig(),
Hourly = new ScheduleConfig()
},
null, // Transformers
new DestinationConfig("TestTable", ["Id"], null),
null,
null)
});
var factory = CreateFactory(config);
// Act
var pipeline = factory.ForTable("TestTable")
.WithUpdateType(UpdateTypes.Mass)
.Build();
// Assert
pipeline.ShouldNotBeNull();
}
[Fact]
public void Builder_WithPreScripts_AddsConfiguredScripts()
{
// Arrange
var config = new PipelinesRoot(
new PipelineSettings("UTC"),
new ScheduleDefaults(),
new Dictionary<string, PipelineConfig>
{
["TestTable"] = new PipelineConfig(
new SourceConfig("lotfinder", "SELECT * FROM Test", null),
new PipelineSchedules
{
Mass = new ScheduleConfig { PrePurge = true },
Daily = new ScheduleConfig(),
Hourly = new ScheduleConfig()
},
null, // Transformers
new DestinationConfig("TestTable", ["Id"], null),
["EXEC sp_BeforeSync"],
null)
});
var factory = CreateFactory(config);
// Act
var pipeline = factory.ForTable("TestTable")
.WithUpdateType(UpdateTypes.Mass)
.Build();
// Assert
pipeline.ShouldNotBeNull();
}
[Fact]
public void Builder_WithPostScripts_AddsConfiguredScripts()
{
// Arrange
var config = new PipelinesRoot(
new PipelineSettings("UTC"),
new ScheduleDefaults(),
new Dictionary<string, PipelineConfig>
{
["TestTable"] = new PipelineConfig(
new SourceConfig("lotfinder", "SELECT * FROM Test", null),
new PipelineSchedules
{
Mass = new ScheduleConfig { PrePurge = true },
Daily = new ScheduleConfig(),
Hourly = new ScheduleConfig()
},
null, // Transformers
new DestinationConfig("TestTable", ["Id"], null),
null,
["UPDATE TestTable SET ProcessedFlag = 1 WHERE ProcessedFlag IS NULL"])
});
var factory = CreateFactory(config);
// Act
var pipeline = factory.ForTable("TestTable")
.WithUpdateType(UpdateTypes.Hourly)
.Build();
// Assert
pipeline.ShouldNotBeNull();
}
#endregion
#region Connection Type Tests
[Theory]
[InlineData("jde")]
[InlineData("cms")]
[InlineData("lotfinder")]
public void Builder_WithValidConnectionType_BuildsPipeline(string connectionType)
{
// Arrange
var config = new PipelinesRoot(
new PipelineSettings("UTC"),
new ScheduleDefaults(),
new Dictionary<string, PipelineConfig>
{
["TestTable"] = new PipelineConfig(
new SourceConfig(connectionType, "SELECT * FROM Test", null),
new PipelineSchedules
{
Mass = new ScheduleConfig { PrePurge = true },
Daily = new ScheduleConfig(),
Hourly = new ScheduleConfig()
},
null, // Transformers
new DestinationConfig("TestTable", ["Id"], null),
null,
null)
});
var factory = CreateFactory(config);
// Act
var pipeline = factory.ForTable("TestTable")
.WithUpdateType(UpdateTypes.Mass)
.Build();
// Assert
pipeline.ShouldNotBeNull();
}
#endregion
#region Settings Tests
[Fact]
public void Factory_WithNullSettings_UsesDefaults()
{
// Arrange - null settings should use defaults
var config = new PipelinesRoot(
null, // Null settings
new ScheduleDefaults(),
new Dictionary<string, PipelineConfig>
{
["TestTable"] = new PipelineConfig(
new SourceConfig("lotfinder", "SELECT * FROM Test", null),
new PipelineSchedules
{
Mass = new ScheduleConfig { PrePurge = true },
Daily = new ScheduleConfig(),
Hourly = new ScheduleConfig()
},
null, // Transformers
new DestinationConfig("TestTable", ["Id"], null),
null,
null)
});
var factory = CreateFactory(config);
// Act
var pipeline = factory.ForTable("TestTable")
.WithUpdateType(UpdateTypes.Hourly)
.Build();
// Assert
pipeline.ShouldNotBeNull();
}
#endregion
#region Helper Methods
private PipelinesRoot CreateValidConfigWithSchedules()
{
return new PipelinesRoot(
new PipelineSettings("UTC"),
new ScheduleDefaults(),
new Dictionary<string, PipelineConfig>
{
["TestTable"] = new PipelineConfig(
new SourceConfig("lotfinder", "SELECT * FROM Test WHERE UpdateDt >= @MinDt",
new Dictionary<string, ParameterConfig>
{
["minDt"] = new ParameterConfig("@MinDt", null, "offset", null)
},
"SELECT * FROM Test"), // MassQuery
new PipelineSchedules
{
Mass = new ScheduleConfig { PrePurge = true, ReIndex = true },
Daily = new ScheduleConfig(),
Hourly = new ScheduleConfig()
},
null, // Transformers
new DestinationConfig("TestTable", ["Id"], null),
null,
null)
});
}
private EtlPipelineFactory CreateFactory(PipelinesRoot config)
{
return new EtlPipelineFactory(_connectionFactory, config, _logger);
}
#endregion
}
NEW/tests/JdeScoping.DataSync.Tests/Services/TableSyncOperationTests.cs
+99 -111
View File
@@ -1,6 +1,7 @@
using System.Data;
using System.Diagnostics.Metrics;
using JdeScoping.Core.Models.Enums;
using JdeScoping.DataSync.Configuration;
using JdeScoping.DataSync.Contracts;
using JdeScoping.DataSync.Etl.Contracts;
using JdeScoping.DataSync.Etl.Pipeline;
@@ -19,7 +20,7 @@ namespace JdeScoping.DataSync.Tests.Services;
/// <summary>
/// Unit tests for TableSyncOperation.
/// Tests that the operation correctly uses the ETL pipeline with UpdateTypes.
/// Tests that the operation correctly uses the ETL pipeline builder.
/// </summary>
public class TableSyncOperationTests
{
@@ -48,33 +49,26 @@ public class TableSyncOperationTests
_metrics = new DataSyncMetrics(meterFactory);
}
#region WithUpdateType Tests
#region Pipeline Builder Tests
[Fact]
public async Task ExecuteAsync_WithUpdateTypesDaily_CallsWithUpdateTypeWithDaily()
public async Task ExecuteAsync_WithDailyUpdateType_CallsBuildWithDailyUpdateType()
{
// Arrange
var task = CreateTask("TestTable", UpdateTypes.Daily);
UpdateTypes? receivedUpdateType = null;
// Pre-create the test pipeline to avoid NSubstitute issues
var testPipeline = CreateTestPipeline();
var mockBuilder = Substitute.For<IEtlPipelineBuilder>();
mockBuilder.WithUpdateType(Arg.Any<UpdateTypes>())
.Returns(callInfo =>
{
receivedUpdateType = callInfo.Arg<UpdateTypes>();
return mockBuilder;
});
mockBuilder.WithMinimumDate(Arg.Any<DateTime?>()).Returns(mockBuilder);
mockBuilder.Build().Returns(testPipeline);
var mockFactory = Substitute.For<IEtlPipelineFactory>();
mockFactory.ForTable(Arg.Any<string>()).Returns(mockBuilder);
mockBuilder.Build(
Arg.Any<EtlPipelineConfig>(),
Arg.Do<UpdateTypes>(ut => receivedUpdateType = ut),
Arg.Any<DateTime?>())
.Returns(testPipeline);
var sut = new TableSyncOperation(
mockFactory,
mockBuilder,
_updateRepository,
_options,
NullLogger<TableSyncOperation>.Instance,
@@ -83,35 +77,28 @@ public class TableSyncOperationTests
// Act
await sut.ExecuteAsync(task);
// Assert - Verify WithUpdateType was called with Daily (not mapped to Incremental)
// Assert
receivedUpdateType.ShouldBe(UpdateTypes.Daily);
}
[Fact]
public async Task ExecuteAsync_WithUpdateTypesHourly_CallsWithUpdateTypeWithHourly()
public async Task ExecuteAsync_WithHourlyUpdateType_CallsBuildWithHourlyUpdateType()
{
// Arrange
var task = CreateTask("TestTable", UpdateTypes.Hourly);
UpdateTypes? receivedUpdateType = null;
// Pre-create the test pipeline to avoid NSubstitute issues
var testPipeline = CreateTestPipeline();
var mockBuilder = Substitute.For<IEtlPipelineBuilder>();
mockBuilder.WithUpdateType(Arg.Any<UpdateTypes>())
.Returns(callInfo =>
{
receivedUpdateType = callInfo.Arg<UpdateTypes>();
return mockBuilder;
});
mockBuilder.WithMinimumDate(Arg.Any<DateTime?>()).Returns(mockBuilder);
mockBuilder.Build().Returns(testPipeline);
var mockFactory = Substitute.For<IEtlPipelineFactory>();
mockFactory.ForTable(Arg.Any<string>()).Returns(mockBuilder);
mockBuilder.Build(
Arg.Any<EtlPipelineConfig>(),
Arg.Do<UpdateTypes>(ut => receivedUpdateType = ut),
Arg.Any<DateTime?>())
.Returns(testPipeline);
var sut = new TableSyncOperation(
mockFactory,
mockBuilder,
_updateRepository,
_options,
NullLogger<TableSyncOperation>.Instance,
@@ -125,30 +112,23 @@ public class TableSyncOperationTests
}
[Fact]
public async Task ExecuteAsync_WithUpdateTypesMass_CallsWithUpdateTypeWithMass()
public async Task ExecuteAsync_WithMassUpdateType_CallsBuildWithMassUpdateType()
{
// Arrange
var task = CreateTask("TestTable", UpdateTypes.Mass);
UpdateTypes? receivedUpdateType = null;
// Pre-create the test pipeline to avoid NSubstitute issues
var testPipeline = CreateTestPipeline();
var mockBuilder = Substitute.For<IEtlPipelineBuilder>();
mockBuilder.WithUpdateType(Arg.Any<UpdateTypes>())
.Returns(callInfo =>
{
receivedUpdateType = callInfo.Arg<UpdateTypes>();
return mockBuilder;
});
mockBuilder.WithMinimumDate(Arg.Any<DateTime?>()).Returns(mockBuilder);
mockBuilder.Build().Returns(testPipeline);
var mockFactory = Substitute.For<IEtlPipelineFactory>();
mockFactory.ForTable(Arg.Any<string>()).Returns(mockBuilder);
mockBuilder.Build(
Arg.Any<EtlPipelineConfig>(),
Arg.Do<UpdateTypes>(ut => receivedUpdateType = ut),
Arg.Any<DateTime?>())
.Returns(testPipeline);
var sut = new TableSyncOperation(
mockFactory,
mockBuilder,
_updateRepository,
_options,
NullLogger<TableSyncOperation>.Instance,
@@ -161,35 +141,24 @@ public class TableSyncOperationTests
receivedUpdateType.ShouldBe(UpdateTypes.Mass);
}
#endregion
#region Pipeline Execution Tests
[Fact]
public async Task ExecuteAsync_CallsForTableWithCorrectTableName()
public async Task ExecuteAsync_CallsBuildWithCorrectPipelineConfig()
{
// Arrange
var task = CreateTask("WorkOrder", UpdateTypes.Daily);
string? receivedTableName = null;
EtlPipelineConfig? receivedConfig = null;
// Pre-create the test pipeline to avoid NSubstitute issues
var testPipeline = CreateTestPipeline();
var mockBuilder = Substitute.For<IEtlPipelineBuilder>();
mockBuilder.WithUpdateType(Arg.Any<UpdateTypes>()).Returns(mockBuilder);
mockBuilder.WithMinimumDate(Arg.Any<DateTime?>()).Returns(mockBuilder);
mockBuilder.Build().Returns(testPipeline);
var mockFactory = Substitute.For<IEtlPipelineFactory>();
mockFactory.ForTable(Arg.Any<string>())
.Returns(callInfo =>
{
receivedTableName = callInfo.Arg<string>();
return mockBuilder;
});
mockBuilder.Build(
Arg.Do<EtlPipelineConfig>(c => receivedConfig = c),
Arg.Any<UpdateTypes>(),
Arg.Any<DateTime?>())
.Returns(testPipeline);
var sut = new TableSyncOperation(
mockFactory,
mockBuilder,
_updateRepository,
_options,
NullLogger<TableSyncOperation>.Instance,
@@ -199,35 +168,29 @@ public class TableSyncOperationTests
await sut.ExecuteAsync(task);
// Assert
receivedTableName.ShouldBe("WorkOrder");
receivedConfig.ShouldNotBeNull();
receivedConfig.Name.ShouldBe("WorkOrder");
}
[Fact]
public async Task ExecuteAsync_CallsWithMinimumDateWithTaskMinimumDt()
public async Task ExecuteAsync_CallsBuildWithCorrectMinimumDate()
{
// Arrange
var minDt = new DateTime(2024, 1, 15, 10, 30, 0, DateTimeKind.Utc);
var task = CreateTask("TestTable", UpdateTypes.Daily, minDt);
DateTime? receivedMinDt = null;
// Pre-create the test pipeline to avoid NSubstitute issues
var testPipeline = CreateTestPipeline();
var mockBuilder = Substitute.For<IEtlPipelineBuilder>();
mockBuilder.WithUpdateType(Arg.Any<UpdateTypes>()).Returns(mockBuilder);
mockBuilder.WithMinimumDate(Arg.Any<DateTime?>())
.Returns(callInfo =>
{
receivedMinDt = callInfo.Arg<DateTime?>();
return mockBuilder;
});
mockBuilder.Build().Returns(testPipeline);
var mockFactory = Substitute.For<IEtlPipelineFactory>();
mockFactory.ForTable(Arg.Any<string>()).Returns(mockBuilder);
mockBuilder.Build(
Arg.Any<EtlPipelineConfig>(),
Arg.Any<UpdateTypes>(),
Arg.Do<DateTime?>(dt => receivedMinDt = dt))
.Returns(testPipeline);
var sut = new TableSyncOperation(
mockFactory,
mockBuilder,
_updateRepository,
_options,
NullLogger<TableSyncOperation>.Instance,
@@ -240,25 +203,54 @@ public class TableSyncOperationTests
receivedMinDt.ShouldBe(minDt);
}
[Fact]
public async Task ExecuteAsync_TaskWithNoPipeline_ThrowsInvalidOperationException()
{
// Arrange
var task = new DataUpdateTask
{
TableName = "TestTable",
SourceSystem = "JDE",
SourceData = "TESTTABLE",
UpdateType = UpdateTypes.Daily,
Pipeline = null // No pipeline!
};
var mockBuilder = Substitute.For<IEtlPipelineBuilder>();
var sut = new TableSyncOperation(
mockBuilder,
_updateRepository,
_options,
NullLogger<TableSyncOperation>.Instance,
_metrics);
// Act & Assert
var ex = await Should.ThrowAsync<InvalidOperationException>(() => sut.ExecuteAsync(task));
ex.Message.ShouldContain("No pipeline configuration");
ex.Message.ShouldContain("TestTable");
}
#endregion
#region Pipeline Execution Tests
[Fact]
public async Task ExecuteAsync_SuccessfulPipeline_CompletesUpdateAsSuccess()
{
// Arrange
var task = CreateTask("TestTable", UpdateTypes.Daily);
// Pre-create the test pipeline to avoid NSubstitute issues
var testPipeline = CreateTestPipeline(totalRows: 100);
var mockBuilder = Substitute.For<IEtlPipelineBuilder>();
mockBuilder.WithUpdateType(Arg.Any<UpdateTypes>()).Returns(mockBuilder);
mockBuilder.WithMinimumDate(Arg.Any<DateTime?>()).Returns(mockBuilder);
mockBuilder.Build().Returns(testPipeline);
var mockFactory = Substitute.For<IEtlPipelineFactory>();
mockFactory.ForTable(Arg.Any<string>()).Returns(mockBuilder);
mockBuilder.Build(
Arg.Any<EtlPipelineConfig>(),
Arg.Any<UpdateTypes>(),
Arg.Any<DateTime?>())
.Returns(testPipeline);
var sut = new TableSyncOperation(
mockFactory,
mockBuilder,
_updateRepository,
_options,
NullLogger<TableSyncOperation>.Instance,
@@ -295,15 +287,14 @@ public class TableSyncOperationTests
var testPipeline = CreateTestPipeline();
var mockBuilder = Substitute.For<IEtlPipelineBuilder>();
mockBuilder.WithUpdateType(Arg.Any<UpdateTypes>()).Returns(mockBuilder);
mockBuilder.WithMinimumDate(Arg.Any<DateTime?>()).Returns(mockBuilder);
mockBuilder.Build().Returns(testPipeline);
var mockFactory = Substitute.For<IEtlPipelineFactory>();
mockFactory.ForTable(Arg.Any<string>()).Returns(mockBuilder);
mockBuilder.Build(
Arg.Any<EtlPipelineConfig>(),
Arg.Any<UpdateTypes>(),
Arg.Any<DateTime?>())
.Returns(testPipeline);
var sut = new TableSyncOperation(
mockFactory,
mockBuilder,
_updateRepository,
_options,
NullLogger<TableSyncOperation>.Instance,
@@ -324,20 +315,17 @@ public class TableSyncOperationTests
{
// Arrange
var task = CreateTask("TestTable", UpdateTypes.Daily);
// Pre-create the test pipeline to avoid NSubstitute issues
var testPipeline = CreateTestPipeline(success: false);
var mockBuilder = Substitute.For<IEtlPipelineBuilder>();
mockBuilder.WithUpdateType(Arg.Any<UpdateTypes>()).Returns(mockBuilder);
mockBuilder.WithMinimumDate(Arg.Any<DateTime?>()).Returns(mockBuilder);
mockBuilder.Build().Returns(testPipeline);
var mockFactory = Substitute.For<IEtlPipelineFactory>();
mockFactory.ForTable(Arg.Any<string>()).Returns(mockBuilder);
mockBuilder.Build(
Arg.Any<EtlPipelineConfig>(),
Arg.Any<UpdateTypes>(),
Arg.Any<DateTime?>())
.Returns(testPipeline);
var sut = new TableSyncOperation(
mockFactory,
mockBuilder,
_updateRepository,
_options,
NullLogger<TableSyncOperation>.Instance,
@@ -366,15 +354,15 @@ public class TableSyncOperationTests
SourceData = tableName.ToUpper(),
UpdateType = updateType,
MinimumDt = minDt,
Config = new DataSourceConfig
Pipeline = new EtlPipelineConfig
{
TableName = tableName,
SourceSystem = "JDE",
SourceData = tableName.ToUpper(),
Name = tableName,
IsEnabled = true,
MassConfig = new ScheduleConfig { Enabled = true, IntervalMinutes = 10080 },
DailyConfig = new ScheduleConfig { Enabled = true, IntervalMinutes = 1440 },
HourlyConfig = new ScheduleConfig { Enabled = true, IntervalMinutes = 60 }
MassSyncIntervalMinutes = 10080,
DailySyncIntervalMinutes = 1440,
HourlySyncIntervalMinutes = 60,
Source = new SourceElement { Connection = "JDE", Query = "SELECT 1" },
Destination = new DestinationElement { Table = tableName, MatchColumns = ["Id"] }
}
};
}
NEW/tests/JdeScoping.DataSync.Tests/SyncOrchestratorTests.cs
+9 -8
View File
@@ -1,8 +1,9 @@
using System.Diagnostics.Metrics;
using JdeScoping.Core.Models.Enums;
using JdeScoping.DataSync.Options;
using JdeScoping.DataSync.Configuration;
using JdeScoping.DataSync.Contracts;
using JdeScoping.DataSync.Models;
using JdeScoping.DataSync.Options;
using JdeScoping.DataSync.Services;
using JdeScoping.DataSync.Telemetry;
using Microsoft.Extensions.DependencyInjection;
@@ -540,15 +541,15 @@ public class SyncOrchestratorTests
SourceData = tableName.ToUpper(),
UpdateType = updateType,
MinimumDt = null,
Config = new DataSourceConfig
Pipeline = new EtlPipelineConfig
{
TableName = tableName,
SourceSystem = "JDE",
SourceData = tableName.ToUpper(),
Name = tableName,
IsEnabled = true,
MassConfig = new ScheduleConfig { Enabled = true, IntervalMinutes = 10080 },
DailyConfig = new ScheduleConfig { Enabled = true, IntervalMinutes = 1440 },
HourlyConfig = new ScheduleConfig { Enabled = true, IntervalMinutes = 60 }
MassSyncIntervalMinutes = 10080,
DailySyncIntervalMinutes = 1440,
HourlySyncIntervalMinutes = 60,
Source = new SourceElement { Connection = "JDE", Query = "SELECT 1" },
Destination = new DestinationElement { Table = tableName, MatchColumns = ["Id"] }
}
};
}
NEW/tests/JdeScoping.DataSync.Tests/WorkProcessorTests.cs
+11 -7
View File
@@ -2,6 +2,10 @@ using System.Diagnostics.Metrics;
using JdeScoping.Core.Interfaces;
using JdeScoping.Core.Models.Enums;
using JdeScoping.Core.Models.Search;
using JdeScoping.DataSync.Configuration;
using EtlPipelineConfig = JdeScoping.DataSync.Configuration.EtlPipelineConfig;
using SourceElement = JdeScoping.DataSync.Configuration.SourceElement;
using DestinationElement = JdeScoping.DataSync.Configuration.DestinationElement;
using JdeScoping.DataSync.Contracts;
using JdeScoping.DataSync.Models;
using JdeScoping.DataSync.Options;
@@ -655,15 +659,15 @@ public class WorkProcessorTests
SourceData = tableName.ToUpper(),
UpdateType = updateType,
MinimumDt = null,
Config = new DataSourceConfig
Pipeline = new EtlPipelineConfig
{
TableName = tableName,
SourceSystem = "JDE",
SourceData = tableName.ToUpper(),
Name = tableName,
IsEnabled = true,
MassConfig = new ScheduleConfig { Enabled = true, IntervalMinutes = 10080 },
DailyConfig = new ScheduleConfig { Enabled = true, IntervalMinutes = 1440 },
HourlyConfig = new ScheduleConfig { Enabled = true, IntervalMinutes = 60 }
MassSyncIntervalMinutes = 10080,
DailySyncIntervalMinutes = 1440,
HourlySyncIntervalMinutes = 60,
Source = new SourceElement { Connection = "JDE", Query = "SELECT 1" },
Destination = new DestinationElement { Table = tableName, MatchColumns = ["Id"] }
}
};
}