refactor(configmanager): migrate to per-file pipeline system
Align ConfigManager with DataSync's per-file pipeline format (pipeline.*.json) by reusing EtlPipelineConfig types directly, eliminating duplicate models and simplifying the codebase. Removes ~3200 lines of obsolete code.
This commit is contained in:
@@ -1,5 +1,6 @@
|
||||
using JdeScoping.ConfigManager.Models;
|
||||
using JdeScoping.ConfigManager.Services;
|
||||
using JdeScoping.DataSync.Configuration;
|
||||
|
||||
namespace JdeScoping.ConfigManager.Tests.Services;
|
||||
|
||||
@@ -47,19 +48,16 @@ public class ValidationServiceTests
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ValidatePipelines_WithDuplicateNames_ReturnsError()
|
||||
public void ValidatePipelines_WithEmptyName_ReturnsError()
|
||||
{
|
||||
// Arrange - duplicate keys not possible in dictionary, but empty names are invalid
|
||||
var config = new PipelinesConfigModel
|
||||
// Arrange
|
||||
var pipelines = new Dictionary<string, EtlPipelineConfig>
|
||||
{
|
||||
Pipelines = new Dictionary<string, PipelineModel>
|
||||
{
|
||||
[""] = new PipelineModel()
|
||||
}
|
||||
[""] = new EtlPipelineConfig()
|
||||
};
|
||||
|
||||
// Act
|
||||
var result = _sut.ValidatePipelines(config);
|
||||
var result = _sut.ValidatePipelines(pipelines);
|
||||
|
||||
// Assert
|
||||
result.IsValid.ShouldBeFalse();
|
||||
@@ -69,22 +67,140 @@ public class ValidationServiceTests
|
||||
public void ValidatePipelines_WithInvalidConnection_ReturnsError()
|
||||
{
|
||||
// Arrange
|
||||
var config = new PipelinesConfigModel
|
||||
var pipelines = new Dictionary<string, EtlPipelineConfig>
|
||||
{
|
||||
Pipelines = new Dictionary<string, PipelineModel>
|
||||
["Test"] = new EtlPipelineConfig
|
||||
{
|
||||
["Test"] = new PipelineModel
|
||||
{
|
||||
Source = new PipelineSource { Connection = "invalid" }
|
||||
}
|
||||
Source = new SourceElement { Connection = "invalid" }
|
||||
}
|
||||
};
|
||||
|
||||
// Act
|
||||
var result = _sut.ValidatePipelines(config);
|
||||
var result = _sut.ValidatePipelines(pipelines);
|
||||
|
||||
// Assert
|
||||
result.IsValid.ShouldBeFalse();
|
||||
result.Errors.ShouldContain(e => e.Contains("Connection"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ValidatePipeline_WithValidConfig_ReturnsNoErrors()
|
||||
{
|
||||
// Arrange
|
||||
var pipeline = new EtlPipelineConfig
|
||||
{
|
||||
Name = "Test",
|
||||
IsEnabled = true,
|
||||
Source = new SourceElement
|
||||
{
|
||||
Connection = "jde",
|
||||
Query = "SELECT * FROM Test"
|
||||
},
|
||||
Destination = new DestinationElement
|
||||
{
|
||||
Table = "TestTable",
|
||||
MatchColumns = ["Id"]
|
||||
},
|
||||
HourlySyncIntervalMinutes = 60
|
||||
};
|
||||
|
||||
// Act
|
||||
var result = _sut.ValidatePipeline(pipeline, "Test");
|
||||
|
||||
// Assert
|
||||
result.IsValid.ShouldBeTrue();
|
||||
result.Errors.ShouldBeEmpty();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ValidatePipeline_WithMissingQuery_ReturnsError()
|
||||
{
|
||||
// Arrange
|
||||
var pipeline = new EtlPipelineConfig
|
||||
{
|
||||
Source = new SourceElement { Connection = "jde", Query = "" },
|
||||
Destination = new DestinationElement { Table = "TestTable" },
|
||||
IsManualOnly = true
|
||||
};
|
||||
|
||||
// Act
|
||||
var result = _sut.ValidatePipeline(pipeline, "Test");
|
||||
|
||||
// Assert
|
||||
result.IsValid.ShouldBeFalse();
|
||||
result.Errors.ShouldContain(e => e.Contains("Query"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ValidatePipeline_WithNoScheduleAndNotManual_ReturnsWarning()
|
||||
{
|
||||
// Arrange
|
||||
var pipeline = new EtlPipelineConfig
|
||||
{
|
||||
Source = new SourceElement { Connection = "jde", Query = "SELECT 1" },
|
||||
Destination = new DestinationElement { Table = "TestTable", MatchColumns = ["Id"] },
|
||||
IsManualOnly = false // No schedules and not manual-only
|
||||
};
|
||||
|
||||
// Act
|
||||
var result = _sut.ValidatePipeline(pipeline, "Test");
|
||||
|
||||
// Assert
|
||||
result.Warnings.ShouldContain(w => w.Contains("No sync schedule"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ValidatePipeline_ManualOnly_DoesNotRequireSchedule()
|
||||
{
|
||||
// Arrange
|
||||
var pipeline = new EtlPipelineConfig
|
||||
{
|
||||
Source = new SourceElement { Connection = "jde", Query = "SELECT 1" },
|
||||
Destination = new DestinationElement { Table = "TestTable", MatchColumns = ["Id"] },
|
||||
IsManualOnly = true
|
||||
};
|
||||
|
||||
// Act
|
||||
var result = _sut.ValidatePipeline(pipeline, "Test");
|
||||
|
||||
// Assert
|
||||
result.Warnings.ShouldNotContain(w => w.Contains("No sync schedule"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ValidatePipeline_WithIntervalBelowMinimum_ReturnsError()
|
||||
{
|
||||
// Arrange
|
||||
var pipeline = new EtlPipelineConfig
|
||||
{
|
||||
Source = new SourceElement { Connection = "jde", Query = "SELECT 1" },
|
||||
Destination = new DestinationElement { Table = "TestTable" },
|
||||
HourlySyncIntervalMinutes = 5 // Below minimum of 15
|
||||
};
|
||||
|
||||
// Act
|
||||
var result = _sut.ValidatePipeline(pipeline, "Test");
|
||||
|
||||
// Assert
|
||||
result.IsValid.ShouldBeFalse();
|
||||
result.Errors.ShouldContain(e => e.Contains("Hourly sync interval"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ValidatePipeline_WithNoMatchColumns_ReturnsWarning()
|
||||
{
|
||||
// Arrange
|
||||
var pipeline = new EtlPipelineConfig
|
||||
{
|
||||
Source = new SourceElement { Connection = "jde", Query = "SELECT 1" },
|
||||
Destination = new DestinationElement { Table = "TestTable", MatchColumns = [] },
|
||||
IsManualOnly = true
|
||||
};
|
||||
|
||||
// Act
|
||||
var result = _sut.ValidatePipeline(pipeline, "Test");
|
||||
|
||||
// Assert
|
||||
result.Warnings.ShouldContain(w => w.Contains("No MatchColumns"));
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user