refactor(configmanager): migrate to per-file pipeline system

Align ConfigManager with DataSync's per-file pipeline format (pipeline.*.json)
by reusing EtlPipelineConfig types directly, eliminating duplicate models and
simplifying the codebase. Removes ~3200 lines of obsolete code.
This commit is contained in:
Joseph Doherty
2026-01-23 02:30:48 -05:00
parent 1b7bb26def
commit ba54a87be5
49 changed files with 1429 additions and 4396 deletions
@@ -1,5 +1,6 @@
using JdeScoping.ConfigManager.Models;
using JdeScoping.ConfigManager.Services;
using JdeScoping.DataSync.Configuration;
namespace JdeScoping.ConfigManager.Tests.Services;
@@ -47,19 +48,16 @@ public class ValidationServiceTests
}
[Fact]
public void ValidatePipelines_WithDuplicateNames_ReturnsError()
public void ValidatePipelines_WithEmptyName_ReturnsError()
{
// Arrange - duplicate keys not possible in dictionary, but empty names are invalid
var config = new PipelinesConfigModel
// Arrange
var pipelines = new Dictionary<string, EtlPipelineConfig>
{
Pipelines = new Dictionary<string, PipelineModel>
{
[""] = new PipelineModel()
}
[""] = new EtlPipelineConfig()
};
// Act
var result = _sut.ValidatePipelines(config);
var result = _sut.ValidatePipelines(pipelines);
// Assert
result.IsValid.ShouldBeFalse();
@@ -69,22 +67,140 @@ public class ValidationServiceTests
public void ValidatePipelines_WithInvalidConnection_ReturnsError()
{
// Arrange
var config = new PipelinesConfigModel
var pipelines = new Dictionary<string, EtlPipelineConfig>
{
Pipelines = new Dictionary<string, PipelineModel>
["Test"] = new EtlPipelineConfig
{
["Test"] = new PipelineModel
{
Source = new PipelineSource { Connection = "invalid" }
}
Source = new SourceElement { Connection = "invalid" }
}
};
// Act
var result = _sut.ValidatePipelines(config);
var result = _sut.ValidatePipelines(pipelines);
// Assert
result.IsValid.ShouldBeFalse();
result.Errors.ShouldContain(e => e.Contains("Connection"));
}
[Fact]
public void ValidatePipeline_WithValidConfig_ReturnsNoErrors()
{
// Arrange
var pipeline = new EtlPipelineConfig
{
Name = "Test",
IsEnabled = true,
Source = new SourceElement
{
Connection = "jde",
Query = "SELECT * FROM Test"
},
Destination = new DestinationElement
{
Table = "TestTable",
MatchColumns = ["Id"]
},
HourlySyncIntervalMinutes = 60
};
// Act
var result = _sut.ValidatePipeline(pipeline, "Test");
// Assert
result.IsValid.ShouldBeTrue();
result.Errors.ShouldBeEmpty();
}
[Fact]
public void ValidatePipeline_WithMissingQuery_ReturnsError()
{
// Arrange
var pipeline = new EtlPipelineConfig
{
Source = new SourceElement { Connection = "jde", Query = "" },
Destination = new DestinationElement { Table = "TestTable" },
IsManualOnly = true
};
// Act
var result = _sut.ValidatePipeline(pipeline, "Test");
// Assert
result.IsValid.ShouldBeFalse();
result.Errors.ShouldContain(e => e.Contains("Query"));
}
[Fact]
public void ValidatePipeline_WithNoScheduleAndNotManual_ReturnsWarning()
{
// Arrange
var pipeline = new EtlPipelineConfig
{
Source = new SourceElement { Connection = "jde", Query = "SELECT 1" },
Destination = new DestinationElement { Table = "TestTable", MatchColumns = ["Id"] },
IsManualOnly = false // No schedules and not manual-only
};
// Act
var result = _sut.ValidatePipeline(pipeline, "Test");
// Assert
result.Warnings.ShouldContain(w => w.Contains("No sync schedule"));
}
[Fact]
public void ValidatePipeline_ManualOnly_DoesNotRequireSchedule()
{
// Arrange
var pipeline = new EtlPipelineConfig
{
Source = new SourceElement { Connection = "jde", Query = "SELECT 1" },
Destination = new DestinationElement { Table = "TestTable", MatchColumns = ["Id"] },
IsManualOnly = true
};
// Act
var result = _sut.ValidatePipeline(pipeline, "Test");
// Assert
result.Warnings.ShouldNotContain(w => w.Contains("No sync schedule"));
}
[Fact]
public void ValidatePipeline_WithIntervalBelowMinimum_ReturnsError()
{
// Arrange
var pipeline = new EtlPipelineConfig
{
Source = new SourceElement { Connection = "jde", Query = "SELECT 1" },
Destination = new DestinationElement { Table = "TestTable" },
HourlySyncIntervalMinutes = 5 // Below minimum of 15
};
// Act
var result = _sut.ValidatePipeline(pipeline, "Test");
// Assert
result.IsValid.ShouldBeFalse();
result.Errors.ShouldContain(e => e.Contains("Hourly sync interval"));
}
[Fact]
public void ValidatePipeline_WithNoMatchColumns_ReturnsWarning()
{
// Arrange
var pipeline = new EtlPipelineConfig
{
Source = new SourceElement { Connection = "jde", Query = "SELECT 1" },
Destination = new DestinationElement { Table = "TestTable", MatchColumns = [] },
IsManualOnly = true
};
// Act
var result = _sut.ValidatePipeline(pipeline, "Test");
// Assert
result.Warnings.ShouldContain(w => w.Contains("No MatchColumns"));
}
}