feat: implement ETL pipeline redesign and ConfigManager improvements

- Add pipeline registry with JSON-based configuration and hot-reload support - Implement manual sync request feature with API, client UI, and database - Improve ConfigManager: connection string dropdown in pipeline editor, step delete/reorder functionality, and fix JSON parsing for ConnectionStrings
2026-01-22 17:48:33 -05:00
parent 5a332232d0
commit 29ac56006d
82 changed files with 6257 additions and 296 deletions
@@ -0,0 +1,483 @@
+using JdeScoping.DataSync.Configuration;
+using JdeScoping.DataSync.Services;
+using Shouldly;
+using Xunit;
+
+namespace JdeScoping.DataSync.Tests.Services;
+
+public class PipelineValidatorTests
+{
+    private readonly IPipelineValidator _validator;
+
+    public PipelineValidatorTests()
+    {
+        _validator = new PipelineValidator();
+    }
+
+    #region Name/Filename Matching
+
+    [Fact]
+    public void Validate_NameMatchesFilename_Passes()
+    {
+        // Arrange
+        var pipeline = CreateValidPipeline("TestPipeline");
+
+        // Act
+        var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json");
+
+        // Assert
+        result.IsValid.ShouldBeTrue();
+        result.Errors.ShouldBeEmpty();
+    }
+
+    [Fact]
+    public void Validate_NameMismatchFilename_Fails()
+    {
+        // Arrange
+        var pipeline = CreateValidPipeline("WrongName");
+
+        // Act
+        var result = _validator.Validate(pipeline, "pipeline.CorrectName.json");
+
+        // Assert
+        result.IsValid.ShouldBeFalse();
+        result.Errors.ShouldContain(e => e.Contains("does not match filename", StringComparison.OrdinalIgnoreCase));
+    }
+
+    [Fact]
+    public void Validate_NameCaseInsensitive_Passes()
+    {
+        // Arrange
+        var pipeline = CreateValidPipeline("testpipeline");
+
+        // Act
+        var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json");
+
+        // Assert
+        result.IsValid.ShouldBeTrue();
+    }
+
+    #endregion
+
+    #region Source Validation
+
+    [Fact]
+    public void Validate_MissingSource_Fails()
+    {
+        // Arrange
+        var pipeline = new EtlPipelineConfig
+        {
+            Name = "TestPipeline",
+            IsEnabled = true,
+            MassSyncIntervalMinutes = 1440,
+            Source = null!,
+            Destination = CreateValidDestination()
+        };
+
+        // Act
+        var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json");
+
+        // Assert
+        result.IsValid.ShouldBeFalse();
+        result.Errors.ShouldContain(e => e.Contains("Source is required", StringComparison.OrdinalIgnoreCase));
+    }
+
+    [Fact]
+    public void Validate_MissingSourceConnection_Fails()
+    {
+        // Arrange
+        var pipeline = new EtlPipelineConfig
+        {
+            Name = "TestPipeline",
+            IsEnabled = true,
+            MassSyncIntervalMinutes = 1440,
+            Source = new SourceElement
+            {
+                Connection = "",
+                Query = "SELECT * FROM table"
+            },
+            Destination = CreateValidDestination()
+        };
+
+        // Act
+        var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json");
+
+        // Assert
+        result.IsValid.ShouldBeFalse();
+        result.Errors.ShouldContain(e => e.Contains("Connection is required", StringComparison.OrdinalIgnoreCase));
+    }
+
+    [Fact]
+    public void Validate_InvalidConnection_Fails()
+    {
+        // Arrange
+        var pipeline = new EtlPipelineConfig
+        {
+            Name = "TestPipeline",
+            IsEnabled = true,
+            MassSyncIntervalMinutes = 1440,
+            Source = new SourceElement
+            {
+                Connection = "invalid_db",
+                Query = "SELECT * FROM table"
+            },
+            Destination = CreateValidDestination()
+        };
+
+        // Act
+        var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json");
+
+        // Assert
+        result.IsValid.ShouldBeFalse();
+        result.Errors.ShouldContain(e => e.Contains("not valid", StringComparison.OrdinalIgnoreCase));
+    }
+
+    [Fact]
+    public void Validate_ValidConnections_Pass()
+    {
+        // Arrange & Act & Assert
+        foreach (var connection in new[] { "jde", "cms", "giw", "lotfinder" })
+        {
+            var pipeline = CreateValidPipeline("TestPipeline");
+            pipeline.Source.Connection = connection;
+
+            var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json");
+            result.IsValid.ShouldBeTrue($"Connection '{connection}' should be valid");
+        }
+    }
+
+    [Fact]
+    public void Validate_MissingSourceQuery_Fails()
+    {
+        // Arrange
+        var pipeline = new EtlPipelineConfig
+        {
+            Name = "TestPipeline",
+            IsEnabled = true,
+            MassSyncIntervalMinutes = 1440,
+            Source = new SourceElement
+            {
+                Connection = "jde",
+                Query = ""
+            },
+            Destination = CreateValidDestination()
+        };
+
+        // Act
+        var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json");
+
+        // Assert
+        result.IsValid.ShouldBeFalse();
+        result.Errors.ShouldContain(e => e.Contains("Query is required", StringComparison.OrdinalIgnoreCase));
+    }
+
+    #endregion
+
+    #region Destination Validation
+
+    [Fact]
+    public void Validate_MissingDestination_Fails()
+    {
+        // Arrange
+        var pipeline = new EtlPipelineConfig
+        {
+            Name = "TestPipeline",
+            IsEnabled = true,
+            MassSyncIntervalMinutes = 1440,
+            Source = CreateValidSource(),
+            Destination = null!
+        };
+
+        // Act
+        var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json");
+
+        // Assert
+        result.IsValid.ShouldBeFalse();
+        result.Errors.ShouldContain(e => e.Contains("Destination is required", StringComparison.OrdinalIgnoreCase));
+    }
+
+    [Fact]
+    public void Validate_MissingDestinationTable_Fails()
+    {
+        // Arrange
+        var pipeline = new EtlPipelineConfig
+        {
+            Name = "TestPipeline",
+            IsEnabled = true,
+            MassSyncIntervalMinutes = 1440,
+            Source = CreateValidSource(),
+            Destination = new DestinationElement
+            {
+                Table = "",
+                MatchColumns = ["Id"]
+            }
+        };
+
+        // Act
+        var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json");
+
+        // Assert
+        result.IsValid.ShouldBeFalse();
+        result.Errors.ShouldContain(e => e.Contains("Table is required", StringComparison.OrdinalIgnoreCase));
+    }
+
+    [Fact]
+    public void Validate_EmptyMatchColumns_Fails()
+    {
+        // Arrange
+        var pipeline = new EtlPipelineConfig
+        {
+            Name = "TestPipeline",
+            IsEnabled = true,
+            MassSyncIntervalMinutes = 1440,
+            Source = CreateValidSource(),
+            Destination = new DestinationElement
+            {
+                Table = "TestTable",
+                MatchColumns = []
+            }
+        };
+
+        // Act
+        var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json");
+
+        // Assert
+        result.IsValid.ShouldBeFalse();
+        result.Errors.ShouldContain(e => e.Contains("MatchColumns", StringComparison.OrdinalIgnoreCase));
+    }
+
+    #endregion
+
+    #region Interval Validation
+
+    [Fact]
+    public void Validate_EnabledWithoutAnyInterval_Fails()
+    {
+        // Arrange
+        var pipeline = new EtlPipelineConfig
+        {
+            Name = "TestPipeline",
+            IsEnabled = true,
+            IsManualOnly = false,
+            // No intervals set
+            Source = CreateValidSource(),
+            Destination = CreateValidDestination()
+        };
+
+        // Act
+        var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json");
+
+        // Assert
+        result.IsValid.ShouldBeFalse();
+        result.Errors.ShouldContain(e => e.Contains("At least one sync interval", StringComparison.OrdinalIgnoreCase));
+    }
+
+    [Fact]
+    public void Validate_ManualOnlyWithoutInterval_Passes()
+    {
+        // Arrange
+        var pipeline = new EtlPipelineConfig
+        {
+            Name = "TestPipeline",
+            IsEnabled = true,
+            IsManualOnly = true,
+            // No intervals set - ok for manual-only
+            Source = CreateValidSource(),
+            Destination = CreateValidDestination()
+        };
+
+        // Act
+        var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json");
+
+        // Assert
+        result.IsValid.ShouldBeTrue();
+    }
+
+    [Fact]
+    public void Validate_DisabledWithoutInterval_Passes()
+    {
+        // Arrange
+        var pipeline = new EtlPipelineConfig
+        {
+            Name = "TestPipeline",
+            IsEnabled = false,
+            // No intervals set - ok for disabled
+            Source = CreateValidSource(),
+            Destination = CreateValidDestination()
+        };
+
+        // Act
+        var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json");
+
+        // Assert
+        result.IsValid.ShouldBeTrue();
+    }
+
+    [Fact]
+    public void Validate_ZeroMassInterval_Fails()
+    {
+        // Arrange
+        var pipeline = CreateValidPipeline("TestPipeline");
+        pipeline.MassSyncIntervalMinutes = 0;
+
+        // Act
+        var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json");
+
+        // Assert
+        result.IsValid.ShouldBeFalse();
+        result.Errors.ShouldContain(e => e.Contains("MassSyncIntervalMinutes must be greater than 0", StringComparison.OrdinalIgnoreCase));
+    }
+
+    [Fact]
+    public void Validate_NegativeInterval_Fails()
+    {
+        // Arrange
+        var pipeline = CreateValidPipeline("TestPipeline");
+        pipeline.DailySyncIntervalMinutes = -60;
+
+        // Act
+        var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json");
+
+        // Assert
+        result.IsValid.ShouldBeFalse();
+        result.Errors.ShouldContain(e => e.Contains("DailySyncIntervalMinutes must be greater than 0", StringComparison.OrdinalIgnoreCase));
+    }
+
+    #endregion
+
+    #region Warning Cases
+
+    [Fact]
+    public void Validate_HourlyWithoutDaily_ReturnsWarning()
+    {
+        // Arrange
+        var pipeline = new EtlPipelineConfig
+        {
+            Name = "TestPipeline",
+            IsEnabled = true,
+            HourlySyncIntervalMinutes = 15,
+            // No daily interval
+            Source = CreateValidSource(),
+            Destination = CreateValidDestination()
+        };
+
+        // Act
+        var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json");
+
+        // Assert
+        result.IsValid.ShouldBeTrue(); // Warnings don't fail validation
+        result.Warnings.ShouldContain(w => w.Contains("daily", StringComparison.OrdinalIgnoreCase));
+    }
+
+    #endregion
+
+    #region Script Validation
+
+    [Fact]
+    public void Validate_PreScriptWithEmptyScript_Fails()
+    {
+        // Arrange
+        var pipeline = CreateValidPipeline("TestPipeline");
+        pipeline.PreScripts = [new ScriptElement { Script = "" }];
+
+        // Act
+        var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json");
+
+        // Assert
+        result.IsValid.ShouldBeFalse();
+        result.Errors.ShouldContain(e => e.Contains("PreScripts", StringComparison.OrdinalIgnoreCase));
+    }
+
+    [Fact]
+    public void Validate_PostScriptWithEmptyScript_Fails()
+    {
+        // Arrange
+        var pipeline = CreateValidPipeline("TestPipeline");
+        pipeline.PostScripts = [new ScriptElement { Script = "" }];
+
+        // Act
+        var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json");
+
+        // Assert
+        result.IsValid.ShouldBeFalse();
+        result.Errors.ShouldContain(e => e.Contains("PostScripts", StringComparison.OrdinalIgnoreCase));
+    }
+
+    [Fact]
+    public void Validate_ValidScripts_Passes()
+    {
+        // Arrange
+        var pipeline = CreateValidPipeline("TestPipeline");
+        pipeline.PreScripts = [new ScriptElement { Script = "TRUNCATE TABLE Staging" }];
+        pipeline.PostScripts = [new ScriptElement { Script = "EXEC ProcessData" }];
+
+        // Act
+        var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json");
+
+        // Assert
+        result.IsValid.ShouldBeTrue();
+    }
+
+    #endregion
+
+    #region Complete Valid Pipeline
+
+    [Fact]
+    public void Validate_CompleteValidPipeline_Passes()
+    {
+        // Arrange
+        var pipeline = new EtlPipelineConfig
+        {
+            Name = "WorkOrder_Curr",
+            IsEnabled = true,
+            MassSyncIntervalMinutes = 1440,
+            DailySyncIntervalMinutes = 60,
+            HourlySyncIntervalMinutes = 15,
+            Source = new SourceElement
+            {
+                Connection = "jde",
+                Query = "SELECT * FROM WorkOrders WHERE ModDate > @lastSync",
+                MassQuery = "SELECT * FROM WorkOrders"
+            },
+            Destination = new DestinationElement
+            {
+                Table = "WorkOrder_Curr",
+                MatchColumns = ["OrderNumber"]
+            }
+        };
+
+        // Act
+        var result = _validator.Validate(pipeline, "pipeline.WorkOrder_Curr.json");
+
+        // Assert
+        result.IsValid.ShouldBeTrue();
+        result.Errors.ShouldBeEmpty();
+    }
+
+    #endregion
+
+    #region Helper Methods
+
+    private static EtlPipelineConfig CreateValidPipeline(string name) => new()
+    {
+        Name = name,
+        IsEnabled = true,
+        MassSyncIntervalMinutes = 1440,
+        Source = CreateValidSource(),
+        Destination = CreateValidDestination()
+    };
+
+    private static SourceElement CreateValidSource() => new()
+    {
+        Connection = "jde",
+        Query = "SELECT * FROM TestTable"
+    };
+
+    private static DestinationElement CreateValidDestination() => new()
+    {
+        Table = "TestTable",
+        MatchColumns = ["Id"]
+    };
+
+    #endregion
+}