Files
Joseph Doherty 29ac56006d feat: implement ETL pipeline redesign and ConfigManager improvements
- Add pipeline registry with JSON-based configuration and hot-reload support
- Implement manual sync request feature with API, client UI, and database
- Improve ConfigManager: connection string dropdown in pipeline editor,
  step delete/reorder functionality, and fix JSON parsing for ConnectionStrings
2026-01-22 17:48:33 -05:00

484 lines
14 KiB
C#

using JdeScoping.DataSync.Configuration;
using JdeScoping.DataSync.Services;
using Shouldly;
using Xunit;
namespace JdeScoping.DataSync.Tests.Services;
public class PipelineValidatorTests
{
private readonly IPipelineValidator _validator;
public PipelineValidatorTests()
{
_validator = new PipelineValidator();
}
#region Name/Filename Matching
[Fact]
public void Validate_NameMatchesFilename_Passes()
{
// Arrange
var pipeline = CreateValidPipeline("TestPipeline");
// Act
var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json");
// Assert
result.IsValid.ShouldBeTrue();
result.Errors.ShouldBeEmpty();
}
[Fact]
public void Validate_NameMismatchFilename_Fails()
{
// Arrange
var pipeline = CreateValidPipeline("WrongName");
// Act
var result = _validator.Validate(pipeline, "pipeline.CorrectName.json");
// Assert
result.IsValid.ShouldBeFalse();
result.Errors.ShouldContain(e => e.Contains("does not match filename", StringComparison.OrdinalIgnoreCase));
}
[Fact]
public void Validate_NameCaseInsensitive_Passes()
{
// Arrange
var pipeline = CreateValidPipeline("testpipeline");
// Act
var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json");
// Assert
result.IsValid.ShouldBeTrue();
}
#endregion
#region Source Validation
[Fact]
public void Validate_MissingSource_Fails()
{
// Arrange
var pipeline = new EtlPipelineConfig
{
Name = "TestPipeline",
IsEnabled = true,
MassSyncIntervalMinutes = 1440,
Source = null!,
Destination = CreateValidDestination()
};
// Act
var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json");
// Assert
result.IsValid.ShouldBeFalse();
result.Errors.ShouldContain(e => e.Contains("Source is required", StringComparison.OrdinalIgnoreCase));
}
[Fact]
public void Validate_MissingSourceConnection_Fails()
{
// Arrange
var pipeline = new EtlPipelineConfig
{
Name = "TestPipeline",
IsEnabled = true,
MassSyncIntervalMinutes = 1440,
Source = new SourceElement
{
Connection = "",
Query = "SELECT * FROM table"
},
Destination = CreateValidDestination()
};
// Act
var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json");
// Assert
result.IsValid.ShouldBeFalse();
result.Errors.ShouldContain(e => e.Contains("Connection is required", StringComparison.OrdinalIgnoreCase));
}
[Fact]
public void Validate_InvalidConnection_Fails()
{
// Arrange
var pipeline = new EtlPipelineConfig
{
Name = "TestPipeline",
IsEnabled = true,
MassSyncIntervalMinutes = 1440,
Source = new SourceElement
{
Connection = "invalid_db",
Query = "SELECT * FROM table"
},
Destination = CreateValidDestination()
};
// Act
var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json");
// Assert
result.IsValid.ShouldBeFalse();
result.Errors.ShouldContain(e => e.Contains("not valid", StringComparison.OrdinalIgnoreCase));
}
[Fact]
public void Validate_ValidConnections_Pass()
{
// Arrange & Act & Assert
foreach (var connection in new[] { "jde", "cms", "giw", "lotfinder" })
{
var pipeline = CreateValidPipeline("TestPipeline");
pipeline.Source.Connection = connection;
var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json");
result.IsValid.ShouldBeTrue($"Connection '{connection}' should be valid");
}
}
[Fact]
public void Validate_MissingSourceQuery_Fails()
{
// Arrange
var pipeline = new EtlPipelineConfig
{
Name = "TestPipeline",
IsEnabled = true,
MassSyncIntervalMinutes = 1440,
Source = new SourceElement
{
Connection = "jde",
Query = ""
},
Destination = CreateValidDestination()
};
// Act
var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json");
// Assert
result.IsValid.ShouldBeFalse();
result.Errors.ShouldContain(e => e.Contains("Query is required", StringComparison.OrdinalIgnoreCase));
}
#endregion
#region Destination Validation
[Fact]
public void Validate_MissingDestination_Fails()
{
// Arrange
var pipeline = new EtlPipelineConfig
{
Name = "TestPipeline",
IsEnabled = true,
MassSyncIntervalMinutes = 1440,
Source = CreateValidSource(),
Destination = null!
};
// Act
var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json");
// Assert
result.IsValid.ShouldBeFalse();
result.Errors.ShouldContain(e => e.Contains("Destination is required", StringComparison.OrdinalIgnoreCase));
}
[Fact]
public void Validate_MissingDestinationTable_Fails()
{
// Arrange
var pipeline = new EtlPipelineConfig
{
Name = "TestPipeline",
IsEnabled = true,
MassSyncIntervalMinutes = 1440,
Source = CreateValidSource(),
Destination = new DestinationElement
{
Table = "",
MatchColumns = ["Id"]
}
};
// Act
var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json");
// Assert
result.IsValid.ShouldBeFalse();
result.Errors.ShouldContain(e => e.Contains("Table is required", StringComparison.OrdinalIgnoreCase));
}
[Fact]
public void Validate_EmptyMatchColumns_Fails()
{
// Arrange
var pipeline = new EtlPipelineConfig
{
Name = "TestPipeline",
IsEnabled = true,
MassSyncIntervalMinutes = 1440,
Source = CreateValidSource(),
Destination = new DestinationElement
{
Table = "TestTable",
MatchColumns = []
}
};
// Act
var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json");
// Assert
result.IsValid.ShouldBeFalse();
result.Errors.ShouldContain(e => e.Contains("MatchColumns", StringComparison.OrdinalIgnoreCase));
}
#endregion
#region Interval Validation
[Fact]
public void Validate_EnabledWithoutAnyInterval_Fails()
{
// Arrange
var pipeline = new EtlPipelineConfig
{
Name = "TestPipeline",
IsEnabled = true,
IsManualOnly = false,
// No intervals set
Source = CreateValidSource(),
Destination = CreateValidDestination()
};
// Act
var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json");
// Assert
result.IsValid.ShouldBeFalse();
result.Errors.ShouldContain(e => e.Contains("At least one sync interval", StringComparison.OrdinalIgnoreCase));
}
[Fact]
public void Validate_ManualOnlyWithoutInterval_Passes()
{
// Arrange
var pipeline = new EtlPipelineConfig
{
Name = "TestPipeline",
IsEnabled = true,
IsManualOnly = true,
// No intervals set - ok for manual-only
Source = CreateValidSource(),
Destination = CreateValidDestination()
};
// Act
var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json");
// Assert
result.IsValid.ShouldBeTrue();
}
[Fact]
public void Validate_DisabledWithoutInterval_Passes()
{
// Arrange
var pipeline = new EtlPipelineConfig
{
Name = "TestPipeline",
IsEnabled = false,
// No intervals set - ok for disabled
Source = CreateValidSource(),
Destination = CreateValidDestination()
};
// Act
var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json");
// Assert
result.IsValid.ShouldBeTrue();
}
[Fact]
public void Validate_ZeroMassInterval_Fails()
{
// Arrange
var pipeline = CreateValidPipeline("TestPipeline");
pipeline.MassSyncIntervalMinutes = 0;
// Act
var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json");
// Assert
result.IsValid.ShouldBeFalse();
result.Errors.ShouldContain(e => e.Contains("MassSyncIntervalMinutes must be greater than 0", StringComparison.OrdinalIgnoreCase));
}
[Fact]
public void Validate_NegativeInterval_Fails()
{
// Arrange
var pipeline = CreateValidPipeline("TestPipeline");
pipeline.DailySyncIntervalMinutes = -60;
// Act
var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json");
// Assert
result.IsValid.ShouldBeFalse();
result.Errors.ShouldContain(e => e.Contains("DailySyncIntervalMinutes must be greater than 0", StringComparison.OrdinalIgnoreCase));
}
#endregion
#region Warning Cases
[Fact]
public void Validate_HourlyWithoutDaily_ReturnsWarning()
{
// Arrange
var pipeline = new EtlPipelineConfig
{
Name = "TestPipeline",
IsEnabled = true,
HourlySyncIntervalMinutes = 15,
// No daily interval
Source = CreateValidSource(),
Destination = CreateValidDestination()
};
// Act
var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json");
// Assert
result.IsValid.ShouldBeTrue(); // Warnings don't fail validation
result.Warnings.ShouldContain(w => w.Contains("daily", StringComparison.OrdinalIgnoreCase));
}
#endregion
#region Script Validation
[Fact]
public void Validate_PreScriptWithEmptyScript_Fails()
{
// Arrange
var pipeline = CreateValidPipeline("TestPipeline");
pipeline.PreScripts = [new ScriptElement { Script = "" }];
// Act
var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json");
// Assert
result.IsValid.ShouldBeFalse();
result.Errors.ShouldContain(e => e.Contains("PreScripts", StringComparison.OrdinalIgnoreCase));
}
[Fact]
public void Validate_PostScriptWithEmptyScript_Fails()
{
// Arrange
var pipeline = CreateValidPipeline("TestPipeline");
pipeline.PostScripts = [new ScriptElement { Script = "" }];
// Act
var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json");
// Assert
result.IsValid.ShouldBeFalse();
result.Errors.ShouldContain(e => e.Contains("PostScripts", StringComparison.OrdinalIgnoreCase));
}
[Fact]
public void Validate_ValidScripts_Passes()
{
// Arrange
var pipeline = CreateValidPipeline("TestPipeline");
pipeline.PreScripts = [new ScriptElement { Script = "TRUNCATE TABLE Staging" }];
pipeline.PostScripts = [new ScriptElement { Script = "EXEC ProcessData" }];
// Act
var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json");
// Assert
result.IsValid.ShouldBeTrue();
}
#endregion
#region Complete Valid Pipeline
[Fact]
public void Validate_CompleteValidPipeline_Passes()
{
// Arrange
var pipeline = new EtlPipelineConfig
{
Name = "WorkOrder_Curr",
IsEnabled = true,
MassSyncIntervalMinutes = 1440,
DailySyncIntervalMinutes = 60,
HourlySyncIntervalMinutes = 15,
Source = new SourceElement
{
Connection = "jde",
Query = "SELECT * FROM WorkOrders WHERE ModDate > @lastSync",
MassQuery = "SELECT * FROM WorkOrders"
},
Destination = new DestinationElement
{
Table = "WorkOrder_Curr",
MatchColumns = ["OrderNumber"]
}
};
// Act
var result = _validator.Validate(pipeline, "pipeline.WorkOrder_Curr.json");
// Assert
result.IsValid.ShouldBeTrue();
result.Errors.ShouldBeEmpty();
}
#endregion
#region Helper Methods
private static EtlPipelineConfig CreateValidPipeline(string name) => new()
{
Name = name,
IsEnabled = true,
MassSyncIntervalMinutes = 1440,
Source = CreateValidSource(),
Destination = CreateValidDestination()
};
private static SourceElement CreateValidSource() => new()
{
Connection = "jde",
Query = "SELECT * FROM TestTable"
};
private static DestinationElement CreateValidDestination() => new()
{
Table = "TestTable",
MatchColumns = ["Id"]
};
#endregion
}