29ac56006d
- Add pipeline registry with JSON-based configuration and hot-reload support - Implement manual sync request feature with API, client UI, and database - Improve ConfigManager: connection string dropdown in pipeline editor, step delete/reorder functionality, and fix JSON parsing for ConnectionStrings
484 lines
14 KiB
C#
484 lines
14 KiB
C#
using JdeScoping.DataSync.Configuration;
|
|
using JdeScoping.DataSync.Services;
|
|
using Shouldly;
|
|
using Xunit;
|
|
|
|
namespace JdeScoping.DataSync.Tests.Services;
|
|
|
|
public class PipelineValidatorTests
|
|
{
|
|
private readonly IPipelineValidator _validator;
|
|
|
|
public PipelineValidatorTests()
|
|
{
|
|
_validator = new PipelineValidator();
|
|
}
|
|
|
|
#region Name/Filename Matching
|
|
|
|
[Fact]
|
|
public void Validate_NameMatchesFilename_Passes()
|
|
{
|
|
// Arrange
|
|
var pipeline = CreateValidPipeline("TestPipeline");
|
|
|
|
// Act
|
|
var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json");
|
|
|
|
// Assert
|
|
result.IsValid.ShouldBeTrue();
|
|
result.Errors.ShouldBeEmpty();
|
|
}
|
|
|
|
[Fact]
|
|
public void Validate_NameMismatchFilename_Fails()
|
|
{
|
|
// Arrange
|
|
var pipeline = CreateValidPipeline("WrongName");
|
|
|
|
// Act
|
|
var result = _validator.Validate(pipeline, "pipeline.CorrectName.json");
|
|
|
|
// Assert
|
|
result.IsValid.ShouldBeFalse();
|
|
result.Errors.ShouldContain(e => e.Contains("does not match filename", StringComparison.OrdinalIgnoreCase));
|
|
}
|
|
|
|
[Fact]
|
|
public void Validate_NameCaseInsensitive_Passes()
|
|
{
|
|
// Arrange
|
|
var pipeline = CreateValidPipeline("testpipeline");
|
|
|
|
// Act
|
|
var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json");
|
|
|
|
// Assert
|
|
result.IsValid.ShouldBeTrue();
|
|
}
|
|
|
|
#endregion
|
|
|
|
#region Source Validation
|
|
|
|
[Fact]
|
|
public void Validate_MissingSource_Fails()
|
|
{
|
|
// Arrange
|
|
var pipeline = new EtlPipelineConfig
|
|
{
|
|
Name = "TestPipeline",
|
|
IsEnabled = true,
|
|
MassSyncIntervalMinutes = 1440,
|
|
Source = null!,
|
|
Destination = CreateValidDestination()
|
|
};
|
|
|
|
// Act
|
|
var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json");
|
|
|
|
// Assert
|
|
result.IsValid.ShouldBeFalse();
|
|
result.Errors.ShouldContain(e => e.Contains("Source is required", StringComparison.OrdinalIgnoreCase));
|
|
}
|
|
|
|
[Fact]
|
|
public void Validate_MissingSourceConnection_Fails()
|
|
{
|
|
// Arrange
|
|
var pipeline = new EtlPipelineConfig
|
|
{
|
|
Name = "TestPipeline",
|
|
IsEnabled = true,
|
|
MassSyncIntervalMinutes = 1440,
|
|
Source = new SourceElement
|
|
{
|
|
Connection = "",
|
|
Query = "SELECT * FROM table"
|
|
},
|
|
Destination = CreateValidDestination()
|
|
};
|
|
|
|
// Act
|
|
var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json");
|
|
|
|
// Assert
|
|
result.IsValid.ShouldBeFalse();
|
|
result.Errors.ShouldContain(e => e.Contains("Connection is required", StringComparison.OrdinalIgnoreCase));
|
|
}
|
|
|
|
[Fact]
|
|
public void Validate_InvalidConnection_Fails()
|
|
{
|
|
// Arrange
|
|
var pipeline = new EtlPipelineConfig
|
|
{
|
|
Name = "TestPipeline",
|
|
IsEnabled = true,
|
|
MassSyncIntervalMinutes = 1440,
|
|
Source = new SourceElement
|
|
{
|
|
Connection = "invalid_db",
|
|
Query = "SELECT * FROM table"
|
|
},
|
|
Destination = CreateValidDestination()
|
|
};
|
|
|
|
// Act
|
|
var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json");
|
|
|
|
// Assert
|
|
result.IsValid.ShouldBeFalse();
|
|
result.Errors.ShouldContain(e => e.Contains("not valid", StringComparison.OrdinalIgnoreCase));
|
|
}
|
|
|
|
[Fact]
|
|
public void Validate_ValidConnections_Pass()
|
|
{
|
|
// Arrange & Act & Assert
|
|
foreach (var connection in new[] { "jde", "cms", "giw", "lotfinder" })
|
|
{
|
|
var pipeline = CreateValidPipeline("TestPipeline");
|
|
pipeline.Source.Connection = connection;
|
|
|
|
var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json");
|
|
result.IsValid.ShouldBeTrue($"Connection '{connection}' should be valid");
|
|
}
|
|
}
|
|
|
|
[Fact]
|
|
public void Validate_MissingSourceQuery_Fails()
|
|
{
|
|
// Arrange
|
|
var pipeline = new EtlPipelineConfig
|
|
{
|
|
Name = "TestPipeline",
|
|
IsEnabled = true,
|
|
MassSyncIntervalMinutes = 1440,
|
|
Source = new SourceElement
|
|
{
|
|
Connection = "jde",
|
|
Query = ""
|
|
},
|
|
Destination = CreateValidDestination()
|
|
};
|
|
|
|
// Act
|
|
var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json");
|
|
|
|
// Assert
|
|
result.IsValid.ShouldBeFalse();
|
|
result.Errors.ShouldContain(e => e.Contains("Query is required", StringComparison.OrdinalIgnoreCase));
|
|
}
|
|
|
|
#endregion
|
|
|
|
#region Destination Validation
|
|
|
|
[Fact]
|
|
public void Validate_MissingDestination_Fails()
|
|
{
|
|
// Arrange
|
|
var pipeline = new EtlPipelineConfig
|
|
{
|
|
Name = "TestPipeline",
|
|
IsEnabled = true,
|
|
MassSyncIntervalMinutes = 1440,
|
|
Source = CreateValidSource(),
|
|
Destination = null!
|
|
};
|
|
|
|
// Act
|
|
var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json");
|
|
|
|
// Assert
|
|
result.IsValid.ShouldBeFalse();
|
|
result.Errors.ShouldContain(e => e.Contains("Destination is required", StringComparison.OrdinalIgnoreCase));
|
|
}
|
|
|
|
[Fact]
|
|
public void Validate_MissingDestinationTable_Fails()
|
|
{
|
|
// Arrange
|
|
var pipeline = new EtlPipelineConfig
|
|
{
|
|
Name = "TestPipeline",
|
|
IsEnabled = true,
|
|
MassSyncIntervalMinutes = 1440,
|
|
Source = CreateValidSource(),
|
|
Destination = new DestinationElement
|
|
{
|
|
Table = "",
|
|
MatchColumns = ["Id"]
|
|
}
|
|
};
|
|
|
|
// Act
|
|
var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json");
|
|
|
|
// Assert
|
|
result.IsValid.ShouldBeFalse();
|
|
result.Errors.ShouldContain(e => e.Contains("Table is required", StringComparison.OrdinalIgnoreCase));
|
|
}
|
|
|
|
[Fact]
|
|
public void Validate_EmptyMatchColumns_Fails()
|
|
{
|
|
// Arrange
|
|
var pipeline = new EtlPipelineConfig
|
|
{
|
|
Name = "TestPipeline",
|
|
IsEnabled = true,
|
|
MassSyncIntervalMinutes = 1440,
|
|
Source = CreateValidSource(),
|
|
Destination = new DestinationElement
|
|
{
|
|
Table = "TestTable",
|
|
MatchColumns = []
|
|
}
|
|
};
|
|
|
|
// Act
|
|
var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json");
|
|
|
|
// Assert
|
|
result.IsValid.ShouldBeFalse();
|
|
result.Errors.ShouldContain(e => e.Contains("MatchColumns", StringComparison.OrdinalIgnoreCase));
|
|
}
|
|
|
|
#endregion
|
|
|
|
#region Interval Validation
|
|
|
|
[Fact]
|
|
public void Validate_EnabledWithoutAnyInterval_Fails()
|
|
{
|
|
// Arrange
|
|
var pipeline = new EtlPipelineConfig
|
|
{
|
|
Name = "TestPipeline",
|
|
IsEnabled = true,
|
|
IsManualOnly = false,
|
|
// No intervals set
|
|
Source = CreateValidSource(),
|
|
Destination = CreateValidDestination()
|
|
};
|
|
|
|
// Act
|
|
var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json");
|
|
|
|
// Assert
|
|
result.IsValid.ShouldBeFalse();
|
|
result.Errors.ShouldContain(e => e.Contains("At least one sync interval", StringComparison.OrdinalIgnoreCase));
|
|
}
|
|
|
|
[Fact]
|
|
public void Validate_ManualOnlyWithoutInterval_Passes()
|
|
{
|
|
// Arrange
|
|
var pipeline = new EtlPipelineConfig
|
|
{
|
|
Name = "TestPipeline",
|
|
IsEnabled = true,
|
|
IsManualOnly = true,
|
|
// No intervals set - ok for manual-only
|
|
Source = CreateValidSource(),
|
|
Destination = CreateValidDestination()
|
|
};
|
|
|
|
// Act
|
|
var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json");
|
|
|
|
// Assert
|
|
result.IsValid.ShouldBeTrue();
|
|
}
|
|
|
|
[Fact]
|
|
public void Validate_DisabledWithoutInterval_Passes()
|
|
{
|
|
// Arrange
|
|
var pipeline = new EtlPipelineConfig
|
|
{
|
|
Name = "TestPipeline",
|
|
IsEnabled = false,
|
|
// No intervals set - ok for disabled
|
|
Source = CreateValidSource(),
|
|
Destination = CreateValidDestination()
|
|
};
|
|
|
|
// Act
|
|
var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json");
|
|
|
|
// Assert
|
|
result.IsValid.ShouldBeTrue();
|
|
}
|
|
|
|
[Fact]
|
|
public void Validate_ZeroMassInterval_Fails()
|
|
{
|
|
// Arrange
|
|
var pipeline = CreateValidPipeline("TestPipeline");
|
|
pipeline.MassSyncIntervalMinutes = 0;
|
|
|
|
// Act
|
|
var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json");
|
|
|
|
// Assert
|
|
result.IsValid.ShouldBeFalse();
|
|
result.Errors.ShouldContain(e => e.Contains("MassSyncIntervalMinutes must be greater than 0", StringComparison.OrdinalIgnoreCase));
|
|
}
|
|
|
|
[Fact]
|
|
public void Validate_NegativeInterval_Fails()
|
|
{
|
|
// Arrange
|
|
var pipeline = CreateValidPipeline("TestPipeline");
|
|
pipeline.DailySyncIntervalMinutes = -60;
|
|
|
|
// Act
|
|
var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json");
|
|
|
|
// Assert
|
|
result.IsValid.ShouldBeFalse();
|
|
result.Errors.ShouldContain(e => e.Contains("DailySyncIntervalMinutes must be greater than 0", StringComparison.OrdinalIgnoreCase));
|
|
}
|
|
|
|
#endregion
|
|
|
|
#region Warning Cases
|
|
|
|
[Fact]
|
|
public void Validate_HourlyWithoutDaily_ReturnsWarning()
|
|
{
|
|
// Arrange
|
|
var pipeline = new EtlPipelineConfig
|
|
{
|
|
Name = "TestPipeline",
|
|
IsEnabled = true,
|
|
HourlySyncIntervalMinutes = 15,
|
|
// No daily interval
|
|
Source = CreateValidSource(),
|
|
Destination = CreateValidDestination()
|
|
};
|
|
|
|
// Act
|
|
var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json");
|
|
|
|
// Assert
|
|
result.IsValid.ShouldBeTrue(); // Warnings don't fail validation
|
|
result.Warnings.ShouldContain(w => w.Contains("daily", StringComparison.OrdinalIgnoreCase));
|
|
}
|
|
|
|
#endregion
|
|
|
|
#region Script Validation
|
|
|
|
[Fact]
|
|
public void Validate_PreScriptWithEmptyScript_Fails()
|
|
{
|
|
// Arrange
|
|
var pipeline = CreateValidPipeline("TestPipeline");
|
|
pipeline.PreScripts = [new ScriptElement { Script = "" }];
|
|
|
|
// Act
|
|
var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json");
|
|
|
|
// Assert
|
|
result.IsValid.ShouldBeFalse();
|
|
result.Errors.ShouldContain(e => e.Contains("PreScripts", StringComparison.OrdinalIgnoreCase));
|
|
}
|
|
|
|
[Fact]
|
|
public void Validate_PostScriptWithEmptyScript_Fails()
|
|
{
|
|
// Arrange
|
|
var pipeline = CreateValidPipeline("TestPipeline");
|
|
pipeline.PostScripts = [new ScriptElement { Script = "" }];
|
|
|
|
// Act
|
|
var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json");
|
|
|
|
// Assert
|
|
result.IsValid.ShouldBeFalse();
|
|
result.Errors.ShouldContain(e => e.Contains("PostScripts", StringComparison.OrdinalIgnoreCase));
|
|
}
|
|
|
|
[Fact]
|
|
public void Validate_ValidScripts_Passes()
|
|
{
|
|
// Arrange
|
|
var pipeline = CreateValidPipeline("TestPipeline");
|
|
pipeline.PreScripts = [new ScriptElement { Script = "TRUNCATE TABLE Staging" }];
|
|
pipeline.PostScripts = [new ScriptElement { Script = "EXEC ProcessData" }];
|
|
|
|
// Act
|
|
var result = _validator.Validate(pipeline, "pipeline.TestPipeline.json");
|
|
|
|
// Assert
|
|
result.IsValid.ShouldBeTrue();
|
|
}
|
|
|
|
#endregion
|
|
|
|
#region Complete Valid Pipeline
|
|
|
|
[Fact]
|
|
public void Validate_CompleteValidPipeline_Passes()
|
|
{
|
|
// Arrange
|
|
var pipeline = new EtlPipelineConfig
|
|
{
|
|
Name = "WorkOrder_Curr",
|
|
IsEnabled = true,
|
|
MassSyncIntervalMinutes = 1440,
|
|
DailySyncIntervalMinutes = 60,
|
|
HourlySyncIntervalMinutes = 15,
|
|
Source = new SourceElement
|
|
{
|
|
Connection = "jde",
|
|
Query = "SELECT * FROM WorkOrders WHERE ModDate > @lastSync",
|
|
MassQuery = "SELECT * FROM WorkOrders"
|
|
},
|
|
Destination = new DestinationElement
|
|
{
|
|
Table = "WorkOrder_Curr",
|
|
MatchColumns = ["OrderNumber"]
|
|
}
|
|
};
|
|
|
|
// Act
|
|
var result = _validator.Validate(pipeline, "pipeline.WorkOrder_Curr.json");
|
|
|
|
// Assert
|
|
result.IsValid.ShouldBeTrue();
|
|
result.Errors.ShouldBeEmpty();
|
|
}
|
|
|
|
#endregion
|
|
|
|
#region Helper Methods
|
|
|
|
private static EtlPipelineConfig CreateValidPipeline(string name) => new()
|
|
{
|
|
Name = name,
|
|
IsEnabled = true,
|
|
MassSyncIntervalMinutes = 1440,
|
|
Source = CreateValidSource(),
|
|
Destination = CreateValidDestination()
|
|
};
|
|
|
|
private static SourceElement CreateValidSource() => new()
|
|
{
|
|
Connection = "jde",
|
|
Query = "SELECT * FROM TestTable"
|
|
};
|
|
|
|
private static DestinationElement CreateValidDestination() => new()
|
|
{
|
|
Table = "TestTable",
|
|
MatchColumns = ["Id"]
|
|
};
|
|
|
|
#endregion
|
|
}
|