feat: implement ETL pipeline redesign and ConfigManager improvements
- Add pipeline registry with JSON-based configuration and hot-reload support - Implement manual sync request feature with API, client UI, and database - Improve ConfigManager: connection string dropdown in pipeline editor, step delete/reorder functionality, and fix JSON parsing for ConnectionStrings
This commit is contained in:
Joseph Doherty
@@ -0,0 +1,289 @@
|
||||
using System.Security.Claims;
|
||||
using JdeScoping.Api.Controllers;
|
||||
using JdeScoping.DataSync.Configuration;
|
||||
using JdeScoping.DataSync.Services;
|
||||
using Microsoft.AspNetCore.Http;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using NSubstitute;
|
||||
using NSubstitute.ExceptionExtensions;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
|
||||
namespace JdeScoping.Api.Tests.Controllers;
|
||||
|
||||
public class PipelineControllerTests
|
||||
{
|
||||
private readonly IPipelineRegistry _pipelineRegistry;
|
||||
private readonly ILogger<PipelineController> _logger;
|
||||
private readonly PipelineController _controller;
|
||||
|
||||
public PipelineControllerTests()
|
||||
{
|
||||
_pipelineRegistry = Substitute.For<IPipelineRegistry>();
|
||||
_logger = Substitute.For<ILogger<PipelineController>>();
|
||||
_controller = new PipelineController(_pipelineRegistry, _logger);
|
||||
SetupAuthenticatedUser("testuser", isAdmin: false);
|
||||
}
|
||||
|
||||
#region GetPipelines Tests
|
||||
|
||||
[Fact]
|
||||
public void GetPipelines_ReturnsEnabledPipelines()
|
||||
{
|
||||
// Arrange
|
||||
var pipelines = new List<EtlPipelineConfig>
|
||||
{
|
||||
CreatePipeline("WorkOrders", massSyncInterval: 1440, dailySyncInterval: 60, hourlySyncInterval: 15),
|
||||
CreatePipeline("Items", massSyncInterval: 1440, dailySyncInterval: 60)
|
||||
};
|
||||
_pipelineRegistry.GetEnabledPipelines().Returns(pipelines);
|
||||
|
||||
// Act
|
||||
var result = _controller.GetPipelines();
|
||||
|
||||
// Assert
|
||||
result.Result.ShouldBeOfType<OkObjectResult>();
|
||||
var okResult = (OkObjectResult)result.Result!;
|
||||
var viewModels = okResult.Value.ShouldBeAssignableTo<List<Api.Contracts.ManualSync.PipelineInfoViewModel>>()!;
|
||||
|
||||
viewModels.Count.ShouldBe(2);
|
||||
viewModels[0].Name.ShouldBe("WorkOrders");
|
||||
viewModels[0].SupportedSyncTypes.ShouldContain("mass");
|
||||
viewModels[0].SupportedSyncTypes.ShouldContain("daily");
|
||||
viewModels[0].SupportedSyncTypes.ShouldContain("hourly");
|
||||
viewModels[1].Name.ShouldBe("Items");
|
||||
viewModels[1].SupportedSyncTypes.ShouldNotContain("hourly");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void GetPipelines_WhenEmpty_ReturnsEmptyList()
|
||||
{
|
||||
// Arrange
|
||||
_pipelineRegistry.GetEnabledPipelines().Returns(new List<EtlPipelineConfig>());
|
||||
|
||||
// Act
|
||||
var result = _controller.GetPipelines();
|
||||
|
||||
// Assert
|
||||
var okResult = (OkObjectResult)result.Result!;
|
||||
var viewModels = okResult.Value.ShouldBeAssignableTo<List<Api.Contracts.ManualSync.PipelineInfoViewModel>>()!;
|
||||
viewModels.ShouldBeEmpty();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void GetPipelines_MapsOnlySupportedSyncTypes()
|
||||
{
|
||||
// Arrange - pipeline with only mass sync
|
||||
var pipelines = new List<EtlPipelineConfig>
|
||||
{
|
||||
CreatePipeline("MassOnly", massSyncInterval: 1440)
|
||||
};
|
||||
_pipelineRegistry.GetEnabledPipelines().Returns(pipelines);
|
||||
|
||||
// Act
|
||||
var result = _controller.GetPipelines();
|
||||
|
||||
// Assert
|
||||
var okResult = (OkObjectResult)result.Result!;
|
||||
var viewModels = okResult.Value.ShouldBeAssignableTo<List<Api.Contracts.ManualSync.PipelineInfoViewModel>>()!;
|
||||
|
||||
viewModels[0].SupportedSyncTypes.Count.ShouldBe(1);
|
||||
viewModels[0].SupportedSyncTypes.ShouldContain("mass");
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region GetStatus Tests
|
||||
|
||||
[Fact]
|
||||
public void GetStatus_ReturnsRegistryMetadata()
|
||||
{
|
||||
// Arrange
|
||||
var allPipelines = new List<EtlPipelineConfig>
|
||||
{
|
||||
CreatePipeline("Pipeline1", isEnabled: true),
|
||||
CreatePipeline("Pipeline2", isEnabled: true),
|
||||
CreatePipeline("Pipeline3", isEnabled: false)
|
||||
};
|
||||
var enabledPipelines = allPipelines.Where(p => p.IsEnabled).ToList();
|
||||
|
||||
_pipelineRegistry.GetAllPipelines().Returns(allPipelines);
|
||||
_pipelineRegistry.GetEnabledPipelines().Returns(enabledPipelines);
|
||||
_pipelineRegistry.Version.Returns(5);
|
||||
_pipelineRegistry.LastLoadedAt.Returns(new DateTime(2024, 1, 15, 10, 30, 0, DateTimeKind.Utc));
|
||||
|
||||
// Act
|
||||
var result = _controller.GetStatus();
|
||||
|
||||
// Assert
|
||||
result.Result.ShouldBeOfType<OkObjectResult>();
|
||||
var okResult = (OkObjectResult)result.Result!;
|
||||
var status = okResult.Value.ShouldBeOfType<PipelineRegistryStatusViewModel>();
|
||||
|
||||
status.Version.ShouldBe(5);
|
||||
status.LastLoadedAt.ShouldBe(new DateTime(2024, 1, 15, 10, 30, 0, DateTimeKind.Utc));
|
||||
status.TotalPipelines.ShouldBe(3);
|
||||
status.EnabledPipelines.ShouldBe(2);
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region ReloadPipelines Tests
|
||||
|
||||
[Fact]
|
||||
public async Task ReloadPipelines_CallsRegistry()
|
||||
{
|
||||
// Arrange
|
||||
SetupAuthenticatedUser("admin", isAdmin: true);
|
||||
_pipelineRegistry.ReloadAsync(Arg.Any<CancellationToken>())
|
||||
.Returns(CreateSuccessfulReloadResult(5, 0));
|
||||
|
||||
// Act
|
||||
await _controller.ReloadPipelines(CancellationToken.None);
|
||||
|
||||
// Assert
|
||||
await _pipelineRegistry.Received(1).ReloadAsync(Arg.Any<CancellationToken>());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ReloadPipelines_ReturnsResult()
|
||||
{
|
||||
// Arrange
|
||||
SetupAuthenticatedUser("admin", isAdmin: true);
|
||||
_pipelineRegistry.ReloadAsync(Arg.Any<CancellationToken>())
|
||||
.Returns(CreateSuccessfulReloadResult(pipelinesLoaded: 10, pipelinesSkipped: 2, previousVersion: 3, newVersion: 4));
|
||||
|
||||
// Act
|
||||
var result = await _controller.ReloadPipelines(CancellationToken.None);
|
||||
|
||||
// Assert
|
||||
result.Result.ShouldBeOfType<OkObjectResult>();
|
||||
var okResult = (OkObjectResult)result.Result!;
|
||||
var reloadResult = okResult.Value.ShouldBeOfType<PipelineReloadResultViewModel>();
|
||||
|
||||
reloadResult.Success.ShouldBeTrue();
|
||||
reloadResult.PipelinesLoaded.ShouldBe(10);
|
||||
reloadResult.PipelinesSkipped.ShouldBe(2);
|
||||
reloadResult.PreviousVersion.ShouldBe(3);
|
||||
reloadResult.NewVersion.ShouldBe(4);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ReloadPipelines_WithErrors_ReturnsErrorDetails()
|
||||
{
|
||||
// Arrange
|
||||
SetupAuthenticatedUser("admin", isAdmin: true);
|
||||
var errors = new List<PipelineLoadError>
|
||||
{
|
||||
new PipelineLoadError
|
||||
{
|
||||
FileName = "pipeline.Bad.json",
|
||||
PipelineName = "Bad",
|
||||
ErrorType = "Validation",
|
||||
Messages = new List<string> { "Missing source", "Missing destination" }
|
||||
}
|
||||
};
|
||||
|
||||
_pipelineRegistry.ReloadAsync(Arg.Any<CancellationToken>())
|
||||
.Returns(new PipelineReloadResult
|
||||
{
|
||||
Success = false,
|
||||
PipelinesLoaded = 5,
|
||||
PipelinesSkipped = 1,
|
||||
PreviousVersion = 1,
|
||||
NewVersion = 1,
|
||||
Errors = errors
|
||||
});
|
||||
|
||||
// Act
|
||||
var result = await _controller.ReloadPipelines(CancellationToken.None);
|
||||
|
||||
// Assert
|
||||
result.Result.ShouldBeOfType<OkObjectResult>();
|
||||
var okResult = (OkObjectResult)result.Result!;
|
||||
var reloadResult = okResult.Value.ShouldBeOfType<PipelineReloadResultViewModel>();
|
||||
|
||||
reloadResult.Success.ShouldBeFalse();
|
||||
reloadResult.Errors.Count.ShouldBe(1);
|
||||
reloadResult.Errors[0].FileName.ShouldBe("pipeline.Bad.json");
|
||||
reloadResult.Errors[0].ErrorType.ShouldBe("Validation");
|
||||
reloadResult.Errors[0].Messages.ShouldContain("Missing source");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ReloadPipelines_WhenExceptionThrown_Returns500()
|
||||
{
|
||||
// Arrange
|
||||
SetupAuthenticatedUser("admin", isAdmin: true);
|
||||
_pipelineRegistry.ReloadAsync(Arg.Any<CancellationToken>())
|
||||
.Throws(new Exception("Unexpected error"));
|
||||
|
||||
// Act
|
||||
var result = await _controller.ReloadPipelines(CancellationToken.None);
|
||||
|
||||
// Assert
|
||||
result.Result.ShouldBeOfType<ObjectResult>();
|
||||
var objectResult = (ObjectResult)result.Result!;
|
||||
objectResult.StatusCode.ShouldBe(StatusCodes.Status500InternalServerError);
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Helper Methods
|
||||
|
||||
private void SetupAuthenticatedUser(string username, bool isAdmin = false)
|
||||
{
|
||||
var claims = new List<Claim>
|
||||
{
|
||||
new(ClaimTypes.Name, username)
|
||||
};
|
||||
|
||||
if (isAdmin)
|
||||
{
|
||||
claims.Add(new Claim(ClaimTypes.Role, "Admin"));
|
||||
}
|
||||
|
||||
var identity = new ClaimsIdentity(claims, "Test");
|
||||
var principal = new ClaimsPrincipal(identity);
|
||||
|
||||
var httpContext = new DefaultHttpContext { User = principal };
|
||||
_controller.ControllerContext = new ControllerContext { HttpContext = httpContext };
|
||||
}
|
||||
|
||||
private static EtlPipelineConfig CreatePipeline(
|
||||
string name,
|
||||
bool isEnabled = true,
|
||||
int? massSyncInterval = null,
|
||||
int? dailySyncInterval = null,
|
||||
int? hourlySyncInterval = null)
|
||||
{
|
||||
return new EtlPipelineConfig
|
||||
{
|
||||
Name = name,
|
||||
IsEnabled = isEnabled,
|
||||
MassSyncIntervalMinutes = massSyncInterval,
|
||||
DailySyncIntervalMinutes = dailySyncInterval,
|
||||
HourlySyncIntervalMinutes = hourlySyncInterval
|
||||
};
|
||||
}
|
||||
|
||||
private static PipelineReloadResult CreateSuccessfulReloadResult(
|
||||
int pipelinesLoaded,
|
||||
int pipelinesSkipped,
|
||||
int previousVersion = 1,
|
||||
int newVersion = 2)
|
||||
{
|
||||
return new PipelineReloadResult
|
||||
{
|
||||
Success = true,
|
||||
PipelinesLoaded = pipelinesLoaded,
|
||||
PipelinesSkipped = pipelinesSkipped,
|
||||
PreviousVersion = previousVersion,
|
||||
NewVersion = newVersion,
|
||||
Errors = new List<PipelineLoadError>()
|
||||
};
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
Reference in New Issue
Block a user