Files
Joseph Doherty 29ac56006d feat: implement ETL pipeline redesign and ConfigManager improvements
- Add pipeline registry with JSON-based configuration and hot-reload support
- Implement manual sync request feature with API, client UI, and database
- Improve ConfigManager: connection string dropdown in pipeline editor,
  step delete/reorder functionality, and fix JSON parsing for ConnectionStrings
2026-01-22 17:48:33 -05:00

290 lines
9.8 KiB
C#

using System.Security.Claims;
using JdeScoping.Api.Controllers;
using JdeScoping.DataSync.Configuration;
using JdeScoping.DataSync.Services;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Logging;
using NSubstitute;
using NSubstitute.ExceptionExtensions;
using Shouldly;
using Xunit;
namespace JdeScoping.Api.Tests.Controllers;
public class PipelineControllerTests
{
private readonly IPipelineRegistry _pipelineRegistry;
private readonly ILogger<PipelineController> _logger;
private readonly PipelineController _controller;
public PipelineControllerTests()
{
_pipelineRegistry = Substitute.For<IPipelineRegistry>();
_logger = Substitute.For<ILogger<PipelineController>>();
_controller = new PipelineController(_pipelineRegistry, _logger);
SetupAuthenticatedUser("testuser", isAdmin: false);
}
#region GetPipelines Tests
[Fact]
public void GetPipelines_ReturnsEnabledPipelines()
{
// Arrange
var pipelines = new List<EtlPipelineConfig>
{
CreatePipeline("WorkOrders", massSyncInterval: 1440, dailySyncInterval: 60, hourlySyncInterval: 15),
CreatePipeline("Items", massSyncInterval: 1440, dailySyncInterval: 60)
};
_pipelineRegistry.GetEnabledPipelines().Returns(pipelines);
// Act
var result = _controller.GetPipelines();
// Assert
result.Result.ShouldBeOfType<OkObjectResult>();
var okResult = (OkObjectResult)result.Result!;
var viewModels = okResult.Value.ShouldBeAssignableTo<List<Api.Contracts.ManualSync.PipelineInfoViewModel>>()!;
viewModels.Count.ShouldBe(2);
viewModels[0].Name.ShouldBe("WorkOrders");
viewModels[0].SupportedSyncTypes.ShouldContain("mass");
viewModels[0].SupportedSyncTypes.ShouldContain("daily");
viewModels[0].SupportedSyncTypes.ShouldContain("hourly");
viewModels[1].Name.ShouldBe("Items");
viewModels[1].SupportedSyncTypes.ShouldNotContain("hourly");
}
[Fact]
public void GetPipelines_WhenEmpty_ReturnsEmptyList()
{
// Arrange
_pipelineRegistry.GetEnabledPipelines().Returns(new List<EtlPipelineConfig>());
// Act
var result = _controller.GetPipelines();
// Assert
var okResult = (OkObjectResult)result.Result!;
var viewModels = okResult.Value.ShouldBeAssignableTo<List<Api.Contracts.ManualSync.PipelineInfoViewModel>>()!;
viewModels.ShouldBeEmpty();
}
[Fact]
public void GetPipelines_MapsOnlySupportedSyncTypes()
{
// Arrange - pipeline with only mass sync
var pipelines = new List<EtlPipelineConfig>
{
CreatePipeline("MassOnly", massSyncInterval: 1440)
};
_pipelineRegistry.GetEnabledPipelines().Returns(pipelines);
// Act
var result = _controller.GetPipelines();
// Assert
var okResult = (OkObjectResult)result.Result!;
var viewModels = okResult.Value.ShouldBeAssignableTo<List<Api.Contracts.ManualSync.PipelineInfoViewModel>>()!;
viewModels[0].SupportedSyncTypes.Count.ShouldBe(1);
viewModels[0].SupportedSyncTypes.ShouldContain("mass");
}
#endregion
#region GetStatus Tests
[Fact]
public void GetStatus_ReturnsRegistryMetadata()
{
// Arrange
var allPipelines = new List<EtlPipelineConfig>
{
CreatePipeline("Pipeline1", isEnabled: true),
CreatePipeline("Pipeline2", isEnabled: true),
CreatePipeline("Pipeline3", isEnabled: false)
};
var enabledPipelines = allPipelines.Where(p => p.IsEnabled).ToList();
_pipelineRegistry.GetAllPipelines().Returns(allPipelines);
_pipelineRegistry.GetEnabledPipelines().Returns(enabledPipelines);
_pipelineRegistry.Version.Returns(5);
_pipelineRegistry.LastLoadedAt.Returns(new DateTime(2024, 1, 15, 10, 30, 0, DateTimeKind.Utc));
// Act
var result = _controller.GetStatus();
// Assert
result.Result.ShouldBeOfType<OkObjectResult>();
var okResult = (OkObjectResult)result.Result!;
var status = okResult.Value.ShouldBeOfType<PipelineRegistryStatusViewModel>();
status.Version.ShouldBe(5);
status.LastLoadedAt.ShouldBe(new DateTime(2024, 1, 15, 10, 30, 0, DateTimeKind.Utc));
status.TotalPipelines.ShouldBe(3);
status.EnabledPipelines.ShouldBe(2);
}
#endregion
#region ReloadPipelines Tests
[Fact]
public async Task ReloadPipelines_CallsRegistry()
{
// Arrange
SetupAuthenticatedUser("admin", isAdmin: true);
_pipelineRegistry.ReloadAsync(Arg.Any<CancellationToken>())
.Returns(CreateSuccessfulReloadResult(5, 0));
// Act
await _controller.ReloadPipelines(CancellationToken.None);
// Assert
await _pipelineRegistry.Received(1).ReloadAsync(Arg.Any<CancellationToken>());
}
[Fact]
public async Task ReloadPipelines_ReturnsResult()
{
// Arrange
SetupAuthenticatedUser("admin", isAdmin: true);
_pipelineRegistry.ReloadAsync(Arg.Any<CancellationToken>())
.Returns(CreateSuccessfulReloadResult(pipelinesLoaded: 10, pipelinesSkipped: 2, previousVersion: 3, newVersion: 4));
// Act
var result = await _controller.ReloadPipelines(CancellationToken.None);
// Assert
result.Result.ShouldBeOfType<OkObjectResult>();
var okResult = (OkObjectResult)result.Result!;
var reloadResult = okResult.Value.ShouldBeOfType<PipelineReloadResultViewModel>();
reloadResult.Success.ShouldBeTrue();
reloadResult.PipelinesLoaded.ShouldBe(10);
reloadResult.PipelinesSkipped.ShouldBe(2);
reloadResult.PreviousVersion.ShouldBe(3);
reloadResult.NewVersion.ShouldBe(4);
}
[Fact]
public async Task ReloadPipelines_WithErrors_ReturnsErrorDetails()
{
// Arrange
SetupAuthenticatedUser("admin", isAdmin: true);
var errors = new List<PipelineLoadError>
{
new PipelineLoadError
{
FileName = "pipeline.Bad.json",
PipelineName = "Bad",
ErrorType = "Validation",
Messages = new List<string> { "Missing source", "Missing destination" }
}
};
_pipelineRegistry.ReloadAsync(Arg.Any<CancellationToken>())
.Returns(new PipelineReloadResult
{
Success = false,
PipelinesLoaded = 5,
PipelinesSkipped = 1,
PreviousVersion = 1,
NewVersion = 1,
Errors = errors
});
// Act
var result = await _controller.ReloadPipelines(CancellationToken.None);
// Assert
result.Result.ShouldBeOfType<OkObjectResult>();
var okResult = (OkObjectResult)result.Result!;
var reloadResult = okResult.Value.ShouldBeOfType<PipelineReloadResultViewModel>();
reloadResult.Success.ShouldBeFalse();
reloadResult.Errors.Count.ShouldBe(1);
reloadResult.Errors[0].FileName.ShouldBe("pipeline.Bad.json");
reloadResult.Errors[0].ErrorType.ShouldBe("Validation");
reloadResult.Errors[0].Messages.ShouldContain("Missing source");
}
[Fact]
public async Task ReloadPipelines_WhenExceptionThrown_Returns500()
{
// Arrange
SetupAuthenticatedUser("admin", isAdmin: true);
_pipelineRegistry.ReloadAsync(Arg.Any<CancellationToken>())
.Throws(new Exception("Unexpected error"));
// Act
var result = await _controller.ReloadPipelines(CancellationToken.None);
// Assert
result.Result.ShouldBeOfType<ObjectResult>();
var objectResult = (ObjectResult)result.Result!;
objectResult.StatusCode.ShouldBe(StatusCodes.Status500InternalServerError);
}
#endregion
#region Helper Methods
private void SetupAuthenticatedUser(string username, bool isAdmin = false)
{
var claims = new List<Claim>
{
new(ClaimTypes.Name, username)
};
if (isAdmin)
{
claims.Add(new Claim(ClaimTypes.Role, "Admin"));
}
var identity = new ClaimsIdentity(claims, "Test");
var principal = new ClaimsPrincipal(identity);
var httpContext = new DefaultHttpContext { User = principal };
_controller.ControllerContext = new ControllerContext { HttpContext = httpContext };
}
private static EtlPipelineConfig CreatePipeline(
string name,
bool isEnabled = true,
int? massSyncInterval = null,
int? dailySyncInterval = null,
int? hourlySyncInterval = null)
{
return new EtlPipelineConfig
{
Name = name,
IsEnabled = isEnabled,
MassSyncIntervalMinutes = massSyncInterval,
DailySyncIntervalMinutes = dailySyncInterval,
HourlySyncIntervalMinutes = hourlySyncInterval
};
}
private static PipelineReloadResult CreateSuccessfulReloadResult(
int pipelinesLoaded,
int pipelinesSkipped,
int previousVersion = 1,
int newVersion = 2)
{
return new PipelineReloadResult
{
Success = true,
PipelinesLoaded = pipelinesLoaded,
PipelinesSkipped = pipelinesSkipped,
PreviousVersion = previousVersion,
NewVersion = newVersion,
Errors = new List<PipelineLoadError>()
};
}
#endregion
}