feat: implement ETL pipeline redesign and ConfigManager improvements

- Add pipeline registry with JSON-based configuration and hot-reload support
- Implement manual sync request feature with API, client UI, and database
- Improve ConfigManager: connection string dropdown in pipeline editor,
  step delete/reorder functionality, and fix JSON parsing for ConnectionStrings
This commit is contained in:
Joseph Doherty
2026-01-22 17:48:33 -05:00
parent 5a332232d0
commit 29ac56006d
82 changed files with 6257 additions and 296 deletions
@@ -0,0 +1,402 @@
using System.Security.Claims;
using JdeScoping.Api.Contracts.ManualSync;
using JdeScoping.Api.Controllers;
using JdeScoping.DataAccess.Services;
using JdeScoping.DataSync.Configuration;
using JdeScoping.DataSync.Services;
using JdeScoping.Domain.Models;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Mvc;
using NSubstitute;
using Shouldly;
using Xunit;
namespace JdeScoping.Api.Tests.Controllers;
public class ManualSyncControllerTests
{
private readonly IManualSyncRequestService _manualSyncRequestService;
private readonly IPipelineRegistry _pipelineRegistry;
private readonly ManualSyncController _controller;
public ManualSyncControllerTests()
{
_manualSyncRequestService = Substitute.For<IManualSyncRequestService>();
_pipelineRegistry = Substitute.For<IPipelineRegistry>();
_controller = new ManualSyncController(_manualSyncRequestService, _pipelineRegistry);
SetupAuthenticatedUser("testuser");
}
#region GetRequests Tests
[Fact]
public async Task GetRequests_ReturnsOkWithRequests()
{
// Arrange
var requests = new List<ManualSyncRequest>
{
CreateRequest(1, "Pipeline1", "mass", "user1", DateTime.UtcNow.AddHours(-2)),
CreateRequest(2, "Pipeline2", "daily", "user2", DateTime.UtcNow.AddHours(-1))
};
_manualSyncRequestService.GetRequestsAsync(false, Arg.Any<CancellationToken>())
.Returns(requests);
// Act
var result = await _controller.GetRequests(false, CancellationToken.None);
// Assert
result.Result.ShouldBeOfType<OkObjectResult>();
var okResult = (OkObjectResult)result.Result!;
var viewModels = okResult.Value.ShouldBeAssignableTo<List<ManualSyncRequestViewModel>>()!;
viewModels.Count.ShouldBe(2);
viewModels[0].Id.ShouldBe(1);
viewModels[0].PipelineName.ShouldBe("Pipeline1");
viewModels[1].Id.ShouldBe(2);
viewModels[1].PipelineName.ShouldBe("Pipeline2");
}
[Fact]
public async Task GetRequests_WithPendingOnlyTrue_PassesPendingOnlyToService()
{
// Arrange
_manualSyncRequestService.GetRequestsAsync(true, Arg.Any<CancellationToken>())
.Returns(new List<ManualSyncRequest>());
// Act
await _controller.GetRequests(true, CancellationToken.None);
// Assert
await _manualSyncRequestService.Received(1)
.GetRequestsAsync(true, Arg.Any<CancellationToken>());
}
[Fact]
public async Task GetRequests_MapsRowVersionToBase64()
{
// Arrange
var rowVersion = new byte[] { 1, 2, 3, 4, 5, 6, 7, 8 };
var request = CreateRequest(1, "Pipeline1", "mass", "user1", DateTime.UtcNow, rowVersion: rowVersion);
_manualSyncRequestService.GetRequestsAsync(false, Arg.Any<CancellationToken>())
.Returns(new List<ManualSyncRequest> { request });
// Act
var result = await _controller.GetRequests(false, CancellationToken.None);
// Assert
var okResult = (OkObjectResult)result.Result!;
var viewModels = okResult.Value.ShouldBeAssignableTo<List<ManualSyncRequestViewModel>>()!;
viewModels[0].RowVersionBase64.ShouldBe(Convert.ToBase64String(rowVersion));
}
#endregion
#region GetPipelines Tests
[Fact]
public void GetPipelines_ReturnsOkWithPipelines()
{
// Arrange
var pipelines = new List<EtlPipelineConfig>
{
CreatePipeline("WorkOrders", massSyncInterval: 1440, dailySyncInterval: 60, hourlySyncInterval: 15),
CreatePipeline("Items", massSyncInterval: 1440, dailySyncInterval: 60)
};
_pipelineRegistry.GetEnabledPipelines().Returns(pipelines);
// Act
var result = _controller.GetPipelines();
// Assert
result.Result.ShouldBeOfType<OkObjectResult>();
var okResult = (OkObjectResult)result.Result!;
var viewModels = okResult.Value.ShouldBeAssignableTo<List<PipelineInfoViewModel>>()!;
viewModels.Count.ShouldBe(2);
viewModels[0].Name.ShouldBe("WorkOrders");
viewModels[0].SupportedSyncTypes.ShouldContain("mass");
viewModels[0].SupportedSyncTypes.ShouldContain("daily");
viewModels[0].SupportedSyncTypes.ShouldContain("hourly");
viewModels[1].Name.ShouldBe("Items");
}
[Fact]
public void GetPipelines_WhenEmpty_ReturnsEmptyList()
{
// Arrange
_pipelineRegistry.GetEnabledPipelines().Returns(new List<EtlPipelineConfig>());
// Act
var result = _controller.GetPipelines();
// Assert
var okResult = (OkObjectResult)result.Result!;
var viewModels = okResult.Value.ShouldBeAssignableTo<List<PipelineInfoViewModel>>()!;
viewModels.ShouldBeEmpty();
}
#endregion
#region CreateRequest Tests
[Fact]
public async Task CreateRequest_WithValidInput_ReturnsCreated()
{
// Arrange
var dto = new CreateManualSyncRequestDto
{
PipelineName = "WorkOrders",
SyncType = "mass"
};
var createdRequest = CreateRequest(42, "WorkOrders", "mass", "testuser", DateTime.UtcNow);
_pipelineRegistry.IsValidPipelineAndSyncType("WorkOrders", "mass").Returns(true);
_manualSyncRequestService.CreateRequestAsync("WorkOrders", "mass", "testuser", Arg.Any<CancellationToken>())
.Returns(createdRequest);
// Act
var result = await _controller.CreateRequest(dto, CancellationToken.None);
// Assert
result.Result.ShouldBeOfType<CreatedAtActionResult>();
var createdResult = (CreatedAtActionResult)result.Result!;
var viewModel = createdResult.Value.ShouldBeOfType<ManualSyncRequestViewModel>();
viewModel.Id.ShouldBe(42);
viewModel.PipelineName.ShouldBe("WorkOrders");
viewModel.SyncType.ShouldBe("mass");
viewModel.RequestedBy.ShouldBe("testuser");
}
[Fact]
public async Task CreateRequest_WithInvalidPipelineOrSyncType_ReturnsBadRequest()
{
// Arrange
var dto = new CreateManualSyncRequestDto
{
PipelineName = "InvalidPipeline",
SyncType = "invalid"
};
_pipelineRegistry.IsValidPipelineAndSyncType("InvalidPipeline", "invalid").Returns(false);
// Act
var result = await _controller.CreateRequest(dto, CancellationToken.None);
// Assert
result.Result.ShouldBeOfType<BadRequestObjectResult>();
var badRequestResult = (BadRequestObjectResult)result.Result!;
badRequestResult.Value.ShouldBeOfType<string>();
((string)badRequestResult.Value!).ShouldContain("Invalid pipeline/sync type combination");
}
[Fact]
public async Task CreateRequest_WhenUnauthenticated_ReturnsUnauthorized()
{
// Arrange
SetupUnauthenticatedUser();
var dto = new CreateManualSyncRequestDto
{
PipelineName = "WorkOrders",
SyncType = "mass"
};
// Act
var result = await _controller.CreateRequest(dto, CancellationToken.None);
// Assert
result.Result.ShouldBeOfType<UnauthorizedResult>();
}
[Fact]
public async Task CreateRequest_PassesCorrectUsernameToService()
{
// Arrange
var dto = new CreateManualSyncRequestDto
{
PipelineName = "WorkOrders",
SyncType = "daily"
};
var createdRequest = CreateRequest(1, "WorkOrders", "daily", "testuser", DateTime.UtcNow);
_pipelineRegistry.IsValidPipelineAndSyncType("WorkOrders", "daily").Returns(true);
_manualSyncRequestService.CreateRequestAsync(Arg.Any<string>(), Arg.Any<string>(), Arg.Any<string>(), Arg.Any<CancellationToken>())
.Returns(createdRequest);
// Act
await _controller.CreateRequest(dto, CancellationToken.None);
// Assert
await _manualSyncRequestService.Received(1)
.CreateRequestAsync("WorkOrders", "daily", "testuser", Arg.Any<CancellationToken>());
}
#endregion
#region CancelRequest Tests
[Fact]
public async Task CancelRequest_WhenSuccessful_ReturnsOk()
{
// Arrange
var rowVersion = new byte[] { 1, 2, 3, 4, 5, 6, 7, 8 };
var dto = new CancelManualSyncRequestDto
{
RowVersionBase64 = Convert.ToBase64String(rowVersion)
};
_manualSyncRequestService.CancelRequestAsync(1, "testuser", Arg.Any<byte[]>(), Arg.Any<CancellationToken>())
.Returns(true);
// Act
var result = await _controller.CancelRequest(1, dto, CancellationToken.None);
// Assert
result.ShouldBeOfType<OkObjectResult>();
}
[Fact]
public async Task CancelRequest_WhenConcurrencyFails_ReturnsConflict()
{
// Arrange
var rowVersion = new byte[] { 1, 2, 3, 4, 5, 6, 7, 8 };
var dto = new CancelManualSyncRequestDto
{
RowVersionBase64 = Convert.ToBase64String(rowVersion)
};
_manualSyncRequestService.CancelRequestAsync(1, "testuser", Arg.Any<byte[]>(), Arg.Any<CancellationToken>())
.Returns(false);
// Act
var result = await _controller.CancelRequest(1, dto, CancellationToken.None);
// Assert
result.ShouldBeOfType<ConflictObjectResult>();
}
[Fact]
public async Task CancelRequest_WithInvalidBase64_ReturnsBadRequest()
{
// Arrange
var dto = new CancelManualSyncRequestDto
{
RowVersionBase64 = "not-valid-base64!!!"
};
// Act
var result = await _controller.CancelRequest(1, dto, CancellationToken.None);
// Assert
result.ShouldBeOfType<BadRequestObjectResult>();
var badRequest = (BadRequestObjectResult)result;
badRequest.Value.ShouldBeOfType<string>();
((string)badRequest.Value!).ShouldContain("Invalid RowVersionBase64 format");
}
[Fact]
public async Task CancelRequest_WhenUnauthenticated_ReturnsUnauthorized()
{
// Arrange
SetupUnauthenticatedUser();
var dto = new CancelManualSyncRequestDto
{
RowVersionBase64 = Convert.ToBase64String(new byte[] { 1, 2, 3 })
};
// Act
var result = await _controller.CancelRequest(1, dto, CancellationToken.None);
// Assert
result.ShouldBeOfType<UnauthorizedResult>();
}
[Fact]
public async Task CancelRequest_PassesCorrectParametersToService()
{
// Arrange
var rowVersion = new byte[] { 10, 20, 30, 40 };
var dto = new CancelManualSyncRequestDto
{
RowVersionBase64 = Convert.ToBase64String(rowVersion)
};
_manualSyncRequestService.CancelRequestAsync(Arg.Any<int>(), Arg.Any<string>(), Arg.Any<byte[]>(), Arg.Any<CancellationToken>())
.Returns(true);
// Act
await _controller.CancelRequest(99, dto, CancellationToken.None);
// Assert
await _manualSyncRequestService.Received(1)
.CancelRequestAsync(
99,
"testuser",
Arg.Is<byte[]>(b => b.SequenceEqual(rowVersion)),
Arg.Any<CancellationToken>());
}
#endregion
#region Helper Methods
private void SetupAuthenticatedUser(string username)
{
var claims = new List<Claim>
{
new(ClaimTypes.Name, username),
new("dn", $"CN={username},DC=example,DC=com")
};
var identity = new ClaimsIdentity(claims, "Test");
var principal = new ClaimsPrincipal(identity);
var httpContext = new DefaultHttpContext { User = principal };
_controller.ControllerContext = new ControllerContext { HttpContext = httpContext };
}
private void SetupUnauthenticatedUser()
{
var identity = new ClaimsIdentity(); // No claims, not authenticated
var principal = new ClaimsPrincipal(identity);
var httpContext = new DefaultHttpContext { User = principal };
_controller.ControllerContext = new ControllerContext { HttpContext = httpContext };
}
private static ManualSyncRequest CreateRequest(
int id,
string pipelineName,
string syncType,
string requestedBy,
DateTime requestDT,
DateTime? completedDT = null,
DateTime? cancelDT = null,
string? cancelledBy = null,
byte[]? rowVersion = null)
{
return new ManualSyncRequest
{
Id = id,
PipelineName = pipelineName,
SyncType = syncType,
RequestedBy = requestedBy,
RequestDT = requestDT,
CompletedDT = completedDT,
CancelDT = cancelDT,
CancelledBy = cancelledBy,
RowVersion = rowVersion ?? new byte[] { 0, 0, 0, 0, 0, 0, 0, 1 }
};
}
private static EtlPipelineConfig CreatePipeline(
string name,
int? massSyncInterval = null,
int? dailySyncInterval = null,
int? hourlySyncInterval = null,
bool isEnabled = true)
{
return new EtlPipelineConfig
{
Name = name,
IsEnabled = isEnabled,
MassSyncIntervalMinutes = massSyncInterval,
DailySyncIntervalMinutes = dailySyncInterval,
HourlySyncIntervalMinutes = hourlySyncInterval
};
}
#endregion
}
@@ -0,0 +1,289 @@
using System.Security.Claims;
using JdeScoping.Api.Controllers;
using JdeScoping.DataSync.Configuration;
using JdeScoping.DataSync.Services;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Logging;
using NSubstitute;
using NSubstitute.ExceptionExtensions;
using Shouldly;
using Xunit;
namespace JdeScoping.Api.Tests.Controllers;
public class PipelineControllerTests
{
private readonly IPipelineRegistry _pipelineRegistry;
private readonly ILogger<PipelineController> _logger;
private readonly PipelineController _controller;
public PipelineControllerTests()
{
_pipelineRegistry = Substitute.For<IPipelineRegistry>();
_logger = Substitute.For<ILogger<PipelineController>>();
_controller = new PipelineController(_pipelineRegistry, _logger);
SetupAuthenticatedUser("testuser", isAdmin: false);
}
#region GetPipelines Tests
[Fact]
public void GetPipelines_ReturnsEnabledPipelines()
{
// Arrange
var pipelines = new List<EtlPipelineConfig>
{
CreatePipeline("WorkOrders", massSyncInterval: 1440, dailySyncInterval: 60, hourlySyncInterval: 15),
CreatePipeline("Items", massSyncInterval: 1440, dailySyncInterval: 60)
};
_pipelineRegistry.GetEnabledPipelines().Returns(pipelines);
// Act
var result = _controller.GetPipelines();
// Assert
result.Result.ShouldBeOfType<OkObjectResult>();
var okResult = (OkObjectResult)result.Result!;
var viewModels = okResult.Value.ShouldBeAssignableTo<List<Api.Contracts.ManualSync.PipelineInfoViewModel>>()!;
viewModels.Count.ShouldBe(2);
viewModels[0].Name.ShouldBe("WorkOrders");
viewModels[0].SupportedSyncTypes.ShouldContain("mass");
viewModels[0].SupportedSyncTypes.ShouldContain("daily");
viewModels[0].SupportedSyncTypes.ShouldContain("hourly");
viewModels[1].Name.ShouldBe("Items");
viewModels[1].SupportedSyncTypes.ShouldNotContain("hourly");
}
[Fact]
public void GetPipelines_WhenEmpty_ReturnsEmptyList()
{
// Arrange
_pipelineRegistry.GetEnabledPipelines().Returns(new List<EtlPipelineConfig>());
// Act
var result = _controller.GetPipelines();
// Assert
var okResult = (OkObjectResult)result.Result!;
var viewModels = okResult.Value.ShouldBeAssignableTo<List<Api.Contracts.ManualSync.PipelineInfoViewModel>>()!;
viewModels.ShouldBeEmpty();
}
[Fact]
public void GetPipelines_MapsOnlySupportedSyncTypes()
{
// Arrange - pipeline with only mass sync
var pipelines = new List<EtlPipelineConfig>
{
CreatePipeline("MassOnly", massSyncInterval: 1440)
};
_pipelineRegistry.GetEnabledPipelines().Returns(pipelines);
// Act
var result = _controller.GetPipelines();
// Assert
var okResult = (OkObjectResult)result.Result!;
var viewModels = okResult.Value.ShouldBeAssignableTo<List<Api.Contracts.ManualSync.PipelineInfoViewModel>>()!;
viewModels[0].SupportedSyncTypes.Count.ShouldBe(1);
viewModels[0].SupportedSyncTypes.ShouldContain("mass");
}
#endregion
#region GetStatus Tests
[Fact]
public void GetStatus_ReturnsRegistryMetadata()
{
// Arrange
var allPipelines = new List<EtlPipelineConfig>
{
CreatePipeline("Pipeline1", isEnabled: true),
CreatePipeline("Pipeline2", isEnabled: true),
CreatePipeline("Pipeline3", isEnabled: false)
};
var enabledPipelines = allPipelines.Where(p => p.IsEnabled).ToList();
_pipelineRegistry.GetAllPipelines().Returns(allPipelines);
_pipelineRegistry.GetEnabledPipelines().Returns(enabledPipelines);
_pipelineRegistry.Version.Returns(5);
_pipelineRegistry.LastLoadedAt.Returns(new DateTime(2024, 1, 15, 10, 30, 0, DateTimeKind.Utc));
// Act
var result = _controller.GetStatus();
// Assert
result.Result.ShouldBeOfType<OkObjectResult>();
var okResult = (OkObjectResult)result.Result!;
var status = okResult.Value.ShouldBeOfType<PipelineRegistryStatusViewModel>();
status.Version.ShouldBe(5);
status.LastLoadedAt.ShouldBe(new DateTime(2024, 1, 15, 10, 30, 0, DateTimeKind.Utc));
status.TotalPipelines.ShouldBe(3);
status.EnabledPipelines.ShouldBe(2);
}
#endregion
#region ReloadPipelines Tests
[Fact]
public async Task ReloadPipelines_CallsRegistry()
{
// Arrange
SetupAuthenticatedUser("admin", isAdmin: true);
_pipelineRegistry.ReloadAsync(Arg.Any<CancellationToken>())
.Returns(CreateSuccessfulReloadResult(5, 0));
// Act
await _controller.ReloadPipelines(CancellationToken.None);
// Assert
await _pipelineRegistry.Received(1).ReloadAsync(Arg.Any<CancellationToken>());
}
[Fact]
public async Task ReloadPipelines_ReturnsResult()
{
// Arrange
SetupAuthenticatedUser("admin", isAdmin: true);
_pipelineRegistry.ReloadAsync(Arg.Any<CancellationToken>())
.Returns(CreateSuccessfulReloadResult(pipelinesLoaded: 10, pipelinesSkipped: 2, previousVersion: 3, newVersion: 4));
// Act
var result = await _controller.ReloadPipelines(CancellationToken.None);
// Assert
result.Result.ShouldBeOfType<OkObjectResult>();
var okResult = (OkObjectResult)result.Result!;
var reloadResult = okResult.Value.ShouldBeOfType<PipelineReloadResultViewModel>();
reloadResult.Success.ShouldBeTrue();
reloadResult.PipelinesLoaded.ShouldBe(10);
reloadResult.PipelinesSkipped.ShouldBe(2);
reloadResult.PreviousVersion.ShouldBe(3);
reloadResult.NewVersion.ShouldBe(4);
}
[Fact]
public async Task ReloadPipelines_WithErrors_ReturnsErrorDetails()
{
// Arrange
SetupAuthenticatedUser("admin", isAdmin: true);
var errors = new List<PipelineLoadError>
{
new PipelineLoadError
{
FileName = "pipeline.Bad.json",
PipelineName = "Bad",
ErrorType = "Validation",
Messages = new List<string> { "Missing source", "Missing destination" }
}
};
_pipelineRegistry.ReloadAsync(Arg.Any<CancellationToken>())
.Returns(new PipelineReloadResult
{
Success = false,
PipelinesLoaded = 5,
PipelinesSkipped = 1,
PreviousVersion = 1,
NewVersion = 1,
Errors = errors
});
// Act
var result = await _controller.ReloadPipelines(CancellationToken.None);
// Assert
result.Result.ShouldBeOfType<OkObjectResult>();
var okResult = (OkObjectResult)result.Result!;
var reloadResult = okResult.Value.ShouldBeOfType<PipelineReloadResultViewModel>();
reloadResult.Success.ShouldBeFalse();
reloadResult.Errors.Count.ShouldBe(1);
reloadResult.Errors[0].FileName.ShouldBe("pipeline.Bad.json");
reloadResult.Errors[0].ErrorType.ShouldBe("Validation");
reloadResult.Errors[0].Messages.ShouldContain("Missing source");
}
[Fact]
public async Task ReloadPipelines_WhenExceptionThrown_Returns500()
{
// Arrange
SetupAuthenticatedUser("admin", isAdmin: true);
_pipelineRegistry.ReloadAsync(Arg.Any<CancellationToken>())
.Throws(new Exception("Unexpected error"));
// Act
var result = await _controller.ReloadPipelines(CancellationToken.None);
// Assert
result.Result.ShouldBeOfType<ObjectResult>();
var objectResult = (ObjectResult)result.Result!;
objectResult.StatusCode.ShouldBe(StatusCodes.Status500InternalServerError);
}
#endregion
#region Helper Methods
private void SetupAuthenticatedUser(string username, bool isAdmin = false)
{
var claims = new List<Claim>
{
new(ClaimTypes.Name, username)
};
if (isAdmin)
{
claims.Add(new Claim(ClaimTypes.Role, "Admin"));
}
var identity = new ClaimsIdentity(claims, "Test");
var principal = new ClaimsPrincipal(identity);
var httpContext = new DefaultHttpContext { User = principal };
_controller.ControllerContext = new ControllerContext { HttpContext = httpContext };
}
private static EtlPipelineConfig CreatePipeline(
string name,
bool isEnabled = true,
int? massSyncInterval = null,
int? dailySyncInterval = null,
int? hourlySyncInterval = null)
{
return new EtlPipelineConfig
{
Name = name,
IsEnabled = isEnabled,
MassSyncIntervalMinutes = massSyncInterval,
DailySyncIntervalMinutes = dailySyncInterval,
HourlySyncIntervalMinutes = hourlySyncInterval
};
}
private static PipelineReloadResult CreateSuccessfulReloadResult(
int pipelinesLoaded,
int pipelinesSkipped,
int previousVersion = 1,
int newVersion = 2)
{
return new PipelineReloadResult
{
Success = true,
PipelinesLoaded = pipelinesLoaded,
PipelinesSkipped = pipelinesSkipped,
PreviousVersion = previousVersion,
NewVersion = newVersion,
Errors = new List<PipelineLoadError>()
};
}
#endregion
}