feat: implement ETL pipeline redesign and ConfigManager improvements
- Add pipeline registry with JSON-based configuration and hot-reload support - Implement manual sync request feature with API, client UI, and database - Improve ConfigManager: connection string dropdown in pipeline editor, step delete/reorder functionality, and fix JSON parsing for ConnectionStrings
This commit is contained in:
Joseph Doherty
@@ -0,0 +1,402 @@
|
||||
using System.Security.Claims;
|
||||
using JdeScoping.Api.Contracts.ManualSync;
|
||||
using JdeScoping.Api.Controllers;
|
||||
using JdeScoping.DataAccess.Services;
|
||||
using JdeScoping.DataSync.Configuration;
|
||||
using JdeScoping.DataSync.Services;
|
||||
using JdeScoping.Domain.Models;
|
||||
using Microsoft.AspNetCore.Http;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using NSubstitute;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
|
||||
namespace JdeScoping.Api.Tests.Controllers;
|
||||
|
||||
public class ManualSyncControllerTests
|
||||
{
|
||||
private readonly IManualSyncRequestService _manualSyncRequestService;
|
||||
private readonly IPipelineRegistry _pipelineRegistry;
|
||||
private readonly ManualSyncController _controller;
|
||||
|
||||
public ManualSyncControllerTests()
|
||||
{
|
||||
_manualSyncRequestService = Substitute.For<IManualSyncRequestService>();
|
||||
_pipelineRegistry = Substitute.For<IPipelineRegistry>();
|
||||
_controller = new ManualSyncController(_manualSyncRequestService, _pipelineRegistry);
|
||||
SetupAuthenticatedUser("testuser");
|
||||
}
|
||||
|
||||
#region GetRequests Tests
|
||||
|
||||
[Fact]
|
||||
public async Task GetRequests_ReturnsOkWithRequests()
|
||||
{
|
||||
// Arrange
|
||||
var requests = new List<ManualSyncRequest>
|
||||
{
|
||||
CreateRequest(1, "Pipeline1", "mass", "user1", DateTime.UtcNow.AddHours(-2)),
|
||||
CreateRequest(2, "Pipeline2", "daily", "user2", DateTime.UtcNow.AddHours(-1))
|
||||
};
|
||||
_manualSyncRequestService.GetRequestsAsync(false, Arg.Any<CancellationToken>())
|
||||
.Returns(requests);
|
||||
|
||||
// Act
|
||||
var result = await _controller.GetRequests(false, CancellationToken.None);
|
||||
|
||||
// Assert
|
||||
result.Result.ShouldBeOfType<OkObjectResult>();
|
||||
var okResult = (OkObjectResult)result.Result!;
|
||||
var viewModels = okResult.Value.ShouldBeAssignableTo<List<ManualSyncRequestViewModel>>()!;
|
||||
viewModels.Count.ShouldBe(2);
|
||||
viewModels[0].Id.ShouldBe(1);
|
||||
viewModels[0].PipelineName.ShouldBe("Pipeline1");
|
||||
viewModels[1].Id.ShouldBe(2);
|
||||
viewModels[1].PipelineName.ShouldBe("Pipeline2");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetRequests_WithPendingOnlyTrue_PassesPendingOnlyToService()
|
||||
{
|
||||
// Arrange
|
||||
_manualSyncRequestService.GetRequestsAsync(true, Arg.Any<CancellationToken>())
|
||||
.Returns(new List<ManualSyncRequest>());
|
||||
|
||||
// Act
|
||||
await _controller.GetRequests(true, CancellationToken.None);
|
||||
|
||||
// Assert
|
||||
await _manualSyncRequestService.Received(1)
|
||||
.GetRequestsAsync(true, Arg.Any<CancellationToken>());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetRequests_MapsRowVersionToBase64()
|
||||
{
|
||||
// Arrange
|
||||
var rowVersion = new byte[] { 1, 2, 3, 4, 5, 6, 7, 8 };
|
||||
var request = CreateRequest(1, "Pipeline1", "mass", "user1", DateTime.UtcNow, rowVersion: rowVersion);
|
||||
_manualSyncRequestService.GetRequestsAsync(false, Arg.Any<CancellationToken>())
|
||||
.Returns(new List<ManualSyncRequest> { request });
|
||||
|
||||
// Act
|
||||
var result = await _controller.GetRequests(false, CancellationToken.None);
|
||||
|
||||
// Assert
|
||||
var okResult = (OkObjectResult)result.Result!;
|
||||
var viewModels = okResult.Value.ShouldBeAssignableTo<List<ManualSyncRequestViewModel>>()!;
|
||||
viewModels[0].RowVersionBase64.ShouldBe(Convert.ToBase64String(rowVersion));
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region GetPipelines Tests
|
||||
|
||||
[Fact]
|
||||
public void GetPipelines_ReturnsOkWithPipelines()
|
||||
{
|
||||
// Arrange
|
||||
var pipelines = new List<EtlPipelineConfig>
|
||||
{
|
||||
CreatePipeline("WorkOrders", massSyncInterval: 1440, dailySyncInterval: 60, hourlySyncInterval: 15),
|
||||
CreatePipeline("Items", massSyncInterval: 1440, dailySyncInterval: 60)
|
||||
};
|
||||
_pipelineRegistry.GetEnabledPipelines().Returns(pipelines);
|
||||
|
||||
// Act
|
||||
var result = _controller.GetPipelines();
|
||||
|
||||
// Assert
|
||||
result.Result.ShouldBeOfType<OkObjectResult>();
|
||||
var okResult = (OkObjectResult)result.Result!;
|
||||
var viewModels = okResult.Value.ShouldBeAssignableTo<List<PipelineInfoViewModel>>()!;
|
||||
viewModels.Count.ShouldBe(2);
|
||||
viewModels[0].Name.ShouldBe("WorkOrders");
|
||||
viewModels[0].SupportedSyncTypes.ShouldContain("mass");
|
||||
viewModels[0].SupportedSyncTypes.ShouldContain("daily");
|
||||
viewModels[0].SupportedSyncTypes.ShouldContain("hourly");
|
||||
viewModels[1].Name.ShouldBe("Items");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void GetPipelines_WhenEmpty_ReturnsEmptyList()
|
||||
{
|
||||
// Arrange
|
||||
_pipelineRegistry.GetEnabledPipelines().Returns(new List<EtlPipelineConfig>());
|
||||
|
||||
// Act
|
||||
var result = _controller.GetPipelines();
|
||||
|
||||
// Assert
|
||||
var okResult = (OkObjectResult)result.Result!;
|
||||
var viewModels = okResult.Value.ShouldBeAssignableTo<List<PipelineInfoViewModel>>()!;
|
||||
viewModels.ShouldBeEmpty();
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region CreateRequest Tests
|
||||
|
||||
[Fact]
|
||||
public async Task CreateRequest_WithValidInput_ReturnsCreated()
|
||||
{
|
||||
// Arrange
|
||||
var dto = new CreateManualSyncRequestDto
|
||||
{
|
||||
PipelineName = "WorkOrders",
|
||||
SyncType = "mass"
|
||||
};
|
||||
var createdRequest = CreateRequest(42, "WorkOrders", "mass", "testuser", DateTime.UtcNow);
|
||||
|
||||
_pipelineRegistry.IsValidPipelineAndSyncType("WorkOrders", "mass").Returns(true);
|
||||
_manualSyncRequestService.CreateRequestAsync("WorkOrders", "mass", "testuser", Arg.Any<CancellationToken>())
|
||||
.Returns(createdRequest);
|
||||
|
||||
// Act
|
||||
var result = await _controller.CreateRequest(dto, CancellationToken.None);
|
||||
|
||||
// Assert
|
||||
result.Result.ShouldBeOfType<CreatedAtActionResult>();
|
||||
var createdResult = (CreatedAtActionResult)result.Result!;
|
||||
var viewModel = createdResult.Value.ShouldBeOfType<ManualSyncRequestViewModel>();
|
||||
viewModel.Id.ShouldBe(42);
|
||||
viewModel.PipelineName.ShouldBe("WorkOrders");
|
||||
viewModel.SyncType.ShouldBe("mass");
|
||||
viewModel.RequestedBy.ShouldBe("testuser");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task CreateRequest_WithInvalidPipelineOrSyncType_ReturnsBadRequest()
|
||||
{
|
||||
// Arrange
|
||||
var dto = new CreateManualSyncRequestDto
|
||||
{
|
||||
PipelineName = "InvalidPipeline",
|
||||
SyncType = "invalid"
|
||||
};
|
||||
_pipelineRegistry.IsValidPipelineAndSyncType("InvalidPipeline", "invalid").Returns(false);
|
||||
|
||||
// Act
|
||||
var result = await _controller.CreateRequest(dto, CancellationToken.None);
|
||||
|
||||
// Assert
|
||||
result.Result.ShouldBeOfType<BadRequestObjectResult>();
|
||||
var badRequestResult = (BadRequestObjectResult)result.Result!;
|
||||
badRequestResult.Value.ShouldBeOfType<string>();
|
||||
((string)badRequestResult.Value!).ShouldContain("Invalid pipeline/sync type combination");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task CreateRequest_WhenUnauthenticated_ReturnsUnauthorized()
|
||||
{
|
||||
// Arrange
|
||||
SetupUnauthenticatedUser();
|
||||
var dto = new CreateManualSyncRequestDto
|
||||
{
|
||||
PipelineName = "WorkOrders",
|
||||
SyncType = "mass"
|
||||
};
|
||||
|
||||
// Act
|
||||
var result = await _controller.CreateRequest(dto, CancellationToken.None);
|
||||
|
||||
// Assert
|
||||
result.Result.ShouldBeOfType<UnauthorizedResult>();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task CreateRequest_PassesCorrectUsernameToService()
|
||||
{
|
||||
// Arrange
|
||||
var dto = new CreateManualSyncRequestDto
|
||||
{
|
||||
PipelineName = "WorkOrders",
|
||||
SyncType = "daily"
|
||||
};
|
||||
var createdRequest = CreateRequest(1, "WorkOrders", "daily", "testuser", DateTime.UtcNow);
|
||||
|
||||
_pipelineRegistry.IsValidPipelineAndSyncType("WorkOrders", "daily").Returns(true);
|
||||
_manualSyncRequestService.CreateRequestAsync(Arg.Any<string>(), Arg.Any<string>(), Arg.Any<string>(), Arg.Any<CancellationToken>())
|
||||
.Returns(createdRequest);
|
||||
|
||||
// Act
|
||||
await _controller.CreateRequest(dto, CancellationToken.None);
|
||||
|
||||
// Assert
|
||||
await _manualSyncRequestService.Received(1)
|
||||
.CreateRequestAsync("WorkOrders", "daily", "testuser", Arg.Any<CancellationToken>());
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region CancelRequest Tests
|
||||
|
||||
[Fact]
|
||||
public async Task CancelRequest_WhenSuccessful_ReturnsOk()
|
||||
{
|
||||
// Arrange
|
||||
var rowVersion = new byte[] { 1, 2, 3, 4, 5, 6, 7, 8 };
|
||||
var dto = new CancelManualSyncRequestDto
|
||||
{
|
||||
RowVersionBase64 = Convert.ToBase64String(rowVersion)
|
||||
};
|
||||
_manualSyncRequestService.CancelRequestAsync(1, "testuser", Arg.Any<byte[]>(), Arg.Any<CancellationToken>())
|
||||
.Returns(true);
|
||||
|
||||
// Act
|
||||
var result = await _controller.CancelRequest(1, dto, CancellationToken.None);
|
||||
|
||||
// Assert
|
||||
result.ShouldBeOfType<OkObjectResult>();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task CancelRequest_WhenConcurrencyFails_ReturnsConflict()
|
||||
{
|
||||
// Arrange
|
||||
var rowVersion = new byte[] { 1, 2, 3, 4, 5, 6, 7, 8 };
|
||||
var dto = new CancelManualSyncRequestDto
|
||||
{
|
||||
RowVersionBase64 = Convert.ToBase64String(rowVersion)
|
||||
};
|
||||
_manualSyncRequestService.CancelRequestAsync(1, "testuser", Arg.Any<byte[]>(), Arg.Any<CancellationToken>())
|
||||
.Returns(false);
|
||||
|
||||
// Act
|
||||
var result = await _controller.CancelRequest(1, dto, CancellationToken.None);
|
||||
|
||||
// Assert
|
||||
result.ShouldBeOfType<ConflictObjectResult>();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task CancelRequest_WithInvalidBase64_ReturnsBadRequest()
|
||||
{
|
||||
// Arrange
|
||||
var dto = new CancelManualSyncRequestDto
|
||||
{
|
||||
RowVersionBase64 = "not-valid-base64!!!"
|
||||
};
|
||||
|
||||
// Act
|
||||
var result = await _controller.CancelRequest(1, dto, CancellationToken.None);
|
||||
|
||||
// Assert
|
||||
result.ShouldBeOfType<BadRequestObjectResult>();
|
||||
var badRequest = (BadRequestObjectResult)result;
|
||||
badRequest.Value.ShouldBeOfType<string>();
|
||||
((string)badRequest.Value!).ShouldContain("Invalid RowVersionBase64 format");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task CancelRequest_WhenUnauthenticated_ReturnsUnauthorized()
|
||||
{
|
||||
// Arrange
|
||||
SetupUnauthenticatedUser();
|
||||
var dto = new CancelManualSyncRequestDto
|
||||
{
|
||||
RowVersionBase64 = Convert.ToBase64String(new byte[] { 1, 2, 3 })
|
||||
};
|
||||
|
||||
// Act
|
||||
var result = await _controller.CancelRequest(1, dto, CancellationToken.None);
|
||||
|
||||
// Assert
|
||||
result.ShouldBeOfType<UnauthorizedResult>();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task CancelRequest_PassesCorrectParametersToService()
|
||||
{
|
||||
// Arrange
|
||||
var rowVersion = new byte[] { 10, 20, 30, 40 };
|
||||
var dto = new CancelManualSyncRequestDto
|
||||
{
|
||||
RowVersionBase64 = Convert.ToBase64String(rowVersion)
|
||||
};
|
||||
_manualSyncRequestService.CancelRequestAsync(Arg.Any<int>(), Arg.Any<string>(), Arg.Any<byte[]>(), Arg.Any<CancellationToken>())
|
||||
.Returns(true);
|
||||
|
||||
// Act
|
||||
await _controller.CancelRequest(99, dto, CancellationToken.None);
|
||||
|
||||
// Assert
|
||||
await _manualSyncRequestService.Received(1)
|
||||
.CancelRequestAsync(
|
||||
99,
|
||||
"testuser",
|
||||
Arg.Is<byte[]>(b => b.SequenceEqual(rowVersion)),
|
||||
Arg.Any<CancellationToken>());
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Helper Methods
|
||||
|
||||
private void SetupAuthenticatedUser(string username)
|
||||
{
|
||||
var claims = new List<Claim>
|
||||
{
|
||||
new(ClaimTypes.Name, username),
|
||||
new("dn", $"CN={username},DC=example,DC=com")
|
||||
};
|
||||
var identity = new ClaimsIdentity(claims, "Test");
|
||||
var principal = new ClaimsPrincipal(identity);
|
||||
|
||||
var httpContext = new DefaultHttpContext { User = principal };
|
||||
_controller.ControllerContext = new ControllerContext { HttpContext = httpContext };
|
||||
}
|
||||
|
||||
private void SetupUnauthenticatedUser()
|
||||
{
|
||||
var identity = new ClaimsIdentity(); // No claims, not authenticated
|
||||
var principal = new ClaimsPrincipal(identity);
|
||||
|
||||
var httpContext = new DefaultHttpContext { User = principal };
|
||||
_controller.ControllerContext = new ControllerContext { HttpContext = httpContext };
|
||||
}
|
||||
|
||||
private static ManualSyncRequest CreateRequest(
|
||||
int id,
|
||||
string pipelineName,
|
||||
string syncType,
|
||||
string requestedBy,
|
||||
DateTime requestDT,
|
||||
DateTime? completedDT = null,
|
||||
DateTime? cancelDT = null,
|
||||
string? cancelledBy = null,
|
||||
byte[]? rowVersion = null)
|
||||
{
|
||||
return new ManualSyncRequest
|
||||
{
|
||||
Id = id,
|
||||
PipelineName = pipelineName,
|
||||
SyncType = syncType,
|
||||
RequestedBy = requestedBy,
|
||||
RequestDT = requestDT,
|
||||
CompletedDT = completedDT,
|
||||
CancelDT = cancelDT,
|
||||
CancelledBy = cancelledBy,
|
||||
RowVersion = rowVersion ?? new byte[] { 0, 0, 0, 0, 0, 0, 0, 1 }
|
||||
};
|
||||
}
|
||||
|
||||
private static EtlPipelineConfig CreatePipeline(
|
||||
string name,
|
||||
int? massSyncInterval = null,
|
||||
int? dailySyncInterval = null,
|
||||
int? hourlySyncInterval = null,
|
||||
bool isEnabled = true)
|
||||
{
|
||||
return new EtlPipelineConfig
|
||||
{
|
||||
Name = name,
|
||||
IsEnabled = isEnabled,
|
||||
MassSyncIntervalMinutes = massSyncInterval,
|
||||
DailySyncIntervalMinutes = dailySyncInterval,
|
||||
HourlySyncIntervalMinutes = hourlySyncInterval
|
||||
};
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
Reference in New Issue
Block a user