feat: implement ETL pipeline redesign and ConfigManager improvements

- Add pipeline registry with JSON-based configuration and hot-reload support - Implement manual sync request feature with API, client UI, and database - Improve ConfigManager: connection string dropdown in pipeline editor, step delete/reorder functionality, and fix JSON parsing for ConnectionStrings
2026-01-22 17:48:33 -05:00
parent 5a332232d0
commit 29ac56006d
82 changed files with 6257 additions and 296 deletions
@@ -0,0 +1,189 @@
+using JdeScoping.Api.Contracts.ManualSync;
+using JdeScoping.DataAccess.Services;
+using JdeScoping.DataSync.Services;
+using Microsoft.AspNetCore.Authorization;
+using Microsoft.AspNetCore.Http;
+using Microsoft.AspNetCore.Mvc;
+
+namespace JdeScoping.Api.Controllers;
+
+/// <summary>
+/// API endpoints for manual data sync request management.
+/// </summary>
+[Route("api/manual-sync")]
+[ApiController]
+[Authorize]
+public class ManualSyncController : ApiControllerBase
+{
+    private readonly IManualSyncRequestService _manualSyncRequestService;
+    private readonly IPipelineRegistry _pipelineRegistry;
+
+    /// <summary>
+    /// Initializes a new instance of the <see cref="ManualSyncController"/> class.
+    /// </summary>
+    /// <param name="manualSyncRequestService">The service for managing manual sync requests.</param>
+    /// <param name="pipelineRegistry">The pipeline registry for querying pipeline information.</param>
+    public ManualSyncController(
+        IManualSyncRequestService manualSyncRequestService,
+        IPipelineRegistry pipelineRegistry)
+    {
+        _manualSyncRequestService = manualSyncRequestService;
+        _pipelineRegistry = pipelineRegistry;
+    }
+
+    /// <summary>
+    /// Gets all manual sync requests, optionally filtered to pending only.
+    /// </summary>
+    /// <param name="pendingOnly">If true, returns only pending requests. Default is false.</param>
+    /// <param name="ct">Cancellation token.</param>
+    /// <returns>A list of manual sync request view models.</returns>
+    [HttpGet]
+    [ProducesResponseType(typeof(List<ManualSyncRequestViewModel>), StatusCodes.Status200OK)]
+    [ProducesResponseType(StatusCodes.Status401Unauthorized)]
+    public async Task<ActionResult<List<ManualSyncRequestViewModel>>> GetRequests(
+        [FromQuery] bool pendingOnly = false,
+        CancellationToken ct = default)
+    {
+        var requests = await _manualSyncRequestService.GetRequestsAsync(pendingOnly, ct);
+
+        var viewModels = requests.Select(r => new ManualSyncRequestViewModel
+        {
+            Id = r.Id,
+            PipelineName = r.PipelineName,
+            SyncType = r.SyncType,
+            RequestDT = r.RequestDT,
+            RequestedBy = r.RequestedBy,
+            CompletedDT = r.CompletedDT,
+            CancelDT = r.CancelDT,
+            CancelledBy = r.CancelledBy,
+            Status = r.Status,
+            RowVersionBase64 = Convert.ToBase64String(r.RowVersion)
+        }).ToList();
+
+        return Ok(viewModels);
+    }
+
+    /// <summary>
+    /// Gets all available pipelines with their supported sync types.
+    /// </summary>
+    /// <returns>A list of pipeline information view models.</returns>
+    [HttpGet("pipelines")]
+    [ProducesResponseType(typeof(List<PipelineInfoViewModel>), StatusCodes.Status200OK)]
+    [ProducesResponseType(StatusCodes.Status401Unauthorized)]
+    public ActionResult<List<PipelineInfoViewModel>> GetPipelines()
+    {
+        var pipelines = _pipelineRegistry.GetEnabledPipelines()
+            .Select(p => new PipelineInfoViewModel
+            {
+                Name = p.Name,
+                SupportedSyncTypes = GetSupportedSyncTypes(p)
+            })
+            .ToList();
+
+        return Ok(pipelines);
+    }
+
+    /// <summary>
+    /// Creates a new manual sync request.
+    /// </summary>
+    /// <param name="dto">The create request containing pipeline name and sync type.</param>
+    /// <param name="ct">Cancellation token.</param>
+    /// <returns>The created manual sync request view model.</returns>
+    [HttpPost]
+    [ProducesResponseType(typeof(ManualSyncRequestViewModel), StatusCodes.Status201Created)]
+    [ProducesResponseType(StatusCodes.Status400BadRequest)]
+    [ProducesResponseType(StatusCodes.Status401Unauthorized)]
+    public async Task<ActionResult<ManualSyncRequestViewModel>> CreateRequest(
+        [FromBody] CreateManualSyncRequestDto dto,
+        CancellationToken ct = default)
+    {
+        var username = User.Identity?.Name;
+        if (string.IsNullOrEmpty(username))
+        {
+            return Unauthorized();
+        }
+
+        // Validate pipeline and sync type combination using the registry
+        if (!_pipelineRegistry.IsValidPipelineAndSyncType(dto.PipelineName, dto.SyncType))
+        {
+            return BadRequest($"Invalid pipeline/sync type combination. Pipeline '{dto.PipelineName}' does not support sync type '{dto.SyncType}'.");
+        }
+
+        var request = await _manualSyncRequestService.CreateRequestAsync(
+            dto.PipelineName,
+            dto.SyncType,
+            username,
+            ct);
+
+        var viewModel = new ManualSyncRequestViewModel
+        {
+            Id = request.Id,
+            PipelineName = request.PipelineName,
+            SyncType = request.SyncType,
+            RequestDT = request.RequestDT,
+            RequestedBy = request.RequestedBy,
+            CompletedDT = request.CompletedDT,
+            CancelDT = request.CancelDT,
+            CancelledBy = request.CancelledBy,
+            Status = request.Status,
+            RowVersionBase64 = Convert.ToBase64String(request.RowVersion)
+        };
+
+        return CreatedAtAction(nameof(GetRequests), viewModel);
+    }
+
+    /// <summary>
+    /// Cancels a pending manual sync request.
+    /// </summary>
+    /// <param name="id">The ID of the request to cancel.</param>
+    /// <param name="dto">The cancel request containing the row version for concurrency.</param>
+    /// <param name="ct">Cancellation token.</param>
+    /// <returns>A success message if cancelled, or a conflict error if already processed.</returns>
+    [HttpPost("{id:int}/cancel")]
+    [ProducesResponseType(typeof(object), StatusCodes.Status200OK)]
+    [ProducesResponseType(StatusCodes.Status401Unauthorized)]
+    [ProducesResponseType(StatusCodes.Status409Conflict)]
+    public async Task<IActionResult> CancelRequest(
+        int id,
+        [FromBody] CancelManualSyncRequestDto dto,
+        CancellationToken ct = default)
+    {
+        var username = User.Identity?.Name;
+        if (string.IsNullOrEmpty(username))
+        {
+            return Unauthorized();
+        }
+
+        byte[] rowVersion;
+        try
+        {
+            rowVersion = Convert.FromBase64String(dto.RowVersionBase64);
+        }
+        catch (FormatException)
+        {
+            return BadRequest("Invalid RowVersionBase64 format.");
+        }
+
+        var cancelled = await _manualSyncRequestService.CancelRequestAsync(
+            id,
+            username,
+            rowVersion,
+            ct);
+
+        if (!cancelled)
+        {
+            return Conflict(new { message = "The request has already been completed or cancelled, or the data has been modified by another user." });
+        }
+
+        return Ok(new { message = "Request cancelled successfully." });
+    }
+
+    private static List<string> GetSupportedSyncTypes(DataSync.Configuration.EtlPipelineConfig pipeline)
+    {
+        var types = new List<string>();
+        if (pipeline.SupportsMassSync) types.Add("mass");
+        if (pipeline.SupportsDailySync) types.Add("daily");
+        if (pipeline.SupportsHourlySync) types.Add("hourly");
+        return types;
+    }
+}