feat: implement ETL pipeline redesign and ConfigManager improvements

- Add pipeline registry with JSON-based configuration and hot-reload support - Implement manual sync request feature with API, client UI, and database - Improve ConfigManager: connection string dropdown in pipeline editor, step delete/reorder functionality, and fix JSON parsing for ConnectionStrings
2026-01-22 17:48:33 -05:00
parent 5a332232d0
commit 29ac56006d
82 changed files with 6257 additions and 296 deletions
@@ -0,0 +1,12 @@
+namespace JdeScoping.Api.Contracts.ManualSync;
+
+/// <summary>
+/// Request to cancel a manual sync request.
+/// </summary>
+public class CancelManualSyncRequestDto
+{
+    /// <summary>
+    /// The row version for optimistic concurrency (Base64 encoded).
+    /// </summary>
+    public string RowVersionBase64 { get; set; } = string.Empty;
+}
@@ -0,0 +1,17 @@
+namespace JdeScoping.Api.Contracts.ManualSync;
+
+/// <summary>
+/// Request to create a new manual sync request.
+/// </summary>
+public class CreateManualSyncRequestDto
+{
+    /// <summary>
+    /// The name of the pipeline to sync.
+    /// </summary>
+    public string PipelineName { get; set; } = string.Empty;
+
+    /// <summary>
+    /// The type of sync to perform (mass, daily, hourly).
+    /// </summary>
+    public string SyncType { get; set; } = string.Empty;
+}
@@ -0,0 +1,57 @@
+namespace JdeScoping.Api.Contracts.ManualSync;
+
+/// <summary>
+/// View model for a manual sync request.
+/// </summary>
+public class ManualSyncRequestViewModel
+{
+    /// <summary>
+    /// The unique identifier for the sync request.
+    /// </summary>
+    public int Id { get; set; }
+
+    /// <summary>
+    /// The name of the pipeline being synced.
+    /// </summary>
+    public string PipelineName { get; set; } = string.Empty;
+
+    /// <summary>
+    /// The type of sync (mass, daily, hourly).
+    /// </summary>
+    public string SyncType { get; set; } = string.Empty;
+
+    /// <summary>
+    /// The date and time the request was made.
+    /// </summary>
+    public DateTime RequestDT { get; set; }
+
+    /// <summary>
+    /// The username of the person who requested the sync.
+    /// </summary>
+    public string RequestedBy { get; set; } = string.Empty;
+
+    /// <summary>
+    /// The date and time the sync completed, if applicable.
+    /// </summary>
+    public DateTime? CompletedDT { get; set; }
+
+    /// <summary>
+    /// The date and time the sync was cancelled, if applicable.
+    /// </summary>
+    public DateTime? CancelDT { get; set; }
+
+    /// <summary>
+    /// The username of the person who cancelled the sync, if applicable.
+    /// </summary>
+    public string? CancelledBy { get; set; }
+
+    /// <summary>
+    /// The current status of the sync request.
+    /// </summary>
+    public string Status { get; set; } = string.Empty;
+
+    /// <summary>
+    /// The row version for optimistic concurrency (Base64 encoded).
+    /// </summary>
+    public string RowVersionBase64 { get; set; } = string.Empty;
+}
@@ -0,0 +1,17 @@
+namespace JdeScoping.Api.Contracts.ManualSync;
+
+/// <summary>
+/// View model for pipeline information.
+/// </summary>
+public class PipelineInfoViewModel
+{
+    /// <summary>
+    /// The name of the pipeline.
+    /// </summary>
+    public string Name { get; set; } = string.Empty;
+
+    /// <summary>
+    /// The sync types supported by this pipeline.
+    /// </summary>
+    public List<string> SupportedSyncTypes { get; set; } = new();
+}
@@ -0,0 +1,189 @@
+using JdeScoping.Api.Contracts.ManualSync;
+using JdeScoping.DataAccess.Services;
+using JdeScoping.DataSync.Services;
+using Microsoft.AspNetCore.Authorization;
+using Microsoft.AspNetCore.Http;
+using Microsoft.AspNetCore.Mvc;
+
+namespace JdeScoping.Api.Controllers;
+
+/// <summary>
+/// API endpoints for manual data sync request management.
+/// </summary>
+[Route("api/manual-sync")]
+[ApiController]
+[Authorize]
+public class ManualSyncController : ApiControllerBase
+{
+    private readonly IManualSyncRequestService _manualSyncRequestService;
+    private readonly IPipelineRegistry _pipelineRegistry;
+
+    /// <summary>
+    /// Initializes a new instance of the <see cref="ManualSyncController"/> class.
+    /// </summary>
+    /// <param name="manualSyncRequestService">The service for managing manual sync requests.</param>
+    /// <param name="pipelineRegistry">The pipeline registry for querying pipeline information.</param>
+    public ManualSyncController(
+        IManualSyncRequestService manualSyncRequestService,
+        IPipelineRegistry pipelineRegistry)
+    {
+        _manualSyncRequestService = manualSyncRequestService;
+        _pipelineRegistry = pipelineRegistry;
+    }
+
+    /// <summary>
+    /// Gets all manual sync requests, optionally filtered to pending only.
+    /// </summary>
+    /// <param name="pendingOnly">If true, returns only pending requests. Default is false.</param>
+    /// <param name="ct">Cancellation token.</param>
+    /// <returns>A list of manual sync request view models.</returns>
+    [HttpGet]
+    [ProducesResponseType(typeof(List<ManualSyncRequestViewModel>), StatusCodes.Status200OK)]
+    [ProducesResponseType(StatusCodes.Status401Unauthorized)]
+    public async Task<ActionResult<List<ManualSyncRequestViewModel>>> GetRequests(
+        [FromQuery] bool pendingOnly = false,
+        CancellationToken ct = default)
+    {
+        var requests = await _manualSyncRequestService.GetRequestsAsync(pendingOnly, ct);
+
+        var viewModels = requests.Select(r => new ManualSyncRequestViewModel
+        {
+            Id = r.Id,
+            PipelineName = r.PipelineName,
+            SyncType = r.SyncType,
+            RequestDT = r.RequestDT,
+            RequestedBy = r.RequestedBy,
+            CompletedDT = r.CompletedDT,
+            CancelDT = r.CancelDT,
+            CancelledBy = r.CancelledBy,
+            Status = r.Status,
+            RowVersionBase64 = Convert.ToBase64String(r.RowVersion)
+        }).ToList();
+
+        return Ok(viewModels);
+    }
+
+    /// <summary>
+    /// Gets all available pipelines with their supported sync types.
+    /// </summary>
+    /// <returns>A list of pipeline information view models.</returns>
+    [HttpGet("pipelines")]
+    [ProducesResponseType(typeof(List<PipelineInfoViewModel>), StatusCodes.Status200OK)]
+    [ProducesResponseType(StatusCodes.Status401Unauthorized)]
+    public ActionResult<List<PipelineInfoViewModel>> GetPipelines()
+    {
+        var pipelines = _pipelineRegistry.GetEnabledPipelines()
+            .Select(p => new PipelineInfoViewModel
+            {
+                Name = p.Name,
+                SupportedSyncTypes = GetSupportedSyncTypes(p)
+            })
+            .ToList();
+
+        return Ok(pipelines);
+    }
+
+    /// <summary>
+    /// Creates a new manual sync request.
+    /// </summary>
+    /// <param name="dto">The create request containing pipeline name and sync type.</param>
+    /// <param name="ct">Cancellation token.</param>
+    /// <returns>The created manual sync request view model.</returns>
+    [HttpPost]
+    [ProducesResponseType(typeof(ManualSyncRequestViewModel), StatusCodes.Status201Created)]
+    [ProducesResponseType(StatusCodes.Status400BadRequest)]
+    [ProducesResponseType(StatusCodes.Status401Unauthorized)]
+    public async Task<ActionResult<ManualSyncRequestViewModel>> CreateRequest(
+        [FromBody] CreateManualSyncRequestDto dto,
+        CancellationToken ct = default)
+    {
+        var username = User.Identity?.Name;
+        if (string.IsNullOrEmpty(username))
+        {
+            return Unauthorized();
+        }
+
+        // Validate pipeline and sync type combination using the registry
+        if (!_pipelineRegistry.IsValidPipelineAndSyncType(dto.PipelineName, dto.SyncType))
+        {
+            return BadRequest($"Invalid pipeline/sync type combination. Pipeline '{dto.PipelineName}' does not support sync type '{dto.SyncType}'.");
+        }
+
+        var request = await _manualSyncRequestService.CreateRequestAsync(
+            dto.PipelineName,
+            dto.SyncType,
+            username,
+            ct);
+
+        var viewModel = new ManualSyncRequestViewModel
+        {
+            Id = request.Id,
+            PipelineName = request.PipelineName,
+            SyncType = request.SyncType,
+            RequestDT = request.RequestDT,
+            RequestedBy = request.RequestedBy,
+            CompletedDT = request.CompletedDT,
+            CancelDT = request.CancelDT,
+            CancelledBy = request.CancelledBy,
+            Status = request.Status,
+            RowVersionBase64 = Convert.ToBase64String(request.RowVersion)
+        };
+
+        return CreatedAtAction(nameof(GetRequests), viewModel);
+    }
+
+    /// <summary>
+    /// Cancels a pending manual sync request.
+    /// </summary>
+    /// <param name="id">The ID of the request to cancel.</param>
+    /// <param name="dto">The cancel request containing the row version for concurrency.</param>
+    /// <param name="ct">Cancellation token.</param>
+    /// <returns>A success message if cancelled, or a conflict error if already processed.</returns>
+    [HttpPost("{id:int}/cancel")]
+    [ProducesResponseType(typeof(object), StatusCodes.Status200OK)]
+    [ProducesResponseType(StatusCodes.Status401Unauthorized)]
+    [ProducesResponseType(StatusCodes.Status409Conflict)]
+    public async Task<IActionResult> CancelRequest(
+        int id,
+        [FromBody] CancelManualSyncRequestDto dto,
+        CancellationToken ct = default)
+    {
+        var username = User.Identity?.Name;
+        if (string.IsNullOrEmpty(username))
+        {
+            return Unauthorized();
+        }
+
+        byte[] rowVersion;
+        try
+        {
+            rowVersion = Convert.FromBase64String(dto.RowVersionBase64);
+        }
+        catch (FormatException)
+        {
+            return BadRequest("Invalid RowVersionBase64 format.");
+        }
+
+        var cancelled = await _manualSyncRequestService.CancelRequestAsync(
+            id,
+            username,
+            rowVersion,
+            ct);
+
+        if (!cancelled)
+        {
+            return Conflict(new { message = "The request has already been completed or cancelled, or the data has been modified by another user." });
+        }
+
+        return Ok(new { message = "Request cancelled successfully." });
+    }
+
+    private static List<string> GetSupportedSyncTypes(DataSync.Configuration.EtlPipelineConfig pipeline)
+    {
+        var types = new List<string>();
+        if (pipeline.SupportsMassSync) types.Add("mass");
+        if (pipeline.SupportsDailySync) types.Add("daily");
+        if (pipeline.SupportsHourlySync) types.Add("hourly");
+        return types;
+    }
+}
@@ -0,0 +1,224 @@
+using JdeScoping.Api.Contracts.ManualSync;
+using JdeScoping.DataSync.Configuration;
+using JdeScoping.DataSync.Services;
+using Microsoft.AspNetCore.Authorization;
+using Microsoft.AspNetCore.Http;
+using Microsoft.AspNetCore.Mvc;
+using Microsoft.Extensions.Logging;
+
+namespace JdeScoping.Api.Controllers;
+
+/// <summary>
+/// API endpoints for pipeline management and hot reload.
+/// </summary>
+[Route("api/pipelines")]
+[ApiController]
+[Authorize]
+public class PipelineController : ApiControllerBase
+{
+    private readonly IPipelineRegistry _pipelineRegistry;
+    private readonly ILogger<PipelineController> _logger;
+
+    /// <summary>
+    /// Initializes a new instance of the <see cref="PipelineController"/> class.
+    /// </summary>
+    /// <param name="pipelineRegistry">The pipeline registry.</param>
+    /// <param name="logger">The logger.</param>
+    public PipelineController(
+        IPipelineRegistry pipelineRegistry,
+        ILogger<PipelineController> logger)
+    {
+        _pipelineRegistry = pipelineRegistry;
+        _logger = logger;
+    }
+
+    /// <summary>
+    /// Gets all enabled pipelines with their supported sync types.
+    /// </summary>
+    /// <returns>A list of pipeline information view models.</returns>
+    [HttpGet]
+    [ProducesResponseType(typeof(List<PipelineInfoViewModel>), StatusCodes.Status200OK)]
+    [ProducesResponseType(StatusCodes.Status401Unauthorized)]
+    public ActionResult<List<PipelineInfoViewModel>> GetPipelines()
+    {
+        var pipelines = _pipelineRegistry.GetEnabledPipelines()
+            .Select(p => new PipelineInfoViewModel
+            {
+                Name = p.Name,
+                SupportedSyncTypes = GetSupportedSyncTypes(p)
+            })
+            .ToList();
+
+        return Ok(pipelines);
+    }
+
+    /// <summary>
+    /// Gets registry metadata including version and last load time.
+    /// </summary>
+    /// <returns>Registry metadata.</returns>
+    [HttpGet("status")]
+    [ProducesResponseType(typeof(PipelineRegistryStatusViewModel), StatusCodes.Status200OK)]
+    [ProducesResponseType(StatusCodes.Status401Unauthorized)]
+    public ActionResult<PipelineRegistryStatusViewModel> GetStatus()
+    {
+        var allPipelines = _pipelineRegistry.GetAllPipelines();
+        var enabledPipelines = _pipelineRegistry.GetEnabledPipelines();
+
+        return Ok(new PipelineRegistryStatusViewModel
+        {
+            Version = _pipelineRegistry.Version,
+            LastLoadedAt = _pipelineRegistry.LastLoadedAt,
+            TotalPipelines = allPipelines.Count,
+            EnabledPipelines = enabledPipelines.Count
+        });
+    }
+
+    /// <summary>
+    /// Reloads all pipeline definitions from disk.
+    /// Requires Admin role.
+    /// </summary>
+    /// <param name="ct">Cancellation token.</param>
+    /// <returns>The reload result.</returns>
+    [HttpPost("reload")]
+    [Authorize(Roles = "Admin")]
+    [ProducesResponseType(typeof(PipelineReloadResultViewModel), StatusCodes.Status200OK)]
+    [ProducesResponseType(StatusCodes.Status401Unauthorized)]
+    [ProducesResponseType(StatusCodes.Status403Forbidden)]
+    [ProducesResponseType(StatusCodes.Status500InternalServerError)]
+    public async Task<ActionResult<PipelineReloadResultViewModel>> ReloadPipelines(CancellationToken ct = default)
+    {
+        var username = User.Identity?.Name ?? "unknown";
+
+        _logger.LogInformation("Pipeline reload requested by {User}", username);
+
+        try
+        {
+            var result = await _pipelineRegistry.ReloadAsync(ct);
+
+            _logger.LogInformation(
+                "Pipeline reload completed: Success={Success}, Loaded={Loaded}, Skipped={Skipped}, Version={Version}",
+                result.Success,
+                result.PipelinesLoaded,
+                result.PipelinesSkipped,
+                result.NewVersion);
+
+            return Ok(new PipelineReloadResultViewModel
+            {
+                Success = result.Success,
+                PipelinesLoaded = result.PipelinesLoaded,
+                PipelinesSkipped = result.PipelinesSkipped,
+                PreviousVersion = result.PreviousVersion,
+                NewVersion = result.NewVersion,
+                Errors = result.Errors.Select(e => new PipelineLoadErrorViewModel
+                {
+                    FileName = e.FileName,
+                    PipelineName = e.PipelineName,
+                    ErrorType = e.ErrorType,
+                    Messages = e.Messages
+                }).ToList()
+            });
+        }
+        catch (Exception ex)
+        {
+            _logger.LogError(ex, "Pipeline reload failed");
+            return StatusCode(StatusCodes.Status500InternalServerError, new { message = "Pipeline reload failed. See server logs for details." });
+        }
+    }
+
+    private static List<string> GetSupportedSyncTypes(EtlPipelineConfig pipeline)
+    {
+        var types = new List<string>();
+        if (pipeline.SupportsMassSync) types.Add("mass");
+        if (pipeline.SupportsDailySync) types.Add("daily");
+        if (pipeline.SupportsHourlySync) types.Add("hourly");
+        return types;
+    }
+}
+
+/// <summary>
+/// View model for pipeline registry status.
+/// </summary>
+public class PipelineRegistryStatusViewModel
+{
+    /// <summary>
+    /// Gets or sets the current registry version.
+    /// </summary>
+    public int Version { get; set; }
+
+    /// <summary>
+    /// Gets or sets the timestamp of the last successful load.
+    /// </summary>
+    public DateTime? LastLoadedAt { get; set; }
+
+    /// <summary>
+    /// Gets or sets the total number of pipelines.
+    /// </summary>
+    public int TotalPipelines { get; set; }
+
+    /// <summary>
+    /// Gets or sets the number of enabled pipelines.
+    /// </summary>
+    public int EnabledPipelines { get; set; }
+}
+
+/// <summary>
+/// View model for pipeline reload result.
+/// </summary>
+public class PipelineReloadResultViewModel
+{
+    /// <summary>
+    /// Gets or sets a value indicating whether the reload was successful.
+    /// </summary>
+    public bool Success { get; set; }
+
+    /// <summary>
+    /// Gets or sets the number of pipelines loaded.
+    /// </summary>
+    public int PipelinesLoaded { get; set; }
+
+    /// <summary>
+    /// Gets or sets the number of pipelines skipped.
+    /// </summary>
+    public int PipelinesSkipped { get; set; }
+
+    /// <summary>
+    /// Gets or sets the previous version.
+    /// </summary>
+    public int PreviousVersion { get; set; }
+
+    /// <summary>
+    /// Gets or sets the new version.
+    /// </summary>
+    public int NewVersion { get; set; }
+
+    /// <summary>
+    /// Gets or sets the list of errors.
+    /// </summary>
+    public List<PipelineLoadErrorViewModel> Errors { get; set; } = [];
+}
+
+/// <summary>
+/// View model for pipeline load error.
+/// </summary>
+public class PipelineLoadErrorViewModel
+{
+    /// <summary>
+    /// Gets or sets the file name.
+    /// </summary>
+    public string FileName { get; set; } = string.Empty;
+
+    /// <summary>
+    /// Gets or sets the pipeline name.
+    /// </summary>
+    public string PipelineName { get; set; } = string.Empty;
+
+    /// <summary>
+    /// Gets or sets the error type.
+    /// </summary>
+    public string ErrorType { get; set; } = string.Empty;
+
+    /// <summary>
+    /// Gets or sets the error messages.
+    /// </summary>
+    public List<string> Messages { get; set; } = [];
+}