feat: implement ETL pipeline redesign and ConfigManager improvements

- Add pipeline registry with JSON-based configuration and hot-reload support - Implement manual sync request feature with API, client UI, and database - Improve ConfigManager: connection string dropdown in pipeline editor, step delete/reorder functionality, and fix JSON parsing for ConnectionStrings
2026-01-22 17:48:33 -05:00
parent 5a332232d0
commit 29ac56006d
82 changed files with 6257 additions and 296 deletions
@@ -152,4 +152,44 @@ public static class ApiRoutes
            $"api/refresh-status?minDT={minDt:yyyy-MM-dd}&maxDT={maxDt:yyyy-MM-dd}";
    }

+    /// <summary>
+    /// Routes for manual sync API endpoints.
+    /// </summary>
+    public static class ManualSync
+    {
+        /// <summary>Base route for manual sync endpoints.</summary>
+        public const string Base = "api/manual-sync";
+
+        /// <summary>Route for pipeline list.</summary>
+        public const string Pipelines = "api/manual-sync/pipelines";
+
+        /// <summary>Route template for cancelling a request (use in controller attributes).</summary>
+        public const string Cancel = "{id:int}/cancel";
+
+        /// <summary>Builds the route to get requests with optional filter.</summary>
+        /// <param name="pendingOnly">If true, returns only pending requests.</param>
+        /// <returns>The formatted route.</returns>
+        public static string GetRequests(bool pendingOnly = false) =>
+            pendingOnly ? $"{Base}?pendingOnly=true" : Base;
+
+        /// <summary>Builds the route to cancel a specific request.</summary>
+        /// <param name="id">The request ID.</param>
+        /// <returns>The formatted route.</returns>
+        public static string GetCancel(int id) => $"{Base}/{id}/cancel";
+    }
+
+    /// <summary>
+    /// Routes for pipeline API endpoints.
+    /// </summary>
+    public static class Pipeline
+    {
+        /// <summary>Base route for pipeline endpoints.</summary>
+        public const string Base = "api/pipelines";
+
+        /// <summary>Route for pipeline status.</summary>
+        public const string Status = "api/pipelines/status";
+
+        /// <summary>Route to reload pipelines.</summary>
+        public const string Reload = "api/pipelines/reload";
+    }
 }
@@ -0,0 +1,42 @@
+using JdeScoping.Core.ApiContracts.Results;
+using JdeScoping.Core.ViewModels;
+
+namespace JdeScoping.Core.ApiContracts;
+
+/// <summary>
+/// Client contract for manual sync API operations.
+/// </summary>
+public interface IManualSyncApiClient
+{
+    /// <summary>
+    /// Gets manual sync requests, optionally filtered to pending only.
+    /// </summary>
+    /// <param name="pendingOnly">If true, returns only pending requests.</param>
+    /// <param name="ct">Cancellation token.</param>
+    /// <returns>A list of manual sync request view models.</returns>
+    Task<ApiResult<IReadOnlyList<ManualSyncRequestViewModel>>> GetRequestsAsync(bool pendingOnly = false, CancellationToken ct = default);
+
+    /// <summary>
+    /// Gets all available pipelines for manual sync.
+    /// </summary>
+    /// <param name="ct">Cancellation token.</param>
+    /// <returns>A list of pipeline information view models.</returns>
+    Task<ApiResult<IReadOnlyList<PipelineInfoViewModel>>> GetPipelinesAsync(CancellationToken ct = default);
+
+    /// <summary>
+    /// Creates a new manual sync request.
+    /// </summary>
+    /// <param name="request">The request details.</param>
+    /// <param name="ct">Cancellation token.</param>
+    /// <returns>The created manual sync request view model.</returns>
+    Task<ApiResult<ManualSyncRequestViewModel>> CreateRequestAsync(CreateManualSyncRequestDto request, CancellationToken ct = default);
+
+    /// <summary>
+    /// Cancels a pending manual sync request.
+    /// </summary>
+    /// <param name="id">The request ID.</param>
+    /// <param name="rowVersionBase64">The row version for optimistic concurrency.</param>
+    /// <param name="ct">Cancellation token.</param>
+    /// <returns>A unit result indicating success or failure.</returns>
+    Task<ApiResult<Unit>> CancelRequestAsync(int id, string rowVersionBase64, CancellationToken ct = default);
+}
@@ -0,0 +1,120 @@
+using JdeScoping.Core.ApiContracts.Results;
+using JdeScoping.Core.ViewModels;
+
+namespace JdeScoping.Core.ApiContracts;
+
+/// <summary>
+/// Client contract for pipeline API operations.
+/// </summary>
+public interface IPipelineApiClient
+{
+    /// <summary>
+    /// Gets all enabled pipelines.
+    /// </summary>
+    /// <param name="ct">Cancellation token.</param>
+    /// <returns>A list of pipeline information view models.</returns>
+    Task<ApiResult<IReadOnlyList<PipelineInfoViewModel>>> GetPipelinesAsync(CancellationToken ct = default);
+
+    /// <summary>
+    /// Gets the pipeline registry status.
+    /// </summary>
+    /// <param name="ct">Cancellation token.</param>
+    /// <returns>The registry status.</returns>
+    Task<ApiResult<PipelineRegistryStatusViewModel>> GetStatusAsync(CancellationToken ct = default);
+
+    /// <summary>
+    /// Reloads all pipeline definitions from disk.
+    /// Requires Admin role.
+    /// </summary>
+    /// <param name="ct">Cancellation token.</param>
+    /// <returns>The reload result.</returns>
+    Task<ApiResult<PipelineReloadResultViewModel>> ReloadPipelinesAsync(CancellationToken ct = default);
+}
+
+/// <summary>
+/// View model for pipeline registry status.
+/// </summary>
+public class PipelineRegistryStatusViewModel
+{
+    /// <summary>
+    /// Gets or sets the current registry version.
+    /// </summary>
+    public int Version { get; set; }
+
+    /// <summary>
+    /// Gets or sets the timestamp of the last successful load.
+    /// </summary>
+    public DateTime? LastLoadedAt { get; set; }
+
+    /// <summary>
+    /// Gets or sets the total number of pipelines.
+    /// </summary>
+    public int TotalPipelines { get; set; }
+
+    /// <summary>
+    /// Gets or sets the number of enabled pipelines.
+    /// </summary>
+    public int EnabledPipelines { get; set; }
+}
+
+/// <summary>
+/// View model for pipeline reload result.
+/// </summary>
+public class PipelineReloadResultViewModel
+{
+    /// <summary>
+    /// Gets or sets a value indicating whether the reload was successful.
+    /// </summary>
+    public bool Success { get; set; }
+
+    /// <summary>
+    /// Gets or sets the number of pipelines loaded.
+    /// </summary>
+    public int PipelinesLoaded { get; set; }
+
+    /// <summary>
+    /// Gets or sets the number of pipelines skipped.
+    /// </summary>
+    public int PipelinesSkipped { get; set; }
+
+    /// <summary>
+    /// Gets or sets the previous version.
+    /// </summary>
+    public int PreviousVersion { get; set; }
+
+    /// <summary>
+    /// Gets or sets the new version.
+    /// </summary>
+    public int NewVersion { get; set; }
+
+    /// <summary>
+    /// Gets or sets the list of errors.
+    /// </summary>
+    public List<PipelineLoadErrorViewModel> Errors { get; set; } = [];
+}
+
+/// <summary>
+/// View model for pipeline load error.
+/// </summary>
+public class PipelineLoadErrorViewModel
+{
+    /// <summary>
+    /// Gets or sets the file name.
+    /// </summary>
+    public string FileName { get; set; } = string.Empty;
+
+    /// <summary>
+    /// Gets or sets the pipeline name.
+    /// </summary>
+    public string PipelineName { get; set; } = string.Empty;
+
+    /// <summary>
+    /// Gets or sets the error type.
+    /// </summary>
+    public string ErrorType { get; set; } = string.Empty;
+
+    /// <summary>
+    /// Gets or sets the error messages.
+    /// </summary>
+    public List<string> Messages { get; set; } = [];
+}
@@ -0,0 +1,17 @@
+namespace JdeScoping.Core.ViewModels;
+
+/// <summary>
+/// Request to create a new manual sync request.
+/// </summary>
+public class CreateManualSyncRequestDto
+{
+    /// <summary>
+    /// The name of the pipeline to sync.
+    /// </summary>
+    public string PipelineName { get; set; } = string.Empty;
+
+    /// <summary>
+    /// The type of sync to perform (mass, daily, hourly).
+    /// </summary>
+    public string SyncType { get; set; } = string.Empty;
+}
@@ -0,0 +1,57 @@
+namespace JdeScoping.Core.ViewModels;
+
+/// <summary>
+/// View model for a manual sync request.
+/// </summary>
+public class ManualSyncRequestViewModel
+{
+    /// <summary>
+    /// The unique identifier for the sync request.
+    /// </summary>
+    public int Id { get; set; }
+
+    /// <summary>
+    /// The name of the pipeline being synced.
+    /// </summary>
+    public string PipelineName { get; set; } = string.Empty;
+
+    /// <summary>
+    /// The type of sync (mass, daily, hourly).
+    /// </summary>
+    public string SyncType { get; set; } = string.Empty;
+
+    /// <summary>
+    /// The date and time the request was made.
+    /// </summary>
+    public DateTime RequestDT { get; set; }
+
+    /// <summary>
+    /// The username of the person who requested the sync.
+    /// </summary>
+    public string RequestedBy { get; set; } = string.Empty;
+
+    /// <summary>
+    /// The date and time the sync completed, if applicable.
+    /// </summary>
+    public DateTime? CompletedDT { get; set; }
+
+    /// <summary>
+    /// The date and time the sync was cancelled, if applicable.
+    /// </summary>
+    public DateTime? CancelDT { get; set; }
+
+    /// <summary>
+    /// The username of the person who cancelled the sync, if applicable.
+    /// </summary>
+    public string? CancelledBy { get; set; }
+
+    /// <summary>
+    /// The current status of the sync request.
+    /// </summary>
+    public string Status { get; set; } = string.Empty;
+
+    /// <summary>
+    /// The row version for optimistic concurrency (Base64 encoded).
+    /// </summary>
+    public string RowVersionBase64 { get; set; } = string.Empty;
+}
@@ -0,0 +1,17 @@
+namespace JdeScoping.Core.ViewModels;
+
+/// <summary>
+/// View model for pipeline information.
+/// </summary>
+public class PipelineInfoViewModel
+{
+    /// <summary>
+    /// The name of the pipeline.
+    /// </summary>
+    public string Name { get; set; } = string.Empty;
+
+    /// <summary>
+    /// The sync types supported by this pipeline.
+    /// </summary>
+    public List<string> SupportedSyncTypes { get; set; } = new();
+}