feat: implement ETL pipeline redesign and ConfigManager improvements

- Add pipeline registry with JSON-based configuration and hot-reload support - Implement manual sync request feature with API, client UI, and database - Improve ConfigManager: connection string dropdown in pipeline editor, step delete/reorder functionality, and fix JSON parsing for ConnectionStrings
2026-01-22 17:48:33 -05:00
parent 5a332232d0
commit 29ac56006d
82 changed files with 6257 additions and 296 deletions
@@ -52,6 +52,9 @@ public static class DataAccessDependencyInjection
        services.AddScoped<IWorkOrderTraversalService, WorkOrderTraversalService>();
        services.AddScoped<ISearchProcessor, SearchProcessor>();

+        // Register manual sync request service (scoped - per request lifetime)
+        services.AddScoped<IManualSyncRequestService, ManualSyncRequestService>();
+
        return services;
    }

@@ -21,6 +21,7 @@

  <ItemGroup>
    <ProjectReference Include="..\JdeScoping.Core\JdeScoping.Core.csproj" />
+    <ProjectReference Include="..\JdeScoping.Domain\JdeScoping.Domain.csproj" />
  </ItemGroup>

 </Project>
@@ -0,0 +1,68 @@
+using JdeScoping.Domain.Models;
+
+namespace JdeScoping.DataAccess.Services;
+
+/// <summary>
+/// Service for managing manual data sync requests.
+/// </summary>
+public interface IManualSyncRequestService
+{
+    /// <summary>
+    /// Gets all manual sync requests, optionally filtered to pending only.
+    /// </summary>
+    /// <param name="pendingOnly">If true, returns only pending requests.</param>
+    /// <param name="ct">Cancellation token.</param>
+    /// <returns>A read-only list of manual sync requests.</returns>
+    Task<IReadOnlyList<ManualSyncRequest>> GetRequestsAsync(
+        bool pendingOnly = false,
+        CancellationToken ct = default);
+
+    /// <summary>
+    /// Gets the next pending request in FIFO order (for processor).
+    /// </summary>
+    /// <param name="ct">Cancellation token.</param>
+    /// <returns>The next pending request, or null if none exists.</returns>
+    Task<ManualSyncRequest?> GetNextPendingRequestAsync(CancellationToken ct = default);
+
+    /// <summary>
+    /// Creates a new manual sync request.
+    /// </summary>
+    /// <param name="pipelineName">The name of the ETL pipeline to sync.</param>
+    /// <param name="syncType">The type of sync (mass, daily, hourly).</param>
+    /// <param name="requestedBy">The username of the requester.</param>
+    /// <param name="ct">Cancellation token.</param>
+    /// <returns>The created manual sync request.</returns>
+    Task<ManualSyncRequest> CreateRequestAsync(
+        string pipelineName,
+        string syncType,
+        string requestedBy,
+        CancellationToken ct = default);
+
+    /// <summary>
+    /// Cancels a pending request using optimistic concurrency.
+    /// Returns true if cancelled, false if already completed/cancelled.
+    /// </summary>
+    /// <param name="id">The request ID.</param>
+    /// <param name="cancelledBy">The username of the user cancelling the request.</param>
+    /// <param name="rowVersion">The row version for optimistic concurrency.</param>
+    /// <param name="ct">Cancellation token.</param>
+    /// <returns>True if the request was cancelled; false if already completed or cancelled.</returns>
+    Task<bool> CancelRequestAsync(
+        int id,
+        string cancelledBy,
+        byte[] rowVersion,
+        CancellationToken ct = default);
+
+    /// <summary>
+    /// Marks a request as completed using optimistic concurrency.
+    /// Called by the sync processor.
+    /// </summary>
+    /// <param name="id">The request ID.</param>
+    /// <param name="rowVersion">The row version for optimistic concurrency.</param>
+    /// <param name="ct">Cancellation token.</param>
+    /// <returns>True if the request was marked completed; false if already completed or cancelled.</returns>
+    Task<bool> CompleteRequestAsync(
+        int id,
+        byte[] rowVersion,
+        CancellationToken ct = default);
+}
@@ -0,0 +1,202 @@
+using Dapper;
+using JdeScoping.DataAccess.Interfaces;
+using JdeScoping.Domain.Models;
+using Microsoft.Extensions.Logging;
+
+namespace JdeScoping.DataAccess.Services;
+
+/// <summary>
+/// Service implementation for managing manual data sync requests.
+/// </summary>
+public sealed class ManualSyncRequestService : IManualSyncRequestService
+{
+    private readonly IDbConnectionFactory _connectionFactory;
+    private readonly ILogger<ManualSyncRequestService> _logger;
+
+    /// <summary>
+    /// Initializes a new instance of the <see cref="ManualSyncRequestService"/> class.
+    /// </summary>
+    /// <param name="connectionFactory">The database connection factory.</param>
+    /// <param name="logger">The logger instance.</param>
+    public ManualSyncRequestService(
+        IDbConnectionFactory connectionFactory,
+        ILogger<ManualSyncRequestService> logger)
+    {
+        _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory));
+        _logger = logger ?? throw new ArgumentNullException(nameof(logger));
+    }
+
+    /// <inheritdoc/>
+    public async Task<IReadOnlyList<ManualSyncRequest>> GetRequestsAsync(
+        bool pendingOnly = false,
+        CancellationToken ct = default)
+    {
+        _logger.LogDebug("Getting manual sync requests (pendingOnly: {PendingOnly})", pendingOnly);
+
+        const string sqlAll = """
+            SELECT ID, PipelineName, SyncType, RequestDT, RequestedBy,
+                   CompletedDT, CancelDT, CancelledBy, RowVersion
+            FROM dbo.ManualSyncRequest
+            ORDER BY RequestDT DESC
+            """;
+
+        const string sqlPending = """
+            SELECT ID, PipelineName, SyncType, RequestDT, RequestedBy,
+                   CompletedDT, CancelDT, CancelledBy, RowVersion
+            FROM dbo.ManualSyncRequest
+            WHERE CompletedDT IS NULL AND CancelDT IS NULL
+            ORDER BY RequestDT ASC
+            """;
+
+        await using var connection = await _connectionFactory.CreateLotFinderConnectionAsync(ct);
+        var results = await connection.QueryAsync<ManualSyncRequest>(
+            pendingOnly ? sqlPending : sqlAll);
+
+        var list = results.ToList();
+        _logger.LogDebug("Retrieved {Count} manual sync requests", list.Count);
+        return list;
+    }
+
+    /// <inheritdoc/>
+    public async Task<ManualSyncRequest?> GetNextPendingRequestAsync(CancellationToken ct = default)
+    {
+        _logger.LogDebug("Getting next pending manual sync request");
+
+        const string sql = """
+            SELECT TOP 1 ID, PipelineName, SyncType, RequestDT, RequestedBy,
+                         CompletedDT, CancelDT, CancelledBy, RowVersion
+            FROM dbo.ManualSyncRequest
+            WHERE CompletedDT IS NULL AND CancelDT IS NULL
+            ORDER BY RequestDT ASC
+            """;
+
+        await using var connection = await _connectionFactory.CreateLotFinderConnectionAsync(ct);
+        var result = await connection.QueryFirstOrDefaultAsync<ManualSyncRequest>(sql);
+
+        if (result != null)
+        {
+            _logger.LogDebug("Found pending request ID {Id} for pipeline {Pipeline}",
+                result.Id, result.PipelineName);
+        }
+        else
+        {
+            _logger.LogDebug("No pending manual sync requests found");
+        }
+
+        return result;
+    }
+
+    /// <inheritdoc/>
+    public async Task<ManualSyncRequest> CreateRequestAsync(
+        string pipelineName,
+        string syncType,
+        string requestedBy,
+        CancellationToken ct = default)
+    {
+        _logger.LogInformation(
+            "Creating manual sync request for pipeline {Pipeline}, type {SyncType}, by {User}",
+            pipelineName, syncType, requestedBy);
+
+        const string sql = """
+            INSERT INTO dbo.ManualSyncRequest (PipelineName, SyncType, RequestedBy)
+            OUTPUT INSERTED.ID, INSERTED.PipelineName, INSERTED.SyncType,
+                   INSERTED.RequestDT, INSERTED.RequestedBy,
+                   INSERTED.CompletedDT, INSERTED.CancelDT, INSERTED.CancelledBy,
+                   INSERTED.RowVersion
+            VALUES (@PipelineName, @SyncType, @RequestedBy)
+            """;
+
+        await using var connection = await _connectionFactory.CreateLotFinderConnectionAsync(ct);
+        var result = await connection.QuerySingleAsync<ManualSyncRequest>(sql, new
+        {
+            PipelineName = pipelineName,
+            SyncType = syncType,
+            RequestedBy = requestedBy
+        });
+
+        _logger.LogInformation("Created manual sync request with ID {Id}", result.Id);
+        return result;
+    }
+
+    /// <inheritdoc/>
+    public async Task<bool> CancelRequestAsync(
+        int id,
+        string cancelledBy,
+        byte[] rowVersion,
+        CancellationToken ct = default)
+    {
+        _logger.LogInformation(
+            "Cancelling manual sync request ID {Id} by {User}",
+            id, cancelledBy);
+
+        const string sql = """
+            UPDATE dbo.ManualSyncRequest
+            SET CancelDT = @CancelDT, CancelledBy = @CancelledBy
+            WHERE ID = @Id AND RowVersion = @RowVersion
+              AND CompletedDT IS NULL AND CancelDT IS NULL
+            """;
+
+        await using var connection = await _connectionFactory.CreateLotFinderConnectionAsync(ct);
+        var affectedRows = await connection.ExecuteAsync(sql, new
+        {
+            Id = id,
+            CancelDT = DateTime.UtcNow,
+            CancelledBy = cancelledBy,
+            RowVersion = rowVersion
+        });
+
+        var success = affectedRows > 0;
+
+        if (success)
+        {
+            _logger.LogInformation("Successfully cancelled manual sync request ID {Id}", id);
+        }
+        else
+        {
+            _logger.LogWarning(
+                "Failed to cancel manual sync request ID {Id} - already completed/cancelled or version mismatch",
+                id);
+        }
+
+        return success;
+    }
+
+    /// <inheritdoc/>
+    public async Task<bool> CompleteRequestAsync(
+        int id,
+        byte[] rowVersion,
+        CancellationToken ct = default)
+    {
+        _logger.LogInformation("Completing manual sync request ID {Id}", id);
+
+        const string sql = """
+            UPDATE dbo.ManualSyncRequest
+            SET CompletedDT = @CompletedDT
+            WHERE ID = @Id AND RowVersion = @RowVersion
+              AND CompletedDT IS NULL AND CancelDT IS NULL
+            """;
+
+        await using var connection = await _connectionFactory.CreateLotFinderConnectionAsync(ct);
+        var affectedRows = await connection.ExecuteAsync(sql, new
+        {
+            Id = id,
+            CompletedDT = DateTime.UtcNow,
+            RowVersion = rowVersion
+        });
+
+        var success = affectedRows > 0;
+
+        if (success)
+        {
+            _logger.LogInformation("Successfully completed manual sync request ID {Id}", id);
+        }
+        else
+        {
+            _logger.LogWarning(
+                "Failed to complete manual sync request ID {Id} - already completed/cancelled or version mismatch",
+                id);
+        }
+
+        return success;
+    }
+}