feat: implement ETL pipeline redesign and ConfigManager improvements

- Add pipeline registry with JSON-based configuration and hot-reload support - Implement manual sync request feature with API, client UI, and database - Improve ConfigManager: connection string dropdown in pipeline editor, step delete/reorder functionality, and fix JSON parsing for ConnectionStrings
2026-01-22 17:48:33 -05:00
parent 5a332232d0
commit 29ac56006d
82 changed files with 6257 additions and 296 deletions
@@ -0,0 +1,89 @@
+using JdeScoping.DataSync.Options;
+using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+
+namespace JdeScoping.DataSync.Services;
+
+/// <summary>
+/// Initializes the pipeline registry at application startup.
+/// Runs as a hosted service to properly handle async loading.
+/// </summary>
+public class PipelineRegistryInitializer : IHostedService
+{
+    private readonly IPipelineRegistry _registry;
+    private readonly IOptions<DataSyncOptions> _options;
+    private readonly IHostApplicationLifetime _lifetime;
+    private readonly ILogger<PipelineRegistryInitializer> _logger;
+
+    /// <summary>
+    /// Initializes a new instance of the <see cref="PipelineRegistryInitializer"/> class.
+    /// </summary>
+    public PipelineRegistryInitializer(
+        IPipelineRegistry registry,
+        IOptions<DataSyncOptions> options,
+        IHostApplicationLifetime lifetime,
+        ILogger<PipelineRegistryInitializer> logger)
+    {
+        _registry = registry ?? throw new ArgumentNullException(nameof(registry));
+        _options = options ?? throw new ArgumentNullException(nameof(options));
+        _lifetime = lifetime ?? throw new ArgumentNullException(nameof(lifetime));
+        _logger = logger ?? throw new ArgumentNullException(nameof(logger));
+    }
+
+    /// <inheritdoc/>
+    public async Task StartAsync(CancellationToken ct)
+    {
+        _logger.LogInformation("Loading pipeline definitions from {Directory}...",
+            _options.Value.PipelinesDirectory);
+
+        var result = await _registry.ReloadAsync(ct);
+
+        if (!result.Success && _options.Value.StrictPipelineValidation)
+        {
+            _logger.LogCritical(
+                "Pipeline validation failed with {ErrorCount} errors. " +
+                "Application will stop. Set StrictPipelineValidation=false to allow.",
+                result.Errors.Count);
+
+            foreach (var error in result.Errors)
+            {
+                _logger.LogError(
+                    "Pipeline {Name} ({File}): [{Type}] {Messages}",
+                    error.PipelineName,
+                    error.FileName,
+                    error.ErrorType,
+                    string.Join("; ", error.Messages));
+            }
+
+            _lifetime.StopApplication();
+            return;
+        }
+
+        if (result.Errors.Count > 0)
+        {
+            _logger.LogWarning(
+                "Pipeline loading completed with {ErrorCount} warnings",
+                result.Errors.Count);
+
+            foreach (var error in result.Errors)
+            {
+                _logger.LogWarning(
+                    "Pipeline {Name} ({File}): [{Type}] {Messages}",
+                    error.PipelineName,
+                    error.FileName,
+                    error.ErrorType,
+                    string.Join("; ", error.Messages));
+            }
+        }
+
+        _logger.LogInformation(
+            "Loaded {Count} pipelines ({Enabled} enabled) - version {Version}",
+            result.PipelinesLoaded,
+            _registry.GetEnabledPipelines().Count,
+            result.NewVersion);
+    }
+
+    /// <inheritdoc/>
+    public Task StopAsync(CancellationToken ct) => Task.CompletedTask;
+}