feat: extract DevEtl to JdeScoping.DataSync.Dev project

- Create JdeScoping.DataSync.Dev for sandbox testing ETL code - Create JdeScoping.DataSync.Dev.Tests for associated tests - Move 22 source files and 8 test files - Update namespaces from DevEtl to Dev - Add both projects to solution
2026-01-06 10:18:09 -05:00
parent 7ad4e3ec1c
commit 81b07ce027
37 changed files with 2458 additions and 83 deletions
@@ -1,37 +0,0 @@
-using JdeScoping.DataAccess.Interfaces;
-using JdeScoping.DataSync.Etl.Destinations;
-using JdeScoping.DataSync.Etl.Models;
-using JdeScoping.DataSync.Etl.Pipeline;
-using JdeScoping.DataSync.Etl.Sources;
-
-namespace JdeScoping.DataSync.DevEtl;
-
-/// <summary>
-/// Development ETL pipeline for the Branch table.
-/// </summary>
-public static class BranchDevEtl
-{
-    public static readonly string TableName = "Branch";
-    public static readonly string CacheFileName = "branch.json.zstd";
-
-    private static readonly JsonColumnSchema[] Schema =
-    [
-        new("Code", typeof(string), IsNullable: false),
-        new("Description", typeof(string), IsNullable: true),
-        new("LastUpdateDT", typeof(DateTime), IsNullable: false),
-    ];
-
-    public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
-    {
-        ArgumentNullException.ThrowIfNull(connectionFactory);
-
-        if (string.IsNullOrWhiteSpace(cacheFilePath))
-            throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath));
-
-        return new EtlPipelineBuilder()
-            .WithName($"{TableName}_Dev")
-            .WithSource(new JsonZstdFileSource(cacheFilePath, Schema))
-            .WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
-            .Build();
-    }
-}
@@ -1,80 +0,0 @@
-using JdeScoping.DataAccess.Interfaces;
-using JdeScoping.DataSync.Etl.Pipeline;
-using JdeScoping.DataSync.Etl.Results;
-using Microsoft.Extensions.Logging;
-
-namespace JdeScoping.DataSync.DevEtl;
-
-/// <summary>
-/// Registry for development ETL pipelines that load from cached JSON files.
-/// </summary>
-public class DevEtlRegistry
-{
-    private readonly IDbConnectionFactory _connectionFactory;
-    private readonly string _cacheDirectory;
-    private readonly ILogger<DevEtlRegistry>? _logger;
-
-    private readonly Dictionary<string, Func<IDbConnectionFactory, string, EtlPipeline>> _pipelineFactories = new(StringComparer.OrdinalIgnoreCase)
-    {
-        [BranchDevEtl.TableName] = (factory, cacheDir) =>
-            BranchDevEtl.Create(factory, Path.Combine(cacheDir, BranchDevEtl.CacheFileName)),
-    };
-
-    public DevEtlRegistry(
-        IDbConnectionFactory connectionFactory,
-        string cacheDirectory,
-        ILogger<DevEtlRegistry>? logger = null)
-    {
-        _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory));
-
-        if (string.IsNullOrWhiteSpace(cacheDirectory))
-            throw new ArgumentException("Cache directory is required.", nameof(cacheDirectory));
-
-        if (!Directory.Exists(cacheDirectory))
-            throw new DirectoryNotFoundException($"Cache directory not found: {cacheDirectory}");
-
-        _cacheDirectory = cacheDirectory;
-        _logger = logger;
-    }
-
-    public IEnumerable<string> GetAvailableTables() => _pipelineFactories.Keys;
-
-    public EtlPipeline GetPipeline(string tableName)
-    {
-        if (!_pipelineFactories.TryGetValue(tableName, out var factory))
-            throw new ArgumentException($"No pipeline registered for table '{tableName}'.", nameof(tableName));
-
-        return factory(_connectionFactory, _cacheDirectory);
-    }
-
-    public async Task<PipelineResult> RunAsync(string tableName, CancellationToken cancellationToken = default)
-    {
-        _logger?.LogInformation("Running dev ETL for {TableName}", tableName);
-
-        var pipeline = GetPipeline(tableName);
-        var result = await pipeline.ExecuteAsync(cancellationToken);
-
-        if (result.Success)
-            _logger?.LogInformation("Completed {TableName}: {Rows} rows in {Elapsed:g}",
-                tableName, result.TotalRows, result.Elapsed);
-        else
-            _logger?.LogError(result.Error, "Failed {TableName}: {Error}",
-                tableName, result.Error?.Message);
-
-        return result;
-    }
-
-    public async Task<IReadOnlyList<PipelineResult>> RunAllAsync(CancellationToken cancellationToken = default)
-    {
-        var results = new List<PipelineResult>();
-
-        foreach (var tableName in GetAvailableTables())
-        {
-            cancellationToken.ThrowIfCancellationRequested();
-            var result = await RunAsync(tableName, cancellationToken);
-            results.Add(result);
-        }
-
-        return results;
-    }
-}