diff --git a/NEW/JdeScoping.slnx b/NEW/JdeScoping.slnx index f10b23c..519e1d6 100644 --- a/NEW/JdeScoping.slnx +++ b/NEW/JdeScoping.slnx @@ -7,6 +7,7 @@ + @@ -20,6 +21,7 @@ + diff --git a/NEW/src/JdeScoping.DataSync/DevEtl/BranchDevEtl.cs b/NEW/src/JdeScoping.DataSync.Dev/BranchDevEtl.cs similarity index 94% rename from NEW/src/JdeScoping.DataSync/DevEtl/BranchDevEtl.cs rename to NEW/src/JdeScoping.DataSync.Dev/BranchDevEtl.cs index ac83fff..584de0e 100644 --- a/NEW/src/JdeScoping.DataSync/DevEtl/BranchDevEtl.cs +++ b/NEW/src/JdeScoping.DataSync.Dev/BranchDevEtl.cs @@ -4,7 +4,7 @@ using JdeScoping.DataSync.Etl.Models; using JdeScoping.DataSync.Etl.Pipeline; using JdeScoping.DataSync.Etl.Sources; -namespace JdeScoping.DataSync.DevEtl; +namespace JdeScoping.DataSync.Dev; /// /// Development ETL pipeline for the Branch table. diff --git a/NEW/src/JdeScoping.DataSync.Dev/DevEtlRegistry.cs b/NEW/src/JdeScoping.DataSync.Dev/DevEtlRegistry.cs new file mode 100644 index 0000000..207b1de --- /dev/null +++ b/NEW/src/JdeScoping.DataSync.Dev/DevEtlRegistry.cs @@ -0,0 +1,188 @@ +using System.Collections.Concurrent; +using JdeScoping.DataAccess.Interfaces; +using JdeScoping.DataSync.Etl.Pipeline; +using JdeScoping.DataSync.Etl.Results; +using Microsoft.Extensions.Logging; + +namespace JdeScoping.DataSync.Dev; + +/// +/// Registry for development ETL pipelines that load from cached JSON files. +/// +public class DevEtlRegistry +{ + private readonly IDbConnectionFactory _connectionFactory; + private readonly string _cacheDirectory; + private readonly ILogger? _logger; + + private readonly Dictionary> _pipelineFactories = new(StringComparer.OrdinalIgnoreCase) + { + // Small tables (< 1 MB) + [BranchDevEtl.TableName] = (factory, cacheDir) => + BranchDevEtl.Create(factory, Path.Combine(cacheDir, BranchDevEtl.CacheFileName)), + [OrgHierarchyDevEtl.TableName] = (factory, cacheDir) => + OrgHierarchyDevEtl.Create(factory, Path.Combine(cacheDir, OrgHierarchyDevEtl.CacheFileName)), + [WorkCenterDevEtl.TableName] = (factory, cacheDir) => + WorkCenterDevEtl.Create(factory, Path.Combine(cacheDir, WorkCenterDevEtl.CacheFileName)), + [ProfitCenterDevEtl.TableName] = (factory, cacheDir) => + ProfitCenterDevEtl.Create(factory, Path.Combine(cacheDir, ProfitCenterDevEtl.CacheFileName)), + // Medium tables (1-20 MB) + [JdeUserDevEtl.TableName] = (factory, cacheDir) => + JdeUserDevEtl.Create(factory, Path.Combine(cacheDir, JdeUserDevEtl.CacheFileName)), + [FunctionCodeDevEtl.TableName] = (factory, cacheDir) => + FunctionCodeDevEtl.Create(factory, Path.Combine(cacheDir, FunctionCodeDevEtl.CacheFileName)), + [ItemDevEtl.TableName] = (factory, cacheDir) => + ItemDevEtl.Create(factory, Path.Combine(cacheDir, ItemDevEtl.CacheFileName)), + [RouteMasterDevEtl.TableName] = (factory, cacheDir) => + RouteMasterDevEtl.Create(factory, Path.Combine(cacheDir, RouteMasterDevEtl.CacheFileName)), + // Large tables (20-200 MB) + [LotDevEtl.TableName] = (factory, cacheDir) => + LotDevEtl.Create(factory, Path.Combine(cacheDir, LotDevEtl.CacheFileName)), + [MisDataDevEtl.TableName] = (factory, cacheDir) => + MisDataDevEtl.Create(factory, Path.Combine(cacheDir, MisDataDevEtl.CacheFileName)), + [WorkOrderCurrDevEtl.TableName] = (factory, cacheDir) => + WorkOrderCurrDevEtl.Create(factory, Path.Combine(cacheDir, WorkOrderCurrDevEtl.CacheFileName)), + [WorkOrderHistDevEtl.TableName] = (factory, cacheDir) => + WorkOrderHistDevEtl.Create(factory, Path.Combine(cacheDir, WorkOrderHistDevEtl.CacheFileName)), + [LotUsageHistDevEtl.TableName] = (factory, cacheDir) => + LotUsageHistDevEtl.Create(factory, Path.Combine(cacheDir, LotUsageHistDevEtl.CacheFileName)), + [WorkOrderComponentHistDevEtl.TableName] = (factory, cacheDir) => + WorkOrderComponentHistDevEtl.Create(factory, Path.Combine(cacheDir, WorkOrderComponentHistDevEtl.CacheFileName)), + // Very large tables (200+ MB) + [WorkOrderStepHistDevEtl.TableName] = (factory, cacheDir) => + WorkOrderStepHistDevEtl.Create(factory, Path.Combine(cacheDir, WorkOrderStepHistDevEtl.CacheFileName)), + [WorkOrderComponentCurrDevEtl.TableName] = (factory, cacheDir) => + WorkOrderComponentCurrDevEtl.Create(factory, Path.Combine(cacheDir, WorkOrderComponentCurrDevEtl.CacheFileName)), + [WorkOrderRoutingDevEtl.TableName] = (factory, cacheDir) => + WorkOrderRoutingDevEtl.Create(factory, Path.Combine(cacheDir, WorkOrderRoutingDevEtl.CacheFileName)), + [LotUsageCurrDevEtl.TableName] = (factory, cacheDir) => + LotUsageCurrDevEtl.Create(factory, Path.Combine(cacheDir, LotUsageCurrDevEtl.CacheFileName)), + [WorkOrderStepCurrDevEtl.TableName] = (factory, cacheDir) => + WorkOrderStepCurrDevEtl.Create(factory, Path.Combine(cacheDir, WorkOrderStepCurrDevEtl.CacheFileName)), + [WorkOrderTimeHistDevEtl.TableName] = (factory, cacheDir) => + WorkOrderTimeHistDevEtl.Create(factory, Path.Combine(cacheDir, WorkOrderTimeHistDevEtl.CacheFileName)), + [WorkOrderTimeCurrDevEtl.TableName] = (factory, cacheDir) => + WorkOrderTimeCurrDevEtl.Create(factory, Path.Combine(cacheDir, WorkOrderTimeCurrDevEtl.CacheFileName)), + }; + + public DevEtlRegistry( + IDbConnectionFactory connectionFactory, + string cacheDirectory, + ILogger? logger = null) + { + _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); + + if (string.IsNullOrWhiteSpace(cacheDirectory)) + throw new ArgumentException("Cache directory is required.", nameof(cacheDirectory)); + + if (!Directory.Exists(cacheDirectory)) + throw new DirectoryNotFoundException($"Cache directory not found: {cacheDirectory}"); + + _cacheDirectory = cacheDirectory; + _logger = logger; + } + + public IEnumerable GetAvailableTables() => _pipelineFactories.Keys; + + public EtlPipeline GetPipeline(string tableName) + { + if (!_pipelineFactories.TryGetValue(tableName, out var factory)) + throw new ArgumentException($"No pipeline registered for table '{tableName}'.", nameof(tableName)); + + return factory(_connectionFactory, _cacheDirectory); + } + + public async Task RunAsync(string tableName, CancellationToken cancellationToken = default) + { + _logger?.LogInformation("Running dev ETL for {TableName}", tableName); + + var pipeline = GetPipeline(tableName); + var result = await pipeline.ExecuteAsync(cancellationToken); + + if (result.Success) + _logger?.LogInformation("Completed {TableName}: {Rows} rows in {Elapsed:g}", + tableName, result.TotalRows, result.Elapsed); + else + _logger?.LogError(result.Error, "Failed {TableName}: {Error}", + tableName, result.Error?.Message); + + return result; + } + + public async Task> RunAllAsync(CancellationToken cancellationToken = default) + { + var results = new List(); + + foreach (var tableName in GetAvailableTables()) + { + cancellationToken.ThrowIfCancellationRequested(); + var result = await RunAsync(tableName, cancellationToken); + results.Add(result); + } + + return results; + } + + /// + /// Runs all dev ETL pipelines with parallelization. + /// Small/medium tables run concurrently, very large tables run sequentially at the end. + /// + /// Maximum concurrent table loads (default 4). + /// Cancellation token. + public async Task> RunAllParallelAsync( + int maxDegreeOfParallelism = 4, + CancellationToken cancellationToken = default) + { + var results = new ConcurrentBag(); + using var semaphore = new SemaphoreSlim(maxDegreeOfParallelism); + + // Separate tables by size - run very large ones sequentially at the end + var smallMediumTables = GetAvailableTables() + .Where(t => !IsVeryLargeTable(t)) + .ToList(); + var veryLargeTables = GetAvailableTables() + .Where(IsVeryLargeTable) + .ToList(); + + _logger?.LogInformation( + "Running {ParallelCount} tables in parallel (max {MaxParallel}), then {SequentialCount} large tables sequentially", + smallMediumTables.Count, maxDegreeOfParallelism, veryLargeTables.Count); + + // Run small/medium tables in parallel + var tasks = smallMediumTables.Select(async tableName => + { + await semaphore.WaitAsync(cancellationToken); + try + { + var result = await RunAsync(tableName, cancellationToken); + results.Add(result); + } + finally + { + semaphore.Release(); + } + }); + + await Task.WhenAll(tasks); + + // Run very large tables sequentially (IO-bound, would contend) + foreach (var tableName in veryLargeTables) + { + cancellationToken.ThrowIfCancellationRequested(); + var result = await RunAsync(tableName, cancellationToken); + results.Add(result); + } + + return results.ToList(); + } + + /// + /// Identifies very large tables that should be loaded sequentially to avoid IO contention. + /// + private static bool IsVeryLargeTable(string tableName) => + tableName.Contains("WorkOrderTime", StringComparison.OrdinalIgnoreCase) || + tableName.Contains("WorkOrderStep", StringComparison.OrdinalIgnoreCase) || + tableName.Contains("WorkOrderRouting", StringComparison.OrdinalIgnoreCase) || + tableName.Contains("WorkOrderComponent", StringComparison.OrdinalIgnoreCase) || + tableName.Contains("LotUsage", StringComparison.OrdinalIgnoreCase); +} diff --git a/NEW/src/JdeScoping.DataSync.Dev/FunctionCodeDevEtl.cs b/NEW/src/JdeScoping.DataSync.Dev/FunctionCodeDevEtl.cs new file mode 100644 index 0000000..11d13c7 --- /dev/null +++ b/NEW/src/JdeScoping.DataSync.Dev/FunctionCodeDevEtl.cs @@ -0,0 +1,38 @@ +using JdeScoping.DataAccess.Interfaces; +using JdeScoping.DataSync.Etl.Destinations; +using JdeScoping.DataSync.Etl.Models; +using JdeScoping.DataSync.Etl.Pipeline; +using JdeScoping.DataSync.Etl.Sources; + +namespace JdeScoping.DataSync.Dev; + +/// +/// Development ETL pipeline for the FunctionCode table. +/// Schema from: Scripts/005_CreateFunctionCodeTable.sql +/// +public static class FunctionCodeDevEtl +{ + public static readonly string TableName = "FunctionCode"; + public static readonly string CacheFileName = "functioncode.json.zstd"; + + private static readonly JsonColumnSchema[] Schema = + [ + new("Code", typeof(string), IsNullable: false), + new("Description", typeof(string), IsNullable: true), + new("LastUpdateDT", typeof(DateTime), IsNullable: false), + ]; + + public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath) + { + ArgumentNullException.ThrowIfNull(connectionFactory); + + if (string.IsNullOrWhiteSpace(cacheFilePath)) + throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath)); + + return new EtlPipelineBuilder() + .WithName($"{TableName}_Dev") + .WithSource(new JsonZstdFileSource(cacheFilePath, Schema)) + .WithDestination(new DbBulkImportDestination(connectionFactory, TableName)) + .Build(); + } +} diff --git a/NEW/src/JdeScoping.DataSync.Dev/ItemDevEtl.cs b/NEW/src/JdeScoping.DataSync.Dev/ItemDevEtl.cs new file mode 100644 index 0000000..8d58929 --- /dev/null +++ b/NEW/src/JdeScoping.DataSync.Dev/ItemDevEtl.cs @@ -0,0 +1,41 @@ +using JdeScoping.DataAccess.Interfaces; +using JdeScoping.DataSync.Etl.Destinations; +using JdeScoping.DataSync.Etl.Models; +using JdeScoping.DataSync.Etl.Pipeline; +using JdeScoping.DataSync.Etl.Sources; + +namespace JdeScoping.DataSync.Dev; + +/// +/// Development ETL pipeline for the Item table. +/// Schema from: Scripts/008_CreateItemTable.sql +/// +public static class ItemDevEtl +{ + public static readonly string TableName = "Item"; + public static readonly string CacheFileName = "item.json.zstd"; + + private static readonly JsonColumnSchema[] Schema = + [ + new("ShortItemNumber", typeof(long), IsNullable: false), + new("ItemNumber", typeof(string), IsNullable: false), + new("Description", typeof(string), IsNullable: true), + new("PlanningFamily", typeof(string), IsNullable: true), + new("StockingType", typeof(string), IsNullable: true), + new("LastUpdateDT", typeof(DateTime), IsNullable: false), + ]; + + public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath) + { + ArgumentNullException.ThrowIfNull(connectionFactory); + + if (string.IsNullOrWhiteSpace(cacheFilePath)) + throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath)); + + return new EtlPipelineBuilder() + .WithName($"{TableName}_Dev") + .WithSource(new JsonZstdFileSource(cacheFilePath, Schema)) + .WithDestination(new DbBulkImportDestination(connectionFactory, TableName)) + .Build(); + } +} diff --git a/NEW/src/JdeScoping.DataSync.Dev/JdeScoping.DataSync.Dev.csproj b/NEW/src/JdeScoping.DataSync.Dev/JdeScoping.DataSync.Dev.csproj new file mode 100644 index 0000000..223ab5c --- /dev/null +++ b/NEW/src/JdeScoping.DataSync.Dev/JdeScoping.DataSync.Dev.csproj @@ -0,0 +1,13 @@ + + + + net10.0 + enable + enable + + + + + + + diff --git a/NEW/src/JdeScoping.DataSync.Dev/JdeUserDevEtl.cs b/NEW/src/JdeScoping.DataSync.Dev/JdeUserDevEtl.cs new file mode 100644 index 0000000..419247c --- /dev/null +++ b/NEW/src/JdeScoping.DataSync.Dev/JdeUserDevEtl.cs @@ -0,0 +1,39 @@ +using JdeScoping.DataAccess.Interfaces; +using JdeScoping.DataSync.Etl.Destinations; +using JdeScoping.DataSync.Etl.Models; +using JdeScoping.DataSync.Etl.Pipeline; +using JdeScoping.DataSync.Etl.Sources; + +namespace JdeScoping.DataSync.Dev; + +/// +/// Development ETL pipeline for the JdeUser table. +/// Schema from: Scripts/009_CreateJdeUserTable.sql +/// +public static class JdeUserDevEtl +{ + public static readonly string TableName = "JdeUser"; + public static readonly string CacheFileName = "jdeuser.json.zstd"; + + private static readonly JsonColumnSchema[] Schema = + [ + new("AddressNumber", typeof(long), IsNullable: false), + new("UserID", typeof(string), IsNullable: true), + new("FullName", typeof(string), IsNullable: false), + new("LastUpdateDT", typeof(DateTime), IsNullable: false), + ]; + + public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath) + { + ArgumentNullException.ThrowIfNull(connectionFactory); + + if (string.IsNullOrWhiteSpace(cacheFilePath)) + throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath)); + + return new EtlPipelineBuilder() + .WithName($"{TableName}_Dev") + .WithSource(new JsonZstdFileSource(cacheFilePath, Schema)) + .WithDestination(new DbBulkImportDestination(connectionFactory, TableName)) + .Build(); + } +} diff --git a/NEW/src/JdeScoping.DataSync.Dev/LotDevEtl.cs b/NEW/src/JdeScoping.DataSync.Dev/LotDevEtl.cs new file mode 100644 index 0000000..643c4e4 --- /dev/null +++ b/NEW/src/JdeScoping.DataSync.Dev/LotDevEtl.cs @@ -0,0 +1,45 @@ +using JdeScoping.DataAccess.Interfaces; +using JdeScoping.DataSync.Etl.Destinations; +using JdeScoping.DataSync.Etl.Models; +using JdeScoping.DataSync.Etl.Pipeline; +using JdeScoping.DataSync.Etl.Sources; + +namespace JdeScoping.DataSync.Dev; + +/// +/// Development ETL pipeline for the Lot table. +/// Schema from: Scripts/013_CreateLotTable.sql +/// +public static class LotDevEtl +{ + public static readonly string TableName = "Lot"; + public static readonly string CacheFileName = "lot.json.zstd"; + + private static readonly JsonColumnSchema[] Schema = + [ + new("LotNumber", typeof(string), IsNullable: false), + new("BranchCode", typeof(string), IsNullable: false), + new("ShortItemNumber", typeof(long), IsNullable: false), + new("ItemNumber", typeof(string), IsNullable: true), + new("SupplierCode", typeof(long), IsNullable: false), + new("StatusCode", typeof(string), IsNullable: true), + new("Memo1", typeof(string), IsNullable: true), + new("Memo2", typeof(string), IsNullable: true), + new("Memo3", typeof(string), IsNullable: true), + new("LastUpdateDT", typeof(DateTime), IsNullable: false), + ]; + + public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath) + { + ArgumentNullException.ThrowIfNull(connectionFactory); + + if (string.IsNullOrWhiteSpace(cacheFilePath)) + throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath)); + + return new EtlPipelineBuilder() + .WithName($"{TableName}_Dev") + .WithSource(new JsonZstdFileSource(cacheFilePath, Schema)) + .WithDestination(new DbBulkImportDestination(connectionFactory, TableName)) + .Build(); + } +} diff --git a/NEW/src/JdeScoping.DataSync.Dev/LotUsageCurrDevEtl.cs b/NEW/src/JdeScoping.DataSync.Dev/LotUsageCurrDevEtl.cs new file mode 100644 index 0000000..7d580e9 --- /dev/null +++ b/NEW/src/JdeScoping.DataSync.Dev/LotUsageCurrDevEtl.cs @@ -0,0 +1,42 @@ +using JdeScoping.DataAccess.Interfaces; +using JdeScoping.DataSync.Etl.Destinations; +using JdeScoping.DataSync.Etl.Models; +using JdeScoping.DataSync.Etl.Pipeline; +using JdeScoping.DataSync.Etl.Sources; + +namespace JdeScoping.DataSync.Dev; + +/// +/// Development ETL pipeline for the LotUsage_Curr table. +/// Schema from: Scripts/024_CreateLotUsageCurrTable.sql +/// +public static class LotUsageCurrDevEtl +{ + public static readonly string TableName = "LotUsage_Curr"; + public static readonly string CacheFileName = "lotusage_curr.json.zstd"; + + private static readonly JsonColumnSchema[] Schema = + [ + new("UniqueID", typeof(long), IsNullable: false), + new("WorkOrderNumber", typeof(long), IsNullable: false), + new("LotNumber", typeof(string), IsNullable: false), + new("BranchCode", typeof(string), IsNullable: true), + new("ShortItemNumber", typeof(long), IsNullable: false), + new("Quantity", typeof(decimal), IsNullable: false), + new("LastUpdateDT", typeof(DateTime), IsNullable: false), + ]; + + public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath) + { + ArgumentNullException.ThrowIfNull(connectionFactory); + + if (string.IsNullOrWhiteSpace(cacheFilePath)) + throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath)); + + return new EtlPipelineBuilder() + .WithName($"{TableName}_Dev") + .WithSource(new JsonZstdFileSource(cacheFilePath, Schema)) + .WithDestination(new DbBulkImportDestination(connectionFactory, TableName)) + .Build(); + } +} diff --git a/NEW/src/JdeScoping.DataSync.Dev/LotUsageHistDevEtl.cs b/NEW/src/JdeScoping.DataSync.Dev/LotUsageHistDevEtl.cs new file mode 100644 index 0000000..7c9079e --- /dev/null +++ b/NEW/src/JdeScoping.DataSync.Dev/LotUsageHistDevEtl.cs @@ -0,0 +1,42 @@ +using JdeScoping.DataAccess.Interfaces; +using JdeScoping.DataSync.Etl.Destinations; +using JdeScoping.DataSync.Etl.Models; +using JdeScoping.DataSync.Etl.Pipeline; +using JdeScoping.DataSync.Etl.Sources; + +namespace JdeScoping.DataSync.Dev; + +/// +/// Development ETL pipeline for the LotUsage_Hist table. +/// Schema from: Scripts/025_CreateLotUsageHistTable.sql +/// +public static class LotUsageHistDevEtl +{ + public static readonly string TableName = "LotUsage_Hist"; + public static readonly string CacheFileName = "lotusage_hist.json.zstd"; + + private static readonly JsonColumnSchema[] Schema = + [ + new("UniqueID", typeof(long), IsNullable: false), + new("WorkOrderNumber", typeof(long), IsNullable: false), + new("LotNumber", typeof(string), IsNullable: false), + new("BranchCode", typeof(string), IsNullable: true), + new("ShortItemNumber", typeof(long), IsNullable: false), + new("Quantity", typeof(decimal), IsNullable: false), + new("LastUpdateDT", typeof(DateTime), IsNullable: false), + ]; + + public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath) + { + ArgumentNullException.ThrowIfNull(connectionFactory); + + if (string.IsNullOrWhiteSpace(cacheFilePath)) + throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath)); + + return new EtlPipelineBuilder() + .WithName($"{TableName}_Dev") + .WithSource(new JsonZstdFileSource(cacheFilePath, Schema)) + .WithDestination(new DbBulkImportDestination(connectionFactory, TableName)) + .Build(); + } +} diff --git a/NEW/src/JdeScoping.DataSync.Dev/MisDataDevEtl.cs b/NEW/src/JdeScoping.DataSync.Dev/MisDataDevEtl.cs new file mode 100644 index 0000000..4d07326 --- /dev/null +++ b/NEW/src/JdeScoping.DataSync.Dev/MisDataDevEtl.cs @@ -0,0 +1,49 @@ +using JdeScoping.DataAccess.Interfaces; +using JdeScoping.DataSync.Etl.Destinations; +using JdeScoping.DataSync.Etl.Models; +using JdeScoping.DataSync.Etl.Pipeline; +using JdeScoping.DataSync.Etl.Sources; + +namespace JdeScoping.DataSync.Dev; + +/// +/// Development ETL pipeline for the MisData table. +/// Schema from: Scripts/012_CreateMisDataTable.sql +/// +public static class MisDataDevEtl +{ + public static readonly string TableName = "MisData"; + public static readonly string CacheFileName = "misdata.json.zstd"; + + private static readonly JsonColumnSchema[] Schema = + [ + new("ItemNumber", typeof(string), IsNullable: false), + new("BranchCode", typeof(string), IsNullable: false), + new("SequenceNumber", typeof(string), IsNullable: false), + new("MisNumber", typeof(string), IsNullable: false), + new("RevID", typeof(string), IsNullable: false), + new("CharNumber", typeof(string), IsNullable: false), + new("TestDescription", typeof(string), IsNullable: true), + new("SamplingType", typeof(string), IsNullable: true), + new("SamplingValue", typeof(string), IsNullable: true), + new("ToolsGauges", typeof(string), IsNullable: true), + new("WorkInstructions", typeof(string), IsNullable: true), + new("Status", typeof(string), IsNullable: false), + new("ReleaseDate", typeof(DateTime), IsNullable: true), + new("ObsoleteDate", typeof(DateTime), IsNullable: true), + ]; + + public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath) + { + ArgumentNullException.ThrowIfNull(connectionFactory); + + if (string.IsNullOrWhiteSpace(cacheFilePath)) + throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath)); + + return new EtlPipelineBuilder() + .WithName($"{TableName}_Dev") + .WithSource(new JsonZstdFileSource(cacheFilePath, Schema)) + .WithDestination(new DbBulkImportDestination(connectionFactory, TableName)) + .Build(); + } +} diff --git a/NEW/src/JdeScoping.DataSync.Dev/OrgHierarchyDevEtl.cs b/NEW/src/JdeScoping.DataSync.Dev/OrgHierarchyDevEtl.cs new file mode 100644 index 0000000..f198da0 --- /dev/null +++ b/NEW/src/JdeScoping.DataSync.Dev/OrgHierarchyDevEtl.cs @@ -0,0 +1,39 @@ +using JdeScoping.DataAccess.Interfaces; +using JdeScoping.DataSync.Etl.Destinations; +using JdeScoping.DataSync.Etl.Models; +using JdeScoping.DataSync.Etl.Pipeline; +using JdeScoping.DataSync.Etl.Sources; + +namespace JdeScoping.DataSync.Dev; + +/// +/// Development ETL pipeline for the OrgHierarchy table. +/// Schema from: Scripts/010_CreateOrgHierarchyTable.sql +/// +public static class OrgHierarchyDevEtl +{ + public static readonly string TableName = "OrgHierarchy"; + public static readonly string CacheFileName = "orghierarchy.json.zstd"; + + private static readonly JsonColumnSchema[] Schema = + [ + new("WorkCenterCode", typeof(string), IsNullable: false), + new("BranchCode", typeof(string), IsNullable: false), + new("ProfitCenterCode", typeof(string), IsNullable: false), + new("LastUpdateDT", typeof(DateTime), IsNullable: false), + ]; + + public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath) + { + ArgumentNullException.ThrowIfNull(connectionFactory); + + if (string.IsNullOrWhiteSpace(cacheFilePath)) + throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath)); + + return new EtlPipelineBuilder() + .WithName($"{TableName}_Dev") + .WithSource(new JsonZstdFileSource(cacheFilePath, Schema)) + .WithDestination(new DbBulkImportDestination(connectionFactory, TableName)) + .Build(); + } +} diff --git a/NEW/src/JdeScoping.DataSync.Dev/ProfitCenterDevEtl.cs b/NEW/src/JdeScoping.DataSync.Dev/ProfitCenterDevEtl.cs new file mode 100644 index 0000000..7dbb96a --- /dev/null +++ b/NEW/src/JdeScoping.DataSync.Dev/ProfitCenterDevEtl.cs @@ -0,0 +1,38 @@ +using JdeScoping.DataAccess.Interfaces; +using JdeScoping.DataSync.Etl.Destinations; +using JdeScoping.DataSync.Etl.Models; +using JdeScoping.DataSync.Etl.Pipeline; +using JdeScoping.DataSync.Etl.Sources; + +namespace JdeScoping.DataSync.Dev; + +/// +/// Development ETL pipeline for the ProfitCenter table. +/// Schema from: Scripts/006_CreateProfitCenterTable.sql +/// +public static class ProfitCenterDevEtl +{ + public static readonly string TableName = "ProfitCenter"; + public static readonly string CacheFileName = "profitcenter.json.zstd"; + + private static readonly JsonColumnSchema[] Schema = + [ + new("Code", typeof(string), IsNullable: false), + new("Description", typeof(string), IsNullable: true), + new("LastUpdateDT", typeof(DateTime), IsNullable: false), + ]; + + public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath) + { + ArgumentNullException.ThrowIfNull(connectionFactory); + + if (string.IsNullOrWhiteSpace(cacheFilePath)) + throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath)); + + return new EtlPipelineBuilder() + .WithName($"{TableName}_Dev") + .WithSource(new JsonZstdFileSource(cacheFilePath, Schema)) + .WithDestination(new DbBulkImportDestination(connectionFactory, TableName)) + .Build(); + } +} diff --git a/NEW/src/JdeScoping.DataSync.Dev/RouteMasterDevEtl.cs b/NEW/src/JdeScoping.DataSync.Dev/RouteMasterDevEtl.cs new file mode 100644 index 0000000..04f8116 --- /dev/null +++ b/NEW/src/JdeScoping.DataSync.Dev/RouteMasterDevEtl.cs @@ -0,0 +1,44 @@ +using JdeScoping.DataAccess.Interfaces; +using JdeScoping.DataSync.Etl.Destinations; +using JdeScoping.DataSync.Etl.Models; +using JdeScoping.DataSync.Etl.Pipeline; +using JdeScoping.DataSync.Etl.Sources; + +namespace JdeScoping.DataSync.Dev; + +/// +/// Development ETL pipeline for the RouteMaster table. +/// Schema from: Scripts/011_CreateRouteMasterTable.sql +/// +public static class RouteMasterDevEtl +{ + public static readonly string TableName = "RouteMaster"; + public static readonly string CacheFileName = "routemaster.json.zstd"; + + private static readonly JsonColumnSchema[] Schema = + [ + new("BranchCode", typeof(string), IsNullable: false), + new("ItemNumber", typeof(string), IsNullable: false), + new("RoutingType", typeof(string), IsNullable: false), + new("SequenceNumber", typeof(decimal), IsNullable: false), + new("FunctionCode", typeof(string), IsNullable: true), + new("WorkCenterCode", typeof(string), IsNullable: true), + new("StartDate", typeof(DateTime), IsNullable: false), + new("EndDate", typeof(DateTime), IsNullable: true), + new("LastUpdateDT", typeof(DateTime), IsNullable: false), + ]; + + public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath) + { + ArgumentNullException.ThrowIfNull(connectionFactory); + + if (string.IsNullOrWhiteSpace(cacheFilePath)) + throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath)); + + return new EtlPipelineBuilder() + .WithName($"{TableName}_Dev") + .WithSource(new JsonZstdFileSource(cacheFilePath, Schema)) + .WithDestination(new DbBulkImportDestination(connectionFactory, TableName)) + .Build(); + } +} diff --git a/NEW/src/JdeScoping.DataSync.Dev/WorkCenterDevEtl.cs b/NEW/src/JdeScoping.DataSync.Dev/WorkCenterDevEtl.cs new file mode 100644 index 0000000..21a121b --- /dev/null +++ b/NEW/src/JdeScoping.DataSync.Dev/WorkCenterDevEtl.cs @@ -0,0 +1,38 @@ +using JdeScoping.DataAccess.Interfaces; +using JdeScoping.DataSync.Etl.Destinations; +using JdeScoping.DataSync.Etl.Models; +using JdeScoping.DataSync.Etl.Pipeline; +using JdeScoping.DataSync.Etl.Sources; + +namespace JdeScoping.DataSync.Dev; + +/// +/// Development ETL pipeline for the WorkCenter table. +/// Schema from: Scripts/007_CreateWorkCenterTable.sql +/// +public static class WorkCenterDevEtl +{ + public static readonly string TableName = "WorkCenter"; + public static readonly string CacheFileName = "workcenter.json.zstd"; + + private static readonly JsonColumnSchema[] Schema = + [ + new("Code", typeof(string), IsNullable: false), + new("Description", typeof(string), IsNullable: true), + new("LastUpdateDT", typeof(DateTime), IsNullable: false), + ]; + + public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath) + { + ArgumentNullException.ThrowIfNull(connectionFactory); + + if (string.IsNullOrWhiteSpace(cacheFilePath)) + throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath)); + + return new EtlPipelineBuilder() + .WithName($"{TableName}_Dev") + .WithSource(new JsonZstdFileSource(cacheFilePath, Schema)) + .WithDestination(new DbBulkImportDestination(connectionFactory, TableName)) + .Build(); + } +} diff --git a/NEW/src/JdeScoping.DataSync.Dev/WorkOrderComponentCurrDevEtl.cs b/NEW/src/JdeScoping.DataSync.Dev/WorkOrderComponentCurrDevEtl.cs new file mode 100644 index 0000000..04a6409 --- /dev/null +++ b/NEW/src/JdeScoping.DataSync.Dev/WorkOrderComponentCurrDevEtl.cs @@ -0,0 +1,42 @@ +using JdeScoping.DataAccess.Interfaces; +using JdeScoping.DataSync.Etl.Destinations; +using JdeScoping.DataSync.Etl.Models; +using JdeScoping.DataSync.Etl.Pipeline; +using JdeScoping.DataSync.Etl.Sources; + +namespace JdeScoping.DataSync.Dev; + +/// +/// Development ETL pipeline for the WorkOrderComponent_Curr table. +/// Schema from: Scripts/021_CreateWorkOrderComponentCurrTable.sql +/// +public static class WorkOrderComponentCurrDevEtl +{ + public static readonly string TableName = "WorkOrderComponent_Curr"; + public static readonly string CacheFileName = "workordercomponent_curr.json.zstd"; + + private static readonly JsonColumnSchema[] Schema = + [ + new("UniqueID", typeof(long), IsNullable: false), + new("WorkOrderNumber", typeof(long), IsNullable: false), + new("LotNumber", typeof(string), IsNullable: false), + new("BranchCode", typeof(string), IsNullable: true), + new("ShortItemNumber", typeof(long), IsNullable: false), + new("Quantity", typeof(decimal), IsNullable: false), + new("LastUpdateDT", typeof(DateTime), IsNullable: false), + ]; + + public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath) + { + ArgumentNullException.ThrowIfNull(connectionFactory); + + if (string.IsNullOrWhiteSpace(cacheFilePath)) + throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath)); + + return new EtlPipelineBuilder() + .WithName($"{TableName}_Dev") + .WithSource(new JsonZstdFileSource(cacheFilePath, Schema)) + .WithDestination(new DbBulkImportDestination(connectionFactory, TableName)) + .Build(); + } +} diff --git a/NEW/src/JdeScoping.DataSync.Dev/WorkOrderComponentHistDevEtl.cs b/NEW/src/JdeScoping.DataSync.Dev/WorkOrderComponentHistDevEtl.cs new file mode 100644 index 0000000..e19ad48 --- /dev/null +++ b/NEW/src/JdeScoping.DataSync.Dev/WorkOrderComponentHistDevEtl.cs @@ -0,0 +1,42 @@ +using JdeScoping.DataAccess.Interfaces; +using JdeScoping.DataSync.Etl.Destinations; +using JdeScoping.DataSync.Etl.Models; +using JdeScoping.DataSync.Etl.Pipeline; +using JdeScoping.DataSync.Etl.Sources; + +namespace JdeScoping.DataSync.Dev; + +/// +/// Development ETL pipeline for the WorkOrderComponent_Hist table. +/// Schema from: Scripts/022_CreateWorkOrderComponentHistTable.sql +/// +public static class WorkOrderComponentHistDevEtl +{ + public static readonly string TableName = "WorkOrderComponent_Hist"; + public static readonly string CacheFileName = "workordercomponent_hist.json.zstd"; + + private static readonly JsonColumnSchema[] Schema = + [ + new("UniqueID", typeof(long), IsNullable: false), + new("WorkOrderNumber", typeof(long), IsNullable: false), + new("LotNumber", typeof(string), IsNullable: false), + new("BranchCode", typeof(string), IsNullable: true), + new("ShortItemNumber", typeof(long), IsNullable: false), + new("Quantity", typeof(decimal), IsNullable: false), + new("LastUpdateDT", typeof(DateTime), IsNullable: false), + ]; + + public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath) + { + ArgumentNullException.ThrowIfNull(connectionFactory); + + if (string.IsNullOrWhiteSpace(cacheFilePath)) + throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath)); + + return new EtlPipelineBuilder() + .WithName($"{TableName}_Dev") + .WithSource(new JsonZstdFileSource(cacheFilePath, Schema)) + .WithDestination(new DbBulkImportDestination(connectionFactory, TableName)) + .Build(); + } +} diff --git a/NEW/src/JdeScoping.DataSync.Dev/WorkOrderCurrDevEtl.cs b/NEW/src/JdeScoping.DataSync.Dev/WorkOrderCurrDevEtl.cs new file mode 100644 index 0000000..eab6217 --- /dev/null +++ b/NEW/src/JdeScoping.DataSync.Dev/WorkOrderCurrDevEtl.cs @@ -0,0 +1,50 @@ +using JdeScoping.DataAccess.Interfaces; +using JdeScoping.DataSync.Etl.Destinations; +using JdeScoping.DataSync.Etl.Models; +using JdeScoping.DataSync.Etl.Pipeline; +using JdeScoping.DataSync.Etl.Sources; + +namespace JdeScoping.DataSync.Dev; + +/// +/// Development ETL pipeline for the WorkOrder_Curr table. +/// Schema from: Scripts/015_CreateWorkOrderCurrTable.sql +/// +public static class WorkOrderCurrDevEtl +{ + public static readonly string TableName = "WorkOrder_Curr"; + public static readonly string CacheFileName = "workorder_curr.json.zstd"; + + private static readonly JsonColumnSchema[] Schema = + [ + new("WorkOrderNumber", typeof(long), IsNullable: false), + new("BranchCode", typeof(string), IsNullable: true), + new("LotNumber", typeof(string), IsNullable: true), + new("ItemNumber", typeof(string), IsNullable: true), + new("ShortItemNumber", typeof(long), IsNullable: false), + new("ParentWorkOrderNumber", typeof(string), IsNullable: true), + new("OrderQuantity", typeof(decimal), IsNullable: false), + new("HeldQuantity", typeof(decimal), IsNullable: false), + new("ShippedQuantity", typeof(decimal), IsNullable: false), + new("StatusCode", typeof(string), IsNullable: true), + new("StatusCodeUpdateDT", typeof(DateTime), IsNullable: true), + new("IssueDate", typeof(DateTime), IsNullable: false), + new("StartDate", typeof(DateTime), IsNullable: false), + new("RoutingType", typeof(string), IsNullable: true), + new("LastUpdateDT", typeof(DateTime), IsNullable: false), + ]; + + public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath) + { + ArgumentNullException.ThrowIfNull(connectionFactory); + + if (string.IsNullOrWhiteSpace(cacheFilePath)) + throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath)); + + return new EtlPipelineBuilder() + .WithName($"{TableName}_Dev") + .WithSource(new JsonZstdFileSource(cacheFilePath, Schema)) + .WithDestination(new DbBulkImportDestination(connectionFactory, TableName)) + .Build(); + } +} diff --git a/NEW/src/JdeScoping.DataSync.Dev/WorkOrderHistDevEtl.cs b/NEW/src/JdeScoping.DataSync.Dev/WorkOrderHistDevEtl.cs new file mode 100644 index 0000000..0c8b3fc --- /dev/null +++ b/NEW/src/JdeScoping.DataSync.Dev/WorkOrderHistDevEtl.cs @@ -0,0 +1,50 @@ +using JdeScoping.DataAccess.Interfaces; +using JdeScoping.DataSync.Etl.Destinations; +using JdeScoping.DataSync.Etl.Models; +using JdeScoping.DataSync.Etl.Pipeline; +using JdeScoping.DataSync.Etl.Sources; + +namespace JdeScoping.DataSync.Dev; + +/// +/// Development ETL pipeline for the WorkOrder_Hist table. +/// Schema from: Scripts/016_CreateWorkOrderHistTable.sql +/// +public static class WorkOrderHistDevEtl +{ + public static readonly string TableName = "WorkOrder_Hist"; + public static readonly string CacheFileName = "workorder_hist.json.zstd"; + + private static readonly JsonColumnSchema[] Schema = + [ + new("WorkOrderNumber", typeof(long), IsNullable: false), + new("BranchCode", typeof(string), IsNullable: true), + new("LotNumber", typeof(string), IsNullable: true), + new("ItemNumber", typeof(string), IsNullable: true), + new("ShortItemNumber", typeof(long), IsNullable: false), + new("ParentWorkOrderNumber", typeof(string), IsNullable: true), + new("OrderQuantity", typeof(decimal), IsNullable: false), + new("HeldQuantity", typeof(decimal), IsNullable: false), + new("ShippedQuantity", typeof(decimal), IsNullable: false), + new("StatusCode", typeof(string), IsNullable: true), + new("StatusCodeUpdateDT", typeof(DateTime), IsNullable: true), + new("IssueDate", typeof(DateTime), IsNullable: false), + new("StartDate", typeof(DateTime), IsNullable: false), + new("RoutingType", typeof(string), IsNullable: true), + new("LastUpdateDT", typeof(DateTime), IsNullable: false), + ]; + + public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath) + { + ArgumentNullException.ThrowIfNull(connectionFactory); + + if (string.IsNullOrWhiteSpace(cacheFilePath)) + throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath)); + + return new EtlPipelineBuilder() + .WithName($"{TableName}_Dev") + .WithSource(new JsonZstdFileSource(cacheFilePath, Schema)) + .WithDestination(new DbBulkImportDestination(connectionFactory, TableName)) + .Build(); + } +} diff --git a/NEW/src/JdeScoping.DataSync.Dev/WorkOrderRoutingDevEtl.cs b/NEW/src/JdeScoping.DataSync.Dev/WorkOrderRoutingDevEtl.cs new file mode 100644 index 0000000..1b01255 --- /dev/null +++ b/NEW/src/JdeScoping.DataSync.Dev/WorkOrderRoutingDevEtl.cs @@ -0,0 +1,48 @@ +using JdeScoping.DataAccess.Interfaces; +using JdeScoping.DataSync.Etl.Destinations; +using JdeScoping.DataSync.Etl.Models; +using JdeScoping.DataSync.Etl.Pipeline; +using JdeScoping.DataSync.Etl.Sources; + +namespace JdeScoping.DataSync.Dev; + +/// +/// Development ETL pipeline for the WorkOrderRouting table. +/// Schema from: Scripts/023_CreateWorkOrderRoutingTable.sql +/// +public static class WorkOrderRoutingDevEtl +{ + public static readonly string TableName = "WorkOrderRouting"; + public static readonly string CacheFileName = "workorderrouting.json.zstd"; + + private static readonly JsonColumnSchema[] Schema = + [ + new("UserID", typeof(string), IsNullable: false), + new("BatchNumber", typeof(string), IsNullable: false), + new("TransactionNumber", typeof(string), IsNullable: false), + new("LineNumber", typeof(int), IsNullable: false), + new("StepNumber", typeof(decimal), IsNullable: false), + new("WorkCenterCode", typeof(string), IsNullable: false), + new("WorkOrderNumber", typeof(long), IsNullable: false), + new("RoutingType", typeof(string), IsNullable: true), + new("BranchCode", typeof(string), IsNullable: true), + new("StepDescription", typeof(string), IsNullable: true), + new("FunctionCode", typeof(string), IsNullable: true), + new("TransactionDate", typeof(DateTime), IsNullable: false), + new("LastUpdateDT", typeof(DateTime), IsNullable: false), + ]; + + public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath) + { + ArgumentNullException.ThrowIfNull(connectionFactory); + + if (string.IsNullOrWhiteSpace(cacheFilePath)) + throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath)); + + return new EtlPipelineBuilder() + .WithName($"{TableName}_Dev") + .WithSource(new JsonZstdFileSource(cacheFilePath, Schema)) + .WithDestination(new DbBulkImportDestination(connectionFactory, TableName)) + .Build(); + } +} diff --git a/NEW/src/JdeScoping.DataSync.Dev/WorkOrderStepCurrDevEtl.cs b/NEW/src/JdeScoping.DataSync.Dev/WorkOrderStepCurrDevEtl.cs new file mode 100644 index 0000000..52c7015 --- /dev/null +++ b/NEW/src/JdeScoping.DataSync.Dev/WorkOrderStepCurrDevEtl.cs @@ -0,0 +1,46 @@ +using JdeScoping.DataAccess.Interfaces; +using JdeScoping.DataSync.Etl.Destinations; +using JdeScoping.DataSync.Etl.Models; +using JdeScoping.DataSync.Etl.Pipeline; +using JdeScoping.DataSync.Etl.Sources; + +namespace JdeScoping.DataSync.Dev; + +/// +/// Development ETL pipeline for the WorkOrderStep_Curr table. +/// Schema from: Scripts/017_CreateWorkOrderStepCurrTable.sql +/// +public static class WorkOrderStepCurrDevEtl +{ + public static readonly string TableName = "WorkOrderStep_Curr"; + public static readonly string CacheFileName = "workorderstep_curr.json.zstd"; + + private static readonly JsonColumnSchema[] Schema = + [ + new("WorkOrderNumber", typeof(long), IsNullable: false), + new("WorkCenterCode", typeof(string), IsNullable: false), + new("StepNumber", typeof(decimal), IsNullable: false), + new("StepTypeCode", typeof(string), IsNullable: false), + new("BranchCode", typeof(string), IsNullable: false), + new("StepDescription", typeof(string), IsNullable: true), + new("StartDT", typeof(DateTime), IsNullable: true), + new("EndDT", typeof(DateTime), IsNullable: true), + new("FunctionCode", typeof(string), IsNullable: true), + new("ScrappedQuantity", typeof(decimal), IsNullable: false), + new("LastUpdateDT", typeof(DateTime), IsNullable: false), + ]; + + public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath) + { + ArgumentNullException.ThrowIfNull(connectionFactory); + + if (string.IsNullOrWhiteSpace(cacheFilePath)) + throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath)); + + return new EtlPipelineBuilder() + .WithName($"{TableName}_Dev") + .WithSource(new JsonZstdFileSource(cacheFilePath, Schema)) + .WithDestination(new DbBulkImportDestination(connectionFactory, TableName)) + .Build(); + } +} diff --git a/NEW/src/JdeScoping.DataSync.Dev/WorkOrderStepHistDevEtl.cs b/NEW/src/JdeScoping.DataSync.Dev/WorkOrderStepHistDevEtl.cs new file mode 100644 index 0000000..63b4be5 --- /dev/null +++ b/NEW/src/JdeScoping.DataSync.Dev/WorkOrderStepHistDevEtl.cs @@ -0,0 +1,46 @@ +using JdeScoping.DataAccess.Interfaces; +using JdeScoping.DataSync.Etl.Destinations; +using JdeScoping.DataSync.Etl.Models; +using JdeScoping.DataSync.Etl.Pipeline; +using JdeScoping.DataSync.Etl.Sources; + +namespace JdeScoping.DataSync.Dev; + +/// +/// Development ETL pipeline for the WorkOrderStep_Hist table. +/// Schema from: Scripts/018_CreateWorkOrderStepHistTable.sql +/// +public static class WorkOrderStepHistDevEtl +{ + public static readonly string TableName = "WorkOrderStep_Hist"; + public static readonly string CacheFileName = "workorderstep_hist.json.zstd"; + + private static readonly JsonColumnSchema[] Schema = + [ + new("WorkOrderNumber", typeof(long), IsNullable: false), + new("WorkCenterCode", typeof(string), IsNullable: false), + new("StepNumber", typeof(decimal), IsNullable: false), + new("StepTypeCode", typeof(string), IsNullable: false), + new("BranchCode", typeof(string), IsNullable: false), + new("StepDescription", typeof(string), IsNullable: true), + new("StartDT", typeof(DateTime), IsNullable: true), + new("EndDT", typeof(DateTime), IsNullable: true), + new("FunctionCode", typeof(string), IsNullable: true), + new("ScrappedQuantity", typeof(decimal), IsNullable: false), + new("LastUpdateDT", typeof(DateTime), IsNullable: false), + ]; + + public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath) + { + ArgumentNullException.ThrowIfNull(connectionFactory); + + if (string.IsNullOrWhiteSpace(cacheFilePath)) + throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath)); + + return new EtlPipelineBuilder() + .WithName($"{TableName}_Dev") + .WithSource(new JsonZstdFileSource(cacheFilePath, Schema)) + .WithDestination(new DbBulkImportDestination(connectionFactory, TableName)) + .Build(); + } +} diff --git a/NEW/src/JdeScoping.DataSync.Dev/WorkOrderTimeCurrDevEtl.cs b/NEW/src/JdeScoping.DataSync.Dev/WorkOrderTimeCurrDevEtl.cs new file mode 100644 index 0000000..14ff1f2 --- /dev/null +++ b/NEW/src/JdeScoping.DataSync.Dev/WorkOrderTimeCurrDevEtl.cs @@ -0,0 +1,43 @@ +using JdeScoping.DataAccess.Interfaces; +using JdeScoping.DataSync.Etl.Destinations; +using JdeScoping.DataSync.Etl.Models; +using JdeScoping.DataSync.Etl.Pipeline; +using JdeScoping.DataSync.Etl.Sources; + +namespace JdeScoping.DataSync.Dev; + +/// +/// Development ETL pipeline for the WorkOrderTime_Curr table. +/// Schema from: Scripts/019_CreateWorkOrderTimeCurrTable.sql +/// +public static class WorkOrderTimeCurrDevEtl +{ + public static readonly string TableName = "WorkOrderTime_Curr"; + public static readonly string CacheFileName = "workordertime_curr.json.zstd"; + + private static readonly JsonColumnSchema[] Schema = + [ + new("UniqueID", typeof(long), IsNullable: false), + new("WorkOrderNumber", typeof(long), IsNullable: false), + new("StepNumber", typeof(decimal), IsNullable: false), + new("WorkCenterCode", typeof(string), IsNullable: false), + new("BranchCode", typeof(string), IsNullable: false), + new("AddressNumber", typeof(long), IsNullable: false), + new("GlDate", typeof(DateTime), IsNullable: true), + new("LastUpdateDT", typeof(DateTime), IsNullable: false), + ]; + + public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath) + { + ArgumentNullException.ThrowIfNull(connectionFactory); + + if (string.IsNullOrWhiteSpace(cacheFilePath)) + throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath)); + + return new EtlPipelineBuilder() + .WithName($"{TableName}_Dev") + .WithSource(new JsonZstdFileSource(cacheFilePath, Schema)) + .WithDestination(new DbBulkImportDestination(connectionFactory, TableName)) + .Build(); + } +} diff --git a/NEW/src/JdeScoping.DataSync.Dev/WorkOrderTimeHistDevEtl.cs b/NEW/src/JdeScoping.DataSync.Dev/WorkOrderTimeHistDevEtl.cs new file mode 100644 index 0000000..ef63f89 --- /dev/null +++ b/NEW/src/JdeScoping.DataSync.Dev/WorkOrderTimeHistDevEtl.cs @@ -0,0 +1,43 @@ +using JdeScoping.DataAccess.Interfaces; +using JdeScoping.DataSync.Etl.Destinations; +using JdeScoping.DataSync.Etl.Models; +using JdeScoping.DataSync.Etl.Pipeline; +using JdeScoping.DataSync.Etl.Sources; + +namespace JdeScoping.DataSync.Dev; + +/// +/// Development ETL pipeline for the WorkOrderTime_Hist table. +/// Schema from: Scripts/020_CreateWorkOrderTimeHistTable.sql +/// +public static class WorkOrderTimeHistDevEtl +{ + public static readonly string TableName = "WorkOrderTime_Hist"; + public static readonly string CacheFileName = "workordertime_hist.json.zstd"; + + private static readonly JsonColumnSchema[] Schema = + [ + new("UniqueID", typeof(long), IsNullable: false), + new("WorkOrderNumber", typeof(long), IsNullable: false), + new("StepNumber", typeof(decimal), IsNullable: false), + new("WorkCenterCode", typeof(string), IsNullable: false), + new("BranchCode", typeof(string), IsNullable: false), + new("AddressNumber", typeof(long), IsNullable: false), + new("GlDate", typeof(DateTime), IsNullable: true), + new("LastUpdateDT", typeof(DateTime), IsNullable: false), + ]; + + public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath) + { + ArgumentNullException.ThrowIfNull(connectionFactory); + + if (string.IsNullOrWhiteSpace(cacheFilePath)) + throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath)); + + return new EtlPipelineBuilder() + .WithName($"{TableName}_Dev") + .WithSource(new JsonZstdFileSource(cacheFilePath, Schema)) + .WithDestination(new DbBulkImportDestination(connectionFactory, TableName)) + .Build(); + } +} diff --git a/NEW/src/JdeScoping.DataSync/DevEtl/DevEtlRegistry.cs b/NEW/src/JdeScoping.DataSync/DevEtl/DevEtlRegistry.cs deleted file mode 100644 index e681043..0000000 --- a/NEW/src/JdeScoping.DataSync/DevEtl/DevEtlRegistry.cs +++ /dev/null @@ -1,80 +0,0 @@ -using JdeScoping.DataAccess.Interfaces; -using JdeScoping.DataSync.Etl.Pipeline; -using JdeScoping.DataSync.Etl.Results; -using Microsoft.Extensions.Logging; - -namespace JdeScoping.DataSync.DevEtl; - -/// -/// Registry for development ETL pipelines that load from cached JSON files. -/// -public class DevEtlRegistry -{ - private readonly IDbConnectionFactory _connectionFactory; - private readonly string _cacheDirectory; - private readonly ILogger? _logger; - - private readonly Dictionary> _pipelineFactories = new(StringComparer.OrdinalIgnoreCase) - { - [BranchDevEtl.TableName] = (factory, cacheDir) => - BranchDevEtl.Create(factory, Path.Combine(cacheDir, BranchDevEtl.CacheFileName)), - }; - - public DevEtlRegistry( - IDbConnectionFactory connectionFactory, - string cacheDirectory, - ILogger? logger = null) - { - _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); - - if (string.IsNullOrWhiteSpace(cacheDirectory)) - throw new ArgumentException("Cache directory is required.", nameof(cacheDirectory)); - - if (!Directory.Exists(cacheDirectory)) - throw new DirectoryNotFoundException($"Cache directory not found: {cacheDirectory}"); - - _cacheDirectory = cacheDirectory; - _logger = logger; - } - - public IEnumerable GetAvailableTables() => _pipelineFactories.Keys; - - public EtlPipeline GetPipeline(string tableName) - { - if (!_pipelineFactories.TryGetValue(tableName, out var factory)) - throw new ArgumentException($"No pipeline registered for table '{tableName}'.", nameof(tableName)); - - return factory(_connectionFactory, _cacheDirectory); - } - - public async Task RunAsync(string tableName, CancellationToken cancellationToken = default) - { - _logger?.LogInformation("Running dev ETL for {TableName}", tableName); - - var pipeline = GetPipeline(tableName); - var result = await pipeline.ExecuteAsync(cancellationToken); - - if (result.Success) - _logger?.LogInformation("Completed {TableName}: {Rows} rows in {Elapsed:g}", - tableName, result.TotalRows, result.Elapsed); - else - _logger?.LogError(result.Error, "Failed {TableName}: {Error}", - tableName, result.Error?.Message); - - return result; - } - - public async Task> RunAllAsync(CancellationToken cancellationToken = default) - { - var results = new List(); - - foreach (var tableName in GetAvailableTables()) - { - cancellationToken.ThrowIfCancellationRequested(); - var result = await RunAsync(tableName, cancellationToken); - results.Add(result); - } - - return results; - } -} diff --git a/NEW/tests/JdeScoping.DataSync.Tests/DevEtl/BranchDevEtlTests.cs b/NEW/tests/JdeScoping.DataSync.Dev.Tests/BranchDevEtlTests.cs similarity index 95% rename from NEW/tests/JdeScoping.DataSync.Tests/DevEtl/BranchDevEtlTests.cs rename to NEW/tests/JdeScoping.DataSync.Dev.Tests/BranchDevEtlTests.cs index 652df92..991c9ad 100644 --- a/NEW/tests/JdeScoping.DataSync.Tests/DevEtl/BranchDevEtlTests.cs +++ b/NEW/tests/JdeScoping.DataSync.Dev.Tests/BranchDevEtlTests.cs @@ -1,14 +1,14 @@ using Dapper; using JdeScoping.DataAccess; using JdeScoping.DataAccess.Interfaces; -using JdeScoping.DataSync.DevEtl; +using JdeScoping.DataSync.Dev; using Microsoft.Data.SqlClient; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging.Abstractions; using NSubstitute; using Shouldly; -namespace JdeScoping.DataSync.Tests.DevEtl; +namespace JdeScoping.DataSync.Dev.Tests; /// /// Integration tests for Branch development ETL. diff --git a/NEW/tests/JdeScoping.DataSync.Dev.Tests/FunctionCodeDevEtlTests.cs b/NEW/tests/JdeScoping.DataSync.Dev.Tests/FunctionCodeDevEtlTests.cs new file mode 100644 index 0000000..44af0f9 --- /dev/null +++ b/NEW/tests/JdeScoping.DataSync.Dev.Tests/FunctionCodeDevEtlTests.cs @@ -0,0 +1,96 @@ +using Dapper; +using JdeScoping.DataAccess; +using JdeScoping.DataAccess.Interfaces; +using JdeScoping.DataSync.Dev; +using Microsoft.Data.SqlClient; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging.Abstractions; +using NSubstitute; +using Shouldly; + +namespace JdeScoping.DataSync.Dev.Tests; + +/// +/// Integration tests for FunctionCode development ETL. +/// +public class FunctionCodeDevEtlTests : IAsyncLifetime +{ + private readonly string _connectionString; + private readonly string _cacheDirectory; + private readonly IDbConnectionFactory _connectionFactory; + + public FunctionCodeDevEtlTests() + { + var config = new ConfigurationBuilder() + .SetBasePath(Directory.GetCurrentDirectory()) + .AddJsonFile("appsettings.json", optional: true) + .AddEnvironmentVariables() + .Build(); + + _connectionString = config.GetConnectionString("LotFinderDB") + ?? throw new InvalidOperationException("LotFinderDB connection string not configured."); + + _cacheDirectory = config["DevEtl:CacheDirectory"] + ?? Path.Combine(Directory.GetCurrentDirectory(), "..", "..", "..", "..", "..", "CACHED_DB_FILES"); + + _connectionFactory = new DbConnectionFactory(config, NullLogger.Instance); + } + + public async Task InitializeAsync() + { + await using var connection = new SqlConnection(_connectionString); + await connection.OpenAsync(); + await connection.ExecuteAsync("TRUNCATE TABLE dbo.FunctionCode"); + } + + public Task DisposeAsync() => Task.CompletedTask; + + [Fact] + public void Create_ReturnsValidPipeline() + { + var cacheFilePath = Path.Combine(_cacheDirectory, FunctionCodeDevEtl.CacheFileName); + if (!File.Exists(cacheFilePath)) return; + + var pipeline = FunctionCodeDevEtl.Create(_connectionFactory, cacheFilePath); + + pipeline.ShouldNotBeNull(); + pipeline.PipelineName.ShouldBe("FunctionCode_Dev"); + } + + [Fact] + public async Task Execute_LoadsData() + { + var cacheFilePath = Path.Combine(_cacheDirectory, FunctionCodeDevEtl.CacheFileName); + if (!File.Exists(cacheFilePath)) return; + + var pipeline = FunctionCodeDevEtl.Create(_connectionFactory, cacheFilePath); + var result = await pipeline.ExecuteAsync(); + + result.Success.ShouldBeTrue(result.Error?.Message ?? "Pipeline should succeed"); + result.TotalRows.ShouldBeGreaterThan(0); + + await using var connection = new SqlConnection(_connectionString); + await connection.OpenAsync(); + var count = await connection.ExecuteScalarAsync("SELECT COUNT(*) FROM dbo.FunctionCode"); + count.ShouldBe((int)result.TotalRows); + } + + [Fact] + public async Task Registry_RunAsync_LoadsTable() + { + if (!Directory.Exists(_cacheDirectory)) return; + + var registry = new DevEtlRegistry(_connectionFactory, _cacheDirectory); + var result = await registry.RunAsync("FunctionCode"); + + result.Success.ShouldBeTrue(result.Error?.Message ?? "Pipeline should succeed"); + result.TotalRows.ShouldBeGreaterThan(0); + } + + [Fact] + public void Create_WithNullConnectionFactory_ThrowsArgumentNullException() + { + var cacheFilePath = Path.Combine(_cacheDirectory, FunctionCodeDevEtl.CacheFileName); + Should.Throw(() => FunctionCodeDevEtl.Create(null!, cacheFilePath)); + } +} diff --git a/NEW/tests/JdeScoping.DataSync.Dev.Tests/ItemDevEtlTests.cs b/NEW/tests/JdeScoping.DataSync.Dev.Tests/ItemDevEtlTests.cs new file mode 100644 index 0000000..a4a0b3e --- /dev/null +++ b/NEW/tests/JdeScoping.DataSync.Dev.Tests/ItemDevEtlTests.cs @@ -0,0 +1,96 @@ +using Dapper; +using JdeScoping.DataAccess; +using JdeScoping.DataAccess.Interfaces; +using JdeScoping.DataSync.Dev; +using Microsoft.Data.SqlClient; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging.Abstractions; +using NSubstitute; +using Shouldly; + +namespace JdeScoping.DataSync.Dev.Tests; + +/// +/// Integration tests for Item development ETL. +/// +public class ItemDevEtlTests : IAsyncLifetime +{ + private readonly string _connectionString; + private readonly string _cacheDirectory; + private readonly IDbConnectionFactory _connectionFactory; + + public ItemDevEtlTests() + { + var config = new ConfigurationBuilder() + .SetBasePath(Directory.GetCurrentDirectory()) + .AddJsonFile("appsettings.json", optional: true) + .AddEnvironmentVariables() + .Build(); + + _connectionString = config.GetConnectionString("LotFinderDB") + ?? throw new InvalidOperationException("LotFinderDB connection string not configured."); + + _cacheDirectory = config["DevEtl:CacheDirectory"] + ?? Path.Combine(Directory.GetCurrentDirectory(), "..", "..", "..", "..", "..", "CACHED_DB_FILES"); + + _connectionFactory = new DbConnectionFactory(config, NullLogger.Instance); + } + + public async Task InitializeAsync() + { + await using var connection = new SqlConnection(_connectionString); + await connection.OpenAsync(); + await connection.ExecuteAsync("TRUNCATE TABLE dbo.Item"); + } + + public Task DisposeAsync() => Task.CompletedTask; + + [Fact] + public void Create_ReturnsValidPipeline() + { + var cacheFilePath = Path.Combine(_cacheDirectory, ItemDevEtl.CacheFileName); + if (!File.Exists(cacheFilePath)) return; + + var pipeline = ItemDevEtl.Create(_connectionFactory, cacheFilePath); + + pipeline.ShouldNotBeNull(); + pipeline.PipelineName.ShouldBe("Item_Dev"); + } + + [Fact] + public async Task Execute_LoadsData() + { + var cacheFilePath = Path.Combine(_cacheDirectory, ItemDevEtl.CacheFileName); + if (!File.Exists(cacheFilePath)) return; + + var pipeline = ItemDevEtl.Create(_connectionFactory, cacheFilePath); + var result = await pipeline.ExecuteAsync(); + + result.Success.ShouldBeTrue(result.Error?.Message ?? "Pipeline should succeed"); + result.TotalRows.ShouldBeGreaterThan(0); + + await using var connection = new SqlConnection(_connectionString); + await connection.OpenAsync(); + var count = await connection.ExecuteScalarAsync("SELECT COUNT(*) FROM dbo.Item"); + count.ShouldBe((int)result.TotalRows); + } + + [Fact] + public async Task Registry_RunAsync_LoadsTable() + { + if (!Directory.Exists(_cacheDirectory)) return; + + var registry = new DevEtlRegistry(_connectionFactory, _cacheDirectory); + var result = await registry.RunAsync("Item"); + + result.Success.ShouldBeTrue(result.Error?.Message ?? "Pipeline should succeed"); + result.TotalRows.ShouldBeGreaterThan(0); + } + + [Fact] + public void Create_WithNullConnectionFactory_ThrowsArgumentNullException() + { + var cacheFilePath = Path.Combine(_cacheDirectory, ItemDevEtl.CacheFileName); + Should.Throw(() => ItemDevEtl.Create(null!, cacheFilePath)); + } +} diff --git a/NEW/tests/JdeScoping.DataSync.Dev.Tests/JdeScoping.DataSync.Dev.Tests.csproj b/NEW/tests/JdeScoping.DataSync.Dev.Tests/JdeScoping.DataSync.Dev.Tests.csproj new file mode 100644 index 0000000..eba4d70 --- /dev/null +++ b/NEW/tests/JdeScoping.DataSync.Dev.Tests/JdeScoping.DataSync.Dev.Tests.csproj @@ -0,0 +1,44 @@ + + + + net10.0 + enable + enable + false + true + + + + + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + + + + + + + + + + + + + + + + + + + + diff --git a/NEW/tests/JdeScoping.DataSync.Dev.Tests/JdeUserDevEtlTests.cs b/NEW/tests/JdeScoping.DataSync.Dev.Tests/JdeUserDevEtlTests.cs new file mode 100644 index 0000000..7b780f0 --- /dev/null +++ b/NEW/tests/JdeScoping.DataSync.Dev.Tests/JdeUserDevEtlTests.cs @@ -0,0 +1,96 @@ +using Dapper; +using JdeScoping.DataAccess; +using JdeScoping.DataAccess.Interfaces; +using JdeScoping.DataSync.Dev; +using Microsoft.Data.SqlClient; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging.Abstractions; +using NSubstitute; +using Shouldly; + +namespace JdeScoping.DataSync.Dev.Tests; + +/// +/// Integration tests for JdeUser development ETL. +/// +public class JdeUserDevEtlTests : IAsyncLifetime +{ + private readonly string _connectionString; + private readonly string _cacheDirectory; + private readonly IDbConnectionFactory _connectionFactory; + + public JdeUserDevEtlTests() + { + var config = new ConfigurationBuilder() + .SetBasePath(Directory.GetCurrentDirectory()) + .AddJsonFile("appsettings.json", optional: true) + .AddEnvironmentVariables() + .Build(); + + _connectionString = config.GetConnectionString("LotFinderDB") + ?? throw new InvalidOperationException("LotFinderDB connection string not configured."); + + _cacheDirectory = config["DevEtl:CacheDirectory"] + ?? Path.Combine(Directory.GetCurrentDirectory(), "..", "..", "..", "..", "..", "CACHED_DB_FILES"); + + _connectionFactory = new DbConnectionFactory(config, NullLogger.Instance); + } + + public async Task InitializeAsync() + { + await using var connection = new SqlConnection(_connectionString); + await connection.OpenAsync(); + await connection.ExecuteAsync("TRUNCATE TABLE dbo.JdeUser"); + } + + public Task DisposeAsync() => Task.CompletedTask; + + [Fact] + public void Create_ReturnsValidPipeline() + { + var cacheFilePath = Path.Combine(_cacheDirectory, JdeUserDevEtl.CacheFileName); + if (!File.Exists(cacheFilePath)) return; + + var pipeline = JdeUserDevEtl.Create(_connectionFactory, cacheFilePath); + + pipeline.ShouldNotBeNull(); + pipeline.PipelineName.ShouldBe("JdeUser_Dev"); + } + + [Fact] + public async Task Execute_LoadsData() + { + var cacheFilePath = Path.Combine(_cacheDirectory, JdeUserDevEtl.CacheFileName); + if (!File.Exists(cacheFilePath)) return; + + var pipeline = JdeUserDevEtl.Create(_connectionFactory, cacheFilePath); + var result = await pipeline.ExecuteAsync(); + + result.Success.ShouldBeTrue(result.Error?.Message ?? "Pipeline should succeed"); + result.TotalRows.ShouldBeGreaterThan(0); + + await using var connection = new SqlConnection(_connectionString); + await connection.OpenAsync(); + var count = await connection.ExecuteScalarAsync("SELECT COUNT(*) FROM dbo.JdeUser"); + count.ShouldBe((int)result.TotalRows); + } + + [Fact] + public async Task Registry_RunAsync_LoadsTable() + { + if (!Directory.Exists(_cacheDirectory)) return; + + var registry = new DevEtlRegistry(_connectionFactory, _cacheDirectory); + var result = await registry.RunAsync("JdeUser"); + + result.Success.ShouldBeTrue(result.Error?.Message ?? "Pipeline should succeed"); + result.TotalRows.ShouldBeGreaterThan(0); + } + + [Fact] + public void Create_WithNullConnectionFactory_ThrowsArgumentNullException() + { + var cacheFilePath = Path.Combine(_cacheDirectory, JdeUserDevEtl.CacheFileName); + Should.Throw(() => JdeUserDevEtl.Create(null!, cacheFilePath)); + } +} diff --git a/NEW/tests/JdeScoping.DataSync.Dev.Tests/OrgHierarchyDevEtlTests.cs b/NEW/tests/JdeScoping.DataSync.Dev.Tests/OrgHierarchyDevEtlTests.cs new file mode 100644 index 0000000..dc880b6 --- /dev/null +++ b/NEW/tests/JdeScoping.DataSync.Dev.Tests/OrgHierarchyDevEtlTests.cs @@ -0,0 +1,166 @@ +using Dapper; +using JdeScoping.DataAccess; +using JdeScoping.DataAccess.Interfaces; +using JdeScoping.DataSync.Dev; +using Microsoft.Data.SqlClient; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging.Abstractions; +using NSubstitute; +using Shouldly; + +namespace JdeScoping.DataSync.Dev.Tests; + +/// +/// Integration tests for OrgHierarchy development ETL. +/// Requires: Local SQL Server, CACHED_DB_FILES directory with orghierarchy.json.zstd +/// +public class OrgHierarchyDevEtlTests : IAsyncLifetime +{ + private readonly string _connectionString; + private readonly string _cacheDirectory; + private readonly IDbConnectionFactory _connectionFactory; + + public OrgHierarchyDevEtlTests() + { + // Load configuration + var config = new ConfigurationBuilder() + .SetBasePath(Directory.GetCurrentDirectory()) + .AddJsonFile("appsettings.json", optional: true) + .AddEnvironmentVariables() + .Build(); + + _connectionString = config.GetConnectionString("LotFinderDB") + ?? throw new InvalidOperationException("LotFinderDB connection string not configured."); + + _cacheDirectory = config["DevEtl:CacheDirectory"] + ?? Path.Combine(Directory.GetCurrentDirectory(), "..", "..", "..", "..", "..", "CACHED_DB_FILES"); + + _connectionFactory = new DbConnectionFactory(config, NullLogger.Instance); + } + + public async Task InitializeAsync() + { + // Ensure OrgHierarchy table is empty before test + await using var connection = new SqlConnection(_connectionString); + await connection.OpenAsync(); + await connection.ExecuteAsync("TRUNCATE TABLE dbo.OrgHierarchy"); + } + + public Task DisposeAsync() => Task.CompletedTask; + + [Fact] + public void Create_ReturnsValidPipeline() + { + // Arrange + var cacheFilePath = Path.Combine(_cacheDirectory, OrgHierarchyDevEtl.CacheFileName); + if (!File.Exists(cacheFilePath)) + { + // Skip test if cache file doesn't exist + return; + } + + // Act + var pipeline = OrgHierarchyDevEtl.Create(_connectionFactory, cacheFilePath); + + // Assert + pipeline.ShouldNotBeNull(); + pipeline.PipelineName.ShouldBe("OrgHierarchy_Dev"); + } + + [Fact] + public async Task Execute_LoadsOrgHierarchyData() + { + // Arrange + var cacheFilePath = Path.Combine(_cacheDirectory, OrgHierarchyDevEtl.CacheFileName); + if (!File.Exists(cacheFilePath)) + { + // Skip test if cache file doesn't exist + return; + } + + var pipeline = OrgHierarchyDevEtl.Create(_connectionFactory, cacheFilePath); + + // Act + var result = await pipeline.ExecuteAsync(); + + // Assert + result.Success.ShouldBeTrue(result.Error?.Message ?? "Pipeline should succeed"); + result.TotalRows.ShouldBeGreaterThan(0, "Should load at least one row"); + + // Verify data in database + await using var connection = new SqlConnection(_connectionString); + await connection.OpenAsync(); + var count = await connection.ExecuteScalarAsync("SELECT COUNT(*) FROM dbo.OrgHierarchy"); + + count.ShouldBe((int)result.TotalRows, "Database row count should match pipeline result"); + } + + [Fact] + public async Task Registry_RunAsync_LoadsOrgHierarchy() + { + // Arrange + if (!Directory.Exists(_cacheDirectory)) + { + // Skip test if cache directory doesn't exist + return; + } + + var registry = new DevEtlRegistry(_connectionFactory, _cacheDirectory); + + // Act + var result = await registry.RunAsync("OrgHierarchy"); + + // Assert + result.Success.ShouldBeTrue(result.Error?.Message ?? "Pipeline should succeed"); + result.TotalRows.ShouldBeGreaterThan(0); + } + + [Fact] + public void Create_WithNullConnectionFactory_ThrowsArgumentNullException() + { + // Arrange + var cacheFilePath = Path.Combine(_cacheDirectory, OrgHierarchyDevEtl.CacheFileName); + + // Act & Assert + Should.Throw(() => OrgHierarchyDevEtl.Create(null!, cacheFilePath)); + } + + [Fact] + public void Create_WithEmptyCacheFilePath_ThrowsArgumentException() + { + // Arrange + var mockFactory = Substitute.For(); + + // Act & Assert + Should.Throw(() => OrgHierarchyDevEtl.Create(mockFactory, string.Empty)); + } + + [Fact] + public void Create_WithNonExistentCacheFile_ThrowsFileNotFoundException() + { + // Arrange + var mockFactory = Substitute.For(); + var nonExistentPath = "/nonexistent/path/orghierarchy.json.zstd"; + + // Act & Assert + Should.Throw(() => OrgHierarchyDevEtl.Create(mockFactory, nonExistentPath)); + } + + [Fact] + public void DevEtlRegistry_GetAvailableTables_IncludesOrgHierarchy() + { + // Arrange + if (!Directory.Exists(_cacheDirectory)) + { + return; + } + + var registry = new DevEtlRegistry(_connectionFactory, _cacheDirectory); + + // Act + var tables = registry.GetAvailableTables().ToList(); + + // Assert + tables.ShouldContain("OrgHierarchy"); + } +} diff --git a/NEW/tests/JdeScoping.DataSync.Dev.Tests/ProfitCenterDevEtlTests.cs b/NEW/tests/JdeScoping.DataSync.Dev.Tests/ProfitCenterDevEtlTests.cs new file mode 100644 index 0000000..2cc5c10 --- /dev/null +++ b/NEW/tests/JdeScoping.DataSync.Dev.Tests/ProfitCenterDevEtlTests.cs @@ -0,0 +1,117 @@ +using Dapper; +using JdeScoping.DataAccess; +using JdeScoping.DataAccess.Interfaces; +using JdeScoping.DataSync.Dev; +using Microsoft.Data.SqlClient; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging.Abstractions; +using NSubstitute; +using Shouldly; + +namespace JdeScoping.DataSync.Dev.Tests; + +/// +/// Integration tests for ProfitCenter development ETL. +/// Requires: Local SQL Server, CACHED_DB_FILES directory with profitcenter.json.zstd +/// +public class ProfitCenterDevEtlTests : IAsyncLifetime +{ + private readonly string _connectionString; + private readonly string _cacheDirectory; + private readonly IDbConnectionFactory _connectionFactory; + + public ProfitCenterDevEtlTests() + { + var config = new ConfigurationBuilder() + .SetBasePath(Directory.GetCurrentDirectory()) + .AddJsonFile("appsettings.json", optional: true) + .AddEnvironmentVariables() + .Build(); + + _connectionString = config.GetConnectionString("LotFinderDB") + ?? throw new InvalidOperationException("LotFinderDB connection string not configured."); + + _cacheDirectory = config["DevEtl:CacheDirectory"] + ?? Path.Combine(Directory.GetCurrentDirectory(), "..", "..", "..", "..", "..", "CACHED_DB_FILES"); + + _connectionFactory = new DbConnectionFactory(config, NullLogger.Instance); + } + + public async Task InitializeAsync() + { + await using var connection = new SqlConnection(_connectionString); + await connection.OpenAsync(); + await connection.ExecuteAsync("TRUNCATE TABLE dbo.ProfitCenter"); + } + + public Task DisposeAsync() => Task.CompletedTask; + + [Fact] + public void Create_ReturnsValidPipeline() + { + var cacheFilePath = Path.Combine(_cacheDirectory, ProfitCenterDevEtl.CacheFileName); + if (!File.Exists(cacheFilePath)) return; + + var pipeline = ProfitCenterDevEtl.Create(_connectionFactory, cacheFilePath); + + pipeline.ShouldNotBeNull(); + pipeline.PipelineName.ShouldBe("ProfitCenter_Dev"); + } + + [Fact] + public async Task Execute_LoadsProfitCenterData() + { + var cacheFilePath = Path.Combine(_cacheDirectory, ProfitCenterDevEtl.CacheFileName); + if (!File.Exists(cacheFilePath)) return; + + var pipeline = ProfitCenterDevEtl.Create(_connectionFactory, cacheFilePath); + + var result = await pipeline.ExecuteAsync(); + + result.Success.ShouldBeTrue(result.Error?.Message ?? "Pipeline should succeed"); + result.TotalRows.ShouldBeGreaterThan(0, "Should load at least one row"); + + await using var connection = new SqlConnection(_connectionString); + await connection.OpenAsync(); + var count = await connection.ExecuteScalarAsync("SELECT COUNT(*) FROM dbo.ProfitCenter"); + + count.ShouldBe((int)result.TotalRows); + } + + [Fact] + public async Task Registry_RunAsync_LoadsProfitCenter() + { + if (!Directory.Exists(_cacheDirectory)) return; + + var registry = new DevEtlRegistry(_connectionFactory, _cacheDirectory); + var result = await registry.RunAsync("ProfitCenter"); + + result.Success.ShouldBeTrue(result.Error?.Message ?? "Pipeline should succeed"); + result.TotalRows.ShouldBeGreaterThan(0); + } + + [Fact] + public void Create_WithNullConnectionFactory_ThrowsArgumentNullException() + { + var cacheFilePath = Path.Combine(_cacheDirectory, ProfitCenterDevEtl.CacheFileName); + Should.Throw(() => ProfitCenterDevEtl.Create(null!, cacheFilePath)); + } + + [Fact] + public void Create_WithEmptyCacheFilePath_ThrowsArgumentException() + { + var mockFactory = Substitute.For(); + Should.Throw(() => ProfitCenterDevEtl.Create(mockFactory, string.Empty)); + } + + [Fact] + public void DevEtlRegistry_GetAvailableTables_IncludesProfitCenter() + { + if (!Directory.Exists(_cacheDirectory)) return; + + var registry = new DevEtlRegistry(_connectionFactory, _cacheDirectory); + var tables = registry.GetAvailableTables().ToList(); + + tables.ShouldContain("ProfitCenter"); + } +} diff --git a/NEW/tests/JdeScoping.DataSync.Dev.Tests/RouteMasterDevEtlTests.cs b/NEW/tests/JdeScoping.DataSync.Dev.Tests/RouteMasterDevEtlTests.cs new file mode 100644 index 0000000..c7fb45c --- /dev/null +++ b/NEW/tests/JdeScoping.DataSync.Dev.Tests/RouteMasterDevEtlTests.cs @@ -0,0 +1,96 @@ +using Dapper; +using JdeScoping.DataAccess; +using JdeScoping.DataAccess.Interfaces; +using JdeScoping.DataSync.Dev; +using Microsoft.Data.SqlClient; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging.Abstractions; +using NSubstitute; +using Shouldly; + +namespace JdeScoping.DataSync.Dev.Tests; + +/// +/// Integration tests for RouteMaster development ETL. +/// +public class RouteMasterDevEtlTests : IAsyncLifetime +{ + private readonly string _connectionString; + private readonly string _cacheDirectory; + private readonly IDbConnectionFactory _connectionFactory; + + public RouteMasterDevEtlTests() + { + var config = new ConfigurationBuilder() + .SetBasePath(Directory.GetCurrentDirectory()) + .AddJsonFile("appsettings.json", optional: true) + .AddEnvironmentVariables() + .Build(); + + _connectionString = config.GetConnectionString("LotFinderDB") + ?? throw new InvalidOperationException("LotFinderDB connection string not configured."); + + _cacheDirectory = config["DevEtl:CacheDirectory"] + ?? Path.Combine(Directory.GetCurrentDirectory(), "..", "..", "..", "..", "..", "CACHED_DB_FILES"); + + _connectionFactory = new DbConnectionFactory(config, NullLogger.Instance); + } + + public async Task InitializeAsync() + { + await using var connection = new SqlConnection(_connectionString); + await connection.OpenAsync(); + await connection.ExecuteAsync("TRUNCATE TABLE dbo.RouteMaster"); + } + + public Task DisposeAsync() => Task.CompletedTask; + + [Fact] + public void Create_ReturnsValidPipeline() + { + var cacheFilePath = Path.Combine(_cacheDirectory, RouteMasterDevEtl.CacheFileName); + if (!File.Exists(cacheFilePath)) return; + + var pipeline = RouteMasterDevEtl.Create(_connectionFactory, cacheFilePath); + + pipeline.ShouldNotBeNull(); + pipeline.PipelineName.ShouldBe("RouteMaster_Dev"); + } + + [Fact] + public async Task Execute_LoadsData() + { + var cacheFilePath = Path.Combine(_cacheDirectory, RouteMasterDevEtl.CacheFileName); + if (!File.Exists(cacheFilePath)) return; + + var pipeline = RouteMasterDevEtl.Create(_connectionFactory, cacheFilePath); + var result = await pipeline.ExecuteAsync(); + + result.Success.ShouldBeTrue(result.Error?.Message ?? "Pipeline should succeed"); + result.TotalRows.ShouldBeGreaterThan(0); + + await using var connection = new SqlConnection(_connectionString); + await connection.OpenAsync(); + var count = await connection.ExecuteScalarAsync("SELECT COUNT(*) FROM dbo.RouteMaster"); + count.ShouldBe((int)result.TotalRows); + } + + [Fact] + public async Task Registry_RunAsync_LoadsTable() + { + if (!Directory.Exists(_cacheDirectory)) return; + + var registry = new DevEtlRegistry(_connectionFactory, _cacheDirectory); + var result = await registry.RunAsync("RouteMaster"); + + result.Success.ShouldBeTrue(result.Error?.Message ?? "Pipeline should succeed"); + result.TotalRows.ShouldBeGreaterThan(0); + } + + [Fact] + public void Create_WithNullConnectionFactory_ThrowsArgumentNullException() + { + var cacheFilePath = Path.Combine(_cacheDirectory, RouteMasterDevEtl.CacheFileName); + Should.Throw(() => RouteMasterDevEtl.Create(null!, cacheFilePath)); + } +} diff --git a/NEW/tests/JdeScoping.DataSync.Dev.Tests/WorkCenterDevEtlTests.cs b/NEW/tests/JdeScoping.DataSync.Dev.Tests/WorkCenterDevEtlTests.cs new file mode 100644 index 0000000..2b59d91 --- /dev/null +++ b/NEW/tests/JdeScoping.DataSync.Dev.Tests/WorkCenterDevEtlTests.cs @@ -0,0 +1,117 @@ +using Dapper; +using JdeScoping.DataAccess; +using JdeScoping.DataAccess.Interfaces; +using JdeScoping.DataSync.Dev; +using Microsoft.Data.SqlClient; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging.Abstractions; +using NSubstitute; +using Shouldly; + +namespace JdeScoping.DataSync.Dev.Tests; + +/// +/// Integration tests for WorkCenter development ETL. +/// Requires: Local SQL Server, CACHED_DB_FILES directory with workcenter.json.zstd +/// +public class WorkCenterDevEtlTests : IAsyncLifetime +{ + private readonly string _connectionString; + private readonly string _cacheDirectory; + private readonly IDbConnectionFactory _connectionFactory; + + public WorkCenterDevEtlTests() + { + var config = new ConfigurationBuilder() + .SetBasePath(Directory.GetCurrentDirectory()) + .AddJsonFile("appsettings.json", optional: true) + .AddEnvironmentVariables() + .Build(); + + _connectionString = config.GetConnectionString("LotFinderDB") + ?? throw new InvalidOperationException("LotFinderDB connection string not configured."); + + _cacheDirectory = config["DevEtl:CacheDirectory"] + ?? Path.Combine(Directory.GetCurrentDirectory(), "..", "..", "..", "..", "..", "CACHED_DB_FILES"); + + _connectionFactory = new DbConnectionFactory(config, NullLogger.Instance); + } + + public async Task InitializeAsync() + { + await using var connection = new SqlConnection(_connectionString); + await connection.OpenAsync(); + await connection.ExecuteAsync("TRUNCATE TABLE dbo.WorkCenter"); + } + + public Task DisposeAsync() => Task.CompletedTask; + + [Fact] + public void Create_ReturnsValidPipeline() + { + var cacheFilePath = Path.Combine(_cacheDirectory, WorkCenterDevEtl.CacheFileName); + if (!File.Exists(cacheFilePath)) return; + + var pipeline = WorkCenterDevEtl.Create(_connectionFactory, cacheFilePath); + + pipeline.ShouldNotBeNull(); + pipeline.PipelineName.ShouldBe("WorkCenter_Dev"); + } + + [Fact] + public async Task Execute_LoadsWorkCenterData() + { + var cacheFilePath = Path.Combine(_cacheDirectory, WorkCenterDevEtl.CacheFileName); + if (!File.Exists(cacheFilePath)) return; + + var pipeline = WorkCenterDevEtl.Create(_connectionFactory, cacheFilePath); + + var result = await pipeline.ExecuteAsync(); + + result.Success.ShouldBeTrue(result.Error?.Message ?? "Pipeline should succeed"); + result.TotalRows.ShouldBeGreaterThan(0, "Should load at least one row"); + + await using var connection = new SqlConnection(_connectionString); + await connection.OpenAsync(); + var count = await connection.ExecuteScalarAsync("SELECT COUNT(*) FROM dbo.WorkCenter"); + + count.ShouldBe((int)result.TotalRows); + } + + [Fact] + public async Task Registry_RunAsync_LoadsWorkCenter() + { + if (!Directory.Exists(_cacheDirectory)) return; + + var registry = new DevEtlRegistry(_connectionFactory, _cacheDirectory); + var result = await registry.RunAsync("WorkCenter"); + + result.Success.ShouldBeTrue(result.Error?.Message ?? "Pipeline should succeed"); + result.TotalRows.ShouldBeGreaterThan(0); + } + + [Fact] + public void Create_WithNullConnectionFactory_ThrowsArgumentNullException() + { + var cacheFilePath = Path.Combine(_cacheDirectory, WorkCenterDevEtl.CacheFileName); + Should.Throw(() => WorkCenterDevEtl.Create(null!, cacheFilePath)); + } + + [Fact] + public void Create_WithEmptyCacheFilePath_ThrowsArgumentException() + { + var mockFactory = Substitute.For(); + Should.Throw(() => WorkCenterDevEtl.Create(mockFactory, string.Empty)); + } + + [Fact] + public void DevEtlRegistry_GetAvailableTables_IncludesWorkCenter() + { + if (!Directory.Exists(_cacheDirectory)) return; + + var registry = new DevEtlRegistry(_connectionFactory, _cacheDirectory); + var tables = registry.GetAvailableTables().ToList(); + + tables.ShouldContain("WorkCenter"); + } +} diff --git a/NEW/tests/JdeScoping.DataSync.Dev.Tests/appsettings.json b/NEW/tests/JdeScoping.DataSync.Dev.Tests/appsettings.json new file mode 100644 index 0000000..3a01c2b --- /dev/null +++ b/NEW/tests/JdeScoping.DataSync.Dev.Tests/appsettings.json @@ -0,0 +1,8 @@ +{ + "ConnectionStrings": { + "LotFinderDB": "Server=localhost,1434;Database=ScopingTool;User Id=scopingapp;Password=Sc0ping@pp_Dev#2024;TrustServerCertificate=true" + }, + "DevEtl": { + "CacheDirectory": "/Users/dohertj2/Desktop/JdeScopingTool/CACHED_DB_FILES" + } +} diff --git a/PLANS/2026-01-06-datasync-dev-extraction-design.md b/PLANS/2026-01-06-datasync-dev-extraction-design.md new file mode 100644 index 0000000..e4858d5 --- /dev/null +++ b/PLANS/2026-01-06-datasync-dev-extraction-design.md @@ -0,0 +1,162 @@ +# Design: Extract DevEtl to JdeScoping.DataSync.Dev + +**Date:** 2026-01-06 +**Status:** Approved + +## Purpose + +Move dev/testing-specific ETL code from `JdeScoping.DataSync` into a separate `JdeScoping.DataSync.Dev` project. This code loads cached JSON/zstd files into SQL Server for local sandbox development and should not be part of the production DataSync assembly. + +## Scope + +### Files to Move (Source) + +**22 files** from `src/JdeScoping.DataSync/DevEtl/` → `src/JdeScoping.DataSync.Dev/` + +- `DevEtlRegistry.cs` +- `BranchDevEtl.cs` +- `FunctionCodeDevEtl.cs` +- `ItemDevEtl.cs` +- `JdeUserDevEtl.cs` +- `LotDevEtl.cs` +- `LotUsageCurrDevEtl.cs` +- `LotUsageHistDevEtl.cs` +- `MisDataDevEtl.cs` +- `OrgHierarchyDevEtl.cs` +- `ProfitCenterDevEtl.cs` +- `RouteMasterDevEtl.cs` +- `WorkCenterDevEtl.cs` +- `WorkOrderComponentCurrDevEtl.cs` +- `WorkOrderComponentHistDevEtl.cs` +- `WorkOrderCurrDevEtl.cs` +- `WorkOrderHistDevEtl.cs` +- `WorkOrderRoutingDevEtl.cs` +- `WorkOrderStepCurrDevEtl.cs` +- `WorkOrderStepHistDevEtl.cs` +- `WorkOrderTimeCurrDevEtl.cs` +- `WorkOrderTimeHistDevEtl.cs` + +### Files to Move (Tests) + +**8 files** from `tests/JdeScoping.DataSync.Tests/DevEtl/` → `tests/JdeScoping.DataSync.Dev.Tests/` + +- `BranchDevEtlTests.cs` +- `FunctionCodeDevEtlTests.cs` +- `ItemDevEtlTests.cs` +- `JdeUserDevEtlTests.cs` +- `OrgHierarchyDevEtlTests.cs` +- `ProfitCenterDevEtlTests.cs` +- `RouteMasterDevEtlTests.cs` +- `WorkCenterDevEtlTests.cs` + +## New Project Structure + +``` +NEW/ +├── src/ +│ └── JdeScoping.DataSync.Dev/ +│ ├── JdeScoping.DataSync.Dev.csproj +│ ├── DevEtlRegistry.cs +│ ├── BranchDevEtl.cs +│ └── ... (20 more files) +└── tests/ + └── JdeScoping.DataSync.Dev.Tests/ + ├── JdeScoping.DataSync.Dev.Tests.csproj + ├── BranchDevEtlTests.cs + └── ... (7 more files) +``` + +## Dependencies + +### JdeScoping.DataSync.Dev.csproj + +```xml + + + + net10.0 + enable + enable + + + + + + + +``` + +Reuses ETL infrastructure from DataSync: +- `EtlPipeline`, `EtlPipelineBuilder` +- `JsonZstdFileSource` +- `DbBulkImportDestination` +- `JsonColumnSchema` + +### JdeScoping.DataSync.Dev.Tests.csproj + +```xml + + + + net10.0 + enable + enable + false + + + + + + + + + + + + + + + +``` + +## Namespace Changes + +All source files: +```csharp +// Before +namespace JdeScoping.DataSync.DevEtl; + +// After +namespace JdeScoping.DataSync.Dev; +``` + +All test files: +```csharp +// Before +using JdeScoping.DataSync.DevEtl; + +// After +using JdeScoping.DataSync.Dev; +``` + +## Cleanup + +After migration, delete: +- `src/JdeScoping.DataSync/DevEtl/` (entire folder) +- `tests/JdeScoping.DataSync.Tests/DevEtl/` (entire folder) + +## Cache Files + +No change to how cache files are located. The `DevEtlRegistry` continues to accept a `cacheDirectory` parameter at runtime. + +## Implementation Steps + +1. Create `src/JdeScoping.DataSync.Dev/JdeScoping.DataSync.Dev.csproj` +2. Move 22 source files from `DevEtl/` to new project +3. Update namespace in all source files +4. Create `tests/JdeScoping.DataSync.Dev.Tests/JdeScoping.DataSync.Dev.Tests.csproj` +5. Move 8 test files to new project +6. Update using statements in test files +7. Add both projects to solution +8. Delete old `DevEtl/` folders +9. Build and run tests to verify diff --git a/PLANS/2026-01-06-datasync-dev-extraction.md b/PLANS/2026-01-06-datasync-dev-extraction.md new file mode 100644 index 0000000..2b41c02 --- /dev/null +++ b/PLANS/2026-01-06-datasync-dev-extraction.md @@ -0,0 +1,389 @@ +# DataSync.Dev Extraction Implementation Plan + +> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task. + +**Goal:** Extract dev/testing ETL code from JdeScoping.DataSync into a separate JdeScoping.DataSync.Dev project with its own test project. + +**Architecture:** Create two new projects (DataSync.Dev and DataSync.Dev.Tests) that depend on the existing DataSync project. Move all DevEtl files, update namespaces, update solution file, then delete original folders. + +**Tech Stack:** .NET 10, xUnit, Shouldly, NSubstitute + +--- + +## Task 1: Create JdeScoping.DataSync.Dev Project + +**Files:** +- Create: `NEW/src/JdeScoping.DataSync.Dev/JdeScoping.DataSync.Dev.csproj` + +**Step 1: Create project directory** + +Run: +```bash +mkdir -p /Users/dohertj2/Desktop/JdeScopingTool/NEW/src/JdeScoping.DataSync.Dev +``` + +**Step 2: Create project file** + +Create `NEW/src/JdeScoping.DataSync.Dev/JdeScoping.DataSync.Dev.csproj`: + +```xml + + + + net10.0 + enable + enable + + + + + + + +``` + +**Step 3: Verify project builds** + +Run: +```bash +dotnet build /Users/dohertj2/Desktop/JdeScopingTool/NEW/src/JdeScoping.DataSync.Dev/JdeScoping.DataSync.Dev.csproj +``` +Expected: Build succeeded + +--- + +## Task 2: Move Source Files to DataSync.Dev + +**Files:** +- Move: `NEW/src/JdeScoping.DataSync/DevEtl/*.cs` → `NEW/src/JdeScoping.DataSync.Dev/` + +**Step 1: Move all 22 source files** + +Run: +```bash +mv /Users/dohertj2/Desktop/JdeScopingTool/NEW/src/JdeScoping.DataSync/DevEtl/*.cs /Users/dohertj2/Desktop/JdeScopingTool/NEW/src/JdeScoping.DataSync.Dev/ +``` + +**Step 2: Remove empty DevEtl directory** + +Run: +```bash +rmdir /Users/dohertj2/Desktop/JdeScopingTool/NEW/src/JdeScoping.DataSync/DevEtl +``` + +**Step 3: Verify files moved** + +Run: +```bash +ls /Users/dohertj2/Desktop/JdeScopingTool/NEW/src/JdeScoping.DataSync.Dev/*.cs | wc -l +``` +Expected: 22 + +--- + +## Task 3: Update Namespaces in Source Files + +**Files:** +- Modify: All 22 `.cs` files in `NEW/src/JdeScoping.DataSync.Dev/` + +**Step 1: Update namespace declarations** + +Replace in all files: +```csharp +namespace JdeScoping.DataSync.DevEtl; +``` + +With: +```csharp +namespace JdeScoping.DataSync.Dev; +``` + +Run: +```bash +cd /Users/dohertj2/Desktop/JdeScopingTool/NEW/src/JdeScoping.DataSync.Dev && \ +sed -i '' 's/namespace JdeScoping\.DataSync\.DevEtl;/namespace JdeScoping.DataSync.Dev;/g' *.cs +``` + +**Step 2: Verify namespace changes** + +Run: +```bash +grep -l "JdeScoping.DataSync.DevEtl" /Users/dohertj2/Desktop/JdeScopingTool/NEW/src/JdeScoping.DataSync.Dev/*.cs +``` +Expected: No output (no files should contain old namespace) + +**Step 3: Verify new namespace exists** + +Run: +```bash +grep -c "namespace JdeScoping.DataSync.Dev;" /Users/dohertj2/Desktop/JdeScopingTool/NEW/src/JdeScoping.DataSync.Dev/*.cs | head -5 +``` +Expected: Each file shows `:1` + +**Step 4: Build to verify** + +Run: +```bash +dotnet build /Users/dohertj2/Desktop/JdeScopingTool/NEW/src/JdeScoping.DataSync.Dev/JdeScoping.DataSync.Dev.csproj +``` +Expected: Build succeeded + +--- + +## Task 4: Create JdeScoping.DataSync.Dev.Tests Project + +**Files:** +- Create: `NEW/tests/JdeScoping.DataSync.Dev.Tests/JdeScoping.DataSync.Dev.Tests.csproj` + +**Step 1: Create test project directory** + +Run: +```bash +mkdir -p /Users/dohertj2/Desktop/JdeScopingTool/NEW/tests/JdeScoping.DataSync.Dev.Tests +``` + +**Step 2: Create test project file** + +Create `NEW/tests/JdeScoping.DataSync.Dev.Tests/JdeScoping.DataSync.Dev.Tests.csproj`: + +```xml + + + + net10.0 + enable + enable + false + true + + + + + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + + + + + + + + + + + + + + + + + + + + +``` + +**Step 3: Copy appsettings.json from existing test project** + +Run: +```bash +cp /Users/dohertj2/Desktop/JdeScopingTool/NEW/tests/JdeScoping.DataSync.Tests/appsettings.json /Users/dohertj2/Desktop/JdeScopingTool/NEW/tests/JdeScoping.DataSync.Dev.Tests/ +``` + +**Step 4: Verify project builds** + +Run: +```bash +dotnet build /Users/dohertj2/Desktop/JdeScopingTool/NEW/tests/JdeScoping.DataSync.Dev.Tests/JdeScoping.DataSync.Dev.Tests.csproj +``` +Expected: Build succeeded + +--- + +## Task 5: Move Test Files to DataSync.Dev.Tests + +**Files:** +- Move: `NEW/tests/JdeScoping.DataSync.Tests/DevEtl/*.cs` → `NEW/tests/JdeScoping.DataSync.Dev.Tests/` + +**Step 1: Move all 8 test files** + +Run: +```bash +mv /Users/dohertj2/Desktop/JdeScopingTool/NEW/tests/JdeScoping.DataSync.Tests/DevEtl/*.cs /Users/dohertj2/Desktop/JdeScopingTool/NEW/tests/JdeScoping.DataSync.Dev.Tests/ +``` + +**Step 2: Remove empty DevEtl directory** + +Run: +```bash +rmdir /Users/dohertj2/Desktop/JdeScopingTool/NEW/tests/JdeScoping.DataSync.Tests/DevEtl +``` + +**Step 3: Verify files moved** + +Run: +```bash +ls /Users/dohertj2/Desktop/JdeScopingTool/NEW/tests/JdeScoping.DataSync.Dev.Tests/*.cs | wc -l +``` +Expected: 8 + +--- + +## Task 6: Update Namespaces and Usings in Test Files + +**Files:** +- Modify: All 8 `.cs` files in `NEW/tests/JdeScoping.DataSync.Dev.Tests/` + +**Step 1: Update namespace declarations** + +Replace in all files: +```csharp +namespace JdeScoping.DataSync.Tests.DevEtl; +``` + +With: +```csharp +namespace JdeScoping.DataSync.Dev.Tests; +``` + +Run: +```bash +cd /Users/dohertj2/Desktop/JdeScopingTool/NEW/tests/JdeScoping.DataSync.Dev.Tests && \ +sed -i '' 's/namespace JdeScoping\.DataSync\.Tests\.DevEtl;/namespace JdeScoping.DataSync.Dev.Tests;/g' *.cs +``` + +**Step 2: Update using statements** + +Replace in all files: +```csharp +using JdeScoping.DataSync.DevEtl; +``` + +With: +```csharp +using JdeScoping.DataSync.Dev; +``` + +Run: +```bash +cd /Users/dohertj2/Desktop/JdeScopingTool/NEW/tests/JdeScoping.DataSync.Dev.Tests && \ +sed -i '' 's/using JdeScoping\.DataSync\.DevEtl;/using JdeScoping.DataSync.Dev;/g' *.cs +``` + +**Step 3: Verify changes** + +Run: +```bash +grep -l "DevEtl" /Users/dohertj2/Desktop/JdeScopingTool/NEW/tests/JdeScoping.DataSync.Dev.Tests/*.cs +``` +Expected: No output (no files should contain "DevEtl") + +**Step 4: Build to verify** + +Run: +```bash +dotnet build /Users/dohertj2/Desktop/JdeScopingTool/NEW/tests/JdeScoping.DataSync.Dev.Tests/JdeScoping.DataSync.Dev.Tests.csproj +``` +Expected: Build succeeded + +--- + +## Task 7: Update Solution File + +**Files:** +- Modify: `NEW/JdeScoping.slnx` + +**Step 1: Add new projects to solution** + +Update `NEW/JdeScoping.slnx` to add two new Project entries: + +In the `/src/` folder section, add: +```xml + +``` + +In the `/tests/` folder section, add: +```xml + +``` + +**Step 2: Verify solution builds** + +Run: +```bash +dotnet build /Users/dohertj2/Desktop/JdeScopingTool/NEW/JdeScoping.slnx +``` +Expected: Build succeeded + +--- + +## Task 8: Run Tests to Verify + +**Step 1: Run new test project** + +Run: +```bash +dotnet test /Users/dohertj2/Desktop/JdeScopingTool/NEW/tests/JdeScoping.DataSync.Dev.Tests/JdeScoping.DataSync.Dev.Tests.csproj --verbosity normal +``` +Expected: All tests pass (some may skip if cache files don't exist) + +**Step 2: Run full solution tests** + +Run: +```bash +dotnet test /Users/dohertj2/Desktop/JdeScopingTool/NEW/JdeScoping.slnx --verbosity minimal +``` +Expected: All tests pass + +--- + +## Task 9: Commit Changes + +**Step 1: Stage all changes** + +Run: +```bash +cd /Users/dohertj2/Desktop/JdeScopingTool && \ +git add NEW/src/JdeScoping.DataSync.Dev/ && \ +git add NEW/tests/JdeScoping.DataSync.Dev.Tests/ && \ +git add NEW/JdeScoping.slnx +``` + +**Step 2: Commit** + +Run: +```bash +git commit -m "feat: extract DevEtl to JdeScoping.DataSync.Dev project + +- Create JdeScoping.DataSync.Dev for sandbox testing ETL code +- Create JdeScoping.DataSync.Dev.Tests for associated tests +- Move 22 source files and 8 test files +- Update namespaces from DevEtl to Dev +- Add both projects to solution" +``` + +--- + +## Summary + +| Task | Description | Files Changed | +|------|-------------|---------------| +| 1 | Create DataSync.Dev project | 1 new csproj | +| 2 | Move source files | 22 files moved | +| 3 | Update source namespaces | 22 files modified | +| 4 | Create DataSync.Dev.Tests project | 1 new csproj + appsettings | +| 5 | Move test files | 8 files moved | +| 6 | Update test namespaces/usings | 8 files modified | +| 7 | Update solution file | 1 file modified | +| 8 | Run tests | Verification | +| 9 | Commit | Git |