feat: extract DevEtl to JdeScoping.DataSync.Dev project

- Create JdeScoping.DataSync.Dev for sandbox testing ETL code
- Create JdeScoping.DataSync.Dev.Tests for associated tests
- Move 22 source files and 8 test files
- Update namespaces from DevEtl to Dev
- Add both projects to solution
This commit is contained in:
Joseph Doherty
2026-01-06 10:18:09 -05:00
parent 7ad4e3ec1c
commit 81b07ce027
37 changed files with 2458 additions and 83 deletions
@@ -0,0 +1,37 @@
using JdeScoping.DataAccess.Interfaces;
using JdeScoping.DataSync.Etl.Destinations;
using JdeScoping.DataSync.Etl.Models;
using JdeScoping.DataSync.Etl.Pipeline;
using JdeScoping.DataSync.Etl.Sources;
namespace JdeScoping.DataSync.Dev;
/// <summary>
/// Development ETL pipeline for the Branch table.
/// </summary>
public static class BranchDevEtl
{
public static readonly string TableName = "Branch";
public static readonly string CacheFileName = "branch.json.zstd";
private static readonly JsonColumnSchema[] Schema =
[
new("Code", typeof(string), IsNullable: false),
new("Description", typeof(string), IsNullable: true),
new("LastUpdateDT", typeof(DateTime), IsNullable: false),
];
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
{
ArgumentNullException.ThrowIfNull(connectionFactory);
if (string.IsNullOrWhiteSpace(cacheFilePath))
throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath));
return new EtlPipelineBuilder()
.WithName($"{TableName}_Dev")
.WithSource(new JsonZstdFileSource(cacheFilePath, Schema))
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
.Build();
}
}
@@ -0,0 +1,188 @@
using System.Collections.Concurrent;
using JdeScoping.DataAccess.Interfaces;
using JdeScoping.DataSync.Etl.Pipeline;
using JdeScoping.DataSync.Etl.Results;
using Microsoft.Extensions.Logging;
namespace JdeScoping.DataSync.Dev;
/// <summary>
/// Registry for development ETL pipelines that load from cached JSON files.
/// </summary>
public class DevEtlRegistry
{
private readonly IDbConnectionFactory _connectionFactory;
private readonly string _cacheDirectory;
private readonly ILogger<DevEtlRegistry>? _logger;
private readonly Dictionary<string, Func<IDbConnectionFactory, string, EtlPipeline>> _pipelineFactories = new(StringComparer.OrdinalIgnoreCase)
{
// Small tables (< 1 MB)
[BranchDevEtl.TableName] = (factory, cacheDir) =>
BranchDevEtl.Create(factory, Path.Combine(cacheDir, BranchDevEtl.CacheFileName)),
[OrgHierarchyDevEtl.TableName] = (factory, cacheDir) =>
OrgHierarchyDevEtl.Create(factory, Path.Combine(cacheDir, OrgHierarchyDevEtl.CacheFileName)),
[WorkCenterDevEtl.TableName] = (factory, cacheDir) =>
WorkCenterDevEtl.Create(factory, Path.Combine(cacheDir, WorkCenterDevEtl.CacheFileName)),
[ProfitCenterDevEtl.TableName] = (factory, cacheDir) =>
ProfitCenterDevEtl.Create(factory, Path.Combine(cacheDir, ProfitCenterDevEtl.CacheFileName)),
// Medium tables (1-20 MB)
[JdeUserDevEtl.TableName] = (factory, cacheDir) =>
JdeUserDevEtl.Create(factory, Path.Combine(cacheDir, JdeUserDevEtl.CacheFileName)),
[FunctionCodeDevEtl.TableName] = (factory, cacheDir) =>
FunctionCodeDevEtl.Create(factory, Path.Combine(cacheDir, FunctionCodeDevEtl.CacheFileName)),
[ItemDevEtl.TableName] = (factory, cacheDir) =>
ItemDevEtl.Create(factory, Path.Combine(cacheDir, ItemDevEtl.CacheFileName)),
[RouteMasterDevEtl.TableName] = (factory, cacheDir) =>
RouteMasterDevEtl.Create(factory, Path.Combine(cacheDir, RouteMasterDevEtl.CacheFileName)),
// Large tables (20-200 MB)
[LotDevEtl.TableName] = (factory, cacheDir) =>
LotDevEtl.Create(factory, Path.Combine(cacheDir, LotDevEtl.CacheFileName)),
[MisDataDevEtl.TableName] = (factory, cacheDir) =>
MisDataDevEtl.Create(factory, Path.Combine(cacheDir, MisDataDevEtl.CacheFileName)),
[WorkOrderCurrDevEtl.TableName] = (factory, cacheDir) =>
WorkOrderCurrDevEtl.Create(factory, Path.Combine(cacheDir, WorkOrderCurrDevEtl.CacheFileName)),
[WorkOrderHistDevEtl.TableName] = (factory, cacheDir) =>
WorkOrderHistDevEtl.Create(factory, Path.Combine(cacheDir, WorkOrderHistDevEtl.CacheFileName)),
[LotUsageHistDevEtl.TableName] = (factory, cacheDir) =>
LotUsageHistDevEtl.Create(factory, Path.Combine(cacheDir, LotUsageHistDevEtl.CacheFileName)),
[WorkOrderComponentHistDevEtl.TableName] = (factory, cacheDir) =>
WorkOrderComponentHistDevEtl.Create(factory, Path.Combine(cacheDir, WorkOrderComponentHistDevEtl.CacheFileName)),
// Very large tables (200+ MB)
[WorkOrderStepHistDevEtl.TableName] = (factory, cacheDir) =>
WorkOrderStepHistDevEtl.Create(factory, Path.Combine(cacheDir, WorkOrderStepHistDevEtl.CacheFileName)),
[WorkOrderComponentCurrDevEtl.TableName] = (factory, cacheDir) =>
WorkOrderComponentCurrDevEtl.Create(factory, Path.Combine(cacheDir, WorkOrderComponentCurrDevEtl.CacheFileName)),
[WorkOrderRoutingDevEtl.TableName] = (factory, cacheDir) =>
WorkOrderRoutingDevEtl.Create(factory, Path.Combine(cacheDir, WorkOrderRoutingDevEtl.CacheFileName)),
[LotUsageCurrDevEtl.TableName] = (factory, cacheDir) =>
LotUsageCurrDevEtl.Create(factory, Path.Combine(cacheDir, LotUsageCurrDevEtl.CacheFileName)),
[WorkOrderStepCurrDevEtl.TableName] = (factory, cacheDir) =>
WorkOrderStepCurrDevEtl.Create(factory, Path.Combine(cacheDir, WorkOrderStepCurrDevEtl.CacheFileName)),
[WorkOrderTimeHistDevEtl.TableName] = (factory, cacheDir) =>
WorkOrderTimeHistDevEtl.Create(factory, Path.Combine(cacheDir, WorkOrderTimeHistDevEtl.CacheFileName)),
[WorkOrderTimeCurrDevEtl.TableName] = (factory, cacheDir) =>
WorkOrderTimeCurrDevEtl.Create(factory, Path.Combine(cacheDir, WorkOrderTimeCurrDevEtl.CacheFileName)),
};
public DevEtlRegistry(
IDbConnectionFactory connectionFactory,
string cacheDirectory,
ILogger<DevEtlRegistry>? logger = null)
{
_connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory));
if (string.IsNullOrWhiteSpace(cacheDirectory))
throw new ArgumentException("Cache directory is required.", nameof(cacheDirectory));
if (!Directory.Exists(cacheDirectory))
throw new DirectoryNotFoundException($"Cache directory not found: {cacheDirectory}");
_cacheDirectory = cacheDirectory;
_logger = logger;
}
public IEnumerable<string> GetAvailableTables() => _pipelineFactories.Keys;
public EtlPipeline GetPipeline(string tableName)
{
if (!_pipelineFactories.TryGetValue(tableName, out var factory))
throw new ArgumentException($"No pipeline registered for table '{tableName}'.", nameof(tableName));
return factory(_connectionFactory, _cacheDirectory);
}
public async Task<PipelineResult> RunAsync(string tableName, CancellationToken cancellationToken = default)
{
_logger?.LogInformation("Running dev ETL for {TableName}", tableName);
var pipeline = GetPipeline(tableName);
var result = await pipeline.ExecuteAsync(cancellationToken);
if (result.Success)
_logger?.LogInformation("Completed {TableName}: {Rows} rows in {Elapsed:g}",
tableName, result.TotalRows, result.Elapsed);
else
_logger?.LogError(result.Error, "Failed {TableName}: {Error}",
tableName, result.Error?.Message);
return result;
}
public async Task<IReadOnlyList<PipelineResult>> RunAllAsync(CancellationToken cancellationToken = default)
{
var results = new List<PipelineResult>();
foreach (var tableName in GetAvailableTables())
{
cancellationToken.ThrowIfCancellationRequested();
var result = await RunAsync(tableName, cancellationToken);
results.Add(result);
}
return results;
}
/// <summary>
/// Runs all dev ETL pipelines with parallelization.
/// Small/medium tables run concurrently, very large tables run sequentially at the end.
/// </summary>
/// <param name="maxDegreeOfParallelism">Maximum concurrent table loads (default 4).</param>
/// <param name="cancellationToken">Cancellation token.</param>
public async Task<IReadOnlyList<PipelineResult>> RunAllParallelAsync(
int maxDegreeOfParallelism = 4,
CancellationToken cancellationToken = default)
{
var results = new ConcurrentBag<PipelineResult>();
using var semaphore = new SemaphoreSlim(maxDegreeOfParallelism);
// Separate tables by size - run very large ones sequentially at the end
var smallMediumTables = GetAvailableTables()
.Where(t => !IsVeryLargeTable(t))
.ToList();
var veryLargeTables = GetAvailableTables()
.Where(IsVeryLargeTable)
.ToList();
_logger?.LogInformation(
"Running {ParallelCount} tables in parallel (max {MaxParallel}), then {SequentialCount} large tables sequentially",
smallMediumTables.Count, maxDegreeOfParallelism, veryLargeTables.Count);
// Run small/medium tables in parallel
var tasks = smallMediumTables.Select(async tableName =>
{
await semaphore.WaitAsync(cancellationToken);
try
{
var result = await RunAsync(tableName, cancellationToken);
results.Add(result);
}
finally
{
semaphore.Release();
}
});
await Task.WhenAll(tasks);
// Run very large tables sequentially (IO-bound, would contend)
foreach (var tableName in veryLargeTables)
{
cancellationToken.ThrowIfCancellationRequested();
var result = await RunAsync(tableName, cancellationToken);
results.Add(result);
}
return results.ToList();
}
/// <summary>
/// Identifies very large tables that should be loaded sequentially to avoid IO contention.
/// </summary>
private static bool IsVeryLargeTable(string tableName) =>
tableName.Contains("WorkOrderTime", StringComparison.OrdinalIgnoreCase) ||
tableName.Contains("WorkOrderStep", StringComparison.OrdinalIgnoreCase) ||
tableName.Contains("WorkOrderRouting", StringComparison.OrdinalIgnoreCase) ||
tableName.Contains("WorkOrderComponent", StringComparison.OrdinalIgnoreCase) ||
tableName.Contains("LotUsage", StringComparison.OrdinalIgnoreCase);
}
@@ -0,0 +1,38 @@
using JdeScoping.DataAccess.Interfaces;
using JdeScoping.DataSync.Etl.Destinations;
using JdeScoping.DataSync.Etl.Models;
using JdeScoping.DataSync.Etl.Pipeline;
using JdeScoping.DataSync.Etl.Sources;
namespace JdeScoping.DataSync.Dev;
/// <summary>
/// Development ETL pipeline for the FunctionCode table.
/// Schema from: Scripts/005_CreateFunctionCodeTable.sql
/// </summary>
public static class FunctionCodeDevEtl
{
public static readonly string TableName = "FunctionCode";
public static readonly string CacheFileName = "functioncode.json.zstd";
private static readonly JsonColumnSchema[] Schema =
[
new("Code", typeof(string), IsNullable: false),
new("Description", typeof(string), IsNullable: true),
new("LastUpdateDT", typeof(DateTime), IsNullable: false),
];
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
{
ArgumentNullException.ThrowIfNull(connectionFactory);
if (string.IsNullOrWhiteSpace(cacheFilePath))
throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath));
return new EtlPipelineBuilder()
.WithName($"{TableName}_Dev")
.WithSource(new JsonZstdFileSource(cacheFilePath, Schema))
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
.Build();
}
}
@@ -0,0 +1,41 @@
using JdeScoping.DataAccess.Interfaces;
using JdeScoping.DataSync.Etl.Destinations;
using JdeScoping.DataSync.Etl.Models;
using JdeScoping.DataSync.Etl.Pipeline;
using JdeScoping.DataSync.Etl.Sources;
namespace JdeScoping.DataSync.Dev;
/// <summary>
/// Development ETL pipeline for the Item table.
/// Schema from: Scripts/008_CreateItemTable.sql
/// </summary>
public static class ItemDevEtl
{
public static readonly string TableName = "Item";
public static readonly string CacheFileName = "item.json.zstd";
private static readonly JsonColumnSchema[] Schema =
[
new("ShortItemNumber", typeof(long), IsNullable: false),
new("ItemNumber", typeof(string), IsNullable: false),
new("Description", typeof(string), IsNullable: true),
new("PlanningFamily", typeof(string), IsNullable: true),
new("StockingType", typeof(string), IsNullable: true),
new("LastUpdateDT", typeof(DateTime), IsNullable: false),
];
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
{
ArgumentNullException.ThrowIfNull(connectionFactory);
if (string.IsNullOrWhiteSpace(cacheFilePath))
throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath));
return new EtlPipelineBuilder()
.WithName($"{TableName}_Dev")
.WithSource(new JsonZstdFileSource(cacheFilePath, Schema))
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
.Build();
}
}
@@ -0,0 +1,13 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\JdeScoping.DataSync\JdeScoping.DataSync.csproj" />
</ItemGroup>
</Project>
@@ -0,0 +1,39 @@
using JdeScoping.DataAccess.Interfaces;
using JdeScoping.DataSync.Etl.Destinations;
using JdeScoping.DataSync.Etl.Models;
using JdeScoping.DataSync.Etl.Pipeline;
using JdeScoping.DataSync.Etl.Sources;
namespace JdeScoping.DataSync.Dev;
/// <summary>
/// Development ETL pipeline for the JdeUser table.
/// Schema from: Scripts/009_CreateJdeUserTable.sql
/// </summary>
public static class JdeUserDevEtl
{
public static readonly string TableName = "JdeUser";
public static readonly string CacheFileName = "jdeuser.json.zstd";
private static readonly JsonColumnSchema[] Schema =
[
new("AddressNumber", typeof(long), IsNullable: false),
new("UserID", typeof(string), IsNullable: true),
new("FullName", typeof(string), IsNullable: false),
new("LastUpdateDT", typeof(DateTime), IsNullable: false),
];
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
{
ArgumentNullException.ThrowIfNull(connectionFactory);
if (string.IsNullOrWhiteSpace(cacheFilePath))
throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath));
return new EtlPipelineBuilder()
.WithName($"{TableName}_Dev")
.WithSource(new JsonZstdFileSource(cacheFilePath, Schema))
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
.Build();
}
}
@@ -0,0 +1,45 @@
using JdeScoping.DataAccess.Interfaces;
using JdeScoping.DataSync.Etl.Destinations;
using JdeScoping.DataSync.Etl.Models;
using JdeScoping.DataSync.Etl.Pipeline;
using JdeScoping.DataSync.Etl.Sources;
namespace JdeScoping.DataSync.Dev;
/// <summary>
/// Development ETL pipeline for the Lot table.
/// Schema from: Scripts/013_CreateLotTable.sql
/// </summary>
public static class LotDevEtl
{
public static readonly string TableName = "Lot";
public static readonly string CacheFileName = "lot.json.zstd";
private static readonly JsonColumnSchema[] Schema =
[
new("LotNumber", typeof(string), IsNullable: false),
new("BranchCode", typeof(string), IsNullable: false),
new("ShortItemNumber", typeof(long), IsNullable: false),
new("ItemNumber", typeof(string), IsNullable: true),
new("SupplierCode", typeof(long), IsNullable: false),
new("StatusCode", typeof(string), IsNullable: true),
new("Memo1", typeof(string), IsNullable: true),
new("Memo2", typeof(string), IsNullable: true),
new("Memo3", typeof(string), IsNullable: true),
new("LastUpdateDT", typeof(DateTime), IsNullable: false),
];
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
{
ArgumentNullException.ThrowIfNull(connectionFactory);
if (string.IsNullOrWhiteSpace(cacheFilePath))
throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath));
return new EtlPipelineBuilder()
.WithName($"{TableName}_Dev")
.WithSource(new JsonZstdFileSource(cacheFilePath, Schema))
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
.Build();
}
}
@@ -0,0 +1,42 @@
using JdeScoping.DataAccess.Interfaces;
using JdeScoping.DataSync.Etl.Destinations;
using JdeScoping.DataSync.Etl.Models;
using JdeScoping.DataSync.Etl.Pipeline;
using JdeScoping.DataSync.Etl.Sources;
namespace JdeScoping.DataSync.Dev;
/// <summary>
/// Development ETL pipeline for the LotUsage_Curr table.
/// Schema from: Scripts/024_CreateLotUsageCurrTable.sql
/// </summary>
public static class LotUsageCurrDevEtl
{
public static readonly string TableName = "LotUsage_Curr";
public static readonly string CacheFileName = "lotusage_curr.json.zstd";
private static readonly JsonColumnSchema[] Schema =
[
new("UniqueID", typeof(long), IsNullable: false),
new("WorkOrderNumber", typeof(long), IsNullable: false),
new("LotNumber", typeof(string), IsNullable: false),
new("BranchCode", typeof(string), IsNullable: true),
new("ShortItemNumber", typeof(long), IsNullable: false),
new("Quantity", typeof(decimal), IsNullable: false),
new("LastUpdateDT", typeof(DateTime), IsNullable: false),
];
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
{
ArgumentNullException.ThrowIfNull(connectionFactory);
if (string.IsNullOrWhiteSpace(cacheFilePath))
throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath));
return new EtlPipelineBuilder()
.WithName($"{TableName}_Dev")
.WithSource(new JsonZstdFileSource(cacheFilePath, Schema))
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
.Build();
}
}
@@ -0,0 +1,42 @@
using JdeScoping.DataAccess.Interfaces;
using JdeScoping.DataSync.Etl.Destinations;
using JdeScoping.DataSync.Etl.Models;
using JdeScoping.DataSync.Etl.Pipeline;
using JdeScoping.DataSync.Etl.Sources;
namespace JdeScoping.DataSync.Dev;
/// <summary>
/// Development ETL pipeline for the LotUsage_Hist table.
/// Schema from: Scripts/025_CreateLotUsageHistTable.sql
/// </summary>
public static class LotUsageHistDevEtl
{
public static readonly string TableName = "LotUsage_Hist";
public static readonly string CacheFileName = "lotusage_hist.json.zstd";
private static readonly JsonColumnSchema[] Schema =
[
new("UniqueID", typeof(long), IsNullable: false),
new("WorkOrderNumber", typeof(long), IsNullable: false),
new("LotNumber", typeof(string), IsNullable: false),
new("BranchCode", typeof(string), IsNullable: true),
new("ShortItemNumber", typeof(long), IsNullable: false),
new("Quantity", typeof(decimal), IsNullable: false),
new("LastUpdateDT", typeof(DateTime), IsNullable: false),
];
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
{
ArgumentNullException.ThrowIfNull(connectionFactory);
if (string.IsNullOrWhiteSpace(cacheFilePath))
throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath));
return new EtlPipelineBuilder()
.WithName($"{TableName}_Dev")
.WithSource(new JsonZstdFileSource(cacheFilePath, Schema))
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
.Build();
}
}
@@ -0,0 +1,49 @@
using JdeScoping.DataAccess.Interfaces;
using JdeScoping.DataSync.Etl.Destinations;
using JdeScoping.DataSync.Etl.Models;
using JdeScoping.DataSync.Etl.Pipeline;
using JdeScoping.DataSync.Etl.Sources;
namespace JdeScoping.DataSync.Dev;
/// <summary>
/// Development ETL pipeline for the MisData table.
/// Schema from: Scripts/012_CreateMisDataTable.sql
/// </summary>
public static class MisDataDevEtl
{
public static readonly string TableName = "MisData";
public static readonly string CacheFileName = "misdata.json.zstd";
private static readonly JsonColumnSchema[] Schema =
[
new("ItemNumber", typeof(string), IsNullable: false),
new("BranchCode", typeof(string), IsNullable: false),
new("SequenceNumber", typeof(string), IsNullable: false),
new("MisNumber", typeof(string), IsNullable: false),
new("RevID", typeof(string), IsNullable: false),
new("CharNumber", typeof(string), IsNullable: false),
new("TestDescription", typeof(string), IsNullable: true),
new("SamplingType", typeof(string), IsNullable: true),
new("SamplingValue", typeof(string), IsNullable: true),
new("ToolsGauges", typeof(string), IsNullable: true),
new("WorkInstructions", typeof(string), IsNullable: true),
new("Status", typeof(string), IsNullable: false),
new("ReleaseDate", typeof(DateTime), IsNullable: true),
new("ObsoleteDate", typeof(DateTime), IsNullable: true),
];
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
{
ArgumentNullException.ThrowIfNull(connectionFactory);
if (string.IsNullOrWhiteSpace(cacheFilePath))
throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath));
return new EtlPipelineBuilder()
.WithName($"{TableName}_Dev")
.WithSource(new JsonZstdFileSource(cacheFilePath, Schema))
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
.Build();
}
}
@@ -0,0 +1,39 @@
using JdeScoping.DataAccess.Interfaces;
using JdeScoping.DataSync.Etl.Destinations;
using JdeScoping.DataSync.Etl.Models;
using JdeScoping.DataSync.Etl.Pipeline;
using JdeScoping.DataSync.Etl.Sources;
namespace JdeScoping.DataSync.Dev;
/// <summary>
/// Development ETL pipeline for the OrgHierarchy table.
/// Schema from: Scripts/010_CreateOrgHierarchyTable.sql
/// </summary>
public static class OrgHierarchyDevEtl
{
public static readonly string TableName = "OrgHierarchy";
public static readonly string CacheFileName = "orghierarchy.json.zstd";
private static readonly JsonColumnSchema[] Schema =
[
new("WorkCenterCode", typeof(string), IsNullable: false),
new("BranchCode", typeof(string), IsNullable: false),
new("ProfitCenterCode", typeof(string), IsNullable: false),
new("LastUpdateDT", typeof(DateTime), IsNullable: false),
];
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
{
ArgumentNullException.ThrowIfNull(connectionFactory);
if (string.IsNullOrWhiteSpace(cacheFilePath))
throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath));
return new EtlPipelineBuilder()
.WithName($"{TableName}_Dev")
.WithSource(new JsonZstdFileSource(cacheFilePath, Schema))
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
.Build();
}
}
@@ -0,0 +1,38 @@
using JdeScoping.DataAccess.Interfaces;
using JdeScoping.DataSync.Etl.Destinations;
using JdeScoping.DataSync.Etl.Models;
using JdeScoping.DataSync.Etl.Pipeline;
using JdeScoping.DataSync.Etl.Sources;
namespace JdeScoping.DataSync.Dev;
/// <summary>
/// Development ETL pipeline for the ProfitCenter table.
/// Schema from: Scripts/006_CreateProfitCenterTable.sql
/// </summary>
public static class ProfitCenterDevEtl
{
public static readonly string TableName = "ProfitCenter";
public static readonly string CacheFileName = "profitcenter.json.zstd";
private static readonly JsonColumnSchema[] Schema =
[
new("Code", typeof(string), IsNullable: false),
new("Description", typeof(string), IsNullable: true),
new("LastUpdateDT", typeof(DateTime), IsNullable: false),
];
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
{
ArgumentNullException.ThrowIfNull(connectionFactory);
if (string.IsNullOrWhiteSpace(cacheFilePath))
throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath));
return new EtlPipelineBuilder()
.WithName($"{TableName}_Dev")
.WithSource(new JsonZstdFileSource(cacheFilePath, Schema))
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
.Build();
}
}
@@ -0,0 +1,44 @@
using JdeScoping.DataAccess.Interfaces;
using JdeScoping.DataSync.Etl.Destinations;
using JdeScoping.DataSync.Etl.Models;
using JdeScoping.DataSync.Etl.Pipeline;
using JdeScoping.DataSync.Etl.Sources;
namespace JdeScoping.DataSync.Dev;
/// <summary>
/// Development ETL pipeline for the RouteMaster table.
/// Schema from: Scripts/011_CreateRouteMasterTable.sql
/// </summary>
public static class RouteMasterDevEtl
{
public static readonly string TableName = "RouteMaster";
public static readonly string CacheFileName = "routemaster.json.zstd";
private static readonly JsonColumnSchema[] Schema =
[
new("BranchCode", typeof(string), IsNullable: false),
new("ItemNumber", typeof(string), IsNullable: false),
new("RoutingType", typeof(string), IsNullable: false),
new("SequenceNumber", typeof(decimal), IsNullable: false),
new("FunctionCode", typeof(string), IsNullable: true),
new("WorkCenterCode", typeof(string), IsNullable: true),
new("StartDate", typeof(DateTime), IsNullable: false),
new("EndDate", typeof(DateTime), IsNullable: true),
new("LastUpdateDT", typeof(DateTime), IsNullable: false),
];
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
{
ArgumentNullException.ThrowIfNull(connectionFactory);
if (string.IsNullOrWhiteSpace(cacheFilePath))
throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath));
return new EtlPipelineBuilder()
.WithName($"{TableName}_Dev")
.WithSource(new JsonZstdFileSource(cacheFilePath, Schema))
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
.Build();
}
}
@@ -0,0 +1,38 @@
using JdeScoping.DataAccess.Interfaces;
using JdeScoping.DataSync.Etl.Destinations;
using JdeScoping.DataSync.Etl.Models;
using JdeScoping.DataSync.Etl.Pipeline;
using JdeScoping.DataSync.Etl.Sources;
namespace JdeScoping.DataSync.Dev;
/// <summary>
/// Development ETL pipeline for the WorkCenter table.
/// Schema from: Scripts/007_CreateWorkCenterTable.sql
/// </summary>
public static class WorkCenterDevEtl
{
public static readonly string TableName = "WorkCenter";
public static readonly string CacheFileName = "workcenter.json.zstd";
private static readonly JsonColumnSchema[] Schema =
[
new("Code", typeof(string), IsNullable: false),
new("Description", typeof(string), IsNullable: true),
new("LastUpdateDT", typeof(DateTime), IsNullable: false),
];
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
{
ArgumentNullException.ThrowIfNull(connectionFactory);
if (string.IsNullOrWhiteSpace(cacheFilePath))
throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath));
return new EtlPipelineBuilder()
.WithName($"{TableName}_Dev")
.WithSource(new JsonZstdFileSource(cacheFilePath, Schema))
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
.Build();
}
}
@@ -0,0 +1,42 @@
using JdeScoping.DataAccess.Interfaces;
using JdeScoping.DataSync.Etl.Destinations;
using JdeScoping.DataSync.Etl.Models;
using JdeScoping.DataSync.Etl.Pipeline;
using JdeScoping.DataSync.Etl.Sources;
namespace JdeScoping.DataSync.Dev;
/// <summary>
/// Development ETL pipeline for the WorkOrderComponent_Curr table.
/// Schema from: Scripts/021_CreateWorkOrderComponentCurrTable.sql
/// </summary>
public static class WorkOrderComponentCurrDevEtl
{
public static readonly string TableName = "WorkOrderComponent_Curr";
public static readonly string CacheFileName = "workordercomponent_curr.json.zstd";
private static readonly JsonColumnSchema[] Schema =
[
new("UniqueID", typeof(long), IsNullable: false),
new("WorkOrderNumber", typeof(long), IsNullable: false),
new("LotNumber", typeof(string), IsNullable: false),
new("BranchCode", typeof(string), IsNullable: true),
new("ShortItemNumber", typeof(long), IsNullable: false),
new("Quantity", typeof(decimal), IsNullable: false),
new("LastUpdateDT", typeof(DateTime), IsNullable: false),
];
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
{
ArgumentNullException.ThrowIfNull(connectionFactory);
if (string.IsNullOrWhiteSpace(cacheFilePath))
throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath));
return new EtlPipelineBuilder()
.WithName($"{TableName}_Dev")
.WithSource(new JsonZstdFileSource(cacheFilePath, Schema))
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
.Build();
}
}
@@ -0,0 +1,42 @@
using JdeScoping.DataAccess.Interfaces;
using JdeScoping.DataSync.Etl.Destinations;
using JdeScoping.DataSync.Etl.Models;
using JdeScoping.DataSync.Etl.Pipeline;
using JdeScoping.DataSync.Etl.Sources;
namespace JdeScoping.DataSync.Dev;
/// <summary>
/// Development ETL pipeline for the WorkOrderComponent_Hist table.
/// Schema from: Scripts/022_CreateWorkOrderComponentHistTable.sql
/// </summary>
public static class WorkOrderComponentHistDevEtl
{
public static readonly string TableName = "WorkOrderComponent_Hist";
public static readonly string CacheFileName = "workordercomponent_hist.json.zstd";
private static readonly JsonColumnSchema[] Schema =
[
new("UniqueID", typeof(long), IsNullable: false),
new("WorkOrderNumber", typeof(long), IsNullable: false),
new("LotNumber", typeof(string), IsNullable: false),
new("BranchCode", typeof(string), IsNullable: true),
new("ShortItemNumber", typeof(long), IsNullable: false),
new("Quantity", typeof(decimal), IsNullable: false),
new("LastUpdateDT", typeof(DateTime), IsNullable: false),
];
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
{
ArgumentNullException.ThrowIfNull(connectionFactory);
if (string.IsNullOrWhiteSpace(cacheFilePath))
throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath));
return new EtlPipelineBuilder()
.WithName($"{TableName}_Dev")
.WithSource(new JsonZstdFileSource(cacheFilePath, Schema))
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
.Build();
}
}
@@ -0,0 +1,50 @@
using JdeScoping.DataAccess.Interfaces;
using JdeScoping.DataSync.Etl.Destinations;
using JdeScoping.DataSync.Etl.Models;
using JdeScoping.DataSync.Etl.Pipeline;
using JdeScoping.DataSync.Etl.Sources;
namespace JdeScoping.DataSync.Dev;
/// <summary>
/// Development ETL pipeline for the WorkOrder_Curr table.
/// Schema from: Scripts/015_CreateWorkOrderCurrTable.sql
/// </summary>
public static class WorkOrderCurrDevEtl
{
public static readonly string TableName = "WorkOrder_Curr";
public static readonly string CacheFileName = "workorder_curr.json.zstd";
private static readonly JsonColumnSchema[] Schema =
[
new("WorkOrderNumber", typeof(long), IsNullable: false),
new("BranchCode", typeof(string), IsNullable: true),
new("LotNumber", typeof(string), IsNullable: true),
new("ItemNumber", typeof(string), IsNullable: true),
new("ShortItemNumber", typeof(long), IsNullable: false),
new("ParentWorkOrderNumber", typeof(string), IsNullable: true),
new("OrderQuantity", typeof(decimal), IsNullable: false),
new("HeldQuantity", typeof(decimal), IsNullable: false),
new("ShippedQuantity", typeof(decimal), IsNullable: false),
new("StatusCode", typeof(string), IsNullable: true),
new("StatusCodeUpdateDT", typeof(DateTime), IsNullable: true),
new("IssueDate", typeof(DateTime), IsNullable: false),
new("StartDate", typeof(DateTime), IsNullable: false),
new("RoutingType", typeof(string), IsNullable: true),
new("LastUpdateDT", typeof(DateTime), IsNullable: false),
];
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
{
ArgumentNullException.ThrowIfNull(connectionFactory);
if (string.IsNullOrWhiteSpace(cacheFilePath))
throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath));
return new EtlPipelineBuilder()
.WithName($"{TableName}_Dev")
.WithSource(new JsonZstdFileSource(cacheFilePath, Schema))
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
.Build();
}
}
@@ -0,0 +1,50 @@
using JdeScoping.DataAccess.Interfaces;
using JdeScoping.DataSync.Etl.Destinations;
using JdeScoping.DataSync.Etl.Models;
using JdeScoping.DataSync.Etl.Pipeline;
using JdeScoping.DataSync.Etl.Sources;
namespace JdeScoping.DataSync.Dev;
/// <summary>
/// Development ETL pipeline for the WorkOrder_Hist table.
/// Schema from: Scripts/016_CreateWorkOrderHistTable.sql
/// </summary>
public static class WorkOrderHistDevEtl
{
public static readonly string TableName = "WorkOrder_Hist";
public static readonly string CacheFileName = "workorder_hist.json.zstd";
private static readonly JsonColumnSchema[] Schema =
[
new("WorkOrderNumber", typeof(long), IsNullable: false),
new("BranchCode", typeof(string), IsNullable: true),
new("LotNumber", typeof(string), IsNullable: true),
new("ItemNumber", typeof(string), IsNullable: true),
new("ShortItemNumber", typeof(long), IsNullable: false),
new("ParentWorkOrderNumber", typeof(string), IsNullable: true),
new("OrderQuantity", typeof(decimal), IsNullable: false),
new("HeldQuantity", typeof(decimal), IsNullable: false),
new("ShippedQuantity", typeof(decimal), IsNullable: false),
new("StatusCode", typeof(string), IsNullable: true),
new("StatusCodeUpdateDT", typeof(DateTime), IsNullable: true),
new("IssueDate", typeof(DateTime), IsNullable: false),
new("StartDate", typeof(DateTime), IsNullable: false),
new("RoutingType", typeof(string), IsNullable: true),
new("LastUpdateDT", typeof(DateTime), IsNullable: false),
];
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
{
ArgumentNullException.ThrowIfNull(connectionFactory);
if (string.IsNullOrWhiteSpace(cacheFilePath))
throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath));
return new EtlPipelineBuilder()
.WithName($"{TableName}_Dev")
.WithSource(new JsonZstdFileSource(cacheFilePath, Schema))
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
.Build();
}
}
@@ -0,0 +1,48 @@
using JdeScoping.DataAccess.Interfaces;
using JdeScoping.DataSync.Etl.Destinations;
using JdeScoping.DataSync.Etl.Models;
using JdeScoping.DataSync.Etl.Pipeline;
using JdeScoping.DataSync.Etl.Sources;
namespace JdeScoping.DataSync.Dev;
/// <summary>
/// Development ETL pipeline for the WorkOrderRouting table.
/// Schema from: Scripts/023_CreateWorkOrderRoutingTable.sql
/// </summary>
public static class WorkOrderRoutingDevEtl
{
public static readonly string TableName = "WorkOrderRouting";
public static readonly string CacheFileName = "workorderrouting.json.zstd";
private static readonly JsonColumnSchema[] Schema =
[
new("UserID", typeof(string), IsNullable: false),
new("BatchNumber", typeof(string), IsNullable: false),
new("TransactionNumber", typeof(string), IsNullable: false),
new("LineNumber", typeof(int), IsNullable: false),
new("StepNumber", typeof(decimal), IsNullable: false),
new("WorkCenterCode", typeof(string), IsNullable: false),
new("WorkOrderNumber", typeof(long), IsNullable: false),
new("RoutingType", typeof(string), IsNullable: true),
new("BranchCode", typeof(string), IsNullable: true),
new("StepDescription", typeof(string), IsNullable: true),
new("FunctionCode", typeof(string), IsNullable: true),
new("TransactionDate", typeof(DateTime), IsNullable: false),
new("LastUpdateDT", typeof(DateTime), IsNullable: false),
];
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
{
ArgumentNullException.ThrowIfNull(connectionFactory);
if (string.IsNullOrWhiteSpace(cacheFilePath))
throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath));
return new EtlPipelineBuilder()
.WithName($"{TableName}_Dev")
.WithSource(new JsonZstdFileSource(cacheFilePath, Schema))
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
.Build();
}
}
@@ -0,0 +1,46 @@
using JdeScoping.DataAccess.Interfaces;
using JdeScoping.DataSync.Etl.Destinations;
using JdeScoping.DataSync.Etl.Models;
using JdeScoping.DataSync.Etl.Pipeline;
using JdeScoping.DataSync.Etl.Sources;
namespace JdeScoping.DataSync.Dev;
/// <summary>
/// Development ETL pipeline for the WorkOrderStep_Curr table.
/// Schema from: Scripts/017_CreateWorkOrderStepCurrTable.sql
/// </summary>
public static class WorkOrderStepCurrDevEtl
{
public static readonly string TableName = "WorkOrderStep_Curr";
public static readonly string CacheFileName = "workorderstep_curr.json.zstd";
private static readonly JsonColumnSchema[] Schema =
[
new("WorkOrderNumber", typeof(long), IsNullable: false),
new("WorkCenterCode", typeof(string), IsNullable: false),
new("StepNumber", typeof(decimal), IsNullable: false),
new("StepTypeCode", typeof(string), IsNullable: false),
new("BranchCode", typeof(string), IsNullable: false),
new("StepDescription", typeof(string), IsNullable: true),
new("StartDT", typeof(DateTime), IsNullable: true),
new("EndDT", typeof(DateTime), IsNullable: true),
new("FunctionCode", typeof(string), IsNullable: true),
new("ScrappedQuantity", typeof(decimal), IsNullable: false),
new("LastUpdateDT", typeof(DateTime), IsNullable: false),
];
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
{
ArgumentNullException.ThrowIfNull(connectionFactory);
if (string.IsNullOrWhiteSpace(cacheFilePath))
throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath));
return new EtlPipelineBuilder()
.WithName($"{TableName}_Dev")
.WithSource(new JsonZstdFileSource(cacheFilePath, Schema))
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
.Build();
}
}
@@ -0,0 +1,46 @@
using JdeScoping.DataAccess.Interfaces;
using JdeScoping.DataSync.Etl.Destinations;
using JdeScoping.DataSync.Etl.Models;
using JdeScoping.DataSync.Etl.Pipeline;
using JdeScoping.DataSync.Etl.Sources;
namespace JdeScoping.DataSync.Dev;
/// <summary>
/// Development ETL pipeline for the WorkOrderStep_Hist table.
/// Schema from: Scripts/018_CreateWorkOrderStepHistTable.sql
/// </summary>
public static class WorkOrderStepHistDevEtl
{
public static readonly string TableName = "WorkOrderStep_Hist";
public static readonly string CacheFileName = "workorderstep_hist.json.zstd";
private static readonly JsonColumnSchema[] Schema =
[
new("WorkOrderNumber", typeof(long), IsNullable: false),
new("WorkCenterCode", typeof(string), IsNullable: false),
new("StepNumber", typeof(decimal), IsNullable: false),
new("StepTypeCode", typeof(string), IsNullable: false),
new("BranchCode", typeof(string), IsNullable: false),
new("StepDescription", typeof(string), IsNullable: true),
new("StartDT", typeof(DateTime), IsNullable: true),
new("EndDT", typeof(DateTime), IsNullable: true),
new("FunctionCode", typeof(string), IsNullable: true),
new("ScrappedQuantity", typeof(decimal), IsNullable: false),
new("LastUpdateDT", typeof(DateTime), IsNullable: false),
];
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
{
ArgumentNullException.ThrowIfNull(connectionFactory);
if (string.IsNullOrWhiteSpace(cacheFilePath))
throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath));
return new EtlPipelineBuilder()
.WithName($"{TableName}_Dev")
.WithSource(new JsonZstdFileSource(cacheFilePath, Schema))
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
.Build();
}
}
@@ -0,0 +1,43 @@
using JdeScoping.DataAccess.Interfaces;
using JdeScoping.DataSync.Etl.Destinations;
using JdeScoping.DataSync.Etl.Models;
using JdeScoping.DataSync.Etl.Pipeline;
using JdeScoping.DataSync.Etl.Sources;
namespace JdeScoping.DataSync.Dev;
/// <summary>
/// Development ETL pipeline for the WorkOrderTime_Curr table.
/// Schema from: Scripts/019_CreateWorkOrderTimeCurrTable.sql
/// </summary>
public static class WorkOrderTimeCurrDevEtl
{
public static readonly string TableName = "WorkOrderTime_Curr";
public static readonly string CacheFileName = "workordertime_curr.json.zstd";
private static readonly JsonColumnSchema[] Schema =
[
new("UniqueID", typeof(long), IsNullable: false),
new("WorkOrderNumber", typeof(long), IsNullable: false),
new("StepNumber", typeof(decimal), IsNullable: false),
new("WorkCenterCode", typeof(string), IsNullable: false),
new("BranchCode", typeof(string), IsNullable: false),
new("AddressNumber", typeof(long), IsNullable: false),
new("GlDate", typeof(DateTime), IsNullable: true),
new("LastUpdateDT", typeof(DateTime), IsNullable: false),
];
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
{
ArgumentNullException.ThrowIfNull(connectionFactory);
if (string.IsNullOrWhiteSpace(cacheFilePath))
throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath));
return new EtlPipelineBuilder()
.WithName($"{TableName}_Dev")
.WithSource(new JsonZstdFileSource(cacheFilePath, Schema))
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
.Build();
}
}
@@ -0,0 +1,43 @@
using JdeScoping.DataAccess.Interfaces;
using JdeScoping.DataSync.Etl.Destinations;
using JdeScoping.DataSync.Etl.Models;
using JdeScoping.DataSync.Etl.Pipeline;
using JdeScoping.DataSync.Etl.Sources;
namespace JdeScoping.DataSync.Dev;
/// <summary>
/// Development ETL pipeline for the WorkOrderTime_Hist table.
/// Schema from: Scripts/020_CreateWorkOrderTimeHistTable.sql
/// </summary>
public static class WorkOrderTimeHistDevEtl
{
public static readonly string TableName = "WorkOrderTime_Hist";
public static readonly string CacheFileName = "workordertime_hist.json.zstd";
private static readonly JsonColumnSchema[] Schema =
[
new("UniqueID", typeof(long), IsNullable: false),
new("WorkOrderNumber", typeof(long), IsNullable: false),
new("StepNumber", typeof(decimal), IsNullable: false),
new("WorkCenterCode", typeof(string), IsNullable: false),
new("BranchCode", typeof(string), IsNullable: false),
new("AddressNumber", typeof(long), IsNullable: false),
new("GlDate", typeof(DateTime), IsNullable: true),
new("LastUpdateDT", typeof(DateTime), IsNullable: false),
];
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
{
ArgumentNullException.ThrowIfNull(connectionFactory);
if (string.IsNullOrWhiteSpace(cacheFilePath))
throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath));
return new EtlPipelineBuilder()
.WithName($"{TableName}_Dev")
.WithSource(new JsonZstdFileSource(cacheFilePath, Schema))
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
.Build();
}
}