feat: extract DevEtl to JdeScoping.DataSync.Dev project
- Create JdeScoping.DataSync.Dev for sandbox testing ETL code - Create JdeScoping.DataSync.Dev.Tests for associated tests - Move 22 source files and 8 test files - Update namespaces from DevEtl to Dev - Add both projects to solution
This commit is contained in:
Joseph Doherty
@@ -7,6 +7,7 @@
|
||||
<Project Path="src/JdeScoping.Database/JdeScoping.Database.csproj" />
|
||||
<Project Path="src/JdeScoping.DataSync.SourceGenerators/JdeScoping.DataSync.SourceGenerators.csproj" />
|
||||
<Project Path="src/JdeScoping.DataSync/JdeScoping.DataSync.csproj" />
|
||||
<Project Path="src/JdeScoping.DataSync.Dev/JdeScoping.DataSync.Dev.csproj" />
|
||||
<Project Path="src/JdeScoping.ExcelIO/JdeScoping.ExcelIO.csproj" />
|
||||
<Project Path="src/JdeScoping.Host/JdeScoping.Host.csproj" />
|
||||
<Project Path="src/JdeScoping.Infrastructure/JdeScoping.Infrastructure.csproj" />
|
||||
@@ -20,6 +21,7 @@
|
||||
<Project Path="tests/JdeScoping.Database.Tests/JdeScoping.Database.Tests.csproj" />
|
||||
<Project Path="tests/JdeScoping.DataSync.IntegrationTests/JdeScoping.DataSync.IntegrationTests.csproj" />
|
||||
<Project Path="tests/JdeScoping.DataSync.Tests/JdeScoping.DataSync.Tests.csproj" />
|
||||
<Project Path="tests/JdeScoping.DataSync.Dev.Tests/JdeScoping.DataSync.Dev.Tests.csproj" />
|
||||
<Project Path="tests/JdeScoping.ExcelIO.Tests/JdeScoping.ExcelIO.Tests.csproj" />
|
||||
<Project Path="tests/JdeScoping.Host.Tests/JdeScoping.Host.Tests.csproj" />
|
||||
<Project Path="tests/JdeScoping.Infrastructure.Tests/JdeScoping.Infrastructure.Tests.csproj" />
|
||||
|
||||
+1
-1
@@ -4,7 +4,7 @@ using JdeScoping.DataSync.Etl.Models;
|
||||
using JdeScoping.DataSync.Etl.Pipeline;
|
||||
using JdeScoping.DataSync.Etl.Sources;
|
||||
|
||||
namespace JdeScoping.DataSync.DevEtl;
|
||||
namespace JdeScoping.DataSync.Dev;
|
||||
|
||||
/// <summary>
|
||||
/// Development ETL pipeline for the Branch table.
|
||||
@@ -0,0 +1,188 @@
|
||||
using System.Collections.Concurrent;
|
||||
using JdeScoping.DataAccess.Interfaces;
|
||||
using JdeScoping.DataSync.Etl.Pipeline;
|
||||
using JdeScoping.DataSync.Etl.Results;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace JdeScoping.DataSync.Dev;
|
||||
|
||||
/// <summary>
|
||||
/// Registry for development ETL pipelines that load from cached JSON files.
|
||||
/// </summary>
|
||||
public class DevEtlRegistry
|
||||
{
|
||||
private readonly IDbConnectionFactory _connectionFactory;
|
||||
private readonly string _cacheDirectory;
|
||||
private readonly ILogger<DevEtlRegistry>? _logger;
|
||||
|
||||
private readonly Dictionary<string, Func<IDbConnectionFactory, string, EtlPipeline>> _pipelineFactories = new(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
// Small tables (< 1 MB)
|
||||
[BranchDevEtl.TableName] = (factory, cacheDir) =>
|
||||
BranchDevEtl.Create(factory, Path.Combine(cacheDir, BranchDevEtl.CacheFileName)),
|
||||
[OrgHierarchyDevEtl.TableName] = (factory, cacheDir) =>
|
||||
OrgHierarchyDevEtl.Create(factory, Path.Combine(cacheDir, OrgHierarchyDevEtl.CacheFileName)),
|
||||
[WorkCenterDevEtl.TableName] = (factory, cacheDir) =>
|
||||
WorkCenterDevEtl.Create(factory, Path.Combine(cacheDir, WorkCenterDevEtl.CacheFileName)),
|
||||
[ProfitCenterDevEtl.TableName] = (factory, cacheDir) =>
|
||||
ProfitCenterDevEtl.Create(factory, Path.Combine(cacheDir, ProfitCenterDevEtl.CacheFileName)),
|
||||
// Medium tables (1-20 MB)
|
||||
[JdeUserDevEtl.TableName] = (factory, cacheDir) =>
|
||||
JdeUserDevEtl.Create(factory, Path.Combine(cacheDir, JdeUserDevEtl.CacheFileName)),
|
||||
[FunctionCodeDevEtl.TableName] = (factory, cacheDir) =>
|
||||
FunctionCodeDevEtl.Create(factory, Path.Combine(cacheDir, FunctionCodeDevEtl.CacheFileName)),
|
||||
[ItemDevEtl.TableName] = (factory, cacheDir) =>
|
||||
ItemDevEtl.Create(factory, Path.Combine(cacheDir, ItemDevEtl.CacheFileName)),
|
||||
[RouteMasterDevEtl.TableName] = (factory, cacheDir) =>
|
||||
RouteMasterDevEtl.Create(factory, Path.Combine(cacheDir, RouteMasterDevEtl.CacheFileName)),
|
||||
// Large tables (20-200 MB)
|
||||
[LotDevEtl.TableName] = (factory, cacheDir) =>
|
||||
LotDevEtl.Create(factory, Path.Combine(cacheDir, LotDevEtl.CacheFileName)),
|
||||
[MisDataDevEtl.TableName] = (factory, cacheDir) =>
|
||||
MisDataDevEtl.Create(factory, Path.Combine(cacheDir, MisDataDevEtl.CacheFileName)),
|
||||
[WorkOrderCurrDevEtl.TableName] = (factory, cacheDir) =>
|
||||
WorkOrderCurrDevEtl.Create(factory, Path.Combine(cacheDir, WorkOrderCurrDevEtl.CacheFileName)),
|
||||
[WorkOrderHistDevEtl.TableName] = (factory, cacheDir) =>
|
||||
WorkOrderHistDevEtl.Create(factory, Path.Combine(cacheDir, WorkOrderHistDevEtl.CacheFileName)),
|
||||
[LotUsageHistDevEtl.TableName] = (factory, cacheDir) =>
|
||||
LotUsageHistDevEtl.Create(factory, Path.Combine(cacheDir, LotUsageHistDevEtl.CacheFileName)),
|
||||
[WorkOrderComponentHistDevEtl.TableName] = (factory, cacheDir) =>
|
||||
WorkOrderComponentHistDevEtl.Create(factory, Path.Combine(cacheDir, WorkOrderComponentHistDevEtl.CacheFileName)),
|
||||
// Very large tables (200+ MB)
|
||||
[WorkOrderStepHistDevEtl.TableName] = (factory, cacheDir) =>
|
||||
WorkOrderStepHistDevEtl.Create(factory, Path.Combine(cacheDir, WorkOrderStepHistDevEtl.CacheFileName)),
|
||||
[WorkOrderComponentCurrDevEtl.TableName] = (factory, cacheDir) =>
|
||||
WorkOrderComponentCurrDevEtl.Create(factory, Path.Combine(cacheDir, WorkOrderComponentCurrDevEtl.CacheFileName)),
|
||||
[WorkOrderRoutingDevEtl.TableName] = (factory, cacheDir) =>
|
||||
WorkOrderRoutingDevEtl.Create(factory, Path.Combine(cacheDir, WorkOrderRoutingDevEtl.CacheFileName)),
|
||||
[LotUsageCurrDevEtl.TableName] = (factory, cacheDir) =>
|
||||
LotUsageCurrDevEtl.Create(factory, Path.Combine(cacheDir, LotUsageCurrDevEtl.CacheFileName)),
|
||||
[WorkOrderStepCurrDevEtl.TableName] = (factory, cacheDir) =>
|
||||
WorkOrderStepCurrDevEtl.Create(factory, Path.Combine(cacheDir, WorkOrderStepCurrDevEtl.CacheFileName)),
|
||||
[WorkOrderTimeHistDevEtl.TableName] = (factory, cacheDir) =>
|
||||
WorkOrderTimeHistDevEtl.Create(factory, Path.Combine(cacheDir, WorkOrderTimeHistDevEtl.CacheFileName)),
|
||||
[WorkOrderTimeCurrDevEtl.TableName] = (factory, cacheDir) =>
|
||||
WorkOrderTimeCurrDevEtl.Create(factory, Path.Combine(cacheDir, WorkOrderTimeCurrDevEtl.CacheFileName)),
|
||||
};
|
||||
|
||||
public DevEtlRegistry(
|
||||
IDbConnectionFactory connectionFactory,
|
||||
string cacheDirectory,
|
||||
ILogger<DevEtlRegistry>? logger = null)
|
||||
{
|
||||
_connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory));
|
||||
|
||||
if (string.IsNullOrWhiteSpace(cacheDirectory))
|
||||
throw new ArgumentException("Cache directory is required.", nameof(cacheDirectory));
|
||||
|
||||
if (!Directory.Exists(cacheDirectory))
|
||||
throw new DirectoryNotFoundException($"Cache directory not found: {cacheDirectory}");
|
||||
|
||||
_cacheDirectory = cacheDirectory;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
public IEnumerable<string> GetAvailableTables() => _pipelineFactories.Keys;
|
||||
|
||||
public EtlPipeline GetPipeline(string tableName)
|
||||
{
|
||||
if (!_pipelineFactories.TryGetValue(tableName, out var factory))
|
||||
throw new ArgumentException($"No pipeline registered for table '{tableName}'.", nameof(tableName));
|
||||
|
||||
return factory(_connectionFactory, _cacheDirectory);
|
||||
}
|
||||
|
||||
public async Task<PipelineResult> RunAsync(string tableName, CancellationToken cancellationToken = default)
|
||||
{
|
||||
_logger?.LogInformation("Running dev ETL for {TableName}", tableName);
|
||||
|
||||
var pipeline = GetPipeline(tableName);
|
||||
var result = await pipeline.ExecuteAsync(cancellationToken);
|
||||
|
||||
if (result.Success)
|
||||
_logger?.LogInformation("Completed {TableName}: {Rows} rows in {Elapsed:g}",
|
||||
tableName, result.TotalRows, result.Elapsed);
|
||||
else
|
||||
_logger?.LogError(result.Error, "Failed {TableName}: {Error}",
|
||||
tableName, result.Error?.Message);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
public async Task<IReadOnlyList<PipelineResult>> RunAllAsync(CancellationToken cancellationToken = default)
|
||||
{
|
||||
var results = new List<PipelineResult>();
|
||||
|
||||
foreach (var tableName in GetAvailableTables())
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var result = await RunAsync(tableName, cancellationToken);
|
||||
results.Add(result);
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Runs all dev ETL pipelines with parallelization.
|
||||
/// Small/medium tables run concurrently, very large tables run sequentially at the end.
|
||||
/// </summary>
|
||||
/// <param name="maxDegreeOfParallelism">Maximum concurrent table loads (default 4).</param>
|
||||
/// <param name="cancellationToken">Cancellation token.</param>
|
||||
public async Task<IReadOnlyList<PipelineResult>> RunAllParallelAsync(
|
||||
int maxDegreeOfParallelism = 4,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var results = new ConcurrentBag<PipelineResult>();
|
||||
using var semaphore = new SemaphoreSlim(maxDegreeOfParallelism);
|
||||
|
||||
// Separate tables by size - run very large ones sequentially at the end
|
||||
var smallMediumTables = GetAvailableTables()
|
||||
.Where(t => !IsVeryLargeTable(t))
|
||||
.ToList();
|
||||
var veryLargeTables = GetAvailableTables()
|
||||
.Where(IsVeryLargeTable)
|
||||
.ToList();
|
||||
|
||||
_logger?.LogInformation(
|
||||
"Running {ParallelCount} tables in parallel (max {MaxParallel}), then {SequentialCount} large tables sequentially",
|
||||
smallMediumTables.Count, maxDegreeOfParallelism, veryLargeTables.Count);
|
||||
|
||||
// Run small/medium tables in parallel
|
||||
var tasks = smallMediumTables.Select(async tableName =>
|
||||
{
|
||||
await semaphore.WaitAsync(cancellationToken);
|
||||
try
|
||||
{
|
||||
var result = await RunAsync(tableName, cancellationToken);
|
||||
results.Add(result);
|
||||
}
|
||||
finally
|
||||
{
|
||||
semaphore.Release();
|
||||
}
|
||||
});
|
||||
|
||||
await Task.WhenAll(tasks);
|
||||
|
||||
// Run very large tables sequentially (IO-bound, would contend)
|
||||
foreach (var tableName in veryLargeTables)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var result = await RunAsync(tableName, cancellationToken);
|
||||
results.Add(result);
|
||||
}
|
||||
|
||||
return results.ToList();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Identifies very large tables that should be loaded sequentially to avoid IO contention.
|
||||
/// </summary>
|
||||
private static bool IsVeryLargeTable(string tableName) =>
|
||||
tableName.Contains("WorkOrderTime", StringComparison.OrdinalIgnoreCase) ||
|
||||
tableName.Contains("WorkOrderStep", StringComparison.OrdinalIgnoreCase) ||
|
||||
tableName.Contains("WorkOrderRouting", StringComparison.OrdinalIgnoreCase) ||
|
||||
tableName.Contains("WorkOrderComponent", StringComparison.OrdinalIgnoreCase) ||
|
||||
tableName.Contains("LotUsage", StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
using JdeScoping.DataAccess.Interfaces;
|
||||
using JdeScoping.DataSync.Etl.Destinations;
|
||||
using JdeScoping.DataSync.Etl.Models;
|
||||
using JdeScoping.DataSync.Etl.Pipeline;
|
||||
using JdeScoping.DataSync.Etl.Sources;
|
||||
|
||||
namespace JdeScoping.DataSync.Dev;
|
||||
|
||||
/// <summary>
|
||||
/// Development ETL pipeline for the FunctionCode table.
|
||||
/// Schema from: Scripts/005_CreateFunctionCodeTable.sql
|
||||
/// </summary>
|
||||
public static class FunctionCodeDevEtl
|
||||
{
|
||||
public static readonly string TableName = "FunctionCode";
|
||||
public static readonly string CacheFileName = "functioncode.json.zstd";
|
||||
|
||||
private static readonly JsonColumnSchema[] Schema =
|
||||
[
|
||||
new("Code", typeof(string), IsNullable: false),
|
||||
new("Description", typeof(string), IsNullable: true),
|
||||
new("LastUpdateDT", typeof(DateTime), IsNullable: false),
|
||||
];
|
||||
|
||||
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(connectionFactory);
|
||||
|
||||
if (string.IsNullOrWhiteSpace(cacheFilePath))
|
||||
throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath));
|
||||
|
||||
return new EtlPipelineBuilder()
|
||||
.WithName($"{TableName}_Dev")
|
||||
.WithSource(new JsonZstdFileSource(cacheFilePath, Schema))
|
||||
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
|
||||
.Build();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
using JdeScoping.DataAccess.Interfaces;
|
||||
using JdeScoping.DataSync.Etl.Destinations;
|
||||
using JdeScoping.DataSync.Etl.Models;
|
||||
using JdeScoping.DataSync.Etl.Pipeline;
|
||||
using JdeScoping.DataSync.Etl.Sources;
|
||||
|
||||
namespace JdeScoping.DataSync.Dev;
|
||||
|
||||
/// <summary>
|
||||
/// Development ETL pipeline for the Item table.
|
||||
/// Schema from: Scripts/008_CreateItemTable.sql
|
||||
/// </summary>
|
||||
public static class ItemDevEtl
|
||||
{
|
||||
public static readonly string TableName = "Item";
|
||||
public static readonly string CacheFileName = "item.json.zstd";
|
||||
|
||||
private static readonly JsonColumnSchema[] Schema =
|
||||
[
|
||||
new("ShortItemNumber", typeof(long), IsNullable: false),
|
||||
new("ItemNumber", typeof(string), IsNullable: false),
|
||||
new("Description", typeof(string), IsNullable: true),
|
||||
new("PlanningFamily", typeof(string), IsNullable: true),
|
||||
new("StockingType", typeof(string), IsNullable: true),
|
||||
new("LastUpdateDT", typeof(DateTime), IsNullable: false),
|
||||
];
|
||||
|
||||
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(connectionFactory);
|
||||
|
||||
if (string.IsNullOrWhiteSpace(cacheFilePath))
|
||||
throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath));
|
||||
|
||||
return new EtlPipelineBuilder()
|
||||
.WithName($"{TableName}_Dev")
|
||||
.WithSource(new JsonZstdFileSource(cacheFilePath, Schema))
|
||||
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
|
||||
.Build();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\JdeScoping.DataSync\JdeScoping.DataSync.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
@@ -0,0 +1,39 @@
|
||||
using JdeScoping.DataAccess.Interfaces;
|
||||
using JdeScoping.DataSync.Etl.Destinations;
|
||||
using JdeScoping.DataSync.Etl.Models;
|
||||
using JdeScoping.DataSync.Etl.Pipeline;
|
||||
using JdeScoping.DataSync.Etl.Sources;
|
||||
|
||||
namespace JdeScoping.DataSync.Dev;
|
||||
|
||||
/// <summary>
|
||||
/// Development ETL pipeline for the JdeUser table.
|
||||
/// Schema from: Scripts/009_CreateJdeUserTable.sql
|
||||
/// </summary>
|
||||
public static class JdeUserDevEtl
|
||||
{
|
||||
public static readonly string TableName = "JdeUser";
|
||||
public static readonly string CacheFileName = "jdeuser.json.zstd";
|
||||
|
||||
private static readonly JsonColumnSchema[] Schema =
|
||||
[
|
||||
new("AddressNumber", typeof(long), IsNullable: false),
|
||||
new("UserID", typeof(string), IsNullable: true),
|
||||
new("FullName", typeof(string), IsNullable: false),
|
||||
new("LastUpdateDT", typeof(DateTime), IsNullable: false),
|
||||
];
|
||||
|
||||
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(connectionFactory);
|
||||
|
||||
if (string.IsNullOrWhiteSpace(cacheFilePath))
|
||||
throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath));
|
||||
|
||||
return new EtlPipelineBuilder()
|
||||
.WithName($"{TableName}_Dev")
|
||||
.WithSource(new JsonZstdFileSource(cacheFilePath, Schema))
|
||||
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
|
||||
.Build();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
using JdeScoping.DataAccess.Interfaces;
|
||||
using JdeScoping.DataSync.Etl.Destinations;
|
||||
using JdeScoping.DataSync.Etl.Models;
|
||||
using JdeScoping.DataSync.Etl.Pipeline;
|
||||
using JdeScoping.DataSync.Etl.Sources;
|
||||
|
||||
namespace JdeScoping.DataSync.Dev;
|
||||
|
||||
/// <summary>
|
||||
/// Development ETL pipeline for the Lot table.
|
||||
/// Schema from: Scripts/013_CreateLotTable.sql
|
||||
/// </summary>
|
||||
public static class LotDevEtl
|
||||
{
|
||||
public static readonly string TableName = "Lot";
|
||||
public static readonly string CacheFileName = "lot.json.zstd";
|
||||
|
||||
private static readonly JsonColumnSchema[] Schema =
|
||||
[
|
||||
new("LotNumber", typeof(string), IsNullable: false),
|
||||
new("BranchCode", typeof(string), IsNullable: false),
|
||||
new("ShortItemNumber", typeof(long), IsNullable: false),
|
||||
new("ItemNumber", typeof(string), IsNullable: true),
|
||||
new("SupplierCode", typeof(long), IsNullable: false),
|
||||
new("StatusCode", typeof(string), IsNullable: true),
|
||||
new("Memo1", typeof(string), IsNullable: true),
|
||||
new("Memo2", typeof(string), IsNullable: true),
|
||||
new("Memo3", typeof(string), IsNullable: true),
|
||||
new("LastUpdateDT", typeof(DateTime), IsNullable: false),
|
||||
];
|
||||
|
||||
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(connectionFactory);
|
||||
|
||||
if (string.IsNullOrWhiteSpace(cacheFilePath))
|
||||
throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath));
|
||||
|
||||
return new EtlPipelineBuilder()
|
||||
.WithName($"{TableName}_Dev")
|
||||
.WithSource(new JsonZstdFileSource(cacheFilePath, Schema))
|
||||
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
|
||||
.Build();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
using JdeScoping.DataAccess.Interfaces;
|
||||
using JdeScoping.DataSync.Etl.Destinations;
|
||||
using JdeScoping.DataSync.Etl.Models;
|
||||
using JdeScoping.DataSync.Etl.Pipeline;
|
||||
using JdeScoping.DataSync.Etl.Sources;
|
||||
|
||||
namespace JdeScoping.DataSync.Dev;
|
||||
|
||||
/// <summary>
|
||||
/// Development ETL pipeline for the LotUsage_Curr table.
|
||||
/// Schema from: Scripts/024_CreateLotUsageCurrTable.sql
|
||||
/// </summary>
|
||||
public static class LotUsageCurrDevEtl
|
||||
{
|
||||
public static readonly string TableName = "LotUsage_Curr";
|
||||
public static readonly string CacheFileName = "lotusage_curr.json.zstd";
|
||||
|
||||
private static readonly JsonColumnSchema[] Schema =
|
||||
[
|
||||
new("UniqueID", typeof(long), IsNullable: false),
|
||||
new("WorkOrderNumber", typeof(long), IsNullable: false),
|
||||
new("LotNumber", typeof(string), IsNullable: false),
|
||||
new("BranchCode", typeof(string), IsNullable: true),
|
||||
new("ShortItemNumber", typeof(long), IsNullable: false),
|
||||
new("Quantity", typeof(decimal), IsNullable: false),
|
||||
new("LastUpdateDT", typeof(DateTime), IsNullable: false),
|
||||
];
|
||||
|
||||
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(connectionFactory);
|
||||
|
||||
if (string.IsNullOrWhiteSpace(cacheFilePath))
|
||||
throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath));
|
||||
|
||||
return new EtlPipelineBuilder()
|
||||
.WithName($"{TableName}_Dev")
|
||||
.WithSource(new JsonZstdFileSource(cacheFilePath, Schema))
|
||||
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
|
||||
.Build();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
using JdeScoping.DataAccess.Interfaces;
|
||||
using JdeScoping.DataSync.Etl.Destinations;
|
||||
using JdeScoping.DataSync.Etl.Models;
|
||||
using JdeScoping.DataSync.Etl.Pipeline;
|
||||
using JdeScoping.DataSync.Etl.Sources;
|
||||
|
||||
namespace JdeScoping.DataSync.Dev;
|
||||
|
||||
/// <summary>
|
||||
/// Development ETL pipeline for the LotUsage_Hist table.
|
||||
/// Schema from: Scripts/025_CreateLotUsageHistTable.sql
|
||||
/// </summary>
|
||||
public static class LotUsageHistDevEtl
|
||||
{
|
||||
public static readonly string TableName = "LotUsage_Hist";
|
||||
public static readonly string CacheFileName = "lotusage_hist.json.zstd";
|
||||
|
||||
private static readonly JsonColumnSchema[] Schema =
|
||||
[
|
||||
new("UniqueID", typeof(long), IsNullable: false),
|
||||
new("WorkOrderNumber", typeof(long), IsNullable: false),
|
||||
new("LotNumber", typeof(string), IsNullable: false),
|
||||
new("BranchCode", typeof(string), IsNullable: true),
|
||||
new("ShortItemNumber", typeof(long), IsNullable: false),
|
||||
new("Quantity", typeof(decimal), IsNullable: false),
|
||||
new("LastUpdateDT", typeof(DateTime), IsNullable: false),
|
||||
];
|
||||
|
||||
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(connectionFactory);
|
||||
|
||||
if (string.IsNullOrWhiteSpace(cacheFilePath))
|
||||
throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath));
|
||||
|
||||
return new EtlPipelineBuilder()
|
||||
.WithName($"{TableName}_Dev")
|
||||
.WithSource(new JsonZstdFileSource(cacheFilePath, Schema))
|
||||
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
|
||||
.Build();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
using JdeScoping.DataAccess.Interfaces;
|
||||
using JdeScoping.DataSync.Etl.Destinations;
|
||||
using JdeScoping.DataSync.Etl.Models;
|
||||
using JdeScoping.DataSync.Etl.Pipeline;
|
||||
using JdeScoping.DataSync.Etl.Sources;
|
||||
|
||||
namespace JdeScoping.DataSync.Dev;
|
||||
|
||||
/// <summary>
|
||||
/// Development ETL pipeline for the MisData table.
|
||||
/// Schema from: Scripts/012_CreateMisDataTable.sql
|
||||
/// </summary>
|
||||
public static class MisDataDevEtl
|
||||
{
|
||||
public static readonly string TableName = "MisData";
|
||||
public static readonly string CacheFileName = "misdata.json.zstd";
|
||||
|
||||
private static readonly JsonColumnSchema[] Schema =
|
||||
[
|
||||
new("ItemNumber", typeof(string), IsNullable: false),
|
||||
new("BranchCode", typeof(string), IsNullable: false),
|
||||
new("SequenceNumber", typeof(string), IsNullable: false),
|
||||
new("MisNumber", typeof(string), IsNullable: false),
|
||||
new("RevID", typeof(string), IsNullable: false),
|
||||
new("CharNumber", typeof(string), IsNullable: false),
|
||||
new("TestDescription", typeof(string), IsNullable: true),
|
||||
new("SamplingType", typeof(string), IsNullable: true),
|
||||
new("SamplingValue", typeof(string), IsNullable: true),
|
||||
new("ToolsGauges", typeof(string), IsNullable: true),
|
||||
new("WorkInstructions", typeof(string), IsNullable: true),
|
||||
new("Status", typeof(string), IsNullable: false),
|
||||
new("ReleaseDate", typeof(DateTime), IsNullable: true),
|
||||
new("ObsoleteDate", typeof(DateTime), IsNullable: true),
|
||||
];
|
||||
|
||||
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(connectionFactory);
|
||||
|
||||
if (string.IsNullOrWhiteSpace(cacheFilePath))
|
||||
throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath));
|
||||
|
||||
return new EtlPipelineBuilder()
|
||||
.WithName($"{TableName}_Dev")
|
||||
.WithSource(new JsonZstdFileSource(cacheFilePath, Schema))
|
||||
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
|
||||
.Build();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
using JdeScoping.DataAccess.Interfaces;
|
||||
using JdeScoping.DataSync.Etl.Destinations;
|
||||
using JdeScoping.DataSync.Etl.Models;
|
||||
using JdeScoping.DataSync.Etl.Pipeline;
|
||||
using JdeScoping.DataSync.Etl.Sources;
|
||||
|
||||
namespace JdeScoping.DataSync.Dev;
|
||||
|
||||
/// <summary>
|
||||
/// Development ETL pipeline for the OrgHierarchy table.
|
||||
/// Schema from: Scripts/010_CreateOrgHierarchyTable.sql
|
||||
/// </summary>
|
||||
public static class OrgHierarchyDevEtl
|
||||
{
|
||||
public static readonly string TableName = "OrgHierarchy";
|
||||
public static readonly string CacheFileName = "orghierarchy.json.zstd";
|
||||
|
||||
private static readonly JsonColumnSchema[] Schema =
|
||||
[
|
||||
new("WorkCenterCode", typeof(string), IsNullable: false),
|
||||
new("BranchCode", typeof(string), IsNullable: false),
|
||||
new("ProfitCenterCode", typeof(string), IsNullable: false),
|
||||
new("LastUpdateDT", typeof(DateTime), IsNullable: false),
|
||||
];
|
||||
|
||||
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(connectionFactory);
|
||||
|
||||
if (string.IsNullOrWhiteSpace(cacheFilePath))
|
||||
throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath));
|
||||
|
||||
return new EtlPipelineBuilder()
|
||||
.WithName($"{TableName}_Dev")
|
||||
.WithSource(new JsonZstdFileSource(cacheFilePath, Schema))
|
||||
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
|
||||
.Build();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
using JdeScoping.DataAccess.Interfaces;
|
||||
using JdeScoping.DataSync.Etl.Destinations;
|
||||
using JdeScoping.DataSync.Etl.Models;
|
||||
using JdeScoping.DataSync.Etl.Pipeline;
|
||||
using JdeScoping.DataSync.Etl.Sources;
|
||||
|
||||
namespace JdeScoping.DataSync.Dev;
|
||||
|
||||
/// <summary>
|
||||
/// Development ETL pipeline for the ProfitCenter table.
|
||||
/// Schema from: Scripts/006_CreateProfitCenterTable.sql
|
||||
/// </summary>
|
||||
public static class ProfitCenterDevEtl
|
||||
{
|
||||
public static readonly string TableName = "ProfitCenter";
|
||||
public static readonly string CacheFileName = "profitcenter.json.zstd";
|
||||
|
||||
private static readonly JsonColumnSchema[] Schema =
|
||||
[
|
||||
new("Code", typeof(string), IsNullable: false),
|
||||
new("Description", typeof(string), IsNullable: true),
|
||||
new("LastUpdateDT", typeof(DateTime), IsNullable: false),
|
||||
];
|
||||
|
||||
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(connectionFactory);
|
||||
|
||||
if (string.IsNullOrWhiteSpace(cacheFilePath))
|
||||
throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath));
|
||||
|
||||
return new EtlPipelineBuilder()
|
||||
.WithName($"{TableName}_Dev")
|
||||
.WithSource(new JsonZstdFileSource(cacheFilePath, Schema))
|
||||
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
|
||||
.Build();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
using JdeScoping.DataAccess.Interfaces;
|
||||
using JdeScoping.DataSync.Etl.Destinations;
|
||||
using JdeScoping.DataSync.Etl.Models;
|
||||
using JdeScoping.DataSync.Etl.Pipeline;
|
||||
using JdeScoping.DataSync.Etl.Sources;
|
||||
|
||||
namespace JdeScoping.DataSync.Dev;
|
||||
|
||||
/// <summary>
|
||||
/// Development ETL pipeline for the RouteMaster table.
|
||||
/// Schema from: Scripts/011_CreateRouteMasterTable.sql
|
||||
/// </summary>
|
||||
public static class RouteMasterDevEtl
|
||||
{
|
||||
public static readonly string TableName = "RouteMaster";
|
||||
public static readonly string CacheFileName = "routemaster.json.zstd";
|
||||
|
||||
private static readonly JsonColumnSchema[] Schema =
|
||||
[
|
||||
new("BranchCode", typeof(string), IsNullable: false),
|
||||
new("ItemNumber", typeof(string), IsNullable: false),
|
||||
new("RoutingType", typeof(string), IsNullable: false),
|
||||
new("SequenceNumber", typeof(decimal), IsNullable: false),
|
||||
new("FunctionCode", typeof(string), IsNullable: true),
|
||||
new("WorkCenterCode", typeof(string), IsNullable: true),
|
||||
new("StartDate", typeof(DateTime), IsNullable: false),
|
||||
new("EndDate", typeof(DateTime), IsNullable: true),
|
||||
new("LastUpdateDT", typeof(DateTime), IsNullable: false),
|
||||
];
|
||||
|
||||
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(connectionFactory);
|
||||
|
||||
if (string.IsNullOrWhiteSpace(cacheFilePath))
|
||||
throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath));
|
||||
|
||||
return new EtlPipelineBuilder()
|
||||
.WithName($"{TableName}_Dev")
|
||||
.WithSource(new JsonZstdFileSource(cacheFilePath, Schema))
|
||||
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
|
||||
.Build();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
using JdeScoping.DataAccess.Interfaces;
|
||||
using JdeScoping.DataSync.Etl.Destinations;
|
||||
using JdeScoping.DataSync.Etl.Models;
|
||||
using JdeScoping.DataSync.Etl.Pipeline;
|
||||
using JdeScoping.DataSync.Etl.Sources;
|
||||
|
||||
namespace JdeScoping.DataSync.Dev;
|
||||
|
||||
/// <summary>
|
||||
/// Development ETL pipeline for the WorkCenter table.
|
||||
/// Schema from: Scripts/007_CreateWorkCenterTable.sql
|
||||
/// </summary>
|
||||
public static class WorkCenterDevEtl
|
||||
{
|
||||
public static readonly string TableName = "WorkCenter";
|
||||
public static readonly string CacheFileName = "workcenter.json.zstd";
|
||||
|
||||
private static readonly JsonColumnSchema[] Schema =
|
||||
[
|
||||
new("Code", typeof(string), IsNullable: false),
|
||||
new("Description", typeof(string), IsNullable: true),
|
||||
new("LastUpdateDT", typeof(DateTime), IsNullable: false),
|
||||
];
|
||||
|
||||
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(connectionFactory);
|
||||
|
||||
if (string.IsNullOrWhiteSpace(cacheFilePath))
|
||||
throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath));
|
||||
|
||||
return new EtlPipelineBuilder()
|
||||
.WithName($"{TableName}_Dev")
|
||||
.WithSource(new JsonZstdFileSource(cacheFilePath, Schema))
|
||||
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
|
||||
.Build();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
using JdeScoping.DataAccess.Interfaces;
|
||||
using JdeScoping.DataSync.Etl.Destinations;
|
||||
using JdeScoping.DataSync.Etl.Models;
|
||||
using JdeScoping.DataSync.Etl.Pipeline;
|
||||
using JdeScoping.DataSync.Etl.Sources;
|
||||
|
||||
namespace JdeScoping.DataSync.Dev;
|
||||
|
||||
/// <summary>
|
||||
/// Development ETL pipeline for the WorkOrderComponent_Curr table.
|
||||
/// Schema from: Scripts/021_CreateWorkOrderComponentCurrTable.sql
|
||||
/// </summary>
|
||||
public static class WorkOrderComponentCurrDevEtl
|
||||
{
|
||||
public static readonly string TableName = "WorkOrderComponent_Curr";
|
||||
public static readonly string CacheFileName = "workordercomponent_curr.json.zstd";
|
||||
|
||||
private static readonly JsonColumnSchema[] Schema =
|
||||
[
|
||||
new("UniqueID", typeof(long), IsNullable: false),
|
||||
new("WorkOrderNumber", typeof(long), IsNullable: false),
|
||||
new("LotNumber", typeof(string), IsNullable: false),
|
||||
new("BranchCode", typeof(string), IsNullable: true),
|
||||
new("ShortItemNumber", typeof(long), IsNullable: false),
|
||||
new("Quantity", typeof(decimal), IsNullable: false),
|
||||
new("LastUpdateDT", typeof(DateTime), IsNullable: false),
|
||||
];
|
||||
|
||||
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(connectionFactory);
|
||||
|
||||
if (string.IsNullOrWhiteSpace(cacheFilePath))
|
||||
throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath));
|
||||
|
||||
return new EtlPipelineBuilder()
|
||||
.WithName($"{TableName}_Dev")
|
||||
.WithSource(new JsonZstdFileSource(cacheFilePath, Schema))
|
||||
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
|
||||
.Build();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
using JdeScoping.DataAccess.Interfaces;
|
||||
using JdeScoping.DataSync.Etl.Destinations;
|
||||
using JdeScoping.DataSync.Etl.Models;
|
||||
using JdeScoping.DataSync.Etl.Pipeline;
|
||||
using JdeScoping.DataSync.Etl.Sources;
|
||||
|
||||
namespace JdeScoping.DataSync.Dev;
|
||||
|
||||
/// <summary>
|
||||
/// Development ETL pipeline for the WorkOrderComponent_Hist table.
|
||||
/// Schema from: Scripts/022_CreateWorkOrderComponentHistTable.sql
|
||||
/// </summary>
|
||||
public static class WorkOrderComponentHistDevEtl
|
||||
{
|
||||
public static readonly string TableName = "WorkOrderComponent_Hist";
|
||||
public static readonly string CacheFileName = "workordercomponent_hist.json.zstd";
|
||||
|
||||
private static readonly JsonColumnSchema[] Schema =
|
||||
[
|
||||
new("UniqueID", typeof(long), IsNullable: false),
|
||||
new("WorkOrderNumber", typeof(long), IsNullable: false),
|
||||
new("LotNumber", typeof(string), IsNullable: false),
|
||||
new("BranchCode", typeof(string), IsNullable: true),
|
||||
new("ShortItemNumber", typeof(long), IsNullable: false),
|
||||
new("Quantity", typeof(decimal), IsNullable: false),
|
||||
new("LastUpdateDT", typeof(DateTime), IsNullable: false),
|
||||
];
|
||||
|
||||
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(connectionFactory);
|
||||
|
||||
if (string.IsNullOrWhiteSpace(cacheFilePath))
|
||||
throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath));
|
||||
|
||||
return new EtlPipelineBuilder()
|
||||
.WithName($"{TableName}_Dev")
|
||||
.WithSource(new JsonZstdFileSource(cacheFilePath, Schema))
|
||||
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
|
||||
.Build();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
using JdeScoping.DataAccess.Interfaces;
|
||||
using JdeScoping.DataSync.Etl.Destinations;
|
||||
using JdeScoping.DataSync.Etl.Models;
|
||||
using JdeScoping.DataSync.Etl.Pipeline;
|
||||
using JdeScoping.DataSync.Etl.Sources;
|
||||
|
||||
namespace JdeScoping.DataSync.Dev;
|
||||
|
||||
/// <summary>
|
||||
/// Development ETL pipeline for the WorkOrder_Curr table.
|
||||
/// Schema from: Scripts/015_CreateWorkOrderCurrTable.sql
|
||||
/// </summary>
|
||||
public static class WorkOrderCurrDevEtl
|
||||
{
|
||||
public static readonly string TableName = "WorkOrder_Curr";
|
||||
public static readonly string CacheFileName = "workorder_curr.json.zstd";
|
||||
|
||||
private static readonly JsonColumnSchema[] Schema =
|
||||
[
|
||||
new("WorkOrderNumber", typeof(long), IsNullable: false),
|
||||
new("BranchCode", typeof(string), IsNullable: true),
|
||||
new("LotNumber", typeof(string), IsNullable: true),
|
||||
new("ItemNumber", typeof(string), IsNullable: true),
|
||||
new("ShortItemNumber", typeof(long), IsNullable: false),
|
||||
new("ParentWorkOrderNumber", typeof(string), IsNullable: true),
|
||||
new("OrderQuantity", typeof(decimal), IsNullable: false),
|
||||
new("HeldQuantity", typeof(decimal), IsNullable: false),
|
||||
new("ShippedQuantity", typeof(decimal), IsNullable: false),
|
||||
new("StatusCode", typeof(string), IsNullable: true),
|
||||
new("StatusCodeUpdateDT", typeof(DateTime), IsNullable: true),
|
||||
new("IssueDate", typeof(DateTime), IsNullable: false),
|
||||
new("StartDate", typeof(DateTime), IsNullable: false),
|
||||
new("RoutingType", typeof(string), IsNullable: true),
|
||||
new("LastUpdateDT", typeof(DateTime), IsNullable: false),
|
||||
];
|
||||
|
||||
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(connectionFactory);
|
||||
|
||||
if (string.IsNullOrWhiteSpace(cacheFilePath))
|
||||
throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath));
|
||||
|
||||
return new EtlPipelineBuilder()
|
||||
.WithName($"{TableName}_Dev")
|
||||
.WithSource(new JsonZstdFileSource(cacheFilePath, Schema))
|
||||
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
|
||||
.Build();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
using JdeScoping.DataAccess.Interfaces;
|
||||
using JdeScoping.DataSync.Etl.Destinations;
|
||||
using JdeScoping.DataSync.Etl.Models;
|
||||
using JdeScoping.DataSync.Etl.Pipeline;
|
||||
using JdeScoping.DataSync.Etl.Sources;
|
||||
|
||||
namespace JdeScoping.DataSync.Dev;
|
||||
|
||||
/// <summary>
|
||||
/// Development ETL pipeline for the WorkOrder_Hist table.
|
||||
/// Schema from: Scripts/016_CreateWorkOrderHistTable.sql
|
||||
/// </summary>
|
||||
public static class WorkOrderHistDevEtl
|
||||
{
|
||||
public static readonly string TableName = "WorkOrder_Hist";
|
||||
public static readonly string CacheFileName = "workorder_hist.json.zstd";
|
||||
|
||||
private static readonly JsonColumnSchema[] Schema =
|
||||
[
|
||||
new("WorkOrderNumber", typeof(long), IsNullable: false),
|
||||
new("BranchCode", typeof(string), IsNullable: true),
|
||||
new("LotNumber", typeof(string), IsNullable: true),
|
||||
new("ItemNumber", typeof(string), IsNullable: true),
|
||||
new("ShortItemNumber", typeof(long), IsNullable: false),
|
||||
new("ParentWorkOrderNumber", typeof(string), IsNullable: true),
|
||||
new("OrderQuantity", typeof(decimal), IsNullable: false),
|
||||
new("HeldQuantity", typeof(decimal), IsNullable: false),
|
||||
new("ShippedQuantity", typeof(decimal), IsNullable: false),
|
||||
new("StatusCode", typeof(string), IsNullable: true),
|
||||
new("StatusCodeUpdateDT", typeof(DateTime), IsNullable: true),
|
||||
new("IssueDate", typeof(DateTime), IsNullable: false),
|
||||
new("StartDate", typeof(DateTime), IsNullable: false),
|
||||
new("RoutingType", typeof(string), IsNullable: true),
|
||||
new("LastUpdateDT", typeof(DateTime), IsNullable: false),
|
||||
];
|
||||
|
||||
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(connectionFactory);
|
||||
|
||||
if (string.IsNullOrWhiteSpace(cacheFilePath))
|
||||
throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath));
|
||||
|
||||
return new EtlPipelineBuilder()
|
||||
.WithName($"{TableName}_Dev")
|
||||
.WithSource(new JsonZstdFileSource(cacheFilePath, Schema))
|
||||
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
|
||||
.Build();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
using JdeScoping.DataAccess.Interfaces;
|
||||
using JdeScoping.DataSync.Etl.Destinations;
|
||||
using JdeScoping.DataSync.Etl.Models;
|
||||
using JdeScoping.DataSync.Etl.Pipeline;
|
||||
using JdeScoping.DataSync.Etl.Sources;
|
||||
|
||||
namespace JdeScoping.DataSync.Dev;
|
||||
|
||||
/// <summary>
|
||||
/// Development ETL pipeline for the WorkOrderRouting table.
|
||||
/// Schema from: Scripts/023_CreateWorkOrderRoutingTable.sql
|
||||
/// </summary>
|
||||
public static class WorkOrderRoutingDevEtl
|
||||
{
|
||||
public static readonly string TableName = "WorkOrderRouting";
|
||||
public static readonly string CacheFileName = "workorderrouting.json.zstd";
|
||||
|
||||
private static readonly JsonColumnSchema[] Schema =
|
||||
[
|
||||
new("UserID", typeof(string), IsNullable: false),
|
||||
new("BatchNumber", typeof(string), IsNullable: false),
|
||||
new("TransactionNumber", typeof(string), IsNullable: false),
|
||||
new("LineNumber", typeof(int), IsNullable: false),
|
||||
new("StepNumber", typeof(decimal), IsNullable: false),
|
||||
new("WorkCenterCode", typeof(string), IsNullable: false),
|
||||
new("WorkOrderNumber", typeof(long), IsNullable: false),
|
||||
new("RoutingType", typeof(string), IsNullable: true),
|
||||
new("BranchCode", typeof(string), IsNullable: true),
|
||||
new("StepDescription", typeof(string), IsNullable: true),
|
||||
new("FunctionCode", typeof(string), IsNullable: true),
|
||||
new("TransactionDate", typeof(DateTime), IsNullable: false),
|
||||
new("LastUpdateDT", typeof(DateTime), IsNullable: false),
|
||||
];
|
||||
|
||||
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(connectionFactory);
|
||||
|
||||
if (string.IsNullOrWhiteSpace(cacheFilePath))
|
||||
throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath));
|
||||
|
||||
return new EtlPipelineBuilder()
|
||||
.WithName($"{TableName}_Dev")
|
||||
.WithSource(new JsonZstdFileSource(cacheFilePath, Schema))
|
||||
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
|
||||
.Build();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
using JdeScoping.DataAccess.Interfaces;
|
||||
using JdeScoping.DataSync.Etl.Destinations;
|
||||
using JdeScoping.DataSync.Etl.Models;
|
||||
using JdeScoping.DataSync.Etl.Pipeline;
|
||||
using JdeScoping.DataSync.Etl.Sources;
|
||||
|
||||
namespace JdeScoping.DataSync.Dev;
|
||||
|
||||
/// <summary>
|
||||
/// Development ETL pipeline for the WorkOrderStep_Curr table.
|
||||
/// Schema from: Scripts/017_CreateWorkOrderStepCurrTable.sql
|
||||
/// </summary>
|
||||
public static class WorkOrderStepCurrDevEtl
|
||||
{
|
||||
public static readonly string TableName = "WorkOrderStep_Curr";
|
||||
public static readonly string CacheFileName = "workorderstep_curr.json.zstd";
|
||||
|
||||
private static readonly JsonColumnSchema[] Schema =
|
||||
[
|
||||
new("WorkOrderNumber", typeof(long), IsNullable: false),
|
||||
new("WorkCenterCode", typeof(string), IsNullable: false),
|
||||
new("StepNumber", typeof(decimal), IsNullable: false),
|
||||
new("StepTypeCode", typeof(string), IsNullable: false),
|
||||
new("BranchCode", typeof(string), IsNullable: false),
|
||||
new("StepDescription", typeof(string), IsNullable: true),
|
||||
new("StartDT", typeof(DateTime), IsNullable: true),
|
||||
new("EndDT", typeof(DateTime), IsNullable: true),
|
||||
new("FunctionCode", typeof(string), IsNullable: true),
|
||||
new("ScrappedQuantity", typeof(decimal), IsNullable: false),
|
||||
new("LastUpdateDT", typeof(DateTime), IsNullable: false),
|
||||
];
|
||||
|
||||
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(connectionFactory);
|
||||
|
||||
if (string.IsNullOrWhiteSpace(cacheFilePath))
|
||||
throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath));
|
||||
|
||||
return new EtlPipelineBuilder()
|
||||
.WithName($"{TableName}_Dev")
|
||||
.WithSource(new JsonZstdFileSource(cacheFilePath, Schema))
|
||||
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
|
||||
.Build();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
using JdeScoping.DataAccess.Interfaces;
|
||||
using JdeScoping.DataSync.Etl.Destinations;
|
||||
using JdeScoping.DataSync.Etl.Models;
|
||||
using JdeScoping.DataSync.Etl.Pipeline;
|
||||
using JdeScoping.DataSync.Etl.Sources;
|
||||
|
||||
namespace JdeScoping.DataSync.Dev;
|
||||
|
||||
/// <summary>
|
||||
/// Development ETL pipeline for the WorkOrderStep_Hist table.
|
||||
/// Schema from: Scripts/018_CreateWorkOrderStepHistTable.sql
|
||||
/// </summary>
|
||||
public static class WorkOrderStepHistDevEtl
|
||||
{
|
||||
public static readonly string TableName = "WorkOrderStep_Hist";
|
||||
public static readonly string CacheFileName = "workorderstep_hist.json.zstd";
|
||||
|
||||
private static readonly JsonColumnSchema[] Schema =
|
||||
[
|
||||
new("WorkOrderNumber", typeof(long), IsNullable: false),
|
||||
new("WorkCenterCode", typeof(string), IsNullable: false),
|
||||
new("StepNumber", typeof(decimal), IsNullable: false),
|
||||
new("StepTypeCode", typeof(string), IsNullable: false),
|
||||
new("BranchCode", typeof(string), IsNullable: false),
|
||||
new("StepDescription", typeof(string), IsNullable: true),
|
||||
new("StartDT", typeof(DateTime), IsNullable: true),
|
||||
new("EndDT", typeof(DateTime), IsNullable: true),
|
||||
new("FunctionCode", typeof(string), IsNullable: true),
|
||||
new("ScrappedQuantity", typeof(decimal), IsNullable: false),
|
||||
new("LastUpdateDT", typeof(DateTime), IsNullable: false),
|
||||
];
|
||||
|
||||
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(connectionFactory);
|
||||
|
||||
if (string.IsNullOrWhiteSpace(cacheFilePath))
|
||||
throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath));
|
||||
|
||||
return new EtlPipelineBuilder()
|
||||
.WithName($"{TableName}_Dev")
|
||||
.WithSource(new JsonZstdFileSource(cacheFilePath, Schema))
|
||||
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
|
||||
.Build();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
using JdeScoping.DataAccess.Interfaces;
|
||||
using JdeScoping.DataSync.Etl.Destinations;
|
||||
using JdeScoping.DataSync.Etl.Models;
|
||||
using JdeScoping.DataSync.Etl.Pipeline;
|
||||
using JdeScoping.DataSync.Etl.Sources;
|
||||
|
||||
namespace JdeScoping.DataSync.Dev;
|
||||
|
||||
/// <summary>
|
||||
/// Development ETL pipeline for the WorkOrderTime_Curr table.
|
||||
/// Schema from: Scripts/019_CreateWorkOrderTimeCurrTable.sql
|
||||
/// </summary>
|
||||
public static class WorkOrderTimeCurrDevEtl
|
||||
{
|
||||
public static readonly string TableName = "WorkOrderTime_Curr";
|
||||
public static readonly string CacheFileName = "workordertime_curr.json.zstd";
|
||||
|
||||
private static readonly JsonColumnSchema[] Schema =
|
||||
[
|
||||
new("UniqueID", typeof(long), IsNullable: false),
|
||||
new("WorkOrderNumber", typeof(long), IsNullable: false),
|
||||
new("StepNumber", typeof(decimal), IsNullable: false),
|
||||
new("WorkCenterCode", typeof(string), IsNullable: false),
|
||||
new("BranchCode", typeof(string), IsNullable: false),
|
||||
new("AddressNumber", typeof(long), IsNullable: false),
|
||||
new("GlDate", typeof(DateTime), IsNullable: true),
|
||||
new("LastUpdateDT", typeof(DateTime), IsNullable: false),
|
||||
];
|
||||
|
||||
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(connectionFactory);
|
||||
|
||||
if (string.IsNullOrWhiteSpace(cacheFilePath))
|
||||
throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath));
|
||||
|
||||
return new EtlPipelineBuilder()
|
||||
.WithName($"{TableName}_Dev")
|
||||
.WithSource(new JsonZstdFileSource(cacheFilePath, Schema))
|
||||
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
|
||||
.Build();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
using JdeScoping.DataAccess.Interfaces;
|
||||
using JdeScoping.DataSync.Etl.Destinations;
|
||||
using JdeScoping.DataSync.Etl.Models;
|
||||
using JdeScoping.DataSync.Etl.Pipeline;
|
||||
using JdeScoping.DataSync.Etl.Sources;
|
||||
|
||||
namespace JdeScoping.DataSync.Dev;
|
||||
|
||||
/// <summary>
|
||||
/// Development ETL pipeline for the WorkOrderTime_Hist table.
|
||||
/// Schema from: Scripts/020_CreateWorkOrderTimeHistTable.sql
|
||||
/// </summary>
|
||||
public static class WorkOrderTimeHistDevEtl
|
||||
{
|
||||
public static readonly string TableName = "WorkOrderTime_Hist";
|
||||
public static readonly string CacheFileName = "workordertime_hist.json.zstd";
|
||||
|
||||
private static readonly JsonColumnSchema[] Schema =
|
||||
[
|
||||
new("UniqueID", typeof(long), IsNullable: false),
|
||||
new("WorkOrderNumber", typeof(long), IsNullable: false),
|
||||
new("StepNumber", typeof(decimal), IsNullable: false),
|
||||
new("WorkCenterCode", typeof(string), IsNullable: false),
|
||||
new("BranchCode", typeof(string), IsNullable: false),
|
||||
new("AddressNumber", typeof(long), IsNullable: false),
|
||||
new("GlDate", typeof(DateTime), IsNullable: true),
|
||||
new("LastUpdateDT", typeof(DateTime), IsNullable: false),
|
||||
];
|
||||
|
||||
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(connectionFactory);
|
||||
|
||||
if (string.IsNullOrWhiteSpace(cacheFilePath))
|
||||
throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath));
|
||||
|
||||
return new EtlPipelineBuilder()
|
||||
.WithName($"{TableName}_Dev")
|
||||
.WithSource(new JsonZstdFileSource(cacheFilePath, Schema))
|
||||
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
|
||||
.Build();
|
||||
}
|
||||
}
|
||||
@@ -1,80 +0,0 @@
|
||||
using JdeScoping.DataAccess.Interfaces;
|
||||
using JdeScoping.DataSync.Etl.Pipeline;
|
||||
using JdeScoping.DataSync.Etl.Results;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace JdeScoping.DataSync.DevEtl;
|
||||
|
||||
/// <summary>
|
||||
/// Registry for development ETL pipelines that load from cached JSON files.
|
||||
/// </summary>
|
||||
public class DevEtlRegistry
|
||||
{
|
||||
private readonly IDbConnectionFactory _connectionFactory;
|
||||
private readonly string _cacheDirectory;
|
||||
private readonly ILogger<DevEtlRegistry>? _logger;
|
||||
|
||||
private readonly Dictionary<string, Func<IDbConnectionFactory, string, EtlPipeline>> _pipelineFactories = new(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
[BranchDevEtl.TableName] = (factory, cacheDir) =>
|
||||
BranchDevEtl.Create(factory, Path.Combine(cacheDir, BranchDevEtl.CacheFileName)),
|
||||
};
|
||||
|
||||
public DevEtlRegistry(
|
||||
IDbConnectionFactory connectionFactory,
|
||||
string cacheDirectory,
|
||||
ILogger<DevEtlRegistry>? logger = null)
|
||||
{
|
||||
_connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory));
|
||||
|
||||
if (string.IsNullOrWhiteSpace(cacheDirectory))
|
||||
throw new ArgumentException("Cache directory is required.", nameof(cacheDirectory));
|
||||
|
||||
if (!Directory.Exists(cacheDirectory))
|
||||
throw new DirectoryNotFoundException($"Cache directory not found: {cacheDirectory}");
|
||||
|
||||
_cacheDirectory = cacheDirectory;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
public IEnumerable<string> GetAvailableTables() => _pipelineFactories.Keys;
|
||||
|
||||
public EtlPipeline GetPipeline(string tableName)
|
||||
{
|
||||
if (!_pipelineFactories.TryGetValue(tableName, out var factory))
|
||||
throw new ArgumentException($"No pipeline registered for table '{tableName}'.", nameof(tableName));
|
||||
|
||||
return factory(_connectionFactory, _cacheDirectory);
|
||||
}
|
||||
|
||||
public async Task<PipelineResult> RunAsync(string tableName, CancellationToken cancellationToken = default)
|
||||
{
|
||||
_logger?.LogInformation("Running dev ETL for {TableName}", tableName);
|
||||
|
||||
var pipeline = GetPipeline(tableName);
|
||||
var result = await pipeline.ExecuteAsync(cancellationToken);
|
||||
|
||||
if (result.Success)
|
||||
_logger?.LogInformation("Completed {TableName}: {Rows} rows in {Elapsed:g}",
|
||||
tableName, result.TotalRows, result.Elapsed);
|
||||
else
|
||||
_logger?.LogError(result.Error, "Failed {TableName}: {Error}",
|
||||
tableName, result.Error?.Message);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
public async Task<IReadOnlyList<PipelineResult>> RunAllAsync(CancellationToken cancellationToken = default)
|
||||
{
|
||||
var results = new List<PipelineResult>();
|
||||
|
||||
foreach (var tableName in GetAvailableTables())
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var result = await RunAsync(tableName, cancellationToken);
|
||||
results.Add(result);
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
}
|
||||
+2
-2
@@ -1,14 +1,14 @@
|
||||
using Dapper;
|
||||
using JdeScoping.DataAccess;
|
||||
using JdeScoping.DataAccess.Interfaces;
|
||||
using JdeScoping.DataSync.DevEtl;
|
||||
using JdeScoping.DataSync.Dev;
|
||||
using Microsoft.Data.SqlClient;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using NSubstitute;
|
||||
using Shouldly;
|
||||
|
||||
namespace JdeScoping.DataSync.Tests.DevEtl;
|
||||
namespace JdeScoping.DataSync.Dev.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Integration tests for Branch development ETL.
|
||||
@@ -0,0 +1,96 @@
|
||||
using Dapper;
|
||||
using JdeScoping.DataAccess;
|
||||
using JdeScoping.DataAccess.Interfaces;
|
||||
using JdeScoping.DataSync.Dev;
|
||||
using Microsoft.Data.SqlClient;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using NSubstitute;
|
||||
using Shouldly;
|
||||
|
||||
namespace JdeScoping.DataSync.Dev.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Integration tests for FunctionCode development ETL.
|
||||
/// </summary>
|
||||
public class FunctionCodeDevEtlTests : IAsyncLifetime
|
||||
{
|
||||
private readonly string _connectionString;
|
||||
private readonly string _cacheDirectory;
|
||||
private readonly IDbConnectionFactory _connectionFactory;
|
||||
|
||||
public FunctionCodeDevEtlTests()
|
||||
{
|
||||
var config = new ConfigurationBuilder()
|
||||
.SetBasePath(Directory.GetCurrentDirectory())
|
||||
.AddJsonFile("appsettings.json", optional: true)
|
||||
.AddEnvironmentVariables()
|
||||
.Build();
|
||||
|
||||
_connectionString = config.GetConnectionString("LotFinderDB")
|
||||
?? throw new InvalidOperationException("LotFinderDB connection string not configured.");
|
||||
|
||||
_cacheDirectory = config["DevEtl:CacheDirectory"]
|
||||
?? Path.Combine(Directory.GetCurrentDirectory(), "..", "..", "..", "..", "..", "CACHED_DB_FILES");
|
||||
|
||||
_connectionFactory = new DbConnectionFactory(config, NullLogger<DbConnectionFactory>.Instance);
|
||||
}
|
||||
|
||||
public async Task InitializeAsync()
|
||||
{
|
||||
await using var connection = new SqlConnection(_connectionString);
|
||||
await connection.OpenAsync();
|
||||
await connection.ExecuteAsync("TRUNCATE TABLE dbo.FunctionCode");
|
||||
}
|
||||
|
||||
public Task DisposeAsync() => Task.CompletedTask;
|
||||
|
||||
[Fact]
|
||||
public void Create_ReturnsValidPipeline()
|
||||
{
|
||||
var cacheFilePath = Path.Combine(_cacheDirectory, FunctionCodeDevEtl.CacheFileName);
|
||||
if (!File.Exists(cacheFilePath)) return;
|
||||
|
||||
var pipeline = FunctionCodeDevEtl.Create(_connectionFactory, cacheFilePath);
|
||||
|
||||
pipeline.ShouldNotBeNull();
|
||||
pipeline.PipelineName.ShouldBe("FunctionCode_Dev");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Execute_LoadsData()
|
||||
{
|
||||
var cacheFilePath = Path.Combine(_cacheDirectory, FunctionCodeDevEtl.CacheFileName);
|
||||
if (!File.Exists(cacheFilePath)) return;
|
||||
|
||||
var pipeline = FunctionCodeDevEtl.Create(_connectionFactory, cacheFilePath);
|
||||
var result = await pipeline.ExecuteAsync();
|
||||
|
||||
result.Success.ShouldBeTrue(result.Error?.Message ?? "Pipeline should succeed");
|
||||
result.TotalRows.ShouldBeGreaterThan(0);
|
||||
|
||||
await using var connection = new SqlConnection(_connectionString);
|
||||
await connection.OpenAsync();
|
||||
var count = await connection.ExecuteScalarAsync<int>("SELECT COUNT(*) FROM dbo.FunctionCode");
|
||||
count.ShouldBe((int)result.TotalRows);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Registry_RunAsync_LoadsTable()
|
||||
{
|
||||
if (!Directory.Exists(_cacheDirectory)) return;
|
||||
|
||||
var registry = new DevEtlRegistry(_connectionFactory, _cacheDirectory);
|
||||
var result = await registry.RunAsync("FunctionCode");
|
||||
|
||||
result.Success.ShouldBeTrue(result.Error?.Message ?? "Pipeline should succeed");
|
||||
result.TotalRows.ShouldBeGreaterThan(0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Create_WithNullConnectionFactory_ThrowsArgumentNullException()
|
||||
{
|
||||
var cacheFilePath = Path.Combine(_cacheDirectory, FunctionCodeDevEtl.CacheFileName);
|
||||
Should.Throw<ArgumentNullException>(() => FunctionCodeDevEtl.Create(null!, cacheFilePath));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
using Dapper;
|
||||
using JdeScoping.DataAccess;
|
||||
using JdeScoping.DataAccess.Interfaces;
|
||||
using JdeScoping.DataSync.Dev;
|
||||
using Microsoft.Data.SqlClient;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using NSubstitute;
|
||||
using Shouldly;
|
||||
|
||||
namespace JdeScoping.DataSync.Dev.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Integration tests for Item development ETL.
|
||||
/// </summary>
|
||||
public class ItemDevEtlTests : IAsyncLifetime
|
||||
{
|
||||
private readonly string _connectionString;
|
||||
private readonly string _cacheDirectory;
|
||||
private readonly IDbConnectionFactory _connectionFactory;
|
||||
|
||||
public ItemDevEtlTests()
|
||||
{
|
||||
var config = new ConfigurationBuilder()
|
||||
.SetBasePath(Directory.GetCurrentDirectory())
|
||||
.AddJsonFile("appsettings.json", optional: true)
|
||||
.AddEnvironmentVariables()
|
||||
.Build();
|
||||
|
||||
_connectionString = config.GetConnectionString("LotFinderDB")
|
||||
?? throw new InvalidOperationException("LotFinderDB connection string not configured.");
|
||||
|
||||
_cacheDirectory = config["DevEtl:CacheDirectory"]
|
||||
?? Path.Combine(Directory.GetCurrentDirectory(), "..", "..", "..", "..", "..", "CACHED_DB_FILES");
|
||||
|
||||
_connectionFactory = new DbConnectionFactory(config, NullLogger<DbConnectionFactory>.Instance);
|
||||
}
|
||||
|
||||
public async Task InitializeAsync()
|
||||
{
|
||||
await using var connection = new SqlConnection(_connectionString);
|
||||
await connection.OpenAsync();
|
||||
await connection.ExecuteAsync("TRUNCATE TABLE dbo.Item");
|
||||
}
|
||||
|
||||
public Task DisposeAsync() => Task.CompletedTask;
|
||||
|
||||
[Fact]
|
||||
public void Create_ReturnsValidPipeline()
|
||||
{
|
||||
var cacheFilePath = Path.Combine(_cacheDirectory, ItemDevEtl.CacheFileName);
|
||||
if (!File.Exists(cacheFilePath)) return;
|
||||
|
||||
var pipeline = ItemDevEtl.Create(_connectionFactory, cacheFilePath);
|
||||
|
||||
pipeline.ShouldNotBeNull();
|
||||
pipeline.PipelineName.ShouldBe("Item_Dev");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Execute_LoadsData()
|
||||
{
|
||||
var cacheFilePath = Path.Combine(_cacheDirectory, ItemDevEtl.CacheFileName);
|
||||
if (!File.Exists(cacheFilePath)) return;
|
||||
|
||||
var pipeline = ItemDevEtl.Create(_connectionFactory, cacheFilePath);
|
||||
var result = await pipeline.ExecuteAsync();
|
||||
|
||||
result.Success.ShouldBeTrue(result.Error?.Message ?? "Pipeline should succeed");
|
||||
result.TotalRows.ShouldBeGreaterThan(0);
|
||||
|
||||
await using var connection = new SqlConnection(_connectionString);
|
||||
await connection.OpenAsync();
|
||||
var count = await connection.ExecuteScalarAsync<int>("SELECT COUNT(*) FROM dbo.Item");
|
||||
count.ShouldBe((int)result.TotalRows);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Registry_RunAsync_LoadsTable()
|
||||
{
|
||||
if (!Directory.Exists(_cacheDirectory)) return;
|
||||
|
||||
var registry = new DevEtlRegistry(_connectionFactory, _cacheDirectory);
|
||||
var result = await registry.RunAsync("Item");
|
||||
|
||||
result.Success.ShouldBeTrue(result.Error?.Message ?? "Pipeline should succeed");
|
||||
result.TotalRows.ShouldBeGreaterThan(0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Create_WithNullConnectionFactory_ThrowsArgumentNullException()
|
||||
{
|
||||
var cacheFilePath = Path.Combine(_cacheDirectory, ItemDevEtl.CacheFileName);
|
||||
Should.Throw<ArgumentNullException>(() => ItemDevEtl.Create(null!, cacheFilePath));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
<IsPackable>false</IsPackable>
|
||||
<IsTestProject>true</IsTestProject>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.14.1" />
|
||||
<PackageReference Include="xunit" Version="2.9.3" />
|
||||
<PackageReference Include="xunit.runner.visualstudio" Version="3.0.2">
|
||||
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
|
||||
<PrivateAssets>all</PrivateAssets>
|
||||
</PackageReference>
|
||||
<PackageReference Include="Shouldly" Version="4.3.0" />
|
||||
<PackageReference Include="NSubstitute" Version="5.3.0" />
|
||||
<PackageReference Include="coverlet.collector" Version="6.0.4">
|
||||
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
|
||||
<PrivateAssets>all</PrivateAssets>
|
||||
</PackageReference>
|
||||
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="10.0.1" />
|
||||
<PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="10.0.1" />
|
||||
<PackageReference Include="Microsoft.Extensions.Configuration.EnvironmentVariables" Version="10.0.1" />
|
||||
<PackageReference Include="Microsoft.Data.SqlClient" Version="6.1.3" />
|
||||
<PackageReference Include="Dapper" Version="2.1.66" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\..\src\JdeScoping.DataAccess\JdeScoping.DataAccess.csproj" />
|
||||
<ProjectReference Include="..\..\src\JdeScoping.DataSync.Dev\JdeScoping.DataSync.Dev.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<Using Include="Xunit" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<None Include="appsettings.json" CopyToOutputDirectory="PreserveNewest" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
@@ -0,0 +1,96 @@
|
||||
using Dapper;
|
||||
using JdeScoping.DataAccess;
|
||||
using JdeScoping.DataAccess.Interfaces;
|
||||
using JdeScoping.DataSync.Dev;
|
||||
using Microsoft.Data.SqlClient;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using NSubstitute;
|
||||
using Shouldly;
|
||||
|
||||
namespace JdeScoping.DataSync.Dev.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Integration tests for JdeUser development ETL.
|
||||
/// </summary>
|
||||
public class JdeUserDevEtlTests : IAsyncLifetime
|
||||
{
|
||||
private readonly string _connectionString;
|
||||
private readonly string _cacheDirectory;
|
||||
private readonly IDbConnectionFactory _connectionFactory;
|
||||
|
||||
public JdeUserDevEtlTests()
|
||||
{
|
||||
var config = new ConfigurationBuilder()
|
||||
.SetBasePath(Directory.GetCurrentDirectory())
|
||||
.AddJsonFile("appsettings.json", optional: true)
|
||||
.AddEnvironmentVariables()
|
||||
.Build();
|
||||
|
||||
_connectionString = config.GetConnectionString("LotFinderDB")
|
||||
?? throw new InvalidOperationException("LotFinderDB connection string not configured.");
|
||||
|
||||
_cacheDirectory = config["DevEtl:CacheDirectory"]
|
||||
?? Path.Combine(Directory.GetCurrentDirectory(), "..", "..", "..", "..", "..", "CACHED_DB_FILES");
|
||||
|
||||
_connectionFactory = new DbConnectionFactory(config, NullLogger<DbConnectionFactory>.Instance);
|
||||
}
|
||||
|
||||
public async Task InitializeAsync()
|
||||
{
|
||||
await using var connection = new SqlConnection(_connectionString);
|
||||
await connection.OpenAsync();
|
||||
await connection.ExecuteAsync("TRUNCATE TABLE dbo.JdeUser");
|
||||
}
|
||||
|
||||
public Task DisposeAsync() => Task.CompletedTask;
|
||||
|
||||
[Fact]
|
||||
public void Create_ReturnsValidPipeline()
|
||||
{
|
||||
var cacheFilePath = Path.Combine(_cacheDirectory, JdeUserDevEtl.CacheFileName);
|
||||
if (!File.Exists(cacheFilePath)) return;
|
||||
|
||||
var pipeline = JdeUserDevEtl.Create(_connectionFactory, cacheFilePath);
|
||||
|
||||
pipeline.ShouldNotBeNull();
|
||||
pipeline.PipelineName.ShouldBe("JdeUser_Dev");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Execute_LoadsData()
|
||||
{
|
||||
var cacheFilePath = Path.Combine(_cacheDirectory, JdeUserDevEtl.CacheFileName);
|
||||
if (!File.Exists(cacheFilePath)) return;
|
||||
|
||||
var pipeline = JdeUserDevEtl.Create(_connectionFactory, cacheFilePath);
|
||||
var result = await pipeline.ExecuteAsync();
|
||||
|
||||
result.Success.ShouldBeTrue(result.Error?.Message ?? "Pipeline should succeed");
|
||||
result.TotalRows.ShouldBeGreaterThan(0);
|
||||
|
||||
await using var connection = new SqlConnection(_connectionString);
|
||||
await connection.OpenAsync();
|
||||
var count = await connection.ExecuteScalarAsync<int>("SELECT COUNT(*) FROM dbo.JdeUser");
|
||||
count.ShouldBe((int)result.TotalRows);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Registry_RunAsync_LoadsTable()
|
||||
{
|
||||
if (!Directory.Exists(_cacheDirectory)) return;
|
||||
|
||||
var registry = new DevEtlRegistry(_connectionFactory, _cacheDirectory);
|
||||
var result = await registry.RunAsync("JdeUser");
|
||||
|
||||
result.Success.ShouldBeTrue(result.Error?.Message ?? "Pipeline should succeed");
|
||||
result.TotalRows.ShouldBeGreaterThan(0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Create_WithNullConnectionFactory_ThrowsArgumentNullException()
|
||||
{
|
||||
var cacheFilePath = Path.Combine(_cacheDirectory, JdeUserDevEtl.CacheFileName);
|
||||
Should.Throw<ArgumentNullException>(() => JdeUserDevEtl.Create(null!, cacheFilePath));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,166 @@
|
||||
using Dapper;
|
||||
using JdeScoping.DataAccess;
|
||||
using JdeScoping.DataAccess.Interfaces;
|
||||
using JdeScoping.DataSync.Dev;
|
||||
using Microsoft.Data.SqlClient;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using NSubstitute;
|
||||
using Shouldly;
|
||||
|
||||
namespace JdeScoping.DataSync.Dev.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Integration tests for OrgHierarchy development ETL.
|
||||
/// Requires: Local SQL Server, CACHED_DB_FILES directory with orghierarchy.json.zstd
|
||||
/// </summary>
|
||||
public class OrgHierarchyDevEtlTests : IAsyncLifetime
|
||||
{
|
||||
private readonly string _connectionString;
|
||||
private readonly string _cacheDirectory;
|
||||
private readonly IDbConnectionFactory _connectionFactory;
|
||||
|
||||
public OrgHierarchyDevEtlTests()
|
||||
{
|
||||
// Load configuration
|
||||
var config = new ConfigurationBuilder()
|
||||
.SetBasePath(Directory.GetCurrentDirectory())
|
||||
.AddJsonFile("appsettings.json", optional: true)
|
||||
.AddEnvironmentVariables()
|
||||
.Build();
|
||||
|
||||
_connectionString = config.GetConnectionString("LotFinderDB")
|
||||
?? throw new InvalidOperationException("LotFinderDB connection string not configured.");
|
||||
|
||||
_cacheDirectory = config["DevEtl:CacheDirectory"]
|
||||
?? Path.Combine(Directory.GetCurrentDirectory(), "..", "..", "..", "..", "..", "CACHED_DB_FILES");
|
||||
|
||||
_connectionFactory = new DbConnectionFactory(config, NullLogger<DbConnectionFactory>.Instance);
|
||||
}
|
||||
|
||||
public async Task InitializeAsync()
|
||||
{
|
||||
// Ensure OrgHierarchy table is empty before test
|
||||
await using var connection = new SqlConnection(_connectionString);
|
||||
await connection.OpenAsync();
|
||||
await connection.ExecuteAsync("TRUNCATE TABLE dbo.OrgHierarchy");
|
||||
}
|
||||
|
||||
public Task DisposeAsync() => Task.CompletedTask;
|
||||
|
||||
[Fact]
|
||||
public void Create_ReturnsValidPipeline()
|
||||
{
|
||||
// Arrange
|
||||
var cacheFilePath = Path.Combine(_cacheDirectory, OrgHierarchyDevEtl.CacheFileName);
|
||||
if (!File.Exists(cacheFilePath))
|
||||
{
|
||||
// Skip test if cache file doesn't exist
|
||||
return;
|
||||
}
|
||||
|
||||
// Act
|
||||
var pipeline = OrgHierarchyDevEtl.Create(_connectionFactory, cacheFilePath);
|
||||
|
||||
// Assert
|
||||
pipeline.ShouldNotBeNull();
|
||||
pipeline.PipelineName.ShouldBe("OrgHierarchy_Dev");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Execute_LoadsOrgHierarchyData()
|
||||
{
|
||||
// Arrange
|
||||
var cacheFilePath = Path.Combine(_cacheDirectory, OrgHierarchyDevEtl.CacheFileName);
|
||||
if (!File.Exists(cacheFilePath))
|
||||
{
|
||||
// Skip test if cache file doesn't exist
|
||||
return;
|
||||
}
|
||||
|
||||
var pipeline = OrgHierarchyDevEtl.Create(_connectionFactory, cacheFilePath);
|
||||
|
||||
// Act
|
||||
var result = await pipeline.ExecuteAsync();
|
||||
|
||||
// Assert
|
||||
result.Success.ShouldBeTrue(result.Error?.Message ?? "Pipeline should succeed");
|
||||
result.TotalRows.ShouldBeGreaterThan(0, "Should load at least one row");
|
||||
|
||||
// Verify data in database
|
||||
await using var connection = new SqlConnection(_connectionString);
|
||||
await connection.OpenAsync();
|
||||
var count = await connection.ExecuteScalarAsync<int>("SELECT COUNT(*) FROM dbo.OrgHierarchy");
|
||||
|
||||
count.ShouldBe((int)result.TotalRows, "Database row count should match pipeline result");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Registry_RunAsync_LoadsOrgHierarchy()
|
||||
{
|
||||
// Arrange
|
||||
if (!Directory.Exists(_cacheDirectory))
|
||||
{
|
||||
// Skip test if cache directory doesn't exist
|
||||
return;
|
||||
}
|
||||
|
||||
var registry = new DevEtlRegistry(_connectionFactory, _cacheDirectory);
|
||||
|
||||
// Act
|
||||
var result = await registry.RunAsync("OrgHierarchy");
|
||||
|
||||
// Assert
|
||||
result.Success.ShouldBeTrue(result.Error?.Message ?? "Pipeline should succeed");
|
||||
result.TotalRows.ShouldBeGreaterThan(0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Create_WithNullConnectionFactory_ThrowsArgumentNullException()
|
||||
{
|
||||
// Arrange
|
||||
var cacheFilePath = Path.Combine(_cacheDirectory, OrgHierarchyDevEtl.CacheFileName);
|
||||
|
||||
// Act & Assert
|
||||
Should.Throw<ArgumentNullException>(() => OrgHierarchyDevEtl.Create(null!, cacheFilePath));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Create_WithEmptyCacheFilePath_ThrowsArgumentException()
|
||||
{
|
||||
// Arrange
|
||||
var mockFactory = Substitute.For<IDbConnectionFactory>();
|
||||
|
||||
// Act & Assert
|
||||
Should.Throw<ArgumentException>(() => OrgHierarchyDevEtl.Create(mockFactory, string.Empty));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Create_WithNonExistentCacheFile_ThrowsFileNotFoundException()
|
||||
{
|
||||
// Arrange
|
||||
var mockFactory = Substitute.For<IDbConnectionFactory>();
|
||||
var nonExistentPath = "/nonexistent/path/orghierarchy.json.zstd";
|
||||
|
||||
// Act & Assert
|
||||
Should.Throw<FileNotFoundException>(() => OrgHierarchyDevEtl.Create(mockFactory, nonExistentPath));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void DevEtlRegistry_GetAvailableTables_IncludesOrgHierarchy()
|
||||
{
|
||||
// Arrange
|
||||
if (!Directory.Exists(_cacheDirectory))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var registry = new DevEtlRegistry(_connectionFactory, _cacheDirectory);
|
||||
|
||||
// Act
|
||||
var tables = registry.GetAvailableTables().ToList();
|
||||
|
||||
// Assert
|
||||
tables.ShouldContain("OrgHierarchy");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,117 @@
|
||||
using Dapper;
|
||||
using JdeScoping.DataAccess;
|
||||
using JdeScoping.DataAccess.Interfaces;
|
||||
using JdeScoping.DataSync.Dev;
|
||||
using Microsoft.Data.SqlClient;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using NSubstitute;
|
||||
using Shouldly;
|
||||
|
||||
namespace JdeScoping.DataSync.Dev.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Integration tests for ProfitCenter development ETL.
|
||||
/// Requires: Local SQL Server, CACHED_DB_FILES directory with profitcenter.json.zstd
|
||||
/// </summary>
|
||||
public class ProfitCenterDevEtlTests : IAsyncLifetime
|
||||
{
|
||||
private readonly string _connectionString;
|
||||
private readonly string _cacheDirectory;
|
||||
private readonly IDbConnectionFactory _connectionFactory;
|
||||
|
||||
public ProfitCenterDevEtlTests()
|
||||
{
|
||||
var config = new ConfigurationBuilder()
|
||||
.SetBasePath(Directory.GetCurrentDirectory())
|
||||
.AddJsonFile("appsettings.json", optional: true)
|
||||
.AddEnvironmentVariables()
|
||||
.Build();
|
||||
|
||||
_connectionString = config.GetConnectionString("LotFinderDB")
|
||||
?? throw new InvalidOperationException("LotFinderDB connection string not configured.");
|
||||
|
||||
_cacheDirectory = config["DevEtl:CacheDirectory"]
|
||||
?? Path.Combine(Directory.GetCurrentDirectory(), "..", "..", "..", "..", "..", "CACHED_DB_FILES");
|
||||
|
||||
_connectionFactory = new DbConnectionFactory(config, NullLogger<DbConnectionFactory>.Instance);
|
||||
}
|
||||
|
||||
public async Task InitializeAsync()
|
||||
{
|
||||
await using var connection = new SqlConnection(_connectionString);
|
||||
await connection.OpenAsync();
|
||||
await connection.ExecuteAsync("TRUNCATE TABLE dbo.ProfitCenter");
|
||||
}
|
||||
|
||||
public Task DisposeAsync() => Task.CompletedTask;
|
||||
|
||||
[Fact]
|
||||
public void Create_ReturnsValidPipeline()
|
||||
{
|
||||
var cacheFilePath = Path.Combine(_cacheDirectory, ProfitCenterDevEtl.CacheFileName);
|
||||
if (!File.Exists(cacheFilePath)) return;
|
||||
|
||||
var pipeline = ProfitCenterDevEtl.Create(_connectionFactory, cacheFilePath);
|
||||
|
||||
pipeline.ShouldNotBeNull();
|
||||
pipeline.PipelineName.ShouldBe("ProfitCenter_Dev");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Execute_LoadsProfitCenterData()
|
||||
{
|
||||
var cacheFilePath = Path.Combine(_cacheDirectory, ProfitCenterDevEtl.CacheFileName);
|
||||
if (!File.Exists(cacheFilePath)) return;
|
||||
|
||||
var pipeline = ProfitCenterDevEtl.Create(_connectionFactory, cacheFilePath);
|
||||
|
||||
var result = await pipeline.ExecuteAsync();
|
||||
|
||||
result.Success.ShouldBeTrue(result.Error?.Message ?? "Pipeline should succeed");
|
||||
result.TotalRows.ShouldBeGreaterThan(0, "Should load at least one row");
|
||||
|
||||
await using var connection = new SqlConnection(_connectionString);
|
||||
await connection.OpenAsync();
|
||||
var count = await connection.ExecuteScalarAsync<int>("SELECT COUNT(*) FROM dbo.ProfitCenter");
|
||||
|
||||
count.ShouldBe((int)result.TotalRows);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Registry_RunAsync_LoadsProfitCenter()
|
||||
{
|
||||
if (!Directory.Exists(_cacheDirectory)) return;
|
||||
|
||||
var registry = new DevEtlRegistry(_connectionFactory, _cacheDirectory);
|
||||
var result = await registry.RunAsync("ProfitCenter");
|
||||
|
||||
result.Success.ShouldBeTrue(result.Error?.Message ?? "Pipeline should succeed");
|
||||
result.TotalRows.ShouldBeGreaterThan(0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Create_WithNullConnectionFactory_ThrowsArgumentNullException()
|
||||
{
|
||||
var cacheFilePath = Path.Combine(_cacheDirectory, ProfitCenterDevEtl.CacheFileName);
|
||||
Should.Throw<ArgumentNullException>(() => ProfitCenterDevEtl.Create(null!, cacheFilePath));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Create_WithEmptyCacheFilePath_ThrowsArgumentException()
|
||||
{
|
||||
var mockFactory = Substitute.For<IDbConnectionFactory>();
|
||||
Should.Throw<ArgumentException>(() => ProfitCenterDevEtl.Create(mockFactory, string.Empty));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void DevEtlRegistry_GetAvailableTables_IncludesProfitCenter()
|
||||
{
|
||||
if (!Directory.Exists(_cacheDirectory)) return;
|
||||
|
||||
var registry = new DevEtlRegistry(_connectionFactory, _cacheDirectory);
|
||||
var tables = registry.GetAvailableTables().ToList();
|
||||
|
||||
tables.ShouldContain("ProfitCenter");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
using Dapper;
|
||||
using JdeScoping.DataAccess;
|
||||
using JdeScoping.DataAccess.Interfaces;
|
||||
using JdeScoping.DataSync.Dev;
|
||||
using Microsoft.Data.SqlClient;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using NSubstitute;
|
||||
using Shouldly;
|
||||
|
||||
namespace JdeScoping.DataSync.Dev.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Integration tests for RouteMaster development ETL.
|
||||
/// </summary>
|
||||
public class RouteMasterDevEtlTests : IAsyncLifetime
|
||||
{
|
||||
private readonly string _connectionString;
|
||||
private readonly string _cacheDirectory;
|
||||
private readonly IDbConnectionFactory _connectionFactory;
|
||||
|
||||
public RouteMasterDevEtlTests()
|
||||
{
|
||||
var config = new ConfigurationBuilder()
|
||||
.SetBasePath(Directory.GetCurrentDirectory())
|
||||
.AddJsonFile("appsettings.json", optional: true)
|
||||
.AddEnvironmentVariables()
|
||||
.Build();
|
||||
|
||||
_connectionString = config.GetConnectionString("LotFinderDB")
|
||||
?? throw new InvalidOperationException("LotFinderDB connection string not configured.");
|
||||
|
||||
_cacheDirectory = config["DevEtl:CacheDirectory"]
|
||||
?? Path.Combine(Directory.GetCurrentDirectory(), "..", "..", "..", "..", "..", "CACHED_DB_FILES");
|
||||
|
||||
_connectionFactory = new DbConnectionFactory(config, NullLogger<DbConnectionFactory>.Instance);
|
||||
}
|
||||
|
||||
public async Task InitializeAsync()
|
||||
{
|
||||
await using var connection = new SqlConnection(_connectionString);
|
||||
await connection.OpenAsync();
|
||||
await connection.ExecuteAsync("TRUNCATE TABLE dbo.RouteMaster");
|
||||
}
|
||||
|
||||
public Task DisposeAsync() => Task.CompletedTask;
|
||||
|
||||
[Fact]
|
||||
public void Create_ReturnsValidPipeline()
|
||||
{
|
||||
var cacheFilePath = Path.Combine(_cacheDirectory, RouteMasterDevEtl.CacheFileName);
|
||||
if (!File.Exists(cacheFilePath)) return;
|
||||
|
||||
var pipeline = RouteMasterDevEtl.Create(_connectionFactory, cacheFilePath);
|
||||
|
||||
pipeline.ShouldNotBeNull();
|
||||
pipeline.PipelineName.ShouldBe("RouteMaster_Dev");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Execute_LoadsData()
|
||||
{
|
||||
var cacheFilePath = Path.Combine(_cacheDirectory, RouteMasterDevEtl.CacheFileName);
|
||||
if (!File.Exists(cacheFilePath)) return;
|
||||
|
||||
var pipeline = RouteMasterDevEtl.Create(_connectionFactory, cacheFilePath);
|
||||
var result = await pipeline.ExecuteAsync();
|
||||
|
||||
result.Success.ShouldBeTrue(result.Error?.Message ?? "Pipeline should succeed");
|
||||
result.TotalRows.ShouldBeGreaterThan(0);
|
||||
|
||||
await using var connection = new SqlConnection(_connectionString);
|
||||
await connection.OpenAsync();
|
||||
var count = await connection.ExecuteScalarAsync<int>("SELECT COUNT(*) FROM dbo.RouteMaster");
|
||||
count.ShouldBe((int)result.TotalRows);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Registry_RunAsync_LoadsTable()
|
||||
{
|
||||
if (!Directory.Exists(_cacheDirectory)) return;
|
||||
|
||||
var registry = new DevEtlRegistry(_connectionFactory, _cacheDirectory);
|
||||
var result = await registry.RunAsync("RouteMaster");
|
||||
|
||||
result.Success.ShouldBeTrue(result.Error?.Message ?? "Pipeline should succeed");
|
||||
result.TotalRows.ShouldBeGreaterThan(0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Create_WithNullConnectionFactory_ThrowsArgumentNullException()
|
||||
{
|
||||
var cacheFilePath = Path.Combine(_cacheDirectory, RouteMasterDevEtl.CacheFileName);
|
||||
Should.Throw<ArgumentNullException>(() => RouteMasterDevEtl.Create(null!, cacheFilePath));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,117 @@
|
||||
using Dapper;
|
||||
using JdeScoping.DataAccess;
|
||||
using JdeScoping.DataAccess.Interfaces;
|
||||
using JdeScoping.DataSync.Dev;
|
||||
using Microsoft.Data.SqlClient;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using NSubstitute;
|
||||
using Shouldly;
|
||||
|
||||
namespace JdeScoping.DataSync.Dev.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Integration tests for WorkCenter development ETL.
|
||||
/// Requires: Local SQL Server, CACHED_DB_FILES directory with workcenter.json.zstd
|
||||
/// </summary>
|
||||
public class WorkCenterDevEtlTests : IAsyncLifetime
|
||||
{
|
||||
private readonly string _connectionString;
|
||||
private readonly string _cacheDirectory;
|
||||
private readonly IDbConnectionFactory _connectionFactory;
|
||||
|
||||
public WorkCenterDevEtlTests()
|
||||
{
|
||||
var config = new ConfigurationBuilder()
|
||||
.SetBasePath(Directory.GetCurrentDirectory())
|
||||
.AddJsonFile("appsettings.json", optional: true)
|
||||
.AddEnvironmentVariables()
|
||||
.Build();
|
||||
|
||||
_connectionString = config.GetConnectionString("LotFinderDB")
|
||||
?? throw new InvalidOperationException("LotFinderDB connection string not configured.");
|
||||
|
||||
_cacheDirectory = config["DevEtl:CacheDirectory"]
|
||||
?? Path.Combine(Directory.GetCurrentDirectory(), "..", "..", "..", "..", "..", "CACHED_DB_FILES");
|
||||
|
||||
_connectionFactory = new DbConnectionFactory(config, NullLogger<DbConnectionFactory>.Instance);
|
||||
}
|
||||
|
||||
public async Task InitializeAsync()
|
||||
{
|
||||
await using var connection = new SqlConnection(_connectionString);
|
||||
await connection.OpenAsync();
|
||||
await connection.ExecuteAsync("TRUNCATE TABLE dbo.WorkCenter");
|
||||
}
|
||||
|
||||
public Task DisposeAsync() => Task.CompletedTask;
|
||||
|
||||
[Fact]
|
||||
public void Create_ReturnsValidPipeline()
|
||||
{
|
||||
var cacheFilePath = Path.Combine(_cacheDirectory, WorkCenterDevEtl.CacheFileName);
|
||||
if (!File.Exists(cacheFilePath)) return;
|
||||
|
||||
var pipeline = WorkCenterDevEtl.Create(_connectionFactory, cacheFilePath);
|
||||
|
||||
pipeline.ShouldNotBeNull();
|
||||
pipeline.PipelineName.ShouldBe("WorkCenter_Dev");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Execute_LoadsWorkCenterData()
|
||||
{
|
||||
var cacheFilePath = Path.Combine(_cacheDirectory, WorkCenterDevEtl.CacheFileName);
|
||||
if (!File.Exists(cacheFilePath)) return;
|
||||
|
||||
var pipeline = WorkCenterDevEtl.Create(_connectionFactory, cacheFilePath);
|
||||
|
||||
var result = await pipeline.ExecuteAsync();
|
||||
|
||||
result.Success.ShouldBeTrue(result.Error?.Message ?? "Pipeline should succeed");
|
||||
result.TotalRows.ShouldBeGreaterThan(0, "Should load at least one row");
|
||||
|
||||
await using var connection = new SqlConnection(_connectionString);
|
||||
await connection.OpenAsync();
|
||||
var count = await connection.ExecuteScalarAsync<int>("SELECT COUNT(*) FROM dbo.WorkCenter");
|
||||
|
||||
count.ShouldBe((int)result.TotalRows);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Registry_RunAsync_LoadsWorkCenter()
|
||||
{
|
||||
if (!Directory.Exists(_cacheDirectory)) return;
|
||||
|
||||
var registry = new DevEtlRegistry(_connectionFactory, _cacheDirectory);
|
||||
var result = await registry.RunAsync("WorkCenter");
|
||||
|
||||
result.Success.ShouldBeTrue(result.Error?.Message ?? "Pipeline should succeed");
|
||||
result.TotalRows.ShouldBeGreaterThan(0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Create_WithNullConnectionFactory_ThrowsArgumentNullException()
|
||||
{
|
||||
var cacheFilePath = Path.Combine(_cacheDirectory, WorkCenterDevEtl.CacheFileName);
|
||||
Should.Throw<ArgumentNullException>(() => WorkCenterDevEtl.Create(null!, cacheFilePath));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Create_WithEmptyCacheFilePath_ThrowsArgumentException()
|
||||
{
|
||||
var mockFactory = Substitute.For<IDbConnectionFactory>();
|
||||
Should.Throw<ArgumentException>(() => WorkCenterDevEtl.Create(mockFactory, string.Empty));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void DevEtlRegistry_GetAvailableTables_IncludesWorkCenter()
|
||||
{
|
||||
if (!Directory.Exists(_cacheDirectory)) return;
|
||||
|
||||
var registry = new DevEtlRegistry(_connectionFactory, _cacheDirectory);
|
||||
var tables = registry.GetAvailableTables().ToList();
|
||||
|
||||
tables.ShouldContain("WorkCenter");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"ConnectionStrings": {
|
||||
"LotFinderDB": "Server=localhost,1434;Database=ScopingTool;User Id=scopingapp;Password=Sc0ping@pp_Dev#2024;TrustServerCertificate=true"
|
||||
},
|
||||
"DevEtl": {
|
||||
"CacheDirectory": "/Users/dohertj2/Desktop/JdeScopingTool/CACHED_DB_FILES"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,162 @@
|
||||
# Design: Extract DevEtl to JdeScoping.DataSync.Dev
|
||||
|
||||
**Date:** 2026-01-06
|
||||
**Status:** Approved
|
||||
|
||||
## Purpose
|
||||
|
||||
Move dev/testing-specific ETL code from `JdeScoping.DataSync` into a separate `JdeScoping.DataSync.Dev` project. This code loads cached JSON/zstd files into SQL Server for local sandbox development and should not be part of the production DataSync assembly.
|
||||
|
||||
## Scope
|
||||
|
||||
### Files to Move (Source)
|
||||
|
||||
**22 files** from `src/JdeScoping.DataSync/DevEtl/` → `src/JdeScoping.DataSync.Dev/`
|
||||
|
||||
- `DevEtlRegistry.cs`
|
||||
- `BranchDevEtl.cs`
|
||||
- `FunctionCodeDevEtl.cs`
|
||||
- `ItemDevEtl.cs`
|
||||
- `JdeUserDevEtl.cs`
|
||||
- `LotDevEtl.cs`
|
||||
- `LotUsageCurrDevEtl.cs`
|
||||
- `LotUsageHistDevEtl.cs`
|
||||
- `MisDataDevEtl.cs`
|
||||
- `OrgHierarchyDevEtl.cs`
|
||||
- `ProfitCenterDevEtl.cs`
|
||||
- `RouteMasterDevEtl.cs`
|
||||
- `WorkCenterDevEtl.cs`
|
||||
- `WorkOrderComponentCurrDevEtl.cs`
|
||||
- `WorkOrderComponentHistDevEtl.cs`
|
||||
- `WorkOrderCurrDevEtl.cs`
|
||||
- `WorkOrderHistDevEtl.cs`
|
||||
- `WorkOrderRoutingDevEtl.cs`
|
||||
- `WorkOrderStepCurrDevEtl.cs`
|
||||
- `WorkOrderStepHistDevEtl.cs`
|
||||
- `WorkOrderTimeCurrDevEtl.cs`
|
||||
- `WorkOrderTimeHistDevEtl.cs`
|
||||
|
||||
### Files to Move (Tests)
|
||||
|
||||
**8 files** from `tests/JdeScoping.DataSync.Tests/DevEtl/` → `tests/JdeScoping.DataSync.Dev.Tests/`
|
||||
|
||||
- `BranchDevEtlTests.cs`
|
||||
- `FunctionCodeDevEtlTests.cs`
|
||||
- `ItemDevEtlTests.cs`
|
||||
- `JdeUserDevEtlTests.cs`
|
||||
- `OrgHierarchyDevEtlTests.cs`
|
||||
- `ProfitCenterDevEtlTests.cs`
|
||||
- `RouteMasterDevEtlTests.cs`
|
||||
- `WorkCenterDevEtlTests.cs`
|
||||
|
||||
## New Project Structure
|
||||
|
||||
```
|
||||
NEW/
|
||||
├── src/
|
||||
│ └── JdeScoping.DataSync.Dev/
|
||||
│ ├── JdeScoping.DataSync.Dev.csproj
|
||||
│ ├── DevEtlRegistry.cs
|
||||
│ ├── BranchDevEtl.cs
|
||||
│ └── ... (20 more files)
|
||||
└── tests/
|
||||
└── JdeScoping.DataSync.Dev.Tests/
|
||||
├── JdeScoping.DataSync.Dev.Tests.csproj
|
||||
├── BranchDevEtlTests.cs
|
||||
└── ... (7 more files)
|
||||
```
|
||||
|
||||
## Dependencies
|
||||
|
||||
### JdeScoping.DataSync.Dev.csproj
|
||||
|
||||
```xml
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\JdeScoping.DataSync\JdeScoping.DataSync.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
```
|
||||
|
||||
Reuses ETL infrastructure from DataSync:
|
||||
- `EtlPipeline`, `EtlPipelineBuilder`
|
||||
- `JsonZstdFileSource`
|
||||
- `DbBulkImportDestination`
|
||||
- `JsonColumnSchema`
|
||||
|
||||
### JdeScoping.DataSync.Dev.Tests.csproj
|
||||
|
||||
```xml
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
<IsPackable>false</IsPackable>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\..\src\JdeScoping.DataSync.Dev\JdeScoping.DataSync.Dev.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="..." />
|
||||
<PackageReference Include="xunit" Version="..." />
|
||||
<PackageReference Include="xunit.runner.visualstudio" Version="..." />
|
||||
<PackageReference Include="Shouldly" Version="..." />
|
||||
<PackageReference Include="NSubstitute" Version="..." />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
```
|
||||
|
||||
## Namespace Changes
|
||||
|
||||
All source files:
|
||||
```csharp
|
||||
// Before
|
||||
namespace JdeScoping.DataSync.DevEtl;
|
||||
|
||||
// After
|
||||
namespace JdeScoping.DataSync.Dev;
|
||||
```
|
||||
|
||||
All test files:
|
||||
```csharp
|
||||
// Before
|
||||
using JdeScoping.DataSync.DevEtl;
|
||||
|
||||
// After
|
||||
using JdeScoping.DataSync.Dev;
|
||||
```
|
||||
|
||||
## Cleanup
|
||||
|
||||
After migration, delete:
|
||||
- `src/JdeScoping.DataSync/DevEtl/` (entire folder)
|
||||
- `tests/JdeScoping.DataSync.Tests/DevEtl/` (entire folder)
|
||||
|
||||
## Cache Files
|
||||
|
||||
No change to how cache files are located. The `DevEtlRegistry` continues to accept a `cacheDirectory` parameter at runtime.
|
||||
|
||||
## Implementation Steps
|
||||
|
||||
1. Create `src/JdeScoping.DataSync.Dev/JdeScoping.DataSync.Dev.csproj`
|
||||
2. Move 22 source files from `DevEtl/` to new project
|
||||
3. Update namespace in all source files
|
||||
4. Create `tests/JdeScoping.DataSync.Dev.Tests/JdeScoping.DataSync.Dev.Tests.csproj`
|
||||
5. Move 8 test files to new project
|
||||
6. Update using statements in test files
|
||||
7. Add both projects to solution
|
||||
8. Delete old `DevEtl/` folders
|
||||
9. Build and run tests to verify
|
||||
@@ -0,0 +1,389 @@
|
||||
# DataSync.Dev Extraction Implementation Plan
|
||||
|
||||
> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task.
|
||||
|
||||
**Goal:** Extract dev/testing ETL code from JdeScoping.DataSync into a separate JdeScoping.DataSync.Dev project with its own test project.
|
||||
|
||||
**Architecture:** Create two new projects (DataSync.Dev and DataSync.Dev.Tests) that depend on the existing DataSync project. Move all DevEtl files, update namespaces, update solution file, then delete original folders.
|
||||
|
||||
**Tech Stack:** .NET 10, xUnit, Shouldly, NSubstitute
|
||||
|
||||
---
|
||||
|
||||
## Task 1: Create JdeScoping.DataSync.Dev Project
|
||||
|
||||
**Files:**
|
||||
- Create: `NEW/src/JdeScoping.DataSync.Dev/JdeScoping.DataSync.Dev.csproj`
|
||||
|
||||
**Step 1: Create project directory**
|
||||
|
||||
Run:
|
||||
```bash
|
||||
mkdir -p /Users/dohertj2/Desktop/JdeScopingTool/NEW/src/JdeScoping.DataSync.Dev
|
||||
```
|
||||
|
||||
**Step 2: Create project file**
|
||||
|
||||
Create `NEW/src/JdeScoping.DataSync.Dev/JdeScoping.DataSync.Dev.csproj`:
|
||||
|
||||
```xml
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\JdeScoping.DataSync\JdeScoping.DataSync.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
```
|
||||
|
||||
**Step 3: Verify project builds**
|
||||
|
||||
Run:
|
||||
```bash
|
||||
dotnet build /Users/dohertj2/Desktop/JdeScopingTool/NEW/src/JdeScoping.DataSync.Dev/JdeScoping.DataSync.Dev.csproj
|
||||
```
|
||||
Expected: Build succeeded
|
||||
|
||||
---
|
||||
|
||||
## Task 2: Move Source Files to DataSync.Dev
|
||||
|
||||
**Files:**
|
||||
- Move: `NEW/src/JdeScoping.DataSync/DevEtl/*.cs` → `NEW/src/JdeScoping.DataSync.Dev/`
|
||||
|
||||
**Step 1: Move all 22 source files**
|
||||
|
||||
Run:
|
||||
```bash
|
||||
mv /Users/dohertj2/Desktop/JdeScopingTool/NEW/src/JdeScoping.DataSync/DevEtl/*.cs /Users/dohertj2/Desktop/JdeScopingTool/NEW/src/JdeScoping.DataSync.Dev/
|
||||
```
|
||||
|
||||
**Step 2: Remove empty DevEtl directory**
|
||||
|
||||
Run:
|
||||
```bash
|
||||
rmdir /Users/dohertj2/Desktop/JdeScopingTool/NEW/src/JdeScoping.DataSync/DevEtl
|
||||
```
|
||||
|
||||
**Step 3: Verify files moved**
|
||||
|
||||
Run:
|
||||
```bash
|
||||
ls /Users/dohertj2/Desktop/JdeScopingTool/NEW/src/JdeScoping.DataSync.Dev/*.cs | wc -l
|
||||
```
|
||||
Expected: 22
|
||||
|
||||
---
|
||||
|
||||
## Task 3: Update Namespaces in Source Files
|
||||
|
||||
**Files:**
|
||||
- Modify: All 22 `.cs` files in `NEW/src/JdeScoping.DataSync.Dev/`
|
||||
|
||||
**Step 1: Update namespace declarations**
|
||||
|
||||
Replace in all files:
|
||||
```csharp
|
||||
namespace JdeScoping.DataSync.DevEtl;
|
||||
```
|
||||
|
||||
With:
|
||||
```csharp
|
||||
namespace JdeScoping.DataSync.Dev;
|
||||
```
|
||||
|
||||
Run:
|
||||
```bash
|
||||
cd /Users/dohertj2/Desktop/JdeScopingTool/NEW/src/JdeScoping.DataSync.Dev && \
|
||||
sed -i '' 's/namespace JdeScoping\.DataSync\.DevEtl;/namespace JdeScoping.DataSync.Dev;/g' *.cs
|
||||
```
|
||||
|
||||
**Step 2: Verify namespace changes**
|
||||
|
||||
Run:
|
||||
```bash
|
||||
grep -l "JdeScoping.DataSync.DevEtl" /Users/dohertj2/Desktop/JdeScopingTool/NEW/src/JdeScoping.DataSync.Dev/*.cs
|
||||
```
|
||||
Expected: No output (no files should contain old namespace)
|
||||
|
||||
**Step 3: Verify new namespace exists**
|
||||
|
||||
Run:
|
||||
```bash
|
||||
grep -c "namespace JdeScoping.DataSync.Dev;" /Users/dohertj2/Desktop/JdeScopingTool/NEW/src/JdeScoping.DataSync.Dev/*.cs | head -5
|
||||
```
|
||||
Expected: Each file shows `:1`
|
||||
|
||||
**Step 4: Build to verify**
|
||||
|
||||
Run:
|
||||
```bash
|
||||
dotnet build /Users/dohertj2/Desktop/JdeScopingTool/NEW/src/JdeScoping.DataSync.Dev/JdeScoping.DataSync.Dev.csproj
|
||||
```
|
||||
Expected: Build succeeded
|
||||
|
||||
---
|
||||
|
||||
## Task 4: Create JdeScoping.DataSync.Dev.Tests Project
|
||||
|
||||
**Files:**
|
||||
- Create: `NEW/tests/JdeScoping.DataSync.Dev.Tests/JdeScoping.DataSync.Dev.Tests.csproj`
|
||||
|
||||
**Step 1: Create test project directory**
|
||||
|
||||
Run:
|
||||
```bash
|
||||
mkdir -p /Users/dohertj2/Desktop/JdeScopingTool/NEW/tests/JdeScoping.DataSync.Dev.Tests
|
||||
```
|
||||
|
||||
**Step 2: Create test project file**
|
||||
|
||||
Create `NEW/tests/JdeScoping.DataSync.Dev.Tests/JdeScoping.DataSync.Dev.Tests.csproj`:
|
||||
|
||||
```xml
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
<IsPackable>false</IsPackable>
|
||||
<IsTestProject>true</IsTestProject>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.14.1" />
|
||||
<PackageReference Include="xunit" Version="2.9.3" />
|
||||
<PackageReference Include="xunit.runner.visualstudio" Version="3.0.2">
|
||||
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
|
||||
<PrivateAssets>all</PrivateAssets>
|
||||
</PackageReference>
|
||||
<PackageReference Include="Shouldly" Version="4.3.0" />
|
||||
<PackageReference Include="NSubstitute" Version="5.3.0" />
|
||||
<PackageReference Include="coverlet.collector" Version="6.0.4">
|
||||
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
|
||||
<PrivateAssets>all</PrivateAssets>
|
||||
</PackageReference>
|
||||
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="10.0.1" />
|
||||
<PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="10.0.1" />
|
||||
<PackageReference Include="Microsoft.Extensions.Configuration.EnvironmentVariables" Version="10.0.1" />
|
||||
<PackageReference Include="Microsoft.Data.SqlClient" Version="6.1.3" />
|
||||
<PackageReference Include="Dapper" Version="2.1.66" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\..\src\JdeScoping.DataAccess\JdeScoping.DataAccess.csproj" />
|
||||
<ProjectReference Include="..\..\src\JdeScoping.DataSync.Dev\JdeScoping.DataSync.Dev.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<Using Include="Xunit" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<None Include="appsettings.json" CopyToOutputDirectory="PreserveNewest" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
```
|
||||
|
||||
**Step 3: Copy appsettings.json from existing test project**
|
||||
|
||||
Run:
|
||||
```bash
|
||||
cp /Users/dohertj2/Desktop/JdeScopingTool/NEW/tests/JdeScoping.DataSync.Tests/appsettings.json /Users/dohertj2/Desktop/JdeScopingTool/NEW/tests/JdeScoping.DataSync.Dev.Tests/
|
||||
```
|
||||
|
||||
**Step 4: Verify project builds**
|
||||
|
||||
Run:
|
||||
```bash
|
||||
dotnet build /Users/dohertj2/Desktop/JdeScopingTool/NEW/tests/JdeScoping.DataSync.Dev.Tests/JdeScoping.DataSync.Dev.Tests.csproj
|
||||
```
|
||||
Expected: Build succeeded
|
||||
|
||||
---
|
||||
|
||||
## Task 5: Move Test Files to DataSync.Dev.Tests
|
||||
|
||||
**Files:**
|
||||
- Move: `NEW/tests/JdeScoping.DataSync.Tests/DevEtl/*.cs` → `NEW/tests/JdeScoping.DataSync.Dev.Tests/`
|
||||
|
||||
**Step 1: Move all 8 test files**
|
||||
|
||||
Run:
|
||||
```bash
|
||||
mv /Users/dohertj2/Desktop/JdeScopingTool/NEW/tests/JdeScoping.DataSync.Tests/DevEtl/*.cs /Users/dohertj2/Desktop/JdeScopingTool/NEW/tests/JdeScoping.DataSync.Dev.Tests/
|
||||
```
|
||||
|
||||
**Step 2: Remove empty DevEtl directory**
|
||||
|
||||
Run:
|
||||
```bash
|
||||
rmdir /Users/dohertj2/Desktop/JdeScopingTool/NEW/tests/JdeScoping.DataSync.Tests/DevEtl
|
||||
```
|
||||
|
||||
**Step 3: Verify files moved**
|
||||
|
||||
Run:
|
||||
```bash
|
||||
ls /Users/dohertj2/Desktop/JdeScopingTool/NEW/tests/JdeScoping.DataSync.Dev.Tests/*.cs | wc -l
|
||||
```
|
||||
Expected: 8
|
||||
|
||||
---
|
||||
|
||||
## Task 6: Update Namespaces and Usings in Test Files
|
||||
|
||||
**Files:**
|
||||
- Modify: All 8 `.cs` files in `NEW/tests/JdeScoping.DataSync.Dev.Tests/`
|
||||
|
||||
**Step 1: Update namespace declarations**
|
||||
|
||||
Replace in all files:
|
||||
```csharp
|
||||
namespace JdeScoping.DataSync.Tests.DevEtl;
|
||||
```
|
||||
|
||||
With:
|
||||
```csharp
|
||||
namespace JdeScoping.DataSync.Dev.Tests;
|
||||
```
|
||||
|
||||
Run:
|
||||
```bash
|
||||
cd /Users/dohertj2/Desktop/JdeScopingTool/NEW/tests/JdeScoping.DataSync.Dev.Tests && \
|
||||
sed -i '' 's/namespace JdeScoping\.DataSync\.Tests\.DevEtl;/namespace JdeScoping.DataSync.Dev.Tests;/g' *.cs
|
||||
```
|
||||
|
||||
**Step 2: Update using statements**
|
||||
|
||||
Replace in all files:
|
||||
```csharp
|
||||
using JdeScoping.DataSync.DevEtl;
|
||||
```
|
||||
|
||||
With:
|
||||
```csharp
|
||||
using JdeScoping.DataSync.Dev;
|
||||
```
|
||||
|
||||
Run:
|
||||
```bash
|
||||
cd /Users/dohertj2/Desktop/JdeScopingTool/NEW/tests/JdeScoping.DataSync.Dev.Tests && \
|
||||
sed -i '' 's/using JdeScoping\.DataSync\.DevEtl;/using JdeScoping.DataSync.Dev;/g' *.cs
|
||||
```
|
||||
|
||||
**Step 3: Verify changes**
|
||||
|
||||
Run:
|
||||
```bash
|
||||
grep -l "DevEtl" /Users/dohertj2/Desktop/JdeScopingTool/NEW/tests/JdeScoping.DataSync.Dev.Tests/*.cs
|
||||
```
|
||||
Expected: No output (no files should contain "DevEtl")
|
||||
|
||||
**Step 4: Build to verify**
|
||||
|
||||
Run:
|
||||
```bash
|
||||
dotnet build /Users/dohertj2/Desktop/JdeScopingTool/NEW/tests/JdeScoping.DataSync.Dev.Tests/JdeScoping.DataSync.Dev.Tests.csproj
|
||||
```
|
||||
Expected: Build succeeded
|
||||
|
||||
---
|
||||
|
||||
## Task 7: Update Solution File
|
||||
|
||||
**Files:**
|
||||
- Modify: `NEW/JdeScoping.slnx`
|
||||
|
||||
**Step 1: Add new projects to solution**
|
||||
|
||||
Update `NEW/JdeScoping.slnx` to add two new Project entries:
|
||||
|
||||
In the `/src/` folder section, add:
|
||||
```xml
|
||||
<Project Path="src/JdeScoping.DataSync.Dev/JdeScoping.DataSync.Dev.csproj" />
|
||||
```
|
||||
|
||||
In the `/tests/` folder section, add:
|
||||
```xml
|
||||
<Project Path="tests/JdeScoping.DataSync.Dev.Tests/JdeScoping.DataSync.Dev.Tests.csproj" />
|
||||
```
|
||||
|
||||
**Step 2: Verify solution builds**
|
||||
|
||||
Run:
|
||||
```bash
|
||||
dotnet build /Users/dohertj2/Desktop/JdeScopingTool/NEW/JdeScoping.slnx
|
||||
```
|
||||
Expected: Build succeeded
|
||||
|
||||
---
|
||||
|
||||
## Task 8: Run Tests to Verify
|
||||
|
||||
**Step 1: Run new test project**
|
||||
|
||||
Run:
|
||||
```bash
|
||||
dotnet test /Users/dohertj2/Desktop/JdeScopingTool/NEW/tests/JdeScoping.DataSync.Dev.Tests/JdeScoping.DataSync.Dev.Tests.csproj --verbosity normal
|
||||
```
|
||||
Expected: All tests pass (some may skip if cache files don't exist)
|
||||
|
||||
**Step 2: Run full solution tests**
|
||||
|
||||
Run:
|
||||
```bash
|
||||
dotnet test /Users/dohertj2/Desktop/JdeScopingTool/NEW/JdeScoping.slnx --verbosity minimal
|
||||
```
|
||||
Expected: All tests pass
|
||||
|
||||
---
|
||||
|
||||
## Task 9: Commit Changes
|
||||
|
||||
**Step 1: Stage all changes**
|
||||
|
||||
Run:
|
||||
```bash
|
||||
cd /Users/dohertj2/Desktop/JdeScopingTool && \
|
||||
git add NEW/src/JdeScoping.DataSync.Dev/ && \
|
||||
git add NEW/tests/JdeScoping.DataSync.Dev.Tests/ && \
|
||||
git add NEW/JdeScoping.slnx
|
||||
```
|
||||
|
||||
**Step 2: Commit**
|
||||
|
||||
Run:
|
||||
```bash
|
||||
git commit -m "feat: extract DevEtl to JdeScoping.DataSync.Dev project
|
||||
|
||||
- Create JdeScoping.DataSync.Dev for sandbox testing ETL code
|
||||
- Create JdeScoping.DataSync.Dev.Tests for associated tests
|
||||
- Move 22 source files and 8 test files
|
||||
- Update namespaces from DevEtl to Dev
|
||||
- Add both projects to solution"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Summary
|
||||
|
||||
| Task | Description | Files Changed |
|
||||
|------|-------------|---------------|
|
||||
| 1 | Create DataSync.Dev project | 1 new csproj |
|
||||
| 2 | Move source files | 22 files moved |
|
||||
| 3 | Update source namespaces | 22 files modified |
|
||||
| 4 | Create DataSync.Dev.Tests project | 1 new csproj + appsettings |
|
||||
| 5 | Move test files | 8 files moved |
|
||||
| 6 | Update test namespaces/usings | 8 files modified |
|
||||
| 7 | Update solution file | 1 file modified |
|
||||
| 8 | Run tests | Verification |
|
||||
| 9 | Commit | Git |
|
||||
Reference in New Issue
Block a user