refactor: remove unused classes and consolidate ViewModels in Core

Remove 9 unused types from Core (duplicate extension classes, TableSpec, ColumnSpec, LotLocation), move ComponentLotViewModel and OperatorViewModel from Client to Core, and refactor DataSync.Dev to use pipeline-based configuration. Fix Login.razor to use UserInfoDto directly.
This commit is contained in:
Joseph Doherty
2026-01-19 00:13:12 -05:00
parent 80057590f4
commit 7e36bb4225
89 changed files with 1049 additions and 2282 deletions
@@ -1,29 +0,0 @@
using JdeScoping.DataAccess.Interfaces;
using JdeScoping.DataSync.Etl.Destinations;
using JdeScoping.DataSync.Etl.Pipeline;
using JdeScoping.DataSync.Dev.Sources;
namespace JdeScoping.DataSync.Dev;
/// <summary>
/// Development ETL pipeline for the Branch table.
/// </summary>
public static class BranchDevEtl
{
public static readonly string TableName = "Branch";
public static readonly string CacheFileName = "branch.pb.zstd";
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
{
ArgumentNullException.ThrowIfNull(connectionFactory);
if (string.IsNullOrWhiteSpace(cacheFilePath))
throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath));
return new EtlPipelineBuilder()
.WithName($"{TableName}_Dev")
.WithSource(new ProtobufZstdFileSource(cacheFilePath))
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
.Build();
}
}
@@ -0,0 +1,58 @@
namespace JdeScoping.DataSync.Dev.Configuration;
/// <summary>
/// Root configuration for development ETL pipelines.
/// </summary>
public record DevPipelinesRoot(
DevPipelineSettings? Settings,
Dictionary<string, DevPipelineConfig> Pipelines)
{
public DevPipelineSettings EffectiveSettings => Settings ?? new DevPipelineSettings();
}
/// <summary>
/// Settings for development pipeline execution.
/// </summary>
public record DevPipelineSettings
{
/// <summary>
/// Size categories for parallel/sequential execution.
/// Very large tables run sequentially to avoid IO contention.
/// </summary>
public SizeCategories? SizeCategories { get; init; }
}
/// <summary>
/// Table size categorization for execution strategy.
/// </summary>
public record SizeCategories
{
/// <summary>Small tables (less than 1 MB) - run in parallel.</summary>
public List<string>? Small { get; init; }
/// <summary>Medium tables (1-20 MB) - run in parallel.</summary>
public List<string>? Medium { get; init; }
/// <summary>Large tables (20-200 MB) - run in parallel.</summary>
public List<string>? Large { get; init; }
/// <summary>Very large tables (200+ MB) - run sequentially.</summary>
public List<string>? VeryLarge { get; init; }
}
/// <summary>
/// Configuration for a single development ETL pipeline.
/// </summary>
public record DevPipelineConfig(
DevSourceConfig Source,
DevDestinationConfig Destination);
/// <summary>
/// Source configuration for a development pipeline (file-based).
/// </summary>
public record DevSourceConfig(string FileName);
/// <summary>
/// Destination configuration for a development pipeline.
/// </summary>
public record DevDestinationConfig(string Table);
@@ -0,0 +1,29 @@
using JdeScoping.DataSync.Etl.Pipeline;
namespace JdeScoping.DataSync.Dev.Contracts;
/// <summary>
/// Factory for creating development ETL pipelines from JSON configuration.
/// </summary>
public interface IDevEtlPipelineFactory
{
/// <summary>
/// Gets the list of available table names.
/// </summary>
IEnumerable<string> GetAvailableTables();
/// <summary>
/// Creates a pipeline for the specified table.
/// </summary>
/// <param name="tableName">The table name.</param>
/// <param name="cacheDirectory">The directory containing cache files.</param>
/// <returns>The configured ETL pipeline.</returns>
EtlPipeline GetPipeline(string tableName, string cacheDirectory);
/// <summary>
/// Checks if a table is categorized as very large (should run sequentially).
/// </summary>
/// <param name="tableName">The table name.</param>
/// <returns>True if the table is very large.</returns>
bool IsVeryLargeTable(string tableName);
}
@@ -1,6 +1,5 @@
using System.Collections.Concurrent;
using JdeScoping.DataAccess.Interfaces;
using JdeScoping.DataSync.Etl.Pipeline;
using JdeScoping.DataSync.Dev.Contracts;
using JdeScoping.DataSync.Etl.Results;
using Microsoft.Extensions.Logging;
@@ -8,69 +7,20 @@ namespace JdeScoping.DataSync.Dev;
/// <summary>
/// Registry for development ETL pipelines that load from cached protobuf files.
/// Uses JSON configuration via IDevEtlPipelineFactory.
/// </summary>
public class DevEtlRegistry
{
private readonly IDbConnectionFactory _connectionFactory;
private readonly IDevEtlPipelineFactory _pipelineFactory;
private readonly string _cacheDirectory;
private readonly ILogger<DevEtlRegistry>? _logger;
private readonly Dictionary<string, Func<IDbConnectionFactory, string, EtlPipeline>> _pipelineFactories = new(StringComparer.OrdinalIgnoreCase)
{
// Small tables (< 1 MB)
[BranchDevEtl.TableName] = (factory, cacheDir) =>
BranchDevEtl.Create(factory, Path.Combine(cacheDir, BranchDevEtl.CacheFileName)),
[OrgHierarchyDevEtl.TableName] = (factory, cacheDir) =>
OrgHierarchyDevEtl.Create(factory, Path.Combine(cacheDir, OrgHierarchyDevEtl.CacheFileName)),
[WorkCenterDevEtl.TableName] = (factory, cacheDir) =>
WorkCenterDevEtl.Create(factory, Path.Combine(cacheDir, WorkCenterDevEtl.CacheFileName)),
[ProfitCenterDevEtl.TableName] = (factory, cacheDir) =>
ProfitCenterDevEtl.Create(factory, Path.Combine(cacheDir, ProfitCenterDevEtl.CacheFileName)),
// Medium tables (1-20 MB)
[JdeUserDevEtl.TableName] = (factory, cacheDir) =>
JdeUserDevEtl.Create(factory, Path.Combine(cacheDir, JdeUserDevEtl.CacheFileName)),
[FunctionCodeDevEtl.TableName] = (factory, cacheDir) =>
FunctionCodeDevEtl.Create(factory, Path.Combine(cacheDir, FunctionCodeDevEtl.CacheFileName)),
[ItemDevEtl.TableName] = (factory, cacheDir) =>
ItemDevEtl.Create(factory, Path.Combine(cacheDir, ItemDevEtl.CacheFileName)),
[RouteMasterDevEtl.TableName] = (factory, cacheDir) =>
RouteMasterDevEtl.Create(factory, Path.Combine(cacheDir, RouteMasterDevEtl.CacheFileName)),
// Large tables (20-200 MB)
[LotDevEtl.TableName] = (factory, cacheDir) =>
LotDevEtl.Create(factory, Path.Combine(cacheDir, LotDevEtl.CacheFileName)),
[MisDataDevEtl.TableName] = (factory, cacheDir) =>
MisDataDevEtl.Create(factory, Path.Combine(cacheDir, MisDataDevEtl.CacheFileName)),
[WorkOrderCurrDevEtl.TableName] = (factory, cacheDir) =>
WorkOrderCurrDevEtl.Create(factory, Path.Combine(cacheDir, WorkOrderCurrDevEtl.CacheFileName)),
[WorkOrderHistDevEtl.TableName] = (factory, cacheDir) =>
WorkOrderHistDevEtl.Create(factory, Path.Combine(cacheDir, WorkOrderHistDevEtl.CacheFileName)),
[LotUsageHistDevEtl.TableName] = (factory, cacheDir) =>
LotUsageHistDevEtl.Create(factory, Path.Combine(cacheDir, LotUsageHistDevEtl.CacheFileName)),
[WorkOrderComponentHistDevEtl.TableName] = (factory, cacheDir) =>
WorkOrderComponentHistDevEtl.Create(factory, Path.Combine(cacheDir, WorkOrderComponentHistDevEtl.CacheFileName)),
// Very large tables (200+ MB)
[WorkOrderStepHistDevEtl.TableName] = (factory, cacheDir) =>
WorkOrderStepHistDevEtl.Create(factory, Path.Combine(cacheDir, WorkOrderStepHistDevEtl.CacheFileName)),
[WorkOrderComponentCurrDevEtl.TableName] = (factory, cacheDir) =>
WorkOrderComponentCurrDevEtl.Create(factory, Path.Combine(cacheDir, WorkOrderComponentCurrDevEtl.CacheFileName)),
[WorkOrderRoutingDevEtl.TableName] = (factory, cacheDir) =>
WorkOrderRoutingDevEtl.Create(factory, Path.Combine(cacheDir, WorkOrderRoutingDevEtl.CacheFileName)),
[LotUsageCurrDevEtl.TableName] = (factory, cacheDir) =>
LotUsageCurrDevEtl.Create(factory, Path.Combine(cacheDir, LotUsageCurrDevEtl.CacheFileName)),
[WorkOrderStepCurrDevEtl.TableName] = (factory, cacheDir) =>
WorkOrderStepCurrDevEtl.Create(factory, Path.Combine(cacheDir, WorkOrderStepCurrDevEtl.CacheFileName)),
[WorkOrderTimeHistDevEtl.TableName] = (factory, cacheDir) =>
WorkOrderTimeHistDevEtl.Create(factory, Path.Combine(cacheDir, WorkOrderTimeHistDevEtl.CacheFileName)),
[WorkOrderTimeCurrDevEtl.TableName] = (factory, cacheDir) =>
WorkOrderTimeCurrDevEtl.Create(factory, Path.Combine(cacheDir, WorkOrderTimeCurrDevEtl.CacheFileName)),
};
public DevEtlRegistry(
IDbConnectionFactory connectionFactory,
IDevEtlPipelineFactory pipelineFactory,
string cacheDirectory,
ILogger<DevEtlRegistry>? logger = null)
{
_connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory));
_pipelineFactory = pipelineFactory ?? throw new ArgumentNullException(nameof(pipelineFactory));
if (string.IsNullOrWhiteSpace(cacheDirectory))
throw new ArgumentException("Cache directory is required.", nameof(cacheDirectory));
@@ -82,21 +32,13 @@ public class DevEtlRegistry
_logger = logger;
}
public IEnumerable<string> GetAvailableTables() => _pipelineFactories.Keys;
public EtlPipeline GetPipeline(string tableName)
{
if (!_pipelineFactories.TryGetValue(tableName, out var factory))
throw new ArgumentException($"No pipeline registered for table '{tableName}'.", nameof(tableName));
return factory(_connectionFactory, _cacheDirectory);
}
public IEnumerable<string> GetAvailableTables() => _pipelineFactory.GetAvailableTables();
public async Task<PipelineResult> RunAsync(string tableName, CancellationToken cancellationToken = default)
{
_logger?.LogInformation("Running dev ETL for {TableName}", tableName);
var pipeline = GetPipeline(tableName);
var pipeline = _pipelineFactory.GetPipeline(tableName, _cacheDirectory);
var result = await pipeline.ExecuteAsync(cancellationToken);
if (result.Success)
@@ -138,10 +80,10 @@ public class DevEtlRegistry
// Separate tables by size - run very large ones sequentially at the end
var smallMediumTables = GetAvailableTables()
.Where(t => !IsVeryLargeTable(t))
.Where(t => !_pipelineFactory.IsVeryLargeTable(t))
.ToList();
var veryLargeTables = GetAvailableTables()
.Where(IsVeryLargeTable)
.Where(t => _pipelineFactory.IsVeryLargeTable(t))
.ToList();
_logger?.LogInformation(
@@ -175,14 +117,4 @@ public class DevEtlRegistry
return results.ToList();
}
/// <summary>
/// Identifies very large tables that should be loaded sequentially to avoid IO contention.
/// </summary>
private static bool IsVeryLargeTable(string tableName) =>
tableName.Contains("WorkOrderTime", StringComparison.OrdinalIgnoreCase) ||
tableName.Contains("WorkOrderStep", StringComparison.OrdinalIgnoreCase) ||
tableName.Contains("WorkOrderRouting", StringComparison.OrdinalIgnoreCase) ||
tableName.Contains("WorkOrderComponent", StringComparison.OrdinalIgnoreCase) ||
tableName.Contains("LotUsage", StringComparison.OrdinalIgnoreCase);
}
@@ -1,29 +0,0 @@
using JdeScoping.DataAccess.Interfaces;
using JdeScoping.DataSync.Etl.Destinations;
using JdeScoping.DataSync.Etl.Pipeline;
using JdeScoping.DataSync.Dev.Sources;
namespace JdeScoping.DataSync.Dev;
/// <summary>
/// Development ETL pipeline for the FunctionCode table.
/// </summary>
public static class FunctionCodeDevEtl
{
public static readonly string TableName = "FunctionCode";
public static readonly string CacheFileName = "functioncode.pb.zstd";
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
{
ArgumentNullException.ThrowIfNull(connectionFactory);
if (string.IsNullOrWhiteSpace(cacheFilePath))
throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath));
return new EtlPipelineBuilder()
.WithName($"{TableName}_Dev")
.WithSource(new ProtobufZstdFileSource(cacheFilePath))
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
.Build();
}
}
@@ -1,29 +0,0 @@
using JdeScoping.DataAccess.Interfaces;
using JdeScoping.DataSync.Etl.Destinations;
using JdeScoping.DataSync.Etl.Pipeline;
using JdeScoping.DataSync.Dev.Sources;
namespace JdeScoping.DataSync.Dev;
/// <summary>
/// Development ETL pipeline for the Item table.
/// </summary>
public static class ItemDevEtl
{
public static readonly string TableName = "Item";
public static readonly string CacheFileName = "item.pb.zstd";
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
{
ArgumentNullException.ThrowIfNull(connectionFactory);
if (string.IsNullOrWhiteSpace(cacheFilePath))
throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath));
return new EtlPipelineBuilder()
.WithName($"{TableName}_Dev")
.WithSource(new ProtobufZstdFileSource(cacheFilePath))
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
.Build();
}
}
@@ -6,6 +6,11 @@
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<InternalsVisibleTo Include="JdeScoping.DataSync.Dev.Tests" />
<InternalsVisibleTo Include="DynamicProxyGenAssembly2" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="protobuf-net-data" Version="4.1.0" />
</ItemGroup>
@@ -14,4 +19,10 @@
<ProjectReference Include="..\JdeScoping.DataSync\JdeScoping.DataSync.csproj" />
</ItemGroup>
<ItemGroup>
<Content Include="Pipelines\dev-pipelines.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
</ItemGroup>
</Project>
@@ -1,29 +0,0 @@
using JdeScoping.DataAccess.Interfaces;
using JdeScoping.DataSync.Etl.Destinations;
using JdeScoping.DataSync.Etl.Pipeline;
using JdeScoping.DataSync.Dev.Sources;
namespace JdeScoping.DataSync.Dev;
/// <summary>
/// Development ETL pipeline for the JdeUser table.
/// </summary>
public static class JdeUserDevEtl
{
public static readonly string TableName = "JdeUser";
public static readonly string CacheFileName = "jdeuser.pb.zstd";
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
{
ArgumentNullException.ThrowIfNull(connectionFactory);
if (string.IsNullOrWhiteSpace(cacheFilePath))
throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath));
return new EtlPipelineBuilder()
.WithName($"{TableName}_Dev")
.WithSource(new ProtobufZstdFileSource(cacheFilePath))
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
.Build();
}
}
@@ -1,29 +0,0 @@
using JdeScoping.DataAccess.Interfaces;
using JdeScoping.DataSync.Etl.Destinations;
using JdeScoping.DataSync.Etl.Pipeline;
using JdeScoping.DataSync.Dev.Sources;
namespace JdeScoping.DataSync.Dev;
/// <summary>
/// Development ETL pipeline for the Lot table.
/// </summary>
public static class LotDevEtl
{
public static readonly string TableName = "Lot";
public static readonly string CacheFileName = "lot.pb.zstd";
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
{
ArgumentNullException.ThrowIfNull(connectionFactory);
if (string.IsNullOrWhiteSpace(cacheFilePath))
throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath));
return new EtlPipelineBuilder()
.WithName($"{TableName}_Dev")
.WithSource(new ProtobufZstdFileSource(cacheFilePath))
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
.Build();
}
}
@@ -1,29 +0,0 @@
using JdeScoping.DataAccess.Interfaces;
using JdeScoping.DataSync.Etl.Destinations;
using JdeScoping.DataSync.Etl.Pipeline;
using JdeScoping.DataSync.Dev.Sources;
namespace JdeScoping.DataSync.Dev;
/// <summary>
/// Development ETL pipeline for the LotUsage_Curr table.
/// </summary>
public static class LotUsageCurrDevEtl
{
public static readonly string TableName = "LotUsage_Curr";
public static readonly string CacheFileName = "lotusage_curr.pb.zstd";
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
{
ArgumentNullException.ThrowIfNull(connectionFactory);
if (string.IsNullOrWhiteSpace(cacheFilePath))
throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath));
return new EtlPipelineBuilder()
.WithName($"{TableName}_Dev")
.WithSource(new ProtobufZstdFileSource(cacheFilePath))
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
.Build();
}
}
@@ -1,29 +0,0 @@
using JdeScoping.DataAccess.Interfaces;
using JdeScoping.DataSync.Etl.Destinations;
using JdeScoping.DataSync.Etl.Pipeline;
using JdeScoping.DataSync.Dev.Sources;
namespace JdeScoping.DataSync.Dev;
/// <summary>
/// Development ETL pipeline for the LotUsage_Hist table.
/// </summary>
public static class LotUsageHistDevEtl
{
public static readonly string TableName = "LotUsage_Hist";
public static readonly string CacheFileName = "lotusage_hist.pb.zstd";
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
{
ArgumentNullException.ThrowIfNull(connectionFactory);
if (string.IsNullOrWhiteSpace(cacheFilePath))
throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath));
return new EtlPipelineBuilder()
.WithName($"{TableName}_Dev")
.WithSource(new ProtobufZstdFileSource(cacheFilePath))
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
.Build();
}
}
@@ -1,29 +0,0 @@
using JdeScoping.DataAccess.Interfaces;
using JdeScoping.DataSync.Etl.Destinations;
using JdeScoping.DataSync.Etl.Pipeline;
using JdeScoping.DataSync.Dev.Sources;
namespace JdeScoping.DataSync.Dev;
/// <summary>
/// Development ETL pipeline for the MisData table.
/// </summary>
public static class MisDataDevEtl
{
public static readonly string TableName = "MisData";
public static readonly string CacheFileName = "misdata.pb.zstd";
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
{
ArgumentNullException.ThrowIfNull(connectionFactory);
if (string.IsNullOrWhiteSpace(cacheFilePath))
throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath));
return new EtlPipelineBuilder()
.WithName($"{TableName}_Dev")
.WithSource(new ProtobufZstdFileSource(cacheFilePath))
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
.Build();
}
}
@@ -0,0 +1,18 @@
namespace JdeScoping.DataSync.Dev.Options;
/// <summary>
/// Configuration options for development ETL pipelines.
/// </summary>
public class DevPipelineOptions
{
/// <summary>
/// Configuration section name.
/// </summary>
public const string SectionName = "DevPipelines";
/// <summary>
/// Path to the dev-pipelines.json configuration file.
/// Relative to the assembly directory.
/// </summary>
public string ConfigPath { get; set; } = "Pipelines/dev-pipelines.json";
}
@@ -1,29 +0,0 @@
using JdeScoping.DataAccess.Interfaces;
using JdeScoping.DataSync.Etl.Destinations;
using JdeScoping.DataSync.Etl.Pipeline;
using JdeScoping.DataSync.Dev.Sources;
namespace JdeScoping.DataSync.Dev;
/// <summary>
/// Development ETL pipeline for the OrgHierarchy table.
/// </summary>
public static class OrgHierarchyDevEtl
{
public static readonly string TableName = "OrgHierarchy";
public static readonly string CacheFileName = "orghierarchy.pb.zstd";
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
{
ArgumentNullException.ThrowIfNull(connectionFactory);
if (string.IsNullOrWhiteSpace(cacheFilePath))
throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath));
return new EtlPipelineBuilder()
.WithName($"{TableName}_Dev")
.WithSource(new ProtobufZstdFileSource(cacheFilePath))
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
.Build();
}
}
@@ -0,0 +1,96 @@
{
"settings": {
"sizeCategories": {
"small": ["Branch", "OrgHierarchy", "WorkCenter", "ProfitCenter"],
"medium": ["JdeUser", "FunctionCode", "Item", "RouteMaster"],
"large": ["Lot", "MisData", "WorkOrder_Curr", "WorkOrder_Hist", "LotUsage_Hist", "WorkOrderComponent_Hist"],
"veryLarge": ["WorkOrderStep_Hist", "WorkOrderComponent_Curr", "WorkOrderRouting", "LotUsage_Curr", "WorkOrderStep_Curr", "WorkOrderTime_Hist", "WorkOrderTime_Curr"]
}
},
"pipelines": {
"Branch": {
"source": { "fileName": "branch.pb.zstd" },
"destination": { "table": "Branch" }
},
"OrgHierarchy": {
"source": { "fileName": "orghierarchy.pb.zstd" },
"destination": { "table": "OrgHierarchy" }
},
"WorkCenter": {
"source": { "fileName": "workcenter.pb.zstd" },
"destination": { "table": "WorkCenter" }
},
"ProfitCenter": {
"source": { "fileName": "profitcenter.pb.zstd" },
"destination": { "table": "ProfitCenter" }
},
"JdeUser": {
"source": { "fileName": "jdeuser.pb.zstd" },
"destination": { "table": "JdeUser" }
},
"FunctionCode": {
"source": { "fileName": "functioncode.pb.zstd" },
"destination": { "table": "FunctionCode" }
},
"Item": {
"source": { "fileName": "item.pb.zstd" },
"destination": { "table": "Item" }
},
"RouteMaster": {
"source": { "fileName": "routemaster.pb.zstd" },
"destination": { "table": "RouteMaster" }
},
"Lot": {
"source": { "fileName": "lot.pb.zstd" },
"destination": { "table": "Lot" }
},
"MisData": {
"source": { "fileName": "misdata.pb.zstd" },
"destination": { "table": "MisData" }
},
"WorkOrder_Curr": {
"source": { "fileName": "workorder_curr.pb.zstd" },
"destination": { "table": "WorkOrder_Curr" }
},
"WorkOrder_Hist": {
"source": { "fileName": "workorder_hist.pb.zstd" },
"destination": { "table": "WorkOrder_Hist" }
},
"LotUsage_Curr": {
"source": { "fileName": "lotusage_curr.pb.zstd" },
"destination": { "table": "LotUsage_Curr" }
},
"LotUsage_Hist": {
"source": { "fileName": "lotusage_hist.pb.zstd" },
"destination": { "table": "LotUsage_Hist" }
},
"WorkOrderComponent_Curr": {
"source": { "fileName": "workordercomponent_curr.pb.zstd" },
"destination": { "table": "WorkOrderComponent_Curr" }
},
"WorkOrderComponent_Hist": {
"source": { "fileName": "workordercomponent_hist.pb.zstd" },
"destination": { "table": "WorkOrderComponent_Hist" }
},
"WorkOrderStep_Curr": {
"source": { "fileName": "workorderstep_curr.pb.zstd" },
"destination": { "table": "WorkOrderStep_Curr" }
},
"WorkOrderStep_Hist": {
"source": { "fileName": "workorderstep_hist.pb.zstd" },
"destination": { "table": "WorkOrderStep_Hist" }
},
"WorkOrderTime_Curr": {
"source": { "fileName": "workordertime_curr.pb.zstd" },
"destination": { "table": "WorkOrderTime_Curr" }
},
"WorkOrderTime_Hist": {
"source": { "fileName": "workordertime_hist.pb.zstd" },
"destination": { "table": "WorkOrderTime_Hist" }
},
"WorkOrderRouting": {
"source": { "fileName": "workorderrouting.pb.zstd" },
"destination": { "table": "WorkOrderRouting" }
}
}
}
@@ -1,29 +0,0 @@
using JdeScoping.DataAccess.Interfaces;
using JdeScoping.DataSync.Etl.Destinations;
using JdeScoping.DataSync.Etl.Pipeline;
using JdeScoping.DataSync.Dev.Sources;
namespace JdeScoping.DataSync.Dev;
/// <summary>
/// Development ETL pipeline for the ProfitCenter table.
/// </summary>
public static class ProfitCenterDevEtl
{
public static readonly string TableName = "ProfitCenter";
public static readonly string CacheFileName = "profitcenter.pb.zstd";
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
{
ArgumentNullException.ThrowIfNull(connectionFactory);
if (string.IsNullOrWhiteSpace(cacheFilePath))
throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath));
return new EtlPipelineBuilder()
.WithName($"{TableName}_Dev")
.WithSource(new ProtobufZstdFileSource(cacheFilePath))
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
.Build();
}
}
@@ -1,29 +0,0 @@
using JdeScoping.DataAccess.Interfaces;
using JdeScoping.DataSync.Etl.Destinations;
using JdeScoping.DataSync.Etl.Pipeline;
using JdeScoping.DataSync.Dev.Sources;
namespace JdeScoping.DataSync.Dev;
/// <summary>
/// Development ETL pipeline for the RouteMaster table.
/// </summary>
public static class RouteMasterDevEtl
{
public static readonly string TableName = "RouteMaster";
public static readonly string CacheFileName = "routemaster.pb.zstd";
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
{
ArgumentNullException.ThrowIfNull(connectionFactory);
if (string.IsNullOrWhiteSpace(cacheFilePath))
throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath));
return new EtlPipelineBuilder()
.WithName($"{TableName}_Dev")
.WithSource(new ProtobufZstdFileSource(cacheFilePath))
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
.Build();
}
}
@@ -0,0 +1,123 @@
using System.Text.Json;
using JdeScoping.DataAccess.Interfaces;
using JdeScoping.DataSync.Dev.Configuration;
using JdeScoping.DataSync.Dev.Contracts;
using JdeScoping.DataSync.Dev.Options;
using JdeScoping.DataSync.Dev.Sources;
using JdeScoping.DataSync.Etl.Destinations;
using JdeScoping.DataSync.Etl.Pipeline;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
namespace JdeScoping.DataSync.Dev.Services;
/// <summary>
/// Factory for creating development ETL pipelines from JSON configuration.
/// </summary>
public class DevEtlPipelineFactory : IDevEtlPipelineFactory
{
private static readonly JsonSerializerOptions JsonOptions = new()
{
PropertyNameCaseInsensitive = true,
ReadCommentHandling = JsonCommentHandling.Skip,
AllowTrailingCommas = true
};
private readonly IDbConnectionFactory _connectionFactory;
private readonly ILogger<EtlPipeline> _logger;
private readonly DevPipelinesRoot _config;
private readonly HashSet<string> _veryLargeTables;
/// <summary>
/// Creates a new development pipeline factory.
/// </summary>
public DevEtlPipelineFactory(
IDbConnectionFactory connectionFactory,
IOptions<DevPipelineOptions> options,
ILogger<EtlPipeline> logger)
{
ArgumentNullException.ThrowIfNull(connectionFactory);
ArgumentNullException.ThrowIfNull(options);
ArgumentNullException.ThrowIfNull(logger);
_connectionFactory = connectionFactory;
_logger = logger;
_config = LoadPipelineConfigs(options.Value.ConfigPath);
_veryLargeTables = BuildVeryLargeTableSet();
}
/// <summary>
/// Creates a new development pipeline factory with pre-loaded config (for testing).
/// </summary>
internal DevEtlPipelineFactory(
IDbConnectionFactory connectionFactory,
DevPipelinesRoot config,
ILogger<EtlPipeline> logger)
{
ArgumentNullException.ThrowIfNull(connectionFactory);
ArgumentNullException.ThrowIfNull(config);
ArgumentNullException.ThrowIfNull(logger);
_connectionFactory = connectionFactory;
_logger = logger;
_config = config;
_veryLargeTables = BuildVeryLargeTableSet();
}
/// <inheritdoc />
public IEnumerable<string> GetAvailableTables() => _config.Pipelines.Keys;
/// <inheritdoc />
public bool IsVeryLargeTable(string tableName) =>
_veryLargeTables.Contains(tableName);
/// <inheritdoc />
public EtlPipeline GetPipeline(string tableName, string cacheDirectory)
{
ArgumentException.ThrowIfNullOrWhiteSpace(tableName);
ArgumentException.ThrowIfNullOrWhiteSpace(cacheDirectory);
if (!_config.Pipelines.TryGetValue(tableName, out var pipelineConfig))
{
throw new InvalidOperationException(
$"No pipeline configured for table: {tableName}. " +
$"Available tables: {string.Join(", ", _config.Pipelines.Keys)}");
}
var cacheFilePath = Path.Combine(cacheDirectory, pipelineConfig.Source.FileName);
return new EtlPipelineBuilder()
.WithName($"{tableName}_Dev")
.WithSource(new ProtobufZstdFileSource(cacheFilePath))
.WithDestination(new DbBulkImportDestination(_connectionFactory, pipelineConfig.Destination.Table))
.WithLogger(_logger)
.Build();
}
private DevPipelinesRoot LoadPipelineConfigs(string configPath)
{
var assemblyDir = Path.GetDirectoryName(typeof(DevEtlPipelineFactory).Assembly.Location)!;
var fullPath = Path.Combine(assemblyDir, configPath);
if (!File.Exists(fullPath))
{
throw new FileNotFoundException(
$"Dev pipeline config not found: {fullPath}. " +
"Ensure the config file is included in the build output.");
}
var json = File.ReadAllText(fullPath);
var root = JsonSerializer.Deserialize<DevPipelinesRoot>(json, JsonOptions)
?? throw new InvalidOperationException("Failed to deserialize dev pipeline config: result was null.");
return root;
}
private HashSet<string> BuildVeryLargeTableSet()
{
var veryLarge = _config.EffectiveSettings.SizeCategories?.VeryLarge;
return veryLarge != null
? new HashSet<string>(veryLarge, StringComparer.OrdinalIgnoreCase)
: new HashSet<string>(StringComparer.OrdinalIgnoreCase);
}
}
@@ -1,29 +0,0 @@
using JdeScoping.DataAccess.Interfaces;
using JdeScoping.DataSync.Etl.Destinations;
using JdeScoping.DataSync.Etl.Pipeline;
using JdeScoping.DataSync.Dev.Sources;
namespace JdeScoping.DataSync.Dev;
/// <summary>
/// Development ETL pipeline for the WorkCenter table.
/// </summary>
public static class WorkCenterDevEtl
{
public static readonly string TableName = "WorkCenter";
public static readonly string CacheFileName = "workcenter.pb.zstd";
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
{
ArgumentNullException.ThrowIfNull(connectionFactory);
if (string.IsNullOrWhiteSpace(cacheFilePath))
throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath));
return new EtlPipelineBuilder()
.WithName($"{TableName}_Dev")
.WithSource(new ProtobufZstdFileSource(cacheFilePath))
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
.Build();
}
}
@@ -1,29 +0,0 @@
using JdeScoping.DataAccess.Interfaces;
using JdeScoping.DataSync.Etl.Destinations;
using JdeScoping.DataSync.Etl.Pipeline;
using JdeScoping.DataSync.Dev.Sources;
namespace JdeScoping.DataSync.Dev;
/// <summary>
/// Development ETL pipeline for the WorkOrderComponent_Curr table.
/// </summary>
public static class WorkOrderComponentCurrDevEtl
{
public static readonly string TableName = "WorkOrderComponent_Curr";
public static readonly string CacheFileName = "workordercomponent_curr.pb.zstd";
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
{
ArgumentNullException.ThrowIfNull(connectionFactory);
if (string.IsNullOrWhiteSpace(cacheFilePath))
throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath));
return new EtlPipelineBuilder()
.WithName($"{TableName}_Dev")
.WithSource(new ProtobufZstdFileSource(cacheFilePath))
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
.Build();
}
}
@@ -1,29 +0,0 @@
using JdeScoping.DataAccess.Interfaces;
using JdeScoping.DataSync.Etl.Destinations;
using JdeScoping.DataSync.Etl.Pipeline;
using JdeScoping.DataSync.Dev.Sources;
namespace JdeScoping.DataSync.Dev;
/// <summary>
/// Development ETL pipeline for the WorkOrderComponent_Hist table.
/// </summary>
public static class WorkOrderComponentHistDevEtl
{
public static readonly string TableName = "WorkOrderComponent_Hist";
public static readonly string CacheFileName = "workordercomponent_hist.pb.zstd";
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
{
ArgumentNullException.ThrowIfNull(connectionFactory);
if (string.IsNullOrWhiteSpace(cacheFilePath))
throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath));
return new EtlPipelineBuilder()
.WithName($"{TableName}_Dev")
.WithSource(new ProtobufZstdFileSource(cacheFilePath))
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
.Build();
}
}
@@ -1,29 +0,0 @@
using JdeScoping.DataAccess.Interfaces;
using JdeScoping.DataSync.Etl.Destinations;
using JdeScoping.DataSync.Etl.Pipeline;
using JdeScoping.DataSync.Dev.Sources;
namespace JdeScoping.DataSync.Dev;
/// <summary>
/// Development ETL pipeline for the WorkOrder_Curr table.
/// </summary>
public static class WorkOrderCurrDevEtl
{
public static readonly string TableName = "WorkOrder_Curr";
public static readonly string CacheFileName = "workorder_curr.pb.zstd";
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
{
ArgumentNullException.ThrowIfNull(connectionFactory);
if (string.IsNullOrWhiteSpace(cacheFilePath))
throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath));
return new EtlPipelineBuilder()
.WithName($"{TableName}_Dev")
.WithSource(new ProtobufZstdFileSource(cacheFilePath))
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
.Build();
}
}
@@ -1,29 +0,0 @@
using JdeScoping.DataAccess.Interfaces;
using JdeScoping.DataSync.Etl.Destinations;
using JdeScoping.DataSync.Etl.Pipeline;
using JdeScoping.DataSync.Dev.Sources;
namespace JdeScoping.DataSync.Dev;
/// <summary>
/// Development ETL pipeline for the WorkOrder_Hist table.
/// </summary>
public static class WorkOrderHistDevEtl
{
public static readonly string TableName = "WorkOrder_Hist";
public static readonly string CacheFileName = "workorder_hist.pb.zstd";
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
{
ArgumentNullException.ThrowIfNull(connectionFactory);
if (string.IsNullOrWhiteSpace(cacheFilePath))
throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath));
return new EtlPipelineBuilder()
.WithName($"{TableName}_Dev")
.WithSource(new ProtobufZstdFileSource(cacheFilePath))
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
.Build();
}
}
@@ -1,29 +0,0 @@
using JdeScoping.DataAccess.Interfaces;
using JdeScoping.DataSync.Etl.Destinations;
using JdeScoping.DataSync.Etl.Pipeline;
using JdeScoping.DataSync.Dev.Sources;
namespace JdeScoping.DataSync.Dev;
/// <summary>
/// Development ETL pipeline for the WorkOrderRouting table.
/// </summary>
public static class WorkOrderRoutingDevEtl
{
public static readonly string TableName = "WorkOrderRouting";
public static readonly string CacheFileName = "workorderrouting.pb.zstd";
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
{
ArgumentNullException.ThrowIfNull(connectionFactory);
if (string.IsNullOrWhiteSpace(cacheFilePath))
throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath));
return new EtlPipelineBuilder()
.WithName($"{TableName}_Dev")
.WithSource(new ProtobufZstdFileSource(cacheFilePath))
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
.Build();
}
}
@@ -1,29 +0,0 @@
using JdeScoping.DataAccess.Interfaces;
using JdeScoping.DataSync.Etl.Destinations;
using JdeScoping.DataSync.Etl.Pipeline;
using JdeScoping.DataSync.Dev.Sources;
namespace JdeScoping.DataSync.Dev;
/// <summary>
/// Development ETL pipeline for the WorkOrderStep_Curr table.
/// </summary>
public static class WorkOrderStepCurrDevEtl
{
public static readonly string TableName = "WorkOrderStep_Curr";
public static readonly string CacheFileName = "workorderstep_curr.pb.zstd";
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
{
ArgumentNullException.ThrowIfNull(connectionFactory);
if (string.IsNullOrWhiteSpace(cacheFilePath))
throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath));
return new EtlPipelineBuilder()
.WithName($"{TableName}_Dev")
.WithSource(new ProtobufZstdFileSource(cacheFilePath))
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
.Build();
}
}
@@ -1,29 +0,0 @@
using JdeScoping.DataAccess.Interfaces;
using JdeScoping.DataSync.Etl.Destinations;
using JdeScoping.DataSync.Etl.Pipeline;
using JdeScoping.DataSync.Dev.Sources;
namespace JdeScoping.DataSync.Dev;
/// <summary>
/// Development ETL pipeline for the WorkOrderStep_Hist table.
/// </summary>
public static class WorkOrderStepHistDevEtl
{
public static readonly string TableName = "WorkOrderStep_Hist";
public static readonly string CacheFileName = "workorderstep_hist.pb.zstd";
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
{
ArgumentNullException.ThrowIfNull(connectionFactory);
if (string.IsNullOrWhiteSpace(cacheFilePath))
throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath));
return new EtlPipelineBuilder()
.WithName($"{TableName}_Dev")
.WithSource(new ProtobufZstdFileSource(cacheFilePath))
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
.Build();
}
}
@@ -1,29 +0,0 @@
using JdeScoping.DataAccess.Interfaces;
using JdeScoping.DataSync.Etl.Destinations;
using JdeScoping.DataSync.Etl.Pipeline;
using JdeScoping.DataSync.Dev.Sources;
namespace JdeScoping.DataSync.Dev;
/// <summary>
/// Development ETL pipeline for the WorkOrderTime_Curr table.
/// </summary>
public static class WorkOrderTimeCurrDevEtl
{
public static readonly string TableName = "WorkOrderTime_Curr";
public static readonly string CacheFileName = "workordertime_curr.pb.zstd";
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
{
ArgumentNullException.ThrowIfNull(connectionFactory);
if (string.IsNullOrWhiteSpace(cacheFilePath))
throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath));
return new EtlPipelineBuilder()
.WithName($"{TableName}_Dev")
.WithSource(new ProtobufZstdFileSource(cacheFilePath))
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
.Build();
}
}
@@ -1,29 +0,0 @@
using JdeScoping.DataAccess.Interfaces;
using JdeScoping.DataSync.Etl.Destinations;
using JdeScoping.DataSync.Etl.Pipeline;
using JdeScoping.DataSync.Dev.Sources;
namespace JdeScoping.DataSync.Dev;
/// <summary>
/// Development ETL pipeline for the WorkOrderTime_Hist table.
/// </summary>
public static class WorkOrderTimeHistDevEtl
{
public static readonly string TableName = "WorkOrderTime_Hist";
public static readonly string CacheFileName = "workordertime_hist.pb.zstd";
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
{
ArgumentNullException.ThrowIfNull(connectionFactory);
if (string.IsNullOrWhiteSpace(cacheFilePath))
throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath));
return new EtlPipelineBuilder()
.WithName($"{TableName}_Dev")
.WithSource(new ProtobufZstdFileSource(cacheFilePath))
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
.Build();
}
}