refactor: remove unused classes and consolidate ViewModels in Core
Remove 9 unused types from Core (duplicate extension classes, TableSpec, ColumnSpec, LotLocation), move ComponentLotViewModel and OperatorViewModel from Client to Core, and refactor DataSync.Dev to use pipeline-based configuration. Fix Login.razor to use UserInfoDto directly.
This commit is contained in:
@@ -1,29 +0,0 @@
|
||||
using JdeScoping.DataAccess.Interfaces;
|
||||
using JdeScoping.DataSync.Etl.Destinations;
|
||||
using JdeScoping.DataSync.Etl.Pipeline;
|
||||
using JdeScoping.DataSync.Dev.Sources;
|
||||
|
||||
namespace JdeScoping.DataSync.Dev;
|
||||
|
||||
/// <summary>
|
||||
/// Development ETL pipeline for the Branch table.
|
||||
/// </summary>
|
||||
public static class BranchDevEtl
|
||||
{
|
||||
public static readonly string TableName = "Branch";
|
||||
public static readonly string CacheFileName = "branch.pb.zstd";
|
||||
|
||||
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(connectionFactory);
|
||||
|
||||
if (string.IsNullOrWhiteSpace(cacheFilePath))
|
||||
throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath));
|
||||
|
||||
return new EtlPipelineBuilder()
|
||||
.WithName($"{TableName}_Dev")
|
||||
.WithSource(new ProtobufZstdFileSource(cacheFilePath))
|
||||
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
|
||||
.Build();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
namespace JdeScoping.DataSync.Dev.Configuration;
|
||||
|
||||
/// <summary>
|
||||
/// Root configuration for development ETL pipelines.
|
||||
/// </summary>
|
||||
public record DevPipelinesRoot(
|
||||
DevPipelineSettings? Settings,
|
||||
Dictionary<string, DevPipelineConfig> Pipelines)
|
||||
{
|
||||
public DevPipelineSettings EffectiveSettings => Settings ?? new DevPipelineSettings();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Settings for development pipeline execution.
|
||||
/// </summary>
|
||||
public record DevPipelineSettings
|
||||
{
|
||||
/// <summary>
|
||||
/// Size categories for parallel/sequential execution.
|
||||
/// Very large tables run sequentially to avoid IO contention.
|
||||
/// </summary>
|
||||
public SizeCategories? SizeCategories { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Table size categorization for execution strategy.
|
||||
/// </summary>
|
||||
public record SizeCategories
|
||||
{
|
||||
/// <summary>Small tables (less than 1 MB) - run in parallel.</summary>
|
||||
public List<string>? Small { get; init; }
|
||||
|
||||
/// <summary>Medium tables (1-20 MB) - run in parallel.</summary>
|
||||
public List<string>? Medium { get; init; }
|
||||
|
||||
/// <summary>Large tables (20-200 MB) - run in parallel.</summary>
|
||||
public List<string>? Large { get; init; }
|
||||
|
||||
/// <summary>Very large tables (200+ MB) - run sequentially.</summary>
|
||||
public List<string>? VeryLarge { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Configuration for a single development ETL pipeline.
|
||||
/// </summary>
|
||||
public record DevPipelineConfig(
|
||||
DevSourceConfig Source,
|
||||
DevDestinationConfig Destination);
|
||||
|
||||
/// <summary>
|
||||
/// Source configuration for a development pipeline (file-based).
|
||||
/// </summary>
|
||||
public record DevSourceConfig(string FileName);
|
||||
|
||||
/// <summary>
|
||||
/// Destination configuration for a development pipeline.
|
||||
/// </summary>
|
||||
public record DevDestinationConfig(string Table);
|
||||
@@ -0,0 +1,29 @@
|
||||
using JdeScoping.DataSync.Etl.Pipeline;
|
||||
|
||||
namespace JdeScoping.DataSync.Dev.Contracts;
|
||||
|
||||
/// <summary>
|
||||
/// Factory for creating development ETL pipelines from JSON configuration.
|
||||
/// </summary>
|
||||
public interface IDevEtlPipelineFactory
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets the list of available table names.
|
||||
/// </summary>
|
||||
IEnumerable<string> GetAvailableTables();
|
||||
|
||||
/// <summary>
|
||||
/// Creates a pipeline for the specified table.
|
||||
/// </summary>
|
||||
/// <param name="tableName">The table name.</param>
|
||||
/// <param name="cacheDirectory">The directory containing cache files.</param>
|
||||
/// <returns>The configured ETL pipeline.</returns>
|
||||
EtlPipeline GetPipeline(string tableName, string cacheDirectory);
|
||||
|
||||
/// <summary>
|
||||
/// Checks if a table is categorized as very large (should run sequentially).
|
||||
/// </summary>
|
||||
/// <param name="tableName">The table name.</param>
|
||||
/// <returns>True if the table is very large.</returns>
|
||||
bool IsVeryLargeTable(string tableName);
|
||||
}
|
||||
@@ -1,6 +1,5 @@
|
||||
using System.Collections.Concurrent;
|
||||
using JdeScoping.DataAccess.Interfaces;
|
||||
using JdeScoping.DataSync.Etl.Pipeline;
|
||||
using JdeScoping.DataSync.Dev.Contracts;
|
||||
using JdeScoping.DataSync.Etl.Results;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
@@ -8,69 +7,20 @@ namespace JdeScoping.DataSync.Dev;
|
||||
|
||||
/// <summary>
|
||||
/// Registry for development ETL pipelines that load from cached protobuf files.
|
||||
/// Uses JSON configuration via IDevEtlPipelineFactory.
|
||||
/// </summary>
|
||||
public class DevEtlRegistry
|
||||
{
|
||||
private readonly IDbConnectionFactory _connectionFactory;
|
||||
private readonly IDevEtlPipelineFactory _pipelineFactory;
|
||||
private readonly string _cacheDirectory;
|
||||
private readonly ILogger<DevEtlRegistry>? _logger;
|
||||
|
||||
private readonly Dictionary<string, Func<IDbConnectionFactory, string, EtlPipeline>> _pipelineFactories = new(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
// Small tables (< 1 MB)
|
||||
[BranchDevEtl.TableName] = (factory, cacheDir) =>
|
||||
BranchDevEtl.Create(factory, Path.Combine(cacheDir, BranchDevEtl.CacheFileName)),
|
||||
[OrgHierarchyDevEtl.TableName] = (factory, cacheDir) =>
|
||||
OrgHierarchyDevEtl.Create(factory, Path.Combine(cacheDir, OrgHierarchyDevEtl.CacheFileName)),
|
||||
[WorkCenterDevEtl.TableName] = (factory, cacheDir) =>
|
||||
WorkCenterDevEtl.Create(factory, Path.Combine(cacheDir, WorkCenterDevEtl.CacheFileName)),
|
||||
[ProfitCenterDevEtl.TableName] = (factory, cacheDir) =>
|
||||
ProfitCenterDevEtl.Create(factory, Path.Combine(cacheDir, ProfitCenterDevEtl.CacheFileName)),
|
||||
// Medium tables (1-20 MB)
|
||||
[JdeUserDevEtl.TableName] = (factory, cacheDir) =>
|
||||
JdeUserDevEtl.Create(factory, Path.Combine(cacheDir, JdeUserDevEtl.CacheFileName)),
|
||||
[FunctionCodeDevEtl.TableName] = (factory, cacheDir) =>
|
||||
FunctionCodeDevEtl.Create(factory, Path.Combine(cacheDir, FunctionCodeDevEtl.CacheFileName)),
|
||||
[ItemDevEtl.TableName] = (factory, cacheDir) =>
|
||||
ItemDevEtl.Create(factory, Path.Combine(cacheDir, ItemDevEtl.CacheFileName)),
|
||||
[RouteMasterDevEtl.TableName] = (factory, cacheDir) =>
|
||||
RouteMasterDevEtl.Create(factory, Path.Combine(cacheDir, RouteMasterDevEtl.CacheFileName)),
|
||||
// Large tables (20-200 MB)
|
||||
[LotDevEtl.TableName] = (factory, cacheDir) =>
|
||||
LotDevEtl.Create(factory, Path.Combine(cacheDir, LotDevEtl.CacheFileName)),
|
||||
[MisDataDevEtl.TableName] = (factory, cacheDir) =>
|
||||
MisDataDevEtl.Create(factory, Path.Combine(cacheDir, MisDataDevEtl.CacheFileName)),
|
||||
[WorkOrderCurrDevEtl.TableName] = (factory, cacheDir) =>
|
||||
WorkOrderCurrDevEtl.Create(factory, Path.Combine(cacheDir, WorkOrderCurrDevEtl.CacheFileName)),
|
||||
[WorkOrderHistDevEtl.TableName] = (factory, cacheDir) =>
|
||||
WorkOrderHistDevEtl.Create(factory, Path.Combine(cacheDir, WorkOrderHistDevEtl.CacheFileName)),
|
||||
[LotUsageHistDevEtl.TableName] = (factory, cacheDir) =>
|
||||
LotUsageHistDevEtl.Create(factory, Path.Combine(cacheDir, LotUsageHistDevEtl.CacheFileName)),
|
||||
[WorkOrderComponentHistDevEtl.TableName] = (factory, cacheDir) =>
|
||||
WorkOrderComponentHistDevEtl.Create(factory, Path.Combine(cacheDir, WorkOrderComponentHistDevEtl.CacheFileName)),
|
||||
// Very large tables (200+ MB)
|
||||
[WorkOrderStepHistDevEtl.TableName] = (factory, cacheDir) =>
|
||||
WorkOrderStepHistDevEtl.Create(factory, Path.Combine(cacheDir, WorkOrderStepHistDevEtl.CacheFileName)),
|
||||
[WorkOrderComponentCurrDevEtl.TableName] = (factory, cacheDir) =>
|
||||
WorkOrderComponentCurrDevEtl.Create(factory, Path.Combine(cacheDir, WorkOrderComponentCurrDevEtl.CacheFileName)),
|
||||
[WorkOrderRoutingDevEtl.TableName] = (factory, cacheDir) =>
|
||||
WorkOrderRoutingDevEtl.Create(factory, Path.Combine(cacheDir, WorkOrderRoutingDevEtl.CacheFileName)),
|
||||
[LotUsageCurrDevEtl.TableName] = (factory, cacheDir) =>
|
||||
LotUsageCurrDevEtl.Create(factory, Path.Combine(cacheDir, LotUsageCurrDevEtl.CacheFileName)),
|
||||
[WorkOrderStepCurrDevEtl.TableName] = (factory, cacheDir) =>
|
||||
WorkOrderStepCurrDevEtl.Create(factory, Path.Combine(cacheDir, WorkOrderStepCurrDevEtl.CacheFileName)),
|
||||
[WorkOrderTimeHistDevEtl.TableName] = (factory, cacheDir) =>
|
||||
WorkOrderTimeHistDevEtl.Create(factory, Path.Combine(cacheDir, WorkOrderTimeHistDevEtl.CacheFileName)),
|
||||
[WorkOrderTimeCurrDevEtl.TableName] = (factory, cacheDir) =>
|
||||
WorkOrderTimeCurrDevEtl.Create(factory, Path.Combine(cacheDir, WorkOrderTimeCurrDevEtl.CacheFileName)),
|
||||
};
|
||||
|
||||
public DevEtlRegistry(
|
||||
IDbConnectionFactory connectionFactory,
|
||||
IDevEtlPipelineFactory pipelineFactory,
|
||||
string cacheDirectory,
|
||||
ILogger<DevEtlRegistry>? logger = null)
|
||||
{
|
||||
_connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory));
|
||||
_pipelineFactory = pipelineFactory ?? throw new ArgumentNullException(nameof(pipelineFactory));
|
||||
|
||||
if (string.IsNullOrWhiteSpace(cacheDirectory))
|
||||
throw new ArgumentException("Cache directory is required.", nameof(cacheDirectory));
|
||||
@@ -82,21 +32,13 @@ public class DevEtlRegistry
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
public IEnumerable<string> GetAvailableTables() => _pipelineFactories.Keys;
|
||||
|
||||
public EtlPipeline GetPipeline(string tableName)
|
||||
{
|
||||
if (!_pipelineFactories.TryGetValue(tableName, out var factory))
|
||||
throw new ArgumentException($"No pipeline registered for table '{tableName}'.", nameof(tableName));
|
||||
|
||||
return factory(_connectionFactory, _cacheDirectory);
|
||||
}
|
||||
public IEnumerable<string> GetAvailableTables() => _pipelineFactory.GetAvailableTables();
|
||||
|
||||
public async Task<PipelineResult> RunAsync(string tableName, CancellationToken cancellationToken = default)
|
||||
{
|
||||
_logger?.LogInformation("Running dev ETL for {TableName}", tableName);
|
||||
|
||||
var pipeline = GetPipeline(tableName);
|
||||
var pipeline = _pipelineFactory.GetPipeline(tableName, _cacheDirectory);
|
||||
var result = await pipeline.ExecuteAsync(cancellationToken);
|
||||
|
||||
if (result.Success)
|
||||
@@ -138,10 +80,10 @@ public class DevEtlRegistry
|
||||
|
||||
// Separate tables by size - run very large ones sequentially at the end
|
||||
var smallMediumTables = GetAvailableTables()
|
||||
.Where(t => !IsVeryLargeTable(t))
|
||||
.Where(t => !_pipelineFactory.IsVeryLargeTable(t))
|
||||
.ToList();
|
||||
var veryLargeTables = GetAvailableTables()
|
||||
.Where(IsVeryLargeTable)
|
||||
.Where(t => _pipelineFactory.IsVeryLargeTable(t))
|
||||
.ToList();
|
||||
|
||||
_logger?.LogInformation(
|
||||
@@ -175,14 +117,4 @@ public class DevEtlRegistry
|
||||
|
||||
return results.ToList();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Identifies very large tables that should be loaded sequentially to avoid IO contention.
|
||||
/// </summary>
|
||||
private static bool IsVeryLargeTable(string tableName) =>
|
||||
tableName.Contains("WorkOrderTime", StringComparison.OrdinalIgnoreCase) ||
|
||||
tableName.Contains("WorkOrderStep", StringComparison.OrdinalIgnoreCase) ||
|
||||
tableName.Contains("WorkOrderRouting", StringComparison.OrdinalIgnoreCase) ||
|
||||
tableName.Contains("WorkOrderComponent", StringComparison.OrdinalIgnoreCase) ||
|
||||
tableName.Contains("LotUsage", StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
@@ -1,29 +0,0 @@
|
||||
using JdeScoping.DataAccess.Interfaces;
|
||||
using JdeScoping.DataSync.Etl.Destinations;
|
||||
using JdeScoping.DataSync.Etl.Pipeline;
|
||||
using JdeScoping.DataSync.Dev.Sources;
|
||||
|
||||
namespace JdeScoping.DataSync.Dev;
|
||||
|
||||
/// <summary>
|
||||
/// Development ETL pipeline for the FunctionCode table.
|
||||
/// </summary>
|
||||
public static class FunctionCodeDevEtl
|
||||
{
|
||||
public static readonly string TableName = "FunctionCode";
|
||||
public static readonly string CacheFileName = "functioncode.pb.zstd";
|
||||
|
||||
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(connectionFactory);
|
||||
|
||||
if (string.IsNullOrWhiteSpace(cacheFilePath))
|
||||
throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath));
|
||||
|
||||
return new EtlPipelineBuilder()
|
||||
.WithName($"{TableName}_Dev")
|
||||
.WithSource(new ProtobufZstdFileSource(cacheFilePath))
|
||||
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
|
||||
.Build();
|
||||
}
|
||||
}
|
||||
@@ -1,29 +0,0 @@
|
||||
using JdeScoping.DataAccess.Interfaces;
|
||||
using JdeScoping.DataSync.Etl.Destinations;
|
||||
using JdeScoping.DataSync.Etl.Pipeline;
|
||||
using JdeScoping.DataSync.Dev.Sources;
|
||||
|
||||
namespace JdeScoping.DataSync.Dev;
|
||||
|
||||
/// <summary>
|
||||
/// Development ETL pipeline for the Item table.
|
||||
/// </summary>
|
||||
public static class ItemDevEtl
|
||||
{
|
||||
public static readonly string TableName = "Item";
|
||||
public static readonly string CacheFileName = "item.pb.zstd";
|
||||
|
||||
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(connectionFactory);
|
||||
|
||||
if (string.IsNullOrWhiteSpace(cacheFilePath))
|
||||
throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath));
|
||||
|
||||
return new EtlPipelineBuilder()
|
||||
.WithName($"{TableName}_Dev")
|
||||
.WithSource(new ProtobufZstdFileSource(cacheFilePath))
|
||||
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
|
||||
.Build();
|
||||
}
|
||||
}
|
||||
@@ -6,6 +6,11 @@
|
||||
<Nullable>enable</Nullable>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<InternalsVisibleTo Include="JdeScoping.DataSync.Dev.Tests" />
|
||||
<InternalsVisibleTo Include="DynamicProxyGenAssembly2" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="protobuf-net-data" Version="4.1.0" />
|
||||
</ItemGroup>
|
||||
@@ -14,4 +19,10 @@
|
||||
<ProjectReference Include="..\JdeScoping.DataSync\JdeScoping.DataSync.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<Content Include="Pipelines\dev-pipelines.json">
|
||||
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
|
||||
</Content>
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
|
||||
@@ -1,29 +0,0 @@
|
||||
using JdeScoping.DataAccess.Interfaces;
|
||||
using JdeScoping.DataSync.Etl.Destinations;
|
||||
using JdeScoping.DataSync.Etl.Pipeline;
|
||||
using JdeScoping.DataSync.Dev.Sources;
|
||||
|
||||
namespace JdeScoping.DataSync.Dev;
|
||||
|
||||
/// <summary>
|
||||
/// Development ETL pipeline for the JdeUser table.
|
||||
/// </summary>
|
||||
public static class JdeUserDevEtl
|
||||
{
|
||||
public static readonly string TableName = "JdeUser";
|
||||
public static readonly string CacheFileName = "jdeuser.pb.zstd";
|
||||
|
||||
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(connectionFactory);
|
||||
|
||||
if (string.IsNullOrWhiteSpace(cacheFilePath))
|
||||
throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath));
|
||||
|
||||
return new EtlPipelineBuilder()
|
||||
.WithName($"{TableName}_Dev")
|
||||
.WithSource(new ProtobufZstdFileSource(cacheFilePath))
|
||||
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
|
||||
.Build();
|
||||
}
|
||||
}
|
||||
@@ -1,29 +0,0 @@
|
||||
using JdeScoping.DataAccess.Interfaces;
|
||||
using JdeScoping.DataSync.Etl.Destinations;
|
||||
using JdeScoping.DataSync.Etl.Pipeline;
|
||||
using JdeScoping.DataSync.Dev.Sources;
|
||||
|
||||
namespace JdeScoping.DataSync.Dev;
|
||||
|
||||
/// <summary>
|
||||
/// Development ETL pipeline for the Lot table.
|
||||
/// </summary>
|
||||
public static class LotDevEtl
|
||||
{
|
||||
public static readonly string TableName = "Lot";
|
||||
public static readonly string CacheFileName = "lot.pb.zstd";
|
||||
|
||||
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(connectionFactory);
|
||||
|
||||
if (string.IsNullOrWhiteSpace(cacheFilePath))
|
||||
throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath));
|
||||
|
||||
return new EtlPipelineBuilder()
|
||||
.WithName($"{TableName}_Dev")
|
||||
.WithSource(new ProtobufZstdFileSource(cacheFilePath))
|
||||
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
|
||||
.Build();
|
||||
}
|
||||
}
|
||||
@@ -1,29 +0,0 @@
|
||||
using JdeScoping.DataAccess.Interfaces;
|
||||
using JdeScoping.DataSync.Etl.Destinations;
|
||||
using JdeScoping.DataSync.Etl.Pipeline;
|
||||
using JdeScoping.DataSync.Dev.Sources;
|
||||
|
||||
namespace JdeScoping.DataSync.Dev;
|
||||
|
||||
/// <summary>
|
||||
/// Development ETL pipeline for the LotUsage_Curr table.
|
||||
/// </summary>
|
||||
public static class LotUsageCurrDevEtl
|
||||
{
|
||||
public static readonly string TableName = "LotUsage_Curr";
|
||||
public static readonly string CacheFileName = "lotusage_curr.pb.zstd";
|
||||
|
||||
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(connectionFactory);
|
||||
|
||||
if (string.IsNullOrWhiteSpace(cacheFilePath))
|
||||
throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath));
|
||||
|
||||
return new EtlPipelineBuilder()
|
||||
.WithName($"{TableName}_Dev")
|
||||
.WithSource(new ProtobufZstdFileSource(cacheFilePath))
|
||||
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
|
||||
.Build();
|
||||
}
|
||||
}
|
||||
@@ -1,29 +0,0 @@
|
||||
using JdeScoping.DataAccess.Interfaces;
|
||||
using JdeScoping.DataSync.Etl.Destinations;
|
||||
using JdeScoping.DataSync.Etl.Pipeline;
|
||||
using JdeScoping.DataSync.Dev.Sources;
|
||||
|
||||
namespace JdeScoping.DataSync.Dev;
|
||||
|
||||
/// <summary>
|
||||
/// Development ETL pipeline for the LotUsage_Hist table.
|
||||
/// </summary>
|
||||
public static class LotUsageHistDevEtl
|
||||
{
|
||||
public static readonly string TableName = "LotUsage_Hist";
|
||||
public static readonly string CacheFileName = "lotusage_hist.pb.zstd";
|
||||
|
||||
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(connectionFactory);
|
||||
|
||||
if (string.IsNullOrWhiteSpace(cacheFilePath))
|
||||
throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath));
|
||||
|
||||
return new EtlPipelineBuilder()
|
||||
.WithName($"{TableName}_Dev")
|
||||
.WithSource(new ProtobufZstdFileSource(cacheFilePath))
|
||||
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
|
||||
.Build();
|
||||
}
|
||||
}
|
||||
@@ -1,29 +0,0 @@
|
||||
using JdeScoping.DataAccess.Interfaces;
|
||||
using JdeScoping.DataSync.Etl.Destinations;
|
||||
using JdeScoping.DataSync.Etl.Pipeline;
|
||||
using JdeScoping.DataSync.Dev.Sources;
|
||||
|
||||
namespace JdeScoping.DataSync.Dev;
|
||||
|
||||
/// <summary>
|
||||
/// Development ETL pipeline for the MisData table.
|
||||
/// </summary>
|
||||
public static class MisDataDevEtl
|
||||
{
|
||||
public static readonly string TableName = "MisData";
|
||||
public static readonly string CacheFileName = "misdata.pb.zstd";
|
||||
|
||||
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(connectionFactory);
|
||||
|
||||
if (string.IsNullOrWhiteSpace(cacheFilePath))
|
||||
throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath));
|
||||
|
||||
return new EtlPipelineBuilder()
|
||||
.WithName($"{TableName}_Dev")
|
||||
.WithSource(new ProtobufZstdFileSource(cacheFilePath))
|
||||
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
|
||||
.Build();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
namespace JdeScoping.DataSync.Dev.Options;
|
||||
|
||||
/// <summary>
|
||||
/// Configuration options for development ETL pipelines.
|
||||
/// </summary>
|
||||
public class DevPipelineOptions
|
||||
{
|
||||
/// <summary>
|
||||
/// Configuration section name.
|
||||
/// </summary>
|
||||
public const string SectionName = "DevPipelines";
|
||||
|
||||
/// <summary>
|
||||
/// Path to the dev-pipelines.json configuration file.
|
||||
/// Relative to the assembly directory.
|
||||
/// </summary>
|
||||
public string ConfigPath { get; set; } = "Pipelines/dev-pipelines.json";
|
||||
}
|
||||
@@ -1,29 +0,0 @@
|
||||
using JdeScoping.DataAccess.Interfaces;
|
||||
using JdeScoping.DataSync.Etl.Destinations;
|
||||
using JdeScoping.DataSync.Etl.Pipeline;
|
||||
using JdeScoping.DataSync.Dev.Sources;
|
||||
|
||||
namespace JdeScoping.DataSync.Dev;
|
||||
|
||||
/// <summary>
|
||||
/// Development ETL pipeline for the OrgHierarchy table.
|
||||
/// </summary>
|
||||
public static class OrgHierarchyDevEtl
|
||||
{
|
||||
public static readonly string TableName = "OrgHierarchy";
|
||||
public static readonly string CacheFileName = "orghierarchy.pb.zstd";
|
||||
|
||||
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(connectionFactory);
|
||||
|
||||
if (string.IsNullOrWhiteSpace(cacheFilePath))
|
||||
throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath));
|
||||
|
||||
return new EtlPipelineBuilder()
|
||||
.WithName($"{TableName}_Dev")
|
||||
.WithSource(new ProtobufZstdFileSource(cacheFilePath))
|
||||
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
|
||||
.Build();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
{
|
||||
"settings": {
|
||||
"sizeCategories": {
|
||||
"small": ["Branch", "OrgHierarchy", "WorkCenter", "ProfitCenter"],
|
||||
"medium": ["JdeUser", "FunctionCode", "Item", "RouteMaster"],
|
||||
"large": ["Lot", "MisData", "WorkOrder_Curr", "WorkOrder_Hist", "LotUsage_Hist", "WorkOrderComponent_Hist"],
|
||||
"veryLarge": ["WorkOrderStep_Hist", "WorkOrderComponent_Curr", "WorkOrderRouting", "LotUsage_Curr", "WorkOrderStep_Curr", "WorkOrderTime_Hist", "WorkOrderTime_Curr"]
|
||||
}
|
||||
},
|
||||
"pipelines": {
|
||||
"Branch": {
|
||||
"source": { "fileName": "branch.pb.zstd" },
|
||||
"destination": { "table": "Branch" }
|
||||
},
|
||||
"OrgHierarchy": {
|
||||
"source": { "fileName": "orghierarchy.pb.zstd" },
|
||||
"destination": { "table": "OrgHierarchy" }
|
||||
},
|
||||
"WorkCenter": {
|
||||
"source": { "fileName": "workcenter.pb.zstd" },
|
||||
"destination": { "table": "WorkCenter" }
|
||||
},
|
||||
"ProfitCenter": {
|
||||
"source": { "fileName": "profitcenter.pb.zstd" },
|
||||
"destination": { "table": "ProfitCenter" }
|
||||
},
|
||||
"JdeUser": {
|
||||
"source": { "fileName": "jdeuser.pb.zstd" },
|
||||
"destination": { "table": "JdeUser" }
|
||||
},
|
||||
"FunctionCode": {
|
||||
"source": { "fileName": "functioncode.pb.zstd" },
|
||||
"destination": { "table": "FunctionCode" }
|
||||
},
|
||||
"Item": {
|
||||
"source": { "fileName": "item.pb.zstd" },
|
||||
"destination": { "table": "Item" }
|
||||
},
|
||||
"RouteMaster": {
|
||||
"source": { "fileName": "routemaster.pb.zstd" },
|
||||
"destination": { "table": "RouteMaster" }
|
||||
},
|
||||
"Lot": {
|
||||
"source": { "fileName": "lot.pb.zstd" },
|
||||
"destination": { "table": "Lot" }
|
||||
},
|
||||
"MisData": {
|
||||
"source": { "fileName": "misdata.pb.zstd" },
|
||||
"destination": { "table": "MisData" }
|
||||
},
|
||||
"WorkOrder_Curr": {
|
||||
"source": { "fileName": "workorder_curr.pb.zstd" },
|
||||
"destination": { "table": "WorkOrder_Curr" }
|
||||
},
|
||||
"WorkOrder_Hist": {
|
||||
"source": { "fileName": "workorder_hist.pb.zstd" },
|
||||
"destination": { "table": "WorkOrder_Hist" }
|
||||
},
|
||||
"LotUsage_Curr": {
|
||||
"source": { "fileName": "lotusage_curr.pb.zstd" },
|
||||
"destination": { "table": "LotUsage_Curr" }
|
||||
},
|
||||
"LotUsage_Hist": {
|
||||
"source": { "fileName": "lotusage_hist.pb.zstd" },
|
||||
"destination": { "table": "LotUsage_Hist" }
|
||||
},
|
||||
"WorkOrderComponent_Curr": {
|
||||
"source": { "fileName": "workordercomponent_curr.pb.zstd" },
|
||||
"destination": { "table": "WorkOrderComponent_Curr" }
|
||||
},
|
||||
"WorkOrderComponent_Hist": {
|
||||
"source": { "fileName": "workordercomponent_hist.pb.zstd" },
|
||||
"destination": { "table": "WorkOrderComponent_Hist" }
|
||||
},
|
||||
"WorkOrderStep_Curr": {
|
||||
"source": { "fileName": "workorderstep_curr.pb.zstd" },
|
||||
"destination": { "table": "WorkOrderStep_Curr" }
|
||||
},
|
||||
"WorkOrderStep_Hist": {
|
||||
"source": { "fileName": "workorderstep_hist.pb.zstd" },
|
||||
"destination": { "table": "WorkOrderStep_Hist" }
|
||||
},
|
||||
"WorkOrderTime_Curr": {
|
||||
"source": { "fileName": "workordertime_curr.pb.zstd" },
|
||||
"destination": { "table": "WorkOrderTime_Curr" }
|
||||
},
|
||||
"WorkOrderTime_Hist": {
|
||||
"source": { "fileName": "workordertime_hist.pb.zstd" },
|
||||
"destination": { "table": "WorkOrderTime_Hist" }
|
||||
},
|
||||
"WorkOrderRouting": {
|
||||
"source": { "fileName": "workorderrouting.pb.zstd" },
|
||||
"destination": { "table": "WorkOrderRouting" }
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,29 +0,0 @@
|
||||
using JdeScoping.DataAccess.Interfaces;
|
||||
using JdeScoping.DataSync.Etl.Destinations;
|
||||
using JdeScoping.DataSync.Etl.Pipeline;
|
||||
using JdeScoping.DataSync.Dev.Sources;
|
||||
|
||||
namespace JdeScoping.DataSync.Dev;
|
||||
|
||||
/// <summary>
|
||||
/// Development ETL pipeline for the ProfitCenter table.
|
||||
/// </summary>
|
||||
public static class ProfitCenterDevEtl
|
||||
{
|
||||
public static readonly string TableName = "ProfitCenter";
|
||||
public static readonly string CacheFileName = "profitcenter.pb.zstd";
|
||||
|
||||
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(connectionFactory);
|
||||
|
||||
if (string.IsNullOrWhiteSpace(cacheFilePath))
|
||||
throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath));
|
||||
|
||||
return new EtlPipelineBuilder()
|
||||
.WithName($"{TableName}_Dev")
|
||||
.WithSource(new ProtobufZstdFileSource(cacheFilePath))
|
||||
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
|
||||
.Build();
|
||||
}
|
||||
}
|
||||
@@ -1,29 +0,0 @@
|
||||
using JdeScoping.DataAccess.Interfaces;
|
||||
using JdeScoping.DataSync.Etl.Destinations;
|
||||
using JdeScoping.DataSync.Etl.Pipeline;
|
||||
using JdeScoping.DataSync.Dev.Sources;
|
||||
|
||||
namespace JdeScoping.DataSync.Dev;
|
||||
|
||||
/// <summary>
|
||||
/// Development ETL pipeline for the RouteMaster table.
|
||||
/// </summary>
|
||||
public static class RouteMasterDevEtl
|
||||
{
|
||||
public static readonly string TableName = "RouteMaster";
|
||||
public static readonly string CacheFileName = "routemaster.pb.zstd";
|
||||
|
||||
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(connectionFactory);
|
||||
|
||||
if (string.IsNullOrWhiteSpace(cacheFilePath))
|
||||
throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath));
|
||||
|
||||
return new EtlPipelineBuilder()
|
||||
.WithName($"{TableName}_Dev")
|
||||
.WithSource(new ProtobufZstdFileSource(cacheFilePath))
|
||||
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
|
||||
.Build();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,123 @@
|
||||
using System.Text.Json;
|
||||
using JdeScoping.DataAccess.Interfaces;
|
||||
using JdeScoping.DataSync.Dev.Configuration;
|
||||
using JdeScoping.DataSync.Dev.Contracts;
|
||||
using JdeScoping.DataSync.Dev.Options;
|
||||
using JdeScoping.DataSync.Dev.Sources;
|
||||
using JdeScoping.DataSync.Etl.Destinations;
|
||||
using JdeScoping.DataSync.Etl.Pipeline;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
namespace JdeScoping.DataSync.Dev.Services;
|
||||
|
||||
/// <summary>
|
||||
/// Factory for creating development ETL pipelines from JSON configuration.
|
||||
/// </summary>
|
||||
public class DevEtlPipelineFactory : IDevEtlPipelineFactory
|
||||
{
|
||||
private static readonly JsonSerializerOptions JsonOptions = new()
|
||||
{
|
||||
PropertyNameCaseInsensitive = true,
|
||||
ReadCommentHandling = JsonCommentHandling.Skip,
|
||||
AllowTrailingCommas = true
|
||||
};
|
||||
|
||||
private readonly IDbConnectionFactory _connectionFactory;
|
||||
private readonly ILogger<EtlPipeline> _logger;
|
||||
private readonly DevPipelinesRoot _config;
|
||||
private readonly HashSet<string> _veryLargeTables;
|
||||
|
||||
/// <summary>
|
||||
/// Creates a new development pipeline factory.
|
||||
/// </summary>
|
||||
public DevEtlPipelineFactory(
|
||||
IDbConnectionFactory connectionFactory,
|
||||
IOptions<DevPipelineOptions> options,
|
||||
ILogger<EtlPipeline> logger)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(connectionFactory);
|
||||
ArgumentNullException.ThrowIfNull(options);
|
||||
ArgumentNullException.ThrowIfNull(logger);
|
||||
|
||||
_connectionFactory = connectionFactory;
|
||||
_logger = logger;
|
||||
_config = LoadPipelineConfigs(options.Value.ConfigPath);
|
||||
_veryLargeTables = BuildVeryLargeTableSet();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates a new development pipeline factory with pre-loaded config (for testing).
|
||||
/// </summary>
|
||||
internal DevEtlPipelineFactory(
|
||||
IDbConnectionFactory connectionFactory,
|
||||
DevPipelinesRoot config,
|
||||
ILogger<EtlPipeline> logger)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(connectionFactory);
|
||||
ArgumentNullException.ThrowIfNull(config);
|
||||
ArgumentNullException.ThrowIfNull(logger);
|
||||
|
||||
_connectionFactory = connectionFactory;
|
||||
_logger = logger;
|
||||
_config = config;
|
||||
_veryLargeTables = BuildVeryLargeTableSet();
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public IEnumerable<string> GetAvailableTables() => _config.Pipelines.Keys;
|
||||
|
||||
/// <inheritdoc />
|
||||
public bool IsVeryLargeTable(string tableName) =>
|
||||
_veryLargeTables.Contains(tableName);
|
||||
|
||||
/// <inheritdoc />
|
||||
public EtlPipeline GetPipeline(string tableName, string cacheDirectory)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(tableName);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(cacheDirectory);
|
||||
|
||||
if (!_config.Pipelines.TryGetValue(tableName, out var pipelineConfig))
|
||||
{
|
||||
throw new InvalidOperationException(
|
||||
$"No pipeline configured for table: {tableName}. " +
|
||||
$"Available tables: {string.Join(", ", _config.Pipelines.Keys)}");
|
||||
}
|
||||
|
||||
var cacheFilePath = Path.Combine(cacheDirectory, pipelineConfig.Source.FileName);
|
||||
|
||||
return new EtlPipelineBuilder()
|
||||
.WithName($"{tableName}_Dev")
|
||||
.WithSource(new ProtobufZstdFileSource(cacheFilePath))
|
||||
.WithDestination(new DbBulkImportDestination(_connectionFactory, pipelineConfig.Destination.Table))
|
||||
.WithLogger(_logger)
|
||||
.Build();
|
||||
}
|
||||
|
||||
private DevPipelinesRoot LoadPipelineConfigs(string configPath)
|
||||
{
|
||||
var assemblyDir = Path.GetDirectoryName(typeof(DevEtlPipelineFactory).Assembly.Location)!;
|
||||
var fullPath = Path.Combine(assemblyDir, configPath);
|
||||
|
||||
if (!File.Exists(fullPath))
|
||||
{
|
||||
throw new FileNotFoundException(
|
||||
$"Dev pipeline config not found: {fullPath}. " +
|
||||
"Ensure the config file is included in the build output.");
|
||||
}
|
||||
|
||||
var json = File.ReadAllText(fullPath);
|
||||
var root = JsonSerializer.Deserialize<DevPipelinesRoot>(json, JsonOptions)
|
||||
?? throw new InvalidOperationException("Failed to deserialize dev pipeline config: result was null.");
|
||||
|
||||
return root;
|
||||
}
|
||||
|
||||
private HashSet<string> BuildVeryLargeTableSet()
|
||||
{
|
||||
var veryLarge = _config.EffectiveSettings.SizeCategories?.VeryLarge;
|
||||
return veryLarge != null
|
||||
? new HashSet<string>(veryLarge, StringComparer.OrdinalIgnoreCase)
|
||||
: new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
}
|
||||
}
|
||||
@@ -1,29 +0,0 @@
|
||||
using JdeScoping.DataAccess.Interfaces;
|
||||
using JdeScoping.DataSync.Etl.Destinations;
|
||||
using JdeScoping.DataSync.Etl.Pipeline;
|
||||
using JdeScoping.DataSync.Dev.Sources;
|
||||
|
||||
namespace JdeScoping.DataSync.Dev;
|
||||
|
||||
/// <summary>
|
||||
/// Development ETL pipeline for the WorkCenter table.
|
||||
/// </summary>
|
||||
public static class WorkCenterDevEtl
|
||||
{
|
||||
public static readonly string TableName = "WorkCenter";
|
||||
public static readonly string CacheFileName = "workcenter.pb.zstd";
|
||||
|
||||
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(connectionFactory);
|
||||
|
||||
if (string.IsNullOrWhiteSpace(cacheFilePath))
|
||||
throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath));
|
||||
|
||||
return new EtlPipelineBuilder()
|
||||
.WithName($"{TableName}_Dev")
|
||||
.WithSource(new ProtobufZstdFileSource(cacheFilePath))
|
||||
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
|
||||
.Build();
|
||||
}
|
||||
}
|
||||
@@ -1,29 +0,0 @@
|
||||
using JdeScoping.DataAccess.Interfaces;
|
||||
using JdeScoping.DataSync.Etl.Destinations;
|
||||
using JdeScoping.DataSync.Etl.Pipeline;
|
||||
using JdeScoping.DataSync.Dev.Sources;
|
||||
|
||||
namespace JdeScoping.DataSync.Dev;
|
||||
|
||||
/// <summary>
|
||||
/// Development ETL pipeline for the WorkOrderComponent_Curr table.
|
||||
/// </summary>
|
||||
public static class WorkOrderComponentCurrDevEtl
|
||||
{
|
||||
public static readonly string TableName = "WorkOrderComponent_Curr";
|
||||
public static readonly string CacheFileName = "workordercomponent_curr.pb.zstd";
|
||||
|
||||
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(connectionFactory);
|
||||
|
||||
if (string.IsNullOrWhiteSpace(cacheFilePath))
|
||||
throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath));
|
||||
|
||||
return new EtlPipelineBuilder()
|
||||
.WithName($"{TableName}_Dev")
|
||||
.WithSource(new ProtobufZstdFileSource(cacheFilePath))
|
||||
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
|
||||
.Build();
|
||||
}
|
||||
}
|
||||
@@ -1,29 +0,0 @@
|
||||
using JdeScoping.DataAccess.Interfaces;
|
||||
using JdeScoping.DataSync.Etl.Destinations;
|
||||
using JdeScoping.DataSync.Etl.Pipeline;
|
||||
using JdeScoping.DataSync.Dev.Sources;
|
||||
|
||||
namespace JdeScoping.DataSync.Dev;
|
||||
|
||||
/// <summary>
|
||||
/// Development ETL pipeline for the WorkOrderComponent_Hist table.
|
||||
/// </summary>
|
||||
public static class WorkOrderComponentHistDevEtl
|
||||
{
|
||||
public static readonly string TableName = "WorkOrderComponent_Hist";
|
||||
public static readonly string CacheFileName = "workordercomponent_hist.pb.zstd";
|
||||
|
||||
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(connectionFactory);
|
||||
|
||||
if (string.IsNullOrWhiteSpace(cacheFilePath))
|
||||
throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath));
|
||||
|
||||
return new EtlPipelineBuilder()
|
||||
.WithName($"{TableName}_Dev")
|
||||
.WithSource(new ProtobufZstdFileSource(cacheFilePath))
|
||||
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
|
||||
.Build();
|
||||
}
|
||||
}
|
||||
@@ -1,29 +0,0 @@
|
||||
using JdeScoping.DataAccess.Interfaces;
|
||||
using JdeScoping.DataSync.Etl.Destinations;
|
||||
using JdeScoping.DataSync.Etl.Pipeline;
|
||||
using JdeScoping.DataSync.Dev.Sources;
|
||||
|
||||
namespace JdeScoping.DataSync.Dev;
|
||||
|
||||
/// <summary>
|
||||
/// Development ETL pipeline for the WorkOrder_Curr table.
|
||||
/// </summary>
|
||||
public static class WorkOrderCurrDevEtl
|
||||
{
|
||||
public static readonly string TableName = "WorkOrder_Curr";
|
||||
public static readonly string CacheFileName = "workorder_curr.pb.zstd";
|
||||
|
||||
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(connectionFactory);
|
||||
|
||||
if (string.IsNullOrWhiteSpace(cacheFilePath))
|
||||
throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath));
|
||||
|
||||
return new EtlPipelineBuilder()
|
||||
.WithName($"{TableName}_Dev")
|
||||
.WithSource(new ProtobufZstdFileSource(cacheFilePath))
|
||||
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
|
||||
.Build();
|
||||
}
|
||||
}
|
||||
@@ -1,29 +0,0 @@
|
||||
using JdeScoping.DataAccess.Interfaces;
|
||||
using JdeScoping.DataSync.Etl.Destinations;
|
||||
using JdeScoping.DataSync.Etl.Pipeline;
|
||||
using JdeScoping.DataSync.Dev.Sources;
|
||||
|
||||
namespace JdeScoping.DataSync.Dev;
|
||||
|
||||
/// <summary>
|
||||
/// Development ETL pipeline for the WorkOrder_Hist table.
|
||||
/// </summary>
|
||||
public static class WorkOrderHistDevEtl
|
||||
{
|
||||
public static readonly string TableName = "WorkOrder_Hist";
|
||||
public static readonly string CacheFileName = "workorder_hist.pb.zstd";
|
||||
|
||||
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(connectionFactory);
|
||||
|
||||
if (string.IsNullOrWhiteSpace(cacheFilePath))
|
||||
throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath));
|
||||
|
||||
return new EtlPipelineBuilder()
|
||||
.WithName($"{TableName}_Dev")
|
||||
.WithSource(new ProtobufZstdFileSource(cacheFilePath))
|
||||
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
|
||||
.Build();
|
||||
}
|
||||
}
|
||||
@@ -1,29 +0,0 @@
|
||||
using JdeScoping.DataAccess.Interfaces;
|
||||
using JdeScoping.DataSync.Etl.Destinations;
|
||||
using JdeScoping.DataSync.Etl.Pipeline;
|
||||
using JdeScoping.DataSync.Dev.Sources;
|
||||
|
||||
namespace JdeScoping.DataSync.Dev;
|
||||
|
||||
/// <summary>
|
||||
/// Development ETL pipeline for the WorkOrderRouting table.
|
||||
/// </summary>
|
||||
public static class WorkOrderRoutingDevEtl
|
||||
{
|
||||
public static readonly string TableName = "WorkOrderRouting";
|
||||
public static readonly string CacheFileName = "workorderrouting.pb.zstd";
|
||||
|
||||
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(connectionFactory);
|
||||
|
||||
if (string.IsNullOrWhiteSpace(cacheFilePath))
|
||||
throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath));
|
||||
|
||||
return new EtlPipelineBuilder()
|
||||
.WithName($"{TableName}_Dev")
|
||||
.WithSource(new ProtobufZstdFileSource(cacheFilePath))
|
||||
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
|
||||
.Build();
|
||||
}
|
||||
}
|
||||
@@ -1,29 +0,0 @@
|
||||
using JdeScoping.DataAccess.Interfaces;
|
||||
using JdeScoping.DataSync.Etl.Destinations;
|
||||
using JdeScoping.DataSync.Etl.Pipeline;
|
||||
using JdeScoping.DataSync.Dev.Sources;
|
||||
|
||||
namespace JdeScoping.DataSync.Dev;
|
||||
|
||||
/// <summary>
|
||||
/// Development ETL pipeline for the WorkOrderStep_Curr table.
|
||||
/// </summary>
|
||||
public static class WorkOrderStepCurrDevEtl
|
||||
{
|
||||
public static readonly string TableName = "WorkOrderStep_Curr";
|
||||
public static readonly string CacheFileName = "workorderstep_curr.pb.zstd";
|
||||
|
||||
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(connectionFactory);
|
||||
|
||||
if (string.IsNullOrWhiteSpace(cacheFilePath))
|
||||
throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath));
|
||||
|
||||
return new EtlPipelineBuilder()
|
||||
.WithName($"{TableName}_Dev")
|
||||
.WithSource(new ProtobufZstdFileSource(cacheFilePath))
|
||||
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
|
||||
.Build();
|
||||
}
|
||||
}
|
||||
@@ -1,29 +0,0 @@
|
||||
using JdeScoping.DataAccess.Interfaces;
|
||||
using JdeScoping.DataSync.Etl.Destinations;
|
||||
using JdeScoping.DataSync.Etl.Pipeline;
|
||||
using JdeScoping.DataSync.Dev.Sources;
|
||||
|
||||
namespace JdeScoping.DataSync.Dev;
|
||||
|
||||
/// <summary>
|
||||
/// Development ETL pipeline for the WorkOrderStep_Hist table.
|
||||
/// </summary>
|
||||
public static class WorkOrderStepHistDevEtl
|
||||
{
|
||||
public static readonly string TableName = "WorkOrderStep_Hist";
|
||||
public static readonly string CacheFileName = "workorderstep_hist.pb.zstd";
|
||||
|
||||
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(connectionFactory);
|
||||
|
||||
if (string.IsNullOrWhiteSpace(cacheFilePath))
|
||||
throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath));
|
||||
|
||||
return new EtlPipelineBuilder()
|
||||
.WithName($"{TableName}_Dev")
|
||||
.WithSource(new ProtobufZstdFileSource(cacheFilePath))
|
||||
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
|
||||
.Build();
|
||||
}
|
||||
}
|
||||
@@ -1,29 +0,0 @@
|
||||
using JdeScoping.DataAccess.Interfaces;
|
||||
using JdeScoping.DataSync.Etl.Destinations;
|
||||
using JdeScoping.DataSync.Etl.Pipeline;
|
||||
using JdeScoping.DataSync.Dev.Sources;
|
||||
|
||||
namespace JdeScoping.DataSync.Dev;
|
||||
|
||||
/// <summary>
|
||||
/// Development ETL pipeline for the WorkOrderTime_Curr table.
|
||||
/// </summary>
|
||||
public static class WorkOrderTimeCurrDevEtl
|
||||
{
|
||||
public static readonly string TableName = "WorkOrderTime_Curr";
|
||||
public static readonly string CacheFileName = "workordertime_curr.pb.zstd";
|
||||
|
||||
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(connectionFactory);
|
||||
|
||||
if (string.IsNullOrWhiteSpace(cacheFilePath))
|
||||
throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath));
|
||||
|
||||
return new EtlPipelineBuilder()
|
||||
.WithName($"{TableName}_Dev")
|
||||
.WithSource(new ProtobufZstdFileSource(cacheFilePath))
|
||||
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
|
||||
.Build();
|
||||
}
|
||||
}
|
||||
@@ -1,29 +0,0 @@
|
||||
using JdeScoping.DataAccess.Interfaces;
|
||||
using JdeScoping.DataSync.Etl.Destinations;
|
||||
using JdeScoping.DataSync.Etl.Pipeline;
|
||||
using JdeScoping.DataSync.Dev.Sources;
|
||||
|
||||
namespace JdeScoping.DataSync.Dev;
|
||||
|
||||
/// <summary>
|
||||
/// Development ETL pipeline for the WorkOrderTime_Hist table.
|
||||
/// </summary>
|
||||
public static class WorkOrderTimeHistDevEtl
|
||||
{
|
||||
public static readonly string TableName = "WorkOrderTime_Hist";
|
||||
public static readonly string CacheFileName = "workordertime_hist.pb.zstd";
|
||||
|
||||
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(connectionFactory);
|
||||
|
||||
if (string.IsNullOrWhiteSpace(cacheFilePath))
|
||||
throw new ArgumentException("Cache file path is required.", nameof(cacheFilePath));
|
||||
|
||||
return new EtlPipelineBuilder()
|
||||
.WithName($"{TableName}_Dev")
|
||||
.WithSource(new ProtobufZstdFileSource(cacheFilePath))
|
||||
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
|
||||
.Build();
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user