dohertj2/jdescopingtool
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: migrate all DevEtl files to protobuf source

Browse Source 
Replace JsonZstdFileSource with ProtobufZstdFileSource across all 20
DevEtl pipeline files. This removes the need for manually defined
JsonColumnSchema arrays as protobuf files embed their schema.

Changes per file:
- Remove using JdeScoping.DataSync.Dev.Models
- Remove private JsonColumnSchema[] Schema array
- Change CacheFileName from .json.zstd to .pb.zstd
- Change WithSource from JsonZstdFileSource to ProtobufZstdFileSource
This commit is contained in:
Joseph Doherty
2026-01-06 16:42:56 -05:00
parent ca63b674f5
commit ead3496cdf
20 changed files with 40 additions and 325 deletions
Download Patch File Download Diff File Expand all files Collapse all files
NEW/src/JdeScoping.DataSync.Dev/FunctionCodeDevEtl.cs
+2 -11
View File
@@ -1,6 +1,5 @@
using JdeScoping.DataAccess.Interfaces;
using JdeScoping.DataSync.Etl.Destinations;
using JdeScoping.DataSync.Dev.Models;
using JdeScoping.DataSync.Etl.Pipeline;
using JdeScoping.DataSync.Dev.Sources;
@@ -8,19 +7,11 @@ namespace JdeScoping.DataSync.Dev;
/// <summary>
/// Development ETL pipeline for the FunctionCode table.
/// Schema from: Scripts/005_CreateFunctionCodeTable.sql
/// </summary>
public static class FunctionCodeDevEtl
{
public static readonly string TableName = "FunctionCode";
public static readonly string CacheFileName = "functioncode.json.zstd";
private static readonly JsonColumnSchema[] Schema =
[
new("Code", typeof(string), IsNullable: false),
new("Description", typeof(string), IsNullable: true),
new("LastUpdateDT", typeof(DateTime), IsNullable: false),
];
public static readonly string CacheFileName = "functioncode.pb.zstd";
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
{
@@ -31,7 +22,7 @@ public static class FunctionCodeDevEtl
return new EtlPipelineBuilder()
.WithName($"{TableName}_Dev")
.WithSource(new JsonZstdFileSource(cacheFilePath, Schema))
.WithSource(new ProtobufZstdFileSource(cacheFilePath))
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
.Build();
}
NEW/src/JdeScoping.DataSync.Dev/ItemDevEtl.cs
+2 -14
View File
@@ -1,6 +1,5 @@
using JdeScoping.DataAccess.Interfaces;
using JdeScoping.DataSync.Etl.Destinations;
using JdeScoping.DataSync.Dev.Models;
using JdeScoping.DataSync.Etl.Pipeline;
using JdeScoping.DataSync.Dev.Sources;
@@ -8,22 +7,11 @@ namespace JdeScoping.DataSync.Dev;
/// <summary>
/// Development ETL pipeline for the Item table.
/// Schema from: Scripts/008_CreateItemTable.sql
/// </summary>
public static class ItemDevEtl
{
public static readonly string TableName = "Item";
public static readonly string CacheFileName = "item.json.zstd";
private static readonly JsonColumnSchema[] Schema =
[
new("ShortItemNumber", typeof(long), IsNullable: false),
new("ItemNumber", typeof(string), IsNullable: false),
new("Description", typeof(string), IsNullable: true),
new("PlanningFamily", typeof(string), IsNullable: true),
new("StockingType", typeof(string), IsNullable: true),
new("LastUpdateDT", typeof(DateTime), IsNullable: false),
];
public static readonly string CacheFileName = "item.pb.zstd";
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
{
@@ -34,7 +22,7 @@ public static class ItemDevEtl
return new EtlPipelineBuilder()
.WithName($"{TableName}_Dev")
.WithSource(new JsonZstdFileSource(cacheFilePath, Schema))
.WithSource(new ProtobufZstdFileSource(cacheFilePath))
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
.Build();
}
NEW/src/JdeScoping.DataSync.Dev/JdeUserDevEtl.cs
+2 -12
View File
@@ -1,6 +1,5 @@
using JdeScoping.DataAccess.Interfaces;
using JdeScoping.DataSync.Etl.Destinations;
using JdeScoping.DataSync.Dev.Models;
using JdeScoping.DataSync.Etl.Pipeline;
using JdeScoping.DataSync.Dev.Sources;
@@ -8,20 +7,11 @@ namespace JdeScoping.DataSync.Dev;
/// <summary>
/// Development ETL pipeline for the JdeUser table.
/// Schema from: Scripts/009_CreateJdeUserTable.sql
/// </summary>
public static class JdeUserDevEtl
{
public static readonly string TableName = "JdeUser";
public static readonly string CacheFileName = "jdeuser.json.zstd";
private static readonly JsonColumnSchema[] Schema =
[
new("AddressNumber", typeof(long), IsNullable: false),
new("UserID", typeof(string), IsNullable: true),
new("FullName", typeof(string), IsNullable: false),
new("LastUpdateDT", typeof(DateTime), IsNullable: false),
];
public static readonly string CacheFileName = "jdeuser.pb.zstd";
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
{
@@ -32,7 +22,7 @@ public static class JdeUserDevEtl
return new EtlPipelineBuilder()
.WithName($"{TableName}_Dev")
.WithSource(new JsonZstdFileSource(cacheFilePath, Schema))
.WithSource(new ProtobufZstdFileSource(cacheFilePath))
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
.Build();
}
NEW/src/JdeScoping.DataSync.Dev/LotDevEtl.cs
+2 -18
View File
@@ -1,6 +1,5 @@
using JdeScoping.DataAccess.Interfaces;
using JdeScoping.DataSync.Etl.Destinations;
using JdeScoping.DataSync.Dev.Models;
using JdeScoping.DataSync.Etl.Pipeline;
using JdeScoping.DataSync.Dev.Sources;
@@ -8,26 +7,11 @@ namespace JdeScoping.DataSync.Dev;
/// <summary>
/// Development ETL pipeline for the Lot table.
/// Schema from: Scripts/013_CreateLotTable.sql
/// </summary>
public static class LotDevEtl
{
public static readonly string TableName = "Lot";
public static readonly string CacheFileName = "lot.json.zstd";
private static readonly JsonColumnSchema[] Schema =
[
new("LotNumber", typeof(string), IsNullable: false),
new("BranchCode", typeof(string), IsNullable: false),
new("ShortItemNumber", typeof(long), IsNullable: false),
new("ItemNumber", typeof(string), IsNullable: true),
new("SupplierCode", typeof(long), IsNullable: false),
new("StatusCode", typeof(string), IsNullable: true),
new("Memo1", typeof(string), IsNullable: true),
new("Memo2", typeof(string), IsNullable: true),
new("Memo3", typeof(string), IsNullable: true),
new("LastUpdateDT", typeof(DateTime), IsNullable: false),
];
public static readonly string CacheFileName = "lot.pb.zstd";
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
{
@@ -38,7 +22,7 @@ public static class LotDevEtl
return new EtlPipelineBuilder()
.WithName($"{TableName}_Dev")
.WithSource(new JsonZstdFileSource(cacheFilePath, Schema))
.WithSource(new ProtobufZstdFileSource(cacheFilePath))
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
.Build();
}
NEW/src/JdeScoping.DataSync.Dev/LotUsageCurrDevEtl.cs
+2 -15
View File
@@ -1,6 +1,5 @@
using JdeScoping.DataAccess.Interfaces;
using JdeScoping.DataSync.Etl.Destinations;
using JdeScoping.DataSync.Dev.Models;
using JdeScoping.DataSync.Etl.Pipeline;
using JdeScoping.DataSync.Dev.Sources;
@@ -8,23 +7,11 @@ namespace JdeScoping.DataSync.Dev;
/// <summary>
/// Development ETL pipeline for the LotUsage_Curr table.
/// Schema from: Scripts/024_CreateLotUsageCurrTable.sql
/// </summary>
public static class LotUsageCurrDevEtl
{
public static readonly string TableName = "LotUsage_Curr";
public static readonly string CacheFileName = "lotusage_curr.json.zstd";
private static readonly JsonColumnSchema[] Schema =
[
new("UniqueID", typeof(long), IsNullable: false),
new("WorkOrderNumber", typeof(long), IsNullable: false),
new("LotNumber", typeof(string), IsNullable: false),
new("BranchCode", typeof(string), IsNullable: true),
new("ShortItemNumber", typeof(long), IsNullable: false),
new("Quantity", typeof(decimal), IsNullable: false),
new("LastUpdateDT", typeof(DateTime), IsNullable: false),
];
public static readonly string CacheFileName = "lotusage_curr.pb.zstd";
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
{
@@ -35,7 +22,7 @@ public static class LotUsageCurrDevEtl
return new EtlPipelineBuilder()
.WithName($"{TableName}_Dev")
.WithSource(new JsonZstdFileSource(cacheFilePath, Schema))
.WithSource(new ProtobufZstdFileSource(cacheFilePath))
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
.Build();
}
NEW/src/JdeScoping.DataSync.Dev/LotUsageHistDevEtl.cs
+2 -15
View File
@@ -1,6 +1,5 @@
using JdeScoping.DataAccess.Interfaces;
using JdeScoping.DataSync.Etl.Destinations;
using JdeScoping.DataSync.Dev.Models;
using JdeScoping.DataSync.Etl.Pipeline;
using JdeScoping.DataSync.Dev.Sources;
@@ -8,23 +7,11 @@ namespace JdeScoping.DataSync.Dev;
/// <summary>
/// Development ETL pipeline for the LotUsage_Hist table.
/// Schema from: Scripts/025_CreateLotUsageHistTable.sql
/// </summary>
public static class LotUsageHistDevEtl
{
public static readonly string TableName = "LotUsage_Hist";
public static readonly string CacheFileName = "lotusage_hist.json.zstd";
private static readonly JsonColumnSchema[] Schema =
[
new("UniqueID", typeof(long), IsNullable: false),
new("WorkOrderNumber", typeof(long), IsNullable: false),
new("LotNumber", typeof(string), IsNullable: false),
new("BranchCode", typeof(string), IsNullable: true),
new("ShortItemNumber", typeof(long), IsNullable: false),
new("Quantity", typeof(decimal), IsNullable: false),
new("LastUpdateDT", typeof(DateTime), IsNullable: false),
];
public static readonly string CacheFileName = "lotusage_hist.pb.zstd";
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
{
@@ -35,7 +22,7 @@ public static class LotUsageHistDevEtl
return new EtlPipelineBuilder()
.WithName($"{TableName}_Dev")
.WithSource(new JsonZstdFileSource(cacheFilePath, Schema))
.WithSource(new ProtobufZstdFileSource(cacheFilePath))
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
.Build();
}
NEW/src/JdeScoping.DataSync.Dev/MisDataDevEtl.cs
+2 -22
View File
@@ -1,6 +1,5 @@
using JdeScoping.DataAccess.Interfaces;
using JdeScoping.DataSync.Etl.Destinations;
using JdeScoping.DataSync.Dev.Models;
using JdeScoping.DataSync.Etl.Pipeline;
using JdeScoping.DataSync.Dev.Sources;
@@ -8,30 +7,11 @@ namespace JdeScoping.DataSync.Dev;
/// <summary>
/// Development ETL pipeline for the MisData table.
/// Schema from: Scripts/012_CreateMisDataTable.sql
/// </summary>
public static class MisDataDevEtl
{
public static readonly string TableName = "MisData";
public static readonly string CacheFileName = "misdata.json.zstd";
private static readonly JsonColumnSchema[] Schema =
[
new("ItemNumber", typeof(string), IsNullable: false),
new("BranchCode", typeof(string), IsNullable: false),
new("SequenceNumber", typeof(string), IsNullable: false),
new("MisNumber", typeof(string), IsNullable: false),
new("RevID", typeof(string), IsNullable: false),
new("CharNumber", typeof(string), IsNullable: false),
new("TestDescription", typeof(string), IsNullable: true),
new("SamplingType", typeof(string), IsNullable: true),
new("SamplingValue", typeof(string), IsNullable: true),
new("ToolsGauges", typeof(string), IsNullable: true),
new("WorkInstructions", typeof(string), IsNullable: true),
new("Status", typeof(string), IsNullable: false),
new("ReleaseDate", typeof(DateTime), IsNullable: true),
new("ObsoleteDate", typeof(DateTime), IsNullable: true),
];
public static readonly string CacheFileName = "misdata.pb.zstd";
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
{
@@ -42,7 +22,7 @@ public static class MisDataDevEtl
return new EtlPipelineBuilder()
.WithName($"{TableName}_Dev")
.WithSource(new JsonZstdFileSource(cacheFilePath, Schema))
.WithSource(new ProtobufZstdFileSource(cacheFilePath))
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
.Build();
}
NEW/src/JdeScoping.DataSync.Dev/OrgHierarchyDevEtl.cs
+2 -12
View File
@@ -1,6 +1,5 @@
using JdeScoping.DataAccess.Interfaces;
using JdeScoping.DataSync.Etl.Destinations;
using JdeScoping.DataSync.Dev.Models;
using JdeScoping.DataSync.Etl.Pipeline;
using JdeScoping.DataSync.Dev.Sources;
@@ -8,20 +7,11 @@ namespace JdeScoping.DataSync.Dev;
/// <summary>
/// Development ETL pipeline for the OrgHierarchy table.
/// Schema from: Scripts/010_CreateOrgHierarchyTable.sql
/// </summary>
public static class OrgHierarchyDevEtl
{
public static readonly string TableName = "OrgHierarchy";
public static readonly string CacheFileName = "orghierarchy.json.zstd";
private static readonly JsonColumnSchema[] Schema =
[
new("WorkCenterCode", typeof(string), IsNullable: false),
new("BranchCode", typeof(string), IsNullable: false),
new("ProfitCenterCode", typeof(string), IsNullable: false),
new("LastUpdateDT", typeof(DateTime), IsNullable: false),
];
public static readonly string CacheFileName = "orghierarchy.pb.zstd";
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
{
@@ -32,7 +22,7 @@ public static class OrgHierarchyDevEtl
return new EtlPipelineBuilder()
.WithName($"{TableName}_Dev")
.WithSource(new JsonZstdFileSource(cacheFilePath, Schema))
.WithSource(new ProtobufZstdFileSource(cacheFilePath))
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
.Build();
}
NEW/src/JdeScoping.DataSync.Dev/ProfitCenterDevEtl.cs
+2 -11
View File
@@ -1,6 +1,5 @@
using JdeScoping.DataAccess.Interfaces;
using JdeScoping.DataSync.Etl.Destinations;
using JdeScoping.DataSync.Dev.Models;
using JdeScoping.DataSync.Etl.Pipeline;
using JdeScoping.DataSync.Dev.Sources;
@@ -8,19 +7,11 @@ namespace JdeScoping.DataSync.Dev;
/// <summary>
/// Development ETL pipeline for the ProfitCenter table.
/// Schema from: Scripts/006_CreateProfitCenterTable.sql
/// </summary>
public static class ProfitCenterDevEtl
{
public static readonly string TableName = "ProfitCenter";
public static readonly string CacheFileName = "profitcenter.json.zstd";
private static readonly JsonColumnSchema[] Schema =
[
new("Code", typeof(string), IsNullable: false),
new("Description", typeof(string), IsNullable: true),
new("LastUpdateDT", typeof(DateTime), IsNullable: false),
];
public static readonly string CacheFileName = "profitcenter.pb.zstd";
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
{
@@ -31,7 +22,7 @@ public static class ProfitCenterDevEtl
return new EtlPipelineBuilder()
.WithName($"{TableName}_Dev")
.WithSource(new JsonZstdFileSource(cacheFilePath, Schema))
.WithSource(new ProtobufZstdFileSource(cacheFilePath))
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
.Build();
}
NEW/src/JdeScoping.DataSync.Dev/RouteMasterDevEtl.cs
+2 -17
View File
@@ -1,6 +1,5 @@
using JdeScoping.DataAccess.Interfaces;
using JdeScoping.DataSync.Etl.Destinations;
using JdeScoping.DataSync.Dev.Models;
using JdeScoping.DataSync.Etl.Pipeline;
using JdeScoping.DataSync.Dev.Sources;
@@ -8,25 +7,11 @@ namespace JdeScoping.DataSync.Dev;
/// <summary>
/// Development ETL pipeline for the RouteMaster table.
/// Schema from: Scripts/011_CreateRouteMasterTable.sql
/// </summary>
public static class RouteMasterDevEtl
{
public static readonly string TableName = "RouteMaster";
public static readonly string CacheFileName = "routemaster.json.zstd";
private static readonly JsonColumnSchema[] Schema =
[
new("BranchCode", typeof(string), IsNullable: false),
new("ItemNumber", typeof(string), IsNullable: false),
new("RoutingType", typeof(string), IsNullable: false),
new("SequenceNumber", typeof(decimal), IsNullable: false),
new("FunctionCode", typeof(string), IsNullable: true),
new("WorkCenterCode", typeof(string), IsNullable: true),
new("StartDate", typeof(DateTime), IsNullable: false),
new("EndDate", typeof(DateTime), IsNullable: true),
new("LastUpdateDT", typeof(DateTime), IsNullable: false),
];
public static readonly string CacheFileName = "routemaster.pb.zstd";
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
{
@@ -37,7 +22,7 @@ public static class RouteMasterDevEtl
return new EtlPipelineBuilder()
.WithName($"{TableName}_Dev")
.WithSource(new JsonZstdFileSource(cacheFilePath, Schema))
.WithSource(new ProtobufZstdFileSource(cacheFilePath))
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
.Build();
}
NEW/src/JdeScoping.DataSync.Dev/WorkCenterDevEtl.cs
+2 -11
View File
@@ -1,6 +1,5 @@
using JdeScoping.DataAccess.Interfaces;
using JdeScoping.DataSync.Etl.Destinations;
using JdeScoping.DataSync.Dev.Models;
using JdeScoping.DataSync.Etl.Pipeline;
using JdeScoping.DataSync.Dev.Sources;
@@ -8,19 +7,11 @@ namespace JdeScoping.DataSync.Dev;
/// <summary>
/// Development ETL pipeline for the WorkCenter table.
/// Schema from: Scripts/007_CreateWorkCenterTable.sql
/// </summary>
public static class WorkCenterDevEtl
{
public static readonly string TableName = "WorkCenter";
public static readonly string CacheFileName = "workcenter.json.zstd";
private static readonly JsonColumnSchema[] Schema =
[
new("Code", typeof(string), IsNullable: false),
new("Description", typeof(string), IsNullable: true),
new("LastUpdateDT", typeof(DateTime), IsNullable: false),
];
public static readonly string CacheFileName = "workcenter.pb.zstd";
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
{
@@ -31,7 +22,7 @@ public static class WorkCenterDevEtl
return new EtlPipelineBuilder()
.WithName($"{TableName}_Dev")
.WithSource(new JsonZstdFileSource(cacheFilePath, Schema))
.WithSource(new ProtobufZstdFileSource(cacheFilePath))
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
.Build();
}
NEW/src/JdeScoping.DataSync.Dev/WorkOrderComponentCurrDevEtl.cs
+2 -15
View File
@@ -1,6 +1,5 @@
using JdeScoping.DataAccess.Interfaces;
using JdeScoping.DataSync.Etl.Destinations;
using JdeScoping.DataSync.Dev.Models;
using JdeScoping.DataSync.Etl.Pipeline;
using JdeScoping.DataSync.Dev.Sources;
@@ -8,23 +7,11 @@ namespace JdeScoping.DataSync.Dev;
/// <summary>
/// Development ETL pipeline for the WorkOrderComponent_Curr table.
/// Schema from: Scripts/021_CreateWorkOrderComponentCurrTable.sql
/// </summary>
public static class WorkOrderComponentCurrDevEtl
{
public static readonly string TableName = "WorkOrderComponent_Curr";
public static readonly string CacheFileName = "workordercomponent_curr.json.zstd";
private static readonly JsonColumnSchema[] Schema =
[
new("UniqueID", typeof(long), IsNullable: false),
new("WorkOrderNumber", typeof(long), IsNullable: false),
new("LotNumber", typeof(string), IsNullable: false),
new("BranchCode", typeof(string), IsNullable: true),
new("ShortItemNumber", typeof(long), IsNullable: false),
new("Quantity", typeof(decimal), IsNullable: false),
new("LastUpdateDT", typeof(DateTime), IsNullable: false),
];
public static readonly string CacheFileName = "workordercomponent_curr.pb.zstd";
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
{
@@ -35,7 +22,7 @@ public static class WorkOrderComponentCurrDevEtl
return new EtlPipelineBuilder()
.WithName($"{TableName}_Dev")
.WithSource(new JsonZstdFileSource(cacheFilePath, Schema))
.WithSource(new ProtobufZstdFileSource(cacheFilePath))
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
.Build();
}
NEW/src/JdeScoping.DataSync.Dev/WorkOrderComponentHistDevEtl.cs
+2 -15
View File
@@ -1,6 +1,5 @@
using JdeScoping.DataAccess.Interfaces;
using JdeScoping.DataSync.Etl.Destinations;
using JdeScoping.DataSync.Dev.Models;
using JdeScoping.DataSync.Etl.Pipeline;
using JdeScoping.DataSync.Dev.Sources;
@@ -8,23 +7,11 @@ namespace JdeScoping.DataSync.Dev;
/// <summary>
/// Development ETL pipeline for the WorkOrderComponent_Hist table.
/// Schema from: Scripts/022_CreateWorkOrderComponentHistTable.sql
/// </summary>
public static class WorkOrderComponentHistDevEtl
{
public static readonly string TableName = "WorkOrderComponent_Hist";
public static readonly string CacheFileName = "workordercomponent_hist.json.zstd";
private static readonly JsonColumnSchema[] Schema =
[
new("UniqueID", typeof(long), IsNullable: false),
new("WorkOrderNumber", typeof(long), IsNullable: false),
new("LotNumber", typeof(string), IsNullable: false),
new("BranchCode", typeof(string), IsNullable: true),
new("ShortItemNumber", typeof(long), IsNullable: false),
new("Quantity", typeof(decimal), IsNullable: false),
new("LastUpdateDT", typeof(DateTime), IsNullable: false),
];
public static readonly string CacheFileName = "workordercomponent_hist.pb.zstd";
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
{
@@ -35,7 +22,7 @@ public static class WorkOrderComponentHistDevEtl
return new EtlPipelineBuilder()
.WithName($"{TableName}_Dev")
.WithSource(new JsonZstdFileSource(cacheFilePath, Schema))
.WithSource(new ProtobufZstdFileSource(cacheFilePath))
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
.Build();
}
NEW/src/JdeScoping.DataSync.Dev/WorkOrderCurrDevEtl.cs
+2 -23
View File
@@ -1,6 +1,5 @@
using JdeScoping.DataAccess.Interfaces;
using JdeScoping.DataSync.Etl.Destinations;
using JdeScoping.DataSync.Dev.Models;
using JdeScoping.DataSync.Etl.Pipeline;
using JdeScoping.DataSync.Dev.Sources;
@@ -8,31 +7,11 @@ namespace JdeScoping.DataSync.Dev;
/// <summary>
/// Development ETL pipeline for the WorkOrder_Curr table.
/// Schema from: Scripts/015_CreateWorkOrderCurrTable.sql
/// </summary>
public static class WorkOrderCurrDevEtl
{
public static readonly string TableName = "WorkOrder_Curr";
public static readonly string CacheFileName = "workorder_curr.json.zstd";
private static readonly JsonColumnSchema[] Schema =
[
new("WorkOrderNumber", typeof(long), IsNullable: false),
new("BranchCode", typeof(string), IsNullable: true),
new("LotNumber", typeof(string), IsNullable: true),
new("ItemNumber", typeof(string), IsNullable: true),
new("ShortItemNumber", typeof(long), IsNullable: false),
new("ParentWorkOrderNumber", typeof(string), IsNullable: true),
new("OrderQuantity", typeof(decimal), IsNullable: false),
new("HeldQuantity", typeof(decimal), IsNullable: false),
new("ShippedQuantity", typeof(decimal), IsNullable: false),
new("StatusCode", typeof(string), IsNullable: true),
new("StatusCodeUpdateDT", typeof(DateTime), IsNullable: true),
new("IssueDate", typeof(DateTime), IsNullable: false),
new("StartDate", typeof(DateTime), IsNullable: false),
new("RoutingType", typeof(string), IsNullable: true),
new("LastUpdateDT", typeof(DateTime), IsNullable: false),
];
public static readonly string CacheFileName = "workorder_curr.pb.zstd";
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
{
@@ -43,7 +22,7 @@ public static class WorkOrderCurrDevEtl
return new EtlPipelineBuilder()
.WithName($"{TableName}_Dev")
.WithSource(new JsonZstdFileSource(cacheFilePath, Schema))
.WithSource(new ProtobufZstdFileSource(cacheFilePath))
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
.Build();
}
NEW/src/JdeScoping.DataSync.Dev/WorkOrderHistDevEtl.cs
+2 -23
View File
@@ -1,6 +1,5 @@
using JdeScoping.DataAccess.Interfaces;
using JdeScoping.DataSync.Etl.Destinations;
using JdeScoping.DataSync.Dev.Models;
using JdeScoping.DataSync.Etl.Pipeline;
using JdeScoping.DataSync.Dev.Sources;
@@ -8,31 +7,11 @@ namespace JdeScoping.DataSync.Dev;
/// <summary>
/// Development ETL pipeline for the WorkOrder_Hist table.
/// Schema from: Scripts/016_CreateWorkOrderHistTable.sql
/// </summary>
public static class WorkOrderHistDevEtl
{
public static readonly string TableName = "WorkOrder_Hist";
public static readonly string CacheFileName = "workorder_hist.json.zstd";
private static readonly JsonColumnSchema[] Schema =
[
new("WorkOrderNumber", typeof(long), IsNullable: false),
new("BranchCode", typeof(string), IsNullable: true),
new("LotNumber", typeof(string), IsNullable: true),
new("ItemNumber", typeof(string), IsNullable: true),
new("ShortItemNumber", typeof(long), IsNullable: false),
new("ParentWorkOrderNumber", typeof(string), IsNullable: true),
new("OrderQuantity", typeof(decimal), IsNullable: false),
new("HeldQuantity", typeof(decimal), IsNullable: false),
new("ShippedQuantity", typeof(decimal), IsNullable: false),
new("StatusCode", typeof(string), IsNullable: true),
new("StatusCodeUpdateDT", typeof(DateTime), IsNullable: true),
new("IssueDate", typeof(DateTime), IsNullable: false),
new("StartDate", typeof(DateTime), IsNullable: false),
new("RoutingType", typeof(string), IsNullable: true),
new("LastUpdateDT", typeof(DateTime), IsNullable: false),
];
public static readonly string CacheFileName = "workorder_hist.pb.zstd";
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
{
@@ -43,7 +22,7 @@ public static class WorkOrderHistDevEtl
return new EtlPipelineBuilder()
.WithName($"{TableName}_Dev")
.WithSource(new JsonZstdFileSource(cacheFilePath, Schema))
.WithSource(new ProtobufZstdFileSource(cacheFilePath))
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
.Build();
}
NEW/src/JdeScoping.DataSync.Dev/WorkOrderRoutingDevEtl.cs
+2 -21
View File
@@ -1,6 +1,5 @@
using JdeScoping.DataAccess.Interfaces;
using JdeScoping.DataSync.Etl.Destinations;
using JdeScoping.DataSync.Dev.Models;
using JdeScoping.DataSync.Etl.Pipeline;
using JdeScoping.DataSync.Dev.Sources;
@@ -8,29 +7,11 @@ namespace JdeScoping.DataSync.Dev;
/// <summary>
/// Development ETL pipeline for the WorkOrderRouting table.
/// Schema from: Scripts/023_CreateWorkOrderRoutingTable.sql
/// </summary>
public static class WorkOrderRoutingDevEtl
{
public static readonly string TableName = "WorkOrderRouting";
public static readonly string CacheFileName = "workorderrouting.json.zstd";
private static readonly JsonColumnSchema[] Schema =
[
new("UserID", typeof(string), IsNullable: false),
new("BatchNumber", typeof(string), IsNullable: false),
new("TransactionNumber", typeof(string), IsNullable: false),
new("LineNumber", typeof(int), IsNullable: false),
new("StepNumber", typeof(decimal), IsNullable: false),
new("WorkCenterCode", typeof(string), IsNullable: false),
new("WorkOrderNumber", typeof(long), IsNullable: false),
new("RoutingType", typeof(string), IsNullable: true),
new("BranchCode", typeof(string), IsNullable: true),
new("StepDescription", typeof(string), IsNullable: true),
new("FunctionCode", typeof(string), IsNullable: true),
new("TransactionDate", typeof(DateTime), IsNullable: false),
new("LastUpdateDT", typeof(DateTime), IsNullable: false),
];
public static readonly string CacheFileName = "workorderrouting.pb.zstd";
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
{
@@ -41,7 +22,7 @@ public static class WorkOrderRoutingDevEtl
return new EtlPipelineBuilder()
.WithName($"{TableName}_Dev")
.WithSource(new JsonZstdFileSource(cacheFilePath, Schema))
.WithSource(new ProtobufZstdFileSource(cacheFilePath))
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
.Build();
}
NEW/src/JdeScoping.DataSync.Dev/WorkOrderStepCurrDevEtl.cs
+2 -19
View File
@@ -1,6 +1,5 @@
using JdeScoping.DataAccess.Interfaces;
using JdeScoping.DataSync.Etl.Destinations;
using JdeScoping.DataSync.Dev.Models;
using JdeScoping.DataSync.Etl.Pipeline;
using JdeScoping.DataSync.Dev.Sources;
@@ -8,27 +7,11 @@ namespace JdeScoping.DataSync.Dev;
/// <summary>
/// Development ETL pipeline for the WorkOrderStep_Curr table.
/// Schema from: Scripts/017_CreateWorkOrderStepCurrTable.sql
/// </summary>
public static class WorkOrderStepCurrDevEtl
{
public static readonly string TableName = "WorkOrderStep_Curr";
public static readonly string CacheFileName = "workorderstep_curr.json.zstd";
private static readonly JsonColumnSchema[] Schema =
[
new("WorkOrderNumber", typeof(long), IsNullable: false),
new("WorkCenterCode", typeof(string), IsNullable: false),
new("StepNumber", typeof(decimal), IsNullable: false),
new("StepTypeCode", typeof(string), IsNullable: false),
new("BranchCode", typeof(string), IsNullable: false),
new("StepDescription", typeof(string), IsNullable: true),
new("StartDT", typeof(DateTime), IsNullable: true),
new("EndDT", typeof(DateTime), IsNullable: true),
new("FunctionCode", typeof(string), IsNullable: true),
new("ScrappedQuantity", typeof(decimal), IsNullable: false),
new("LastUpdateDT", typeof(DateTime), IsNullable: false),
];
public static readonly string CacheFileName = "workorderstep_curr.pb.zstd";
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
{
@@ -39,7 +22,7 @@ public static class WorkOrderStepCurrDevEtl
return new EtlPipelineBuilder()
.WithName($"{TableName}_Dev")
.WithSource(new JsonZstdFileSource(cacheFilePath, Schema))
.WithSource(new ProtobufZstdFileSource(cacheFilePath))
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
.Build();
}
NEW/src/JdeScoping.DataSync.Dev/WorkOrderStepHistDevEtl.cs
+2 -19
View File
@@ -1,6 +1,5 @@
using JdeScoping.DataAccess.Interfaces;
using JdeScoping.DataSync.Etl.Destinations;
using JdeScoping.DataSync.Dev.Models;
using JdeScoping.DataSync.Etl.Pipeline;
using JdeScoping.DataSync.Dev.Sources;
@@ -8,27 +7,11 @@ namespace JdeScoping.DataSync.Dev;
/// <summary>
/// Development ETL pipeline for the WorkOrderStep_Hist table.
/// Schema from: Scripts/018_CreateWorkOrderStepHistTable.sql
/// </summary>
public static class WorkOrderStepHistDevEtl
{
public static readonly string TableName = "WorkOrderStep_Hist";
public static readonly string CacheFileName = "workorderstep_hist.json.zstd";
private static readonly JsonColumnSchema[] Schema =
[
new("WorkOrderNumber", typeof(long), IsNullable: false),
new("WorkCenterCode", typeof(string), IsNullable: false),
new("StepNumber", typeof(decimal), IsNullable: false),
new("StepTypeCode", typeof(string), IsNullable: false),
new("BranchCode", typeof(string), IsNullable: false),
new("StepDescription", typeof(string), IsNullable: true),
new("StartDT", typeof(DateTime), IsNullable: true),
new("EndDT", typeof(DateTime), IsNullable: true),
new("FunctionCode", typeof(string), IsNullable: true),
new("ScrappedQuantity", typeof(decimal), IsNullable: false),
new("LastUpdateDT", typeof(DateTime), IsNullable: false),
];
public static readonly string CacheFileName = "workorderstep_hist.pb.zstd";
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
{
@@ -39,7 +22,7 @@ public static class WorkOrderStepHistDevEtl
return new EtlPipelineBuilder()
.WithName($"{TableName}_Dev")
.WithSource(new JsonZstdFileSource(cacheFilePath, Schema))
.WithSource(new ProtobufZstdFileSource(cacheFilePath))
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
.Build();
}
NEW/src/JdeScoping.DataSync.Dev/WorkOrderTimeCurrDevEtl.cs
+2 -16
View File
@@ -1,6 +1,5 @@
using JdeScoping.DataAccess.Interfaces;
using JdeScoping.DataSync.Etl.Destinations;
using JdeScoping.DataSync.Dev.Models;
using JdeScoping.DataSync.Etl.Pipeline;
using JdeScoping.DataSync.Dev.Sources;
@@ -8,24 +7,11 @@ namespace JdeScoping.DataSync.Dev;
/// <summary>
/// Development ETL pipeline for the WorkOrderTime_Curr table.
/// Schema from: Scripts/019_CreateWorkOrderTimeCurrTable.sql
/// </summary>
public static class WorkOrderTimeCurrDevEtl
{
public static readonly string TableName = "WorkOrderTime_Curr";
public static readonly string CacheFileName = "workordertime_curr.json.zstd";
private static readonly JsonColumnSchema[] Schema =
[
new("UniqueID", typeof(long), IsNullable: false),
new("WorkOrderNumber", typeof(long), IsNullable: false),
new("StepNumber", typeof(decimal), IsNullable: false),
new("WorkCenterCode", typeof(string), IsNullable: false),
new("BranchCode", typeof(string), IsNullable: false),
new("AddressNumber", typeof(long), IsNullable: false),
new("GlDate", typeof(DateTime), IsNullable: true),
new("LastUpdateDT", typeof(DateTime), IsNullable: false),
];
public static readonly string CacheFileName = "workordertime_curr.pb.zstd";
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
{
@@ -36,7 +22,7 @@ public static class WorkOrderTimeCurrDevEtl
return new EtlPipelineBuilder()
.WithName($"{TableName}_Dev")
.WithSource(new JsonZstdFileSource(cacheFilePath, Schema))
.WithSource(new ProtobufZstdFileSource(cacheFilePath))
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
.Build();
}
NEW/src/JdeScoping.DataSync.Dev/WorkOrderTimeHistDevEtl.cs
+2 -16
View File
@@ -1,6 +1,5 @@
using JdeScoping.DataAccess.Interfaces;
using JdeScoping.DataSync.Etl.Destinations;
using JdeScoping.DataSync.Dev.Models;
using JdeScoping.DataSync.Etl.Pipeline;
using JdeScoping.DataSync.Dev.Sources;
@@ -8,24 +7,11 @@ namespace JdeScoping.DataSync.Dev;
/// <summary>
/// Development ETL pipeline for the WorkOrderTime_Hist table.
/// Schema from: Scripts/020_CreateWorkOrderTimeHistTable.sql
/// </summary>
public static class WorkOrderTimeHistDevEtl
{
public static readonly string TableName = "WorkOrderTime_Hist";
public static readonly string CacheFileName = "workordertime_hist.json.zstd";
private static readonly JsonColumnSchema[] Schema =
[
new("UniqueID", typeof(long), IsNullable: false),
new("WorkOrderNumber", typeof(long), IsNullable: false),
new("StepNumber", typeof(decimal), IsNullable: false),
new("WorkCenterCode", typeof(string), IsNullable: false),
new("BranchCode", typeof(string), IsNullable: false),
new("AddressNumber", typeof(long), IsNullable: false),
new("GlDate", typeof(DateTime), IsNullable: true),
new("LastUpdateDT", typeof(DateTime), IsNullable: false),
];
public static readonly string CacheFileName = "workordertime_hist.pb.zstd";
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
{
@@ -36,7 +22,7 @@ public static class WorkOrderTimeHistDevEtl
return new EtlPipelineBuilder()
.WithName($"{TableName}_Dev")
.WithSource(new JsonZstdFileSource(cacheFilePath, Schema))
.WithSource(new ProtobufZstdFileSource(cacheFilePath))
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
.Build();
}