diff --git a/NEW/src/JdeScoping.DataSync.Dev/FunctionCodeDevEtl.cs b/NEW/src/JdeScoping.DataSync.Dev/FunctionCodeDevEtl.cs index 9d9102c..4d1fa28 100644 --- a/NEW/src/JdeScoping.DataSync.Dev/FunctionCodeDevEtl.cs +++ b/NEW/src/JdeScoping.DataSync.Dev/FunctionCodeDevEtl.cs @@ -1,6 +1,5 @@ using JdeScoping.DataAccess.Interfaces; using JdeScoping.DataSync.Etl.Destinations; -using JdeScoping.DataSync.Dev.Models; using JdeScoping.DataSync.Etl.Pipeline; using JdeScoping.DataSync.Dev.Sources; @@ -8,19 +7,11 @@ namespace JdeScoping.DataSync.Dev; /// /// Development ETL pipeline for the FunctionCode table. -/// Schema from: Scripts/005_CreateFunctionCodeTable.sql /// public static class FunctionCodeDevEtl { public static readonly string TableName = "FunctionCode"; - public static readonly string CacheFileName = "functioncode.json.zstd"; - - private static readonly JsonColumnSchema[] Schema = - [ - new("Code", typeof(string), IsNullable: false), - new("Description", typeof(string), IsNullable: true), - new("LastUpdateDT", typeof(DateTime), IsNullable: false), - ]; + public static readonly string CacheFileName = "functioncode.pb.zstd"; public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath) { @@ -31,7 +22,7 @@ public static class FunctionCodeDevEtl return new EtlPipelineBuilder() .WithName($"{TableName}_Dev") - .WithSource(new JsonZstdFileSource(cacheFilePath, Schema)) + .WithSource(new ProtobufZstdFileSource(cacheFilePath)) .WithDestination(new DbBulkImportDestination(connectionFactory, TableName)) .Build(); } diff --git a/NEW/src/JdeScoping.DataSync.Dev/ItemDevEtl.cs b/NEW/src/JdeScoping.DataSync.Dev/ItemDevEtl.cs index 7b3b645..3d98e36 100644 --- a/NEW/src/JdeScoping.DataSync.Dev/ItemDevEtl.cs +++ b/NEW/src/JdeScoping.DataSync.Dev/ItemDevEtl.cs @@ -1,6 +1,5 @@ using JdeScoping.DataAccess.Interfaces; using JdeScoping.DataSync.Etl.Destinations; -using JdeScoping.DataSync.Dev.Models; using JdeScoping.DataSync.Etl.Pipeline; using JdeScoping.DataSync.Dev.Sources; @@ -8,22 +7,11 @@ namespace JdeScoping.DataSync.Dev; /// /// Development ETL pipeline for the Item table. -/// Schema from: Scripts/008_CreateItemTable.sql /// public static class ItemDevEtl { public static readonly string TableName = "Item"; - public static readonly string CacheFileName = "item.json.zstd"; - - private static readonly JsonColumnSchema[] Schema = - [ - new("ShortItemNumber", typeof(long), IsNullable: false), - new("ItemNumber", typeof(string), IsNullable: false), - new("Description", typeof(string), IsNullable: true), - new("PlanningFamily", typeof(string), IsNullable: true), - new("StockingType", typeof(string), IsNullable: true), - new("LastUpdateDT", typeof(DateTime), IsNullable: false), - ]; + public static readonly string CacheFileName = "item.pb.zstd"; public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath) { @@ -34,7 +22,7 @@ public static class ItemDevEtl return new EtlPipelineBuilder() .WithName($"{TableName}_Dev") - .WithSource(new JsonZstdFileSource(cacheFilePath, Schema)) + .WithSource(new ProtobufZstdFileSource(cacheFilePath)) .WithDestination(new DbBulkImportDestination(connectionFactory, TableName)) .Build(); } diff --git a/NEW/src/JdeScoping.DataSync.Dev/JdeUserDevEtl.cs b/NEW/src/JdeScoping.DataSync.Dev/JdeUserDevEtl.cs index 5759c20..3529e7d 100644 --- a/NEW/src/JdeScoping.DataSync.Dev/JdeUserDevEtl.cs +++ b/NEW/src/JdeScoping.DataSync.Dev/JdeUserDevEtl.cs @@ -1,6 +1,5 @@ using JdeScoping.DataAccess.Interfaces; using JdeScoping.DataSync.Etl.Destinations; -using JdeScoping.DataSync.Dev.Models; using JdeScoping.DataSync.Etl.Pipeline; using JdeScoping.DataSync.Dev.Sources; @@ -8,20 +7,11 @@ namespace JdeScoping.DataSync.Dev; /// /// Development ETL pipeline for the JdeUser table. -/// Schema from: Scripts/009_CreateJdeUserTable.sql /// public static class JdeUserDevEtl { public static readonly string TableName = "JdeUser"; - public static readonly string CacheFileName = "jdeuser.json.zstd"; - - private static readonly JsonColumnSchema[] Schema = - [ - new("AddressNumber", typeof(long), IsNullable: false), - new("UserID", typeof(string), IsNullable: true), - new("FullName", typeof(string), IsNullable: false), - new("LastUpdateDT", typeof(DateTime), IsNullable: false), - ]; + public static readonly string CacheFileName = "jdeuser.pb.zstd"; public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath) { @@ -32,7 +22,7 @@ public static class JdeUserDevEtl return new EtlPipelineBuilder() .WithName($"{TableName}_Dev") - .WithSource(new JsonZstdFileSource(cacheFilePath, Schema)) + .WithSource(new ProtobufZstdFileSource(cacheFilePath)) .WithDestination(new DbBulkImportDestination(connectionFactory, TableName)) .Build(); } diff --git a/NEW/src/JdeScoping.DataSync.Dev/LotDevEtl.cs b/NEW/src/JdeScoping.DataSync.Dev/LotDevEtl.cs index 52a2411..c21a31f 100644 --- a/NEW/src/JdeScoping.DataSync.Dev/LotDevEtl.cs +++ b/NEW/src/JdeScoping.DataSync.Dev/LotDevEtl.cs @@ -1,6 +1,5 @@ using JdeScoping.DataAccess.Interfaces; using JdeScoping.DataSync.Etl.Destinations; -using JdeScoping.DataSync.Dev.Models; using JdeScoping.DataSync.Etl.Pipeline; using JdeScoping.DataSync.Dev.Sources; @@ -8,26 +7,11 @@ namespace JdeScoping.DataSync.Dev; /// /// Development ETL pipeline for the Lot table. -/// Schema from: Scripts/013_CreateLotTable.sql /// public static class LotDevEtl { public static readonly string TableName = "Lot"; - public static readonly string CacheFileName = "lot.json.zstd"; - - private static readonly JsonColumnSchema[] Schema = - [ - new("LotNumber", typeof(string), IsNullable: false), - new("BranchCode", typeof(string), IsNullable: false), - new("ShortItemNumber", typeof(long), IsNullable: false), - new("ItemNumber", typeof(string), IsNullable: true), - new("SupplierCode", typeof(long), IsNullable: false), - new("StatusCode", typeof(string), IsNullable: true), - new("Memo1", typeof(string), IsNullable: true), - new("Memo2", typeof(string), IsNullable: true), - new("Memo3", typeof(string), IsNullable: true), - new("LastUpdateDT", typeof(DateTime), IsNullable: false), - ]; + public static readonly string CacheFileName = "lot.pb.zstd"; public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath) { @@ -38,7 +22,7 @@ public static class LotDevEtl return new EtlPipelineBuilder() .WithName($"{TableName}_Dev") - .WithSource(new JsonZstdFileSource(cacheFilePath, Schema)) + .WithSource(new ProtobufZstdFileSource(cacheFilePath)) .WithDestination(new DbBulkImportDestination(connectionFactory, TableName)) .Build(); } diff --git a/NEW/src/JdeScoping.DataSync.Dev/LotUsageCurrDevEtl.cs b/NEW/src/JdeScoping.DataSync.Dev/LotUsageCurrDevEtl.cs index 42cf9fe..4a8199c 100644 --- a/NEW/src/JdeScoping.DataSync.Dev/LotUsageCurrDevEtl.cs +++ b/NEW/src/JdeScoping.DataSync.Dev/LotUsageCurrDevEtl.cs @@ -1,6 +1,5 @@ using JdeScoping.DataAccess.Interfaces; using JdeScoping.DataSync.Etl.Destinations; -using JdeScoping.DataSync.Dev.Models; using JdeScoping.DataSync.Etl.Pipeline; using JdeScoping.DataSync.Dev.Sources; @@ -8,23 +7,11 @@ namespace JdeScoping.DataSync.Dev; /// /// Development ETL pipeline for the LotUsage_Curr table. -/// Schema from: Scripts/024_CreateLotUsageCurrTable.sql /// public static class LotUsageCurrDevEtl { public static readonly string TableName = "LotUsage_Curr"; - public static readonly string CacheFileName = "lotusage_curr.json.zstd"; - - private static readonly JsonColumnSchema[] Schema = - [ - new("UniqueID", typeof(long), IsNullable: false), - new("WorkOrderNumber", typeof(long), IsNullable: false), - new("LotNumber", typeof(string), IsNullable: false), - new("BranchCode", typeof(string), IsNullable: true), - new("ShortItemNumber", typeof(long), IsNullable: false), - new("Quantity", typeof(decimal), IsNullable: false), - new("LastUpdateDT", typeof(DateTime), IsNullable: false), - ]; + public static readonly string CacheFileName = "lotusage_curr.pb.zstd"; public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath) { @@ -35,7 +22,7 @@ public static class LotUsageCurrDevEtl return new EtlPipelineBuilder() .WithName($"{TableName}_Dev") - .WithSource(new JsonZstdFileSource(cacheFilePath, Schema)) + .WithSource(new ProtobufZstdFileSource(cacheFilePath)) .WithDestination(new DbBulkImportDestination(connectionFactory, TableName)) .Build(); } diff --git a/NEW/src/JdeScoping.DataSync.Dev/LotUsageHistDevEtl.cs b/NEW/src/JdeScoping.DataSync.Dev/LotUsageHistDevEtl.cs index 1a2604a..80fc338 100644 --- a/NEW/src/JdeScoping.DataSync.Dev/LotUsageHistDevEtl.cs +++ b/NEW/src/JdeScoping.DataSync.Dev/LotUsageHistDevEtl.cs @@ -1,6 +1,5 @@ using JdeScoping.DataAccess.Interfaces; using JdeScoping.DataSync.Etl.Destinations; -using JdeScoping.DataSync.Dev.Models; using JdeScoping.DataSync.Etl.Pipeline; using JdeScoping.DataSync.Dev.Sources; @@ -8,23 +7,11 @@ namespace JdeScoping.DataSync.Dev; /// /// Development ETL pipeline for the LotUsage_Hist table. -/// Schema from: Scripts/025_CreateLotUsageHistTable.sql /// public static class LotUsageHistDevEtl { public static readonly string TableName = "LotUsage_Hist"; - public static readonly string CacheFileName = "lotusage_hist.json.zstd"; - - private static readonly JsonColumnSchema[] Schema = - [ - new("UniqueID", typeof(long), IsNullable: false), - new("WorkOrderNumber", typeof(long), IsNullable: false), - new("LotNumber", typeof(string), IsNullable: false), - new("BranchCode", typeof(string), IsNullable: true), - new("ShortItemNumber", typeof(long), IsNullable: false), - new("Quantity", typeof(decimal), IsNullable: false), - new("LastUpdateDT", typeof(DateTime), IsNullable: false), - ]; + public static readonly string CacheFileName = "lotusage_hist.pb.zstd"; public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath) { @@ -35,7 +22,7 @@ public static class LotUsageHistDevEtl return new EtlPipelineBuilder() .WithName($"{TableName}_Dev") - .WithSource(new JsonZstdFileSource(cacheFilePath, Schema)) + .WithSource(new ProtobufZstdFileSource(cacheFilePath)) .WithDestination(new DbBulkImportDestination(connectionFactory, TableName)) .Build(); } diff --git a/NEW/src/JdeScoping.DataSync.Dev/MisDataDevEtl.cs b/NEW/src/JdeScoping.DataSync.Dev/MisDataDevEtl.cs index f9f238a..4bf73e6 100644 --- a/NEW/src/JdeScoping.DataSync.Dev/MisDataDevEtl.cs +++ b/NEW/src/JdeScoping.DataSync.Dev/MisDataDevEtl.cs @@ -1,6 +1,5 @@ using JdeScoping.DataAccess.Interfaces; using JdeScoping.DataSync.Etl.Destinations; -using JdeScoping.DataSync.Dev.Models; using JdeScoping.DataSync.Etl.Pipeline; using JdeScoping.DataSync.Dev.Sources; @@ -8,30 +7,11 @@ namespace JdeScoping.DataSync.Dev; /// /// Development ETL pipeline for the MisData table. -/// Schema from: Scripts/012_CreateMisDataTable.sql /// public static class MisDataDevEtl { public static readonly string TableName = "MisData"; - public static readonly string CacheFileName = "misdata.json.zstd"; - - private static readonly JsonColumnSchema[] Schema = - [ - new("ItemNumber", typeof(string), IsNullable: false), - new("BranchCode", typeof(string), IsNullable: false), - new("SequenceNumber", typeof(string), IsNullable: false), - new("MisNumber", typeof(string), IsNullable: false), - new("RevID", typeof(string), IsNullable: false), - new("CharNumber", typeof(string), IsNullable: false), - new("TestDescription", typeof(string), IsNullable: true), - new("SamplingType", typeof(string), IsNullable: true), - new("SamplingValue", typeof(string), IsNullable: true), - new("ToolsGauges", typeof(string), IsNullable: true), - new("WorkInstructions", typeof(string), IsNullable: true), - new("Status", typeof(string), IsNullable: false), - new("ReleaseDate", typeof(DateTime), IsNullable: true), - new("ObsoleteDate", typeof(DateTime), IsNullable: true), - ]; + public static readonly string CacheFileName = "misdata.pb.zstd"; public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath) { @@ -42,7 +22,7 @@ public static class MisDataDevEtl return new EtlPipelineBuilder() .WithName($"{TableName}_Dev") - .WithSource(new JsonZstdFileSource(cacheFilePath, Schema)) + .WithSource(new ProtobufZstdFileSource(cacheFilePath)) .WithDestination(new DbBulkImportDestination(connectionFactory, TableName)) .Build(); } diff --git a/NEW/src/JdeScoping.DataSync.Dev/OrgHierarchyDevEtl.cs b/NEW/src/JdeScoping.DataSync.Dev/OrgHierarchyDevEtl.cs index 45bf0a7..a3f8aad 100644 --- a/NEW/src/JdeScoping.DataSync.Dev/OrgHierarchyDevEtl.cs +++ b/NEW/src/JdeScoping.DataSync.Dev/OrgHierarchyDevEtl.cs @@ -1,6 +1,5 @@ using JdeScoping.DataAccess.Interfaces; using JdeScoping.DataSync.Etl.Destinations; -using JdeScoping.DataSync.Dev.Models; using JdeScoping.DataSync.Etl.Pipeline; using JdeScoping.DataSync.Dev.Sources; @@ -8,20 +7,11 @@ namespace JdeScoping.DataSync.Dev; /// /// Development ETL pipeline for the OrgHierarchy table. -/// Schema from: Scripts/010_CreateOrgHierarchyTable.sql /// public static class OrgHierarchyDevEtl { public static readonly string TableName = "OrgHierarchy"; - public static readonly string CacheFileName = "orghierarchy.json.zstd"; - - private static readonly JsonColumnSchema[] Schema = - [ - new("WorkCenterCode", typeof(string), IsNullable: false), - new("BranchCode", typeof(string), IsNullable: false), - new("ProfitCenterCode", typeof(string), IsNullable: false), - new("LastUpdateDT", typeof(DateTime), IsNullable: false), - ]; + public static readonly string CacheFileName = "orghierarchy.pb.zstd"; public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath) { @@ -32,7 +22,7 @@ public static class OrgHierarchyDevEtl return new EtlPipelineBuilder() .WithName($"{TableName}_Dev") - .WithSource(new JsonZstdFileSource(cacheFilePath, Schema)) + .WithSource(new ProtobufZstdFileSource(cacheFilePath)) .WithDestination(new DbBulkImportDestination(connectionFactory, TableName)) .Build(); } diff --git a/NEW/src/JdeScoping.DataSync.Dev/ProfitCenterDevEtl.cs b/NEW/src/JdeScoping.DataSync.Dev/ProfitCenterDevEtl.cs index d3ac3e1..e11e55a 100644 --- a/NEW/src/JdeScoping.DataSync.Dev/ProfitCenterDevEtl.cs +++ b/NEW/src/JdeScoping.DataSync.Dev/ProfitCenterDevEtl.cs @@ -1,6 +1,5 @@ using JdeScoping.DataAccess.Interfaces; using JdeScoping.DataSync.Etl.Destinations; -using JdeScoping.DataSync.Dev.Models; using JdeScoping.DataSync.Etl.Pipeline; using JdeScoping.DataSync.Dev.Sources; @@ -8,19 +7,11 @@ namespace JdeScoping.DataSync.Dev; /// /// Development ETL pipeline for the ProfitCenter table. -/// Schema from: Scripts/006_CreateProfitCenterTable.sql /// public static class ProfitCenterDevEtl { public static readonly string TableName = "ProfitCenter"; - public static readonly string CacheFileName = "profitcenter.json.zstd"; - - private static readonly JsonColumnSchema[] Schema = - [ - new("Code", typeof(string), IsNullable: false), - new("Description", typeof(string), IsNullable: true), - new("LastUpdateDT", typeof(DateTime), IsNullable: false), - ]; + public static readonly string CacheFileName = "profitcenter.pb.zstd"; public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath) { @@ -31,7 +22,7 @@ public static class ProfitCenterDevEtl return new EtlPipelineBuilder() .WithName($"{TableName}_Dev") - .WithSource(new JsonZstdFileSource(cacheFilePath, Schema)) + .WithSource(new ProtobufZstdFileSource(cacheFilePath)) .WithDestination(new DbBulkImportDestination(connectionFactory, TableName)) .Build(); } diff --git a/NEW/src/JdeScoping.DataSync.Dev/RouteMasterDevEtl.cs b/NEW/src/JdeScoping.DataSync.Dev/RouteMasterDevEtl.cs index 717b1ad..1fe8651 100644 --- a/NEW/src/JdeScoping.DataSync.Dev/RouteMasterDevEtl.cs +++ b/NEW/src/JdeScoping.DataSync.Dev/RouteMasterDevEtl.cs @@ -1,6 +1,5 @@ using JdeScoping.DataAccess.Interfaces; using JdeScoping.DataSync.Etl.Destinations; -using JdeScoping.DataSync.Dev.Models; using JdeScoping.DataSync.Etl.Pipeline; using JdeScoping.DataSync.Dev.Sources; @@ -8,25 +7,11 @@ namespace JdeScoping.DataSync.Dev; /// /// Development ETL pipeline for the RouteMaster table. -/// Schema from: Scripts/011_CreateRouteMasterTable.sql /// public static class RouteMasterDevEtl { public static readonly string TableName = "RouteMaster"; - public static readonly string CacheFileName = "routemaster.json.zstd"; - - private static readonly JsonColumnSchema[] Schema = - [ - new("BranchCode", typeof(string), IsNullable: false), - new("ItemNumber", typeof(string), IsNullable: false), - new("RoutingType", typeof(string), IsNullable: false), - new("SequenceNumber", typeof(decimal), IsNullable: false), - new("FunctionCode", typeof(string), IsNullable: true), - new("WorkCenterCode", typeof(string), IsNullable: true), - new("StartDate", typeof(DateTime), IsNullable: false), - new("EndDate", typeof(DateTime), IsNullable: true), - new("LastUpdateDT", typeof(DateTime), IsNullable: false), - ]; + public static readonly string CacheFileName = "routemaster.pb.zstd"; public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath) { @@ -37,7 +22,7 @@ public static class RouteMasterDevEtl return new EtlPipelineBuilder() .WithName($"{TableName}_Dev") - .WithSource(new JsonZstdFileSource(cacheFilePath, Schema)) + .WithSource(new ProtobufZstdFileSource(cacheFilePath)) .WithDestination(new DbBulkImportDestination(connectionFactory, TableName)) .Build(); } diff --git a/NEW/src/JdeScoping.DataSync.Dev/WorkCenterDevEtl.cs b/NEW/src/JdeScoping.DataSync.Dev/WorkCenterDevEtl.cs index c0c9490..501ca28 100644 --- a/NEW/src/JdeScoping.DataSync.Dev/WorkCenterDevEtl.cs +++ b/NEW/src/JdeScoping.DataSync.Dev/WorkCenterDevEtl.cs @@ -1,6 +1,5 @@ using JdeScoping.DataAccess.Interfaces; using JdeScoping.DataSync.Etl.Destinations; -using JdeScoping.DataSync.Dev.Models; using JdeScoping.DataSync.Etl.Pipeline; using JdeScoping.DataSync.Dev.Sources; @@ -8,19 +7,11 @@ namespace JdeScoping.DataSync.Dev; /// /// Development ETL pipeline for the WorkCenter table. -/// Schema from: Scripts/007_CreateWorkCenterTable.sql /// public static class WorkCenterDevEtl { public static readonly string TableName = "WorkCenter"; - public static readonly string CacheFileName = "workcenter.json.zstd"; - - private static readonly JsonColumnSchema[] Schema = - [ - new("Code", typeof(string), IsNullable: false), - new("Description", typeof(string), IsNullable: true), - new("LastUpdateDT", typeof(DateTime), IsNullable: false), - ]; + public static readonly string CacheFileName = "workcenter.pb.zstd"; public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath) { @@ -31,7 +22,7 @@ public static class WorkCenterDevEtl return new EtlPipelineBuilder() .WithName($"{TableName}_Dev") - .WithSource(new JsonZstdFileSource(cacheFilePath, Schema)) + .WithSource(new ProtobufZstdFileSource(cacheFilePath)) .WithDestination(new DbBulkImportDestination(connectionFactory, TableName)) .Build(); } diff --git a/NEW/src/JdeScoping.DataSync.Dev/WorkOrderComponentCurrDevEtl.cs b/NEW/src/JdeScoping.DataSync.Dev/WorkOrderComponentCurrDevEtl.cs index a24303f..0868249 100644 --- a/NEW/src/JdeScoping.DataSync.Dev/WorkOrderComponentCurrDevEtl.cs +++ b/NEW/src/JdeScoping.DataSync.Dev/WorkOrderComponentCurrDevEtl.cs @@ -1,6 +1,5 @@ using JdeScoping.DataAccess.Interfaces; using JdeScoping.DataSync.Etl.Destinations; -using JdeScoping.DataSync.Dev.Models; using JdeScoping.DataSync.Etl.Pipeline; using JdeScoping.DataSync.Dev.Sources; @@ -8,23 +7,11 @@ namespace JdeScoping.DataSync.Dev; /// /// Development ETL pipeline for the WorkOrderComponent_Curr table. -/// Schema from: Scripts/021_CreateWorkOrderComponentCurrTable.sql /// public static class WorkOrderComponentCurrDevEtl { public static readonly string TableName = "WorkOrderComponent_Curr"; - public static readonly string CacheFileName = "workordercomponent_curr.json.zstd"; - - private static readonly JsonColumnSchema[] Schema = - [ - new("UniqueID", typeof(long), IsNullable: false), - new("WorkOrderNumber", typeof(long), IsNullable: false), - new("LotNumber", typeof(string), IsNullable: false), - new("BranchCode", typeof(string), IsNullable: true), - new("ShortItemNumber", typeof(long), IsNullable: false), - new("Quantity", typeof(decimal), IsNullable: false), - new("LastUpdateDT", typeof(DateTime), IsNullable: false), - ]; + public static readonly string CacheFileName = "workordercomponent_curr.pb.zstd"; public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath) { @@ -35,7 +22,7 @@ public static class WorkOrderComponentCurrDevEtl return new EtlPipelineBuilder() .WithName($"{TableName}_Dev") - .WithSource(new JsonZstdFileSource(cacheFilePath, Schema)) + .WithSource(new ProtobufZstdFileSource(cacheFilePath)) .WithDestination(new DbBulkImportDestination(connectionFactory, TableName)) .Build(); } diff --git a/NEW/src/JdeScoping.DataSync.Dev/WorkOrderComponentHistDevEtl.cs b/NEW/src/JdeScoping.DataSync.Dev/WorkOrderComponentHistDevEtl.cs index 365f2a8..ca0888b 100644 --- a/NEW/src/JdeScoping.DataSync.Dev/WorkOrderComponentHistDevEtl.cs +++ b/NEW/src/JdeScoping.DataSync.Dev/WorkOrderComponentHistDevEtl.cs @@ -1,6 +1,5 @@ using JdeScoping.DataAccess.Interfaces; using JdeScoping.DataSync.Etl.Destinations; -using JdeScoping.DataSync.Dev.Models; using JdeScoping.DataSync.Etl.Pipeline; using JdeScoping.DataSync.Dev.Sources; @@ -8,23 +7,11 @@ namespace JdeScoping.DataSync.Dev; /// /// Development ETL pipeline for the WorkOrderComponent_Hist table. -/// Schema from: Scripts/022_CreateWorkOrderComponentHistTable.sql /// public static class WorkOrderComponentHistDevEtl { public static readonly string TableName = "WorkOrderComponent_Hist"; - public static readonly string CacheFileName = "workordercomponent_hist.json.zstd"; - - private static readonly JsonColumnSchema[] Schema = - [ - new("UniqueID", typeof(long), IsNullable: false), - new("WorkOrderNumber", typeof(long), IsNullable: false), - new("LotNumber", typeof(string), IsNullable: false), - new("BranchCode", typeof(string), IsNullable: true), - new("ShortItemNumber", typeof(long), IsNullable: false), - new("Quantity", typeof(decimal), IsNullable: false), - new("LastUpdateDT", typeof(DateTime), IsNullable: false), - ]; + public static readonly string CacheFileName = "workordercomponent_hist.pb.zstd"; public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath) { @@ -35,7 +22,7 @@ public static class WorkOrderComponentHistDevEtl return new EtlPipelineBuilder() .WithName($"{TableName}_Dev") - .WithSource(new JsonZstdFileSource(cacheFilePath, Schema)) + .WithSource(new ProtobufZstdFileSource(cacheFilePath)) .WithDestination(new DbBulkImportDestination(connectionFactory, TableName)) .Build(); } diff --git a/NEW/src/JdeScoping.DataSync.Dev/WorkOrderCurrDevEtl.cs b/NEW/src/JdeScoping.DataSync.Dev/WorkOrderCurrDevEtl.cs index c786cb4..839b6b6 100644 --- a/NEW/src/JdeScoping.DataSync.Dev/WorkOrderCurrDevEtl.cs +++ b/NEW/src/JdeScoping.DataSync.Dev/WorkOrderCurrDevEtl.cs @@ -1,6 +1,5 @@ using JdeScoping.DataAccess.Interfaces; using JdeScoping.DataSync.Etl.Destinations; -using JdeScoping.DataSync.Dev.Models; using JdeScoping.DataSync.Etl.Pipeline; using JdeScoping.DataSync.Dev.Sources; @@ -8,31 +7,11 @@ namespace JdeScoping.DataSync.Dev; /// /// Development ETL pipeline for the WorkOrder_Curr table. -/// Schema from: Scripts/015_CreateWorkOrderCurrTable.sql /// public static class WorkOrderCurrDevEtl { public static readonly string TableName = "WorkOrder_Curr"; - public static readonly string CacheFileName = "workorder_curr.json.zstd"; - - private static readonly JsonColumnSchema[] Schema = - [ - new("WorkOrderNumber", typeof(long), IsNullable: false), - new("BranchCode", typeof(string), IsNullable: true), - new("LotNumber", typeof(string), IsNullable: true), - new("ItemNumber", typeof(string), IsNullable: true), - new("ShortItemNumber", typeof(long), IsNullable: false), - new("ParentWorkOrderNumber", typeof(string), IsNullable: true), - new("OrderQuantity", typeof(decimal), IsNullable: false), - new("HeldQuantity", typeof(decimal), IsNullable: false), - new("ShippedQuantity", typeof(decimal), IsNullable: false), - new("StatusCode", typeof(string), IsNullable: true), - new("StatusCodeUpdateDT", typeof(DateTime), IsNullable: true), - new("IssueDate", typeof(DateTime), IsNullable: false), - new("StartDate", typeof(DateTime), IsNullable: false), - new("RoutingType", typeof(string), IsNullable: true), - new("LastUpdateDT", typeof(DateTime), IsNullable: false), - ]; + public static readonly string CacheFileName = "workorder_curr.pb.zstd"; public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath) { @@ -43,7 +22,7 @@ public static class WorkOrderCurrDevEtl return new EtlPipelineBuilder() .WithName($"{TableName}_Dev") - .WithSource(new JsonZstdFileSource(cacheFilePath, Schema)) + .WithSource(new ProtobufZstdFileSource(cacheFilePath)) .WithDestination(new DbBulkImportDestination(connectionFactory, TableName)) .Build(); } diff --git a/NEW/src/JdeScoping.DataSync.Dev/WorkOrderHistDevEtl.cs b/NEW/src/JdeScoping.DataSync.Dev/WorkOrderHistDevEtl.cs index 272d6c8..8c5d146 100644 --- a/NEW/src/JdeScoping.DataSync.Dev/WorkOrderHistDevEtl.cs +++ b/NEW/src/JdeScoping.DataSync.Dev/WorkOrderHistDevEtl.cs @@ -1,6 +1,5 @@ using JdeScoping.DataAccess.Interfaces; using JdeScoping.DataSync.Etl.Destinations; -using JdeScoping.DataSync.Dev.Models; using JdeScoping.DataSync.Etl.Pipeline; using JdeScoping.DataSync.Dev.Sources; @@ -8,31 +7,11 @@ namespace JdeScoping.DataSync.Dev; /// /// Development ETL pipeline for the WorkOrder_Hist table. -/// Schema from: Scripts/016_CreateWorkOrderHistTable.sql /// public static class WorkOrderHistDevEtl { public static readonly string TableName = "WorkOrder_Hist"; - public static readonly string CacheFileName = "workorder_hist.json.zstd"; - - private static readonly JsonColumnSchema[] Schema = - [ - new("WorkOrderNumber", typeof(long), IsNullable: false), - new("BranchCode", typeof(string), IsNullable: true), - new("LotNumber", typeof(string), IsNullable: true), - new("ItemNumber", typeof(string), IsNullable: true), - new("ShortItemNumber", typeof(long), IsNullable: false), - new("ParentWorkOrderNumber", typeof(string), IsNullable: true), - new("OrderQuantity", typeof(decimal), IsNullable: false), - new("HeldQuantity", typeof(decimal), IsNullable: false), - new("ShippedQuantity", typeof(decimal), IsNullable: false), - new("StatusCode", typeof(string), IsNullable: true), - new("StatusCodeUpdateDT", typeof(DateTime), IsNullable: true), - new("IssueDate", typeof(DateTime), IsNullable: false), - new("StartDate", typeof(DateTime), IsNullable: false), - new("RoutingType", typeof(string), IsNullable: true), - new("LastUpdateDT", typeof(DateTime), IsNullable: false), - ]; + public static readonly string CacheFileName = "workorder_hist.pb.zstd"; public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath) { @@ -43,7 +22,7 @@ public static class WorkOrderHistDevEtl return new EtlPipelineBuilder() .WithName($"{TableName}_Dev") - .WithSource(new JsonZstdFileSource(cacheFilePath, Schema)) + .WithSource(new ProtobufZstdFileSource(cacheFilePath)) .WithDestination(new DbBulkImportDestination(connectionFactory, TableName)) .Build(); } diff --git a/NEW/src/JdeScoping.DataSync.Dev/WorkOrderRoutingDevEtl.cs b/NEW/src/JdeScoping.DataSync.Dev/WorkOrderRoutingDevEtl.cs index b9019d6..6b1c5f1 100644 --- a/NEW/src/JdeScoping.DataSync.Dev/WorkOrderRoutingDevEtl.cs +++ b/NEW/src/JdeScoping.DataSync.Dev/WorkOrderRoutingDevEtl.cs @@ -1,6 +1,5 @@ using JdeScoping.DataAccess.Interfaces; using JdeScoping.DataSync.Etl.Destinations; -using JdeScoping.DataSync.Dev.Models; using JdeScoping.DataSync.Etl.Pipeline; using JdeScoping.DataSync.Dev.Sources; @@ -8,29 +7,11 @@ namespace JdeScoping.DataSync.Dev; /// /// Development ETL pipeline for the WorkOrderRouting table. -/// Schema from: Scripts/023_CreateWorkOrderRoutingTable.sql /// public static class WorkOrderRoutingDevEtl { public static readonly string TableName = "WorkOrderRouting"; - public static readonly string CacheFileName = "workorderrouting.json.zstd"; - - private static readonly JsonColumnSchema[] Schema = - [ - new("UserID", typeof(string), IsNullable: false), - new("BatchNumber", typeof(string), IsNullable: false), - new("TransactionNumber", typeof(string), IsNullable: false), - new("LineNumber", typeof(int), IsNullable: false), - new("StepNumber", typeof(decimal), IsNullable: false), - new("WorkCenterCode", typeof(string), IsNullable: false), - new("WorkOrderNumber", typeof(long), IsNullable: false), - new("RoutingType", typeof(string), IsNullable: true), - new("BranchCode", typeof(string), IsNullable: true), - new("StepDescription", typeof(string), IsNullable: true), - new("FunctionCode", typeof(string), IsNullable: true), - new("TransactionDate", typeof(DateTime), IsNullable: false), - new("LastUpdateDT", typeof(DateTime), IsNullable: false), - ]; + public static readonly string CacheFileName = "workorderrouting.pb.zstd"; public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath) { @@ -41,7 +22,7 @@ public static class WorkOrderRoutingDevEtl return new EtlPipelineBuilder() .WithName($"{TableName}_Dev") - .WithSource(new JsonZstdFileSource(cacheFilePath, Schema)) + .WithSource(new ProtobufZstdFileSource(cacheFilePath)) .WithDestination(new DbBulkImportDestination(connectionFactory, TableName)) .Build(); } diff --git a/NEW/src/JdeScoping.DataSync.Dev/WorkOrderStepCurrDevEtl.cs b/NEW/src/JdeScoping.DataSync.Dev/WorkOrderStepCurrDevEtl.cs index 67d569b..5bebbb5 100644 --- a/NEW/src/JdeScoping.DataSync.Dev/WorkOrderStepCurrDevEtl.cs +++ b/NEW/src/JdeScoping.DataSync.Dev/WorkOrderStepCurrDevEtl.cs @@ -1,6 +1,5 @@ using JdeScoping.DataAccess.Interfaces; using JdeScoping.DataSync.Etl.Destinations; -using JdeScoping.DataSync.Dev.Models; using JdeScoping.DataSync.Etl.Pipeline; using JdeScoping.DataSync.Dev.Sources; @@ -8,27 +7,11 @@ namespace JdeScoping.DataSync.Dev; /// /// Development ETL pipeline for the WorkOrderStep_Curr table. -/// Schema from: Scripts/017_CreateWorkOrderStepCurrTable.sql /// public static class WorkOrderStepCurrDevEtl { public static readonly string TableName = "WorkOrderStep_Curr"; - public static readonly string CacheFileName = "workorderstep_curr.json.zstd"; - - private static readonly JsonColumnSchema[] Schema = - [ - new("WorkOrderNumber", typeof(long), IsNullable: false), - new("WorkCenterCode", typeof(string), IsNullable: false), - new("StepNumber", typeof(decimal), IsNullable: false), - new("StepTypeCode", typeof(string), IsNullable: false), - new("BranchCode", typeof(string), IsNullable: false), - new("StepDescription", typeof(string), IsNullable: true), - new("StartDT", typeof(DateTime), IsNullable: true), - new("EndDT", typeof(DateTime), IsNullable: true), - new("FunctionCode", typeof(string), IsNullable: true), - new("ScrappedQuantity", typeof(decimal), IsNullable: false), - new("LastUpdateDT", typeof(DateTime), IsNullable: false), - ]; + public static readonly string CacheFileName = "workorderstep_curr.pb.zstd"; public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath) { @@ -39,7 +22,7 @@ public static class WorkOrderStepCurrDevEtl return new EtlPipelineBuilder() .WithName($"{TableName}_Dev") - .WithSource(new JsonZstdFileSource(cacheFilePath, Schema)) + .WithSource(new ProtobufZstdFileSource(cacheFilePath)) .WithDestination(new DbBulkImportDestination(connectionFactory, TableName)) .Build(); } diff --git a/NEW/src/JdeScoping.DataSync.Dev/WorkOrderStepHistDevEtl.cs b/NEW/src/JdeScoping.DataSync.Dev/WorkOrderStepHistDevEtl.cs index 387e205..92ebd87 100644 --- a/NEW/src/JdeScoping.DataSync.Dev/WorkOrderStepHistDevEtl.cs +++ b/NEW/src/JdeScoping.DataSync.Dev/WorkOrderStepHistDevEtl.cs @@ -1,6 +1,5 @@ using JdeScoping.DataAccess.Interfaces; using JdeScoping.DataSync.Etl.Destinations; -using JdeScoping.DataSync.Dev.Models; using JdeScoping.DataSync.Etl.Pipeline; using JdeScoping.DataSync.Dev.Sources; @@ -8,27 +7,11 @@ namespace JdeScoping.DataSync.Dev; /// /// Development ETL pipeline for the WorkOrderStep_Hist table. -/// Schema from: Scripts/018_CreateWorkOrderStepHistTable.sql /// public static class WorkOrderStepHistDevEtl { public static readonly string TableName = "WorkOrderStep_Hist"; - public static readonly string CacheFileName = "workorderstep_hist.json.zstd"; - - private static readonly JsonColumnSchema[] Schema = - [ - new("WorkOrderNumber", typeof(long), IsNullable: false), - new("WorkCenterCode", typeof(string), IsNullable: false), - new("StepNumber", typeof(decimal), IsNullable: false), - new("StepTypeCode", typeof(string), IsNullable: false), - new("BranchCode", typeof(string), IsNullable: false), - new("StepDescription", typeof(string), IsNullable: true), - new("StartDT", typeof(DateTime), IsNullable: true), - new("EndDT", typeof(DateTime), IsNullable: true), - new("FunctionCode", typeof(string), IsNullable: true), - new("ScrappedQuantity", typeof(decimal), IsNullable: false), - new("LastUpdateDT", typeof(DateTime), IsNullable: false), - ]; + public static readonly string CacheFileName = "workorderstep_hist.pb.zstd"; public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath) { @@ -39,7 +22,7 @@ public static class WorkOrderStepHistDevEtl return new EtlPipelineBuilder() .WithName($"{TableName}_Dev") - .WithSource(new JsonZstdFileSource(cacheFilePath, Schema)) + .WithSource(new ProtobufZstdFileSource(cacheFilePath)) .WithDestination(new DbBulkImportDestination(connectionFactory, TableName)) .Build(); } diff --git a/NEW/src/JdeScoping.DataSync.Dev/WorkOrderTimeCurrDevEtl.cs b/NEW/src/JdeScoping.DataSync.Dev/WorkOrderTimeCurrDevEtl.cs index 7d4088e..ae2d735 100644 --- a/NEW/src/JdeScoping.DataSync.Dev/WorkOrderTimeCurrDevEtl.cs +++ b/NEW/src/JdeScoping.DataSync.Dev/WorkOrderTimeCurrDevEtl.cs @@ -1,6 +1,5 @@ using JdeScoping.DataAccess.Interfaces; using JdeScoping.DataSync.Etl.Destinations; -using JdeScoping.DataSync.Dev.Models; using JdeScoping.DataSync.Etl.Pipeline; using JdeScoping.DataSync.Dev.Sources; @@ -8,24 +7,11 @@ namespace JdeScoping.DataSync.Dev; /// /// Development ETL pipeline for the WorkOrderTime_Curr table. -/// Schema from: Scripts/019_CreateWorkOrderTimeCurrTable.sql /// public static class WorkOrderTimeCurrDevEtl { public static readonly string TableName = "WorkOrderTime_Curr"; - public static readonly string CacheFileName = "workordertime_curr.json.zstd"; - - private static readonly JsonColumnSchema[] Schema = - [ - new("UniqueID", typeof(long), IsNullable: false), - new("WorkOrderNumber", typeof(long), IsNullable: false), - new("StepNumber", typeof(decimal), IsNullable: false), - new("WorkCenterCode", typeof(string), IsNullable: false), - new("BranchCode", typeof(string), IsNullable: false), - new("AddressNumber", typeof(long), IsNullable: false), - new("GlDate", typeof(DateTime), IsNullable: true), - new("LastUpdateDT", typeof(DateTime), IsNullable: false), - ]; + public static readonly string CacheFileName = "workordertime_curr.pb.zstd"; public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath) { @@ -36,7 +22,7 @@ public static class WorkOrderTimeCurrDevEtl return new EtlPipelineBuilder() .WithName($"{TableName}_Dev") - .WithSource(new JsonZstdFileSource(cacheFilePath, Schema)) + .WithSource(new ProtobufZstdFileSource(cacheFilePath)) .WithDestination(new DbBulkImportDestination(connectionFactory, TableName)) .Build(); } diff --git a/NEW/src/JdeScoping.DataSync.Dev/WorkOrderTimeHistDevEtl.cs b/NEW/src/JdeScoping.DataSync.Dev/WorkOrderTimeHistDevEtl.cs index 7d987cb..3c290d5 100644 --- a/NEW/src/JdeScoping.DataSync.Dev/WorkOrderTimeHistDevEtl.cs +++ b/NEW/src/JdeScoping.DataSync.Dev/WorkOrderTimeHistDevEtl.cs @@ -1,6 +1,5 @@ using JdeScoping.DataAccess.Interfaces; using JdeScoping.DataSync.Etl.Destinations; -using JdeScoping.DataSync.Dev.Models; using JdeScoping.DataSync.Etl.Pipeline; using JdeScoping.DataSync.Dev.Sources; @@ -8,24 +7,11 @@ namespace JdeScoping.DataSync.Dev; /// /// Development ETL pipeline for the WorkOrderTime_Hist table. -/// Schema from: Scripts/020_CreateWorkOrderTimeHistTable.sql /// public static class WorkOrderTimeHistDevEtl { public static readonly string TableName = "WorkOrderTime_Hist"; - public static readonly string CacheFileName = "workordertime_hist.json.zstd"; - - private static readonly JsonColumnSchema[] Schema = - [ - new("UniqueID", typeof(long), IsNullable: false), - new("WorkOrderNumber", typeof(long), IsNullable: false), - new("StepNumber", typeof(decimal), IsNullable: false), - new("WorkCenterCode", typeof(string), IsNullable: false), - new("BranchCode", typeof(string), IsNullable: false), - new("AddressNumber", typeof(long), IsNullable: false), - new("GlDate", typeof(DateTime), IsNullable: true), - new("LastUpdateDT", typeof(DateTime), IsNullable: false), - ]; + public static readonly string CacheFileName = "workordertime_hist.pb.zstd"; public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath) { @@ -36,7 +22,7 @@ public static class WorkOrderTimeHistDevEtl return new EtlPipelineBuilder() .WithName($"{TableName}_Dev") - .WithSource(new JsonZstdFileSource(cacheFilePath, Schema)) + .WithSource(new ProtobufZstdFileSource(cacheFilePath)) .WithDestination(new DbBulkImportDestination(connectionFactory, TableName)) .Build(); }