refactor: migrate all DevEtl files to protobuf source
Replace JsonZstdFileSource with ProtobufZstdFileSource across all 20 DevEtl pipeline files. This removes the need for manually defined JsonColumnSchema arrays as protobuf files embed their schema. Changes per file: - Remove using JdeScoping.DataSync.Dev.Models - Remove private JsonColumnSchema[] Schema array - Change CacheFileName from .json.zstd to .pb.zstd - Change WithSource from JsonZstdFileSource to ProtobufZstdFileSource
This commit is contained in:
@@ -1,6 +1,5 @@
|
||||
using JdeScoping.DataAccess.Interfaces;
|
||||
using JdeScoping.DataSync.Etl.Destinations;
|
||||
using JdeScoping.DataSync.Dev.Models;
|
||||
using JdeScoping.DataSync.Etl.Pipeline;
|
||||
using JdeScoping.DataSync.Dev.Sources;
|
||||
|
||||
@@ -8,24 +7,11 @@ namespace JdeScoping.DataSync.Dev;
|
||||
|
||||
/// <summary>
|
||||
/// Development ETL pipeline for the WorkOrderTime_Curr table.
|
||||
/// Schema from: Scripts/019_CreateWorkOrderTimeCurrTable.sql
|
||||
/// </summary>
|
||||
public static class WorkOrderTimeCurrDevEtl
|
||||
{
|
||||
public static readonly string TableName = "WorkOrderTime_Curr";
|
||||
public static readonly string CacheFileName = "workordertime_curr.json.zstd";
|
||||
|
||||
private static readonly JsonColumnSchema[] Schema =
|
||||
[
|
||||
new("UniqueID", typeof(long), IsNullable: false),
|
||||
new("WorkOrderNumber", typeof(long), IsNullable: false),
|
||||
new("StepNumber", typeof(decimal), IsNullable: false),
|
||||
new("WorkCenterCode", typeof(string), IsNullable: false),
|
||||
new("BranchCode", typeof(string), IsNullable: false),
|
||||
new("AddressNumber", typeof(long), IsNullable: false),
|
||||
new("GlDate", typeof(DateTime), IsNullable: true),
|
||||
new("LastUpdateDT", typeof(DateTime), IsNullable: false),
|
||||
];
|
||||
public static readonly string CacheFileName = "workordertime_curr.pb.zstd";
|
||||
|
||||
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
|
||||
{
|
||||
@@ -36,7 +22,7 @@ public static class WorkOrderTimeCurrDevEtl
|
||||
|
||||
return new EtlPipelineBuilder()
|
||||
.WithName($"{TableName}_Dev")
|
||||
.WithSource(new JsonZstdFileSource(cacheFilePath, Schema))
|
||||
.WithSource(new ProtobufZstdFileSource(cacheFilePath))
|
||||
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
|
||||
.Build();
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user