refactor: migrate all DevEtl files to protobuf source
Replace JsonZstdFileSource with ProtobufZstdFileSource across all 20 DevEtl pipeline files. This removes the need for manually defined JsonColumnSchema arrays as protobuf files embed their schema. Changes per file: - Remove using JdeScoping.DataSync.Dev.Models - Remove private JsonColumnSchema[] Schema array - Change CacheFileName from .json.zstd to .pb.zstd - Change WithSource from JsonZstdFileSource to ProtobufZstdFileSource
This commit is contained in:
@@ -1,6 +1,5 @@
|
||||
using JdeScoping.DataAccess.Interfaces;
|
||||
using JdeScoping.DataSync.Etl.Destinations;
|
||||
using JdeScoping.DataSync.Dev.Models;
|
||||
using JdeScoping.DataSync.Etl.Pipeline;
|
||||
using JdeScoping.DataSync.Dev.Sources;
|
||||
|
||||
@@ -8,19 +7,11 @@ namespace JdeScoping.DataSync.Dev;
|
||||
|
||||
/// <summary>
|
||||
/// Development ETL pipeline for the FunctionCode table.
|
||||
/// Schema from: Scripts/005_CreateFunctionCodeTable.sql
|
||||
/// </summary>
|
||||
public static class FunctionCodeDevEtl
|
||||
{
|
||||
public static readonly string TableName = "FunctionCode";
|
||||
public static readonly string CacheFileName = "functioncode.json.zstd";
|
||||
|
||||
private static readonly JsonColumnSchema[] Schema =
|
||||
[
|
||||
new("Code", typeof(string), IsNullable: false),
|
||||
new("Description", typeof(string), IsNullable: true),
|
||||
new("LastUpdateDT", typeof(DateTime), IsNullable: false),
|
||||
];
|
||||
public static readonly string CacheFileName = "functioncode.pb.zstd";
|
||||
|
||||
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
|
||||
{
|
||||
@@ -31,7 +22,7 @@ public static class FunctionCodeDevEtl
|
||||
|
||||
return new EtlPipelineBuilder()
|
||||
.WithName($"{TableName}_Dev")
|
||||
.WithSource(new JsonZstdFileSource(cacheFilePath, Schema))
|
||||
.WithSource(new ProtobufZstdFileSource(cacheFilePath))
|
||||
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
|
||||
.Build();
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user