refactor: migrate all DevEtl files to protobuf source

Replace JsonZstdFileSource with ProtobufZstdFileSource across all 20
DevEtl pipeline files. This removes the need for manually defined
JsonColumnSchema arrays as protobuf files embed their schema.

Changes per file:
- Remove using JdeScoping.DataSync.Dev.Models
- Remove private JsonColumnSchema[] Schema array
- Change CacheFileName from .json.zstd to .pb.zstd
- Change WithSource from JsonZstdFileSource to ProtobufZstdFileSource
This commit is contained in:
Joseph Doherty
2026-01-06 16:42:56 -05:00
parent ca63b674f5
commit ead3496cdf
20 changed files with 40 additions and 325 deletions
@@ -1,6 +1,5 @@
using JdeScoping.DataAccess.Interfaces;
using JdeScoping.DataSync.Etl.Destinations;
using JdeScoping.DataSync.Dev.Models;
using JdeScoping.DataSync.Etl.Pipeline;
using JdeScoping.DataSync.Dev.Sources;
@@ -8,19 +7,11 @@ namespace JdeScoping.DataSync.Dev;
/// <summary>
/// Development ETL pipeline for the FunctionCode table.
/// Schema from: Scripts/005_CreateFunctionCodeTable.sql
/// </summary>
public static class FunctionCodeDevEtl
{
public static readonly string TableName = "FunctionCode";
public static readonly string CacheFileName = "functioncode.json.zstd";
private static readonly JsonColumnSchema[] Schema =
[
new("Code", typeof(string), IsNullable: false),
new("Description", typeof(string), IsNullable: true),
new("LastUpdateDT", typeof(DateTime), IsNullable: false),
];
public static readonly string CacheFileName = "functioncode.pb.zstd";
public static EtlPipeline Create(IDbConnectionFactory connectionFactory, string cacheFilePath)
{
@@ -31,7 +22,7 @@ public static class FunctionCodeDevEtl
return new EtlPipelineBuilder()
.WithName($"{TableName}_Dev")
.WithSource(new JsonZstdFileSource(cacheFilePath, Schema))
.WithSource(new ProtobufZstdFileSource(cacheFilePath))
.WithDestination(new DbBulkImportDestination(connectionFactory, TableName))
.Build();
}