Joseph Doherty
89 lines
2.9 KiB
C#
89 lines
2.9 KiB
C#
using System.Diagnostics;
|
|
using JdeScoping.DataSync.Dev;
|
|
using JdeScoping.DataSync.Dev.Options;
|
|
using JdeScoping.DataSync.Dev.Services;
|
|
using JdeScoping.DevLoader;
|
|
using Microsoft.Extensions.Logging;
|
|
using Microsoft.Extensions.Options;
|
|
using Serilog;
|
|
|
|
const string DefaultConnectionString =
|
|
"Server=localhost,1434;Database=ScopingTool;User Id=scopingapp;Password=Sc0ping@pp_Dev#2024;TrustServerCertificate=true;Encrypt=false";
|
|
|
|
// --- Parse arguments ---
|
|
string? cacheDir = null;
|
|
string connectionString = DefaultConnectionString;
|
|
|
|
for (var i = 0; i < args.Length; i++)
|
|
{
|
|
switch (args[i])
|
|
{
|
|
case "--cache-dir" when i + 1 < args.Length:
|
|
cacheDir = args[++i];
|
|
break;
|
|
case "--connection-string" when i + 1 < args.Length:
|
|
connectionString = args[++i];
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (string.IsNullOrWhiteSpace(cacheDir))
|
|
{
|
|
Console.Error.WriteLine("Usage: jdescoping-devloader --cache-dir <path> [--connection-string <cs>]");
|
|
return 1;
|
|
}
|
|
|
|
// --- Set up Serilog ---
|
|
Log.Logger = new LoggerConfiguration()
|
|
.MinimumLevel.Information()
|
|
.WriteTo.Console(outputTemplate: "[{Timestamp:HH:mm:ss} {Level:u3}] {Message:lj}{NewLine}{Exception}")
|
|
.CreateLogger();
|
|
|
|
using var loggerFactory = LoggerFactory.Create(builder =>
|
|
builder.AddSerilog(Log.Logger, dispose: false));
|
|
|
|
var logger = loggerFactory.CreateLogger<DevEtlRegistry>();
|
|
var pipelineLogger = loggerFactory.CreateLogger<JdeScoping.DataSync.Etl.Pipeline.EtlPipeline>();
|
|
|
|
// --- Build components ---
|
|
var connectionFactory = new SimpleDbConnectionFactory(connectionString);
|
|
var options = Options.Create(new DevPipelineOptions());
|
|
var pipelineFactory = new DevEtlPipelineFactory(connectionFactory, options, pipelineLogger);
|
|
var registry = new DevEtlRegistry(pipelineFactory, cacheDir, logger);
|
|
|
|
// --- Run ---
|
|
var tables = registry.GetAvailableTables().ToList();
|
|
Log.Information("Found {Count} tables to load from {Dir}", tables.Count, cacheDir);
|
|
Log.Information("Tables: {Tables}", string.Join(", ", tables));
|
|
|
|
var sw = Stopwatch.StartNew();
|
|
var results = await registry.RunAllParallelAsync(maxDegreeOfParallelism: 4);
|
|
sw.Stop();
|
|
|
|
// --- Report ---
|
|
Log.Information("=== Dev ETL Complete ({Elapsed:g}) ===", sw.Elapsed);
|
|
|
|
var succeeded = 0;
|
|
var failed = 0;
|
|
long totalRows = 0;
|
|
|
|
foreach (var r in results.OrderBy(r => r.Success ? 0 : 1))
|
|
{
|
|
var tableName = r.Steps.FirstOrDefault()?.StepName?.Replace("_Dev", "") ?? "Unknown";
|
|
if (r.Success)
|
|
{
|
|
succeeded++;
|
|
totalRows += r.TotalRows;
|
|
Log.Information(" OK {Table,-30} {Rows,10:N0} rows ({Elapsed:g})", tableName, r.TotalRows, r.Elapsed);
|
|
}
|
|
else
|
|
{
|
|
failed++;
|
|
Log.Error(" FAIL {Table,-30} {Error}", tableName, r.Error?.Message);
|
|
}
|
|
}
|
|
|
|
Log.Information("Summary: {Succeeded} succeeded, {Failed} failed, {TotalRows:N0} total rows", succeeded, failed, totalRows);
|
|
|
|
return failed > 0 ? 1 : 0;
|