feat(etl): implement EtlPipeline and EtlPipelineBuilder

Add pipeline orchestration for ETL operations: - EtlPipeline: executes source -> transform -> destination flow - EtlPipelineBuilder: fluent builder for pipeline configuration - Supports pre/post scripts, multiple transformers - Returns PipelineResult with step-by-step timing
2026-01-03 09:31:32 -05:00
parent 644e884b21
commit 4c16e62661
3 changed files with 328 additions and 0 deletions
@@ -0,0 +1,145 @@
+using System.Data;
+using JdeScoping.DataSync.Etl.Contracts;
+using JdeScoping.DataSync.Etl.Pipeline;
+using JdeScoping.DataSync.Etl.Results;
+using Microsoft.Extensions.Logging.Abstractions;
+using NSubstitute;
+using NSubstitute.ExceptionExtensions;
+
+namespace JdeScoping.DataSync.Tests.Etl.Pipeline;
+
+public class EtlPipelineTests
+{
+    [Fact]
+    public async Task ExecuteAsync_SuccessfulPipeline_ReturnsSuccessResult()
+    {
+        var source = CreateMockSource();
+        var destination = CreateMockDestination(100);
+
+        var pipeline = new EtlPipelineBuilder()
+            .WithName("TestPipeline")
+            .WithSource(source)
+            .WithDestination(destination)
+            .WithLogger(NullLogger<EtlPipeline>.Instance)
+            .Build();
+
+        var result = await pipeline.ExecuteAsync();
+
+        Assert.True(result.Success);
+        Assert.Equal(100, result.TotalRows);
+        Assert.Null(result.Error);
+    }
+
+    [Fact]
+    public async Task ExecuteAsync_WithPreScript_RunsScriptBeforeDestination()
+    {
+        var callOrder = new List<string>();
+        var source = CreateMockSource();
+        var destination = CreateMockDestination(100);
+        destination.When(d => d.WriteAsync(Arg.Any<IDataReader>(), Arg.Any<CancellationToken>()))
+            .Do(_ => callOrder.Add("destination"));
+
+        var preScript = Substitute.For<IScriptRunner>();
+        preScript.ScriptName.Returns("PreScript");
+        preScript.When(s => s.ExecuteAsync(Arg.Any<CancellationToken>()))
+            .Do(_ => callOrder.Add("prescript"));
+
+        var pipeline = new EtlPipelineBuilder()
+            .WithName("TestPipeline")
+            .WithSource(source)
+            .WithDestination(destination)
+            .WithPreScript(preScript)
+            .WithLogger(NullLogger<EtlPipeline>.Instance)
+            .Build();
+
+        await pipeline.ExecuteAsync();
+
+        Assert.Equal(new[] { "prescript", "destination" }, callOrder);
+    }
+
+    [Fact]
+    public async Task ExecuteAsync_DestinationFails_ReturnsFailedResult()
+    {
+        var source = CreateMockSource();
+        var destination = Substitute.For<IImportDestination>();
+        destination.DestinationName.Returns("FailingDest");
+        destination.WriteAsync(Arg.Any<IDataReader>(), Arg.Any<CancellationToken>())
+            .ThrowsAsync(new InvalidOperationException("Destination failed"));
+
+        var pipeline = new EtlPipelineBuilder()
+            .WithName("TestPipeline")
+            .WithSource(source)
+            .WithDestination(destination)
+            .WithLogger(NullLogger<EtlPipeline>.Instance)
+            .Build();
+
+        var result = await pipeline.ExecuteAsync();
+
+        Assert.False(result.Success);
+        Assert.NotNull(result.Error);
+        Assert.IsType<InvalidOperationException>(result.Error);
+    }
+
+    [Fact]
+    public async Task ExecuteAsync_TracksStepResults()
+    {
+        var source = CreateMockSource();
+        var destination = CreateMockDestination(100);
+
+        var pipeline = new EtlPipelineBuilder()
+            .WithName("TestPipeline")
+            .WithSource(source)
+            .WithDestination(destination)
+            .WithLogger(NullLogger<EtlPipeline>.Instance)
+            .Build();
+
+        var result = await pipeline.ExecuteAsync();
+
+        Assert.Equal(2, result.Steps.Count);
+        Assert.Equal("Source", result.Steps[0].StepType);
+        Assert.Equal("Destination", result.Steps[1].StepType);
+    }
+
+    [Fact]
+    public void Build_WithoutSource_ThrowsInvalidOperationException()
+    {
+        var destination = CreateMockDestination(100);
+        var builder = new EtlPipelineBuilder()
+            .WithName("TestPipeline")
+            .WithDestination(destination);
+
+        Assert.Throws<InvalidOperationException>(() => builder.Build());
+    }
+
+    [Fact]
+    public void Build_WithoutDestination_ThrowsInvalidOperationException()
+    {
+        var source = CreateMockSource();
+        var builder = new EtlPipelineBuilder()
+            .WithName("TestPipeline")
+            .WithSource(source);
+
+        Assert.Throws<InvalidOperationException>(() => builder.Build());
+    }
+
+    private static IImportSource CreateMockSource()
+    {
+        var reader = Substitute.For<IDataReader>();
+        reader.Read().Returns(false);
+        reader.FieldCount.Returns(0);
+
+        var source = Substitute.For<IImportSource>();
+        source.SourceName.Returns("MockSource");
+        source.ReadDataAsync(Arg.Any<CancellationToken>()).Returns(Task.FromResult(reader));
+        return source;
+    }
+
+    private static IImportDestination CreateMockDestination(long rows)
+    {
+        var destination = Substitute.For<IImportDestination>();
+        destination.DestinationName.Returns("MockDestination");
+        destination.WriteAsync(Arg.Any<IDataReader>(), Arg.Any<CancellationToken>())
+            .Returns(Task.FromResult(new DestinationResult(rows, 1, TimeSpan.FromSeconds(1))));
+        return destination;
+    }
+}