feat(etl): add result models for pipeline execution

2026-01-03 08:58:06 -05:00
parent ec4c8fab87
commit dac3d216fd
4 changed files with 213 additions and 0 deletions
@@ -0,0 +1,12 @@
+namespace JdeScoping.DataSync.Etl.Results;
+
+/// <summary>
+/// Represents the result of writing data to a destination.
+/// </summary>
+/// <param name="RowsProcessed">The number of rows successfully written to the destination.</param>
+/// <param name="BatchCount">The number of batches used to write the data.</param>
+/// <param name="Elapsed">The total time taken to write all batches.</param>
+public record DestinationResult(
+    long RowsProcessed,
+    int BatchCount,
+    TimeSpan Elapsed);
@@ -0,0 +1,38 @@
+namespace JdeScoping.DataSync.Etl.Results;
+
+/// <summary>
+/// Represents the complete result of an ETL pipeline execution.
+/// </summary>
+/// <param name="Success">Indicates whether the pipeline completed successfully.</param>
+/// <param name="TotalRows">The total number of rows processed by the pipeline.</param>
+/// <param name="Elapsed">The total time taken to execute the pipeline.</param>
+/// <param name="Steps">The results of each step executed in the pipeline.</param>
+/// <param name="Error">The exception that caused the pipeline to fail, if any.</param>
+public record PipelineResult(
+    bool Success,
+    long TotalRows,
+    TimeSpan Elapsed,
+    IReadOnlyList<StepResult> Steps,
+    Exception? Error = null)
+{
+    /// <summary>
+    /// Creates a successful pipeline result.
+    /// </summary>
+    /// <param name="totalRows">The total number of rows processed.</param>
+    /// <param name="elapsed">The total execution time.</param>
+    /// <param name="steps">The results of each step.</param>
+    /// <returns>A PipelineResult indicating success.</returns>
+    public static PipelineResult Succeeded(long totalRows, TimeSpan elapsed, IReadOnlyList<StepResult> steps)
+        => new(true, totalRows, elapsed, steps);
+
+    /// <summary>
+    /// Creates a failed pipeline result.
+    /// </summary>
+    /// <param name="totalRows">The number of rows processed before failure.</param>
+    /// <param name="elapsed">The total execution time including failure.</param>
+    /// <param name="steps">The results of steps executed before failure.</param>
+    /// <param name="error">The exception that caused the failure.</param>
+    /// <returns>A PipelineResult indicating failure with error details.</returns>
+    public static PipelineResult Failed(long totalRows, TimeSpan elapsed, IReadOnlyList<StepResult> steps, Exception error)
+        => new(false, totalRows, elapsed, steps, error);
+}
@@ -0,0 +1,14 @@
+namespace JdeScoping.DataSync.Etl.Results;
+
+/// <summary>
+/// Represents the result of a single pipeline step execution.
+/// </summary>
+/// <param name="StepName">The name of the step that was executed.</param>
+/// <param name="StepType">The type of step (e.g., "Transform", "Load").</param>
+/// <param name="RowsAffected">The number of rows affected by this step.</param>
+/// <param name="Elapsed">The time taken to execute this step.</param>
+public record StepResult(
+    string StepName,
+    string StepType,
+    long RowsAffected,
+    TimeSpan Elapsed);
@@ -0,0 +1,149 @@
+using JdeScoping.DataSync.Etl.Results;
+using Shouldly;
+
+namespace JdeScoping.DataSync.Tests.Etl.Results;
+
+public class PipelineResultTests
+{
+    [Fact]
+    public void Succeeded_Creates_Result_With_Success_True()
+    {
+        // Arrange
+        var steps = new List<StepResult>
+        {
+            new("ExtractStep", "Extract", 100, TimeSpan.FromSeconds(1)),
+            new("TransformStep", "Transform", 100, TimeSpan.FromSeconds(2)),
+            new("LoadStep", "Load", 100, TimeSpan.FromSeconds(3))
+        };
+        var elapsed = TimeSpan.FromSeconds(6);
+
+        // Act
+        var result = PipelineResult.Succeeded(100, elapsed, steps);
+
+        // Assert
+        result.Success.ShouldBeTrue();
+        result.TotalRows.ShouldBe(100);
+        result.Elapsed.ShouldBe(elapsed);
+        result.Steps.ShouldBe(steps);
+        result.Error.ShouldBeNull();
+    }
+
+    [Fact]
+    public void Succeeded_With_Empty_Steps_Creates_Valid_Result()
+    {
+        // Arrange
+        var steps = Array.Empty<StepResult>();
+        var elapsed = TimeSpan.Zero;
+
+        // Act
+        var result = PipelineResult.Succeeded(0, elapsed, steps);
+
+        // Assert
+        result.Success.ShouldBeTrue();
+        result.TotalRows.ShouldBe(0);
+        result.Elapsed.ShouldBe(TimeSpan.Zero);
+        result.Steps.ShouldBeEmpty();
+        result.Error.ShouldBeNull();
+    }
+
+    [Fact]
+    public void Failed_Creates_Result_With_Success_False()
+    {
+        // Arrange
+        var steps = new List<StepResult>
+        {
+            new("ExtractStep", "Extract", 50, TimeSpan.FromSeconds(1))
+        };
+        var elapsed = TimeSpan.FromSeconds(1);
+        var error = new InvalidOperationException("Pipeline failed during transform");
+
+        // Act
+        var result = PipelineResult.Failed(50, elapsed, steps, error);
+
+        // Assert
+        result.Success.ShouldBeFalse();
+        result.TotalRows.ShouldBe(50);
+        result.Elapsed.ShouldBe(elapsed);
+        result.Steps.ShouldBe(steps);
+        result.Error.ShouldBe(error);
+    }
+
+    [Fact]
+    public void Failed_Preserves_Error_Details()
+    {
+        // Arrange
+        var innerException = new TimeoutException("Database timeout");
+        var error = new InvalidOperationException("Pipeline failed", innerException);
+        var steps = Array.Empty<StepResult>();
+
+        // Act
+        var result = PipelineResult.Failed(0, TimeSpan.FromSeconds(30), steps, error);
+
+        // Assert
+        result.Error.ShouldNotBeNull();
+        result.Error.Message.ShouldBe("Pipeline failed");
+        result.Error.InnerException.ShouldBe(innerException);
+    }
+
+    [Fact]
+    public void StepResult_Stores_All_Properties_Correctly()
+    {
+        // Arrange & Act
+        var stepResult = new StepResult("MyStep", "Transform", 500, TimeSpan.FromMilliseconds(250));
+
+        // Assert
+        stepResult.StepName.ShouldBe("MyStep");
+        stepResult.StepType.ShouldBe("Transform");
+        stepResult.RowsAffected.ShouldBe(500);
+        stepResult.Elapsed.ShouldBe(TimeSpan.FromMilliseconds(250));
+    }
+
+    [Fact]
+    public void DestinationResult_Stores_All_Properties_Correctly()
+    {
+        // Arrange & Act
+        var destResult = new DestinationResult(1000, 10, TimeSpan.FromSeconds(5));
+
+        // Assert
+        destResult.RowsProcessed.ShouldBe(1000);
+        destResult.BatchCount.ShouldBe(10);
+        destResult.Elapsed.ShouldBe(TimeSpan.FromSeconds(5));
+    }
+
+    [Fact]
+    public void PipelineResult_Records_Are_Equal_When_Properties_Match()
+    {
+        // Arrange
+        var steps = new List<StepResult> { new("Step1", "Extract", 100, TimeSpan.FromSeconds(1)) };
+        var elapsed = TimeSpan.FromSeconds(1);
+
+        // Act
+        var result1 = PipelineResult.Succeeded(100, elapsed, steps);
+        var result2 = PipelineResult.Succeeded(100, elapsed, steps);
+
+        // Assert
+        result1.ShouldBe(result2);
+    }
+
+    [Fact]
+    public void StepResult_Records_Are_Equal_When_Properties_Match()
+    {
+        // Arrange & Act
+        var step1 = new StepResult("Step", "Load", 100, TimeSpan.FromSeconds(1));
+        var step2 = new StepResult("Step", "Load", 100, TimeSpan.FromSeconds(1));
+
+        // Assert
+        step1.ShouldBe(step2);
+    }
+
+    [Fact]
+    public void DestinationResult_Records_Are_Equal_When_Properties_Match()
+    {
+        // Arrange & Act
+        var dest1 = new DestinationResult(500, 5, TimeSpan.FromSeconds(2));
+        var dest2 = new DestinationResult(500, 5, TimeSpan.FromSeconds(2));
+
+        // Assert
+        dest1.ShouldBe(dest2);
+    }
+}