feat(etl): add result models for pipeline execution
This commit is contained in:
@@ -0,0 +1,12 @@
|
|||||||
|
namespace JdeScoping.DataSync.Etl.Results;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Represents the result of writing data to a destination.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="RowsProcessed">The number of rows successfully written to the destination.</param>
|
||||||
|
/// <param name="BatchCount">The number of batches used to write the data.</param>
|
||||||
|
/// <param name="Elapsed">The total time taken to write all batches.</param>
|
||||||
|
public record DestinationResult(
|
||||||
|
long RowsProcessed,
|
||||||
|
int BatchCount,
|
||||||
|
TimeSpan Elapsed);
|
||||||
@@ -0,0 +1,38 @@
|
|||||||
|
namespace JdeScoping.DataSync.Etl.Results;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Represents the complete result of an ETL pipeline execution.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="Success">Indicates whether the pipeline completed successfully.</param>
|
||||||
|
/// <param name="TotalRows">The total number of rows processed by the pipeline.</param>
|
||||||
|
/// <param name="Elapsed">The total time taken to execute the pipeline.</param>
|
||||||
|
/// <param name="Steps">The results of each step executed in the pipeline.</param>
|
||||||
|
/// <param name="Error">The exception that caused the pipeline to fail, if any.</param>
|
||||||
|
public record PipelineResult(
|
||||||
|
bool Success,
|
||||||
|
long TotalRows,
|
||||||
|
TimeSpan Elapsed,
|
||||||
|
IReadOnlyList<StepResult> Steps,
|
||||||
|
Exception? Error = null)
|
||||||
|
{
|
||||||
|
/// <summary>
|
||||||
|
/// Creates a successful pipeline result.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="totalRows">The total number of rows processed.</param>
|
||||||
|
/// <param name="elapsed">The total execution time.</param>
|
||||||
|
/// <param name="steps">The results of each step.</param>
|
||||||
|
/// <returns>A PipelineResult indicating success.</returns>
|
||||||
|
public static PipelineResult Succeeded(long totalRows, TimeSpan elapsed, IReadOnlyList<StepResult> steps)
|
||||||
|
=> new(true, totalRows, elapsed, steps);
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Creates a failed pipeline result.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="totalRows">The number of rows processed before failure.</param>
|
||||||
|
/// <param name="elapsed">The total execution time including failure.</param>
|
||||||
|
/// <param name="steps">The results of steps executed before failure.</param>
|
||||||
|
/// <param name="error">The exception that caused the failure.</param>
|
||||||
|
/// <returns>A PipelineResult indicating failure with error details.</returns>
|
||||||
|
public static PipelineResult Failed(long totalRows, TimeSpan elapsed, IReadOnlyList<StepResult> steps, Exception error)
|
||||||
|
=> new(false, totalRows, elapsed, steps, error);
|
||||||
|
}
|
||||||
@@ -0,0 +1,14 @@
|
|||||||
|
namespace JdeScoping.DataSync.Etl.Results;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Represents the result of a single pipeline step execution.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="StepName">The name of the step that was executed.</param>
|
||||||
|
/// <param name="StepType">The type of step (e.g., "Transform", "Load").</param>
|
||||||
|
/// <param name="RowsAffected">The number of rows affected by this step.</param>
|
||||||
|
/// <param name="Elapsed">The time taken to execute this step.</param>
|
||||||
|
public record StepResult(
|
||||||
|
string StepName,
|
||||||
|
string StepType,
|
||||||
|
long RowsAffected,
|
||||||
|
TimeSpan Elapsed);
|
||||||
@@ -0,0 +1,149 @@
|
|||||||
|
using JdeScoping.DataSync.Etl.Results;
|
||||||
|
using Shouldly;
|
||||||
|
|
||||||
|
namespace JdeScoping.DataSync.Tests.Etl.Results;
|
||||||
|
|
||||||
|
public class PipelineResultTests
|
||||||
|
{
|
||||||
|
[Fact]
|
||||||
|
public void Succeeded_Creates_Result_With_Success_True()
|
||||||
|
{
|
||||||
|
// Arrange
|
||||||
|
var steps = new List<StepResult>
|
||||||
|
{
|
||||||
|
new("ExtractStep", "Extract", 100, TimeSpan.FromSeconds(1)),
|
||||||
|
new("TransformStep", "Transform", 100, TimeSpan.FromSeconds(2)),
|
||||||
|
new("LoadStep", "Load", 100, TimeSpan.FromSeconds(3))
|
||||||
|
};
|
||||||
|
var elapsed = TimeSpan.FromSeconds(6);
|
||||||
|
|
||||||
|
// Act
|
||||||
|
var result = PipelineResult.Succeeded(100, elapsed, steps);
|
||||||
|
|
||||||
|
// Assert
|
||||||
|
result.Success.ShouldBeTrue();
|
||||||
|
result.TotalRows.ShouldBe(100);
|
||||||
|
result.Elapsed.ShouldBe(elapsed);
|
||||||
|
result.Steps.ShouldBe(steps);
|
||||||
|
result.Error.ShouldBeNull();
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void Succeeded_With_Empty_Steps_Creates_Valid_Result()
|
||||||
|
{
|
||||||
|
// Arrange
|
||||||
|
var steps = Array.Empty<StepResult>();
|
||||||
|
var elapsed = TimeSpan.Zero;
|
||||||
|
|
||||||
|
// Act
|
||||||
|
var result = PipelineResult.Succeeded(0, elapsed, steps);
|
||||||
|
|
||||||
|
// Assert
|
||||||
|
result.Success.ShouldBeTrue();
|
||||||
|
result.TotalRows.ShouldBe(0);
|
||||||
|
result.Elapsed.ShouldBe(TimeSpan.Zero);
|
||||||
|
result.Steps.ShouldBeEmpty();
|
||||||
|
result.Error.ShouldBeNull();
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void Failed_Creates_Result_With_Success_False()
|
||||||
|
{
|
||||||
|
// Arrange
|
||||||
|
var steps = new List<StepResult>
|
||||||
|
{
|
||||||
|
new("ExtractStep", "Extract", 50, TimeSpan.FromSeconds(1))
|
||||||
|
};
|
||||||
|
var elapsed = TimeSpan.FromSeconds(1);
|
||||||
|
var error = new InvalidOperationException("Pipeline failed during transform");
|
||||||
|
|
||||||
|
// Act
|
||||||
|
var result = PipelineResult.Failed(50, elapsed, steps, error);
|
||||||
|
|
||||||
|
// Assert
|
||||||
|
result.Success.ShouldBeFalse();
|
||||||
|
result.TotalRows.ShouldBe(50);
|
||||||
|
result.Elapsed.ShouldBe(elapsed);
|
||||||
|
result.Steps.ShouldBe(steps);
|
||||||
|
result.Error.ShouldBe(error);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void Failed_Preserves_Error_Details()
|
||||||
|
{
|
||||||
|
// Arrange
|
||||||
|
var innerException = new TimeoutException("Database timeout");
|
||||||
|
var error = new InvalidOperationException("Pipeline failed", innerException);
|
||||||
|
var steps = Array.Empty<StepResult>();
|
||||||
|
|
||||||
|
// Act
|
||||||
|
var result = PipelineResult.Failed(0, TimeSpan.FromSeconds(30), steps, error);
|
||||||
|
|
||||||
|
// Assert
|
||||||
|
result.Error.ShouldNotBeNull();
|
||||||
|
result.Error.Message.ShouldBe("Pipeline failed");
|
||||||
|
result.Error.InnerException.ShouldBe(innerException);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void StepResult_Stores_All_Properties_Correctly()
|
||||||
|
{
|
||||||
|
// Arrange & Act
|
||||||
|
var stepResult = new StepResult("MyStep", "Transform", 500, TimeSpan.FromMilliseconds(250));
|
||||||
|
|
||||||
|
// Assert
|
||||||
|
stepResult.StepName.ShouldBe("MyStep");
|
||||||
|
stepResult.StepType.ShouldBe("Transform");
|
||||||
|
stepResult.RowsAffected.ShouldBe(500);
|
||||||
|
stepResult.Elapsed.ShouldBe(TimeSpan.FromMilliseconds(250));
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void DestinationResult_Stores_All_Properties_Correctly()
|
||||||
|
{
|
||||||
|
// Arrange & Act
|
||||||
|
var destResult = new DestinationResult(1000, 10, TimeSpan.FromSeconds(5));
|
||||||
|
|
||||||
|
// Assert
|
||||||
|
destResult.RowsProcessed.ShouldBe(1000);
|
||||||
|
destResult.BatchCount.ShouldBe(10);
|
||||||
|
destResult.Elapsed.ShouldBe(TimeSpan.FromSeconds(5));
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void PipelineResult_Records_Are_Equal_When_Properties_Match()
|
||||||
|
{
|
||||||
|
// Arrange
|
||||||
|
var steps = new List<StepResult> { new("Step1", "Extract", 100, TimeSpan.FromSeconds(1)) };
|
||||||
|
var elapsed = TimeSpan.FromSeconds(1);
|
||||||
|
|
||||||
|
// Act
|
||||||
|
var result1 = PipelineResult.Succeeded(100, elapsed, steps);
|
||||||
|
var result2 = PipelineResult.Succeeded(100, elapsed, steps);
|
||||||
|
|
||||||
|
// Assert
|
||||||
|
result1.ShouldBe(result2);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void StepResult_Records_Are_Equal_When_Properties_Match()
|
||||||
|
{
|
||||||
|
// Arrange & Act
|
||||||
|
var step1 = new StepResult("Step", "Load", 100, TimeSpan.FromSeconds(1));
|
||||||
|
var step2 = new StepResult("Step", "Load", 100, TimeSpan.FromSeconds(1));
|
||||||
|
|
||||||
|
// Assert
|
||||||
|
step1.ShouldBe(step2);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void DestinationResult_Records_Are_Equal_When_Properties_Match()
|
||||||
|
{
|
||||||
|
// Arrange & Act
|
||||||
|
var dest1 = new DestinationResult(500, 5, TimeSpan.FromSeconds(2));
|
||||||
|
var dest2 = new DestinationResult(500, 5, TimeSpan.FromSeconds(2));
|
||||||
|
|
||||||
|
// Assert
|
||||||
|
dest1.ShouldBe(dest2);
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user