feat(etl): add core ETL pipeline interfaces
Add the four core interfaces for the ETL pipeline: - IImportSource: defines data sources for reading - IDataTransformer: defines data transformation layer - IImportDestination: defines data write destinations - IScriptRunner: defines script execution capability
This commit is contained in:
@@ -0,0 +1,21 @@
|
||||
using System.Data;
|
||||
|
||||
namespace JdeScoping.DataSync.Etl.Contracts;
|
||||
|
||||
/// <summary>
|
||||
/// Defines a transformer that can modify or enhance data during the ETL process.
|
||||
/// </summary>
|
||||
public interface IDataTransformer
|
||||
{
|
||||
/// <summary>
|
||||
/// Transforms the source data reader into a new data reader with modifications applied.
|
||||
/// </summary>
|
||||
/// <param name="source">The source data reader to transform.</param>
|
||||
/// <returns>A new IDataReader with transformations applied.</returns>
|
||||
IDataReader Transform(IDataReader source);
|
||||
|
||||
/// <summary>
|
||||
/// Gets the name of this transformer for logging and identification.
|
||||
/// </summary>
|
||||
string TransformerName { get; }
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
using System.Data;
|
||||
using JdeScoping.DataSync.Etl.Results;
|
||||
|
||||
namespace JdeScoping.DataSync.Etl.Contracts;
|
||||
|
||||
/// <summary>
|
||||
/// Defines a destination to which data can be written during import operations.
|
||||
/// </summary>
|
||||
public interface IImportDestination
|
||||
{
|
||||
/// <summary>
|
||||
/// Writes data from the source reader to the destination asynchronously.
|
||||
/// </summary>
|
||||
/// <param name="source">The data reader containing the data to write.</param>
|
||||
/// <param name="cancellationToken">Token to cancel the operation.</param>
|
||||
/// <returns>A result containing statistics about the write operation.</returns>
|
||||
Task<DestinationResult> WriteAsync(
|
||||
IDataReader source,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Gets the name of this destination for logging and identification.
|
||||
/// </summary>
|
||||
string DestinationName { get; }
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
using System.Data;
|
||||
|
||||
namespace JdeScoping.DataSync.Etl.Contracts;
|
||||
|
||||
/// <summary>
|
||||
/// Defines a source from which data can be read for import operations.
|
||||
/// </summary>
|
||||
public interface IImportSource : IAsyncDisposable
|
||||
{
|
||||
/// <summary>
|
||||
/// Reads data from the source asynchronously.
|
||||
/// </summary>
|
||||
/// <param name="cancellationToken">Token to cancel the operation.</param>
|
||||
/// <returns>An IDataReader providing access to the source data.</returns>
|
||||
Task<IDataReader> ReadDataAsync(CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Gets the name of this data source for logging and identification.
|
||||
/// </summary>
|
||||
string SourceName { get; }
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
namespace JdeScoping.DataSync.Etl.Contracts;
|
||||
|
||||
/// <summary>
|
||||
/// Defines a runner that can execute scripts as part of the ETL process.
|
||||
/// </summary>
|
||||
public interface IScriptRunner
|
||||
{
|
||||
/// <summary>
|
||||
/// Executes the script asynchronously.
|
||||
/// </summary>
|
||||
/// <param name="cancellationToken">Token to cancel the operation.</param>
|
||||
Task ExecuteAsync(CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Gets the name of this script for logging and identification.
|
||||
/// </summary>
|
||||
string ScriptName { get; }
|
||||
}
|
||||
Reference in New Issue
Block a user