docs: add XML documentation and ConfigManager implementation plans
Add comprehensive XML documentation (param/returns tags) across 132 source files to improve IntelliSense and API discoverability. Include ConfigManager design documents and implementation plans for phases 1-9.
This commit is contained in:
@@ -5,7 +5,9 @@ public record PipelinesRoot(
|
||||
ScheduleDefaults? ScheduleDefaults, // Optional - defaults applied if missing
|
||||
Dictionary<string, PipelineConfig> Pipelines)
|
||||
{
|
||||
/// <summary>Gets the effective pipeline settings, using defaults if not specified.</summary>
|
||||
public PipelineSettings EffectiveSettings => Settings ?? new PipelineSettings();
|
||||
/// <summary>Gets the effective schedule defaults, using defaults if not specified.</summary>
|
||||
public ScheduleDefaults EffectiveScheduleDefaults => ScheduleDefaults ?? new ScheduleDefaults();
|
||||
}
|
||||
|
||||
|
||||
@@ -33,6 +33,7 @@ public record ScheduleConfig
|
||||
/// <summary>
|
||||
/// Merges this config with defaults. Non-null/non-default values in this config override defaults.
|
||||
/// </summary>
|
||||
/// <param name="defaults">The default configuration to merge with.</param>
|
||||
public ScheduleConfig MergeWith(ScheduleConfig defaults)
|
||||
{
|
||||
return new ScheduleConfig
|
||||
@@ -92,7 +93,18 @@ public record ScheduleDefaults
|
||||
/// </summary>
|
||||
public record PipelineSchedules
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets or initializes the Mass schedule configuration override.
|
||||
/// </summary>
|
||||
public ScheduleConfig? Mass { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or initializes the Daily schedule configuration override.
|
||||
/// </summary>
|
||||
public ScheduleConfig? Daily { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or initializes the Hourly schedule configuration override.
|
||||
/// </summary>
|
||||
public ScheduleConfig? Hourly { get; init; }
|
||||
}
|
||||
|
||||
@@ -11,6 +11,8 @@ public static class EtlServiceCollectionExtensions
|
||||
/// <summary>
|
||||
/// Adds ETL pipeline services to the service collection.
|
||||
/// </summary>
|
||||
/// <param name="services">The service collection to add ETL services to.</param>
|
||||
/// <returns>The service collection for method chaining.</returns>
|
||||
public static IServiceCollection AddEtlPipeline(this IServiceCollection services)
|
||||
{
|
||||
// Register the builder as transient so each request gets a fresh builder
|
||||
|
||||
@@ -15,8 +15,21 @@ public class EtlPipeline
|
||||
private readonly IReadOnlyList<IScriptRunner> _postScripts;
|
||||
private readonly ILogger<EtlPipeline> _logger;
|
||||
|
||||
/// <summary>
|
||||
/// Gets the name of the pipeline.
|
||||
/// </summary>
|
||||
public string PipelineName { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the EtlPipeline.
|
||||
/// </summary>
|
||||
/// <param name="name">The name of the pipeline.</param>
|
||||
/// <param name="source">The data source for the pipeline.</param>
|
||||
/// <param name="transformers">The data transformers to apply.</param>
|
||||
/// <param name="destination">The destination for the transformed data.</param>
|
||||
/// <param name="preScripts">Scripts to run before the pipeline executes.</param>
|
||||
/// <param name="postScripts">Scripts to run after the pipeline executes.</param>
|
||||
/// <param name="logger">The logger for recording pipeline execution.</param>
|
||||
internal EtlPipeline(
|
||||
string name,
|
||||
IImportSource source,
|
||||
@@ -35,6 +48,10 @@ public class EtlPipeline
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Executes the ETL pipeline, running all steps in sequence.
|
||||
/// </summary>
|
||||
/// <param name="cancellationToken">The cancellation token.</param>
|
||||
public async Task<PipelineResult> ExecuteAsync(CancellationToken cancellationToken = default)
|
||||
{
|
||||
var steps = new List<StepResult>();
|
||||
|
||||
@@ -15,18 +15,33 @@ public class EtlPipelineBuilder
|
||||
private ILogger<EtlPipeline>? _logger;
|
||||
private int _defaultCommandTimeoutSeconds = 600;
|
||||
|
||||
/// <summary>
|
||||
/// Sets the pipeline name.
|
||||
/// </summary>
|
||||
/// <param name="name">The pipeline name.</param>
|
||||
/// <returns>The builder for method chaining.</returns>
|
||||
public EtlPipelineBuilder WithName(string name)
|
||||
{
|
||||
_name = name ?? throw new ArgumentNullException(nameof(name));
|
||||
return this;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Sets the data source for the pipeline.
|
||||
/// </summary>
|
||||
/// <param name="source">The import source.</param>
|
||||
/// <returns>The builder for method chaining.</returns>
|
||||
public EtlPipelineBuilder WithSource(IImportSource source)
|
||||
{
|
||||
_source = source ?? throw new ArgumentNullException(nameof(source));
|
||||
return this;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Adds a data transformer to the pipeline.
|
||||
/// </summary>
|
||||
/// <param name="transformer">The data transformer.</param>
|
||||
/// <returns>The builder for method chaining.</returns>
|
||||
public EtlPipelineBuilder WithTransformer(IDataTransformer transformer)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(transformer);
|
||||
@@ -34,12 +49,22 @@ public class EtlPipelineBuilder
|
||||
return this;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Sets the destination for the transformed data.
|
||||
/// </summary>
|
||||
/// <param name="destination">The import destination.</param>
|
||||
/// <returns>The builder for method chaining.</returns>
|
||||
public EtlPipelineBuilder WithDestination(IImportDestination destination)
|
||||
{
|
||||
_destination = destination ?? throw new ArgumentNullException(nameof(destination));
|
||||
return this;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Adds a pre-execution script to run before the pipeline starts.
|
||||
/// </summary>
|
||||
/// <param name="script">The script runner.</param>
|
||||
/// <returns>The builder for method chaining.</returns>
|
||||
public EtlPipelineBuilder WithPreScript(IScriptRunner script)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(script);
|
||||
@@ -47,6 +72,11 @@ public class EtlPipelineBuilder
|
||||
return this;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Adds a post-execution script to run after the pipeline completes.
|
||||
/// </summary>
|
||||
/// <param name="script">The script runner.</param>
|
||||
/// <returns>The builder for method chaining.</returns>
|
||||
public EtlPipelineBuilder WithPostScript(IScriptRunner script)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(script);
|
||||
@@ -54,6 +84,11 @@ public class EtlPipelineBuilder
|
||||
return this;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Sets the logger for the pipeline.
|
||||
/// </summary>
|
||||
/// <param name="logger">The logger instance.</param>
|
||||
/// <returns>The builder for method chaining.</returns>
|
||||
public EtlPipelineBuilder WithLogger(ILogger<EtlPipeline> logger)
|
||||
{
|
||||
_logger = logger;
|
||||
@@ -63,6 +98,11 @@ public class EtlPipelineBuilder
|
||||
// TODO: Currently this timeout value is stored but not passed to destinations.
|
||||
// In the future, the pipeline should pass this timeout to destinations that support it.
|
||||
// For now, destinations use their own default timeout (600 seconds).
|
||||
/// <summary>
|
||||
/// Sets the command timeout for database operations.
|
||||
/// </summary>
|
||||
/// <param name="timeout">The timeout duration (must be between 0 and 24 hours).</param>
|
||||
/// <returns>The builder for method chaining.</returns>
|
||||
public EtlPipelineBuilder WithCommandTimeout(TimeSpan timeout)
|
||||
{
|
||||
if (timeout < TimeSpan.Zero || timeout > TimeSpan.FromHours(24))
|
||||
@@ -72,6 +112,11 @@ public class EtlPipelineBuilder
|
||||
return this;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Builds and returns the configured ETL pipeline.
|
||||
/// </summary>
|
||||
/// <returns>The constructed ETL pipeline.</returns>
|
||||
/// <exception cref="InvalidOperationException">Thrown when source or destination is not configured.</exception>
|
||||
public EtlPipeline Build()
|
||||
{
|
||||
if (_source == null)
|
||||
|
||||
@@ -9,6 +9,7 @@ public static class CommonScripts
|
||||
/// Parses a table name, extracting schema if present.
|
||||
/// Supports: "Table", "dbo.Table", "[dbo].[Table]"
|
||||
/// </summary>
|
||||
/// <param name="tableName">The table name to parse.</param>
|
||||
public static (string Schema, string Table) ParseTableName(string tableName)
|
||||
{
|
||||
var cleaned = tableName.Replace("[", "").Replace("]", "");
|
||||
@@ -21,12 +22,19 @@ public static class CommonScripts
|
||||
/// <summary>
|
||||
/// Formats a table name as a properly quoted [schema].[table] identifier.
|
||||
/// </summary>
|
||||
/// <param name="tableName">The table name to format.</param>
|
||||
public static string FormatQualifiedTableName(string tableName)
|
||||
{
|
||||
var (schema, table) = ParseTableName(tableName);
|
||||
return $"[{schema}].[{table}]";
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates a script runner that disables all indexes on a table.
|
||||
/// </summary>
|
||||
/// <param name="factory">The database connection factory.</param>
|
||||
/// <param name="tableName">The table name to disable indexes for.</param>
|
||||
/// <param name="timeoutSeconds">The execution timeout in seconds (default 300).</param>
|
||||
public static IScriptRunner DisableIndexes(
|
||||
IDbConnectionFactory factory,
|
||||
string tableName,
|
||||
@@ -54,6 +62,12 @@ IF LEN(@sql) > 0 EXEC sp_executesql @sql;";
|
||||
timeoutSeconds: timeoutSeconds);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates a script runner that rebuilds all indexes on a table.
|
||||
/// </summary>
|
||||
/// <param name="factory">The database connection factory.</param>
|
||||
/// <param name="tableName">The table name to rebuild indexes for.</param>
|
||||
/// <param name="timeoutSeconds">The execution timeout in seconds (default 3600).</param>
|
||||
public static IScriptRunner RebuildIndexes(
|
||||
IDbConnectionFactory factory,
|
||||
string tableName,
|
||||
@@ -70,6 +84,12 @@ EXEC sp_executesql @sql;";
|
||||
timeoutSeconds: timeoutSeconds);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates a script runner that updates statistics on a table.
|
||||
/// </summary>
|
||||
/// <param name="factory">The database connection factory.</param>
|
||||
/// <param name="tableName">The table name to update statistics for.</param>
|
||||
/// <param name="timeoutSeconds">The execution timeout in seconds (default 600).</param>
|
||||
public static IScriptRunner UpdateStatistics(
|
||||
IDbConnectionFactory factory,
|
||||
string tableName,
|
||||
@@ -86,6 +106,14 @@ EXEC sp_executesql @sql;";
|
||||
timeoutSeconds: timeoutSeconds);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates a script runner for executing custom SQL.
|
||||
/// </summary>
|
||||
/// <param name="factory">The database connection factory.</param>
|
||||
/// <param name="sql">The SQL statement to execute.</param>
|
||||
/// <param name="name">The name/description of the script.</param>
|
||||
/// <param name="parameters">Optional parameters for the SQL statement.</param>
|
||||
/// <param name="timeoutSeconds">The execution timeout in seconds (default 30).</param>
|
||||
public static IScriptRunner CustomSql(
|
||||
IDbConnectionFactory factory,
|
||||
string sql,
|
||||
|
||||
@@ -4,6 +4,9 @@ using JdeScoping.DataSync.Etl.Contracts;
|
||||
|
||||
namespace JdeScoping.DataSync.Etl.Scripts;
|
||||
|
||||
/// <summary>
|
||||
/// SQL script runner that executes SQL commands against the database.
|
||||
/// </summary>
|
||||
public class SqlScriptRunner : IScriptRunner
|
||||
{
|
||||
private readonly IDbConnectionFactory _connectionFactory;
|
||||
@@ -11,8 +14,19 @@ public class SqlScriptRunner : IScriptRunner
|
||||
private readonly object? _parameters;
|
||||
private readonly int _timeoutSeconds;
|
||||
|
||||
/// <summary>
|
||||
/// The name of this script.
|
||||
/// </summary>
|
||||
public string ScriptName { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="SqlScriptRunner"/> class.
|
||||
/// </summary>
|
||||
/// <param name="connectionFactory">The database connection factory.</param>
|
||||
/// <param name="sql">The SQL command to execute.</param>
|
||||
/// <param name="name">The optional name of the script.</param>
|
||||
/// <param name="parameters">The optional parameters for the SQL command.</param>
|
||||
/// <param name="timeoutSeconds">The command timeout in seconds.</param>
|
||||
public SqlScriptRunner(
|
||||
IDbConnectionFactory connectionFactory,
|
||||
string sql,
|
||||
@@ -30,6 +44,10 @@ public class SqlScriptRunner : IScriptRunner
|
||||
ScriptName = name ?? "SqlScript";
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Executes the SQL script asynchronously.
|
||||
/// </summary>
|
||||
/// <param name="cancellationToken">The cancellation token.</param>
|
||||
public async Task ExecuteAsync(CancellationToken cancellationToken = default)
|
||||
{
|
||||
await using var connection = await _connectionFactory.CreateLotFinderConnectionAsync(cancellationToken);
|
||||
|
||||
@@ -31,36 +31,53 @@ public abstract class DataTransformerBase : IDataTransformer
|
||||
/// Gets the field count from the source reader.
|
||||
/// Override to add or remove fields.
|
||||
/// </summary>
|
||||
/// <param name="source">The source data reader.</param>
|
||||
/// <returns>The field count.</returns>
|
||||
public virtual int GetFieldCount(IDataReader source) => source.FieldCount;
|
||||
|
||||
/// <summary>
|
||||
/// Gets the name of a field at the specified ordinal.
|
||||
/// Override to rename fields.
|
||||
/// </summary>
|
||||
/// <param name="ordinal">The field ordinal.</param>
|
||||
/// <param name="source">The source data reader.</param>
|
||||
/// <returns>The field name.</returns>
|
||||
public virtual string GetName(int ordinal, IDataReader source) => source.GetName(ordinal);
|
||||
|
||||
/// <summary>
|
||||
/// Gets the type of a field at the specified ordinal.
|
||||
/// Override to change field types.
|
||||
/// </summary>
|
||||
/// <param name="ordinal">The field ordinal.</param>
|
||||
/// <param name="source">The source data reader.</param>
|
||||
/// <returns>The field type.</returns>
|
||||
public virtual Type GetFieldType(int ordinal, IDataReader source) => source.GetFieldType(ordinal);
|
||||
|
||||
/// <summary>
|
||||
/// Gets the value of a field at the specified ordinal.
|
||||
/// Override to transform values.
|
||||
/// </summary>
|
||||
/// <param name="ordinal">The field ordinal.</param>
|
||||
/// <param name="source">The source data reader.</param>
|
||||
/// <returns>The field value.</returns>
|
||||
public virtual object GetValue(int ordinal, IDataReader source) => source.GetValue(ordinal);
|
||||
|
||||
/// <summary>
|
||||
/// Gets the ordinal of a field by name.
|
||||
/// Override to support renamed fields.
|
||||
/// </summary>
|
||||
/// <param name="name">The field name.</param>
|
||||
/// <param name="source">The source data reader.</param>
|
||||
/// <returns>The field ordinal.</returns>
|
||||
public virtual int GetOrdinal(string name, IDataReader source) => source.GetOrdinal(name);
|
||||
|
||||
/// <summary>
|
||||
/// Checks if a field value is DBNull.
|
||||
/// Override to handle null transformations.
|
||||
/// </summary>
|
||||
/// <param name="ordinal">The field ordinal.</param>
|
||||
/// <param name="source">The source data reader.</param>
|
||||
/// <returns>True if the field value is DBNull; otherwise, false.</returns>
|
||||
public virtual bool IsDBNull(int ordinal, IDataReader source) => source.IsDBNull(ordinal);
|
||||
|
||||
/// <summary>
|
||||
@@ -76,6 +93,13 @@ public abstract class DataTransformerBase : IDataTransformer
|
||||
/// Gets bytes from a field at the specified ordinal.
|
||||
/// Throws NotSupportedException for computed columns (where MapOrdinal returns -1).
|
||||
/// </summary>
|
||||
/// <param name="ordinal">The field ordinal.</param>
|
||||
/// <param name="fieldOffset">The offset in the field data.</param>
|
||||
/// <param name="buffer">The buffer to copy bytes into.</param>
|
||||
/// <param name="bufferOffset">The offset in the buffer where copying begins.</param>
|
||||
/// <param name="length">The maximum number of bytes to copy.</param>
|
||||
/// <param name="source">The source data reader.</param>
|
||||
/// <returns>The number of bytes actually copied.</returns>
|
||||
public virtual long GetBytes(int ordinal, long fieldOffset, byte[]? buffer,
|
||||
int bufferOffset, int length, IDataReader source)
|
||||
{
|
||||
@@ -90,6 +114,13 @@ public abstract class DataTransformerBase : IDataTransformer
|
||||
/// Gets characters from a field at the specified ordinal.
|
||||
/// Throws NotSupportedException for computed columns (where MapOrdinal returns -1).
|
||||
/// </summary>
|
||||
/// <param name="ordinal">The field ordinal.</param>
|
||||
/// <param name="fieldOffset">The offset in the field data.</param>
|
||||
/// <param name="buffer">The buffer to copy characters into.</param>
|
||||
/// <param name="bufferOffset">The offset in the buffer where copying begins.</param>
|
||||
/// <param name="length">The maximum number of characters to copy.</param>
|
||||
/// <param name="source">The source data reader.</param>
|
||||
/// <returns>The number of characters actually copied.</returns>
|
||||
public virtual long GetChars(int ordinal, long fieldOffset, char[]? buffer,
|
||||
int bufferOffset, int length, IDataReader source)
|
||||
{
|
||||
@@ -104,6 +135,9 @@ public abstract class DataTransformerBase : IDataTransformer
|
||||
/// Gets nested data reader for a field at the specified ordinal.
|
||||
/// Throws NotSupportedException for computed columns (where MapOrdinal returns -1).
|
||||
/// </summary>
|
||||
/// <param name="ordinal">The field ordinal.</param>
|
||||
/// <param name="source">The source data reader.</param>
|
||||
/// <returns>A nested data reader for the field.</returns>
|
||||
public virtual IDataReader GetData(int ordinal, IDataReader source)
|
||||
{
|
||||
var sourceOrdinal = MapOrdinal(ordinal, source);
|
||||
@@ -117,6 +151,9 @@ public abstract class DataTransformerBase : IDataTransformer
|
||||
/// Gets the data type name for a field at the specified ordinal.
|
||||
/// Throws NotSupportedException for computed columns (where MapOrdinal returns -1).
|
||||
/// </summary>
|
||||
/// <param name="ordinal">The field ordinal.</param>
|
||||
/// <param name="source">The source data reader.</param>
|
||||
/// <returns>The data type name.</returns>
|
||||
public virtual string GetDataTypeName(int ordinal, IDataReader source)
|
||||
{
|
||||
var sourceOrdinal = MapOrdinal(ordinal, source);
|
||||
|
||||
@@ -11,6 +11,11 @@ internal sealed class TransformingDataReader : IDataReader
|
||||
private readonly IDataReader _source;
|
||||
private readonly DataTransformerBase _transformer;
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="TransformingDataReader"/> class.
|
||||
/// </summary>
|
||||
/// <param name="source">The underlying data reader to wrap.</param>
|
||||
/// <param name="transformer">The transformer to apply to field values.</param>
|
||||
public TransformingDataReader(IDataReader source, DataTransformerBase transformer)
|
||||
{
|
||||
_source = source ?? throw new ArgumentNullException(nameof(source));
|
||||
@@ -18,41 +23,196 @@ internal sealed class TransformingDataReader : IDataReader
|
||||
}
|
||||
|
||||
// Properties and methods delegated to transformer
|
||||
/// <summary>
|
||||
/// Gets the number of columns in the current row.
|
||||
/// </summary>
|
||||
public int FieldCount => _transformer.GetFieldCount(_source);
|
||||
|
||||
/// <summary>
|
||||
/// Gets the name of the specified column.
|
||||
/// </summary>
|
||||
/// <param name="i">The zero-based column ordinal.</param>
|
||||
/// <returns>The name of the specified column.</returns>
|
||||
public string GetName(int i) => _transformer.GetName(i, _source);
|
||||
|
||||
/// <summary>
|
||||
/// Gets the data type of the specified column.
|
||||
/// </summary>
|
||||
/// <param name="i">The zero-based column ordinal.</param>
|
||||
/// <returns>The data type of the specified column.</returns>
|
||||
public Type GetFieldType(int i) => _transformer.GetFieldType(i, _source);
|
||||
|
||||
/// <summary>
|
||||
/// Gets the column ordinal given the name of the column.
|
||||
/// </summary>
|
||||
/// <param name="name">The name of the column.</param>
|
||||
/// <returns>The zero-based column ordinal.</returns>
|
||||
public int GetOrdinal(string name) => _transformer.GetOrdinal(name, _source);
|
||||
|
||||
/// <summary>
|
||||
/// Gets the value of the specified column as an object, applying transformations.
|
||||
/// </summary>
|
||||
/// <param name="i">The zero-based column ordinal.</param>
|
||||
/// <returns>The transformed value of the specified column.</returns>
|
||||
public object GetValue(int i) => _transformer.GetValue(i, _source);
|
||||
|
||||
/// <summary>
|
||||
/// Gets whether the value of the specified column is null.
|
||||
/// </summary>
|
||||
/// <param name="i">The zero-based column ordinal.</param>
|
||||
/// <returns>True if the value is null; otherwise, false.</returns>
|
||||
public bool IsDBNull(int i) => _transformer.IsDBNull(i, _source);
|
||||
|
||||
/// <summary>
|
||||
/// Gets the value of the column at the specified ordinal, applying transformations.
|
||||
/// </summary>
|
||||
/// <param name="i">The zero-based column ordinal.</param>
|
||||
/// <returns>The transformed value of the column.</returns>
|
||||
public object this[int i] => GetValue(i);
|
||||
|
||||
/// <summary>
|
||||
/// Gets the value of the column with the specified name, applying transformations.
|
||||
/// </summary>
|
||||
/// <param name="name">The name of the column.</param>
|
||||
/// <returns>The transformed value of the column.</returns>
|
||||
public object this[string name] => GetValue(GetOrdinal(name));
|
||||
|
||||
// Row navigation - delegated directly to source
|
||||
/// <summary>
|
||||
/// Advances the reader to the next record.
|
||||
/// </summary>
|
||||
/// <returns>True if there are more rows; otherwise, false.</returns>
|
||||
public bool Read() => _source.Read();
|
||||
|
||||
/// <summary>
|
||||
/// Advances the reader to the next result set.
|
||||
/// </summary>
|
||||
/// <returns>True if there are more result sets; otherwise, false.</returns>
|
||||
public bool NextResult() => _source.NextResult();
|
||||
|
||||
/// <summary>
|
||||
/// Gets the nesting depth of the current row.
|
||||
/// </summary>
|
||||
public int Depth => _source.Depth;
|
||||
|
||||
/// <summary>
|
||||
/// Gets whether the reader is closed.
|
||||
/// </summary>
|
||||
public bool IsClosed => _source.IsClosed;
|
||||
|
||||
/// <summary>
|
||||
/// Gets the number of rows affected by the last operation.
|
||||
/// </summary>
|
||||
public int RecordsAffected => _source.RecordsAffected;
|
||||
|
||||
/// <summary>
|
||||
/// Closes the reader.
|
||||
/// </summary>
|
||||
public void Close() => _source.Close();
|
||||
|
||||
/// <summary>
|
||||
/// Releases all resources used by the reader.
|
||||
/// </summary>
|
||||
public void Dispose() => _source.Dispose();
|
||||
|
||||
// Typed accessors - use GetValue for transformation support
|
||||
/// <summary>
|
||||
/// Gets the value of the specified column as a boolean.
|
||||
/// </summary>
|
||||
/// <param name="i">The zero-based column ordinal.</param>
|
||||
/// <returns>The boolean value of the column.</returns>
|
||||
public bool GetBoolean(int i) => (bool)GetValue(i);
|
||||
|
||||
/// <summary>
|
||||
/// Gets the value of the specified column as a byte.
|
||||
/// </summary>
|
||||
/// <param name="i">The zero-based column ordinal.</param>
|
||||
/// <returns>The byte value of the column.</returns>
|
||||
public byte GetByte(int i) => (byte)GetValue(i);
|
||||
|
||||
/// <summary>
|
||||
/// Gets the value of the specified column as a character.
|
||||
/// </summary>
|
||||
/// <param name="i">The zero-based column ordinal.</param>
|
||||
/// <returns>The character value of the column.</returns>
|
||||
public char GetChar(int i) => (char)GetValue(i);
|
||||
|
||||
/// <summary>
|
||||
/// Gets the value of the specified column as a DateTime.
|
||||
/// </summary>
|
||||
/// <param name="i">The zero-based column ordinal.</param>
|
||||
/// <returns>The DateTime value of the column.</returns>
|
||||
public DateTime GetDateTime(int i) => (DateTime)GetValue(i);
|
||||
|
||||
/// <summary>
|
||||
/// Gets the value of the specified column as a decimal.
|
||||
/// </summary>
|
||||
/// <param name="i">The zero-based column ordinal.</param>
|
||||
/// <returns>The decimal value of the column.</returns>
|
||||
public decimal GetDecimal(int i) => (decimal)GetValue(i);
|
||||
|
||||
/// <summary>
|
||||
/// Gets the value of the specified column as a double.
|
||||
/// </summary>
|
||||
/// <param name="i">The zero-based column ordinal.</param>
|
||||
/// <returns>The double value of the column.</returns>
|
||||
public double GetDouble(int i) => (double)GetValue(i);
|
||||
|
||||
/// <summary>
|
||||
/// Gets the value of the specified column as a float.
|
||||
/// </summary>
|
||||
/// <param name="i">The zero-based column ordinal.</param>
|
||||
/// <returns>The float value of the column.</returns>
|
||||
public float GetFloat(int i) => (float)GetValue(i);
|
||||
|
||||
/// <summary>
|
||||
/// Gets the value of the specified column as a GUID.
|
||||
/// </summary>
|
||||
/// <param name="i">The zero-based column ordinal.</param>
|
||||
/// <returns>The GUID value of the column.</returns>
|
||||
public Guid GetGuid(int i) => (Guid)GetValue(i);
|
||||
|
||||
/// <summary>
|
||||
/// Gets the value of the specified column as a 16-bit signed integer.
|
||||
/// </summary>
|
||||
/// <param name="i">The zero-based column ordinal.</param>
|
||||
/// <returns>The 16-bit signed integer value of the column.</returns>
|
||||
public short GetInt16(int i) => (short)GetValue(i);
|
||||
|
||||
/// <summary>
|
||||
/// Gets the value of the specified column as a 32-bit signed integer.
|
||||
/// </summary>
|
||||
/// <param name="i">The zero-based column ordinal.</param>
|
||||
/// <returns>The 32-bit signed integer value of the column.</returns>
|
||||
public int GetInt32(int i) => (int)GetValue(i);
|
||||
|
||||
/// <summary>
|
||||
/// Gets the value of the specified column as a 64-bit signed integer.
|
||||
/// </summary>
|
||||
/// <param name="i">The zero-based column ordinal.</param>
|
||||
/// <returns>The 64-bit signed integer value of the column.</returns>
|
||||
public long GetInt64(int i) => (long)GetValue(i);
|
||||
|
||||
/// <summary>
|
||||
/// Gets the value of the specified column as a string.
|
||||
/// </summary>
|
||||
/// <param name="i">The zero-based column ordinal.</param>
|
||||
/// <returns>The string value of the column.</returns>
|
||||
public string GetString(int i) => (string)GetValue(i);
|
||||
|
||||
// Schema and bulk data access - delegated to transformer
|
||||
/// <summary>
|
||||
/// Gets the name of the type for the specified column.
|
||||
/// </summary>
|
||||
/// <param name="i">The zero-based column ordinal.</param>
|
||||
/// <returns>The name of the data type of the specified column.</returns>
|
||||
public string GetDataTypeName(int i) => _transformer.GetDataTypeName(i, _source);
|
||||
|
||||
/// <summary>
|
||||
/// Populates an object array with the values of the current row.
|
||||
/// </summary>
|
||||
/// <param name="values">An array of objects to populate with column values.</param>
|
||||
/// <returns>The number of values populated.</returns>
|
||||
public int GetValues(object[] values)
|
||||
{
|
||||
var count = Math.Min(values.Length, FieldCount);
|
||||
@@ -61,13 +221,40 @@ internal sealed class TransformingDataReader : IDataReader
|
||||
return count;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the byte values of the specified column starting at the given offset.
|
||||
/// </summary>
|
||||
/// <param name="i">The zero-based column ordinal.</param>
|
||||
/// <param name="fieldOffset">The offset within the field.</param>
|
||||
/// <param name="buffer">The buffer to read bytes into.</param>
|
||||
/// <param name="bufferoffset">The offset in the buffer to start writing.</param>
|
||||
/// <param name="length">The maximum number of bytes to read.</param>
|
||||
/// <returns>The number of bytes read.</returns>
|
||||
public long GetBytes(int i, long fieldOffset, byte[]? buffer, int bufferoffset, int length)
|
||||
=> _transformer.GetBytes(i, fieldOffset, buffer, bufferoffset, length, _source);
|
||||
|
||||
/// <summary>
|
||||
/// Gets the character values of the specified column starting at the given offset.
|
||||
/// </summary>
|
||||
/// <param name="i">The zero-based column ordinal.</param>
|
||||
/// <param name="fieldoffset">The offset within the field.</param>
|
||||
/// <param name="buffer">The buffer to read characters into.</param>
|
||||
/// <param name="bufferoffset">The offset in the buffer to start writing.</param>
|
||||
/// <param name="length">The maximum number of characters to read.</param>
|
||||
/// <returns>The number of characters read.</returns>
|
||||
public long GetChars(int i, long fieldoffset, char[]? buffer, int bufferoffset, int length)
|
||||
=> _transformer.GetChars(i, fieldoffset, buffer, bufferoffset, length, _source);
|
||||
|
||||
/// <summary>
|
||||
/// Gets a nested data reader for the specified column.
|
||||
/// </summary>
|
||||
/// <param name="i">The zero-based column ordinal.</param>
|
||||
/// <returns>A nested data reader for the specified column.</returns>
|
||||
public IDataReader GetData(int i) => _transformer.GetData(i, _source);
|
||||
|
||||
/// <summary>
|
||||
/// Gets a DataTable that describes the schema of the result set.
|
||||
/// </summary>
|
||||
/// <returns>A DataTable describing the result set schema, or null if no schema is available.</returns>
|
||||
public DataTable? GetSchemaTable() => _source.GetSchemaTable();
|
||||
}
|
||||
|
||||
@@ -13,6 +13,7 @@ public class DataSyncHealthCheck : IHealthCheck
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="DataSyncHealthCheck"/> class.
|
||||
/// </summary>
|
||||
/// <param name="repository">The data update repository.</param>
|
||||
public DataSyncHealthCheck(IDataUpdateRepository repository)
|
||||
{
|
||||
_repository = repository ?? throw new ArgumentNullException(nameof(repository));
|
||||
|
||||
@@ -3,5 +3,9 @@ namespace JdeScoping.DataSync.Options;
|
||||
public class PipelineOptions
|
||||
{
|
||||
public const string SectionName = "Pipelines";
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the path to the pipeline configuration file.
|
||||
/// </summary>
|
||||
public string ConfigPath { get; set; } = "Pipelines/pipelines.json";
|
||||
}
|
||||
|
||||
@@ -22,6 +22,8 @@ public class DataUpdateRepository : IDataUpdateRepository
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="DataUpdateRepository"/> class.
|
||||
/// </summary>
|
||||
/// <param name="connectionFactory">The database connection factory.</param>
|
||||
/// <param name="logger">The logger instance.</param>
|
||||
public DataUpdateRepository(
|
||||
IDbConnectionFactory connectionFactory,
|
||||
ILogger<DataUpdateRepository> logger)
|
||||
|
||||
@@ -53,6 +53,9 @@ public class EtlPipelineFactory : IEtlPipelineFactory
|
||||
/// <summary>
|
||||
/// Creates a new pipeline factory with a pre-loaded configuration (for testing).
|
||||
/// </summary>
|
||||
/// <param name="connectionFactory">Factory for creating database connections.</param>
|
||||
/// <param name="config">Pre-loaded pipeline configuration.</param>
|
||||
/// <param name="logger">Logger for pipeline execution.</param>
|
||||
internal EtlPipelineFactory(
|
||||
IDbConnectionFactory connectionFactory,
|
||||
PipelinesRoot config,
|
||||
@@ -167,6 +170,15 @@ public class EtlPipelineFactory : IEtlPipelineFactory
|
||||
private UpdateTypes _updateType = UpdateTypes.Hourly;
|
||||
private DateTime? _minDtOverride;
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the PipelineBuilder class.
|
||||
/// </summary>
|
||||
/// <param name="connectionFactory">Factory for creating database connections.</param>
|
||||
/// <param name="tableName">The name of the table for this pipeline.</param>
|
||||
/// <param name="config">The pipeline configuration.</param>
|
||||
/// <param name="settings">Global pipeline settings.</param>
|
||||
/// <param name="scheduleDefaults">Default schedule configuration.</param>
|
||||
/// <param name="logger">Logger for pipeline execution.</param>
|
||||
public PipelineBuilder(
|
||||
IDbConnectionFactory connectionFactory,
|
||||
string tableName,
|
||||
@@ -183,18 +195,32 @@ public class EtlPipelineFactory : IEtlPipelineFactory
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Specifies the update type for this pipeline.
|
||||
/// </summary>
|
||||
/// <param name="updateType">The type of update (Mass, Daily, or Hourly).</param>
|
||||
/// <returns>The builder for fluent configuration.</returns>
|
||||
public IEtlPipelineBuilder WithUpdateType(UpdateTypes updateType)
|
||||
{
|
||||
_updateType = updateType;
|
||||
return this;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Specifies the minimum date for incremental data extraction.
|
||||
/// </summary>
|
||||
/// <param name="minDt">The minimum date, or null for no filter.</param>
|
||||
/// <returns>The builder for fluent configuration.</returns>
|
||||
public IEtlPipelineBuilder WithMinimumDate(DateTime? minDt)
|
||||
{
|
||||
_minDtOverride = minDt;
|
||||
return this;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Builds and returns the configured ETL pipeline.
|
||||
/// </summary>
|
||||
/// <returns>A configured ETL pipeline ready for execution.</returns>
|
||||
public EtlPipeline Build()
|
||||
{
|
||||
return BuildWithSchedules();
|
||||
|
||||
@@ -21,6 +21,9 @@ public class ScheduleChecker : IScheduleChecker
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="ScheduleChecker"/> class.
|
||||
/// </summary>
|
||||
/// <param name="repository">Repository for data update records.</param>
|
||||
/// <param name="options">Data sync configuration options.</param>
|
||||
/// <param name="logger">Logger instance.</param>
|
||||
public ScheduleChecker(
|
||||
IDataUpdateRepository repository,
|
||||
IOptions<DataSyncOptions> options,
|
||||
|
||||
@@ -21,6 +21,15 @@ public class SearchExecutionService : ISearchExecutionService
|
||||
private readonly WorkProcessorOptions _options;
|
||||
private readonly ILogger<SearchExecutionService> _logger;
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the SearchExecutionService.
|
||||
/// </summary>
|
||||
/// <param name="searchRepository">The repository for search data access.</param>
|
||||
/// <param name="searchProcessor">The processor for executing search queries.</param>
|
||||
/// <param name="excelExportService">The service for generating Excel export files.</param>
|
||||
/// <param name="notificationService">The service for sending search status notifications.</param>
|
||||
/// <param name="options">The work processor configuration options.</param>
|
||||
/// <param name="logger">The logger for recording search execution events.</param>
|
||||
public SearchExecutionService(
|
||||
ISearchRepository searchRepository,
|
||||
ISearchProcessor searchProcessor,
|
||||
|
||||
@@ -21,6 +21,11 @@ public class SyncOrchestrator : ISyncOrchestrator
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="SyncOrchestrator"/> class.
|
||||
/// </summary>
|
||||
/// <param name="scopeFactory">Factory for creating service scopes.</param>
|
||||
/// <param name="scheduleChecker">Checker for determining pending sync tasks.</param>
|
||||
/// <param name="options">Data sync configuration options.</param>
|
||||
/// <param name="logger">Logger instance.</param>
|
||||
/// <param name="metrics">Metrics collector for sync operations.</param>
|
||||
public SyncOrchestrator(
|
||||
IServiceScopeFactory scopeFactory,
|
||||
IScheduleChecker scheduleChecker,
|
||||
|
||||
@@ -24,6 +24,11 @@ public class TableSyncOperation : ITableSyncOperation
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="TableSyncOperation"/> class.
|
||||
/// </summary>
|
||||
/// <param name="pipelineFactory">Factory for creating ETL pipelines.</param>
|
||||
/// <param name="updateRepository">Repository for managing data update records.</param>
|
||||
/// <param name="options">Data sync configuration options.</param>
|
||||
/// <param name="logger">Logger for operation events.</param>
|
||||
/// <param name="metrics">Metrics collector for operation tracking.</param>
|
||||
public TableSyncOperation(
|
||||
IEtlPipelineFactory pipelineFactory,
|
||||
IDataUpdateRepository updateRepository,
|
||||
|
||||
@@ -18,6 +18,7 @@ public class DataSyncMetrics
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="DataSyncMetrics"/> class.
|
||||
/// </summary>
|
||||
/// <param name="meterFactory">The meter factory for creating metrics.</param>
|
||||
public DataSyncMetrics(IMeterFactory meterFactory)
|
||||
{
|
||||
var meter = meterFactory.Create("JdeScoping.DataSync");
|
||||
@@ -61,6 +62,8 @@ public class DataSyncMetrics
|
||||
/// <summary>
|
||||
/// Records that a sync operation has started.
|
||||
/// </summary>
|
||||
/// <param name="tableName">The name of the table being synced.</param>
|
||||
/// <param name="updateType">The type of update (e.g., Mass, Daily, Hourly).</param>
|
||||
public void RecordOperationStarted(string tableName, string updateType)
|
||||
{
|
||||
_operationsStarted.Add(1,
|
||||
@@ -71,6 +74,10 @@ public class DataSyncMetrics
|
||||
/// <summary>
|
||||
/// Records that a sync operation completed successfully.
|
||||
/// </summary>
|
||||
/// <param name="tableName">The name of the table being synced.</param>
|
||||
/// <param name="updateType">The type of update (e.g., Mass, Daily, Hourly).</param>
|
||||
/// <param name="recordCount">The number of records processed.</param>
|
||||
/// <param name="durationSeconds">The operation duration in seconds.</param>
|
||||
public void RecordOperationCompleted(string tableName, string updateType, long recordCount, double durationSeconds)
|
||||
{
|
||||
var tags = new KeyValuePair<string, object?>[]
|
||||
@@ -87,6 +94,8 @@ public class DataSyncMetrics
|
||||
/// <summary>
|
||||
/// Records that a sync operation failed.
|
||||
/// </summary>
|
||||
/// <param name="tableName">The name of the table being synced.</param>
|
||||
/// <param name="updateType">The type of update (e.g., Mass, Daily, Hourly).</param>
|
||||
public void RecordOperationFailed(string tableName, string updateType)
|
||||
{
|
||||
_operationsFailed.Add(1,
|
||||
@@ -105,6 +114,9 @@ public class DataSyncMetrics
|
||||
/// <summary>
|
||||
/// Records completion of a sync cycle.
|
||||
/// </summary>
|
||||
/// <param name="successCount">Number of successful operations in the cycle.</param>
|
||||
/// <param name="failedCount">Number of failed operations in the cycle.</param>
|
||||
/// <param name="durationSeconds">The cycle duration in seconds.</param>
|
||||
public void RecordCycleCompleted(int successCount, int failedCount, double durationSeconds)
|
||||
{
|
||||
_cyclesCompleted.Add(1,
|
||||
|
||||
@@ -26,6 +26,10 @@ public class WorkProcessor : BackgroundService
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="WorkProcessor"/> class.
|
||||
/// </summary>
|
||||
/// <param name="scopeFactory">Factory for creating service scopes.</param>
|
||||
/// <param name="options">Configuration options for work processing.</param>
|
||||
/// <param name="logger">Logger for recording work processor events.</param>
|
||||
/// <param name="metrics">Metrics collector for tracking work cycles.</param>
|
||||
public WorkProcessor(
|
||||
IServiceScopeFactory scopeFactory,
|
||||
IOptions<WorkProcessorOptions> options,
|
||||
|
||||
Reference in New Issue
Block a user