docs: add XML documentation and ConfigManager implementation plans

Add comprehensive XML documentation (param/returns tags) across 132 source
files to improve IntelliSense and API discoverability. Include ConfigManager
design documents and implementation plans for phases 1-9.
This commit is contained in:
Joseph Doherty
2026-01-20 02:26:26 -05:00
parent c044337539
commit d49330e697
136 changed files with 9181 additions and 4 deletions
@@ -5,7 +5,9 @@ public record PipelinesRoot(
ScheduleDefaults? ScheduleDefaults, // Optional - defaults applied if missing
Dictionary<string, PipelineConfig> Pipelines)
{
/// <summary>Gets the effective pipeline settings, using defaults if not specified.</summary>
public PipelineSettings EffectiveSettings => Settings ?? new PipelineSettings();
/// <summary>Gets the effective schedule defaults, using defaults if not specified.</summary>
public ScheduleDefaults EffectiveScheduleDefaults => ScheduleDefaults ?? new ScheduleDefaults();
}
@@ -33,6 +33,7 @@ public record ScheduleConfig
/// <summary>
/// Merges this config with defaults. Non-null/non-default values in this config override defaults.
/// </summary>
/// <param name="defaults">The default configuration to merge with.</param>
public ScheduleConfig MergeWith(ScheduleConfig defaults)
{
return new ScheduleConfig
@@ -92,7 +93,18 @@ public record ScheduleDefaults
/// </summary>
public record PipelineSchedules
{
/// <summary>
/// Gets or initializes the Mass schedule configuration override.
/// </summary>
public ScheduleConfig? Mass { get; init; }
/// <summary>
/// Gets or initializes the Daily schedule configuration override.
/// </summary>
public ScheduleConfig? Daily { get; init; }
/// <summary>
/// Gets or initializes the Hourly schedule configuration override.
/// </summary>
public ScheduleConfig? Hourly { get; init; }
}
@@ -11,6 +11,8 @@ public static class EtlServiceCollectionExtensions
/// <summary>
/// Adds ETL pipeline services to the service collection.
/// </summary>
/// <param name="services">The service collection to add ETL services to.</param>
/// <returns>The service collection for method chaining.</returns>
public static IServiceCollection AddEtlPipeline(this IServiceCollection services)
{
// Register the builder as transient so each request gets a fresh builder
@@ -15,8 +15,21 @@ public class EtlPipeline
private readonly IReadOnlyList<IScriptRunner> _postScripts;
private readonly ILogger<EtlPipeline> _logger;
/// <summary>
/// Gets the name of the pipeline.
/// </summary>
public string PipelineName { get; }
/// <summary>
/// Initializes a new instance of the EtlPipeline.
/// </summary>
/// <param name="name">The name of the pipeline.</param>
/// <param name="source">The data source for the pipeline.</param>
/// <param name="transformers">The data transformers to apply.</param>
/// <param name="destination">The destination for the transformed data.</param>
/// <param name="preScripts">Scripts to run before the pipeline executes.</param>
/// <param name="postScripts">Scripts to run after the pipeline executes.</param>
/// <param name="logger">The logger for recording pipeline execution.</param>
internal EtlPipeline(
string name,
IImportSource source,
@@ -35,6 +48,10 @@ public class EtlPipeline
_logger = logger;
}
/// <summary>
/// Executes the ETL pipeline, running all steps in sequence.
/// </summary>
/// <param name="cancellationToken">The cancellation token.</param>
public async Task<PipelineResult> ExecuteAsync(CancellationToken cancellationToken = default)
{
var steps = new List<StepResult>();
@@ -15,18 +15,33 @@ public class EtlPipelineBuilder
private ILogger<EtlPipeline>? _logger;
private int _defaultCommandTimeoutSeconds = 600;
/// <summary>
/// Sets the pipeline name.
/// </summary>
/// <param name="name">The pipeline name.</param>
/// <returns>The builder for method chaining.</returns>
public EtlPipelineBuilder WithName(string name)
{
_name = name ?? throw new ArgumentNullException(nameof(name));
return this;
}
/// <summary>
/// Sets the data source for the pipeline.
/// </summary>
/// <param name="source">The import source.</param>
/// <returns>The builder for method chaining.</returns>
public EtlPipelineBuilder WithSource(IImportSource source)
{
_source = source ?? throw new ArgumentNullException(nameof(source));
return this;
}
/// <summary>
/// Adds a data transformer to the pipeline.
/// </summary>
/// <param name="transformer">The data transformer.</param>
/// <returns>The builder for method chaining.</returns>
public EtlPipelineBuilder WithTransformer(IDataTransformer transformer)
{
ArgumentNullException.ThrowIfNull(transformer);
@@ -34,12 +49,22 @@ public class EtlPipelineBuilder
return this;
}
/// <summary>
/// Sets the destination for the transformed data.
/// </summary>
/// <param name="destination">The import destination.</param>
/// <returns>The builder for method chaining.</returns>
public EtlPipelineBuilder WithDestination(IImportDestination destination)
{
_destination = destination ?? throw new ArgumentNullException(nameof(destination));
return this;
}
/// <summary>
/// Adds a pre-execution script to run before the pipeline starts.
/// </summary>
/// <param name="script">The script runner.</param>
/// <returns>The builder for method chaining.</returns>
public EtlPipelineBuilder WithPreScript(IScriptRunner script)
{
ArgumentNullException.ThrowIfNull(script);
@@ -47,6 +72,11 @@ public class EtlPipelineBuilder
return this;
}
/// <summary>
/// Adds a post-execution script to run after the pipeline completes.
/// </summary>
/// <param name="script">The script runner.</param>
/// <returns>The builder for method chaining.</returns>
public EtlPipelineBuilder WithPostScript(IScriptRunner script)
{
ArgumentNullException.ThrowIfNull(script);
@@ -54,6 +84,11 @@ public class EtlPipelineBuilder
return this;
}
/// <summary>
/// Sets the logger for the pipeline.
/// </summary>
/// <param name="logger">The logger instance.</param>
/// <returns>The builder for method chaining.</returns>
public EtlPipelineBuilder WithLogger(ILogger<EtlPipeline> logger)
{
_logger = logger;
@@ -63,6 +98,11 @@ public class EtlPipelineBuilder
// TODO: Currently this timeout value is stored but not passed to destinations.
// In the future, the pipeline should pass this timeout to destinations that support it.
// For now, destinations use their own default timeout (600 seconds).
/// <summary>
/// Sets the command timeout for database operations.
/// </summary>
/// <param name="timeout">The timeout duration (must be between 0 and 24 hours).</param>
/// <returns>The builder for method chaining.</returns>
public EtlPipelineBuilder WithCommandTimeout(TimeSpan timeout)
{
if (timeout < TimeSpan.Zero || timeout > TimeSpan.FromHours(24))
@@ -72,6 +112,11 @@ public class EtlPipelineBuilder
return this;
}
/// <summary>
/// Builds and returns the configured ETL pipeline.
/// </summary>
/// <returns>The constructed ETL pipeline.</returns>
/// <exception cref="InvalidOperationException">Thrown when source or destination is not configured.</exception>
public EtlPipeline Build()
{
if (_source == null)
@@ -9,6 +9,7 @@ public static class CommonScripts
/// Parses a table name, extracting schema if present.
/// Supports: "Table", "dbo.Table", "[dbo].[Table]"
/// </summary>
/// <param name="tableName">The table name to parse.</param>
public static (string Schema, string Table) ParseTableName(string tableName)
{
var cleaned = tableName.Replace("[", "").Replace("]", "");
@@ -21,12 +22,19 @@ public static class CommonScripts
/// <summary>
/// Formats a table name as a properly quoted [schema].[table] identifier.
/// </summary>
/// <param name="tableName">The table name to format.</param>
public static string FormatQualifiedTableName(string tableName)
{
var (schema, table) = ParseTableName(tableName);
return $"[{schema}].[{table}]";
}
/// <summary>
/// Creates a script runner that disables all indexes on a table.
/// </summary>
/// <param name="factory">The database connection factory.</param>
/// <param name="tableName">The table name to disable indexes for.</param>
/// <param name="timeoutSeconds">The execution timeout in seconds (default 300).</param>
public static IScriptRunner DisableIndexes(
IDbConnectionFactory factory,
string tableName,
@@ -54,6 +62,12 @@ IF LEN(@sql) > 0 EXEC sp_executesql @sql;";
timeoutSeconds: timeoutSeconds);
}
/// <summary>
/// Creates a script runner that rebuilds all indexes on a table.
/// </summary>
/// <param name="factory">The database connection factory.</param>
/// <param name="tableName">The table name to rebuild indexes for.</param>
/// <param name="timeoutSeconds">The execution timeout in seconds (default 3600).</param>
public static IScriptRunner RebuildIndexes(
IDbConnectionFactory factory,
string tableName,
@@ -70,6 +84,12 @@ EXEC sp_executesql @sql;";
timeoutSeconds: timeoutSeconds);
}
/// <summary>
/// Creates a script runner that updates statistics on a table.
/// </summary>
/// <param name="factory">The database connection factory.</param>
/// <param name="tableName">The table name to update statistics for.</param>
/// <param name="timeoutSeconds">The execution timeout in seconds (default 600).</param>
public static IScriptRunner UpdateStatistics(
IDbConnectionFactory factory,
string tableName,
@@ -86,6 +106,14 @@ EXEC sp_executesql @sql;";
timeoutSeconds: timeoutSeconds);
}
/// <summary>
/// Creates a script runner for executing custom SQL.
/// </summary>
/// <param name="factory">The database connection factory.</param>
/// <param name="sql">The SQL statement to execute.</param>
/// <param name="name">The name/description of the script.</param>
/// <param name="parameters">Optional parameters for the SQL statement.</param>
/// <param name="timeoutSeconds">The execution timeout in seconds (default 30).</param>
public static IScriptRunner CustomSql(
IDbConnectionFactory factory,
string sql,
@@ -4,6 +4,9 @@ using JdeScoping.DataSync.Etl.Contracts;
namespace JdeScoping.DataSync.Etl.Scripts;
/// <summary>
/// SQL script runner that executes SQL commands against the database.
/// </summary>
public class SqlScriptRunner : IScriptRunner
{
private readonly IDbConnectionFactory _connectionFactory;
@@ -11,8 +14,19 @@ public class SqlScriptRunner : IScriptRunner
private readonly object? _parameters;
private readonly int _timeoutSeconds;
/// <summary>
/// The name of this script.
/// </summary>
public string ScriptName { get; }
/// <summary>
/// Initializes a new instance of the <see cref="SqlScriptRunner"/> class.
/// </summary>
/// <param name="connectionFactory">The database connection factory.</param>
/// <param name="sql">The SQL command to execute.</param>
/// <param name="name">The optional name of the script.</param>
/// <param name="parameters">The optional parameters for the SQL command.</param>
/// <param name="timeoutSeconds">The command timeout in seconds.</param>
public SqlScriptRunner(
IDbConnectionFactory connectionFactory,
string sql,
@@ -30,6 +44,10 @@ public class SqlScriptRunner : IScriptRunner
ScriptName = name ?? "SqlScript";
}
/// <summary>
/// Executes the SQL script asynchronously.
/// </summary>
/// <param name="cancellationToken">The cancellation token.</param>
public async Task ExecuteAsync(CancellationToken cancellationToken = default)
{
await using var connection = await _connectionFactory.CreateLotFinderConnectionAsync(cancellationToken);
@@ -31,36 +31,53 @@ public abstract class DataTransformerBase : IDataTransformer
/// Gets the field count from the source reader.
/// Override to add or remove fields.
/// </summary>
/// <param name="source">The source data reader.</param>
/// <returns>The field count.</returns>
public virtual int GetFieldCount(IDataReader source) => source.FieldCount;
/// <summary>
/// Gets the name of a field at the specified ordinal.
/// Override to rename fields.
/// </summary>
/// <param name="ordinal">The field ordinal.</param>
/// <param name="source">The source data reader.</param>
/// <returns>The field name.</returns>
public virtual string GetName(int ordinal, IDataReader source) => source.GetName(ordinal);
/// <summary>
/// Gets the type of a field at the specified ordinal.
/// Override to change field types.
/// </summary>
/// <param name="ordinal">The field ordinal.</param>
/// <param name="source">The source data reader.</param>
/// <returns>The field type.</returns>
public virtual Type GetFieldType(int ordinal, IDataReader source) => source.GetFieldType(ordinal);
/// <summary>
/// Gets the value of a field at the specified ordinal.
/// Override to transform values.
/// </summary>
/// <param name="ordinal">The field ordinal.</param>
/// <param name="source">The source data reader.</param>
/// <returns>The field value.</returns>
public virtual object GetValue(int ordinal, IDataReader source) => source.GetValue(ordinal);
/// <summary>
/// Gets the ordinal of a field by name.
/// Override to support renamed fields.
/// </summary>
/// <param name="name">The field name.</param>
/// <param name="source">The source data reader.</param>
/// <returns>The field ordinal.</returns>
public virtual int GetOrdinal(string name, IDataReader source) => source.GetOrdinal(name);
/// <summary>
/// Checks if a field value is DBNull.
/// Override to handle null transformations.
/// </summary>
/// <param name="ordinal">The field ordinal.</param>
/// <param name="source">The source data reader.</param>
/// <returns>True if the field value is DBNull; otherwise, false.</returns>
public virtual bool IsDBNull(int ordinal, IDataReader source) => source.IsDBNull(ordinal);
/// <summary>
@@ -76,6 +93,13 @@ public abstract class DataTransformerBase : IDataTransformer
/// Gets bytes from a field at the specified ordinal.
/// Throws NotSupportedException for computed columns (where MapOrdinal returns -1).
/// </summary>
/// <param name="ordinal">The field ordinal.</param>
/// <param name="fieldOffset">The offset in the field data.</param>
/// <param name="buffer">The buffer to copy bytes into.</param>
/// <param name="bufferOffset">The offset in the buffer where copying begins.</param>
/// <param name="length">The maximum number of bytes to copy.</param>
/// <param name="source">The source data reader.</param>
/// <returns>The number of bytes actually copied.</returns>
public virtual long GetBytes(int ordinal, long fieldOffset, byte[]? buffer,
int bufferOffset, int length, IDataReader source)
{
@@ -90,6 +114,13 @@ public abstract class DataTransformerBase : IDataTransformer
/// Gets characters from a field at the specified ordinal.
/// Throws NotSupportedException for computed columns (where MapOrdinal returns -1).
/// </summary>
/// <param name="ordinal">The field ordinal.</param>
/// <param name="fieldOffset">The offset in the field data.</param>
/// <param name="buffer">The buffer to copy characters into.</param>
/// <param name="bufferOffset">The offset in the buffer where copying begins.</param>
/// <param name="length">The maximum number of characters to copy.</param>
/// <param name="source">The source data reader.</param>
/// <returns>The number of characters actually copied.</returns>
public virtual long GetChars(int ordinal, long fieldOffset, char[]? buffer,
int bufferOffset, int length, IDataReader source)
{
@@ -104,6 +135,9 @@ public abstract class DataTransformerBase : IDataTransformer
/// Gets nested data reader for a field at the specified ordinal.
/// Throws NotSupportedException for computed columns (where MapOrdinal returns -1).
/// </summary>
/// <param name="ordinal">The field ordinal.</param>
/// <param name="source">The source data reader.</param>
/// <returns>A nested data reader for the field.</returns>
public virtual IDataReader GetData(int ordinal, IDataReader source)
{
var sourceOrdinal = MapOrdinal(ordinal, source);
@@ -117,6 +151,9 @@ public abstract class DataTransformerBase : IDataTransformer
/// Gets the data type name for a field at the specified ordinal.
/// Throws NotSupportedException for computed columns (where MapOrdinal returns -1).
/// </summary>
/// <param name="ordinal">The field ordinal.</param>
/// <param name="source">The source data reader.</param>
/// <returns>The data type name.</returns>
public virtual string GetDataTypeName(int ordinal, IDataReader source)
{
var sourceOrdinal = MapOrdinal(ordinal, source);
@@ -11,6 +11,11 @@ internal sealed class TransformingDataReader : IDataReader
private readonly IDataReader _source;
private readonly DataTransformerBase _transformer;
/// <summary>
/// Initializes a new instance of the <see cref="TransformingDataReader"/> class.
/// </summary>
/// <param name="source">The underlying data reader to wrap.</param>
/// <param name="transformer">The transformer to apply to field values.</param>
public TransformingDataReader(IDataReader source, DataTransformerBase transformer)
{
_source = source ?? throw new ArgumentNullException(nameof(source));
@@ -18,41 +23,196 @@ internal sealed class TransformingDataReader : IDataReader
}
// Properties and methods delegated to transformer
/// <summary>
/// Gets the number of columns in the current row.
/// </summary>
public int FieldCount => _transformer.GetFieldCount(_source);
/// <summary>
/// Gets the name of the specified column.
/// </summary>
/// <param name="i">The zero-based column ordinal.</param>
/// <returns>The name of the specified column.</returns>
public string GetName(int i) => _transformer.GetName(i, _source);
/// <summary>
/// Gets the data type of the specified column.
/// </summary>
/// <param name="i">The zero-based column ordinal.</param>
/// <returns>The data type of the specified column.</returns>
public Type GetFieldType(int i) => _transformer.GetFieldType(i, _source);
/// <summary>
/// Gets the column ordinal given the name of the column.
/// </summary>
/// <param name="name">The name of the column.</param>
/// <returns>The zero-based column ordinal.</returns>
public int GetOrdinal(string name) => _transformer.GetOrdinal(name, _source);
/// <summary>
/// Gets the value of the specified column as an object, applying transformations.
/// </summary>
/// <param name="i">The zero-based column ordinal.</param>
/// <returns>The transformed value of the specified column.</returns>
public object GetValue(int i) => _transformer.GetValue(i, _source);
/// <summary>
/// Gets whether the value of the specified column is null.
/// </summary>
/// <param name="i">The zero-based column ordinal.</param>
/// <returns>True if the value is null; otherwise, false.</returns>
public bool IsDBNull(int i) => _transformer.IsDBNull(i, _source);
/// <summary>
/// Gets the value of the column at the specified ordinal, applying transformations.
/// </summary>
/// <param name="i">The zero-based column ordinal.</param>
/// <returns>The transformed value of the column.</returns>
public object this[int i] => GetValue(i);
/// <summary>
/// Gets the value of the column with the specified name, applying transformations.
/// </summary>
/// <param name="name">The name of the column.</param>
/// <returns>The transformed value of the column.</returns>
public object this[string name] => GetValue(GetOrdinal(name));
// Row navigation - delegated directly to source
/// <summary>
/// Advances the reader to the next record.
/// </summary>
/// <returns>True if there are more rows; otherwise, false.</returns>
public bool Read() => _source.Read();
/// <summary>
/// Advances the reader to the next result set.
/// </summary>
/// <returns>True if there are more result sets; otherwise, false.</returns>
public bool NextResult() => _source.NextResult();
/// <summary>
/// Gets the nesting depth of the current row.
/// </summary>
public int Depth => _source.Depth;
/// <summary>
/// Gets whether the reader is closed.
/// </summary>
public bool IsClosed => _source.IsClosed;
/// <summary>
/// Gets the number of rows affected by the last operation.
/// </summary>
public int RecordsAffected => _source.RecordsAffected;
/// <summary>
/// Closes the reader.
/// </summary>
public void Close() => _source.Close();
/// <summary>
/// Releases all resources used by the reader.
/// </summary>
public void Dispose() => _source.Dispose();
// Typed accessors - use GetValue for transformation support
/// <summary>
/// Gets the value of the specified column as a boolean.
/// </summary>
/// <param name="i">The zero-based column ordinal.</param>
/// <returns>The boolean value of the column.</returns>
public bool GetBoolean(int i) => (bool)GetValue(i);
/// <summary>
/// Gets the value of the specified column as a byte.
/// </summary>
/// <param name="i">The zero-based column ordinal.</param>
/// <returns>The byte value of the column.</returns>
public byte GetByte(int i) => (byte)GetValue(i);
/// <summary>
/// Gets the value of the specified column as a character.
/// </summary>
/// <param name="i">The zero-based column ordinal.</param>
/// <returns>The character value of the column.</returns>
public char GetChar(int i) => (char)GetValue(i);
/// <summary>
/// Gets the value of the specified column as a DateTime.
/// </summary>
/// <param name="i">The zero-based column ordinal.</param>
/// <returns>The DateTime value of the column.</returns>
public DateTime GetDateTime(int i) => (DateTime)GetValue(i);
/// <summary>
/// Gets the value of the specified column as a decimal.
/// </summary>
/// <param name="i">The zero-based column ordinal.</param>
/// <returns>The decimal value of the column.</returns>
public decimal GetDecimal(int i) => (decimal)GetValue(i);
/// <summary>
/// Gets the value of the specified column as a double.
/// </summary>
/// <param name="i">The zero-based column ordinal.</param>
/// <returns>The double value of the column.</returns>
public double GetDouble(int i) => (double)GetValue(i);
/// <summary>
/// Gets the value of the specified column as a float.
/// </summary>
/// <param name="i">The zero-based column ordinal.</param>
/// <returns>The float value of the column.</returns>
public float GetFloat(int i) => (float)GetValue(i);
/// <summary>
/// Gets the value of the specified column as a GUID.
/// </summary>
/// <param name="i">The zero-based column ordinal.</param>
/// <returns>The GUID value of the column.</returns>
public Guid GetGuid(int i) => (Guid)GetValue(i);
/// <summary>
/// Gets the value of the specified column as a 16-bit signed integer.
/// </summary>
/// <param name="i">The zero-based column ordinal.</param>
/// <returns>The 16-bit signed integer value of the column.</returns>
public short GetInt16(int i) => (short)GetValue(i);
/// <summary>
/// Gets the value of the specified column as a 32-bit signed integer.
/// </summary>
/// <param name="i">The zero-based column ordinal.</param>
/// <returns>The 32-bit signed integer value of the column.</returns>
public int GetInt32(int i) => (int)GetValue(i);
/// <summary>
/// Gets the value of the specified column as a 64-bit signed integer.
/// </summary>
/// <param name="i">The zero-based column ordinal.</param>
/// <returns>The 64-bit signed integer value of the column.</returns>
public long GetInt64(int i) => (long)GetValue(i);
/// <summary>
/// Gets the value of the specified column as a string.
/// </summary>
/// <param name="i">The zero-based column ordinal.</param>
/// <returns>The string value of the column.</returns>
public string GetString(int i) => (string)GetValue(i);
// Schema and bulk data access - delegated to transformer
/// <summary>
/// Gets the name of the type for the specified column.
/// </summary>
/// <param name="i">The zero-based column ordinal.</param>
/// <returns>The name of the data type of the specified column.</returns>
public string GetDataTypeName(int i) => _transformer.GetDataTypeName(i, _source);
/// <summary>
/// Populates an object array with the values of the current row.
/// </summary>
/// <param name="values">An array of objects to populate with column values.</param>
/// <returns>The number of values populated.</returns>
public int GetValues(object[] values)
{
var count = Math.Min(values.Length, FieldCount);
@@ -61,13 +221,40 @@ internal sealed class TransformingDataReader : IDataReader
return count;
}
/// <summary>
/// Gets the byte values of the specified column starting at the given offset.
/// </summary>
/// <param name="i">The zero-based column ordinal.</param>
/// <param name="fieldOffset">The offset within the field.</param>
/// <param name="buffer">The buffer to read bytes into.</param>
/// <param name="bufferoffset">The offset in the buffer to start writing.</param>
/// <param name="length">The maximum number of bytes to read.</param>
/// <returns>The number of bytes read.</returns>
public long GetBytes(int i, long fieldOffset, byte[]? buffer, int bufferoffset, int length)
=> _transformer.GetBytes(i, fieldOffset, buffer, bufferoffset, length, _source);
/// <summary>
/// Gets the character values of the specified column starting at the given offset.
/// </summary>
/// <param name="i">The zero-based column ordinal.</param>
/// <param name="fieldoffset">The offset within the field.</param>
/// <param name="buffer">The buffer to read characters into.</param>
/// <param name="bufferoffset">The offset in the buffer to start writing.</param>
/// <param name="length">The maximum number of characters to read.</param>
/// <returns>The number of characters read.</returns>
public long GetChars(int i, long fieldoffset, char[]? buffer, int bufferoffset, int length)
=> _transformer.GetChars(i, fieldoffset, buffer, bufferoffset, length, _source);
/// <summary>
/// Gets a nested data reader for the specified column.
/// </summary>
/// <param name="i">The zero-based column ordinal.</param>
/// <returns>A nested data reader for the specified column.</returns>
public IDataReader GetData(int i) => _transformer.GetData(i, _source);
/// <summary>
/// Gets a DataTable that describes the schema of the result set.
/// </summary>
/// <returns>A DataTable describing the result set schema, or null if no schema is available.</returns>
public DataTable? GetSchemaTable() => _source.GetSchemaTable();
}
@@ -13,6 +13,7 @@ public class DataSyncHealthCheck : IHealthCheck
/// <summary>
/// Initializes a new instance of the <see cref="DataSyncHealthCheck"/> class.
/// </summary>
/// <param name="repository">The data update repository.</param>
public DataSyncHealthCheck(IDataUpdateRepository repository)
{
_repository = repository ?? throw new ArgumentNullException(nameof(repository));
@@ -3,5 +3,9 @@ namespace JdeScoping.DataSync.Options;
public class PipelineOptions
{
public const string SectionName = "Pipelines";
/// <summary>
/// Gets or sets the path to the pipeline configuration file.
/// </summary>
public string ConfigPath { get; set; } = "Pipelines/pipelines.json";
}
@@ -22,6 +22,8 @@ public class DataUpdateRepository : IDataUpdateRepository
/// <summary>
/// Initializes a new instance of the <see cref="DataUpdateRepository"/> class.
/// </summary>
/// <param name="connectionFactory">The database connection factory.</param>
/// <param name="logger">The logger instance.</param>
public DataUpdateRepository(
IDbConnectionFactory connectionFactory,
ILogger<DataUpdateRepository> logger)
@@ -53,6 +53,9 @@ public class EtlPipelineFactory : IEtlPipelineFactory
/// <summary>
/// Creates a new pipeline factory with a pre-loaded configuration (for testing).
/// </summary>
/// <param name="connectionFactory">Factory for creating database connections.</param>
/// <param name="config">Pre-loaded pipeline configuration.</param>
/// <param name="logger">Logger for pipeline execution.</param>
internal EtlPipelineFactory(
IDbConnectionFactory connectionFactory,
PipelinesRoot config,
@@ -167,6 +170,15 @@ public class EtlPipelineFactory : IEtlPipelineFactory
private UpdateTypes _updateType = UpdateTypes.Hourly;
private DateTime? _minDtOverride;
/// <summary>
/// Initializes a new instance of the PipelineBuilder class.
/// </summary>
/// <param name="connectionFactory">Factory for creating database connections.</param>
/// <param name="tableName">The name of the table for this pipeline.</param>
/// <param name="config">The pipeline configuration.</param>
/// <param name="settings">Global pipeline settings.</param>
/// <param name="scheduleDefaults">Default schedule configuration.</param>
/// <param name="logger">Logger for pipeline execution.</param>
public PipelineBuilder(
IDbConnectionFactory connectionFactory,
string tableName,
@@ -183,18 +195,32 @@ public class EtlPipelineFactory : IEtlPipelineFactory
_logger = logger;
}
/// <summary>
/// Specifies the update type for this pipeline.
/// </summary>
/// <param name="updateType">The type of update (Mass, Daily, or Hourly).</param>
/// <returns>The builder for fluent configuration.</returns>
public IEtlPipelineBuilder WithUpdateType(UpdateTypes updateType)
{
_updateType = updateType;
return this;
}
/// <summary>
/// Specifies the minimum date for incremental data extraction.
/// </summary>
/// <param name="minDt">The minimum date, or null for no filter.</param>
/// <returns>The builder for fluent configuration.</returns>
public IEtlPipelineBuilder WithMinimumDate(DateTime? minDt)
{
_minDtOverride = minDt;
return this;
}
/// <summary>
/// Builds and returns the configured ETL pipeline.
/// </summary>
/// <returns>A configured ETL pipeline ready for execution.</returns>
public EtlPipeline Build()
{
return BuildWithSchedules();
@@ -21,6 +21,9 @@ public class ScheduleChecker : IScheduleChecker
/// <summary>
/// Initializes a new instance of the <see cref="ScheduleChecker"/> class.
/// </summary>
/// <param name="repository">Repository for data update records.</param>
/// <param name="options">Data sync configuration options.</param>
/// <param name="logger">Logger instance.</param>
public ScheduleChecker(
IDataUpdateRepository repository,
IOptions<DataSyncOptions> options,
@@ -21,6 +21,15 @@ public class SearchExecutionService : ISearchExecutionService
private readonly WorkProcessorOptions _options;
private readonly ILogger<SearchExecutionService> _logger;
/// <summary>
/// Initializes a new instance of the SearchExecutionService.
/// </summary>
/// <param name="searchRepository">The repository for search data access.</param>
/// <param name="searchProcessor">The processor for executing search queries.</param>
/// <param name="excelExportService">The service for generating Excel export files.</param>
/// <param name="notificationService">The service for sending search status notifications.</param>
/// <param name="options">The work processor configuration options.</param>
/// <param name="logger">The logger for recording search execution events.</param>
public SearchExecutionService(
ISearchRepository searchRepository,
ISearchProcessor searchProcessor,
@@ -21,6 +21,11 @@ public class SyncOrchestrator : ISyncOrchestrator
/// <summary>
/// Initializes a new instance of the <see cref="SyncOrchestrator"/> class.
/// </summary>
/// <param name="scopeFactory">Factory for creating service scopes.</param>
/// <param name="scheduleChecker">Checker for determining pending sync tasks.</param>
/// <param name="options">Data sync configuration options.</param>
/// <param name="logger">Logger instance.</param>
/// <param name="metrics">Metrics collector for sync operations.</param>
public SyncOrchestrator(
IServiceScopeFactory scopeFactory,
IScheduleChecker scheduleChecker,
@@ -24,6 +24,11 @@ public class TableSyncOperation : ITableSyncOperation
/// <summary>
/// Initializes a new instance of the <see cref="TableSyncOperation"/> class.
/// </summary>
/// <param name="pipelineFactory">Factory for creating ETL pipelines.</param>
/// <param name="updateRepository">Repository for managing data update records.</param>
/// <param name="options">Data sync configuration options.</param>
/// <param name="logger">Logger for operation events.</param>
/// <param name="metrics">Metrics collector for operation tracking.</param>
public TableSyncOperation(
IEtlPipelineFactory pipelineFactory,
IDataUpdateRepository updateRepository,
@@ -18,6 +18,7 @@ public class DataSyncMetrics
/// <summary>
/// Initializes a new instance of the <see cref="DataSyncMetrics"/> class.
/// </summary>
/// <param name="meterFactory">The meter factory for creating metrics.</param>
public DataSyncMetrics(IMeterFactory meterFactory)
{
var meter = meterFactory.Create("JdeScoping.DataSync");
@@ -61,6 +62,8 @@ public class DataSyncMetrics
/// <summary>
/// Records that a sync operation has started.
/// </summary>
/// <param name="tableName">The name of the table being synced.</param>
/// <param name="updateType">The type of update (e.g., Mass, Daily, Hourly).</param>
public void RecordOperationStarted(string tableName, string updateType)
{
_operationsStarted.Add(1,
@@ -71,6 +74,10 @@ public class DataSyncMetrics
/// <summary>
/// Records that a sync operation completed successfully.
/// </summary>
/// <param name="tableName">The name of the table being synced.</param>
/// <param name="updateType">The type of update (e.g., Mass, Daily, Hourly).</param>
/// <param name="recordCount">The number of records processed.</param>
/// <param name="durationSeconds">The operation duration in seconds.</param>
public void RecordOperationCompleted(string tableName, string updateType, long recordCount, double durationSeconds)
{
var tags = new KeyValuePair<string, object?>[]
@@ -87,6 +94,8 @@ public class DataSyncMetrics
/// <summary>
/// Records that a sync operation failed.
/// </summary>
/// <param name="tableName">The name of the table being synced.</param>
/// <param name="updateType">The type of update (e.g., Mass, Daily, Hourly).</param>
public void RecordOperationFailed(string tableName, string updateType)
{
_operationsFailed.Add(1,
@@ -105,6 +114,9 @@ public class DataSyncMetrics
/// <summary>
/// Records completion of a sync cycle.
/// </summary>
/// <param name="successCount">Number of successful operations in the cycle.</param>
/// <param name="failedCount">Number of failed operations in the cycle.</param>
/// <param name="durationSeconds">The cycle duration in seconds.</param>
public void RecordCycleCompleted(int successCount, int failedCount, double durationSeconds)
{
_cyclesCompleted.Add(1,
@@ -26,6 +26,10 @@ public class WorkProcessor : BackgroundService
/// <summary>
/// Initializes a new instance of the <see cref="WorkProcessor"/> class.
/// </summary>
/// <param name="scopeFactory">Factory for creating service scopes.</param>
/// <param name="options">Configuration options for work processing.</param>
/// <param name="logger">Logger for recording work processor events.</param>
/// <param name="metrics">Metrics collector for tracking work cycles.</param>
public WorkProcessor(
IServiceScopeFactory scopeFactory,
IOptions<WorkProcessorOptions> options,